From c3b1518405cd3cae033efa42f51dcf96be1ba052 Mon Sep 17 00:00:00 2001 From: Michal Fupso Date: Wed, 11 Sep 2024 16:00:36 -0700 Subject: [PATCH 01/12] Add loadbalancer kube-controller --- api/pkg/apis/projectcalico/v3/constants.go | 7 + api/pkg/apis/projectcalico/v3/ippool.go | 15 +- .../projectcalico/v3/kubecontrollersconfig.go | 7 + .../projectcalico/v3/zz_generated.deepcopy.go | 21 + api/pkg/openapi/openapi_generated.go | 34 +- .../pkg/storage/calico/ippool_storage.go | 2 + calicoctl/calicoctl/commands/ipam/check.go | 25 +- .../resourcemgr/kubecontrollersconfig_test.go | 3 + .../calico-kube-controllers-rbac.yaml | 11 + .../templates/calico-kube-controllers.yaml | 4 +- .../operator.tigera.io_installations_crd.yaml | 8 + kube-controllers/cmd/kube-controllers/main.go | 6 + kube-controllers/pkg/config/config.go | 3 +- kube-controllers/pkg/config/config_test.go | 14 +- kube-controllers/pkg/config/runconfig.go | 27 +- .../loadbalancer/loadbalancer_controller.go | 700 ++++++++++++++++++ .../loadbalancer_controller_fv_test.go | 274 +++++++ .../loadbalancer/loadbalancer_suite_test.go | 38 + .../loadbalancer_controller_utils.go | 43 ++ .../crd/crd.projectcalico.org_ippools.yaml | 4 + ...ico.org_kubecontrollersconfigurations.yaml | 14 + libcalico-go/lib/apis/v1/ippool.go | 5 + libcalico-go/lib/apis/v1/openapi_generated.go | 7 + libcalico-go/lib/apis/v3/openapi_generated.go | 7 + .../lib/backend/k8s/resources/ippool.go | 3 +- libcalico-go/lib/backend/model/block.go | 3 +- libcalico-go/lib/backend/model/ippool.go | 20 +- libcalico-go/lib/clientv3/ippool.go | 29 +- libcalico-go/lib/clientv3/ippool_e2e_test.go | 11 +- .../lib/clientv3/kubecontrollersconfig.go | 8 +- libcalico-go/lib/ipam/ipam.go | 48 +- libcalico-go/lib/upgrade/converters/ippool.go | 14 +- libcalico-go/lib/validator/v3/validator.go | 47 +- .../lib/validator/v3/validator_test.go | 82 +- manifests/calico-bpf.yaml | 31 +- manifests/calico-etcd.yaml | 2 +- manifests/calico-policy-only.yaml | 31 +- manifests/calico-typha.yaml | 31 +- manifests/calico-vxlan.yaml | 31 +- manifests/calico.yaml | 31 +- manifests/canal-etcd.yaml | 2 +- manifests/canal.yaml | 31 +- manifests/crds.yaml | 18 + manifests/flannel-migration/calico.yaml | 31 +- .../ocp/crd.projectcalico.org_ippools.yaml | 4 + ...ico.org_kubecontrollersconfigurations.yaml | 14 + .../operator.tigera.io_installations_crd.yaml | 8 + manifests/operator-crds.yaml | 26 + manifests/tigera-operator.yaml | 26 + node/tests/k8st/infra/calico-kdd.yaml | 13 +- 50 files changed, 1821 insertions(+), 53 deletions(-) create mode 100644 kube-controllers/pkg/controllers/loadbalancer/loadbalancer_controller.go create mode 100644 kube-controllers/pkg/controllers/loadbalancer/loadbalancer_controller_fv_test.go create mode 100644 kube-controllers/pkg/controllers/loadbalancer/loadbalancer_suite_test.go create mode 100644 kube-controllers/tests/testutils/loadbalancer_controller_utils.go diff --git a/api/pkg/apis/projectcalico/v3/constants.go b/api/pkg/apis/projectcalico/v3/constants.go index 289204d0cad..3f7742cf1eb 100644 --- a/api/pkg/apis/projectcalico/v3/constants.go +++ b/api/pkg/apis/projectcalico/v3/constants.go @@ -53,4 +53,11 @@ const ( // Enum options for enable/disable fields Enabled = "Enabled" Disabled = "Disabled" + + // Enum options for AllServices/RequestedServicesOnly + AllServices = "AllServices" + RequestedServicesOnly = "RequestedServicesOnly" + + // Host name used for Service LoadBalancer + VirtualLoadBalancer = "virtual-load-balancer" ) diff --git a/api/pkg/apis/projectcalico/v3/ippool.go b/api/pkg/apis/projectcalico/v3/ippool.go index 969abb61558..c5f97e17292 100644 --- a/api/pkg/apis/projectcalico/v3/ippool.go +++ b/api/pkg/apis/projectcalico/v3/ippool.go @@ -85,13 +85,17 @@ type IPPoolSpec struct { // AllowedUse controls what the IP pool will be used for. If not specified or empty, defaults to // ["Tunnel", "Workload"] for back-compatibility AllowedUses []IPPoolAllowedUse `json:"allowedUses,omitempty" validate:"omitempty"` + + // Determines the mode how IP addresses should be assigned from this pool + AssignmentMode AssignmentMode `json:"assignmentMode,omitempty" validate:"omitempty,assignmentMode"` } type IPPoolAllowedUse string const ( - IPPoolAllowedUseWorkload IPPoolAllowedUse = "Workload" - IPPoolAllowedUseTunnel IPPoolAllowedUse = "Tunnel" + IPPoolAllowedUseWorkload IPPoolAllowedUse = "Workload" + IPPoolAllowedUseTunnel IPPoolAllowedUse = "Tunnel" + IPPoolAllowedUseLoadBalancer IPPoolAllowedUse = "LoadBalancer" ) type VXLANMode string @@ -120,6 +124,13 @@ const ( CrossSubnet EncapMode = "cross-subnet" ) +type AssignmentMode string + +const ( + Automatic AssignmentMode = "Automatic" + Manual AssignmentMode = "Manual" +) + const DefaultMode = Always type IPIPConfiguration struct { diff --git a/api/pkg/apis/projectcalico/v3/kubecontrollersconfig.go b/api/pkg/apis/projectcalico/v3/kubecontrollersconfig.go index d2805b7ca2c..56c38e384a9 100644 --- a/api/pkg/apis/projectcalico/v3/kubecontrollersconfig.go +++ b/api/pkg/apis/projectcalico/v3/kubecontrollersconfig.go @@ -84,6 +84,9 @@ type ControllersConfig struct { // Namespace enables and configures the namespace controller. Enabled by default, set to nil to disable. Namespace *NamespaceControllerConfig `json:"namespace,omitempty"` + + // LoadBalancer enables and configures the LoadBalancer controller. Enabled by default, set to nil to disable. + LoadBalancer *LoadBalancerControllerConfig `json:"loadBalancer,omitempty"` } // NodeControllerConfig configures the node controller, which automatically cleans up configuration @@ -137,6 +140,10 @@ type NamespaceControllerConfig struct { ReconcilerPeriod *metav1.Duration `json:"reconcilerPeriod,omitempty" validate:"omitempty"` } +type LoadBalancerControllerConfig struct { + AssignIPs string `json:"assignIPs,omitempty" validate:"omitempty,assignIPs"` +} + // KubeControllersConfigurationStatus represents the status of the configuration. It's useful for admins to // be able to see the actual config that was applied, which can be modified by environment variables on the // kube-controllers process. diff --git a/api/pkg/apis/projectcalico/v3/zz_generated.deepcopy.go b/api/pkg/apis/projectcalico/v3/zz_generated.deepcopy.go index cbdebff3f2d..a3815da3621 100644 --- a/api/pkg/apis/projectcalico/v3/zz_generated.deepcopy.go +++ b/api/pkg/apis/projectcalico/v3/zz_generated.deepcopy.go @@ -926,6 +926,11 @@ func (in *ControllersConfig) DeepCopyInto(out *ControllersConfig) { *out = new(NamespaceControllerConfig) (*in).DeepCopyInto(*out) } + if in.LoadBalancer != nil { + in, out := &in.LoadBalancer, &out.LoadBalancer + *out = new(LoadBalancerControllerConfig) + **out = **in + } return } @@ -2354,6 +2359,22 @@ func (in *KubeControllersConfigurationStatus) DeepCopy() *KubeControllersConfigu return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *LoadBalancerControllerConfig) DeepCopyInto(out *LoadBalancerControllerConfig) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LoadBalancerControllerConfig. +func (in *LoadBalancerControllerConfig) DeepCopy() *LoadBalancerControllerConfig { + if in == nil { + return nil + } + out := new(LoadBalancerControllerConfig) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *NamespaceControllerConfig) DeepCopyInto(out *NamespaceControllerConfig) { *out = *in diff --git a/api/pkg/openapi/openapi_generated.go b/api/pkg/openapi/openapi_generated.go index ba2ea1f23f7..08c736b31ba 100644 --- a/api/pkg/openapi/openapi_generated.go +++ b/api/pkg/openapi/openapi_generated.go @@ -85,6 +85,7 @@ func GetOpenAPIDefinitions(ref common.ReferenceCallback) map[string]common.OpenA "github.com/projectcalico/api/pkg/apis/projectcalico/v3.KubeControllersConfigurationList": schema_pkg_apis_projectcalico_v3_KubeControllersConfigurationList(ref), "github.com/projectcalico/api/pkg/apis/projectcalico/v3.KubeControllersConfigurationSpec": schema_pkg_apis_projectcalico_v3_KubeControllersConfigurationSpec(ref), "github.com/projectcalico/api/pkg/apis/projectcalico/v3.KubeControllersConfigurationStatus": schema_pkg_apis_projectcalico_v3_KubeControllersConfigurationStatus(ref), + "github.com/projectcalico/api/pkg/apis/projectcalico/v3.LoadBalancerControllerConfig": schema_pkg_apis_projectcalico_v3_LoadBalancerControllerConfig(ref), "github.com/projectcalico/api/pkg/apis/projectcalico/v3.NamespaceControllerConfig": schema_pkg_apis_projectcalico_v3_NamespaceControllerConfig(ref), "github.com/projectcalico/api/pkg/apis/projectcalico/v3.NetworkPolicy": schema_pkg_apis_projectcalico_v3_NetworkPolicy(ref), "github.com/projectcalico/api/pkg/apis/projectcalico/v3.NetworkPolicyList": schema_pkg_apis_projectcalico_v3_NetworkPolicyList(ref), @@ -2035,11 +2036,17 @@ func schema_pkg_apis_projectcalico_v3_ControllersConfig(ref common.ReferenceCall Ref: ref("github.com/projectcalico/api/pkg/apis/projectcalico/v3.NamespaceControllerConfig"), }, }, + "loadBalancer": { + SchemaProps: spec.SchemaProps{ + Description: "LoadBalancer enables and configures the LoadBalancer controller. Enabled by default, set to nil to disable.", + Ref: ref("github.com/projectcalico/api/pkg/apis/projectcalico/v3.LoadBalancerControllerConfig"), + }, + }, }, }, }, Dependencies: []string{ - "github.com/projectcalico/api/pkg/apis/projectcalico/v3.NamespaceControllerConfig", "github.com/projectcalico/api/pkg/apis/projectcalico/v3.NodeControllerConfig", "github.com/projectcalico/api/pkg/apis/projectcalico/v3.PolicyControllerConfig", "github.com/projectcalico/api/pkg/apis/projectcalico/v3.ServiceAccountControllerConfig", "github.com/projectcalico/api/pkg/apis/projectcalico/v3.WorkloadEndpointControllerConfig"}, + "github.com/projectcalico/api/pkg/apis/projectcalico/v3.LoadBalancerControllerConfig", "github.com/projectcalico/api/pkg/apis/projectcalico/v3.NamespaceControllerConfig", "github.com/projectcalico/api/pkg/apis/projectcalico/v3.NodeControllerConfig", "github.com/projectcalico/api/pkg/apis/projectcalico/v3.PolicyControllerConfig", "github.com/projectcalico/api/pkg/apis/projectcalico/v3.ServiceAccountControllerConfig", "github.com/projectcalico/api/pkg/apis/projectcalico/v3.WorkloadEndpointControllerConfig"}, } } @@ -4244,6 +4251,13 @@ func schema_pkg_apis_projectcalico_v3_IPPoolSpec(ref common.ReferenceCallback) c }, }, }, + "assignmentMode": { + SchemaProps: spec.SchemaProps{ + Description: "Determines the mode how IP addresses should be assigned from this pool", + Type: []string{"string"}, + Format: "", + }, + }, }, Required: []string{"cidr"}, }, @@ -4561,6 +4575,24 @@ func schema_pkg_apis_projectcalico_v3_KubeControllersConfigurationStatus(ref com } } +func schema_pkg_apis_projectcalico_v3_LoadBalancerControllerConfig(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Type: []string{"object"}, + Properties: map[string]spec.Schema{ + "assignIPs": { + SchemaProps: spec.SchemaProps{ + Type: []string{"string"}, + Format: "", + }, + }, + }, + }, + }, + } +} + func schema_pkg_apis_projectcalico_v3_NamespaceControllerConfig(ref common.ReferenceCallback) common.OpenAPIDefinition { return common.OpenAPIDefinition{ Schema: spec.Schema{ diff --git a/apiserver/pkg/storage/calico/ippool_storage.go b/apiserver/pkg/storage/calico/ippool_storage.go index b43b6468b95..b278662183d 100644 --- a/apiserver/pkg/storage/calico/ippool_storage.go +++ b/apiserver/pkg/storage/calico/ippool_storage.go @@ -108,5 +108,7 @@ func (gc IPPoolConverter) convertToAAPIList(libcalicoListObject resourceListObje if matched, err := pred.Matches(&aapiIPPool); err == nil && matched { aapiIPPoolList.Items = append(aapiIPPoolList.Items, aapiIPPool) } + } + } diff --git a/calicoctl/calicoctl/commands/ipam/check.go b/calicoctl/calicoctl/commands/ipam/check.go index 4efc6139ded..db0e7c833af 100644 --- a/calicoctl/calicoctl/commands/ipam/check.go +++ b/calicoctl/calicoctl/commands/ipam/check.go @@ -23,7 +23,9 @@ import ( "sort" "strings" - docopt "github.com/docopt/docopt-go" + "github.com/docopt/docopt-go" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/kubernetes" @@ -255,6 +257,27 @@ func (c *IPAMChecker) checkIPAM(ctx context.Context) error { fmt.Println() } + { + fmt.Println("Loading all service load balancer.") + service, err := c.k8sClient.CoreV1().Services("").List(ctx, metav1.ListOptions{}) + if err != nil { + return err + } + + var lengthLoadBalancer int + + for _, svc := range service.Items { + if svc.Spec.Type == corev1.ServiceTypeLoadBalancer && svc.Status.LoadBalancer.Ingress != nil { + lengthLoadBalancer++ + for _, ingress := range svc.Status.LoadBalancer.Ingress { + c.recordInUseIP(ingress.IP, svc, svc.Name) + } + } + } + fmt.Printf("Found %d service load balancer.\n", lengthLoadBalancer) + fmt.Println() + } + { fmt.Println("Loading all workload endpoints.") weps, err := c.v3Client.WorkloadEndpoints().List(ctx, options.ListOptions{}) diff --git a/calicoctl/calicoctl/resourcemgr/kubecontrollersconfig_test.go b/calicoctl/calicoctl/resourcemgr/kubecontrollersconfig_test.go index f8df69a9a0f..e171cbd4ddf 100644 --- a/calicoctl/calicoctl/resourcemgr/kubecontrollersconfig_test.go +++ b/calicoctl/calicoctl/resourcemgr/kubecontrollersconfig_test.go @@ -49,6 +49,8 @@ spec: reconcilerPeriod: 4m namespace: reconcilerPeriod: 5m + loadbalancer: + assignIPs: AllServices ` resources, err := createResources(text) Expect(err).NotTo(HaveOccurred()) @@ -71,6 +73,7 @@ spec: To(Equal(&api.ServiceAccountControllerConfig{ReconcilerPeriod: &v1.Duration{Duration: time.Minute * 4}})) Expect(kcc.Spec.Controllers.Namespace). To(Equal(&api.NamespaceControllerConfig{ReconcilerPeriod: &v1.Duration{Duration: time.Minute * 5}})) + Expect(kcc.Spec.Controllers.LoadBalancer.AssignIPs).To(Equal(&api.LoadBalancerControllerConfig{AssignIPs: api.AllServices})) // Status Expect(kcc.Status.EnvironmentVars).To(BeNil()) diff --git a/charts/calico/templates/calico-kube-controllers-rbac.yaml b/charts/calico/templates/calico-kube-controllers-rbac.yaml index 5a7398c488e..2b07d047b52 100644 --- a/charts/calico/templates/calico-kube-controllers-rbac.yaml +++ b/charts/calico/templates/calico-kube-controllers-rbac.yaml @@ -64,6 +64,16 @@ rules: - get - list - watch + # Services are monitored for service LoadBalancer + - apiGroups: [""] + resources: + - services + - services/status + verbs: + - get + - list + - update + - watch # IPAM resources are manipulated in response to node and block updates, as well as periodic triggers. - apiGroups: ["crd.projectcalico.org"] resources: @@ -75,6 +85,7 @@ rules: - blockaffinities - ipamblocks - ipamhandles + - ipamconfigs - tiers verbs: - get diff --git a/charts/calico/templates/calico-kube-controllers.yaml b/charts/calico/templates/calico-kube-controllers.yaml index bb6834bf276..4c1a95a13e9 100644 --- a/charts/calico/templates/calico-kube-controllers.yaml +++ b/charts/calico/templates/calico-kube-controllers.yaml @@ -71,7 +71,7 @@ spec: key: etcd_cert # Choose which controllers to run. - name: ENABLED_CONTROLLERS - value: policy,namespace,serviceaccount,workloadendpoint,node + value: policy,namespace,serviceaccount,workloadendpoint,node,loadbalancer {{- if .Values.kubeControllers.env }} {{ toYaml .Values.kubeControllers.env | indent 12 }} {{- end }} @@ -111,7 +111,7 @@ spec: env: # Choose which controllers to run. - name: ENABLED_CONTROLLERS - value: node + value: node,loadbalancer - name: DATASTORE_TYPE value: kubernetes livenessProbe: diff --git a/charts/tigera-operator/crds/operator.tigera.io_installations_crd.yaml b/charts/tigera-operator/crds/operator.tigera.io_installations_crd.yaml index a90bc6b52b0..f8bc238901b 100644 --- a/charts/tigera-operator/crds/operator.tigera.io_installations_crd.yaml +++ b/charts/tigera-operator/crds/operator.tigera.io_installations_crd.yaml @@ -1129,6 +1129,14 @@ spec: NodeSelector specifies the node selector that will be set for the IP Pool. Default: 'all()' type: string + assignmentMode: + description: AssignmentMode determines if IP addresses from this pool should be + assigned automatically or on request only + enum: + - Automatic + - Manual + type: string + default: Automatic required: - cidr type: object diff --git a/kube-controllers/cmd/kube-controllers/main.go b/kube-controllers/cmd/kube-controllers/main.go index 54c2ecb20ee..82a635d87de 100644 --- a/kube-controllers/cmd/kube-controllers/main.go +++ b/kube-controllers/cmd/kube-controllers/main.go @@ -23,6 +23,7 @@ import ( "strings" "time" + "github.com/projectcalico/calico/kube-controllers/pkg/controllers/loadbalancer" "github.com/prometheus/client_golang/prometheus/promhttp" log "github.com/sirupsen/logrus" "go.etcd.io/etcd/client/pkg/v3/srv" @@ -454,6 +455,11 @@ func (cc *controllerControl) InitControllers(ctx context.Context, cfg config.Run serviceAccountController := serviceaccount.NewServiceAccountController(ctx, k8sClientset, calicoClient, *cfg.Controllers.ServiceAccount) cc.controllers["ServiceAccount"] = serviceAccountController } + + if cfg.Controllers.LoadBalancer != nil { + loadBalancerController := loadbalancer.NewLoadBalancerController(ctx, k8sClientset, calicoClient, *cfg.Controllers.LoadBalancer) + cc.controllers["LoadBalancer"] = loadBalancerController + } } // registerInformers registers the given informers, if not already registered. Registered informers diff --git a/kube-controllers/pkg/config/config.go b/kube-controllers/pkg/config/config.go index e4e33451f0f..2b4aaaae849 100644 --- a/kube-controllers/pkg/config/config.go +++ b/kube-controllers/pkg/config/config.go @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2020 Tigera, Inc. All rights reserved. +// Copyright (c) 2017-2024 Tigera, Inc. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -40,6 +40,7 @@ type Config struct { ProfileWorkers int `default:"1" split_words:"true"` PolicyWorkers int `default:"1" split_words:"true"` NodeWorkers int `default:"1" split_words:"true"` + LoadBalancerWorkers int `default:"1" split_words:"true"` // Path to a kubeconfig file to use for accessing the k8s API. Kubeconfig string `default:"" split_words:"false"` diff --git a/kube-controllers/pkg/config/config_test.go b/kube-controllers/pkg/config/config_test.go index b4b2be7633d..b973c55ac59 100644 --- a/kube-controllers/pkg/config/config_test.go +++ b/kube-controllers/pkg/config/config_test.go @@ -1,4 +1,4 @@ -// Copyright (c) 2017, 2020 Tigera, Inc. All rights reserved. +// Copyright (c) 2017 - 2024 Tigera, Inc. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -94,6 +94,7 @@ var _ = Describe("Config", func() { Expect(cfg.WorkloadEndpointWorkers).To(Equal(1)) Expect(cfg.ProfileWorkers).To(Equal(1)) Expect(cfg.PolicyWorkers).To(Equal(1)) + Expect(cfg.LoadBalancerWorkers).To(Equal(1)) Expect(cfg.Kubeconfig).To(Equal("")) }) @@ -142,6 +143,10 @@ var _ = Describe("Config", func() { ReconcilerPeriod: time.Minute * 5, NumberOfWorkers: 1, })) + Expect(rc.LoadBalancer).To(Equal(&config.LoadBalancerControllerConfig{ + NumberOfWorkers: 1, + AssignIPs: v3.AllServices, + })) close(done) }) @@ -200,6 +205,9 @@ var _ = Describe("Config", func() { ReconcilerPeriod: &v1.Duration{Duration: time.Second * 32}}, ServiceAccount: &v3.ServiceAccountControllerConfig{ ReconcilerPeriod: &v1.Duration{Duration: time.Second * 33}}, + LoadBalancer: &v3.LoadBalancerControllerConfig{ + AssignIPs: v3.RequestedServicesOnly, + }, }, } m = &mockKCC{get: kcc} @@ -240,6 +248,10 @@ var _ = Describe("Config", func() { ReconcilerPeriod: time.Second * 33, NumberOfWorkers: 1, })) + Expect(rc.LoadBalancer).To(Equal(&config.LoadBalancerControllerConfig{ + NumberOfWorkers: 1, + AssignIPs: v3.RequestedServicesOnly, + })) close(done) }) diff --git a/kube-controllers/pkg/config/runconfig.go b/kube-controllers/pkg/config/runconfig.go index 0cbfeeb49cb..9944dbf3e2c 100644 --- a/kube-controllers/pkg/config/runconfig.go +++ b/kube-controllers/pkg/config/runconfig.go @@ -1,4 +1,4 @@ -// Copyright (c) 2020 Tigera, Inc. All rights reserved. +// Copyright (c) 2024 Tigera, Inc. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -68,6 +68,9 @@ func init() { Namespace: &v3.NamespaceControllerConfig{ ReconcilerPeriod: &v1.Duration{Duration: time.Minute * 5}, }, + LoadBalancer: &v3.LoadBalancerControllerConfig{ + AssignIPs: v3.AllServices, + }, }, } } @@ -90,6 +93,7 @@ type ControllersConfig struct { WorkloadEndpoint *GenericControllerConfig ServiceAccount *GenericControllerConfig Namespace *GenericControllerConfig + LoadBalancer *LoadBalancerControllerConfig } type GenericControllerConfig struct { @@ -110,6 +114,13 @@ type NodeControllerConfig struct { LeakGracePeriod *v1.Duration } +type LoadBalancerControllerConfig struct { + NumberOfWorkers int + + //Determines if LoadBalancer controller will auto-assign ip addresses or only if asked to do so via annotation + AssignIPs string +} + type RunConfigController struct { out chan RunConfig } @@ -358,6 +369,11 @@ func mergeConfig(envVars map[string]string, envCfg Config, apiCfg v3.KubeControl rc.Namespace.NumberOfWorkers = envCfg.ProfileWorkers } + if rc.LoadBalancer != nil { + rc.LoadBalancer.NumberOfWorkers = envCfg.LoadBalancerWorkers + rc.LoadBalancer.AssignIPs = apiCfg.Controllers.LoadBalancer.AssignIPs + } + return rCfg, status } @@ -511,6 +527,7 @@ func mergeEnabledControllers(envVars map[string]string, status *v3.KubeControlle w := ac.WorkloadEndpoint s := ac.ServiceAccount ns := ac.Namespace + lb := ac.LoadBalancer v, p := envVars[EnvEnabledControllers] if p { @@ -532,6 +549,9 @@ func mergeEnabledControllers(envVars map[string]string, status *v3.KubeControlle case "serviceaccount": rc.ServiceAccount = &GenericControllerConfig{} sc.ServiceAccount = &v3.ServiceAccountControllerConfig{} + case "loadbalancer": + rc.LoadBalancer = &LoadBalancerControllerConfig{} + sc.LoadBalancer = &v3.LoadBalancerControllerConfig{} case "flannelmigration": log.WithField(EnvEnabledControllers, v).Fatal("cannot run flannelmigration with other controllers") default: @@ -570,6 +590,11 @@ func mergeEnabledControllers(envVars map[string]string, status *v3.KubeControlle rc.Namespace = &GenericControllerConfig{} sc.Namespace = &v3.NamespaceControllerConfig{} } + + if lb != nil { + rc.LoadBalancer = &LoadBalancerControllerConfig{} + sc.LoadBalancer = &v3.LoadBalancerControllerConfig{} + } } // Set reconciler periods, if enabled diff --git a/kube-controllers/pkg/controllers/loadbalancer/loadbalancer_controller.go b/kube-controllers/pkg/controllers/loadbalancer/loadbalancer_controller.go new file mode 100644 index 00000000000..a2c765c8e49 --- /dev/null +++ b/kube-controllers/pkg/controllers/loadbalancer/loadbalancer_controller.go @@ -0,0 +1,700 @@ +// Copyright (c) 2019-2024 Tigera, Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package loadbalancer + +import ( + "context" + "errors" + "fmt" + "net" + "reflect" + "slices" + "strings" + "time" + + api "github.com/projectcalico/api/pkg/apis/projectcalico/v3" + "github.com/projectcalico/calico/kube-controllers/pkg/config" + "github.com/projectcalico/calico/kube-controllers/pkg/controllers/controller" + "github.com/projectcalico/calico/kube-controllers/pkg/controllers/node" + bapi "github.com/projectcalico/calico/libcalico-go/lib/backend/api" + "github.com/projectcalico/calico/libcalico-go/lib/backend/model" + "github.com/projectcalico/calico/libcalico-go/lib/ipam" + log "github.com/sirupsen/logrus" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + k8svalidation "k8s.io/apimachinery/pkg/util/validation" + "k8s.io/apimachinery/pkg/util/wait" + + rcache "github.com/projectcalico/calico/kube-controllers/pkg/cache" + client "github.com/projectcalico/calico/libcalico-go/lib/clientv3" + + cnet "github.com/projectcalico/calico/libcalico-go/lib/net" + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/fields" + uruntime "k8s.io/apimachinery/pkg/util/runtime" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/cache" +) + +const ( + annotationIpv4Pools = "projectcalico.org/ipv4pools" + annotationIpv6Pools = "projectcalico.org/ipv6pools" + annotationLoadBalancerIp = "projectcalico.org/loadBalancerIPs" + timer = 1 * time.Minute +) + +// loadBalancerController implements the Controller interface for managing Kubernetes services +// and endpoints, syncing them to the Calico datastore as NetworkSet. +type loadBalancerController struct { + informer cache.Controller + lbResourceCache rcache.ResourceCache + calicoClient client.Interface + dataFeed *node.DataFeed + ctx context.Context + cfg config.LoadBalancerControllerConfig + clientSet *kubernetes.Clientset + syncerUpdates chan interface{} + syncStatus bapi.SyncStatus + syncChan chan interface{} + ipamBlocks map[string]model.KVPair + ipPools map[string]api.IPPool +} + +// NewLoadBalancerController returns a controller which manages Service LoadBalancer objects. +func NewLoadBalancerController(ctx context.Context, clientset *kubernetes.Clientset, calicoClient client.Interface, cfg config.LoadBalancerControllerConfig) controller.Controller { + // set up service informer + svcWatcher := cache.NewListWatchFromClient(clientset.CoreV1().RESTClient(), "services", "", fields.Everything()) + + listFunc := func() (map[string]interface{}, error) { + log.Debugf("Listing profiles from Calico datastore") + filteredServices := make(map[string]interface{}) + + // Get all profile objects from Calico datastore. + serviceList, err := clientset.CoreV1().Services("").List(ctx, metav1.ListOptions{}) + if err != nil { + return nil, err + } + + // Filter out only services of type LoadBalancer + for _, svc := range serviceList.Items { + if svc.Spec.Type == v1.ServiceTypeLoadBalancer { + key := createHandle(&svc) + filteredServices[key] = svc + } + } + log.Debugf("Found %d Service LoadBalancer in Calico datastore", len(filteredServices)) + return filteredServices, nil + } + + cacheArgs := rcache.ResourceCacheArgs{ + ListFunc: listFunc, + ObjectType: reflect.TypeOf(&v1.Service{}), + LogTypeDesc: "Service", + } + ccache := rcache.NewResourceCache(cacheArgs) + + svcHandler := cache.ResourceEventHandlerFuncs{ + AddFunc: func(obj interface{}) { + log.Debugf("Got ADD event for service") + svc := obj.(*v1.Service) + if isCalicoManagedLoadBalancer(svc, cfg.AssignIPs) { + ccache.Set(createHandle(svc), svc) + } + }, + UpdateFunc: func(oldObj, newObj interface{}) { + log.Debugf("Got UPDATE event for service") + log.Debugf("Old object: %v", oldObj) + log.Debugf("New object: %v", newObj) + svcNew := newObj.(*v1.Service) + svcOld := oldObj.(*v1.Service) + + // Service changed type, we need to release the addresses used by the service + if svcNew.Spec.Type != v1.ServiceTypeLoadBalancer && + svcOld.Spec.Type == v1.ServiceTypeLoadBalancer { + ccache.Delete(createHandle(svcOld)) + } else if svcOld.Annotations[annotationIpv4Pools] != svcNew.Annotations[annotationIpv4Pools] || + svcOld.Annotations[annotationIpv6Pools] != svcNew.Annotations[annotationIpv6Pools] || + svcOld.Annotations[annotationLoadBalancerIp] != svcNew.Annotations[annotationLoadBalancerIp] { + // Calico annotations have changed, get new address based on new conditions. + ccache.Set(createHandle(svcNew), svcNew) + } + }, + DeleteFunc: func(obj interface{}) { + log.Debugf("Got DELETE event for service") + svc := obj.(*v1.Service) + if isCalicoManagedLoadBalancer(svc, cfg.AssignIPs) { + ccache.Delete(createHandle(svc)) + } + }, + } + _, informer := cache.NewIndexerInformer(svcWatcher, &v1.Service{}, 0, svcHandler, cache.Indexers{}) + + return &loadBalancerController{ + calicoClient: calicoClient, + ctx: ctx, + cfg: cfg, + clientSet: clientset, + dataFeed: node.NewDataFeed(calicoClient), + lbResourceCache: ccache, + informer: informer, + syncerUpdates: make(chan interface{}), + syncChan: make(chan interface{}), + ipamBlocks: make(map[string]model.KVPair), + ipPools: make(map[string]api.IPPool), + } +} + +// Run starts the controller. +func (c *loadBalancerController) Run(stopCh chan struct{}) { + defer uruntime.HandleCrash() + + // Start the Kubernetes informer, which will start syncing with the Kubernetes API. + log.Info("Starting LoadBalancer controller") + go c.informer.Run(stopCh) + + // Wait until we are in sync with the Kubernetes API before starting the + // resource cache. + log.Debug("Waiting to sync with Kubernetes API (Service)") + for !c.informer.HasSynced() { + time.Sleep(100 * time.Millisecond) + } + log.Debug("Finished syncing with Kubernetes API (Service)") + c.lbResourceCache.Run(timer.String()) + + for i := 0; i < c.cfg.NumberOfWorkers; i++ { + go wait.Until(c.runWorker, time.Second, stopCh) + } + + log.Info("LoadBalancer controller is now running") + + // Register to the syncer datafeed for updates to ippools and ipamBlocks + c.RegisterWith(c.dataFeed) + c.dataFeed.Start() + go c.acceptScheduledRequests(stopCh) + + <-stopCh + log.Info("Stopping Service controller") +} + +func (c *loadBalancerController) RegisterWith(f *node.DataFeed) { + f.RegisterForNotification(model.BlockKey{}, c.onUpdate) + f.RegisterForNotification(model.ResourceKey{}, c.onUpdate) + f.RegisterForSyncStatus(c.onStatusUpdate) +} + +func (c *loadBalancerController) onStatusUpdate(s bapi.SyncStatus) { + c.syncerUpdates <- s +} + +func (c *loadBalancerController) onUpdate(update bapi.Update) { + switch update.KVPair.Key.(type) { + case model.ResourceKey: + switch update.KVPair.Key.(model.ResourceKey).Kind { + case api.KindIPPool: + c.syncerUpdates <- update.KVPair + } + case model.BlockKey: + c.syncerUpdates <- update.KVPair + default: + log.Warnf("Unexpected kind received over syncer: %s", update.KVPair.Key) + } +} + +func (c *loadBalancerController) acceptScheduledRequests(stopCh <-chan struct{}) { + log.Infof("Will run periodic IPAM sync every %s", timer) + t := time.NewTicker(timer) + for { + select { + case update := <-c.syncerUpdates: + c.handleUpdate(update) + case <-t.C: + log.Infof("Running periodic IPAM sync of Service LoadBalancer") + c.syncIPAM() + case <-stopCh: + return + } + } +} + +func (c *loadBalancerController) handleUpdate(update interface{}) { + switch update := update.(type) { + case bapi.SyncStatus: + c.syncStatus = update + switch update { + case bapi.InSync: + log.WithField("status", update).Info("Syncer is InSync, kicking sync channel") + kick(c.syncChan) + } + return + case model.KVPair: + switch update.Key.(type) { + case model.ResourceKey: + switch update.Key.(model.ResourceKey).Kind { + case api.KindIPPool: + log.Infof("Running periodic IPAM sync of Service LoadBalancer") + c.handleIPPoolUpdate(update) + return + } + case model.BlockKey: + c.handleBlockUpdate(update) + return + } + } + +} + +func (c *loadBalancerController) handleBlockUpdate(kvp model.KVPair) { + if kvp.Value != nil { + host := kvp.Value.(*model.AllocationBlock).Affinity + if host != nil && *host == fmt.Sprintf("host:%s", api.VirtualLoadBalancer) { + c.ipamBlocks[kvp.Key.String()] = kvp + } + } else { + delete(c.ipamBlocks, kvp.Key.String()) + } +} + +func (c *loadBalancerController) handleIPPoolUpdate(kvp model.KVPair) { + log.Infof("Addind IPPOOL") + if kvp.Value != nil { + pool := kvp.Value.(*api.IPPool) + if slices.Contains(pool.Spec.AllowedUses, api.IPPoolAllowedUseLoadBalancer) { + c.ipPools[kvp.Key.String()] = *pool + } + } else { + delete(c.ipPools, kvp.Key.String()) + } +} + +// syncIpam has two main uses. It functions as a garbage collection for leaked IP addresses from Service LoadBalancer +// The other use case is to update IPs for any Service LoadBalancer that do not have IPs assigned, this could be caused by the user +// creating Service LoadBalancer before any valid pools were created +func (c *loadBalancerController) syncIPAM() { + // Garbage collection + // Check all ipamBlocks with loadBalancer affinity + for _, block := range c.ipamBlocks { + attributes := block.Value.(*model.AllocationBlock).Attributes + for _, attr := range attributes { + obj, empty := c.lbResourceCache.Get(*attr.AttrPrimary) + service := obj.(*v1.Service) + if !empty { + // Service with handle exists, we need to check that all assigned IPs with the handle are still in use by the service + log.Infof("Service found for handle: %s. Check if all IPs allocated by the handle are in use.", *attr.AttrPrimary) + ips, err := c.calicoClient.IPAM().IPsByHandle(c.ctx, *attr.AttrPrimary) + if err != nil { + log.Errorf("Error getting IPs for handle: %s", *attr.AttrPrimary) + break + } + for _, ingressIP := range service.Status.LoadBalancer.Ingress { + inUse := false + for _, handleIP := range ips { + if handleIP.String() == ingressIP.IP { + inUse = true + } + if !inUse { + releaseOptions := ipam.ReleaseOptions{ + Address: ingressIP.IP, + } + _, err = c.calicoClient.IPAM().ReleaseIPs(c.ctx, releaseOptions) + if err != nil { + log.Errorf("Error releasing IP(%s) for service: %s", ingressIP.IP, service.Name) + } + } + } + } + } else { + // Service no longer exists, leak confirmed. Release all IPs allocated with the specific handle + log.Infof("Service not found for handle: %s. Releasing unused IPs", *attr.AttrPrimary) + err := c.releaseIP(*attr.AttrPrimary) + if err != nil { + log.Errorf("Error releasing IPAM for handle %s: %s", *attr.AttrPrimary, err) + } + } + } + } + + // Check that all services have assigned IPs as requested, skip if there are no ippools + if len(c.ipPools) != 0 { + for _, key := range c.lbResourceCache.ListKeys() { + obj, empty := c.lbResourceCache.Get(key) + if !empty { + service := obj.(*v1.Service) + if isCalicoManagedLoadBalancer(service, c.cfg.AssignIPs) { + if service.Status.LoadBalancer.Ingress == nil || + (len(service.Status.LoadBalancer.Ingress) == 1 && + (*service.Spec.IPFamilyPolicy == v1.IPFamilyPolicyRequireDualStack) || *service.Spec.IPFamilyPolicy == v1.IPFamilyPolicyPreferDualStack) { + err := c.assignIP(service) + if err != nil { + log.Errorf("Error assigning IP to service %s: %s", service.Name, err) + } + } + } + } + } + } +} + +func (c *loadBalancerController) runWorker() { + for c.processNextItem() { + } +} + +func (c *loadBalancerController) processNextItem() bool { + workqueue := c.lbResourceCache.GetQueue() + key, quit := workqueue.Get() + if quit { + return false + } + + err := c.syncToDatastore(key.(string)) + c.handleErr(err, key.(string)) + + workqueue.Done(key) + return true +} + +// syncToDatastore processes the next item in queue +func (c *loadBalancerController) syncToDatastore(key string) error { + obj, exists := c.lbResourceCache.Get(key) + + if !exists { + // Service does not exist release the IPs + return c.releaseIP(key) + } else { + svc := obj.(*v1.Service) + // We don't have IP, assign one + if svc.Status.LoadBalancer.Ingress == nil { + return c.assignIP(svc) + } else { + // Service was updated + return c.checkStatus(svc) + } + } +} + +func (c *loadBalancerController) handleErr(err error, key string) { + workqueue := c.lbResourceCache.GetQueue() + if err == nil { + // Forget about the #AddRateLimited history of the key on every successful synchronization. + // This ensures that future processing of updates for this key is not delayed because of + // an outdated error history. + workqueue.Forget(key) + return + } + + // This controller retries 5 times if something goes wrong. After that, it stops trying. + if workqueue.NumRequeues(key) < 5 { + // Re-enqueue the key rate limited. Based on the rate limiter on the + // queue and the re-enqueue history, the key will be processed later again. + log.WithError(err).Errorf("Error syncing Service LoadBalancer %v: %v", key, err) + workqueue.AddRateLimited(key) + return + } + + workqueue.Forget(key) + + // Report to an external entity that, even after several retries, we could not successfully process this key + uruntime.HandleError(err) + log.WithError(err).Errorf("Dropping Service LoadBalancer %q out of the queue: %v", key, err) +} + +// assignIP tries to assign IP address for Service. +func (c *loadBalancerController) assignIP(svc *v1.Service) error { + if len(c.ipPools) == 0 { + return nil + } + handle := createHandle(svc) + + metadataAttrs := map[string]string{ + ipam.AttributeService: svc.Name, + ipam.AttributeNamespace: svc.Namespace, + ipam.AttributeType: string(svc.Spec.Type), + ipam.AttributeTimestamp: time.Now().UTC().String(), + } + + // User requested specific IP, attempt to allocate + if svc.Annotations[annotationLoadBalancerIp] != "" { + ipAddrs, err := validateAnnotation(parseAnnotation(svc.Annotations[annotationLoadBalancerIp])) + if err != nil { + return err + } + + ingress := svc.Status.LoadBalancer.Ingress + + for _, addrs := range ipAddrs { + for _, lbingress := range ingress { + // We must be trying to assign missing address due to an error, + // skip this assignment as it's already assigned and move onto the next one + if lbingress.IP == addrs.String() { + continue + } + } + + ipamArgs := ipam.AssignIPArgs{ + IP: addrs, + Hostname: api.VirtualLoadBalancer, + HandleID: &handle, + Attrs: metadataAttrs, + } + + err = c.calicoClient.IPAM().AssignIP(c.ctx, ipamArgs) + if err != nil { + log.WithField("ip", addrs).WithError(err).Warn("failed to assign ip to node") + return err + } + + ingress = append(ingress, v1.LoadBalancerIngress{IP: addrs.String()}) + } + + svc.Status.LoadBalancer.Ingress = ingress + _, err = c.clientSet.CoreV1().Services(svc.Namespace).UpdateStatus(c.ctx, svc, metav1.UpdateOptions{}) + return err + } + + // Build AssignArgs based on Service IP family attr + num4 := 0 + num6 := 0 + + for _, ipFamily := range svc.Spec.IPFamilies { + if ipFamily == v1.IPv4Protocol { + num4++ + } + if ipFamily == v1.IPv6Protocol { + num6++ + } + } + + // Check if IP from ipfamily is already assigned, skip it as we're trying to assign only the missing one. + // This can happen when error happened during the initial assignment, and now we're trying to assign ip again from the syncIPAM func + ingressList := svc.Status.LoadBalancer.Ingress + if ingressList != nil { + for _, ingress := range ingressList { + if ip := cnet.ParseIP(ingress.IP); ip != nil { + if ip.To4() != nil { + num4 = 0 + } + if ip.To16() != nil { + num6 = 0 + } + } + } + + if num4 == 0 && num6 == 0 { + return errors.New("No new IPs to assign, Service already has ipv4 and ipv6 address") + } + } + + args := ipam.AutoAssignArgs{ + Num4: num4, + Num6: num6, + IntendedUse: api.IPPoolAllowedUseLoadBalancer, + Hostname: api.VirtualLoadBalancer, + HandleID: &handle, + Attrs: metadataAttrs, + } + + if svc.Annotations[annotationIpv4Pools] != "" { + args.IPv4Pools, _ = c.resolvePools(parseAnnotation(svc.Annotations[annotationIpv4Pools]), true) + if args.IPv4Pools == nil { + return errors.New("no IPv4 pools found from annotation") + } + } + + if svc.Annotations[annotationIpv6Pools] != "" { + args.IPv6Pools, _ = c.resolvePools(parseAnnotation(svc.Annotations[annotationIpv6Pools]), false) + if args.IPv6Pools == nil { + return errors.New("no IPv6 pools found from annotation") + } + } + + v4Assignments, v6assignments, err := c.calicoClient.IPAM().AutoAssign(c.ctx, args) + if err != nil { + log.WithField("svc", svc.Name).WithError(err).Warn("error on assigning IPAM to service") + } + + var ingress []v1.LoadBalancerIngress + + if v4Assignments != nil { + for _, assignment := range v4Assignments.IPs { + ingress = append(ingress, v1.LoadBalancerIngress{ + IP: assignment.IP.String(), + }) + } + } + + if v6assignments != nil { + for _, assignment := range v6assignments.IPs { + ingress = append(ingress, v1.LoadBalancerIngress{ + IP: assignment.IP.String(), + }) + } + } + + svc.Status.LoadBalancer.Ingress = ingress + + _, err = c.clientSet.CoreV1().Services(svc.Namespace).UpdateStatus(c.ctx, svc, metav1.UpdateOptions{}) + if err != nil { + log.Infof("LoadBalancer Error updating service %s/%s: %v", svc.Namespace, svc.Name, err) + return err + } + return nil +} + +// releaseIP tries to release all IPs allocated with the Service unique handle +func (c *loadBalancerController) releaseIP(handle string) error { + log.Info("Service type LoadBalancer deleted, releasing assigned IP address") + + err := c.calicoClient.IPAM().ReleaseByHandle(c.ctx, handle) + if err != nil { + log.Errorf("error on removing assigned IP for handle %s", handle) + return err + } + return nil +} + +// checkStatus determines what has changed about the Service. Service can change it's type, if that happens +// we want to release the IP previously used. If calico annotations have changed we try to assign new IP address to the Service +func (c *loadBalancerController) checkStatus(svc *v1.Service) error { + // Service type has changed, release the ip by the handle + if svc.Spec.Type != v1.ServiceTypeLoadBalancer { + return c.releaseIP(createHandle(svc)) + } else { + // Type is load balancer, this means calico annotations have changed, and we need to re-assign IPs from this service + err := c.releaseIP(createHandle(svc)) + if err != nil { + return err + } + // We can only assign new IP if we are still managing the address assignment + if isCalicoManagedLoadBalancer(svc, c.cfg.AssignIPs) { + return c.assignIP(svc) + } else { + // No longer managed by Calico, clean up our assigned IP from Service + svc.Status.LoadBalancer.Ingress = []v1.LoadBalancerIngress{} + + _, err = c.clientSet.CoreV1().Services(svc.Namespace).UpdateStatus(c.ctx, svc, metav1.UpdateOptions{}) + if err != nil { + return err + } + } + return nil + } +} + +// resolvePools valid IPpool range when specific pool is requested by the user +func (c *loadBalancerController) resolvePools(pools []string, isv4 bool) ([]cnet.IPNet, error) { + // Iterate through the provided pools. If it parses as a CIDR, just use that. + // If it does not parse as a CIDR, then attempt to lookup an IP pool with a matching name. + result := []cnet.IPNet{} + for _, p := range pools { + _, cidr, err := net.ParseCIDR(p) + if err != nil { + // Didn't parse as a CIDR - check if it's the name + // of a configured IP pool. + for _, ipp := range c.ipPools { + if ipp.Name == p { + // Found a match. Use the CIDR from the matching pool. + _, cidr, err = net.ParseCIDR(ipp.Spec.CIDR) + if err != nil { + return nil, fmt.Errorf("failed to parse IP pool cidr: %s", err) + } + log.Infof("Resolved pool name %s to cidr %s", ipp.Name, cidr) + } + } + + if cidr == nil { + // Unable to resolve this pool to a CIDR - return an error. + return nil, fmt.Errorf("error parsing pool %q: %s", p, err) + } + } + + ip := cidr.IP + if isv4 && ip.To4() == nil { + return nil, fmt.Errorf("%q isn't a IPv4 address", ip) + } + if !isv4 && ip.To4() != nil { + return nil, fmt.Errorf("%q isn't a IPv6 address", ip) + } + result = append(result, cnet.IPNet{IPNet: *cidr}) + } + return result, nil +} + +// isCalicoManagedLoadBalancer returns if Calico should try to assign IP address for the LoadBalancer +// We assign IPs only if the loadBalancer controller assignIP is set to AllService +// or in RequestedOnlyServices if the service has calico annotation +func isCalicoManagedLoadBalancer(svc *v1.Service, assignIPs string) bool { + if svc.Spec.Type == v1.ServiceTypeLoadBalancer { + if assignIPs == api.AllServices { + return true + } + + if assignIPs == api.RequestedServicesOnly { + if svc.Annotations[annotationIpv4Pools] != "" || + svc.Annotations[annotationIpv6Pools] != "" || + svc.Annotations[annotationLoadBalancerIp] != "" { + return true + } + } + } + return false +} + +// validateAnnotation checks if the ips specified in the calico annotation are valid. +// Each service can have at most one ipv4 and one ipv6 address +func validateAnnotation(ipAddrs []string) ([]cnet.IP, error) { + parsedIPs := make([]cnet.IP, 0) + ipv4 := 0 + ipv6 := 0 + for _, ipAddr := range ipAddrs { + curr := cnet.ParseIP(ipAddr) + if curr == nil { + return nil, errors.New(fmt.Sprintf("Could not parse %s as a valid IP address", curr)) + } + if curr.To4() != nil { + ipv4++ + } else if curr.To16() != nil { + ipv6++ + } + parsedIPs = append(parsedIPs, *curr) + } + + if ipv6 > 1 || ipv4 > 1 { + return nil, errors.New(fmt.Sprintf("At max only one ipv4 and one ipv6 address can be specified. Recieved %d ipv4 and %d ipv6 addresses", ipv4, ipv6)) + } + return parsedIPs, nil +} + +func parseAnnotation(str string) []string { + str = strings.TrimPrefix(str, "[") + str = strings.TrimSuffix(str, "]") + return strings.Split(str, ",") +} + +// createHandle Returns a handle to use for IP allocation for the service +func createHandle(svc *v1.Service) string { + handle := strings.ToLower(fmt.Sprintf("lb.%s.%s", svc.Name, svc.UID)) + if len(handle) > k8svalidation.DNS1123SubdomainMaxLength { + return handle[:k8svalidation.DNS1123SubdomainMaxLength] + } + return handle +} + +func kick(c chan<- interface{}) { + select { + case c <- nil: + // pass + default: + // pass + } +} diff --git a/kube-controllers/pkg/controllers/loadbalancer/loadbalancer_controller_fv_test.go b/kube-controllers/pkg/controllers/loadbalancer/loadbalancer_controller_fv_test.go new file mode 100644 index 00000000000..b3a1f9424b0 --- /dev/null +++ b/kube-controllers/pkg/controllers/loadbalancer/loadbalancer_controller_fv_test.go @@ -0,0 +1,274 @@ +// Copyright (c) 2024 Tigera, Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package loadbalancer_test + +import ( + "context" + "os" + "time" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + v3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" + "github.com/projectcalico/calico/felix/fv/containers" + "github.com/projectcalico/calico/kube-controllers/tests/testutils" + "github.com/projectcalico/calico/libcalico-go/lib/apiconfig" + client "github.com/projectcalico/calico/libcalico-go/lib/clientv3" + "github.com/projectcalico/calico/libcalico-go/lib/json" + "github.com/projectcalico/calico/libcalico-go/lib/options" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/client-go/kubernetes" +) + +var _ = Describe("Calico loadbalancer controller FV tests (etcd mode)", func() { + var ( + etcd *containers.Container + loadbalancercontroller *containers.Container + apiserver *containers.Container + calicoClient client.Interface + k8sClient *kubernetes.Clientset + controllerManager *containers.Container + ) + + const testNamespace = "test-loadbalancer-ns" + + basicIpPool := v3.IPPool{ + ObjectMeta: metav1.ObjectMeta{ + Name: "loadbalancer-ippool", + }, + Spec: v3.IPPoolSpec{ + CIDR: "1.2.3.0/24", + BlockSize: 26, + NodeSelector: "all()", + AllowedUses: []v3.IPPoolAllowedUse{v3.IPPoolAllowedUseLoadBalancer}, + }, + } + + specificIpPool := v3.IPPool{ + ObjectMeta: metav1.ObjectMeta{ + Name: "loadbalancer-ippool-specific", + }, + Spec: v3.IPPoolSpec{ + CIDR: "4.4.4.4/32", + BlockSize: 32, + NodeSelector: "all()", + AllowedUses: []v3.IPPoolAllowedUse{v3.IPPoolAllowedUseLoadBalancer}, + AssignmentMode: v3.Manual, + }, + } + + basicService := v1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: "basic-service", + }, + Spec: v1.ServiceSpec{ + Type: v1.ServiceTypeLoadBalancer, + Ports: []v1.ServicePort{ + { + Port: 8787, + Name: "default", + TargetPort: intstr.FromInt32(9797), + }, + }, + Selector: map[string]string{"app": "my-app"}, + }, + } + + serviceIpv4PoolSpecified := v1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: "basic-ipv4-pool-specified", + Annotations: map[string]string{ + "projectcalico.org/ipv4pools": specificIpPool.Name, + }, + }, + Spec: v1.ServiceSpec{ + Type: v1.ServiceTypeLoadBalancer, + Ports: []v1.ServicePort{ + { + Port: 8787, + Name: "default", + TargetPort: intstr.FromInt32(9797), + }, + }, + Selector: map[string]string{"app": "my-app"}, + }, + } + + serviceIpAddressSpecified := v1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: "basic-ip-address-specified", + Annotations: map[string]string{ + "projectcalico.org/loadBalancerIPs": "1.2.3.100", + }, + }, + Spec: v1.ServiceSpec{ + Type: v1.ServiceTypeLoadBalancer, + Ports: []v1.ServicePort{ + { + Port: 8787, + Name: "default", + TargetPort: intstr.FromInt32(9797), + }, + }, + Selector: map[string]string{"app": "my-app"}, + }, + } + + BeforeEach(func() { + // Run etcd. + etcd = testutils.RunEtcd() + calicoClient = testutils.GetCalicoClient(apiconfig.EtcdV3, etcd.IP, "") + + // Run apiserver. + apiserver = testutils.RunK8sApiserver(etcd.IP) + + // Write out a kubeconfig file + kconfigfile, err := os.CreateTemp("", "ginkgo-loadbalancercontroller") + Expect(err).NotTo(HaveOccurred()) + defer os.Remove(kconfigfile.Name()) + data := testutils.BuildKubeconfig(apiserver.IP) + _, err = kconfigfile.Write([]byte(data)) + Expect(err).NotTo(HaveOccurred()) + + // Make the kubeconfig readable by the container. + Expect(kconfigfile.Chmod(os.ModePerm)).NotTo(HaveOccurred()) + + loadbalancercontroller = testutils.RunLoadBalancerController(apiconfig.EtcdV3, etcd.IP, kconfigfile.Name(), "") + + k8sClient, err = testutils.GetK8sClient(kconfigfile.Name()) + Expect(err).NotTo(HaveOccurred()) + + // Wait for the apiserver to be available. + Eventually(func() error { + _, err := k8sClient.CoreV1().Services(testNamespace).List(context.Background(), metav1.ListOptions{}) + return err + }, 30*time.Second, 1*time.Second).Should(BeNil()) + + // Run controller manager. Empirically it can take around 10s until the + // controller manager is ready to create default service accounts, even + // when the k8s image has already been downloaded to run the API + // server. We use Eventually to allow for possible delay when doing + // initial pod creation below. + controllerManager = testutils.RunK8sControllerManager(apiserver.IP) + }) + + AfterEach(func() { + controllerManager.Stop() + loadbalancercontroller.Stop() + apiserver.Stop() + etcd.Stop() + }) + + Context("Service LoadBalancer FV tests - LoadBalancer AllServices mode", func() { + BeforeEach(func() { + nsName := testNamespace + ns := &v1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: nsName, + }, + Spec: v1.NamespaceSpec{}, + } + _, err := k8sClient.CoreV1().Namespaces().Create(context.Background(), ns, metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + + _, err = calicoClient.IPPools().Create(context.Background(), &basicIpPool, options.SetOptions{}) + Expect(err).NotTo(HaveOccurred()) + + _, err = calicoClient.IPPools().Create(context.Background(), &specificIpPool, options.SetOptions{}) + Expect(err).NotTo(HaveOccurred()) + }) + + It("Should assign IP from available IP pool", func() { + _, err := k8sClient.CoreV1().Services(testNamespace).Create(context.Background(), &basicService, metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + + Eventually(func() []v1.LoadBalancerIngress { + service, _ := k8sClient.CoreV1().Services(testNamespace).Get(context.Background(), basicService.Name, metav1.GetOptions{}) + return service.Status.LoadBalancer.Ingress + }, time.Second*15, 500*time.Millisecond).ShouldNot(BeEmpty()) + }) + + It("Should assign IP from specified IP pool", func() { + _, err := k8sClient.CoreV1().Services(testNamespace).Create(context.Background(), &serviceIpv4PoolSpecified, metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + + Eventually(func() []v1.LoadBalancerIngress { + service, _ := k8sClient.CoreV1().Services(testNamespace).Get(context.Background(), serviceIpv4PoolSpecified.Name, metav1.GetOptions{}) + return service.Status.LoadBalancer.Ingress + }, time.Second*15, 500*time.Millisecond).Should(Not(BeEmpty())) + + service, _ := k8sClient.CoreV1().Services(testNamespace).Get(context.Background(), serviceIpv4PoolSpecified.Name, metav1.GetOptions{}) + Expect(service.Status.LoadBalancer.Ingress[0].IP).Should(Equal("4.4.4.4")) + }) + + It("Should assign specific IP address", func() { + _, err := k8sClient.CoreV1().Services(testNamespace).Create(context.Background(), &serviceIpAddressSpecified, metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + + Eventually(func() []v1.LoadBalancerIngress { + service, _ := k8sClient.CoreV1().Services(testNamespace).Get(context.Background(), serviceIpAddressSpecified.Name, metav1.GetOptions{}) + return service.Status.LoadBalancer.Ingress + }, time.Second*15, 500*time.Millisecond).Should(Not(BeEmpty())) + + service, _ := k8sClient.CoreV1().Services(testNamespace).Get(context.Background(), serviceIpAddressSpecified.Name, metav1.GetOptions{}) + Expect(service.Status.LoadBalancer.Ingress[0].IP).Should(Equal("1.2.3.100")) + }) + + It("Should remove IP assignment when Service type is changed from LoadBalancer to NodePort", func() { + _, err := k8sClient.CoreV1().Services(testNamespace).Create(context.Background(), &serviceIpv4PoolSpecified, metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + + Eventually(func() []v1.LoadBalancerIngress { + service, _ := k8sClient.CoreV1().Services(testNamespace).Get(context.Background(), serviceIpv4PoolSpecified.Name, metav1.GetOptions{}) + return service.Status.LoadBalancer.Ingress + }, time.Second*15, 500*time.Millisecond).Should(Not(BeEmpty())) + + service, err := k8sClient.CoreV1().Services(testNamespace).Get(context.Background(), serviceIpv4PoolSpecified.Name, metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + Expect(service.Status.LoadBalancer.Ingress[0].IP).Should(Equal("4.4.4.4")) + + svcPatch := map[string]interface{}{} + spec := map[string]interface{}{} + svcPatch["spec"] = spec + spec["type"] = v1.ServiceTypeNodePort + patch, err := json.Marshal(svcPatch) + Expect(err).NotTo(HaveOccurred()) + + service, err = k8sClient.CoreV1().Services(testNamespace).Patch(context.Background(), service.Name, types.StrategicMergePatchType, patch, metav1.PatchOptions{}) + Expect(err).NotTo(HaveOccurred()) + Expect(service.Spec.Type).To(Equal(v1.ServiceTypeNodePort)) + Expect(service.Status.LoadBalancer.Ingress).Should(BeEmpty()) + }) + + It("Should not assign IP if there is no LoadBalancer IP pool", func() { + _, err := calicoClient.IPPools().Delete(context.Background(), specificIpPool.Name, options.DeleteOptions{}) + Expect(err).NotTo(HaveOccurred()) + + _, err = calicoClient.IPPools().Delete(context.Background(), basicIpPool.Name, options.DeleteOptions{}) + Expect(err).NotTo(HaveOccurred()) + + _, err = k8sClient.CoreV1().Services(testNamespace).Create(context.Background(), &basicService, metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + + Eventually(func() []v1.LoadBalancerIngress { + service, _ := k8sClient.CoreV1().Services(testNamespace).Get(context.Background(), basicService.Name, metav1.GetOptions{}) + return service.Status.LoadBalancer.Ingress + }, time.Second*15, 500*time.Millisecond).Should(BeEmpty()) + }) + }) +}) diff --git a/kube-controllers/pkg/controllers/loadbalancer/loadbalancer_suite_test.go b/kube-controllers/pkg/controllers/loadbalancer/loadbalancer_suite_test.go new file mode 100644 index 00000000000..f284ab387b0 --- /dev/null +++ b/kube-controllers/pkg/controllers/loadbalancer/loadbalancer_suite_test.go @@ -0,0 +1,38 @@ +// Copyright (c) 2024 Tigera, Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package loadbalancer + +import ( + "testing" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + "github.com/sirupsen/logrus" + + "github.com/projectcalico/calico/libcalico-go/lib/testutils" + + "github.com/onsi/ginkgo/reporters" +) + +func init() { + testutils.HookLogrusForGinkgo() + logrus.SetLevel(logrus.DebugLevel) +} + +func Test(t *testing.T) { + RegisterFailHandler(Fail) + junitReporter := reporters.NewJUnitReporter("../../report/loadbalancer_controller_suite.xml") + RunSpecsWithDefaultAndCustomReporters(t, "LoadBalancer controller suite", []Reporter{junitReporter}) +} diff --git a/kube-controllers/tests/testutils/loadbalancer_controller_utils.go b/kube-controllers/tests/testutils/loadbalancer_controller_utils.go new file mode 100644 index 00000000000..823af95ac41 --- /dev/null +++ b/kube-controllers/tests/testutils/loadbalancer_controller_utils.go @@ -0,0 +1,43 @@ +// Copyright (c) 2017 Tigera, Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// The utils in this file are specific to the policy controller, +// and are not expected to be shared across projects. + +package testutils + +import ( + "fmt" + "os" + + "github.com/projectcalico/calico/felix/fv/containers" + "github.com/projectcalico/calico/libcalico-go/lib/apiconfig" +) + +func RunLoadBalancerController(datastoreType apiconfig.DatastoreType, etcdIP, kconfigfile, ctrls string) *containers.Container { + if ctrls == "" { + // Default to all controllers. + ctrls = "workloadendpoint,namespace,policy,node,serviceaccount,loadbalancer" + } + return containers.Run("calico-kube-controllers", + containers.RunOpts{AutoRemove: true}, + "-e", fmt.Sprintf("ETCD_ENDPOINTS=http://%s:2379", etcdIP), + "-e", fmt.Sprintf("DATASTORE_TYPE=%s", datastoreType), + "-e", fmt.Sprintf("ENABLED_CONTROLLERS=%s", ctrls), + "-e", fmt.Sprintf("KUBECONFIG=%s", kconfigfile), + "-e", "LOG_LEVEL=debug", + "-e", "RECONCILER_PERIOD=10s", + "-v", fmt.Sprintf("%s:%s", kconfigfile, kconfigfile), + os.Getenv("CONTAINER_NAME")) +} diff --git a/libcalico-go/config/crd/crd.projectcalico.org_ippools.yaml b/libcalico-go/config/crd/crd.projectcalico.org_ippools.yaml index 83311f963ad..e1258e63bf7 100644 --- a/libcalico-go/config/crd/crd.projectcalico.org_ippools.yaml +++ b/libcalico-go/config/crd/crd.projectcalico.org_ippools.yaml @@ -37,6 +37,10 @@ spec: items: type: string type: array + assignmentMode: + description: Determines the mode how IP addresses should be assigned + from this pool + type: string blockSize: description: The block size to use for IP address assignments from this pool. Defaults to 26 for IPv4 and 122 for IPv6. diff --git a/libcalico-go/config/crd/crd.projectcalico.org_kubecontrollersconfigurations.yaml b/libcalico-go/config/crd/crd.projectcalico.org_kubecontrollersconfigurations.yaml index 504de3e39f5..9a3c2252436 100644 --- a/libcalico-go/config/crd/crd.projectcalico.org_kubecontrollersconfigurations.yaml +++ b/libcalico-go/config/crd/crd.projectcalico.org_kubecontrollersconfigurations.yaml @@ -36,6 +36,13 @@ spec: description: Controllers enables and configures individual Kubernetes controllers properties: + loadBalancer: + description: LoadBalancer enables and configures the LoadBalancer + controller. Enabled by default, set to nil to disable. + properties: + assignIPs: + type: string + type: object namespace: description: Namespace enables and configures the namespace controller. Enabled by default, set to nil to disable. @@ -145,6 +152,13 @@ spec: description: Controllers enables and configures individual Kubernetes controllers properties: + loadBalancer: + description: LoadBalancer enables and configures the LoadBalancer + controller. Enabled by default, set to nil to disable. + properties: + assignIPs: + type: string + type: object namespace: description: Namespace enables and configures the namespace controller. Enabled by default, set to nil to disable. diff --git a/libcalico-go/lib/apis/v1/ippool.go b/libcalico-go/lib/apis/v1/ippool.go index 7365bac4547..37cc7ff782b 100644 --- a/libcalico-go/lib/apis/v1/ippool.go +++ b/libcalico-go/lib/apis/v1/ippool.go @@ -17,6 +17,8 @@ package v1 import ( "fmt" + v3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" + "github.com/projectcalico/calico/libcalico-go/lib/apis/v1/unversioned" "github.com/projectcalico/calico/libcalico-go/lib/backend/encap" "github.com/projectcalico/calico/libcalico-go/lib/net" @@ -63,6 +65,9 @@ type IPPoolSpec struct { // When disabled is true, Calico IPAM will not assign addresses from this pool. Disabled bool `json:"disabled,omitempty"` + + // Enables/Disables automatic IP address assignment from this pool for pods and service.LoadBalancer + AssignmentMode v3.AssignmentMode `json:"assignmentMode,omitempty"` } type IPIPConfiguration struct { diff --git a/libcalico-go/lib/apis/v1/openapi_generated.go b/libcalico-go/lib/apis/v1/openapi_generated.go index 4b0530f8aca..593779e28d3 100644 --- a/libcalico-go/lib/apis/v1/openapi_generated.go +++ b/libcalico-go/lib/apis/v1/openapi_generated.go @@ -929,6 +929,13 @@ func schema_libcalico_go_lib_apis_v1_IPPoolSpec(ref common.ReferenceCallback) co Format: "", }, }, + "assignmentMode": { + SchemaProps: spec.SchemaProps{ + Description: "Enables/Disables automatic IP address assignment from this pool for pods and service.LoadBalancer", + Type: []string{"string"}, + Format: "", + }, + }, }, }, }, diff --git a/libcalico-go/lib/apis/v3/openapi_generated.go b/libcalico-go/lib/apis/v3/openapi_generated.go index b4d15e457a9..053278aa960 100644 --- a/libcalico-go/lib/apis/v3/openapi_generated.go +++ b/libcalico-go/lib/apis/v3/openapi_generated.go @@ -955,6 +955,13 @@ func schema_libcalico_go_lib_apis_v1_IPPoolSpec(ref common.ReferenceCallback) co Format: "", }, }, + "assignmentMode": { + SchemaProps: spec.SchemaProps{ + Description: "Enables/Disables automatic IP address assignment from this pool for pods and service.LoadBalancer", + Type: []string{"string"}, + Format: "", + }, + }, }, }, }, diff --git a/libcalico-go/lib/backend/k8s/resources/ippool.go b/libcalico-go/lib/backend/k8s/resources/ippool.go index 6d9612143c3..232eb2240c6 100644 --- a/libcalico-go/lib/backend/k8s/resources/ippool.go +++ b/libcalico-go/lib/backend/k8s/resources/ippool.go @@ -1,4 +1,4 @@ -// Copyright (c) 2016-2017 Tigera, Inc. All rights reserved. +// Copyright (c) 2016-2024 Tigera, Inc. All rights reserved. // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -131,6 +131,7 @@ func IPPoolV3ToV1(kvp *model.KVPair) (*model.KVPair, error) { IPAM: !v3res.Spec.Disabled, Disabled: v3res.Spec.Disabled, DisableBGPExport: v3res.Spec.DisableBGPExport, + AssignmentMode: v3res.Spec.AssignmentMode, }, Revision: kvp.Revision, }, nil diff --git a/libcalico-go/lib/backend/model/block.go b/libcalico-go/lib/backend/model/block.go index e06b46d59f6..2d185961153 100644 --- a/libcalico-go/lib/backend/model/block.go +++ b/libcalico-go/lib/backend/model/block.go @@ -1,4 +1,4 @@ -// Copyright (c) 2016-2021 Tigera, Inc. All rights reserved. +// Copyright (c) 2016-2024 Tigera, Inc. All rights reserved. // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -33,6 +33,7 @@ const ( IPAMBlockAttributeNamespace = "namespace" IPAMBlockAttributeNode = "node" IPAMBlockAttributeType = "type" + IPAMBLockAttributeService = "service" IPAMBlockAttributeTypeIPIP = "ipipTunnelAddress" IPAMBlockAttributeTypeVXLAN = "vxlanTunnelAddress" IPAMBlockAttributeTypeVXLANV6 = "vxlanV6TunnelAddress" diff --git a/libcalico-go/lib/backend/model/ippool.go b/libcalico-go/lib/backend/model/ippool.go index 15d5c75d167..3b19320cf32 100644 --- a/libcalico-go/lib/backend/model/ippool.go +++ b/libcalico-go/lib/backend/model/ippool.go @@ -1,4 +1,4 @@ -// Copyright (c) 2016,2021 Tigera, Inc. All rights reserved. +// Copyright (c) 2016-2024 Tigera, Inc. All rights reserved. // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -20,6 +20,7 @@ import ( "regexp" "strings" + v3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" log "github.com/sirupsen/logrus" "github.com/projectcalico/calico/libcalico-go/lib/backend/encap" @@ -96,12 +97,13 @@ func (options IPPoolListOptions) KeyFromDefaultPath(path string) Key { } type IPPool struct { - CIDR net.IPNet `json:"cidr"` - IPIPInterface string `json:"ipip"` - IPIPMode encap.Mode `json:"ipip_mode"` - VXLANMode encap.Mode `json:"vxlan_mode"` - Masquerade bool `json:"masquerade"` - IPAM bool `json:"ipam"` - Disabled bool `json:"disabled"` - DisableBGPExport bool `json:"disableBGPExport"` + CIDR net.IPNet `json:"cidr"` + IPIPInterface string `json:"ipip"` + IPIPMode encap.Mode `json:"ipip_mode"` + VXLANMode encap.Mode `json:"vxlan_mode"` + Masquerade bool `json:"masquerade"` + IPAM bool `json:"ipam"` + Disabled bool `json:"disabled"` + DisableBGPExport bool `json:"disableBGPExport"` + AssignmentMode v3.AssignmentMode `json:"assignment_mode"` } diff --git a/libcalico-go/lib/clientv3/ippool.go b/libcalico-go/lib/clientv3/ippool.go index 6bd1572e467..f50d6ad4b8e 100644 --- a/libcalico-go/lib/clientv3/ippool.go +++ b/libcalico-go/lib/clientv3/ippool.go @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2021 Tigera, Inc. All rights reserved. +// Copyright (c) 2017-2024 Tigera, Inc. All rights reserved. // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -283,6 +283,10 @@ func convertIpPoolFromStorage(pool *apiv3.IPPool) error { pool.Spec.VXLANMode = apiv3.VXLANModeNever } + if pool.Spec.AssignmentMode == "" { + pool.Spec.AssignmentMode = apiv3.Automatic + } + return nil } @@ -353,6 +357,29 @@ func (r ipPools) validateAndSetDefaults(ctx context.Context, new, old *apiv3.IPP new.Spec.AllowedUses = []apiv3.IPPoolAllowedUse{apiv3.IPPoolAllowedUseWorkload, apiv3.IPPoolAllowedUseTunnel} } + // Update, check if previously AllowedUses was LoadBalancer it has not changed + if old != nil { + oldLoadBalancer := false + for _, u := range old.Spec.AllowedUses { + if u == apiv3.IPPoolAllowedUseLoadBalancer { + oldLoadBalancer = true + } + } + for _, u := range new.Spec.AllowedUses { + if u != apiv3.IPPoolAllowedUseLoadBalancer && oldLoadBalancer { + return cerrors.ErrorValidation{ErroredFields: []cerrors.ErroredField{{ + Name: "IPPool.Spec.AllowedUses", + Reason: "IPPool allowed use cannot be changed from LoadBalacer to Workload or Tunnel", + Value: new.Spec.AllowedUses, + }}} + } + } + } + + if new.Spec.AssignmentMode == "" { + new.Spec.AssignmentMode = apiv3.Automatic + } + // Check that the blockSize hasn't changed since updates are not supported. if old != nil && old.Spec.BlockSize != new.Spec.BlockSize { errFields = append(errFields, cerrors.ErroredField{ diff --git a/libcalico-go/lib/clientv3/ippool_e2e_test.go b/libcalico-go/lib/clientv3/ippool_e2e_test.go index d2e3640fdf5..77270795a80 100644 --- a/libcalico-go/lib/clientv3/ippool_e2e_test.go +++ b/libcalico-go/lib/clientv3/ippool_e2e_test.go @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2021 Tigera, Inc. All rights reserved. +// Copyright (c) 2017-2024 Tigera, Inc. All rights reserved. // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -100,6 +100,15 @@ var _ = testutils.E2eDatastoreDescribe("IPPool tests", testutils.DatastoreAll, f AllowedUses: []apiv3.IPPoolAllowedUse{apiv3.IPPoolAllowedUseWorkload, apiv3.IPPoolAllowedUseTunnel}, } + //spec4 := apiv3.IPPoolSpec{ + // CIDR: "1.2.3.0/4", + // IPIPMode: apiv3.IPIPModeNever, + // VXLANMode: apiv3.VXLANModeNever, + // BlockSize: 26, + // NodeSelector: "all()", + // AllowedUses: []apiv3.IPPoolAllowedUse{apiv3.IPPoolAllowedUseLoadBalancer}, + //} + It("should error when creating an IPPool with no name", func() { c, err := clientv3.New(config) Expect(err).NotTo(HaveOccurred()) diff --git a/libcalico-go/lib/clientv3/kubecontrollersconfig.go b/libcalico-go/lib/clientv3/kubecontrollersconfig.go index 972af86e3bd..09067dbd29d 100644 --- a/libcalico-go/lib/clientv3/kubecontrollersconfig.go +++ b/libcalico-go/lib/clientv3/kubecontrollersconfig.go @@ -1,4 +1,4 @@ -// Copyright (c) 2020 Tigera, Inc. All rights reserved. +// Copyright (c) 2024 Tigera, Inc. All rights reserved. // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -54,6 +54,12 @@ func (r kubeControllersConfiguration) fillDefaults(res *apiv3.KubeControllersCon res.Spec.Controllers.Node.LeakGracePeriod = &metav1.Duration{Duration: 15 * time.Minute} } } + + if res.Spec.Controllers.LoadBalancer != nil { + if res.Spec.Controllers.LoadBalancer.AssignIPs == "" { + res.Spec.Controllers.LoadBalancer.AssignIPs = apiv3.AllServices + } + } } // Create takes the representation of a KubeControllersConfiguration and creates it. diff --git a/libcalico-go/lib/ipam/ipam.go b/libcalico-go/lib/ipam/ipam.go index c08cbbda8a3..3225e1f6ec5 100644 --- a/libcalico-go/lib/ipam/ipam.go +++ b/libcalico-go/lib/ipam/ipam.go @@ -1,4 +1,4 @@ -// Copyright (c) 2016-2021 Tigera, Inc. All rights reserved. +// Copyright (c) 2016-2024 Tigera, Inc. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -23,11 +23,10 @@ import ( "strings" "time" + v3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" log "github.com/sirupsen/logrus" "golang.org/x/sync/semaphore" - v3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" - libapiv3 "github.com/projectcalico/calico/libcalico-go/lib/apis/v3" bapi "github.com/projectcalico/calico/libcalico-go/lib/backend/api" "github.com/projectcalico/calico/libcalico-go/lib/backend/model" @@ -51,6 +50,7 @@ const ( AttributeNode = model.IPAMBlockAttributeNode AttributeTimestamp = model.IPAMBlockAttributeTimestamp AttributeType = model.IPAMBlockAttributeType + AttributeService = model.IPAMBLockAttributeService AttributeTypeIPIP = model.IPAMBlockAttributeTypeIPIP AttributeTypeVXLAN = model.IPAMBlockAttributeTypeVXLAN AttributeTypeVXLANV6 = model.IPAMBlockAttributeTypeVXLANV6 @@ -257,6 +257,19 @@ func (c ipamClient) determinePools(ctx context.Context, requestedPoolNets []net. log.Debugf("enabled pools: %v", enabledPools) log.Debugf("requested pools: %v", requestedPoolNets) + // No pools requested, we should only pick from pools that have assignmentMode: Automatic + if requestedPoolNets == nil { + log.Debug("No pools were specified, using pools with AssignmentMode: Automatic") + automaticAssignPools := []v3.IPPool{} + for _, pool := range enabledPools { + if pool.Spec.AssignmentMode == v3.Automatic { + automaticAssignPools = append(automaticAssignPools, pool) + } + } + enabledPools = automaticAssignPools + log.Debugf("automatic pools: %v", enabledPools) + } + // Build a map so we can lookup existing pools. pm := map[string]v3.IPPool{} @@ -330,16 +343,27 @@ func (c ipamClient) determinePools(ctx context.Context, requestedPoolNets []net. // selects this node. It returns matching pools, list of host-affine blocks and any error encountered. func (c ipamClient) prepareAffinityBlocksForHost(ctx context.Context, requestedPools []net.IPNet, version int, host string, rsvdAttr *HostReservedAttr, use v3.IPPoolAllowedUse) ([]v3.IPPool, []net.IPNet, error) { // Retrieve node for given hostname to use for ip pool node selection - node, err := c.client.Get(ctx, model.ResourceKey{Kind: libapiv3.KindNode, Name: host}, "") - if err != nil { - log.WithError(err).WithField("node", host).Error("failed to get node for host") - return nil, nil, err - } + var node *model.KVPair + var err error + var v3n *libapiv3.Node + var ok bool + // Regular node, continue as usual { + if host != v3.VirtualLoadBalancer { + node, err = c.client.Get(ctx, model.ResourceKey{Kind: libapiv3.KindNode, Name: host}, "") + if err != nil { + log.WithError(err).WithField("node", host).Error("failed to get node for host") + return nil, nil, err + } - // Make sure the returned value is OK. - v3n, ok := node.Value.(*libapiv3.Node) - if !ok { - return nil, nil, fmt.Errorf("Datastore returned malformed node object") + // Make sure the returned value is OK. + v3n, ok = node.Value.(*libapiv3.Node) + if !ok { + return nil, nil, fmt.Errorf("Datastore returned malformed node object") + } + } else { + // Special case for Service LoadBalancer that is affined to virtual node + v3n = libapiv3.NewNode() + v3n.Name = v3.VirtualLoadBalancer } maxPrefixLen, err := getMaxPrefixLen(version, rsvdAttr) diff --git a/libcalico-go/lib/upgrade/converters/ippool.go b/libcalico-go/lib/upgrade/converters/ippool.go index f6409aa640f..a4a073bc881 100644 --- a/libcalico-go/lib/upgrade/converters/ippool.go +++ b/libcalico-go/lib/upgrade/converters/ippool.go @@ -50,12 +50,13 @@ func (_ IPPool) APIV1ToBackendV1(rIn unversioned.Resource) (*model.KVPair, error CIDR: p.Metadata.CIDR, }, Value: &model.IPPool{ - CIDR: p.Metadata.CIDR, - IPIPInterface: ipipInterface, - IPIPMode: ipipMode, - Masquerade: p.Spec.NATOutgoing, - IPAM: !p.Spec.Disabled, - Disabled: p.Spec.Disabled, + CIDR: p.Metadata.CIDR, + IPIPInterface: ipipInterface, + IPIPMode: ipipMode, + Masquerade: p.Spec.NATOutgoing, + IPAM: !p.Spec.Disabled, + Disabled: p.Spec.Disabled, + AssignmentMode: p.Spec.AssignmentMode, }, } @@ -82,6 +83,7 @@ func (_ IPPool) BackendV1ToAPIV3(kvp *model.KVPair) (Resource, error) { apiv3.IPPoolAllowedUseWorkload, apiv3.IPPoolAllowedUseTunnel, }, + AssignmentMode: pool.AssignmentMode, } // Set the blocksize based on IP address family. diff --git a/libcalico-go/lib/validator/v3/validator.go b/libcalico-go/lib/validator/v3/validator.go index 6150534ec10..bb02d315196 100644 --- a/libcalico-go/lib/validator/v3/validator.go +++ b/libcalico-go/lib/validator/v3/validator.go @@ -95,6 +95,8 @@ var ( protocolRegex = regexp.MustCompile("^(TCP|UDP|ICMP|ICMPv6|SCTP|UDPLite)$") ipipModeRegex = regexp.MustCompile("^(Always|CrossSubnet|Never)$") vxlanModeRegex = regexp.MustCompile("^(Always|CrossSubnet|Never)$") + assignmentModeRegex = regexp.MustCompile("^(Automatic|Manual)$") + assignIPsRegex = regexp.MustCompile("^(AllServices|RequestedServicesOnly)$") logLevelRegex = regexp.MustCompile("^(Debug|Info|Warning|Error|Fatal)$") bpfLogLevelRegex = regexp.MustCompile("^(Debug|Info|Off)$") bpfServiceModeRegex = regexp.MustCompile("^(Tunnel|DSR)$") @@ -183,6 +185,8 @@ func init() { registerFieldValidator("ipVersion", validateIPVersion) registerFieldValidator("ipIpMode", validateIPIPMode) registerFieldValidator("vxlanMode", validateVXLANMode) + registerFieldValidator("assignmentMode", validateAssignmentMode) + registerFieldValidator("assignIPs", validateAssignIPs) registerFieldValidator("policyType", validatePolicyType) registerFieldValidator("logLevel", validateLogLevel) registerFieldValidator("bpfLogLevel", validateBPFLogLevel) @@ -453,6 +457,18 @@ func validateVXLANMode(fl validator.FieldLevel) bool { return vxlanModeRegex.MatchString(s) } +func validateAssignmentMode(fl validator.FieldLevel) bool { + s := fl.Field().String() + log.Debugf("Validate Assignemnt Mode: %s", s) + return assignmentModeRegex.MatchString(s) +} + +func validateAssignIPs(fl validator.FieldLevel) bool { + s := fl.Field().String() + log.Debugf("Validate Assign IPs: %s", s) + return assignIPsRegex.MatchString(s) +} + func RegexValidator(desc string, rx *regexp.Regexp) func(fl validator.FieldLevel) bool { return func(fl validator.FieldLevel) bool { s := fl.Field().String() @@ -1095,6 +1111,13 @@ func validateIPPoolSpec(structLevel validator.StructLevel) { // Normalize the CIDR before persisting. pool.CIDR = cidr.String() + isLoadBalancer := false + for _, u := range pool.AllowedUses { + if u == api.IPPoolAllowedUseLoadBalancer { + isLoadBalancer = true + } + } + // IPIP cannot be enabled for IPv6. if cidr.Version() == 6 && pool.IPIPMode != api.IPIPModeNever { structLevel.ReportError(reflect.ValueOf(pool.IPIPMode), @@ -1104,7 +1127,13 @@ func validateIPPoolSpec(structLevel validator.StructLevel) { // Cannot have both VXLAN and IPIP on the same IP pool. if ipipModeEnabled(pool.IPIPMode) && vxLanModeEnabled(pool.VXLANMode) { structLevel.ReportError(reflect.ValueOf(pool.IPIPMode), - "IPpool.IPIPMode", "", reason("IPIPMode and VXLANMode cannot both be enabled on the same IP pool"), "") + "IPpool.IPIPMode", "", reason("IPIPMode and VXLANMode cannot be enabled on LoadBalancer IP pool"), "") + } + + // Cannot have VXLAN and IPIP enabled on LoadBalancer IP pool. + if isLoadBalancer && (ipipModeEnabled(pool.IPIPMode) || vxLanModeEnabled(pool.VXLANMode)) { + structLevel.ReportError(reflect.ValueOf(pool.IPIPMode), + "IPpool.IPIPMode", "", reason("IPIPMode or VXLANMode cannot both be enabled on AllowedUses LoadBalancer IP pool"), "") } // Default the blockSize @@ -1159,13 +1188,29 @@ func validateIPPoolSpec(structLevel validator.StructLevel) { // Allowed use must be one of the enums. for _, a := range pool.AllowedUses { switch a { + case api.IPPoolAllowedUseLoadBalancer: + continue case api.IPPoolAllowedUseWorkload, api.IPPoolAllowedUseTunnel: + if isLoadBalancer { + structLevel.ReportError(reflect.ValueOf(pool.AllowedUses), + "IPpool.AllowedUses", "", reason("LoadBalancer cannot be used at the same time as: "+string(a)), "") + } continue default: structLevel.ReportError(reflect.ValueOf(pool.AllowedUses), "IPpool.AllowedUses", "", reason("unknown use: "+string(a)), "") } } + + if isLoadBalancer && pool.DisableBGPExport { + structLevel.ReportError(reflect.ValueOf(pool.CIDR), + "IPpool.DisableBGPExport", "", reason("IP Pool with AllowedUse LoadBalancer has to have DisableBGPExport set to true"), "") + } + + if isLoadBalancer && pool.NodeSelector != "all()" { + structLevel.ReportError(reflect.ValueOf(pool.CIDR), + "IPpool.NodeSelector", "", reason("IP Pool with AllowedUse LoadBalancer has to have node selector set to all()"), "") + } } func vxLanModeEnabled(mode api.VXLANMode) bool { diff --git a/libcalico-go/lib/validator/v3/validator_test.go b/libcalico-go/lib/validator/v3/validator_test.go index 5491c83c53b..acd74d0c00f 100644 --- a/libcalico-go/lib/validator/v3/validator_test.go +++ b/libcalico-go/lib/validator/v3/validator_test.go @@ -1040,6 +1040,17 @@ func init() { }, }, }, true), + Entry("should reject IP pool with invalid allowed uses combination", + api.IPPool{ + ObjectMeta: v1.ObjectMeta{Name: "pool.name"}, + Spec: api.IPPoolSpec{ + CIDR: netv4_4, + AllowedUses: []api.IPPoolAllowedUse{ + api.IPPoolAllowedUseLoadBalancer, + api.IPPoolAllowedUseTunnel, + }, + }, + }, false), Entry("should reject IP pool with invalid allowed uses", api.IPPool{ ObjectMeta: v1.ObjectMeta{Name: "pool.name"}, @@ -1050,7 +1061,66 @@ func init() { }, }, }, false), - + Entry("should accept IP pool with valid AssignmentMode", + api.IPPool{ + ObjectMeta: v1.ObjectMeta{Name: "pool.name"}, + Spec: api.IPPoolSpec{ + CIDR: netv4_4, + AssignmentMode: api.Automatic, + }, + }, true), + Entry("should reject IP pool with invalid assignment mode", + api.IPPool{ + ObjectMeta: v1.ObjectMeta{Name: "pool.name"}, + Spec: api.IPPoolSpec{ + CIDR: netv4_4, + AssignmentMode: "Garbage", + }, + }, false), + Entry("should reject IP pool with LoadBlancer and disableBGPExport true", + api.IPPool{ + ObjectMeta: v1.ObjectMeta{Name: "pool.name"}, + Spec: api.IPPoolSpec{ + CIDR: netv4_4, + AllowedUses: []api.IPPoolAllowedUse{ + api.IPPoolAllowedUseLoadBalancer, + }, + DisableBGPExport: Vtrue, + }, + }, false), + Entry("should reject IP pool with LoadBlancer and VXLAN mode enabled", + api.IPPool{ + ObjectMeta: v1.ObjectMeta{Name: "pool.name"}, + Spec: api.IPPoolSpec{ + CIDR: netv4_4, + AllowedUses: []api.IPPoolAllowedUse{ + api.IPPoolAllowedUseLoadBalancer, + }, + VXLANMode: api.VXLANModeAlways, + }, + }, false), + Entry("should reject IP pool with LoadBlancer and IPIP mode enabled", + api.IPPool{ + ObjectMeta: v1.ObjectMeta{Name: "pool.name"}, + Spec: api.IPPoolSpec{ + CIDR: netv4_4, + AllowedUses: []api.IPPoolAllowedUse{ + api.IPPoolAllowedUseLoadBalancer, + }, + IPIPMode: api.IPIPModeAlways, + }, + }, false), + Entry("should reject IP pool with LoadBlancer and nodeSelector other than all()", + api.IPPool{ + ObjectMeta: v1.ObjectMeta{Name: "pool.name"}, + Spec: api.IPPoolSpec{ + CIDR: netv4_4, + AllowedUses: []api.IPPoolAllowedUse{ + api.IPPoolAllowedUseLoadBalancer, + }, + NodeSelector: "!all()", + }, + }, false), // (API) IPReservation Entry("should accept IPReservation with an IP", api.IPReservation{ @@ -3115,6 +3185,7 @@ func init() { WorkloadEndpoint: &api.WorkloadEndpointControllerConfig{}, ServiceAccount: &api.ServiceAccountControllerConfig{}, Namespace: &api.NamespaceControllerConfig{}, + LoadBalancer: &api.LoadBalancerControllerConfig{}, }}, true, ), Entry("should accept valid reconciliation period on node", @@ -3147,6 +3218,15 @@ func init() { Entry("should accept valid reconciliation period on namespace", api.NamespaceControllerConfig{ReconcilerPeriod: &v1.Duration{Duration: time.Second * 330}}, true, ), + Entry("should accept valid assignIPs value for LoadBalancer config", + api.LoadBalancerControllerConfig{AssignIPs: api.AllServices}, true, + ), + Entry("should accept valid assignIPs value for LoadBalancer config", + api.LoadBalancerControllerConfig{AssignIPs: api.RequestedServicesOnly}, true, + ), + Entry("should not accept invalid assignIPs value for LoadBalancer config", + api.LoadBalancerControllerConfig{AssignIPs: "incorrect-value"}, false, + ), // BGP Communities validation in BGPConfigurationSpec Entry("should not accept community when PrefixAdvertisement is empty", api.BGPConfigurationSpec{ diff --git a/manifests/calico-bpf.yaml b/manifests/calico-bpf.yaml index 56114dfeb2e..84ae2eb040a 100644 --- a/manifests/calico-bpf.yaml +++ b/manifests/calico-bpf.yaml @@ -3242,6 +3242,10 @@ spec: items: type: string type: array + assignmentMode: + description: Determines the mode how IP addresses should be assigned + from this pool + type: string blockSize: description: The block size to use for IP address assignments from this pool. Defaults to 26 for IPv4 and 122 for IPv6. @@ -3406,6 +3410,13 @@ spec: description: Controllers enables and configures individual Kubernetes controllers properties: + loadBalancer: + description: LoadBalancer enables and configures the LoadBalancer + controller. Enabled by default, set to nil to disable. + properties: + assignIPs: + type: string + type: object namespace: description: Namespace enables and configures the namespace controller. Enabled by default, set to nil to disable. @@ -3515,6 +3526,13 @@ spec: description: Controllers enables and configures individual Kubernetes controllers properties: + loadBalancer: + description: LoadBalancer enables and configures the LoadBalancer + controller. Enabled by default, set to nil to disable. + properties: + assignIPs: + type: string + type: object namespace: description: Namespace enables and configures the namespace controller. Enabled by default, set to nil to disable. @@ -4611,6 +4629,16 @@ rules: - get - list - watch + # Services are monitored for service LoadBalancer + - apiGroups: [""] + resources: + - services + - services/status + verbs: + - get + - list + - update + - watch # IPAM resources are manipulated in response to node and block updates, as well as periodic triggers. - apiGroups: ["crd.projectcalico.org"] resources: @@ -4622,6 +4650,7 @@ rules: - blockaffinities - ipamblocks - ipamhandles + - ipamconfigs - tiers verbs: - get @@ -5327,7 +5356,7 @@ spec: env: # Choose which controllers to run. - name: ENABLED_CONTROLLERS - value: node + value: node,loadbalancer - name: DATASTORE_TYPE value: kubernetes livenessProbe: diff --git a/manifests/calico-etcd.yaml b/manifests/calico-etcd.yaml index ea6d5976b39..77f3598794e 100644 --- a/manifests/calico-etcd.yaml +++ b/manifests/calico-etcd.yaml @@ -652,7 +652,7 @@ spec: key: etcd_cert # Choose which controllers to run. - name: ENABLED_CONTROLLERS - value: policy,namespace,serviceaccount,workloadendpoint,node + value: policy,namespace,serviceaccount,workloadendpoint,node,loadbalancer volumeMounts: # Mount in the etcd TLS secrets. - mountPath: /calico-secrets diff --git a/manifests/calico-policy-only.yaml b/manifests/calico-policy-only.yaml index d123e1eb1a1..3c2718c181b 100644 --- a/manifests/calico-policy-only.yaml +++ b/manifests/calico-policy-only.yaml @@ -3252,6 +3252,10 @@ spec: items: type: string type: array + assignmentMode: + description: Determines the mode how IP addresses should be assigned + from this pool + type: string blockSize: description: The block size to use for IP address assignments from this pool. Defaults to 26 for IPv4 and 122 for IPv6. @@ -3416,6 +3420,13 @@ spec: description: Controllers enables and configures individual Kubernetes controllers properties: + loadBalancer: + description: LoadBalancer enables and configures the LoadBalancer + controller. Enabled by default, set to nil to disable. + properties: + assignIPs: + type: string + type: object namespace: description: Namespace enables and configures the namespace controller. Enabled by default, set to nil to disable. @@ -3525,6 +3536,13 @@ spec: description: Controllers enables and configures individual Kubernetes controllers properties: + loadBalancer: + description: LoadBalancer enables and configures the LoadBalancer + controller. Enabled by default, set to nil to disable. + properties: + assignIPs: + type: string + type: object namespace: description: Namespace enables and configures the namespace controller. Enabled by default, set to nil to disable. @@ -4621,6 +4639,16 @@ rules: - get - list - watch + # Services are monitored for service LoadBalancer + - apiGroups: [""] + resources: + - services + - services/status + verbs: + - get + - list + - update + - watch # IPAM resources are manipulated in response to node and block updates, as well as periodic triggers. - apiGroups: ["crd.projectcalico.org"] resources: @@ -4632,6 +4660,7 @@ rules: - blockaffinities - ipamblocks - ipamhandles + - ipamconfigs - tiers verbs: - get @@ -5209,7 +5238,7 @@ spec: env: # Choose which controllers to run. - name: ENABLED_CONTROLLERS - value: node + value: node,loadbalancer - name: DATASTORE_TYPE value: kubernetes livenessProbe: diff --git a/manifests/calico-typha.yaml b/manifests/calico-typha.yaml index a7786076f41..0c990b68a8f 100644 --- a/manifests/calico-typha.yaml +++ b/manifests/calico-typha.yaml @@ -3253,6 +3253,10 @@ spec: items: type: string type: array + assignmentMode: + description: Determines the mode how IP addresses should be assigned + from this pool + type: string blockSize: description: The block size to use for IP address assignments from this pool. Defaults to 26 for IPv4 and 122 for IPv6. @@ -3417,6 +3421,13 @@ spec: description: Controllers enables and configures individual Kubernetes controllers properties: + loadBalancer: + description: LoadBalancer enables and configures the LoadBalancer + controller. Enabled by default, set to nil to disable. + properties: + assignIPs: + type: string + type: object namespace: description: Namespace enables and configures the namespace controller. Enabled by default, set to nil to disable. @@ -3526,6 +3537,13 @@ spec: description: Controllers enables and configures individual Kubernetes controllers properties: + loadBalancer: + description: LoadBalancer enables and configures the LoadBalancer + controller. Enabled by default, set to nil to disable. + properties: + assignIPs: + type: string + type: object namespace: description: Namespace enables and configures the namespace controller. Enabled by default, set to nil to disable. @@ -4622,6 +4640,16 @@ rules: - get - list - watch + # Services are monitored for service LoadBalancer + - apiGroups: [""] + resources: + - services + - services/status + verbs: + - get + - list + - update + - watch # IPAM resources are manipulated in response to node and block updates, as well as periodic triggers. - apiGroups: ["crd.projectcalico.org"] resources: @@ -4633,6 +4661,7 @@ rules: - blockaffinities - ipamblocks - ipamhandles + - ipamconfigs - tiers verbs: - get @@ -5328,7 +5357,7 @@ spec: env: # Choose which controllers to run. - name: ENABLED_CONTROLLERS - value: node + value: node,loadbalancer - name: DATASTORE_TYPE value: kubernetes livenessProbe: diff --git a/manifests/calico-vxlan.yaml b/manifests/calico-vxlan.yaml index 0cdf85e9c02..2730ed22a39 100644 --- a/manifests/calico-vxlan.yaml +++ b/manifests/calico-vxlan.yaml @@ -3237,6 +3237,10 @@ spec: items: type: string type: array + assignmentMode: + description: Determines the mode how IP addresses should be assigned + from this pool + type: string blockSize: description: The block size to use for IP address assignments from this pool. Defaults to 26 for IPv4 and 122 for IPv6. @@ -3401,6 +3405,13 @@ spec: description: Controllers enables and configures individual Kubernetes controllers properties: + loadBalancer: + description: LoadBalancer enables and configures the LoadBalancer + controller. Enabled by default, set to nil to disable. + properties: + assignIPs: + type: string + type: object namespace: description: Namespace enables and configures the namespace controller. Enabled by default, set to nil to disable. @@ -3510,6 +3521,13 @@ spec: description: Controllers enables and configures individual Kubernetes controllers properties: + loadBalancer: + description: LoadBalancer enables and configures the LoadBalancer + controller. Enabled by default, set to nil to disable. + properties: + assignIPs: + type: string + type: object namespace: description: Namespace enables and configures the namespace controller. Enabled by default, set to nil to disable. @@ -4606,6 +4624,16 @@ rules: - get - list - watch + # Services are monitored for service LoadBalancer + - apiGroups: [""] + resources: + - services + - services/status + verbs: + - get + - list + - update + - watch # IPAM resources are manipulated in response to node and block updates, as well as periodic triggers. - apiGroups: ["crd.projectcalico.org"] resources: @@ -4617,6 +4645,7 @@ rules: - blockaffinities - ipamblocks - ipamhandles + - ipamconfigs - tiers verbs: - get @@ -5284,7 +5313,7 @@ spec: env: # Choose which controllers to run. - name: ENABLED_CONTROLLERS - value: node + value: node,loadbalancer - name: DATASTORE_TYPE value: kubernetes livenessProbe: diff --git a/manifests/calico.yaml b/manifests/calico.yaml index 93ac6912012..0a5dad7c91f 100644 --- a/manifests/calico.yaml +++ b/manifests/calico.yaml @@ -3237,6 +3237,10 @@ spec: items: type: string type: array + assignmentMode: + description: Determines the mode how IP addresses should be assigned + from this pool + type: string blockSize: description: The block size to use for IP address assignments from this pool. Defaults to 26 for IPv4 and 122 for IPv6. @@ -3401,6 +3405,13 @@ spec: description: Controllers enables and configures individual Kubernetes controllers properties: + loadBalancer: + description: LoadBalancer enables and configures the LoadBalancer + controller. Enabled by default, set to nil to disable. + properties: + assignIPs: + type: string + type: object namespace: description: Namespace enables and configures the namespace controller. Enabled by default, set to nil to disable. @@ -3510,6 +3521,13 @@ spec: description: Controllers enables and configures individual Kubernetes controllers properties: + loadBalancer: + description: LoadBalancer enables and configures the LoadBalancer + controller. Enabled by default, set to nil to disable. + properties: + assignIPs: + type: string + type: object namespace: description: Namespace enables and configures the namespace controller. Enabled by default, set to nil to disable. @@ -4606,6 +4624,16 @@ rules: - get - list - watch + # Services are monitored for service LoadBalancer + - apiGroups: [""] + resources: + - services + - services/status + verbs: + - get + - list + - update + - watch # IPAM resources are manipulated in response to node and block updates, as well as periodic triggers. - apiGroups: ["crd.projectcalico.org"] resources: @@ -4617,6 +4645,7 @@ rules: - blockaffinities - ipamblocks - ipamhandles + - ipamconfigs - tiers verbs: - get @@ -5286,7 +5315,7 @@ spec: env: # Choose which controllers to run. - name: ENABLED_CONTROLLERS - value: node + value: node,loadbalancer - name: DATASTORE_TYPE value: kubernetes livenessProbe: diff --git a/manifests/canal-etcd.yaml b/manifests/canal-etcd.yaml index 2b7ef05e0d7..e5593d88067 100644 --- a/manifests/canal-etcd.yaml +++ b/manifests/canal-etcd.yaml @@ -812,7 +812,7 @@ spec: key: etcd_cert # Choose which controllers to run. - name: ENABLED_CONTROLLERS - value: policy,namespace,serviceaccount,workloadendpoint,node + value: policy,namespace,serviceaccount,workloadendpoint,node,loadbalancer volumeMounts: # Mount in the etcd TLS secrets. - mountPath: /calico-secrets diff --git a/manifests/canal.yaml b/manifests/canal.yaml index 55154c2d41e..5adc3239a37 100644 --- a/manifests/canal.yaml +++ b/manifests/canal.yaml @@ -3254,6 +3254,10 @@ spec: items: type: string type: array + assignmentMode: + description: Determines the mode how IP addresses should be assigned + from this pool + type: string blockSize: description: The block size to use for IP address assignments from this pool. Defaults to 26 for IPv4 and 122 for IPv6. @@ -3418,6 +3422,13 @@ spec: description: Controllers enables and configures individual Kubernetes controllers properties: + loadBalancer: + description: LoadBalancer enables and configures the LoadBalancer + controller. Enabled by default, set to nil to disable. + properties: + assignIPs: + type: string + type: object namespace: description: Namespace enables and configures the namespace controller. Enabled by default, set to nil to disable. @@ -3527,6 +3538,13 @@ spec: description: Controllers enables and configures individual Kubernetes controllers properties: + loadBalancer: + description: LoadBalancer enables and configures the LoadBalancer + controller. Enabled by default, set to nil to disable. + properties: + assignIPs: + type: string + type: object namespace: description: Namespace enables and configures the namespace controller. Enabled by default, set to nil to disable. @@ -4623,6 +4641,16 @@ rules: - get - list - watch + # Services are monitored for service LoadBalancer + - apiGroups: [""] + resources: + - services + - services/status + verbs: + - get + - list + - update + - watch # IPAM resources are manipulated in response to node and block updates, as well as periodic triggers. - apiGroups: ["crd.projectcalico.org"] resources: @@ -4634,6 +4662,7 @@ rules: - blockaffinities - ipamblocks - ipamhandles + - ipamconfigs - tiers verbs: - get @@ -5284,7 +5313,7 @@ spec: env: # Choose which controllers to run. - name: ENABLED_CONTROLLERS - value: node + value: node,loadbalancer - name: DATASTORE_TYPE value: kubernetes livenessProbe: diff --git a/manifests/crds.yaml b/manifests/crds.yaml index 3eefa5183f9..540bb48a2ca 100644 --- a/manifests/crds.yaml +++ b/manifests/crds.yaml @@ -3147,6 +3147,10 @@ spec: items: type: string type: array + assignmentMode: + description: Determines the mode how IP addresses should be assigned + from this pool + type: string blockSize: description: The block size to use for IP address assignments from this pool. Defaults to 26 for IPv4 and 122 for IPv6. @@ -3311,6 +3315,13 @@ spec: description: Controllers enables and configures individual Kubernetes controllers properties: + loadBalancer: + description: LoadBalancer enables and configures the LoadBalancer + controller. Enabled by default, set to nil to disable. + properties: + assignIPs: + type: string + type: object namespace: description: Namespace enables and configures the namespace controller. Enabled by default, set to nil to disable. @@ -3420,6 +3431,13 @@ spec: description: Controllers enables and configures individual Kubernetes controllers properties: + loadBalancer: + description: LoadBalancer enables and configures the LoadBalancer + controller. Enabled by default, set to nil to disable. + properties: + assignIPs: + type: string + type: object namespace: description: Namespace enables and configures the namespace controller. Enabled by default, set to nil to disable. diff --git a/manifests/flannel-migration/calico.yaml b/manifests/flannel-migration/calico.yaml index d945af160eb..d5a33f59b2d 100644 --- a/manifests/flannel-migration/calico.yaml +++ b/manifests/flannel-migration/calico.yaml @@ -3237,6 +3237,10 @@ spec: items: type: string type: array + assignmentMode: + description: Determines the mode how IP addresses should be assigned + from this pool + type: string blockSize: description: The block size to use for IP address assignments from this pool. Defaults to 26 for IPv4 and 122 for IPv6. @@ -3401,6 +3405,13 @@ spec: description: Controllers enables and configures individual Kubernetes controllers properties: + loadBalancer: + description: LoadBalancer enables and configures the LoadBalancer + controller. Enabled by default, set to nil to disable. + properties: + assignIPs: + type: string + type: object namespace: description: Namespace enables and configures the namespace controller. Enabled by default, set to nil to disable. @@ -3510,6 +3521,13 @@ spec: description: Controllers enables and configures individual Kubernetes controllers properties: + loadBalancer: + description: LoadBalancer enables and configures the LoadBalancer + controller. Enabled by default, set to nil to disable. + properties: + assignIPs: + type: string + type: object namespace: description: Namespace enables and configures the namespace controller. Enabled by default, set to nil to disable. @@ -4606,6 +4624,16 @@ rules: - get - list - watch + # Services are monitored for service LoadBalancer + - apiGroups: [""] + resources: + - services + - services/status + verbs: + - get + - list + - update + - watch # IPAM resources are manipulated in response to node and block updates, as well as periodic triggers. - apiGroups: ["crd.projectcalico.org"] resources: @@ -4617,6 +4645,7 @@ rules: - blockaffinities - ipamblocks - ipamhandles + - ipamconfigs - tiers verbs: - get @@ -5286,7 +5315,7 @@ spec: env: # Choose which controllers to run. - name: ENABLED_CONTROLLERS - value: node + value: node,loadbalancer - name: DATASTORE_TYPE value: kubernetes livenessProbe: diff --git a/manifests/ocp/crd.projectcalico.org_ippools.yaml b/manifests/ocp/crd.projectcalico.org_ippools.yaml index 57a5aef4fcd..9970a74783c 100644 --- a/manifests/ocp/crd.projectcalico.org_ippools.yaml +++ b/manifests/ocp/crd.projectcalico.org_ippools.yaml @@ -37,6 +37,10 @@ spec: items: type: string type: array + assignmentMode: + description: Determines the mode how IP addresses should be assigned + from this pool + type: string blockSize: description: The block size to use for IP address assignments from this pool. Defaults to 26 for IPv4 and 122 for IPv6. diff --git a/manifests/ocp/crd.projectcalico.org_kubecontrollersconfigurations.yaml b/manifests/ocp/crd.projectcalico.org_kubecontrollersconfigurations.yaml index a3dcac3d659..70a36f93bdc 100644 --- a/manifests/ocp/crd.projectcalico.org_kubecontrollersconfigurations.yaml +++ b/manifests/ocp/crd.projectcalico.org_kubecontrollersconfigurations.yaml @@ -36,6 +36,13 @@ spec: description: Controllers enables and configures individual Kubernetes controllers properties: + loadBalancer: + description: LoadBalancer enables and configures the LoadBalancer + controller. Enabled by default, set to nil to disable. + properties: + assignIPs: + type: string + type: object namespace: description: Namespace enables and configures the namespace controller. Enabled by default, set to nil to disable. @@ -145,6 +152,13 @@ spec: description: Controllers enables and configures individual Kubernetes controllers properties: + loadBalancer: + description: LoadBalancer enables and configures the LoadBalancer + controller. Enabled by default, set to nil to disable. + properties: + assignIPs: + type: string + type: object namespace: description: Namespace enables and configures the namespace controller. Enabled by default, set to nil to disable. diff --git a/manifests/ocp/operator.tigera.io_installations_crd.yaml b/manifests/ocp/operator.tigera.io_installations_crd.yaml index a4e5120e781..9ba6593c0b5 100644 --- a/manifests/ocp/operator.tigera.io_installations_crd.yaml +++ b/manifests/ocp/operator.tigera.io_installations_crd.yaml @@ -1129,6 +1129,14 @@ spec: NodeSelector specifies the node selector that will be set for the IP Pool. Default: 'all()' type: string + assignmentMode: + description: AssignmentMode determines if IP addresses from this pool should be + assigned automatically or on request only + enum: + - Automatic + - Manual + type: string + default: Automatic required: - cidr type: object diff --git a/manifests/operator-crds.yaml b/manifests/operator-crds.yaml index 3ef82f26def..711f737f474 100644 --- a/manifests/operator-crds.yaml +++ b/manifests/operator-crds.yaml @@ -2588,6 +2588,14 @@ spec: NodeSelector specifies the node selector that will be set for the IP Pool. Default: 'all()' type: string + assignmentMode: + description: AssignmentMode determines if IP addresses from this pool should be + assigned automatically or on request only + enum: + - Automatic + - Manual + type: string + default: Automatic required: - cidr type: object @@ -19676,6 +19684,10 @@ spec: items: type: string type: array + assignmentMode: + description: Determines the mode how IP addresses should be assigned + from this pool + type: string blockSize: description: The block size to use for IP address assignments from this pool. Defaults to 26 for IPv4 and 122 for IPv6. @@ -19840,6 +19852,13 @@ spec: description: Controllers enables and configures individual Kubernetes controllers properties: + loadBalancer: + description: LoadBalancer enables and configures the LoadBalancer + controller. Enabled by default, set to nil to disable. + properties: + assignIPs: + type: string + type: object namespace: description: Namespace enables and configures the namespace controller. Enabled by default, set to nil to disable. @@ -19949,6 +19968,13 @@ spec: description: Controllers enables and configures individual Kubernetes controllers properties: + loadBalancer: + description: LoadBalancer enables and configures the LoadBalancer + controller. Enabled by default, set to nil to disable. + properties: + assignIPs: + type: string + type: object namespace: description: Namespace enables and configures the namespace controller. Enabled by default, set to nil to disable. diff --git a/manifests/tigera-operator.yaml b/manifests/tigera-operator.yaml index a6955b8f954..a30474331bd 100644 --- a/manifests/tigera-operator.yaml +++ b/manifests/tigera-operator.yaml @@ -3166,6 +3166,10 @@ spec: items: type: string type: array + assignmentMode: + description: Determines the mode how IP addresses should be assigned + from this pool + type: string blockSize: description: The block size to use for IP address assignments from this pool. Defaults to 26 for IPv4 and 122 for IPv6. @@ -3332,6 +3336,13 @@ spec: description: Controllers enables and configures individual Kubernetes controllers properties: + loadBalancer: + description: LoadBalancer enables and configures the LoadBalancer + controller. Enabled by default, set to nil to disable. + properties: + assignIPs: + type: string + type: object namespace: description: Namespace enables and configures the namespace controller. Enabled by default, set to nil to disable. @@ -3441,6 +3452,13 @@ spec: description: Controllers enables and configures individual Kubernetes controllers properties: + loadBalancer: + description: LoadBalancer enables and configures the LoadBalancer + controller. Enabled by default, set to nil to disable. + properties: + assignIPs: + type: string + type: object namespace: description: Namespace enables and configures the namespace controller. Enabled by default, set to nil to disable. @@ -7107,6 +7125,14 @@ spec: NodeSelector specifies the node selector that will be set for the IP Pool. Default: 'all()' type: string + assignmentMode: + description: AssignmentMode determines if IP addresses from this pool should be + assigned automatically or on request only + enum: + - Automatic + - Manual + type: string + default: Automatic required: - cidr type: object diff --git a/node/tests/k8st/infra/calico-kdd.yaml b/node/tests/k8st/infra/calico-kdd.yaml index 68780c87db7..fdfd9e1fb12 100644 --- a/node/tests/k8st/infra/calico-kdd.yaml +++ b/node/tests/k8st/infra/calico-kdd.yaml @@ -114,6 +114,16 @@ rules: - get - list - watch + # Services are monitored for service LoadBalancer + - apiGroups: [""] + resources: + - services + - services/status + verbs: + - get + - list + - update + - watch # IPAM resources are manipulated in response to node and block updates, as well as periodic triggers. - apiGroups: ["crd.projectcalico.org"] resources: @@ -125,6 +135,7 @@ rules: - blockaffinities - ipamblocks - ipamhandles + - ipamconfigs - tiers verbs: - get @@ -794,7 +805,7 @@ spec: env: # Choose which controllers to run. - name: ENABLED_CONTROLLERS - value: node + value: node,loadbalancer - name: DATASTORE_TYPE value: kubernetes livenessProbe: From 2865a5432defcff9c587a3decae217594544a750 Mon Sep 17 00:00:00 2001 From: Michal Fupso Date: Thu, 12 Sep 2024 08:03:34 -0700 Subject: [PATCH 02/12] calicoctl crds --- calicoctl/calicoctl/commands/crds/crds.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/calicoctl/calicoctl/commands/crds/crds.go b/calicoctl/calicoctl/commands/crds/crds.go index a9d24782dff..28fa137519f 100644 --- a/calicoctl/calicoctl/commands/crds/crds.go +++ b/calicoctl/calicoctl/commands/crds/crds.go @@ -30,9 +30,9 @@ const ( ipamblocks = "apiVersion: apiextensions.k8s.io/v1\nkind: CustomResourceDefinition\nmetadata:\n name: ipamblocks.crd.projectcalico.org\nspec:\n group: crd.projectcalico.org\n names:\n kind: IPAMBlock\n listKind: IPAMBlockList\n plural: ipamblocks\n singular: ipamblock\n preserveUnknownFields: false\n scope: Cluster\n versions:\n - name: v1\n schema:\n openAPIV3Schema:\n properties:\n apiVersion:\n description: 'APIVersion defines the versioned schema of this representation\n of an object. Servers should convert recognized schemas to the latest\n internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'\n type: string\n kind:\n description: 'Kind is a string value representing the REST resource this\n object represents. Servers may infer this from the endpoint the client\n submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'\n type: string\n metadata:\n type: object\n spec:\n description: IPAMBlockSpec contains the specification for an IPAMBlock\n resource.\n properties:\n affinity:\n description: Affinity of the block, if this block has one. If set,\n it will be of the form \"host:\". If not set, this block\n is not affine to a host.\n type: string\n allocations:\n description: Array of allocations in-use within this block. nil entries\n mean the allocation is free. For non-nil entries at index i, the\n index is the ordinal of the allocation within this block and the\n value is the index of the associated attributes in the Attributes\n array.\n items:\n type: integer\n # TODO: This nullable is manually added in. We should update controller-gen\n # to handle []*int properly itself.\n nullable: true\n type: array\n attributes:\n description: Attributes is an array of arbitrary metadata associated\n with allocations in the block. To find attributes for a given allocation,\n use the value of the allocation's entry in the Allocations array\n as the index of the element in this array.\n items:\n properties:\n handle_id:\n type: string\n secondary:\n additionalProperties:\n type: string\n type: object\n type: object\n type: array\n cidr:\n description: The block's CIDR.\n type: string\n deleted:\n description: Deleted is an internal boolean used to workaround a limitation\n in the Kubernetes API whereby deletion will not return a conflict\n error if the block has been updated. It should not be set manually.\n type: boolean\n sequenceNumber:\n default: 0\n description: We store a sequence number that is updated each time\n the block is written. Each allocation will also store the sequence\n number of the block at the time of its creation. When releasing\n an IP, passing the sequence number associated with the allocation\n allows us to protect against a race condition and ensure the IP\n hasn't been released and re-allocated since the release request.\n format: int64\n type: integer\n sequenceNumberForAllocation:\n additionalProperties:\n format: int64\n type: integer\n description: Map of allocated ordinal within the block to sequence\n number of the block at the time of allocation. Kubernetes does not\n allow numerical keys for maps, so the key is cast to a string.\n type: object\n strictAffinity:\n description: StrictAffinity on the IPAMBlock is deprecated and no\n longer used by the code. Use IPAMConfig StrictAffinity instead.\n type: boolean\n unallocated:\n description: Unallocated is an ordered list of allocations which are\n free in the block.\n items:\n type: integer\n type: array\n required:\n - allocations\n - attributes\n - cidr\n - strictAffinity\n - unallocated\n type: object\n type: object\n served: true\n storage: true\nstatus:\n acceptedNames:\n kind: \"\"\n plural: \"\"\n conditions: []\n storedVersions: []\n" ipamconfigs = "apiVersion: apiextensions.k8s.io/v1\nkind: CustomResourceDefinition\nmetadata:\n name: ipamconfigs.crd.projectcalico.org\nspec:\n group: crd.projectcalico.org\n names:\n kind: IPAMConfig\n listKind: IPAMConfigList\n plural: ipamconfigs\n singular: ipamconfig\n preserveUnknownFields: false\n scope: Cluster\n versions:\n - name: v1\n schema:\n openAPIV3Schema:\n properties:\n apiVersion:\n description: 'APIVersion defines the versioned schema of this representation\n of an object. Servers should convert recognized schemas to the latest\n internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'\n type: string\n kind:\n description: 'Kind is a string value representing the REST resource this\n object represents. Servers may infer this from the endpoint the client\n submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'\n type: string\n metadata:\n type: object\n spec:\n description: IPAMConfigSpec contains the specification for an IPAMConfig\n resource.\n properties:\n autoAllocateBlocks:\n type: boolean\n maxBlocksPerHost:\n description: MaxBlocksPerHost, if non-zero, is the max number of blocks\n that can be affine to each host.\n maximum: 2147483647\n minimum: 0\n type: integer\n strictAffinity:\n type: boolean\n required:\n - autoAllocateBlocks\n - strictAffinity\n type: object\n type: object\n served: true\n storage: true\nstatus:\n acceptedNames:\n kind: \"\"\n plural: \"\"\n conditions: []\n storedVersions: []\n" ipamhandles = "apiVersion: apiextensions.k8s.io/v1\nkind: CustomResourceDefinition\nmetadata:\n name: ipamhandles.crd.projectcalico.org\nspec:\n group: crd.projectcalico.org\n names:\n kind: IPAMHandle\n listKind: IPAMHandleList\n plural: ipamhandles\n singular: ipamhandle\n preserveUnknownFields: false\n scope: Cluster\n versions:\n - name: v1\n schema:\n openAPIV3Schema:\n properties:\n apiVersion:\n description: 'APIVersion defines the versioned schema of this representation\n of an object. Servers should convert recognized schemas to the latest\n internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'\n type: string\n kind:\n description: 'Kind is a string value representing the REST resource this\n object represents. Servers may infer this from the endpoint the client\n submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'\n type: string\n metadata:\n type: object\n spec:\n description: IPAMHandleSpec contains the specification for an IPAMHandle\n resource.\n properties:\n block:\n additionalProperties:\n type: integer\n type: object\n deleted:\n type: boolean\n handleID:\n type: string\n required:\n - block\n - handleID\n type: object\n type: object\n served: true\n storage: true\nstatus:\n acceptedNames:\n kind: \"\"\n plural: \"\"\n conditions: []\n storedVersions: []\n" - ippools = "apiVersion: apiextensions.k8s.io/v1\nkind: CustomResourceDefinition\nmetadata:\n name: ippools.crd.projectcalico.org\nspec:\n group: crd.projectcalico.org\n names:\n kind: IPPool\n listKind: IPPoolList\n plural: ippools\n singular: ippool\n preserveUnknownFields: false\n scope: Cluster\n versions:\n - name: v1\n schema:\n openAPIV3Schema:\n properties:\n apiVersion:\n description: 'APIVersion defines the versioned schema of this representation\n of an object. Servers should convert recognized schemas to the latest\n internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'\n type: string\n kind:\n description: 'Kind is a string value representing the REST resource this\n object represents. Servers may infer this from the endpoint the client\n submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'\n type: string\n metadata:\n type: object\n spec:\n description: IPPoolSpec contains the specification for an IPPool resource.\n properties:\n allowedUses:\n description: AllowedUse controls what the IP pool will be used for. If\n not specified or empty, defaults to [\"Tunnel\", \"Workload\"] for back-compatibility\n items:\n type: string\n type: array\n blockSize:\n description: The block size to use for IP address assignments from\n this pool. Defaults to 26 for IPv4 and 122 for IPv6.\n type: integer\n cidr:\n description: The pool CIDR.\n type: string\n disableBGPExport:\n description: 'Disable exporting routes from this IP Pool''s CIDR over\n BGP. [Default: false]'\n type: boolean\n disabled:\n description: When disabled is true, Calico IPAM will not assign addresses\n from this pool.\n type: boolean\n ipip:\n description: 'Deprecated: this field is only used for APIv1 backwards\n compatibility. Setting this field is not allowed, this field is\n for internal use only.'\n properties:\n enabled:\n description: When enabled is true, ipip tunneling will be used\n to deliver packets to destinations within this pool.\n type: boolean\n mode:\n description: The IPIP mode. This can be one of \"always\" or \"cross-subnet\". A\n mode of \"always\" will also use IPIP tunneling for routing to\n destination IP addresses within this pool. A mode of \"cross-subnet\"\n will only use IPIP tunneling when the destination node is on\n a different subnet to the originating node. The default value\n (if not specified) is \"always\".\n type: string\n type: object\n ipipMode:\n description: Contains configuration for IPIP tunneling for this pool.\n If not specified, then this is defaulted to \"Never\" (i.e. IPIP tunneling\n is disabled).\n type: string\n nat-outgoing:\n description: 'Deprecated: this field is only used for APIv1 backwards\n compatibility. Setting this field is not allowed, this field is\n for internal use only.'\n type: boolean\n natOutgoing:\n description: When natOutgoing is true, packets sent from Calico networked\n containers in this pool to destinations outside of this pool will\n be masqueraded.\n type: boolean\n nodeSelector:\n description: Allows IPPool to allocate for a specific node by label\n selector.\n type: string\n vxlanMode:\n description: Contains configuration for VXLAN tunneling for this pool.\n If not specified, then this is defaulted to \"Never\" (i.e. VXLAN\n tunneling is disabled).\n type: string\n required:\n - cidr\n type: object\n type: object\n served: true\n storage: true\nstatus:\n acceptedNames:\n kind: \"\"\n plural: \"\"\n conditions: []\n storedVersions: []\n" + ippools = "apiVersion: apiextensions.k8s.io/v1\nkind: CustomResourceDefinition\nmetadata:\n name: ippools.crd.projectcalico.org\nspec:\n group: crd.projectcalico.org\n names:\n kind: IPPool\n listKind: IPPoolList\n plural: ippools\n singular: ippool\n preserveUnknownFields: false\n scope: Cluster\n versions:\n - name: v1\n schema:\n openAPIV3Schema:\n properties:\n apiVersion:\n description: 'APIVersion defines the versioned schema of this representation\n of an object. Servers should convert recognized schemas to the latest\n internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'\n type: string\n kind:\n description: 'Kind is a string value representing the REST resource this\n object represents. Servers may infer this from the endpoint the client\n submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'\n type: string\n metadata:\n type: object\n spec:\n description: IPPoolSpec contains the specification for an IPPool resource.\n properties:\n allowedUses:\n description: AllowedUse controls what the IP pool will be used for. If\n not specified or empty, defaults to [\"Tunnel\", \"Workload\"] for back-compatibility\n items:\n type: string\n type: array\n assignmentMode:\n description: Determines the mode how IP addresses should be assigned\n from this pool\n type: string\n blockSize:\n description: The block size to use for IP address assignments from\n this pool. Defaults to 26 for IPv4 and 122 for IPv6.\n type: integer\n cidr:\n description: The pool CIDR.\n type: string\n disableBGPExport:\n description: 'Disable exporting routes from this IP Pool''s CIDR over\n BGP. [Default: false]'\n type: boolean\n disabled:\n description: When disabled is true, Calico IPAM will not assign addresses\n from this pool.\n type: boolean\n ipip:\n description: 'Deprecated: this field is only used for APIv1 backwards\n compatibility. Setting this field is not allowed, this field is\n for internal use only.'\n properties:\n enabled:\n description: When enabled is true, ipip tunneling will be used\n to deliver packets to destinations within this pool.\n type: boolean\n mode:\n description: The IPIP mode. This can be one of \"always\" or \"cross-subnet\". A\n mode of \"always\" will also use IPIP tunneling for routing to\n destination IP addresses within this pool. A mode of \"cross-subnet\"\n will only use IPIP tunneling when the destination node is on\n a different subnet to the originating node. The default value\n (if not specified) is \"always\".\n type: string\n type: object\n ipipMode:\n description: Contains configuration for IPIP tunneling for this pool.\n If not specified, then this is defaulted to \"Never\" (i.e. IPIP tunneling\n is disabled).\n type: string\n nat-outgoing:\n description: 'Deprecated: this field is only used for APIv1 backwards\n compatibility. Setting this field is not allowed, this field is\n for internal use only.'\n type: boolean\n natOutgoing:\n description: When natOutgoing is true, packets sent from Calico networked\n containers in this pool to destinations outside of this pool will\n be masqueraded.\n type: boolean\n nodeSelector:\n description: Allows IPPool to allocate for a specific node by label\n selector.\n type: string\n vxlanMode:\n description: Contains configuration for VXLAN tunneling for this pool.\n If not specified, then this is defaulted to \"Never\" (i.e. VXLAN\n tunneling is disabled).\n type: string\n required:\n - cidr\n type: object\n type: object\n served: true\n storage: true\nstatus:\n acceptedNames:\n kind: \"\"\n plural: \"\"\n conditions: []\n storedVersions: []\n" ipreservations = "apiVersion: apiextensions.k8s.io/v1\nkind: CustomResourceDefinition\nmetadata:\n annotations:\n controller-gen.kubebuilder.io/version: (devel)\n creationTimestamp: null\n name: ipreservations.crd.projectcalico.org\nspec:\n group: crd.projectcalico.org\n names:\n kind: IPReservation\n listKind: IPReservationList\n plural: ipreservations\n singular: ipreservation\n preserveUnknownFields: false\n scope: Cluster\n versions:\n - name: v1\n schema:\n openAPIV3Schema:\n properties:\n apiVersion:\n description: 'APIVersion defines the versioned schema of this representation\n of an object. Servers should convert recognized schemas to the latest\n internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'\n type: string\n kind:\n description: 'Kind is a string value representing the REST resource this\n object represents. Servers may infer this from the endpoint the client\n submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'\n type: string\n metadata:\n type: object\n spec:\n description: IPReservationSpec contains the specification for an IPReservation\n resource.\n properties:\n reservedCIDRs:\n description: ReservedCIDRs is a list of CIDRs and/or IP addresses\n that Calico IPAM will exclude from new allocations.\n items:\n type: string\n type: array\n type: object\n type: object\n served: true\n storage: true\nstatus:\n acceptedNames:\n kind: \"\"\n plural: \"\"\n conditions: []\n storedVersions: []\n" - kubecontrollersconfigurations = "apiVersion: apiextensions.k8s.io/v1\nkind: CustomResourceDefinition\nmetadata:\n name: kubecontrollersconfigurations.crd.projectcalico.org\nspec:\n group: crd.projectcalico.org\n names:\n kind: KubeControllersConfiguration\n listKind: KubeControllersConfigurationList\n plural: kubecontrollersconfigurations\n singular: kubecontrollersconfiguration\n preserveUnknownFields: false\n scope: Cluster\n versions:\n - name: v1\n schema:\n openAPIV3Schema:\n properties:\n apiVersion:\n description: 'APIVersion defines the versioned schema of this representation\n of an object. Servers should convert recognized schemas to the latest\n internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'\n type: string\n kind:\n description: 'Kind is a string value representing the REST resource this\n object represents. Servers may infer this from the endpoint the client\n submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'\n type: string\n metadata:\n type: object\n spec:\n description: KubeControllersConfigurationSpec contains the values of the\n Kubernetes controllers configuration.\n properties:\n controllers:\n description: Controllers enables and configures individual Kubernetes\n controllers\n properties:\n namespace:\n description: Namespace enables and configures the namespace controller.\n Enabled by default, set to nil to disable.\n properties:\n reconcilerPeriod:\n description: 'ReconcilerPeriod is the period to perform reconciliation\n with the Calico datastore. [Default: 5m]'\n type: string\n type: object\n node:\n description: Node enables and configures the node controller.\n Enabled by default, set to nil to disable.\n properties:\n hostEndpoint:\n description: HostEndpoint controls syncing nodes to host endpoints.\n Disabled by default, set to nil to disable.\n properties:\n autoCreate:\n description: 'AutoCreate enables automatic creation of\n host endpoints for every node. [Default: Disabled]'\n type: string\n type: object\n leakGracePeriod:\n description: 'LeakGracePeriod is the period used by the controller\n to determine if an IP address has been leaked. Set to 0\n to disable IP garbage collection. [Default: 15m]'\n type: string\n reconcilerPeriod:\n description: 'ReconcilerPeriod is the period to perform reconciliation\n with the Calico datastore. [Default: 5m]'\n type: string\n syncLabels:\n description: 'SyncLabels controls whether to copy Kubernetes\n node labels to Calico nodes. [Default: Enabled]'\n type: string\n type: object\n policy:\n description: Policy enables and configures the policy controller.\n Enabled by default, set to nil to disable.\n properties:\n reconcilerPeriod:\n description: 'ReconcilerPeriod is the period to perform reconciliation\n with the Calico datastore. [Default: 5m]'\n type: string\n type: object\n serviceAccount:\n description: ServiceAccount enables and configures the service\n account controller. Enabled by default, set to nil to disable.\n properties:\n reconcilerPeriod:\n description: 'ReconcilerPeriod is the period to perform reconciliation\n with the Calico datastore. [Default: 5m]'\n type: string\n type: object\n workloadEndpoint:\n description: WorkloadEndpoint enables and configures the workload\n endpoint controller. Enabled by default, set to nil to disable.\n properties:\n reconcilerPeriod:\n description: 'ReconcilerPeriod is the period to perform reconciliation\n with the Calico datastore. [Default: 5m]'\n type: string\n type: object\n type: object\n debugProfilePort:\n description: DebugProfilePort configures the port to serve memory\n and cpu profiles on. If not specified, profiling is disabled.\n format: int32\n type: integer\n etcdV3CompactionPeriod:\n description: 'EtcdV3CompactionPeriod is the period between etcdv3\n compaction requests. Set to 0 to disable. [Default: 10m]'\n type: string\n healthChecks:\n description: 'HealthChecks enables or disables support for health\n checks [Default: Enabled]'\n type: string\n logSeverityScreen:\n description: 'LogSeverityScreen is the log severity above which logs\n are sent to the stdout. [Default: Info]'\n type: string\n prometheusMetricsPort:\n description: 'PrometheusMetricsPort is the TCP port that the Prometheus\n metrics server should bind to. Set to 0 to disable. [Default: 9094]'\n type: integer\n required:\n - controllers\n type: object\n status:\n description: KubeControllersConfigurationStatus represents the status\n of the configuration. It's useful for admins to be able to see the actual\n config that was applied, which can be modified by environment variables\n on the kube-controllers process.\n properties:\n environmentVars:\n additionalProperties:\n type: string\n description: EnvironmentVars contains the environment variables on\n the kube-controllers that influenced the RunningConfig.\n type: object\n runningConfig:\n description: RunningConfig contains the effective config that is running\n in the kube-controllers pod, after merging the API resource with\n any environment variables.\n properties:\n controllers:\n description: Controllers enables and configures individual Kubernetes\n controllers\n properties:\n namespace:\n description: Namespace enables and configures the namespace\n controller. Enabled by default, set to nil to disable.\n properties:\n reconcilerPeriod:\n description: 'ReconcilerPeriod is the period to perform\n reconciliation with the Calico datastore. [Default:\n 5m]'\n type: string\n type: object\n node:\n description: Node enables and configures the node controller.\n Enabled by default, set to nil to disable.\n properties:\n hostEndpoint:\n description: HostEndpoint controls syncing nodes to host\n endpoints. Disabled by default, set to nil to disable.\n properties:\n autoCreate:\n description: 'AutoCreate enables automatic creation\n of host endpoints for every node. [Default: Disabled]'\n type: string\n type: object\n leakGracePeriod:\n description: 'LeakGracePeriod is the period used by the\n controller to determine if an IP address has been leaked.\n Set to 0 to disable IP garbage collection. [Default:\n 15m]'\n type: string\n reconcilerPeriod:\n description: 'ReconcilerPeriod is the period to perform\n reconciliation with the Calico datastore. [Default:\n 5m]'\n type: string\n syncLabels:\n description: 'SyncLabels controls whether to copy Kubernetes\n node labels to Calico nodes. [Default: Enabled]'\n type: string\n type: object\n policy:\n description: Policy enables and configures the policy controller.\n Enabled by default, set to nil to disable.\n properties:\n reconcilerPeriod:\n description: 'ReconcilerPeriod is the period to perform\n reconciliation with the Calico datastore. [Default:\n 5m]'\n type: string\n type: object\n serviceAccount:\n description: ServiceAccount enables and configures the service\n account controller. Enabled by default, set to nil to disable.\n properties:\n reconcilerPeriod:\n description: 'ReconcilerPeriod is the period to perform\n reconciliation with the Calico datastore. [Default:\n 5m]'\n type: string\n type: object\n workloadEndpoint:\n description: WorkloadEndpoint enables and configures the workload\n endpoint controller. Enabled by default, set to nil to disable.\n properties:\n reconcilerPeriod:\n description: 'ReconcilerPeriod is the period to perform\n reconciliation with the Calico datastore. [Default:\n 5m]'\n type: string\n type: object\n type: object\n debugProfilePort:\n description: DebugProfilePort configures the port to serve memory\n and cpu profiles on. If not specified, profiling is disabled.\n format: int32\n type: integer\n etcdV3CompactionPeriod:\n description: 'EtcdV3CompactionPeriod is the period between etcdv3\n compaction requests. Set to 0 to disable. [Default: 10m]'\n type: string\n healthChecks:\n description: 'HealthChecks enables or disables support for health\n checks [Default: Enabled]'\n type: string\n logSeverityScreen:\n description: 'LogSeverityScreen is the log severity above which\n logs are sent to the stdout. [Default: Info]'\n type: string\n prometheusMetricsPort:\n description: 'PrometheusMetricsPort is the TCP port that the Prometheus\n metrics server should bind to. Set to 0 to disable. [Default:\n 9094]'\n type: integer\n required:\n - controllers\n type: object\n type: object\n type: object\n served: true\n storage: true\nstatus:\n acceptedNames:\n kind: \"\"\n plural: \"\"\n conditions: []\n storedVersions: []\n" + kubecontrollersconfigurations = "apiVersion: apiextensions.k8s.io/v1\nkind: CustomResourceDefinition\nmetadata:\n name: kubecontrollersconfigurations.crd.projectcalico.org\nspec:\n group: crd.projectcalico.org\n names:\n kind: KubeControllersConfiguration\n listKind: KubeControllersConfigurationList\n plural: kubecontrollersconfigurations\n singular: kubecontrollersconfiguration\n preserveUnknownFields: false\n scope: Cluster\n versions:\n - name: v1\n schema:\n openAPIV3Schema:\n properties:\n apiVersion:\n description: 'APIVersion defines the versioned schema of this representation\n of an object. Servers should convert recognized schemas to the latest\n internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'\n type: string\n kind:\n description: 'Kind is a string value representing the REST resource this\n object represents. Servers may infer this from the endpoint the client\n submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'\n type: string\n metadata:\n type: object\n spec:\n description: KubeControllersConfigurationSpec contains the values of the\n Kubernetes controllers configuration.\n properties:\n controllers:\n description: Controllers enables and configures individual Kubernetes\n controllers\n properties:\n loadBalancer:\n description: LoadBalancer enables and configures the LoadBalancer\n controller. Enabled by default, set to nil to disable.\n properties:\n assignIPs:\n type: string\n type: object\n namespace:\n description: Namespace enables and configures the namespace controller.\n Enabled by default, set to nil to disable.\n properties:\n reconcilerPeriod:\n description: 'ReconcilerPeriod is the period to perform reconciliation\n with the Calico datastore. [Default: 5m]'\n type: string\n type: object\n node:\n description: Node enables and configures the node controller.\n Enabled by default, set to nil to disable.\n properties:\n hostEndpoint:\n description: HostEndpoint controls syncing nodes to host endpoints.\n Disabled by default, set to nil to disable.\n properties:\n autoCreate:\n description: 'AutoCreate enables automatic creation of\n host endpoints for every node. [Default: Disabled]'\n type: string\n type: object\n leakGracePeriod:\n description: 'LeakGracePeriod is the period used by the controller\n to determine if an IP address has been leaked. Set to 0\n to disable IP garbage collection. [Default: 15m]'\n type: string\n reconcilerPeriod:\n description: 'ReconcilerPeriod is the period to perform reconciliation\n with the Calico datastore. [Default: 5m]'\n type: string\n syncLabels:\n description: 'SyncLabels controls whether to copy Kubernetes\n node labels to Calico nodes. [Default: Enabled]'\n type: string\n type: object\n policy:\n description: Policy enables and configures the policy controller.\n Enabled by default, set to nil to disable.\n properties:\n reconcilerPeriod:\n description: 'ReconcilerPeriod is the period to perform reconciliation\n with the Calico datastore. [Default: 5m]'\n type: string\n type: object\n serviceAccount:\n description: ServiceAccount enables and configures the service\n account controller. Enabled by default, set to nil to disable.\n properties:\n reconcilerPeriod:\n description: 'ReconcilerPeriod is the period to perform reconciliation\n with the Calico datastore. [Default: 5m]'\n type: string\n type: object\n workloadEndpoint:\n description: WorkloadEndpoint enables and configures the workload\n endpoint controller. Enabled by default, set to nil to disable.\n properties:\n reconcilerPeriod:\n description: 'ReconcilerPeriod is the period to perform reconciliation\n with the Calico datastore. [Default: 5m]'\n type: string\n type: object\n type: object\n debugProfilePort:\n description: DebugProfilePort configures the port to serve memory\n and cpu profiles on. If not specified, profiling is disabled.\n format: int32\n type: integer\n etcdV3CompactionPeriod:\n description: 'EtcdV3CompactionPeriod is the period between etcdv3\n compaction requests. Set to 0 to disable. [Default: 10m]'\n type: string\n healthChecks:\n description: 'HealthChecks enables or disables support for health\n checks [Default: Enabled]'\n type: string\n logSeverityScreen:\n description: 'LogSeverityScreen is the log severity above which logs\n are sent to the stdout. [Default: Info]'\n type: string\n prometheusMetricsPort:\n description: 'PrometheusMetricsPort is the TCP port that the Prometheus\n metrics server should bind to. Set to 0 to disable. [Default: 9094]'\n type: integer\n required:\n - controllers\n type: object\n status:\n description: KubeControllersConfigurationStatus represents the status\n of the configuration. It's useful for admins to be able to see the actual\n config that was applied, which can be modified by environment variables\n on the kube-controllers process.\n properties:\n environmentVars:\n additionalProperties:\n type: string\n description: EnvironmentVars contains the environment variables on\n the kube-controllers that influenced the RunningConfig.\n type: object\n runningConfig:\n description: RunningConfig contains the effective config that is running\n in the kube-controllers pod, after merging the API resource with\n any environment variables.\n properties:\n controllers:\n description: Controllers enables and configures individual Kubernetes\n controllers\n properties:\n loadBalancer:\n description: LoadBalancer enables and configures the LoadBalancer\n controller. Enabled by default, set to nil to disable.\n properties:\n assignIPs:\n type: string\n type: object\n namespace:\n description: Namespace enables and configures the namespace\n controller. Enabled by default, set to nil to disable.\n properties:\n reconcilerPeriod:\n description: 'ReconcilerPeriod is the period to perform\n reconciliation with the Calico datastore. [Default:\n 5m]'\n type: string\n type: object\n node:\n description: Node enables and configures the node controller.\n Enabled by default, set to nil to disable.\n properties:\n hostEndpoint:\n description: HostEndpoint controls syncing nodes to host\n endpoints. Disabled by default, set to nil to disable.\n properties:\n autoCreate:\n description: 'AutoCreate enables automatic creation\n of host endpoints for every node. [Default: Disabled]'\n type: string\n type: object\n leakGracePeriod:\n description: 'LeakGracePeriod is the period used by the\n controller to determine if an IP address has been leaked.\n Set to 0 to disable IP garbage collection. [Default:\n 15m]'\n type: string\n reconcilerPeriod:\n description: 'ReconcilerPeriod is the period to perform\n reconciliation with the Calico datastore. [Default:\n 5m]'\n type: string\n syncLabels:\n description: 'SyncLabels controls whether to copy Kubernetes\n node labels to Calico nodes. [Default: Enabled]'\n type: string\n type: object\n policy:\n description: Policy enables and configures the policy controller.\n Enabled by default, set to nil to disable.\n properties:\n reconcilerPeriod:\n description: 'ReconcilerPeriod is the period to perform\n reconciliation with the Calico datastore. [Default:\n 5m]'\n type: string\n type: object\n serviceAccount:\n description: ServiceAccount enables and configures the service\n account controller. Enabled by default, set to nil to disable.\n properties:\n reconcilerPeriod:\n description: 'ReconcilerPeriod is the period to perform\n reconciliation with the Calico datastore. [Default:\n 5m]'\n type: string\n type: object\n workloadEndpoint:\n description: WorkloadEndpoint enables and configures the workload\n endpoint controller. Enabled by default, set to nil to disable.\n properties:\n reconcilerPeriod:\n description: 'ReconcilerPeriod is the period to perform\n reconciliation with the Calico datastore. [Default:\n 5m]'\n type: string\n type: object\n type: object\n debugProfilePort:\n description: DebugProfilePort configures the port to serve memory\n and cpu profiles on. If not specified, profiling is disabled.\n format: int32\n type: integer\n etcdV3CompactionPeriod:\n description: 'EtcdV3CompactionPeriod is the period between etcdv3\n compaction requests. Set to 0 to disable. [Default: 10m]'\n type: string\n healthChecks:\n description: 'HealthChecks enables or disables support for health\n checks [Default: Enabled]'\n type: string\n logSeverityScreen:\n description: 'LogSeverityScreen is the log severity above which\n logs are sent to the stdout. [Default: Info]'\n type: string\n prometheusMetricsPort:\n description: 'PrometheusMetricsPort is the TCP port that the Prometheus\n metrics server should bind to. Set to 0 to disable. [Default:\n 9094]'\n type: integer\n required:\n - controllers\n type: object\n type: object\n type: object\n served: true\n storage: true\nstatus:\n acceptedNames:\n kind: \"\"\n plural: \"\"\n conditions: []\n storedVersions: []\n" networkpolicies = "apiVersion: apiextensions.k8s.io/v1\nkind: CustomResourceDefinition\nmetadata:\n name: networkpolicies.crd.projectcalico.org\nspec:\n group: crd.projectcalico.org\n names:\n kind: NetworkPolicy\n listKind: NetworkPolicyList\n plural: networkpolicies\n singular: networkpolicy\n preserveUnknownFields: false\n scope: Namespaced\n versions:\n - name: v1\n schema:\n openAPIV3Schema:\n properties:\n apiVersion:\n description: 'APIVersion defines the versioned schema of this representation\n of an object. Servers should convert recognized schemas to the latest\n internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'\n type: string\n kind:\n description: 'Kind is a string value representing the REST resource this\n object represents. Servers may infer this from the endpoint the client\n submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'\n type: string\n metadata:\n type: object\n spec:\n properties:\n egress:\n description: The ordered set of egress rules. Each rule contains\n a set of packet match criteria and a corresponding action to apply.\n items:\n description: \"A Rule encapsulates a set of match criteria and an\n action. Both selector-based security Policy and security Profiles\n reference rules - separated out as a list of rules for both ingress\n and egress packet matching. \\n Each positive match criteria has\n a negated version, prefixed with \\\"Not\\\". All the match criteria\n within a rule must be satisfied for a packet to match. A single\n rule can contain the positive and negative version of a match\n and both must be satisfied for the rule to match.\"\n properties:\n action:\n type: string\n destination:\n description: Destination contains the match criteria that apply\n to destination entity.\n properties:\n namespaceSelector:\n description: \"NamespaceSelector is an optional field that\n contains a selector expression. Only traffic that originates\n from (or terminates at) endpoints within the selected\n namespaces will be matched. When both NamespaceSelector\n and another selector are defined on the same rule, then\n only workload endpoints that are matched by both selectors\n will be selected by the rule. \\n For NetworkPolicy, an\n empty NamespaceSelector implies that the Selector is limited\n to selecting only workload endpoints in the same namespace\n as the NetworkPolicy. \\n For NetworkPolicy, `global()`\n NamespaceSelector implies that the Selector is limited\n to selecting only GlobalNetworkSet or HostEndpoint. \\n\n For GlobalNetworkPolicy, an empty NamespaceSelector implies\n the Selector applies to workload endpoints across all\n namespaces.\"\n type: string\n nets:\n description: Nets is an optional field that restricts the\n rule to only apply to traffic that originates from (or\n terminates at) IP addresses in any of the given subnets.\n items:\n type: string\n type: array\n notNets:\n description: NotNets is the negated version of the Nets\n field.\n items:\n type: string\n type: array\n notPorts:\n description: NotPorts is the negated version of the Ports\n field. Since only some protocols have ports, if any ports\n are specified it requires the Protocol match in the Rule\n to be set to \"TCP\" or \"UDP\".\n items:\n anyOf:\n - type: integer\n - type: string\n pattern: ^.*\n x-kubernetes-int-or-string: true\n type: array\n notSelector:\n description: NotSelector is the negated version of the Selector\n field. See Selector field for subtleties with negated\n selectors.\n type: string\n ports:\n description: \"Ports is an optional field that restricts\n the rule to only apply to traffic that has a source (destination)\n port that matches one of these ranges/values. This value\n is a list of integers or strings that represent ranges\n of ports. \\n Since only some protocols have ports, if\n any ports are specified it requires the Protocol match\n in the Rule to be set to \\\"TCP\\\" or \\\"UDP\\\".\"\n items:\n anyOf:\n - type: integer\n - type: string\n pattern: ^.*\n x-kubernetes-int-or-string: true\n type: array\n selector:\n description: \"Selector is an optional field that contains\n a selector expression (see Policy for sample syntax).\n \\ Only traffic that originates from (terminates at) endpoints\n matching the selector will be matched. \\n Note that: in\n addition to the negated version of the Selector (see NotSelector\n below), the selector expression syntax itself supports\n negation. The two types of negation are subtly different.\n One negates the set of matched endpoints, the other negates\n the whole match: \\n \\tSelector = \\\"!has(my_label)\\\" matches\n packets that are from other Calico-controlled \\tendpoints\n that do not have the label \\\"my_label\\\". \\n \\tNotSelector\n = \\\"has(my_label)\\\" matches packets that are not from\n Calico-controlled \\tendpoints that do have the label \\\"my_label\\\".\n \\n The effect is that the latter will accept packets from\n non-Calico sources whereas the former is limited to packets\n from Calico-controlled endpoints.\"\n type: string\n serviceAccounts:\n description: ServiceAccounts is an optional field that restricts\n the rule to only apply to traffic that originates from\n (or terminates at) a pod running as a matching service\n account.\n properties:\n names:\n description: Names is an optional field that restricts\n the rule to only apply to traffic that originates\n from (or terminates at) a pod running as a service\n account whose name is in the list.\n items:\n type: string\n type: array\n selector:\n description: Selector is an optional field that restricts\n the rule to only apply to traffic that originates\n from (or terminates at) a pod running as a service\n account that matches the given label selector. If\n both Names and Selector are specified then they are\n AND'ed.\n type: string\n type: object\n services:\n description: \"Services is an optional field that contains\n options for matching Kubernetes Services. If specified,\n only traffic that originates from or terminates at endpoints\n within the selected service(s) will be matched, and only\n to/from each endpoint's port. \\n Services cannot be specified\n on the same rule as Selector, NotSelector, NamespaceSelector,\n Nets, NotNets or ServiceAccounts. \\n Ports and NotPorts\n can only be specified with Services on ingress rules.\"\n properties:\n name:\n description: Name specifies the name of a Kubernetes\n Service to match.\n type: string\n namespace:\n description: Namespace specifies the namespace of the\n given Service. If left empty, the rule will match\n within this policy's namespace.\n type: string\n type: object\n type: object\n http:\n description: HTTP contains match criteria that apply to HTTP\n requests.\n properties:\n methods:\n description: Methods is an optional field that restricts\n the rule to apply only to HTTP requests that use one of\n the listed HTTP Methods (e.g. GET, PUT, etc.) Multiple\n methods are OR'd together.\n items:\n type: string\n type: array\n paths:\n description: 'Paths is an optional field that restricts\n the rule to apply to HTTP requests that use one of the\n listed HTTP Paths. Multiple paths are OR''d together.\n e.g: - exact: /foo - prefix: /bar NOTE: Each entry may\n ONLY specify either a `exact` or a `prefix` match. The\n validator will check for it.'\n items:\n description: 'HTTPPath specifies an HTTP path to match.\n It may be either of the form: exact: : which matches\n the path exactly or prefix: : which matches\n the path prefix'\n properties:\n exact:\n type: string\n prefix:\n type: string\n type: object\n type: array\n type: object\n icmp:\n description: ICMP is an optional field that restricts the rule\n to apply to a specific type and code of ICMP traffic. This\n should only be specified if the Protocol field is set to \"ICMP\"\n or \"ICMPv6\".\n properties:\n code:\n description: Match on a specific ICMP code. If specified,\n the Type value must also be specified. This is a technical\n limitation imposed by the kernel's iptables firewall,\n which Calico uses to enforce the rule.\n type: integer\n type:\n description: Match on a specific ICMP type. For example\n a value of 8 refers to ICMP Echo Request (i.e. pings).\n type: integer\n type: object\n ipVersion:\n description: IPVersion is an optional field that restricts the\n rule to only match a specific IP version.\n type: integer\n metadata:\n description: Metadata contains additional information for this\n rule\n properties:\n annotations:\n additionalProperties:\n type: string\n description: Annotations is a set of key value pairs that\n give extra information about the rule\n type: object\n type: object\n notICMP:\n description: NotICMP is the negated version of the ICMP field.\n properties:\n code:\n description: Match on a specific ICMP code. If specified,\n the Type value must also be specified. This is a technical\n limitation imposed by the kernel's iptables firewall,\n which Calico uses to enforce the rule.\n type: integer\n type:\n description: Match on a specific ICMP type. For example\n a value of 8 refers to ICMP Echo Request (i.e. pings).\n type: integer\n type: object\n notProtocol:\n anyOf:\n - type: integer\n - type: string\n description: NotProtocol is the negated version of the Protocol\n field.\n pattern: ^.*\n x-kubernetes-int-or-string: true\n protocol:\n anyOf:\n - type: integer\n - type: string\n description: \"Protocol is an optional field that restricts the\n rule to only apply to traffic of a specific IP protocol. Required\n if any of the EntityRules contain Ports (because ports only\n apply to certain protocols). \\n Must be one of these string\n values: \\\"TCP\\\", \\\"UDP\\\", \\\"ICMP\\\", \\\"ICMPv6\\\", \\\"SCTP\\\",\n \\\"UDPLite\\\" or an integer in the range 1-255.\"\n pattern: ^.*\n x-kubernetes-int-or-string: true\n source:\n description: Source contains the match criteria that apply to\n source entity.\n properties:\n namespaceSelector:\n description: \"NamespaceSelector is an optional field that\n contains a selector expression. Only traffic that originates\n from (or terminates at) endpoints within the selected\n namespaces will be matched. When both NamespaceSelector\n and another selector are defined on the same rule, then\n only workload endpoints that are matched by both selectors\n will be selected by the rule. \\n For NetworkPolicy, an\n empty NamespaceSelector implies that the Selector is limited\n to selecting only workload endpoints in the same namespace\n as the NetworkPolicy. \\n For NetworkPolicy, `global()`\n NamespaceSelector implies that the Selector is limited\n to selecting only GlobalNetworkSet or HostEndpoint. \\n\n For GlobalNetworkPolicy, an empty NamespaceSelector implies\n the Selector applies to workload endpoints across all\n namespaces.\"\n type: string\n nets:\n description: Nets is an optional field that restricts the\n rule to only apply to traffic that originates from (or\n terminates at) IP addresses in any of the given subnets.\n items:\n type: string\n type: array\n notNets:\n description: NotNets is the negated version of the Nets\n field.\n items:\n type: string\n type: array\n notPorts:\n description: NotPorts is the negated version of the Ports\n field. Since only some protocols have ports, if any ports\n are specified it requires the Protocol match in the Rule\n to be set to \"TCP\" or \"UDP\".\n items:\n anyOf:\n - type: integer\n - type: string\n pattern: ^.*\n x-kubernetes-int-or-string: true\n type: array\n notSelector:\n description: NotSelector is the negated version of the Selector\n field. See Selector field for subtleties with negated\n selectors.\n type: string\n ports:\n description: \"Ports is an optional field that restricts\n the rule to only apply to traffic that has a source (destination)\n port that matches one of these ranges/values. This value\n is a list of integers or strings that represent ranges\n of ports. \\n Since only some protocols have ports, if\n any ports are specified it requires the Protocol match\n in the Rule to be set to \\\"TCP\\\" or \\\"UDP\\\".\"\n items:\n anyOf:\n - type: integer\n - type: string\n pattern: ^.*\n x-kubernetes-int-or-string: true\n type: array\n selector:\n description: \"Selector is an optional field that contains\n a selector expression (see Policy for sample syntax).\n \\ Only traffic that originates from (terminates at) endpoints\n matching the selector will be matched. \\n Note that: in\n addition to the negated version of the Selector (see NotSelector\n below), the selector expression syntax itself supports\n negation. The two types of negation are subtly different.\n One negates the set of matched endpoints, the other negates\n the whole match: \\n \\tSelector = \\\"!has(my_label)\\\" matches\n packets that are from other Calico-controlled \\tendpoints\n that do not have the label \\\"my_label\\\". \\n \\tNotSelector\n = \\\"has(my_label)\\\" matches packets that are not from\n Calico-controlled \\tendpoints that do have the label \\\"my_label\\\".\n \\n The effect is that the latter will accept packets from\n non-Calico sources whereas the former is limited to packets\n from Calico-controlled endpoints.\"\n type: string\n serviceAccounts:\n description: ServiceAccounts is an optional field that restricts\n the rule to only apply to traffic that originates from\n (or terminates at) a pod running as a matching service\n account.\n properties:\n names:\n description: Names is an optional field that restricts\n the rule to only apply to traffic that originates\n from (or terminates at) a pod running as a service\n account whose name is in the list.\n items:\n type: string\n type: array\n selector:\n description: Selector is an optional field that restricts\n the rule to only apply to traffic that originates\n from (or terminates at) a pod running as a service\n account that matches the given label selector. If\n both Names and Selector are specified then they are\n AND'ed.\n type: string\n type: object\n services:\n description: \"Services is an optional field that contains\n options for matching Kubernetes Services. If specified,\n only traffic that originates from or terminates at endpoints\n within the selected service(s) will be matched, and only\n to/from each endpoint's port. \\n Services cannot be specified\n on the same rule as Selector, NotSelector, NamespaceSelector,\n Nets, NotNets or ServiceAccounts. \\n Ports and NotPorts\n can only be specified with Services on ingress rules.\"\n properties:\n name:\n description: Name specifies the name of a Kubernetes\n Service to match.\n type: string\n namespace:\n description: Namespace specifies the namespace of the\n given Service. If left empty, the rule will match\n within this policy's namespace.\n type: string\n type: object\n type: object\n required:\n - action\n type: object\n type: array\n ingress:\n description: The ordered set of ingress rules. Each rule contains\n a set of packet match criteria and a corresponding action to apply.\n items:\n description: \"A Rule encapsulates a set of match criteria and an\n action. Both selector-based security Policy and security Profiles\n reference rules - separated out as a list of rules for both ingress\n and egress packet matching. \\n Each positive match criteria has\n a negated version, prefixed with \\\"Not\\\". All the match criteria\n within a rule must be satisfied for a packet to match. A single\n rule can contain the positive and negative version of a match\n and both must be satisfied for the rule to match.\"\n properties:\n action:\n type: string\n destination:\n description: Destination contains the match criteria that apply\n to destination entity.\n properties:\n namespaceSelector:\n description: \"NamespaceSelector is an optional field that\n contains a selector expression. Only traffic that originates\n from (or terminates at) endpoints within the selected\n namespaces will be matched. When both NamespaceSelector\n and another selector are defined on the same rule, then\n only workload endpoints that are matched by both selectors\n will be selected by the rule. \\n For NetworkPolicy, an\n empty NamespaceSelector implies that the Selector is limited\n to selecting only workload endpoints in the same namespace\n as the NetworkPolicy. \\n For NetworkPolicy, `global()`\n NamespaceSelector implies that the Selector is limited\n to selecting only GlobalNetworkSet or HostEndpoint. \\n\n For GlobalNetworkPolicy, an empty NamespaceSelector implies\n the Selector applies to workload endpoints across all\n namespaces.\"\n type: string\n nets:\n description: Nets is an optional field that restricts the\n rule to only apply to traffic that originates from (or\n terminates at) IP addresses in any of the given subnets.\n items:\n type: string\n type: array\n notNets:\n description: NotNets is the negated version of the Nets\n field.\n items:\n type: string\n type: array\n notPorts:\n description: NotPorts is the negated version of the Ports\n field. Since only some protocols have ports, if any ports\n are specified it requires the Protocol match in the Rule\n to be set to \"TCP\" or \"UDP\".\n items:\n anyOf:\n - type: integer\n - type: string\n pattern: ^.*\n x-kubernetes-int-or-string: true\n type: array\n notSelector:\n description: NotSelector is the negated version of the Selector\n field. See Selector field for subtleties with negated\n selectors.\n type: string\n ports:\n description: \"Ports is an optional field that restricts\n the rule to only apply to traffic that has a source (destination)\n port that matches one of these ranges/values. This value\n is a list of integers or strings that represent ranges\n of ports. \\n Since only some protocols have ports, if\n any ports are specified it requires the Protocol match\n in the Rule to be set to \\\"TCP\\\" or \\\"UDP\\\".\"\n items:\n anyOf:\n - type: integer\n - type: string\n pattern: ^.*\n x-kubernetes-int-or-string: true\n type: array\n selector:\n description: \"Selector is an optional field that contains\n a selector expression (see Policy for sample syntax).\n \\ Only traffic that originates from (terminates at) endpoints\n matching the selector will be matched. \\n Note that: in\n addition to the negated version of the Selector (see NotSelector\n below), the selector expression syntax itself supports\n negation. The two types of negation are subtly different.\n One negates the set of matched endpoints, the other negates\n the whole match: \\n \\tSelector = \\\"!has(my_label)\\\" matches\n packets that are from other Calico-controlled \\tendpoints\n that do not have the label \\\"my_label\\\". \\n \\tNotSelector\n = \\\"has(my_label)\\\" matches packets that are not from\n Calico-controlled \\tendpoints that do have the label \\\"my_label\\\".\n \\n The effect is that the latter will accept packets from\n non-Calico sources whereas the former is limited to packets\n from Calico-controlled endpoints.\"\n type: string\n serviceAccounts:\n description: ServiceAccounts is an optional field that restricts\n the rule to only apply to traffic that originates from\n (or terminates at) a pod running as a matching service\n account.\n properties:\n names:\n description: Names is an optional field that restricts\n the rule to only apply to traffic that originates\n from (or terminates at) a pod running as a service\n account whose name is in the list.\n items:\n type: string\n type: array\n selector:\n description: Selector is an optional field that restricts\n the rule to only apply to traffic that originates\n from (or terminates at) a pod running as a service\n account that matches the given label selector. If\n both Names and Selector are specified then they are\n AND'ed.\n type: string\n type: object\n services:\n description: \"Services is an optional field that contains\n options for matching Kubernetes Services. If specified,\n only traffic that originates from or terminates at endpoints\n within the selected service(s) will be matched, and only\n to/from each endpoint's port. \\n Services cannot be specified\n on the same rule as Selector, NotSelector, NamespaceSelector,\n Nets, NotNets or ServiceAccounts. \\n Ports and NotPorts\n can only be specified with Services on ingress rules.\"\n properties:\n name:\n description: Name specifies the name of a Kubernetes\n Service to match.\n type: string\n namespace:\n description: Namespace specifies the namespace of the\n given Service. If left empty, the rule will match\n within this policy's namespace.\n type: string\n type: object\n type: object\n http:\n description: HTTP contains match criteria that apply to HTTP\n requests.\n properties:\n methods:\n description: Methods is an optional field that restricts\n the rule to apply only to HTTP requests that use one of\n the listed HTTP Methods (e.g. GET, PUT, etc.) Multiple\n methods are OR'd together.\n items:\n type: string\n type: array\n paths:\n description: 'Paths is an optional field that restricts\n the rule to apply to HTTP requests that use one of the\n listed HTTP Paths. Multiple paths are OR''d together.\n e.g: - exact: /foo - prefix: /bar NOTE: Each entry may\n ONLY specify either a `exact` or a `prefix` match. The\n validator will check for it.'\n items:\n description: 'HTTPPath specifies an HTTP path to match.\n It may be either of the form: exact: : which matches\n the path exactly or prefix: : which matches\n the path prefix'\n properties:\n exact:\n type: string\n prefix:\n type: string\n type: object\n type: array\n type: object\n icmp:\n description: ICMP is an optional field that restricts the rule\n to apply to a specific type and code of ICMP traffic. This\n should only be specified if the Protocol field is set to \"ICMP\"\n or \"ICMPv6\".\n properties:\n code:\n description: Match on a specific ICMP code. If specified,\n the Type value must also be specified. This is a technical\n limitation imposed by the kernel's iptables firewall,\n which Calico uses to enforce the rule.\n type: integer\n type:\n description: Match on a specific ICMP type. For example\n a value of 8 refers to ICMP Echo Request (i.e. pings).\n type: integer\n type: object\n ipVersion:\n description: IPVersion is an optional field that restricts the\n rule to only match a specific IP version.\n type: integer\n metadata:\n description: Metadata contains additional information for this\n rule\n properties:\n annotations:\n additionalProperties:\n type: string\n description: Annotations is a set of key value pairs that\n give extra information about the rule\n type: object\n type: object\n notICMP:\n description: NotICMP is the negated version of the ICMP field.\n properties:\n code:\n description: Match on a specific ICMP code. If specified,\n the Type value must also be specified. This is a technical\n limitation imposed by the kernel's iptables firewall,\n which Calico uses to enforce the rule.\n type: integer\n type:\n description: Match on a specific ICMP type. For example\n a value of 8 refers to ICMP Echo Request (i.e. pings).\n type: integer\n type: object\n notProtocol:\n anyOf:\n - type: integer\n - type: string\n description: NotProtocol is the negated version of the Protocol\n field.\n pattern: ^.*\n x-kubernetes-int-or-string: true\n protocol:\n anyOf:\n - type: integer\n - type: string\n description: \"Protocol is an optional field that restricts the\n rule to only apply to traffic of a specific IP protocol. Required\n if any of the EntityRules contain Ports (because ports only\n apply to certain protocols). \\n Must be one of these string\n values: \\\"TCP\\\", \\\"UDP\\\", \\\"ICMP\\\", \\\"ICMPv6\\\", \\\"SCTP\\\",\n \\\"UDPLite\\\" or an integer in the range 1-255.\"\n pattern: ^.*\n x-kubernetes-int-or-string: true\n source:\n description: Source contains the match criteria that apply to\n source entity.\n properties:\n namespaceSelector:\n description: \"NamespaceSelector is an optional field that\n contains a selector expression. Only traffic that originates\n from (or terminates at) endpoints within the selected\n namespaces will be matched. When both NamespaceSelector\n and another selector are defined on the same rule, then\n only workload endpoints that are matched by both selectors\n will be selected by the rule. \\n For NetworkPolicy, an\n empty NamespaceSelector implies that the Selector is limited\n to selecting only workload endpoints in the same namespace\n as the NetworkPolicy. \\n For NetworkPolicy, `global()`\n NamespaceSelector implies that the Selector is limited\n to selecting only GlobalNetworkSet or HostEndpoint. \\n\n For GlobalNetworkPolicy, an empty NamespaceSelector implies\n the Selector applies to workload endpoints across all\n namespaces.\"\n type: string\n nets:\n description: Nets is an optional field that restricts the\n rule to only apply to traffic that originates from (or\n terminates at) IP addresses in any of the given subnets.\n items:\n type: string\n type: array\n notNets:\n description: NotNets is the negated version of the Nets\n field.\n items:\n type: string\n type: array\n notPorts:\n description: NotPorts is the negated version of the Ports\n field. Since only some protocols have ports, if any ports\n are specified it requires the Protocol match in the Rule\n to be set to \"TCP\" or \"UDP\".\n items:\n anyOf:\n - type: integer\n - type: string\n pattern: ^.*\n x-kubernetes-int-or-string: true\n type: array\n notSelector:\n description: NotSelector is the negated version of the Selector\n field. See Selector field for subtleties with negated\n selectors.\n type: string\n ports:\n description: \"Ports is an optional field that restricts\n the rule to only apply to traffic that has a source (destination)\n port that matches one of these ranges/values. This value\n is a list of integers or strings that represent ranges\n of ports. \\n Since only some protocols have ports, if\n any ports are specified it requires the Protocol match\n in the Rule to be set to \\\"TCP\\\" or \\\"UDP\\\".\"\n items:\n anyOf:\n - type: integer\n - type: string\n pattern: ^.*\n x-kubernetes-int-or-string: true\n type: array\n selector:\n description: \"Selector is an optional field that contains\n a selector expression (see Policy for sample syntax).\n \\ Only traffic that originates from (terminates at) endpoints\n matching the selector will be matched. \\n Note that: in\n addition to the negated version of the Selector (see NotSelector\n below), the selector expression syntax itself supports\n negation. The two types of negation are subtly different.\n One negates the set of matched endpoints, the other negates\n the whole match: \\n \\tSelector = \\\"!has(my_label)\\\" matches\n packets that are from other Calico-controlled \\tendpoints\n that do not have the label \\\"my_label\\\". \\n \\tNotSelector\n = \\\"has(my_label)\\\" matches packets that are not from\n Calico-controlled \\tendpoints that do have the label \\\"my_label\\\".\n \\n The effect is that the latter will accept packets from\n non-Calico sources whereas the former is limited to packets\n from Calico-controlled endpoints.\"\n type: string\n serviceAccounts:\n description: ServiceAccounts is an optional field that restricts\n the rule to only apply to traffic that originates from\n (or terminates at) a pod running as a matching service\n account.\n properties:\n names:\n description: Names is an optional field that restricts\n the rule to only apply to traffic that originates\n from (or terminates at) a pod running as a service\n account whose name is in the list.\n items:\n type: string\n type: array\n selector:\n description: Selector is an optional field that restricts\n the rule to only apply to traffic that originates\n from (or terminates at) a pod running as a service\n account that matches the given label selector. If\n both Names and Selector are specified then they are\n AND'ed.\n type: string\n type: object\n services:\n description: \"Services is an optional field that contains\n options for matching Kubernetes Services. If specified,\n only traffic that originates from or terminates at endpoints\n within the selected service(s) will be matched, and only\n to/from each endpoint's port. \\n Services cannot be specified\n on the same rule as Selector, NotSelector, NamespaceSelector,\n Nets, NotNets or ServiceAccounts. \\n Ports and NotPorts\n can only be specified with Services on ingress rules.\"\n properties:\n name:\n description: Name specifies the name of a Kubernetes\n Service to match.\n type: string\n namespace:\n description: Namespace specifies the namespace of the\n given Service. If left empty, the rule will match\n within this policy's namespace.\n type: string\n type: object\n type: object\n required:\n - action\n type: object\n type: array\n order:\n description: Order is an optional field that specifies the order in\n which the policy is applied. Policies with higher \"order\" are applied\n after those with lower order within the same tier. If the order\n is omitted, it may be considered to be \"infinite\" - i.e. the policy\n will be applied last. Policies with identical order will be applied\n in alphanumerical order based on the Policy \"Name\" within the tier.\n type: number\n performanceHints:\n description: \"PerformanceHints contains a list of hints to Calico's\n policy engine to help process the policy more efficiently. Hints\n never change the enforcement behaviour of the policy. \\n Currently,\n the only available hint is \\\"AssumeNeededOnEveryNode\\\". When that\n hint is set on a policy, Felix will act as if the policy matches\n a local endpoint even if it does not. This is useful for \\\"preloading\\\"\n any large static policies that are known to be used on every node.\n If the policy is _not_ used on a particular node then the work done\n to preload the policy (and to maintain it) is wasted.\"\n items:\n type: string\n type: array\n selector:\n description: \"The selector is an expression used to pick out the endpoints\n that the policy should be applied to. \\n Selector expressions follow\n this syntax: \\n \\tlabel == \\\"string_literal\\\" -> comparison, e.g.\n my_label == \\\"foo bar\\\" \\tlabel != \\\"string_literal\\\" -> not\n equal; also matches if label is not present \\tlabel in { \\\"a\\\",\n \\\"b\\\", \\\"c\\\", ... } -> true if the value of label X is one of\n \\\"a\\\", \\\"b\\\", \\\"c\\\" \\tlabel not in { \\\"a\\\", \\\"b\\\", \\\"c\\\", ... }\n \\ -> true if the value of label X is not one of \\\"a\\\", \\\"b\\\", \\\"c\\\"\n \\thas(label_name) -> True if that label is present \\t! expr ->\n negation of expr \\texpr && expr -> Short-circuit and \\texpr ||\n expr -> Short-circuit or \\t( expr ) -> parens for grouping \\tall()\n or the empty selector -> matches all endpoints. \\n Label names are\n allowed to contain alphanumerics, -, _ and /. String literals are\n more permissive but they do not support escape characters. \\n Examples\n (with made-up labels): \\n \\ttype == \\\"webserver\\\" && deployment\n == \\\"prod\\\" \\ttype in {\\\"frontend\\\", \\\"backend\\\"} \\tdeployment !=\n \\\"dev\\\" \\t! has(label_name)\"\n type: string\n serviceAccountSelector:\n description: ServiceAccountSelector is an optional field for an expression\n used to select a pod based on service accounts.\n type: string\n tier:\n description: The name of the tier that this policy belongs to. If\n this is omitted, the default tier (name is \"default\") is assumed. The\n specified tier must exist in order to create security policies within\n the tier, the \"default\" tier is created automatically if it does\n not exist, this means for deployments requiring only a single Tier,\n the tier name may be omitted on all policy management requests.\n type: string\n types:\n description: \"Types indicates whether this policy applies to ingress,\n or to egress, or to both. When not explicitly specified (and so\n the value on creation is empty or nil), Calico defaults Types according\n to what Ingress and Egress are present in the policy. The default\n is: \\n - [ PolicyTypeIngress ], if there are no Egress rules (including\n the case where there are also no Ingress rules) \\n - [ PolicyTypeEgress\n ], if there are Egress rules but no Ingress rules \\n - [ PolicyTypeIngress,\n PolicyTypeEgress ], if there are both Ingress and Egress rules.\n \\n When the policy is read back again, Types will always be one\n of these values, never empty or nil.\"\n items:\n description: PolicyType enumerates the possible values of the PolicySpec\n Types field.\n type: string\n type: array\n type: object\n type: object\n served: true\n storage: true\nstatus:\n acceptedNames:\n kind: \"\"\n plural: \"\"\n conditions: []\n storedVersions: []\n" networksets = "apiVersion: apiextensions.k8s.io/v1\nkind: CustomResourceDefinition\nmetadata:\n name: networksets.crd.projectcalico.org\nspec:\n group: crd.projectcalico.org\n names:\n kind: NetworkSet\n listKind: NetworkSetList\n plural: networksets\n singular: networkset\n preserveUnknownFields: false\n scope: Namespaced\n versions:\n - name: v1\n schema:\n openAPIV3Schema:\n description: NetworkSet is the Namespaced-equivalent of the GlobalNetworkSet.\n properties:\n apiVersion:\n description: 'APIVersion defines the versioned schema of this representation\n of an object. Servers should convert recognized schemas to the latest\n internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'\n type: string\n kind:\n description: 'Kind is a string value representing the REST resource this\n object represents. Servers may infer this from the endpoint the client\n submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'\n type: string\n metadata:\n type: object\n spec:\n description: NetworkSetSpec contains the specification for a NetworkSet\n resource.\n properties:\n nets:\n description: The list of IP networks that belong to this set.\n items:\n type: string\n type: array\n type: object\n type: object\n served: true\n storage: true\nstatus:\n acceptedNames:\n kind: \"\"\n plural: \"\"\n conditions: []\n storedVersions: []\n" tiers = "apiVersion: apiextensions.k8s.io/v1\nkind: CustomResourceDefinition\nmetadata:\n annotations:\n controller-gen.kubebuilder.io/version: (devel)\n creationTimestamp: null\n name: tiers.crd.projectcalico.org\nspec:\n group: crd.projectcalico.org\n names:\n kind: Tier\n listKind: TierList\n plural: tiers\n singular: tier\n scope: Cluster\n versions:\n - name: v1\n schema:\n openAPIV3Schema:\n properties:\n apiVersion:\n description: 'APIVersion defines the versioned schema of this representation\n of an object. Servers should convert recognized schemas to the latest\n internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'\n type: string\n kind:\n description: 'Kind is a string value representing the REST resource this\n object represents. Servers may infer this from the endpoint the client\n submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'\n type: string\n metadata:\n type: object\n spec:\n description: TierSpec contains the specification for a security policy\n tier resource.\n properties:\n order:\n description: Order is an optional field that specifies the order in\n which the tier is applied. Tiers with higher \"order\" are applied\n after those with lower order. If the order is omitted, it may be\n considered to be \"infinite\" - i.e. the tier will be applied last. Tiers\n with identical order will be applied in alphanumerical order based\n on the Tier \"Name\".\n type: number\n type: object\n type: object\n served: true\n storage: true\nstatus:\n acceptedNames:\n kind: \"\"\n plural: \"\"\n conditions: []\n storedVersions: []\n" From dc729fd0818af1fd8613bb2c62f45bf618e987b5 Mon Sep 17 00:00:00 2001 From: Michal Fupso Date: Mon, 16 Sep 2024 07:53:38 -0700 Subject: [PATCH 03/12] LoadBalancer kube-controller update --- api/pkg/apis/projectcalico/v3/constants.go | 4 - api/pkg/apis/projectcalico/v3/ippool.go | 1 + .../projectcalico/v3/kubecontrollersconfig.go | 9 +- calicoctl/calicoctl/commands/ipam/check.go | 9 +- kube-controllers/cmd/kube-controllers/main.go | 7 +- kube-controllers/pkg/config/config.go | 1 - kube-controllers/pkg/config/config_test.go | 7 +- kube-controllers/pkg/config/runconfig.go | 7 +- .../loadbalancer/loadbalancer_controller.go | 632 +++++++++--------- .../pkg/controllers/node/controller.go | 5 +- libcalico-go/lib/backend/model/block.go | 2 +- libcalico-go/lib/clientv3/ippool.go | 3 +- libcalico-go/lib/clientv3/ippool_e2e_test.go | 9 - libcalico-go/lib/ipam/ipam.go | 33 +- .../lib/ipam/ipam_block_reader_writer.go | 6 +- 15 files changed, 371 insertions(+), 364 deletions(-) diff --git a/api/pkg/apis/projectcalico/v3/constants.go b/api/pkg/apis/projectcalico/v3/constants.go index 3f7742cf1eb..6d98397f374 100644 --- a/api/pkg/apis/projectcalico/v3/constants.go +++ b/api/pkg/apis/projectcalico/v3/constants.go @@ -54,10 +54,6 @@ const ( Enabled = "Enabled" Disabled = "Disabled" - // Enum options for AllServices/RequestedServicesOnly - AllServices = "AllServices" - RequestedServicesOnly = "RequestedServicesOnly" - // Host name used for Service LoadBalancer VirtualLoadBalancer = "virtual-load-balancer" ) diff --git a/api/pkg/apis/projectcalico/v3/ippool.go b/api/pkg/apis/projectcalico/v3/ippool.go index c5f97e17292..1b74c825412 100644 --- a/api/pkg/apis/projectcalico/v3/ippool.go +++ b/api/pkg/apis/projectcalico/v3/ippool.go @@ -124,6 +124,7 @@ const ( CrossSubnet EncapMode = "cross-subnet" ) +// +kubebuilder:validation:Enum=Automatic;Manual type AssignmentMode string const ( diff --git a/api/pkg/apis/projectcalico/v3/kubecontrollersconfig.go b/api/pkg/apis/projectcalico/v3/kubecontrollersconfig.go index 56c38e384a9..39565555065 100644 --- a/api/pkg/apis/projectcalico/v3/kubecontrollersconfig.go +++ b/api/pkg/apis/projectcalico/v3/kubecontrollersconfig.go @@ -141,9 +141,16 @@ type NamespaceControllerConfig struct { } type LoadBalancerControllerConfig struct { - AssignIPs string `json:"assignIPs,omitempty" validate:"omitempty,assignIPs"` + AssignIPs AssignIPs `json:"assignIPs,omitempty" validate:"omitempty,assignIPs"` } +type AssignIPs string + +const ( + AllServices AssignIPs = "AllServices" + RequestedServicesOnly AssignIPs = "RequestedServicesOnly" +) + // KubeControllersConfigurationStatus represents the status of the configuration. It's useful for admins to // be able to see the actual config that was applied, which can be modified by environment variables on the // kube-controllers process. diff --git a/calicoctl/calicoctl/commands/ipam/check.go b/calicoctl/calicoctl/commands/ipam/check.go index db0e7c833af..74180ce975f 100644 --- a/calicoctl/calicoctl/commands/ipam/check.go +++ b/calicoctl/calicoctl/commands/ipam/check.go @@ -24,6 +24,7 @@ import ( "strings" "github.com/docopt/docopt-go" + "github.com/projectcalico/calico/kube-controllers/pkg/controllers/loadbalancer" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" @@ -264,10 +265,16 @@ func (c *IPAMChecker) checkIPAM(ctx context.Context) error { return err } + kubeControllerConfig, err := c.v3Client.KubeControllersConfiguration().Get(ctx, "default", options.GetOptions{}) + if err != nil { + return err + } + var lengthLoadBalancer int for _, svc := range service.Items { - if svc.Spec.Type == corev1.ServiceTypeLoadBalancer && svc.Status.LoadBalancer.Ingress != nil { + if svc.Spec.Type == corev1.ServiceTypeLoadBalancer && + loadbalancer.IsCalicoManagedLoadBalancer(&svc, kubeControllerConfig.Spec.Controllers.LoadBalancer.AssignIPs) { lengthLoadBalancer++ for _, ingress := range svc.Status.LoadBalancer.Ingress { c.recordInUseIP(ingress.IP, svc, svc.Name) diff --git a/kube-controllers/cmd/kube-controllers/main.go b/kube-controllers/cmd/kube-controllers/main.go index 82a635d87de..6bd6b406e6e 100644 --- a/kube-controllers/cmd/kube-controllers/main.go +++ b/kube-controllers/cmd/kube-controllers/main.go @@ -431,6 +431,8 @@ func (cc *controllerControl) InitControllers(ctx context.Context, cfg config.Run factory := informers.NewSharedInformerFactory(k8sClientset, 0) podInformer := factory.Core().V1().Pods().Informer() nodeInformer := factory.Core().V1().Nodes().Informer() + serviceInformer := factory.Core().V1().Services().Informer() + dataFeed := node.NewDataFeed(calicoClient) if cfg.Controllers.WorkloadEndpoint != nil { podController := pod.NewPodController(ctx, k8sClientset, calicoClient, *cfg.Controllers.WorkloadEndpoint, podInformer) @@ -447,7 +449,7 @@ func (cc *controllerControl) InitControllers(ctx context.Context, cfg config.Run cc.controllers["NetworkPolicy"] = policyController } if cfg.Controllers.Node != nil { - nodeController := node.NewNodeController(ctx, k8sClientset, calicoClient, *cfg.Controllers.Node, nodeInformer, podInformer) + nodeController := node.NewNodeController(ctx, k8sClientset, calicoClient, *cfg.Controllers.Node, nodeInformer, podInformer, dataFeed) cc.controllers["Node"] = nodeController cc.registerInformers(podInformer, nodeInformer) } @@ -457,8 +459,9 @@ func (cc *controllerControl) InitControllers(ctx context.Context, cfg config.Run } if cfg.Controllers.LoadBalancer != nil { - loadBalancerController := loadbalancer.NewLoadBalancerController(ctx, k8sClientset, calicoClient, *cfg.Controllers.LoadBalancer) + loadBalancerController := loadbalancer.NewLoadBalancerController(k8sClientset, calicoClient, *cfg.Controllers.LoadBalancer, serviceInformer, dataFeed) cc.controllers["LoadBalancer"] = loadBalancerController + cc.registerInformers(serviceInformer) } } diff --git a/kube-controllers/pkg/config/config.go b/kube-controllers/pkg/config/config.go index 2b4aaaae849..65fab642ad9 100644 --- a/kube-controllers/pkg/config/config.go +++ b/kube-controllers/pkg/config/config.go @@ -40,7 +40,6 @@ type Config struct { ProfileWorkers int `default:"1" split_words:"true"` PolicyWorkers int `default:"1" split_words:"true"` NodeWorkers int `default:"1" split_words:"true"` - LoadBalancerWorkers int `default:"1" split_words:"true"` // Path to a kubeconfig file to use for accessing the k8s API. Kubeconfig string `default:"" split_words:"false"` diff --git a/kube-controllers/pkg/config/config_test.go b/kube-controllers/pkg/config/config_test.go index b973c55ac59..b1676cbdcee 100644 --- a/kube-controllers/pkg/config/config_test.go +++ b/kube-controllers/pkg/config/config_test.go @@ -94,7 +94,6 @@ var _ = Describe("Config", func() { Expect(cfg.WorkloadEndpointWorkers).To(Equal(1)) Expect(cfg.ProfileWorkers).To(Equal(1)) Expect(cfg.PolicyWorkers).To(Equal(1)) - Expect(cfg.LoadBalancerWorkers).To(Equal(1)) Expect(cfg.Kubeconfig).To(Equal("")) }) @@ -144,8 +143,7 @@ var _ = Describe("Config", func() { NumberOfWorkers: 1, })) Expect(rc.LoadBalancer).To(Equal(&config.LoadBalancerControllerConfig{ - NumberOfWorkers: 1, - AssignIPs: v3.AllServices, + AssignIPs: v3.AllServices, })) close(done) }) @@ -249,8 +247,7 @@ var _ = Describe("Config", func() { NumberOfWorkers: 1, })) Expect(rc.LoadBalancer).To(Equal(&config.LoadBalancerControllerConfig{ - NumberOfWorkers: 1, - AssignIPs: v3.RequestedServicesOnly, + AssignIPs: v3.RequestedServicesOnly, })) close(done) }) diff --git a/kube-controllers/pkg/config/runconfig.go b/kube-controllers/pkg/config/runconfig.go index 9944dbf3e2c..8479fda8daf 100644 --- a/kube-controllers/pkg/config/runconfig.go +++ b/kube-controllers/pkg/config/runconfig.go @@ -115,10 +115,8 @@ type NodeControllerConfig struct { } type LoadBalancerControllerConfig struct { - NumberOfWorkers int - - //Determines if LoadBalancer controller will auto-assign ip addresses or only if asked to do so via annotation - AssignIPs string + // AssignIPs indicates if LoadBalancer controller will auto-assign ip addresses or only if asked to do so via annotation + AssignIPs v3.AssignIPs } type RunConfigController struct { @@ -370,7 +368,6 @@ func mergeConfig(envVars map[string]string, envCfg Config, apiCfg v3.KubeControl } if rc.LoadBalancer != nil { - rc.LoadBalancer.NumberOfWorkers = envCfg.LoadBalancerWorkers rc.LoadBalancer.AssignIPs = apiCfg.Controllers.LoadBalancer.AssignIPs } diff --git a/kube-controllers/pkg/controllers/loadbalancer/loadbalancer_controller.go b/kube-controllers/pkg/controllers/loadbalancer/loadbalancer_controller.go index a2c765c8e49..38dc27f94a1 100644 --- a/kube-controllers/pkg/controllers/loadbalancer/loadbalancer_controller.go +++ b/kube-controllers/pkg/controllers/loadbalancer/loadbalancer_controller.go @@ -16,10 +16,12 @@ package loadbalancer import ( "context" + "crypto/sha256" + "encoding/base64" "errors" "fmt" "net" - "reflect" + "regexp" "slices" "strings" "time" @@ -30,20 +32,18 @@ import ( "github.com/projectcalico/calico/kube-controllers/pkg/controllers/node" bapi "github.com/projectcalico/calico/libcalico-go/lib/backend/api" "github.com/projectcalico/calico/libcalico-go/lib/backend/model" - "github.com/projectcalico/calico/libcalico-go/lib/ipam" - log "github.com/sirupsen/logrus" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - k8svalidation "k8s.io/apimachinery/pkg/util/validation" - "k8s.io/apimachinery/pkg/util/wait" - - rcache "github.com/projectcalico/calico/kube-controllers/pkg/cache" client "github.com/projectcalico/calico/libcalico-go/lib/clientv3" - + "github.com/projectcalico/calico/libcalico-go/lib/ipam" + "github.com/projectcalico/calico/libcalico-go/lib/json" cnet "github.com/projectcalico/calico/libcalico-go/lib/net" + log "github.com/sirupsen/logrus" v1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/fields" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" uruntime "k8s.io/apimachinery/pkg/util/runtime" + k8svalidation "k8s.io/apimachinery/pkg/util/validation" "k8s.io/client-go/kubernetes" + v1lister "k8s.io/client-go/listers/core/v1" "k8s.io/client-go/tools/cache" ) @@ -51,136 +51,124 @@ const ( annotationIpv4Pools = "projectcalico.org/ipv4pools" annotationIpv6Pools = "projectcalico.org/ipv6pools" annotationLoadBalancerIp = "projectcalico.org/loadBalancerIPs" - timer = 1 * time.Minute + timer = 5 * time.Minute +) + +type serviceObject struct { + handle string + updateType serviceUpdateType + service *v1.Service +} + +type serviceUpdateType string + +const ( + serviceUpdateTypeADD serviceUpdateType = "ADD" + serviceUpdateTypeUPDATE serviceUpdateType = "UPDATE" + serviceUpdateTypeDELETE serviceUpdateType = "DELETE" ) // loadBalancerController implements the Controller interface for managing Kubernetes services // and endpoints, syncing them to the Calico datastore as NetworkSet. type loadBalancerController struct { - informer cache.Controller - lbResourceCache rcache.ResourceCache - calicoClient client.Interface - dataFeed *node.DataFeed - ctx context.Context - cfg config.LoadBalancerControllerConfig - clientSet *kubernetes.Clientset - syncerUpdates chan interface{} - syncStatus bapi.SyncStatus - syncChan chan interface{} - ipamBlocks map[string]model.KVPair - ipPools map[string]api.IPPool + calicoClient client.Interface + dataFeed *node.DataFeed + cfg config.LoadBalancerControllerConfig + clientSet *kubernetes.Clientset + syncerUpdates chan interface{} + syncStatus bapi.SyncStatus + syncChan chan interface{} + ipamBlocks map[string]model.KVPair + ipPools map[string]api.IPPool + serviceInformer cache.SharedIndexInformer + serviceLister v1lister.ServiceLister + servicesToUpdate map[string]serviceObject } // NewLoadBalancerController returns a controller which manages Service LoadBalancer objects. -func NewLoadBalancerController(ctx context.Context, clientset *kubernetes.Clientset, calicoClient client.Interface, cfg config.LoadBalancerControllerConfig) controller.Controller { - // set up service informer - svcWatcher := cache.NewListWatchFromClient(clientset.CoreV1().RESTClient(), "services", "", fields.Everything()) - - listFunc := func() (map[string]interface{}, error) { - log.Debugf("Listing profiles from Calico datastore") - filteredServices := make(map[string]interface{}) - - // Get all profile objects from Calico datastore. - serviceList, err := clientset.CoreV1().Services("").List(ctx, metav1.ListOptions{}) - if err != nil { - return nil, err - } - - // Filter out only services of type LoadBalancer - for _, svc := range serviceList.Items { - if svc.Spec.Type == v1.ServiceTypeLoadBalancer { - key := createHandle(&svc) - filteredServices[key] = svc - } - } - log.Debugf("Found %d Service LoadBalancer in Calico datastore", len(filteredServices)) - return filteredServices, nil - } - - cacheArgs := rcache.ResourceCacheArgs{ - ListFunc: listFunc, - ObjectType: reflect.TypeOf(&v1.Service{}), - LogTypeDesc: "Service", - } - ccache := rcache.NewResourceCache(cacheArgs) - - svcHandler := cache.ResourceEventHandlerFuncs{ - AddFunc: func(obj interface{}) { - log.Debugf("Got ADD event for service") - svc := obj.(*v1.Service) - if isCalicoManagedLoadBalancer(svc, cfg.AssignIPs) { - ccache.Set(createHandle(svc), svc) +func NewLoadBalancerController(clientset *kubernetes.Clientset, calicoClient client.Interface, cfg config.LoadBalancerControllerConfig, serviceInformer cache.SharedIndexInformer, dataFeed *node.DataFeed) controller.Controller { + lbc := &loadBalancerController{ + calicoClient: calicoClient, + cfg: cfg, + clientSet: clientset, + dataFeed: dataFeed, + syncerUpdates: make(chan interface{}), + syncChan: make(chan interface{}, 1), + ipamBlocks: make(map[string]model.KVPair), + ipPools: make(map[string]api.IPPool), + serviceInformer: serviceInformer, + serviceLister: v1lister.NewServiceLister(serviceInformer.GetIndexer()), + servicesToUpdate: make(map[string]serviceObject), + } + + lbc.RegisterWith(lbc.dataFeed) + lbc.dataFeed.Start() + + _, err := lbc.serviceInformer.AddEventHandler(cache.ResourceEventHandlerFuncs{ + DeleteFunc: func(obj interface{}) { + if svc, ok := obj.(*v1.Service); ok { + if IsCalicoManagedLoadBalancer(svc, cfg.AssignIPs) { + lbc.syncerUpdates <- serviceObject{ + updateType: serviceUpdateTypeDELETE, + } + } } }, UpdateFunc: func(oldObj, newObj interface{}) { - log.Debugf("Got UPDATE event for service") - log.Debugf("Old object: %v", oldObj) - log.Debugf("New object: %v", newObj) svcNew := newObj.(*v1.Service) svcOld := oldObj.(*v1.Service) - - // Service changed type, we need to release the addresses used by the service + handle, err := createHandle(svcNew) + if err != nil { + log.WithError(err).Error("Error creating load balancer handle") + return + } if svcNew.Spec.Type != v1.ServiceTypeLoadBalancer && svcOld.Spec.Type == v1.ServiceTypeLoadBalancer { - ccache.Delete(createHandle(svcOld)) + lbc.syncerUpdates <- serviceObject{ + updateType: serviceUpdateTypeDELETE, + } } else if svcOld.Annotations[annotationIpv4Pools] != svcNew.Annotations[annotationIpv4Pools] || svcOld.Annotations[annotationIpv6Pools] != svcNew.Annotations[annotationIpv6Pools] || svcOld.Annotations[annotationLoadBalancerIp] != svcNew.Annotations[annotationLoadBalancerIp] { // Calico annotations have changed, get new address based on new conditions. - ccache.Set(createHandle(svcNew), svcNew) + lbc.syncerUpdates <- serviceObject{ + handle: handle, + updateType: serviceUpdateTypeUPDATE, + service: svcNew, + } + } else if svcNew.Status.LoadBalancer.Ingress == nil && IsCalicoManagedLoadBalancer(svcNew, cfg.AssignIPs) { + lbc.syncerUpdates <- serviceObject{ + handle: handle, + updateType: serviceUpdateTypeADD, + } } }, - DeleteFunc: func(obj interface{}) { - log.Debugf("Got DELETE event for service") + AddFunc: func(obj interface{}) { svc := obj.(*v1.Service) - if isCalicoManagedLoadBalancer(svc, cfg.AssignIPs) { - ccache.Delete(createHandle(svc)) + if IsCalicoManagedLoadBalancer(svc, cfg.AssignIPs) { + lbc.syncerUpdates <- serviceObject{ + updateType: serviceUpdateTypeADD, + } } }, + }) + if err != nil { + log.WithError(err).Fatal("Failed to add event handle for Service LoadBalancer") + return nil } - _, informer := cache.NewIndexerInformer(svcWatcher, &v1.Service{}, 0, svcHandler, cache.Indexers{}) - - return &loadBalancerController{ - calicoClient: calicoClient, - ctx: ctx, - cfg: cfg, - clientSet: clientset, - dataFeed: node.NewDataFeed(calicoClient), - lbResourceCache: ccache, - informer: informer, - syncerUpdates: make(chan interface{}), - syncChan: make(chan interface{}), - ipamBlocks: make(map[string]model.KVPair), - ipPools: make(map[string]api.IPPool), - } + return lbc } // Run starts the controller. func (c *loadBalancerController) Run(stopCh chan struct{}) { defer uruntime.HandleCrash() - // Start the Kubernetes informer, which will start syncing with the Kubernetes API. - log.Info("Starting LoadBalancer controller") - go c.informer.Run(stopCh) - - // Wait until we are in sync with the Kubernetes API before starting the - // resource cache. log.Debug("Waiting to sync with Kubernetes API (Service)") - for !c.informer.HasSynced() { - time.Sleep(100 * time.Millisecond) - } - log.Debug("Finished syncing with Kubernetes API (Service)") - c.lbResourceCache.Run(timer.String()) - - for i := 0; i < c.cfg.NumberOfWorkers; i++ { - go wait.Until(c.runWorker, time.Second, stopCh) + if !cache.WaitForNamedCacheSync("loadbalancer", stopCh, c.serviceInformer.HasSynced) { + log.Info("Failed to sync resources, received signal for controller to shut down.") + return } - log.Info("LoadBalancer controller is now running") - - // Register to the syncer datafeed for updates to ippools and ipamBlocks - c.RegisterWith(c.dataFeed) - c.dataFeed.Start() go c.acceptScheduledRequests(stopCh) <-stopCh @@ -221,6 +209,8 @@ func (c *loadBalancerController) acceptScheduledRequests(stopCh <-chan struct{}) case <-t.C: log.Infof("Running periodic IPAM sync of Service LoadBalancer") c.syncIPAM() + case <-c.syncChan: + c.syncIPAM() case <-stopCh: return } @@ -242,22 +232,25 @@ func (c *loadBalancerController) handleUpdate(update interface{}) { case model.ResourceKey: switch update.Key.(model.ResourceKey).Kind { case api.KindIPPool: - log.Infof("Running periodic IPAM sync of Service LoadBalancer") c.handleIPPoolUpdate(update) + kick(c.syncChan) return } case model.BlockKey: c.handleBlockUpdate(update) return } + case serviceObject: + c.handleServiceUpdate(update) + kick(c.syncChan) + return } - } func (c *loadBalancerController) handleBlockUpdate(kvp model.KVPair) { if kvp.Value != nil { host := kvp.Value.(*model.AllocationBlock).Affinity - if host != nil && *host == fmt.Sprintf("host:%s", api.VirtualLoadBalancer) { + if host != nil && *host == fmt.Sprintf("virtual:%s", api.VirtualLoadBalancer) { c.ipamBlocks[kvp.Key.String()] = kvp } } else { @@ -266,7 +259,6 @@ func (c *loadBalancerController) handleBlockUpdate(kvp model.KVPair) { } func (c *loadBalancerController) handleIPPoolUpdate(kvp model.KVPair) { - log.Infof("Addind IPPOOL") if kvp.Value != nil { pool := kvp.Value.(*api.IPPool) if slices.Contains(pool.Spec.AllowedUses, api.IPPoolAllowedUseLoadBalancer) { @@ -277,48 +269,64 @@ func (c *loadBalancerController) handleIPPoolUpdate(kvp model.KVPair) { } } +func (c *loadBalancerController) handleServiceUpdate(svcObj serviceObject) { + switch svcObj.updateType { + case serviceUpdateTypeUPDATE: + c.servicesToUpdate[svcObj.handle] = svcObj + } +} + // syncIpam has two main uses. It functions as a garbage collection for leaked IP addresses from Service LoadBalancer // The other use case is to update IPs for any Service LoadBalancer that do not have IPs assigned, this could be caused by the user // creating Service LoadBalancer before any valid pools were created func (c *loadBalancerController) syncIPAM() { + + services, err := c.getServiceLoadBalancerList() + if err != nil { + log.WithError(err).Error("Skipping IPAM sync") + } + // Garbage collection - // Check all ipamBlocks with loadBalancer affinity - for _, block := range c.ipamBlocks { - attributes := block.Value.(*model.AllocationBlock).Attributes - for _, attr := range attributes { - obj, empty := c.lbResourceCache.Get(*attr.AttrPrimary) - service := obj.(*v1.Service) - if !empty { - // Service with handle exists, we need to check that all assigned IPs with the handle are still in use by the service - log.Infof("Service found for handle: %s. Check if all IPs allocated by the handle are in use.", *attr.AttrPrimary) - ips, err := c.calicoClient.IPAM().IPsByHandle(c.ctx, *attr.AttrPrimary) - if err != nil { - log.Errorf("Error getting IPs for handle: %s", *attr.AttrPrimary) - break - } - for _, ingressIP := range service.Status.LoadBalancer.Ingress { - inUse := false - for _, handleIP := range ips { - if handleIP.String() == ingressIP.IP { - inUse = true - } - if !inUse { - releaseOptions := ipam.ReleaseOptions{ - Address: ingressIP.IP, + // Check all ipamBlocks with loadBalancer affinity, and release unused allocated IPs + // Skip if there is service scheduled for update. ip will be released during the update and we will run GC right after the un-allocation + if len(c.servicesToUpdate) == 0 { + log.Info("Running Service LoadBalancer IP garbage collection") + for _, block := range c.ipamBlocks { + attributes := block.Value.(*model.AllocationBlock).Attributes + for _, attr := range attributes { + if svc, exists := services[*attr.AttrPrimary]; exists { + // Service with handle exists, we need to check that all assigned IPs with the handle are still in use by the svc + log.Debugf("Service found for handle: %s. Check if all IPs allocated by the handle are in use.", *attr.AttrPrimary) + ips, err := c.calicoClient.IPAM().IPsByHandle(context.Background(), *attr.AttrPrimary) + if err != nil { + log.Errorf("Error getting IPs for handle: %s", *attr.AttrPrimary) + } + for _, ingressIP := range svc.Status.LoadBalancer.Ingress { + inUse := false + for _, handleIP := range ips { + if handleIP.String() == ingressIP.IP { + log.Debugf("IP %s in use, skipping", handleIP.String()) + inUse = true } - _, err = c.calicoClient.IPAM().ReleaseIPs(c.ctx, releaseOptions) - if err != nil { - log.Errorf("Error releasing IP(%s) for service: %s", ingressIP.IP, service.Name) + if !inUse { + log.Debugf("IP %s not in use, releasing", handleIP.String()) + releaseOptions := ipam.ReleaseOptions{ + Address: ingressIP.IP, + } + _, err = c.calicoClient.IPAM().ReleaseIPs(context.Background(), releaseOptions) + if err != nil { + log.Errorf("Error releasing IP(%s) for svc: %s", ingressIP.IP, svc.Name) + } } } } - } - } else { - // Service no longer exists, leak confirmed. Release all IPs allocated with the specific handle - log.Infof("Service not found for handle: %s. Releasing unused IPs", *attr.AttrPrimary) - err := c.releaseIP(*attr.AttrPrimary) - if err != nil { - log.Errorf("Error releasing IPAM for handle %s: %s", *attr.AttrPrimary, err) + } else { + // Service no longer exists, leak confirmed. Release all IPs allocated with the specific handle + log.Infof("Service not found for handle: %s. Releasing unused IPs", *attr.AttrPrimary) + err := c.releaseIP(*attr.AttrPrimary) + if err != nil { + log.Errorf("Error releasing IPAM for handle %s: %s", *attr.AttrPrimary, err) + } } } } @@ -326,95 +334,59 @@ func (c *loadBalancerController) syncIPAM() { // Check that all services have assigned IPs as requested, skip if there are no ippools if len(c.ipPools) != 0 { - for _, key := range c.lbResourceCache.ListKeys() { - obj, empty := c.lbResourceCache.Get(key) - if !empty { - service := obj.(*v1.Service) - if isCalicoManagedLoadBalancer(service, c.cfg.AssignIPs) { - if service.Status.LoadBalancer.Ingress == nil || - (len(service.Status.LoadBalancer.Ingress) == 1 && - (*service.Spec.IPFamilyPolicy == v1.IPFamilyPolicyRequireDualStack) || *service.Spec.IPFamilyPolicy == v1.IPFamilyPolicyPreferDualStack) { - err := c.assignIP(service) - if err != nil { - log.Errorf("Error assigning IP to service %s: %s", service.Name, err) - } - } + for _, svc := range services { + if svc.Status.LoadBalancer.Ingress == nil || + (len(svc.Status.LoadBalancer.Ingress) == 1 && + (*svc.Spec.IPFamilyPolicy == v1.IPFamilyPolicyRequireDualStack) || *svc.Spec.IPFamilyPolicy == v1.IPFamilyPolicyPreferDualStack) { + err = c.assignIP(&svc) + if err != nil { + log.WithError(err).Errorf("Error assigning IP to svc: %s", svc.Name) } } } } -} - -func (c *loadBalancerController) runWorker() { - for c.processNextItem() { - } -} - -func (c *loadBalancerController) processNextItem() bool { - workqueue := c.lbResourceCache.GetQueue() - key, quit := workqueue.Get() - if quit { - return false - } - - err := c.syncToDatastore(key.(string)) - c.handleErr(err, key.(string)) - workqueue.Done(key) - return true -} - -// syncToDatastore processes the next item in queue -func (c *loadBalancerController) syncToDatastore(key string) error { - obj, exists := c.lbResourceCache.Get(key) - - if !exists { - // Service does not exist release the IPs - return c.releaseIP(key) - } else { - svc := obj.(*v1.Service) - // We don't have IP, assign one - if svc.Status.LoadBalancer.Ingress == nil { - return c.assignIP(svc) - } else { - // Service was updated - return c.checkStatus(svc) + if len(c.servicesToUpdate) != 0 { + for key, svcObj := range c.servicesToUpdate { + switch svcObj.updateType { + case serviceUpdateTypeUPDATE: + err = c.releaseIP(svcObj.handle) + if err != nil { + log.WithError(err).Error("Error releasing Service IP") + continue + } + svc, err := c.clientSet.CoreV1().Services(svcObj.service.Namespace).Get(context.Background(), svcObj.service.Name, metav1.GetOptions{}) + if err != nil { + log.WithError(err).Errorf("Error getting Service %s", svcObj.service.Name) + continue + } + svc.Status.LoadBalancer.Ingress = []v1.LoadBalancerIngress{} + _, err = c.clientSet.CoreV1().Services(svc.Namespace).UpdateStatus(context.Background(), svc, metav1.UpdateOptions{}) + if err != nil { + // We have assigned IP to the service, but were not able to update the status. The IP will be picked up by GC during the next sync, and we will try to assign a new IP + log.WithError(err).Error("Error updating Service IP") + continue + } + delete(c.servicesToUpdate, key) + } } } } -func (c *loadBalancerController) handleErr(err error, key string) { - workqueue := c.lbResourceCache.GetQueue() - if err == nil { - // Forget about the #AddRateLimited history of the key on every successful synchronization. - // This ensures that future processing of updates for this key is not delayed because of - // an outdated error history. - workqueue.Forget(key) - return - } - - // This controller retries 5 times if something goes wrong. After that, it stops trying. - if workqueue.NumRequeues(key) < 5 { - // Re-enqueue the key rate limited. Based on the rate limiter on the - // queue and the re-enqueue history, the key will be processed later again. - log.WithError(err).Errorf("Error syncing Service LoadBalancer %v: %v", key, err) - workqueue.AddRateLimited(key) - return - } - - workqueue.Forget(key) - - // Report to an external entity that, even after several retries, we could not successfully process this key - uruntime.HandleError(err) - log.WithError(err).Errorf("Dropping Service LoadBalancer %q out of the queue: %v", key, err) -} - // assignIP tries to assign IP address for Service. func (c *loadBalancerController) assignIP(svc *v1.Service) error { if len(c.ipPools) == 0 { return nil } - handle := createHandle(svc) + handle, err := createHandle(svc) + if err != nil { + return err + } + + loadBalancerIps, ipv4Pools, ipv6Pools, err := c.parseAnnotations(svc.Annotations) + if err != nil { + return err + } metadataAttrs := map[string]string{ ipam.AttributeService: svc.Name, @@ -424,15 +396,11 @@ func (c *loadBalancerController) assignIP(svc *v1.Service) error { } // User requested specific IP, attempt to allocate - if svc.Annotations[annotationLoadBalancerIp] != "" { - ipAddrs, err := validateAnnotation(parseAnnotation(svc.Annotations[annotationLoadBalancerIp])) - if err != nil { - return err - } + if len(loadBalancerIps) != 0 { ingress := svc.Status.LoadBalancer.Ingress - for _, addrs := range ipAddrs { + for _, addrs := range loadBalancerIps { for _, lbingress := range ingress { // We must be trying to assign missing address due to an error, // skip this assignment as it's already assigned and move onto the next one @@ -448,7 +416,7 @@ func (c *loadBalancerController) assignIP(svc *v1.Service) error { Attrs: metadataAttrs, } - err = c.calicoClient.IPAM().AssignIP(c.ctx, ipamArgs) + err = c.calicoClient.IPAM().AssignIP(context.Background(), ipamArgs) if err != nil { log.WithField("ip", addrs).WithError(err).Warn("failed to assign ip to node") return err @@ -458,7 +426,7 @@ func (c *loadBalancerController) assignIP(svc *v1.Service) error { } svc.Status.LoadBalancer.Ingress = ingress - _, err = c.clientSet.CoreV1().Services(svc.Namespace).UpdateStatus(c.ctx, svc, metav1.UpdateOptions{}) + _, err = c.clientSet.CoreV1().Services(svc.Namespace).UpdateStatus(context.Background(), svc, metav1.UpdateOptions{}) return err } @@ -504,21 +472,15 @@ func (c *loadBalancerController) assignIP(svc *v1.Service) error { Attrs: metadataAttrs, } - if svc.Annotations[annotationIpv4Pools] != "" { - args.IPv4Pools, _ = c.resolvePools(parseAnnotation(svc.Annotations[annotationIpv4Pools]), true) - if args.IPv4Pools == nil { - return errors.New("no IPv4 pools found from annotation") - } + if len(ipv4Pools) != 0 { + args.IPv4Pools = ipv4Pools } - if svc.Annotations[annotationIpv6Pools] != "" { - args.IPv6Pools, _ = c.resolvePools(parseAnnotation(svc.Annotations[annotationIpv6Pools]), false) - if args.IPv6Pools == nil { - return errors.New("no IPv6 pools found from annotation") - } + if len(ipv6Pools) != 0 { + args.IPv6Pools = ipv6Pools } - v4Assignments, v6assignments, err := c.calicoClient.IPAM().AutoAssign(c.ctx, args) + v4Assignments, v6assignments, err := c.calicoClient.IPAM().AutoAssign(context.Background(), args) if err != nil { log.WithField("svc", svc.Name).WithError(err).Warn("error on assigning IPAM to service") } @@ -543,9 +505,9 @@ func (c *loadBalancerController) assignIP(svc *v1.Service) error { svc.Status.LoadBalancer.Ingress = ingress - _, err = c.clientSet.CoreV1().Services(svc.Namespace).UpdateStatus(c.ctx, svc, metav1.UpdateOptions{}) + _, err = c.clientSet.CoreV1().Services(svc.Namespace).UpdateStatus(context.Background(), svc, metav1.UpdateOptions{}) if err != nil { - log.Infof("LoadBalancer Error updating service %s/%s: %v", svc.Namespace, svc.Name, err) + log.Errorf("LoadBalancer Error updating service %s/%s: %v", svc.Namespace, svc.Name, err) return err } return nil @@ -555,60 +517,53 @@ func (c *loadBalancerController) assignIP(svc *v1.Service) error { func (c *loadBalancerController) releaseIP(handle string) error { log.Info("Service type LoadBalancer deleted, releasing assigned IP address") - err := c.calicoClient.IPAM().ReleaseByHandle(c.ctx, handle) + err := c.calicoClient.IPAM().ReleaseByHandle(context.Background(), handle) if err != nil { log.Errorf("error on removing assigned IP for handle %s", handle) return err } + return nil } -// checkStatus determines what has changed about the Service. Service can change it's type, if that happens -// we want to release the IP previously used. If calico annotations have changed we try to assign new IP address to the Service -func (c *loadBalancerController) checkStatus(svc *v1.Service) error { - // Service type has changed, release the ip by the handle - if svc.Spec.Type != v1.ServiceTypeLoadBalancer { - return c.releaseIP(createHandle(svc)) - } else { - // Type is load balancer, this means calico annotations have changed, and we need to re-assign IPs from this service - err := c.releaseIP(createHandle(svc)) - if err != nil { - return err - } - // We can only assign new IP if we are still managing the address assignment - if isCalicoManagedLoadBalancer(svc, c.cfg.AssignIPs) { - return c.assignIP(svc) - } else { - // No longer managed by Calico, clean up our assigned IP from Service - svc.Status.LoadBalancer.Ingress = []v1.LoadBalancerIngress{} - - _, err = c.clientSet.CoreV1().Services(svc.Namespace).UpdateStatus(c.ctx, svc, metav1.UpdateOptions{}) +func (c *loadBalancerController) getServiceLoadBalancerList() (map[string]v1.Service, error) { + services, err := c.serviceLister.Services("").List(labels.Everything()) + if err != nil { + log.WithError(err).Error("Error getting svc list") + return nil, err + } + + loadBalancerServices := make(map[string]v1.Service) + for _, svc := range services { + if IsCalicoManagedLoadBalancer(svc, c.cfg.AssignIPs) { + handle, err := createHandle(svc) if err != nil { - return err + return nil, err } + loadBalancerServices[handle] = *svc } - return nil } + return loadBalancerServices, nil } // resolvePools valid IPpool range when specific pool is requested by the user -func (c *loadBalancerController) resolvePools(pools []string, isv4 bool) ([]cnet.IPNet, error) { - // Iterate through the provided pools. If it parses as a CIDR, just use that. +func (c *loadBalancerController) resolvePools(poolIDs []string, isv4 bool) ([]cnet.IPNet, error) { + // Iterate through the provided poolIDs. If it parses as a CIDR, just use that. // If it does not parse as a CIDR, then attempt to lookup an IP pool with a matching name. - result := []cnet.IPNet{} - for _, p := range pools { + poolCIDRs := []cnet.IPNet{} + for _, p := range poolIDs { _, cidr, err := net.ParseCIDR(p) if err != nil { // Didn't parse as a CIDR - check if it's the name // of a configured IP pool. - for _, ipp := range c.ipPools { - if ipp.Name == p { + for _, ipPool := range c.ipPools { + if ipPool.Name == p { // Found a match. Use the CIDR from the matching pool. - _, cidr, err = net.ParseCIDR(ipp.Spec.CIDR) + _, cidr, err = net.ParseCIDR(ipPool.Spec.CIDR) if err != nil { return nil, fmt.Errorf("failed to parse IP pool cidr: %s", err) } - log.Infof("Resolved pool name %s to cidr %s", ipp.Name, cidr) + log.Infof("Resolved pool name %s to cidr %s", ipPool.Name, cidr) } } @@ -625,69 +580,116 @@ func (c *loadBalancerController) resolvePools(pools []string, isv4 bool) ([]cnet if !isv4 && ip.To4() != nil { return nil, fmt.Errorf("%q isn't a IPv6 address", ip) } - result = append(result, cnet.IPNet{IPNet: *cidr}) + poolCIDRs = append(poolCIDRs, cnet.IPNet{IPNet: *cidr}) } - return result, nil + return poolCIDRs, nil } -// isCalicoManagedLoadBalancer returns if Calico should try to assign IP address for the LoadBalancer +// IsCalicoManagedLoadBalancer returns if Calico should try to assign IP address for the LoadBalancer // We assign IPs only if the loadBalancer controller assignIP is set to AllService // or in RequestedOnlyServices if the service has calico annotation -func isCalicoManagedLoadBalancer(svc *v1.Service, assignIPs string) bool { - if svc.Spec.Type == v1.ServiceTypeLoadBalancer { - if assignIPs == api.AllServices { - return true - } +func IsCalicoManagedLoadBalancer(svc *v1.Service, assignIPs api.AssignIPs) bool { + if svc.Spec.Type != v1.ServiceTypeLoadBalancer { + return false + } - if assignIPs == api.RequestedServicesOnly { - if svc.Annotations[annotationIpv4Pools] != "" || - svc.Annotations[annotationIpv6Pools] != "" || - svc.Annotations[annotationLoadBalancerIp] != "" { - return true - } - } + if assignIPs == api.AllServices { + return true + } + + if svc.Annotations[annotationIpv4Pools] != "" || + svc.Annotations[annotationIpv6Pools] != "" || + svc.Annotations[annotationLoadBalancerIp] != "" { + return true } return false } // validateAnnotation checks if the ips specified in the calico annotation are valid. // Each service can have at most one ipv4 and one ipv6 address -func validateAnnotation(ipAddrs []string) ([]cnet.IP, error) { - parsedIPs := make([]cnet.IP, 0) - ipv4 := 0 - ipv6 := 0 - for _, ipAddr := range ipAddrs { - curr := cnet.ParseIP(ipAddr) - if curr == nil { - return nil, errors.New(fmt.Sprintf("Could not parse %s as a valid IP address", curr)) - } - if curr.To4() != nil { - ipv4++ - } else if curr.To16() != nil { - ipv6++ - } - parsedIPs = append(parsedIPs, *curr) - } +func (c *loadBalancerController) parseAnnotations(annotations map[string]string) ([]cnet.IP, []cnet.IPNet, []cnet.IPNet, error) { + loadBalancerIPs := []cnet.IP{} + ipv4Pools := []cnet.IPNet{} + ipv6Pools := []cnet.IPNet{} + + for key, annotation := range annotations { + switch key { + case annotationIpv4Pools: + poolCIDRs := []string{} + err := json.Unmarshal([]byte(annotation), &poolCIDRs) + if err != nil { + return nil, nil, nil, err + } + ipv4Pools, err = c.resolvePools(poolCIDRs, true) + if err != nil { + return nil, nil, nil, err + } + if ipv4Pools == nil { + return nil, nil, nil, fmt.Errorf("failed to resolve pools for IPv4 addresses from annotation") + } + case annotationIpv6Pools: + poolCIDRs := []string{} + err := json.Unmarshal([]byte(annotation), &poolCIDRs) + if err != nil { + return nil, nil, nil, err + } + ipv6Pools, err = c.resolvePools(poolCIDRs, false) + if err != nil { + return nil, nil, nil, err + } + if ipv6Pools == nil { + return nil, nil, nil, fmt.Errorf("failed to resolve pools for IPv6 addresses from annotation") + } + case annotationLoadBalancerIp: + ipAddrs := []string{} + err := json.Unmarshal([]byte(annotation), &ipAddrs) + if err != nil { + return nil, nil, nil, err + } + ipv4 := 0 + ipv6 := 0 + for _, ipAddr := range ipAddrs { + curr := cnet.ParseIP(ipAddr) + if curr == nil { + return nil, nil, nil, errors.New(fmt.Sprintf("Could not parse %s as a valid IP address", ipAddr)) + } + if curr.To4() != nil { + ipv4++ + } else if curr.To16() != nil { + ipv6++ + } + loadBalancerIPs = append(loadBalancerIPs, *curr) + } - if ipv6 > 1 || ipv4 > 1 { - return nil, errors.New(fmt.Sprintf("At max only one ipv4 and one ipv6 address can be specified. Recieved %d ipv4 and %d ipv6 addresses", ipv4, ipv6)) + if ipv6 > 1 || ipv4 > 1 { + return nil, nil, nil, errors.New(fmt.Sprintf("At max only one ipv4 and one ipv6 address can be specified. Recieved %d ipv4 and %d ipv6 addresses", ipv4, ipv6)) + } + } } - return parsedIPs, nil -} - -func parseAnnotation(str string) []string { - str = strings.TrimPrefix(str, "[") - str = strings.TrimSuffix(str, "]") - return strings.Split(str, ",") + return loadBalancerIPs, ipv4Pools, ipv6Pools, nil } // createHandle Returns a handle to use for IP allocation for the service -func createHandle(svc *v1.Service) string { - handle := strings.ToLower(fmt.Sprintf("lb.%s.%s", svc.Name, svc.UID)) - if len(handle) > k8svalidation.DNS1123SubdomainMaxLength { - return handle[:k8svalidation.DNS1123SubdomainMaxLength] +func createHandle(svc *v1.Service) (string, error) { + prefix := "lb-" + handle := strings.ToLower(fmt.Sprintf("%s-%s-%s", svc.Name, svc.Namespace, svc.UID)) + + hasher := sha256.New() + _, err := hasher.Write([]byte(handle)) + if err != nil { + log.WithError(err).Panic("Failed to generate hash from handle") + return "", err + } + + hash := base64.RawURLEncoding.EncodeToString(hasher.Sum(nil)) + regex := regexp.MustCompile("([-_.])") + hash = regex.ReplaceAllString(hash, "") + handle = prefix + hash + if len(handle) > k8svalidation.DNS1123LabelMaxLength { + handle = handle[:k8svalidation.DNS1123LabelMaxLength] } - return handle + + return handle, nil } func kick(c chan<- interface{}) { diff --git a/kube-controllers/pkg/controllers/node/controller.go b/kube-controllers/pkg/controllers/node/controller.go index 68170c2c991..e59475e8f8b 100644 --- a/kube-controllers/pkg/controllers/node/controller.go +++ b/kube-controllers/pkg/controllers/node/controller.go @@ -67,12 +67,13 @@ func NewNodeController(ctx context.Context, k8sClientset *kubernetes.Clientset, calicoClient client.Interface, cfg config.NodeControllerConfig, - nodeInformer, podInformer cache.SharedIndexInformer) controller.Controller { + nodeInformer, podInformer cache.SharedIndexInformer, + dataFeed *DataFeed) controller.Controller { nc := &NodeController{ ctx: ctx, calicoClient: calicoClient, k8sClientset: k8sClientset, - dataFeed: NewDataFeed(calicoClient), + dataFeed: dataFeed, nodeInformer: nodeInformer, podInformer: podInformer, } diff --git a/libcalico-go/lib/backend/model/block.go b/libcalico-go/lib/backend/model/block.go index 2d185961153..dcfc49c3e1e 100644 --- a/libcalico-go/lib/backend/model/block.go +++ b/libcalico-go/lib/backend/model/block.go @@ -33,7 +33,7 @@ const ( IPAMBlockAttributeNamespace = "namespace" IPAMBlockAttributeNode = "node" IPAMBlockAttributeType = "type" - IPAMBLockAttributeService = "service" + IPAMBlockAttributeService = "service" IPAMBlockAttributeTypeIPIP = "ipipTunnelAddress" IPAMBlockAttributeTypeVXLAN = "vxlanTunnelAddress" IPAMBlockAttributeTypeVXLANV6 = "vxlanV6TunnelAddress" diff --git a/libcalico-go/lib/clientv3/ippool.go b/libcalico-go/lib/clientv3/ippool.go index f50d6ad4b8e..d850caa76ec 100644 --- a/libcalico-go/lib/clientv3/ippool.go +++ b/libcalico-go/lib/clientv3/ippool.go @@ -283,7 +283,8 @@ func convertIpPoolFromStorage(pool *apiv3.IPPool) error { pool.Spec.VXLANMode = apiv3.VXLANModeNever } - if pool.Spec.AssignmentMode == "" { + assignmentMode := pool.Spec.AssignmentMode + if &assignmentMode == nil { pool.Spec.AssignmentMode = apiv3.Automatic } diff --git a/libcalico-go/lib/clientv3/ippool_e2e_test.go b/libcalico-go/lib/clientv3/ippool_e2e_test.go index 77270795a80..dbb9deaf1de 100644 --- a/libcalico-go/lib/clientv3/ippool_e2e_test.go +++ b/libcalico-go/lib/clientv3/ippool_e2e_test.go @@ -100,15 +100,6 @@ var _ = testutils.E2eDatastoreDescribe("IPPool tests", testutils.DatastoreAll, f AllowedUses: []apiv3.IPPoolAllowedUse{apiv3.IPPoolAllowedUseWorkload, apiv3.IPPoolAllowedUseTunnel}, } - //spec4 := apiv3.IPPoolSpec{ - // CIDR: "1.2.3.0/4", - // IPIPMode: apiv3.IPIPModeNever, - // VXLANMode: apiv3.VXLANModeNever, - // BlockSize: 26, - // NodeSelector: "all()", - // AllowedUses: []apiv3.IPPoolAllowedUse{apiv3.IPPoolAllowedUseLoadBalancer}, - //} - It("should error when creating an IPPool with no name", func() { c, err := clientv3.New(config) Expect(err).NotTo(HaveOccurred()) diff --git a/libcalico-go/lib/ipam/ipam.go b/libcalico-go/lib/ipam/ipam.go index 3225e1f6ec5..c15f4eef89a 100644 --- a/libcalico-go/lib/ipam/ipam.go +++ b/libcalico-go/lib/ipam/ipam.go @@ -50,7 +50,7 @@ const ( AttributeNode = model.IPAMBlockAttributeNode AttributeTimestamp = model.IPAMBlockAttributeTimestamp AttributeType = model.IPAMBlockAttributeType - AttributeService = model.IPAMBLockAttributeService + AttributeService = model.IPAMBlockAttributeService AttributeTypeIPIP = model.IPAMBlockAttributeTypeIPIP AttributeTypeVXLAN = model.IPAMBlockAttributeTypeVXLAN AttributeTypeVXLANV6 = model.IPAMBlockAttributeTypeVXLANV6 @@ -257,19 +257,6 @@ func (c ipamClient) determinePools(ctx context.Context, requestedPoolNets []net. log.Debugf("enabled pools: %v", enabledPools) log.Debugf("requested pools: %v", requestedPoolNets) - // No pools requested, we should only pick from pools that have assignmentMode: Automatic - if requestedPoolNets == nil { - log.Debug("No pools were specified, using pools with AssignmentMode: Automatic") - automaticAssignPools := []v3.IPPool{} - for _, pool := range enabledPools { - if pool.Spec.AssignmentMode == v3.Automatic { - automaticAssignPools = append(automaticAssignPools, pool) - } - } - enabledPools = automaticAssignPools - log.Debugf("automatic pools: %v", enabledPools) - } - // Build a map so we can lookup existing pools. pm := map[string]v3.IPPool{} @@ -320,6 +307,9 @@ func (c ipamClient) determinePools(ctx context.Context, requestedPoolNets []net. // We only want to use IP pools which actually match this node, so do a filter based on // selector. for _, pool := range enabledPools { + if requestedPoolNets == nil && pool.Spec.AssignmentMode != v3.Automatic { + continue + } var matches bool matches, err = SelectsNode(pool, node) if err != nil { @@ -347,7 +337,7 @@ func (c ipamClient) prepareAffinityBlocksForHost(ctx context.Context, requestedP var err error var v3n *libapiv3.Node var ok bool - // Regular node, continue as usual { + // Regular node, continue as usual if host != v3.VirtualLoadBalancer { node, err = c.client.Get(ctx, model.ResourceKey{Kind: libapiv3.KindNode, Name: host}, "") if err != nil { @@ -1671,7 +1661,7 @@ func (c ipamClient) releaseByHandle(ctx context.Context, blockCIDR net.IPNet, op // KVPair read from before. No need to update the Value since we // have been directly manipulating the value referenced by the KVPair. logCtx.Debug("Updating block to release IPs") - _, err = c.blockReaderWriter.updateBlock(ctx, obj) + obj, err = c.blockReaderWriter.updateBlock(ctx, obj) if err != nil { if _, ok := err.(cerrors.ErrorResourceUpdateConflict); ok { // Comparison failed - retry. @@ -1693,6 +1683,17 @@ func (c ipamClient) releaseByHandle(ctx context.Context, blockCIDR net.IPNet, op if err = c.ensureConsistentAffinity(ctx, block.AllocationBlock); err != nil { logCtx.WithError(err).Warn("Error ensuring consistent affinity but IP already released. Returning no error.") } + + // If this is loadBalancer we delete the block without waiting + if *block.Affinity == "virtual:virtual-load-balancer" { + block = allocationBlock{obj.Value.(*model.AllocationBlock)} + if block.empty() { + err = c.blockReaderWriter.deleteBlock(ctx, obj) + if err != nil { + return err + } + } + } return nil } return errors.New("Hit max retries") diff --git a/libcalico-go/lib/ipam/ipam_block_reader_writer.go b/libcalico-go/lib/ipam/ipam_block_reader_writer.go index b95bd8662c8..214dfbcfb3b 100644 --- a/libcalico-go/lib/ipam/ipam_block_reader_writer.go +++ b/libcalico-go/lib/ipam/ipam_block_reader_writer.go @@ -211,7 +211,11 @@ func (rw blockReaderWriter) claimAffineBlock(ctx context.Context, aff *model.KVP logCtx := log.WithFields(log.Fields{"host": host, "subnet": subnet}) // Create the new block. - affinityKeyStr := "host:" + host + prefix := "host:" + if host == v3.VirtualLoadBalancer { + prefix = "virtual:" + } + affinityKeyStr := prefix + host block := newBlock(subnet, rsvdAttr) block.Affinity = &affinityKeyStr From 694ce2d6a37089288320e3f3272d7c85a40787cc Mon Sep 17 00:00:00 2001 From: Michal Fupso Date: Mon, 16 Sep 2024 10:47:50 -0700 Subject: [PATCH 04/12] gen files --- .../config/crd/crd.projectcalico.org_ippools.yaml | 3 +++ .../syncersv1/bgpsyncer/bgpsyncer_e2e_test.go | 13 +++++++------ 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/libcalico-go/config/crd/crd.projectcalico.org_ippools.yaml b/libcalico-go/config/crd/crd.projectcalico.org_ippools.yaml index e1258e63bf7..df10e8c6196 100644 --- a/libcalico-go/config/crd/crd.projectcalico.org_ippools.yaml +++ b/libcalico-go/config/crd/crd.projectcalico.org_ippools.yaml @@ -40,6 +40,9 @@ spec: assignmentMode: description: Determines the mode how IP addresses should be assigned from this pool + enum: + - Automatic + - Manual type: string blockSize: description: The block size to use for IP address assignments from diff --git a/libcalico-go/lib/backend/syncersv1/bgpsyncer/bgpsyncer_e2e_test.go b/libcalico-go/lib/backend/syncersv1/bgpsyncer/bgpsyncer_e2e_test.go index 23064eb56b5..e2bcaf84d13 100644 --- a/libcalico-go/lib/backend/syncersv1/bgpsyncer/bgpsyncer_e2e_test.go +++ b/libcalico-go/lib/backend/syncersv1/bgpsyncer/bgpsyncer_e2e_test.go @@ -181,12 +181,13 @@ var _ = testutils.E2eDatastoreDescribe("BGP syncer tests", testutils.DatastoreAl syncTester.ExpectData(model.KVPair{ Key: poolKeyV1, Value: &model.IPPool{ - CIDR: poolCIDRNet, - IPIPInterface: "tunl0", - IPIPMode: encap.CrossSubnet, - Masquerade: true, - IPAM: true, - Disabled: false, + CIDR: poolCIDRNet, + IPIPInterface: "tunl0", + IPIPMode: encap.CrossSubnet, + Masquerade: true, + IPAM: true, + Disabled: false, + AssignmentMode: apiv3.Automatic, }, Revision: pool.ResourceVersion, }) From 059fa11f35743847c2dadf8962cfc2d7cc8a9555 Mon Sep 17 00:00:00 2001 From: Michal Fupso Date: Mon, 16 Sep 2024 10:49:46 -0700 Subject: [PATCH 05/12] gen files --- calicoctl/calicoctl/commands/ipam/check.go | 3 +- .../resourcemgr/kubecontrollersconfig_test.go | 4 +- .../loadbalancer_controller_fv_test.go | 53 ++++++++++++++++++- 3 files changed, 56 insertions(+), 4 deletions(-) diff --git a/calicoctl/calicoctl/commands/ipam/check.go b/calicoctl/calicoctl/commands/ipam/check.go index 74180ce975f..4d35f156ead 100644 --- a/calicoctl/calicoctl/commands/ipam/check.go +++ b/calicoctl/calicoctl/commands/ipam/check.go @@ -24,12 +24,13 @@ import ( "strings" "github.com/docopt/docopt-go" - "github.com/projectcalico/calico/kube-controllers/pkg/controllers/loadbalancer" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/kubernetes" + "github.com/projectcalico/calico/kube-controllers/pkg/controllers/loadbalancer" + "github.com/projectcalico/calico/libcalico-go/lib/set" "github.com/projectcalico/calico/libcalico-go/lib/ipam" diff --git a/calicoctl/calicoctl/resourcemgr/kubecontrollersconfig_test.go b/calicoctl/calicoctl/resourcemgr/kubecontrollersconfig_test.go index e171cbd4ddf..13584bbdf58 100644 --- a/calicoctl/calicoctl/resourcemgr/kubecontrollersconfig_test.go +++ b/calicoctl/calicoctl/resourcemgr/kubecontrollersconfig_test.go @@ -73,7 +73,8 @@ spec: To(Equal(&api.ServiceAccountControllerConfig{ReconcilerPeriod: &v1.Duration{Duration: time.Minute * 4}})) Expect(kcc.Spec.Controllers.Namespace). To(Equal(&api.NamespaceControllerConfig{ReconcilerPeriod: &v1.Duration{Duration: time.Minute * 5}})) - Expect(kcc.Spec.Controllers.LoadBalancer.AssignIPs).To(Equal(&api.LoadBalancerControllerConfig{AssignIPs: api.AllServices})) + Expect(kcc.Spec.Controllers.LoadBalancer). + To(Equal(&api.LoadBalancerControllerConfig{AssignIPs: api.AllServices})) // Status Expect(kcc.Status.EnvironmentVars).To(BeNil()) @@ -82,6 +83,7 @@ spec: Expect(kcc.Status.RunningConfig.Controllers.WorkloadEndpoint).To(BeNil()) Expect(kcc.Status.RunningConfig.Controllers.ServiceAccount).To(BeNil()) Expect(kcc.Status.RunningConfig.Controllers.Namespace).To(BeNil()) + Expect(kcc.Status.RunningConfig.Controllers.LoadBalancer).To(BeNil()) Expect(kcc.Status.RunningConfig.LogSeverityScreen).To(Equal("")) Expect(kcc.Status.RunningConfig.HealthChecks).To(Equal("")) Expect(kcc.Status.RunningConfig.EtcdV3CompactionPeriod).To(BeNil()) diff --git a/kube-controllers/pkg/controllers/loadbalancer/loadbalancer_controller_fv_test.go b/kube-controllers/pkg/controllers/loadbalancer/loadbalancer_controller_fv_test.go index b3a1f9424b0..7443952387c 100644 --- a/kube-controllers/pkg/controllers/loadbalancer/loadbalancer_controller_fv_test.go +++ b/kube-controllers/pkg/controllers/loadbalancer/loadbalancer_controller_fv_test.go @@ -16,6 +16,7 @@ package loadbalancer_test import ( "context" + "fmt" "os" "time" @@ -93,7 +94,7 @@ var _ = Describe("Calico loadbalancer controller FV tests (etcd mode)", func() { ObjectMeta: metav1.ObjectMeta{ Name: "basic-ipv4-pool-specified", Annotations: map[string]string{ - "projectcalico.org/ipv4pools": specificIpPool.Name, + "projectcalico.org/ipv4pools": fmt.Sprintf("[\"%s\"]", specificIpPool.Name), }, }, Spec: v1.ServiceSpec{ @@ -113,7 +114,7 @@ var _ = Describe("Calico loadbalancer controller FV tests (etcd mode)", func() { ObjectMeta: metav1.ObjectMeta{ Name: "basic-ip-address-specified", Annotations: map[string]string{ - "projectcalico.org/loadBalancerIPs": "1.2.3.100", + "projectcalico.org/loadBalancerIPs": "[\"1.2.3.100\"]", }, }, Spec: v1.ServiceSpec{ @@ -270,5 +271,53 @@ var _ = Describe("Calico loadbalancer controller FV tests (etcd mode)", func() { return service.Status.LoadBalancer.Ingress }, time.Second*15, 500*time.Millisecond).Should(BeEmpty()) }) + + It("Should assign IP after LoadBalancer IP pool is created", func() { + _, err := calicoClient.IPPools().Delete(context.Background(), specificIpPool.Name, options.DeleteOptions{}) + Expect(err).NotTo(HaveOccurred()) + + _, err = calicoClient.IPPools().Delete(context.Background(), basicIpPool.Name, options.DeleteOptions{}) + Expect(err).NotTo(HaveOccurred()) + + _, err = k8sClient.CoreV1().Services(testNamespace).Create(context.Background(), &basicService, metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + + Eventually(func() []v1.LoadBalancerIngress { + service, _ := k8sClient.CoreV1().Services(testNamespace).Get(context.Background(), basicService.Name, metav1.GetOptions{}) + return service.Status.LoadBalancer.Ingress + }, time.Second*15, 500*time.Millisecond).Should(BeEmpty()) + + _, err = calicoClient.IPPools().Create(context.Background(), &basicIpPool, options.SetOptions{}) + Expect(err).NotTo(HaveOccurred()) + + Eventually(func() []v1.LoadBalancerIngress { + service, _ := k8sClient.CoreV1().Services(testNamespace).Get(context.Background(), basicService.Name, metav1.GetOptions{}) + return service.Status.LoadBalancer.Ingress + }, time.Second*15, 500*time.Millisecond).ShouldNot(BeEmpty()) + }) + + It("Should assign IP after LoadBalancer IP pool is created", func() { + _, err := calicoClient.IPPools().Delete(context.Background(), specificIpPool.Name, options.DeleteOptions{}) + Expect(err).NotTo(HaveOccurred()) + + _, err = calicoClient.IPPools().Delete(context.Background(), basicIpPool.Name, options.DeleteOptions{}) + Expect(err).NotTo(HaveOccurred()) + + _, err = k8sClient.CoreV1().Services(testNamespace).Create(context.Background(), &basicService, metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + + Eventually(func() []v1.LoadBalancerIngress { + service, _ := k8sClient.CoreV1().Services(testNamespace).Get(context.Background(), basicService.Name, metav1.GetOptions{}) + return service.Status.LoadBalancer.Ingress + }, time.Second*15, 500*time.Millisecond).Should(BeEmpty()) + + _, err = calicoClient.IPPools().Create(context.Background(), &basicIpPool, options.SetOptions{}) + Expect(err).NotTo(HaveOccurred()) + + Eventually(func() []v1.LoadBalancerIngress { + service, _ := k8sClient.CoreV1().Services(testNamespace).Get(context.Background(), basicService.Name, metav1.GetOptions{}) + return service.Status.LoadBalancer.Ingress + }, time.Second*15, 500*time.Millisecond).ShouldNot(BeEmpty()) + }) }) }) From 26d89a9888f4acb2fa5182aeb518b0410f3b6d1a Mon Sep 17 00:00:00 2001 From: Michal Fupso Date: Mon, 16 Sep 2024 10:52:59 -0700 Subject: [PATCH 06/12] calico gen manifest --- manifests/calico-bpf.yaml | 3 +++ manifests/calico-policy-only.yaml | 3 +++ manifests/calico-typha.yaml | 3 +++ manifests/calico-vxlan.yaml | 3 +++ manifests/calico.yaml | 3 +++ manifests/canal.yaml | 3 +++ manifests/crds.yaml | 3 +++ manifests/flannel-migration/calico.yaml | 3 +++ manifests/ocp/crd.projectcalico.org_ippools.yaml | 3 +++ manifests/operator-crds.yaml | 3 +++ manifests/tigera-operator.yaml | 3 +++ 11 files changed, 33 insertions(+) diff --git a/manifests/calico-bpf.yaml b/manifests/calico-bpf.yaml index 84ae2eb040a..77951dba519 100644 --- a/manifests/calico-bpf.yaml +++ b/manifests/calico-bpf.yaml @@ -3245,6 +3245,9 @@ spec: assignmentMode: description: Determines the mode how IP addresses should be assigned from this pool + enum: + - Automatic + - Manual type: string blockSize: description: The block size to use for IP address assignments from diff --git a/manifests/calico-policy-only.yaml b/manifests/calico-policy-only.yaml index 3c2718c181b..cb53347c80a 100644 --- a/manifests/calico-policy-only.yaml +++ b/manifests/calico-policy-only.yaml @@ -3255,6 +3255,9 @@ spec: assignmentMode: description: Determines the mode how IP addresses should be assigned from this pool + enum: + - Automatic + - Manual type: string blockSize: description: The block size to use for IP address assignments from diff --git a/manifests/calico-typha.yaml b/manifests/calico-typha.yaml index 0c990b68a8f..964640c4de6 100644 --- a/manifests/calico-typha.yaml +++ b/manifests/calico-typha.yaml @@ -3256,6 +3256,9 @@ spec: assignmentMode: description: Determines the mode how IP addresses should be assigned from this pool + enum: + - Automatic + - Manual type: string blockSize: description: The block size to use for IP address assignments from diff --git a/manifests/calico-vxlan.yaml b/manifests/calico-vxlan.yaml index 2730ed22a39..e7d6c4831d8 100644 --- a/manifests/calico-vxlan.yaml +++ b/manifests/calico-vxlan.yaml @@ -3240,6 +3240,9 @@ spec: assignmentMode: description: Determines the mode how IP addresses should be assigned from this pool + enum: + - Automatic + - Manual type: string blockSize: description: The block size to use for IP address assignments from diff --git a/manifests/calico.yaml b/manifests/calico.yaml index 0a5dad7c91f..ec48ff0d070 100644 --- a/manifests/calico.yaml +++ b/manifests/calico.yaml @@ -3240,6 +3240,9 @@ spec: assignmentMode: description: Determines the mode how IP addresses should be assigned from this pool + enum: + - Automatic + - Manual type: string blockSize: description: The block size to use for IP address assignments from diff --git a/manifests/canal.yaml b/manifests/canal.yaml index 50c40238271..2aa93107447 100644 --- a/manifests/canal.yaml +++ b/manifests/canal.yaml @@ -3257,6 +3257,9 @@ spec: assignmentMode: description: Determines the mode how IP addresses should be assigned from this pool + enum: + - Automatic + - Manual type: string blockSize: description: The block size to use for IP address assignments from diff --git a/manifests/crds.yaml b/manifests/crds.yaml index 540bb48a2ca..543fa940e73 100644 --- a/manifests/crds.yaml +++ b/manifests/crds.yaml @@ -3150,6 +3150,9 @@ spec: assignmentMode: description: Determines the mode how IP addresses should be assigned from this pool + enum: + - Automatic + - Manual type: string blockSize: description: The block size to use for IP address assignments from diff --git a/manifests/flannel-migration/calico.yaml b/manifests/flannel-migration/calico.yaml index d5a33f59b2d..c7831538928 100644 --- a/manifests/flannel-migration/calico.yaml +++ b/manifests/flannel-migration/calico.yaml @@ -3240,6 +3240,9 @@ spec: assignmentMode: description: Determines the mode how IP addresses should be assigned from this pool + enum: + - Automatic + - Manual type: string blockSize: description: The block size to use for IP address assignments from diff --git a/manifests/ocp/crd.projectcalico.org_ippools.yaml b/manifests/ocp/crd.projectcalico.org_ippools.yaml index 9970a74783c..15e1c1fbcaa 100644 --- a/manifests/ocp/crd.projectcalico.org_ippools.yaml +++ b/manifests/ocp/crd.projectcalico.org_ippools.yaml @@ -40,6 +40,9 @@ spec: assignmentMode: description: Determines the mode how IP addresses should be assigned from this pool + enum: + - Automatic + - Manual type: string blockSize: description: The block size to use for IP address assignments from diff --git a/manifests/operator-crds.yaml b/manifests/operator-crds.yaml index 711f737f474..bc8231e7b5c 100644 --- a/manifests/operator-crds.yaml +++ b/manifests/operator-crds.yaml @@ -19687,6 +19687,9 @@ spec: assignmentMode: description: Determines the mode how IP addresses should be assigned from this pool + enum: + - Automatic + - Manual type: string blockSize: description: The block size to use for IP address assignments from diff --git a/manifests/tigera-operator.yaml b/manifests/tigera-operator.yaml index a30474331bd..52d8b1eb12b 100644 --- a/manifests/tigera-operator.yaml +++ b/manifests/tigera-operator.yaml @@ -3169,6 +3169,9 @@ spec: assignmentMode: description: Determines the mode how IP addresses should be assigned from this pool + enum: + - Automatic + - Manual type: string blockSize: description: The block size to use for IP address assignments from From e97579e6446eca8c60b8e62c09a4d7d79fa5d7eb Mon Sep 17 00:00:00 2001 From: Michal Fupso Date: Mon, 16 Sep 2024 10:56:18 -0700 Subject: [PATCH 07/12] calicoctl crds --- calicoctl/calicoctl/commands/crds/crds.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/calicoctl/calicoctl/commands/crds/crds.go b/calicoctl/calicoctl/commands/crds/crds.go index 28fa137519f..1be51e3b6db 100644 --- a/calicoctl/calicoctl/commands/crds/crds.go +++ b/calicoctl/calicoctl/commands/crds/crds.go @@ -30,7 +30,7 @@ const ( ipamblocks = "apiVersion: apiextensions.k8s.io/v1\nkind: CustomResourceDefinition\nmetadata:\n name: ipamblocks.crd.projectcalico.org\nspec:\n group: crd.projectcalico.org\n names:\n kind: IPAMBlock\n listKind: IPAMBlockList\n plural: ipamblocks\n singular: ipamblock\n preserveUnknownFields: false\n scope: Cluster\n versions:\n - name: v1\n schema:\n openAPIV3Schema:\n properties:\n apiVersion:\n description: 'APIVersion defines the versioned schema of this representation\n of an object. Servers should convert recognized schemas to the latest\n internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'\n type: string\n kind:\n description: 'Kind is a string value representing the REST resource this\n object represents. Servers may infer this from the endpoint the client\n submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'\n type: string\n metadata:\n type: object\n spec:\n description: IPAMBlockSpec contains the specification for an IPAMBlock\n resource.\n properties:\n affinity:\n description: Affinity of the block, if this block has one. If set,\n it will be of the form \"host:\". If not set, this block\n is not affine to a host.\n type: string\n allocations:\n description: Array of allocations in-use within this block. nil entries\n mean the allocation is free. For non-nil entries at index i, the\n index is the ordinal of the allocation within this block and the\n value is the index of the associated attributes in the Attributes\n array.\n items:\n type: integer\n # TODO: This nullable is manually added in. We should update controller-gen\n # to handle []*int properly itself.\n nullable: true\n type: array\n attributes:\n description: Attributes is an array of arbitrary metadata associated\n with allocations in the block. To find attributes for a given allocation,\n use the value of the allocation's entry in the Allocations array\n as the index of the element in this array.\n items:\n properties:\n handle_id:\n type: string\n secondary:\n additionalProperties:\n type: string\n type: object\n type: object\n type: array\n cidr:\n description: The block's CIDR.\n type: string\n deleted:\n description: Deleted is an internal boolean used to workaround a limitation\n in the Kubernetes API whereby deletion will not return a conflict\n error if the block has been updated. It should not be set manually.\n type: boolean\n sequenceNumber:\n default: 0\n description: We store a sequence number that is updated each time\n the block is written. Each allocation will also store the sequence\n number of the block at the time of its creation. When releasing\n an IP, passing the sequence number associated with the allocation\n allows us to protect against a race condition and ensure the IP\n hasn't been released and re-allocated since the release request.\n format: int64\n type: integer\n sequenceNumberForAllocation:\n additionalProperties:\n format: int64\n type: integer\n description: Map of allocated ordinal within the block to sequence\n number of the block at the time of allocation. Kubernetes does not\n allow numerical keys for maps, so the key is cast to a string.\n type: object\n strictAffinity:\n description: StrictAffinity on the IPAMBlock is deprecated and no\n longer used by the code. Use IPAMConfig StrictAffinity instead.\n type: boolean\n unallocated:\n description: Unallocated is an ordered list of allocations which are\n free in the block.\n items:\n type: integer\n type: array\n required:\n - allocations\n - attributes\n - cidr\n - strictAffinity\n - unallocated\n type: object\n type: object\n served: true\n storage: true\nstatus:\n acceptedNames:\n kind: \"\"\n plural: \"\"\n conditions: []\n storedVersions: []\n" ipamconfigs = "apiVersion: apiextensions.k8s.io/v1\nkind: CustomResourceDefinition\nmetadata:\n name: ipamconfigs.crd.projectcalico.org\nspec:\n group: crd.projectcalico.org\n names:\n kind: IPAMConfig\n listKind: IPAMConfigList\n plural: ipamconfigs\n singular: ipamconfig\n preserveUnknownFields: false\n scope: Cluster\n versions:\n - name: v1\n schema:\n openAPIV3Schema:\n properties:\n apiVersion:\n description: 'APIVersion defines the versioned schema of this representation\n of an object. Servers should convert recognized schemas to the latest\n internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'\n type: string\n kind:\n description: 'Kind is a string value representing the REST resource this\n object represents. Servers may infer this from the endpoint the client\n submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'\n type: string\n metadata:\n type: object\n spec:\n description: IPAMConfigSpec contains the specification for an IPAMConfig\n resource.\n properties:\n autoAllocateBlocks:\n type: boolean\n maxBlocksPerHost:\n description: MaxBlocksPerHost, if non-zero, is the max number of blocks\n that can be affine to each host.\n maximum: 2147483647\n minimum: 0\n type: integer\n strictAffinity:\n type: boolean\n required:\n - autoAllocateBlocks\n - strictAffinity\n type: object\n type: object\n served: true\n storage: true\nstatus:\n acceptedNames:\n kind: \"\"\n plural: \"\"\n conditions: []\n storedVersions: []\n" ipamhandles = "apiVersion: apiextensions.k8s.io/v1\nkind: CustomResourceDefinition\nmetadata:\n name: ipamhandles.crd.projectcalico.org\nspec:\n group: crd.projectcalico.org\n names:\n kind: IPAMHandle\n listKind: IPAMHandleList\n plural: ipamhandles\n singular: ipamhandle\n preserveUnknownFields: false\n scope: Cluster\n versions:\n - name: v1\n schema:\n openAPIV3Schema:\n properties:\n apiVersion:\n description: 'APIVersion defines the versioned schema of this representation\n of an object. Servers should convert recognized schemas to the latest\n internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'\n type: string\n kind:\n description: 'Kind is a string value representing the REST resource this\n object represents. Servers may infer this from the endpoint the client\n submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'\n type: string\n metadata:\n type: object\n spec:\n description: IPAMHandleSpec contains the specification for an IPAMHandle\n resource.\n properties:\n block:\n additionalProperties:\n type: integer\n type: object\n deleted:\n type: boolean\n handleID:\n type: string\n required:\n - block\n - handleID\n type: object\n type: object\n served: true\n storage: true\nstatus:\n acceptedNames:\n kind: \"\"\n plural: \"\"\n conditions: []\n storedVersions: []\n" - ippools = "apiVersion: apiextensions.k8s.io/v1\nkind: CustomResourceDefinition\nmetadata:\n name: ippools.crd.projectcalico.org\nspec:\n group: crd.projectcalico.org\n names:\n kind: IPPool\n listKind: IPPoolList\n plural: ippools\n singular: ippool\n preserveUnknownFields: false\n scope: Cluster\n versions:\n - name: v1\n schema:\n openAPIV3Schema:\n properties:\n apiVersion:\n description: 'APIVersion defines the versioned schema of this representation\n of an object. Servers should convert recognized schemas to the latest\n internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'\n type: string\n kind:\n description: 'Kind is a string value representing the REST resource this\n object represents. Servers may infer this from the endpoint the client\n submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'\n type: string\n metadata:\n type: object\n spec:\n description: IPPoolSpec contains the specification for an IPPool resource.\n properties:\n allowedUses:\n description: AllowedUse controls what the IP pool will be used for. If\n not specified or empty, defaults to [\"Tunnel\", \"Workload\"] for back-compatibility\n items:\n type: string\n type: array\n assignmentMode:\n description: Determines the mode how IP addresses should be assigned\n from this pool\n type: string\n blockSize:\n description: The block size to use for IP address assignments from\n this pool. Defaults to 26 for IPv4 and 122 for IPv6.\n type: integer\n cidr:\n description: The pool CIDR.\n type: string\n disableBGPExport:\n description: 'Disable exporting routes from this IP Pool''s CIDR over\n BGP. [Default: false]'\n type: boolean\n disabled:\n description: When disabled is true, Calico IPAM will not assign addresses\n from this pool.\n type: boolean\n ipip:\n description: 'Deprecated: this field is only used for APIv1 backwards\n compatibility. Setting this field is not allowed, this field is\n for internal use only.'\n properties:\n enabled:\n description: When enabled is true, ipip tunneling will be used\n to deliver packets to destinations within this pool.\n type: boolean\n mode:\n description: The IPIP mode. This can be one of \"always\" or \"cross-subnet\". A\n mode of \"always\" will also use IPIP tunneling for routing to\n destination IP addresses within this pool. A mode of \"cross-subnet\"\n will only use IPIP tunneling when the destination node is on\n a different subnet to the originating node. The default value\n (if not specified) is \"always\".\n type: string\n type: object\n ipipMode:\n description: Contains configuration for IPIP tunneling for this pool.\n If not specified, then this is defaulted to \"Never\" (i.e. IPIP tunneling\n is disabled).\n type: string\n nat-outgoing:\n description: 'Deprecated: this field is only used for APIv1 backwards\n compatibility. Setting this field is not allowed, this field is\n for internal use only.'\n type: boolean\n natOutgoing:\n description: When natOutgoing is true, packets sent from Calico networked\n containers in this pool to destinations outside of this pool will\n be masqueraded.\n type: boolean\n nodeSelector:\n description: Allows IPPool to allocate for a specific node by label\n selector.\n type: string\n vxlanMode:\n description: Contains configuration for VXLAN tunneling for this pool.\n If not specified, then this is defaulted to \"Never\" (i.e. VXLAN\n tunneling is disabled).\n type: string\n required:\n - cidr\n type: object\n type: object\n served: true\n storage: true\nstatus:\n acceptedNames:\n kind: \"\"\n plural: \"\"\n conditions: []\n storedVersions: []\n" + ippools = "apiVersion: apiextensions.k8s.io/v1\nkind: CustomResourceDefinition\nmetadata:\n name: ippools.crd.projectcalico.org\nspec:\n group: crd.projectcalico.org\n names:\n kind: IPPool\n listKind: IPPoolList\n plural: ippools\n singular: ippool\n preserveUnknownFields: false\n scope: Cluster\n versions:\n - name: v1\n schema:\n openAPIV3Schema:\n properties:\n apiVersion:\n description: 'APIVersion defines the versioned schema of this representation\n of an object. Servers should convert recognized schemas to the latest\n internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'\n type: string\n kind:\n description: 'Kind is a string value representing the REST resource this\n object represents. Servers may infer this from the endpoint the client\n submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'\n type: string\n metadata:\n type: object\n spec:\n description: IPPoolSpec contains the specification for an IPPool resource.\n properties:\n allowedUses:\n description: AllowedUse controls what the IP pool will be used for. If\n not specified or empty, defaults to [\"Tunnel\", \"Workload\"] for back-compatibility\n items:\n type: string\n type: array\n assignmentMode:\n description: Determines the mode how IP addresses should be assigned\n from this pool\n enum:\n - Automatic\n - Manual\n type: string\n blockSize:\n description: The block size to use for IP address assignments from\n this pool. Defaults to 26 for IPv4 and 122 for IPv6.\n type: integer\n cidr:\n description: The pool CIDR.\n type: string\n disableBGPExport:\n description: 'Disable exporting routes from this IP Pool''s CIDR over\n BGP. [Default: false]'\n type: boolean\n disabled:\n description: When disabled is true, Calico IPAM will not assign addresses\n from this pool.\n type: boolean\n ipip:\n description: 'Deprecated: this field is only used for APIv1 backwards\n compatibility. Setting this field is not allowed, this field is\n for internal use only.'\n properties:\n enabled:\n description: When enabled is true, ipip tunneling will be used\n to deliver packets to destinations within this pool.\n type: boolean\n mode:\n description: The IPIP mode. This can be one of \"always\" or \"cross-subnet\". A\n mode of \"always\" will also use IPIP tunneling for routing to\n destination IP addresses within this pool. A mode of \"cross-subnet\"\n will only use IPIP tunneling when the destination node is on\n a different subnet to the originating node. The default value\n (if not specified) is \"always\".\n type: string\n type: object\n ipipMode:\n description: Contains configuration for IPIP tunneling for this pool.\n If not specified, then this is defaulted to \"Never\" (i.e. IPIP tunneling\n is disabled).\n type: string\n nat-outgoing:\n description: 'Deprecated: this field is only used for APIv1 backwards\n compatibility. Setting this field is not allowed, this field is\n for internal use only.'\n type: boolean\n natOutgoing:\n description: When natOutgoing is true, packets sent from Calico networked\n containers in this pool to destinations outside of this pool will\n be masqueraded.\n type: boolean\n nodeSelector:\n description: Allows IPPool to allocate for a specific node by label\n selector.\n type: string\n vxlanMode:\n description: Contains configuration for VXLAN tunneling for this pool.\n If not specified, then this is defaulted to \"Never\" (i.e. VXLAN\n tunneling is disabled).\n type: string\n required:\n - cidr\n type: object\n type: object\n served: true\n storage: true\nstatus:\n acceptedNames:\n kind: \"\"\n plural: \"\"\n conditions: []\n storedVersions: []\n" ipreservations = "apiVersion: apiextensions.k8s.io/v1\nkind: CustomResourceDefinition\nmetadata:\n annotations:\n controller-gen.kubebuilder.io/version: (devel)\n creationTimestamp: null\n name: ipreservations.crd.projectcalico.org\nspec:\n group: crd.projectcalico.org\n names:\n kind: IPReservation\n listKind: IPReservationList\n plural: ipreservations\n singular: ipreservation\n preserveUnknownFields: false\n scope: Cluster\n versions:\n - name: v1\n schema:\n openAPIV3Schema:\n properties:\n apiVersion:\n description: 'APIVersion defines the versioned schema of this representation\n of an object. Servers should convert recognized schemas to the latest\n internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'\n type: string\n kind:\n description: 'Kind is a string value representing the REST resource this\n object represents. Servers may infer this from the endpoint the client\n submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'\n type: string\n metadata:\n type: object\n spec:\n description: IPReservationSpec contains the specification for an IPReservation\n resource.\n properties:\n reservedCIDRs:\n description: ReservedCIDRs is a list of CIDRs and/or IP addresses\n that Calico IPAM will exclude from new allocations.\n items:\n type: string\n type: array\n type: object\n type: object\n served: true\n storage: true\nstatus:\n acceptedNames:\n kind: \"\"\n plural: \"\"\n conditions: []\n storedVersions: []\n" kubecontrollersconfigurations = "apiVersion: apiextensions.k8s.io/v1\nkind: CustomResourceDefinition\nmetadata:\n name: kubecontrollersconfigurations.crd.projectcalico.org\nspec:\n group: crd.projectcalico.org\n names:\n kind: KubeControllersConfiguration\n listKind: KubeControllersConfigurationList\n plural: kubecontrollersconfigurations\n singular: kubecontrollersconfiguration\n preserveUnknownFields: false\n scope: Cluster\n versions:\n - name: v1\n schema:\n openAPIV3Schema:\n properties:\n apiVersion:\n description: 'APIVersion defines the versioned schema of this representation\n of an object. Servers should convert recognized schemas to the latest\n internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'\n type: string\n kind:\n description: 'Kind is a string value representing the REST resource this\n object represents. Servers may infer this from the endpoint the client\n submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'\n type: string\n metadata:\n type: object\n spec:\n description: KubeControllersConfigurationSpec contains the values of the\n Kubernetes controllers configuration.\n properties:\n controllers:\n description: Controllers enables and configures individual Kubernetes\n controllers\n properties:\n loadBalancer:\n description: LoadBalancer enables and configures the LoadBalancer\n controller. Enabled by default, set to nil to disable.\n properties:\n assignIPs:\n type: string\n type: object\n namespace:\n description: Namespace enables and configures the namespace controller.\n Enabled by default, set to nil to disable.\n properties:\n reconcilerPeriod:\n description: 'ReconcilerPeriod is the period to perform reconciliation\n with the Calico datastore. [Default: 5m]'\n type: string\n type: object\n node:\n description: Node enables and configures the node controller.\n Enabled by default, set to nil to disable.\n properties:\n hostEndpoint:\n description: HostEndpoint controls syncing nodes to host endpoints.\n Disabled by default, set to nil to disable.\n properties:\n autoCreate:\n description: 'AutoCreate enables automatic creation of\n host endpoints for every node. [Default: Disabled]'\n type: string\n type: object\n leakGracePeriod:\n description: 'LeakGracePeriod is the period used by the controller\n to determine if an IP address has been leaked. Set to 0\n to disable IP garbage collection. [Default: 15m]'\n type: string\n reconcilerPeriod:\n description: 'ReconcilerPeriod is the period to perform reconciliation\n with the Calico datastore. [Default: 5m]'\n type: string\n syncLabels:\n description: 'SyncLabels controls whether to copy Kubernetes\n node labels to Calico nodes. [Default: Enabled]'\n type: string\n type: object\n policy:\n description: Policy enables and configures the policy controller.\n Enabled by default, set to nil to disable.\n properties:\n reconcilerPeriod:\n description: 'ReconcilerPeriod is the period to perform reconciliation\n with the Calico datastore. [Default: 5m]'\n type: string\n type: object\n serviceAccount:\n description: ServiceAccount enables and configures the service\n account controller. Enabled by default, set to nil to disable.\n properties:\n reconcilerPeriod:\n description: 'ReconcilerPeriod is the period to perform reconciliation\n with the Calico datastore. [Default: 5m]'\n type: string\n type: object\n workloadEndpoint:\n description: WorkloadEndpoint enables and configures the workload\n endpoint controller. Enabled by default, set to nil to disable.\n properties:\n reconcilerPeriod:\n description: 'ReconcilerPeriod is the period to perform reconciliation\n with the Calico datastore. [Default: 5m]'\n type: string\n type: object\n type: object\n debugProfilePort:\n description: DebugProfilePort configures the port to serve memory\n and cpu profiles on. If not specified, profiling is disabled.\n format: int32\n type: integer\n etcdV3CompactionPeriod:\n description: 'EtcdV3CompactionPeriod is the period between etcdv3\n compaction requests. Set to 0 to disable. [Default: 10m]'\n type: string\n healthChecks:\n description: 'HealthChecks enables or disables support for health\n checks [Default: Enabled]'\n type: string\n logSeverityScreen:\n description: 'LogSeverityScreen is the log severity above which logs\n are sent to the stdout. [Default: Info]'\n type: string\n prometheusMetricsPort:\n description: 'PrometheusMetricsPort is the TCP port that the Prometheus\n metrics server should bind to. Set to 0 to disable. [Default: 9094]'\n type: integer\n required:\n - controllers\n type: object\n status:\n description: KubeControllersConfigurationStatus represents the status\n of the configuration. It's useful for admins to be able to see the actual\n config that was applied, which can be modified by environment variables\n on the kube-controllers process.\n properties:\n environmentVars:\n additionalProperties:\n type: string\n description: EnvironmentVars contains the environment variables on\n the kube-controllers that influenced the RunningConfig.\n type: object\n runningConfig:\n description: RunningConfig contains the effective config that is running\n in the kube-controllers pod, after merging the API resource with\n any environment variables.\n properties:\n controllers:\n description: Controllers enables and configures individual Kubernetes\n controllers\n properties:\n loadBalancer:\n description: LoadBalancer enables and configures the LoadBalancer\n controller. Enabled by default, set to nil to disable.\n properties:\n assignIPs:\n type: string\n type: object\n namespace:\n description: Namespace enables and configures the namespace\n controller. Enabled by default, set to nil to disable.\n properties:\n reconcilerPeriod:\n description: 'ReconcilerPeriod is the period to perform\n reconciliation with the Calico datastore. [Default:\n 5m]'\n type: string\n type: object\n node:\n description: Node enables and configures the node controller.\n Enabled by default, set to nil to disable.\n properties:\n hostEndpoint:\n description: HostEndpoint controls syncing nodes to host\n endpoints. Disabled by default, set to nil to disable.\n properties:\n autoCreate:\n description: 'AutoCreate enables automatic creation\n of host endpoints for every node. [Default: Disabled]'\n type: string\n type: object\n leakGracePeriod:\n description: 'LeakGracePeriod is the period used by the\n controller to determine if an IP address has been leaked.\n Set to 0 to disable IP garbage collection. [Default:\n 15m]'\n type: string\n reconcilerPeriod:\n description: 'ReconcilerPeriod is the period to perform\n reconciliation with the Calico datastore. [Default:\n 5m]'\n type: string\n syncLabels:\n description: 'SyncLabels controls whether to copy Kubernetes\n node labels to Calico nodes. [Default: Enabled]'\n type: string\n type: object\n policy:\n description: Policy enables and configures the policy controller.\n Enabled by default, set to nil to disable.\n properties:\n reconcilerPeriod:\n description: 'ReconcilerPeriod is the period to perform\n reconciliation with the Calico datastore. [Default:\n 5m]'\n type: string\n type: object\n serviceAccount:\n description: ServiceAccount enables and configures the service\n account controller. Enabled by default, set to nil to disable.\n properties:\n reconcilerPeriod:\n description: 'ReconcilerPeriod is the period to perform\n reconciliation with the Calico datastore. [Default:\n 5m]'\n type: string\n type: object\n workloadEndpoint:\n description: WorkloadEndpoint enables and configures the workload\n endpoint controller. Enabled by default, set to nil to disable.\n properties:\n reconcilerPeriod:\n description: 'ReconcilerPeriod is the period to perform\n reconciliation with the Calico datastore. [Default:\n 5m]'\n type: string\n type: object\n type: object\n debugProfilePort:\n description: DebugProfilePort configures the port to serve memory\n and cpu profiles on. If not specified, profiling is disabled.\n format: int32\n type: integer\n etcdV3CompactionPeriod:\n description: 'EtcdV3CompactionPeriod is the period between etcdv3\n compaction requests. Set to 0 to disable. [Default: 10m]'\n type: string\n healthChecks:\n description: 'HealthChecks enables or disables support for health\n checks [Default: Enabled]'\n type: string\n logSeverityScreen:\n description: 'LogSeverityScreen is the log severity above which\n logs are sent to the stdout. [Default: Info]'\n type: string\n prometheusMetricsPort:\n description: 'PrometheusMetricsPort is the TCP port that the Prometheus\n metrics server should bind to. Set to 0 to disable. [Default:\n 9094]'\n type: integer\n required:\n - controllers\n type: object\n type: object\n type: object\n served: true\n storage: true\nstatus:\n acceptedNames:\n kind: \"\"\n plural: \"\"\n conditions: []\n storedVersions: []\n" networkpolicies = "apiVersion: apiextensions.k8s.io/v1\nkind: CustomResourceDefinition\nmetadata:\n name: networkpolicies.crd.projectcalico.org\nspec:\n group: crd.projectcalico.org\n names:\n kind: NetworkPolicy\n listKind: NetworkPolicyList\n plural: networkpolicies\n singular: networkpolicy\n preserveUnknownFields: false\n scope: Namespaced\n versions:\n - name: v1\n schema:\n openAPIV3Schema:\n properties:\n apiVersion:\n description: 'APIVersion defines the versioned schema of this representation\n of an object. Servers should convert recognized schemas to the latest\n internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'\n type: string\n kind:\n description: 'Kind is a string value representing the REST resource this\n object represents. Servers may infer this from the endpoint the client\n submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'\n type: string\n metadata:\n type: object\n spec:\n properties:\n egress:\n description: The ordered set of egress rules. Each rule contains\n a set of packet match criteria and a corresponding action to apply.\n items:\n description: \"A Rule encapsulates a set of match criteria and an\n action. Both selector-based security Policy and security Profiles\n reference rules - separated out as a list of rules for both ingress\n and egress packet matching. \\n Each positive match criteria has\n a negated version, prefixed with \\\"Not\\\". All the match criteria\n within a rule must be satisfied for a packet to match. A single\n rule can contain the positive and negative version of a match\n and both must be satisfied for the rule to match.\"\n properties:\n action:\n type: string\n destination:\n description: Destination contains the match criteria that apply\n to destination entity.\n properties:\n namespaceSelector:\n description: \"NamespaceSelector is an optional field that\n contains a selector expression. Only traffic that originates\n from (or terminates at) endpoints within the selected\n namespaces will be matched. When both NamespaceSelector\n and another selector are defined on the same rule, then\n only workload endpoints that are matched by both selectors\n will be selected by the rule. \\n For NetworkPolicy, an\n empty NamespaceSelector implies that the Selector is limited\n to selecting only workload endpoints in the same namespace\n as the NetworkPolicy. \\n For NetworkPolicy, `global()`\n NamespaceSelector implies that the Selector is limited\n to selecting only GlobalNetworkSet or HostEndpoint. \\n\n For GlobalNetworkPolicy, an empty NamespaceSelector implies\n the Selector applies to workload endpoints across all\n namespaces.\"\n type: string\n nets:\n description: Nets is an optional field that restricts the\n rule to only apply to traffic that originates from (or\n terminates at) IP addresses in any of the given subnets.\n items:\n type: string\n type: array\n notNets:\n description: NotNets is the negated version of the Nets\n field.\n items:\n type: string\n type: array\n notPorts:\n description: NotPorts is the negated version of the Ports\n field. Since only some protocols have ports, if any ports\n are specified it requires the Protocol match in the Rule\n to be set to \"TCP\" or \"UDP\".\n items:\n anyOf:\n - type: integer\n - type: string\n pattern: ^.*\n x-kubernetes-int-or-string: true\n type: array\n notSelector:\n description: NotSelector is the negated version of the Selector\n field. See Selector field for subtleties with negated\n selectors.\n type: string\n ports:\n description: \"Ports is an optional field that restricts\n the rule to only apply to traffic that has a source (destination)\n port that matches one of these ranges/values. This value\n is a list of integers or strings that represent ranges\n of ports. \\n Since only some protocols have ports, if\n any ports are specified it requires the Protocol match\n in the Rule to be set to \\\"TCP\\\" or \\\"UDP\\\".\"\n items:\n anyOf:\n - type: integer\n - type: string\n pattern: ^.*\n x-kubernetes-int-or-string: true\n type: array\n selector:\n description: \"Selector is an optional field that contains\n a selector expression (see Policy for sample syntax).\n \\ Only traffic that originates from (terminates at) endpoints\n matching the selector will be matched. \\n Note that: in\n addition to the negated version of the Selector (see NotSelector\n below), the selector expression syntax itself supports\n negation. The two types of negation are subtly different.\n One negates the set of matched endpoints, the other negates\n the whole match: \\n \\tSelector = \\\"!has(my_label)\\\" matches\n packets that are from other Calico-controlled \\tendpoints\n that do not have the label \\\"my_label\\\". \\n \\tNotSelector\n = \\\"has(my_label)\\\" matches packets that are not from\n Calico-controlled \\tendpoints that do have the label \\\"my_label\\\".\n \\n The effect is that the latter will accept packets from\n non-Calico sources whereas the former is limited to packets\n from Calico-controlled endpoints.\"\n type: string\n serviceAccounts:\n description: ServiceAccounts is an optional field that restricts\n the rule to only apply to traffic that originates from\n (or terminates at) a pod running as a matching service\n account.\n properties:\n names:\n description: Names is an optional field that restricts\n the rule to only apply to traffic that originates\n from (or terminates at) a pod running as a service\n account whose name is in the list.\n items:\n type: string\n type: array\n selector:\n description: Selector is an optional field that restricts\n the rule to only apply to traffic that originates\n from (or terminates at) a pod running as a service\n account that matches the given label selector. If\n both Names and Selector are specified then they are\n AND'ed.\n type: string\n type: object\n services:\n description: \"Services is an optional field that contains\n options for matching Kubernetes Services. If specified,\n only traffic that originates from or terminates at endpoints\n within the selected service(s) will be matched, and only\n to/from each endpoint's port. \\n Services cannot be specified\n on the same rule as Selector, NotSelector, NamespaceSelector,\n Nets, NotNets or ServiceAccounts. \\n Ports and NotPorts\n can only be specified with Services on ingress rules.\"\n properties:\n name:\n description: Name specifies the name of a Kubernetes\n Service to match.\n type: string\n namespace:\n description: Namespace specifies the namespace of the\n given Service. If left empty, the rule will match\n within this policy's namespace.\n type: string\n type: object\n type: object\n http:\n description: HTTP contains match criteria that apply to HTTP\n requests.\n properties:\n methods:\n description: Methods is an optional field that restricts\n the rule to apply only to HTTP requests that use one of\n the listed HTTP Methods (e.g. GET, PUT, etc.) Multiple\n methods are OR'd together.\n items:\n type: string\n type: array\n paths:\n description: 'Paths is an optional field that restricts\n the rule to apply to HTTP requests that use one of the\n listed HTTP Paths. Multiple paths are OR''d together.\n e.g: - exact: /foo - prefix: /bar NOTE: Each entry may\n ONLY specify either a `exact` or a `prefix` match. The\n validator will check for it.'\n items:\n description: 'HTTPPath specifies an HTTP path to match.\n It may be either of the form: exact: : which matches\n the path exactly or prefix: : which matches\n the path prefix'\n properties:\n exact:\n type: string\n prefix:\n type: string\n type: object\n type: array\n type: object\n icmp:\n description: ICMP is an optional field that restricts the rule\n to apply to a specific type and code of ICMP traffic. This\n should only be specified if the Protocol field is set to \"ICMP\"\n or \"ICMPv6\".\n properties:\n code:\n description: Match on a specific ICMP code. If specified,\n the Type value must also be specified. This is a technical\n limitation imposed by the kernel's iptables firewall,\n which Calico uses to enforce the rule.\n type: integer\n type:\n description: Match on a specific ICMP type. For example\n a value of 8 refers to ICMP Echo Request (i.e. pings).\n type: integer\n type: object\n ipVersion:\n description: IPVersion is an optional field that restricts the\n rule to only match a specific IP version.\n type: integer\n metadata:\n description: Metadata contains additional information for this\n rule\n properties:\n annotations:\n additionalProperties:\n type: string\n description: Annotations is a set of key value pairs that\n give extra information about the rule\n type: object\n type: object\n notICMP:\n description: NotICMP is the negated version of the ICMP field.\n properties:\n code:\n description: Match on a specific ICMP code. If specified,\n the Type value must also be specified. This is a technical\n limitation imposed by the kernel's iptables firewall,\n which Calico uses to enforce the rule.\n type: integer\n type:\n description: Match on a specific ICMP type. For example\n a value of 8 refers to ICMP Echo Request (i.e. pings).\n type: integer\n type: object\n notProtocol:\n anyOf:\n - type: integer\n - type: string\n description: NotProtocol is the negated version of the Protocol\n field.\n pattern: ^.*\n x-kubernetes-int-or-string: true\n protocol:\n anyOf:\n - type: integer\n - type: string\n description: \"Protocol is an optional field that restricts the\n rule to only apply to traffic of a specific IP protocol. Required\n if any of the EntityRules contain Ports (because ports only\n apply to certain protocols). \\n Must be one of these string\n values: \\\"TCP\\\", \\\"UDP\\\", \\\"ICMP\\\", \\\"ICMPv6\\\", \\\"SCTP\\\",\n \\\"UDPLite\\\" or an integer in the range 1-255.\"\n pattern: ^.*\n x-kubernetes-int-or-string: true\n source:\n description: Source contains the match criteria that apply to\n source entity.\n properties:\n namespaceSelector:\n description: \"NamespaceSelector is an optional field that\n contains a selector expression. Only traffic that originates\n from (or terminates at) endpoints within the selected\n namespaces will be matched. When both NamespaceSelector\n and another selector are defined on the same rule, then\n only workload endpoints that are matched by both selectors\n will be selected by the rule. \\n For NetworkPolicy, an\n empty NamespaceSelector implies that the Selector is limited\n to selecting only workload endpoints in the same namespace\n as the NetworkPolicy. \\n For NetworkPolicy, `global()`\n NamespaceSelector implies that the Selector is limited\n to selecting only GlobalNetworkSet or HostEndpoint. \\n\n For GlobalNetworkPolicy, an empty NamespaceSelector implies\n the Selector applies to workload endpoints across all\n namespaces.\"\n type: string\n nets:\n description: Nets is an optional field that restricts the\n rule to only apply to traffic that originates from (or\n terminates at) IP addresses in any of the given subnets.\n items:\n type: string\n type: array\n notNets:\n description: NotNets is the negated version of the Nets\n field.\n items:\n type: string\n type: array\n notPorts:\n description: NotPorts is the negated version of the Ports\n field. Since only some protocols have ports, if any ports\n are specified it requires the Protocol match in the Rule\n to be set to \"TCP\" or \"UDP\".\n items:\n anyOf:\n - type: integer\n - type: string\n pattern: ^.*\n x-kubernetes-int-or-string: true\n type: array\n notSelector:\n description: NotSelector is the negated version of the Selector\n field. See Selector field for subtleties with negated\n selectors.\n type: string\n ports:\n description: \"Ports is an optional field that restricts\n the rule to only apply to traffic that has a source (destination)\n port that matches one of these ranges/values. This value\n is a list of integers or strings that represent ranges\n of ports. \\n Since only some protocols have ports, if\n any ports are specified it requires the Protocol match\n in the Rule to be set to \\\"TCP\\\" or \\\"UDP\\\".\"\n items:\n anyOf:\n - type: integer\n - type: string\n pattern: ^.*\n x-kubernetes-int-or-string: true\n type: array\n selector:\n description: \"Selector is an optional field that contains\n a selector expression (see Policy for sample syntax).\n \\ Only traffic that originates from (terminates at) endpoints\n matching the selector will be matched. \\n Note that: in\n addition to the negated version of the Selector (see NotSelector\n below), the selector expression syntax itself supports\n negation. The two types of negation are subtly different.\n One negates the set of matched endpoints, the other negates\n the whole match: \\n \\tSelector = \\\"!has(my_label)\\\" matches\n packets that are from other Calico-controlled \\tendpoints\n that do not have the label \\\"my_label\\\". \\n \\tNotSelector\n = \\\"has(my_label)\\\" matches packets that are not from\n Calico-controlled \\tendpoints that do have the label \\\"my_label\\\".\n \\n The effect is that the latter will accept packets from\n non-Calico sources whereas the former is limited to packets\n from Calico-controlled endpoints.\"\n type: string\n serviceAccounts:\n description: ServiceAccounts is an optional field that restricts\n the rule to only apply to traffic that originates from\n (or terminates at) a pod running as a matching service\n account.\n properties:\n names:\n description: Names is an optional field that restricts\n the rule to only apply to traffic that originates\n from (or terminates at) a pod running as a service\n account whose name is in the list.\n items:\n type: string\n type: array\n selector:\n description: Selector is an optional field that restricts\n the rule to only apply to traffic that originates\n from (or terminates at) a pod running as a service\n account that matches the given label selector. If\n both Names and Selector are specified then they are\n AND'ed.\n type: string\n type: object\n services:\n description: \"Services is an optional field that contains\n options for matching Kubernetes Services. If specified,\n only traffic that originates from or terminates at endpoints\n within the selected service(s) will be matched, and only\n to/from each endpoint's port. \\n Services cannot be specified\n on the same rule as Selector, NotSelector, NamespaceSelector,\n Nets, NotNets or ServiceAccounts. \\n Ports and NotPorts\n can only be specified with Services on ingress rules.\"\n properties:\n name:\n description: Name specifies the name of a Kubernetes\n Service to match.\n type: string\n namespace:\n description: Namespace specifies the namespace of the\n given Service. If left empty, the rule will match\n within this policy's namespace.\n type: string\n type: object\n type: object\n required:\n - action\n type: object\n type: array\n ingress:\n description: The ordered set of ingress rules. Each rule contains\n a set of packet match criteria and a corresponding action to apply.\n items:\n description: \"A Rule encapsulates a set of match criteria and an\n action. Both selector-based security Policy and security Profiles\n reference rules - separated out as a list of rules for both ingress\n and egress packet matching. \\n Each positive match criteria has\n a negated version, prefixed with \\\"Not\\\". All the match criteria\n within a rule must be satisfied for a packet to match. A single\n rule can contain the positive and negative version of a match\n and both must be satisfied for the rule to match.\"\n properties:\n action:\n type: string\n destination:\n description: Destination contains the match criteria that apply\n to destination entity.\n properties:\n namespaceSelector:\n description: \"NamespaceSelector is an optional field that\n contains a selector expression. Only traffic that originates\n from (or terminates at) endpoints within the selected\n namespaces will be matched. When both NamespaceSelector\n and another selector are defined on the same rule, then\n only workload endpoints that are matched by both selectors\n will be selected by the rule. \\n For NetworkPolicy, an\n empty NamespaceSelector implies that the Selector is limited\n to selecting only workload endpoints in the same namespace\n as the NetworkPolicy. \\n For NetworkPolicy, `global()`\n NamespaceSelector implies that the Selector is limited\n to selecting only GlobalNetworkSet or HostEndpoint. \\n\n For GlobalNetworkPolicy, an empty NamespaceSelector implies\n the Selector applies to workload endpoints across all\n namespaces.\"\n type: string\n nets:\n description: Nets is an optional field that restricts the\n rule to only apply to traffic that originates from (or\n terminates at) IP addresses in any of the given subnets.\n items:\n type: string\n type: array\n notNets:\n description: NotNets is the negated version of the Nets\n field.\n items:\n type: string\n type: array\n notPorts:\n description: NotPorts is the negated version of the Ports\n field. Since only some protocols have ports, if any ports\n are specified it requires the Protocol match in the Rule\n to be set to \"TCP\" or \"UDP\".\n items:\n anyOf:\n - type: integer\n - type: string\n pattern: ^.*\n x-kubernetes-int-or-string: true\n type: array\n notSelector:\n description: NotSelector is the negated version of the Selector\n field. See Selector field for subtleties with negated\n selectors.\n type: string\n ports:\n description: \"Ports is an optional field that restricts\n the rule to only apply to traffic that has a source (destination)\n port that matches one of these ranges/values. This value\n is a list of integers or strings that represent ranges\n of ports. \\n Since only some protocols have ports, if\n any ports are specified it requires the Protocol match\n in the Rule to be set to \\\"TCP\\\" or \\\"UDP\\\".\"\n items:\n anyOf:\n - type: integer\n - type: string\n pattern: ^.*\n x-kubernetes-int-or-string: true\n type: array\n selector:\n description: \"Selector is an optional field that contains\n a selector expression (see Policy for sample syntax).\n \\ Only traffic that originates from (terminates at) endpoints\n matching the selector will be matched. \\n Note that: in\n addition to the negated version of the Selector (see NotSelector\n below), the selector expression syntax itself supports\n negation. The two types of negation are subtly different.\n One negates the set of matched endpoints, the other negates\n the whole match: \\n \\tSelector = \\\"!has(my_label)\\\" matches\n packets that are from other Calico-controlled \\tendpoints\n that do not have the label \\\"my_label\\\". \\n \\tNotSelector\n = \\\"has(my_label)\\\" matches packets that are not from\n Calico-controlled \\tendpoints that do have the label \\\"my_label\\\".\n \\n The effect is that the latter will accept packets from\n non-Calico sources whereas the former is limited to packets\n from Calico-controlled endpoints.\"\n type: string\n serviceAccounts:\n description: ServiceAccounts is an optional field that restricts\n the rule to only apply to traffic that originates from\n (or terminates at) a pod running as a matching service\n account.\n properties:\n names:\n description: Names is an optional field that restricts\n the rule to only apply to traffic that originates\n from (or terminates at) a pod running as a service\n account whose name is in the list.\n items:\n type: string\n type: array\n selector:\n description: Selector is an optional field that restricts\n the rule to only apply to traffic that originates\n from (or terminates at) a pod running as a service\n account that matches the given label selector. If\n both Names and Selector are specified then they are\n AND'ed.\n type: string\n type: object\n services:\n description: \"Services is an optional field that contains\n options for matching Kubernetes Services. If specified,\n only traffic that originates from or terminates at endpoints\n within the selected service(s) will be matched, and only\n to/from each endpoint's port. \\n Services cannot be specified\n on the same rule as Selector, NotSelector, NamespaceSelector,\n Nets, NotNets or ServiceAccounts. \\n Ports and NotPorts\n can only be specified with Services on ingress rules.\"\n properties:\n name:\n description: Name specifies the name of a Kubernetes\n Service to match.\n type: string\n namespace:\n description: Namespace specifies the namespace of the\n given Service. If left empty, the rule will match\n within this policy's namespace.\n type: string\n type: object\n type: object\n http:\n description: HTTP contains match criteria that apply to HTTP\n requests.\n properties:\n methods:\n description: Methods is an optional field that restricts\n the rule to apply only to HTTP requests that use one of\n the listed HTTP Methods (e.g. GET, PUT, etc.) Multiple\n methods are OR'd together.\n items:\n type: string\n type: array\n paths:\n description: 'Paths is an optional field that restricts\n the rule to apply to HTTP requests that use one of the\n listed HTTP Paths. Multiple paths are OR''d together.\n e.g: - exact: /foo - prefix: /bar NOTE: Each entry may\n ONLY specify either a `exact` or a `prefix` match. The\n validator will check for it.'\n items:\n description: 'HTTPPath specifies an HTTP path to match.\n It may be either of the form: exact: : which matches\n the path exactly or prefix: : which matches\n the path prefix'\n properties:\n exact:\n type: string\n prefix:\n type: string\n type: object\n type: array\n type: object\n icmp:\n description: ICMP is an optional field that restricts the rule\n to apply to a specific type and code of ICMP traffic. This\n should only be specified if the Protocol field is set to \"ICMP\"\n or \"ICMPv6\".\n properties:\n code:\n description: Match on a specific ICMP code. If specified,\n the Type value must also be specified. This is a technical\n limitation imposed by the kernel's iptables firewall,\n which Calico uses to enforce the rule.\n type: integer\n type:\n description: Match on a specific ICMP type. For example\n a value of 8 refers to ICMP Echo Request (i.e. pings).\n type: integer\n type: object\n ipVersion:\n description: IPVersion is an optional field that restricts the\n rule to only match a specific IP version.\n type: integer\n metadata:\n description: Metadata contains additional information for this\n rule\n properties:\n annotations:\n additionalProperties:\n type: string\n description: Annotations is a set of key value pairs that\n give extra information about the rule\n type: object\n type: object\n notICMP:\n description: NotICMP is the negated version of the ICMP field.\n properties:\n code:\n description: Match on a specific ICMP code. If specified,\n the Type value must also be specified. This is a technical\n limitation imposed by the kernel's iptables firewall,\n which Calico uses to enforce the rule.\n type: integer\n type:\n description: Match on a specific ICMP type. For example\n a value of 8 refers to ICMP Echo Request (i.e. pings).\n type: integer\n type: object\n notProtocol:\n anyOf:\n - type: integer\n - type: string\n description: NotProtocol is the negated version of the Protocol\n field.\n pattern: ^.*\n x-kubernetes-int-or-string: true\n protocol:\n anyOf:\n - type: integer\n - type: string\n description: \"Protocol is an optional field that restricts the\n rule to only apply to traffic of a specific IP protocol. Required\n if any of the EntityRules contain Ports (because ports only\n apply to certain protocols). \\n Must be one of these string\n values: \\\"TCP\\\", \\\"UDP\\\", \\\"ICMP\\\", \\\"ICMPv6\\\", \\\"SCTP\\\",\n \\\"UDPLite\\\" or an integer in the range 1-255.\"\n pattern: ^.*\n x-kubernetes-int-or-string: true\n source:\n description: Source contains the match criteria that apply to\n source entity.\n properties:\n namespaceSelector:\n description: \"NamespaceSelector is an optional field that\n contains a selector expression. Only traffic that originates\n from (or terminates at) endpoints within the selected\n namespaces will be matched. When both NamespaceSelector\n and another selector are defined on the same rule, then\n only workload endpoints that are matched by both selectors\n will be selected by the rule. \\n For NetworkPolicy, an\n empty NamespaceSelector implies that the Selector is limited\n to selecting only workload endpoints in the same namespace\n as the NetworkPolicy. \\n For NetworkPolicy, `global()`\n NamespaceSelector implies that the Selector is limited\n to selecting only GlobalNetworkSet or HostEndpoint. \\n\n For GlobalNetworkPolicy, an empty NamespaceSelector implies\n the Selector applies to workload endpoints across all\n namespaces.\"\n type: string\n nets:\n description: Nets is an optional field that restricts the\n rule to only apply to traffic that originates from (or\n terminates at) IP addresses in any of the given subnets.\n items:\n type: string\n type: array\n notNets:\n description: NotNets is the negated version of the Nets\n field.\n items:\n type: string\n type: array\n notPorts:\n description: NotPorts is the negated version of the Ports\n field. Since only some protocols have ports, if any ports\n are specified it requires the Protocol match in the Rule\n to be set to \"TCP\" or \"UDP\".\n items:\n anyOf:\n - type: integer\n - type: string\n pattern: ^.*\n x-kubernetes-int-or-string: true\n type: array\n notSelector:\n description: NotSelector is the negated version of the Selector\n field. See Selector field for subtleties with negated\n selectors.\n type: string\n ports:\n description: \"Ports is an optional field that restricts\n the rule to only apply to traffic that has a source (destination)\n port that matches one of these ranges/values. This value\n is a list of integers or strings that represent ranges\n of ports. \\n Since only some protocols have ports, if\n any ports are specified it requires the Protocol match\n in the Rule to be set to \\\"TCP\\\" or \\\"UDP\\\".\"\n items:\n anyOf:\n - type: integer\n - type: string\n pattern: ^.*\n x-kubernetes-int-or-string: true\n type: array\n selector:\n description: \"Selector is an optional field that contains\n a selector expression (see Policy for sample syntax).\n \\ Only traffic that originates from (terminates at) endpoints\n matching the selector will be matched. \\n Note that: in\n addition to the negated version of the Selector (see NotSelector\n below), the selector expression syntax itself supports\n negation. The two types of negation are subtly different.\n One negates the set of matched endpoints, the other negates\n the whole match: \\n \\tSelector = \\\"!has(my_label)\\\" matches\n packets that are from other Calico-controlled \\tendpoints\n that do not have the label \\\"my_label\\\". \\n \\tNotSelector\n = \\\"has(my_label)\\\" matches packets that are not from\n Calico-controlled \\tendpoints that do have the label \\\"my_label\\\".\n \\n The effect is that the latter will accept packets from\n non-Calico sources whereas the former is limited to packets\n from Calico-controlled endpoints.\"\n type: string\n serviceAccounts:\n description: ServiceAccounts is an optional field that restricts\n the rule to only apply to traffic that originates from\n (or terminates at) a pod running as a matching service\n account.\n properties:\n names:\n description: Names is an optional field that restricts\n the rule to only apply to traffic that originates\n from (or terminates at) a pod running as a service\n account whose name is in the list.\n items:\n type: string\n type: array\n selector:\n description: Selector is an optional field that restricts\n the rule to only apply to traffic that originates\n from (or terminates at) a pod running as a service\n account that matches the given label selector. If\n both Names and Selector are specified then they are\n AND'ed.\n type: string\n type: object\n services:\n description: \"Services is an optional field that contains\n options for matching Kubernetes Services. If specified,\n only traffic that originates from or terminates at endpoints\n within the selected service(s) will be matched, and only\n to/from each endpoint's port. \\n Services cannot be specified\n on the same rule as Selector, NotSelector, NamespaceSelector,\n Nets, NotNets or ServiceAccounts. \\n Ports and NotPorts\n can only be specified with Services on ingress rules.\"\n properties:\n name:\n description: Name specifies the name of a Kubernetes\n Service to match.\n type: string\n namespace:\n description: Namespace specifies the namespace of the\n given Service. If left empty, the rule will match\n within this policy's namespace.\n type: string\n type: object\n type: object\n required:\n - action\n type: object\n type: array\n order:\n description: Order is an optional field that specifies the order in\n which the policy is applied. Policies with higher \"order\" are applied\n after those with lower order within the same tier. If the order\n is omitted, it may be considered to be \"infinite\" - i.e. the policy\n will be applied last. Policies with identical order will be applied\n in alphanumerical order based on the Policy \"Name\" within the tier.\n type: number\n performanceHints:\n description: \"PerformanceHints contains a list of hints to Calico's\n policy engine to help process the policy more efficiently. Hints\n never change the enforcement behaviour of the policy. \\n Currently,\n the only available hint is \\\"AssumeNeededOnEveryNode\\\". When that\n hint is set on a policy, Felix will act as if the policy matches\n a local endpoint even if it does not. This is useful for \\\"preloading\\\"\n any large static policies that are known to be used on every node.\n If the policy is _not_ used on a particular node then the work done\n to preload the policy (and to maintain it) is wasted.\"\n items:\n type: string\n type: array\n selector:\n description: \"The selector is an expression used to pick out the endpoints\n that the policy should be applied to. \\n Selector expressions follow\n this syntax: \\n \\tlabel == \\\"string_literal\\\" -> comparison, e.g.\n my_label == \\\"foo bar\\\" \\tlabel != \\\"string_literal\\\" -> not\n equal; also matches if label is not present \\tlabel in { \\\"a\\\",\n \\\"b\\\", \\\"c\\\", ... } -> true if the value of label X is one of\n \\\"a\\\", \\\"b\\\", \\\"c\\\" \\tlabel not in { \\\"a\\\", \\\"b\\\", \\\"c\\\", ... }\n \\ -> true if the value of label X is not one of \\\"a\\\", \\\"b\\\", \\\"c\\\"\n \\thas(label_name) -> True if that label is present \\t! expr ->\n negation of expr \\texpr && expr -> Short-circuit and \\texpr ||\n expr -> Short-circuit or \\t( expr ) -> parens for grouping \\tall()\n or the empty selector -> matches all endpoints. \\n Label names are\n allowed to contain alphanumerics, -, _ and /. String literals are\n more permissive but they do not support escape characters. \\n Examples\n (with made-up labels): \\n \\ttype == \\\"webserver\\\" && deployment\n == \\\"prod\\\" \\ttype in {\\\"frontend\\\", \\\"backend\\\"} \\tdeployment !=\n \\\"dev\\\" \\t! has(label_name)\"\n type: string\n serviceAccountSelector:\n description: ServiceAccountSelector is an optional field for an expression\n used to select a pod based on service accounts.\n type: string\n tier:\n description: The name of the tier that this policy belongs to. If\n this is omitted, the default tier (name is \"default\") is assumed. The\n specified tier must exist in order to create security policies within\n the tier, the \"default\" tier is created automatically if it does\n not exist, this means for deployments requiring only a single Tier,\n the tier name may be omitted on all policy management requests.\n type: string\n types:\n description: \"Types indicates whether this policy applies to ingress,\n or to egress, or to both. When not explicitly specified (and so\n the value on creation is empty or nil), Calico defaults Types according\n to what Ingress and Egress are present in the policy. The default\n is: \\n - [ PolicyTypeIngress ], if there are no Egress rules (including\n the case where there are also no Ingress rules) \\n - [ PolicyTypeEgress\n ], if there are Egress rules but no Ingress rules \\n - [ PolicyTypeIngress,\n PolicyTypeEgress ], if there are both Ingress and Egress rules.\n \\n When the policy is read back again, Types will always be one\n of these values, never empty or nil.\"\n items:\n description: PolicyType enumerates the possible values of the PolicySpec\n Types field.\n type: string\n type: array\n type: object\n type: object\n served: true\n storage: true\nstatus:\n acceptedNames:\n kind: \"\"\n plural: \"\"\n conditions: []\n storedVersions: []\n" From 78ae5bd9832cec488746f43e604472dd21424d1a Mon Sep 17 00:00:00 2001 From: Michal Fupso Date: Mon, 16 Sep 2024 15:51:18 -0700 Subject: [PATCH 08/12] Test updates --- calicoctl/calicoctl/commands/ipam/check.go | 8 +++----- calicoctl/tests/st/utils/data.py | 9 +++++++++ calicoctl/tests/st/utils/v1_data.py | 9 ++++++--- kube-controllers/cmd/kube-controllers/main.go | 2 +- kube-controllers/pkg/config/config_test.go | 3 +++ kube-controllers/pkg/config/runconfig.go | 5 ++++- .../loadbalancer/loadbalancer_controller.go | 4 ++-- .../loadbalancer/loadbalancer_controller_fv_test.go | 12 +++++++----- .../syncersv1/felixsyncer/felixsyncer_e2e_test.go | 13 +++++++------ .../tunnelipsyncer/tunnelipsyncer_e2e_test.go | 13 +++++++------ libcalico-go/lib/ipam/ipam.go | 11 +++++++---- 11 files changed, 56 insertions(+), 33 deletions(-) diff --git a/calicoctl/calicoctl/commands/ipam/check.go b/calicoctl/calicoctl/commands/ipam/check.go index 4d35f156ead..88ff202474c 100644 --- a/calicoctl/calicoctl/commands/ipam/check.go +++ b/calicoctl/calicoctl/commands/ipam/check.go @@ -24,13 +24,12 @@ import ( "strings" "github.com/docopt/docopt-go" + "github.com/projectcalico/calico/kube-controllers/pkg/controllers/loadbalancer" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/kubernetes" - "github.com/projectcalico/calico/kube-controllers/pkg/controllers/loadbalancer" - "github.com/projectcalico/calico/libcalico-go/lib/set" "github.com/projectcalico/calico/libcalico-go/lib/ipam" @@ -261,7 +260,7 @@ func (c *IPAMChecker) checkIPAM(ctx context.Context) error { { fmt.Println("Loading all service load balancer.") - service, err := c.k8sClient.CoreV1().Services("").List(ctx, metav1.ListOptions{}) + services, err := c.k8sClient.CoreV1().Services("").List(ctx, metav1.ListOptions{}) if err != nil { return err } @@ -272,8 +271,7 @@ func (c *IPAMChecker) checkIPAM(ctx context.Context) error { } var lengthLoadBalancer int - - for _, svc := range service.Items { + for _, svc := range services.Items { if svc.Spec.Type == corev1.ServiceTypeLoadBalancer && loadbalancer.IsCalicoManagedLoadBalancer(&svc, kubeControllerConfig.Spec.Controllers.LoadBalancer.AssignIPs) { lengthLoadBalancer++ diff --git a/calicoctl/tests/st/utils/data.py b/calicoctl/tests/st/utils/data.py index ca389afcd41..0b31bc0cf76 100644 --- a/calicoctl/tests/st/utils/data.py +++ b/calicoctl/tests/st/utils/data.py @@ -50,6 +50,7 @@ 'blockSize': 27, 'allowedUses': ["Workload", "Tunnel"], 'nodeSelector': "foo == 'bar'", + 'assignmentMode': "Automatic", } } @@ -75,6 +76,7 @@ 'vxlanMode': 'Always', 'allowedUses': ["Workload", "Tunnel"], 'nodeSelector': "all()", + 'assignmentMode': "Automatic", } } @@ -92,6 +94,7 @@ 'allowedUses': ["Workload", "Tunnel"], 'nodeSelector': "foo == 'bar'", 'disabled': True, + 'assignmentMode': "Automatic", } } @@ -111,6 +114,7 @@ 'allowedUses': ["Workload", "Tunnel"], 'nodeSelector': "foo == 'bar'", 'disabled': True, + 'assignmentMode': "Automatic", } } @@ -127,6 +131,7 @@ 'blockSize': 27, 'allowedUses': ["Workload", "Tunnel"], 'nodeSelector': "foo == 'bar'", + 'assignmentMode': "Automatic", } } @@ -143,6 +148,7 @@ 'blockSize': 27, 'allowedUses': ["Workload", "Tunnel"], 'nodeSelector': "foo == 'bar'", + 'assignmentMode': "Automatic", } } @@ -159,6 +165,7 @@ 'blockSize': 27, 'allowedUses': ["Workload", "Tunnel"], 'nodeSelector': "foo == 'bar'", + 'assignmentMode': "Automatic", } } @@ -175,6 +182,7 @@ 'blockSize': 27, 'allowedUses': ["Workload", "Tunnel"], 'nodeSelector': "foo == 'bar'", + 'assignmentMode': "Automatic", } } @@ -191,6 +199,7 @@ 'blockSize': 123, 'allowedUses': ["Workload", "Tunnel"], 'nodeSelector': "all()", + 'assignmentMode': "Automatic", } } diff --git a/calicoctl/tests/st/utils/v1_data.py b/calicoctl/tests/st/utils/v1_data.py index 601f56313d9..231ff10f857 100644 --- a/calicoctl/tests/st/utils/v1_data.py +++ b/calicoctl/tests/st/utils/v1_data.py @@ -115,7 +115,8 @@ 'metadata': {'cidr': '10.1.0.0/26'}, 'spec': {'disabled': False, 'ipip': {'enabled': True, 'mode': 'cross-subnet'}, - 'nat-outgoing': True} + 'nat-outgoing': True, + 'assignmentMode': "Automatic",} } data['ippool_v4_large'] = { 'apiVersion': 'v1', @@ -123,7 +124,8 @@ 'metadata': {'cidr': '10.0.0.0/8'}, 'spec': {'disabled': False, 'ipip': {'enabled': True, 'mode': 'always'}, - 'nat-outgoing': True} + 'nat-outgoing': True, + 'assignmentMode': "Automatic",} } data['ippool_mixed'] = { 'apiVersion': 'v1', @@ -131,7 +133,8 @@ 'metadata': {'cidr': '2006::/64'}, 'spec': {'disabled': False, 'ipip': {'enabled': False, 'mode': 'always'}, - 'nat-outgoing': False} + 'nat-outgoing': False, + 'assignmentMode': "Automatic",} } data['node_long_name'] = { 'apiVersion': 'v1', diff --git a/kube-controllers/cmd/kube-controllers/main.go b/kube-controllers/cmd/kube-controllers/main.go index 6bd6b406e6e..f7d31388ac5 100644 --- a/kube-controllers/cmd/kube-controllers/main.go +++ b/kube-controllers/cmd/kube-controllers/main.go @@ -23,7 +23,6 @@ import ( "strings" "time" - "github.com/projectcalico/calico/kube-controllers/pkg/controllers/loadbalancer" "github.com/prometheus/client_golang/prometheus/promhttp" log "github.com/sirupsen/logrus" "go.etcd.io/etcd/client/pkg/v3/srv" @@ -39,6 +38,7 @@ import ( "github.com/projectcalico/calico/kube-controllers/pkg/config" "github.com/projectcalico/calico/kube-controllers/pkg/controllers/controller" "github.com/projectcalico/calico/kube-controllers/pkg/controllers/flannelmigration" + "github.com/projectcalico/calico/kube-controllers/pkg/controllers/loadbalancer" "github.com/projectcalico/calico/kube-controllers/pkg/controllers/namespace" "github.com/projectcalico/calico/kube-controllers/pkg/controllers/networkpolicy" "github.com/projectcalico/calico/kube-controllers/pkg/controllers/node" diff --git a/kube-controllers/pkg/config/config_test.go b/kube-controllers/pkg/config/config_test.go index b1676cbdcee..ed5f4c730e5 100644 --- a/kube-controllers/pkg/config/config_test.go +++ b/kube-controllers/pkg/config/config_test.go @@ -171,6 +171,9 @@ var _ = Describe("Config", func() { ReconcilerPeriod: &v1.Duration{Duration: time.Minute * 5}})) Expect(c.ServiceAccount).To(Equal(&v3.ServiceAccountControllerConfig{ ReconcilerPeriod: &v1.Duration{Duration: time.Minute * 5}})) + Expect(c.LoadBalancer).To(Equal(&v3.LoadBalancerControllerConfig{ + AssignIPs: v3.AllServices, + })) close(done) }) }) diff --git a/kube-controllers/pkg/config/runconfig.go b/kube-controllers/pkg/config/runconfig.go index 8479fda8daf..fd3012ae42f 100644 --- a/kube-controllers/pkg/config/runconfig.go +++ b/kube-controllers/pkg/config/runconfig.go @@ -368,7 +368,10 @@ func mergeConfig(envVars map[string]string, envCfg Config, apiCfg v3.KubeControl } if rc.LoadBalancer != nil { - rc.LoadBalancer.AssignIPs = apiCfg.Controllers.LoadBalancer.AssignIPs + if apiCfg.Controllers.LoadBalancer != nil { + rc.LoadBalancer.AssignIPs = apiCfg.Controllers.LoadBalancer.AssignIPs + status.RunningConfig.Controllers.LoadBalancer.AssignIPs = apiCfg.Controllers.LoadBalancer.AssignIPs + } } return rCfg, status diff --git a/kube-controllers/pkg/controllers/loadbalancer/loadbalancer_controller.go b/kube-controllers/pkg/controllers/loadbalancer/loadbalancer_controller.go index 38dc27f94a1..c5c9a78bb98 100644 --- a/kube-controllers/pkg/controllers/loadbalancer/loadbalancer_controller.go +++ b/kube-controllers/pkg/controllers/loadbalancer/loadbalancer_controller.go @@ -651,7 +651,7 @@ func (c *loadBalancerController) parseAnnotations(annotations map[string]string) for _, ipAddr := range ipAddrs { curr := cnet.ParseIP(ipAddr) if curr == nil { - return nil, nil, nil, errors.New(fmt.Sprintf("Could not parse %s as a valid IP address", ipAddr)) + return nil, nil, nil, fmt.Errorf("Could not parse %s as a valid IP address", ipAddr) } if curr.To4() != nil { ipv4++ @@ -662,7 +662,7 @@ func (c *loadBalancerController) parseAnnotations(annotations map[string]string) } if ipv6 > 1 || ipv4 > 1 { - return nil, nil, nil, errors.New(fmt.Sprintf("At max only one ipv4 and one ipv6 address can be specified. Recieved %d ipv4 and %d ipv6 addresses", ipv4, ipv6)) + return nil, nil, nil, fmt.Errorf("At max only one ipv4 and one ipv6 address can be specified. Recieved %d ipv4 and %d ipv6 addresses", ipv4, ipv6) } } } diff --git a/kube-controllers/pkg/controllers/loadbalancer/loadbalancer_controller_fv_test.go b/kube-controllers/pkg/controllers/loadbalancer/loadbalancer_controller_fv_test.go index 7443952387c..94b7d9fcaf8 100644 --- a/kube-controllers/pkg/controllers/loadbalancer/loadbalancer_controller_fv_test.go +++ b/kube-controllers/pkg/controllers/loadbalancer/loadbalancer_controller_fv_test.go @@ -22,18 +22,20 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/client-go/kubernetes" + v3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" + "github.com/projectcalico/calico/felix/fv/containers" "github.com/projectcalico/calico/kube-controllers/tests/testutils" "github.com/projectcalico/calico/libcalico-go/lib/apiconfig" client "github.com/projectcalico/calico/libcalico-go/lib/clientv3" "github.com/projectcalico/calico/libcalico-go/lib/json" "github.com/projectcalico/calico/libcalico-go/lib/options" - v1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/types" - "k8s.io/apimachinery/pkg/util/intstr" - "k8s.io/client-go/kubernetes" ) var _ = Describe("Calico loadbalancer controller FV tests (etcd mode)", func() { diff --git a/libcalico-go/lib/backend/syncersv1/felixsyncer/felixsyncer_e2e_test.go b/libcalico-go/lib/backend/syncersv1/felixsyncer/felixsyncer_e2e_test.go index 3dcf02d351a..b71f1df0c71 100644 --- a/libcalico-go/lib/backend/syncersv1/felixsyncer/felixsyncer_e2e_test.go +++ b/libcalico-go/lib/backend/syncersv1/felixsyncer/felixsyncer_e2e_test.go @@ -462,12 +462,13 @@ var _ = testutils.E2eDatastoreDescribe("Felix syncer tests", testutils.Datastore syncTester.ExpectData(model.KVPair{ Key: model.IPPoolKey{CIDR: net.MustParseCIDR("192.124.0.0/21")}, Value: &model.IPPool{ - CIDR: poolCIDRNet, - IPIPInterface: "tunl0", - IPIPMode: encap.CrossSubnet, - Masquerade: true, - IPAM: true, - Disabled: false, + CIDR: poolCIDRNet, + IPIPInterface: "tunl0", + IPIPMode: encap.CrossSubnet, + Masquerade: true, + IPAM: true, + Disabled: false, + AssignmentMode: apiv3.Automatic, }, Revision: pool.ResourceVersion, }) diff --git a/libcalico-go/lib/backend/syncersv1/tunnelipsyncer/tunnelipsyncer_e2e_test.go b/libcalico-go/lib/backend/syncersv1/tunnelipsyncer/tunnelipsyncer_e2e_test.go index 34a623add2a..b0c75d2af94 100644 --- a/libcalico-go/lib/backend/syncersv1/tunnelipsyncer/tunnelipsyncer_e2e_test.go +++ b/libcalico-go/lib/backend/syncersv1/tunnelipsyncer/tunnelipsyncer_e2e_test.go @@ -101,12 +101,13 @@ var _ = testutils.E2eDatastoreDescribe("Tunnel IP allocation syncer tests", test syncTester.ExpectData(model.KVPair{ Key: poolKeyV1, Value: &model.IPPool{ - CIDR: poolCIDRNet, - IPIPInterface: "tunl0", - IPIPMode: encap.CrossSubnet, - Masquerade: true, - IPAM: true, - Disabled: false, + CIDR: poolCIDRNet, + IPIPInterface: "tunl0", + IPIPMode: encap.CrossSubnet, + Masquerade: true, + IPAM: true, + Disabled: false, + AssignmentMode: apiv3.Automatic, }, Revision: pool.ResourceVersion, }) diff --git a/libcalico-go/lib/ipam/ipam.go b/libcalico-go/lib/ipam/ipam.go index c15f4eef89a..6a733fab540 100644 --- a/libcalico-go/lib/ipam/ipam.go +++ b/libcalico-go/lib/ipam/ipam.go @@ -56,6 +56,9 @@ const ( AttributeTypeVXLANV6 = model.IPAMBlockAttributeTypeVXLANV6 AttributeTypeWireguard = model.IPAMBlockAttributeTypeWireguard AttributeTypeWireguardV6 = model.IPAMBlockAttributeTypeWireguardV6 + + // Host affinity used for Service LoadBalancer + loadBalancerAffinityHost = "virtual:virtual-load-balancer" ) var ( @@ -305,7 +308,7 @@ func (c ipamClient) determinePools(ctx context.Context, requestedPoolNets []net. // At this point, we've determined the set of enabled IP pools which are valid for use. // We only want to use IP pools which actually match this node, so do a filter based on - // selector. + // selector. Additionally, we check the ippools assignmentMode type so we don't use ips from Manual pool when no pool was specified for _, pool := range enabledPools { if requestedPoolNets == nil && pool.Spec.AssignmentMode != v3.Automatic { continue @@ -337,8 +340,8 @@ func (c ipamClient) prepareAffinityBlocksForHost(ctx context.Context, requestedP var err error var v3n *libapiv3.Node var ok bool - // Regular node, continue as usual - if host != v3.VirtualLoadBalancer { + // Use is not LoadBalancer, continue as normal with node + if use != v3.IPPoolAllowedUseLoadBalancer { node, err = c.client.Get(ctx, model.ResourceKey{Kind: libapiv3.KindNode, Name: host}, "") if err != nil { log.WithError(err).WithField("node", host).Error("failed to get node for host") @@ -1685,7 +1688,7 @@ func (c ipamClient) releaseByHandle(ctx context.Context, blockCIDR net.IPNet, op } // If this is loadBalancer we delete the block without waiting - if *block.Affinity == "virtual:virtual-load-balancer" { + if *block.Affinity == loadBalancerAffinityHost { block = allocationBlock{obj.Value.(*model.AllocationBlock)} if block.empty() { err = c.blockReaderWriter.deleteBlock(ctx, obj) From aa6da2d13bfc1a6159761cde4bb2b9ee5e7e26fe Mon Sep 17 00:00:00 2001 From: Michal Fupso Date: Mon, 16 Sep 2024 16:28:57 -0700 Subject: [PATCH 09/12] imports --- calicoctl/calicoctl/commands/ipam/check.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/calicoctl/calicoctl/commands/ipam/check.go b/calicoctl/calicoctl/commands/ipam/check.go index 88ff202474c..453b3c6d439 100644 --- a/calicoctl/calicoctl/commands/ipam/check.go +++ b/calicoctl/calicoctl/commands/ipam/check.go @@ -24,12 +24,13 @@ import ( "strings" "github.com/docopt/docopt-go" - "github.com/projectcalico/calico/kube-controllers/pkg/controllers/loadbalancer" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/kubernetes" + "github.com/projectcalico/calico/kube-controllers/pkg/controllers/loadbalancer" + "github.com/projectcalico/calico/libcalico-go/lib/set" "github.com/projectcalico/calico/libcalico-go/lib/ipam" From d1ab7f3d4f52eb4d35c55b8ea603160ffba6611b Mon Sep 17 00:00:00 2001 From: Michal Fupso Date: Tue, 24 Sep 2024 08:27:39 -0700 Subject: [PATCH 10/12] Loadbalancer IPAM updates --- kube-controllers/cmd/kube-controllers/main.go | 6 + .../flannelmigration/ipam_migrator.go | 13 +- .../loadbalancer/loadbalancer_controller.go | 680 ++++++++++++------ .../loadbalancer_controller_fv_test.go | 24 - .../pkg/controllers/node/controller.go | 3 - .../pkg/controllers/node/fake_client.go | 2 +- libcalico-go/lib/clientv3/common_test.go | 13 +- libcalico-go/lib/clientv3/ippool.go | 3 +- libcalico-go/lib/clientv3/ippool_e2e_test.go | 42 +- .../clientv3/ippool_kdd_conversion_test.go | 112 +-- libcalico-go/lib/clientv3/node_e2e_test.go | 6 +- libcalico-go/lib/ipam/interface.go | 2 +- libcalico-go/lib/ipam/ipam.go | 72 +- libcalico-go/lib/ipam/ipam_block.go | 10 +- .../lib/ipam/ipam_block_reader_writer.go | 21 +- .../lib/ipam/ipam_block_reader_writer_test.go | 17 +- libcalico-go/lib/ipam/ipam_test.go | 13 +- libcalico-go/lib/ipam/ipam_types.go | 3 + 18 files changed, 675 insertions(+), 367 deletions(-) diff --git a/kube-controllers/cmd/kube-controllers/main.go b/kube-controllers/cmd/kube-controllers/main.go index f7d31388ac5..42a4f010425 100644 --- a/kube-controllers/cmd/kube-controllers/main.go +++ b/kube-controllers/cmd/kube-controllers/main.go @@ -432,8 +432,14 @@ func (cc *controllerControl) InitControllers(ctx context.Context, cfg config.Run podInformer := factory.Core().V1().Pods().Informer() nodeInformer := factory.Core().V1().Nodes().Informer() serviceInformer := factory.Core().V1().Services().Informer() + dataFeed := node.NewDataFeed(calicoClient) + // Only start the datafeed if node or loadbalancer controller is enabled + if cfg.Controllers.LoadBalancer != nil || cfg.Controllers.Node != nil { + dataFeed.Start() + } + if cfg.Controllers.WorkloadEndpoint != nil { podController := pod.NewPodController(ctx, k8sClientset, calicoClient, *cfg.Controllers.WorkloadEndpoint, podInformer) cc.controllers["Pod"] = podController diff --git a/kube-controllers/pkg/controllers/flannelmigration/ipam_migrator.go b/kube-controllers/pkg/controllers/flannelmigration/ipam_migrator.go index f5691e69001..00233b61cb4 100644 --- a/kube-controllers/pkg/controllers/flannelmigration/ipam_migrator.go +++ b/kube-controllers/pkg/controllers/flannelmigration/ipam_migrator.go @@ -19,18 +19,18 @@ import ( "encoding/json" "fmt" - "github.com/projectcalico/calico/libcalico-go/lib/ipam" - v1 "k8s.io/api/core/v1" "k8s.io/client-go/kubernetes" - api "github.com/projectcalico/api/pkg/apis/projectcalico/v3" log "github.com/sirupsen/logrus" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + api "github.com/projectcalico/api/pkg/apis/projectcalico/v3" + libapi "github.com/projectcalico/calico/libcalico-go/lib/apis/v3" client "github.com/projectcalico/calico/libcalico-go/lib/clientv3" cerrors "github.com/projectcalico/calico/libcalico-go/lib/errors" + "github.com/projectcalico/calico/libcalico-go/lib/ipam" cnet "github.com/projectcalico/calico/libcalico-go/lib/net" "github.com/projectcalico/calico/libcalico-go/lib/options" ) @@ -182,8 +182,13 @@ func (m ipamMigrator) SetupCalicoIPAMForNode(node *v1.Node) error { vtepMac := fvm.VtepMAC log.Infof("node %s has vxlan setup from Flannel (vtepMac: '%s', vtepIP: '%s').", node.Name, vtepMac, vtepIP.String()) + affinityCfg := ipam.AffinityConfig{ + Host: node.Name, + AffinityType: ipam.AffinityTypeHost, + } + // Allocate Calico IPAM blocks for node. - claimed, failed, err := m.calicoClient.IPAM().ClaimAffinity(m.ctx, *cidr, node.Name) + claimed, failed, err := m.calicoClient.IPAM().ClaimAffinity(m.ctx, *cidr, affinityCfg) if err != nil { log.WithError(err).Errorf("Failed to claim IPAM blocks for node %s, claimed %d, failed %d", node.Name, len(claimed), len(failed)) return err diff --git a/kube-controllers/pkg/controllers/loadbalancer/loadbalancer_controller.go b/kube-controllers/pkg/controllers/loadbalancer/loadbalancer_controller.go index c5c9a78bb98..fc64b62a826 100644 --- a/kube-controllers/pkg/controllers/loadbalancer/loadbalancer_controller.go +++ b/kube-controllers/pkg/controllers/loadbalancer/loadbalancer_controller.go @@ -23,10 +23,23 @@ import ( "net" "regexp" "slices" + "strconv" "strings" "time" + log "github.com/sirupsen/logrus" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/types" + uruntime "k8s.io/apimachinery/pkg/util/runtime" + k8svalidation "k8s.io/apimachinery/pkg/util/validation" + "k8s.io/client-go/kubernetes" + v1lister "k8s.io/client-go/listers/core/v1" + "k8s.io/client-go/tools/cache" + api "github.com/projectcalico/api/pkg/apis/projectcalico/v3" + "github.com/projectcalico/calico/kube-controllers/pkg/config" "github.com/projectcalico/calico/kube-controllers/pkg/controllers/controller" "github.com/projectcalico/calico/kube-controllers/pkg/controllers/node" @@ -36,28 +49,21 @@ import ( "github.com/projectcalico/calico/libcalico-go/lib/ipam" "github.com/projectcalico/calico/libcalico-go/lib/json" cnet "github.com/projectcalico/calico/libcalico-go/lib/net" - log "github.com/sirupsen/logrus" - v1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/labels" - uruntime "k8s.io/apimachinery/pkg/util/runtime" - k8svalidation "k8s.io/apimachinery/pkg/util/validation" - "k8s.io/client-go/kubernetes" - v1lister "k8s.io/client-go/listers/core/v1" - "k8s.io/client-go/tools/cache" + "github.com/projectcalico/calico/libcalico-go/lib/options" ) const ( - annotationIpv4Pools = "projectcalico.org/ipv4pools" - annotationIpv6Pools = "projectcalico.org/ipv6pools" - annotationLoadBalancerIp = "projectcalico.org/loadBalancerIPs" + annotationIPv4Pools = "projectcalico.org/ipv4pools" + annotationIPv6Pools = "projectcalico.org/ipv6pools" + annotationLoadBalancerIP = "projectcalico.org/loadBalancerIPs" timer = 5 * time.Minute ) type serviceObject struct { - handle string - updateType serviceUpdateType - service *v1.Service + handle string + updateType serviceUpdateType + namespacedName types.NamespacedName + service *v1.Service } type serviceUpdateType string @@ -68,89 +74,139 @@ const ( serviceUpdateTypeDELETE serviceUpdateType = "DELETE" ) +type allocationTracker struct { + servicesByIp map[string]types.NamespacedName + ipsByService map[types.NamespacedName]map[string]bool +} + +func (t *allocationTracker) assignAddress(svc *v1.Service, ip string) { + namespacedName := types.NamespacedName{ + Namespace: svc.Namespace, + Name: svc.Name, + } + t.servicesByIp[ip] = namespacedName + if t.ipsByService[namespacedName] == nil { + t.ipsByService[namespacedName] = make(map[string]bool) + t.ipsByService[namespacedName][ip] = true + } else { + t.ipsByService[namespacedName][ip] = true + } +} + +func (t *allocationTracker) releaseAddress(svc *v1.Service, ip string) { + namespacedName := types.NamespacedName{ + Namespace: svc.Namespace, + Name: svc.Name, + } + delete(t.servicesByIp, ip) + delete(t.ipsByService[namespacedName], ip) +} + +func (t *allocationTracker) deleteService(namespacedName types.NamespacedName) { + for ip := range t.ipsByService[namespacedName] { + delete(t.servicesByIp, ip) + } + delete(t.ipsByService, namespacedName) +} + // loadBalancerController implements the Controller interface for managing Kubernetes services // and endpoints, syncing them to the Calico datastore as NetworkSet. type loadBalancerController struct { - calicoClient client.Interface - dataFeed *node.DataFeed - cfg config.LoadBalancerControllerConfig - clientSet *kubernetes.Clientset - syncerUpdates chan interface{} - syncStatus bapi.SyncStatus - syncChan chan interface{} - ipamBlocks map[string]model.KVPair - ipPools map[string]api.IPPool - serviceInformer cache.SharedIndexInformer - serviceLister v1lister.ServiceLister - servicesToUpdate map[string]serviceObject + calicoClient client.Interface + dataFeed *node.DataFeed + cfg config.LoadBalancerControllerConfig + clientSet *kubernetes.Clientset + syncerUpdates chan interface{} + syncStatus bapi.SyncStatus + syncChan chan interface{} + serviceUpdates chan serviceObject + ipamBlocks map[string]model.KVPair + ipPools map[string]api.IPPool + serviceInformer cache.SharedIndexInformer + serviceLister v1lister.ServiceLister + allocationTracker allocationTracker } // NewLoadBalancerController returns a controller which manages Service LoadBalancer objects. func NewLoadBalancerController(clientset *kubernetes.Clientset, calicoClient client.Interface, cfg config.LoadBalancerControllerConfig, serviceInformer cache.SharedIndexInformer, dataFeed *node.DataFeed) controller.Controller { lbc := &loadBalancerController{ - calicoClient: calicoClient, - cfg: cfg, - clientSet: clientset, - dataFeed: dataFeed, - syncerUpdates: make(chan interface{}), - syncChan: make(chan interface{}, 1), - ipamBlocks: make(map[string]model.KVPair), - ipPools: make(map[string]api.IPPool), - serviceInformer: serviceInformer, - serviceLister: v1lister.NewServiceLister(serviceInformer.GetIndexer()), - servicesToUpdate: make(map[string]serviceObject), + calicoClient: calicoClient, + cfg: cfg, + clientSet: clientset, + dataFeed: dataFeed, + syncerUpdates: make(chan interface{}), + syncChan: make(chan interface{}, 1), + serviceUpdates: make(chan serviceObject, 1), + ipamBlocks: make(map[string]model.KVPair), + ipPools: make(map[string]api.IPPool), + serviceInformer: serviceInformer, + serviceLister: v1lister.NewServiceLister(serviceInformer.GetIndexer()), + allocationTracker: allocationTracker{ + servicesByIp: make(map[string]types.NamespacedName), + ipsByService: make(map[types.NamespacedName]map[string]bool), + }, } lbc.RegisterWith(lbc.dataFeed) - lbc.dataFeed.Start() - _, err := lbc.serviceInformer.AddEventHandler(cache.ResourceEventHandlerFuncs{ - DeleteFunc: func(obj interface{}) { - if svc, ok := obj.(*v1.Service); ok { - if IsCalicoManagedLoadBalancer(svc, cfg.AssignIPs) { - lbc.syncerUpdates <- serviceObject{ - updateType: serviceUpdateTypeDELETE, - } - } + serviceAdd := func(obj interface{}) { + if svc, ok := obj.(*v1.Service); ok { + handle, err := createHandle(svc) + if err != nil { + log.WithError(err).Error("Error creating handle for service") + return } - }, - UpdateFunc: func(oldObj, newObj interface{}) { - svcNew := newObj.(*v1.Service) - svcOld := oldObj.(*v1.Service) - handle, err := createHandle(svcNew) + lbc.serviceUpdates <- serviceObject{ + handle: handle, + updateType: serviceUpdateTypeADD, + namespacedName: types.NamespacedName{ + Namespace: svc.Namespace, + Name: svc.Name, + }, + } + } + } + + serviceUpdate := func(objNew interface{}, objOld interface{}) { + if svc, ok := objNew.(*v1.Service); ok { + handle, err := createHandle(svc) if err != nil { - log.WithError(err).Error("Error creating load balancer handle") + log.WithError(err).Error("Error creating handle for service") return } - if svcNew.Spec.Type != v1.ServiceTypeLoadBalancer && - svcOld.Spec.Type == v1.ServiceTypeLoadBalancer { - lbc.syncerUpdates <- serviceObject{ - updateType: serviceUpdateTypeDELETE, - } - } else if svcOld.Annotations[annotationIpv4Pools] != svcNew.Annotations[annotationIpv4Pools] || - svcOld.Annotations[annotationIpv6Pools] != svcNew.Annotations[annotationIpv6Pools] || - svcOld.Annotations[annotationLoadBalancerIp] != svcNew.Annotations[annotationLoadBalancerIp] { - // Calico annotations have changed, get new address based on new conditions. - lbc.syncerUpdates <- serviceObject{ - handle: handle, - updateType: serviceUpdateTypeUPDATE, - service: svcNew, - } - } else if svcNew.Status.LoadBalancer.Ingress == nil && IsCalicoManagedLoadBalancer(svcNew, cfg.AssignIPs) { - lbc.syncerUpdates <- serviceObject{ - handle: handle, - updateType: serviceUpdateTypeADD, - } + lbc.serviceUpdates <- serviceObject{ + handle: handle, + updateType: serviceUpdateTypeUPDATE, + namespacedName: types.NamespacedName{ + Namespace: svc.Namespace, + Name: svc.Name, + }, } - }, - AddFunc: func(obj interface{}) { - svc := obj.(*v1.Service) - if IsCalicoManagedLoadBalancer(svc, cfg.AssignIPs) { - lbc.syncerUpdates <- serviceObject{ - updateType: serviceUpdateTypeADD, - } + } + } + + serviceDelete := func(objNew interface{}) { + if svc, ok := objNew.(*v1.Service); ok { + handle, err := createHandle(svc) + if err != nil { + log.WithError(err).Error("Error creating handle for service") + return } - }, + lbc.serviceUpdates <- serviceObject{ + handle: handle, + updateType: serviceUpdateTypeDELETE, + namespacedName: types.NamespacedName{ + Namespace: svc.Namespace, + Name: svc.Name, + }, + } + } + } + + _, err := lbc.serviceInformer.AddEventHandler(cache.ResourceEventHandlerFuncs{ + AddFunc: serviceAdd, + UpdateFunc: serviceUpdate, + DeleteFunc: serviceDelete, }) if err != nil { log.WithError(err).Fatal("Failed to add event handle for Service LoadBalancer") @@ -169,6 +225,31 @@ func (c *loadBalancerController) Run(stopCh chan struct{}) { return } + // Load services and assigned IPs into cache + svcObjs, err := c.getServiceObjectList() + if err != nil { + log.WithError(err).Fatal("Failed to get service objects") + } + for _, svcObj := range svcObjs { + if IsCalicoManagedLoadBalancer(svcObj.service, c.cfg.AssignIPs) { + for _, ingress := range svcObj.service.Status.LoadBalancer.Ingress { + c.allocationTracker.assignAddress(svcObj.service, ingress.IP) + } + } + } + + // Load LoadBalancer ipPools into cache + ippools, err := c.calicoClient.IPPools().List(context.Background(), options.ListOptions{}) + if err != nil { + log.Error("Failed to get IpPools.") + return + } + for _, pool := range ippools.Items { + if slices.Contains(pool.Spec.AllowedUses, api.IPPoolAllowedUseLoadBalancer) { + c.ipPools[pool.Name] = pool + } + } + go c.acceptScheduledRequests(stopCh) <-stopCh @@ -211,6 +292,11 @@ func (c *loadBalancerController) acceptScheduledRequests(stopCh <-chan struct{}) c.syncIPAM() case <-c.syncChan: c.syncIPAM() + case svcObj := <-c.serviceUpdates: + err := c.syncService(svcObj) + if err != nil { + log.WithError(err).Error("Error syncing service object, will retry during next IPAM sync") + } case <-stopCh: return } @@ -240,17 +326,14 @@ func (c *loadBalancerController) handleUpdate(update interface{}) { c.handleBlockUpdate(update) return } - case serviceObject: - c.handleServiceUpdate(update) - kick(c.syncChan) - return } } func (c *loadBalancerController) handleBlockUpdate(kvp model.KVPair) { if kvp.Value != nil { - host := kvp.Value.(*model.AllocationBlock).Affinity - if host != nil && *host == fmt.Sprintf("virtual:%s", api.VirtualLoadBalancer) { + affinity := kvp.Value.(*model.AllocationBlock).Affinity + + if affinity != nil && *affinity == fmt.Sprintf("%s:%s", ipam.AffinityTypeVirtual, api.VirtualLoadBalancer) { c.ipamBlocks[kvp.Key.String()] = kvp } } else { @@ -269,125 +352,255 @@ func (c *loadBalancerController) handleIPPoolUpdate(kvp model.KVPair) { } } -func (c *loadBalancerController) handleServiceUpdate(svcObj serviceObject) { - switch svcObj.updateType { - case serviceUpdateTypeUPDATE: - c.servicesToUpdate[svcObj.handle] = svcObj - } -} - // syncIpam has two main uses. It functions as a garbage collection for leaked IP addresses from Service LoadBalancer // The other use case is to update IPs for any Service LoadBalancer that do not have IPs assigned, this could be caused by the user // creating Service LoadBalancer before any valid pools were created func (c *loadBalancerController) syncIPAM() { - - services, err := c.getServiceLoadBalancerList() + svcObjs, err := c.getServiceObjectList() if err != nil { - log.WithError(err).Error("Skipping IPAM sync") - } - - // Garbage collection - // Check all ipamBlocks with loadBalancer affinity, and release unused allocated IPs - // Skip if there is service scheduled for update. ip will be released during the update and we will run GC right after the un-allocation - if len(c.servicesToUpdate) == 0 { - log.Info("Running Service LoadBalancer IP garbage collection") - for _, block := range c.ipamBlocks { - attributes := block.Value.(*model.AllocationBlock).Attributes - for _, attr := range attributes { - if svc, exists := services[*attr.AttrPrimary]; exists { - // Service with handle exists, we need to check that all assigned IPs with the handle are still in use by the svc - log.Debugf("Service found for handle: %s. Check if all IPs allocated by the handle are in use.", *attr.AttrPrimary) - ips, err := c.calicoClient.IPAM().IPsByHandle(context.Background(), *attr.AttrPrimary) - if err != nil { - log.Errorf("Error getting IPs for handle: %s", *attr.AttrPrimary) - } - for _, ingressIP := range svc.Status.LoadBalancer.Ingress { - inUse := false - for _, handleIP := range ips { - if handleIP.String() == ingressIP.IP { - log.Debugf("IP %s in use, skipping", handleIP.String()) - inUse = true - } - if !inUse { - log.Debugf("IP %s not in use, releasing", handleIP.String()) - releaseOptions := ipam.ReleaseOptions{ - Address: ingressIP.IP, - } - _, err = c.calicoClient.IPAM().ReleaseIPs(context.Background(), releaseOptions) - if err != nil { - log.Errorf("Error releasing IP(%s) for svc: %s", ingressIP.IP, svc.Name) - } - } - } - } - } else { - // Service no longer exists, leak confirmed. Release all IPs allocated with the specific handle - log.Infof("Service not found for handle: %s. Releasing unused IPs", *attr.AttrPrimary) - err := c.releaseIP(*attr.AttrPrimary) - if err != nil { - log.Errorf("Error releasing IPAM for handle %s: %s", *attr.AttrPrimary, err) - } - } - } + return + } + + for _, svcObj := range svcObjs { + err = c.syncService(svcObj) + if err != nil { + log.WithError(err).Errorf("Failed to sync service %s/%s", svcObj.service.Namespace, svcObj.service.Name) + continue } } - // Check that all services have assigned IPs as requested, skip if there are no ippools - if len(c.ipPools) != 0 { - for _, svc := range services { - if svc.Status.LoadBalancer.Ingress == nil || - (len(svc.Status.LoadBalancer.Ingress) == 1 && - (*svc.Spec.IPFamilyPolicy == v1.IPFamilyPolicyRequireDualStack) || *svc.Spec.IPFamilyPolicy == v1.IPFamilyPolicyPreferDualStack) { - err = c.assignIP(&svc) + for _, block := range c.ipamBlocks { + b := block.Value.(*model.AllocationBlock) + for key := range b.SequenceNumberForAllocation { + ordinal, err := strconv.Atoi(key) + if err != nil { + log.WithError(err).Errorf("Failed to parse Ordinal for block %s", key) + } + ip := b.OrdinalToIP(ordinal) + + if _, ok := c.allocationTracker.servicesByIp[ip.String()]; !ok { + log.Infof("Found allocated IP, but not in use. Will release IP: %s", ip.String()) + releaseOptions := ipam.ReleaseOptions{ + Address: ip.String(), + } + _, err := c.calicoClient.IPAM().ReleaseIPs(context.Background(), releaseOptions) if err != nil { - log.WithError(err).Errorf("Error assigning IP to svc: %s", svc.Name) + log.WithError(err).Errorf("Failed to release IP %s", ip.String()) + continue } } } } +} - if len(c.servicesToUpdate) != 0 { - for key, svcObj := range c.servicesToUpdate { - switch svcObj.updateType { - case serviceUpdateTypeUPDATE: - err = c.releaseIP(svcObj.handle) +// syncService does the following: +// - Releases any IP addresses in the IPAM DB associated with the Service that are not in the Service status. +// - Allocates any addresses necessary to satisfy the Service LB request +// - Updates the controllers internal state tracking of which IP addresses are allocated. +// - Updates the IP addresses in the Service Status to match the IPAM DB. +func (c *loadBalancerController) syncService(svcObj serviceObject) error { + var err error + var svc *v1.Service + serviceUpdated := false + reassignIP := false + svcStatus := make(map[string]v1.LoadBalancerIngress) + + switch svcObj.updateType { + case serviceUpdateTypeDELETE: + if c.allocationTracker.ipsByService[svcObj.namespacedName] != nil { + err = c.releaseIPByHandle(svcObj) + if err != nil { + log.WithError(err).Errorf("Failed to release IP for %s/%s", svcObj.namespacedName.Namespace, svcObj.namespacedName.Name) + return err + } + c.allocationTracker.deleteService(svcObj.namespacedName) + } + return nil + case serviceUpdateTypeADD, serviceUpdateTypeUPDATE: + if svcObj.service != nil { + svc = svcObj.service + } else { + svc, err = c.serviceLister.Services(svcObj.namespacedName.Namespace).Get(svcObj.namespacedName.Name) + if err != nil { + log.WithError(err).Errorf("Failed to get service %s/%s", svcObj.namespacedName.Namespace, svcObj.namespacedName.Name) + return err + } + } + + // Get IPs in use from service + for _, ingress := range svc.Status.LoadBalancer.Ingress { + svcStatus[ingress.IP] = ingress + } + + // Clear any IPs that we have marked as assigned, but are not in service status + for ip := range c.allocationTracker.ipsByService[svcObj.namespacedName] { + if _, ok := svcStatus[ip]; !ok { + err = c.releaseIP(ip) if err != nil { - log.WithError(err).Error("Error releasing Service IP") continue } - svc, err := c.clientSet.CoreV1().Services(svcObj.service.Namespace).Get(context.Background(), svcObj.service.Name, metav1.GetOptions{}) + c.allocationTracker.releaseAddress(svc, ip) + } + } + + //Service has no IP in status, if the service is managed by calico we will try to assign IPs bellow + if len(svcStatus) == 0 { + reassignIP = true + } + + if svc.Spec.Type != v1.ServiceTypeLoadBalancer { + // Service type has changed, release the ip assigned by calico + if c.allocationTracker.ipsByService[svcObj.namespacedName] != nil { + err = c.releaseIPByHandle(svcObj) if err != nil { - log.WithError(err).Errorf("Error getting Service %s", svcObj.service.Name) - continue + return err } - svc.Status.LoadBalancer.Ingress = []v1.LoadBalancerIngress{} - _, err = c.clientSet.CoreV1().Services(svc.Namespace).UpdateStatus(context.Background(), svc, metav1.UpdateOptions{}) + c.allocationTracker.deleteService(svcObj.namespacedName) + return nil + } else { + // Service is not a type of LoadBalancer, we can skip the update + return nil + } + } + + loadBalancerIPs, ipv4pools, ipv6pools, err := c.parseAnnotations(svc.Annotations) + if err != nil { + log.WithError(err).Errorf("Failed to parse annotations for service %s/%s", svc.Namespace, svc.Name) + return err + } + + // Calico assigned IP previously, no longer managed by us, release IPs assigned by calico + if loadBalancerIPs == nil && + ipv4pools == nil && + ipv6pools == nil && + !IsCalicoManagedLoadBalancer(svc, c.cfg.AssignIPs) && + c.allocationTracker.ipsByService[svcObj.namespacedName] != nil { + for ip := range c.allocationTracker.ipsByService[svcObj.namespacedName] { + err = c.releaseIP(ip) if err != nil { - // We have assigned IP to the service, but were not able to update the status. The IP will be picked up by GC during the next sync, and we will try to assign a new IP - log.WithError(err).Error("Error updating Service IP") + log.WithError(err).Errorf("Failed to release IP for %s/%s", svc.Namespace, svc.Name) continue } - delete(c.servicesToUpdate, key) + delete(svcStatus, ip) + c.allocationTracker.releaseAddress(svc, ip) + serviceUpdated = true + } + } + + // Check that service has assigned IP equal to the ones on annotations + if loadBalancerIPs != nil { + lbIPs := make(map[string]bool) + for _, ip := range loadBalancerIPs { + lbIPs[ip.String()] = true + } + for ip := range svcStatus { + if _, ok := lbIPs[ip]; !ok { + err = c.releaseIP(ip) + if err != nil { + log.WithError(err).Errorf("Failed to release IP for %s/%s", svc.Namespace, svc.Name) + return err + } + delete(svcStatus, ip) + c.allocationTracker.releaseAddress(svc, ip) + reassignIP = true + serviceUpdated = true + } + } + } else if ipv4pools != nil || ipv6pools != nil { + // If pool annotations are specified, we need to check that the IPs assigned are from the specified pools + for ip := range svcStatus { + if !isIpInIppool(ip, ipv4pools) && !isIpInIppool(ip, ipv6pools) { + err = c.releaseIP(ip) + if err != nil { + log.WithError(err).Errorf("Failed to release IP for %s/%s", svc.Namespace, svc.Name) + return err + } + delete(svcStatus, ip) + c.allocationTracker.releaseAddress(svc, ip) + reassignIP = true + serviceUpdated = true + } + } + } else { + // No annotations are specified, check that the IPs assigned aren't from Manual pool from earlier assignment + for ip := range svcStatus { + pool, err := c.localIppoolFromIp(ip) + if err != nil { + return err + } + if pool != nil { + if pool.Spec.AssignmentMode == api.Manual { + err = c.releaseIP(ip) + if err != nil { + log.WithError(err).Errorf("Failed to release IP for %s/%s", svc.Namespace, svc.Name) + return err + } + delete(svcStatus, ip) + c.allocationTracker.releaseAddress(svc, ip) + reassignIP = true + serviceUpdated = true + } + } + } + } + } + + // Service is not in sync, we try to assign new IPs + if reassignIP && IsCalicoManagedLoadBalancer(svc, c.cfg.AssignIPs) { + svcIngress := []v1.LoadBalancerIngress{} + for _, ingress := range svcStatus { + svcIngress = append(svcIngress, ingress) + } + svc.Status.LoadBalancer.Ingress = svcIngress + assignedIPs, err := c.assignIP(svc) + if err != nil { + log.WithError(err).Errorf("Failed to assign IP for %s/%s", svc.Namespace, svc.Name) + return err + } + for _, ip := range assignedIPs { + svcStatus[ip] = v1.LoadBalancerIngress{ + IP: ip, } + c.allocationTracker.assignAddress(svc, ip) } + serviceUpdated = true } + + // If there were no changes to the service during sync, we skip the Status update + if serviceUpdated { + svcIngress := []v1.LoadBalancerIngress{} + for _, ingress := range svcStatus { + svcIngress = append(svcIngress, ingress) + } + svc.Status.LoadBalancer.Ingress = svcIngress + + _, err = c.clientSet.CoreV1().Services(svc.Namespace).UpdateStatus(context.Background(), svc, metav1.UpdateOptions{}) + if err != nil { + log.WithError(err).Errorf("Failed to update service status %s/%s", svc.Namespace, svc.Name) + return err + } + } + return nil } // assignIP tries to assign IP address for Service. -func (c *loadBalancerController) assignIP(svc *v1.Service) error { +func (c *loadBalancerController) assignIP(svc *v1.Service) ([]string, error) { if len(c.ipPools) == 0 { - return nil + return nil, nil } + handle, err := createHandle(svc) if err != nil { - return err + return nil, err } loadBalancerIps, ipv4Pools, ipv6Pools, err := c.parseAnnotations(svc.Annotations) if err != nil { - return err + return nil, err } + var assignedIPs []string + metadataAttrs := map[string]string{ ipam.AttributeService: svc.Name, ipam.AttributeNamespace: svc.Namespace, @@ -396,38 +609,39 @@ func (c *loadBalancerController) assignIP(svc *v1.Service) error { } // User requested specific IP, attempt to allocate - if len(loadBalancerIps) != 0 { - + if loadBalancerIps != nil { ingress := svc.Status.LoadBalancer.Ingress for _, addrs := range loadBalancerIps { + skipAddrs := false for _, lbingress := range ingress { // We must be trying to assign missing address due to an error, // skip this assignment as it's already assigned and move onto the next one if lbingress.IP == addrs.String() { - continue + skipAddrs = true } } + if skipAddrs { + continue + } + ipamArgs := ipam.AssignIPArgs{ - IP: addrs, - Hostname: api.VirtualLoadBalancer, - HandleID: &handle, - Attrs: metadataAttrs, + IP: addrs, + Hostname: api.VirtualLoadBalancer, + HandleID: &handle, + Attrs: metadataAttrs, + IntendedUse: api.VirtualLoadBalancer, } err = c.calicoClient.IPAM().AssignIP(context.Background(), ipamArgs) if err != nil { log.WithField("ip", addrs).WithError(err).Warn("failed to assign ip to node") - return err + return nil, err } - - ingress = append(ingress, v1.LoadBalancerIngress{IP: addrs.String()}) + assignedIPs = append(assignedIPs, addrs.String()) } - - svc.Status.LoadBalancer.Ingress = ingress - _, err = c.clientSet.CoreV1().Services(svc.Namespace).UpdateStatus(context.Background(), svc, metav1.UpdateOptions{}) - return err + return assignedIPs, err } // Build AssignArgs based on Service IP family attr @@ -459,7 +673,7 @@ func (c *loadBalancerController) assignIP(svc *v1.Service) error { } if num4 == 0 && num6 == 0 { - return errors.New("No new IPs to assign, Service already has ipv4 and ipv6 address") + return nil, errors.New("No new IPs to assign, Service already has ipv4 and ipv6 address") } } @@ -472,75 +686,82 @@ func (c *loadBalancerController) assignIP(svc *v1.Service) error { Attrs: metadataAttrs, } - if len(ipv4Pools) != 0 { + if ipv4Pools != nil { args.IPv4Pools = ipv4Pools } - if len(ipv6Pools) != 0 { + if ipv6Pools != nil { args.IPv6Pools = ipv6Pools } v4Assignments, v6assignments, err := c.calicoClient.IPAM().AutoAssign(context.Background(), args) if err != nil { - log.WithField("svc", svc.Name).WithError(err).Warn("error on assigning IPAM to service") + log.WithField("svc", svc.Name).WithError(err).Warn("error on assigning IP address to service") + return nil, err } - var ingress []v1.LoadBalancerIngress - if v4Assignments != nil { for _, assignment := range v4Assignments.IPs { - ingress = append(ingress, v1.LoadBalancerIngress{ - IP: assignment.IP.String(), - }) + assignedIPs = append(assignedIPs, assignment.IP.String()) } } if v6assignments != nil { for _, assignment := range v6assignments.IPs { - ingress = append(ingress, v1.LoadBalancerIngress{ - IP: assignment.IP.String(), - }) + assignedIPs = append(assignedIPs, assignment.IP.String()) } } - svc.Status.LoadBalancer.Ingress = ingress + return assignedIPs, nil +} + +// releaseIPByHandle tries to release all IPs allocated with the Service unique handle +func (c *loadBalancerController) releaseIPByHandle(svcObj serviceObject) error { + log.Info("Service type LoadBalancer deleted, releasing assigned IP address") - _, err = c.clientSet.CoreV1().Services(svc.Namespace).UpdateStatus(context.Background(), svc, metav1.UpdateOptions{}) + err := c.calicoClient.IPAM().ReleaseByHandle(context.Background(), svcObj.handle) if err != nil { - log.Errorf("LoadBalancer Error updating service %s/%s: %v", svc.Namespace, svc.Name, err) + log.Errorf("error on removing assigned IP for handle %s", svcObj.handle) return err } return nil } -// releaseIP tries to release all IPs allocated with the Service unique handle -func (c *loadBalancerController) releaseIP(handle string) error { - log.Info("Service type LoadBalancer deleted, releasing assigned IP address") - - err := c.calicoClient.IPAM().ReleaseByHandle(context.Background(), handle) +func (c *loadBalancerController) releaseIP(ip string) error { + releaseOptions := ipam.ReleaseOptions{ + Address: ip, + } + _, err := c.calicoClient.IPAM().ReleaseIPs(context.Background(), releaseOptions) if err != nil { - log.Errorf("error on removing assigned IP for handle %s", handle) + log.Errorf("error on removing assigned IP %s", ip) return err } - return nil } -func (c *loadBalancerController) getServiceLoadBalancerList() (map[string]v1.Service, error) { +func (c *loadBalancerController) getServiceObjectList() (map[string]serviceObject, error) { services, err := c.serviceLister.Services("").List(labels.Everything()) if err != nil { - log.WithError(err).Error("Error getting svc list") + log.WithError(err).Error("Error getting service list") return nil, err } - loadBalancerServices := make(map[string]v1.Service) + loadBalancerServices := make(map[string]serviceObject) for _, svc := range services { if IsCalicoManagedLoadBalancer(svc, c.cfg.AssignIPs) { handle, err := createHandle(svc) if err != nil { return nil, err } - loadBalancerServices[handle] = *svc + loadBalancerServices[handle] = serviceObject{ + handle: handle, + updateType: serviceUpdateTypeUPDATE, + namespacedName: types.NamespacedName{ + Namespace: svc.Namespace, + Name: svc.Name, + }, + service: svc, + } } } return loadBalancerServices, nil @@ -597,9 +818,9 @@ func IsCalicoManagedLoadBalancer(svc *v1.Service, assignIPs api.AssignIPs) bool return true } - if svc.Annotations[annotationIpv4Pools] != "" || - svc.Annotations[annotationIpv6Pools] != "" || - svc.Annotations[annotationLoadBalancerIp] != "" { + if svc.Annotations[annotationIPv4Pools] != "" || + svc.Annotations[annotationIPv6Pools] != "" || + svc.Annotations[annotationLoadBalancerIP] != "" { return true } return false @@ -608,13 +829,13 @@ func IsCalicoManagedLoadBalancer(svc *v1.Service, assignIPs api.AssignIPs) bool // validateAnnotation checks if the ips specified in the calico annotation are valid. // Each service can have at most one ipv4 and one ipv6 address func (c *loadBalancerController) parseAnnotations(annotations map[string]string) ([]cnet.IP, []cnet.IPNet, []cnet.IPNet, error) { - loadBalancerIPs := []cnet.IP{} - ipv4Pools := []cnet.IPNet{} - ipv6Pools := []cnet.IPNet{} + var loadBalancerIPs []cnet.IP + var ipv4Pools []cnet.IPNet + var ipv6Pools []cnet.IPNet for key, annotation := range annotations { switch key { - case annotationIpv4Pools: + case annotationIPv4Pools: poolCIDRs := []string{} err := json.Unmarshal([]byte(annotation), &poolCIDRs) if err != nil { @@ -627,7 +848,7 @@ func (c *loadBalancerController) parseAnnotations(annotations map[string]string) if ipv4Pools == nil { return nil, nil, nil, fmt.Errorf("failed to resolve pools for IPv4 addresses from annotation") } - case annotationIpv6Pools: + case annotationIPv6Pools: poolCIDRs := []string{} err := json.Unmarshal([]byte(annotation), &poolCIDRs) if err != nil { @@ -640,7 +861,7 @@ func (c *loadBalancerController) parseAnnotations(annotations map[string]string) if ipv6Pools == nil { return nil, nil, nil, fmt.Errorf("failed to resolve pools for IPv6 addresses from annotation") } - case annotationLoadBalancerIp: + case annotationLoadBalancerIP: ipAddrs := []string{} err := json.Unmarshal([]byte(annotation), &ipAddrs) if err != nil { @@ -692,6 +913,33 @@ func createHandle(svc *v1.Service) (string, error) { return handle, nil } +func isIpInIppool(ipAddr string, cidrs []cnet.IPNet) bool { + if cidrs == nil { + return false + } + ip := net.ParseIP(ipAddr) + for _, cidr := range cidrs { + if cidr.Contains(ip) { + return true + } + } + return false +} + +func (c *loadBalancerController) localIppoolFromIp(ipAddr string) (*api.IPPool, error) { + ip := net.ParseIP(ipAddr) + for _, ippool := range c.ipPools { + _, cidr, err := net.ParseCIDR(ippool.Spec.CIDR) + if err != nil { + return nil, err + } + if cidr.Contains(ip) { + return &ippool, nil + } + } + return nil, nil +} + func kick(c chan<- interface{}) { select { case c <- nil: diff --git a/kube-controllers/pkg/controllers/loadbalancer/loadbalancer_controller_fv_test.go b/kube-controllers/pkg/controllers/loadbalancer/loadbalancer_controller_fv_test.go index 94b7d9fcaf8..566085bbfaf 100644 --- a/kube-controllers/pkg/controllers/loadbalancer/loadbalancer_controller_fv_test.go +++ b/kube-controllers/pkg/controllers/loadbalancer/loadbalancer_controller_fv_test.go @@ -297,29 +297,5 @@ var _ = Describe("Calico loadbalancer controller FV tests (etcd mode)", func() { return service.Status.LoadBalancer.Ingress }, time.Second*15, 500*time.Millisecond).ShouldNot(BeEmpty()) }) - - It("Should assign IP after LoadBalancer IP pool is created", func() { - _, err := calicoClient.IPPools().Delete(context.Background(), specificIpPool.Name, options.DeleteOptions{}) - Expect(err).NotTo(HaveOccurred()) - - _, err = calicoClient.IPPools().Delete(context.Background(), basicIpPool.Name, options.DeleteOptions{}) - Expect(err).NotTo(HaveOccurred()) - - _, err = k8sClient.CoreV1().Services(testNamespace).Create(context.Background(), &basicService, metav1.CreateOptions{}) - Expect(err).NotTo(HaveOccurred()) - - Eventually(func() []v1.LoadBalancerIngress { - service, _ := k8sClient.CoreV1().Services(testNamespace).Get(context.Background(), basicService.Name, metav1.GetOptions{}) - return service.Status.LoadBalancer.Ingress - }, time.Second*15, 500*time.Millisecond).Should(BeEmpty()) - - _, err = calicoClient.IPPools().Create(context.Background(), &basicIpPool, options.SetOptions{}) - Expect(err).NotTo(HaveOccurred()) - - Eventually(func() []v1.LoadBalancerIngress { - service, _ := k8sClient.CoreV1().Services(testNamespace).Get(context.Background(), basicService.Name, metav1.GetOptions{}) - return service.Status.LoadBalancer.Ingress - }, time.Second*15, 500*time.Millisecond).ShouldNot(BeEmpty()) - }) }) }) diff --git a/kube-controllers/pkg/controllers/node/controller.go b/kube-controllers/pkg/controllers/node/controller.go index a03d5c28e2f..96edaea2cf8 100644 --- a/kube-controllers/pkg/controllers/node/controller.go +++ b/kube-controllers/pkg/controllers/node/controller.go @@ -130,9 +130,6 @@ func NewNodeController(ctx context.Context, return nil } - // Start the Calico data feed. - nc.dataFeed.Start() - return nc } diff --git a/kube-controllers/pkg/controllers/node/fake_client.go b/kube-controllers/pkg/controllers/node/fake_client.go index 760502d6f37..2c3ab352788 100644 --- a/kube-controllers/pkg/controllers/node/fake_client.go +++ b/kube-controllers/pkg/controllers/node/fake_client.go @@ -283,7 +283,7 @@ func (f *fakeIPAMClient) ReleaseByHandle(ctx context.Context, handleID string) e // ClaimAffinity claims affinity to the given host for all blocks // within the given CIDR. The given CIDR must fall within a configured // pool. If an empty string is passed as the host, then the value returned by os.Hostname is used. -func (f *fakeIPAMClient) ClaimAffinity(ctx context.Context, cidr cnet.IPNet, host string) ([]cnet.IPNet, []cnet.IPNet, error) { +func (f *fakeIPAMClient) ClaimAffinity(ctx context.Context, cidr cnet.IPNet, affinityCfg ipam.AffinityConfig) ([]cnet.IPNet, []cnet.IPNet, error) { panic("not implemented") // TODO: Implement } diff --git a/libcalico-go/lib/clientv3/common_test.go b/libcalico-go/lib/clientv3/common_test.go index 7c54d4cb653..663101a2129 100644 --- a/libcalico-go/lib/clientv3/common_test.go +++ b/libcalico-go/lib/clientv3/common_test.go @@ -40,12 +40,13 @@ var _ = testutils.E2eDatastoreDescribe("Common resource tests", testutils.Datast ctx := context.Background() name1 := "ippool-1" spec1 := apiv3.IPPoolSpec{ - CIDR: "1.2.3.0/24", - IPIPMode: apiv3.IPIPModeAlways, - VXLANMode: apiv3.VXLANModeNever, - BlockSize: 26, - NodeSelector: "all()", - AllowedUses: []apiv3.IPPoolAllowedUse{apiv3.IPPoolAllowedUseWorkload}, + CIDR: "1.2.3.0/24", + IPIPMode: apiv3.IPIPModeAlways, + VXLANMode: apiv3.VXLANModeNever, + BlockSize: 26, + NodeSelector: "all()", + AllowedUses: []apiv3.IPPoolAllowedUse{apiv3.IPPoolAllowedUseWorkload}, + AssignmentMode: apiv3.Automatic, } c, err := clientv3.New(config) Expect(err).NotTo(HaveOccurred()) diff --git a/libcalico-go/lib/clientv3/ippool.go b/libcalico-go/lib/clientv3/ippool.go index d850caa76ec..f50d6ad4b8e 100644 --- a/libcalico-go/lib/clientv3/ippool.go +++ b/libcalico-go/lib/clientv3/ippool.go @@ -283,8 +283,7 @@ func convertIpPoolFromStorage(pool *apiv3.IPPool) error { pool.Spec.VXLANMode = apiv3.VXLANModeNever } - assignmentMode := pool.Spec.AssignmentMode - if &assignmentMode == nil { + if pool.Spec.AssignmentMode == "" { pool.Spec.AssignmentMode = apiv3.Automatic } diff --git a/libcalico-go/lib/clientv3/ippool_e2e_test.go b/libcalico-go/lib/clientv3/ippool_e2e_test.go index dbb9deaf1de..06c6bca9e81 100644 --- a/libcalico-go/lib/clientv3/ippool_e2e_test.go +++ b/libcalico-go/lib/clientv3/ippool_e2e_test.go @@ -47,12 +47,13 @@ var _ = testutils.E2eDatastoreDescribe("IPPool tests", testutils.DatastoreAll, f name2 := "ippool-2" name3 := "ippool-3" spec1 := apiv3.IPPoolSpec{ - CIDR: "1.2.3.0/24", - IPIPMode: apiv3.IPIPModeAlways, - VXLANMode: apiv3.VXLANModeNever, - BlockSize: 26, - NodeSelector: "all()", - AllowedUses: []apiv3.IPPoolAllowedUse{apiv3.IPPoolAllowedUseWorkload, apiv3.IPPoolAllowedUseTunnel}, + CIDR: "1.2.3.0/24", + IPIPMode: apiv3.IPIPModeAlways, + VXLANMode: apiv3.VXLANModeNever, + BlockSize: 26, + NodeSelector: "all()", + AllowedUses: []apiv3.IPPoolAllowedUse{apiv3.IPPoolAllowedUseWorkload, apiv3.IPPoolAllowedUseTunnel}, + AssignmentMode: apiv3.Automatic, } spec1_2 := apiv3.IPPoolSpec{ CIDR: "1.2.3.0/24", @@ -63,6 +64,7 @@ var _ = testutils.E2eDatastoreDescribe("IPPool tests", testutils.DatastoreAll, f NodeSelector: `foo == "bar"`, AllowedUses: []apiv3.IPPoolAllowedUse{apiv3.IPPoolAllowedUseWorkload, apiv3.IPPoolAllowedUseTunnel}, DisableBGPExport: true, + AssignmentMode: apiv3.Automatic, } spec2 := apiv3.IPPoolSpec{ CIDR: "2001::/120", @@ -73,6 +75,7 @@ var _ = testutils.E2eDatastoreDescribe("IPPool tests", testutils.DatastoreAll, f NodeSelector: "all()", AllowedUses: []apiv3.IPPoolAllowedUse{apiv3.IPPoolAllowedUseWorkload, apiv3.IPPoolAllowedUseTunnel}, DisableBGPExport: true, + AssignmentMode: apiv3.Automatic, } spec2_1 := apiv3.IPPoolSpec{ CIDR: "2001::/120", @@ -82,22 +85,25 @@ var _ = testutils.E2eDatastoreDescribe("IPPool tests", testutils.DatastoreAll, f NodeSelector: "all()", AllowedUses: []apiv3.IPPoolAllowedUse{apiv3.IPPoolAllowedUseWorkload, apiv3.IPPoolAllowedUseTunnel}, DisableBGPExport: false, + AssignmentMode: apiv3.Automatic, } spec3 := apiv3.IPPoolSpec{ - CIDR: "1.2.3.0/24", - IPIPMode: "", - VXLANMode: "", - BlockSize: 26, - NodeSelector: "all()", - AllowedUses: []apiv3.IPPoolAllowedUse{apiv3.IPPoolAllowedUseWorkload, apiv3.IPPoolAllowedUseTunnel}, + CIDR: "1.2.3.0/24", + IPIPMode: "", + VXLANMode: "", + BlockSize: 26, + NodeSelector: "all()", + AllowedUses: []apiv3.IPPoolAllowedUse{apiv3.IPPoolAllowedUseWorkload, apiv3.IPPoolAllowedUseTunnel}, + AssignmentMode: apiv3.Automatic, } spec3_1 := apiv3.IPPoolSpec{ - CIDR: "1.2.3.0/24", - IPIPMode: apiv3.IPIPModeNever, - VXLANMode: apiv3.VXLANModeNever, - BlockSize: 26, - NodeSelector: "all()", - AllowedUses: []apiv3.IPPoolAllowedUse{apiv3.IPPoolAllowedUseWorkload, apiv3.IPPoolAllowedUseTunnel}, + CIDR: "1.2.3.0/24", + IPIPMode: apiv3.IPIPModeNever, + VXLANMode: apiv3.VXLANModeNever, + BlockSize: 26, + NodeSelector: "all()", + AllowedUses: []apiv3.IPPoolAllowedUse{apiv3.IPPoolAllowedUseWorkload, apiv3.IPPoolAllowedUseTunnel}, + AssignmentMode: apiv3.Automatic, } It("should error when creating an IPPool with no name", func() { diff --git a/libcalico-go/lib/clientv3/ippool_kdd_conversion_test.go b/libcalico-go/lib/clientv3/ippool_kdd_conversion_test.go index acd91eb251a..c4146912d0e 100644 --- a/libcalico-go/lib/clientv3/ippool_kdd_conversion_test.go +++ b/libcalico-go/lib/clientv3/ippool_kdd_conversion_test.go @@ -40,13 +40,14 @@ var _ = testutils.E2eDatastoreDescribe("IPPool KDD v1 to v3 migration tests", te name2 := "ippool-2" spec1_v3 := apiv3.IPPoolSpec{ - CIDR: "1.2.3.0/24", - NATOutgoing: true, - IPIPMode: apiv3.IPIPModeCrossSubnet, - VXLANMode: apiv3.VXLANModeNever, - BlockSize: 26, - NodeSelector: "all()", - AllowedUses: []apiv3.IPPoolAllowedUse{apiv3.IPPoolAllowedUseWorkload, apiv3.IPPoolAllowedUseTunnel}, + CIDR: "1.2.3.0/24", + NATOutgoing: true, + IPIPMode: apiv3.IPIPModeCrossSubnet, + VXLANMode: apiv3.VXLANModeNever, + BlockSize: 26, + NodeSelector: "all()", + AllowedUses: []apiv3.IPPoolAllowedUse{apiv3.IPPoolAllowedUseWorkload, apiv3.IPPoolAllowedUseTunnel}, + AssignmentMode: apiv3.Automatic, } kvp1 := &model.KVPair{ Key: model.ResourceKey{ @@ -71,19 +72,21 @@ var _ = testutils.E2eDatastoreDescribe("IPPool KDD v1 to v3 migration tests", te Enabled: true, Mode: encap.CrossSubnet, }, - BlockSize: 26, + BlockSize: 26, + AssignmentMode: apiv3.Automatic, }, }, } spec2_v3 := apiv3.IPPoolSpec{ - CIDR: "2001::/120", - NATOutgoing: true, - IPIPMode: apiv3.IPIPModeNever, - VXLANMode: apiv3.VXLANModeNever, - BlockSize: 122, - NodeSelector: "all()", - AllowedUses: []apiv3.IPPoolAllowedUse{apiv3.IPPoolAllowedUseWorkload, apiv3.IPPoolAllowedUseTunnel}, + CIDR: "2001::/120", + NATOutgoing: true, + IPIPMode: apiv3.IPIPModeNever, + VXLANMode: apiv3.VXLANModeNever, + BlockSize: 122, + NodeSelector: "all()", + AllowedUses: []apiv3.IPPoolAllowedUse{apiv3.IPPoolAllowedUseWorkload, apiv3.IPPoolAllowedUseTunnel}, + AssignmentMode: apiv3.Automatic, } kvp2 := &model.KVPair{ Key: model.ResourceKey{ @@ -108,18 +111,20 @@ var _ = testutils.E2eDatastoreDescribe("IPPool KDD v1 to v3 migration tests", te IPIP: &apiv3.IPIPConfiguration{ Enabled: false, }, + AssignmentMode: apiv3.Automatic, }, }, } spec3_v3 := apiv3.IPPoolSpec{ - CIDR: "1.1.1.0/24", - NATOutgoing: false, - IPIPMode: apiv3.IPIPModeAlways, - VXLANMode: apiv3.VXLANModeNever, - BlockSize: 26, - NodeSelector: "all()", - AllowedUses: []apiv3.IPPoolAllowedUse{apiv3.IPPoolAllowedUseWorkload, apiv3.IPPoolAllowedUseTunnel}, + CIDR: "1.1.1.0/24", + NATOutgoing: false, + IPIPMode: apiv3.IPIPModeAlways, + VXLANMode: apiv3.VXLANModeNever, + BlockSize: 26, + NodeSelector: "all()", + AllowedUses: []apiv3.IPPoolAllowedUse{apiv3.IPPoolAllowedUseWorkload, apiv3.IPPoolAllowedUseTunnel}, + AssignmentMode: apiv3.Automatic, } kvp3 := &model.KVPair{ Key: model.ResourceKey{ @@ -141,20 +146,22 @@ var _ = testutils.E2eDatastoreDescribe("IPPool KDD v1 to v3 migration tests", te IPIP: &apiv3.IPIPConfiguration{ Enabled: true, }, - BlockSize: 26, - NodeSelector: "all()", + BlockSize: 26, + NodeSelector: "all()", + AssignmentMode: apiv3.Automatic, }, }, } spec5_v3 := apiv3.IPPoolSpec{ - CIDR: "1.2.3.0/24", - NATOutgoing: true, - IPIPMode: apiv3.IPIPModeAlways, - VXLANMode: apiv3.VXLANModeNever, - BlockSize: 26, - NodeSelector: "all()", - AllowedUses: []apiv3.IPPoolAllowedUse{apiv3.IPPoolAllowedUseWorkload, apiv3.IPPoolAllowedUseTunnel}, + CIDR: "1.2.3.0/24", + NATOutgoing: true, + IPIPMode: apiv3.IPIPModeAlways, + VXLANMode: apiv3.VXLANModeNever, + BlockSize: 26, + NodeSelector: "all()", + AllowedUses: []apiv3.IPPoolAllowedUse{apiv3.IPPoolAllowedUseWorkload, apiv3.IPPoolAllowedUseTunnel}, + AssignmentMode: apiv3.Automatic, } kvp5 := &model.KVPair{ Key: model.ResourceKey{ @@ -177,22 +184,24 @@ var _ = testutils.E2eDatastoreDescribe("IPPool KDD v1 to v3 migration tests", te Enabled: true, Mode: encap.Always, }, - NATOutgoing: true, - NATOutgoingV1: false, - BlockSize: 26, - NodeSelector: "all()", + NATOutgoing: true, + NATOutgoingV1: false, + BlockSize: 26, + NodeSelector: "all()", + AssignmentMode: apiv3.Automatic, }, }, } spec6_v3 := apiv3.IPPoolSpec{ - CIDR: "1.2.3.0/24", - NATOutgoing: true, - IPIPMode: apiv3.IPIPModeCrossSubnet, - VXLANMode: apiv3.VXLANModeNever, - BlockSize: 26, - NodeSelector: "has(x)", - AllowedUses: []apiv3.IPPoolAllowedUse{apiv3.IPPoolAllowedUseWorkload, apiv3.IPPoolAllowedUseTunnel}, + CIDR: "1.2.3.0/24", + NATOutgoing: true, + IPIPMode: apiv3.IPIPModeCrossSubnet, + VXLANMode: apiv3.VXLANModeNever, + BlockSize: 26, + NodeSelector: "has(x)", + AllowedUses: []apiv3.IPPoolAllowedUse{apiv3.IPPoolAllowedUseWorkload, apiv3.IPPoolAllowedUseTunnel}, + AssignmentMode: apiv3.Automatic, } kvp6 := &model.KVPair{ Key: model.ResourceKey{ @@ -208,15 +217,16 @@ var _ = testutils.E2eDatastoreDescribe("IPPool KDD v1 to v3 migration tests", te Name: name1, }, Spec: apiv3.IPPoolSpec{ - CIDR: "1.2.3.0/24", - Disabled: false, - VXLANMode: apiv3.VXLANModeNever, - IPIPMode: apiv3.IPIPModeCrossSubnet, - IPIP: nil, - NATOutgoing: false, - NATOutgoingV1: true, - BlockSize: 26, - NodeSelector: "has(x)", + CIDR: "1.2.3.0/24", + Disabled: false, + VXLANMode: apiv3.VXLANModeNever, + IPIPMode: apiv3.IPIPModeCrossSubnet, + IPIP: nil, + NATOutgoing: false, + NATOutgoingV1: true, + BlockSize: 26, + NodeSelector: "has(x)", + AssignmentMode: apiv3.Automatic, }, }, } diff --git a/libcalico-go/lib/clientv3/node_e2e_test.go b/libcalico-go/lib/clientv3/node_e2e_test.go index 0fb5bfcfeed..dd40e68978b 100644 --- a/libcalico-go/lib/clientv3/node_e2e_test.go +++ b/libcalico-go/lib/clientv3/node_e2e_test.go @@ -284,7 +284,11 @@ var _ = testutils.E2eDatastoreDescribe("Node tests (etcdv3)", testutils.Datastor Mask: net.IPMask{255, 255, 255, 0}, }, } - _, _, err = c.IPAM().ClaimAffinity(ctx, affBlock, name1) + affinityCfg := ipam.AffinityConfig{ + AffinityType: ipam.AffinityTypeHost, + Host: name1, + } + _, _, err = c.IPAM().ClaimAffinity(ctx, affBlock, affinityCfg) Expect(err).NotTo(HaveOccurred()) handle := "myhandle" diff --git a/libcalico-go/lib/ipam/interface.go b/libcalico-go/lib/ipam/interface.go index c7c6bf0f40e..bdd16088ead 100644 --- a/libcalico-go/lib/ipam/interface.go +++ b/libcalico-go/lib/ipam/interface.go @@ -59,7 +59,7 @@ type Interface interface { // ClaimAffinity claims affinity to the given host for all blocks // within the given CIDR. The given CIDR must fall within a configured // pool. If an empty string is passed as the host, then the value returned by os.Hostname is used. - ClaimAffinity(ctx context.Context, cidr cnet.IPNet, host string) ([]cnet.IPNet, []cnet.IPNet, error) + ClaimAffinity(ctx context.Context, cidr cnet.IPNet, affinityCfg AffinityConfig) ([]cnet.IPNet, []cnet.IPNet, error) // ReleaseAffinity releases affinity for all blocks within the given CIDR // on the given host. If an empty string is passed as the host, then the diff --git a/libcalico-go/lib/ipam/ipam.go b/libcalico-go/lib/ipam/ipam.go index 6a733fab540..4a52158a40d 100644 --- a/libcalico-go/lib/ipam/ipam.go +++ b/libcalico-go/lib/ipam/ipam.go @@ -23,10 +23,11 @@ import ( "strings" "time" - v3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" log "github.com/sirupsen/logrus" "golang.org/x/sync/semaphore" + v3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" + libapiv3 "github.com/projectcalico/calico/libcalico-go/lib/apis/v3" bapi "github.com/projectcalico/calico/libcalico-go/lib/backend/api" "github.com/projectcalico/calico/libcalico-go/lib/backend/model" @@ -147,7 +148,7 @@ func (c ipamClient) AutoAssign(ctx context.Context, args AutoAssignArgs) (*IPAMA // getBlockFromAffinity returns the block referenced by the given affinity, attempting to create it if // it does not exist. getBlockFromAffinity will delete the provided affinity if it does not match the actual // affinity of the block. -func (c ipamClient) getBlockFromAffinity(ctx context.Context, aff *model.KVPair, rsvdAttr *HostReservedAttr) (*model.KVPair, error) { +func (c ipamClient) getBlockFromAffinity(ctx context.Context, aff *model.KVPair, rsvdAttr *HostReservedAttr, affinityCfg AffinityConfig) (*model.KVPair, error) { // Parse out affinity data. cidr := aff.Key.(model.BlockAffinityKey).CIDR host := aff.Key.(model.BlockAffinityKey).Host @@ -177,7 +178,7 @@ func (c ipamClient) getBlockFromAffinity(ctx context.Context, aff *model.KVPair, // Claim the block, which will also confirm the affinity. logCtx.Info("Attempting to claim the block") - b, err := c.blockReaderWriter.claimAffineBlock(ctx, aff, *cfg, rsvdAttr) + b, err := c.blockReaderWriter.claimAffineBlock(ctx, aff, *cfg, rsvdAttr, affinityCfg) if err != nil { logCtx.WithError(err).Warn("Error claiming block") return nil, err @@ -470,7 +471,7 @@ func filterPoolsByUse(pools []v3.IPPool, use v3.IPPoolAllowedUse) []v3.IPPool { type blockAssignState struct { client ipamClient version int - host string + affinityCfg AffinityConfig pools []v3.IPPool remainingAffineBlocks []net.IPNet hostReservedAttr *HostReservedAttr @@ -486,7 +487,7 @@ type blockAssignState struct { // and assign affinity. // It returns a block, a boolean if block is newly claimed and any error encountered. func (s *blockAssignState) findOrClaimBlock(ctx context.Context, minFreeIps int) (*model.KVPair, bool, error) { - logCtx := log.WithFields(log.Fields{"host": s.host}) + logCtx := log.WithFields(log.Fields{"Host": s.affinityCfg.Host}) // First, we try to find a block from one of the existing host-affine blocks. for len(s.remainingAffineBlocks) > 0 { @@ -504,14 +505,14 @@ func (s *blockAssignState) findOrClaimBlock(ctx context.Context, minFreeIps int) for i := 0; i < datastoreRetries; i++ { // Get the affinity. logCtx.Infof("Trying affinity for %s", cidr) - aff, err := s.client.blockReaderWriter.queryAffinity(ctx, s.host, cidr, "") + aff, err := s.client.blockReaderWriter.queryAffinity(ctx, s.affinityCfg.Host, cidr, "") if err != nil { logCtx.WithError(err).Warnf("Error getting affinity") break } // Get the block which is referenced by the affinity, creating it if necessary. - b, err := s.client.getBlockFromAffinity(ctx, aff, s.hostReservedAttr) + b, err := s.client.getBlockFromAffinity(ctx, aff, s.hostReservedAttr, s.affinityCfg) if err != nil { // Couldn't get a block for this affinity. if _, ok := err.(cerrors.ErrorResourceUpdateConflict); ok { @@ -553,7 +554,7 @@ func (s *blockAssignState) findOrClaimBlock(ctx context.Context, minFreeIps int) // allocated affine block. This may happen due to a race condition where another process on the host allocates a new block // after we decide that a new block is required to satisfy this request, but before we actually allocate a new block. logCtx.Info("Tried all affine blocks. Looking for an affine block with space, or a new unclaimed block") - subnet, err := s.client.blockReaderWriter.findUsableBlock(ctx, s.host, s.version, s.pools, s.reservations, *config) + subnet, err := s.client.blockReaderWriter.findUsableBlock(ctx, s.affinityCfg.Host, s.version, s.pools, s.reservations, *config) if err != nil { if _, ok := err.(noFreeBlocksError); ok { // No free blocks. Break. @@ -563,12 +564,12 @@ func (s *blockAssignState) findOrClaimBlock(ctx context.Context, minFreeIps int) } return nil, false, err } - logCtx := log.WithFields(log.Fields{"host": s.host, "subnet": subnet}) + logCtx := log.WithFields(log.Fields{"host": s.affinityCfg.Host, "subnet": subnet}) logCtx.Info("Found unclaimed block") for j := 0; j < datastoreRetries; j++ { // We found an unclaimed block - claim affinity for it. - pa, err := s.client.blockReaderWriter.getPendingAffinity(ctx, s.host, *subnet) + pa, err := s.client.blockReaderWriter.getPendingAffinity(ctx, s.affinityCfg.Host, *subnet) if err != nil { if _, ok := err.(cerrors.ErrorResourceUpdateConflict); ok { logCtx.WithError(err).Debug("CAS error claiming pending affinity, retry") @@ -579,7 +580,7 @@ func (s *blockAssignState) findOrClaimBlock(ctx context.Context, minFreeIps int) } // We have an affinity - try to get the block. - b, err := s.client.getBlockFromAffinity(ctx, pa, s.hostReservedAttr) + b, err := s.client.getBlockFromAffinity(ctx, pa, s.hostReservedAttr, s.affinityCfg) if err != nil { if _, ok := err.(cerrors.ErrorResourceUpdateConflict); ok { logCtx.WithError(err).Debug("CAS error getting block, retry") @@ -663,6 +664,15 @@ func (c ipamClient) autoAssign(ctx context.Context, num int, handleID *string, a return nil, ErrUseRequired } + affinityCfg := AffinityConfig{ + AffinityType: AffinityTypeHost, + Host: host, + } + + if use == v3.IPPoolAllowedUseLoadBalancer { + affinityCfg.AffinityType = AffinityTypeVirtual + } + // Load the set of reserved IPs/CIDRs. reservations, err := c.getReservedIPs(ctx) if err != nil { @@ -719,7 +729,7 @@ func (c ipamClient) autoAssign(ctx context.Context, num int, handleID *string, a s := &blockAssignState{ client: c, version: version, - host: host, + affinityCfg: affinityCfg, pools: pools, remainingAffineBlocks: affBlocks, hostReservedAttr: rsvdAttr, @@ -877,6 +887,15 @@ func (c ipamClient) AssignIP(ctx context.Context, args AssignIPArgs) error { } log.Infof("Assigning IP %s to host: %s", args.IP, hostname) + affinityCfg := AffinityConfig{ + AffinityType: AffinityTypeHost, + Host: hostname, + } + + //if args.IntendedUse == v3.IPPoolAllowedUseLoadBalancer { + // affinityCfg.AffinityType = AffinityTypeVirtual + //} + pool, err := c.blockReaderWriter.getPoolForIP(args.IP, nil) if err != nil { return err @@ -917,7 +936,7 @@ func (c ipamClient) AssignIP(ctx context.Context, args AssignIPArgs) error { return err } - obj, err = c.blockReaderWriter.claimAffineBlock(ctx, pa, *cfg, args.HostReservedAttr) + obj, err = c.blockReaderWriter.claimAffineBlock(ctx, pa, *cfg, args.HostReservedAttr, affinityCfg) if err != nil { if _, ok := err.(*errBlockClaimConflict); ok { log.Warningf("Someone else claimed block %s before us", blockCIDR.String()) @@ -1239,8 +1258,8 @@ func (c ipamClient) assignFromExistingBlock(ctx context.Context, block *model.KV // pool. Returns a list of blocks that were claimed, as well as a // list of blocks that were claimed by another host. // If an empty string is passed as the host, then the hostname is automatically detected. -func (c ipamClient) ClaimAffinity(ctx context.Context, cidr net.IPNet, host string) ([]net.IPNet, []net.IPNet, error) { - logCtx := log.WithFields(log.Fields{"host": host, "cidr": cidr}) +func (c ipamClient) ClaimAffinity(ctx context.Context, cidr net.IPNet, affinityCfg AffinityConfig) ([]net.IPNet, []net.IPNet, error) { + logCtx := log.WithFields(log.Fields{"host": affinityCfg.Host, "cidr": cidr}) // Verify the requested CIDR falls within a configured pool. pool, err := c.blockReaderWriter.getPoolForIP(net.IP{IP: cidr.IP}, nil) @@ -1259,7 +1278,7 @@ func (c ipamClient) ClaimAffinity(ctx context.Context, cidr net.IPNet, host stri } // Determine the hostname to use. - hostname, err := decideHostname(host) + hostname, err := decideHostname(affinityCfg.Host) if err != nil { return nil, nil, err } @@ -1289,7 +1308,7 @@ func (c ipamClient) ClaimAffinity(ctx context.Context, cidr net.IPNet, host stri } // Once we have the affinity, claim the block, which will confirm the affinity. - _, err = c.blockReaderWriter.claimAffineBlock(ctx, pa, *cfg, nil) + _, err = c.blockReaderWriter.claimAffineBlock(ctx, pa, *cfg, nil, affinityCfg) if err != nil { if _, ok := err.(cerrors.ErrorResourceUpdateConflict); ok { logCtx.WithError(err).Debug("CAS error claiming affine block - retry") @@ -2126,7 +2145,12 @@ func (c ipamClient) EnsureBlock(ctx context.Context, args BlockArgs) (*net.IPNet if err != nil { return nil, nil, err } - log.Infof("Ensure block for host %s, ipv4 attr %v ipv6 attr %v", hostname, args.HostReservedAttrIPv4s, args.HostReservedAttrIPv6s) + log.Infof("Ensure block for Host %s, ipv4 attr %v ipv6 attr %v", hostname, args.HostReservedAttrIPv4s, args.HostReservedAttrIPv6s) + + affinityCfg := AffinityConfig{ + AffinityType: AffinityTypeHost, + Host: hostname, + } var v4Net, v6Net *net.IPNet @@ -2136,7 +2160,7 @@ func (c ipamClient) EnsureBlock(ctx context.Context, args BlockArgs) (*net.IPNet return nil, nil, fmt.Errorf("provided IPv4 IPPools list contains one or more IPv6 IPPools") } } - v4Net, err = c.ensureBlock(ctx, args.HostReservedAttrIPv4s, args.IPv4Pools, 4, hostname) + v4Net, err = c.ensureBlock(ctx, args.HostReservedAttrIPv4s, args.IPv4Pools, 4, affinityCfg) if err != nil { log.Errorf("Error ensure IPv4 block: %v", err) return nil, nil, err @@ -2149,7 +2173,7 @@ func (c ipamClient) EnsureBlock(ctx context.Context, args BlockArgs) (*net.IPNet return nil, nil, fmt.Errorf("provided IPv6 IPPools list contains one or more IPv4 IPPools") } } - v6Net, err = c.ensureBlock(ctx, args.HostReservedAttrIPv6s, args.IPv4Pools, 6, hostname) + v6Net, err = c.ensureBlock(ctx, args.HostReservedAttrIPv6s, args.IPv4Pools, 6, affinityCfg) if err != nil { log.Errorf("Error ensure IPv6 block: %v", err) return nil, nil, err @@ -2192,13 +2216,13 @@ func getMaxPrefixLen(version int, attrs *HostReservedAttr) (int, error) { return maxPrefixLen, nil } -func (c ipamClient) ensureBlock(ctx context.Context, rsvdAttr *HostReservedAttr, requestedPools []net.IPNet, version int, host string) (*net.IPNet, error) { +func (c ipamClient) ensureBlock(ctx context.Context, rsvdAttr *HostReservedAttr, requestedPools []net.IPNet, version int, affinityCfg AffinityConfig) (*net.IPNet, error) { // This function is similar to autoAssign except it does not allocate ips. - logCtx := log.WithFields(log.Fields{"host": host}) + logCtx := log.WithFields(log.Fields{"host": affinityCfg.Host}) logCtx.Info("Looking up existing affinities for host") - pools, affBlocks, err := c.prepareAffinityBlocksForHost(ctx, requestedPools, version, host, rsvdAttr, v3.IPPoolAllowedUseWorkload) + pools, affBlocks, err := c.prepareAffinityBlocksForHost(ctx, requestedPools, version, affinityCfg.Host, rsvdAttr, v3.IPPoolAllowedUseWorkload) if err != nil { return nil, err } @@ -2214,7 +2238,7 @@ func (c ipamClient) ensureBlock(ctx context.Context, rsvdAttr *HostReservedAttr, s := &blockAssignState{ client: c, version: version, - host: host, + affinityCfg: affinityCfg, pools: pools, remainingAffineBlocks: affBlocks, hostReservedAttr: rsvdAttr, diff --git a/libcalico-go/lib/ipam/ipam_block.go b/libcalico-go/lib/ipam/ipam_block.go index 70b727d0e87..0a6dab53cbd 100644 --- a/libcalico-go/lib/ipam/ipam_block.go +++ b/libcalico-go/lib/ipam/ipam_block.go @@ -190,13 +190,21 @@ func (b *allocationBlock) assign(affinityCheck bool, address cnet.IP, handleID * // hostAffinityMatches checks if the provided host matches the provided affinity. func hostAffinityMatches(host string, block *model.AllocationBlock) bool { - return *block.Affinity == "host:"+host + if strings.HasPrefix(*block.Affinity, "host:") { + return *block.Affinity == "host:"+host + } else if strings.HasPrefix(*block.Affinity, "virtual:") { + return *block.Affinity == "virtual:"+host + } + return false } func getHostAffinity(block *model.AllocationBlock) string { if block.Affinity != nil && strings.HasPrefix(*block.Affinity, "host:") { return strings.TrimPrefix(*block.Affinity, "host:") } + if block.Affinity != nil && strings.HasPrefix(*block.Affinity, "virtual:") { + return strings.TrimPrefix(*block.Affinity, "virtual:") + } return "" } diff --git a/libcalico-go/lib/ipam/ipam_block_reader_writer.go b/libcalico-go/lib/ipam/ipam_block_reader_writer.go index 214dfbcfb3b..9815869c609 100644 --- a/libcalico-go/lib/ipam/ipam_block_reader_writer.go +++ b/libcalico-go/lib/ipam/ipam_block_reader_writer.go @@ -34,6 +34,18 @@ import ( cnet "github.com/projectcalico/calico/libcalico-go/lib/net" ) +type AffinityConfig struct { + AffinityType AffinityType + Host string +} + +type AffinityType string + +const ( + AffinityTypeHost AffinityType = "host" + AffinityTypeVirtual AffinityType = "virtual" +) + type blockReaderWriter struct { client bapi.Client pools PoolAccessorInterface @@ -204,18 +216,13 @@ func (rw blockReaderWriter) getPendingAffinity(ctx context.Context, host string, // claimAffineBlock claims the provided block using the given pending affinity. If successful, it will confirm the affinity. If another host // steals the block, claimAffineBlock will attempt to delete the provided pending affinity. -func (rw blockReaderWriter) claimAffineBlock(ctx context.Context, aff *model.KVPair, config IPAMConfig, rsvdAttr *HostReservedAttr) (*model.KVPair, error) { +func (rw blockReaderWriter) claimAffineBlock(ctx context.Context, aff *model.KVPair, config IPAMConfig, rsvdAttr *HostReservedAttr, affinityCfg AffinityConfig) (*model.KVPair, error) { // Pull out relevant fields. subnet := aff.Key.(model.BlockAffinityKey).CIDR host := aff.Key.(model.BlockAffinityKey).Host logCtx := log.WithFields(log.Fields{"host": host, "subnet": subnet}) - // Create the new block. - prefix := "host:" - if host == v3.VirtualLoadBalancer { - prefix = "virtual:" - } - affinityKeyStr := prefix + host + affinityKeyStr := fmt.Sprintf("%s:%s", affinityCfg.AffinityType, host) block := newBlock(subnet, rsvdAttr) block.Affinity = &affinityKeyStr diff --git a/libcalico-go/lib/ipam/ipam_block_reader_writer_test.go b/libcalico-go/lib/ipam/ipam_block_reader_writer_test.go index 0c9cfa12d13..070272a7593 100644 --- a/libcalico-go/lib/ipam/ipam_block_reader_writer_test.go +++ b/libcalico-go/lib/ipam/ipam_block_reader_writer_test.go @@ -212,7 +212,7 @@ var _ = testutils.E2eDatastoreDescribe("IPAM affine block allocation tests", tes } }) - By("assigning from host twice the number of available blocks all at once", func() { + By("assigning from Host twice the number of available blocks all at once", func() { wg := sync.WaitGroup{} var testErr error @@ -384,7 +384,11 @@ var _ = testutils.E2eDatastoreDescribe("IPAM affine block allocation tests", tes defer wg.Done() testhost := "same-host" - success, failed, err := ic.ClaimAffinity(ctx, *net, testhost) + affinityCfg := AffinityConfig{ + AffinityType: AffinityTypeHost, + Host: testhost, + } + success, failed, err := ic.ClaimAffinity(ctx, *net, affinityCfg) if err != nil { log.WithError(err).Errorf("ClaimAffinity failed for host %s", testhost) testErr = err @@ -510,7 +514,8 @@ var _ = testutils.E2eDatastoreDescribe("IPAM affine block allocation tests", tes Expect(err).NotTo(HaveOccurred()) config := IPAMConfig{} - _, err = rw.claimAffineBlock(ctx, pa, config, nil) + affinityCfg := AffinityConfig{AffinityType: AffinityTypeHost, Host: hostA} + _, err = rw.claimAffineBlock(ctx, pa, config, nil, affinityCfg) Expect(err).NotTo(BeNil()) // Should hit a resource update conflict. @@ -911,7 +916,8 @@ var _ = testutils.E2eDatastoreDescribe("IPAM affine block allocation tests", tes Expect(err).NotTo(HaveOccurred()) config := IPAMConfig{} - _, err = rw.claimAffineBlock(ctx, pa, config, nil) + affinityCfg := AffinityConfig{AffinityType: AffinityTypeHost, Host: host} + _, err = rw.claimAffineBlock(ctx, pa, config, nil, affinityCfg) Expect(err).NotTo(HaveOccurred()) }) @@ -920,7 +926,8 @@ var _ = testutils.E2eDatastoreDescribe("IPAM affine block allocation tests", tes Expect(err).NotTo(HaveOccurred()) config := IPAMConfig{} - _, err = rw.claimAffineBlock(ctx, pa, config, nil) + affinityCfg := AffinityConfig{AffinityType: AffinityTypeHost, Host: host} + _, err = rw.claimAffineBlock(ctx, pa, config, nil, affinityCfg) Expect(err).NotTo(HaveOccurred()) }) diff --git a/libcalico-go/lib/ipam/ipam_test.go b/libcalico-go/lib/ipam/ipam_test.go index 79f7ec06e3e..9efbb2d26b7 100644 --- a/libcalico-go/lib/ipam/ipam_test.go +++ b/libcalico-go/lib/ipam/ipam_test.go @@ -2382,19 +2382,24 @@ var _ = testutils.E2eDatastoreDescribe("IPAM tests", testutils.DatastoreAll, fun cfg, err := ic.GetIPAMConfig(context.Background()) Expect(err).NotTo(HaveOccurred()) + affinityCfg := AffinityConfig{ + AffinityType: AffinityTypeHost, + Host: host, + } + // Claim affinity on two blocks for _, blockCIDR := range affBlocks { pa, err := ic.(*ipamClient).blockReaderWriter.getPendingAffinity(ctx, host, blockCIDR) Expect(err).NotTo(HaveOccurred()) - _, err = ic.(*ipamClient).blockReaderWriter.claimAffineBlock(ctx, pa, *cfg, rsvdAttr) + _, err = ic.(*ipamClient).blockReaderWriter.claimAffineBlock(ctx, pa, *cfg, rsvdAttr, affinityCfg) Expect(err).NotTo(HaveOccurred()) } s = &blockAssignState{ client: *ic.(*ipamClient), version: 4, - host: host, + affinityCfg: affinityCfg, pools: pools, remainingAffineBlocks: affBlocks, hostReservedAttr: rsvdAttr, @@ -3148,7 +3153,9 @@ var _ = testutils.E2eDatastoreDescribe("IPAM tests", testutils.DatastoreAll, fun assignIPutil(ic, args.assignIP, "host-a") - outClaimed, outFailed, outError := ic.ClaimAffinity(context.Background(), inIPNet, args.host) + affinityCfg := AffinityConfig{AffinityType: AffinityTypeHost, Host: args.host} + + outClaimed, outFailed, outError := ic.ClaimAffinity(context.Background(), inIPNet, affinityCfg) log.Println("Claimed IP blocks: ", outClaimed) log.Println("Failed to claim IP blocks: ", outFailed) diff --git a/libcalico-go/lib/ipam/ipam_types.go b/libcalico-go/lib/ipam/ipam_types.go index 10bece12b5f..5580c13cf7f 100644 --- a/libcalico-go/lib/ipam/ipam_types.go +++ b/libcalico-go/lib/ipam/ipam_types.go @@ -42,6 +42,9 @@ type AssignIPArgs struct { // If specified, the attributes of reserved IPv4 addresses in the block. HostReservedAttr *HostReservedAttr + + // The intended use for the IP address. Used to determine the affinityType of the host. + IntendedUse v3.IPPoolAllowedUse } // AutoAssignArgs defines the set of arguments for assigning one or more From 3e80ab89c548206bf5062d77ba2ed24d9caeabe2 Mon Sep 17 00:00:00 2001 From: Michal Fupso Date: Tue, 24 Sep 2024 13:13:01 -0700 Subject: [PATCH 11/12] libcalico-go tests --- libcalico-go/lib/backend/k8s/k8s_test.go | 10 ++++++---- libcalico-go/lib/ipam/ipam.go | 6 +++--- libcalico-go/lib/ipam/ipam_test.go | 17 +++++++++-------- 3 files changed, 18 insertions(+), 15 deletions(-) diff --git a/libcalico-go/lib/backend/k8s/k8s_test.go b/libcalico-go/lib/backend/k8s/k8s_test.go index 6f6504a4a3c..ffc37b143d8 100644 --- a/libcalico-go/lib/backend/k8s/k8s_test.go +++ b/libcalico-go/lib/backend/k8s/k8s_test.go @@ -2531,9 +2531,10 @@ var _ = testutils.E2eDatastoreDescribe("Test Syncer API for Kubernetes backend", Name: "192-16-0-0-16", }, Spec: apiv3.IPPoolSpec{ - CIDR: cidr, - IPIPMode: apiv3.IPIPModeCrossSubnet, - Disabled: true, + CIDR: cidr, + IPIPMode: apiv3.IPIPModeCrossSubnet, + Disabled: true, + AssignmentMode: apiv3.Automatic, }, }, } @@ -3132,7 +3133,8 @@ var _ = testutils.E2eDatastoreDescribe("Test Watch support", testutils.Datastore Name: name, }, Spec: apiv3.IPPoolSpec{ - CIDR: cidr, + CIDR: cidr, + AssignmentMode: apiv3.Automatic, }, }, } diff --git a/libcalico-go/lib/ipam/ipam.go b/libcalico-go/lib/ipam/ipam.go index 4a52158a40d..25cd8b344f5 100644 --- a/libcalico-go/lib/ipam/ipam.go +++ b/libcalico-go/lib/ipam/ipam.go @@ -892,9 +892,9 @@ func (c ipamClient) AssignIP(ctx context.Context, args AssignIPArgs) error { Host: hostname, } - //if args.IntendedUse == v3.IPPoolAllowedUseLoadBalancer { - // affinityCfg.AffinityType = AffinityTypeVirtual - //} + if args.IntendedUse == v3.IPPoolAllowedUseLoadBalancer { + affinityCfg.AffinityType = AffinityTypeVirtual + } pool, err := c.blockReaderWriter.getPoolForIP(args.IP, nil) if err != nil { diff --git a/libcalico-go/lib/ipam/ipam_test.go b/libcalico-go/lib/ipam/ipam_test.go index 9efbb2d26b7..00e0e7b3355 100644 --- a/libcalico-go/lib/ipam/ipam_test.go +++ b/libcalico-go/lib/ipam/ipam_test.go @@ -54,11 +54,12 @@ type ipPoolAccessor struct { } type pool struct { - cidr string - blockSize int - enabled bool - nodeSelector string - allowedUses []v3.IPPoolAllowedUse + cidr string + blockSize int + enabled bool + nodeSelector string + allowedUses []v3.IPPoolAllowedUse + assignmentMode v3.AssignmentMode } func (i *ipPoolAccessor) GetEnabledPools(ipVersion int) ([]v3.IPPool, error) { @@ -3397,15 +3398,15 @@ func deleteAllPools() { } func applyPool(cidr string, enabled bool, nodeSelector string) { - ipPools.pools[cidr] = pool{enabled: enabled, nodeSelector: nodeSelector} + ipPools.pools[cidr] = pool{enabled: enabled, nodeSelector: nodeSelector, assignmentMode: v3.Automatic} } func applyPoolWithUses(cidr string, enabled bool, nodeSelector string, uses []v3.IPPoolAllowedUse) { - ipPools.pools[cidr] = pool{enabled: enabled, nodeSelector: nodeSelector, allowedUses: uses} + ipPools.pools[cidr] = pool{enabled: enabled, nodeSelector: nodeSelector, allowedUses: uses, assignmentMode: v3.Automatic} } func applyPoolWithBlockSize(cidr string, enabled bool, nodeSelector string, blockSize int) { - ipPools.pools[cidr] = pool{enabled: enabled, nodeSelector: nodeSelector, blockSize: blockSize} + ipPools.pools[cidr] = pool{enabled: enabled, nodeSelector: nodeSelector, blockSize: blockSize, assignmentMode: v3.Automatic} } func deletePool(cidr string) { From 6f0222fd47b1cc225fb66f493c0042776f20cdec Mon Sep 17 00:00:00 2001 From: Michal Fupso Date: Fri, 27 Sep 2024 13:12:31 -0700 Subject: [PATCH 12/12] Test updates --- .../loadbalancer/loadbalancer_controller.go | 24 +- .../loadbalancer_controller_fv_test.go | 206 +++++++++++++++--- libcalico-go/lib/ipam/ipam_block.go | 8 +- .../lib/ipam/ipam_block_reader_writer.go | 12 - libcalico-go/lib/ipam/ipam_types.go | 12 + 5 files changed, 208 insertions(+), 54 deletions(-) diff --git a/kube-controllers/pkg/controllers/loadbalancer/loadbalancer_controller.go b/kube-controllers/pkg/controllers/loadbalancer/loadbalancer_controller.go index fc64b62a826..9b423edc721 100644 --- a/kube-controllers/pkg/controllers/loadbalancer/loadbalancer_controller.go +++ b/kube-controllers/pkg/controllers/loadbalancer/loadbalancer_controller.go @@ -41,7 +41,6 @@ import ( api "github.com/projectcalico/api/pkg/apis/projectcalico/v3" "github.com/projectcalico/calico/kube-controllers/pkg/config" - "github.com/projectcalico/calico/kube-controllers/pkg/controllers/controller" "github.com/projectcalico/calico/kube-controllers/pkg/controllers/node" bapi "github.com/projectcalico/calico/libcalico-go/lib/backend/api" "github.com/projectcalico/calico/libcalico-go/lib/backend/model" @@ -115,7 +114,7 @@ type loadBalancerController struct { calicoClient client.Interface dataFeed *node.DataFeed cfg config.LoadBalancerControllerConfig - clientSet *kubernetes.Clientset + clientSet kubernetes.Interface syncerUpdates chan interface{} syncStatus bapi.SyncStatus syncChan chan interface{} @@ -128,7 +127,7 @@ type loadBalancerController struct { } // NewLoadBalancerController returns a controller which manages Service LoadBalancer objects. -func NewLoadBalancerController(clientset *kubernetes.Clientset, calicoClient client.Interface, cfg config.LoadBalancerControllerConfig, serviceInformer cache.SharedIndexInformer, dataFeed *node.DataFeed) controller.Controller { +func NewLoadBalancerController(clientset kubernetes.Interface, calicoClient client.Interface, cfg config.LoadBalancerControllerConfig, serviceInformer cache.SharedIndexInformer, dataFeed *node.DataFeed) *loadBalancerController { lbc := &loadBalancerController{ calicoClient: calicoClient, cfg: cfg, @@ -415,6 +414,7 @@ func (c *loadBalancerController) syncService(svcObj serviceObject) error { } c.allocationTracker.deleteService(svcObj.namespacedName) } + kick(c.syncChan) return nil case serviceUpdateTypeADD, serviceUpdateTypeUPDATE: if svcObj.service != nil { @@ -456,6 +456,7 @@ func (c *loadBalancerController) syncService(svcObj serviceObject) error { return err } c.allocationTracker.deleteService(svcObj.namespacedName) + kick(c.syncChan) return nil } else { // Service is not a type of LoadBalancer, we can skip the update @@ -555,15 +556,16 @@ func (c *loadBalancerController) syncService(svcObj serviceObject) error { assignedIPs, err := c.assignIP(svc) if err != nil { log.WithError(err).Errorf("Failed to assign IP for %s/%s", svc.Namespace, svc.Name) - return err - } - for _, ip := range assignedIPs { - svcStatus[ip] = v1.LoadBalancerIngress{ - IP: ip, + + } else { + for _, ip := range assignedIPs { + svcStatus[ip] = v1.LoadBalancerIngress{ + IP: ip, + } + c.allocationTracker.assignAddress(svc, ip) } - c.allocationTracker.assignAddress(svc, ip) + serviceUpdated = true } - serviceUpdated = true } // If there were no changes to the service during sync, we skip the Status update @@ -631,7 +633,7 @@ func (c *loadBalancerController) assignIP(svc *v1.Service) ([]string, error) { Hostname: api.VirtualLoadBalancer, HandleID: &handle, Attrs: metadataAttrs, - IntendedUse: api.VirtualLoadBalancer, + IntendedUse: api.IPPoolAllowedUseLoadBalancer, } err = c.calicoClient.IPAM().AssignIP(context.Background(), ipamArgs) diff --git a/kube-controllers/pkg/controllers/loadbalancer/loadbalancer_controller_fv_test.go b/kube-controllers/pkg/controllers/loadbalancer/loadbalancer_controller_fv_test.go index 566085bbfaf..2c53063deff 100644 --- a/kube-controllers/pkg/controllers/loadbalancer/loadbalancer_controller_fv_test.go +++ b/kube-controllers/pkg/controllers/loadbalancer/loadbalancer_controller_fv_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package loadbalancer_test +package loadbalancer import ( "context" @@ -50,24 +50,42 @@ var _ = Describe("Calico loadbalancer controller FV tests (etcd mode)", func() { const testNamespace = "test-loadbalancer-ns" - basicIpPool := v3.IPPool{ + v4poolManualIP := "1.1.1.1" + specificIpFromAutomaticPool := "1.2.3.100" + v4poolManualSpecifcIP := "4.4.4.4" + + v4poolManual := v3.IPPool{ + ObjectMeta: metav1.ObjectMeta{ + Name: "v4pool-manual", + }, + Spec: v3.IPPoolSpec{ + CIDR: fmt.Sprintf("%s/32", v4poolManualIP), + BlockSize: 32, + NodeSelector: "all()", + AllowedUses: []v3.IPPoolAllowedUse{v3.IPPoolAllowedUseLoadBalancer}, + AssignmentMode: v3.Manual, + }, + } + + v4poolAutomatic := v3.IPPool{ ObjectMeta: metav1.ObjectMeta{ - Name: "loadbalancer-ippool", + Name: "v4pool-automatic", }, Spec: v3.IPPoolSpec{ - CIDR: "1.2.3.0/24", - BlockSize: 26, - NodeSelector: "all()", - AllowedUses: []v3.IPPoolAllowedUse{v3.IPPoolAllowedUseLoadBalancer}, + CIDR: "1.2.3.0/24", + BlockSize: 26, + NodeSelector: "all()", + AllowedUses: []v3.IPPoolAllowedUse{v3.IPPoolAllowedUseLoadBalancer}, + AssignmentMode: v3.Automatic, }, } - specificIpPool := v3.IPPool{ + v4poolManualSpecific := v3.IPPool{ ObjectMeta: metav1.ObjectMeta{ Name: "loadbalancer-ippool-specific", }, Spec: v3.IPPoolSpec{ - CIDR: "4.4.4.4/32", + CIDR: fmt.Sprintf("%s/32", v4poolManualSpecifcIP), BlockSize: 32, NodeSelector: "all()", AllowedUses: []v3.IPPoolAllowedUse{v3.IPPoolAllowedUseLoadBalancer}, @@ -96,7 +114,7 @@ var _ = Describe("Calico loadbalancer controller FV tests (etcd mode)", func() { ObjectMeta: metav1.ObjectMeta{ Name: "basic-ipv4-pool-specified", Annotations: map[string]string{ - "projectcalico.org/ipv4pools": fmt.Sprintf("[\"%s\"]", specificIpPool.Name), + "projectcalico.org/ipv4pools": fmt.Sprintf("[\"%s\"]", v4poolManualSpecific.Name), }, }, Spec: v1.ServiceSpec{ @@ -116,7 +134,7 @@ var _ = Describe("Calico loadbalancer controller FV tests (etcd mode)", func() { ObjectMeta: metav1.ObjectMeta{ Name: "basic-ip-address-specified", Annotations: map[string]string{ - "projectcalico.org/loadBalancerIPs": "[\"1.2.3.100\"]", + "projectcalico.org/loadBalancerIPs": fmt.Sprintf("[\"%s\"]", specificIpFromAutomaticPool), }, }, Spec: v1.ServiceSpec{ @@ -189,10 +207,13 @@ var _ = Describe("Calico loadbalancer controller FV tests (etcd mode)", func() { _, err := k8sClient.CoreV1().Namespaces().Create(context.Background(), ns, metav1.CreateOptions{}) Expect(err).NotTo(HaveOccurred()) - _, err = calicoClient.IPPools().Create(context.Background(), &basicIpPool, options.SetOptions{}) + _, err = calicoClient.IPPools().Create(context.Background(), &v4poolAutomatic, options.SetOptions{}) Expect(err).NotTo(HaveOccurred()) - _, err = calicoClient.IPPools().Create(context.Background(), &specificIpPool, options.SetOptions{}) + _, err = calicoClient.IPPools().Create(context.Background(), &v4poolManual, options.SetOptions{}) + Expect(err).NotTo(HaveOccurred()) + + _, err = calicoClient.IPPools().Create(context.Background(), &v4poolManualSpecific, options.SetOptions{}) Expect(err).NotTo(HaveOccurred()) }) @@ -215,7 +236,8 @@ var _ = Describe("Calico loadbalancer controller FV tests (etcd mode)", func() { return service.Status.LoadBalancer.Ingress }, time.Second*15, 500*time.Millisecond).Should(Not(BeEmpty())) - service, _ := k8sClient.CoreV1().Services(testNamespace).Get(context.Background(), serviceIpv4PoolSpecified.Name, metav1.GetOptions{}) + service, err := k8sClient.CoreV1().Services(testNamespace).Get(context.Background(), serviceIpv4PoolSpecified.Name, metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) Expect(service.Status.LoadBalancer.Ingress[0].IP).Should(Equal("4.4.4.4")) }) @@ -228,12 +250,21 @@ var _ = Describe("Calico loadbalancer controller FV tests (etcd mode)", func() { return service.Status.LoadBalancer.Ingress }, time.Second*15, 500*time.Millisecond).Should(Not(BeEmpty())) - service, _ := k8sClient.CoreV1().Services(testNamespace).Get(context.Background(), serviceIpAddressSpecified.Name, metav1.GetOptions{}) - Expect(service.Status.LoadBalancer.Ingress[0].IP).Should(Equal("1.2.3.100")) + service, err := k8sClient.CoreV1().Services(testNamespace).Get(context.Background(), serviceIpAddressSpecified.Name, metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + Expect(service.Status.LoadBalancer.Ingress[0].IP).Should(Equal(specificIpFromAutomaticPool)) }) It("Should remove IP assignment when Service type is changed from LoadBalancer to NodePort", func() { - _, err := k8sClient.CoreV1().Services(testNamespace).Create(context.Background(), &serviceIpv4PoolSpecified, metav1.CreateOptions{}) + _, err := k8sClient.CoreV1().Services(testNamespace).Create(context.Background(), &basicService, metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + + Eventually(func() []v1.LoadBalancerIngress { + service, _ := k8sClient.CoreV1().Services(testNamespace).Get(context.Background(), basicService.Name, metav1.GetOptions{}) + return service.Status.LoadBalancer.Ingress + }, time.Second*15, 500*time.Millisecond).Should(Not(BeEmpty())) + + _, err = k8sClient.CoreV1().Services(testNamespace).Create(context.Background(), &serviceIpv4PoolSpecified, metav1.CreateOptions{}) Expect(err).NotTo(HaveOccurred()) Eventually(func() []v1.LoadBalancerIngress { @@ -241,10 +272,11 @@ var _ = Describe("Calico loadbalancer controller FV tests (etcd mode)", func() { return service.Status.LoadBalancer.Ingress }, time.Second*15, 500*time.Millisecond).Should(Not(BeEmpty())) - service, err := k8sClient.CoreV1().Services(testNamespace).Get(context.Background(), serviceIpv4PoolSpecified.Name, metav1.GetOptions{}) + serviceSpecific, err := k8sClient.CoreV1().Services(testNamespace).Get(context.Background(), serviceIpv4PoolSpecified.Name, metav1.GetOptions{}) Expect(err).NotTo(HaveOccurred()) - Expect(service.Status.LoadBalancer.Ingress[0].IP).Should(Equal("4.4.4.4")) + Expect(serviceSpecific.Status.LoadBalancer.Ingress[0].IP).Should(Equal(v4poolManualSpecifcIP)) + // Update the service type to NodePort, LoadBalancer controller should release the IP svcPatch := map[string]interface{}{} spec := map[string]interface{}{} svcPatch["spec"] = spec @@ -252,17 +284,31 @@ var _ = Describe("Calico loadbalancer controller FV tests (etcd mode)", func() { patch, err := json.Marshal(svcPatch) Expect(err).NotTo(HaveOccurred()) - service, err = k8sClient.CoreV1().Services(testNamespace).Patch(context.Background(), service.Name, types.StrategicMergePatchType, patch, metav1.PatchOptions{}) + serviceSpecific, err = k8sClient.CoreV1().Services(testNamespace).Patch(context.Background(), serviceSpecific.Name, types.StrategicMergePatchType, patch, metav1.PatchOptions{}) Expect(err).NotTo(HaveOccurred()) - Expect(service.Spec.Type).To(Equal(v1.ServiceTypeNodePort)) - Expect(service.Status.LoadBalancer.Ingress).Should(BeEmpty()) + Expect(serviceSpecific.Spec.Type).To(Equal(v1.ServiceTypeNodePort)) + Expect(serviceSpecific.Status.LoadBalancer.Ingress).Should(BeEmpty()) + + // Update annotation for the basic service, we should be able to assign the IP from specific service that was released + serviceBasic, err := k8sClient.CoreV1().Services(testNamespace).Get(context.Background(), basicService.Name, metav1.GetOptions{}) + serviceBasic.Annotations = map[string]string{ + "projectcalico.org/loadBalancerIPs": fmt.Sprintf("[\"%s\"]", v4poolManualSpecifcIP), + } + + serviceBasic, err = k8sClient.CoreV1().Services(testNamespace).Update(context.Background(), serviceBasic, metav1.UpdateOptions{}) + Expect(err).NotTo(HaveOccurred()) + + Eventually(func() string { + service, _ := k8sClient.CoreV1().Services(testNamespace).Get(context.Background(), basicService.Name, metav1.GetOptions{}) + return service.Status.LoadBalancer.Ingress[0].IP + }, time.Second*15, 500*time.Millisecond).Should(Equal(v4poolManualSpecifcIP)) }) It("Should not assign IP if there is no LoadBalancer IP pool", func() { - _, err := calicoClient.IPPools().Delete(context.Background(), specificIpPool.Name, options.DeleteOptions{}) + _, err := calicoClient.IPPools().Delete(context.Background(), v4poolManualSpecific.Name, options.DeleteOptions{}) Expect(err).NotTo(HaveOccurred()) - _, err = calicoClient.IPPools().Delete(context.Background(), basicIpPool.Name, options.DeleteOptions{}) + _, err = calicoClient.IPPools().Delete(context.Background(), v4poolAutomatic.Name, options.DeleteOptions{}) Expect(err).NotTo(HaveOccurred()) _, err = k8sClient.CoreV1().Services(testNamespace).Create(context.Background(), &basicService, metav1.CreateOptions{}) @@ -275,10 +321,10 @@ var _ = Describe("Calico loadbalancer controller FV tests (etcd mode)", func() { }) It("Should assign IP after LoadBalancer IP pool is created", func() { - _, err := calicoClient.IPPools().Delete(context.Background(), specificIpPool.Name, options.DeleteOptions{}) + _, err := calicoClient.IPPools().Delete(context.Background(), v4poolManualSpecific.Name, options.DeleteOptions{}) Expect(err).NotTo(HaveOccurred()) - _, err = calicoClient.IPPools().Delete(context.Background(), basicIpPool.Name, options.DeleteOptions{}) + _, err = calicoClient.IPPools().Delete(context.Background(), v4poolAutomatic.Name, options.DeleteOptions{}) Expect(err).NotTo(HaveOccurred()) _, err = k8sClient.CoreV1().Services(testNamespace).Create(context.Background(), &basicService, metav1.CreateOptions{}) @@ -289,7 +335,7 @@ var _ = Describe("Calico loadbalancer controller FV tests (etcd mode)", func() { return service.Status.LoadBalancer.Ingress }, time.Second*15, 500*time.Millisecond).Should(BeEmpty()) - _, err = calicoClient.IPPools().Create(context.Background(), &basicIpPool, options.SetOptions{}) + _, err = calicoClient.IPPools().Create(context.Background(), &v4poolAutomatic, options.SetOptions{}) Expect(err).NotTo(HaveOccurred()) Eventually(func() []v1.LoadBalancerIngress { @@ -297,5 +343,111 @@ var _ = Describe("Calico loadbalancer controller FV tests (etcd mode)", func() { return service.Status.LoadBalancer.Ingress }, time.Second*15, 500*time.Millisecond).ShouldNot(BeEmpty()) }) + + It("Should update service IP after pool annotation has been added", func() { + _, err := k8sClient.CoreV1().Services(testNamespace).Create(context.Background(), &basicService, metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + + Eventually(func() []v1.LoadBalancerIngress { + service, _ := k8sClient.CoreV1().Services(testNamespace).Get(context.Background(), basicService.Name, metav1.GetOptions{}) + return service.Status.LoadBalancer.Ingress + }, time.Second*15, 500*time.Millisecond).Should(Not(BeEmpty())) + + service, err := k8sClient.CoreV1().Services(testNamespace).Get(context.Background(), basicService.Name, metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + + service.Annotations = map[string]string{ + "projectcalico.org/ipv4pools": "[\"v4pool-manual\"]", + } + + service, err = k8sClient.CoreV1().Services(testNamespace).Update(context.Background(), service, metav1.UpdateOptions{}) + Expect(err).NotTo(HaveOccurred()) + + Eventually(func() string { + service, _ := k8sClient.CoreV1().Services(testNamespace).Get(context.Background(), basicService.Name, metav1.GetOptions{}) + return service.Status.LoadBalancer.Ingress[0].IP + }, time.Second*15, 500*time.Millisecond).Should(Equal(v4poolManualIP)) + }) + + It("Should update service IP after ip annotation has been added", func() { + _, err := k8sClient.CoreV1().Services(testNamespace).Create(context.Background(), &basicService, metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + + Eventually(func() []v1.LoadBalancerIngress { + service, _ := k8sClient.CoreV1().Services(testNamespace).Get(context.Background(), basicService.Name, metav1.GetOptions{}) + return service.Status.LoadBalancer.Ingress + }, time.Second*15, 500*time.Millisecond).Should(Not(BeEmpty())) + + service, err := k8sClient.CoreV1().Services(testNamespace).Get(context.Background(), basicService.Name, metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + + service.Annotations = map[string]string{ + "projectcalico.org/loadBalancerIPs": fmt.Sprintf("[\"%s\"]", specificIpFromAutomaticPool), + } + + service, err = k8sClient.CoreV1().Services(testNamespace).Update(context.Background(), service, metav1.UpdateOptions{}) + Expect(err).NotTo(HaveOccurred()) + + Eventually(func() string { + service, _ := k8sClient.CoreV1().Services(testNamespace).Get(context.Background(), basicService.Name, metav1.GetOptions{}) + return service.Status.LoadBalancer.Ingress[0].IP + }, time.Second*15, 500*time.Millisecond).Should(Equal(specificIpFromAutomaticPool)) + }) + + It("Should update service IP after the address has been released from another service", func() { + // Create service with IP annotation + _, err := k8sClient.CoreV1().Services(testNamespace).Create(context.Background(), &serviceIpAddressSpecified, metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + + Eventually(func() []v1.LoadBalancerIngress { + service, _ := k8sClient.CoreV1().Services(testNamespace).Get(context.Background(), serviceIpAddressSpecified.Name, metav1.GetOptions{}) + return service.Status.LoadBalancer.Ingress + }, time.Second*15, 500*time.Millisecond).Should(Not(BeEmpty())) + + // Check that the assigned IP is the one specified in the annotation + service, err := k8sClient.CoreV1().Services(testNamespace).Get(context.Background(), serviceIpAddressSpecified.Name, metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + Expect(service.Status.LoadBalancer.Ingress[0].IP).Should(Equal(specificIpFromAutomaticPool)) + + // Create a service with no annotation + _, err = k8sClient.CoreV1().Services(testNamespace).Create(context.Background(), &serviceIpv4PoolSpecified, metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + + Eventually(func() []v1.LoadBalancerIngress { + service, _ := k8sClient.CoreV1().Services(testNamespace).Get(context.Background(), serviceIpv4PoolSpecified.Name, metav1.GetOptions{}) + return service.Status.LoadBalancer.Ingress + }, time.Second*15, 500*time.Millisecond).Should(Not(BeEmpty())) + + service, err = k8sClient.CoreV1().Services(testNamespace).Get(context.Background(), serviceIpv4PoolSpecified.Name, metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + + // Update the service to have the same IP as the service we have created above + service.Annotations = map[string]string{ + "projectcalico.org/loadBalancerIPs": fmt.Sprintf("[\"%s\"]", specificIpFromAutomaticPool), + } + + service, err = k8sClient.CoreV1().Services(testNamespace).Update(context.Background(), service, metav1.UpdateOptions{}) + Expect(err).NotTo(HaveOccurred()) + + // The service ingress should be empty + Eventually(func() []v1.LoadBalancerIngress { + service, _ := k8sClient.CoreV1().Services(testNamespace).Get(context.Background(), serviceIpv4PoolSpecified.Name, metav1.GetOptions{}) + return service.Status.LoadBalancer.Ingress + }, time.Second*15, 500*time.Millisecond).Should(BeEmpty()) + + err = k8sClient.CoreV1().Services(testNamespace).Delete(context.Background(), serviceIpAddressSpecified.Name, metav1.DeleteOptions{}) + Expect(err).NotTo(HaveOccurred()) + + // After we deleted the other service we should be able to have the IP assigned to this one + Eventually(func() []v1.LoadBalancerIngress { + service, _ := k8sClient.CoreV1().Services(testNamespace).Get(context.Background(), serviceIpv4PoolSpecified.Name, metav1.GetOptions{}) + return service.Status.LoadBalancer.Ingress + }, time.Second*15, 500*time.Millisecond).Should(Not(BeEmpty())) + + service, err = k8sClient.CoreV1().Services(testNamespace).Get(context.Background(), serviceIpv4PoolSpecified.Name, metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + Expect(service.Status.LoadBalancer.Ingress[0].IP).Should(Equal(specificIpFromAutomaticPool)) + + }) }) }) diff --git a/libcalico-go/lib/ipam/ipam_block.go b/libcalico-go/lib/ipam/ipam_block.go index 0a6dab53cbd..c4d264bd608 100644 --- a/libcalico-go/lib/ipam/ipam_block.go +++ b/libcalico-go/lib/ipam/ipam_block.go @@ -199,11 +199,11 @@ func hostAffinityMatches(host string, block *model.AllocationBlock) bool { } func getHostAffinity(block *model.AllocationBlock) string { - if block.Affinity != nil && strings.HasPrefix(*block.Affinity, "host:") { - return strings.TrimPrefix(*block.Affinity, "host:") + if block.Affinity != nil && strings.HasPrefix(*block.Affinity, fmt.Sprintf("%s:", AffinityTypeHost)) { + return strings.TrimPrefix(*block.Affinity, fmt.Sprintf("%s:", AffinityTypeHost)) } - if block.Affinity != nil && strings.HasPrefix(*block.Affinity, "virtual:") { - return strings.TrimPrefix(*block.Affinity, "virtual:") + if block.Affinity != nil && strings.HasPrefix(*block.Affinity, fmt.Sprintf("%s:", AffinityTypeVirtual)) { + return strings.TrimPrefix(*block.Affinity, fmt.Sprintf("%s:", AffinityTypeVirtual)) } return "" } diff --git a/libcalico-go/lib/ipam/ipam_block_reader_writer.go b/libcalico-go/lib/ipam/ipam_block_reader_writer.go index 9815869c609..7fa29fc8d4c 100644 --- a/libcalico-go/lib/ipam/ipam_block_reader_writer.go +++ b/libcalico-go/lib/ipam/ipam_block_reader_writer.go @@ -34,18 +34,6 @@ import ( cnet "github.com/projectcalico/calico/libcalico-go/lib/net" ) -type AffinityConfig struct { - AffinityType AffinityType - Host string -} - -type AffinityType string - -const ( - AffinityTypeHost AffinityType = "host" - AffinityTypeVirtual AffinityType = "virtual" -) - type blockReaderWriter struct { client bapi.Client pools PoolAccessorInterface diff --git a/libcalico-go/lib/ipam/ipam_types.go b/libcalico-go/lib/ipam/ipam_types.go index 5580c13cf7f..3837c2b3dba 100644 --- a/libcalico-go/lib/ipam/ipam_types.go +++ b/libcalico-go/lib/ipam/ipam_types.go @@ -188,6 +188,18 @@ type ReleaseOptions struct { SequenceNumber *uint64 } +type AffinityConfig struct { + AffinityType AffinityType + Host string +} + +type AffinityType string + +const ( + AffinityTypeHost AffinityType = "host" + AffinityTypeVirtual AffinityType = "virtual" +) + func (opts *ReleaseOptions) AsNetIP() (*cnet.IP, error) { ip := cnet.ParseIP(opts.Address) if ip != nil {