From 1c1cbb3897fd553c9ea186fa87f54842aec8d9b5 Mon Sep 17 00:00:00 2001 From: Reed Schalo Date: Mon, 30 Sep 2024 10:26:21 -0700 Subject: [PATCH 1/9] fix: update cluster role to allow reading CRDs --- charts/karpenter/templates/clusterrole-core.yaml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/charts/karpenter/templates/clusterrole-core.yaml b/charts/karpenter/templates/clusterrole-core.yaml index 9ef0dc300d5b..d20509e597f6 100644 --- a/charts/karpenter/templates/clusterrole-core.yaml +++ b/charts/karpenter/templates/clusterrole-core.yaml @@ -41,11 +41,9 @@ rules: - apiGroups: ["apps"] resources: ["daemonsets", "deployments", "replicasets", "statefulsets"] verbs: ["list", "watch"] - {{- if .Values.webhook.enabled }} - apiGroups: ["apiextensions.k8s.io"] resources: ["customresourcedefinitions"] - verbs: ["watch", "list"] - {{- end }} + verbs: ["get", "watch", "list"] - apiGroups: ["policy"] resources: ["poddisruptionbudgets"] verbs: ["get", "list", "watch"] @@ -75,6 +73,7 @@ rules: {{- if .Values.webhook.enabled }} - apiGroups: ["apiextensions.k8s.io"] resources: ["customresourcedefinitions"] + resourceNames: ["ec2nodeclasses.karpenter.k8s.aws", "nodepools.karpenter.sh", "nodeclaims.karpenter.sh"] verbs: ["update"] {{- end }} {{- with .Values.additionalClusterRoleRules -}} From bc3fdcda0a832538c5d3921dcdf3002c649905a5 Mon Sep 17 00:00:00 2001 From: Reed Schalo Date: Mon, 30 Sep 2024 10:42:12 -0700 Subject: [PATCH 2/9] add back conditional to read permissions --- charts/karpenter/templates/clusterrole-core.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/charts/karpenter/templates/clusterrole-core.yaml b/charts/karpenter/templates/clusterrole-core.yaml index d20509e597f6..dc0ee1e670cb 100644 --- a/charts/karpenter/templates/clusterrole-core.yaml +++ b/charts/karpenter/templates/clusterrole-core.yaml @@ -41,9 +41,11 @@ rules: - apiGroups: ["apps"] resources: ["daemonsets", "deployments", "replicasets", "statefulsets"] verbs: ["list", "watch"] + {{- if .Values.webhook.enabled }} - apiGroups: ["apiextensions.k8s.io"] resources: ["customresourcedefinitions"] verbs: ["get", "watch", "list"] + {{- end }} - apiGroups: ["policy"] resources: ["poddisruptionbudgets"] verbs: ["get", "list", "watch"] From a09bea7a743a44d51ed4ae18763012260f239df6 Mon Sep 17 00:00:00 2001 From: Reed Schalo Date: Mon, 30 Sep 2024 11:24:18 -0700 Subject: [PATCH 3/9] move migration controllers to provider --- pkg/controllers/controllers.go | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/pkg/controllers/controllers.go b/pkg/controllers/controllers.go index 1f6411bea07a..681edeae1088 100644 --- a/pkg/controllers/controllers.go +++ b/pkg/controllers/controllers.go @@ -20,7 +20,9 @@ import ( "github.com/awslabs/operatorpkg/controller" "github.com/awslabs/operatorpkg/status" "sigs.k8s.io/controller-runtime/pkg/manager" + karpv1 "sigs.k8s.io/karpenter/pkg/apis/v1" "sigs.k8s.io/karpenter/pkg/cloudprovider" + migrationcrd "sigs.k8s.io/karpenter/pkg/controllers/migration/crd" migration "sigs.k8s.io/karpenter/pkg/controllers/migration/resource" v1 "github.com/aws/karpenter-provider-aws/pkg/apis/v1" @@ -38,6 +40,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/karpenter/pkg/events" + karpoptions "sigs.k8s.io/karpenter/pkg/operator/options" "github.com/aws/karpenter-provider-aws/pkg/cache" "github.com/aws/karpenter-provider-aws/pkg/controllers/interruption" @@ -68,7 +71,12 @@ func NewControllers(ctx context.Context, mgr manager.Manager, sess *session.Sess controllerspricing.NewController(pricingProvider), controllersinstancetype.NewController(instanceTypeProvider), status.NewController[*v1.EC2NodeClass](kubeClient, mgr.GetEventRecorderFor("karpenter")), - migration.NewController[*v1.EC2NodeClass](kubeClient), + } + if !karpoptions.FromContext(ctx).DisableWebhook { + controllers = append(controllers, migration.NewController[*karpv1.NodeClaim](kubeClient)) + controllers = append(controllers, migration.NewController[*karpv1.NodePool](kubeClient)) + controllers = append(controllers, migration.NewController[*v1.EC2NodeClass](kubeClient)) + controllers = append(controllers, migrationcrd.NewController(kubeClient, cloudProvider)) } if options.FromContext(ctx).InterruptionQueue != "" { sqsapi := servicesqs.New(sess) From 7e87bbe2b2e5360a7877abb23fcd57b3f7ce76e4 Mon Sep 17 00:00:00 2001 From: Reed Schalo Date: Mon, 30 Sep 2024 11:29:39 -0700 Subject: [PATCH 4/9] move crd status to webhook enabled block --- charts/karpenter/templates/clusterrole-core.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/karpenter/templates/clusterrole-core.yaml b/charts/karpenter/templates/clusterrole-core.yaml index dc0ee1e670cb..9e8e1375976b 100644 --- a/charts/karpenter/templates/clusterrole-core.yaml +++ b/charts/karpenter/templates/clusterrole-core.yaml @@ -68,11 +68,11 @@ rules: - apiGroups: [""] resources: ["pods"] verbs: ["delete"] + {{- if .Values.webhook.enabled }} - apiGroups: ["apiextensions.k8s.io"] resources: ["customresourcedefinitions/status"] resourceNames: ["ec2nodeclasses.karpenter.k8s.aws", "nodepools.karpenter.sh", "nodeclaims.karpenter.sh"] verbs: ["patch"] - {{- if .Values.webhook.enabled }} - apiGroups: ["apiextensions.k8s.io"] resources: ["customresourcedefinitions"] resourceNames: ["ec2nodeclasses.karpenter.k8s.aws", "nodepools.karpenter.sh", "nodeclaims.karpenter.sh"] From 054d06693c8272c6cf0285f4ff33e4b675b834b6 Mon Sep 17 00:00:00 2001 From: Reed Schalo Date: Mon, 30 Sep 2024 12:29:28 -0700 Subject: [PATCH 5/9] remove print statment --- test/suites/integration/migration_test.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/test/suites/integration/migration_test.go b/test/suites/integration/migration_test.go index 0f641970e8f1..b9bf026aedcf 100644 --- a/test/suites/integration/migration_test.go +++ b/test/suites/integration/migration_test.go @@ -15,7 +15,6 @@ limitations under the License. package integration_test import ( - "fmt" "time" apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" @@ -88,7 +87,6 @@ var _ = Describe("EC2NodeClass Migration Controller", func() { crd := &apiextensionsv1.CustomResourceDefinition{} Eventually(func(g Gomega) { g.Expect(env.Client.Get(env.Context, client.ObjectKeyFromObject(item), crd)).To(Succeed()) - fmt.Println(crd.Status.StoredVersions) g.Expect(crd.Status.StoredVersions).To(HaveExactElements("v1")) }).WithTimeout(time.Second * 10).Should(Succeed()) } From 4e2dc94a295046dbe04bda2a04e7a622c315ead6 Mon Sep 17 00:00:00 2001 From: Reed Schalo Date: Mon, 30 Sep 2024 13:36:07 -0700 Subject: [PATCH 6/9] move controllers back to core --- pkg/controllers/controllers.go | 5 ----- 1 file changed, 5 deletions(-) diff --git a/pkg/controllers/controllers.go b/pkg/controllers/controllers.go index 681edeae1088..18b19c89237c 100644 --- a/pkg/controllers/controllers.go +++ b/pkg/controllers/controllers.go @@ -20,9 +20,7 @@ import ( "github.com/awslabs/operatorpkg/controller" "github.com/awslabs/operatorpkg/status" "sigs.k8s.io/controller-runtime/pkg/manager" - karpv1 "sigs.k8s.io/karpenter/pkg/apis/v1" "sigs.k8s.io/karpenter/pkg/cloudprovider" - migrationcrd "sigs.k8s.io/karpenter/pkg/controllers/migration/crd" migration "sigs.k8s.io/karpenter/pkg/controllers/migration/resource" v1 "github.com/aws/karpenter-provider-aws/pkg/apis/v1" @@ -73,10 +71,7 @@ func NewControllers(ctx context.Context, mgr manager.Manager, sess *session.Sess status.NewController[*v1.EC2NodeClass](kubeClient, mgr.GetEventRecorderFor("karpenter")), } if !karpoptions.FromContext(ctx).DisableWebhook { - controllers = append(controllers, migration.NewController[*karpv1.NodeClaim](kubeClient)) - controllers = append(controllers, migration.NewController[*karpv1.NodePool](kubeClient)) controllers = append(controllers, migration.NewController[*v1.EC2NodeClass](kubeClient)) - controllers = append(controllers, migrationcrd.NewController(kubeClient, cloudProvider)) } if options.FromContext(ctx).InterruptionQueue != "" { sqsapi := servicesqs.New(sess) From 18d6f993bdea3b54c3d7de33d9c114772842fa5d Mon Sep 17 00:00:00 2001 From: Reed Schalo Date: Mon, 30 Sep 2024 13:53:33 -0700 Subject: [PATCH 7/9] add context --- cmd/controller/main.go | 1 + 1 file changed, 1 insertion(+) diff --git a/cmd/controller/main.go b/cmd/controller/main.go index 9a51e4537aef..78a454e1bc6c 100644 --- a/cmd/controller/main.go +++ b/cmd/controller/main.go @@ -44,6 +44,7 @@ func main() { op. WithControllers(ctx, corecontrollers.NewControllers( + ctx, op.Manager, op.Clock, op.GetClient(), From 91b2af972b66adebb52e2ccdad03e374b3a009b7 Mon Sep 17 00:00:00 2001 From: Reed Schalo Date: Mon, 30 Sep 2024 14:37:06 -0700 Subject: [PATCH 8/9] bump core --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 6545bd85d158..ae2cf4a828bb 100644 --- a/go.mod +++ b/go.mod @@ -32,7 +32,7 @@ require ( k8s.io/utils v0.0.0-20240102154912-e7106e64919e knative.dev/pkg v0.0.0-20231010144348-ca8c009405dd sigs.k8s.io/controller-runtime v0.18.4 - sigs.k8s.io/karpenter v1.0.2 + sigs.k8s.io/karpenter v1.0.3-0.20240930210524-ced4d37eda82 sigs.k8s.io/yaml v1.4.0 ) diff --git a/go.sum b/go.sum index 8b54b60a7272..f68e5c977fe3 100644 --- a/go.sum +++ b/go.sum @@ -763,6 +763,8 @@ sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMm sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= sigs.k8s.io/karpenter v1.0.2 h1:llX4Sb6cLZmSZImMqFktZ3O+M8+nlc8xRcAEtfQVzCI= sigs.k8s.io/karpenter v1.0.2/go.mod h1:3NLmsnHHw8p4VutpjTOPUZyhE3qH6yGTs8O94Lsu8uw= +sigs.k8s.io/karpenter v1.0.3-0.20240930210524-ced4d37eda82 h1:kKll2aUiWasrhrIkZF2fwSC7T5snCirfTQV3mjn0rH4= +sigs.k8s.io/karpenter v1.0.3-0.20240930210524-ced4d37eda82/go.mod h1:3NLmsnHHw8p4VutpjTOPUZyhE3qH6yGTs8O94Lsu8uw= sigs.k8s.io/structured-merge-diff/v4 v4.4.1 h1:150L+0vs/8DA78h1u02ooW1/fFq/Lwr+sGiqlzvrtq4= sigs.k8s.io/structured-merge-diff/v4 v4.4.1/go.mod h1:N8hJocpFajUSSeSJ9bOZ77VzejKZaXsTtZo4/u7Io08= sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E= From b2e2ca48fa7061b2ada7f4ad0ff5b3be484c197c Mon Sep 17 00:00:00 2001 From: Reed Schalo Date: Mon, 30 Sep 2024 14:46:18 -0700 Subject: [PATCH 9/9] fix go.sum --- go.sum | 2 -- 1 file changed, 2 deletions(-) diff --git a/go.sum b/go.sum index f68e5c977fe3..a033be8ef239 100644 --- a/go.sum +++ b/go.sum @@ -761,8 +761,6 @@ sigs.k8s.io/controller-runtime v0.18.4 h1:87+guW1zhvuPLh1PHybKdYFLU0YJp4FhJRmiHv sigs.k8s.io/controller-runtime v0.18.4/go.mod h1:TVoGrfdpbA9VRFaRnKgk9P5/atA0pMwq+f+msb9M8Sg= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= -sigs.k8s.io/karpenter v1.0.2 h1:llX4Sb6cLZmSZImMqFktZ3O+M8+nlc8xRcAEtfQVzCI= -sigs.k8s.io/karpenter v1.0.2/go.mod h1:3NLmsnHHw8p4VutpjTOPUZyhE3qH6yGTs8O94Lsu8uw= sigs.k8s.io/karpenter v1.0.3-0.20240930210524-ced4d37eda82 h1:kKll2aUiWasrhrIkZF2fwSC7T5snCirfTQV3mjn0rH4= sigs.k8s.io/karpenter v1.0.3-0.20240930210524-ced4d37eda82/go.mod h1:3NLmsnHHw8p4VutpjTOPUZyhE3qH6yGTs8O94Lsu8uw= sigs.k8s.io/structured-merge-diff/v4 v4.4.1 h1:150L+0vs/8DA78h1u02ooW1/fFq/Lwr+sGiqlzvrtq4=