From af796698f8ae9549043969da6f27a3864e1d251a Mon Sep 17 00:00:00 2001 From: Christian Schlotter Date: Fri, 23 Feb 2024 17:28:20 +0100 Subject: [PATCH] debug cert-manager --- Makefile | 2 +- scripts/ci-e2e.sh | 14 +++- test/e2e/clusterctl_upgrade_test.go | 2 +- test/framework/alltypes_helpers.go | 72 ++++++++++++++++++- .../clusterctl/clusterctl_helpers.go | 30 ++++++++ test/framework/controller_helpers.go | 18 +++++ 6 files changed, 132 insertions(+), 6 deletions(-) diff --git a/Makefile b/Makefile index 92c0390ed554..ed2575ad4ccf 100644 --- a/Makefile +++ b/Makefile @@ -874,7 +874,7 @@ test-test-extension-junit: $(SETUP_ENVTEST) $(GOTESTSUM) ## Run unit and integra .PHONY: test-e2e test-e2e: $(GINKGO) generate-e2e-templates ## Run the end-to-end tests $(GINKGO) -v --trace -poll-progress-after=$(GINKGO_POLL_PROGRESS_AFTER) \ - -poll-progress-interval=$(GINKGO_POLL_PROGRESS_INTERVAL) --tags=e2e --focus="$(GINKGO_FOCUS)" \ + -poll-progress-interval=$(GINKGO_POLL_PROGRESS_INTERVAL) --tags=e2e --focus="$(GINKGO_FOCUS)" --label-filter=Foo \ $(_SKIP_ARGS) --nodes=$(GINKGO_NODES) --timeout=$(GINKGO_TIMEOUT) --no-color=$(GINKGO_NOCOLOR) \ --output-dir="$(ARTIFACTS)" --junit-report="junit.e2e_suite.1.xml" $(GINKGO_ARGS) $(ROOT_DIR)/$(TEST_DIR)/e2e -- \ -e2e.artifacts-folder="$(ARTIFACTS)" \ diff --git a/scripts/ci-e2e.sh b/scripts/ci-e2e.sh index 242e972c0998..b7ae54a6e65f 100755 --- a/scripts/ci-e2e.sh +++ b/scripts/ci-e2e.sh @@ -114,6 +114,14 @@ docker events > "${ARTIFACTS_LOCAL}/docker-events.txt" 2>&1 & ctr -n moby events > "${ARTIFACTS_LOCAL}/containerd-events.txt" 2>&1 & # Run e2e tests -mkdir -p "$ARTIFACTS" -echo "+ run tests!" -make test-e2e + +ORIGINAL_ARTIFACTS="${ARTIFACTS}" + +for i in {1..10}; do + ARTIFACTS="${ORIGINAL_ARTIFACTS}/${i}" + echo "Using artifacts location: $ARTIFACTS" + mkdir -p "$ARTIFACTS" + echo "+ run tests!" + + make test-e2e +done diff --git a/test/e2e/clusterctl_upgrade_test.go b/test/e2e/clusterctl_upgrade_test.go index 652d35b07510..8aab4efea7d9 100644 --- a/test/e2e/clusterctl_upgrade_test.go +++ b/test/e2e/clusterctl_upgrade_test.go @@ -26,7 +26,7 @@ import ( "sigs.k8s.io/cluster-api/test/framework" ) -var _ = Describe("When testing clusterctl upgrades (v0.4=>current)", func() { +var _ = Describe("When testing clusterctl upgrades (v0.4=>current) [PR-Blocking]", Label("Foo"), func() { ClusterctlUpgradeSpec(ctx, func() ClusterctlUpgradeSpecInput { return ClusterctlUpgradeSpecInput{ E2EConfig: e2eConfig, diff --git a/test/framework/alltypes_helpers.go b/test/framework/alltypes_helpers.go index d5ce38bb5f5c..e316b7418805 100644 --- a/test/framework/alltypes_helpers.go +++ b/test/framework/alltypes_helpers.go @@ -42,6 +42,42 @@ import ( . "sigs.k8s.io/cluster-api/test/framework/ginkgoextensions" ) +// GetGlobalCAPIResourcesInput is the input for GetCAPIResources. +type GetGlobalCAPIResourcesInput struct { + Lister Lister + Namespace string +} + +// GetGlobalCAPIResources reads all the CAPI resources in a namespace. +// This list includes all the types belonging to CAPI providers. +func GetGlobalCAPIResources(ctx context.Context, input GetGlobalCAPIResourcesInput) []*unstructured.Unstructured { + Expect(ctx).NotTo(BeNil(), "ctx is required for GetGlobalCAPIResources") + Expect(input.Lister).NotTo(BeNil(), "input.Lister is required for GetGlobalCAPIResources") + + types := getGlobalClusterAPITypes(ctx, input.Lister) + + objList := []*unstructured.Unstructured{} + for i := range types { + typeMeta := types[i] + typeList := new(unstructured.UnstructuredList) + typeList.SetAPIVersion(typeMeta.APIVersion) + typeList.SetKind(typeMeta.Kind) + + if err := input.Lister.List(ctx, typeList); err != nil { + if apierrors.IsNotFound(err) { + continue + } + Fail(fmt.Sprintf("failed to list %q resources: %v", typeList.GroupVersionKind(), err)) + } + for i := range typeList.Items { + obj := typeList.Items[i] + objList = append(objList, &obj) + } + } + + return objList +} + // GetCAPIResourcesInput is the input for GetCAPIResources. type GetCAPIResourcesInput struct { Lister Lister @@ -79,6 +115,35 @@ func GetCAPIResources(ctx context.Context, input GetCAPIResourcesInput) []*unstr return objList } +// getClusterAPITypes returns the list of TypeMeta to be considered for the move discovery phase. +// This list includes all the types belonging to CAPI providers. +func getGlobalClusterAPITypes(ctx context.Context, lister Lister) []metav1.TypeMeta { + discoveredTypes := []metav1.TypeMeta{} + + crdList := &apiextensionsv1.CustomResourceDefinitionList{} + Eventually(func() error { + return lister.List(ctx, crdList, client.HasLabels{"clusterctl.cluster.x-k8s.io/core"}) + }, retryableOperationTimeout, retryableOperationInterval).Should(Succeed(), "failed to list CRDs for CAPI providers") + + for _, crd := range crdList.Items { + for _, version := range crd.Spec.Versions { + if !version.Storage { + continue + } + + discoveredTypes = append(discoveredTypes, metav1.TypeMeta{ + Kind: crd.Spec.Names.Kind, + APIVersion: metav1.GroupVersion{ + Group: crd.Spec.Group, + Version: version.Name, + }.String(), + }) + } + } + + return discoveredTypes +} + // getClusterAPITypes returns the list of TypeMeta to be considered for the move discovery phase. // This list includes all the types belonging to CAPI providers. func getClusterAPITypes(ctx context.Context, lister Lister) []metav1.TypeMeta { @@ -104,6 +169,7 @@ func getClusterAPITypes(ctx context.Context, lister Lister) []metav1.TypeMeta { }) } } + return discoveredTypes } @@ -126,7 +192,11 @@ func DumpAllResources(ctx context.Context, input DumpAllResourcesInput) { Namespace: input.Namespace, }) - for i := range resources { + globalResources := GetGlobalCAPIResources(ctx, GetGlobalCAPIResourcesInput{ + Lister: input.Lister, + }) + + for i := range append(resources, globalResources...) { r := resources[i] dumpObject(r, input.LogPath) } diff --git a/test/framework/clusterctl/clusterctl_helpers.go b/test/framework/clusterctl/clusterctl_helpers.go index d970a5e3f229..31ffd64dde73 100644 --- a/test/framework/clusterctl/clusterctl_helpers.go +++ b/test/framework/clusterctl/clusterctl_helpers.go @@ -18,6 +18,7 @@ package clusterctl import ( "context" + "fmt" "os" "path/filepath" "time" @@ -128,6 +129,35 @@ func InitManagementClusterAndWatchControllerLogs(ctx context.Context, input Init }) } } + + certManagerDeployments := framework.GetCertManagerDeployments(ctx, framework.GetControllerDeploymentsInput{ + Lister: client, + }) + for _, deployment := range certManagerDeployments { + fmt.Printf("Pod status of %s/%s:\n%++v\n", deployment.Namespace, deployment.Name, deployment.Status) + // framework.WaitForDeploymentsAvailable(ctx, framework.WaitForDeploymentsAvailableInput{ + // Getter: client, + // Deployment: deployment, + // }, intervals...) + + // Start streaming logs from all controller providers + framework.WatchDeploymentLogsByName(ctx, framework.WatchDeploymentLogsByNameInput{ + GetLister: client, + Cache: input.ClusterProxy.GetCache(ctx), + ClientSet: input.ClusterProxy.GetClientSet(), + Deployment: deployment, + LogPath: filepath.Join(input.LogFolder, "logs", deployment.GetNamespace()), + }) + + // if !input.DisableMetricsCollection { + // framework.WatchPodMetrics(ctx, framework.WatchPodMetricsInput{ + // GetLister: client, + // ClientSet: input.ClusterProxy.GetClientSet(), + // Deployment: deployment, + // MetricsPath: filepath.Join(input.LogFolder, "metrics", deployment.GetNamespace()), + // }) + // } + } } // UpgradeManagementClusterAndWaitInput is the input type for UpgradeManagementClusterAndWait. diff --git a/test/framework/controller_helpers.go b/test/framework/controller_helpers.go index a4327a37264d..6e88510c81ff 100644 --- a/test/framework/controller_helpers.go +++ b/test/framework/controller_helpers.go @@ -21,6 +21,7 @@ import ( . "github.com/onsi/gomega" appsv1 "k8s.io/api/apps/v1" + "sigs.k8s.io/controller-runtime/pkg/client" ) // GetControllerDeploymentsInput is the input for GetControllerDeployments. @@ -46,6 +47,23 @@ func GetControllerDeployments(ctx context.Context, input GetControllerDeployment return deployments } +// GetCertManagerDeployments returns all the deployment for the cluster API controllers existing in a management cluster. +func GetCertManagerDeployments(ctx context.Context, input GetControllerDeploymentsInput) []*appsv1.Deployment { + deploymentList := &appsv1.DeploymentList{} + Eventually(func() error { + return input.Lister.List(ctx, deploymentList, client.InNamespace("cert-manager")) + }, retryableOperationTimeout, retryableOperationInterval).Should(Succeed(), "Failed to list deployments for the cluster API controllers") + + deployments := make([]*appsv1.Deployment, 0, len(deploymentList.Items)) + for i := range deploymentList.Items { + d := &deploymentList.Items[i] + if !skipDeployment(d, input.ExcludeNamespaces) { + deployments = append(deployments, d) + } + } + return deployments +} + func skipDeployment(d *appsv1.Deployment, excludeNamespaces []string) bool { if !d.DeletionTimestamp.IsZero() { return true