clusterctl: validate no objects exist from CRDs before deleting them

cmd/clusterctl/client/cluster/components.go

@@ -17,14 +17,17 @@ limitations under the License.
 package cluster
 
 import (
+	"context"
 	"fmt"
 	"strings"
 
 	"github.com/pkg/errors"
 	corev1 "k8s.io/api/core/v1"
+	apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/apimachinery/pkg/runtime/schema"
 	kerrors "k8s.io/apimachinery/pkg/util/errors"
 	"k8s.io/apimachinery/pkg/util/sets"
 	"sigs.k8s.io/controller-runtime/pkg/client"
@@ -65,6 +68,9 @@ type ComponentsClient interface {
 	// DeleteWebhookNamespace deletes the core provider webhook namespace (eg. capi-webhook-system).
 	// This is required when upgrading to v1alpha4 where webhooks are included in the controller itself.
 	DeleteWebhookNamespace() error
+
+	// ValidateNoObjectsExist checks if custom resources of the custom resource definitions exist and returns an error if so.
+	ValidateNoObjectsExist(ctx context.Context, provider clusterctlv1.Provider) error
 }
 
 // providerComponents implements ComponentsClient.
@@ -256,6 +262,59 @@ func (p *providerComponents) DeleteWebhookNamespace() error {
 	return nil
 }
 
+func (p *providerComponents) ValidateNoObjectsExist(ctx context.Context, provider clusterctlv1.Provider) error {
+	log := logf.Log
+	log.Info("Checking for CRs", "Provider", provider.Name, "Version", provider.Version, "Namespace", provider.Namespace)
+
+	proxyClient, err := p.proxy.NewClient()
+	if err != nil {
+		return err
+	}
+
+	// Fetch all the components belonging to a provider.
+	// We want that the delete operation is able to clean-up everything.
+	labels := map[string]string{
+		clusterctlv1.ClusterctlLabel: "",
+		clusterv1.ProviderNameLabel:  provider.ManifestLabel(),
+	}
+
+	customResources := &apiextensionsv1.CustomResourceDefinitionList{}
+	if err := proxyClient.List(ctx, customResources, client.MatchingLabels(labels)); err != nil {
+		return err
+	}
+
+	// Filter the resources according to the delete options
+	crsHavingObjects := []string{}
+	for _, crd := range customResources.Items {
+		crd := crd
+		storageVersion, err := storageVersionForCRD(&crd)
+		if err != nil {
+			return err
+		}
+
+		list := &unstructured.UnstructuredList{}
+		list.SetGroupVersionKind(schema.GroupVersionKind{
+			Group:   crd.Spec.Group,
+			Version: storageVersion,
+			Kind:    crd.Spec.Names.ListKind,
+		})
+
+		if err := proxyClient.List(ctx, list); err != nil {
+			return err
+		}
+
+		if len(list.Items) > 0 {
+			crsHavingObjects = append(crsHavingObjects, crd.Kind)
+		}
+	}
+
+	if len(crsHavingObjects) > 0 {
+		return fmt.Errorf("found existing objects for provider CRDs %q: [%s]. Please delete these objects first before running clusterctl delete with --include-crd", provider.GetName(), strings.Join(crsHavingObjects, ", "))
+	}
+
+	return nil
+}
+
 // newComponentsClient returns a providerComponents.
 func newComponentsClient(proxy Proxy) *providerComponents {
 	return &providerComponents{

cmd/clusterctl/client/cluster/components_test.go

@@ -24,6 +24,7 @@ import (
 	. "github.com/onsi/gomega"
 	corev1 "k8s.io/api/core/v1"
 	rbacv1 "k8s.io/api/rbac/v1"
+	apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
@@ -494,3 +495,78 @@ func Test_providerComponents_Create(t *testing.T) {
 		})
 	}
 }
+
+func Test_providerComponents_ValidateNoObjectsExist(t *testing.T) {
+	labels := map[string]string{
+		clusterv1.ProviderNameLabel: "infrastructure-infra",
+	}
+
+	crd := &apiextensionsv1.CustomResourceDefinition{
+		TypeMeta: metav1.TypeMeta{
+			Kind:       "CustomResourceDefinition",
+			APIVersion: apiextensionsv1.SchemeGroupVersion.Identifier(),
+		},
+		ObjectMeta: metav1.ObjectMeta{
+			Name:   "crd1",
+			Labels: labels,
+		},
+		Spec: apiextensionsv1.CustomResourceDefinitionSpec{
+			Group: "some.group",
+			Names: apiextensionsv1.CustomResourceDefinitionNames{
+				ListKind: "SomeCRDList",
+				Kind:     "SomeCRD",
+			},
+			Versions: []apiextensionsv1.CustomResourceDefinitionVersion{
+				{Name: "v1", Storage: true},
+			},
+		},
+	}
+	crd.ObjectMeta.Labels[clusterctlv1.ClusterctlLabel] = ""
+
+	cr := &unstructured.Unstructured{}
+	cr.SetAPIVersion("some.group/v1")
+	cr.SetKind("SomeCRD")
+	cr.SetName("cr1")
+
+	tests := []struct {
+		name     string
+		provider clusterctlv1.Provider
+		initObjs []client.Object
+		wantErr  bool
+	}{
+		{
+			name:     "No objects exist",
+			provider: clusterctlv1.Provider{ObjectMeta: metav1.ObjectMeta{Name: "infrastructure-infra", Namespace: "ns1"}, ProviderName: "infra", Type: string(clusterctlv1.InfrastructureProviderType)},
+			initObjs: []client.Object{},
+			wantErr:  false,
+		},
+		{
+			name:     "CRD exists but no objects",
+			provider: clusterctlv1.Provider{ObjectMeta: metav1.ObjectMeta{Name: "infrastructure-infra", Namespace: "ns1"}, ProviderName: "infra", Type: string(clusterctlv1.InfrastructureProviderType)},
+			initObjs: []client.Object{
+				crd,
+			},
+			wantErr: false,
+		},
+		{
+			name:     "CRD exists but and also objects",
+			provider: clusterctlv1.Provider{ObjectMeta: metav1.ObjectMeta{Name: "infrastructure-infra", Namespace: "ns1"}, ProviderName: "infra", Type: string(clusterctlv1.InfrastructureProviderType)},
+			initObjs: []client.Object{
+				crd,
+				cr,
+			},
+			wantErr: true,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			proxy := test.NewFakeProxy().WithObjs(tt.initObjs...)
+
+			c := newComponentsClient(proxy)
+
+			if err := c.ValidateNoObjectsExist(context.Background(), tt.provider); (err != nil) != tt.wantErr {
+				t.Errorf("providerComponents.ValidateNoObjectsExist() error = %v, wantErr %v", err, tt.wantErr)
+			}
+		})
+	}
+}

cmd/clusterctl/client/delete.go

@@ -17,8 +17,11 @@ limitations under the License.
 package client
 
 import (
+	"context"
+
 	"github.com/pkg/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	kerrors "k8s.io/apimachinery/pkg/util/errors"
 
 	clusterctlv1 "sigs.k8s.io/cluster-api/cmd/clusterctl/api/v1alpha3"
 	"sigs.k8s.io/cluster-api/cmd/clusterctl/client/cluster"
@@ -154,6 +157,19 @@ func (c *clusterctlClient) Delete(options DeleteOptions) error {
 		}
 	}
 
+	if options.IncludeCRDs {
+		errList := []error{}
+		for _, provider := range providersToDelete {
+			err = clusterClient.ProviderComponents().ValidateNoObjectsExist(context.TODO(), provider)
+			if err != nil {
+				errList = append(errList, err)
+			}
+		}
+		if len(errList) > 0 {
+			return kerrors.NewAggregate(errList)
+		}
+	}
+
 	// Delete the selected providers.
 	for _, provider := range providersToDelete {
 		if err := clusterClient.ProviderComponents().Delete(cluster.DeleteOptions{Provider: provider, IncludeNamespace: options.IncludeNamespace, IncludeCRDs: options.IncludeCRDs, SkipInventory: options.SkipInventory}); err != nil {

cmd/clusterctl/client/delete_test.go

@@ -66,6 +66,27 @@ func Test_clusterctlClient_Delete(t *testing.T) {
 			wantProviders: sets.Set[string]{},
 			wantErr:       false,
 		},
+		{
+			name: "Delete all the providers including CRDs",
+			fields: fields{
+				client: fakeClusterForDelete(),
+			},
+			args: args{
+				options: DeleteOptions{
+					Kubeconfig:              Kubeconfig{Path: "kubeconfig", Context: "mgmt-context"},
+					IncludeNamespace:        false,
+					IncludeCRDs:             true,
+					SkipInventory:           false,
+					CoreProvider:            "",
+					BootstrapProviders:      nil,
+					InfrastructureProviders: nil,
+					ControlPlaneProviders:   nil,
+					DeleteAll:               true, // delete all the providers
+				},
+			},
+			wantProviders: sets.Set[string]{},
+			wantErr:       false,
+		},
 		{
 			name: "Delete single provider auto-detect namespace",
 			fields: fields{