Add e2e2 (#370)

e2e2/.dockerignore

@@ -0,0 +1 @@
+_bin/

e2e2/.gitignore

@@ -0,0 +1,4 @@
+_bin
+_rundir
+_artifacts
+.kuberlr

e2e2/Makefile

@@ -0,0 +1,4 @@
+BIN_DIR ?= _bin
+
+$(BIN_DIR)/runner: runner.go
+	go build -o $(BIN_DIR)/runner runner.go

e2e2/go.mod

@@ -0,0 +1,83 @@
+module github.com/aws/aws-k8s-tester/e2e2
+
+go 1.21
+
+require (
+	k8s.io/api v0.28.3
+	k8s.io/apimachinery v0.28.3
+	k8s.io/cli-runtime v0.28.3
+	k8s.io/client-go v0.28.3
+	sigs.k8s.io/e2e-framework v0.3.0
+)
+
+require (
+	github.com/aws/aws-sdk-go-v2 v1.22.0 // indirect
+	github.com/aws/aws-sdk-go-v2/config v1.20.0 // indirect
+	github.com/aws/aws-sdk-go-v2/credentials v1.14.0 // indirect
+	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.0 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.0 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.0 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/ini v1.4.0 // indirect
+	github.com/aws/aws-sdk-go-v2/service/eks v1.31.0 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.0 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sso v1.16.0 // indirect
+	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.18.0 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sts v1.24.0 // indirect
+	github.com/aws/smithy-go v1.16.0 // indirect
+	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/emicklei/go-restful/v3 v3.10.2 // indirect
+	github.com/evanphx/json-patch v4.12.0+incompatible // indirect
+	github.com/evanphx/json-patch/v5 v5.6.0 // indirect
+	github.com/go-errors/errors v1.4.2 // indirect
+	github.com/go-logr/logr v1.2.4 // indirect
+	github.com/go-openapi/jsonpointer v0.19.6 // indirect
+	github.com/go-openapi/jsonreference v0.20.2 // indirect
+	github.com/go-openapi/swag v0.22.3 // indirect
+	github.com/gogo/protobuf v1.3.2 // indirect
+	github.com/golang/protobuf v1.5.3 // indirect
+	github.com/google/gnostic-models v0.6.8 // indirect
+	github.com/google/go-cmp v0.5.9 // indirect
+	github.com/google/gofuzz v1.2.0 // indirect
+	github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect
+	github.com/google/uuid v1.3.0 // indirect
+	github.com/imdario/mergo v0.3.15 // indirect
+	github.com/jmespath/go-jmespath v0.4.0 // indirect
+	github.com/josharian/intern v1.0.0 // indirect
+	github.com/json-iterator/go v1.1.12 // indirect
+	github.com/kubeflow/common v0.4.6 // indirect
+	github.com/kubeflow/mpi-operator v0.4.0 // indirect
+	github.com/mailru/easyjson v0.7.7 // indirect
+	github.com/moby/spdystream v0.2.0 // indirect
+	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
+	github.com/modern-go/reflect2 v1.0.2 // indirect
+	github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 // indirect
+	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
+	github.com/pkg/errors v0.9.1 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/spf13/pflag v1.0.5 // indirect
+	github.com/stretchr/testify v1.8.4 // indirect
+	github.com/vladimirvivien/gexe v0.2.0 // indirect
+	github.com/xlab/treeprint v1.2.0 // indirect
+	go.starlark.net v0.0.0-20230525235612-a134d8f9ddca // indirect
+	golang.org/x/net v0.17.0 // indirect
+	golang.org/x/oauth2 v0.8.0 // indirect
+	golang.org/x/sync v0.2.0 // indirect
+	golang.org/x/sys v0.13.0 // indirect
+	golang.org/x/term v0.13.0 // indirect
+	golang.org/x/text v0.13.0 // indirect
+	golang.org/x/time v0.3.0 // indirect
+	google.golang.org/appengine v1.6.7 // indirect
+	google.golang.org/protobuf v1.30.0 // indirect
+	gopkg.in/inf.v0 v0.9.1 // indirect
+	gopkg.in/yaml.v2 v2.4.0 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
+	k8s.io/klog/v2 v2.100.1 // indirect
+	k8s.io/kube-openapi v0.0.0-20230717233707-2695361300d9 // indirect
+	k8s.io/utils v0.0.0-20230406110748-d93618cff8a2 // indirect
+	sigs.k8s.io/controller-runtime v0.15.1 // indirect
+	sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
+	sigs.k8s.io/kustomize/api v0.13.5-0.20230601165947-6ce0bf390ce3 // indirect
+	sigs.k8s.io/kustomize/kyaml v0.14.3-0.20230601165947-6ce0bf390ce3 // indirect
+	sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect
+	sigs.k8s.io/yaml v1.3.0 // indirect
+)

e2e2/internal/framework_extensions/client.go

@@ -0,0 +1,136 @@
+package frameworkext
+
+import (
+	"context"
+	"os"
+
+	"k8s.io/apimachinery/pkg/api/meta"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/cli-runtime/pkg/resource"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/rest"
+	"k8s.io/client-go/restmapper"
+	"sigs.k8s.io/e2e-framework/klient/decoder"
+	"sigs.k8s.io/e2e-framework/klient/k8s"
+)
+
+// ApplyFiles creates Kubernetes objects contained in manifest file(s), in a manner similar to `kubectl apply -f`
+// Multiple objects may be in each manifest file.
+// The manifest files are processed in order.
+func ApplyFiles(restConfig *rest.Config, manifestFiles []string) error {
+	for _, manifestFile := range manifestFiles {
+		if err := ApplyFile(restConfig, manifestFile); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// ApplyFile creates Kubernetes objects contained in a manifest file, in a manner similar to `kubectl apply -f`
+// Multiple objects may be in the manifest file.
+func ApplyFile(restConfig *rest.Config, manifestFile string) error {
+	f, err := os.Open(manifestFile)
+	if err != nil {
+		return err
+	}
+	objs, err := decoder.DecodeAll(context.TODO(), f)
+	if err != nil {
+		return err
+	}
+	return processObjects(restConfig, objs, func(client *resource.Helper, obj k8s.Object) error {
+		namespace, err := meta.NewAccessor().Namespace(obj)
+		if err != nil {
+			return err
+		}
+		if namespace == "" {
+			namespace = "default"
+		}
+		_, err = client.Create(namespace, false, obj)
+		return err
+	})
+}
+
+// DeleteFiles deletes Kubernetes objects contained in manifest file(s), in a manner similar to `kubectl delete -f`
+// Multiple objects may be in each manifest file.
+// The manifest files are processed in order.
+func DeleteFiles(restConfig *rest.Config, manifestFiles []string) error {
+	for _, manifestFile := range manifestFiles {
+		if err := DeleteFile(restConfig, manifestFile); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// DeleteFile deletes Kubernetes objects contained in a manifest file, in a manner similar to `kubectl delete -f`
+// Multiple objects may be in the manifest file.
+func DeleteFile(restConfig *rest.Config, manifestFile string) error {
+	f, err := os.Open(manifestFile)
+	if err != nil {
+		return err
+	}
+	objs, err := decoder.DecodeAll(context.TODO(), f)
+	if err != nil {
+		return err
+	}
+	return processObjects(restConfig, objs, func(client *resource.Helper, obj k8s.Object) error {
+		name, err := meta.NewAccessor().Name(obj)
+		if err != nil {
+			return err
+		}
+		namespace, err := meta.NewAccessor().Namespace(obj)
+		if err != nil {
+			return err
+		}
+		if namespace == "" {
+			namespace = "default"
+		}
+		_, err = client.Delete(namespace, name)
+		return err
+	})
+}
+
+// processObjects applies a processFunc to each object, supplying it a dynamically-typed client appropriate for the object
+func processObjects(restConfig *rest.Config, objs []k8s.Object, processFunc func(client *resource.Helper, obj k8s.Object) error) error {
+	clientset, err := kubernetes.NewForConfig(restConfig)
+	if err != nil {
+		return err
+	}
+	groupResources, err := restmapper.GetAPIGroupResources(clientset.Discovery())
+	if err != nil {
+		return err
+	}
+	rm := restmapper.NewDiscoveryRESTMapper(groupResources)
+	for _, obj := range objs {
+		client, err := newResourceHelper(restConfig, rm, obj)
+		if err != nil {
+			return err
+		}
+		processFunc(client, obj)
+	}
+	return nil
+}
+
+func newResourceHelper(restConfig *rest.Config, rm meta.RESTMapper, obj runtime.Object) (*resource.Helper, error) {
+	gvk := obj.GetObjectKind().GroupVersionKind()
+	gk := schema.GroupKind{Group: gvk.Group, Kind: gvk.Kind}
+	mapping, err := rm.RESTMapping(gk, gvk.Version)
+	if err != nil {
+		return nil, err
+	}
+	gv := mapping.GroupVersionKind.GroupVersion()
+	restConfig.ContentConfig = resource.UnstructuredPlusDefaultContentConfig()
+	restConfig.GroupVersion = &gv
+	if len(gv.Group) == 0 {
+		restConfig.APIPath = "/api"
+	} else {
+		restConfig.APIPath = "/apis"
+	}
+	restClient, err := rest.RESTClientFor(restConfig)
+	if err != nil {
+		return nil, err
+	}
+
+	return resource.NewHelper(restClient, mapping), nil
+}

e2e2/internal/framework_extensions/conditions.go

@@ -0,0 +1,43 @@
+package frameworkext
+
+import (
+	"context"
+
+	appsv1 "k8s.io/api/apps/v1"
+	apimachinerywait "k8s.io/apimachinery/pkg/util/wait"
+
+	"sigs.k8s.io/e2e-framework/klient/k8s"
+	"sigs.k8s.io/e2e-framework/klient/k8s/resources"
+)
+
+type ConditionExtension struct {
+	resources *resources.Resources
+}
+
+func NewConditionExtension(r *resources.Resources) *ConditionExtension {
+	return &ConditionExtension{resources: r}
+}
+
+// ResourceMatch is a helper function used to check if the resource under question has met a pre-defined state. This can
+// be leveraged for checking fields on a resource that may not be immediately present upon creation.
+func (c *ConditionExtension) ResourceMatch(obj k8s.Object, matchFetcher func(object k8s.Object) bool) apimachinerywait.ConditionWithContextFunc {
+	return func(ctx context.Context) (done bool, err error) {
+		if err := c.resources.Get(ctx, obj.GetName(), obj.GetNamespace(), obj); err != nil {
+			return false, err
+		}
+		return matchFetcher(obj), nil
+	}
+}
+
+func (c *ConditionExtension) DaemonSetReady(daemonset k8s.Object) apimachinerywait.ConditionWithContextFunc {
+	return func(ctx context.Context) (done bool, err error) {
+		if err := c.resources.Get(ctx, daemonset.GetName(), daemonset.GetNamespace(), daemonset); err != nil {
+			return false, err
+		}
+		status := daemonset.(*appsv1.DaemonSet).Status
+		if status.NumberReady == status.DesiredNumberScheduled && status.NumberUnavailable == 0 {
+			done = true
+		}
+		return
+	}
+}

e2e2/internal/framework_extensions/doc.go

@@ -0,0 +1,2 @@
+// Package frameworkext contains extensions to sigs.k8s.io/e2e-framework
+package frameworkext

e2e2/scripts/go-test.sh

@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+
+set -o errexit
+set -o pipefail
+set -o nounset
+set -o xtrace
+
+if [ "$#" -eq 0 ]; then
+  echo "Usage: $0 TEST_PATTERN [TEST_FLAGS]"
+  exit 1
+fi
+
+TEST_PATTERN="${1}"
+shift
+
+# -count=1 is not a cacheable flag, so the test will always be executed
+go test --timeout 0 -v -count=1 "${TEST_PATTERN}" ${@}

e2e2/test/cases/nvidia/main_test.go

@@ -0,0 +1,82 @@
+package nvidia
+
+import (
+	"context"
+	"log"
+	"os"
+	"slices"
+	"testing"
+	"time"
+
+	fwext "github.com/aws/aws-k8s-tester/e2e2/internal/framework_extensions"
+	appsv1 "k8s.io/api/apps/v1"
+	v1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"sigs.k8s.io/e2e-framework/klient/wait"
+	"sigs.k8s.io/e2e-framework/klient/wait/conditions"
+	"sigs.k8s.io/e2e-framework/pkg/env"
+	"sigs.k8s.io/e2e-framework/pkg/envconf"
+)
+
+var (
+	testenv env.Environment
+)
+
+func TestMain(m *testing.M) {
+	cfg, err := envconf.NewFromFlags()
+	if err != nil {
+		log.Fatalf("failed to initialize test environment: %v", err)
+	}
+	testenv = env.NewWithConfig(cfg)
+
+	// all NVIDIA tests require the device plugin and MPI operator
+	manifests := []string{
+		"manifests/nvidia-device-plugin.yaml",
+		"manifests/mpi-operator.yaml",
+	}
+
+	testenv.Setup(
+		func(ctx context.Context, config *envconf.Config) (context.Context, error) {
+			err := fwext.ApplyFiles(config.Client().RESTConfig(), manifests)
+			if err != nil {
+				return ctx, err
+			}
+			return ctx, nil
+		},
+		func(ctx context.Context, config *envconf.Config) (context.Context, error) {
+			dep := appsv1.Deployment{
+				ObjectMeta: metav1.ObjectMeta{Name: "mpi-operator", Namespace: "mpi-operator"},
+			}
+			err := wait.For(conditions.New(config.Client().Resources()).DeploymentConditionMatch(&dep, appsv1.DeploymentAvailable, v1.ConditionTrue),
+				wait.WithTimeout(time.Minute*5))
+			if err != nil {
+				return ctx, err
+			}
+			return ctx, nil
+		},
+		func(ctx context.Context, config *envconf.Config) (context.Context, error) {
+			ds := appsv1.DaemonSet{
+				ObjectMeta: metav1.ObjectMeta{Name: "nvidia-device-plugin-daemonset", Namespace: "kube-system"},
+			}
+			err := wait.For(fwext.NewConditionExtension(config.Client().Resources()).DaemonSetReady(&ds),
+				wait.WithTimeout(time.Minute*5))
+			if err != nil {
+				return ctx, err
+			}
+			return ctx, nil
+		},
+	)
+
+	testenv.Finish(
+		func(ctx context.Context, config *envconf.Config) (context.Context, error) {
+			slices.Reverse(manifests)
+			err := fwext.DeleteFiles(config.Client().RESTConfig(), manifests)
+			if err != nil {
+				return ctx, err
+			}
+			return ctx, nil
+		},
+	)
+
+	os.Exit(testenv.Run(m))
+}