diff --git a/Makefile.oss.prow b/Makefile.oss.prow index afd141db9e..45962b8393 100644 --- a/Makefile.oss.prow +++ b/Makefile.oss.prow @@ -47,11 +47,6 @@ KUSTOMIZE_COMPONENTS_DIR := e2e/testdata/hydration/$(KUSTOMIZE_COMPONENTS_PACKAG KUSTOMIZE_COMPONENTS_PUBLIC_AR_IMAGE := $(LOCATION)-docker.pkg.dev/$(TEST_INFRA_PROJECT)/config-sync-test-public/$(KUSTOMIZE_COMPONENTS_PACKAGE_NAME) KUSTOMIZE_COMPONENTS_PUBLIC_GCR_IMAGE := gcr.io/$(TEST_INFRA_PROJECT)/$(KUSTOMIZE_COMPONENTS_PACKAGE_NAME) -BOOKINFO_REPO_PACKAGE_NAME := namespace-repo-bookinfo -BOOKINFO_REPO_DIR := e2e/testdata/$(BOOKINFO_REPO_PACKAGE_NAME) -# namespace-repo-bookinfo images (singleton test-infra registry) -BOOKINFO_REPO_PUBLIC_AR_IMAGE := $(LOCATION)-docker.pkg.dev/$(TEST_INFRA_PROJECT)/config-sync-test-public/$(BOOKINFO_REPO_PACKAGE_NAME) - # This target is run as a singleton against the ci-artifacts project, since # these require for the registries to be public. .PHONY: push-test-oci-images-public @@ -59,7 +54,6 @@ push-test-oci-images-public: "$(CRANE)" @gcloud $(GCLOUD_QUIET) auth configure-docker $(LOCATION)-docker.pkg.dev,gcr.io cd $(KUSTOMIZE_COMPONENTS_DIR) && crane append -f <(tar -f - -c .) -t $(KUSTOMIZE_COMPONENTS_PUBLIC_GCR_IMAGE) cd $(KUSTOMIZE_COMPONENTS_DIR) && crane append -f <(tar -f - -c .) -t $(KUSTOMIZE_COMPONENTS_PUBLIC_AR_IMAGE) - cd $(BOOKINFO_REPO_DIR) && crane append -f <(tar -f - -c .) -t $(BOOKINFO_REPO_PUBLIC_AR_IMAGE) # The following targets are used to provision test resources in a prow environment diff --git a/build/prow/e2e/Dockerfile b/build/prow/e2e/Dockerfile index 4d7493bb35..17c260983b 100644 --- a/build/prow/e2e/Dockerfile +++ b/build/prow/e2e/Dockerfile @@ -21,7 +21,7 @@ ARG DOCKER_CLI_IMAGE FROM ${GCLOUD_IMAGE} as gcloud-install RUN apt-get update && apt-get install -y \ - kubectl google-cloud-sdk-gke-gcloud-auth-plugin + kubectl google-cloud-cli-gke-gcloud-auth-plugin FROM ${GOLANG_IMAGE} as builder diff --git a/e2e/flags.go b/e2e/flags.go index 93a439ad2e..0e9d844423 100644 --- a/e2e/flags.go +++ b/e2e/flags.go @@ -19,6 +19,7 @@ package e2e import ( "flag" "fmt" + "slices" "strings" "testing" @@ -47,6 +48,33 @@ func newStringListFlag(name string, def []string, usage string) *[]string { return &lf.arr } +// stringEnum is a string flag with a set of allowed values +type stringEnum struct { + value string + allowedValues []string +} + +func (i *stringEnum) String() string { + return i.value +} + +func (i *stringEnum) Set(value string) error { + if !slices.Contains(i.allowedValues, value) { + return fmt.Errorf("%s not in allowed values: %s", value, i.allowedValues) + } + i.value = value + return nil +} + +func newStringEnum(name string, defaultVal string, usage string, allowed []string) *string { + lf := &stringEnum{ + value: defaultVal, + allowedValues: allowed, + } + flag.Var(lf, name, fmt.Sprintf("%s Allowed values: %s", usage, allowed)) + return &lf.value +} + // E2E enables running end-to-end tests. var E2E = flag.Bool("e2e", false, "If true, run end-to-end tests.") @@ -101,16 +129,19 @@ var ShareTestEnv = flag.Bool("share-test-env", false, "Specify that the test is using a shared test environment instead of fresh installation per test case.") // GitProvider is the provider that hosts the Git repositories. -var GitProvider = flag.String("git-provider", Local, - "The git provider that hosts the Git repositories. Defaults to local.") +var GitProvider = newStringEnum("git-provider", util.EnvString("E2E_GIT_PROVIDER", Local), + "The git provider that hosts the Git repositories. Defaults to Local.", + []string{Local, Bitbucket, GitLab, CSR}) // OCIProvider is the provider that hosts the OCI repositories. -var OCIProvider = flag.String("oci-provider", Local, - "The registry provider that hosts the OCI repositories. Defaults to local.") +var OCIProvider = newStringEnum("oci-provider", util.EnvString("E2E_OCI_PROVIDER", Local), + "The registry provider that hosts the OCI repositories. Defaults to Local.", + []string{Local, ArtifactRegistry}) -// HelmProvider is the provider that hosts the OCI repositories. -var HelmProvider = flag.String("helm-provider", Local, - "The registry provider that hosts the helm packages. Defaults to local.") +// HelmProvider is the provider that hosts the helm repositories. +var HelmProvider = newStringEnum("helm-provider", util.EnvString("E2E_HELM_PROVIDER", Local), + "The registry provider that hosts the helm repositories. Defaults to Local.", + []string{Local, ArtifactRegistry}) // TestFeatures is the list of features to run. var TestFeatures = flag.String("test-features", "", @@ -250,6 +281,8 @@ const ( GitLab = "gitlab" // CSR indicates using Google Cloud Source Repositories to host the repositories. CSR = "csr" + // ArtifactRegistry indicates using Google Artifact Registry to host the repositories. + ArtifactRegistry = "gar" ) // NumParallel returns the number of parallel test threads diff --git a/e2e/nomostest/artifactregistry/crane.go b/e2e/nomostest/artifactregistry/crane.go deleted file mode 100644 index 51b41f43cd..0000000000 --- a/e2e/nomostest/artifactregistry/crane.go +++ /dev/null @@ -1,92 +0,0 @@ -// Copyright 2022 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package artifactregistry - -import ( - "fmt" - "os" - "path/filepath" - - "kpt.dev/configsync/e2e/nomostest" -) - -// SetupCraneImage creates a new CraneImage for use during an e2e test, along -// with its registry, repository, and login. -// Returns a reference to the Image object and any errors. -func SetupCraneImage(nt *nomostest.NT, name, version string) (*CraneImage, error) { - image, err := SetupImage(nt, name, version) - if err != nil { - return nil, err - } - craneImage := &CraneImage{ - Image: image, - } - if err := craneImage.RegistryLogin(); err != nil { - return nil, err - } - return craneImage, nil -} - -// CraneImage represents a remote OCI-based crane-build image -type CraneImage struct { - *Image -} - -// RegistryLogin will log into the registry with crane using a gcloud auth token. -func (r *CraneImage) RegistryLogin() error { - var err error - authCmd := r.Shell.Command("gcloud", "auth", "print-access-token") - loginCmd := r.Shell.Command("crane", "auth", "login", - "-u", "oauth2accesstoken", "--password-stdin", - r.RegistryHost()) - loginCmd.Stdin, err = authCmd.StdoutPipe() - if err != nil { - return fmt.Errorf("creating STDOUT pipe: %w", err) - } - if err := loginCmd.Start(); err != nil { - return fmt.Errorf("starting login command: %w", err) - } - if err := authCmd.Run(); err != nil { - return fmt.Errorf("running print-access-token command: %w", err) - } - if err := loginCmd.Wait(); err != nil { - return fmt.Errorf("waiting for login command: %w", err) - } - return nil -} - -// Push will package and push the image located at r.BuildPath to the remote registry. -func (r *CraneImage) Push() error { - r.Logger.Infof("Packaging image: %s:%s", r.Name, r.Version) - // Use Tgz instead of just Tar so it's smaller. - parentPath := filepath.Dir(r.BuildPath) - localSourceTgzPath := filepath.Join(parentPath, fmt.Sprintf("%s-%s.tgz", r.Name, r.Version)) - if _, err := r.Shell.ExecWithDebug("tar", "-cvzf", localSourceTgzPath, r.BuildPath); err != nil { - return fmt.Errorf("packaging image: %w", err) - } - r.Logger.Infof("Pushing image: %s:%s", r.Name, r.Version) - _, err := r.Shell.ExecWithDebug("crane", "append", - "-f", localSourceTgzPath, - "-t", fmt.Sprintf("%s:%s", r.Address(), r.Version)) - if err != nil { - return fmt.Errorf("pushing image: %w", err) - } - // Remove local tgz to allow re-push with the same name & version. - // No need to defer. The whole TmpDir will be deleted in test cleanup. - if err := os.Remove(localSourceTgzPath); err != nil { - return fmt.Errorf("deleting local image package: %w", err) - } - return nil -} diff --git a/e2e/nomostest/artifactregistry/helm.go b/e2e/nomostest/artifactregistry/helm.go deleted file mode 100644 index 612955ed86..0000000000 --- a/e2e/nomostest/artifactregistry/helm.go +++ /dev/null @@ -1,167 +0,0 @@ -// Copyright 2022 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package artifactregistry - -import ( - "fmt" - "os" - "path/filepath" - - "kpt.dev/configsync/e2e/nomostest" - "sigs.k8s.io/yaml" -) - -// SetupHelmChart creates a new HelmChart for use during an e2e test. -// Returns a reference to the Image object and any errors. -func SetupHelmChart(nt *nomostest.NT, name, version string) (*HelmChart, error) { - image, err := SetupImage(nt, name, version) - if err != nil { - return nil, err - } - helmChart := &HelmChart{ - Image: image, - } - if err := helmChart.RegistryLogin(); err != nil { - return nil, err - } - return helmChart, nil -} - -// PushHelmChart pushes a new helm chart for use during an e2e test. -// Returns a reference to the RemoteHelmChart object and any errors. -func PushHelmChart(nt *nomostest.NT, name, version string) (*HelmChart, error) { - chart, err := SetupHelmChart(nt, name, version) - if err != nil { - return nil, err - } - artifactPath := fmt.Sprintf("../testdata/helm-charts/%s", name) - if err := chart.CopyLocalPackage(artifactPath); err != nil { - return nil, err - } - if err := chart.Push(); err != nil { - return nil, err - } - return chart, nil -} - -// HelmChart represents a remote OCI-based helm chart -type HelmChart struct { - Image *Image -} - -// RegistryLogin will log into the registry with helm using a gcloud auth token. -func (r *HelmChart) RegistryLogin() error { - var err error - authCmd := r.Image.Shell.Command("gcloud", "auth", "print-access-token") - loginCmd := r.Image.Shell.Command("helm", "registry", "login", - "-u", "oauth2accesstoken", "--password-stdin", - r.Image.RegistryHost()) - loginCmd.Stdin, err = authCmd.StdoutPipe() - if err != nil { - return fmt.Errorf("creating STDOUT pipe: %w", err) - } - if err := loginCmd.Start(); err != nil { - return fmt.Errorf("starting login command: %w", err) - } - if err := authCmd.Run(); err != nil { - return fmt.Errorf("running print-access-token command: %w", err) - } - if err := loginCmd.Wait(); err != nil { - return fmt.Errorf("waiting for login command: %w", err) - } - return nil -} - -// CopyLocalPackage accepts a local path to a helm chart and recursively copies -// it to r.BuildPath, modifying the name of the copied chart from its original -// name to r.Name and version to r.Version. -func (r *HelmChart) CopyLocalPackage(chartPath string) error { - if err := r.Image.CopyLocalPackage(chartPath); err != nil { - return fmt.Errorf("copying helm chart: %v", err) - } - - r.Image.Logger.Infof("Updating helm chart name & version: %s:%s", r.Image.Name, r.Image.Version) - chartFilePath := filepath.Join(r.Image.BuildPath, "Chart.yaml") - err := updateYAMLFile(chartFilePath, func(chartMap map[string]interface{}) error { - chartMap["name"] = r.Image.Name - chartMap["version"] = r.Image.Version - return nil - }) - if err != nil { - return fmt.Errorf("updating Chart.yaml: %v", err) - } - return nil -} - -func updateYAMLFile(name string, updateFn func(map[string]interface{}) error) error { - chartBytes, err := os.ReadFile(name) - if err != nil { - return fmt.Errorf("reading file: %s: %w", name, err) - } - var chartManifest map[string]interface{} - if err := yaml.Unmarshal(chartBytes, &chartManifest); err != nil { - return fmt.Errorf("parsing yaml file: %s: %w", name, err) - } - if err := updateFn(chartManifest); err != nil { - return fmt.Errorf("updating yaml map for %s: %w", name, err) - } - chartBytes, err = yaml.Marshal(chartManifest) - if err != nil { - return fmt.Errorf("formatting yaml for %s: %w", name, err) - } - if err := os.WriteFile(name, chartBytes, os.ModePerm); err != nil { - return fmt.Errorf("writing file: %s: %w", name, err) - } - return nil -} - -// SetVersion updates the local version of the helm chart to the specified -// version with a random suffix -func (r *HelmChart) SetVersion(version string) error { - if err := r.Image.SetVersion(version); err != nil { - return err - } - version = r.Image.Version - chartFilePath := filepath.Join(r.Image.BuildPath, "Chart.yaml") - err := updateYAMLFile(chartFilePath, func(chartMap map[string]interface{}) error { - chartMap["version"] = version - return nil - }) - if err != nil { - return fmt.Errorf("updating Chart.yaml: %v", err) - } - return nil -} - -// Push will package and push the helm chart located at r.BuildPath to the remote registry. -// Use helm to push, instead of crane, to better simulate the user workflow. -func (r *HelmChart) Push() error { - r.Image.Logger.Infof("Packaging helm chart: %s:%s", r.Image.Name, r.Image.Version) - parentPath := filepath.Dir(r.Image.BuildPath) - if _, err := r.Image.Shell.Helm("package", r.Image.BuildPath, "--destination", parentPath+string(filepath.Separator)); err != nil { - return fmt.Errorf("packaging helm chart: %w", err) - } - r.Image.Logger.Infof("Pushing helm chart: %s:%s", r.Image.Name, r.Image.Version) - chartFile := filepath.Join(parentPath, fmt.Sprintf("%s-%s.tgz", r.Image.Name, r.Image.Version)) - if _, err := r.Image.Shell.Helm("push", chartFile, r.Image.RepositoryOCI()); err != nil { - return fmt.Errorf("pushing helm chart: %w", err) - } - // Remove local tgz to allow re-push with the same name & version. - // No need to defer. The whole TmpDir will be deleted in test cleanup. - if err := os.Remove(chartFile); err != nil { - return fmt.Errorf("deleting local helm chart package: %w", err) - } - return nil -} diff --git a/e2e/nomostest/artifactregistry/image.go b/e2e/nomostest/artifactregistry/image.go deleted file mode 100644 index 5614a1c8c9..0000000000 --- a/e2e/nomostest/artifactregistry/image.go +++ /dev/null @@ -1,273 +0,0 @@ -// Copyright 2022 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package artifactregistry - -import ( - "errors" - "fmt" - "io/fs" - "os" - "path/filepath" - "strings" - - "github.com/ettle/strcase" - "kpt.dev/configsync/e2e" - "kpt.dev/configsync/e2e/nomostest" - "kpt.dev/configsync/e2e/nomostest/testlogger" - "kpt.dev/configsync/e2e/nomostest/testshell" - "sigs.k8s.io/kustomize/kyaml/copyutil" -) - -// DefaultLocation is the default location in which to host Artifact Registry -// repositories. In this case, `us`, which is a multi-region location. -const DefaultLocation = "us" - -// RegistryReaderAccountName is the name of the google service account -// with permission to read from Artifact Registry. -const RegistryReaderAccountName = "e2e-test-ar-reader" - -// RegistryReaderAccountEmail returns the email of the google service account -// with permission to read from Artifact Registry. -func RegistryReaderAccountEmail() string { - return fmt.Sprintf("%s@%s.iam.gserviceaccount.com", - RegistryReaderAccountName, *e2e.GCPProject) -} - -// SetupImage constructs a new Image for use during an e2e test, along with its -// repository. Image will be deleted when the test ends. -// Returns a reference to the Image object and any errors. -func SetupImage(nt *nomostest.NT, name, version string) (*Image, error) { - if name == "" { - return nil, fmt.Errorf("image name must not be empty") - } - - // Version will error out if it's empty or longer than 20 characters - if err := validateImageVersion(version); err != nil { - return nil, err - } - chart := &Image{ - Shell: nt.Shell, - Logger: nt.Logger, - Project: *e2e.GCPProject, - // Store images redundantly across regions in the US. - Location: DefaultLocation, - // Use cluster name to avoid overlap. - RepositoryName: fmt.Sprintf("config-sync-e2e-test--%s", nt.ClusterName), - // Use chart name to avoid overlap. - BuildPath: filepath.Join(nt.TmpDir, name), - // Use test name to avoid overlap. Truncate to 40 characters. - Name: generateImageName(nt, name), - Version: version, - } - nt.T.Cleanup(func() { - if err := chart.Delete(); err != nil { - nt.T.Errorf(err.Error()) - } - }) - if err := chart.CreateRepository(); err != nil { - return nil, err - } - if err := chart.CleanBuildPath(); err != nil { - return nil, err - } - // Setting up gcloud as auth helper for Docker _should_ work with both - // helm and crane, but in practice, sometimes the auth helper errors - // or hangs when pushing to a new repository. - // TODO: Test gcloud auth helper and crane/helm login separately - // if err := chart.ConfigureAuthHelper(); err != nil { - // return nil, err - // } - return chart, nil -} - -// Image represents a remote OCI image in Artifact Registry -type Image struct { - // Shell is a helper utility to execute shell commands in a test. - Shell *testshell.TestShell - - // Logger to write logs to - Logger *testlogger.TestLogger - - // Project in which to store the image - Project string - - // Location to store the image - Location string - - // RepositoryName in which to store the image - RepositoryName string - - // name is the name of the image - Name string - - // version is the version of the image - Version string - - // BuildPath is a local directory from which Image will read, package, and push the image from - BuildPath string -} - -// RegistryHost returns the domain of the artifact registry -func (r *Image) RegistryHost() string { - return fmt.Sprintf("%s-docker.pkg.dev", r.Location) -} - -// RepositoryAddress returns the domain and path to the chart repository -func (r *Image) RepositoryAddress() string { - return fmt.Sprintf("%s/%s/%s", r.RegistryHost(), r.Project, r.RepositoryName) -} - -// RepositoryOCI returns the repository address with the oci:// scheme prefix. -func (r *Image) RepositoryOCI() string { - return fmt.Sprintf("oci://%s", r.RepositoryAddress()) -} - -// Address returns the domain and path to the image -func (r *Image) Address() string { - return fmt.Sprintf("%s/%s", r.RepositoryAddress(), r.Name) -} - -// AddressWithTag returns the domain, path, name, and tag of the image -func (r *Image) AddressWithTag() string { - return fmt.Sprintf("%s/%s:%s", r.RepositoryAddress(), r.Name, r.Version) -} - -// CreateRepository uses gcloud to create the repository, if it doesn't exist. -func (r *Image) CreateRepository() error { - out, err := r.Shell.ExecWithDebug("gcloud", "artifacts", "repositories", - "describe", r.RepositoryName, - "--location", r.Location, - "--project", r.Project) - if err != nil { - if !strings.Contains(string(out), "NOT_FOUND") { - return fmt.Errorf("failed to describe image repository: %w", err) - } - // repository does not exist, continue with creation - } else { - // repository already exists, skip creation - return nil - } - - r.Logger.Info("Creating image repository") - _, err = r.Shell.ExecWithDebug("gcloud", "artifacts", "repositories", - "create", r.RepositoryName, - "--repository-format", "docker", - "--location", r.Location, - "--project", r.Project) - if err != nil { - return fmt.Errorf("failed to create image repository: %w", err) - } - return nil -} - -// ConfigureAuthHelper configures the local docker client to use gcloud for -// image registry authorization. Helm uses the docker config. -func (r *Image) ConfigureAuthHelper() error { - r.Logger.Info("Updating Docker config to use gcloud for auth") - if _, err := r.Shell.ExecWithDebug("gcloud", "auth", "configure-docker", r.RegistryHost()); err != nil { - return fmt.Errorf("failed to configure docker auth: %w", err) - } - return nil -} - -// CleanBuildPath creates the r.BuildPath if it doesn't exist and deletes all -// contents if it does. -func (r *Image) CleanBuildPath() error { - r.Logger.Infof("Cleaning build path: %s", r.BuildPath) - if _, err := os.Stat(r.BuildPath); err != nil { - if !errors.Is(err, fs.ErrNotExist) { - return err - } - } else { - if err := os.RemoveAll(r.BuildPath); err != nil { - return fmt.Errorf("deleting package directory: %v", err) - } - } - if err := os.MkdirAll(r.BuildPath, os.ModePerm); err != nil { - return fmt.Errorf("creating package directory: %v", err) - } - return nil -} - -// CopyLocalPackage accepts a local path to a package and recursively copies it -// to r.BuildPath. -func (r *Image) CopyLocalPackage(pkgPath string) error { - r.Logger.Infof("Copying package from test artifacts: %s", pkgPath) - if err := copyutil.CopyDir(pkgPath, r.BuildPath); err != nil { - return fmt.Errorf("copying package directory: %v", err) - } - return nil -} - -// SetName updates the local name of the image to the specified name with a -// random suffix -func (r *Image) SetName(nt *nomostest.NT, name string) { - name = generateImageName(nt, name) - r.Logger.Infof("Updating image name to %q", name) - r.Name = name -} - -// SetVersion updates the local version of the image to the specified tag with a -// random suffix -func (r *Image) SetVersion(version string) error { - if err := validateImageVersion(version); err != nil { - return err - } - r.Version = version - return nil -} - -// Delete the package from the remote registry, including all versions and tags. -func (r *Image) Delete() error { - r.Logger.Infof("Deleting image: %s", r.Name) - if _, err := r.Shell.ExecWithDebug("gcloud", "artifacts", "docker", "images", "delete", r.Address(), "--delete-tags", "--project", r.Project); err != nil { - return fmt.Errorf("deleting image from registry: %w", err) - } - return nil -} - -// creates a chart name from the chart name, test name, and timestamp. -// Result will be no more than 40 characters and can function as a k8s metadata.name. -// Chart name and version must be less than 63 characters combined. -func generateImageName(nt *nomostest.NT, chartName string) string { - testName := strcase.ToKebab(nt.T.Name()) - // The table-driven test sets the test name as a nested subdirectory, - // resulting in the Chart.yaml file not being located at the root directory, - // thus leading to a push error due to the missing Chart.yaml. - testName = strings.ReplaceAll(testName, "/", "-") - if len(chartName) > 20 { - chartName = chartName[:20] - chartName = strings.Trim(chartName, "-") - } - chartName = fmt.Sprintf("%s-%s", chartName, testName) - if len(chartName) > 40 { - chartName = chartName[:40] - chartName = strings.Trim(chartName, "-") - } - return chartName -} - -// validateImageVersion will validate if chart version string is not empty and -// is 20 characters maximum and can function as a k8s metadata.name. -// Chart name and version must be less than 63 characters combined. -func validateImageVersion(chartVersion string) error { - if chartVersion == "" { - return fmt.Errorf("image version must not be empty") - } - if len(chartVersion) > 20 { - return fmt.Errorf("chart version string %q should not exceed 20 characters", chartVersion) - } - return nil -} diff --git a/e2e/nomostest/artifactregistry/yaml.go b/e2e/nomostest/artifactregistry/yaml.go deleted file mode 100644 index be04627b34..0000000000 --- a/e2e/nomostest/artifactregistry/yaml.go +++ /dev/null @@ -1,54 +0,0 @@ -// Copyright 2023 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package artifactregistry - -import ( - "os" - "path/filepath" - - "github.com/pkg/errors" - "k8s.io/apimachinery/pkg/runtime" - jserializer "k8s.io/apimachinery/pkg/runtime/serializer/json" - "kpt.dev/configsync/e2e/nomostest" - "kpt.dev/configsync/pkg/syncer/reconcile" - "sigs.k8s.io/controller-runtime/pkg/client" -) - -// WriteObjectYAMLFile formats the objects as YAML and writes it to the -// specified file. Creates parent directories as needed. -func WriteObjectYAMLFile(nt *nomostest.NT, path string, obj client.Object) error { - fileMode := os.ModePerm - yamlSerializer := jserializer.NewYAMLSerializer(jserializer.DefaultMetaFactory, nt.Scheme, nt.Scheme) - var err error - uObj, err := reconcile.AsUnstructuredSanitized(obj) - if err != nil { - return errors.Wrap(err, "sanitizing object") - } - // Encode converts to json then yaml, to handle "omitempty" directives. - bytes, err := runtime.Encode(yamlSerializer, uObj) - if err != nil { - return errors.Wrap(err, "encoding object as yaml") - } - dirPath := filepath.Dir(path) - if dirPath != "." { - if err := os.MkdirAll(dirPath, fileMode); err != nil { - return errors.Wrapf(err, "making parent directories: %s", dirPath) - } - } - if err := os.WriteFile(path, bytes, fileMode); err != nil { - return errors.Wrapf(err, "writing file: %s", path) - } - return nil -} diff --git a/e2e/nomostest/config_sync.go b/e2e/nomostest/config_sync.go index 5a998dc42c..069392b3b3 100644 --- a/e2e/nomostest/config_sync.go +++ b/e2e/nomostest/config_sync.go @@ -36,6 +36,7 @@ import ( "kpt.dev/configsync/e2e/nomostest/gitproviders" "kpt.dev/configsync/e2e/nomostest/metrics" "kpt.dev/configsync/e2e/nomostest/ntopts" + "kpt.dev/configsync/e2e/nomostest/registryproviders" "kpt.dev/configsync/e2e/nomostest/taskgroup" "kpt.dev/configsync/e2e/nomostest/testpredicates" "kpt.dev/configsync/e2e/nomostest/testwatcher" @@ -69,6 +70,11 @@ const ( // clusterRoleName imitates a user-created (Cluster)Role for NS reconcilers. clusterRoleName = "cs-e2e" + + // helmPeriodOverride overrides the default helm-sync period for responsiveness, + // particularly during errors. The current default period is 1 hour even during + // a sync error. TODO: revisit after b/321790360 + helmPeriodOverride = 15 * time.Second ) var ( @@ -816,6 +822,114 @@ func RepoSyncObjectV1Beta1(nn types.NamespacedName, repoURL string, sourceFormat return rs } +// RootSyncObjectOCI returns a RootSync object that syncs the provided OCIImage. +func (nt *NT) RootSyncObjectOCI(name string, image *registryproviders.OCIImage) *v1beta1.RootSync { + rs := RootSyncObjectV1Beta1FromRootRepo(nt, name) + rs.Spec.SourceType = string(v1beta1.OciSource) + rs.Spec.Oci = &v1beta1.Oci{ + Image: image.FloatingBranchTag(), + Auth: configsync.AuthNone, + } + switch *e2e.OCIProvider { + case e2e.Local: + // Local provider requires CA cert because host is not well known + rs.Spec.Oci.CACertSecretRef = &v1beta1.SecretReference{ + Name: PublicCertSecretName(RegistrySyncSource), + } + case e2e.ArtifactRegistry: + // AR provider requires auth because the registry is private + rs.Spec.Oci.Auth = configsync.AuthGCPServiceAccount + rs.Spec.Oci.GCPServiceAccountEmail = registryproviders.ArtifactRegistryReaderEmail() + default: + nt.T.Fatalf("Unrecognized OCIProvider: %s", *e2e.OCIProvider) + } + return rs +} + +// RepoSyncObjectOCI returns a RepoSync object that syncs the provided OCIImage. +func (nt *NT) RepoSyncObjectOCI(nn types.NamespacedName, image *registryproviders.OCIImage) *v1beta1.RepoSync { + rs := RepoSyncObjectV1Beta1FromNonRootRepo(nt, nn) + rs.Spec.SourceType = string(v1beta1.OciSource) + rs.Spec.Oci = &v1beta1.Oci{ + Image: image.FloatingBranchTag(), + Auth: configsync.AuthNone, + } + switch *e2e.OCIProvider { + case e2e.Local: + // Local provider requires CA cert because host is not well known + rs.Spec.Oci.CACertSecretRef = &v1beta1.SecretReference{ + Name: PublicCertSecretName(RegistrySyncSource), + } + case e2e.ArtifactRegistry: + // AR provider requires auth because the registry is private + rs.Spec.Oci.Auth = configsync.AuthGCPServiceAccount + rs.Spec.Oci.GCPServiceAccountEmail = registryproviders.ArtifactRegistryReaderEmail() + default: + nt.T.Fatalf("Unrecognized OCIProvider: %s", *e2e.OCIProvider) + } + return rs +} + +// RootSyncObjectHelm returns a RootSync object that syncs the provided HelmPackage +func (nt *NT) RootSyncObjectHelm(name string, chart *registryproviders.HelmPackage) *v1beta1.RootSync { + rs := RootSyncObjectV1Beta1FromRootRepo(nt, name) + rs.Spec.SourceType = string(v1beta1.HelmSource) + rs.Spec.Helm = &v1beta1.HelmRootSync{ + HelmBase: v1beta1.HelmBase{ + Repo: nt.HelmProvider.SyncURL(chart.Name), + Chart: chart.Name, + Version: chart.Version, + Auth: configsync.AuthNone, + Period: metav1.Duration{Duration: helmPeriodOverride}, + }, + } + switch *e2e.HelmProvider { + case e2e.Local: + // Local provider requires CA cert because host is not well known + rs.Spec.Helm.CACertSecretRef = &v1beta1.SecretReference{ + Name: PublicCertSecretName(RegistrySyncSource), + } + case e2e.ArtifactRegistry: + // AR provider requires auth because the registry is private + rs.Spec.Helm.Auth = configsync.AuthGCPServiceAccount + rs.Spec.Helm.GCPServiceAccountEmail = registryproviders.ArtifactRegistryReaderEmail() + default: + nt.T.Fatalf("Unrecognized HelmProvider: %s", *e2e.OCIProvider) + } + return rs +} + +// RepoSyncObjectHelm returns a RepoSync object that syncs the provided HelmPackage +func (nt *NT) RepoSyncObjectHelm(nn types.NamespacedName, chart *registryproviders.HelmPackage) *v1beta1.RepoSync { + rs := RepoSyncObjectV1Beta1FromNonRootRepo(nt, nn) + rs.Spec.Git = nil + rs.Spec.SourceType = string(v1beta1.HelmSource) + rs.Spec.Helm = &v1beta1.HelmRepoSync{ + HelmBase: v1beta1.HelmBase{ + Repo: nt.HelmProvider.SyncURL(chart.Name), + Chart: chart.Name, + Version: chart.Version, + Auth: configsync.AuthNone, + // Override the default period of 1 hour for responsiveness + Period: metav1.Duration{Duration: helmPeriodOverride}, + }, + } + switch *e2e.HelmProvider { + case e2e.Local: + // Local provider requires CA cert because host is not well known + rs.Spec.Helm.CACertSecretRef = &v1beta1.SecretReference{ + Name: PublicCertSecretName(RegistrySyncSource), + } + case e2e.ArtifactRegistry: + // AR provider requires auth because the registry is private + rs.Spec.Helm.Auth = configsync.AuthGCPServiceAccount + rs.Spec.Helm.GCPServiceAccountEmail = registryproviders.ArtifactRegistryReaderEmail() + default: + nt.T.Fatalf("Unrecognized HelmProvider: %s", *e2e.OCIProvider) + } + return rs +} + // RepoSyncObjectV1Beta1FromNonRootRepo returns a v1beta1 RepoSync object which // uses a repo from nt.NonRootRepos. func RepoSyncObjectV1Beta1FromNonRootRepo(nt *NT, nn types.NamespacedName) *v1beta1.RepoSync { diff --git a/e2e/nomostest/gitproviders/repository.go b/e2e/nomostest/gitproviders/repository.go index 51cfd02c89..4cda5616df 100644 --- a/e2e/nomostest/gitproviders/repository.go +++ b/e2e/nomostest/gitproviders/repository.go @@ -499,6 +499,28 @@ func (g *Repository) Copy(sourceDir, destDir string) error { return err } +// UseHelmChart copies the files from the provided helm chart to the current +// repository. +func (g *Repository) UseHelmChart(chart string) error { + if err := g.RemoveAll(); err != nil { + return err + } + if err := g.Copy(fmt.Sprintf("../testdata/helm-charts/%s/.", chart), "."); err != nil { + return err + } + if _, err := g.Git("add", "."); err != nil { + return err + } + return nil +} + +// RemoveAll removes all files in the repository. +func (g *Repository) RemoveAll() error { + return g.BulkGit( + []string{"rm", "-f", "-r", "*"}, + ) +} + // Remove deletes `file` from the git repository. // If `file` is a directory, deletes the directory. // Returns error if the file does not exist. diff --git a/e2e/nomostest/new.go b/e2e/nomostest/new.go index bcf0a2219a..77f6374468 100644 --- a/e2e/nomostest/new.go +++ b/e2e/nomostest/new.go @@ -300,8 +300,8 @@ func FreshTestEnv(t nomostesting.NTB, opts *ntopts.New) *NT { RemoteRepositories: make(map[types.NamespacedName]*gitproviders.Repository), WebhookDisabled: &webhookDisabled, GitProvider: gitproviders.NewGitProvider(t, *e2e.GitProvider, logger), - OCIProvider: registryproviders.NewOCIProvider(*e2e.OCIProvider), - HelmProvider: registryproviders.NewHelmProvider(*e2e.HelmProvider), + OCIProvider: registryproviders.NewOCIProvider(*e2e.OCIProvider, opts.ClusterName, shell), + HelmProvider: registryproviders.NewHelmProvider(*e2e.HelmProvider, opts.ClusterName, shell), } // TODO: Try speeding up the reconciler and hydration polling. @@ -463,11 +463,7 @@ func setupTestCase(nt *NT, opts *ntopts.New) { // Cleanup all images before the port forward gets torn down. nt.T.Cleanup(func() { for _, image := range nt.ociImages { - remoteURL, err := nt.OCIProvider.PushURL(image.Name) - if err != nil { - nt.T.Fatal(err) - } - if err := image.Delete(remoteURL); err != nil { + if err := image.Delete(); err != nil { nt.T.Error(err) } } @@ -475,11 +471,7 @@ func setupTestCase(nt *NT, opts *ntopts.New) { }) nt.T.Cleanup(func() { for _, helmPackage := range nt.helmPackages { - remoteURL, err := nt.HelmProvider.PushURL(helmPackage.Name) - if err != nil { - nt.T.Fatal(err) - } - if err := helmPackage.Delete(remoteURL); err != nil { + if err := helmPackage.Delete(); err != nil { nt.T.Error(err) } } diff --git a/e2e/nomostest/ntopts/new.go b/e2e/nomostest/ntopts/new.go index 5f65898618..68eb6c0d95 100644 --- a/e2e/nomostest/ntopts/new.go +++ b/e2e/nomostest/ntopts/new.go @@ -101,6 +101,14 @@ func RequireKind(t testing.NTB) Opt { return func(opt *New) {} } +// RequireHelmArtifactRegistry requires the --helm-provider flag to be set to artifact_registry +func RequireHelmArtifactRegistry(t testing.NTB) Opt { + if *e2e.HelmProvider != e2e.ArtifactRegistry { + t.Skip("The --helm-provider flag must be set to `artifact_registry` to run this test.") + } + return func(opt *New) {} +} + // WithInitialCommit creates the initialCommit before the first sync func WithInitialCommit(initialCommit Commit) func(opt *New) { return func(opt *New) { diff --git a/e2e/nomostest/registry_server.go b/e2e/nomostest/registry_server.go index cd11d9c443..2f7c0c47e6 100644 --- a/e2e/nomostest/registry_server.go +++ b/e2e/nomostest/registry_server.go @@ -59,6 +59,20 @@ const ( ) func setupRegistry(nt *NT) error { + nt.T.Cleanup(func() { + if err := nt.OCIProvider.Teardown(); err != nil { + nt.T.Error(err) + } + if err := nt.HelmProvider.Teardown(); err != nil { + nt.T.Error(err) + } + }) + if err := nt.OCIProvider.Setup(); err != nil { + return err + } + if err := nt.HelmProvider.Setup(); err != nil { + return err + } if *e2e.OCIProvider == e2e.Local || *e2e.HelmProvider == e2e.Local { if err := nt.KubeClient.Create(fake.NamespaceObject(TestRegistryNamespace)); err != nil { return err @@ -129,14 +143,14 @@ func (nt *NT) BuildAndPushOCIImage(repository *gitproviders.Repository) (*regist // BuildAndPushHelmPackage uses the current file system state of the provided Repository // to build a helm package and push it to the current HelmProvider. The resulting // HelmPackage object can be used to set the spec.oci.image field on the RSync. -func (nt *NT) BuildAndPushHelmPackage(repository *gitproviders.Repository) (*registryproviders.HelmPackage, error) { +func (nt *NT) BuildAndPushHelmPackage(repository *gitproviders.Repository, options ...registryproviders.HelmOption) (*registryproviders.HelmPackage, error) { // Construct artifactDir using TmpDir. TmpDir is scoped to each test case and // cleaned up after the test. artifactDir := filepath.Join(nt.TmpDir, "artifacts", "helm", repository.Name) if err := os.MkdirAll(artifactDir, os.ModePerm); err != nil { return nil, fmt.Errorf("creating artifact dir: %w", err) } - helmPackage, err := registryproviders.BuildHelmPackage(artifactDir, nt.Shell, repository, nt.HelmProvider) + helmPackage, err := registryproviders.BuildHelmPackage(artifactDir, nt.Shell, repository, nt.HelmProvider, options...) if err != nil { return nil, err } diff --git a/e2e/nomostest/registryproviders/artifact_registry.go b/e2e/nomostest/registryproviders/artifact_registry.go new file mode 100644 index 0000000000..697d68d0cb --- /dev/null +++ b/e2e/nomostest/registryproviders/artifact_registry.go @@ -0,0 +1,211 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package registryproviders + +import ( + "fmt" + "strings" + + "kpt.dev/configsync/e2e" + "kpt.dev/configsync/e2e/nomostest/testshell" +) + +// DefaultLocation is the default location in which to host Artifact Registry +// repositories. In this case, `us`, which is a multi-region location. +const DefaultLocation = "us" + +// ArtifactRegistryReaderName is the name of the google service account +// with permission to read from Artifact Registry. +const ArtifactRegistryReaderName = "e2e-test-ar-reader" + +// ArtifactRegistryReaderEmail returns the email of the google service account +// with permission to read from Artifact Registry. +func ArtifactRegistryReaderEmail() string { + return fmt.Sprintf("%s@%s.iam.gserviceaccount.com", + ArtifactRegistryReaderName, *e2e.GCPProject) +} + +// ArtifactRegistryProvider is the provider type for Google Artifact Registry +type ArtifactRegistryProvider struct { + // project in which to store the image + project string + // location to store the image + location string + // repositoryName in which to store the image + repositoryName string + // repositorySuffix is a suffix appended after the repository name. This enables + // separating different artifact types within the repository + // (e.g. helm chart vs basic oci image). This is not strictly necessary but + // provides a guardrail to help prevent gotchas when mixing usage of helm-sync + // and oci-sync. + repositorySuffix string + // shell used for invoking CLI tools + shell *testshell.TestShell +} + +// Type returns the provider type. +func (a *ArtifactRegistryProvider) Type() string { + return e2e.ArtifactRegistry +} + +// Teardown preforms teardown +// Does nothing currently. We may want to have this delete the repository to +// minimize side effects. +func (a *ArtifactRegistryProvider) Teardown() error { + return nil +} + +// registryHost returns the domain of the artifact registry +func (a *ArtifactRegistryProvider) registryHost() string { + return fmt.Sprintf("%s-docker.pkg.dev", a.location) +} + +// repositoryAddress returns the domain and path to the chart repository +func (a *ArtifactRegistryProvider) repositoryAddress() string { + return fmt.Sprintf("%s/%s/%s/%s", a.registryHost(), a.project, a.repositoryName, a.repositorySuffix) +} + +// createRepository uses gcloud to create the repository, if it doesn't exist. +func (a *ArtifactRegistryProvider) createRepository() error { + out, err := a.shell.ExecWithDebug("gcloud", "artifacts", "repositories", + "describe", a.repositoryName, + "--location", a.location, + "--project", a.project) + if err != nil { + if !strings.Contains(string(out), "NOT_FOUND") { + return fmt.Errorf("failed to describe image repository: %w", err) + } + // repository does not exist, continue with creation + } else { + // repository already exists, skip creation + return nil + } + + _, err = a.shell.ExecWithDebug("gcloud", "artifacts", "repositories", + "create", a.repositoryName, + "--repository-format", "docker", + "--location", a.location, + "--project", a.project) + if err != nil { + return fmt.Errorf("failed to create image repository: %w", err) + } + return nil +} + +// deleteImage the package from the remote registry, including all versions and tags. +func (a *ArtifactRegistryProvider) deleteImage(name, digest string) error { + imageURL := fmt.Sprintf("%s/%s@%s", a.repositoryAddress(), name, digest) + if _, err := a.shell.ExecWithDebug("gcloud", "artifacts", "docker", "images", "delete", imageURL, "--delete-tags", "--project", a.project); err != nil { + return fmt.Errorf("deleting image from registry: %w", err) + } + return nil +} + +// ArtifactRegistryOCIProvider provides methods for interacting with the test registry-server +// using the oci-sync interface. +type ArtifactRegistryOCIProvider struct { + ArtifactRegistryProvider +} + +func (a *ArtifactRegistryOCIProvider) registryLogin() error { + var err error + authCmd := a.shell.Command("gcloud", "auth", "print-access-token") + loginCmd := a.shell.Command("crane", "auth", "login", + "-u", "oauth2accesstoken", "--password-stdin", + a.registryHost()) + loginCmd.Stdin, err = authCmd.StdoutPipe() + if err != nil { + return fmt.Errorf("creating STDOUT pipe: %w", err) + } + if err := loginCmd.Start(); err != nil { + return fmt.Errorf("starting login command: %w", err) + } + if err := authCmd.Run(); err != nil { + return fmt.Errorf("running print-access-token command: %w", err) + } + if err := loginCmd.Wait(); err != nil { + return fmt.Errorf("waiting for login command: %w", err) + } + return nil +} + +// Setup performs setup +func (a *ArtifactRegistryOCIProvider) Setup() error { + if err := a.registryLogin(); err != nil { + return err + } + return a.createRepository() +} + +// PushURL returns a URL for pushing images to the remote registry. +// name refers to the repo name in the format of / of RootSync|RepoSync. +func (a *ArtifactRegistryOCIProvider) PushURL(name string) (string, error) { + return fmt.Sprintf("%s/%s", a.repositoryAddress(), name), nil +} + +// SyncURL returns a URL for Config Sync to sync from using OCI. +// name refers to the repo name in the format of / of RootSync|RepoSync. +func (a *ArtifactRegistryOCIProvider) SyncURL(name string) string { + return fmt.Sprintf("%s/%s", a.repositoryAddress(), name) +} + +// ArtifactRegistryHelmProvider provides methods for interacting with the test registry-server +// using the helm-sync interface. +type ArtifactRegistryHelmProvider struct { + ArtifactRegistryProvider +} + +func (a *ArtifactRegistryHelmProvider) registryLogin() error { + var err error + authCmd := a.shell.Command("gcloud", "auth", "print-access-token") + loginCmd := a.shell.Command("helm", "registry", "login", + "-u", "oauth2accesstoken", "--password-stdin", + a.registryHost()) + loginCmd.Stdin, err = authCmd.StdoutPipe() + if err != nil { + return fmt.Errorf("creating STDOUT pipe: %w", err) + } + if err := loginCmd.Start(); err != nil { + return fmt.Errorf("starting login command: %w", err) + } + if err := authCmd.Run(); err != nil { + return fmt.Errorf("running print-access-token command: %w", err) + } + if err := loginCmd.Wait(); err != nil { + return fmt.Errorf("waiting for login command: %w", err) + } + return nil +} + +// Setup performs required setup for the helm AR provider. +// This requires setting up authentication for the helm CLI. +func (a *ArtifactRegistryHelmProvider) Setup() error { + if err := a.registryLogin(); err != nil { + return err + } + return a.createRepository() +} + +// PushURL returns a URL for pushing images to the remote registry. +// The name parameter is ignored because helm CLI appends the chart name to the image. +func (a *ArtifactRegistryHelmProvider) PushURL(_ string) (string, error) { + return a.SyncURL(""), nil +} + +// SyncURL returns a URL for Config Sync to sync from using OCI. +// The name parameter is ignored because helm CLI appends the chart name to the image. +func (a *ArtifactRegistryHelmProvider) SyncURL(_ string) string { + return fmt.Sprintf("oci://%s", a.repositoryAddress()) +} diff --git a/e2e/nomostest/registryproviders/helm.go b/e2e/nomostest/registryproviders/helm.go index 32e75e1492..e4eb3b87af 100644 --- a/e2e/nomostest/registryproviders/helm.go +++ b/e2e/nomostest/registryproviders/helm.go @@ -30,11 +30,30 @@ import ( // use an auto-incrementing index to create unique file names for tarballs var helmIndex int +type helmOptions struct { + version string +} + +// HelmOption is an optional parameter when building a helm chart. +type HelmOption func(options *helmOptions) + +// HelmChartVersion builds the chart with the specified version. +func HelmChartVersion(version string) func(options *helmOptions) { + return func(options *helmOptions) { + options.version = version + } +} + // BuildHelmPackage creates a new OCIImage object and associated tarball using the provided // Repository. The contents of the git repository will be bundled into a tarball // at the artifactDir. The resulting OCIImage object can be pushed to a remote // registry using its Push method. -func BuildHelmPackage(artifactDir string, shell *testshell.TestShell, repository *gitproviders.Repository, provider RegistryProvider) (*HelmPackage, error) { +func BuildHelmPackage(artifactDir string, shell *testshell.TestShell, repository *gitproviders.Repository, provider RegistryProvider, opts ...HelmOption) (*HelmPackage, error) { + options := helmOptions{} + for _, opt := range opts { + opt(&options) + } + commitHash, err := repository.Hash() if err != nil { return nil, fmt.Errorf("getting hash: %w", err) @@ -46,7 +65,10 @@ func BuildHelmPackage(artifactDir string, shell *testshell.TestShell, repository // replace / with - to avoid creating nested directories/paths name := strings.Replace(repository.Name, "/", "-", -1) // helm forces a semver, but we can append the branch to create a floating tag - version := fmt.Sprintf("v1.0.0-%s", branch) + version := options.version + if version == "" { + version = fmt.Sprintf("v1.0.0-%s", branch) + } // Use branch/hash for context and imageIndex to enforce file name uniqueness. // This avoids file name collision even if the test builds an image twice with // a dirty repo state. @@ -74,6 +96,7 @@ func BuildHelmPackage(artifactDir string, shell *testshell.TestShell, repository syncURL: provider.SyncURL(name), branch: branch, shell: shell, + provider: provider, } return helmPackage, nil } @@ -114,6 +137,7 @@ type HelmPackage struct { Version string Digest string shell *testshell.TestShell + provider RegistryProvider } // Push the image to the remote registry using the provided registry endpoint. @@ -132,11 +156,7 @@ func (h *HelmPackage) Push(registry string) error { } // Delete the image from the remote registry using the provided registry endpoint. -func (h *HelmPackage) Delete(registry string) error { - imageTag := fmt.Sprintf("%s/%s@%s", strings.TrimPrefix(registry, "oci://"), h.Name, h.Digest) - _, err := h.shell.ExecWithDebug("crane", "delete", imageTag) - if err != nil { - return fmt.Errorf("deleting image: %w", err) - } - return nil +func (h *HelmPackage) Delete() error { + // How to delete images varies by provider, so delegate deletion to the provider. + return h.provider.deleteImage(h.Name, h.Digest) } diff --git a/e2e/nomostest/registryproviders/image.go b/e2e/nomostest/registryproviders/image.go index eadd9550f5..eabd228cf1 100644 --- a/e2e/nomostest/registryproviders/image.go +++ b/e2e/nomostest/registryproviders/image.go @@ -54,6 +54,7 @@ func BuildImage(artifactDir string, shell *testshell.TestShell, repository *gitp syncURL: provider.SyncURL(repository.Name), branch: branch, shell: shell, + provider: provider, } return image, nil } @@ -69,6 +70,7 @@ type OCIImage struct { Name string Digest string shell *testshell.TestShell + provider RegistryProvider } // FloatingBranchTag returns the floating tag that initially points to this image. @@ -106,11 +108,7 @@ func (o *OCIImage) Push(registry string) error { } // Delete the image from the remote registry using the provided registry endpoint. -func (o *OCIImage) Delete(registry string) error { - imageTag := fmt.Sprintf("%s@%s", registry, o.Digest) - _, err := o.shell.ExecWithDebug("crane", "delete", imageTag) - if err != nil { - return fmt.Errorf("deleting image: %w", err) - } - return nil +func (o *OCIImage) Delete() error { + // How to delete images varies by provider, so delegate deletion to the provider. + return o.provider.deleteImage(o.Name, o.Digest) } diff --git a/e2e/nomostest/registryproviders/local.go b/e2e/nomostest/registryproviders/local.go index a65f94b8c2..51ce3f079c 100644 --- a/e2e/nomostest/registryproviders/local.go +++ b/e2e/nomostest/registryproviders/local.go @@ -19,6 +19,7 @@ import ( "kpt.dev/configsync/e2e" "kpt.dev/configsync/e2e/nomostest/portforwarder" + "kpt.dev/configsync/e2e/nomostest/testshell" ) // LocalProvider refers to the test registry-server running on the same test cluster. @@ -26,6 +27,18 @@ type LocalProvider struct { // PortForwarder is a port forwarder to the in-cluster registry server. // This is used to communicate from the tests to the in-cluster registry server. PortForwarder *portforwarder.PortForwarder + // shell is used for invoking command line utilities + shell *testshell.TestShell + // repositoryName is the name of the repository. For LocalProvider, + // this doesn't require explicit repository creation but is just part of the + // path. + repositoryName string + // repositorySuffix is a suffix appended after the repository name. This enables + // separating different artifact types within the repository + // (e.g. helm chart vs basic oci image). This is not strictly necessary but + // provides a guardrail to help prevent gotchas when mixing usage of helm-sync + // and oci-sync. + repositorySuffix string } // Type returns the provider type. @@ -33,6 +46,63 @@ func (l *LocalProvider) Type() string { return e2e.Local } +// Setup performs setup for LocalProvider (no-op) +func (l *LocalProvider) Setup() error { + return nil +} + +// Teardown performs teardown for LocalProvider (no-op) +func (l *LocalProvider) Teardown() error { + return nil +} + +// localAddress returns the local port forwarded address that proxies to the +// in-cluster git server. For use from the test framework to push to the registry. +func (l *LocalProvider) localAddress(name string) (string, error) { + if l.PortForwarder == nil { + return "", fmt.Errorf("PortForwarder must be set for LocalProvider.localAddress()") + } + port, err := l.PortForwarder.LocalPort() + if err != nil { + return "", err + } + return l.localAddressWithPort(port, name), nil +} + +// localAddressWithPort returns the local port forwarded address that proxies to the +// in-cluster git server. For use from the test framework to push to the registry. +// Accepts a port parameter for use from the on ready callback. +func (l *LocalProvider) localAddressWithPort(localPort int, name string) string { + address := fmt.Sprintf("localhost:%d/%s/%s", localPort, l.repositoryName, l.repositorySuffix) + if name != "" { + address = fmt.Sprintf("%s/%s", address, name) + } + return address +} + +// inClusterAddress returns the address of the registry service from within the +// cluster. Fit for use from the *-sync containers inside the cluster. +func (l *LocalProvider) inClusterAddress(name string) string { + address := fmt.Sprintf("test-registry-server.test-registry-system/%s/%s", l.repositoryName, l.repositorySuffix) + if name != "" { + address = fmt.Sprintf("%s/%s", address, name) + } + return address +} + +// deleteImage the package from the remote registry, including all versions and tags. +func (l *LocalProvider) deleteImage(name, digest string) error { + localAddress, err := l.localAddress(name) + if err != nil { + return err + } + imageTag := fmt.Sprintf("%s@%s", localAddress, digest) + if _, err := l.shell.ExecWithDebug("crane", "delete", imageTag); err != nil { + return fmt.Errorf("deleting image: %w", err) + } + return nil +} + // LocalOCIProvider provides methods for interacting with the test registry-server // using the oci-sync interface. type LocalOCIProvider struct { @@ -42,27 +112,20 @@ type LocalOCIProvider struct { // PushURL returns a URL for pushing images to the remote registry. // name refers to the repo name in the format of / of RootSync|RepoSync. func (l *LocalOCIProvider) PushURL(name string) (string, error) { - if l.PortForwarder == nil { - return "", fmt.Errorf("PortForwarder must be set for LocalProvider.OCIPushURL()") - } - port, err := l.PortForwarder.LocalPort() - if err != nil { - return "", err - } - return l.PushURLWithPort(port, name), nil + return l.localAddress(name) } // PushURLWithPort returns a URL for pushing images to the remote registry. // localPort refers to the local port the PortForwarder is listening on. // name refers to the repo name in the format of / of RootSync|RepoSync. func (l *LocalOCIProvider) PushURLWithPort(localPort int, name string) string { - return fmt.Sprintf("localhost:%d/oci/%s", localPort, name) + return l.localAddressWithPort(localPort, name) } // SyncURL returns a URL for Config Sync to sync from using OCI. // name refers to the repo name in the format of / of RootSync|RepoSync. func (l *LocalOCIProvider) SyncURL(name string) string { - return fmt.Sprintf("test-registry-server.test-registry-system/oci/%s", name) + return l.inClusterAddress(name) } // LocalHelmProvider provides methods for interacting with the test registry-server @@ -72,27 +135,24 @@ type LocalHelmProvider struct { } // PushURL returns a URL for pushing images to the remote registry. -// name refers to the repo name in the format of / of RootSync|RepoSync. -func (l *LocalHelmProvider) PushURL(name string) (string, error) { - if l.PortForwarder == nil { - return "", fmt.Errorf("PortForwarder must be set for LocalProvider.OCIPushURL()") - } - port, err := l.PortForwarder.LocalPort() +// The name parameter is ignored because helm CLI appends the chart name to the image. +func (l *LocalHelmProvider) PushURL(_ string) (string, error) { + localAddress, err := l.localAddress("") if err != nil { - return "", err + return localAddress, err } - return l.PushURLWithPort(port, name), nil + return fmt.Sprintf("oci://%s", localAddress), nil } // PushURLWithPort returns a URL for pushing images to the remote registry. // localPort refers to the local port the PortForwarder is listening on. -// name refers to the repo name in the format of / of RootSync|RepoSync. -func (l *LocalHelmProvider) PushURLWithPort(localPort int, name string) string { - return fmt.Sprintf("oci://localhost:%d/helm/%s", localPort, name) +// The name parameter is ignored because helm CLI appends the chart name to the image. +func (l *LocalHelmProvider) PushURLWithPort(localPort int, _ string) string { + return fmt.Sprintf("oci://%s", l.localAddressWithPort(localPort, "")) } // SyncURL returns a URL for Config Sync to sync from using helm. -// name refers to the repo name in the format of / of RootSync|RepoSync. -func (l *LocalHelmProvider) SyncURL(name string) string { - return fmt.Sprintf("oci://test-registry-server.test-registry-system/helm/%s", name) +// The name parameter is ignored because helm CLI appends the chart name to the image. +func (l *LocalHelmProvider) SyncURL(_ string) string { + return fmt.Sprintf("oci://%s", l.inClusterAddress("")) } diff --git a/e2e/nomostest/registryproviders/registry_provider.go b/e2e/nomostest/registryproviders/registry_provider.go index de628127c5..10b286bc1f 100644 --- a/e2e/nomostest/registryproviders/registry_provider.go +++ b/e2e/nomostest/registryproviders/registry_provider.go @@ -14,6 +14,13 @@ package registryproviders +import ( + "fmt" + + "kpt.dev/configsync/e2e" + "kpt.dev/configsync/e2e/nomostest/testshell" +) + // RegistryProvider is an interface for the remote Git providers. type RegistryProvider interface { Type() string @@ -28,24 +35,70 @@ type RegistryProvider interface { // SyncURL returns the registry URL for Config Sync to sync from using OCI. // name refers to the repo name in the format of / of RootSync|RepoSync. SyncURL(name string) string + // Setup is used to perform initialization of the RegistryProvider before the + // tests begin. + Setup() error + // Teardown is used to perform cleanup of the RegistryProvider after test + // completion. + Teardown() error + // deleteImage is used to delegate image deletion to the RegistryProvider + // implementation. This is needed because the delete interface varies by + // provider. + deleteImage(name, digest string) error } // NewOCIProvider creates a RegistryProvider for the specific OCI provider type. // This enables writing tests that can run against multiple types of registries. -func NewOCIProvider(provider string) RegistryProvider { +func NewOCIProvider(provider, repoName string, shell *testshell.TestShell) RegistryProvider { + repositoryName := fmt.Sprintf("config-sync-e2e-test--%s", repoName) + repositorySuffix := "oci" switch provider { - // TODO: Refactor existing Artifact Registry impl to this interface + case e2e.ArtifactRegistry: + return &ArtifactRegistryOCIProvider{ + ArtifactRegistryProvider{ + project: *e2e.GCPProject, + location: DefaultLocation, + // Use cluster name to avoid overlap. + repositoryName: repositoryName, + repositorySuffix: repositorySuffix, + shell: shell, + }, + } default: - return &LocalOCIProvider{} + return &LocalOCIProvider{ + LocalProvider{ + repositoryName: repositoryName, + repositorySuffix: repositorySuffix, + shell: shell, + }, + } } } // NewHelmProvider creates a RegistryProvider for the specific helm provider type. // This enables writing tests that can run against multiple types of registries. -func NewHelmProvider(provider string) RegistryProvider { +func NewHelmProvider(provider, repoName string, shell *testshell.TestShell) RegistryProvider { + repositoryName := fmt.Sprintf("config-sync-e2e-test--%s", repoName) + repositorySuffix := "helm" switch provider { - // TODO: Refactor existing Artifact Registry impl to this interface + case e2e.ArtifactRegistry: + return &ArtifactRegistryHelmProvider{ + ArtifactRegistryProvider{ + project: *e2e.GCPProject, + location: DefaultLocation, + // Use cluster name to avoid overlap. + repositoryName: repositoryName, + repositorySuffix: repositorySuffix, + shell: shell, + }, + } default: - return &LocalHelmProvider{} + return &LocalHelmProvider{ + LocalProvider{ + repositoryName: repositoryName, + repositorySuffix: repositorySuffix, + shell: shell, + }, + } } } diff --git a/e2e/nomostest/ssh.go b/e2e/nomostest/ssh.go index f18e46e07a..848b708b2b 100644 --- a/e2e/nomostest/ssh.go +++ b/e2e/nomostest/ssh.go @@ -327,7 +327,7 @@ func CreateNamespaceSecret(nt *NT, ns string) error { return err } } - if nt.OCIProvider.Type() == e2e.Local { + if nt.OCIProvider.Type() == e2e.Local || nt.HelmProvider.Type() == e2e.Local { caCertPathVal := nt.registryCACertPath if len(caCertPathVal) == 0 { caCertPathVal = caCertPath(nt, RegistrySyncSource) diff --git a/e2e/testcases/helm_sync_test.go b/e2e/testcases/helm_sync_test.go index c7c2e3e978..2a765d7a69 100644 --- a/e2e/testcases/helm_sync_test.go +++ b/e2e/testcases/helm_sync_test.go @@ -27,19 +27,19 @@ import ( "time" appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/utils/pointer" "kpt.dev/configsync/e2e" "kpt.dev/configsync/e2e/nomostest" - "kpt.dev/configsync/e2e/nomostest/artifactregistry" - "kpt.dev/configsync/e2e/nomostest/iam" "kpt.dev/configsync/e2e/nomostest/ntopts" "kpt.dev/configsync/e2e/nomostest/policy" + "kpt.dev/configsync/e2e/nomostest/registryproviders" nomostesting "kpt.dev/configsync/e2e/nomostest/testing" "kpt.dev/configsync/e2e/nomostest/testpredicates" - "kpt.dev/configsync/e2e/nomostest/workloadidentity" "kpt.dev/configsync/pkg/api/configsync" "kpt.dev/configsync/pkg/api/configsync/v1beta1" "kpt.dev/configsync/pkg/core" @@ -401,21 +401,23 @@ func TestHelmDefaultNamespace(t *testing.T) { nt := nomostest.New(t, nomostesting.SyncSource, ntopts.Unstructured, - ntopts.RequireGKE(t), ) - rs := fake.RootSyncObjectV1Beta1(configsync.RootSyncName) - - chart, err := artifactregistry.PushHelmChart(nt, privateSimpleHelmChart, privateSimpleHelmChartVersion) + nt.Must(nt.RootRepos[configsync.RootSyncName].UseHelmChart(privateSimpleHelmChart)) + chart, err := nt.BuildAndPushHelmPackage(nt.RootRepos[configsync.RootSyncName]) if err != nil { nt.T.Fatalf("failed to push helm chart: %v", err) } - nt.T.Log("Update RootSync to sync from a private Artifact Registry") - nt.MustMergePatch(rs, fmt.Sprintf(`{"spec": {"sourceType": "%s", "git": null, "helm": {"repo": "%s", "chart": "%s", "version": "%s", "auth": "gcpserviceaccount", "gcpServiceAccountEmail": "%s", "namespace": "", "deployNamespace": ""}}}`, - v1beta1.HelmSource, chart.Image.RepositoryOCI(), chart.Image.Name, chart.Image.Version, gsaARReaderEmail())) - err = nt.WatchForAllSyncs(nomostest.WithRootSha1Func(nomostest.HelmChartVersionShaFn(chart.Image.Version)), - nomostest.WithSyncDirectoryMap(map[types.NamespacedName]string{nomostest.DefaultRootRepoNamespacedName: chart.Image.Name})) + nt.T.Log("Update RootSync to sync from a helm chart") + // Switch from Git to Helm + rootSyncHelm := nt.RootSyncObjectHelm(configsync.RootSyncName, chart) + nt.T.Log("Manually update the RepoSync object to sync from helm") + if err := nt.KubeClient.Apply(rootSyncHelm); err != nil { + nt.T.Fatal(err) + } + err = nt.WatchForAllSyncs(nomostest.WithRootSha1Func(nomostest.HelmChartVersionShaFn(chart.Version)), + nomostest.WithSyncDirectoryMap(map[types.NamespacedName]string{nomostest.DefaultRootRepoNamespacedName: chart.Name})) if err != nil { nt.T.Fatal(err) } @@ -447,46 +449,48 @@ func TestHelmLatestVersion(t *testing.T) { nt := nomostest.New(t, nomostesting.WorkloadIdentity, ntopts.Unstructured, - ntopts.RequireGKE(t), ) - rs := fake.RootSyncObjectV1Beta1(configsync.RootSyncName) - chart, err := artifactregistry.PushHelmChart(nt, privateSimpleHelmChart, privateSimpleHelmChartVersion) + nt.Must(nt.RootRepos[configsync.RootSyncName].UseHelmChart(privateSimpleHelmChart)) + newVersion := "1.0.0" + chart, err := nt.BuildAndPushHelmPackage(nt.RootRepos[configsync.RootSyncName], registryproviders.HelmChartVersion(newVersion)) if err != nil { nt.T.Fatalf("failed to push helm chart: %v", err) } - nt.T.Log("Update RootSync to sync from a private Artifact Registry") - nt.MustMergePatch(rs, fmt.Sprintf(`{"spec": {"sourceType": "%s", "helm": {"chart": "%s", "repo": "%s", "version": "", "period": "5s", "auth": "gcpserviceaccount", "gcpServiceAccountEmail": "%s", "deployNamespace": "simple"}, "git": null}}`, - v1beta1.HelmSource, chart.Image.Name, chart.Image.RepositoryOCI(), gsaARReaderEmail())) + rs := nt.RootSyncObjectHelm(configsync.RootSyncName, chart) + rs.Spec.Helm.Version = "" + rs.Spec.Helm.DeployNamespace = "simple" + rs.Spec.Helm.Period = metav1.Duration{Duration: 5 * time.Second} + + nt.T.Log("Update RootSync to sync from a helm chart") + if err := nt.KubeClient.Apply(rs); err != nil { + nt.T.Fatal(err) + } if err = nt.Watcher.WatchObject(kinds.Deployment(), "deploy-default", "simple", - []testpredicates.Predicate{testpredicates.HasLabel("version", chart.Image.Version)}); err != nil { - nt.T.Error(err) + []testpredicates.Predicate{testpredicates.HasLabel("version", chart.Version)}); err != nil { + nt.T.Fatal(err) } // helm-sync automatically detects and updates to the new helm chart version - newVersion := "2.5.9" - if err := chart.SetVersion(newVersion); err != nil { - nt.T.Fatal(err) - } - if err := chart.Push(); err != nil { - nt.T.Fatal("failed to push helm chart update: %v", err) + newVersion = "2.5.9" + chart, err = nt.BuildAndPushHelmPackage(nt.RootRepos[configsync.RootSyncName], registryproviders.HelmChartVersion(newVersion)) + if err != nil { + nt.T.Fatalf("failed to push helm chart: %v", err) } if err = nt.Watcher.WatchObject(kinds.Deployment(), "deploy-default", "simple", - []testpredicates.Predicate{testpredicates.HasLabel("version", chart.Image.Version)}); err != nil { - nt.T.Error(err) + []testpredicates.Predicate{testpredicates.HasLabel("version", chart.Version)}); err != nil { + nt.T.Fatal(err) } newVersion = "3.0.0" - if err := chart.SetVersion(newVersion); err != nil { - nt.T.Fatal(err) - } - if err := chart.Push(); err != nil { - nt.T.Fatal("failed to push helm chart update: %v", err) + chart, err = nt.BuildAndPushHelmPackage(nt.RootRepos[configsync.RootSyncName], registryproviders.HelmChartVersion(newVersion)) + if err != nil { + nt.T.Fatalf("failed to push helm chart: %v", err) } if err = nt.Watcher.WatchObject(kinds.Deployment(), "deploy-default", "simple", - []testpredicates.Predicate{testpredicates.HasLabel("version", chart.Image.Version)}); err != nil { - nt.T.Error(err) + []testpredicates.Predicate{testpredicates.HasLabel("version", chart.Version)}); err != nil { + nt.T.Fatal(err) } } @@ -512,48 +516,45 @@ func TestHelmVersionRange(t *testing.T) { // TestHelmNamespaceRepo verifies RepoSync does not sync the helm chart with cluster-scoped resources. It also verifies that RepoSync can successfully // sync the namespace scoped resources, and assign the RepoSync namespace to these resources. -// This test will work only with following pre-requisites: +// Running this test on Artifact Registry has following pre-requisites: // Google service account `e2e-test-ar-reader@${GCP_PROJECT}.iam.gserviceaccount.com` is created with `roles/artifactregistry.reader` for accessing images in Artifact Registry. func TestHelmNamespaceRepo(t *testing.T) { repoSyncNN := nomostest.RepoSyncNN(testNs, configsync.RepoSyncName) - nt := nomostest.New(t, nomostesting.SyncSource, ntopts.RequireGKE(t), + nt := nomostest.New(t, nomostesting.SyncSource, ntopts.RepoSyncPermissions(policy.AllAdmin()), // NS reconciler manages a bunch of resources. ntopts.NamespaceRepo(repoSyncNN.Namespace, repoSyncNN.Name)) - nt.T.Log("Update RepoSync to sync from a public Helm Chart") - rs := nomostest.RepoSyncObjectV1Beta1FromNonRootRepo(nt, repoSyncNN) - rs.Spec.SourceType = string(v1beta1.HelmSource) - rs.Spec.Helm = &v1beta1.HelmRepoSync{HelmBase: v1beta1.HelmBase{ - Repo: publicHelmRepo, - Chart: publicHelmChart, - Auth: configsync.AuthNone, - Version: publicHelmChartVersion, - }} + + nt.T.Log("Push cluster-scoped resources to helm repo") + nt.Must(nt.NonRootRepos[repoSyncNN].Add("templates/ns.yaml", fake.NamespaceObject("foo-ns"))) + chart, err := nt.BuildAndPushHelmPackage(nt.NonRootRepos[repoSyncNN]) + if err != nil { + nt.T.Fatalf("failed to push helm chart: %v", err) + } + + nt.T.Log("Update RepoSync to sync from helm repo, should fail due to cluster-scope resource") + rs := nt.RepoSyncObjectHelm(repoSyncNN, chart) nt.Must(nt.RootRepos[configsync.RootSyncName].Add(nomostest.StructuredNSPath(repoSyncNN.Namespace, repoSyncNN.Name), rs)) - nt.Must(nt.RootRepos[configsync.RootSyncName].CommitAndPush("Update RepoSync to sync from a public Helm Chart with cluster-scoped type")) + nt.Must(nt.RootRepos[configsync.RootSyncName].CommitAndPush("Update RepoSync to sync from a Helm Chart with cluster-scoped resources")) nt.WaitForRepoSyncSourceError(repoSyncNN.Namespace, repoSyncNN.Name, nonhierarchical.BadScopeErrCode, "must be Namespace-scoped type") - chart, err := artifactregistry.PushHelmChart(nt, privateNSHelmChart, privateNSHelmChartVersion) + nt.T.Log("Remove cluster-scope resource from helm repo, add namespace-scope resource") + nt.Must(nt.NonRootRepos[repoSyncNN].Remove("templates/ns.yaml")) + nt.Must(nt.NonRootRepos[repoSyncNN].Add("templates/cm.yaml", fake.ConfigMapObject(core.Name("foo-cm")))) + validChart, err := nt.BuildAndPushHelmPackage(nt.NonRootRepos[repoSyncNN], registryproviders.HelmChartVersion("v1.1.0")) if err != nil { nt.T.Fatalf("failed to push helm chart: %v", err) } - - nt.T.Log("Update RepoSync to sync from a private Artifact Registry") - rs.Spec.Helm = &v1beta1.HelmRepoSync{HelmBase: v1beta1.HelmBase{ - Repo: chart.Image.RepositoryOCI(), - Chart: chart.Image.Name, - Auth: configsync.AuthGCPServiceAccount, - GCPServiceAccountEmail: gsaARReaderEmail(), - Version: chart.Image.Version, - ReleaseName: "test", - }} + rs = nt.RepoSyncObjectHelm(repoSyncNN, validChart) nt.Must(nt.RootRepos[configsync.RootSyncName].Add(nomostest.StructuredNSPath(repoSyncNN.Namespace, repoSyncNN.Name), rs)) - nt.Must(nt.RootRepos[configsync.RootSyncName].CommitAndPush("Update RepoSync to sync from a private Helm Chart without cluster scoped resources")) - err = nt.WatchForAllSyncs(nomostest.WithRepoSha1Func(nomostest.HelmChartVersionShaFn(chart.Image.Version)), nomostest.WithSyncDirectoryMap(map[types.NamespacedName]string{repoSyncNN: chart.Image.Name})) + nt.Must(nt.RootRepos[configsync.RootSyncName].CommitAndPush("Update RepoSync to sync from a Helm Chart with namespace-scoped resources")) + + err = nt.WatchForAllSyncs(nomostest.WithRepoSha1Func(nomostest.HelmChartVersionShaFn(validChart.Version)), + nomostest.WithSyncDirectoryMap(map[types.NamespacedName]string{repoSyncNN: validChart.Name})) if err != nil { nt.T.Fatal(err) } - if err := nt.Validate(rs.Spec.Helm.ReleaseName+"-"+chart.Image.Name, testNs, &appsv1.Deployment{}); err != nil { - nt.T.Error(err) + if err := nt.Validate("foo-cm", repoSyncNN.Namespace, &corev1.ConfigMap{}); err != nil { + nt.T.Fatal(err) } } @@ -564,30 +565,23 @@ func TestHelmNamespaceRepo(t *testing.T) { // Google service account `e2e-test-ar-reader@${GCP_PROJECT}.iam.gserviceaccount.com` is created with `roles/artifactregistry.reader` for accessing images in Artifact Registry. func TestHelmConfigMapNamespaceRepo(t *testing.T) { repoSyncNN := nomostest.RepoSyncNN(testNs, configsync.RepoSyncName) - nt := nomostest.New(t, nomostesting.SyncSource, ntopts.RequireGKE(t), + nt := nomostest.New(t, nomostesting.SyncSource, ntopts.RepoSyncPermissions(policy.AppsAdmin(), policy.CoreAdmin()), ntopts.NamespaceRepo(repoSyncNN.Namespace, repoSyncNN.Name)) - rs := nomostest.RepoSyncObjectV1Beta1FromNonRootRepo(nt, repoSyncNN) cmName := "helm-cm-ns-repo-1" - chart, err := artifactregistry.PushHelmChart(nt, privateNSHelmChart, privateNSHelmChartVersion) + nt.Must(nt.NonRootRepos[repoSyncNN].UseHelmChart(privateNSHelmChart)) + chart, err := nt.BuildAndPushHelmPackage(nt.NonRootRepos[repoSyncNN]) if err != nil { nt.T.Fatalf("failed to push helm chart: %v", err) } - nt.T.Log("Update RepoSync to sync from a private Artifact Registry") - rs.Spec.SourceType = string(v1beta1.HelmSource) - rs.Spec.Helm = &v1beta1.HelmRepoSync{HelmBase: v1beta1.HelmBase{ - Repo: chart.Image.RepositoryOCI(), - Chart: chart.Image.Name, - Auth: configsync.AuthGCPServiceAccount, - GCPServiceAccountEmail: gsaARReaderEmail(), - Version: chart.Image.Version, - ReleaseName: "test", - ValuesFileRefs: []v1beta1.ValuesFileRef{{Name: cmName, DataKey: "foo.yaml"}}, - }} + nt.T.Log("Update RepoSync to sync from a helm chart") + rs := nt.RepoSyncObjectHelm(repoSyncNN, chart) + rs.Spec.Helm.ReleaseName = "test" + rs.Spec.Helm.ValuesFileRefs = []v1beta1.ValuesFileRef{{Name: cmName, DataKey: "foo.yaml"}} nt.Must(nt.RootRepos[configsync.RootSyncName].Add(nomostest.StructuredNSPath(repoSyncNN.Namespace, repoSyncNN.Name), rs)) - nt.Must(nt.RootRepos[configsync.RootSyncName].CommitAndPush("Update RepoSync to sync from a private Helm Chart without cluster scoped resources")) + nt.Must(nt.RootRepos[configsync.RootSyncName].CommitAndPush("Update RepoSync to sync from a Helm Chart without cluster scoped resources")) nt.WaitForRepoSyncStalledError(rs.Namespace, rs.Name, "Validation", "KNV1061: RepoSyncs must reference valid ConfigMaps in spec.helm.valuesFileRefs: ConfigMap \"helm-cm-ns-repo-1\" not found") nt.T.Log("Create a ConfigMap that is not immutable (which should not be allowed)") @@ -653,12 +647,12 @@ func TestHelmConfigMapNamespaceRepo(t *testing.T) { nt.Must(nt.RootRepos[configsync.RootSyncName].CommitAndPush("Update RepoSync to reference new ConfigMap")) err = nt.WatchForAllSyncs( - nomostest.WithRepoSha1Func(nomostest.HelmChartVersionShaFn(chart.Image.Version)), - nomostest.WithSyncDirectoryMap(map[types.NamespacedName]string{repoSyncNN: chart.Image.Name})) + nomostest.WithRepoSha1Func(nomostest.HelmChartVersionShaFn(chart.Version)), + nomostest.WithSyncDirectoryMap(map[types.NamespacedName]string{repoSyncNN: chart.Name})) if err != nil { nt.T.Fatal(err) } - if err := nt.Validate(rs.Spec.Helm.ReleaseName+"-"+chart.Image.Name, testNs, &appsv1.Deployment{}, + if err := nt.Validate(rs.Spec.Helm.ReleaseName+"-"+chart.Name, testNs, &appsv1.Deployment{}, testpredicates.HasLabel("labelsTest", "foo")); err != nil { nt.T.Fatal(err) } @@ -671,9 +665,10 @@ func TestHelmConfigMapNamespaceRepo(t *testing.T) { // - `roles/artifactregistry.reader` for access image in Artifact Registry. func TestHelmGCENode(t *testing.T) { nt := nomostest.New(t, nomostesting.SyncSource, ntopts.Unstructured, - ntopts.RequireGKE(t), ntopts.GCENodeTest) + ntopts.RequireGKE(t), ntopts.GCENodeTest, ntopts.RequireHelmArtifactRegistry(t)) - chart, err := artifactregistry.PushHelmChart(nt, privateCoreDNSHelmChart, privateCoreDNSHelmChartVersion) + nt.Must(nt.RootRepos[configsync.RootSyncName].UseHelmChart(privateCoreDNSHelmChart)) + chart, err := nt.BuildAndPushHelmPackage(nt.RootRepos[configsync.RootSyncName]) if err != nil { nt.T.Fatalf("failed to push helm chart: %v", err) } @@ -681,14 +676,14 @@ func TestHelmGCENode(t *testing.T) { rs := fake.RootSyncObjectV1Beta1(configsync.RootSyncName) nt.T.Log("Update RootSync to sync from a private Artifact Registry") nt.MustMergePatch(rs, fmt.Sprintf(`{"spec": {"sourceType": "%s", "helm": {"repo": "%s", "chart": "%s", "auth": "gcenode", "version": "%s", "releaseName": "my-coredns", "namespace": "coredns"}, "git": null}}`, - v1beta1.HelmSource, chart.Image.RepositoryOCI(), chart.Image.Name, chart.Image.Version)) + v1beta1.HelmSource, nt.HelmProvider.SyncURL(chart.Name), chart.Name, chart.Version)) err = nt.WatchForAllSyncs( - nomostest.WithRootSha1Func(nomostest.HelmChartVersionShaFn(chart.Image.Version)), - nomostest.WithSyncDirectoryMap(map[types.NamespacedName]string{nomostest.DefaultRootRepoNamespacedName: chart.Image.Name})) + nomostest.WithRootSha1Func(nomostest.HelmChartVersionShaFn(chart.Version)), + nomostest.WithSyncDirectoryMap(map[types.NamespacedName]string{nomostest.DefaultRootRepoNamespacedName: chart.Name})) if err != nil { nt.T.Fatal(err) } - if err := nt.Validate(fmt.Sprintf("my-coredns-%s", chart.Image.Name), "coredns", &appsv1.Deployment{}, + if err := nt.Validate(fmt.Sprintf("my-coredns-%s", chart.Name), "coredns", &appsv1.Deployment{}, testpredicates.DeploymentContainerPullPolicyEquals("coredns", "IfNotPresent")); err != nil { nt.T.Error(err) } @@ -725,13 +720,14 @@ func TestHelmARTokenAuth(t *testing.T) { nomostesting.SyncSource, ntopts.Unstructured, ntopts.RequireGKE(t), + ntopts.RequireHelmArtifactRegistry(t), ) rs := fake.RootSyncObjectV1Beta1(configsync.RootSyncName) gsaKeySecretID := "config-sync-ci-ar-key" - gsaEmail := artifactregistry.RegistryReaderAccountEmail() - gsaName := artifactregistry.RegistryReaderAccountName + gsaEmail := registryproviders.ArtifactRegistryReaderEmail() + gsaName := registryproviders.ArtifactRegistryReaderName gsaKeyFilePath, err := fetchServiceAccountKeyFile(nt, *e2e.GCPProject, gsaKeySecretID, gsaEmail, gsaName) if err != nil { nt.T.Fatal(err) @@ -749,82 +745,49 @@ func TestHelmARTokenAuth(t *testing.T) { nt.MustKubectl("delete", "secret", "foo", "-n", configsync.ControllerNamespace, "--ignore-not-found") }) - chart, err := artifactregistry.PushHelmChart(nt, privateCoreDNSHelmChart, privateCoreDNSHelmChartVersion) + nt.Must(nt.RootRepos[configsync.RootSyncName].UseHelmChart(privateCoreDNSHelmChart)) + chart, err := nt.BuildAndPushHelmPackage(nt.RootRepos[configsync.RootSyncName]) if err != nil { nt.T.Fatalf("failed to push helm chart: %v", err) } nt.T.Log("Update RootSync to sync from a private Artifact Registry") nt.MustMergePatch(rs, fmt.Sprintf(`{"spec": {"sourceType": "%s", "git": null, "helm": {"repo": "%s", "chart": "%s", "auth": "token", "version": "%s", "releaseName": "my-coredns", "namespace": "coredns", "secretRef": {"name" : "foo"}}}}`, - v1beta1.HelmSource, chart.Image.RepositoryOCI(), chart.Image.Name, chart.Image.Version)) + v1beta1.HelmSource, nt.HelmProvider.SyncURL(chart.Name), chart.Name, chart.Version)) err = nt.WatchForAllSyncs( - nomostest.WithRootSha1Func(nomostest.HelmChartVersionShaFn(chart.Image.Version)), - nomostest.WithSyncDirectoryMap(map[types.NamespacedName]string{nomostest.DefaultRootRepoNamespacedName: chart.Image.Name})) + nomostest.WithRootSha1Func(nomostest.HelmChartVersionShaFn(chart.Version)), + nomostest.WithSyncDirectoryMap(map[types.NamespacedName]string{nomostest.DefaultRootRepoNamespacedName: chart.Name})) if err != nil { nt.T.Fatal(err) } - if err := nt.Validate(fmt.Sprintf("my-coredns-%s", chart.Image.Name), "coredns", &appsv1.Deployment{}); err != nil { + if err := nt.Validate(fmt.Sprintf("my-coredns-%s", chart.Name), "coredns", &appsv1.Deployment{}); err != nil { nt.T.Error(err) } } // TestHelmEmptyChart verifies Config Sync can apply an empty Helm chart. -// -// Requirements: -// 1. Helm auth https://cloud.google.com/artifact-registry/docs/helm/authentication -// 2. gcloud auth login OR gcloud auth activate-service-account ACCOUNT --key-file=KEY-FILE -// 3. Artifact Registry repo: us-docker.pkg.dev/${GCP_PROJECT}/config-sync-test-private -// 4. GKE cluster with Workload Identity -// 5. Google Service Account: e2e-test-ar-reader@${GCP_PROJECT}.iam.gserviceaccount.com -// 6. IAM for the GSA to read from the Artifact Registry repo -// 7. IAM for the test runner to write to Artifact Registry repo -// 8. gcloud & helm func TestHelmEmptyChart(t *testing.T) { nt := nomostest.New(t, nomostesting.SyncSource, ntopts.Unstructured, - ntopts.RequireGKE(t), ) - if err := workloadidentity.ValidateEnabled(nt); err != nil { - nt.T.Fatal(err) - } - gsaEmail := artifactregistry.RegistryReaderAccountEmail() - if err := iam.ValidateServiceAccountExists(nt, gsaEmail); err != nil { - nt.T.Fatal(err) - } - - chart, err := artifactregistry.PushHelmChart(nt, "empty", "v1.0.0") + chart, err := nt.BuildAndPushHelmPackage(nt.RootRepos[configsync.RootSyncName]) if err != nil { nt.T.Fatalf("failed to push helm chart: %v", err) } - nt.T.Logf("Updating RootSync to sync from the Helm chart: %s:%s", chart.Image.Name, chart.Image.Version) - rs := fake.RootSyncObjectV1Beta1(configsync.RootSyncName) - nt.MustMergePatch(rs, fmt.Sprintf(`{ - "spec": { - "sourceType": %q, - "helm": { - "repo": %q, - "chart": %q, - "version": %q, - "auth": "gcpserviceaccount", - "gcpServiceAccountEmail": %q - }, - "git": null - } - }`, - v1beta1.HelmSource, - chart.Image.RepositoryOCI(), - chart.Image.Name, - chart.Image.Version, - gsaEmail)) + nt.T.Logf("Updating RootSync to sync from the Helm chart: %s:%s", chart.Name, chart.Version) + rs := nt.RootSyncObjectHelm(configsync.RootSyncName, chart) + if err := nt.KubeClient.Apply(rs); err != nil { + nt.T.Fatal(err) + } // Validate that the chart syncs without error err = nt.WatchForAllSyncs( - nomostest.WithRootSha1Func(nomostest.HelmChartVersionShaFn(chart.Image.Version)), + nomostest.WithRootSha1Func(nomostest.HelmChartVersionShaFn(chart.Version)), nomostest.WithSyncDirectoryMap(map[types.NamespacedName]string{ - nomostest.DefaultRootRepoNamespacedName: chart.Image.Name, + nomostest.DefaultRootRepoNamespacedName: chart.Name, })) if err != nil { nt.T.Fatal(err) diff --git a/e2e/testcases/oci_sync_test.go b/e2e/testcases/oci_sync_test.go index 5a4b37e2a6..59609d7d78 100644 --- a/e2e/testcases/oci_sync_test.go +++ b/e2e/testcases/oci_sync_test.go @@ -34,12 +34,11 @@ import ( "kpt.dev/configsync/e2e/nomostest/workloadidentity" "kpt.dev/configsync/pkg/api/configsync" "kpt.dev/configsync/pkg/api/configsync/v1beta1" + "kpt.dev/configsync/pkg/core" "kpt.dev/configsync/pkg/declared" - "kpt.dev/configsync/pkg/importer/filesystem" "kpt.dev/configsync/pkg/kinds" "kpt.dev/configsync/pkg/metadata" "kpt.dev/configsync/pkg/testing/fake" - "sigs.k8s.io/cli-utils/pkg/common" "sigs.k8s.io/controller-runtime/pkg/client" ) @@ -55,9 +54,6 @@ const ( // publicARImage pulls the public OCI image by the default `latest` tag publicARImage = nomostesting.ConfigSyncTestPublicRegistry + "/kustomize-components" - - // bookinfoARImage pulls the public OCI image by the default `latest` tag - bookinfoARImage = nomostesting.ConfigSyncTestPublicRegistry + "/namespace-repo-bookinfo" ) // privateGCRImage pulls the private OCI image by tag @@ -89,7 +85,7 @@ func TestPublicOCI(t *testing.T) { nt.MustMergePatch(rs, fmt.Sprintf(`{"spec": {"sourceType": "%s", "oci": {"image": "%s", "auth": "none"}, "git": null}}`, v1beta1.OciSource, publicARImage)) err := nt.WatchForAllSyncs( - nomostest.WithRootSha1Func(imageDigestFunc(publicARImage)), + nomostest.WithRootSha1Func(imageDigestFuncByName(publicARImage)), nomostest.WithSyncDirectoryMap(map[types.NamespacedName]string{ nomostest.DefaultRootRepoNamespacedName: ".", })) @@ -102,7 +98,7 @@ func TestPublicOCI(t *testing.T) { nt.T.Logf("Update RootSync to sync %s from a public OCI image in GCR", tenant) nt.MustMergePatch(rs, fmt.Sprintf(`{"spec": {"oci": {"image": "%s", "dir": "%s"}}}`, publicGCRImage, tenant)) err = nt.WatchForAllSyncs( - nomostest.WithRootSha1Func(imageDigestFunc(publicGCRImage)), + nomostest.WithRootSha1Func(imageDigestFuncByName(publicGCRImage)), nomostest.WithSyncDirectoryMap(map[types.NamespacedName]string{ nomostest.DefaultRootRepoNamespacedName: "tenant-a", })) @@ -133,7 +129,7 @@ func TestGCENodeOCI(t *testing.T) { nt.MustMergePatch(rs, fmt.Sprintf(`{"spec": {"sourceType": "%s", "oci": {"dir": "%s", "image": "%s", "auth": "gcenode"}, "git": null}}`, v1beta1.OciSource, tenant, privateARImage())) err := nt.WatchForAllSyncs( - nomostest.WithRootSha1Func(imageDigestFunc(privateARImage())), + nomostest.WithRootSha1Func(imageDigestFuncByName(privateARImage())), nomostest.WithSyncDirectoryMap(map[types.NamespacedName]string{ nomostest.DefaultRootRepoNamespacedName: tenant, })) @@ -146,7 +142,7 @@ func TestGCENodeOCI(t *testing.T) { nt.T.Log("Update RootSync to sync from an OCI image in a private Google Container Registry") nt.MustMergePatch(rs, fmt.Sprintf(`{"spec": {"oci": {"image": "%s", "dir": "%s"}}}`, privateGCRImage(), tenant)) err = nt.WatchForAllSyncs( - nomostest.WithRootSha1Func(imageDigestFunc(privateGCRImage())), + nomostest.WithRootSha1Func(imageDigestFuncByName(privateGCRImage())), nomostest.WithSyncDirectoryMap(map[types.NamespacedName]string{ nomostest.DefaultRootRepoNamespacedName: tenant, })) @@ -156,122 +152,59 @@ func TestGCENodeOCI(t *testing.T) { kustomizecomponents.ValidateAllTenants(nt, string(declared.RootReconciler), "../base", tenant) } -func TestSwitchFromGitToOci(t *testing.T) { +func TestSwitchFromGitToOciCentralized(t *testing.T) { + namespace := testNs nt := nomostest.New(t, nomostesting.SyncSource, ntopts.Unstructured, + ntopts.NamespaceRepo(namespace, configsync.RepoSyncName), // bookinfo image contains RoleBinding // bookinfo repo contains ServiceAccount ntopts.RepoSyncPermissions(policy.RBACAdmin(), policy.CoreAdmin()), ) var err error - namespace := "bookinfo" - managerScope := string(declared.RootReconciler) // file path to the RepoSync config in the root repository. - repoSyncPath := "acme/reposync-bookinfo.yaml" + repoSyncPath := nomostest.StructuredNSPath(namespace, configsync.RepoSyncName) rsNN := types.NamespacedName{ Name: configsync.RepoSyncName, Namespace: namespace, } - repoSyncCRPath := "acme/cluster/cr.yaml" - repoSyncRBPath := fmt.Sprintf("acme/namespaces/%s/rb-%s.yaml", rsNN.Namespace, rsNN.Name) - rsCR := nt.RepoSyncClusterRole() - rsRB := nomostest.RepoSyncRoleBinding(rsNN) - repoSyncGit := nomostest.RepoSyncObjectV1Beta1(rsNN, "", filesystem.SourceFormatUnstructured) - repoSyncGit.Spec.Git = &v1beta1.Git{ - Repo: "https://github.com/config-sync-examples/namespace-repo-bookinfo", - Branch: "main", - Auth: configsync.AuthNone, - } - // Ensure the RoleBinding & ClusterRole are deleted after the RepoSync - if err := nomostest.SetDependencies(repoSyncGit, rsRB, rsCR); err != nil { + bookinfoSA := fake.ServiceAccountObject("bookinfo-sa", core.Namespace(namespace)) + bookinfoRole := fake.RoleObject(core.Name("bookinfo-admin")) + // OCI image will only contain the bookinfo-admin role + nt.Must(nt.NonRootRepos[rsNN].Add("acme/role.yaml", bookinfoRole)) + image, err := nt.BuildAndPushOCIImage(nt.NonRootRepos[rsNN]) + if err != nil { nt.T.Fatal(err) } + // Remote git branch will only contain the bookinfo-sa ServiceAccount + nt.Must(nt.NonRootRepos[rsNN].Remove("acme/role.yaml")) + nt.Must(nt.NonRootRepos[rsNN].Add("acme/sa.yaml", bookinfoSA)) + nt.Must(nt.NonRootRepos[rsNN].CommitAndPush("Add ServiceAccount")) - // To facilitate cleanup, add the implicit namespace explicitly. - // This ensures the implicit namespace is deleted in the right order. - t.Cleanup(func() { - implicitNs := &corev1.Namespace{} - implicitNs.Name = namespace - nt.Must(nt.RootRepos[configsync.RootSyncName].Add(repoSyncPath, implicitNs)) - nt.Must(nt.RootRepos[configsync.RootSyncName].CommitAndPush("add implicit namespace explicitly")) - if err := nt.WatchForAllSyncs(); err != nil { - nt.T.Fatal(err) - } - }) - - // Verify the central controlled configuration: switch from Git to OCI - // Backward compatibility check. Previously managed RepoSync objects without sourceType should still work. - nt.T.Log("Add the RepoSync object to the Root Repo") - nt.Must(nt.RootRepos[configsync.RootSyncName].Add(repoSyncPath, repoSyncGit)) - nt.Must(nt.RootRepos[configsync.RootSyncName].Add(repoSyncCRPath, rsCR)) - nt.Must(nt.RootRepos[configsync.RootSyncName].Add(repoSyncRBPath, rsRB)) - nt.Must(nt.RootRepos[configsync.RootSyncName].CommitAndPush("configure RepoSync in the root repository")) - // nt.WaitForRepoSyncs only waits for the root repo being synced because the reposync is not tracked by nt. if err := nt.WatchForAllSyncs(); err != nil { nt.T.Fatal(err) } - nt.T.Log("Verify an implicit namespace is created") - implicitNs := &corev1.Namespace{} - if err := nt.Validate(namespace, "", implicitNs, - testpredicates.HasAnnotation(metadata.ResourceManagerKey, managerScope), - testpredicates.HasAnnotation(common.LifecycleDeleteAnnotation, common.PreventDeletion)); err != nil { - nt.T.Error(err) - } - if err := nt.Validate(configsync.RepoSyncName, namespace, &v1beta1.RepoSync{}, isSourceType(v1beta1.GitSource)); err != nil { - nt.T.Error(err) - } - nt.T.Log("Verify the namespace objects are synced") - err = nt.WatchForSync(kinds.RepoSyncV1Beta1(), configsync.RepoSyncName, namespace, - nomostest.RemoteNsRepoSha1Fn, nomostest.RepoSyncHasStatusSyncCommit, nil) - if err != nil { - nt.T.Fatal(err) - } if err := nt.Validate("bookinfo-sa", namespace, &corev1.ServiceAccount{}, testpredicates.HasAnnotation(metadata.ResourceManagerKey, namespace)); err != nil { - nt.T.Error(err) + nt.T.Fatal(err) } - // To facilitate cleanup, revert the RepoSync to this known good state. - // This way, the RootSync finalizer will delete the RepoSync cleanly and in - // the right order. - t.Cleanup(func() { - nt.Must(nt.RootRepos[configsync.RootSyncName].Add(repoSyncPath, repoSyncGit)) - nt.Must(nt.RootRepos[configsync.RootSyncName].Add(repoSyncCRPath, rsCR)) - nt.Must(nt.RootRepos[configsync.RootSyncName].Add(repoSyncRBPath, rsRB)) - nt.Must(nt.RootRepos[configsync.RootSyncName].CommitAndPush("revert RepoSync")) - if err := nt.WatchForAllSyncs(); err != nil { - nt.T.Fatal(err) - } - }) - // Switch from Git to OCI nt.T.Log("Update the RepoSync object to sync from OCI") - repoSyncOCI := repoSyncGit.DeepCopy() - repoSyncOCI.Spec.Git = nil - repoSyncOCI.Spec.SourceType = string(v1beta1.OciSource) - imageURL := bookinfoARImage - repoSyncOCI.Spec.Oci = &v1beta1.Oci{ - Image: imageURL, - Auth: configsync.AuthNone, - } - // Ensure the RoleBinding & ClusterRole are deleted after the RepoSync - if err := nomostest.SetDependencies(repoSyncOCI, rsRB, rsCR); err != nil { - nt.T.Fatal(err) - } + repoSyncOCI := nt.RepoSyncObjectOCI(rsNN, image) nt.Must(nt.RootRepos[configsync.RootSyncName].Add(repoSyncPath, repoSyncOCI)) nt.Must(nt.RootRepos[configsync.RootSyncName].CommitAndPush("configure RepoSync to sync from OCI in the root repository")) - if err := nt.WatchForAllSyncs(); err != nil { + + if err := nt.WatchForAllSyncs( + nomostest.WithRepoSha1Func(imageDigestFuncByDigest(image.Digest)), + nomostest.WithSyncDirectoryMap(map[types.NamespacedName]string{ + rsNN: ".", + })); err != nil { nt.T.Fatal(err) } if err := nt.Validate(configsync.RepoSyncName, namespace, &v1beta1.RepoSync{}, isSourceType(v1beta1.OciSource)); err != nil { nt.T.Error(err) } - nt.T.Log("Verify the namespace objects are updated") - err = nt.WatchForSync(kinds.RepoSyncV1Beta1(), configsync.RepoSyncName, namespace, - imageDigestFunc(imageURL), nomostest.RepoSyncHasStatusSyncCommit, nil) - if err != nil { - nt.T.Fatal(err) - } if err := nt.Validate("bookinfo-admin", namespace, &rbacv1.Role{}, testpredicates.HasAnnotation(metadata.ResourceManagerKey, namespace)); err != nil { nt.T.Error(err) @@ -279,67 +212,70 @@ func TestSwitchFromGitToOci(t *testing.T) { if err := nt.ValidateNotFound("bookinfo-sa", namespace, &corev1.ServiceAccount{}); err != nil { nt.T.Error(err) } +} - // Verify the manual configuration: switch from Git to OCI - nt.T.Log("Remove RepoSync from the root repository") - nt.Must(nt.RootRepos[configsync.RootSyncName].Remove(repoSyncPath)) - nt.Must(nt.RootRepos[configsync.RootSyncName].CommitAndPush("remove RepoSync from the root repository")) - if err := nt.WatchForAllSyncs(); err != nil { - nt.T.Fatal(err) - } - nt.T.Log("Verify the RepoSync object doesn't exist") - if err := nt.ValidateNotFound(configsync.RepoSyncName, namespace, &v1beta1.RepoSync{}); err != nil { - nt.T.Error(err) +func TestSwitchFromGitToOciDelegated(t *testing.T) { + namespace := testNs + nt := nomostest.New(t, nomostesting.SyncSource, ntopts.Unstructured, + ntopts.WithDelegatedControl, + ntopts.NamespaceRepo(namespace, configsync.RepoSyncName), + // bookinfo image contains RoleBinding + // bookinfo repo contains ServiceAccount + ntopts.RepoSyncPermissions(policy.RBACAdmin(), policy.CoreAdmin()), + ) + rsNN := types.NamespacedName{ + Name: configsync.RepoSyncName, + Namespace: namespace, } - // Verify the default sourceType is set when not specified. - nt.T.Log("Revert the RepoSync object to sync from Git") - if err := nt.KubeClient.Create(repoSyncGit.DeepCopy()); err != nil { + + bookinfoSA := fake.ServiceAccountObject("bookinfo-sa", core.Namespace(namespace)) + bookinfoRole := fake.RoleObject(core.Name("bookinfo-admin")) + // OCI image will only contain the bookinfo-admin Role + nt.Must(nt.NonRootRepos[rsNN].Add("acme/role.yaml", bookinfoRole)) + image, err := nt.BuildAndPushOCIImage(nt.NonRootRepos[rsNN]) + if err != nil { nt.T.Fatal(err) } - if err := nt.Validate(configsync.RepoSyncName, namespace, &v1beta1.RepoSync{}, isSourceType(v1beta1.GitSource)); err != nil { - nt.T.Error(err) - } - nt.T.Log("Verify the namespace objects are synced") - err = nt.WatchForSync(kinds.RepoSyncV1Beta1(), configsync.RepoSyncName, namespace, - nomostest.RemoteNsRepoSha1Fn, nomostest.RepoSyncHasStatusSyncCommit, nil) - if err != nil { + // Remote git branch will only contain the bookinfo-sa ServiceAccount + nt.Must(nt.NonRootRepos[rsNN].Remove("acme/role.yaml")) + nt.Must(nt.NonRootRepos[rsNN].Add("acme/sa.yaml", bookinfoSA)) + nt.Must(nt.NonRootRepos[rsNN].CommitAndPush("Add ServiceAccount")) + + if err := nt.WatchForAllSyncs(); err != nil { nt.T.Fatal(err) } + + // Verify the manual configuration: switch from Git to OCI + // Verify the default sourceType is set when not specified. if err := nt.Validate("bookinfo-sa", namespace, &corev1.ServiceAccount{}, testpredicates.HasAnnotation(metadata.ResourceManagerKey, namespace)); err != nil { - nt.T.Error(err) + nt.T.Fatal(err) } if err := nt.ValidateNotFound("bookinfo-admin", namespace, &rbacv1.Role{}); err != nil { - nt.T.Error(err) + nt.T.Fatal(err) } // Switch from Git to OCI + repoSyncOCI := nt.RepoSyncObjectOCI(rsNN, image) nt.T.Log("Manually update the RepoSync object to sync from OCI") - nt.MustMergePatch(repoSyncOCI.DeepCopy(), fmt.Sprintf(`{"spec": {"sourceType": "%s", "oci": {"image": "%s", "auth": "%s"}, "helm": null, "git": null}}`, - v1beta1.OciSource, imageURL, configsync.AuthNone)) + if err := nt.KubeClient.Apply(repoSyncOCI); err != nil { + nt.T.Fatal(err) + } if err := nt.Validate(configsync.RepoSyncName, namespace, &v1beta1.RepoSync{}, isSourceType(v1beta1.OciSource)); err != nil { - nt.T.Error(err) + nt.T.Fatal(err) } nt.T.Log("Verify the namespace objects are synced") err = nt.WatchForSync(kinds.RepoSyncV1Beta1(), configsync.RepoSyncName, namespace, - imageDigestFunc(imageURL), nomostest.RepoSyncHasStatusSyncCommit, nil) + imageDigestFuncByDigest(image.Digest), nomostest.RepoSyncHasStatusSyncCommit, nil) if err != nil { nt.T.Fatal(err) } if err := nt.Validate("bookinfo-admin", namespace, &rbacv1.Role{}, testpredicates.HasAnnotation(metadata.ResourceManagerKey, namespace)); err != nil { - nt.T.Error(err) + nt.T.Fatal(err) } if err := nt.ValidateNotFound("bookinfo-sa", namespace, &corev1.ServiceAccount{}); err != nil { - nt.T.Error(err) + nt.T.Fatal(err) } - nt.T.Cleanup(func() { - // Reset RepoSync OCI config to be valid and managed by RootSync - nt.Must(nt.RootRepos[configsync.RootSyncName].Add(repoSyncPath, repoSyncOCI)) - nt.Must(nt.RootRepos[configsync.RootSyncName].CommitAndPush("re-configure RepoSync to sync from OCI in the root repository")) - if err := nt.WatchForAllSyncs(); err != nil { - nt.T.Fatal(err) - } - }) // Invalid cases rs := fake.RepoSyncObjectV1Beta1(namespace, configsync.RepoSyncName) @@ -442,9 +378,9 @@ func getImageDigest(nt *nomostest.NT, imageName string) (string, error) { return hex, nil } -// imageDigestFunc wraps getImageDigest to return a Sha1Func that caches the +// imageDigestFuncByName wraps getImageDigest to return a Sha1Func that caches the // image digest, to avoid polling the image registry unnecessarily. -func imageDigestFunc(imageName string) nomostest.Sha1Func { +func imageDigestFuncByName(imageName string) nomostest.Sha1Func { var cached bool var digest string var err error @@ -458,6 +394,14 @@ func imageDigestFunc(imageName string) nomostest.Sha1Func { } } +// imageDigestFuncByDigest uses the provided digest to return a valid Sha1Func +func imageDigestFuncByDigest(digest string) func(nt *nomostest.NT, nn types.NamespacedName) (string, error) { + return func(nt *nomostest.NT, nn types.NamespacedName) (string, error) { + // The RSync status does not include the sha256: prefix + return strings.TrimPrefix(digest, "sha256:"), nil + } +} + /* // archiveAndPushOCIImage tars and extracts (untar) image files to target directory. // The desired version or digest must be in the imageName, and the resolved image sha256 digest is returned. diff --git a/e2e/testcases/private_cert_secret_test.go b/e2e/testcases/private_cert_secret_test.go index 47608d1b1b..ef3b804237 100644 --- a/e2e/testcases/private_cert_secret_test.go +++ b/e2e/testcases/private_cert_secret_test.go @@ -17,7 +17,6 @@ package e2e import ( "encoding/base64" "fmt" - "strings" "testing" "time" @@ -413,10 +412,7 @@ func TestOCICACertSecretRefRootRepo(t *testing.T) { nt.T.Log("Add caCertSecretRef to RootSync") nt.MustMergePatch(rs, caCertSecretPatch(v1beta1.OciSource, caCertSecret)) err = nt.WatchForAllSyncs( - nomostest.WithRootSha1Func(func(nt *nomostest.NT, nn types.NamespacedName) (string, error) { - // the RSync status does not include the sha256: prefix - return strings.TrimPrefix(image.Digest, "sha256:"), nil - }), + nomostest.WithRootSha1Func(imageDigestFuncByDigest(image.Digest)), nomostest.WithSyncDirectoryMap(map[types.NamespacedName]string{ nomostest.DefaultRootRepoNamespacedName: ".", })) @@ -467,10 +463,7 @@ func TestOCICACertSecretRefNamespaceRepo(t *testing.T) { nomostest.StructuredNSPath(nn.Namespace, nn.Name), rs)) nt.Must(nt.RootRepos[configsync.RootSyncName].CommitAndPush("Set the CA cert for the RepoSync")) err = nt.WatchForAllSyncs( - nomostest.WithRepoSha1Func(func(nt *nomostest.NT, nn types.NamespacedName) (string, error) { - // the RSync status does not include the sha256: prefix - return strings.TrimPrefix(image.Digest, "sha256:"), nil - }), + nomostest.WithRepoSha1Func(imageDigestFuncByDigest(image.Digest)), nomostest.WithSyncDirectoryMap(map[types.NamespacedName]string{ nn: ".", })) diff --git a/e2e/testcases/stress_test.go b/e2e/testcases/stress_test.go index 1bf808eb55..4d9e41b03b 100644 --- a/e2e/testcases/stress_test.go +++ b/e2e/testcases/stress_test.go @@ -33,10 +33,10 @@ import ( "k8s.io/apimachinery/pkg/types" "k8s.io/utils/pointer" "kpt.dev/configsync/e2e/nomostest" - "kpt.dev/configsync/e2e/nomostest/artifactregistry" "kpt.dev/configsync/e2e/nomostest/gitproviders" "kpt.dev/configsync/e2e/nomostest/iam" "kpt.dev/configsync/e2e/nomostest/ntopts" + "kpt.dev/configsync/e2e/nomostest/registryproviders" nomostesting "kpt.dev/configsync/e2e/nomostest/testing" "kpt.dev/configsync/e2e/nomostest/testpredicates" "kpt.dev/configsync/e2e/nomostest/testresourcegroup" @@ -467,22 +467,17 @@ func TestStressMemoryUsageGit(t *testing.T) { // 7. IAM for the test runner to write to Artifact Registry repo func TestStressMemoryUsageOCI(t *testing.T) { nt := nomostest.New(t, nomostesting.WorkloadIdentity, ntopts.Unstructured, - ntopts.StressTest, ntopts.RequireGKE(t), + ntopts.StressTest, ntopts.WithReconcileTimeout(configsync.DefaultReconcileTimeout)) if err := workloadidentity.ValidateEnabled(nt); err != nil { nt.T.Fatal(err) } - gsaEmail := artifactregistry.RegistryReaderAccountEmail() + gsaEmail := registryproviders.ArtifactRegistryReaderEmail() if err := iam.ValidateServiceAccountExists(nt, gsaEmail); err != nil { nt.T.Fatal(err) } - image, err := artifactregistry.SetupCraneImage(nt, "anvil-set", "v1.0.0") - if err != nil { - nt.T.Fatal(err) - } - ns := "stress-test-ns" kind := "Anvil" @@ -490,12 +485,16 @@ func TestStressMemoryUsageOCI(t *testing.T) { crCount := 50 nt.T.Logf("Adding a test namespace, %d crds, and %d objects per crd", crdCount, crCount) - nt.Must(artifactregistry.WriteObjectYAMLFile(nt, fmt.Sprintf("%s/ns-%s.yaml", image.BuildPath, ns), fake.NamespaceObject(ns))) + // Remove safety namespace/clusterrole + nt.Must(nt.RootRepos[configsync.RootSyncName].RemoveAll()) + nt.Must(nt.RootRepos[configsync.RootSyncName].Add( + fmt.Sprintf("ns-%s.yaml", ns), fake.NamespaceObject(ns))) for i := 1; i <= crdCount; i++ { group := fmt.Sprintf("acme-%d.com", i) crd := fakeCRD(kind, group) - nt.Must(artifactregistry.WriteObjectYAMLFile(nt, fmt.Sprintf("%s/crd-%s.yaml", image.BuildPath, crd.Name), crd)) + nt.Must(nt.RootRepos[configsync.RootSyncName].Add( + fmt.Sprintf("crd-%s.yaml", crd.Name), crd)) gvk := schema.GroupVersionKind{ Group: group, Kind: kind, @@ -503,25 +502,27 @@ func TestStressMemoryUsageOCI(t *testing.T) { } for j := 1; j <= crCount; j++ { cr := fakeCR(fmt.Sprintf("%s-%d", strings.ToLower(kind), j), ns, gvk) - nt.Must(artifactregistry.WriteObjectYAMLFile(nt, fmt.Sprintf("%s/namespaces/%s/%s-%s.yaml", image.BuildPath, ns, crd.Name, cr.GetName()), cr)) + nt.Must(nt.RootRepos[configsync.RootSyncName].Add( + fmt.Sprintf("namespaces/%s/%s-%s.yaml", ns, crd.Name, cr.GetName()), cr)) } } + nt.Must(nt.RootRepos[configsync.RootSyncName].CommitAndPush("Add namespaces, CRDs, and CRs")) - if err := image.Push(); err != nil { + image, err := nt.BuildAndPushOCIImage(nt.RootRepos[configsync.RootSyncName]) + if err != nil { nt.T.Fatal(err) } - imageURL := image.AddressWithTag() - nt.T.Log("Update RootSync to sync from the OCI image in Artifact Registry") - rs := fake.RootSyncObjectV1Beta1(configsync.RootSyncName) - nt.MustMergePatch(rs, fmt.Sprintf(`{"spec": {"sourceType": "%s", "oci": {"dir": ".", "image": "%s", "auth": "gcpserviceaccount", "gcpServiceAccountEmail": "%s"}, "git": null}}`, - v1beta1.OciSource, imageURL, gsaEmail)) + rs := nt.RootSyncObjectOCI(configsync.RootSyncName, image) + if err := nt.KubeClient.Apply(rs); err != nil { + nt.T.Fatal(err) + } // Validate that the resources sync without the reconciler running out of // memory, getting OOMKilled, and crash looping. err = nt.WatchForAllSyncs( - nomostest.WithRootSha1Func(imageDigestFunc(imageURL)), + nomostest.WithRootSha1Func(imageDigestFuncByDigest(image.Digest)), nomostest.WithSyncDirectoryMap(map[types.NamespacedName]string{ nomostest.DefaultRootRepoNamespacedName: ".", })) @@ -542,25 +543,17 @@ func TestStressMemoryUsageOCI(t *testing.T) { client.InNamespace(ns)) } - emptyImage, err := artifactregistry.SetupCraneImage(nt, "empty", "v1.0.0") + nt.T.Log("Remove all files and publish an empty OCI image") + nt.Must(nt.RootRepos[configsync.RootSyncName].RemoveAll()) + emptyImage, err := nt.BuildAndPushOCIImage(nt.RootRepos[configsync.RootSyncName]) if err != nil { nt.T.Fatal(err) } - if err := emptyImage.Push(); err != nil { - nt.T.Fatal(err) - } - - emptyImageURL := emptyImage.AddressWithTag() - - nt.T.Log("Removing resources from OCI") - nt.MustMergePatch(rs, fmt.Sprintf(`{"spec": {"oci": {"image": "%s"}}}`, - emptyImageURL)) - // Validate that the resources sync without the reconciler running out of // memory, getting OOMKilled, and crash looping. err = nt.WatchForAllSyncs( - nomostest.WithRootSha1Func(imageDigestFunc(emptyImageURL)), + nomostest.WithRootSha1Func(imageDigestFuncByDigest(emptyImage.Digest)), nomostest.WithSyncDirectoryMap(map[types.NamespacedName]string{ nomostest.DefaultRootRepoNamespacedName: ".", })) @@ -584,20 +577,21 @@ func TestStressMemoryUsageOCI(t *testing.T) { // 8. gcloud & helm func TestStressMemoryUsageHelm(t *testing.T) { nt := nomostest.New(t, nomostesting.WorkloadIdentity, ntopts.Unstructured, - ntopts.StressTest, ntopts.RequireGKE(t), + ntopts.StressTest, ntopts.WithReconcileTimeout(30*time.Second)) if err := workloadidentity.ValidateEnabled(nt); err != nil { nt.T.Fatal(err) } - gsaEmail := artifactregistry.RegistryReaderAccountEmail() + gsaEmail := registryproviders.ArtifactRegistryReaderEmail() if err := iam.ValidateServiceAccountExists(nt, gsaEmail); err != nil { nt.T.Fatal(err) } - chart, err := artifactregistry.PushHelmChart(nt, "anvil-set", "v1.0.0") + nt.Must(nt.RootRepos[configsync.RootSyncName].UseHelmChart("anvil-set")) + chart, err := nt.BuildAndPushHelmPackage(nt.RootRepos[configsync.RootSyncName]) if err != nil { - nt.T.Fatal(err) + nt.T.Fatalf("failed to push helm chart: %v", err) } ns := "stress-test-ns" @@ -606,45 +600,24 @@ func TestStressMemoryUsageHelm(t *testing.T) { crCount := 50 kind := "Anvil" - nt.T.Logf("Updating RootSync to sync from the Helm chart: %s:%s", chart.Image.Name, chart.Image.Version) - rs := fake.RootSyncObjectV1Beta1(configsync.RootSyncName) - nt.MustMergePatch(rs, fmt.Sprintf(`{ - "spec": { - "sourceType": %q, - "helm": { - "repo": %q, - "chart": %q, - "version": %q, - "namespace": %q, - "auth": "gcpserviceaccount", - "gcpServiceAccountEmail": %q, - "values": { - "resources": %d, - "replicas": %d - } - }, - "git": null, - "override": { - "apiServerTimeout": "30s" - } - } - }`, - v1beta1.HelmSource, - chart.Image.RepositoryOCI(), - chart.Image.Name, - chart.Image.Version, - ns, - gsaEmail, - crdCount, - crCount)) + nt.T.Logf("Updating RootSync to sync from the Helm chart: %s:%s", chart.Name, chart.Version) + rs := nt.RootSyncObjectHelm(configsync.RootSyncName, chart) + rs.Spec.Helm.Namespace = ns + rs.Spec.Helm.Values = &apiextensionsv1.JSON{ + Raw: []byte(fmt.Sprintf(`{"resources": %d, "replicas": %d}`, crdCount, crCount)), + } + rs.Spec.SafeOverride().APIServerTimeout = &metav1.Duration{Duration: 30 * time.Second} + if err := nt.KubeClient.Apply(rs); err != nil { + nt.T.Fatal(err) + } // Validate that the resources sync without the reconciler running out of // memory, getting OOMKilled, and crash looping. err = nt.WatchForAllSyncs( nomostest.WithTimeout(5*time.Minute), - nomostest.WithRootSha1Func(nomostest.HelmChartVersionShaFn(chart.Image.Version)), + nomostest.WithRootSha1Func(nomostest.HelmChartVersionShaFn(chart.Version)), nomostest.WithSyncDirectoryMap(map[types.NamespacedName]string{ - nomostest.DefaultRootRepoNamespacedName: chart.Image.Name, + nomostest.DefaultRootRepoNamespacedName: chart.Name, })) if err != nil { nt.T.Fatal(err) @@ -663,22 +636,24 @@ func TestStressMemoryUsageHelm(t *testing.T) { client.InNamespace(ns)) } - emptyChart, err := artifactregistry.PushHelmChart(nt, "empty", "v1.0.0") + nt.Must(nt.RootRepos[configsync.RootSyncName].UseHelmChart("empty")) + emptyChart, err := nt.BuildAndPushHelmPackage(nt.RootRepos[configsync.RootSyncName], + registryproviders.HelmChartVersion("v1.1.0")) if err != nil { - nt.T.Fatal(err) + nt.T.Fatalf("failed to push helm chart: %v", err) } - nt.T.Logf("Updating RootSync to sync from the Helm chart: %s:%s", emptyChart.Image.Name, emptyChart.Image.Version) + nt.T.Logf("Updating RootSync to sync from the empty Helm chart: %s:%s", emptyChart.Name, emptyChart.Version) nt.MustMergePatch(rs, fmt.Sprintf(`{"spec": {"helm": {"chart": %q, "version": %q, "namespace": null, "values": null}}}`, - emptyChart.Image.Name, emptyChart.Image.Version)) + emptyChart.Name, emptyChart.Version)) // Validate that the resources sync without the reconciler running out of // memory, getting OOMKilled, and crash looping. err = nt.WatchForAllSyncs( nomostest.WithTimeout(5*time.Minute), - nomostest.WithRootSha1Func(nomostest.HelmChartVersionShaFn(emptyChart.Image.Version)), + nomostest.WithRootSha1Func(nomostest.HelmChartVersionShaFn(emptyChart.Version)), nomostest.WithSyncDirectoryMap(map[types.NamespacedName]string{ - nomostest.DefaultRootRepoNamespacedName: emptyChart.Image.Name, + nomostest.DefaultRootRepoNamespacedName: emptyChart.Name, })) if err != nil { nt.T.Fatal(err) diff --git a/e2e/testcases/workload_identity_test.go b/e2e/testcases/workload_identity_test.go index 55f82434ca..713bb7e02f 100644 --- a/e2e/testcases/workload_identity_test.go +++ b/e2e/testcases/workload_identity_test.go @@ -24,10 +24,10 @@ import ( "k8s.io/apimachinery/pkg/types" "kpt.dev/configsync/e2e" "kpt.dev/configsync/e2e/nomostest" - "kpt.dev/configsync/e2e/nomostest/artifactregistry" "kpt.dev/configsync/e2e/nomostest/iam" "kpt.dev/configsync/e2e/nomostest/kustomizecomponents" "kpt.dev/configsync/e2e/nomostest/ntopts" + "kpt.dev/configsync/e2e/nomostest/registryproviders" nomostesting "kpt.dev/configsync/e2e/nomostest/testing" "kpt.dev/configsync/e2e/nomostest/testpredicates" "kpt.dev/configsync/e2e/nomostest/testutils" @@ -50,16 +50,17 @@ import ( // 4. IAM permission and IAM policy binding are created. func TestWorkloadIdentity(t *testing.T) { testCases := []struct { - name string - fleetWITest bool - crossProject bool - sourceRepo string - sourceChart string - sourceVersion string - sourceType v1beta1.SourceType - gsaEmail string - rootCommitFn nomostest.Sha1Func - testKSAMigration bool + name string + fleetWITest bool + crossProject bool + sourceRepo string + sourceChart string + sourceVersion string + sourceType v1beta1.SourceType + gsaEmail string + rootCommitFn nomostest.Sha1Func + testKSAMigration bool + requireHelmArtifactRegistry bool }{ { name: "Authenticate to Git repo on CSR with GKE WI", @@ -95,7 +96,7 @@ func TestWorkloadIdentity(t *testing.T) { sourceRepo: privateARImage(), sourceType: v1beta1.OciSource, gsaEmail: gsaARReaderEmail(), - rootCommitFn: imageDigestFunc(privateARImage()), + rootCommitFn: imageDigestFuncByDigest(privateARImage()), testKSAMigration: true, }, { @@ -105,7 +106,7 @@ func TestWorkloadIdentity(t *testing.T) { sourceRepo: privateGCRImage(), sourceType: v1beta1.OciSource, gsaEmail: gsaGCRReaderEmail(), - rootCommitFn: imageDigestFunc(privateGCRImage()), + rootCommitFn: imageDigestFuncByDigest(privateGCRImage()), }, { name: "Authenticate to OCI image on AR with Fleet WI in the same project", @@ -114,7 +115,7 @@ func TestWorkloadIdentity(t *testing.T) { sourceRepo: privateARImage(), sourceType: v1beta1.OciSource, gsaEmail: gsaARReaderEmail(), - rootCommitFn: imageDigestFunc(privateARImage()), + rootCommitFn: imageDigestFuncByDigest(privateARImage()), testKSAMigration: true, }, { @@ -124,7 +125,7 @@ func TestWorkloadIdentity(t *testing.T) { sourceRepo: privateGCRImage(), sourceType: v1beta1.OciSource, gsaEmail: gsaGCRReaderEmail(), - rootCommitFn: imageDigestFunc(privateGCRImage()), + rootCommitFn: imageDigestFuncByDigest(privateGCRImage()), }, { name: "Authenticate to OCI image on AR with Fleet WI across project", @@ -133,7 +134,7 @@ func TestWorkloadIdentity(t *testing.T) { sourceRepo: privateARImage(), sourceType: v1beta1.OciSource, gsaEmail: gsaARReaderEmail(), - rootCommitFn: imageDigestFunc(privateARImage()), + rootCommitFn: imageDigestFuncByDigest(privateARImage()), testKSAMigration: true, }, { @@ -143,47 +144,54 @@ func TestWorkloadIdentity(t *testing.T) { sourceRepo: privateGCRImage(), sourceType: v1beta1.OciSource, gsaEmail: gsaGCRReaderEmail(), - rootCommitFn: imageDigestFunc(privateGCRImage()), + rootCommitFn: imageDigestFuncByDigest(privateGCRImage()), }, { - name: "Authenticate to Helm chart on AR with GKE WI", - fleetWITest: false, - crossProject: false, - sourceVersion: privateCoreDNSHelmChartVersion, - sourceChart: privateCoreDNSHelmChart, - sourceType: v1beta1.HelmSource, - gsaEmail: gsaARReaderEmail(), - rootCommitFn: nomostest.HelmChartVersionShaFn(privateCoreDNSHelmChartVersion), - testKSAMigration: true, + name: "Authenticate to Helm chart on AR with GKE WI", + fleetWITest: false, + crossProject: false, + sourceVersion: privateCoreDNSHelmChartVersion, + sourceChart: privateCoreDNSHelmChart, + sourceType: v1beta1.HelmSource, + gsaEmail: gsaARReaderEmail(), + rootCommitFn: nomostest.HelmChartVersionShaFn(privateCoreDNSHelmChartVersion), + testKSAMigration: true, + requireHelmArtifactRegistry: true, }, { - name: "Authenticate to Helm chart on AR with Fleet WI in the same project", - fleetWITest: true, - crossProject: false, - sourceVersion: privateCoreDNSHelmChartVersion, - sourceChart: privateCoreDNSHelmChart, - sourceType: v1beta1.HelmSource, - gsaEmail: gsaARReaderEmail(), - rootCommitFn: nomostest.HelmChartVersionShaFn(privateCoreDNSHelmChartVersion), - testKSAMigration: true, + name: "Authenticate to Helm chart on AR with Fleet WI in the same project", + fleetWITest: true, + crossProject: false, + sourceVersion: privateCoreDNSHelmChartVersion, + sourceChart: privateCoreDNSHelmChart, + sourceType: v1beta1.HelmSource, + gsaEmail: gsaARReaderEmail(), + rootCommitFn: nomostest.HelmChartVersionShaFn(privateCoreDNSHelmChartVersion), + testKSAMigration: true, + requireHelmArtifactRegistry: true, }, { - name: "Authenticate to Helm chart on AR with Fleet WI across project", - fleetWITest: true, - crossProject: true, - sourceVersion: privateCoreDNSHelmChartVersion, - sourceChart: privateCoreDNSHelmChart, - sourceType: v1beta1.HelmSource, - gsaEmail: gsaARReaderEmail(), - rootCommitFn: nomostest.HelmChartVersionShaFn(privateCoreDNSHelmChartVersion), - testKSAMigration: true, + name: "Authenticate to Helm chart on AR with Fleet WI across project", + fleetWITest: true, + crossProject: true, + sourceVersion: privateCoreDNSHelmChartVersion, + sourceChart: privateCoreDNSHelmChart, + sourceType: v1beta1.HelmSource, + gsaEmail: gsaARReaderEmail(), + rootCommitFn: nomostest.HelmChartVersionShaFn(privateCoreDNSHelmChartVersion), + testKSAMigration: true, + requireHelmArtifactRegistry: true, }, } for _, tc := range testCases { tc := tc t.Run(tc.name, func(t *testing.T) { - nt := nomostest.New(t, nomostesting.WorkloadIdentity, ntopts.Unstructured, ntopts.RequireGKE(t)) + opts := []ntopts.Opt{ntopts.Unstructured, ntopts.RequireGKE(t)} + if tc.requireHelmArtifactRegistry { + opts = append(opts, ntopts.RequireHelmArtifactRegistry(t)) + } + nt := nomostest.New(t, nomostesting.WorkloadIdentity, opts...) if err := workloadidentity.ValidateEnabled(nt); err != nil { nt.T.Fatal(err) } @@ -243,15 +251,16 @@ func TestWorkloadIdentity(t *testing.T) { // For helm charts, we need to push the chart to the AR before configuring the RootSync if tc.sourceType == v1beta1.HelmSource { - chart, err := artifactregistry.PushHelmChart(nt, tc.sourceChart, tc.sourceVersion) + nt.Must(nt.RootRepos[configsync.RootSyncName].UseHelmChart(tc.sourceChart)) + chart, err := nt.BuildAndPushHelmPackage(nt.RootRepos[configsync.RootSyncName], registryproviders.HelmChartVersion(tc.sourceVersion)) if err != nil { nt.T.Fatalf("failed to push helm chart: %v", err) } - tc.sourceRepo = chart.Image.RepositoryOCI() - tc.sourceChart = chart.Image.Name - tc.sourceVersion = chart.Image.Version - tc.rootCommitFn = nomostest.HelmChartVersionShaFn(chart.Image.Version) + tc.sourceRepo = nt.HelmProvider.SyncURL(chart.Name) + tc.sourceChart = chart.Name + tc.sourceVersion = chart.Version + tc.rootCommitFn = nomostest.HelmChartVersionShaFn(chart.Version) } // Reuse the RootSync instead of creating a new one so that testing resources can be cleaned up after the test. @@ -325,7 +334,9 @@ func migrateFromGSAtoKSA(nt *nomostest.NT, rs *v1beta1.RootSync, ksaRef types.Na sourceChart := "" if v1beta1.SourceType(rs.Spec.SourceType) == v1beta1.HelmSource { // Change the source repo to guarantee new resources can be reconciled with k8sserviceaccount - chart, err := artifactregistry.PushHelmChart(nt, privateSimpleHelmChart, privateSimpleHelmChartVersion) + nt.Must(nt.RootRepos[configsync.RootSyncName].UseHelmChart(privateSimpleHelmChart)) + chart, err := nt.BuildAndPushHelmPackage(nt.RootRepos[configsync.RootSyncName], registryproviders.HelmChartVersion(privateSimpleHelmChartVersion)) + if err != nil { nt.T.Fatalf("failed to push helm chart: %v", err) } @@ -339,11 +350,11 @@ func migrateFromGSAtoKSA(nt *nomostest.NT, rs *v1beta1.RootSync, ksaRef types.Na } } }`, - chart.Image.RepositoryOCI(), - chart.Image.Name, - chart.Image.Version)) - rootCommitFn = nomostest.HelmChartVersionShaFn(chart.Image.Version) - sourceChart = chart.Image.Name + nt.HelmProvider.SyncURL(chart.Name), + chart.Name, + chart.Version)) + rootCommitFn = nomostest.HelmChartVersionShaFn(chart.Version) + sourceChart = chart.Name } else { // The OCI image contains 3 tenants. The RootSync is only configured to sync // with the `tenant-a` directory. The migration flow changes the sync