From ad09b2e4d1a9e173ab8c95e560926a1173b83c3e Mon Sep 17 00:00:00 2001 From: Rasmus Rask Date: Tue, 15 Oct 2024 14:37:07 +0200 Subject: [PATCH] Add support for simulating `.Capabilities.KubeVersion` (#13) * Use KubeVersion from file, restructure, cleaner messages * Update argo-workflows simulation syntax * Add `metrics-server` test files * Update README with KubeVersion capability simulation * Cleanup feedback/messages * Remove built-in API version from metrics-server simulation * Minor updates to README * README update --- README.md | 45 ++++++---- flux-helm-diff.sh | 86 ++++++++++++------- .../base/metrics-server/helm.yaml | 46 ++++++++++ .../base/argo-workflows/helm.yaml | 8 +- .../base/metrics-server/helm.yaml | 42 +++++++++ 5 files changed, 175 insertions(+), 52 deletions(-) create mode 100644 test/base/infrastructure/base/metrics-server/helm.yaml create mode 100644 test/head/infrastructure/base/metrics-server/helm.yaml diff --git a/README.md b/README.md index 47cfd75..5380291 100644 --- a/README.md +++ b/README.md @@ -22,7 +22,7 @@ Combine with these awesome projects for maximum workflow smoothness: - [Inputs](#inputs) - [Outputs](#outputs) - [Usage](#usage) -- [Dry-running/Emulating API Capabilities](#dry-runningemulating-api-capabilities) +- [Simulating Capabilities](#simulating-capabilities) - [Example Output/PR comment](#example-outputpr-comment) - [Testing](#testing) @@ -188,20 +188,25 @@ Optionally, cause check to fail, if any Helm file failed to render: See [example-workflow.yaml](example-workflow.yaml) for coherent example. +## Simulating Capabilities -## Dry-running/Emulating API Capabilities +When installing, Helm can access the Kubernetes version and available Kubernetes APIs and versions, through "[Built-in Objects](https://helm.sh/docs/chart_template_guide/builtin_objects/)". -When installing, Helm can access the available Kubernetes APIs and versions, through "[Built-in Objects](https://helm.sh/docs/chart_template_guide/builtin_objects/)". +This enable charts to deploy custom resources, or tweak properties as needed, based on the features introduced in specific Kubernetes versions, or APIs offered in the cluster. -This enable charts to deploy custom resources, or tweak properties as needed, based on the APIs offered in the cluster. For example, starting with `argo-workflows` chart 0.41.0, the `ServiceMonitor` resource doesn't even get deployed, if [`.Capabilities.APIVersions.Has`](https://github.com/argoproj/argo-helm/blob/argo-workflows-0.41.0/charts/argo-workflows/templates/controller/workflow-controller-servicemonitor.yaml#L2) doesn't contain [`monitoring.coreos.com/v1`](https://github.com/argoproj/argo-helm/blob/argo-workflows-0.41.0/charts/argo-workflows/templates/_helpers.tpl#L200). +For example, starting with `argo-workflows` chart 0.41.0, the `ServiceMonitor` resource doesn't even get deployed, if [`.Capabilities.APIVersions.Has`](https://github.com/argoproj/argo-helm/blob/argo-workflows-0.41.0/charts/argo-workflows/templates/controller/workflow-controller-servicemonitor.yaml#L2) doesn't contain [`monitoring.coreos.com/v1`](https://github.com/argoproj/argo-helm/blob/argo-workflows-0.41.0/charts/argo-workflows/templates/_helpers.tpl#L200). -This does however also make it difficult to dry-run (using the `helm template` command), with no cluster access. As a workaround, it's possible to specify API version to be used when running the `template` command as commented YAML. The comments has to be the last in the file and must have the document start `---` above. Example: +By default, Helm will use the Kubernetes version included in the tool, and the built-in API versions in this version. These are printed in the beginning of the logs of the GitHub Action for clarity. + +If needed, the Kubernetes version used for templating can be overriden, and *additional* API versions can be specified in the `helm.yaml` file in the form of a small commented YAML document. The comments has to be the last in the file and must have the document start `---` marker above. Example: ```yaml --- -# helm-api-versions: -# - myapi/v0 -# - monitoring.coreos.com/v1 +# flux-helm-diff: +# kube-version: 1.30 +# api-versions: +# - myapi/v0 +# - monitoring.coreos.com/v1 ``` You can verify that the APIs are read correctly from the log output of the "Helm diff" step of the action: @@ -209,7 +214,8 @@ You can verify that the APIs are read correctly from the log output of the "Helm ``` Processing file "infrastructure/base/argo-workflows/helm.yaml" (...) -head API versions: myapi/v0,monitoring.coreos.com/v1 +head simulate Kube version: 1.30 +head API versions: myapi/v0,monitoring.coreos.com/v1 (...) ``` @@ -302,13 +308,14 @@ GITHUB_OUTPUT=debug.out HELM_FILES="${helm_files[@]}" TEST=1 ./flux-helm-diff.sh ### Testing files -| Name | Scenario tested | Expected output | -| ----------------------- | ---------------------------------------------------------------------------------------- | --------------------------------------------------------------- | -| `argo-workflows` | Read API from comment in helm file (otherwise `ServiceMonitor` resource is not rendered) | Diff shows change to `ServiceMonitor`, instead of being removed | -| `dcgm-exporter` | Chart added in `head` that doesn't exist in `base` | Diff shows entire rendered template as added | -| `metaflow` | Very non-standard way of publishing charts (not sure if should be supported) | TBD | -| `nvidia-device-plugin` | HelmRepository (using `https`), minor chart version bump | Diff (with potentially breaking `nodeAffinity`) | -| `podinfo` | Unknown repository type (`HelmTypoRepository`) | `Unrecognised repo type` | -| `weave-gitops-helm2oci` | Repository type changed from HelmRepository (type `oci`) to OCIRepository | No changes | -| `weave-gitops-helmrepo` | HelmRepository with type `oci` | Diff | -| `weave-gitops-ocirepo` | OCIRepository | Diff | +| Name | Scenario tested | Expected output | +| ----------------------- | ---------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------ | +| `argo-workflows` | Read API from comment in helm file (otherwise `ServiceMonitor` resource is not rendered) | Diff shows change to `ServiceMonitor`, instead of being removed | +| `dcgm-exporter` | Chart added in `head` that doesn't exist in `base` | Diff shows entire rendered template as added | +| `metaflow` | Very non-standard way of publishing charts (not sure if should be supported) | TBD | +| `metrics-server` | Old Kube Version simulated in `base`, using Helm's default in `head` | `PodDisruptionBudget` changes API version from `policy/v1beta1` to `policy/v1` | +| `nvidia-device-plugin` | HelmRepository (using `https`), minor chart version bump | Diff (with potentially breaking `nodeAffinity`) | +| `podinfo` | Unknown repository type (`HelmTypoRepository`) | `Unrecognised repo type` | +| `weave-gitops-helm2oci` | Repository type changed from HelmRepository (type `oci`) to OCIRepository | No changes | +| `weave-gitops-helmrepo` | HelmRepository with type `oci` | Diff | +| `weave-gitops-ocirepo` | OCIRepository | Diff | diff --git a/flux-helm-diff.sh b/flux-helm-diff.sh index 93360f0..96011f7 100755 --- a/flux-helm-diff.sh +++ b/flux-helm-diff.sh @@ -35,7 +35,7 @@ helm_template() { if [[ -z "${1}" ]]; then echo "Error: Need ${ref} file name to template" >&2 - output_msg CAUTION "Error: Need \`${ref}\` file name to template" + output_msg CAUTION "Error: Need \`${ref}\` file name to template." return 1 fi @@ -49,10 +49,10 @@ helm_template() { if [ ! -f "${helm_file}" ]; then echo "${ref} file \"${helm_file}\" not found" >&2 if [[ "${ref}" == "base" ]]; then - output_msg TIP "File \`${helm_file}\` not found in \`${ref}\` ref, looks like a new Helm file" + output_msg TIP "Helm file not found in \`${ref}\` ref, looks like a new Helm file." return else - output_msg CAUTION "Error: File \`${helm_file}\` not found in \`${ref}\` ref, cannot produce diff" + output_msg CAUTION "Error: Helm file not found in \`${ref}\` ref, cannot produce diff." return 1 fi fi @@ -104,14 +104,32 @@ helm_template() { else echo "Unrecognised ${ref} repo type" >&2 if [[ "${ref}" == "base" ]]; then - output_msg TIP "Unable to determine \`${ref}\` repo type, not rendering template" + output_msg TIP "Unable to determine \`${ref}\` repo type, not rendering template." return else - output_msg CAUTION "Error: Unable to determine \`${ref}\` repo type, cannot produce diff" + output_msg CAUTION "Error: Unable to determine \`${ref}\` repo type, cannot produce diff." return 1 fi fi + # Use Capabilities.KubeVersion + kube_version=$(yq '. | foot_comment' "${helm_file}" | yq '.flux-helm-diff.kube-version') + [[ "${kube_version}" == "null" ]] && kube_version="" + + # Use Capabilities.APIVersions + mapfile -t api_versions < <(yq '. | foot_comment' "${helm_file}" | yq '.flux-helm-diff.api-versions[]') + + # Let's see what information we got out about the chart... + echo "${ref} repo type: ${repo_type}" >&2 + echo "${ref} repo name: ${repo_name}" >&2 + echo "${ref} repo/chart URL: ${url}" >&2 + echo "${ref} chart name: ${chart_name}" >&2 + echo "${ref} chart version: ${chart_version}" >&2 + echo "${ref} release name: ${release_name}" >&2 + echo "${ref} release namespace: ${release_namespace}" >&2 + echo "${ref} simulate Kube version: ${kube_version}" >&2 + echo "${ref} simulate API versions: $(IFS=,; echo "${api_versions[*]}")" >&2 + # Download chart release_id="${chart_name}-${chart_version}" chart_temp_path="./tmp/${release_name}-${release_id}-${ref}" @@ -129,7 +147,7 @@ helm_template() { chart_file="${chart_temp_path}/${release_id}.tgz" helm pull ${helm_pull_args[@]} --version "${chart_version}" -d "${chart_temp_path}" || { echo "Helm failed to pull \"${url}\" to \"${chart_temp_path}\"" >&2 - output_msg CAUTION "Helm failed to pull \`${url}\` to \`${chart_temp_path}\`" + output_msg CAUTION "Helm failed to pull \`${url}\` to \`${chart_temp_path}\`." return 1 } else @@ -137,7 +155,7 @@ helm_template() { chart_file="${chart_temp_path}/asset.tgz" curl --no-progress-meter -Lo "${chart_file}" "${url}" || { echo "cURL failed to download \"${url}\" to \"${chart_file}\"" >&2 - output_msg CAUTION "cURL failed to download \`${url}\` to \`${chart_file}\`" + output_msg CAUTION "cURL failed to download \`${url}\` to \`${chart_file}\`." return 1 } fi @@ -152,33 +170,38 @@ helm_template() { chart_path="${chart_temp_path}/${chart_name}" fi - # Use Capabilities.APIVersions - mapfile -t api_versions < <(yq '. | foot_comment' "${helm_file}" | yq '.helm-api-versions[]') + # Check if chart is using .Capabilities.KubeVersion + grep -R --include='*.yaml' --include='*.yml' --include='*.tpl' ".Capabilities.KubeVersion" "${chart_temp_path}" > /dev/null && { + echo "${ref} uses \".Capabilities.KubeVersion\"" >&2 + if [[ ${#kube_version} -eq 0 ]]; then + output_msg TIP "Chart in \`${ref}\` ref uses [\`.Capabilities.KubeVersion\`](https://helm.sh/docs/chart_template_guide/builtin_objects/) but is not specifying a Kubernetes version to simulate." \ + "See [Dry-running/simulating Capabilities](https://github.com/marketplace/actions/flux-helm-diff#simulating-capabilities) for details. Will use Helm's default Kubernetes version: \`${helm_default_kube_version}\`." + helm_kube_version=() # treat as array, to avoid adding single-quotes + else + output_msg TIP "Chart in \`${ref}\` ref uses [\`.Capabilities.KubeVersion\`](https://helm.sh/docs/chart_template_guide/builtin_objects/) and is simulating the following Kubernetes version: \`${kube_version}\`" + helm_kube_version=(--kube-version "${kube_version}") # treat as array, to avoid adding single-quotes + fi + } - # Let's see what information we got out about the chart... - echo "${ref} repo type: ${repo_type}" >&2 - echo "${ref} repo name: ${repo_name}" >&2 - echo "${ref} repo/chart URL: ${url}" >&2 - echo "${ref} chart name: ${chart_name}" >&2 - echo "${ref} chart version: ${chart_version}" >&2 - echo "${ref} release name: ${release_name}" >&2 - echo "${ref} release namespace: ${release_namespace}" >&2 - echo "${ref} API versions: $(IFS=,; echo "${api_versions[*]}")" >&2 - - # Inspect rendered manifests + # Check if chart is using .Capabilities.APIVersions grep -R --include='*.yaml' --include='*.yml' --include='*.tpl' ".Capabilities.APIVersions" "${chart_temp_path}" > /dev/null && { - echo "Warning: Chart uses \".Capabilities.APIVersions\"" >&2 - output_msg WARNING "Chart in \`${ref}\` ref uses the \`.Capabilities.APIVersions\` [built-in template object](https://helm.sh/docs/chart_template_guide/builtin_objects/), which can affect rendered manifests." \ - "See [Flux Helm Diff read-me](https://github.com/marketplace/actions/flux-helm-diff#dry-runningemulating-api-capabilities) for details and workaround." + echo "${ref} uses \".Capabilities.APIVersions\"" >&2 + if [[ ${#api_versions[@]} -eq 0 ]]; then + output_msg IMPORTANT "Chart in \`${ref}\` ref uses [\`.Capabilities.APIVersions\`](https://helm.sh/docs/chart_template_guide/builtin_objects/) but is not specifying any APIs to simulate." \ + "Only the built-in API versions are available for templating. See [Dry-running/simulating Capabilities](https://github.com/marketplace/actions/flux-helm-diff#simulating-capabilities) for details and workaround." + else + output_msg TIP "Chart in \`${ref}\` ref uses [\`.Capabilities.APIVersions\`](https://helm.sh/docs/chart_template_guide/builtin_objects/) and is simulating the following APIs:" \ + "$(printf "\`%s\`\n" "${api_versions[@]}")" + fi } # Render template return_code=0 - template_out=$(helm template "${release_name}" "${chart_path}" -n "${release_namespace}" -f <(echo "${chart_values}") --api-versions "$(IFS=,; echo "${api_versions[*]}")" 2>&1) || return_code=$? + template_out=$(helm template "${release_name}" "${chart_path}" -n "${release_namespace}" -f <(echo "${chart_values}") --api-versions "$(IFS=,; echo "${api_versions[*]}")" ${helm_kube_version[@]} 2>&1) || return_code=$? rm -rf "${chart_temp_path}" if [ $return_code -ne 0 ]; then echo "$template_out" >&2 - output_msg CAUTION "Error rendering \`${ref}\` ref: \`${template_out}\`" + output_msg CAUTION "Error rendering \`${ref}\` ref: \`${template_out}\`." return 1 fi @@ -187,6 +210,13 @@ helm_template() { echo "$template_clean" } +# Get default Helm capabilities +helm_capabilities=$(helm template --repo https://abstrask.github.io/helm-charts helm-capabilities) || true +helm_default_kube_version=$(yq '.helmCapabilities.kubeVersion' <<< "${helm_capabilities}") +echo -e "\nHelm default Kubernetes version: ${helm_default_kube_version}" +kube_api_versions=$(yq '.helmCapabilities.apiVersions' <<< "${helm_capabilities}") +echo -e "\nHelm built-in API versions:\n${kube_api_versions}" + EOF=$(dd if=/dev/urandom bs=15 count=1 status=none | base64) echo "markdown<<$EOF" > "$GITHUB_OUTPUT" echo "## Flux Helm diffs" >> "$GITHUB_OUTPUT" @@ -196,11 +226,7 @@ for helm_file in "${helm_files[@]}"; do # Begin output echo -e "\nProcessing file \"$helm_file\"" - { - echo - echo "### \`${helm_file}\`" - echo - } >> "$GITHUB_OUTPUT" + echo -e "\n### \`${helm_file}\`\n" >> "$GITHUB_OUTPUT" # Template before base_out=$(helm_template "base/${helm_file}") || { diff --git a/test/base/infrastructure/base/metrics-server/helm.yaml b/test/base/infrastructure/base/metrics-server/helm.yaml new file mode 100644 index 0000000..a94b3c0 --- /dev/null +++ b/test/base/infrastructure/base/metrics-server/helm.yaml @@ -0,0 +1,46 @@ +--- +apiVersion: source.toolkit.fluxcd.io/v1 +kind: HelmRepository +metadata: + name: metrics-server + namespace: metrics-server +spec: + interval: 15m + url: https://kubernetes-sigs.github.io/metrics-server/ + +--- +apiVersion: helm.toolkit.fluxcd.io/v2 +kind: HelmRelease +metadata: + name: metrics-server + namespace: metrics-server +spec: + interval: 5m + dependsOn: + - name: aws-lb-controller + namespace: aws-lb-controller + targetNamespace: metrics-server + chart: + spec: + chart: metrics-server + version: "3.12.2" + sourceRef: + kind: HelmRepository + name: metrics-server + interval: 15m + install: + skipCRDs: true + values: + podDisruptionBudget: + enabled: true + metrics: + enabled: true + serviceMonitor: + enabled: true + interval: "120s" + additionalLabels: + instance: primary + +--- +# flux-helm-diff: +# kube-version: 1.20 diff --git a/test/head/infrastructure/base/argo-workflows/helm.yaml b/test/head/infrastructure/base/argo-workflows/helm.yaml index ff277ed..be45417 100644 --- a/test/head/infrastructure/base/argo-workflows/helm.yaml +++ b/test/head/infrastructure/base/argo-workflows/helm.yaml @@ -91,6 +91,8 @@ spec: name: argo-workflows --- -# helm-api-versions: -# - myapi/v0 -# - monitoring.coreos.com/v1 +# flux-helm-diff: +# kube-version: 1.30 +# api-versions: +# - myapi/v0 +# - monitoring.coreos.com/v1 diff --git a/test/head/infrastructure/base/metrics-server/helm.yaml b/test/head/infrastructure/base/metrics-server/helm.yaml new file mode 100644 index 0000000..db31aa9 --- /dev/null +++ b/test/head/infrastructure/base/metrics-server/helm.yaml @@ -0,0 +1,42 @@ +--- +apiVersion: source.toolkit.fluxcd.io/v1 +kind: HelmRepository +metadata: + name: metrics-server + namespace: metrics-server +spec: + interval: 15m + url: https://kubernetes-sigs.github.io/metrics-server/ + +--- +apiVersion: helm.toolkit.fluxcd.io/v2 +kind: HelmRelease +metadata: + name: metrics-server + namespace: metrics-server +spec: + interval: 5m + dependsOn: + - name: aws-lb-controller + namespace: aws-lb-controller + targetNamespace: metrics-server + chart: + spec: + chart: metrics-server + version: "3.12.2" + sourceRef: + kind: HelmRepository + name: metrics-server + interval: 15m + install: + skipCRDs: true + values: + podDisruptionBudget: + enabled: true + metrics: + enabled: true + serviceMonitor: + enabled: true + interval: "120s" + additionalLabels: + instance: primary