[Index] Sync index.yaml with OCI releases #56
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: '[Index] Sync index.yaml with OCI releases' | |
| on: | |
| schedule: | |
| - cron: "*/30 * * * *" | |
| # Remove all permissions by default. | |
| permissions: {} | |
| jobs: | |
| find-new-releases: | |
| runs-on: ubuntu-latest | |
| name: Find new releases | |
| outputs: | |
| new-releases: ${{ steps.get-new-releases.outputs.new-releases }} | |
| permissions: | |
| contents: read | |
| steps: | |
| - id: checkout-repo | |
| name: Checkout repo | |
| uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 | |
| with: | |
| ref: index | |
| path: index | |
| - uses: oras-project/setup-oras@9c92598691bfef1424de2f8fae81941568f5889c | |
| - id: get-oci-index | |
| name: Get OCI index | |
| run: | | |
| oras pull registry-1.docker.io/bitnamicharts/charts-index:latest | |
| cat charts-index.json | yq -P | yq eval '. | .entries[] |= .versions' > ./oci_index.yaml | |
| - id: get-charts-index | |
| name: Get Charts index | |
| run: | | |
| cp index/bitnami/index.yaml ./charts_index.yaml | |
| - id: merge | |
| name: Generate merged index | |
| run: | | |
| yq eval-all '. as $item ireduce ({}; . *+ $item )' charts_index.yaml oci_index.yaml > duplicates_index.yaml | |
| yq eval '.entries[] |= unique_by(.name + .version)' duplicates_index.yaml > merged_index.yaml | |
| - id: get-new-releases | |
| name: Find new versions | |
| run: | | |
| yq eval '.entries[][] | .name + ":" + .version' charts_index.yaml |sort| uniq > charts_index_releases | |
| yq eval '.entries[][] | .name + ":" + .version' merged_index.yaml | sort| uniq > merged_index_releases | |
| new_releases="$(comm -13 charts_index_releases merged_index_releases | tr "\n" " " | sed 's/ $//')" | |
| if [ -n "${new_releases}" ]; then | |
| echo "Found new releases: ${new_releases}" | |
| else | |
| echo "No new releases detected" | |
| fi | |
| echo "new-releases=$new_releases" >> $GITHUB_OUTPUT | |
| update-index-and-promotions: | |
| runs-on: ubuntu-latest | |
| needs: | |
| - find-new-releases | |
| name: Update index and push promotions | |
| if: ${{ needs.find-new-releases.outputs.new-releases != '' }} | |
| steps: | |
| - name: Install helm | |
| run: | | |
| HELM_TARBALL="helm-v3.8.1-linux-amd64.tar.gz" | |
| curl -SsLfO "https://get.helm.sh/${HELM_TARBALL}" && sudo tar xf "$HELM_TARBALL" --strip-components 1 -C /usr/local/bin | |
| # Install file plugin | |
| helm plugin add https://github.com/zoobab/helm_file_repo | |
| - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 | |
| with: | |
| ref: 'index' | |
| path: index | |
| # The token is persisted in the local git config and enables scripts to run authenticated git commands. | |
| token: ${{ secrets.BITNAMI_BOT_TOKEN }} | |
| - id: update-index-and-promotions | |
| name: Pull charts and update index | |
| env: | |
| AWS_ACCESS_KEY_ID: ${{ secrets.AWS_PROMOTION_ACCESS_KEY_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_PROMOTION_SECRET_ACCESS_KEY }} | |
| AWS_PROMOTION_BUCKET: ${{ secrets.AWS_PROMOTION_BUCKET }} | |
| AWS_MAX_ATTEMPTS: 3 | |
| AWS_DEFAULT_REGION: us-east-1 | |
| NEW_RELEASES: ${{ needs.find-new-releases.outputs.new-releases }} | |
| run: | | |
| # Promotions template | |
| promotion_json_template=$(cat << "EOF" | |
| { | |
| "platform_id": $platform_id, | |
| "application": $chart_name, | |
| "external_id": "\($chart_name):\($chart_version)", | |
| "version": $app_version, | |
| "bundled_os_version": $bundled_os_version, | |
| "properties": { | |
| "chart_url": "oci://registry-1.docker.io/bitnamicharts/\($chart_name):\($chart_version)", | |
| "containers": $image_list, | |
| "github_repository": "bitnami/charts/tree/main/bitnami/\($chart_name)", | |
| } | |
| } | |
| EOF | |
| ) | |
| cd index | |
| # Configure git | |
| git config user.name "Bitnami Containers" | |
| git config user.email "bitnami-bot@vmware.com" | |
| read -r -a new_releases_arr <<< $NEW_RELEASES | |
| for release in "${new_releases_arr[@]}"; do | |
| read -r -a release_arr <<< "$(tr ':' ' ' <<< "$release")" | |
| chart_name="${release_arr[0]}" | |
| chart_version="${release_arr[1]}" | |
| ## Update index | |
| # Download published asset | |
| mkdir ../download | |
| helm pull "oci://registry-1.docker.io/bitnamicharts/${chart_name}" --version "${chart_version}" --destination ../download | |
| # Rebuild index | |
| helm repo index --url oci://registry-1.docker.io/bitnamicharts --merge bitnami/index.yaml ../download | |
| # Replace .tgz in URL with OCI tag | |
| sed -i "s|oci://registry-1.docker.io/bitnamicharts/$chart_name-$chart_version.tgz|oci://registry-1.docker.io/bitnamicharts/$chart_name:$chart_version|" ../download/index.yaml | |
| # Check index integrity | |
| if [[ $(stat -c%s bitnami/index.yaml) -gt $(stat -c%s ../download/index.yaml) ]]; then | |
| echo "New index.yaml file is shorter than the current one" | |
| exit 1 | |
| fi | |
| # Check repo can be loaded | |
| if ! helm repo add cache file://../download/ ; then | |
| echo "New index.yaml file can't be used as a file" | |
| exit 1 | |
| else | |
| # Remove the repo | |
| helm repo remove cache | |
| fi | |
| cp ../download/index.yaml bitnami/index.yaml | |
| ## Build and push promotions | |
| tar -xzf ../download/${chart_name}-${chart_version}.tgz -C ../download | |
| image_list="[]" | |
| # Get image list (removing the registry) | |
| for chart_yaml in $(find "../download/${chart_name}" -name "Chart.yaml"); do | |
| image_list_aux="$(yq '.annotations.images' "${chart_yaml}" | yq '[ .[] | .image | sub("^[^/]+/", "") ] | tojson')" | |
| image_list="$(jq -c --null-input --argjson arr1 "$image_list" --argjson arr2 "$image_list_aux" '$arr1 + $arr2 | unique')" | |
| done | |
| app_version=$(yq '.appVersion' "../download/${chart_name}/Chart.yaml") | |
| release_date="$(date -u +"%Y/%m/%d")" | |
| file_prefix="$(date -u +"%s%3N-${chart_name}-${app_version}")" | |
| # Build JSON files | |
| jq --null-input \ | |
| --arg platform_id "bitnami-chart-debian-x64" \ | |
| --arg chart_name "${chart_name}" \ | |
| --arg app_version "${app_version}" \ | |
| --arg chart_version "${chart_version}" \ | |
| --arg bundled_os_version "12" \ | |
| --argjson image_list "${image_list}" "${promotion_json_template}" > "${file_prefix}-bitnami-chart-debian-x64.json" | |
| jq --null-input \ | |
| --arg platform_id "vmware-chart-debian-x64" \ | |
| --arg chart_name "${chart_name}" \ | |
| --arg app_version "${app_version}" \ | |
| --arg chart_version "${chart_version}" \ | |
| --arg bundled_os_version "12" \ | |
| --argjson image_list "${image_list}" "${promotion_json_template}" | jq '.properties += {"alias_platform_from": "bitnami-chart-debian-x64"}' > "${file_prefix}-vmware-chart-debian-x64.json" | |
| # Upload files to the release bucket | |
| aws s3 cp "${file_prefix}-bitnami-chart-debian-x64.json" "s3://${AWS_PROMOTION_BUCKET}/releases/${release_date}/${file_prefix}-bitnami-chart-debian-x64.json" | |
| aws s3 cp "${file_prefix}-vmware-chart-debian-x64.json" "s3://${AWS_PROMOTION_BUCKET}/releases/${release_date}/${file_prefix}-vmware-chart-debian-x64.json" | |
| # Remove chart files | |
| rm -rf ../download | |
| done | |
| # Avoid overriding index branch when remote commit does not match our checkout commit | |
| current_commit_id=$(git rev-parse index) | |
| # Push changes | |
| git add bitnami/index.yaml && git commit --signoff --amend --no-edit | |
| git push origin index --force-with-lease=index:${current_commit_id} |