use tmp file to when editing metadata json #115
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Deploy connector to dockerhub, release cli on github | |
| on: | |
| pull_request: | |
| branches: | |
| - main | |
| push: | |
| branches: | |
| - main | |
| tags: | |
| - 'v*' | |
| workflow_dispatch: | |
| # Defines two custom environment variables for the workflow. These are used for the Container registry domain, and a name for the Docker image that this workflow builds. | |
| env: | |
| REGISTRY: ghcr.io | |
| IMAGE_NAME: ${{ github.repository }} | |
| jobs: | |
| build-and-push-image: | |
| runs-on: ubuntu-latest | |
| # Sets the permissions granted to the `GITHUB_TOKEN` for the actions in this job. | |
| permissions: | |
| contents: read | |
| packages: write | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| # Uses the `docker/login-action` action to log in to the Container registry registry using the account and password that will publish the packages. Once published, the packages are scoped to the account defined here. | |
| - name: Log in to the Container registry | |
| uses: docker/login-action@v3 | |
| with: | |
| registry: ${{ env.REGISTRY }} | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| # This step uses [docker/metadata-action](https://github.com/docker/metadata-action#about) to extract tags and labels that will be applied to the specified image. The `id` "meta" allows the output of this step to be referenced in a subsequent step. The `images` value provides the base name for the tags and labels. | |
| - name: Extract metadata (tags, labels) for Docker | |
| id: meta | |
| uses: docker/metadata-action@v5 | |
| with: | |
| images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} | |
| # This step uses the `docker/build-push-action` action to build the image, based on your repository's `Dockerfile`. If the build succeeds, it pushes the image to GitHub Packages. | |
| # It uses the `context` parameter to define the build's context as the set of files located in the specified path. For more information, see "[Usage](https://github.com/docker/build-push-action#usage)" in the README of the `docker/build-push-action` repository. | |
| # It uses the `tags` and `labels` parameters to tag and label the image with the output from the "meta" step. | |
| - name: Build and push Docker image | |
| uses: docker/build-push-action@v5 | |
| with: | |
| context: . | |
| push: ${{ startsWith(github.ref, 'refs/tags/v') }} | |
| tags: ${{ steps.meta.outputs.tags }} | |
| labels: ${{ steps.meta.outputs.labels }} | |
| build-cli-binaries: | |
| name: build the CLI binaries | |
| strategy: | |
| matrix: | |
| include: | |
| - runner: ubuntu-latest | |
| target: x86_64-unknown-linux-musl | |
| rustflags: -C target-feature=+crt-static | |
| linux-packages: musl-tools | |
| - runner: ubuntu-latest | |
| target: aarch64-unknown-linux-musl | |
| rustflags: -C target-feature=+crt-static | |
| linux-packages: gcc-aarch64-linux-gnu musl-tools | |
| linker: /usr/bin/aarch64-linux-gnu-gcc | |
| - runner: macos-latest | |
| target: x86_64-apple-darwin | |
| - runner: macos-latest | |
| target: aarch64-apple-darwin | |
| - runner: windows-latest | |
| target: x86_64-pc-windows-msvc | |
| rustflags: -C target-feature=+crt-static | |
| extension: .exe | |
| runs-on: ${{ matrix.runner }} | |
| env: | |
| CARGO_BUILD_TARGET: ${{ matrix.target }} | |
| CARGO_NET_GIT_FETCH_WITH_CLI: "true" | |
| RUSTFLAGS: "-D warnings ${{ matrix.rustflags }}" | |
| defaults: | |
| run: | |
| shell: bash | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: install protoc | |
| uses: arduino/setup-protoc@v3 | |
| with: | |
| version: "25.x" | |
| repo-token: ${{ secrets.GITHUB_TOKEN }} | |
| - name: install tools | |
| run: | | |
| rustup show | |
| rustup target add ${{ matrix.target }} | |
| - name: install other packages required | |
| if: matrix.linux-packages | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install -y ${{ matrix.linux-packages }} | |
| - uses: Swatinem/rust-cache@v2 | |
| with: | |
| shared-key: "build" # share the cache across jobs | |
| - name: build the CLI | |
| run: | | |
| # If we're on a tag, use the tag name as the release version. | |
| if [[ "$GITHUB_REF_TYPE" == 'tag' ]]; then | |
| # Ensure that the version specified in Cargo.toml is the same as the tag (with a 'v' prefix). | |
| CARGO_VERSION="$(cargo metadata --format-version=1 | jq -r '.packages | .[] | select(.name == "ndc-clickhouse-cli") | .version')" | |
| echo "Git tag: ${GITHUB_REF_NAME}" | |
| echo "Cargo version: ${CARGO_VERSION}" | |
| if [[ "${GITHUB_REF_NAME}" != "v${CARGO_VERSION}" ]]; then | |
| echo >&2 "The Git tag is \"${GITHUB_REF_NAME}\", but the version in Cargo.toml is \"${CARGO_VERSION}\"." | |
| echo >&2 'These must be the same, with a "v" prefix for the tag. Aborting.' | |
| exit 1 | |
| fi | |
| export RELEASE_VERSION="$GITHUB_REF_NAME" | |
| echo "RELEASE_VERSION = ${RELEASE_VERSION}" | |
| fi | |
| if [[ -n '${{ matrix.linker }}' ]]; then | |
| TARGET_SCREAMING="$(echo '${{ matrix.target }}' | tr '[:lower:]' '[:upper:]' | tr '-' '_')" | |
| echo "CARGO_TARGET_${TARGET_SCREAMING}_LINKER"='${{ matrix.linker }}' | |
| declare "CARGO_TARGET_${TARGET_SCREAMING}_LINKER"='${{ matrix.linker }}' | |
| export "CARGO_TARGET_${TARGET_SCREAMING}_LINKER" | |
| fi | |
| echo "Building for target: ${CARGO_BUILD_TARGET}" | |
| cargo build --locked --release --package ndc-clickhouse-cli | |
| mkdir -p release | |
| mv -v target/${{ matrix.target }}/release/ndc-clickhouse-cli release/ndc-clickhouse-cli-${{ matrix.target }}${{ matrix.extension }} | |
| - uses: actions/upload-artifact@v4 | |
| with: | |
| name: ndc-clickhouse-cli-${{ matrix.target }}${{ matrix.extension }} | |
| path: release | |
| if-no-files-found: error | |
| release: | |
| name: release to GitHub | |
| needs: | |
| - build-and-push-image | |
| - build-cli-binaries | |
| runs-on: ubuntu-latest | |
| if: ${{ startsWith(github.ref, 'refs/tags/v') }} | |
| steps: | |
| - uses: actions-rust-lang/setup-rust-toolchain@v1 | |
| with: | |
| rustflags: "" # defaults to "-D warnings", set to empty string to allow warnings | |
| - uses: actions/checkout@v4 | |
| - uses: actions/download-artifact@v4 | |
| with: | |
| path: release/artifacts | |
| merge-multiple: true | |
| - name: generate CLI manifest | |
| run: | | |
| set -evo pipefail | |
| ROOT="$(pwd)" | |
| export CLI_VERSION="$GITHUB_REF_NAME" | |
| export LINUX_AMD64_SHA256=$(sha256sum ${ROOT}/release/artifacts/ndc-clickhouse-cli-x86_64-unknown-linux-musl | cut -f1 -d' ') | |
| export MACOS_AMD64_SHA256=$(sha256sum ${ROOT}/release/artifacts/ndc-clickhouse-cli-x86_64-apple-darwin | cut -f1 -d' ') | |
| export WINDOWS_AMD64_SHA256=$(sha256sum ${ROOT}/release/artifacts/ndc-clickhouse-cli-x86_64-pc-windows-msvc.exe | cut -f1 -d' ') | |
| export LINUX_ARM64_SHA256=$(sha256sum ${ROOT}/release/artifacts/ndc-clickhouse-cli-aarch64-unknown-linux-musl | cut -f1 -d' ') | |
| export MACOS_ARM64_SHA256=$(sha256sum ${ROOT}/release/artifacts/ndc-clickhouse-cli-aarch64-apple-darwin | cut -f1 -d' ') | |
| mkdir -p "${ROOT}/release/" | |
| cat "${ROOT}/ci/templates/manifest.yaml" | envsubst > "${ROOT}/release/manifest.yaml" | |
| - uses: actions/upload-artifact@v4 | |
| with: | |
| name: manifest.yaml | |
| path: release/manifest.yaml | |
| if-no-files-found: error | |
| - name: Build connector definition | |
| run: | | |
| set -evo pipefail | |
| ROOT="$(pwd)" | |
| export DOCKER_IMAGE="ghcr.io/hasura/ndc-clickhouse:$GITHUB_REF_NAME" | |
| export CLI_VERSION=$GITHUB_REF_NAME | |
| mkdir -p "${ROOT}/release/connector-definition/.hasura-connector/" | |
| cat "${ROOT}/ci/templates/connector-metadata.yaml" | envsubst > "${ROOT}/release/connector-definition/.hasura-connector/connector-metadata.yaml" | |
| cargo run --package ndc-clickhouse-cli -- --connector-context-path "${ROOT}/release/connector-definition" init | |
| tar -czvf "${ROOT}/release/artifacts/connector-definition.tgz" --directory "${ROOT}/release/connector-definition/" . | |
| - uses: actions/upload-artifact@v4 | |
| with: | |
| name: connector-definition.tgz | |
| path: ./release/artifacts/connector-definition.tgz | |
| compression-level: 0 # Already compressed | |
| - name: Get version from tag | |
| id: get-version | |
| run: | | |
| echo "tagged_version=${GITHUB_REF#refs/tags/v}" >> $GITHUB_OUTPUT | |
| shell: bash | |
| - uses: mindsers/changelog-reader-action@v2 | |
| id: changelog-reader | |
| with: | |
| version: ${{ steps.get-version.outputs.tagged_version }} | |
| path: ./CHANGELOG.md | |
| - name: create a release | |
| uses: ncipollo/release-action@v1 | |
| with: | |
| draft: false | |
| tag: v${{ steps.get-version.outputs.tagged_version }} | |
| body: ${{ steps.changelog-reader.outputs.changes }} | |
| artifacts: release/artifacts/* | |
| create-cli-plugins-index-pr: | |
| name: Create CLI Plugin Index PR | |
| needs: | |
| - release | |
| runs-on: ubuntu-latest | |
| env: | |
| GH_TOKEN: ${{ secrets.HASURA_BOT_TOKEN }} | |
| if: ${{ startsWith(github.ref, 'refs/tags/v') }} | |
| steps: | |
| - name: check out this repository | |
| uses: actions/checkout@v4 | |
| with: | |
| path: ndc-clickhouse | |
| - name: check out cli-plugins-index | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: hasura/cli-plugins-index | |
| path: cli-plugins-index | |
| token: ${{ secrets.HASURA_BOT_TOKEN }} | |
| - uses: actions/download-artifact@v4 | |
| with: | |
| path: release/artifacts | |
| merge-multiple: true | |
| - name: create cli-plugins-index PR | |
| run: | | |
| set -evo pipefail | |
| ROOT="$(pwd)" | |
| # we assume GITHUB_REF_NAME is the version number prefixed with `v`. This is checked in previous steps | |
| BRANCH_NAME="clickhouse/release-$GITHUB_REF_NAME" | |
| RELEASE_HASH="$(cd ndc-clickhouse && git rev-parse HEAD)" | |
| # Change working directory to the target folder | |
| cd cli-plugins-index | |
| git config --global user.name "hasura-bot" | |
| git config --global user.email "accounts@hasura.io" | |
| # Create a new feature branch for the changes. | |
| git checkout -b $BRANCH_NAME | |
| mkdir "plugins/clickhouse/$GITHUB_REF_NAME" | |
| cp "${ROOT}/release/artifacts/manifest.yaml" "${ROOT}/cli-plugins-index/plugins/clickhouse/$GITHUB_REF_NAME/manifest.yaml" | |
| git add . | |
| git commit -m "Release ClickHouse $GITHUB_REF_NAME" | |
| git push origin $BRANCH_NAME --force | |
| # create a pull-requests containing the updates. | |
| gh pr create \ | |
| --body "Commit in ndc-clickhouse: https://github.com/hasura/ndc-clickhouse/commit/$RELEASE_HASH" \ | |
| --title "Release ClickHouse $GITHUB_REF_NAME" \ | |
| --head "$BRANCH_NAME" \ | |
| --base "master" | |
| create-ndc-hub-pr: | |
| name: Create NDC-Hub PR | |
| needs: | |
| - release | |
| runs-on: ubuntu-latest | |
| env: | |
| GH_TOKEN: ${{ secrets.HASURA_BOT_TOKEN }} | |
| if: ${{ startsWith(github.ref, 'refs/tags/v') }} | |
| steps: | |
| - name: check out this repository | |
| uses: actions/checkout@v4 | |
| with: | |
| path: ndc-clickhouse | |
| - name: check out ndc-hub | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: hasura/ndc-hub | |
| path: ndc-hub | |
| token: ${{ secrets.HASURA_BOT_TOKEN }} | |
| - uses: actions/download-artifact@v4 | |
| with: | |
| path: release/artifacts | |
| merge-multiple: true | |
| - name: create ndc-hub PR | |
| run: | | |
| set -evo pipefail | |
| ROOT="$(pwd)" | |
| # we assume GITHUB_REF_NAME is the version number prefixed with `v`. This is checked in previous steps | |
| BRANCH_NAME="clickhouse/release-$GITHUB_REF_NAME" | |
| # These exported env vars are used for templating | |
| # Release version is the same as tag name, minus leading v. | |
| export RELEASE_VERSION="${GITHUB_REF_NAME:1}" | |
| export RELEASE_HASH="$(cd ndc-clickhouse && git rev-parse HEAD)" | |
| export CONNECTOR_DEFINITION_HASH=$(sha256sum ${ROOT}/release/artifacts/connector-definition.tgz | cut -f1 -d' ') | |
| # Change working directory to the target folder | |
| cd ndc-hub | |
| git config --global user.name "hasura-bot" | |
| git config --global user.email "accounts@hasura.io" | |
| # Create a new feature branch for the changes. | |
| git checkout -b $BRANCH_NAME | |
| # create new connector definition | |
| mkdir "${ROOT}/ndc-hub/registry/hasura/clickhouse/releases/v$RELEASE_VERSION" | |
| cat "${ROOT}/ndc-clickhouse/ci/templates/connector-packaging.json" | envsubst > "${ROOT}/ndc-hub/registry/hasura/clickhouse/releases/v$RELEASE_VERSION/connector-packaging.json" | |
| # modify metadata file to add new entry | |
| UPDATED_METADATA= jq --arg RELEASE_VERSION "$GITHUB_REF_NAME" '.overview.latest_version = $RELEASE_VERSION' "${ROOT}/ndc-hub/registry/hasura/clickhouse/metadata.json" | | |
| jq --arg RELEASE_VERSION "$GITHUB_REF_NAME" --arg RELEASE_HASH "$RELEASE_HASH" '.source_code.version |= [{tag: $RELEASE_VERSION, hash: $RELEASE_HASH, is_verified: true}] + .' > metadata.tmp.json | |
| mv metadata.tmp.json "${ROOT}/ndc-hub/registry/hasura/clickhouse/metadata.json" | |
| git add . | |
| git commit -m "Release ClickHouse $GITHUB_REF_NAME" | |
| git push origin $BRANCH_NAME --force | |
| # create a pull-requests containing the updates. | |
| gh pr create \ | |
| --body "Commit in ndc-clickhouse: https://github.com/hasura/ndc-clickhouse/commit/$RELEASE_HASH" \ | |
| --title "Release ClickHouse $GITHUB_REF_NAME" \ | |
| --head "$BRANCH_NAME" \ | |
| --base "main" |