Skip to content

removed --debug and added a --platform flag to the smoke test #401

removed --debug and added a --platform flag to the smoke test

removed --debug and added a --platform flag to the smoke test #401

name: Build And Push Docker Image
on:
push:
branches:
- main
- luke/manual-deploy
env:
DOCKER_REPOSITORY: quay.io/unstructured-io/unstructured-api
DOCKER_BUILD_REPOSITORY: quay.io/unstructured-io/build-unstructured-api
PACKAGE: "unstructured-api"
PIPELINE_FAMILY: "general"
PIP_VERSION: "22.2.1"
PYTHON_VERSION: "3.10"
jobs:
setup:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: actions/cache@v4
id: virtualenv-cache
with:
path: |
.venv
key: ci-venv-${{ env.PIPELINE_FAMILY }}-${{ hashFiles('requirements/test.txt') }}
- name: Set up Python ${{ env.PYTHON_VERSION }}
uses: actions/setup-python@v5
with:
python-version: ${{ env.PYTHON_VERSION }}
- name: Setup virtual environment (no cache hit)
if: steps.virtualenv-cache.outputs.cache-hit != 'true'
run: |
python${{ env.PYTHON_VERSION }} -m venv .venv
source .venv/bin/activate
make install-ci
set-short-sha:
runs-on: ubuntu-latest
outputs:
short_sha: ${{ steps.set_short_sha.outputs.short_sha }}
steps:
- name: Set Short SHA
id: set_short_sha
run: echo "short_sha=$(echo ${{ github.sha }} | cut -c1-7)" >> $GITHUB_OUTPUT
build-images:
strategy:
matrix:
arch: ["arm64", "amd64"]
runs-on: ubuntu-latest-m
needs: [setup, set-short-sha]
env:
SHORT_SHA: ${{ needs.set-short-sha.outputs.short_sha }}
DOCKER_PLATFORM: linux/${{ matrix.arch }}
steps:
- name: Set up QEMU
uses: docker/setup-qemu-action@v3
with:
platforms: all
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
with:
driver: ${{ matrix.arch == 'amd64' && 'docker' || 'docker-container' }}
#driver: ${{ 'docker-container' }}
- name: Checkout code
uses: actions/checkout@v4
- name: Login to Quay.io
uses: docker/login-action@v3
with:
registry: quay.io
username: ${{ secrets.QUAY_IO_ROBOT_USERNAME }}
password: ${{ secrets.QUAY_IO_ROBOT_TOKEN }}
- name: Build image
run: |
# Clear some space (https://github.com/actions/runner-images/issues/2840)
sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/share/boost
DOCKER_BUILDKIT=1 docker buildx build --load -f Dockerfile \
--platform=${{ matrix.arch }} \
--build-arg PIP_VERSION=$PIP_VERSION \
--build-arg BUILDKIT_INLINE_CACHE=1 \
--build-arg PIPELINE_PACKAGE=${{ env.PIPELINE_FAMILY }} \
--provenance=false \
--progress plain \
--cache-from $DOCKER_BUILD_REPOSITORY:${{ matrix.arch }} \
-t $DOCKER_BUILD_REPOSITORY:${{ matrix.arch }}-$SHORT_SHA .
- name: Set virtualenv cache
uses: actions/cache@v4
id: virtualenv-cache
with:
path: |
.venv
key: ci-venv-${{ env.PIPELINE_FAMILY }}-${{ hashFiles('requirements/test.txt') }}
- name: Test image
run: |
source .venv/bin/activate
export DOCKER_IMAGE="$DOCKER_BUILD_REPOSITORY:${{ matrix.arch }}-$SHORT_SHA"
if [ "$DOCKER_PLATFORM" == "linux/arm64" ]; then
SKIP_INFERENCE_TESTS=true make docker-test
else
make docker-test
fi
- name: Push image
run: |
# write to the build repository to cache for the publish-images job
docker push $DOCKER_BUILD_REPOSITORY:${{ matrix.arch }}-$SHORT_SHA
publish-images:
runs-on: ubuntu-latest-m
needs: [setup, set-short-sha, build-images]
env:
SHORT_SHA: ${{ needs.set-short-sha.outputs.short_sha }}
steps:
- name: Checkout code
uses: actions/checkout@v4
- name: Set SHORT_SHA
run: echo "SHORT_SHA=$(git rev-parse --short HEAD)" >> $GITHUB_ENV
- name: Login to Quay.io
uses: docker/login-action@v3
with:
registry: quay.io
username: ${{ secrets.QUAY_IO_ROBOT_USERNAME }}
password: ${{ secrets.QUAY_IO_ROBOT_TOKEN }}
- name: Pull AMD image
run: |
docker pull $DOCKER_BUILD_REPOSITORY:amd64-$SHORT_SHA
- name: Pull ARM image
run: |
docker pull $DOCKER_BUILD_REPOSITORY:arm64-$SHORT_SHA
- name: Push AMD and ARM tags
run: |
# these are used to construct the final manifest but also cache-from in subsequent runs
docker tag $DOCKER_BUILD_REPOSITORY:amd64-$SHORT_SHA $DOCKER_BUILD_REPOSITORY:amd64
docker push $DOCKER_BUILD_REPOSITORY:amd64
docker tag $DOCKER_BUILD_REPOSITORY:arm64-$SHORT_SHA $DOCKER_BUILD_REPOSITORY:arm64
docker push $DOCKER_BUILD_REPOSITORY:arm64
- name: Push multiarch manifest
run: |
docker manifest create ${DOCKER_REPOSITORY}:latest $DOCKER_BUILD_REPOSITORY:amd64 $DOCKER_BUILD_REPOSITORY:arm64
docker manifest push $DOCKER_REPOSITORY:latest
docker manifest create ${DOCKER_REPOSITORY}:$SHORT_SHA $DOCKER_BUILD_REPOSITORY:amd64 $DOCKER_BUILD_REPOSITORY:arm64
docker manifest push $DOCKER_REPOSITORY:$SHORT_SHA
VERSION=$(grep -m1 version preprocessing-pipeline-family.yaml | cut -d ' ' -f2)
docker manifest create ${DOCKER_REPOSITORY}:$VERSION $DOCKER_BUILD_REPOSITORY:amd64 $DOCKER_BUILD_REPOSITORY:arm64
docker manifest push ${DOCKER_REPOSITORY}:$VERSION