diff --git a/.github/workflows/dev-image-build.yaml b/.github/workflows/dev-image-build.yaml index b8e67af21..b8f0e576c 100644 --- a/.github/workflows/dev-image-build.yaml +++ b/.github/workflows/dev-image-build.yaml @@ -39,6 +39,9 @@ jobs: username: ${{ secrets.DOCKERHUB_TOKEN }} password: ${{ secrets.DOCKERHUB_PASSWD }} + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + - name: Set up Docker Buildx id: buildx uses: docker/setup-buildx-action@v3 diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 9135fbd93..8b6f4ea17 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -51,11 +51,14 @@ jobs: username: ${{ secrets.DOCKERHUB_TOKEN }} # Password or personal access token used to log against the Docker registry password: ${{ secrets.DOCKERHUB_PASSWD }} - + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + - name: Set up Docker Buildx id: buildx uses: docker/setup-buildx-action@v3 - + - name: Setup protoc # You may pin to the exact commit or the version. # uses: arduino/setup-protoc@64c0c85d18e984422218383b81c52f8b077404d3 diff --git a/docker/Dockerfile b/docker/Dockerfile index d8504de98..15929370d 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,25 +1,43 @@ ARG GOLANG_IMAGE +ARG NVIDIA_RUNTIME_IMAGE=nvidia/cuda:12.2.0-base-ubuntu22.04 ARG NVIDIA_IMAGE + FROM $GOLANG_IMAGE AS build +# cause cgo required now just use QEMU to multi-arch compile +# FIXME could switch CXX/CC/AR +# with {GNUARCH}-linux-gnu-g++/{GNUARCH}-linux-gnu-gcc/{GNUARCH}-linux-gnu-gcc-ar +# or zig cc instead. FROM $GOLANG_IMAGE AS GOBUILD ADD . /k8s-vgpu ARG GOPROXY=https://goproxy.cn,direct RUN apt-get update && apt-get -y install libhwloc-dev libdrm-dev -RUN cd /k8s-vgpu && make all +ARG TARGETARCH +RUN cd /k8s-vgpu && CGO_ENABLED=1 GOARCH=${TARGETARCH} make all FROM $NVIDIA_IMAGE AS NVBUILD COPY ./libvgpu /libvgpu WORKDIR /libvgpu RUN apt-get -y update && apt-get -y install wget -RUN wget https://cmake.org/files/v3.19/cmake-3.19.8-Linux-x86_64.tar.gz -RUN tar -xf cmake-3.19.8-Linux-x86_64.tar.gz -RUN cp /libvgpu/cmake-3.19.8-Linux-x86_64/bin/cmake /libvgpu/cmake-3.19.8-Linux-x86_64/bin/cmake3 -ENV PATH="/libvgpu/cmake-3.19.8-Linux-x86_64/bin:${PATH}" + +# install cmake in arch-insensitive path +RUN set -eux; \ + \ + case $(uname -m) in \ + x86_64) CMAKE_DIR=cmake-3.19.8-Linux-x86_64;; \ + aarch64) CMAKE_DIR=cmake-3.19.8-Linux-aarch64;; \ + *) echo "unsupported architecture"; exit 1 ;; \ + esac; \ + wget https://cmake.org/files/v3.19/${CMAKE_DIR}.tar.gz; \ + tar -xf ${CMAKE_DIR}.tar.gz; \ + mv /libvgpu/${CMAKE_DIR} /libvgpu/cmake-3.19.8; \ + cp /libvgpu/cmake-3.19.8/bin/cmake /libvgpu/cmake-3.19.8/bin/cmake3 + +ENV PATH="/libvgpu/cmake-3.19.8/bin:${PATH}" RUN apt-get -y install openssl libssl-dev RUN bash ./build.sh -FROM nvidia/cuda:12.2.0-base-ubuntu22.04 +FROM ${NVIDIA_RUNTIME_IMAGE} ENV NVIDIA_DISABLE_REQUIRE="true" ENV NVIDIA_VISIBLE_DEVICES=all ENV NVIDIA_DRIVER_CAPABILITIES=utility @@ -27,7 +45,7 @@ ENV NVIDIA_DRIVER_CAPABILITIES=utility ARG VERSION LABEL version="$VERSION" LABEL maintainer="opensource@4paradigm.com" -COPY ./LICENSE /k8s-vgpu/LICENSE +#COPY ./LICENSE /k8s-vgpu/LICENSE COPY --from=GOBUILD /k8s-vgpu/bin /k8s-vgpu/bin COPY ./docker/entrypoint.sh /k8s-vgpu/bin/entrypoint.sh COPY ./lib /k8s-vgpu/lib diff --git a/hack/build.sh b/hack/build.sh index a84ddbab1..d47237811 100755 --- a/hack/build.sh +++ b/hack/build.sh @@ -22,29 +22,25 @@ export SHORT_VERSION export COMMIT_CODE export VERSION="${SHORT_VERSION}-${COMMIT_CODE}" export LATEST_VERSION="latest" -export GOLANG_IMAGE="golang:1.21-bullseye" -export NVIDIA_IMAGE="nvidia/cuda:12.2.0-devel-ubuntu20.04" +export GOLANG_IMAGE=${GOLANG_IMAGE:-"golang:1.21-bullseye"} +export NVIDIA_IMAGE=${NVIDIA_IMAGE:-"nvidia/cuda:12.2.0-devel-ubuntu20.04"} +export NVIDIA_RUNTIME_IMAGE=${NVIDIA_RUNTIME_IMAGE:-"nvidia/cuda:12.2.0-base-ubuntu22.04"} export DEST_DIR="/usr/local" IMAGE=${IMAGE-"projecthami/hami"} -function go_build() { - [[ -z "$J" ]] && J=$(nproc | awk '{print int(($0 + 1)/ 2)}') - make -j$J +function docker_buildx_push() { + docker buildx build --push \ + --platform linux/amd64 \ + --platform linux/arm64 \ + --tag "${IMAGE}:${SHORT_VERSION}" \ + --tag "${IMAGE}:${LATEST_VERSION}" \ + --build-arg VERSION="${VERSION}" \ + --build-arg GOLANG_IMAGE=${GOLANG_IMAGE} \ + --build-arg NVIDIA_IMAGE=${NVIDIA_IMAGE} \ + --build-arg NVIDIA_RUNTIME_IMAGE=${NVIDIA_RUNTIME_IMAGE} \ + --build-arg DEST_DIR=${DEST_DIR} \ + -f docker/Dockerfile . } -function docker_build() { - docker build --build-arg VERSION="${VERSION}" --build-arg GOLANG_IMAGE=${GOLANG_IMAGE} --build-arg NVIDIA_IMAGE=${NVIDIA_IMAGE} --build-arg DEST_DIR=${DEST_DIR} -t "${IMAGE}:${VERSION}" -f docker/Dockerfile . - docker tag "${IMAGE}:${VERSION}" "${IMAGE}:${SHORT_VERSION}" - docker tag "${IMAGE}:${VERSION}" "${IMAGE}:${LATEST_VERSION}" -} - -function docker_push() { - #docker push "${IMAGE}:${VERSION}" - docker push "${IMAGE}:${SHORT_VERSION}" - docker push "${IMAGE}:${LATEST_VERSION}" -} - -go_build -docker_build -docker_push \ No newline at end of file +docker_buildx_push