setup multi-arch build

.github/workflows/dev-image-build.yaml

@@ -39,6 +39,9 @@ jobs:
           username: ${{ secrets.DOCKERHUB_TOKEN }}
           password: ${{ secrets.DOCKERHUB_PASSWD }}
 
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+
       - name: Set up Docker Buildx
         id: buildx
         uses: docker/setup-buildx-action@v3

.github/workflows/main.yml

@@ -51,11 +51,14 @@ jobs:
             username: ${{ secrets.DOCKERHUB_TOKEN }}
             # Password or personal access token used to log against the Docker registry
             password: ${{ secrets.DOCKERHUB_PASSWD }}
-
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+
       - name: Set up Docker Buildx
         id: buildx
         uses: docker/setup-buildx-action@v3
-      
+
       - name: Setup protoc
         # You may pin to the exact commit or the version.
         # uses: arduino/setup-protoc@64c0c85d18e984422218383b81c52f8b077404d3

docker/Dockerfile

@@ -1,33 +1,51 @@
 ARG GOLANG_IMAGE
+ARG NVIDIA_RUNTIME_IMAGE=nvidia/cuda:12.2.0-base-ubuntu22.04
 ARG NVIDIA_IMAGE
+
 FROM $GOLANG_IMAGE AS build
 
+# cause cgo required now just use QEMU to multi-arch compile
+# FIXME could switch CXX/CC/AR
+#   with {GNUARCH}-linux-gnu-g++/{GNUARCH}-linux-gnu-gcc/{GNUARCH}-linux-gnu-gcc-ar
+#   or zig cc instead.
 FROM $GOLANG_IMAGE AS GOBUILD
 ADD . /k8s-vgpu
 ARG GOPROXY=https://goproxy.cn,direct
 RUN apt-get update && apt-get -y install libhwloc-dev libdrm-dev
-RUN cd /k8s-vgpu && make all
+ARG TARGETARCH
+RUN cd /k8s-vgpu && CGO_ENABLED=1 GOARCH=${TARGETARCH} make all
 
 FROM $NVIDIA_IMAGE AS NVBUILD
 COPY ./libvgpu /libvgpu
 WORKDIR /libvgpu
 RUN apt-get -y update && apt-get -y install wget
-RUN wget https://cmake.org/files/v3.19/cmake-3.19.8-Linux-x86_64.tar.gz
-RUN tar -xf cmake-3.19.8-Linux-x86_64.tar.gz
-RUN cp /libvgpu/cmake-3.19.8-Linux-x86_64/bin/cmake /libvgpu/cmake-3.19.8-Linux-x86_64/bin/cmake3
-ENV PATH="/libvgpu/cmake-3.19.8-Linux-x86_64/bin:${PATH}"
+
+# install cmake in arch-insensitive path
+RUN set -eux; \
+    \
+    case $(uname -m) in \
+        x86_64) CMAKE_DIR=cmake-3.19.8-Linux-x86_64;; \
+        aarch64) CMAKE_DIR=cmake-3.19.8-Linux-aarch64;; \
+        *) echo "unsupported architecture"; exit 1 ;; \
+    esac; \
+    wget https://cmake.org/files/v3.19/${CMAKE_DIR}.tar.gz; \
+    tar -xf ${CMAKE_DIR}.tar.gz; \
+    mv /libvgpu/${CMAKE_DIR} /libvgpu/cmake-3.19.8; \
+    cp /libvgpu/cmake-3.19.8/bin/cmake /libvgpu/cmake-3.19.8/bin/cmake3
+
+ENV PATH="/libvgpu/cmake-3.19.8/bin:${PATH}"
 RUN apt-get -y install openssl libssl-dev
 RUN bash ./build.sh
 
-FROM nvidia/cuda:12.2.0-base-ubuntu22.04
+FROM ${NVIDIA_RUNTIME_IMAGE}
 ENV NVIDIA_DISABLE_REQUIRE="true"
 ENV NVIDIA_VISIBLE_DEVICES=all
 ENV NVIDIA_DRIVER_CAPABILITIES=utility
 
 ARG VERSION
 LABEL version="$VERSION"
 LABEL maintainer="opensource@4paradigm.com"
-COPY ./LICENSE /k8s-vgpu/LICENSE
+#COPY ./LICENSE /k8s-vgpu/LICENSE
 COPY --from=GOBUILD /k8s-vgpu/bin /k8s-vgpu/bin
 COPY ./docker/entrypoint.sh /k8s-vgpu/bin/entrypoint.sh
 COPY ./lib /k8s-vgpu/lib

hack/build.sh

@@ -22,29 +22,25 @@ export SHORT_VERSION
 export COMMIT_CODE
 export VERSION="${SHORT_VERSION}-${COMMIT_CODE}"
 export LATEST_VERSION="latest"
-export GOLANG_IMAGE="golang:1.21-bullseye"
-export NVIDIA_IMAGE="nvidia/cuda:12.2.0-devel-ubuntu20.04"
+export GOLANG_IMAGE=${GOLANG_IMAGE:-"golang:1.21-bullseye"}
+export NVIDIA_IMAGE=${NVIDIA_IMAGE:-"nvidia/cuda:12.2.0-devel-ubuntu20.04"}
+export NVIDIA_RUNTIME_IMAGE=${NVIDIA_RUNTIME_IMAGE:-"nvidia/cuda:12.2.0-base-ubuntu22.04"}
 export DEST_DIR="/usr/local"
 
 IMAGE=${IMAGE-"projecthami/hami"}
 
-function go_build() {
-  [[ -z "$J" ]] && J=$(nproc | awk '{print int(($0 + 1)/ 2)}')
-  make -j$J
+function docker_buildx_push() {
+    docker buildx build --push \
+      --platform linux/amd64 \
+      --platform linux/arm64 \
+      --tag "${IMAGE}:${SHORT_VERSION}" \
+      --tag "${IMAGE}:${LATEST_VERSION}" \
+      --build-arg VERSION="${VERSION}" \
+      --build-arg GOLANG_IMAGE=${GOLANG_IMAGE} \
+      --build-arg NVIDIA_IMAGE=${NVIDIA_IMAGE} \
+      --build-arg NVIDIA_RUNTIME_IMAGE=${NVIDIA_RUNTIME_IMAGE} \
+      --build-arg DEST_DIR=${DEST_DIR} \
+      -f docker/Dockerfile .
 }
 
-function docker_build() {
-    docker build --build-arg VERSION="${VERSION}" --build-arg GOLANG_IMAGE=${GOLANG_IMAGE} --build-arg NVIDIA_IMAGE=${NVIDIA_IMAGE} --build-arg DEST_DIR=${DEST_DIR} -t "${IMAGE}:${VERSION}" -f docker/Dockerfile .
-    docker tag "${IMAGE}:${VERSION}" "${IMAGE}:${SHORT_VERSION}"
-    docker tag "${IMAGE}:${VERSION}" "${IMAGE}:${LATEST_VERSION}"
-}
-
-function docker_push() {
-    #docker push "${IMAGE}:${VERSION}"
-    docker push "${IMAGE}:${SHORT_VERSION}"
-    docker push "${IMAGE}:${LATEST_VERSION}"
-}
-
-go_build
-docker_build
-docker_push
+docker_buildx_push