diff --git a/core/iwasm/libraries/wasi-nn/README.md b/core/iwasm/libraries/wasi-nn/README.md index 5536f6d57b..99a7664676 100644 --- a/core/iwasm/libraries/wasi-nn/README.md +++ b/core/iwasm/libraries/wasi-nn/README.md @@ -103,7 +103,6 @@ docker run \ wasi-nn-cpu \ --dir=/ \ --env="TARGET=cpu" \ - --native-lib=/lib/libwasi-nn-tflite.so \ /assets/test_tensorflow.wasm ``` @@ -119,7 +118,6 @@ docker run \ wasi-nn-nvidia-gpu \ --dir=/ \ --env="TARGET=gpu" \ - --native-lib=/lib/libwasi-nn-tflite.so \ /assets/test_tensorflow.wasm ``` @@ -131,7 +129,6 @@ docker run \ wasi-nn-vx-delegate \ --dir=/ \ --env="TARGET=gpu" \ - --native-lib=/lib/libwasi-nn-tflite.so \ /assets/test_tensorflow_quantized.wasm ``` @@ -147,7 +144,6 @@ docker run \ wasi-nn-tpu \ --dir=/ \ --env="TARGET=tpu" \ - --native-lib=/lib/libwasi-nn-tflite.so \ /assets/test_tensorflow_quantized.wasm ``` @@ -155,8 +151,8 @@ docker run \ Supported: -- Graph encoding: `tensorflowlite`. -- Execution target: `cpu`, `gpu` and `tpu`. +- Graph encoding: `tensorflowlite`, `openvino` and `ggml` +- Execution target: `cpu` for all. `gpu` and `tpu` for `tensorflowlite`. - Tensor type: `fp32`. ## Smoke test diff --git a/core/iwasm/libraries/wasi-nn/cmake/Findcjson.cmake b/core/iwasm/libraries/wasi-nn/cmake/Findcjson.cmake index 1136f41adc..6c921bbc93 100644 --- a/core/iwasm/libraries/wasi-nn/cmake/Findcjson.cmake +++ b/core/iwasm/libraries/wasi-nn/cmake/Findcjson.cmake @@ -4,13 +4,21 @@ include(FetchContent) set(CJSON_SOURCE_DIR "${WAMR_ROOT_DIR}/core/deps/cjson") - -FetchContent_Declare( - cjson - GIT_REPOSITORY https://github.com/DaveGamble/cJSON.git - GIT_TAG v1.7.18 - SOURCE_DIR ${CJSON_SOURCE_DIR} -) +if(EXISTS ${CJSON_SOURCE_DIR}) + message("Use existed source code under ${CJSON_SOURCE_DIR}") + FetchContent_Declare( + cjson + SOURCE_DIR ${CJSON_SOURCE_DIR} + ) +else() + message("download source code and store it at ${CJSON_SOURCE_DIR}") + FetchContent_Declare( + cjson + GIT_REPOSITORY https://github.com/DaveGamble/cJSON.git + GIT_TAG v1.7.18 + SOURCE_DIR ${CJSON_SOURCE_DIR} + ) +endif() set(ENABLE_CJSON_TEST OFF CACHE INTERNAL "Turn off tests") set(ENABLE_CJSON_UNINSTALL OFF CACHE INTERNAL "Turn off uninstall to avoid targets conflict") diff --git a/core/iwasm/libraries/wasi-nn/cmake/Findllamacpp.cmake b/core/iwasm/libraries/wasi-nn/cmake/Findllamacpp.cmake index 431e15db59..8f4f8d1aa7 100644 --- a/core/iwasm/libraries/wasi-nn/cmake/Findllamacpp.cmake +++ b/core/iwasm/libraries/wasi-nn/cmake/Findllamacpp.cmake @@ -4,13 +4,21 @@ include(FetchContent) set(LLAMA_SOURCE_DIR "${WAMR_ROOT_DIR}/core/deps/llama.cpp") - -FetchContent_Declare( - llamacpp - GIT_REPOSITORY https://github.com/ggerganov/llama.cpp.git - GIT_TAG b3573 - SOURCE_DIR ${LLAMA_SOURCE_DIR} -) +if(EXISTS ${LLAMA_SOURCE_DIR}) + message("Use existed source code under ${LLAMA_SOURCE_DIR}") + FetchContent_Declare( + llamacpp + SOURCE_DIR ${LLAMA_SOURCE_DIR} + ) +else() + message("download source code and store it at ${LLAMA_SOURCE_DIR}") + FetchContent_Declare( + llamacpp + GIT_REPOSITORY https://github.com/ggerganov/llama.cpp.git + GIT_TAG b3573 + SOURCE_DIR ${LLAMA_SOURCE_DIR} + ) +endif() set(LLAMA_BUILD_TESTS OFF) set(LLAMA_BUILD_EXAMPLES OFF) diff --git a/core/iwasm/libraries/wasi-nn/cmake/Findtensorflow_lite.cmake b/core/iwasm/libraries/wasi-nn/cmake/Findtensorflow_lite.cmake index 39480741d3..d2b3f74e04 100644 --- a/core/iwasm/libraries/wasi-nn/cmake/Findtensorflow_lite.cmake +++ b/core/iwasm/libraries/wasi-nn/cmake/Findtensorflow_lite.cmake @@ -4,20 +4,32 @@ include(FetchContent) set(TFLITE_SOURCE_DIR "${WAMR_ROOT_DIR}/core/deps/tensorflow-src") +if(EXISTS ${TFLITE_SOURCE_DIR}) + message("Use existed source code under ${TFLITE_SOURCE_DIR}") + FetchContent_Declare( + tensorflow_lite + SOURCE_DIR ${TFLITE_SOURCE_DIR} + SOURCE_SUBDIR tensorflow/lite + ) +else() + message("download source code and store it at ${TFLITE_SOURCE_DIR}") + FetchContent_Declare( + tensorflow_lite + GIT_REPOSITORY https://github.com/tensorflow/tensorflow.git + GIT_TAG v2.12.0 + GIT_SHALLOW ON + GIT_PROGRESS ON + SOURCE_DIR ${TFLITE_SOURCE_DIR} + SOURCE_SUBDIR tensorflow/lite + PATCH_COMMAND git apply ${CMAKE_CURRENT_LIST_DIR}/add_telemetry.patch + ) +endif() -FetchContent_Declare( - tensorflow_lite - GIT_REPOSITORY https://github.com/tensorflow/tensorflow.git - GIT_TAG v2.12.0 - GIT_SHALLOW ON - GIT_PROGRESS ON - SOURCE_DIR ${TFLITE_SOURCE_DIR} - SOURCE_SUBDIR tensorflow/lite -) if(WAMR_BUILD_WASI_NN_ENABLE_GPU EQUAL 1) set(TFLITE_ENABLE_GPU ON) endif() + if (CMAKE_SIZEOF_VOID_P EQUAL 4) set(TFLITE_ENABLE_XNNPACK OFF) endif() diff --git a/core/iwasm/libraries/wasi-nn/cmake/add_telemetry.patch b/core/iwasm/libraries/wasi-nn/cmake/add_telemetry.patch new file mode 100644 index 0000000000..8dbf5c355c --- /dev/null +++ b/core/iwasm/libraries/wasi-nn/cmake/add_telemetry.patch @@ -0,0 +1,12 @@ +diff --git a/tensorflow/lite/CMakeLists.txt b/tensorflow/lite/CMakeLists.txt +index c71a3925ac..39591a3bd7 100644 +--- a/tensorflow/lite/CMakeLists.txt ++++ b/tensorflow/lite/CMakeLists.txt +@@ -493,6 +493,7 @@ set(TFLITE_PROFILER_SRCS + ${TFLITE_SOURCE_DIR}/profiling/root_profiler.h + ${TFLITE_SOURCE_DIR}/profiling/root_profiler.cc + ${TFLITE_SOURCE_DIR}/profiling/telemetry/profiler.cc ++ ${TFLITE_SOURCE_DIR}/profiling/telemetry/telemetry.cc + ) + if(CMAKE_SYSTEM_NAME MATCHES "Android") + list(APPEND TFLITE_PROFILER_SRCS diff --git a/core/iwasm/libraries/wasi-nn/test/Dockerfile.cpu b/core/iwasm/libraries/wasi-nn/test/Dockerfile.cpu index a33bebff51..67bfbfe0e5 100644 --- a/core/iwasm/libraries/wasi-nn/test/Dockerfile.cpu +++ b/core/iwasm/libraries/wasi-nn/test/Dockerfile.cpu @@ -14,19 +14,32 @@ WORKDIR /usr/local/share/ca-certificates/cacert.org RUN wget -qP /usr/local/share/ca-certificates/cacert.org http://www.cacert.org/certs/root.crt http://www.cacert.org/certs/class3.crt \ && update-ca-certificates +# need a newer cmake +RUN apt-get purge -y cmake + +ARG CMAKE_VER=3.27.0 +RUN wget https://github.com/Kitware/CMake/releases/download/v${CMAKE_VER}/cmake-${CMAKE_VER}-Linux-x86_64.sh \ + -q -O /tmp/cmake-install.sh \ + && chmod u+x /tmp/cmake-install.sh \ + && mkdir /opt/cmake-${CMAKE_VER} \ + && /tmp/cmake-install.sh --skip-license --prefix=/opt/cmake-${CMAKE_VER} \ + && rm /tmp/cmake-install.sh \ + && ln -s /opt/cmake-${CMAKE_VER}/bin/* /usr/local/bin + WORKDIR /home/wamr COPY . . RUN git config --global http.sslCAinfo /etc/ssl/certs/ca-certificates.crt WORKDIR /home/wamr/product-mini/platforms/linux RUN rm -rf build \ - && cmake -S . -B build -DWAMR_BUILD_WASI_NN=1 \ + && cmake -S . -B build\ + -DWAMR_BUILD_WASI_NN=1 -DWAMR_BUILD_WASI_NN_TFLITE=1\ && cmake --build build -j "$(grep -c ^processor /proc/cpuinfo)" FROM ubuntu:22.04 -COPY --from=base /home/wamr/product-mini/platforms/linux/build/iwasm /usr/bin/iwasm -COPY --from=base /home/wamr/product-mini/platforms/linux/build/libiwasm.so /lib/libiwasm.so -COPY --from=base /home/wamr/product-mini/platforms/linux/build/libwasi-nn-*.so /lib/ +COPY --from=base /home/wamr/product-mini/platforms/linux/build/iwasm /usr/bin +COPY --from=base /home/wamr/product-mini/platforms/linux/build/lib*.so /usr/lib +ENV LD_LIBRARY_PATH=/usr/lib ENTRYPOINT [ "iwasm" ] diff --git a/core/iwasm/libraries/wasi-nn/test/Dockerfile.nvidia-gpu b/core/iwasm/libraries/wasi-nn/test/Dockerfile.nvidia-gpu index 49ee51a15e..8fe82510a6 100644 --- a/core/iwasm/libraries/wasi-nn/test/Dockerfile.nvidia-gpu +++ b/core/iwasm/libraries/wasi-nn/test/Dockerfile.nvidia-gpu @@ -14,14 +14,26 @@ WORKDIR /usr/local/share/ca-certificates/cacert.org RUN wget -qP /usr/local/share/ca-certificates/cacert.org http://www.cacert.org/certs/root.crt http://www.cacert.org/certs/class3.crt \ && update-ca-certificates +# need a newer cmake +RUN apt-get purge -y cmake + +ARG CMAKE_VER=3.27.0 +RUN wget https://github.com/Kitware/CMake/releases/download/v${CMAKE_VER}/cmake-${CMAKE_VER}-Linux-x86_64.sh \ + -q -O /tmp/cmake-install.sh \ + && chmod u+x /tmp/cmake-install.sh \ + && mkdir /opt/cmake-${CMAKE_VER} \ + && /tmp/cmake-install.sh --skip-license --prefix=/opt/cmake-${CMAKE_VER} \ + && rm /tmp/cmake-install.sh \ + && ln -s /opt/cmake-${CMAKE_VER}/bin/* /usr/local/bin + WORKDIR /home/wamr COPY . . RUN git config --global http.sslCAinfo /etc/ssl/certs/ca-certificates.crt -WORKDIR /home/wamr/product-mini/platforms/linux/build +WORKDIR /home/wamr/product-mini/platforms/linux RUN rm -rf build \ && cmake -S . -B build \ - -DWAMR_BUILD_WASI_NN=1 \ + -DWAMR_BUILD_WASI_NN=1 -DWAMR_BUILD_WASI_NN_TFLITE=1\ -DWAMR_BUILD_WASI_NN_ENABLE_GPU=1 \ && cmake --build build -j "$(grep -c ^processor /proc/cpuinfo)" @@ -40,8 +52,8 @@ RUN mkdir -p /etc/OpenCL/vendors && \ ENV NVIDIA_VISIBLE_DEVICES=all ENV NVIDIA_DRIVER_CAPABILITIES=compute,utility -COPY --from=base /home/wamr/product-mini/platforms/linux/build/iwasm /usr/bin/iwasm -COPY --from=base /home/wamr/product-mini/platforms/linux/build/libiwasm.so /lib/libiwasm.so -COPY --from=base /home/wamr/product-mini/platforms/linux/build/libwasi-nn-*.so /lib/ +COPY --from=base /home/wamr/product-mini/platforms/linux/build/iwasm /usr/bin +COPY --from=base /home/wamr/product-mini/platforms/linux/build/lib*.so /usr/lib +ENV LD_LIBRARY_PATH=/usr/lib ENTRYPOINT [ "iwasm" ] diff --git a/core/iwasm/libraries/wasi-nn/test/Dockerfile.tpu b/core/iwasm/libraries/wasi-nn/test/Dockerfile.tpu index a0ae29a63f..6a7dc15341 100644 --- a/core/iwasm/libraries/wasi-nn/test/Dockerfile.tpu +++ b/core/iwasm/libraries/wasi-nn/test/Dockerfile.tpu @@ -14,22 +14,35 @@ WORKDIR /usr/local/share/ca-certificates/cacert.org RUN wget -qP /usr/local/share/ca-certificates/cacert.org http://www.cacert.org/certs/root.crt http://www.cacert.org/certs/class3.crt \ && update-ca-certificates +# need a newer cmake +RUN apt-get purge -y cmake + +ARG CMAKE_VER=3.27.0 +RUN wget https://github.com/Kitware/CMake/releases/download/v${CMAKE_VER}/cmake-${CMAKE_VER}-Linux-x86_64.sh \ + -q -O /tmp/cmake-install.sh \ + && chmod u+x /tmp/cmake-install.sh \ + && mkdir /opt/cmake-${CMAKE_VER} \ + && /tmp/cmake-install.sh --skip-license --prefix=/opt/cmake-${CMAKE_VER} \ + && rm /tmp/cmake-install.sh \ + && ln -s /opt/cmake-${CMAKE_VER}/bin/* /usr/local/bin + WORKDIR /home/wamr COPY . . RUN git config --global http.sslCAinfo /etc/ssl/certs/ca-certificates.crt WORKDIR /home/wamr/product-mini/platforms/linux RUN rm -rf build \ - && cmake -S . -B build -DWAMR_BUILD_WASI_NN=1 \ - -DWAMR_BUILD_WASI_NN=1 \ + && cmake -S . -B build\ + -DWAMR_BUILD_WASI_NN=1\ + -DWAMR_BUILD_WASI_NN_TFLITE=1\ -DWAMR_BUILD_WASI_NN_ENABLE_EXTERNAL_DELEGATE=1 \ -DWAMR_BUILD_WASI_NN_EXTERNAL_DELEGATE_PATH="libedgetpu.so.1.0" \ -DWAMR_BUILD_WASI_NN_ENABLE_GPU=1 \ && cmake --build build -j "$(grep -c ^processor /proc/cpuinfo)" -RUN cp /home/wamr/product-mini/platforms/linux/build/iwasm /usr/bin/iwasm \ - && cp /home/wamr/product-mini/platforms/linux/build/libiwasm.so /lib/libiwasm.so \ - && cp /home/wamr/product-mini/platforms/linux/build/libwasi-nn-*.so /lib/ +RUN cp /home/wamr/core/iwasm/libraries/wasi-nn/test/build/iwasm /run/iwasm \ + && cp /home/wamr/product-mini/platforms/linux/build/lib*.so /usr/lib +ENV LD_LIBRARY_PATH=/usr/lib WORKDIR /assets ENTRYPOINT [ "iwasm" ] diff --git a/core/iwasm/libraries/wasi-nn/test/Dockerfile.vx-delegate b/core/iwasm/libraries/wasi-nn/test/Dockerfile.vx-delegate index f078045b94..e05b30119d 100644 --- a/core/iwasm/libraries/wasi-nn/test/Dockerfile.vx-delegate +++ b/core/iwasm/libraries/wasi-nn/test/Dockerfile.vx-delegate @@ -21,6 +21,18 @@ RUN apt-get update && apt-get install -y wget ca-certificates --no-install-recom && update-ca-certificates \ && git config --global http.sslCAinfo /etc/ssl/certs/ca-certificates.crt +# need a newer cmake +RUN apt-get purge -y cmake + +ARG CMAKE_VER=3.27.0 +RUN wget https://github.com/Kitware/CMake/releases/download/v${CMAKE_VER}/cmake-${CMAKE_VER}-Linux-x86_64.sh \ + -q -O /tmp/cmake-install.sh \ + && chmod u+x /tmp/cmake-install.sh \ + && mkdir /opt/cmake-${CMAKE_VER} \ + && /tmp/cmake-install.sh --skip-license --prefix=/opt/cmake-${CMAKE_VER} \ + && rm /tmp/cmake-install.sh \ + && ln -s /opt/cmake-${CMAKE_VER}/bin/* /usr/local/bin + # Build TensorFlow Lite VX delegate default built for x86-64 simulator WORKDIR /tmp RUN git clone https://github.com/VeriSilicon/TIM-VX.git tim-vx \ @@ -89,7 +101,6 @@ ENV VSIMULATOR_CONFIG=czl ENV LD_LIBRARY_PATH=/tmp/tim-vx/prebuilt-sdk/x86_64_linux/lib:/usr/local/lib:/lib/x86_64-linux-gnu/:/lib64/:/usr/lib:$LD_LIBRARY_PATH - # Build WASI-NN WORKDIR /home/wamr @@ -102,12 +113,14 @@ RUN cmake \ -DCMAKE_LIBRARY_PATH=${CMAKE_LIBRARY_PATH}:/usr/local/lib/ \ -DCMAKE_INCLUDE_PATH=${CMAKE_INCLUDE_PATH}:/usr/local/include/ \ -DWAMR_BUILD_WASI_NN=1 \ + -DWAMR_BUILD_WASI_NN_TFLITE=1\ -DWAMR_BUILD_WASI_NN_ENABLE_EXT=1 \ -DWASI_NN_EXT_DELEGATE_PATH="/usr/lib/libvx_delegate.so" \ .. RUN make -j "$(grep -c ^processor /proc/cpuinfo)" -RUN cp /home/wamr/core/iwasm/libraries/wasi-nn/test/build/iwasm /run/iwasm +RUN cp /home/wamr/core/iwasm/libraries/wasi-nn/test/build/iwasm /run/iwasm \ + && cp /home/wamr/product-mini/platforms/linux/build/lib*.so /usr/lib ENTRYPOINT [ "/run/iwasm" ] diff --git a/core/iwasm/libraries/wasi-nn/test/build.sh b/core/iwasm/libraries/wasi-nn/test/build.sh index da6cfa5f8e..dda400f161 100755 --- a/core/iwasm/libraries/wasi-nn/test/build.sh +++ b/core/iwasm/libraries/wasi-nn/test/build.sh @@ -8,9 +8,9 @@ CURR_PATH=$(cd $(dirname $0) && pwd -P) # WASM application that uses WASI-NN /opt/wasi-sdk/bin/clang \ + --target=wasm32-wasi \ + -DNN_LOG_LEVEL=1 \ -Wl,--allow-undefined \ - -Wl,--strip-all,--no-entry \ - --sysroot=/opt/wasi-sdk/share/wasi-sysroot \ -I../include -I../src/utils \ -o test_tensorflow.wasm \ test_tensorflow.c utils.c @@ -28,9 +28,9 @@ python3 sum.py cd ${CURR_PATH} /opt/wasi-sdk/bin/clang \ + --target=wasm32-wasi \ + -DNN_LOG_LEVEL=1 \ -Wl,--allow-undefined \ - -Wl,--strip-all,--no-entry \ - --sysroot=/opt/wasi-sdk/share/wasi-sysroot \ -I../include -I../src/utils \ -o test_tensorflow_quantized.wasm \ test_tensorflow_quantized.c utils.c diff --git a/core/iwasm/libraries/wasi-nn/test/requirements.txt b/core/iwasm/libraries/wasi-nn/test/requirements.txt index 177c22c19f..1643b91b00 100644 --- a/core/iwasm/libraries/wasi-nn/test/requirements.txt +++ b/core/iwasm/libraries/wasi-nn/test/requirements.txt @@ -1,2 +1,2 @@ tensorflow==2.12.1 -numpy==1.26.4 +numpy==1.24.4 diff --git a/core/iwasm/libraries/wasi-nn/test/utils.c b/core/iwasm/libraries/wasi-nn/test/utils.c index c499adc5b9..9e43ec9854 100644 --- a/core/iwasm/libraries/wasi-nn/test/utils.c +++ b/core/iwasm/libraries/wasi-nn/test/utils.c @@ -5,6 +5,7 @@ #include "utils.h" #include "logger.h" +#include "wasi_nn.h" #include #include @@ -57,7 +58,7 @@ wasm_load(char *model_name, graph *g, execution_target target) wasi_nn_error wasm_load_by_name(const char *model_name, graph *g) { - wasm_nn_error res = load_by_name(model_name, g); + wasi_nn_error res = load_by_name(model_name, g); return res; }