Merge branch 'main' into main

.pipelines/ci.yml

@@ -851,10 +851,9 @@ stages:
         architecture: 'x64'
       displayName: "Use Python 3.12"
 
-    # iOS xcframework build doesn't work with CMake 3.25.1, pin to 3.25.0
     - script: |
-        python -m pip install cmake==3.25.0
-      displayName: "Install CMake 3.25.0"
+        python -m pip install cmake
+      displayName: "Install CMake"
 
     - template: templates/set-package-version-variable-step.yml
       parameters:

.pipelines/ios_packaging.yml

@@ -25,10 +25,9 @@ jobs:
       addToPath: true
       architecture: "x64"
 
-  # iOS xcframework build doesn't work with CMake 3.25.1, pin to 3.25.0
   - script: |
-      python -m pip install cmake==3.25.0
-    displayName: "Install CMake 3.25.0"
+      python -m pip install cmake
+    displayName: "Install CMake"
 
   - template: templates/install-appcenter.yml
 

.pipelines/templates/build-package-for-ios-cocoapods.yml

@@ -78,10 +78,9 @@ jobs:
       addToPath: true
       architecture: "x64"
 
-  # iOS xcframework build doesn't work with CMake 3.25.1, pin to 3.25.0
   - script: |
-      python -m pip install cmake==3.25.0
-    displayName: "Install CMake 3.25.0"
+      python -m pip install cmake
+    displayName: "Install CMake"
 
   - template: set-package-version-variable-step.yml
     parameters:

.pipelines/templates/build-package-for-linux.yml

@@ -39,7 +39,7 @@ jobs:
         sudo apt-get install -y libssh-dev
       displayName: Install OpenSSL for Azure custom ops
 
-    # NOTE: on arm64 machine, CMake version needs to be updated since we now require CMake 3.25 or newer.
+    # NOTE: on arm64 machine, CMake version needs to be updated since we now require CMake 3.28 or newer.
     - ${{ if eq(parameters.OrtExtensionsArch, 'x64') }}:
       - bash: |
           export CFLAGS="${{parameters.OrtExtensionsCFlags}}"

.pipelines/templates/ios-framework-build.yml

@@ -38,10 +38,9 @@ jobs:
           addToPath: true
           architecture: "x64"
 
-      # iOS xcframework build doesn't work with CMake 3.25.1, pin to 3.25.0
       - script: |
-          python -m pip install cmake==3.25.0
-        displayName: "Install CMake 3.25.0"
+          python -m pip install cmake
+        displayName: "Install CMake"
 
       - template: set-package-version-variable-step.yml
         parameters:

CMakeLists.txt

@@ -2,7 +2,7 @@
 # Licensed under the MIT License.
 
 # Minimum CMake required
-cmake_minimum_required(VERSION 3.25)
+cmake_minimum_required(VERSION 3.28)
 project(onnxruntime_extensions LANGUAGES C CXX)
 
 # set(CMAKE_VERBOSE_MAKEFILE ON)
@@ -334,6 +334,20 @@ endif()
 file(GLOB TARGET_SRC_NOEXCEPTION "base/*.h" "base/*.cc")
 file(GLOB TARGET_SRC "operators/*.cc" "operators/*.h" "include/*.h" "include/*.hpp")
 
+if(OCOS_ENABLE_DLIB)
+  set(DLIB_ISO_CPP_ONLY ON CACHE INTERNAL "")
+  set(DLIB_NO_GUI_SUPPORT ON CACHE INTERNAL "")
+  set(DLIB_USE_CUDA OFF CACHE INTERNAL "")
+  set(DLIB_USE_LAPACK OFF CACHE INTERNAL "")
+  set(DLIB_USE_BLAS OFF CACHE INTERNAL "")
+  include(dlib)
+
+  # Ideally, dlib should be included as file(GLOB TARGET_SRC_DLIB "${dlib_SOURCE_DIR}/dlib/all/source.cpp")
+  # To avoid the unintentional using some unwanted component, we only include the test_for_odr_violations.cpp
+  # to check if there is any violation in build configuration to ensure compiling some dlib source files correctly.
+  file(GLOB TARGET_SRC_DLIB "${dlib_SOURCE_DIR}/dlib/test_for_odr_violations.cpp")
+endif()
+
 if(OCOS_ENABLE_TF_STRING)
   set(farmhash_SOURCE_DIR ${PROJECT_SOURCE_DIR}/cmake/externals/farmhash)
   file(GLOB TARGET_SRC_KERNELS "operators/text/*.cc" "operators/text/*.h*")
@@ -342,8 +356,12 @@ if(OCOS_ENABLE_TF_STRING)
 endif()
 
 if(OCOS_ENABLE_AUDIO)
+  if (NOT OCOS_ENABLE_DLIB)
+    message(FATAL_ERROR "Audio operators require DLIB to be enabled.")
+  endif()
   include(dr_libs)
   file(GLOB TARGET_SRC_AUDIO "operators/audio/*.*")
+  list(APPEND TARGET_SRC_AUDIO ${dlib_SOURCE_DIR}/dlib/fft/fft.cpp)
   list(APPEND TARGET_SRC_NOEXCEPTION ${TARGET_SRC_AUDIO})
 endif()
 
@@ -353,20 +371,10 @@ if(OCOS_ENABLE_RE2_REGEX)
 endif()
 
 if(OCOS_ENABLE_MATH)
-  if(OCOS_ENABLE_DLIB)
-    set(DLIB_ISO_CPP_ONLY ON CACHE INTERNAL "")
-    set(DLIB_NO_GUI_SUPPORT ON CACHE INTERNAL "")
-    set(DLIB_USE_CUDA OFF CACHE INTERNAL "")
-    set(DLIB_USE_LAPACK OFF CACHE INTERNAL "")
-    set(DLIB_USE_BLAS OFF CACHE INTERNAL "")
-    include(dlib)
-
-    # Ideally, dlib should be included as
-    # file(GLOB TARGET_SRC_DLIB "${dlib_SOURCE_DIR}/dlib/all/source.cpp")
-    # To avoid the unintentional using some unwanted component, only include
-    file(GLOB TARGET_SRC_DLIB "${dlib_SOURCE_DIR}/dlib/test_for_odr_violations.cpp")
-    file(GLOB TARGET_SRC_INVERSE "operators/math/dlib/*.cc" "operators/math/dlib/*.h*")
+  if (NOT OCOS_ENABLE_DLIB)
+    message(FATAL_ERROR "Math operators require DLIB to be enabled.")
   endif()
+  file(GLOB TARGET_SRC_INVERSE "operators/math/dlib/*.cc" "operators/math/dlib/*.h*")
 
   file(GLOB TARGET_SRC_MATH "operators/math/*.cc" "operators/math/*.h*")
   if(OCOS_USE_CUDA)
@@ -658,12 +666,14 @@ if(OCOS_ENABLE_VISION)
     elseif(CMAKE_SYSTEM_NAME STREQUAL "Darwin" OR CMAKE_SYSTEM_NAME STREQUAL "iOS")
       # Use ImageIO on Apple platforms
       set(_DEFAULT_CODEC_ENABLE OFF)
-      target_link_libraries(ocos_operators PRIVATE "-framework CoreFoundation" "-framework CoreGraphics" "-framework ImageIO")
+      target_link_libraries(ocos_operators PRIVATE "-framework CoreFoundation" "-framework CoreGraphics" "-framework CoreServices" "-framework ImageIO")
     endif()
   endif()
 
-  set(_DEFAULT_CODEC_ENABLE ON) # libpng and libjpeg can be optional after EncodeImage with native support too.
   if(_DEFAULT_CODEC_ENABLE)
+    if (NOT OCOS_ENABLE_DLIB)
+      message(FATAL_ERROR "Vision operators require DLIB to be enabled.") # for now, we need dlib for image processing
+    endif()
     include(ext_imgcodecs)  
     target_include_directories(ocos_operators PUBLIC ${libPNG_SOURCE_DIR} ${libJPEG_SOURCE_DIR})
     target_link_libraries(ocos_operators PUBLIC ${PNG_LIBRARY} ${JPEG_LIBRARY})
@@ -752,7 +762,7 @@ if(OCOS_ENABLE_C_API)
     file(GLOB audio_TARGET_SRC "shared/api/c_api_feature_extraction.*" "shared/api/speech_*")
     list(APPEND _TARGET_LIB_SRC ${audio_TARGET_SRC})
   endif()
-  if(OCOS_ENABLE_DLIB)
+  if(OCOS_ENABLE_VISION)
     file(GLOB cv2_TARGET_SRC "shared/api/c_api_processor.*" "shared/api/image_*.*")
     list(APPEND _TARGET_LIB_SRC ${cv2_TARGET_SRC})
   endif()
@@ -867,8 +877,8 @@ target_link_libraries(ortcustomops PUBLIC ocos_operators)
 if(OCOS_BUILD_SHARED_LIB)
   file(GLOB shared_TARGET_SRC "shared/*.cc" "shared/*.h")
   if (OCOS_ENABLE_C_API)
-    if (NOT _HAS_TOKENIZER OR NOT OCOS_ENABLE_AUDIO)
-      message(FATAL_ERROR "Shared library build requires GPT2_TOKENIZER, AUDIO to be enabled.")
+    if (NOT _HAS_TOKENIZER OR NOT OCOS_ENABLE_AUDIO OR NOT OCOS_ENABLE_VISION)
+      message(FATAL_ERROR "Shared library build requires GPT2_TOKENIZER, AUDIO, VISION to be enabled.")
     endif()
     list(APPEND shared_TARGET_SRC "shared/extensions_c.def")
   else()

ThirdPartyNotices.txt

@@ -45,7 +45,7 @@ blingfire 0831265c1aca95ca02eca5bf1155e4251e545328
 
 _____
 
-dlib a12824d42584e292ecb3bad05c4b32c2015a7b89
+dlib v19.24.6
 
 Boost Software License - Version 1.0 - August 17th, 2003
 
@@ -106,7 +106,7 @@ Google Inc.
 Samsung Electronics
 Stefano Rivera <stefano.rivera@gmail.com>
 
-Dominic Battr� <battre@chromium.org>
+Dominic Battré <battre@chromium.org>
 Doug Kwan <dougkwan@google.com>
 Dmitriy Vyukov <dvyukov@google.com>
 John Millikin <jmillikin@gmail.com>

cgmanifest.json

@@ -34,7 +34,7 @@
       "component": {
         "type": "git",
         "git": {
-          "commitHash": "a12824d42584e292ecb3bad05c4b32c2015a7b89",
+          "commitHash": "v19.24.6",
           "repositoryUrl": "https://github.com/davisking/dlib.git"
         }
       }
@@ -87,6 +87,38 @@
         "comments": "v3.10.5"
       }
     },
+    {
+      "component": {
+        "type": "other",
+        "other": {
+          "name": "libjpeg",
+          "version": "release 9e of 16-Jan-2022",
+          "downloadUrl": "https://github.com/davisking/dlib/tree/v19.24.6/dlib/external/libjpeg"
+        },
+        "comments": "for vision domain"
+      }
+    },
+    {
+      "component": {
+        "type": "other",
+        "other": {
+          "name": "libpng",
+          "version": "1.6.37",
+          "downloadUrl": "https://github.com/davisking/dlib/tree/v19.24.6/dlib/external/libpng"
+        },
+        "comments": "for vision domain"
+      }
+    },
+    {
+      "component": {
+        "other": {
+          "name": "zlib",
+          "version": "1.2.11",
+          "downloadUrl": "https://github.com/davisking/dlib/tree/v19.24.6/dlib/external/zlib"
+        },
+        "comments": "for vision domain"
+      }
+    },
     {
       "component": {
         "type": "git",

cmake/externals/dlib.cmake

@@ -1,8 +1,8 @@
-FetchContent_Declare(dlib
-    GIT_REPOSITORY https://github.com/davisking/dlib.git
-    # there is non an official tag which supports STFT,
-    # choose a relatively stable commit id for that.
-    GIT_TAG        a12824d42584e292ecb3bad05c4b32c2015a7b89
+FetchContent_Declare(
+    dlib
+    URL https://github.com/davisking/dlib/archive/refs/tags/v19.24.6.zip
+    URL_HASH SHA1=59b1fb4e9909697c646e4f74e94871dacf49f0bf
+    DOWNLOAD_EXTRACT_TIMESTAMP TRUE
     SOURCE_SUBDIR  not_set
 )
 

include/ort_c_to_cpp.h

@@ -63,6 +63,25 @@ class API {
     instance()->ReleaseStatus(ptr);
   }
 
+  static OrtStatusPtr GetOpAttributeString(const OrtApi& api,
+    const OrtKernelInfo& info, const char* name, std::string& value) noexcept {
+    size_t size = 0;
+    OrtStatus* status = api.KernelInfoGetAttribute_string(&info, name, nullptr, &size);
+    if (status == nullptr) {
+      value.resize(size);
+      status = api.KernelInfoGetAttribute_string(&info, name, &value[0], &size);
+      value.resize(size - 1);  // remove the terminating character '\0'
+      if (status != nullptr) {
+        return status; // some unexpected error
+      }
+    } else {
+      // ignore the error, as the attribute is optional
+      api.ReleaseStatus(status);
+    }
+
+    return nullptr;
+  }
+
   template <typename T>
   static OrtStatusPtr KernelInfoGetAttribute(const OrtKernelInfo& info, const char* name, T& value) noexcept;
 

include/ortx_cpp_helper.h

@@ -17,6 +17,7 @@ class OrtxDeleter {
   }
 };
 
+
 /**
  * @brief A smart pointer class that manages the lifetime of an OrtxObject.
  * 
@@ -64,34 +65,27 @@ class OrtxObjectPtr : public std::unique_ptr<T, OrtxDeleter<T>> {
    */
   extError_t Code() const { return err_; }
 
- private:
-  extError_t err_ = kOrtxOK; /**< The error code associated with the creation of the OrtxObject. */
-};
+  struct PointerAssigner {
+    OrtxObject* obj_{};
+    OrtxObjectPtr<T>& ptr_;
+    PointerAssigner(OrtxObjectPtr<T>& ptr) : ptr_(ptr){};
 
-template <typename T>
-struct PointerAssigner {
-  OrtxObject* obj_{};
-  OrtxObjectPtr<T>& ptr_;
-  PointerAssigner(OrtxObjectPtr<T>& ptr) : ptr_(ptr){};
+    ~PointerAssigner() { ptr_.reset(static_cast<T*>(obj_)); };
 
-  ~PointerAssigner() { ptr_.reset(static_cast<T*>(obj_)); };
-
-  operator T**() { return reinterpret_cast<T**>(&obj_); };
-};
+    operator T**() { return reinterpret_cast<T**>(&obj_); };
+  };
 
 /**
- * @brief A wrapper function for OrtxObjectPtr that can be used as a function parameter on creation.
+ * @brief A wrapper function for OrtxObjectPtr that can be used as a function parameter of T**.
  * 
  * This function creates a PointerAssigner object for the given OrtxObjectPtr. The PointerAssigner
  * object can be used to assign a pointer value to the OrtxObjectPtr.
  * 
- * @tparam T The type of the object pointed to by the OrtxObjectPtr.
- * @param ptr The OrtxObjectPtr to create the PointerAssigner for.
- * @return A PointerAssigner object for the given OrtxObjectPtr.
  */
-template <typename T>
-PointerAssigner<T> ptr(OrtxObjectPtr<T>& ptr) {
-  return PointerAssigner<T>{ptr};
+  PointerAssigner ToBeAssigned() { return PointerAssigner{*this}; }
+
+ private:
+  extError_t err_ = kOrtxOK; /**< The error code associated with the creation of the OrtxObject. */
 };
 
 }  // namespace ort_extensions

include/ortx_extractor.h

@@ -6,7 +6,6 @@
 
 typedef OrtxObject OrtxFeatureExtractor;
 typedef OrtxObject OrtxRawAudios;
-typedef OrtxObject OrtxTensorResult;
 
 #ifdef __cplusplus
 extern "C" {

include/ortx_tokenizer.h

@@ -13,6 +13,33 @@ typedef OrtxObject OrtxStringArray;
 typedef OrtxObject OrtxTokenId2DArray;
 typedef OrtxObject OrtxDetokenizerCache;
 
+struct OrtxTokenizerBlob {
+  const char* config_json_blob;
+  const char* vocab_json_blob;
+  const char* token_module_blob;
+  const char* raw_model_blob;
+  const char* reserved_blob_1;
+
+  const size_t config_blob_len;
+  const size_t vocab_blob_len;
+  const size_t token_module_blob_len;
+  const size_t raw_model_blob_len;
+  const size_t reserved_blob_1_len;
+
+#ifdef __cplusplus
+  OrtxTokenizerBlob(const std::string_view& config_json_blob,
+                    const std::string_view& vocab_json_blob,
+                    const std::string_view& token_module_blob,
+                    const std::string_view& raw_model_blob)
+      : config_json_blob(config_json_blob.data()), vocab_json_blob(vocab_json_blob.data()),
+        token_module_blob(token_module_blob.data()), raw_model_blob(raw_model_blob.data()),
+        config_blob_len(config_json_blob.size()),
+        vocab_blob_len(vocab_json_blob.size()), token_module_blob_len(token_module_blob.size()),
+        raw_model_blob_len(raw_model_blob.size()), reserved_blob_1(nullptr),
+        reserved_blob_1_len(0) {}
+#endif
+};
+
 
 #ifdef __cplusplus
 extern "C" {
@@ -26,6 +53,15 @@ extern "C" {
  */
 extError_t ORTX_API_CALL OrtxCreateTokenizer(OrtxTokenizer** tokenizer, const char* tokenizer_path);
 
+/** \brief Create a tokenizer object with the specified tokenizer blob
+ *
+ * \param tokenizer Pointer to store the created tokenizer object
+ * \param tokenizer_blob Pointer to the tokenizer blob
+ * \return Error code indicating the success or failure of the operation
+ */
+extError_t ORTX_API_CALL OrtxCreateTokenizerFromBlob(OrtxTokenizer** tokenizer, const struct OrtxTokenizerBlob* tokenizer_blob);
+
+
 /** \brief Tokenize the input using the specified tokenizer
  *
  * \param tokenizer Pointer to the tokenizer object