Merge remote-tracking branch 'origin/develop' into feature/stdfs

# Conflicts: # CMakeLists.txt # include/metall/basic_manager.hpp # include/metall/kernel/manager_kernel.hpp # include/metall/kernel/manager_kernel_impl.ipp # test/kernel/multimanager_test.cpp
LLNL · Jan 16, 2024 · a698e98 · a698e98
2 parents f24face + 3a0ae6f
commit a698e98
Show file tree

Hide file tree

Showing 23 changed files with 2,040 additions and 1,205 deletions.
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -92,6 +92,7 @@ set(COMPILER_DEFS "" CACHE STRING "A list of Metall compile definitions to be ad
 
 # ---------- Experimental options ---------- #
 set(UMAP_ROOT "" CACHE PATH "UMap installed root directory")
+set(PRIVATEER_ROOT "" CACHE PATH "Privateer installed root directory")
 
 option(ONLY_DOWNLOAD_GTEST "Only downloading Google Test" OFF)
 option(SKIP_DOWNLOAD_GTEST "Skip downloading Google Test" OFF)
@@ -179,6 +180,11 @@ if (UMAP_ROOT)
     endif ()
 endif ()
 
+# ---------- Privateer ---------- #
+if (PRIVATEER_ROOT)
+    message(STATUS "Privateer Root is: ${PRIVATEER_ROOT}")
+    find_library(LIBPRIVATEER NAMES privateer PATHS ${PRIVATEER_ROOT}/lib)
+endif ()
 
 # ---------- Boost ---------- #
 # Disable the boost-cmake feature (BoostConfig.cmake or boost-config.cmake) since
@@ -265,6 +271,37 @@ function(common_setup_for_metall_executable name)
         endif ()
     endif ()
     # --------------------
+
+    # ----- Privateer----- #
+    if (PRIVATEER_ROOT)
+        target_include_directories(${name} PRIVATE ${PRIVATEER_ROOT}/include)
+        if (LIBPRIVATEER)
+            # 1) Privateer Dependencies
+            FIND_PACKAGE(OpenSSL)
+            if (OpenSSL_FOUND)
+                target_link_libraries(${name} PRIVATE OpenSSL::SSL)
+                target_link_libraries(${name} PRIVATE OpenSSL::Crypto)
+            endif ()
+            target_link_libraries(${name} PRIVATE rt)
+            FIND_PACKAGE(OpenMP REQUIRED)
+            if (OpenMP_CXX_FOUND)
+                target_link_libraries(${name} PRIVATE OpenMP::OpenMP_CXX)
+            else ()
+                message(FATAL_ERROR "OpenMP is required to build Metall with Privateer")
+            endif ()
+            if (ZSTD_ROOT)
+                find_library(LIBZSTD NAMES zstd PATHS ${ZSTD_ROOT}/lib)
+                target_include_directories(${name} PRIVATE ${ZSTD_ROOT}/lib)
+                target_link_libraries(${name} PRIVATE ${LIBZSTD})
+                target_compile_definitions(${name} PRIVATE USE_COMPRESSION)
+            endif ()
+
+            # 2) Link Privateer
+            target_link_libraries(${name} PRIVATE ${LIBPRIVATEER})
+            target_compile_definitions(${name} PRIVATE METALL_USE_PRIVATEER)
+        endif ()
+    endif ()
+    # --------------------
 endfunction()
 
 function(add_metall_executable name source)

diff --git a/include/metall/basic_manager.hpp b/include/metall/basic_manager.hpp
diff --git a/include/metall/defs.hpp b/include/metall/defs.hpp
@@ -49,7 +49,7 @@
 #endif
 
 // --------------------
-// Macros for the default segment storage manager
+// Macros for the default segment storage
 // --------------------
 
 /// \def METALL_SEGMENT_BLOCK_SIZE

diff --git a/include/metall/detail/file.hpp b/include/metall/detail/file.hpp
@@ -179,7 +179,9 @@ inline bool create_file(const fs::path &file_path) {
   const int fd =
       ::open(file_path.c_str(), O_RDWR | O_CREAT | O_TRUNC, S_IRUSR | S_IWUSR);
   if (fd == -1) {
-    logger::perror(logger::level::error, __FILE__, __LINE__, "open");
+    std::stringstream ss;
+    ss << "Failed to create: " << file_path;
+    logger::perror(logger::level::error, __FILE__, __LINE__, ss.str().c_str());
     return false;
   }
 
@@ -456,8 +458,10 @@ inline bool copy_files_in_directory_in_parallel_helper(
     const std::function<bool(const fs::path &, const fs::path &)> &copy_func) {
   std::vector<fs::path> src_file_names;
   if (!get_regular_file_names(source_dir_path, &src_file_names)) {
+    std::stringstream ss;
+    ss << "Failed to get file list in " << source_dir_path;
     logger::out(logger::level::error, __FILE__, __LINE__,
-                "Failed to get file list");
+                ss.str().c_str());
     return false;
   }
 

diff --git a/include/metall/detail/mmap.hpp b/include/metall/detail/mmap.hpp
@@ -250,8 +250,9 @@ inline bool munmap(const int fd, void *const addr, const size_t length,
 }
 
 inline bool map_with_prot_none(void *const addr, const size_t length) {
-  return (os_mmap(addr, length, PROT_NONE, MAP_PRIVATE | MAP_ANONYMOUS, -1,
-                  0) == addr);
+  auto flag = MAP_PRIVATE | MAP_ANONYMOUS;
+  if (addr != nullptr) flag |= MAP_FIXED;
+  return (os_mmap(addr, length, PROT_NONE, flag, -1, 0) == addr);
 }
 
 inline bool os_mprotect(void *const addr, const size_t length, const int prot) {