From 22d30d0e8a93bef749a0a47d79db7507f693aae9 Mon Sep 17 00:00:00 2001 From: Keita Iwabuchi Date: Mon, 17 Apr 2023 16:28:22 -0700 Subject: [PATCH 01/12] (CI Test) Increase Boost to 1.82 --- .github/workflows/ci-test.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci-test.yml b/.github/workflows/ci-test.yml index 604e8d83..0e89a0a3 100644 --- a/.github/workflows/ci-test.yml +++ b/.github/workflows/ci-test.yml @@ -3,7 +3,7 @@ name: CI Test on: [push, pull_request] jobs: - bst1-80-0: + bst1-82-0: runs-on: ubuntu-latest env: METALL_LIMIT_MAKE_PARALLELS: 8 @@ -12,9 +12,9 @@ jobs: - name: Test run: | pushd /dev/shm - wget -q https://boostorg.jfrog.io/artifactory/main/release/1.80.0/source/boost_1_80_0.tar.gz + wget -q https://boostorg.jfrog.io/artifactory/main/release/1.82.0/source/boost_1_82_0.tar.gz mkdir boost - tar xf boost_1_80_0.tar.gz -C boost --strip-components 1 + tar xf boost_1_82_0.tar.gz -C boost --strip-components 1 export BOOST_ROOT=${PWD}/boost popd export METALL_TEST_DIR=${GITHUB_JOB} From aba0654f0f4cd36bb28ec93ca54c297a7bba44f9 Mon Sep 17 00:00:00 2001 From: Keita Iwabuchi Date: Mon, 17 Apr 2023 16:33:50 -0700 Subject: [PATCH 02/12] (github ci test) use gcc 12 --- .github/workflows/ci-test.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci-test.yml b/.github/workflows/ci-test.yml index 0e89a0a3..c2e69efd 100644 --- a/.github/workflows/ci-test.yml +++ b/.github/workflows/ci-test.yml @@ -18,8 +18,8 @@ jobs: export BOOST_ROOT=${PWD}/boost popd export METALL_TEST_DIR=${GITHUB_JOB} - export CC=gcc-10 - export CXX=g++-10 + export CC=gcc-12 + export CXX=g++-12 cd $GITHUB_WORKSPACE bash ./scripts/CI/build_and_test.sh @@ -38,7 +38,7 @@ jobs: export BOOST_ROOT=${PWD}/boost popd export METALL_TEST_DIR=${GITHUB_JOB} - export CC=gcc-10 - export CXX=g++-10 + export CC=gcc-12 + export CXX=g++-12 cd $GITHUB_WORKSPACE bash ./scripts/CI/build_and_test.sh From 21df93b825089734a460c2bf972ee2997aea1b08 Mon Sep 17 00:00:00 2001 From: Keita Iwabuchi Date: Mon, 17 Apr 2023 16:38:24 -0700 Subject: [PATCH 03/12] (github ci test) use checkout v3 --- .github/workflows/ci-test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci-test.yml b/.github/workflows/ci-test.yml index c2e69efd..6d7d0122 100644 --- a/.github/workflows/ci-test.yml +++ b/.github/workflows/ci-test.yml @@ -8,7 +8,7 @@ jobs: env: METALL_LIMIT_MAKE_PARALLELS: 8 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Test run: | pushd /dev/shm @@ -28,7 +28,7 @@ jobs: env: METALL_LIMIT_MAKE_PARALLELS: 8 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Test run: | pushd /dev/shm From 64108fa209562b8953c0809cc56d06ff05635f84 Mon Sep 17 00:00:00 2001 From: Keita Iwabuchi Date: Sat, 6 May 2023 10:57:12 -0700 Subject: [PATCH 04/12] (gitlab ci) add boost 1.82 and 1.81; use gcc 12. --- .gitlab-ci.yml | 86 +++++++++++++++++++++++++++++--------------------- 1 file changed, 50 insertions(+), 36 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index a041a5f4..9b3cd52d 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -27,6 +27,8 @@ install_boost: script: - hostname - pwd +# - spack install boost@1.82.0 +# - spack install boost@1.81.0 # - spack install boost@1.80.0 # - spack install boost@1.79.0 # - spack install boost@1.78.0 @@ -61,110 +63,122 @@ install_boost: - export METALL_TEST_DIR="/dev/shm/metall_test-${CI_CONCURRENT_ID}-${CI_PIPELINE_IID}" - srun -N1 -ppdebug bash ./scripts/CI/build_and_test.sh -build_gcc10.2.1_bst1.80.0: +build_gcc12.1.1_bst1.82.0: extends: .build variables: - GCC_VERSION: "10.2.1" + GCC_VERSION: "12.1.1" + BOOST_VERSION: "1.82.0" + +build_gcc12.1.1_bst1.81.0: + extends: .build + variables: + GCC_VERSION: "12.1.1" + BOOST_VERSION: "1.81.0" + +build_gcc12.1.1_bst1.80.0: + extends: .build + variables: + GCC_VERSION: "12.1.1" BOOST_VERSION: "1.80.0" -build_gcc10.2.1_bst1.79.0: +build_gcc12.1.1_bst1.79.0: extends: .build variables: - GCC_VERSION: "10.2.1" + GCC_VERSION: "12.1.1" BOOST_VERSION: "1.79.0" -build_gcc10.2.1_bst1.78.0: +build_gcc12.1.1_bst1.78.0: extends: .build variables: - GCC_VERSION: "10.2.1" + GCC_VERSION: "12.1.1" BOOST_VERSION: "1.78.0" -build_gcc10.2.1_bst1.77.0: +build_gcc12.1.1_bst1.77.0: extends: .build variables: - GCC_VERSION: "10.2.1" + GCC_VERSION: "12.1.1" BOOST_VERSION: "1.77.0" -build_gcc10.2.1_bst1.76.0: +build_gcc12.1.1_bst1.76.0: extends: .build variables: - GCC_VERSION: "10.2.1" + GCC_VERSION: "12.1.1" BOOST_VERSION: "1.76.0" -build_gcc10.2.1_bst1.75.0: +build_gcc12.1.1_bst1.75.0: extends: .build variables: - GCC_VERSION: "10.2.1" + GCC_VERSION: "12.1.1" BOOST_VERSION: "1.75.0" -build_gcc10.2.1_bst1.74.0: +build_gcc12.1.1_bst1.74.0: extends: .build variables: - GCC_VERSION: "10.2.1" + GCC_VERSION: "12.1.1" BOOST_VERSION: "1.74.0" -build_gcc10.2.1_bst1.73.0: +build_gcc12.1.1_bst1.73.0: extends: .build variables: - GCC_VERSION: "10.2.1" + GCC_VERSION: "12.1.1" BOOST_VERSION: "1.73.0" -build_gcc10.2.1_bst1.72.0: +build_gcc12.1.1_bst1.72.0: extends: .build variables: - GCC_VERSION: "10.2.1" + GCC_VERSION: "12.1.1" BOOST_VERSION: "1.72.0" -build_gcc10.2.1_bst1.71.0: +build_gcc12.1.1_bst1.71.0: extends: .build variables: - GCC_VERSION: "10.2.1" + GCC_VERSION: "12.1.1" BOOST_VERSION: "1.71.0" -build_gcc10.2.1_bst1.70.0: +build_gcc12.1.1_bst1.70.0: extends: .build variables: - GCC_VERSION: "10.2.1" + GCC_VERSION: "12.1.1" BOOST_VERSION: "1.70.0" -build_gcc10.2.1_bst1.69.0: +build_gcc12.1.1_bst1.69.0: extends: .build variables: - GCC_VERSION: "10.2.1" + GCC_VERSION: "12.1.1" BOOST_VERSION: "1.69.0" -build_gcc10.2.1_bst1.68.0: +build_gcc12.1.1_bst1.68.0: extends: .build variables: - GCC_VERSION: "10.2.1" + GCC_VERSION: "12.1.1" BOOST_VERSION: "1.68.0" -build_gcc10.2.1_bst1.67.0: +build_gcc12.1.1_bst1.67.0: extends: .build variables: - GCC_VERSION: "10.2.1" + GCC_VERSION: "12.1.1" BOOST_VERSION: "1.67.0" -build_gcc10.2.1_bst1.66.0: +build_gcc12.1.1_bst1.66.0: extends: .build variables: - GCC_VERSION: "10.2.1" + GCC_VERSION: "12.1.1" BOOST_VERSION: "1.66.0" -build_gcc10.2.1_bst1.65.1: +build_gcc12.1.1_bst1.65.1: extends: .build variables: - GCC_VERSION: "10.2.1" + GCC_VERSION: "12.1.1" BOOST_VERSION: "1.65.1" -build_gcc10.2.1_bst1.65.0: +build_gcc12.1.1_bst1.65.0: extends: .build variables: - GCC_VERSION: "10.2.1" + GCC_VERSION: "12.1.1" BOOST_VERSION: "1.65.0" -build_gcc10.2.1_bst1.64.0: +build_gcc12.1.1_bst1.64.0: extends: .build variables: - GCC_VERSION: "10.2.1" + GCC_VERSION: "12.1.1" BOOST_VERSION: "1.64.0" \ No newline at end of file From 6d53894e1f14d1f1d0f54a001da8029d5580cf66 Mon Sep 17 00:00:00 2001 From: Keita Iwabuchi Date: Sat, 6 May 2023 10:59:30 -0700 Subject: [PATCH 05/12] [skip ci] (JSON) fix doxygen document --- include/metall/json/json_fwd.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/metall/json/json_fwd.hpp b/include/metall/json/json_fwd.hpp index 5346b796..99000f82 100644 --- a/include/metall/json/json_fwd.hpp +++ b/include/metall/json/json_fwd.hpp @@ -17,7 +17,7 @@ #define METALL_LINK_WITH_BOOST_JSON /// \brief Include guard for boost/json/src.hpp -#define METALL_JSON_BOOST_JSON_SRC_INCLUDED +#define METALL_BOOST_JSON_SRC_INCLUDED #endif #if METALL_LINK_WITH_BOOST_JSON From fca9610e1de65172fd07e215a6d91ee10b423915 Mon Sep 17 00:00:00 2001 From: Keita Iwabuchi Date: Sat, 6 May 2023 11:26:35 -0700 Subject: [PATCH 06/12] (gitlab ci test) automatically find spack arch info. --- .gitlab-ci.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 9b3cd52d..421277ce 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -16,7 +16,6 @@ stages: variables: GTEST_COLOR: "1" GIT_DEPTH: 0 - SPACK_ARCH_INFO: "arch=linux-rhel7-broadwell" install_boost: @@ -59,7 +58,7 @@ install_boost: script: - echo "=== build section ===" - module load gcc/${GCC_VERSION} - - spack load boost@${BOOST_VERSION} ${SPACK_ARCH_INFO} + - spack load boost@${BOOST_VERSION} arch=$(spack arch) - export METALL_TEST_DIR="/dev/shm/metall_test-${CI_CONCURRENT_ID}-${CI_PIPELINE_IID}" - srun -N1 -ppdebug bash ./scripts/CI/build_and_test.sh From 10c557472d75d3073427632cd7641fc4a7296aab Mon Sep 17 00:00:00 2001 From: Keita Iwabuchi Date: Sat, 6 May 2023 11:48:04 -0700 Subject: [PATCH 07/12] (gitlab ci test) Install Boost on the fly. --- .gitlab-ci.yml | 25 ++----------------------- 1 file changed, 2 insertions(+), 23 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 421277ce..1087c231 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -25,29 +25,8 @@ install_boost: - shell script: - hostname - - pwd -# - spack install boost@1.82.0 -# - spack install boost@1.81.0 -# - spack install boost@1.80.0 -# - spack install boost@1.79.0 -# - spack install boost@1.78.0 -# - spack install boost@1.77.0 -# - spack install boost@1.76.0 -# - spack install boost@1.75.0 -# - spack install boost@1.74.0 -# - spack install boost@1.73.0 -# - spack install boost@1.72.0 -# - spack install boost@1.71.0 -# - spack install boost@1.70.0 -# - spack install boost@1.69.0 -# - spack install boost@1.68.0 -# - spack install boost@1.67.0 -# - spack install boost@1.66.0 -# - spack install boost@1.65.1 -# - spack install boost@1.65.0 -# - spack install boost@1.64.0 -# - spack clean - - spack find + - srun -N1 -ppdebug spack install boost@1.82.0 boost@1.81.0 boost@1.80.0 boost@1.79.0 boost@1.78.0 boost@1.77.0 boost@1.76.0 boost@1.75.0 boost@1.74.0 boost@1.73.0 boost@1.72.0 boost@1.71.0 boost@1.70.0 boost@1.69.0 boost@1.68.0 boost@1.67.0 boost@1.66.0 boost@1.65.1 boost@1.65.0 boost@1.64.0 + - spack clean .build: From ebc0dda38231217c2c2326e36dc31df19deab0fe Mon Sep 17 00:00:00 2001 From: Keita Iwabuchi Date: Mon, 15 May 2023 18:17:07 -0700 Subject: [PATCH 08/12] Update examples. --- example/CMakeLists.txt | 2 + example/README.md | 59 +++++++++++++---- example/allocator_aware_type.cpp | 102 ++++++++++++++++++++++++++++++ example/complex_map.cpp | 7 +- example/multilevel_containers.cpp | 11 +--- example/simple.cpp | 6 +- example/string_map.cpp | 16 ++--- include/metall/metall.hpp | 4 ++ 8 files changed, 168 insertions(+), 39 deletions(-) create mode 100644 example/allocator_aware_type.cpp diff --git a/example/CMakeLists.txt b/example/CMakeLists.txt index 5c847020..a665f17c 100644 --- a/example/CMakeLists.txt +++ b/example/CMakeLists.txt @@ -32,6 +32,8 @@ add_metall_executable(concurrent_map concurrent_map.cpp) add_metall_executable(metall_containers metall_containers.cpp) +add_metall_executable(allocator_aware_type allocator_aware_type.cpp) + if (BUILD_C) add_c_executable(c_api c_api.c) target_link_libraries(c_api PRIVATE metall_c) diff --git a/example/README.md b/example/README.md index c4d9a24b..99791d07 100644 --- a/example/README.md +++ b/example/README.md @@ -1,44 +1,69 @@ -****# List of Examples +# List of Examples To build examples see a [page](https://metall.readthedocs.io/en/latest/advanced_build/cmake/) hosted on Read the Docs. ### Basic examples * [simple.cpp](simple.cpp) - * Allocating a vector container. -* [vector_of_vectors.cpp](vector_of_vectors.cpp) - * Nested (multi-level) containers. + * Allocating a vector container. + +* [snapshot.cpp](snapshot.cpp) + + * Taking snapshot * [offset_pointer.cpp](offset_pointer.cpp) - * Using the offset pointer to store pointer persistently. -* [snapshot.cpp](snapshot.cpp) - * Taking snapshot + * Using the offset pointer to store pointer persistently. + + +### STL Container with Custom Allocator + +In the examples below, we occasionally use STL-compatible containers exist in [metall/container/](../include/metall/container/) for better readability. +Those containers are just alias of the corresponding containers in Boost.Container and do not have Metall-specific features. +Therefore, the data structures in the examples below are more like how to use STL containers with custom allocators rather than how to use containers with just Metall. * [string.cpp](string.cpp) - * How to use the STL string with Metal + + * STL string container with Metall. + +* [vector_of_vectors.cpp](vector_of_vectors.cpp) + + * Nested (multi-level) containers. + +* [multilevel_containers.cpp](multilevel_containers.cpp) + + * Another multi-level STL container example. * [string_map.cpp](string_map.cpp) - * How to use a STL map container of which key is STL string + + * How to use an STL map container of which key is STL string. * [complex_map.cpp](complex_map.cpp) - * A complex STL map container with Metall -* [multilevel_containers.cpp](multilevel_containers.cpp) - * A multi-level STL container with Metall + * A complex STL map container with Metall. + +* [allocator_aware_type.cpp](allocator_aware_type.cpp) + + * How to implement your own container (data structure) that A) can be stored in another container; B) holds another container inside. + + +### Metall (advanced) * [datastore_description.cpp](datastore_description.cpp) + * Set and get datastore description * [object_attribute.cpp](object_attribute.cpp) + * How to access object attributes * [object_attribute_api_list.cpp](object_attribute_api_list.cpp) + * List of API to manipulate object attributes -### Metall Container +### Metall's Original Container * [metall_containers.cpp](metall_containers.cpp) * List of containers that use Metall at the default allocators. @@ -46,24 +71,29 @@ To build examples see a [page](https://metall.readthedocs.io/en/latest/advanced_ ### Graph * [csr_graph.cpp](csr_graph.cpp) + * A CSR graph data structure (two 1D arrays) with Metall * [adjacency_list_graph.cpp](adjacency_list_graph.cpp) + * An adjacency-lis data structure (multi-level STL container) with Metall ### Concurrent Data Structure (experimental implementation) * [static_mutex.cpp](static_mutex.cpp) + * How to use our [mutex_lock function](../include/metall/utility/mutex.hpp) * [concurrent_map.cpp](concurrent_map.cpp) + * How to use our [persistent concurrent_map class](../include/metall/container/concurrent_map.hpp) ### Fallback Allocator * [fallback_allocator_adaptor.cpp](fallback_allocator_adaptor.cpp) + * How to use [fallback_allocator_adaptor](../include/metall/utility/fallback_allocator_adaptor.hpp) @@ -77,5 +107,6 @@ To build examples see a [page](https://metall.readthedocs.io/en/latest/advanced_ * One can set a MPI CXX compiler to use by using 'MPI_CXX_COMPILE' CMake option. -## C API +### C API + * [c_api.cpp](c_api.c) diff --git a/example/allocator_aware_type.cpp b/example/allocator_aware_type.cpp new file mode 100644 index 00000000..9a5cfc88 --- /dev/null +++ b/example/allocator_aware_type.cpp @@ -0,0 +1,102 @@ +// Copyright 2023 Lawrence Livermore National Security, LLC and other Metall +// Project Developers. See the top-level COPYRIGHT file for details. +// +// SPDX-License-Identifier: (Apache-2.0 OR MIT) + +// This example shows how to create an allocator-aware type, which takes an +// allocator and can be stored inside an STL container. + +#include +#include + +#include +#include + +#include + +// A simple allocator-aware class, which uses an allocator to allocate internal +// contents and can be stored in an STL container. +// +// This class holds a string (i.e., STL container), which requires an allocator. +// 'Alloc' can be any allocator type compatible with the STL allocator +// requirement. +// +// This class is implemented just following STL's standards — no special +// requirements by Metall. We use Metall container for convenience but another +// string container, such as boost::container::string, is okay. +template +struct key_value_pair { + public: + // First key point of this example + // Required to be an allocator-aware type + using allocator_type = Alloc; + + // Use the given allocator type in the key + using key_type = metall::container::basic_string< + char, std::char_traits, + typename std::allocator_traits::template rebind_alloc< + char>>; + + // Default constructor + explicit key_value_pair(const allocator_type &alloc = allocator_type()) + : key(alloc) {} + + // Copy and move constructors + key_value_pair(const key_value_pair &) = default; + key_value_pair(key_value_pair &&) = default; + + // Here is another key point of this example. + // Allocator-extended copy and move constructors + key_value_pair(const key_value_pair &other, const allocator_type &alloc) + : key(other.key, alloc) {} + key_value_pair(key_value_pair &&other, const allocator_type &alloc) + : key(std::move(other.key), alloc) {} + + // Assignment operators. + // Not required to be an allocator-aware type, but we implement them for + // the purpose of this example. + key_value_pair &operator=(const key_value_pair &) = default; + key_value_pair &operator=(key_value_pair &&) = default; + + key_type key; + int value; +}; + +// We use Metall in this example but any STL compatible allocator can be used. +template +using alloc_t = metall::manager::allocator_type; + +using kv_t = key_value_pair>; + +// A vector of kv_t objects. +// Use scoped_allocator_adaptor to wrap up an actual allocator type so that +// the inner containers (allocator-aware types) use the same allocator. See: +// https://en.cppreference.com/w/cpp/memory/scoped_allocator_adaptor +using vec_t = + metall::container::vector>>; + +int main() { + { + metall::manager manager(metall::create_only, "/tmp/metall-dir"); + auto *vec = manager.construct("vec")(manager.get_allocator()); + + vec->resize(2); + + vec->at(0).key = "key0"; + vec->at(0).value = 10; + + vec->at(1).key = "key1"; + vec->at(1).value = 100; + } + + { + metall::manager manager(metall::open_read_only, "/tmp/metall-dir"); + auto *vec = manager.find("vec").first; + for (const auto &e : *vec) { + std::cout << e.key << " : " << e.value << std::endl; + } + } + + return 0; +} \ No newline at end of file diff --git a/example/complex_map.cpp b/example/complex_map.cpp index 62e28fe4..8c9849cf 100644 --- a/example/complex_map.cpp +++ b/example/complex_map.cpp @@ -22,7 +22,6 @@ using key_type = int; // Mapped type template struct mapped_type { - public: using allocator_type = _allocator_type; bc::vector> vec; @@ -56,13 +55,11 @@ int main() { auto pmap = manager.construct("map")(manager.get_allocator<>()); - (*pmap)[0] = metall_map_type::mapped_type(manager.get_allocator<>()); + (*pmap)[0]; pmap->at(0).vec.push_back(0); pmap->at(0).str = "hello, world 0"; - pmap->try_emplace(1); // Scoped_allocator_adopter passes an allocator - // object to mapped_type? pmap->try_emplace(1, - // manager.get_allocator<>()); does not work? + pmap->try_emplace(1); pmap->at(1).vec.push_back(1); pmap->at(1).str = "hello, world 1"; } diff --git a/example/multilevel_containers.cpp b/example/multilevel_containers.cpp index 1e94a487..bba0d3b9 100644 --- a/example/multilevel_containers.cpp +++ b/example/multilevel_containers.cpp @@ -53,18 +53,13 @@ using map_type = int main() { { // Create a new metall object - // Note that Metall creates sparse files so that it does not use physical - // memory space until required - metall::manager manager(metall::create_only, - "/tmp/datastore"); // Prefix of the backing files + metall::manager manager(metall::create_only, "/tmp/datastore"); // Construct an object of map_type map_type* pmap = - manager.construct("map") // Name of the constructed object - (manager.get_allocator<>()); // Call map_type's constructor with Metall - // allocator object + manager.construct("map")(manager.get_allocator<>()); - // You can use the container as usual // + // Can use the container as usual vector_type vec1(manager.get_allocator()); vec1.push_back('a'); (*pmap)[30].emplace(20, vec1); diff --git a/example/simple.cpp b/example/simple.cpp index 912fc9af..19eb7d22 100644 --- a/example/simple.cpp +++ b/example/simple.cpp @@ -35,7 +35,9 @@ int main() { // ---------- Assume exit and restart the program at this point ---------- // - { + /// consistent() returns true if a Metall data store exits at the path and + /// was closed property. + if (metall::manager::consistent("/tmp/dir")) { // Reattach the manager instance metall::manager manager(metall::open_only, "/tmp/dir"); @@ -49,6 +51,8 @@ int main() { std::cout << (*pvec)[1] << std::endl; // Will print "10" manager.destroy("vec"); // Destroy the instance + } else { + std::cerr << "Cannot open a Metall data store" << std::endl; } return 0; diff --git a/example/string_map.cpp b/example/string_map.cpp index c3b30b37..0486857c 100644 --- a/example/string_map.cpp +++ b/example/string_map.cpp @@ -4,7 +4,6 @@ // SPDX-License-Identifier: (Apache-2.0 OR MIT) #include -#include #include #include #include @@ -18,9 +17,8 @@ void string_int_map() { using value_type = std::pair; // Mapped type using string_int_map = - boost::container::map, - metall::manager::allocator_type>; + boost::container::map, + metall::manager::scoped_allocator_type>; { metall::manager manager(metall::create_only, "/tmp/datastore"); @@ -50,13 +48,9 @@ void int_string_map() { using value_type = std::pair; // Mapped type - // Scoped allocator is required to force to use user given allocators in inner - // containers - using map_allocator_type = boost::container::scoped_allocator_adaptor< - metall::manager::allocator_type>; - using int_string_map = - boost::container::map, - map_allocator_type>; + using map_allocator_type = metall::manager::scoped_allocator_type; + using int_string_map = boost::container::map, map_allocator_type>; { metall::manager manager(metall::create_only, "/tmp/datastore"); diff --git a/include/metall/metall.hpp b/include/metall/metall.hpp index d725a462..1b23f46b 100644 --- a/include/metall/metall.hpp +++ b/include/metall/metall.hpp @@ -26,6 +26,10 @@ using manager = basic_manager<>; /// \example multilevel_containers.cpp /// This is an example of how to use a multi-level STL container with Metall. +/// \example allocator_aware_type.cpp +/// This example shows how to create an allocator-aware type, which takes an +/// allocator and can be stored inside an STL container. + /// \example simple.cpp /// This is a simple example of how to use Metall. From 7b283e9ed7343d522b1e6dae3b7d9952f8f486c7 Mon Sep 17 00:00:00 2001 From: Keita Iwabuchi Date: Mon, 12 Jun 2023 19:18:32 -0700 Subject: [PATCH 09/12] Add Boost open-addressing set/map containers --- example/metall_containers.cpp | 29 ++++++++++++++++++- .../metall/container/unordered_flat_map.hpp | 28 ++++++++++++++++++ .../metall/container/unordered_flat_set.hpp | 28 ++++++++++++++++++ .../metall/container/unordered_node_map.hpp | 28 ++++++++++++++++++ .../metall/container/unordered_node_set.hpp | 28 ++++++++++++++++++ 5 files changed, 140 insertions(+), 1 deletion(-) create mode 100644 include/metall/container/unordered_flat_map.hpp create mode 100644 include/metall/container/unordered_flat_set.hpp create mode 100644 include/metall/container/unordered_node_map.hpp create mode 100644 include/metall/container/unordered_node_set.hpp diff --git a/example/metall_containers.cpp b/example/metall_containers.cpp index c1077f82..1ab379d6 100644 --- a/example/metall_containers.cpp +++ b/example/metall_containers.cpp @@ -16,6 +16,18 @@ #include #include +// Boost 1.81 or later is required +#if BOOST_VERSION >= 108100 +#include +#include +#endif + +// Boost 1.82 or later is required +#if BOOST_VERSION >= 108200 +#include +#include +#endif + using namespace metall; namespace mc = metall::container; @@ -33,7 +45,8 @@ int main() { mg.construct>("set")(mg.get_allocator()); mg.construct>("multiset")(mg.get_allocator()); - mg.construct>("umap")(mg.get_allocator()); + mg.construct>("unordered_map")( + mg.get_allocator()); mg.construct>("unordered_multimap")( mg.get_allocator()); @@ -41,6 +54,20 @@ int main() { mg.construct>("unordered_multiset")( mg.get_allocator()); +#if BOOST_VERSION >= 108100 + mg.construct>("unordered_flat_map")( + mg.get_allocator()); + mg.construct>("unordered_flat_set")( + mg.get_allocator()); +#endif + +#if BOOST_VERSION >= 108200 + mg.construct>("unordered_node_map")( + mg.get_allocator()); + mg.construct>("unordered_node_set")( + mg.get_allocator()); +#endif + mg.construct>("vector")(mg.get_allocator()); mg.construct>("stack")(mg.get_allocator()); diff --git a/include/metall/container/unordered_flat_map.hpp b/include/metall/container/unordered_flat_map.hpp new file mode 100644 index 00000000..ecca824c --- /dev/null +++ b/include/metall/container/unordered_flat_map.hpp @@ -0,0 +1,28 @@ +// Copyright 2023 Lawrence Livermore National Security, LLC and other Metall +// Project Developers. See the top-level COPYRIGHT file for details. +// +// SPDX-License-Identifier: (Apache-2.0 OR MIT) + +#ifndef METALL_CONTAINER_UNORDERED_FLAT_MAP_HPP +#define METALL_CONTAINER_UNORDERED_FLAT_MAP_HPP + +#include + +static_assert(BOOST_VERSION >= 108100, "Unsupported Boost version"); +#include + +#include + +namespace metall::container { + +/// \brief An unordered_flat_map container that uses Metall as its default +/// allocator. +template , + class KeyEqual = std::equal_to, + class Allocator = manager::allocator_type>> +using unordered_flat_map = + boost::unordered_flat_map; + +} // namespace metall::container + +#endif // METALL_CONTAINER_UNORDERED_FLAT_MAP_HPP diff --git a/include/metall/container/unordered_flat_set.hpp b/include/metall/container/unordered_flat_set.hpp new file mode 100644 index 00000000..24c63264 --- /dev/null +++ b/include/metall/container/unordered_flat_set.hpp @@ -0,0 +1,28 @@ +// Copyright 2023 Lawrence Livermore National Security, LLC and other Metall +// Project Developers. See the top-level COPYRIGHT file for details. +// +// SPDX-License-Identifier: (Apache-2.0 OR MIT) + +#ifndef METALL_CONTAINER_UNORDERED_FLAT_SET_HPP +#define METALL_CONTAINER_UNORDERED_FLAT_SET_HPP + +#include + +static_assert(BOOST_VERSION >= 108100, "Unsupported Boost version"); +#include + +#include + +namespace metall::container { + +/// \brief An unordered_flat_set container that uses Metall as its default +/// allocator. +template , + class KeyEqual = std::equal_to, + class Allocator = manager::allocator_type> +using unordered_flat_set = + boost::unordered_flat_set; + +} // namespace metall::container + +#endif // METALL_CONTAINER_UNORDERED_FLAT_SET_HPP diff --git a/include/metall/container/unordered_node_map.hpp b/include/metall/container/unordered_node_map.hpp new file mode 100644 index 00000000..07af8ea8 --- /dev/null +++ b/include/metall/container/unordered_node_map.hpp @@ -0,0 +1,28 @@ +// Copyright 2023 Lawrence Livermore National Security, LLC and other Metall +// Project Developers. See the top-level COPYRIGHT file for details. +// +// SPDX-License-Identifier: (Apache-2.0 OR MIT) + +#ifndef METALL_CONTAINER_UNORDERED_NODE_MAP_HPP +#define METALL_CONTAINER_UNORDERED_NODE_MAP_HPP + +#include + +static_assert(BOOST_VERSION >= 108200, "Unsupported Boost version"); +#include + +#include + +namespace metall::container { + +/// \brief An unordered_node_map container that uses Metall as its default +/// allocator. +template , + class KeyEqual = std::equal_to, + class Allocator = manager::allocator_type>> +using unordered_node_map = + boost::unordered_node_map; + +} // namespace metall::container + +#endif // METALL_CONTAINER_UNORDERED_NODE_MAP_HPP diff --git a/include/metall/container/unordered_node_set.hpp b/include/metall/container/unordered_node_set.hpp new file mode 100644 index 00000000..2616a12e --- /dev/null +++ b/include/metall/container/unordered_node_set.hpp @@ -0,0 +1,28 @@ +// Copyright 2023 Lawrence Livermore National Security, LLC and other Metall +// Project Developers. See the top-level COPYRIGHT file for details. +// +// SPDX-License-Identifier: (Apache-2.0 OR MIT) + +#ifndef METALL_CONTAINER_UNORDERED_NODE_SET_HPP +#define METALL_CONTAINER_UNORDERED_NODE_SET_HPP + +#include + +static_assert(BOOST_VERSION >= 108200, "Unsupported Boost version"); +#include + +#include + +namespace metall::container { + +/// \brief An unordered_node_set container that uses Metall as its default +/// allocator. +template , + class KeyEqual = std::equal_to, + class Allocator = manager::allocator_type> +using unordered_node_set = + boost::unordered_node_set; + +} // namespace metall::container + +#endif // METALL_CONTAINER_UNORDERED_NODE_SET_HPP From 38bd902a87e30fc7edd2d5983ce7cbd33d7f8ea1 Mon Sep 17 00:00:00 2001 From: Keita Iwabuchi Date: Fri, 16 Jun 2023 21:43:18 -0700 Subject: [PATCH 10/12] Remove warnings and brush up on code --- example/datastore_description.cpp | 2 +- .../container/experimental/jgraph/jgraph.hpp | 2 +- include/metall/container/string_key_store.hpp | 2 +- include/metall/detail/hash.hpp | 73 ++++++++++++++----- include/metall/detail/utilities.hpp | 19 +++++ .../segment_storage/mmap_segment_storage.hpp | 7 +- 6 files changed, 79 insertions(+), 26 deletions(-) diff --git a/example/datastore_description.cpp b/example/datastore_description.cpp index b0d9b98e..21df681b 100644 --- a/example/datastore_description.cpp +++ b/example/datastore_description.cpp @@ -15,7 +15,7 @@ int main() { metall::manager manager(metall::create_only, "/tmp/dir"); std::string buf; - const auto ret = manager.get_description(&buf); + [[maybe_unused]] const auto ret = manager.get_description(&buf); assert(!ret && buf.empty()); // 'ret' is false and 'buf' is empty since // description has not been created yet. diff --git a/include/metall/container/experimental/jgraph/jgraph.hpp b/include/metall/container/experimental/jgraph/jgraph.hpp index 5d415623..a8fccd7e 100644 --- a/include/metall/container/experimental/jgraph/jgraph.hpp +++ b/include/metall/container/experimental/jgraph/jgraph.hpp @@ -440,7 +440,7 @@ class jgraph { internal_id_type priv_generate_edge_id() { return ++m_max_edge_id; } static internal_id_type priv_hash_id(const std::string_view &id) { - return metall::mtlldetail::MurmurHash64A(id.data(), id.length(), 1234); + return metall::mtlldetail::murmur_hash_64a(id.data(), id.length(), 1234); } vertex_storage_type m_vertex_storage; diff --git a/include/metall/container/string_key_store.hpp b/include/metall/container/string_key_store.hpp index ef48567d..c707dbf5 100644 --- a/include/metall/container/string_key_store.hpp +++ b/include/metall/container/string_key_store.hpp @@ -329,7 +329,7 @@ class string_key_store { #ifdef METALL_CONTAINER_STRING_KEY_STORE_USE_SIMPLE_HASH internal_id_type hash = key.empty() ? 0 : (uint8_t)key[0] % 2; #else - auto hash = (internal_id_type)metall::mtlldetail::MurmurHash64A( + auto hash = (internal_id_type)metall::mtlldetail::murmur_hash_64a( key.data(), (int)key.length(), seed); #endif if (hash == k_max_internal_id) { diff --git a/include/metall/detail/hash.hpp b/include/metall/detail/hash.hpp index abc52dbc..241bc219 100644 --- a/include/metall/detail/hash.hpp +++ b/include/metall/detail/hash.hpp @@ -1,4 +1,4 @@ -// Copyright 2020 Lawrence Livermore National Security, LLC and other Metall +// Copyright 2023 Lawrence Livermore National Security, LLC and other Metall // Project Developers. See the top-level COPYRIGHT file for details. // // SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -8,20 +8,47 @@ #include +#include + namespace metall::mtlldetail { // ----------------------------------------------------------------------------- -// This also contains public domain code from MurmurHash2. +// This file contains public domain code from MurmurHash2. // From the MurmurHash2 header: // // MurmurHash2 was written by Austin Appleby, and is placed in the public // domain. The author hereby disclaims copyright to this source code. -// //----------------------------------------------------------------------------- -inline constexpr uint64_t MurmurHash64A(const void *key, const int len, - const uint64_t seed) { - constexpr uint64_t m = 0xc6a4a7935bd1e995ULL; - constexpr int r = 47; + +namespace mmhdtl { +//----------------------------------------------------------------------------- +// Block read - on little-endian machines this is a single load, +// while on big-endian or unknown machines the byte accesses should +// still get optimized into the most efficient instruction. +static inline uint64_t murmurhash_getblock(const uint64_t *p) { +#if defined(__BYTE_ORDER__) && (__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__) + return *p; +#else + const uint8_t *c = (const uint8_t *)p; + return (uint64_t)c[0] | (uint64_t)c[1] << 8 | (uint64_t)c[2] << 16 | + (uint64_t)c[3] << 24 | (uint64_t)c[4] << 32 | (uint64_t)c[5] << 40 | + (uint64_t)c[6] << 48 | (uint64_t)c[7] << 56; +#endif +} + +} // namespace mmhdtl + +/// \brief MurmurHash2 64-bit hash for 64-bit platforms. +/// \param key The key to hash. +/// \param len Length of the key in byte. +/// \param seed A seed value used for hashing. +/// \return A hash value. +METALL_PRAGMA_IGNORE_GCC_UNINIT_WARNING_BEGIN +inline uint64_t murmur_hash_64a(const void *key, int len, + uint64_t seed) noexcept { + + const uint64_t m = 0xc6a4a7935bd1e995LLU; + const int r = 47; uint64_t h = seed ^ (len * m); @@ -29,7 +56,7 @@ inline constexpr uint64_t MurmurHash64A(const void *key, const int len, const uint64_t *end = data + (len / 8); while (data != end) { - uint64_t k = *data++; + uint64_t k = mmhdtl::murmurhash_getblock(data++); k *= m; k ^= k >> r; @@ -44,7 +71,7 @@ inline constexpr uint64_t MurmurHash64A(const void *key, const int len, switch (len & 7) { case 7: h ^= uint64_t(data2[6]) << 48; - [[fallthrough]]; + [[fallthrough]]; case 6: h ^= uint64_t(data2[5]) << 40; [[fallthrough]]; @@ -71,24 +98,32 @@ inline constexpr uint64_t MurmurHash64A(const void *key, const int len, return h; } +METALL_PRAGMA_IGNORE_GCC_UNINIT_WARNING_END + +/// \brief Alias of murmur_hash_64a. +/// \warning This function is deprecated. Use murmur_hash_64a instead. +inline uint64_t MurmurHash64A(const void *key, int len, + uint64_t seed) noexcept { + return murmur_hash_64a(key, len, seed); +} -/// \brief Hash a value of type T -/// \tparam T The type of a value to hash -/// \tparam seed A seed value used for hashing +/// \brief Hash a value of type T. Provides the same interface as std::hash. +/// \tparam T The type of a value to hash. +/// \tparam seed A seed value used for hashing. template struct hash { - uint64_t operator()(const T &key) const { - return MurmurHash64A(&key, sizeof(T), seed); + inline uint64_t operator()(const T &key) const noexcept { + return murmur_hash_64a(&key, sizeof(T), seed); } }; -/// \brief Hash string data -/// \tparam string_type A string class -/// \tparam seed A seed value used for hashing +/// \brief Hash string data. +/// \tparam string_type A string class. +/// \tparam seed A seed value used for hashing. template struct string_hash { - uint64_t operator()(const string_type &key) const { - return MurmurHash64A( + inline uint64_t operator()(const string_type &key) const noexcept { + return murmur_hash_64a( key.c_str(), key.length() * sizeof(typename string_type::value_type), seed); } diff --git a/include/metall/detail/utilities.hpp b/include/metall/detail/utilities.hpp index a0327a0f..ca9b2400 100644 --- a/include/metall/detail/utilities.hpp +++ b/include/metall/detail/utilities.hpp @@ -14,6 +14,25 @@ #include +#ifndef METALL_PRAGMA_IGNORE_GCC_UNINIT_WARNING_BEGIN +#if defined(__GNUC__) and !defined(__clang__) +#define METALL_PRAGMA_IGNORE_GCC_UNINIT_WARNING_BEGIN \ + _Pragma("GCC diagnostic push") \ + _Pragma("GCC diagnostic ignored \"-Wmaybe-uninitialized\"") +#else +#define METALL_PRAGMA_IGNORE_GCC_UNINIT_WARNING_BEGIN +#endif +#endif + +#ifndef METALL_PRAGMA_IGNORE_GCC_UNINIT_WARNING_END +#if defined(__GNUC__) and !defined(__clang__) +#define METALL_PRAGMA_IGNORE_GCC_UNINIT_WARNING_END \ + _Pragma("GCC diagnostic pop") +#else +#define METALL_PRAGMA_IGNORE_GCC_UNINIT_WARNING_END +#endif +#endif + namespace metall::mtlldetail { /// \brief Computes the next power of 2 diff --git a/include/metall/kernel/segment_storage/mmap_segment_storage.hpp b/include/metall/kernel/segment_storage/mmap_segment_storage.hpp index 73286e4b..013b5067 100644 --- a/include/metall/kernel/segment_storage/mmap_segment_storage.hpp +++ b/include/metall/kernel/segment_storage/mmap_segment_storage.hpp @@ -300,9 +300,8 @@ class mmap_segment_storage { mdtl::round_up(std::min(vm_region_size, block_size), page_size()); m_base_path = base_path; m_vm_region_size = mdtl::round_down(vm_region_size, page_size()); - m_segment = - mdtl::round_up(reinterpret_cast(vm_region), page_size()) + - reinterpret_cast(0); + m_segment = reinterpret_cast( + mdtl::round_up(reinterpret_cast(vm_region), page_size())); m_read_only = false; // Create the first block so that we can assume that there is a block always @@ -338,7 +337,7 @@ class mmap_segment_storage { m_vm_region_size = mdtl::round_down(vm_region_size, page_size()); ; m_segment = reinterpret_cast( - mdtl::round_up(reinterpret_cast(vm_region), page_size())); + mdtl::round_up(reinterpret_cast(vm_region), page_size())); m_read_only = read_only; // Maps block files one by one From 71dac9c20f9bab45e5ad6898cafd568ba222f226 Mon Sep 17 00:00:00 2001 From: Keita Iwabuchi Date: Fri, 16 Jun 2023 21:43:53 -0700 Subject: [PATCH 11/12] Add logger example --- example/CMakeLists.txt | 2 ++ example/README.md | 4 ++++ example/logger.cpp | 20 +++++++++++++++++++ include/metall/metall.hpp | 3 +++ verification/logger/CMakeLists.txt | 2 +- .../logger/{logger.cpp => verify_logger.cpp} | 0 6 files changed, 30 insertions(+), 1 deletion(-) create mode 100644 example/logger.cpp rename verification/logger/{logger.cpp => verify_logger.cpp} (100%) diff --git a/example/CMakeLists.txt b/example/CMakeLists.txt index a665f17c..6d413745 100644 --- a/example/CMakeLists.txt +++ b/example/CMakeLists.txt @@ -34,6 +34,8 @@ add_metall_executable(metall_containers metall_containers.cpp) add_metall_executable(allocator_aware_type allocator_aware_type.cpp) +add_metall_executable(logger logger.cpp) + if (BUILD_C) add_c_executable(c_api c_api.c) target_link_libraries(c_api PRIVATE metall_c) diff --git a/example/README.md b/example/README.md index 99791d07..f5da344b 100644 --- a/example/README.md +++ b/example/README.md @@ -50,6 +50,10 @@ Therefore, the data structures in the examples below are more like how to use ST ### Metall (advanced) +* [logger.cpp](logger.cpp) + + * How to use Metall's logger. + * [datastore_description.cpp](datastore_description.cpp) * Set and get datastore description diff --git a/example/logger.cpp b/example/logger.cpp new file mode 100644 index 00000000..efc98653 --- /dev/null +++ b/example/logger.cpp @@ -0,0 +1,20 @@ +// Copyright 2023 Lawrence Livermore National Security, LLC and other Metall +// Project Developers. See the top-level COPYRIGHT file for details. +// +// SPDX-License-Identifier: (Apache-2.0 OR MIT) + +#include + +int main() { + // Set the log level to , for example, verbose. + // The log level can be changed at any time. + metall::logger::set_log_level(metall::logger::level::verbose); + + // Enable the program to abort when a critical log message is displayed. + metall::logger::abort_on_critical_error(true); + + // Do Metall operations + metall::manager manager(metall::create_only, "/tmp/dir"); + + return 0; +} \ No newline at end of file diff --git a/include/metall/metall.hpp b/include/metall/metall.hpp index 1b23f46b..7f1d86a7 100644 --- a/include/metall/metall.hpp +++ b/include/metall/metall.hpp @@ -54,6 +54,9 @@ using manager = basic_manager<>; /// \example csr_graph.cpp /// This is an example of how to use a CSR graph data structure with Metall. +/// \example logger.cpp +/// This is an example of how to use Metall's logger. + /// \example datastore_description.cpp /// This is an example of how to set and get datastore description. diff --git a/verification/logger/CMakeLists.txt b/verification/logger/CMakeLists.txt index 3ead5f63..88e53691 100644 --- a/verification/logger/CMakeLists.txt +++ b/verification/logger/CMakeLists.txt @@ -1 +1 @@ -add_metall_executable(logger logger.cpp) \ No newline at end of file +add_metall_executable(verify_logger verify_logger.cpp) \ No newline at end of file diff --git a/verification/logger/logger.cpp b/verification/logger/verify_logger.cpp similarity index 100% rename from verification/logger/logger.cpp rename to verification/logger/verify_logger.cpp From 642172b8835ad8b53011b6ba3a2cfc80d10dc343 Mon Sep 17 00:00:00 2001 From: Keita Iwabuchi Date: Sun, 16 Jul 2023 11:50:14 -0700 Subject: [PATCH 12/12] Release v0.26 --- CMakeLists.txt | 2 +- docs/Doxyfile.in | 2 +- include/metall/version.hpp | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 60d89b35..16c14325 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -17,7 +17,7 @@ endif() # Metall general configuration # -------------------------------------------------------------------------------- # project(Metall - VERSION 0.25 + VERSION 0.26 DESCRIPTION "A persistent memory allocator for data-centric analytics" HOMEPAGE_URL "https://github.com/LLNL/metall") diff --git a/docs/Doxyfile.in b/docs/Doxyfile.in index 53ae902a..8c7af008 100644 --- a/docs/Doxyfile.in +++ b/docs/Doxyfile.in @@ -38,7 +38,7 @@ PROJECT_NAME = "Metall" # could be handy for archiving the generated documentation or if some version # control system is used. -PROJECT_NUMBER = v0.25 +PROJECT_NUMBER = v0.26 # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewer a diff --git a/include/metall/version.hpp b/include/metall/version.hpp index 8994bca2..0aac9c1e 100644 --- a/include/metall/version.hpp +++ b/include/metall/version.hpp @@ -14,7 +14,7 @@ /// METALL_VERSION / 100 % 1000 // the minor version. /// METALL_VERSION % 100 // the patch level. /// \endcode -#define METALL_VERSION 2500 +#define METALL_VERSION 2600 namespace metall { /// \brief Variable type to handle a version data.