Merge branch 'main' into 804-log-consistencydisplay-the-pinning-polic…

…y-options-same-as-block-cache-options-metadata-cache-options

.github/workflows/build_windows.yml

@@ -99,10 +99,7 @@ jobs:
         shell: powershell
         run: |    
             cd $Env:THIRDPARTY_HOME
-            curl   https://zlib.net/zlib13.zip -o  zlib13.zip
-            Expand-Archive -Path zlib13.zip -DestinationPath zlib-tmp
-            mv .\zlib-tmp\zlib-1.3\ .
-            rmdir zlib-tmp
+            git clone https://github.com/madler/zlib.git -b v1.3 zlib-1.3
             cd zlib-1.3\contrib\vstudio\vc14
             devenv zlibvc.sln /upgrade
             cp ../../../zlib.h .

.github/workflows/check_license_and_history.yml

@@ -3,8 +3,9 @@ name: Check License and History
 on:  # this workflow is planned to be called by the ci_pipeline and it will compare the PR files with the main
   workflow_call:
   workflow_dispatch:
-  #pull_request_review:
-  #  types: [submitted]
+  push:
+  pull_request_target:
+
 
 jobs:
   changedfiles:

.github/workflows/ci_pipeline.yml

@@ -201,10 +201,7 @@ jobs:
         shell: powershell
         run: |    
             cd $Env:THIRDPARTY_HOME
-            curl   https://zlib.net/zlib13.zip -o  zlib13.zip
-            Expand-Archive -Path zlib13.zip -DestinationPath zlib-tmp
-            mv .\zlib-tmp\zlib-1.3\ .
-            rmdir zlib-tmp
+            git clone https://github.com/madler/zlib.git -b v1.3 zlib-1.3
             cd zlib-1.3\contrib\vstudio\vc14
             devenv zlibvc.sln /upgrade
             cp ../../../zlib.h .

.github/workflows/qa-tests.yml

@@ -54,3 +54,19 @@ jobs:
               make clean && ${{ matrix.short_test }}
               ;;
           esac
+
+      - name: Save the DB if the test failed
+        if:  ${{ failure() }}
+        id: savedb 
+        run:  | 
+          ls -l /tmp/rocksdb_crashtest_* 
+          tar -zcvf /tmp/db4debug.tar.gz /tmp/rocksdb_crashtest_* 
+          ls -l /tmp/db4debug.tar.gz
+
+      - name: Upload the DB
+        uses: actions/upload-artifact@v3
+        if: ${{ always() && steps.savedb.outcome != 'skipped' && steps.savedb.outcome != 'failure' }}
+        with:
+          name: testdb 
+          retention-days: 3
+          path: /tmp/db4debug.tar.gz

CMakeLists.txt

@@ -1,3 +1,17 @@
+# Copyright (C) 2023 Speedb Ltd. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 # Prerequisites for Windows:
 #     This cmake build is for Windows 64-bit only.
 #
@@ -573,6 +587,10 @@ if(HAVE_AUXV_GETAUXVAL)
   add_definitions(-DROCKSDB_AUXV_GETAUXVAL_PRESENT)
 endif()
 
+if(MEMORY_REPORTING AND HAVE_MALLOC_USABLE_SIZE)
+  add_definitions(-DMEMORY_REPORTING)
+endif()
+
 set(FSYNC_MODE AUTO CACHE STRING "Enable RTTI in builds")
 set_property(CACHE FSYNC_MODE PROPERTY STRINGS AUTO FULL BARRIER OFF)
 if(NOT FSYNC_MODE STREQUAL "OFF")

HISTORY.md

@@ -1,36 +1,58 @@
 # Speedb Change Log 
 
 ## Unreleased
+
+### New Features 
+
+### Enhancements
+
+### Bug Fixes
+* LOG Consistency:Display the pinning policy options same as block cache options / metadata cache options (#804).
+
+### Miscellaneous
+* WriteController logging: Remove redundant reports when WC is not shared between dbs
+
+
+## Incaberry 2.8.0 (31/1/2024)
 Based on RocksDB 8.6.7
 
 ### New Features 
 * Added ConfigOptions::compare_to.  When set, this value causes only values that have been changed to be part of the serialized output (#648).
 * Rebase on RocksDB 8.6.7
 
 ### Enhancements
+* Added memory reporting to the logs to be able to monitor which component in the Arena is using the memory and CacheAllocation overall memory usage, print some statistics. This feature requires compiling with MEMORY_REPORTING flag (#481)
 * Added a kUseBaseAddress flag and GetBaseOffset flag to OptionTypeInfo.  If this flag is set and a function is used for processing options, the function is passed the base address of the struct rather than the specific field (#397)
 * Export GetFlushReasonString/GetCompactionReasonString in listener.h (#785).
 * Enabled speedb features in C and Java (#722)
 * stress test: Add the ability to trace write operations. Controlled by flag trace_ops (on by default). Trace files will be written to the expected values dir.
-
+* LOG Enhancement: Have a separate LOG entry per CF Stats. This ensures that no CF stats data is lost in case the size of the combined CF stats text exceeds the LOG's threshold (#534).
 
 ### Bug Fixes
+* Fix a bug in db_stress where non existence parameters have checked with enable_speedb_features.
+* Added IsRefreshIterSupported() to memtable_rep, to publish if the memtable support Refresh() of the iterator.
+Refresh() will return status NotSupported for memtables that do not support Refresh().
+IsAllowRefresh() has been added.
+db_stress has been updated as well to take into account that some memtables do not support Refresh()
 * fix conflicts between db_bench flags and enable speedb features flag(#743).
 * Proactive Flushes: Fix a race in the ShouldInitiateAnotherFlushMemOnly that may cause the method to return an incorrect answer (#758).
 * Stall deadlock consists small cfs (#637).
 * Fix CI failure after changing compation_readahead_size default to 0 (#794).
 * Compaction: Restore SetupForCompaction functionality. Specifically, hint POSIX_FADV_NORMAL for compaction input files.See https://github.com/speedb-io/speedb/issues/787 for full details.
 * stress test: Fix TestIterateAgainstExpected not supporting 0 iterations. TestIterateAgainstExpected was not designed to support value of 0 in FLAGS_num_iterations.
 RocksDB has a value of 10 by default and we've added the option to randomize the values from 0 to 100 in https://github.com/speedb-io/speedb/commit/434692a63318036a3995a53001337f18bf467903
-* LOG Consistency:Display the pinning policy options same as block cache options / metadata cache options (#804).
+* Add more checks for using db_stress with --enable_speedb_features=true
+* Proactive Flushes: Have the initiator return a correct answer when it was requested to initate a flush (#812).
+* stress test: Adding a trace file by default in PR https://github.com/speedb-io/speedb/pull/797 has revealed some incompatibilities between the trace file and several configurations (more details in https://github.com/speedb-io/speedb/issues/813). Keep the trace file and remove the IsDone assertion.
+
 
 ### Miscellaneous
 * Remove leftover references to ROCKSDB_LITE (#755).
 * Options: Set compaction_readahead_size default to 0. The current default of 2Mb is not optimal for most of our use cases. Having a value of 0 means that the FS will use its default size for prefetching (true only with https://github.com/speedb-io/speedb/pull/788).
 * Options: Set level_compaction_dynamic_level_bytes as false by default. This flag is not working properly with Speedb. see https://github.com/speedb-io/speedb/issues/786 for more details.
-* stress test: Disable hash speedb memtable and enable_speedb_features from testing until issues are solved.
+* zlib: Update links to zlib 1.3 in CI and Makefile since the link in zlib.net is dead.
 
-## Hazlenut 2.7.0 (27/10/2023)
+## Hazelnut 2.7.0 (27/10/2023)
 Based on RocksDB 8.1.1
 
 ### New Features 

Makefile

@@ -1,3 +1,17 @@
+# Copyright (C) 2023 Speedb Ltd. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 # Copyright (c) 2011 The LevelDB Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file. See the AUTHORS file for names of contributors.
@@ -2003,7 +2017,7 @@ SHA256_CMD = sha256sum
 
 ZLIB_VER ?= 1.3
 ZLIB_SHA256 ?= ff0ba4c292013dbc27530b3a81e1f9a813cd39de01ca5e0f8bf355702efa593e
-ZLIB_DOWNLOAD_BASE ?= http://zlib.net
+ZLIB_DOWNLOAD_BASE ?= https://github.com/madler/zlib/releases/download/v1.3
 BZIP2_VER ?= 1.0.8
 BZIP2_SHA256 ?= ab5a03176ee106d3f0fa90e381da478ddae405918153cca248e682cd0c4a2269
 BZIP2_DOWNLOAD_BASE ?= http://sourceware.org/pub/bzip2

build_tools/build_detect_platform

@@ -1,5 +1,19 @@
 #!/usr/bin/env bash
+
+# Copyright (C) 2023 Speedb Ltd. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
 #
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 # Detects OS we're compiling on and outputs a file specified by the first
 # argument, which in turn gets read while processing Makefile.
 #
@@ -456,7 +470,11 @@ EOF
 EOF
         if [ "$?" = 0 ]; then
             COMMON_FLAGS="$COMMON_FLAGS -DROCKSDB_MALLOC_USABLE_SIZE"
+            if test $MEMORY_REPORTING; then
+                COMMON_FLAGS="$COMMON_FLAGS -DMEMORY_REPORTING"
+            fi
         fi
+
     fi
 
     if ! test $ROCKSDB_DISABLE_MEMKIND; then

db/arena_wrapped_db_iter.cc

@@ -1,4 +1,22 @@
+// Copyright (C) 2023 Speedb Ltd. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 //  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+//  This source code is licensed under both the GPLv2 (found in the
+//  COPYING file in the root directory) and Apache 2.0 License
+//  (found in the LICENSE.Apache file in the root directory).
+
 //  This source code is licensed under both the GPLv2 (found in the
 //  COPYING file in the root directory) and Apache 2.0 License
 //  (found in the LICENSE.Apache file in the root directory).
@@ -37,7 +55,8 @@ void ArenaWrappedDBIter::Init(
     const SequenceNumber& sequence, uint64_t max_sequential_skip_in_iteration,
     uint64_t version_number, ReadCallback* read_callback, DBImpl* db_impl,
     ColumnFamilyData* cfd, bool expose_blob_index, bool allow_refresh) {
-  auto mem = arena_.AllocateAligned(sizeof(DBIter));
+  auto mem = arena_.AllocateAligned(
+      sizeof(DBIter), ArenaTracker::ArenaStats::ArenaWrappedDBIter);
   db_iter_ =
       new (mem) DBIter(env, read_options, ioptions, mutable_cf_options,
                        ioptions.user_comparator, /* iter */ nullptr, version,
@@ -154,7 +173,10 @@ ArenaWrappedDBIter* NewArenaWrappedDbIterator(
   ArenaWrappedDBIter* iter = new ArenaWrappedDBIter();
   iter->Init(env, read_options, ioptions, mutable_cf_options, version, sequence,
              max_sequential_skip_in_iterations, version_number, read_callback,
-             db_impl, cfd, expose_blob_index, allow_refresh);
+             db_impl, cfd, expose_blob_index,
+             !ioptions.memtable_factory->IsRefreshIterSupported()
+                 ? false
+                 : allow_refresh);
   if (db_impl != nullptr && cfd != nullptr && allow_refresh) {
     iter->StoreRefreshInfo(db_impl, cfd, read_callback, expose_blob_index);
   }

db/arena_wrapped_db_iter.h

@@ -1,3 +1,17 @@
+// Copyright (C) 2023 Speedb Ltd. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 //  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under both the GPLv2 (found in the
 //  COPYING file in the root directory) and Apache 2.0 License
@@ -76,6 +90,7 @@ class ArenaWrappedDBIter : public Iterator {
   Status status() const override { return db_iter_->status(); }
   Slice timestamp() const override { return db_iter_->timestamp(); }
   bool IsBlob() const { return db_iter_->IsBlob(); }
+  bool IsAllowRefresh() override { return allow_refresh_; }
 
   Status GetProperty(std::string prop_name, std::string* prop) override;
 

db/column_family.cc

@@ -1123,8 +1123,8 @@ WriteStallCondition ColumnFamilyData::RecalculateWriteStallConditions(
       ROCKS_LOG_WARN(
           ioptions_.logger,
           "[%s] Stalling writes because we have %d immutable memtables "
-          "(waiting for flush), max_write_buffer_number is set to %d "
-          "rate %" PRIu64,
+          "(waiting for flush), max_write_buffer_number is set to %d. "
+          "delayed write rate: %" PRIu64,
           name_.c_str(), imm()->NumNotFlushed(),
           mutable_cf_options.max_write_buffer_number,
           write_controller->delayed_write_rate());
@@ -1146,8 +1146,8 @@ WriteStallCondition ColumnFamilyData::RecalculateWriteStallConditions(
             1);
       }
       ROCKS_LOG_WARN(ioptions_.logger,
-                     "[%s] Stalling writes because we have %d level-0 files "
-                     "rate %" PRIu64,
+                     "[%s] Stalling writes because we have %d level-0 files. "
+                     "delayed write rate: %" PRIu64,
                      name_.c_str(), vstorage->l0_delay_trigger_count(),
                      write_controller->delayed_write_rate());
     } else if (write_stall_condition == WriteStallCondition::kDelayed &&
@@ -1175,7 +1175,7 @@ WriteStallCondition ColumnFamilyData::RecalculateWriteStallConditions(
       ROCKS_LOG_WARN(
           ioptions_.logger,
           "[%s] Stalling writes because of estimated pending compaction "
-          "bytes %" PRIu64 " rate %" PRIu64,
+          "bytes %" PRIu64 ". delayed write rate: %" PRIu64,
           name_.c_str(), vstorage->estimated_compaction_needed_bytes(),
           write_controller->delayed_write_rate());
     } else {