Enable Speedb features (#543)

HISTORY.md

@@ -21,6 +21,11 @@
 ## Fig v2.5.0 (06/14/2023)
 Based on RocksDB 8.1.1
 
+### New Features
+ * Enable Speedb Features : Speedb users currently configure the database manually. New Speedb users are required to spend a lot of effort reading the documentation of the Speedb features.
+ The purpose of this feature is to help users enable and set Speedb options easily to a default configuration.
+ The SharedOptions class was added to improve the usability of multiple databases cases by arranging shared options.(#543)
+
 ### New Features
 * Delay writes gradually based on memory usage of the WriteBufferManager (WBM).
 Before this PR, setting allow_stall in the WBM's constructor meant that writes are completely stopped when the WBM's memory usage exceeds its quota. The goal here is to gradually decrease

build_tools/check-sources.sh

@@ -32,7 +32,8 @@ fi
 git grep -n 'using namespace' -- ':!build_tools' ':!docs' \
     ':!third-party/folly/folly/lang/Align.h' \
     ':!third-party/gtest-1.8.1/fused-src/gtest/gtest.h' \
-    ':!examples/speedb_with_ttl_example.cc'
+    ':!examples/speedb_with_ttl_example.cc' \
+    ':!examples/enable_speedb_features_example.cc'
 if [ "$?" != "1" ]; then
   echo '^^^^ Do not use "using namespace"'
   BAD=1

examples/Makefile

@@ -20,7 +20,8 @@ CFLAGS += -Wstrict-prototypes
 .PHONY: clean static_lib
 
 all: simple_example column_families_example compaction_filter_example compact_files_example c_simple_example optimistic_transaction_example \
-	 transaction_example options_file_example rocksdb_backup_restore_example speedb_is_awesome_example speedb_with_ttl_example
+	 transaction_example options_file_example rocksdb_backup_restore_example speedb_is_awesome_example speedb_with_ttl_example \
+	 enable_speedb_features_example
 
 simple_example: static_lib simple_example.cc
 	$(CXX) $(CXXFLAGS) $@.cc -o$@ ../$(LIBNAME).a -I../include -O2 -std=c++17 $(PLATFORM_LDFLAGS) $(PLATFORM_CXXFLAGS) $(EXEC_LDFLAGS)
@@ -55,6 +56,8 @@ multi_processes_example: static_lib multi_processes_example.cc
 speedb_is_awesome_example: static_lib speedb_is_awesome_example.cc
 	$(CXX) $(CXXFLAGS) $@.cc -o$@ ../$(LIBNAME).a -I../include -O2 -std=c++17 $(PLATFORM_LDFLAGS) $(PLATFORM_CXXFLAGS) $(EXEC_LDFLAGS)
 
+enable_speedb_features_example: static_lib enable_speedb_features_example.cc
+	$(CXX) $(CXXFLAGS) $@.cc -o$@ ../$(LIBNAME).a -I../include -O2 -std=c++17 $(PLATFORM_LDFLAGS) $(PLATFORM_CXXFLAGS) $(EXEC_LDFLAGS)
 speedb_with_ttl_example: static_lib speedb_with_ttl_example.cc
 	$(CXX) $(CXXFLAGS) $@.cc -o$@ ../$(LIBNAME).a -I../include -O2 -std=c++17 $(PLATFORM_LDFLAGS) $(PLATFORM_CXXFLAGS) $(EXEC_LDFLAGS)
 
@@ -64,7 +67,7 @@ rocksdb_backup_restore_example: static_lib rocksdb_backup_restore_example.cc
 clean:
 	rm -rf ./simple_example ./column_families_example ./compact_files_example ./compaction_filter_example ./c_simple_example c_simple_example.o \
 		   ./optimistic_transaction_example ./transaction_example ./options_file_example ./multi_processes_example ./rocksdb_backup_restore_example \
-		   ./speedb_is_awesome_example ./speedb_with_ttl_example
+		   ./speedb_is_awesome_example ./speedb_with_ttl_example ./enable_speedb_features_example
 
 static_lib:
 	LIBNAME="$(LIBNAME)" $(MAKE) -C .. static_lib

examples/enable_speedb_features_example.cc

@@ -0,0 +1,154 @@
+// Copyright (C) 2022 Speedb Ltd. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <cstdio>
+#include <iostream>
+#include <string>
+
+#include "rocksdb/compression_type.h"
+#include "rocksdb/db.h"
+#include "rocksdb/options.h"
+#include "rocksdb/slice.h"
+
+using namespace ROCKSDB_NAMESPACE;
+
+#if defined(OS_WIN)
+std::string kDBPath1 = "C:\\Windows\\TEMP\\enable_speedb_features_example1";
+std::string kDBPath2 = "C:\\Windows\\TEMP\\enable_speedb_features_example2";
+std::string kDBPath3 = "C:\\Windows\\TEMP\\enable_speedb_features_example3";
+std::string kDBPath4 = "C:\\Windows\\TEMP\\enable_speedb_features_example4";
+#else
+std::string kDBPath1 = "/tmp/enable_speedb_features_example1";
+std::string kDBPath2 = "/tmp/enable_speedb_features_example2";
+std::string kDBPath3 = "/tmp/enable_speedb_features_example3";
+std::string kDBPath4 = "/tmp/enable_speedb_features_example4";
+#endif
+
+int main() {
+  DB *db1 = nullptr;
+  DB *db2 = nullptr;
+  DB *db3 = nullptr;
+  DB *db4 = nullptr;
+  Options op1;
+  Options op2;
+  Options op3;
+  Options op4;
+  size_t total_ram_size_bytes = 512 * 1024 * 1024;
+  size_t delayed_write_rate = 256 * 1024 * 1024;
+  size_t total_threads = 8;
+
+  // define SharedOptions object for each databases group
+  SharedOptions so1(total_ram_size_bytes, total_threads, delayed_write_rate);
+
+  // customize each options file except SpeedbSharedOptiopns members
+  // as listed in the definition of SpeedbSharedOptiopns in options.h
+  op1.create_if_missing = true;
+  op1.compression = rocksdb::kNoCompression;
+  //...
+  op1.EnableSpeedbFeatures(so1);
+
+  op2.create_if_missing = true;
+  op2.compression = rocksdb::kZlibCompression;
+  //...
+  op2.EnableSpeedbFeatures(so1);
+
+  // open the databases
+  Status s = DB::Open(op1, kDBPath1, &db1);
+  if (!s.ok()) {
+    std::cerr << s.ToString() << std::endl;
+    return 1;
+  }
+
+  s = DB::Open(op2, kDBPath2, &db2);
+  if (!s.ok()) {
+    std::cerr << s.ToString() << std::endl;
+    return 1;
+  }
+  std::cout << "DBs group 1 was created" << std::endl;
+
+  // do the same for any group of databases
+  total_ram_size_bytes = 1024 * 1024 * 1024;
+  delayed_write_rate = 128 * 1024 * 1024;
+  total_threads = 4;
+  SharedOptions so2(total_ram_size_bytes, total_threads, delayed_write_rate);
+
+  // again customize each options object except SharedOptiopns members
+  op3.create_if_missing = true;
+  op3.compaction_style = rocksdb::kCompactionStyleUniversal;
+  //...
+  op3.EnableSpeedbFeatures(so2);
+
+  op4.create_if_missing = true;
+  op4.compaction_style = rocksdb::kCompactionStyleLevel;
+  //...
+  op4.EnableSpeedbFeatures(so2);
+
+  // open the databases
+  s = DB::Open(op3, kDBPath3, &db3);
+  if (!s.ok()) {
+    std::cerr << s.ToString() << std::endl;
+    return 1;
+  }
+
+  s = DB::Open(op4, kDBPath4, &db4);
+  if (!s.ok()) {
+    std::cerr << s.ToString() << std::endl;
+    return 1;
+  }
+  std::cout << "DBs group 2 was created" << std::endl;
+
+  // creation of column family
+  rocksdb::ColumnFamilyOptions cfo3(op3);
+  rocksdb::ColumnFamilyHandle *cf;
+  // coustomize it except SpeedbSharedOptiopns members
+
+  // call EnableSpeedbFeaturesCF and supply for it the same SharedOptions
+  // object as the DB, so2 this time.
+  cfo3.EnableSpeedbFeaturesCF(so2);
+  // create the cf
+  s = db3->CreateColumnFamily(cfo3, "new_cf", &cf);
+  if (!s.ok()) {
+    std::cerr << s.ToString() << std::endl;
+    return 1;
+  }
+  std::cout << "new_cf was created in db3" << std::endl;
+
+  s = db3->DropColumnFamily(cf);
+  if (!s.ok()) {
+    std::cerr << s.ToString() << std::endl;
+    return 1;
+  }
+  db3->DestroyColumnFamilyHandle(cf);
+  if (!s.ok()) {
+    std::cerr << s.ToString() << std::endl;
+    return 1;
+  }
+  std::cout << "new_cf was destroyed" << std::endl;
+
+  s = db1->Close();
+  assert(s.ok());
+  s = db2->Close();
+  assert(s.ok());
+  s = db3->Close();
+  assert(s.ok());
+  s = db4->Close();
+  assert(s.ok());
+
+  delete db1;
+  delete db2;
+  delete db3;
+  delete db4;
+
+  return 0;
+}

examples/speedb_is_awesome_example.cc

@@ -54,5 +54,6 @@ int main() {
   // close DB
   s = db->Close();
   assert(s.ok());
+  delete db;
   return 0;
 }

include/rocksdb/options.h

@@ -55,8 +55,9 @@ class Statistics;
 class InternalKeyComparator;
 class WalFilter;
 class WriteBufferManager;
-class FileSystem;
 class WriteController;
+class FileSystem;
+class SharedOptions;
 
 struct Options;
 struct DbPath;
@@ -104,6 +105,18 @@ struct ColumnFamilyOptions : public AdvancedColumnFamilyOptions {
   ColumnFamilyOptions* OptimizeUniversalStyleCompaction(
       uint64_t memtable_memory_budget = 512 * 1024 * 1024);
 
+  // Default values for some parameters in ColumnFamilyOptions are not
+  // optimized for Speedb features, As a starting point for configuring
+  // Speedb Features.
+  // please avoid changing:
+  // write_buffer_size, cache, write_controller, write_buffer_manager,
+  // table_factory, memtable_factory.
+  // the function might override any of those major options, some more options
+  // might be overridden please read the code.
+  // use example can be found in  enable_speedb_features_example.cc
+  // bucket count is initialized to 0; max_write_buffer_number is initialized to
+  // 32
+  ColumnFamilyOptions* EnableSpeedbFeaturesCF(SharedOptions& shared_options);
   // -------------------
   // Parameters that affect behavior
 
@@ -480,6 +493,19 @@ struct DBOptions {
   // bottlenecked by RocksDB.
   DBOptions* IncreaseParallelism(int total_threads = 16);
 
+  // Enable Speedb features function for DBOptions
+  //
+  // please avoid changing:
+  // write_buffer_size cache, write_controller, delayed_write_rate
+  // bytes_per_sync, write_buffer_manager, use_dynamic_delay table_factory and
+  // memtable_factory we will initialize and configure those.
+  // the function might override any of those major options, some more options
+  // might be overridden please read the code.
+  // use example can be fuond in  enable_speedb_features_example.cc
+  DBOptions* EnableSpeedbFeaturesDB(SharedOptions& shared_options);
+
+  // #endif  // ROCKSDB_LITE
+
   // If true, the database will be created if it is missing.
   // Default: false
   bool create_if_missing = false;
@@ -1502,6 +1528,17 @@ struct Options : public DBOptions, public ColumnFamilyOptions {
   // Use this if your DB is very small (like under 1GB) and you don't want to
   // spend lots of memory for memtables.
   Options* OptimizeForSmallDb();
+  // Default values for some parameters in Options are not
+  // optimized for Speedb features, As a starting point for configuring
+  // Speedb Features.
+  // if you choose to use it you should not change:
+  // total_ram_size_bytes, max_background_jobs, delayed_write_rate,
+  // write_buffer_size cache, write_controller,
+  // write_buffer_manager,bytes_per_sync, use_dynamic_delay table_factory and
+  // memtable_factory we will initialize and configure those.
+  // the function might overide any of those.
+  // use example can be found in  enable_speedb_features_example.cc
+  Options* EnableSpeedbFeatures(SharedOptions& shared_options);
 
   // Disable some checks that should not be necessary in the absence of
   // software logic errors or CPU+memory hardware errors. This can improve
@@ -2191,4 +2228,37 @@ struct WaitForCompactOptions {
   std::chrono::microseconds timeout = std::chrono::microseconds::zero();
 };
 
+// use this class to arrange multiple db shared options as a group
+// this class includes all the shared_ptrs from DBOptions.
+// it is also includes initialization for Speedb features
+// more info and use example can be found in  enable_speedb_features_example.cc
+class SharedOptions {
+ public:
+  SharedOptions();
+  SharedOptions(size_t total_ram_size_bytes, size_t total_threads,
+                size_t delayed_write_rate = 256 * 1024 * 1024ul);
+  size_t GetTotalThreads() { return total_threads_; }
+  size_t GetTotalRamSizeBytes() { return total_ram_size_bytes_; }
+  size_t GetDelayedWriteRate() { return delayed_write_rate_; }
+  // this function will increase write buffer manager by increased_by amount
+  // as long as the result is not bigger than the maximum size of
+  // total_ram_size_ /4
+  void IncreaseWriteBufferSize(size_t increase_by);
+
+  std::shared_ptr<Cache> cache = nullptr;
+  std::shared_ptr<WriteController> write_controller = nullptr;
+  std::shared_ptr<WriteBufferManager> write_buffer_manager = nullptr;
+  Env* env = Env::Default();
+  std::shared_ptr<RateLimiter> rate_limiter = nullptr;
+  std::shared_ptr<SstFileManager> sst_file_manager = nullptr;
+  std::shared_ptr<Logger> info_log = nullptr;
+  std::vector<std::shared_ptr<EventListener>> listeners;
+  std::shared_ptr<FileChecksumGenFactory> file_checksum_gen_factory = nullptr;
+
+ private:
+  size_t total_threads_ = 0;
+  size_t total_ram_size_bytes_ = 0;
+  size_t delayed_write_rate_ = 0;
+};
+
 }  // namespace ROCKSDB_NAMESPACE