From 84cf9c73ed8e587b084b2494ec7fa422be498062 Mon Sep 17 00:00:00 2001 From: garyschulte Date: Fri, 25 Oct 2024 10:31:04 -0700 Subject: [PATCH] merge rebase squash commit for preimage store Signed-off-by: garyschulte --- CHANGELOG.md | 5 + .../org/hyperledger/besu/cli/BesuCommand.java | 3 + .../options/storage/DataStorageOptions.java | 13 ++ .../storage/DiffBasedSubStorageOptions.java | 16 +- .../besu/services/BesuConfigurationImpl.java | 7 +- .../stable/DataStorageOptionsTest.java | 14 -- .../src/test/resources/everything_config.toml | 3 +- build.gradle | 2 +- .../keyvalue/KeyValueSegmentIdentifier.java | 3 +- .../KeyValueStorageProviderBuilder.java | 15 +- .../WorldStatePreimageKeyValueStorage.java | 10 + .../bonsai/storage/BonsaiPreImageProxy.java | 67 ------ .../BonsaiWorldStateKeyValueStorage.java | 7 - .../storage/CachingPreImageStorage.java | 195 ++++++++++++++++++ .../bonsai/worldview/BonsaiWorldState.java | 15 +- .../DiffBasedWorldStateKeyValueStorage.java | 68 ++++++ .../common/worldview/DiffBasedWorldState.java | 17 +- .../common/worldview/DiffBasedWorldView.java | 4 + .../DiffBasedWorldStateUpdateAccumulator.java | 31 ++- .../worldstate/DataStorageConfiguration.java | 12 +- .../worldstate/WorldStatePreimageStorage.java | 1 + .../bonsai/AbstractIsolationTests.java | 7 +- .../BonsaiReferenceTestUpdateAccumulator.java | 22 +- .../BonsaiReferenceTestWorldState.java | 34 +-- .../BonsaiReferenceTestWorldStateStorage.java | 87 -------- .../evm/worldstate/AbstractWorldUpdater.java | 4 +- plugin-api/build.gradle | 2 +- .../storage/DataStorageConfiguration.java | 11 +- .../RocksDBKeyValueStorageFactory.java | 4 +- .../BaseVersionedStorageFormat.java | 2 +- .../PrivacyVersionedStorageFormat.java | 2 +- .../rocksdb/NearestKeyValueStorageTest.java | 2 +- ...ksDBKeyValuePrivacyStorageFactoryTest.java | 2 +- .../RocksDBKeyValueStorageFactoryTest.java | 2 +- 34 files changed, 406 insertions(+), 283 deletions(-) delete mode 100644 ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/storage/BonsaiPreImageProxy.java create mode 100644 ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/storage/CachingPreImageStorage.java delete mode 100644 ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/BonsaiReferenceTestWorldStateStorage.java diff --git a/CHANGELOG.md b/CHANGELOG.md index f862ed866fe..4dccf056f37 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,11 @@ - Added isLabelsObserved to LabelledGauge in plugin-api. Default implementation returns false. ### Breaking Changes +- Removed deprecated experimental feature names + - `--Xsnapsync-synchronizer-flat-db-healing-enabled` in favor of `--Xbonsai-full-flat-db-enabled` + - `--Xbonsai-limit-trie-logs-enabled` and `--Xbonsai-trie-log-pruning-enabled` in favor of `--bonsai-limit-trie-logs-enabled` + - `--Xbonsai-trie-logs-pruning-window-size` in favor of `--bonsai-trie-logs-pruning-window-size` + ### Upcoming Breaking Changes diff --git a/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java b/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java index 886e3d68383..aa75c25c01b 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java @@ -2624,6 +2624,9 @@ public void setIgnorableStorageSegments() { if (!unstableChainPruningOptions.getChainDataPruningEnabled()) { rocksDBPlugin.addIgnorableSegmentIdentifier(KeyValueSegmentIdentifier.CHAIN_PRUNER_STATE); } + if (!dataStorageOptions.toDomainObject().getHashPreImageStorageEnabled()) { + rocksDBPlugin.addIgnorableSegmentIdentifier(KeyValueSegmentIdentifier.HASH_PREIMAGE_STORE); + } } private void validatePostMergeCheckpointBlockRequirements() { diff --git a/besu/src/main/java/org/hyperledger/besu/cli/options/storage/DataStorageOptions.java b/besu/src/main/java/org/hyperledger/besu/cli/options/storage/DataStorageOptions.java index fbc57f50def..e7cb1372377 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/options/storage/DataStorageOptions.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/options/storage/DataStorageOptions.java @@ -14,6 +14,7 @@ */ package org.hyperledger.besu.cli.options.storage; +import static org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration.DEFAULT_HASH_PREIMAGE_STORAGE_ENABLED; import static org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration.DEFAULT_RECEIPT_COMPACTION_ENABLED; import org.hyperledger.besu.cli.options.CLIOptions; @@ -35,6 +36,9 @@ public class DataStorageOptions implements CLIOptions private static final String DATA_STORAGE_FORMAT = "--data-storage-format"; + /** Feature to store hash preimages as part of worldstate */ + public static final String HASH_STORE_PREIMAGE_ENABLED = "--hash-preimage-storage-enabled"; + // Use Bonsai DB @Option( names = {DATA_STORAGE_FORMAT}, @@ -49,6 +53,13 @@ public class DataStorageOptions implements CLIOptions fallbackValue = "true") private Boolean receiptCompactionEnabled = DEFAULT_RECEIPT_COMPACTION_ENABLED; + @CommandLine.Option( + names = {HASH_STORE_PREIMAGE_ENABLED}, + description = + "Format to store trie data in. Either FOREST or BONSAI (default: ${DEFAULT-VALUE}).", + arity = "1") + private Boolean hashStorePreimagesEnabled = DEFAULT_HASH_PREIMAGE_STORAGE_ENABLED; + /** * Options specific to diff-based storage modes. Holds the necessary parameters to configure * diff-based storage, such as the Bonsai mode or Verkle in the future. @@ -88,6 +99,7 @@ public static DataStorageOptions fromConfig(final DataStorageConfiguration domai final DataStorageOptions dataStorageOptions = DataStorageOptions.create(); dataStorageOptions.dataStorageFormat = domainObject.getDataStorageFormat(); dataStorageOptions.receiptCompactionEnabled = domainObject.getReceiptCompactionEnabled(); + dataStorageOptions.hashStorePreimagesEnabled = domainObject.getHashPreImageStorageEnabled(); dataStorageOptions.diffBasedSubStorageOptions = DiffBasedSubStorageOptions.fromConfig(domainObject.getDiffBasedSubStorageConfiguration()); return dataStorageOptions; @@ -99,6 +111,7 @@ public DataStorageConfiguration toDomainObject() { ImmutableDataStorageConfiguration.builder() .dataStorageFormat(dataStorageFormat) .receiptCompactionEnabled(receiptCompactionEnabled) + .hashPreImageStorageEnabled(hashStorePreimagesEnabled) .diffBasedSubStorageConfiguration(diffBasedSubStorageOptions.toDomainObject()); return builder.build(); } diff --git a/besu/src/main/java/org/hyperledger/besu/cli/options/storage/DiffBasedSubStorageOptions.java b/besu/src/main/java/org/hyperledger/besu/cli/options/storage/DiffBasedSubStorageOptions.java index c95a7668272..5717786b09b 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/options/storage/DiffBasedSubStorageOptions.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/options/storage/DiffBasedSubStorageOptions.java @@ -55,26 +55,16 @@ public class DiffBasedSubStorageOptions implements CLIOptions