List witness proof elements in order (#1055)

go/carmen/database_test.go

@@ -23,6 +23,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/Fantom-foundation/Carmen/go/database/mpt"
 	"github.com/Fantom-foundation/Carmen/go/database/mpt/io"
 	"github.com/Fantom-foundation/Carmen/go/state/gostate"
 
@@ -1216,26 +1217,35 @@ func TestDatabase_GetProof_Extract_SubProofs(t *testing.T) {
 			}
 
 			// extract storage nodes only
-			gotStorageElements, _, complete := recovered.GetStorageElements(root, addr, keys...)
-			if !complete {
-				t.Errorf("proof is not complete")
-			}
-			// first block's account has no storage, others do
-			if j > 0 && len(gotStorageElements) == 0 {
-				t.Errorf("no storage elements")
-			}
+			allStorageElements := []Bytes{}
+			for _, key := range keys {
+				gotStorageElements, complete := recovered.GetStorageElements(root, addr, key)
+				if !complete {
+					t.Errorf("proof is not complete")
+				}
+				// first block's account has no storage, others do
+				if j > 0 && len(gotStorageElements) == 0 {
+					t.Errorf("no storage elements")
+				}
+
+				// both proofs must be distinct
+				for _, accountElement := range gotAccount.GetElements() {
+					for _, storageElement := range gotStorageElements {
+						if accountElement == storageElement {
+							t.Errorf("account and storage proofs must be distinct")
+						}
+					}
+				}
 
-			// both proofs must be distinct
-			for _, accountElement := range gotAccount.GetElements() {
 				for _, storageElement := range gotStorageElements {
-					if accountElement == storageElement {
-						t.Errorf("account and storage proofs must be distinct")
+					if storageElement != mpt.EmptyNodeEthereumEncoding {
+						allStorageElements = append(allStorageElements, storageElement)
 					}
 				}
 			}
 
 			// putting nodes together must provide the original proof
-			merged := CreateWitnessProofFromNodes(append(gotStorageElements, gotAccount.GetElements()...)...)
+			merged := CreateWitnessProofFromNodes(append(allStorageElements, gotAccount.GetElements()...)...)
 
 			gotElements := merged.GetElements()
 			wantElements := shadowProofs[addr].GetElements()

go/carmen/example_test.go

@@ -481,10 +481,10 @@ func ExampleWitnessProof_GetStorageElements() {
 	// ------- Recover Proof and Split into Account and Storage Proof  -------
 
 	proof := carmen.CreateWitnessProofFromNodes(elements...)
-	accountProof, accountComplete := proof.Extract(rootHash, carmen.Address{1})
-	storageElements, storageRoot, storageComplete := proof.GetStorageElements(rootHash, carmen.Address{1}, carmen.Key{1})
+	accountProofElements, storageRoot, accountComplete := proof.GetAccountElements(rootHash, carmen.Address{1})
+	storageElements, storageComplete := proof.GetStorageElements(rootHash, carmen.Address{1}, carmen.Key{1})
 
-	fmt.Printf("Account proof is complete: %v and has %d elements\n", accountComplete, len(accountProof.GetElements()))
+	fmt.Printf("Account proof is complete: %v and has %d elements\n", accountComplete, len(accountProofElements))
 	fmt.Printf("Storage proof is complete: %v and has %d elements and root: %v\n", storageComplete, len(storageElements), storageRoot)
 
 	// Output: Account proof is complete: true and has 1 elements

go/carmen/proof.go

@@ -48,15 +48,18 @@ type WitnessProof interface {
 	// GetElements returns serialised elements of the witness proof.
 	GetElements() []Bytes
 
-	// GetStorageElements returns serialised elements of the witness proof for a given account
-	// and selected storage locations from this proof.
+	// GetAccountElements returns serialised elements of the witness proof for a selected account and
+	// the root of the account's storage trie. The final return parameter indicates whether everything that
+	// was requested could be covered. If so, it is set to true, otherwise it is set to false.
+	GetAccountElements(root Hash, address Address) ([]Bytes, Hash, bool)
+
+	// GetStorageElements returns serialised elements of the witness proof for a selected
+	// storage location within an account.
 	// The resulting elements contains only the storage part of the account.
-	// For this reason, the second parameter of this method returns the storage root for this storage
-	// as any proving and other operations on the resulting proof must be done related to the storage root.
 	// This method returns a copy that contains only the data necessary for proving storage keys.
 	// The third return parameter indicates whether everything that was requested could be covered.
 	// If so, it is set to true, otherwise it is set to false.
-	GetStorageElements(root Hash, address Address, keys ...Key) ([]Bytes, Hash, bool)
+	GetStorageElements(root Hash, address Address, key Key) ([]Bytes, bool)
 
 	// GetBalance extracts a balance from the witness proof for the input root hash and the address.
 	// If the witness proof contains the requested account for the input address for the given root hash, it returns its balance.
@@ -117,16 +120,16 @@ func (w witnessProof) GetElements() []Bytes {
 	return w.proof.GetElements()
 }
 
-func (w witnessProof) GetStorageElements(root Hash, address Address, keys ...Key) ([]Bytes, Hash, bool) {
-	commonKeys := make([]common.Key, len(keys))
-	for i, k := range keys {
-		commonKeys[i] = common.Key(k)
-	}
-
-	resProof, storageRoot, complete := w.proof.GetStorageElements(common.Hash(root), common.Address(address), commonKeys...)
+func (w witnessProof) GetAccountElements(root Hash, address Address) ([]Bytes, Hash, bool) {
+	resProof, storageRoot, complete := w.proof.GetAccountElements(common.Hash(root), common.Address(address))
 	return resProof, Hash(storageRoot), complete
 }
 
+func (w witnessProof) GetStorageElements(root Hash, address Address, key Key) ([]Bytes, bool) {
+	resProof, complete := w.proof.GetStorageElements(common.Hash(root), common.Address(address), common.Key(key))
+	return resProof, complete
+}
+
 func (w witnessProof) IsValid() bool {
 	return w.proof.IsValid()
 }

go/common/witness/proof.go

@@ -40,15 +40,18 @@ type Proof interface {
 	// GetElements returns serialised elements of the witness proof.
 	GetElements() []immutable.Bytes
 
-	// GetStorageElements returns serialised elements of the witness proof for a given account
-	// and selected storage locations from this proof.
+	// GetAccountElements returns serialised elements of the witness proof for a selected account and
+	// the root of the account's storage trie. The final return parameter indicates whether everything that
+	// was requested could be covered. If so, it is set to true, otherwise it is set to false.
+	GetAccountElements(root common.Hash, address common.Address) ([]immutable.Bytes, common.Hash, bool)
+
+	// GetStorageElements returns serialised elements of the witness proof for a selected
+	// storage location within an account.
 	// The resulting elements contains only the storage part of the account.
-	// For this reason, the second parameter of this method returns the storage root for this storage
-	// as any proving and other operations on the resulting proof must be done related to the storage root.
 	// This method returns a copy that contains only the data necessary for proving storage keys.
 	// The third return parameter indicates whether everything that was requested could be covered.
 	// If so, it is set to true, otherwise it is set to false.
-	GetStorageElements(root common.Hash, address common.Address, keys ...common.Key) ([]immutable.Bytes, common.Hash, bool)
+	GetStorageElements(root common.Hash, address common.Address, key common.Key) ([]immutable.Bytes, bool)
 
 	// GetBalance extracts a balance from the witness proof for the input root hash and the address.
 	// If the witness proof contains the requested account for the input address for the given root hash, it returns its balance.

go/database/mpt/hasher.go

@@ -19,6 +19,7 @@ import (
 	"sync"
 
 	"github.com/Fantom-foundation/Carmen/go/common"
+	"github.com/Fantom-foundation/Carmen/go/common/immutable"
 	"github.com/Fantom-foundation/Carmen/go/database/mpt/rlp"
 	"github.com/Fantom-foundation/Carmen/go/database/mpt/shared"
 )
@@ -245,7 +246,8 @@ func makeEthereumLikeHasher() hasher {
 
 type ethHasher struct{}
 
-var EmptyNodeEthereumHash = common.Keccak256(rlp.Encode(rlp.String{}))
+var EmptyNodeEthereumEncoding = immutable.NewBytes(rlp.Encode(rlp.String{}))
+var EmptyNodeEthereumHash = common.Keccak256(EmptyNodeEthereumEncoding.ToBytes())
 
 func (h ethHasher) updateHashes(
 	ref *NodeReference,

go/database/mpt/proof.go

@@ -14,12 +14,13 @@ import (
 	"bytes"
 	"errors"
 	"fmt"
-	"github.com/Fantom-foundation/Carmen/go/common/immutable"
-	"github.com/Fantom-foundation/Carmen/go/common/witness"
 	"slices"
 	"sort"
 	"strings"
 
+	"github.com/Fantom-foundation/Carmen/go/common/immutable"
+	"github.com/Fantom-foundation/Carmen/go/common/witness"
+
 	"github.com/Fantom-foundation/Carmen/go/common"
 	"github.com/Fantom-foundation/Carmen/go/common/amount"
 	"github.com/Fantom-foundation/Carmen/go/common/tribool"
@@ -268,23 +269,40 @@ func (p WitnessProof) GetElements() []immutable.Bytes {
 	return res
 }
 
-func (p WitnessProof) GetStorageElements(root common.Hash, address common.Address, keys ...common.Key) ([]immutable.Bytes, common.Hash, bool) {
-	visitor := &proofCollectingVisitor{}
+func (p WitnessProof) GetAccountElements(root common.Hash, address common.Address) ([]immutable.Bytes, common.Hash, bool) {
+	visitor := &proofPathRecordingVisitor{}
 	found, complete, err := visitWitnessPathTo(p.proofDb, root, addressToHashedNibbles(address), visitor)
-	if err != nil || !found {
-		return []immutable.Bytes{}, common.Hash{}, complete
+	if err != nil || !complete {
+		return []immutable.Bytes{}, common.Hash{}, false
+	}
+	storageHash := EmptyNodeEthereumHash
+	if found {
+		storageHash = visitor.visitedAccount.storageHash
+	}
+	return visitor.path, storageHash, complete
+}
+
+func (p WitnessProof) GetStorageElements(root common.Hash, address common.Address, key common.Key) ([]immutable.Bytes, bool) {
+	visitor := &proofPathRecordingVisitor{}
+	found, complete, err := visitWitnessPathTo(p.proofDb, root, addressToHashedNibbles(address), visitor)
+	if !complete || err != nil {
+		return []immutable.Bytes{}, false
 	}
 
+	// If the account does not exist, its storage is empty, and this can be proven.
+	if !found {
+		return []immutable.Bytes{EmptyNodeEthereumEncoding}, true
+	}
+
+	// If an account was found, a storage proof can be extracted.
 	storageRoot := visitor.visitedAccount.storageHash
-	visitor.visited = make(proofDb)
-	for _, key := range keys {
-		_, keyComplete, err := visitWitnessPathTo(p.proofDb, storageRoot, keyToHashedPathNibbles(key), visitor)
-		if err != nil || !keyComplete {
-			complete = false
-		}
+	visitor.path = nil
+	_, keyComplete, err := visitWitnessPathTo(p.proofDb, storageRoot, keyToHashedPathNibbles(key), visitor)
+	if err != nil {
+		return []immutable.Bytes{}, false
 	}
 
-	return WitnessProof{visitor.visited}.GetElements(), storageRoot, complete
+	return visitor.path, keyComplete
 }
 
 // proofExtractionVisitor is a visitor that visits MPT nodes and creates a witness proof.
@@ -487,9 +505,20 @@ func (v *proofCollectingVisitor) Visit(hash common.Hash, rlpNode rlpEncodedNode,
 	if !isEmbedded && v.visited != nil {
 		v.visited[hash] = rlpNode
 	}
+}
+
+type proofPathRecordingVisitor struct {
+	path           []immutable.Bytes  // all visited RLP encoded nodes
+	visitedAccount decodedAccountNode // the last visited account node
+}
+
+func (v *proofPathRecordingVisitor) Visit(hash common.Hash, rlpNode rlpEncodedNode, node Node, isEmbedded bool) {
 	if account, ok := node.(*decodedAccountNode); ok {
 		v.visitedAccount = *account
 	}
+	if !isEmbedded {
+		v.path = append(v.path, immutable.NewBytes(rlpNode))
+	}
 }
 
 // createNibblesFromKeyPrefix creates a nibble path from the input key and the number of nibbles.