Ethereum JS and Go codec fixtures and tests

_keccak256_fixtures_src/eth-access-list-tx.eth-tx

@@ -0,0 +1,2 @@
+
��

�a���OSt�����*���S1g~n�
+�UD�r�Y��OSt�����*���S1g~n��B�����,�V� D�P�0�G6�M�o�L������(�r �}	�H�����@'��B��Y���֔�OSt�����*���S1g~n��
��Q�O+03X�XqW?��b����^�@Ek&`�2�����v�O������j�������8b�u!

_keccak256_fixtures_src/eth-dynamic-fee-tx.eth-tx

@@ -0,0 +1,2 @@
+��

�a���OSt�����*���S1g~n�
+�UD�r�Y��OSt�����*���S1g~n��B�����,�V� D�P�0�G6�M�o�L������(�r �}	�H�����@'��B��Y���֔�OSt�����*���S1g~n��
��Q�O+03X�XqW?��b����^�@Ek&`�2�����v�O������j�������8b�u!

_keccak256_fixtures_src/eth-legacy-tx.eth-tx

@@ -0,0 +1,2 @@
+�a
��OSt�����*���S1g~n�
+�UD����
UG&6}+���������}����K��J���2�u�	_x���C��, �#��b��$R�

_keccak256_fixtures_src/eth-log-trie-extension-node.eth-receipt-log-trie

@@ -0,0 +1 @@
+�B���T�b��E��8o�K���H@�6/���c����E z}/�o�\�d�^��h��I���6U

_keccak256_fixtures_src/eth-receipt-trie-extension-node.eth-tx-receipt-trie

@@ -0,0 +1 @@
+�B���T�b��E��8o�K���H@�6/���c����E z}/�o�\�d�^��h��I���6U

_keccak256_fixtures_src/eth-state-trie-contract-leaf-node.eth-state-trie

@@ -0,0 +1 @@
+�i�1e�t�̟z�,\֖�IM|4Mx��:���N�E�F�D
��4��'��v�!��_�{�/����3���gj����^���{E�����س�;H�j�浑����
'

_keccak256_fixtures_src/eth-state-trie-extension-node.eth-state-trie

@@ -0,0 +1 @@
+�B���T�b��E��8o�K���H@�6/���c����E z}/�o�\�d�^��h��I���6U

_keccak256_fixtures_src/eth-storage-trie-branch-node-with-embedded-storage-leaf-and-value.eth-storage-trie

@@ -0,0 +1 @@
+����6x�z(6FB)���}f��w�e�����d�Ɋ�����۸��ė�PC�66Wi��8v������1��a	וɇ1e�t��
��������}t�^���Rj�k���m�L4���0ě�!��~���


_keccak256_fixtures_src/eth-storage-trie-extension-node.eth-storage-trie

@@ -0,0 +1 @@
+�B���T�b��E��8o�K���H@�6/���c����E z}/�o�\�d�^��h��I���6U

_keccak256_fixtures_src/eth-storage-trie-leaf-node.eth-storage-trie

@@ -0,0 +1 @@
+��9��T�b��E��8o�K���H@�6/���c��p<K+��W��C)��F�

_keccak256_fixtures_src/eth-tx-trie-extension-node.eth-tx-trie

@@ -0,0 +1 @@
+�B���T�b��E��8o�K���H@�6/���c����E z}/�o�\�d�^��h��I���6U

go/codecs_test.go

@@ -11,6 +11,8 @@ import (
 	"strings"
 	"testing"
 
+	"github.com/ipld/go-ipld-prime/linking"
+
 	"github.com/ipfs/go-cid"
 	"github.com/ipld/go-ipld-prime"
 	cidlink "github.com/ipld/go-ipld-prime/linking/cid"
@@ -34,22 +36,53 @@ func TestFixtures(t *testing.T) {
 			continue
 		}
 		t.Run(fixtureName, func(t *testing.T) {
-			data, err := loadFixture(fixtureName)
+			data, err := loadFixture(rootFixturePath, fixtureName, codecs, linkSystem)
+			if err != nil {
+				t.Fatalf("failed to load fixture: %v", err)
+			}
+			for fromCodec := range data {
+				for toCodec := range data {
+					msg := fmt.Sprintf("decode(%v)->encode(%v)", fromCodec, toCodec)
+					verifyCid(t, msg, data[fromCodec].value, codecs[toCodec], data[toCodec].cid, linkSystem)
+				}
+			}
+		})
+	}
+}
+
+func TestEthFixtures(t *testing.T) {
+	dirs, err := os.ReadDir(rootKeccak256FixturePath)
+	if err != nil {
+		t.Fatalf("failed to open fixtures dir: %v", err)
+	}
+
+	for _, dir := range dirs {
+		fixtureName := dir.Name()
+		if !dir.IsDir() {
+			continue
+		}
+		if reason, blacklisted := FixtureBlacklist[fixtureName]; blacklisted {
+			fmt.Printf("Skipping fixture '%v': %v\n", fixtureName, reason)
+			continue
+		}
+		t.Run(fixtureName, func(t *testing.T) {
+			ethLinkSystem := setupEthLinkSystem()
+			data, err := loadFixture(rootKeccak256FixturePath, fixtureName, ethCodecs, ethLinkSystem)
 			if err != nil {
 				t.Fatalf("failed to load fixture: %v", err)
 			}
 			for fromCodec := range data {
 				for toCodec := range data {
 					msg := fmt.Sprintf("decode(%v)->encode(%v)", fromCodec, toCodec)
-					verifyCid(t, msg, data[fromCodec].value, codecs[toCodec], data[toCodec].cid)
+					verifyCid(t, msg, data[fromCodec].value, ethCodecs[toCodec], data[toCodec].cid, ethLinkSystem)
 				}
 			}
 		})
 	}
 }
 
-func verifyCid(t *testing.T, desc string, node ipld.Node, toEnc ipld.LinkPrototype, expected cid.Cid) {
-	actual, err := nodeToCid(toEnc, node)
+func verifyCid(t *testing.T, desc string, node ipld.Node, toEnc ipld.LinkPrototype, expected cid.Cid, ls linking.LinkSystem) {
+	actual, err := nodeToCid(ls, toEnc, node)
 	if err != nil {
 		t.Fatalf("failed to convert node to CID: %v", err)
 	}

go/fixtures.go

@@ -7,13 +7,29 @@ import (
 	"path/filepath"
 	"strings"
 
+	"github.com/ipld/go-ipld-prime/linking"
+	"github.com/ipld/go-ipld-prime/multicodec"
+
 	"github.com/ipfs/go-cid"
 	_ "github.com/ipld/go-codec-dagpb"
 	"github.com/ipld/go-ipld-prime"
 	_ "github.com/ipld/go-ipld-prime/codec/dagcbor"
 	_ "github.com/ipld/go-ipld-prime/codec/dagjson"
 	cidlink "github.com/ipld/go-ipld-prime/linking/cid"
 	basicnode "github.com/ipld/go-ipld-prime/node/basic"
+
+	dageth "github.com/vulcanize/go-codec-dageth"
+	"github.com/vulcanize/go-codec-dageth/header"
+	"github.com/vulcanize/go-codec-dageth/log"
+	"github.com/vulcanize/go-codec-dageth/log_trie"
+	"github.com/vulcanize/go-codec-dageth/rct"
+	"github.com/vulcanize/go-codec-dageth/rct_trie"
+	account "github.com/vulcanize/go-codec-dageth/state_account"
+	"github.com/vulcanize/go-codec-dageth/state_trie"
+	"github.com/vulcanize/go-codec-dageth/storage_trie"
+	"github.com/vulcanize/go-codec-dageth/tx"
+	"github.com/vulcanize/go-codec-dageth/tx_trie"
+	"github.com/vulcanize/go-codec-dageth/uncles"
 )
 
 type codecName = string
@@ -44,15 +60,134 @@ var dagJsonLp = cidlink.LinkPrototype{Prefix: cid.Prefix{
 	MhType:   0x12,   // "sha2-256"
 	MhLength: 32,
 }}
+var ethHeaderLp = cidlink.LinkPrototype{Prefix: cid.Prefix{
+	Version:  1,
+	Codec:    0x90, // "eth-block"
+	MhType:   0x1b, // "keccak-256"
+	MhLength: 32,
+}}
+var ethUnclesLp = cidlink.LinkPrototype{Prefix: cid.Prefix{
+	Version:  1,
+	Codec:    0x91, // "eth-block-list"
+	MhType:   0x1b, // "keccak-256"
+	MhLength: 32,
+}}
+var ethTxLp = cidlink.LinkPrototype{Prefix: cid.Prefix{
+	Version:  1,
+	Codec:    0x93, // "eth-tx"
+	MhType:   0x1b, // "keccak-256"
+	MhLength: 32,
+}}
+var ethTxTrieLp = cidlink.LinkPrototype{Prefix: cid.Prefix{
+	Version:  1,
+	Codec:    0x92, // "eth-tx-trie"
+	MhType:   0x1b, // "keccak-256"
+	MhLength: 32,
+}}
+var ethRctLp = cidlink.LinkPrototype{Prefix: cid.Prefix{
+	Version:  1,
+	Codec:    0x95, // "eth-tx-receipt"
+	MhType:   0x1b, // "keccak-256"
+	MhLength: 32,
+}}
+var ethRctTrieLp = cidlink.LinkPrototype{Prefix: cid.Prefix{
+	Version:  1,
+	Codec:    0x94, // "eth-tx-receipt-trie"
+	MhType:   0x1b, // "keccak-256"
+	MhLength: 32,
+}}
+var ethLogLp = cidlink.LinkPrototype{Prefix: cid.Prefix{
+	Version:  1,
+	Codec:    0x9a, // "eth-receipt-log"
+	MhType:   0x1b, // "keccak-256"
+	MhLength: 32,
+}}
+var ethLogTrieLp = cidlink.LinkPrototype{Prefix: cid.Prefix{
+	Version:  1,
+	Codec:    0x99, // "eth-receipt-log-trie"
+	MhType:   0x1b, // "keccak-256"
+	MhLength: 32,
+}}
+var ethStateTrieLp = cidlink.LinkPrototype{Prefix: cid.Prefix{
+	Version:  1,
+	Codec:    0x96, // "eth-state-trie"
+	MhType:   0x1b, // "keccak-256"
+	MhLength: 32,
+}}
+var ethStorageTrieLp = cidlink.LinkPrototype{Prefix: cid.Prefix{
+	Version:  1,
+	Codec:    0x98, // "eth-storage-trie"
+	MhType:   0x1b, // "keccak-256"
+	MhLength: 32,
+}}
+var ethStateAccountLp = cidlink.LinkPrototype{Prefix: cid.Prefix{
+	Version:  1,
+	Codec:    0x97, // "eth-account-snapshot"
+	MhType:   0x1b, // "keccak-256"
+	MhLength: 32,
+}}
 var codecs = map[codecName]ipld.LinkPrototype{
 	"dag-pb":   dagPbLp,
 	"dag-cbor": dagCborLp,
 	"dag-json": dagJsonLp,
 }
+var ethCodecs = map[codecName]ipld.LinkPrototype{
+	"eth-block":            ethHeaderLp,
+	"eth-block-list":       ethUnclesLp,
+	"eth-tx":               ethTxLp,
+	"eth-tx-trie":          ethTxTrieLp,
+	"eth-tx-receipt":       ethRctLp,
+	"eth-tx-receipt-trie":  ethRctTrieLp,
+	"eth-receipt-log":      ethLogLp,
+	"eth-receipt-log-trie": ethLogTrieLp,
+	"eth-account-snapshot": ethStateAccountLp,
+	"eth-state-trie":       ethStateTrieLp,
+	"eth-storage-trie":     ethStorageTrieLp,
+}
 var linkSystem = cidlink.DefaultLinkSystem()
 
-func loadFixture(dir string) (fixtureSet, error) {
-	files, err := os.ReadDir("../fixtures/" + dir)
+func setupEthLinkSystem() linking.LinkSystem {
+	ethRegistry := multicodec.Registry{}
+	ethRegistry.RegisterDecoder(0x90, header.Decode)
+	ethRegistry.RegisterDecoder(0x91, uncles.Decode)
+	ethRegistry.RegisterDecoder(0x92, tx_trie.Decode)
+	ethRegistry.RegisterDecoder(0x93, tx.Decode)
+	ethRegistry.RegisterDecoder(0x94, rct_trie.Decode)
+	ethRegistry.RegisterDecoder(0x95, rct.Decode)
+	ethRegistry.RegisterDecoder(0x96, state_trie.Decode)
+	ethRegistry.RegisterDecoder(0x97, account.Decode)
+	ethRegistry.RegisterDecoder(0x98, storage_trie.Decode)
+	ethRegistry.RegisterDecoder(0x99, log_trie.Decode)
+	ethRegistry.RegisterDecoder(0x9a, log.Decode)
+
+	ethRegistry.RegisterEncoder(0x90, header.Encode)
+	ethRegistry.RegisterEncoder(0x91, uncles.Encode)
+	ethRegistry.RegisterEncoder(0x92, tx_trie.Encode)
+	ethRegistry.RegisterEncoder(0x93, tx.Encode)
+	ethRegistry.RegisterEncoder(0x94, rct_trie.Encode)
+	ethRegistry.RegisterEncoder(0x95, rct.Encode)
+	ethRegistry.RegisterEncoder(0x96, state_trie.Encode)
+	ethRegistry.RegisterEncoder(0x97, account.Encode)
+	ethRegistry.RegisterEncoder(0x98, storage_trie.Encode)
+	ethRegistry.RegisterEncoder(0x99, log_trie.Encode)
+	ethRegistry.RegisterEncoder(0x9a, log.Encode)
+
+	return cidlink.LinkSystemUsingMulticodecRegistry(ethRegistry)
+}
+
+var ethTypeSlab = map[string]ipld.NodePrototype{
+	"eth-tx-trie":          dageth.Type.TrieNode,
+	"eth-tx-receipt-trie":  dageth.Type.TrieNode,
+	"eth-state-trie":       dageth.Type.TrieNode,
+	"eth-storage-trie":     dageth.Type.TrieNode,
+	"eth-receipt-log-trie": dageth.Type.TrieNode,
+}
+
+var rootFixturePath = "../fixtures/"
+var rootKeccak256FixturePath = "../keccak256_fixtures/"
+
+func loadFixture(rootPath, dir string, codecMap map[codecName]ipld.LinkPrototype, ls linking.LinkSystem) (fixtureSet, error) {
+	files, err := os.ReadDir(rootPath + dir)
 	fixtures := make(fixtureSet)
 	if err != nil {
 		return fixtures, err
@@ -67,17 +202,21 @@ func loadFixture(dir string) (fixtureSet, error) {
 		if err != nil {
 			return fixtures, err
 		}
-		byts, err := os.ReadFile("../fixtures/" + dir + "/" + file.Name())
+		byts, err := os.ReadFile(rootPath + dir + "/" + file.Name())
 		if err != nil {
 			return fixtures, err
 		}
 		ext = strings.TrimLeft(ext, ".")
 		na := basicnode.Prototype.Any.NewBuilder()
-		lp, ok := codecs[ext]
+		nodePrototype, ok := ethTypeSlab[ext]
+		if ok {
+			na = nodePrototype.NewBuilder()
+		}
+		lp, ok := codecMap[ext]
 		if !ok {
 			fmt.Printf("unknown codec '%v' for fixture '%v'\n", ext, dir)
 		}
-		decoder, err := linkSystem.DecoderChooser(lp.BuildLink(make([]byte, 32)))
+		decoder, err := ls.DecoderChooser(lp.BuildLink(make([]byte, 32)))
 		if err != nil {
 			return fixtures, err
 		}
@@ -94,12 +233,12 @@ func loadFixture(dir string) (fixtureSet, error) {
 	return fixtures, nil
 }
 
-func nodeToCid(lp ipld.LinkPrototype, node ipld.Node) (cid.Cid, error) {
-	encoder, err := linkSystem.EncoderChooser(lp)
+func nodeToCid(ls linking.LinkSystem, lp ipld.LinkPrototype, node ipld.Node) (cid.Cid, error) {
+	encoder, err := ls.EncoderChooser(lp)
 	if err != nil {
 		return cid.Cid{}, fmt.Errorf("could not choose an encoder: %v", err)
 	}
-	hasher, err := linkSystem.HasherChooser(lp)
+	hasher, err := ls.HasherChooser(lp)
 	if err != nil {
 		return cid.Cid{}, fmt.Errorf("could not choose a hasher: %v", err)
 	}

go/go.mod

@@ -6,24 +6,61 @@ require (
 	github.com/ipfs/go-cid v0.4.1
 	github.com/ipld/go-codec-dagpb v1.6.0
 	github.com/ipld/go-ipld-prime v0.20.0
+	github.com/vulcanize/go-codec-dageth v0.1.2
 )
 
 require (
-	github.com/gopherjs/gopherjs v1.17.2 // indirect
-	github.com/jtolds/gls v4.20.0+incompatible // indirect
+	github.com/DataDog/zstd v1.5.2 // indirect
+	github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6 // indirect
+	github.com/VictoriaMetrics/fastcache v1.6.0 // indirect
+	github.com/beorn7/perks v1.0.1 // indirect
+	github.com/btcsuite/btcd/btcec/v2 v2.2.0 // indirect
+	github.com/btcsuite/btcd/chaincfg/chainhash v1.0.2 // indirect
+	github.com/cespare/xxhash/v2 v2.2.0 // indirect
+	github.com/cockroachdb/errors v1.9.1 // indirect
+	github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b // indirect
+	github.com/cockroachdb/pebble v0.0.0-20230209160836-829675f94811 // indirect
+	github.com/cockroachdb/redact v1.1.3 // indirect
+	github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1 // indirect
+	github.com/ethereum/go-ethereum v1.11.6 // indirect
+	github.com/getsentry/sentry-go v0.18.0 // indirect
+	github.com/go-ole/go-ole v1.2.1 // indirect
+	github.com/go-stack/stack v1.8.1 // indirect
+	github.com/gofrs/flock v0.8.1 // indirect
+	github.com/gogo/protobuf v1.3.2 // indirect
+	github.com/golang/protobuf v1.5.2 // indirect
+	github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb // indirect
+	github.com/holiman/uint256 v1.2.2-0.20230321075855-87b91420868c // indirect
+	github.com/klauspost/compress v1.15.15 // indirect
 	github.com/klauspost/cpuid/v2 v2.0.9 // indirect
+	github.com/kr/pretty v0.3.1 // indirect
+	github.com/kr/text v0.2.0 // indirect
+	github.com/mattn/go-runewidth v0.0.9 // indirect
+	github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
 	github.com/minio/sha256-simd v1.0.0 // indirect
 	github.com/mr-tron/base58 v1.2.0 // indirect
 	github.com/multiformats/go-base32 v0.0.3 // indirect
 	github.com/multiformats/go-base36 v0.1.0 // indirect
 	github.com/multiformats/go-multibase v0.0.3 // indirect
 	github.com/multiformats/go-multihash v0.2.1 // indirect
 	github.com/multiformats/go-varint v0.0.6 // indirect
+	github.com/olekukonko/tablewriter v0.0.5 // indirect
+	github.com/pkg/errors v0.9.1 // indirect
 	github.com/polydawn/refmt v0.0.0-20201211092308-30ac6d18308e // indirect
-	github.com/smartystreets/assertions v1.13.0 // indirect
+	github.com/prometheus/client_golang v1.14.0 // indirect
+	github.com/prometheus/client_model v0.3.0 // indirect
+	github.com/prometheus/common v0.39.0 // indirect
+	github.com/prometheus/procfs v0.9.0 // indirect
+	github.com/rogpeppe/go-internal v1.9.0 // indirect
+	github.com/shirou/gopsutil v3.21.4-0.20210419000835-c7a38de76ee5+incompatible // indirect
 	github.com/spaolacci/murmur3 v1.1.0 // indirect
+	github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 // indirect
+	github.com/tklauser/go-sysconf v0.3.5 // indirect
+	github.com/tklauser/numcpus v0.2.2 // indirect
 	golang.org/x/crypto v0.1.0 // indirect
-	golang.org/x/sys v0.1.0 // indirect
+	golang.org/x/exp v0.0.0-20230206171751-46f607a40771 // indirect
+	golang.org/x/sys v0.6.0 // indirect
+	golang.org/x/text v0.8.0 // indirect
 	google.golang.org/protobuf v1.28.1 // indirect
 	lukechampine.com/blake3 v1.1.6 // indirect
 )