Skip to content

Commit

Permalink
devnet: fix node startup on macOS (erigontech#8569)
Browse files Browse the repository at this point in the history 
* call getEnode before NodeStarted to make sure it is ready for RPC
calls
* fix connection error detection on macOS
* use a non-default p2p port to avoid conflicts
* disable bor milestones on local heimdall
* generate node keys for static peers config
  • Loading branch information
@battlmonstr
battlmonstr authored Oct 26, 2023
1 parent 043ccef commit f1c81dc
Show file tree
Hide file tree
Showing 6 changed files with 161 additions and 101 deletions.
56 changes: 39 additions & 17 deletions cmd/devnet/args/node.go
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,11 @@
package args

import (
"crypto/ecdsa"
"encoding/hex"
"fmt"
"github.com/ledgerwatch/erigon/crypto"
"github.com/ledgerwatch/erigon/p2p/enode"
"math/big"
"net"
"path/filepath"
Expand Down Expand Up @@ -44,7 +48,11 @@ type Node struct {
StaticPeers string `arg:"--staticpeers" json:"staticpeers,omitempty"`
WithoutHeimdall bool `arg:"--bor.withoutheimdall" flag:"" default:"false" json:"bor.withoutheimdall,omitempty"`
HeimdallGRpc string `arg:"--bor.heimdallgRPC" json:"bor.heimdallgRPC,omitempty"`
WithHeimdallMilestones bool `arg:"--bor.milestone" json:"bor.milestone"`
VMDebug bool `arg:"--vmdebug" flag:"" default:"false" json:"dmdebug"`

NodeKey *ecdsa.PrivateKey `arg:"-"`
NodeKeyHex string `arg:"--nodekeyhex" json:"nodekeyhex,omitempty"`
}

func (node *Node) configure(base Node, nodeNumber int) error {
Expand All @@ -62,14 +70,19 @@ func (node *Node) configure(base Node, nodeNumber int) error {

node.StaticPeers = base.StaticPeers

var err error
node.NodeKey, err = crypto.GenerateKey()
if err != nil {
return err
}
node.NodeKeyHex = hex.EncodeToString(crypto.FromECDSA(node.NodeKey))

node.Metrics = base.Metrics
node.MetricsPort = base.MetricsPort
node.MetricsAddr = base.MetricsAddr

node.Snapshots = base.Snapshots

var err error

node.PrivateApiAddr, _, err = portFromBase(base.PrivateApiAddr, nodeNumber, 1)

if err != nil {
Expand All @@ -86,13 +99,24 @@ func (node *Node) configure(base Node, nodeNumber int) error {

node.Port = base.Port + nodeNumber

node.WithHeimdallMilestones = base.WithHeimdallMilestones

return nil
}

func (node Node) ChainID() *big.Int {
func (node *Node) ChainID() *big.Int {
return &big.Int{}
}

func (node *Node) GetHttpPort() int {
return node.HttpPort
}

func (node *Node) GetEnodeURL() string {
port := node.Port
return enode.NewV4(&node.NodeKey.PublicKey, net.ParseIP("127.0.0.1"), port, port).URLv4()
}

type BlockProducer struct {
Node
Mine bool `arg:"--mine" flag:"true"`
Expand All @@ -105,11 +129,10 @@ type BlockProducer struct {
account *accounts.Account
}

func (m BlockProducer) Configure(baseNode Node, nodeNumber int) (int, interface{}, error) {
func (m *BlockProducer) Configure(baseNode Node, nodeNumber int) (interface{}, error) {
err := m.configure(baseNode, nodeNumber)

if err != nil {
return -1, nil, err
return nil, err
}

switch m.Chain {
Expand All @@ -131,18 +154,18 @@ func (m BlockProducer) Configure(baseNode Node, nodeNumber int) (int, interface{
m.Etherbase = m.account.Address.Hex()
}

return m.HttpPort, m, nil
return m, nil
}

func (n BlockProducer) Name() string {
func (n *BlockProducer) Name() string {
return n.Node.Name
}

func (n BlockProducer) Account() *accounts.Account {
func (n *BlockProducer) Account() *accounts.Account {
return n.account
}

func (n BlockProducer) IsBlockProducer() bool {
func (n *BlockProducer) IsBlockProducer() bool {
return true
}

Expand All @@ -153,25 +176,24 @@ type NonBlockProducer struct {
NoDiscover string `arg:"--nodiscover" flag:"" default:"true" json:"nodiscover"`
}

func (n NonBlockProducer) Configure(baseNode Node, nodeNumber int) (int, interface{}, error) {
func (n *NonBlockProducer) Configure(baseNode Node, nodeNumber int) (interface{}, error) {
err := n.configure(baseNode, nodeNumber)

if err != nil {
return -1, nil, err
return nil, err
}

return n.HttpPort, n, nil
return n, nil
}

func (n NonBlockProducer) Name() string {
func (n *NonBlockProducer) Name() string {
return n.Node.Name
}

func (n NonBlockProducer) IsBlockProducer() bool {
func (n *NonBlockProducer) IsBlockProducer() bool {
return false
}

func (n NonBlockProducer) Account() *accounts.Account {
func (n *NonBlockProducer) Account() *accounts.Account {
return nil
}

Expand Down
44 changes: 40 additions & 4 deletions cmd/devnet/args/node_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -162,8 +162,26 @@ func producingNodeArgs(dataDir string, nodeNumber int) []string {
authrpcPortArg, _ := parameterFromArgument("--authrpc.port", "8551")
natArg, _ := parameterFromArgument("--nat", "none")
accountSlotsArg, _ := parameterFromArgument("--txpool.accountslots", "16")

return []string{buildDirArg, dataDirArg, chainType, privateApiAddr, httpPortArg, authrpcPortArg, mine, httpApi, ws, natArg, devPeriod, consoleVerbosity, p2pProtocol, downloaderArg, accountSlotsArg}
withHeimdallMilestonesArg, _ := parameterFromArgument("--bor.milestone", "false")

return []string{
buildDirArg,
dataDirArg,
chainType,
privateApiAddr,
httpPortArg,
authrpcPortArg,
mine,
httpApi,
ws,
natArg,
devPeriod,
consoleVerbosity,
p2pProtocol,
downloaderArg,
accountSlotsArg,
withHeimdallMilestonesArg,
}
}

// nonMiningNodeArgs returns custom args for starting a non-mining node
Expand All @@ -182,6 +200,24 @@ func nonProducingNodeArgs(dataDir string, nodeNumber int, enode string) []string
authrpcPortArg, _ := parameterFromArgument("--authrpc.port", "8551")
natArg, _ := parameterFromArgument("--nat", "none")
ws := wsArg

return []string{buildDirArg, dataDirArg, chainType, privateApiAddr, httpPortArg, authrpcPortArg, httpApi, ws, natArg, staticPeers, noDiscover, consoleVerbosity, torrentPort, p2pProtocol, downloaderArg}
withHeimdallMilestonesArg, _ := parameterFromArgument("--bor.milestone", "false")

return []string{
buildDirArg,
dataDirArg,
chainType,
privateApiAddr,
httpPortArg,
authrpcPortArg,
httpApi,
ws,
natArg,
staticPeers,
noDiscover,
consoleVerbosity,
torrentPort,
p2pProtocol,
downloaderArg,
withHeimdallMilestonesArg,
}
}
99 changes: 32 additions & 67 deletions cmd/devnet/devnet/network.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,9 +4,9 @@ import (
"context"
"errors"
"fmt"
"github.com/ledgerwatch/erigon/cmd/utils"
"math/big"
"net"
"net/url"
"os"
"reflect"
"strings"
Expand Down Expand Up @@ -40,6 +40,7 @@ type Network struct {
BorStateSyncDelay time.Duration
BorPeriod time.Duration
BorMinBlockSize int
BorWithMilestones *bool
wg sync.WaitGroup
peers []string
namedNodes map[string]Node
Expand All @@ -55,11 +56,6 @@ func (nw *Network) ChainID() *big.Int {

// Start starts the process for multiple erigon nodes running on the dev chain
func (nw *Network) Start(ctx context.Context) error {

type configurable interface {
Configure(baseNode args.Node, nodeNumber int) (int, interface{}, error)
}

for _, service := range nw.Services {
if err := service.Start(ctx); err != nil {
nw.Stop()
Expand All @@ -76,35 +72,45 @@ func (nw *Network) Start(ctx context.Context) error {
Snapshots: nw.Snapshots,
}

if nw.BorWithMilestones != nil {
baseNode.WithHeimdallMilestones = *nw.BorWithMilestones
} else {
baseNode.WithHeimdallMilestones = utils.WithHeimdallMilestones.Value
}

cliCtx := CliContext(ctx)

metricsEnabled := cliCtx.Bool("metrics")
metricsNode := cliCtx.Int("metrics.node")
nw.namedNodes = map[string]Node{}

for i, node := range nw.Nodes {
if configurable, ok := node.(configurable); ok {

for i, nodeConfig := range nw.Nodes {
{
base := baseNode

if metricsEnabled && metricsNode == i {
base.Metrics = true
base.MetricsPort = cliCtx.Int("metrics.port")
}
base.StaticPeers = strings.Join(nw.peers, ",")

nodePort, args, err := configurable.Configure(base, i)

if err == nil {
node, err = nw.createNode(fmt.Sprintf("%s:%d", nw.BaseRPCHost, nodePort), args)
argsObj, err := nodeConfig.Configure(base, i)
if err != nil {
nw.Stop()
return err
}

nodePort := nodeConfig.GetHttpPort()
nodeAddr := fmt.Sprintf("%s:%d", nw.BaseRPCHost, nodePort)

node, err := nw.createNode(nodeAddr, argsObj)
if err != nil {
nw.Stop()
return err
}

nw.Nodes[i] = node
nw.namedNodes[node.Name()] = node
nw.peers = append(nw.peers, nodeConfig.GetEnodeURL())

for _, service := range nw.Services {
service.NodeCreated(ctx, node)
Expand All @@ -114,7 +120,6 @@ func (nw *Network) Start(ctx context.Context) error {

for _, node := range nw.Nodes {
err := nw.startNode(node)

if err != nil {
nw.Stop()
return err
Expand All @@ -123,25 +128,6 @@ func (nw *Network) Start(ctx context.Context) error {
for _, service := range nw.Services {
service.NodeStarted(ctx, node)
}

// get the enode of the node
// - note this has the side effect of waiting for the node to start
enode, err := getEnode(node)

if err != nil {
if errors.Is(err, devnetutils.ErrInvalidEnodeString) {
continue
}

nw.Stop()
return err
}

nw.peers = append(nw.peers, enode)

// TODO do we need to call AddPeer to the nodes to make them aware of this one
// the current model only works for an appending node network where the peers gossip
// connections - not sure if this is the case ?
}

return nil
Expand Down Expand Up @@ -201,28 +187,10 @@ func (nw *Network) startNode(n Node) error {
node := n.(*node)

args, err := args.AsArgs(node.args)

if err != nil {
return err
}

if len(nw.peers) > 0 {
peersIndex := -1

for i, arg := range args {
if strings.HasPrefix(arg, "--staticpeers") {
peersIndex = i
break
}
}

if peersIndex >= 0 {
args[peersIndex] = args[peersIndex] + "," + strings.Join(nw.peers, ",")
} else {
args = append(args, "--staticpeers="+strings.Join(nw.peers, ","))
}
}

go func() {
nw.Logger.Info("Running node", "name", node.Name(), "args", args)

Expand Down Expand Up @@ -254,6 +222,14 @@ func (nw *Network) startNode(n Node) error {
return nil
}

func isConnectionError(err error) bool {
var opErr *net.OpError
if errors.As(err, &opErr) {
return opErr.Op == "dial"
}
return false
}

// getEnode returns the enode of the netowrk node
func getEnode(n Node) (string, error) {
reqCount := 0
Expand All @@ -268,21 +244,10 @@ func getEnode(n Node) (string, error) {
}
}

if reqCount < 10 {
var urlErr *url.Error
if errors.As(err, &urlErr) {
var opErr *net.OpError
if errors.As(urlErr.Err, &opErr) {
var callErr *os.SyscallError
if errors.As(opErr.Err, &callErr) {
if strings.HasPrefix(callErr.Syscall, "connect") {
reqCount++
time.Sleep(time.Duration(devnetutils.RandomInt(5)) * time.Second)
continue
}
}
}
}
if isConnectionError(err) && (reqCount < 10) {
reqCount++
time.Sleep(time.Duration(devnetutils.RandomInt(5)) * time.Second)
continue
}

return "", err
Expand Down
Loading

0 comments on commit f1c81dc

Please sign in to comment.