Skip to content

Commit

Permalink
BFT chain unit tests: configure - remove node (#4801)
Browse files Browse the repository at this point in the history 
* BFT chain unit tests: reconfigure - remove node

Signed-off-by: May Rosenbaum <mayro1595@gmail.com>

* BFT chain unit tests: reconfigure - remove node without stop the node

Signed-off-by: May Rosenbaum <mayro1595@gmail.com>

---------

Signed-off-by: May Rosenbaum <mayro1595@gmail.com>
  • Loading branch information
@MayRosenbaum
MayRosenbaum authored Apr 17, 2024
1 parent b715a98 commit 21d6460
Show file tree
Hide file tree
Showing 2 changed files with 224 additions and 4 deletions.
210 changes: 206 additions & 4 deletions orderer/consensus/smartbft/chain_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -195,7 +195,9 @@ func TestSyncNode(t *testing.T) {
// 2. Submit a TX and wait for the TX to be received by all nodes
// 3. Add new node to the network
// 4. Submit a TX and wait for the TX to be received by all nodes
func TestAddNode(t *testing.T) {
// 5. Remove node from the network
// 6. Submit a TX and wait for the TX to be received by all nodes
func TestAddAndRemoveNode(t *testing.T) {
dir := t.TempDir()
channelId := "testchannel"

Expand Down Expand Up @@ -238,6 +240,38 @@ func TestAddNode(t *testing.T) {
for _, node := range nodeMap {
node.State.WaitLedgerHeightToBe(4)
}

// get leader id and ask for the last config block
leaderId := networkSetupInfo.GetAgreedLeader()
lastConfigBlock := nodesMap[leaderId].GetConfigBlock(networkSetupInfo.configInfo.numsOfConfigBlocks[len(networkSetupInfo.configInfo.numsOfConfigBlocks)-1])

// stop the node that should be removed
nodeToRemove := nodesMap[uint64(5)]
nodeToRemove.Stop()

// send a new config block to all nodes, to remove node number 5
env = removeNodeFromConfig(t, lastConfigBlock, 5, networkSetupInfo.channelId)
err = networkSetupInfo.SendTxToAllAvailableNodes(env)
require.NoError(t, err)
for _, node := range nodeMap {
if node.NodeId == nodeToRemove.NodeId {
continue
}
node.State.WaitLedgerHeightToBe(5)
}

// remove node from the network
nodesMap = networkSetupInfo.RemoveNode(uint64(5))
require.Equal(t, len(nodesMap), 4)
require.Equal(t, len(networkSetupInfo.nodeIdToNode), 4)

// send a tx to all nodes again and wait the tx will be added to each ledger
env = createEndorserTxEnvelope("TEST_REMOVAL_OF_NODE", channelId)
err = networkSetupInfo.SendTxToAllAvailableNodes(env)
require.NoError(t, err)
for _, node := range nodeMap {
node.State.WaitLedgerHeightToBe(6)
}
}

// Scenario:
Expand All @@ -247,7 +281,7 @@ func TestAddNode(t *testing.T) {
// 4. Add new node to the network
// 5. Start the old leader and make sure he has synced with other nodes about the config change
// 4. Submit a TX and wait for the TX to be received by all nodes
func TestReconfigurationWhileNodeIsDown(t *testing.T) {
func TestAddNodeWhileAnotherNodeIsDown(t *testing.T) {
dir := t.TempDir()
channelId := "testchannel"

Expand Down Expand Up @@ -313,6 +347,89 @@ func TestReconfigurationWhileNodeIsDown(t *testing.T) {
}
}

// Scenario:
// 1. Start a network of 4 nodes
// 2. Submit a TX and wait for the TX to be received by all nodes
// 3. Add new node to the network
// 4. Submit a TX and wait for the TX to be received by all nodes
// 5. Remove node from the network
// 6. Submit a TX and wait for the TX to be received by all nodes
// NOTE: in this scenario node 5 is not stopped and is only removed from the list of nodes, in such a case nodes 1-4 should ignore its messages.
func TestAddAndRemoveNodeWithoutStop(t *testing.T) {
dir := t.TempDir()
channelId := "testchannel"

// start a network
networkSetupInfo := NewNetworkSetupInfo(t, channelId, dir)
nodeMap := networkSetupInfo.CreateNodes(4)
networkSetupInfo.StartAllNodes()

// wait until all nodes have the genesis block in their ledger
for _, node := range nodeMap {
node.State.WaitLedgerHeightToBe(1)
}

// send a tx to all nodes and wait the tx will be added to each ledger
env := createEndorserTxEnvelope("TEST_MESSAGE #1", channelId)
err := networkSetupInfo.SendTxToAllAvailableNodes(env)
require.NoError(t, err)
for _, node := range nodeMap {
node.State.WaitLedgerHeightToBe(2)
}

// send a new config block to all nodes, to notice them about the new node
env = addNodeToConfig(t, networkSetupInfo.genesisBlock, 5, networkSetupInfo.tlsCA, networkSetupInfo.dir, networkSetupInfo.channelId)
err = networkSetupInfo.SendTxToAllAvailableNodes(env)
require.NoError(t, err)
for _, node := range nodeMap {
node.State.WaitLedgerHeightToBe(3)
}

// add new node to the network
nodesMap, newNode := networkSetupInfo.AddNewNode()
newNode.Start()
require.Equal(t, len(nodesMap), 5)
require.Equal(t, len(networkSetupInfo.nodeIdToNode), 5)

// send a tx to all nodes again and wait the tx will be added to each ledger
env = createEndorserTxEnvelope("TEST_ADDITION_OF_NODE", channelId)
err = networkSetupInfo.SendTxToAllAvailableNodes(env)
require.NoError(t, err)
for _, node := range nodeMap {
node.State.WaitLedgerHeightToBe(4)
}

// get leader id and ask for the last config block
leaderId := networkSetupInfo.GetAgreedLeader()
lastConfigBlock := nodesMap[leaderId].GetConfigBlock(networkSetupInfo.configInfo.numsOfConfigBlocks[len(networkSetupInfo.configInfo.numsOfConfigBlocks)-1])

nodeToRemove := nodesMap[uint64(5)]

// send a new config block to all nodes, to remove node number 5
env = removeNodeFromConfig(t, lastConfigBlock, 5, networkSetupInfo.channelId)
err = networkSetupInfo.SendTxToAllAvailableNodes(env)
require.NoError(t, err)
for _, node := range nodeMap {
if node.NodeId == nodeToRemove.NodeId {
continue
}
node.State.WaitLedgerHeightToBe(5)
}

// remove node from the network
nodesMap = networkSetupInfo.RemoveNode(uint64(5))
require.Equal(t, len(nodesMap), 4)
require.Equal(t, len(networkSetupInfo.nodeIdToNode), 4)

// send a tx to all nodes again and wait the tx will be added to each ledger
env = createEndorserTxEnvelope("TEST_REMOVAL_OF_NODE", channelId)
err = networkSetupInfo.SendTxToAllAvailableNodes(env)
require.NoError(t, err)
for _, node := range nodeMap {
node.State.WaitLedgerHeightToBe(6)
}
}

func createEndorserTxEnvelope(message string, channelId string) *cb.Envelope {
return &cb.Envelope{
Payload: protoutil.MarshalOrPanic(&cb.Payload{
Expand Down Expand Up @@ -345,7 +462,7 @@ func generateNonce() []byte {
return nonceBuf.Bytes()
}

// addNodeToConfig creates a config block based on the last config block. It is useful for addition or removal of a node
// addNodeToConfig creates a config block based on the last config block. It is useful for addition of a node
func addNodeToConfig(t *testing.T, lastConfigBlock *cb.Block, nodeId uint32, tlsCA tlsgen.CA, certDir string, channelId string) *cb.Envelope {
// copy the last config block
clonedLastConfigBlock := proto.Clone(lastConfigBlock).(*cb.Block)
Expand Down Expand Up @@ -389,7 +506,8 @@ func addNodeToConfig(t *testing.T, lastConfigBlock *cb.Block, nodeId uint32, tls
// update organization endpoints
ordererEndpoints := configEnv.Config.ChannelGroup.Groups[channelconfig.OrdererGroupKey].Groups["SampleOrg"].Values["Endpoints"].Value
ordererEndpointsVal := &cb.OrdererAddresses{}
proto.Unmarshal(ordererEndpoints, ordererEndpointsVal)
err = proto.Unmarshal(ordererEndpoints, ordererEndpointsVal)
require.NoError(t, err)
ordererAddresses := ordererEndpointsVal.Addresses
ordererAddresses = append(ordererAddresses, fmt.Sprintf("%s:%d", newOrderer.Host, newOrderer.Port))
configEnv.Config.ChannelGroup.Groups[channelconfig.OrdererGroupKey].Groups["SampleOrg"].Values["Endpoints"].Value = protoutil.MarshalOrPanic(&cb.OrdererAddresses{
Expand Down Expand Up @@ -430,3 +548,87 @@ func addNodeToConfig(t *testing.T, lastConfigBlock *cb.Block, nodeId uint32, tls
}),
}
}

// removeNodeFromConfig creates a config block based on the last config block. It is useful for removal of a node
func removeNodeFromConfig(t *testing.T, lastConfigBlock *cb.Block, nodeId uint32, channelId string) *cb.Envelope {
// copy the last config block
clonedLastConfigBlock := proto.Clone(lastConfigBlock).(*cb.Block)

// fetch the ConfigEnvelope from the block
env := protoutil.UnmarshalEnvelopeOrPanic(clonedLastConfigBlock.Data.Data[0])
payload := protoutil.UnmarshalPayloadOrPanic(env.Payload)
configEnv, err := protoutil.UnmarshalConfigEnvelope(payload.Data)
require.NoError(t, err)
originalConfigEnv := proto.Clone(configEnv).(*cb.ConfigEnvelope)

// update the consenter mapping to exclude the new node
currentOrderers := &cb.Orderers{}
err = proto.Unmarshal(configEnv.Config.ChannelGroup.Groups[channelconfig.OrdererGroupKey].Values[channelconfig.OrderersKey].Value, currentOrderers)
require.NoError(t, err)
var newConsenterMapping []*cb.Consenter
var ordererAddress string
for i, consenter := range currentOrderers.ConsenterMapping {
if consenter.Id == nodeId {
newConsenterMapping = append(currentOrderers.ConsenterMapping[:i], currentOrderers.ConsenterMapping[i+1:]...)
ordererAddress = fmt.Sprintf("%s:%d", consenter.Host, consenter.Port)
break
}
}

currentOrderers.ConsenterMapping = newConsenterMapping
configEnv.Config.ChannelGroup.Groups[channelconfig.OrdererGroupKey].Values[channelconfig.OrderersKey] = &cb.ConfigValue{
Version: configEnv.Config.ChannelGroup.Groups[channelconfig.OrdererGroupKey].Values[channelconfig.OrderersKey].Version + 1,
Value: protoutil.MarshalOrPanic(currentOrderers),
ModPolicy: channelconfig.AdminsPolicyKey,
}

// update organization endpoints to exclude the node's endpoint
ordererEndpoints := configEnv.Config.ChannelGroup.Groups[channelconfig.OrdererGroupKey].Groups["SampleOrg"].Values["Endpoints"].Value
ordererEndpointsVal := &cb.OrdererAddresses{}
err = proto.Unmarshal(ordererEndpoints, ordererEndpointsVal)
require.NoError(t, err)
ordererAddresses := ordererEndpointsVal.Addresses
var newOrdererAddresses []string
for i, address := range ordererAddresses {
if address == ordererAddress {
newOrdererAddresses = append(ordererAddresses[:i], ordererAddresses[i+1:]...)
}
}
configEnv.Config.ChannelGroup.Groups[channelconfig.OrdererGroupKey].Groups["SampleOrg"].Values["Endpoints"].Value = protoutil.MarshalOrPanic(&cb.OrdererAddresses{
Addresses: newOrdererAddresses,
})

// increase the sequence
configEnv.Config.Sequence = configEnv.Config.Sequence + 1

// calculate config update tx
configUpdate, err := update.Compute(originalConfigEnv.Config, configEnv.Config)
require.NoError(t, err)
signerSerializer := smartBFTMocks.NewSignerSerializer(t)
signerSerializer.EXPECT().Serialize().RunAndReturn(
func() ([]byte, error) {
return []byte{1, 2, 3}, nil
}).Maybe()
signerSerializer.EXPECT().Sign(mock.Anything).RunAndReturn(
func(message []byte) ([]byte, error) {
return message, nil
}).Maybe()
configUpdateTx, err := protoutil.CreateSignedEnvelope(cb.HeaderType_CONFIG_UPDATE, channelId, signerSerializer, configUpdate, 0, 0)
require.NoError(t, err)

// return the updated Envelope
return &cb.Envelope{
Payload: protoutil.MarshalOrPanic(&cb.Payload{
Data: protoutil.MarshalOrPanic(&cb.ConfigEnvelope{
Config: configEnv.Config,
LastUpdate: configUpdateTx,
}),
Header: &cb.Header{
ChannelHeader: protoutil.MarshalOrPanic(&cb.ChannelHeader{
Type: int32(cb.HeaderType_CONFIG),
ChannelId: channelId,
}),
},
}),
}
}
18 changes: 18 additions & 0 deletions orderer/consensus/smartbft/util_network_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -128,6 +128,18 @@ func (ns *NetworkSetupInfo) AddNewNode() (map[uint64]*Node, *Node) {
return ns.nodeIdToNode, ns.nodeIdToNode[newNodeId]
}

func (ns *NetworkSetupInfo) RemoveNode(num uint64) map[uint64]*Node {
ns.t.Logf("Removing node %v from the network", num)
delete(ns.nodeIdToNode, num)

// update all nodes about the nodes map
for nodeId := range ns.nodeIdToNode {
ns.nodeIdToNode[nodeId].nodesMap = ns.nodeIdToNode
}

return ns.nodeIdToNode
}

func (ns *NetworkSetupInfo) StartAllNodes() {
ns.t.Logf("Starting nodes in the network")
for _, node := range ns.nodeIdToNode {
Expand Down Expand Up @@ -280,6 +292,12 @@ func (n *Node) SendTx(tx *cb.Envelope) error {
return n.Chain.Order(tx, n.State.Sequence)
}

func (n *Node) GetConfigBlock(num uint64) *cb.Block {
n.lock.RLock()
defer n.lock.RUnlock()
return n.State.ledgerArray[num]
}

func isConfigTx(envelope *cb.Envelope) bool {
if envelope == nil {
return false
Expand Down

0 comments on commit 21d6460

Please sign in to comment.