diff --git a/orderer/consensus/smartbft/chain_test.go b/orderer/consensus/smartbft/chain_test.go index af198aeec5e..a7b5afce5dc 100644 --- a/orderer/consensus/smartbft/chain_test.go +++ b/orderer/consensus/smartbft/chain_test.go @@ -195,7 +195,9 @@ func TestSyncNode(t *testing.T) { // 2. Submit a TX and wait for the TX to be received by all nodes // 3. Add new node to the network // 4. Submit a TX and wait for the TX to be received by all nodes -func TestAddNode(t *testing.T) { +// 5. Remove node from the network +// 6. Submit a TX and wait for the TX to be received by all nodes +func TestAddAndRemoveNode(t *testing.T) { dir := t.TempDir() channelId := "testchannel" @@ -238,6 +240,38 @@ func TestAddNode(t *testing.T) { for _, node := range nodeMap { node.State.WaitLedgerHeightToBe(4) } + + // get leader id and ask for the last config block + leaderId := networkSetupInfo.GetAgreedLeader() + lastConfigBlock := nodesMap[leaderId].GetConfigBlock(networkSetupInfo.configInfo.numsOfConfigBlocks[len(networkSetupInfo.configInfo.numsOfConfigBlocks)-1]) + + // stop the node that should be removed + nodeToRemove := nodesMap[uint64(5)] + nodeToRemove.Stop() + + // send a new config block to all nodes, to remove node number 5 + env = removeNodeFromConfig(t, lastConfigBlock, 5, networkSetupInfo.channelId) + err = networkSetupInfo.SendTxToAllAvailableNodes(env) + require.NoError(t, err) + for _, node := range nodeMap { + if node.NodeId == nodeToRemove.NodeId { + continue + } + node.State.WaitLedgerHeightToBe(5) + } + + // remove node from the network + nodesMap = networkSetupInfo.RemoveNode(uint64(5)) + require.Equal(t, len(nodesMap), 4) + require.Equal(t, len(networkSetupInfo.nodeIdToNode), 4) + + // send a tx to all nodes again and wait the tx will be added to each ledger + env = createEndorserTxEnvelope("TEST_REMOVAL_OF_NODE", channelId) + err = networkSetupInfo.SendTxToAllAvailableNodes(env) + require.NoError(t, err) + for _, node := range nodeMap { + node.State.WaitLedgerHeightToBe(6) + } } // Scenario: @@ -247,7 +281,7 @@ func TestAddNode(t *testing.T) { // 4. Add new node to the network // 5. Start the old leader and make sure he has synced with other nodes about the config change // 4. Submit a TX and wait for the TX to be received by all nodes -func TestReconfigurationWhileNodeIsDown(t *testing.T) { +func TestAddNodeWhileAnotherNodeIsDown(t *testing.T) { dir := t.TempDir() channelId := "testchannel" @@ -313,6 +347,89 @@ func TestReconfigurationWhileNodeIsDown(t *testing.T) { } } +// Scenario: +// 1. Start a network of 4 nodes +// 2. Submit a TX and wait for the TX to be received by all nodes +// 3. Add new node to the network +// 4. Submit a TX and wait for the TX to be received by all nodes +// 5. Remove node from the network +// 6. Submit a TX and wait for the TX to be received by all nodes +// NOTE: in this scenario node 5 is not stopped and is only removed from the list of nodes, in such a case nodes 1-4 should ignore its messages. +func TestAddAndRemoveNodeWithoutStop(t *testing.T) { + dir := t.TempDir() + channelId := "testchannel" + + // start a network + networkSetupInfo := NewNetworkSetupInfo(t, channelId, dir) + nodeMap := networkSetupInfo.CreateNodes(4) + networkSetupInfo.StartAllNodes() + + // wait until all nodes have the genesis block in their ledger + for _, node := range nodeMap { + node.State.WaitLedgerHeightToBe(1) + } + + // send a tx to all nodes and wait the tx will be added to each ledger + env := createEndorserTxEnvelope("TEST_MESSAGE #1", channelId) + err := networkSetupInfo.SendTxToAllAvailableNodes(env) + require.NoError(t, err) + for _, node := range nodeMap { + node.State.WaitLedgerHeightToBe(2) + } + + // send a new config block to all nodes, to notice them about the new node + env = addNodeToConfig(t, networkSetupInfo.genesisBlock, 5, networkSetupInfo.tlsCA, networkSetupInfo.dir, networkSetupInfo.channelId) + err = networkSetupInfo.SendTxToAllAvailableNodes(env) + require.NoError(t, err) + for _, node := range nodeMap { + node.State.WaitLedgerHeightToBe(3) + } + + // add new node to the network + nodesMap, newNode := networkSetupInfo.AddNewNode() + newNode.Start() + require.Equal(t, len(nodesMap), 5) + require.Equal(t, len(networkSetupInfo.nodeIdToNode), 5) + + // send a tx to all nodes again and wait the tx will be added to each ledger + env = createEndorserTxEnvelope("TEST_ADDITION_OF_NODE", channelId) + err = networkSetupInfo.SendTxToAllAvailableNodes(env) + require.NoError(t, err) + for _, node := range nodeMap { + node.State.WaitLedgerHeightToBe(4) + } + + // get leader id and ask for the last config block + leaderId := networkSetupInfo.GetAgreedLeader() + lastConfigBlock := nodesMap[leaderId].GetConfigBlock(networkSetupInfo.configInfo.numsOfConfigBlocks[len(networkSetupInfo.configInfo.numsOfConfigBlocks)-1]) + + nodeToRemove := nodesMap[uint64(5)] + + // send a new config block to all nodes, to remove node number 5 + env = removeNodeFromConfig(t, lastConfigBlock, 5, networkSetupInfo.channelId) + err = networkSetupInfo.SendTxToAllAvailableNodes(env) + require.NoError(t, err) + for _, node := range nodeMap { + if node.NodeId == nodeToRemove.NodeId { + continue + } + node.State.WaitLedgerHeightToBe(5) + } + + // remove node from the network + nodesMap = networkSetupInfo.RemoveNode(uint64(5)) + require.Equal(t, len(nodesMap), 4) + require.Equal(t, len(networkSetupInfo.nodeIdToNode), 4) + + // send a tx to all nodes again and wait the tx will be added to each ledger + env = createEndorserTxEnvelope("TEST_REMOVAL_OF_NODE", channelId) + err = networkSetupInfo.SendTxToAllAvailableNodes(env) + require.NoError(t, err) + for _, node := range nodeMap { + node.State.WaitLedgerHeightToBe(6) + } +} + func createEndorserTxEnvelope(message string, channelId string) *cb.Envelope { return &cb.Envelope{ Payload: protoutil.MarshalOrPanic(&cb.Payload{ @@ -345,7 +462,7 @@ func generateNonce() []byte { return nonceBuf.Bytes() } -// addNodeToConfig creates a config block based on the last config block. It is useful for addition or removal of a node +// addNodeToConfig creates a config block based on the last config block. It is useful for addition of a node func addNodeToConfig(t *testing.T, lastConfigBlock *cb.Block, nodeId uint32, tlsCA tlsgen.CA, certDir string, channelId string) *cb.Envelope { // copy the last config block clonedLastConfigBlock := proto.Clone(lastConfigBlock).(*cb.Block) @@ -389,7 +506,8 @@ func addNodeToConfig(t *testing.T, lastConfigBlock *cb.Block, nodeId uint32, tls // update organization endpoints ordererEndpoints := configEnv.Config.ChannelGroup.Groups[channelconfig.OrdererGroupKey].Groups["SampleOrg"].Values["Endpoints"].Value ordererEndpointsVal := &cb.OrdererAddresses{} - proto.Unmarshal(ordererEndpoints, ordererEndpointsVal) + err = proto.Unmarshal(ordererEndpoints, ordererEndpointsVal) + require.NoError(t, err) ordererAddresses := ordererEndpointsVal.Addresses ordererAddresses = append(ordererAddresses, fmt.Sprintf("%s:%d", newOrderer.Host, newOrderer.Port)) configEnv.Config.ChannelGroup.Groups[channelconfig.OrdererGroupKey].Groups["SampleOrg"].Values["Endpoints"].Value = protoutil.MarshalOrPanic(&cb.OrdererAddresses{ @@ -430,3 +548,87 @@ func addNodeToConfig(t *testing.T, lastConfigBlock *cb.Block, nodeId uint32, tls }), } } + +// removeNodeFromConfig creates a config block based on the last config block. It is useful for removal of a node +func removeNodeFromConfig(t *testing.T, lastConfigBlock *cb.Block, nodeId uint32, channelId string) *cb.Envelope { + // copy the last config block + clonedLastConfigBlock := proto.Clone(lastConfigBlock).(*cb.Block) + + // fetch the ConfigEnvelope from the block + env := protoutil.UnmarshalEnvelopeOrPanic(clonedLastConfigBlock.Data.Data[0]) + payload := protoutil.UnmarshalPayloadOrPanic(env.Payload) + configEnv, err := protoutil.UnmarshalConfigEnvelope(payload.Data) + require.NoError(t, err) + originalConfigEnv := proto.Clone(configEnv).(*cb.ConfigEnvelope) + + // update the consenter mapping to exclude the new node + currentOrderers := &cb.Orderers{} + err = proto.Unmarshal(configEnv.Config.ChannelGroup.Groups[channelconfig.OrdererGroupKey].Values[channelconfig.OrderersKey].Value, currentOrderers) + require.NoError(t, err) + var newConsenterMapping []*cb.Consenter + var ordererAddress string + for i, consenter := range currentOrderers.ConsenterMapping { + if consenter.Id == nodeId { + newConsenterMapping = append(currentOrderers.ConsenterMapping[:i], currentOrderers.ConsenterMapping[i+1:]...) + ordererAddress = fmt.Sprintf("%s:%d", consenter.Host, consenter.Port) + break + } + } + + currentOrderers.ConsenterMapping = newConsenterMapping + configEnv.Config.ChannelGroup.Groups[channelconfig.OrdererGroupKey].Values[channelconfig.OrderersKey] = &cb.ConfigValue{ + Version: configEnv.Config.ChannelGroup.Groups[channelconfig.OrdererGroupKey].Values[channelconfig.OrderersKey].Version + 1, + Value: protoutil.MarshalOrPanic(currentOrderers), + ModPolicy: channelconfig.AdminsPolicyKey, + } + + // update organization endpoints to exclude the node's endpoint + ordererEndpoints := configEnv.Config.ChannelGroup.Groups[channelconfig.OrdererGroupKey].Groups["SampleOrg"].Values["Endpoints"].Value + ordererEndpointsVal := &cb.OrdererAddresses{} + err = proto.Unmarshal(ordererEndpoints, ordererEndpointsVal) + require.NoError(t, err) + ordererAddresses := ordererEndpointsVal.Addresses + var newOrdererAddresses []string + for i, address := range ordererAddresses { + if address == ordererAddress { + newOrdererAddresses = append(ordererAddresses[:i], ordererAddresses[i+1:]...) + } + } + configEnv.Config.ChannelGroup.Groups[channelconfig.OrdererGroupKey].Groups["SampleOrg"].Values["Endpoints"].Value = protoutil.MarshalOrPanic(&cb.OrdererAddresses{ + Addresses: newOrdererAddresses, + }) + + // increase the sequence + configEnv.Config.Sequence = configEnv.Config.Sequence + 1 + + // calculate config update tx + configUpdate, err := update.Compute(originalConfigEnv.Config, configEnv.Config) + require.NoError(t, err) + signerSerializer := smartBFTMocks.NewSignerSerializer(t) + signerSerializer.EXPECT().Serialize().RunAndReturn( + func() ([]byte, error) { + return []byte{1, 2, 3}, nil + }).Maybe() + signerSerializer.EXPECT().Sign(mock.Anything).RunAndReturn( + func(message []byte) ([]byte, error) { + return message, nil + }).Maybe() + configUpdateTx, err := protoutil.CreateSignedEnvelope(cb.HeaderType_CONFIG_UPDATE, channelId, signerSerializer, configUpdate, 0, 0) + require.NoError(t, err) + + // return the updated Envelope + return &cb.Envelope{ + Payload: protoutil.MarshalOrPanic(&cb.Payload{ + Data: protoutil.MarshalOrPanic(&cb.ConfigEnvelope{ + Config: configEnv.Config, + LastUpdate: configUpdateTx, + }), + Header: &cb.Header{ + ChannelHeader: protoutil.MarshalOrPanic(&cb.ChannelHeader{ + Type: int32(cb.HeaderType_CONFIG), + ChannelId: channelId, + }), + }, + }), + } +} diff --git a/orderer/consensus/smartbft/util_network_test.go b/orderer/consensus/smartbft/util_network_test.go index a4403ef3f41..42f02708b80 100644 --- a/orderer/consensus/smartbft/util_network_test.go +++ b/orderer/consensus/smartbft/util_network_test.go @@ -128,6 +128,18 @@ func (ns *NetworkSetupInfo) AddNewNode() (map[uint64]*Node, *Node) { return ns.nodeIdToNode, ns.nodeIdToNode[newNodeId] } +func (ns *NetworkSetupInfo) RemoveNode(num uint64) map[uint64]*Node { + ns.t.Logf("Removing node %v from the network", num) + delete(ns.nodeIdToNode, num) + + // update all nodes about the nodes map + for nodeId := range ns.nodeIdToNode { + ns.nodeIdToNode[nodeId].nodesMap = ns.nodeIdToNode + } + + return ns.nodeIdToNode +} + func (ns *NetworkSetupInfo) StartAllNodes() { ns.t.Logf("Starting nodes in the network") for _, node := range ns.nodeIdToNode { @@ -280,6 +292,12 @@ func (n *Node) SendTx(tx *cb.Envelope) error { return n.Chain.Order(tx, n.State.Sequence) } +func (n *Node) GetConfigBlock(num uint64) *cb.Block { + n.lock.RLock() + defer n.lock.RUnlock() + return n.State.ledgerArray[num] +} + func isConfigTx(envelope *cb.Envelope) bool { if envelope == nil { return false