Initial backend updates to conform to new function signatures in core

Dockerfile

@@ -28,12 +28,14 @@ COPY backend/countries countries
 COPY backend/main.go   .
 
 # include core src
-COPY core/bls      ../core/bls
-COPY core/cmd      ../core/cmd
-COPY core/desohash ../core/desohash
-COPY core/lib      ../core/lib
-COPY core/migrate  ../core/migrate
-COPY core/scripts  ../core/scripts
+COPY core/bls         ../core/bls
+COPY core/cmd         ../core/cmd
+COPY core/collections ../core/collections
+COPY core/consensus   ../core/consensus
+COPY core/desohash    ../core/desohash
+COPY core/lib         ../core/lib
+COPY core/migrate     ../core/migrate
+COPY core/scripts     ../core/scripts
 
 RUN ../core/scripts/install-relic.sh
 

go.mod

@@ -53,6 +53,7 @@ require (
 	github.com/deso-protocol/go-merkle-tree v1.0.0 // indirect
 	github.com/dgraph-io/ristretto v0.1.1 // indirect
 	github.com/dustin/go-humanize v1.0.0 // indirect
+	github.com/emirpasic/gods v1.18.1 // indirect
 	github.com/ethereum/go-ethereum v1.9.25 // indirect
 	github.com/fatih/color v1.13.0 // indirect
 	github.com/fsnotify/fsnotify v1.4.9 // indirect

go.sum

@@ -164,6 +164,8 @@ github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4
 github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
 github.com/dvyukov/go-fuzz v0.0.0-20200318091601-be3528f3a813/go.mod h1:11Gm+ccJnvAhCNLlf5+cS9KjtbaD5I5zaZpFMsTHWTw=
 github.com/edsrzf/mmap-go v0.0.0-20160512033002-935e0e8a636c/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M=
+github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc=
+github.com/emirpasic/gods v1.18.1/go.mod h1:8tpGGwCnJ5H4r6BWwaV6OrWmMoPhUl5jm/FMNAnJvWQ=
 github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=

routes/access_group_test.go

@@ -97,7 +97,7 @@ func ExecuteRequest(t *testing.T, apiServer *APIServer, routePath string, reques
 func TestAPIAccessGroupBaseGroupMembership(t *testing.T) {
 	assert := assert.New(t)
 
-	apiServer, _, _ := newTestAPIServer(t, "" /*globalStateRemoteNode*/)
+	apiServer, _, _ := newTestAPIServer(t, "" /*globalStateRemoteNode*/, false)
 
 	// form the request for RoutePathGetAllUserAccessGroups
 	values := GetAccessGroupsRequest{PublicKeyBase58Check: senderPkString}

routes/admin_node.go

@@ -109,13 +109,13 @@ func (fes *APIServer) _handleNodeControlGetInfo(
 	{
 		desoNodeStatus.LatestHeaderHeight = desoHeaderTip.Height
 		desoNodeStatus.LatestHeaderHash = hex.EncodeToString(desoHeaderTip.Hash[:])
-		desoNodeStatus.LatestHeaderTstampSecs = uint32(desoHeaderTip.Header.TstampSecs)
+		desoNodeStatus.LatestHeaderTstampSecs = uint32(desoHeaderTip.Header.GetTstampSecs())
 	}
 	// Main block chain fields
 	{
 		desoNodeStatus.LatestBlockHeight = desoBlockTip.Height
 		desoNodeStatus.LatestBlockHash = hex.EncodeToString(desoBlockTip.Hash[:])
-		desoNodeStatus.LatestBlockTstampSecs = uint32(desoBlockTip.Header.TstampSecs)
+		desoNodeStatus.LatestBlockTstampSecs = uint32(desoBlockTip.Header.GetTstampSecs())
 	}
 	if fes.TXIndex != nil {
 		// TxIndex status

routes/admin_transaction.go

@@ -44,6 +44,10 @@ type GetGlobalParamsResponse struct {
 	// are included when generating a new Proof-of-Stake leader schedule.
 	LeaderScheduleMaxNumValidators uint64 `safeForLogging:"true"`
 
+	// ValidatorSetMaxNumValidators is the maximum number of validators that
+	// are included when generating a new Proof-of-Stake validator set.
+	ValidatorSetMaxNumValidators uint64 `safeForLogging:"true"`
+
 	// EpochDurationNumBlocks is the number of blocks included in one epoch.
 	EpochDurationNumBlocks uint64 `safeForLogging:"true"`
 
@@ -78,6 +82,7 @@ func (fes *APIServer) GetGlobalParams(ww http.ResponseWriter, req *http.Request)
 		StakeLockupEpochDuration:               globalParamsEntry.StakeLockupEpochDuration,
 		ValidatorJailEpochDuration:             globalParamsEntry.ValidatorJailEpochDuration,
 		LeaderScheduleMaxNumValidators:         globalParamsEntry.LeaderScheduleMaxNumValidators,
+		ValidatorSetMaxNumValidators:           globalParamsEntry.ValidatorSetMaxNumValidators,
 		EpochDurationNumBlocks:                 globalParamsEntry.EpochDurationNumBlocks,
 		JailInactiveValidatorGracePeriodEpochs: globalParamsEntry.JailInactiveValidatorGracePeriodEpochs,
 	}
@@ -121,6 +126,10 @@ type UpdateGlobalParamsRequest struct {
 	// are included when generating a new Proof-of-Stake leader schedule.
 	LeaderScheduleMaxNumValidators uint64 `safeForLogging:"true"`
 
+	// ValidatorSetMaxNumValidators is the maximum number of validators that
+	// are included when generating a new Proof-of-Stake validator set.
+	ValidatorSetMaxNumValidators uint64 `safeForLogging:"true"`
+
 	// EpochDurationNumBlocks is the number of blocks included in one epoch.
 	EpochDurationNumBlocks uint64 `safeForLogging:"true"`
 
@@ -237,6 +246,11 @@ func (fes *APIServer) UpdateGlobalParams(ww http.ResponseWriter, req *http.Reque
 		extraData[lib.LeaderScheduleMaxNumValidatorsKey] = lib.UintToBuf(requestData.LeaderScheduleMaxNumValidators)
 	}
 
+	if requestData.ValidatorSetMaxNumValidators > 0 &&
+		requestData.ValidatorSetMaxNumValidators != globalParamsEntry.ValidatorSetMaxNumValidators {
+		extraData[lib.ValidatorSetMaxNumValidatorsKey] = lib.UintToBuf(requestData.ValidatorSetMaxNumValidators)
+	}
+
 	if requestData.EpochDurationNumBlocks > 0 &&
 		requestData.EpochDurationNumBlocks != globalParamsEntry.EpochDurationNumBlocks {
 		extraData[lib.EpochDurationNumBlocksKey] = lib.UintToBuf(requestData.EpochDurationNumBlocks)

routes/admin_transaction_test.go

@@ -100,6 +100,7 @@ func TestUpdateGlobalParams(t *testing.T) {
 			StakeLockupEpochDuration:               4,
 			ValidatorJailEpochDuration:             4,
 			LeaderScheduleMaxNumValidators:         101,
+			ValidatorSetMaxNumValidators:           102,
 			EpochDurationNumBlocks:                 3601,
 			JailInactiveValidatorGracePeriodEpochs: 49,
 			MinFeeRateNanosPerKB:                   99,
@@ -112,6 +113,7 @@ func TestUpdateGlobalParams(t *testing.T) {
 		require.Equal(t, globalParams.StakeLockupEpochDuration, uint64(4))
 		require.Equal(t, globalParams.ValidatorJailEpochDuration, uint64(4))
 		require.Equal(t, globalParams.LeaderScheduleMaxNumValidators, uint64(101))
+		require.Equal(t, globalParams.ValidatorSetMaxNumValidators, uint64(102))
 		require.Equal(t, globalParams.EpochDurationNumBlocks, uint64(3601))
 		require.Equal(t, globalParams.JailInactiveValidatorGracePeriodEpochs, uint64(49))
 	}
@@ -131,6 +133,7 @@ func TestUpdateGlobalParams(t *testing.T) {
 		require.Equal(t, globalParams.StakeLockupEpochDuration, uint64(4))
 		require.Equal(t, globalParams.ValidatorJailEpochDuration, uint64(4))
 		require.Equal(t, globalParams.LeaderScheduleMaxNumValidators, uint64(101))
+		require.Equal(t, globalParams.ValidatorSetMaxNumValidators, uint64(102))
 		require.Equal(t, globalParams.EpochDurationNumBlocks, uint64(3601))
 		require.Equal(t, globalParams.JailInactiveValidatorGracePeriodEpochs, uint64(50))
 	}

routes/exchange.go

@@ -132,7 +132,7 @@ func _headerToResponse(header *lib.MsgDeSoHeader, hash string) *HeaderResponse {
 		Version:                  header.Version,
 		PrevBlockHashHex:         header.PrevBlockHash.String(),
 		TransactionMerkleRootHex: header.TransactionMerkleRoot.String(),
-		TstampSecs:               header.TstampSecs,
+		TstampSecs:               header.GetTstampSecs(),
 		Height:                   header.Height,
 		Nonce:                    header.Nonce,
 		ExtraNonce:               header.ExtraNonce,
@@ -666,7 +666,7 @@ func APITransactionToResponse(
 	if block != nil && block.Header != nil {
 		ret.BlockInfo = &TransactionBlockInfo{
 			Height:        block.Header.Height,
-			TimestampSecs: block.Header.TstampSecs,
+			TimestampSecs: block.Header.GetTstampSecs(),
 		}
 	}
 

routes/exchange_test.go

@@ -4,8 +4,9 @@ import (
 	"bytes"
 	"encoding/hex"
 	"encoding/json"
-	chainlib "github.com/btcsuite/btcd/blockchain"
+	"github.com/deso-protocol/backend/config"
 	coreCmd "github.com/deso-protocol/core/cmd"
+	"github.com/deso-protocol/core/lib"
 	"github.com/google/uuid"
 	"io"
 	"io/ioutil"
@@ -14,10 +15,6 @@ import (
 	"net/http/httptest"
 	"os"
 	"testing"
-	"time"
-
-	"github.com/deso-protocol/backend/config"
-	"github.com/deso-protocol/core/lib"
 
 	"github.com/dgraph-io/badger/v3"
 
@@ -78,106 +75,7 @@ func GetTestBadgerDb(t *testing.T) (_db *badger.DB, _dir string) {
 	return db, dir
 }
 
-func NewLowDifficultyBlockchain(t *testing.T) (*lib.Blockchain, *lib.DeSoParams, *badger.DB, string) {
-
-	// Set the number of txns per view regeneration to one while creating the txns
-	lib.ReadOnlyUtxoViewRegenerationIntervalTxns = 1
-
-	return NewLowDifficultyBlockchainWithParams(t, &lib.DeSoTestnetParams)
-}
-
-func NewLowDifficultyBlockchainWithParams(t *testing.T, params *lib.DeSoParams) (
-	*lib.Blockchain, *lib.DeSoParams, *badger.DB, string) {
-
-	// Set the number of txns per view regeneration to one while creating the txns
-	lib.ReadOnlyUtxoViewRegenerationIntervalTxns = 1
-
-	db, dir := GetTestBadgerDb(t)
-	timesource := chainlib.NewMedianTime()
-
-	// Set some special parameters for testing. If the blocks above are changed
-	// these values should be updated to reflect the latest testnet values.
-	paramsCopy := *params
-	paramsCopy.GenesisBlock = &lib.MsgDeSoBlock{
-		Header: &lib.MsgDeSoHeader{
-			Version:               0,
-			PrevBlockHash:         lib.MustDecodeHexBlockHash("0000000000000000000000000000000000000000000000000000000000000000"),
-			TransactionMerkleRoot: lib.MustDecodeHexBlockHash("097158f0d27e6d10565c4dc696c784652c3380e0ff8382d3599a4d18b782e965"),
-			TstampSecs:            uint64(1560735050),
-			Height:                uint64(0),
-			Nonce:                 uint64(0),
-			// No ExtraNonce is set in the genesis block
-		},
-		Txns: []*lib.MsgDeSoTxn{
-			{
-				TxInputs:  []*lib.DeSoInput{},
-				TxOutputs: []*lib.DeSoOutput{},
-				TxnMeta: &lib.BlockRewardMetadataa{
-					ExtraData: []byte("They came here, to the new world. World 2.0, version 1776."),
-				},
-				// A signature is not required for BLOCK_REWARD transactions since they
-				// don't spend anything.
-			},
-		},
-	}
-	paramsCopy.MinDifficultyTargetHex = "999999948931e5874cf66a74c0fda790dd8c7458243d400324511a4c71f54faa"
-	paramsCopy.MinChainWorkHex = "0000000000000000000000000000000000000000000000000000000000000000"
-	paramsCopy.MiningIterationsPerCycle = 500
-	// Set maturity to 2 blocks so we can test spending on short chains. The
-	// tests rely on the maturity equaling exactly two blocks (i.e. being
-	// two times the time between blocks).
-	paramsCopy.TimeBetweenBlocks = 2 * time.Second
-	paramsCopy.BlockRewardMaturity = time.Second * 4
-	paramsCopy.TimeBetweenDifficultyRetargets = 100 * time.Second
-	paramsCopy.MaxDifficultyRetargetFactor = 2
-	paramsCopy.SeedBalances = []*lib.DeSoOutput{
-		{
-			PublicKey:   lib.MustBase58CheckDecode(moneyPkString),
-			AmountNanos: uint64(2000000 * lib.NanosPerUnit),
-		},
-	}
-
-	// Temporarily modify the seed balances to make a specific public
-	// key have some DeSo
-	chain, err := lib.NewBlockchain([]string{blockSignerPk}, 0, 0,
-		&paramsCopy, timesource, db, nil, nil, nil, false)
-	if err != nil {
-		log.Fatal(err)
-	}
-
-	return chain, &paramsCopy, db, dir
-}
-
-func NewTestMiner(t *testing.T, chain *lib.Blockchain, params *lib.DeSoParams, isSender bool) (*lib.DeSoMempool, *lib.DeSoMiner) {
-	assert := assert.New(t)
-	require := require.New(t)
-	_ = assert
-	_ = require
-
-	mempool := lib.NewDeSoMempool(
-		chain, 0, /* rateLimitFeeRateNanosPerKB */
-		0 /* minFeeRateNanosPerKB */, "", true,
-		"" /*dataDir*/, "")
-	minerPubKeys := []string{}
-	if isSender {
-		minerPubKeys = append(minerPubKeys, senderPkString)
-	} else {
-		minerPubKeys = append(minerPubKeys, recipientPkString)
-	}
-
-	blockProducer, err := lib.NewDeSoBlockProducer(
-		0, 1,
-		blockSignerSeed,
-		mempool, chain,
-		params, nil)
-	require.NoError(err)
-
-	newMiner, err := lib.NewDeSoMiner(minerPubKeys, 1 /*numThreads*/, blockProducer, params)
-	require.NoError(err)
-	return mempool, newMiner
-}
-
-func newTestAPIServer(t *testing.T, globalStateRemoteNode string) (*APIServer, *APIServer, *lib.DeSoMiner) {
+func newTestAPIServer(t *testing.T, globalStateRemoteNode string, txindex bool) (*APIServer, *APIServer, *lib.DeSoMiner) {
 	assert := assert.New(t)
 	require := require.New(t)
 	_, _ = assert, require
@@ -202,7 +100,7 @@ func newTestAPIServer(t *testing.T, globalStateRemoteNode string) (*APIServer, *
 	coreConfig := coreCmd.LoadConfig()
 	coreConfig.Params = &lib.DeSoTestnetParams
 	coreConfig.DataDirectory = badgerDir
-	coreConfig.TXIndex = true
+	coreConfig.TXIndex = txindex
 	coreConfig.MinerPublicKeys = []string{senderPkString}
 	coreConfig.NumMiningThreads = 1
 	coreConfig.HyperSync = false
@@ -257,7 +155,7 @@ func TestAPI(t *testing.T) {
 	require := require.New(t)
 	_, _ = assert, require
 
-	apiServer, _, miner := newTestAPIServer(t, "" /*globalStateRemoteNode*/)
+	apiServer, _, miner := newTestAPIServer(t, "" /*globalStateRemoteNode*/, false)
 
 	{
 		request, _ := http.NewRequest("GET", RoutePathAPIBase, nil)

routes/global_state_test.go

@@ -18,7 +18,7 @@ func TestGlobalStateServicePutGetDeleteWithDB(t *testing.T) {
 	_, _ = assert, require
 
 	apiServer, _, _ := newTestAPIServer(
-		t, "" /*globalStateRemoteNode*/)
+		t, "" /*globalStateRemoteNode*/, false)
 
 	// Getting when no value is present should return nil without an
 	// error.
@@ -62,7 +62,7 @@ func TestGlobalStateServicePutGetDeleteWithRemoteNode(t *testing.T) {
 	_, _ = assert, require
 
 	apiServer, _, _ := newTestAPIServer(
-		t, "" /*globalStateRemoteNode*/)
+		t, "" /*globalStateRemoteNode*/, false)
 
 	// Getting when no value is present should return nil without an
 	// error.
@@ -189,7 +189,7 @@ func TestGlobalStateServiceURLCreation(t *testing.T) {
 	_, _ = assert, require
 
 	apiServer, _, _ := newTestAPIServer(
-		t, "https://deso.com:17001" /*globalStateRemoteNode*/)
+		t, "https://deso.com:17001" /*globalStateRemoteNode*/, false)
 
 	{
 		url, _, err := apiServer.GlobalState.CreateGetRequest([]byte("woo"))

routes/miner.go

@@ -187,7 +187,7 @@ func (fes *APIServer) SubmitBlock(ww http.ResponseWriter, req *http.Request) {
 	// TODO: Signature checking slows things down because it acquires the ChainLock.
 	// The optimal solution is to check signatures in a way that doesn't acquire the
 	// ChainLock, which is what Bitcoin Core does.
-	isMainChain, isOrphan, err := fes.blockchain.ProcessBlock(
+	isMainChain, isOrphan, _, err := fes.blockchain.ProcessBlock(
 		blockFound, true /*verifySignatures*/)
 	glog.V(1).Infof("Called ProcessBlock: isMainChain=(%v), isOrphan=(%v), err=(%v)",
 		isMainChain, isOrphan, err)

routes/transaction.go

@@ -3953,6 +3953,7 @@ func (fes *APIServer) simulateSubmitTransaction(utxoView *lib.UtxoView, txn *lib
 		txn.Hash(),
 		0,
 		bestHeight,
+		0,
 		false,
 		false,
 	)