Support broken database cluster on startup

[bbrodriges]

Error pops up while running newly added test with -race flag

$ go test -v -race -run=^TestCluster_BrokenAtStartup$ ./...
You use x86_64 version of selected tool.
=== RUN   TestCluster_BrokenAtStartup
=== RUN   TestCluster_BrokenAtStartup/broken_primary
==================
WARNING: DATA RACE
Read at 0x00c0001901d8 by goroutine 21:
  github.com/DATA-DOG/go-sqlmock.(*rowSets).Next()
      /Users/gzuykov/go/pkg/mod/github.com/!d!a!t!a-!d!o!g/go-sqlmock@v1.5.0/rows.go:45 +0xa8
  database/sql.(*Rows).nextLocked()
      /Users/gzuykov/.ya/tools/v4/1217047391/src/database/sql/sql.go:2967 +0x1d0
  database/sql.(*Rows).Next.func1()
      /Users/gzuykov/.ya/tools/v4/1217047391/src/database/sql/sql.go:2945 +0x48
  database/sql.withLock()
      /Users/gzuykov/.ya/tools/v4/1217047391/src/database/sql/sql.go:3396 +0x74
  database/sql.(*Rows).Next()
      /Users/gzuykov/.ya/tools/v4/1217047391/src/database/sql/sql.go:2944 +0x7c
  database/sql.(*Row).Scan()
      /Users/gzuykov/.ya/tools/v4/1217047391/src/database/sql/sql.go:3333 +0x108
  golang.yandex/hasql/checkers.Check()
      /Users/gzuykov/dev/github.com/yandex/go-hasql/checkers/check.go:29 +0xd8
  golang.yandex/hasql/checkers.PostgreSQL()
      /Users/gzuykov/dev/github.com/yandex/go-hasql/checkers/postgresql.go:26 +0x54
  golang.yandex/hasql.checkExecutor.func1()
      /Users/gzuykov/dev/github.com/yandex/go-hasql/check_nodes.go:149 +0x80
  golang.yandex/hasql.checkNodes.func1()
      /Users/gzuykov/dev/github.com/yandex/go-hasql/check_nodes.go:109 +0x88

Previous write at 0x00c0001901d8 by goroutine 20:
  github.com/DATA-DOG/go-sqlmock.(*rowSets).Next()
      /Users/gzuykov/go/pkg/mod/github.com/!d!a!t!a-!d!o!g/go-sqlmock@v1.5.0/rows.go:45 +0xc0
  database/sql.(*Rows).nextLocked()
      /Users/gzuykov/.ya/tools/v4/1217047391/src/database/sql/sql.go:2967 +0x1d0
  database/sql.(*Rows).Next.func1()
      /Users/gzuykov/.ya/tools/v4/1217047391/src/database/sql/sql.go:2945 +0x48
  database/sql.withLock()
      /Users/gzuykov/.ya/tools/v4/1217047391/src/database/sql/sql.go:3396 +0x74
  database/sql.(*Rows).Next()
      /Users/gzuykov/.ya/tools/v4/1217047391/src/database/sql/sql.go:2944 +0x7c
  database/sql.(*Row).Scan()
      /Users/gzuykov/.ya/tools/v4/1217047391/src/database/sql/sql.go:3333 +0x108
  golang.yandex/hasql/checkers.Check()
      /Users/gzuykov/dev/github.com/yandex/go-hasql/checkers/check.go:29 +0xd8
  golang.yandex/hasql/checkers.PostgreSQL()
      /Users/gzuykov/dev/github.com/yandex/go-hasql/checkers/postgresql.go:26 +0x54
  golang.yandex/hasql.checkExecutor.func1()
      /Users/gzuykov/dev/github.com/yandex/go-hasql/check_nodes.go:149 +0x80
  golang.yandex/hasql.checkNodes.func1()
      /Users/gzuykov/dev/github.com/yandex/go-hasql/check_nodes.go:109 +0x88

Goroutine 21 (running) created at:
  golang.yandex/hasql.checkNodes()
      /Users/gzuykov/dev/github.com/yandex/go-hasql/check_nodes.go:106 +0x450
  golang.yandex/hasql.(*Cluster).updateNodes()
      /Users/gzuykov/dev/github.com/yandex/go-hasql/cluster.go:321 +0x1b4
  golang.yandex/hasql.(*Cluster).backgroundNodesUpdate()
      /Users/gzuykov/dev/github.com/yandex/go-hasql/cluster.go:297 +0x30

diff --git a/cluster_test.go b/cluster_test.go
index 10c22eb..c816f2f 100644
--- a/cluster_test.go
+++ b/cluster_test.go
@@ -29,6 +29,8 @@ import (
        "github.com/gofrs/uuid"
        "github.com/stretchr/testify/assert"
        "github.com/stretchr/testify/require"
+
+       "golang.yandex/hasql/checkers"
 )

 func TestNewCluster(t *testing.T) {
@@ -543,6 +545,89 @@ func TestCluster_WaitForStandbyPreferred(t *testing.T) {
        }
 }

+// TestCluster_BrokenAtStartup checks if broken at startup cluster behaves properly
+func TestCluster_BrokenAtStartup(t *testing.T) {
+       isPrimaryRow := sqlmock.NewRows([]string{`NOT pg_is_in_recovery()`}).AddRow(true)
+       isStandbyRow := sqlmock.NewRows([]string{`NOT pg_is_in_recovery()`}).AddRow(false)
+
+       testCases := []struct {
+               name  string
+               nodes []Node
+       }{
+               {
+                       name: "broken_primary",
+                       nodes: func() []Node {
+                               db1, mock1, _ := sqlmock.New() // primary
+                               db2, mock2, _ := sqlmock.New() // standby
+                               db3, mock3, _ := sqlmock.New() // standby
+
+                               mock1.ExpectQuery(`SELECT NOT pg_is_in_recovery()`).WillDelayFor(20 * time.Millisecond)
+                               mock2.ExpectQuery(`SELECT NOT pg_is_in_recovery()`).WillReturnRows(isStandbyRow)
+                               mock3.ExpectQuery(`SELECT NOT pg_is_in_recovery()`).WillReturnRows(isStandbyRow)
+
+                               return []Node{
+                                       NewNode("primary", db1),
+                                       NewNode("standby1", db2),
+                                       NewNode("standby2", db3),
+                               }
+                       }(),
+               },
+               {
Goroutine 20 (finished) created at:
  golang.yandex/hasql.checkNodes()
      /Users/gzuykov/dev/github.com/yandex/go-hasql/check_nodes.go:106 +0x450
  golang.yandex/hasql.(*Cluster).updateNodes()
      /Users/gzuykov/dev/github.com/yandex/go-hasql/cluster.go:321 +0x1b4
  golang.yandex/hasql.(*Cluster).backgroundNodesUpdate()
      /Users/gzuykov/dev/github.com/yandex/go-hasql/cluster.go:297 +0x30
==================
    testing.go:1152: race detected during execution of test
=== RUN   TestCluster_BrokenAtStartup/broken_standby
=== RUN   TestCluster_BrokenAtStartup/broken_standbys
    cluster_test.go:624:
        	Error Trace:	cluster_test.go:624
        	Error:      	Received unexpected error:
        	            	context deadline exceeded
        	Test:       	TestCluster_BrokenAtStartup/broken_standbys
    cluster_test.go:625:
        	Error Trace:	cluster_test.go:625
        	Error:      	Expected value not to be nil.
        	Test:       	TestCluster_BrokenAtStartup/broken_standbys
    cluster_test.go:626:
        	Error Trace:	cluster_test.go:626
        	Error:      	Should NOT be empty, but was []
        	Test:       	TestCluster_BrokenAtStartup/broken_standbys
=== CONT  TestCluster_BrokenAtStartup
    testing.go:1152: race detected during execution of test
--- FAIL: TestCluster_BrokenAtStartup (0.01s)
    --- FAIL: TestCluster_BrokenAtStartup/broken_primary (0.00s)
    --- PASS: TestCluster_BrokenAtStartup/broken_standby (0.00s)
    --- FAIL: TestCluster_BrokenAtStartup/broken_standbys (0.01s)
=== CONT
    testing.go:1152: race detected during execution of test
FAIL
FAIL	golang.yandex/hasql	0.286s
?   	golang.yandex/hasql/checkers	[no test files]
testing: warning: no tests to run
PASS
ok  	golang.yandex/hasql/sqlx	(cached) [no tests to run]
FAIL

[sidh]

Race is triggered because sqlmock.NewRows is not thread-safe. You must create one for each WillReturnRows.

I will look into the issue further.