Merge pull request #1262 from tigrisdata/main

Beta release

.github/workflows/push-docker-debug.yaml

@@ -1,18 +1,33 @@
-name: docker image
+name: publish docker image
 
 on:
   pull_request:
+    paths:
+      - docker/Dockerfile*
+      - scripts/install_*.sh
+      - .github/workflows/push-docker-*.yaml
     types:
       - labeled
+  merge_group:
+    paths:
+      - docker/Dockerfile*
+      - scripts/install_*.sh
+      - .github/workflows/push-docker-*.yaml
+  push:
+    branches:
+      - main
+  release:
+    types: [published]
 
 jobs:
-  debug:
+  debug-image:
     if: ${{ github.event.label.name == 'debug image' }}
     uses: ./.github/workflows/push-docker-image.yaml
     secrets: inherit
     with:
-      docker_repository: "tigrisdata/tigris-debug"
-      quay_repository: "quay.io/tigrisdata/tigris-debug"
+      docker_repository: tigrisdata/tigris-debug
+      quay_repository: quay.io/tigrisdata/tigris-debug
       file: docker/Dockerfile
-      platforms: amd64
-      event_name: push
+      platforms: amd64,arm64
+      event_name: ${{ github.event_name }}
+      build_args: WITH_DEBUG=1

.github/workflows/push-docker-image.yaml

@@ -13,6 +13,8 @@ on:
         type: string
       event_name:
         type: string
+      build_args:
+        type: string
 
 jobs:
   build-and-push:
@@ -78,3 +80,4 @@ jobs:
           push: ${{ inputs.event_name != 'pull_request' }}
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
+          build-args: ${{ inputs.build_args }}

Makefile

@@ -9,11 +9,22 @@ PROTO_DIR=${API_DIR}/proto/server/${V}
 DATA_PROTO_DIR=internal
 LINT_TIMEOUT=5m
 
+# Generic build params
+BUILD_PARAM=-tags=release -ldflags "-X 'github.com/tigrisdata/tigris/util.Version=$(VERSION)' -X 'github.com/tigrisdata/tigris/util.BuildHash=$(GIT_HASH)'" $(shell printenv BUILD_PARAM)
+
 # Needed to be able to build amd64 binaries on MacOS M1
 DOCKER_DIR=test/docker
 DOCKER_COMPOSE=COMPOSE_DOCKER_CLI_BUILD=1 DOCKER_BUILDKIT=1 docker compose -f ${DOCKER_DIR}/docker-compose.yml
 CGO_ENABLED=1
 
+# Enables debugging when compiling binaries
+# Translates _shell_ variable WITH_DEBUG to _Makefile_ variable WITH_DEBUG
+WITH_DEBUG=$(shell printenv WITH_DEBUG) 
+DEBUG_PARAM=-gcflags "all=-N -l"
+
+# Build parameters for testing
+TEST_PARAM=-cover -race -tags=test,integration,tigris_http,tigris_grpc $(shell printenv TEST_PARAM)
+
 # Mocks required for unit tests
 MOCK_INTERFACES=\
 "server/services/v1/billing:Provider" \
@@ -84,13 +95,9 @@ deps:
 	/bin/bash scripts/install_build_deps.sh
 	/bin/bash scripts/install_test_deps.sh
 
-
 # The following targets are for API code generation.
 .PRECIOUS: ${PROTO_DIR}/%_openapi.yaml ${PROTO_DIR}/%.proto
 
-BUILD_PARAM=-tags=release -ldflags "-X 'github.com/tigrisdata/tigris/util.Version=$(VERSION)' -X 'github.com/tigrisdata/tigris/util.BuildHash=$(GIT_HASH)'" $(shell printenv BUILD_PARAM)
-TEST_PARAM=-cover -race -tags=test,integration,tigris_http,tigris_grpc $(shell printenv TEST_PARAM)
-
 ${PROTO_DIR}/%.proto:
 	git submodule update --init --recursive
 
@@ -128,8 +135,13 @@ clean_mocks:
 # Build the server binary.
 server: server/service
 server/service: $(GO_SRC) generate
+ifeq ($(strip $(WITH_DEBUG)),)
 	CGO_ENABLED=$(CGO_ENABLED) go build $(BUILD_PARAM) -o server/service ./server
+else
+	CGO_ENABLED=$(CGO_ENABLED) go build $(BUILD_PARAM) $(DEBUG_PARAM) -o server/service ./server
+endif
 
+# Builds the "admin" command
 admin: cmd/admin/admin
 cmd/admin/admin: cmd/admin/*.go cmd/admin/cmd/*.go
 	CGO_ENABLED=$(CGO_ENABLED) go build $(BUILD_PARAM) -o cmd/admin/admin ./cmd/admin

docker/Dockerfile

@@ -22,7 +22,11 @@ COPY . /build
 # This is used to compile with integration coverage enabled
 ARG BUILD_PARAM_ARG=""
 
-RUN --mount=type=cache,target=/root/.cache/go-build rm -f server/service && BUILD_PARAM=${BUILD_PARAM_ARG} make bins
+# This sets the Makefile target that is called with go build. Defaults to bins -> server.
+ARG BUILD_TARGET="bins"
+
+RUN --mount=type=cache,target=/root/.cache/go-build rm -f server/service && \
+    BUILD_PARAM=${BUILD_PARAM_ARG} make ${BUILD_TARGET}
 
 
 FROM ubuntu:20.04 AS server

docker/Dockerfile.base

@@ -32,6 +32,9 @@ RUN mkdir -p /build /go/src/tigris
 COPY scripts/install_go.sh /build
 RUN sh /build/install_go.sh
 
+COPY scripts/install_delve.sh /build
+RUN sh /build/install_delve.sh
+
 #Download deps once, during docker build. Rebuild only on go.mod change
 ENV PATH="${PATH}:/root/go/bin:/usr/local/go/bin"
 COPY scripts/install_build_deps.sh /build

scripts/install_delve.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+# Copyright 2022-2023 Tigris Data, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+# This intended to be run duing Docker build only
+
+set -e
+
+DELVE_SRC=github.com/go-delve/delve
+DELVE_PKG=${DELVE_SRC}/cmd/dlv@latest
+
+export PATH=$PATH:/usr/local/go/bin
+
+go install ${DELVE_PKG}

server/config/options.go

@@ -290,7 +290,7 @@ type Billing struct {
 }
 
 type Metronome struct {
-	Enabled       bool          `json:"enabled"        mapstructure:"enabled"        yaml:"enabled"`
+	Disabled      bool          `json:"disabled"       mapstructure:"disabled"       yaml:"disabled"`
 	URL           string        `json:"url"            mapstructure:"url"            yaml:"url"`
 	ApiKey        string        `json:"api_key"        mapstructure:"api_key"        yaml:"api_key"`
 	DefaultPlan   string        `json:"default_plan"   mapstructure:"default_plan"   yaml:"default_plan"`
@@ -300,7 +300,7 @@ type Metronome struct {
 type BilledMetrics = map[string]string
 
 type BillingReporter struct {
-	Enabled         bool          `json:"enabled"          mapstructure:"enabled"          yaml:"enabled"`
+	Disabled        bool          `json:"disabled"         mapstructure:"disabled"         yaml:"disabled"`
 	RefreshInterval time.Duration `json:"refresh_interval" mapstructure:"refresh_interval" yaml:"refresh_interval"`
 }
 
@@ -354,14 +354,12 @@ var DefaultConfig = Config{
 	},
 	Billing: Billing{
 		Metronome: Metronome{
-			Enabled: false,
-			URL:     "https://api.metronome.com/v1",
-			ApiKey:  "replace_me",
+			URL:    "https://api.metronome.com/v1",
+			ApiKey: "replace_me",
 			// random placeholder UUID and not an actual plan
 			DefaultPlan: "47eda90f-d2e8-4184-8955-cb3a6467782b",
 		},
 		Reporter: BillingReporter{
-			Enabled:         false,
 			RefreshInterval: time.Second * 60, // 60 seconds
 		},
 	},
@@ -620,6 +618,8 @@ type SearchConfig struct {
 	Compression       bool `json:"compression"         mapstructure:"compression"         yaml:"compression"`
 	IgnoreExtraFields bool `json:"ignore_extra_fields" mapstructure:"ignore_extra_fields" yaml:"ignore_extra_fields"`
 	LogFilter         bool `json:"log_filter"          mapstructure:"log_filter"          yaml:"log_filter"`
+
+	DoNotReloadMetadata bool `json:"do_not_reload_metadata" mapstructure:"do_not_reload_metadata" yaml:"do_not_reload_metadata"`
 }
 
 type SecondaryIndexConfig struct {

server/metadata/tenant.go

@@ -1228,18 +1228,17 @@ func (m *TenantManager) DeleteTenant(ctx context.Context, tx transaction.Tx, ten
 func (tenant *Tenant) CreateBranch(ctx context.Context, tx transaction.Tx, projName string, dbName *DatabaseName) error {
 	tenant.Lock()
 	defer tenant.Unlock()
-
-	dbMeta, err := tenant.MetaStore.Database().Get(ctx, tx, tenant.namespace.Id(), projName)
-	if err != nil {
-		return err
-	}
-
 	// first get the project
 	proj, ok := tenant.projects[projName]
 	if !ok {
 		return NewProjectNotFoundErr(projName)
 	}
 
+	dbMeta, err := tenant.MetaStore.Database().Get(ctx, tx, tenant.namespace.Id(), projName)
+	if err != nil {
+		return err
+	}
+
 	if _, ok = proj.databaseBranches[dbName.Name()]; ok {
 		return NewDatabaseBranchExistsErr(dbName.Branch())
 	}
@@ -1272,15 +1271,15 @@ func (tenant *Tenant) DeleteBranch(ctx context.Context, tx transaction.Tx, projN
 	tenant.Lock()
 	defer tenant.Unlock()
 
-	if dbBranch.IsMainBranch() {
-		return NewMetadataError(ErrCodeCannotDeleteBranch, "'main' database cannot be deleted.")
-	}
-
 	proj, found := tenant.projects[projName]
 	if !found {
 		return NewProjectNotFoundErr(projName)
 	}
 
+	if dbBranch.IsMainBranch() {
+		return NewMetadataError(ErrCodeCannotDeleteBranch, "'main' database cannot be deleted.")
+	}
+
 	return tenant.deleteBranch(ctx, tx, proj, dbBranch)
 }
 

server/metadata/tenant_test.go

@@ -267,6 +267,7 @@ func TestTenantManager_DatabaseBranches(t *testing.T) {
 	require.NoError(t, err)
 	err = tenant.CreateProject(ctx, tx, tenantProj2, nil)
 	require.NoError(t, err)
+	unknownProject := "project_not_exists"
 
 	databases := (&Project{}).GetDatabaseWithBranches()
 	require.Len(t, databases, 0)
@@ -278,6 +279,7 @@ func TestTenantManager_DatabaseBranches(t *testing.T) {
 	require.NoError(t, tenant.CreateBranch(ctx, tx, tenantProj1, NewDatabaseNameWithBranch(tenantProj1, "branch2")))
 	require.NoError(t, tenant.CreateBranch(ctx, tx, tenantProj2, NewDatabaseNameWithBranch(tenantProj2, "branch2")))
 	require.NoError(t, tenant.CreateBranch(ctx, tx, tenantProj1, NewDatabaseNameWithBranch(tenantProj1, "branch3")))
+	require.ErrorContains(t, tenant.CreateBranch(ctx, tx, unknownProject, NewDatabaseNameWithBranch(unknownProject, "branch1")), "project doesn't exist")
 
 	// reload again to get all the branches
 	require.NoError(t, tenant.reload(ctx, tx, nil, nil))
@@ -335,6 +337,7 @@ func TestTenantManager_DatabaseBranches(t *testing.T) {
 	require.NoError(t, tenant.DeleteBranch(ctx, tx, tenantProj1, NewDatabaseNameWithBranch(tenantProj1, "branch1")))
 	require.NoError(t, tenant.DeleteBranch(ctx, tx, tenantProj2, NewDatabaseNameWithBranch(tenantProj2, "branch1")))
 	require.NoError(t, tenant.DeleteBranch(ctx, tx, tenantProj1, NewDatabaseNameWithBranch(tenantProj1, "branch2")))
+	require.ErrorContains(t, tenant.DeleteBranch(ctx, tx, unknownProject, NewDatabaseNameWithBranch(unknownProject, "branch1")), "project doesn't exist")
 
 	require.NoError(t, tenant.reload(ctx, tx, nil, nil))
 	require.NoError(t, tx.Commit(ctx))

server/services/v1/billing/provider.go

@@ -64,7 +64,7 @@ func (w WindowSize) String() string {
 }
 
 func NewProvider() Provider {
-	if config.DefaultConfig.Billing.Metronome.Enabled {
+	if !config.DefaultConfig.Billing.Metronome.Disabled {
 		svc, err := NewMetronomeProvider(config.DefaultConfig.Billing.Metronome)
 		if !ulog.E(err) {
 			return svc

server/services/v1/billing/provider_test.go

@@ -22,24 +22,27 @@ import (
 )
 
 func TestNewProvider(t *testing.T) {
-	t.Run("billing is disabled", func(t *testing.T) {
+	t.Run("billing is enabled by default", func(t *testing.T) {
+		require.False(t, config.DefaultConfig.Billing.Metronome.Disabled)
 		provider := NewProvider()
 
 		_, ok := provider.(*noop)
-		require.True(t, ok)
+		require.False(t, ok)
 
 		_, ok = provider.(*Metronome)
-		require.False(t, ok)
+		require.True(t, ok)
 	})
 
-	t.Run("billing is enabled", func(t *testing.T) {
-		config.DefaultConfig.Billing.Metronome.Enabled = true
+	t.Run("billing is disabled", func(t *testing.T) {
+		config.DefaultConfig.Billing.Metronome.Disabled = true
 		provider := NewProvider()
 
 		_, ok := provider.(*noop)
-		require.False(t, ok)
+		require.True(t, ok)
 
 		_, ok = provider.(*Metronome)
-		require.True(t, ok)
+		require.False(t, ok)
+		// reset
+		config.DefaultConfig.Billing.Metronome.Disabled = false
 	})
 }

server/services/v1/billing/reporter.go

@@ -62,7 +62,7 @@ func NewUsageReporter(
 }
 
 func (r *UsageReporter) Start() {
-	if config.DefaultConfig.Billing.Reporter.Enabled {
+	if !config.DefaultConfig.Billing.Reporter.Disabled {
 		go r.refreshLoop()
 	}
 }

server/services/v1/search.go

@@ -22,6 +22,7 @@ import (
 	"github.com/go-chi/chi/v5"
 	"github.com/grpc-ecosystem/grpc-gateway/v2/runtime"
 	api "github.com/tigrisdata/tigris/api/server/v1"
+	"github.com/tigrisdata/tigris/server/config"
 	"github.com/tigrisdata/tigris/server/metadata"
 	"github.com/tigrisdata/tigris/server/request"
 	"github.com/tigrisdata/tigris/server/services/v1/search"
@@ -120,7 +121,8 @@ func (s *searchService) DeleteIndex(ctx context.Context, req *api.DeleteIndexReq
 	runner.SetDeleteIndexReq(req)
 
 	resp, err := s.sessions.TxExecute(ctx, runner, search.SessionOptions{
-		IncVersion: true,
+		IncVersion:          !config.DefaultConfig.Search.DoNotReloadMetadata,
+		DoNotReloadMetadata: config.DefaultConfig.Search.DoNotReloadMetadata,
 	})
 	if err != nil {
 		return nil, err

server/services/v1/search/session.go

@@ -26,7 +26,8 @@ import (
 )
 
 type SessionOptions struct {
-	IncVersion bool
+	IncVersion          bool
+	DoNotReloadMetadata bool
 }
 
 type Session interface {
@@ -97,8 +98,10 @@ func (sessions *SessionManager) TxExecute(ctx context.Context, runner TxRunner,
 		return Response{}, errors.NotFound("tenant '%s' not found", namespace)
 	}
 
-	if err = sessions.TrackVersion(ctx, tenant); err != nil {
-		return Response{}, err
+	if !option.DoNotReloadMetadata {
+		if err = sessions.TrackVersion(ctx, tenant); err != nil {
+			return Response{}, err
+		}
 	}
 
 	tx, err := sessions.txMgr.StartTx(ctx)