refactor(category): ignore partially deleted data dir (#509)

alpacahq · Sep 19, 2021 · df956c0 · df956c0
1 parent a015238
commit df956c0
Show file tree

Hide file tree

Showing 5 changed files with 15 additions and 6 deletions.
diff --git a/catalog/catalog.go b/catalog/catalog.go
@@ -1,6 +1,7 @@
 package catalog
 
 import (
+	"errors"
 	"fmt"
 	"io/ioutil"
 	"os"
@@ -10,6 +11,8 @@ import (
 	"strings"
 	"sync"
 
+	"github.com/alpacahq/marketstore/v4/utils/log"
+
 	"github.com/alpacahq/marketstore/v4/utils/io"
 )
 
@@ -55,7 +58,7 @@ func load(rootDmap *sync.Map, d *Directory, subPath, rootPath string) error {
 	catFilePath := subPath + "/" + "category_name"
 	catname, err := ioutil.ReadFile(catFilePath)
 	if err != nil {
-		return &ErrCategoryFileNotFound{filePath: catFilePath, msg: io.GetCallerFileContext(0) + err.Error()}
+		return ErrCategoryFileNotFound{filePath: catFilePath, msg: io.GetCallerFileContext(0) + err.Error()}
 	}
 	d.category = string(catname)
 
@@ -76,7 +79,13 @@ func load(rootDmap *sync.Map, d *Directory, subPath, rootPath string) error {
 
 			d.datafile = nil
 			if err := load(rootDmap, d.subDirs[itemName], leafPath, rootPath); err != nil {
-				return fmt.Errorf(io.GetCallerFileContext(0) + ", " + err.Error())
+				var e ErrCategoryFileNotFound
+				if errors.As(err, &e) {
+					log.Warn(fmt.Sprintf("category_name file not found under the directory."+
+						"%s will be ignored:%v", leafPath, err.Error()))
+				} else {
+					return fmt.Errorf(io.GetCallerFileContext(0) + ", " + err.Error())
+				}
 			}
 		} else if filepath.Ext(leafPath) == ".bin" {
 			rootDmap.Store(d.pathToItemName, d)

diff --git a/catalog/catalog_test.go b/catalog/catalog_test.go
@@ -185,7 +185,7 @@ func TestAddAndRemoveDataItem(t *testing.T) {
 func TestAddAndRemoveDataItemFromEmptyDirectory(t *testing.T) {
 	rootDir, _ := ioutil.TempDir("", "catalog_test-TestAddAndRemoveDataItemFromEmptyDirectory")
 	catalogDir, err := catalog.NewDirectory(rootDir)
-	var e *catalog.ErrCategoryFileNotFound
+	var e catalog.ErrCategoryFileNotFound
 	if err != nil && !errors.As(err, &e) {
 		t.Fatal("failed to create a catalog dir.err=" + err.Error())
 		return

diff --git a/catalog/errors.go b/catalog/errors.go
@@ -59,6 +59,6 @@ type ErrCategoryFileNotFound struct {
 	msg      string
 }
 
-func (e *ErrCategoryFileNotFound) Error() string {
+func (e ErrCategoryFileNotFound) Error() string {
 	return "Could not find a category_name file under:" + e.filePath + ", msg=" + e.msg
 }
diff --git a/executor/all_test.go b/executor/all_test.go
@@ -43,7 +43,7 @@ func TestAddDir(t *testing.T) {
 
 	// make catelog directory
 	catDir, err := NewDirectory(tempRootDir)
-	var e *ErrCategoryFileNotFound
+	var e ErrCategoryFileNotFound
 	if err != nil && !errors.As(err, &e) {
 		t.Fatal("failed to create a catalog dir.err=" + err.Error())
 		return

diff --git a/executor/instance.go b/executor/instance.go
@@ -70,7 +70,7 @@ func NewInstanceSetup(relRootDir string, rs ReplicationSender, tm []*trigger.Tri
 	if initCatalog {
 		ThisInstance.CatalogDir, err = catalog.NewDirectory(rootDir)
 		if err != nil {
-			var e *catalog.ErrCategoryFileNotFound
+			var e catalog.ErrCategoryFileNotFound
 			if errors.As(err, &e) {
 				log.Debug("new root directory found:" + rootDir)
 			} else {