Improve HardlinkDir performance

Makefile

@@ -108,6 +108,10 @@ test-fuzz: export DL_SKIP_SSL_VERIFICATION=1
 test-fuzz: reset-db
 	go run cmd/fuzz-test/main.go --host $(GRPC_HOST) --iterations 1000 --projects 5
 
+bench: export DB_URI = postgres://$(DB_USER):$(DB_PASS)@$(DB_HOST):5432/dl_tests
+bench: migrate
+	cd test && go test -bench . -run=^#
+
 reset-db: migrate
 	psql $(DB_URI) -c "truncate dl.objects; truncate dl.contents; truncate dl.projects; truncate dl.cache_versions;"
 

default.nix

@@ -17,7 +17,7 @@ buildGoModule rec {
   version = "0.7.11";
   src = ./.;
   proxyVendor = true; # Fixes: cannot query module due to -mod=vendor running make install
-  vendorHash = "sha256-AeCAM/MQ/zb+C8MNnGe2Dv/vWoSpea4AhE49X/vLNHg=";
+  vendorHash = "sha256-3XVou4NW28QxIe1t0gawbCoyl/3X22PoQdJOjcG0nDM=";
 
   outputs = [ "out" "client" "server" "migrations" ];
 

internal/files/writer.go

@@ -201,15 +201,32 @@ func HardlinkDir(olddir, newdir string) error {
 		}
 	}
 
-	return filepath.Walk(olddir, func(oldpath string, info fs.FileInfo, err error) error {
+	rootInfo, err := os.Lstat(olddir)
+	if err != nil {
+		return fmt.Errorf("cannot stat olddir %v: %w", olddir, err)
+	}
+
+	err = os.MkdirAll(newdir, rootInfo.Mode())
+	if err != nil {
+		return fmt.Errorf("cannot create new root dir %v: %w", olddir, err)
+	}
+
+	return filepath.WalkDir(olddir, func(oldpath string, d os.DirEntry, err error) error {
 		if err != nil {
-			return fmt.Errorf("failed to walk dir: %v, %w", info, err)
+			return fmt.Errorf("failed to walk dir: %v, %w", oldpath, err)
+		}
+		if oldpath == olddir {
+			return nil
 		}
 
 		newpath := filepath.Join(newdir, strings.TrimPrefix(oldpath, olddir))
 
-		if info.IsDir() {
-			err := os.MkdirAll(newpath, info.Mode())
+		if d.IsDir() {
+			info, err := d.Info()
+			if err != nil {
+				return fmt.Errorf("cannot access directory info %v: %w", oldpath, err)
+			}
+			err = os.Mkdir(newpath, info.Mode())
 			if err != nil {
 				return fmt.Errorf("cannot create dir %v: %w", newpath, err)
 			}

test/hardlink_dir_benchmark_test.go

@@ -0,0 +1,45 @@
+package test
+
+import (
+	"fmt"
+	"os"
+	"path"
+	"testing"
+
+	"github.com/gadget-inc/dateilager/internal/files"
+	"github.com/stretchr/testify/require"
+)
+
+func TestHardlinkDir(t *testing.T) {
+	wd, err := os.Getwd()
+	require.NoError(t, err, "os.Getwd() failed")
+
+	bigDir := path.Join(wd, "../js/node_modules")
+	tmpDir := emptyTmpDir(t)
+	defer os.RemoveAll(tmpDir)
+
+	copyDir := path.Join(tmpDir, "node_modules")
+	err = files.HardlinkDir(bigDir, copyDir)
+	require.NoError(t, err, "HardlinkDir failed")
+
+	err = CompareDirectories(bigDir, copyDir)
+	require.NoError(t, err, "compareDirectories %s vs %s failed", bigDir, tmpDir)
+}
+
+func BenchmarkHardlinkDir(b *testing.B) {
+	wd, err := os.Getwd()
+	if err != nil {
+		b.Error(err)
+	}
+
+	bigDir := path.Join(wd, "../js/node_modules")
+	tmpDir := emptyTmpDir(b)
+	defer os.RemoveAll(tmpDir)
+
+	for n := 0; n < b.N; n++ {
+		err := files.HardlinkDir(bigDir, path.Join(tmpDir, "node_modules", fmt.Sprintf("%d", n)))
+		if err != nil {
+			b.Error(err)
+		}
+	}
+}

test/shared_test.go

@@ -4,6 +4,7 @@ import (
 	"archive/tar"
 	"bytes"
 	"context"
+	"crypto/sha256"
 	"fmt"
 	"io"
 	"io/fs"
@@ -275,9 +276,11 @@ func writeFile(t *testing.T, dir string, path string, content string) {
 	require.NoError(t, err, "write file %v", path)
 }
 
-func emptyTmpDir(t *testing.T) string {
+func emptyTmpDir(t testing.TB) string {
 	dir, err := os.MkdirTemp("", "dateilager_tests_")
-	require.NoError(t, err, "create temp dir")
+	if err != nil {
+		t.Fatal(err)
+	}
 
 	return dir
 }
@@ -764,3 +767,94 @@ func formatPtr(p *int64) string {
 	}
 	return fmt.Sprint(*p)
 }
+
+// CompareDirectories compares the contents and permissions of two directories recursively.
+func CompareDirectories(dir1, dir2 string) error {
+	files1 := make(map[string]os.FileInfo)
+	err := filepath.Walk(dir1, func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+		relPath, err := filepath.Rel(dir1, path)
+		if err != nil {
+			return err
+		}
+		files1[relPath] = info
+		return nil
+	})
+	if err != nil {
+		return err
+	}
+
+	err = filepath.Walk(dir2, func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+		relPath, err := filepath.Rel(dir2, path)
+		if err != nil {
+			return err
+		}
+		if origInfo, ok := files1[relPath]; ok {
+			if info.IsDir() && origInfo.IsDir() {
+				// Only compare directories for existence, not content
+				delete(files1, relPath)
+				return nil
+			}
+			if !compareFileInfo(origInfo, info) {
+				return fmt.Errorf("file permissions differ for %s: %o vs %o", relPath, origInfo.Mode()&os.ModePerm, info.Mode()&os.ModePerm)
+			}
+			if equal, err := compareFileContents(filepath.Join(dir1, relPath), filepath.Join(dir2, relPath)); err != nil {
+				return fmt.Errorf("error comparing contents of %s: %v", relPath, err)
+			} else if !equal {
+				return fmt.Errorf("contents differ for %s", relPath)
+			}
+			delete(files1, relPath)
+		} else {
+			return fmt.Errorf("extra file found in directory 2: %s", path)
+		}
+		return nil
+	})
+	if err != nil {
+		return err
+	}
+	for file := range files1 {
+		err = fmt.Errorf("file missing in directory 2: %s", file)
+	}
+	return err
+}
+
+// compareFileInfo compares the permissions and other metadata of two files.
+func compareFileInfo(info1, info2 os.FileInfo) bool {
+	return (info1.Mode() & os.ModePerm) == (info2.Mode() & os.ModePerm)
+}
+
+// compareFileContents compares the contents of two files.
+func compareFileContents(file1, file2 string) (bool, error) {
+	f1, err := os.Open(file1)
+	if err != nil {
+		return false, err
+	}
+	defer f1.Close()
+	f2, err := os.Open(file2)
+	if err != nil {
+		return false, err
+	}
+	defer f2.Close()
+
+	if info1, _ := f1.Stat(); info1.IsDir() {
+		return true, nil // Skip directories in content comparison
+	}
+	if info2, _ := f2.Stat(); info2.IsDir() {
+		return true, nil // Skip directories in content comparison
+	}
+
+	hash1, hash2 := sha256.New(), sha256.New()
+	if _, err := io.Copy(hash1, f1); err != nil {
+		return false, err
+	}
+	if _, err := io.Copy(hash2, f2); err != nil {
+		return false, err
+	}
+
+	return bytes.Equal(hash1.Sum(nil), hash2.Sum(nil)), nil
+}