Storage Garbage Collector

main.go

@@ -28,12 +28,15 @@ func main() {
 	dataStoreRAM := ramCache.DataStore{
 		Conf: conf,
 		Ch:   make(chan ramCache.File, 10000),
+		Log:  logs,
 	}
 	go dataStoreRAM.RamFileManager()
+
 	dataStoreStorage := storageCache.DataStore{
 		Conf: conf,
-		Ch:   make(chan storageCache.File, 10000),
+		Log:  logs,
 	}
+	go dataStoreStorage.StorageFileManager()
 
 	h := Handler{conf: conf, dataStoreRAM: &dataStoreRAM, dataStoreStorage: &dataStoreStorage, log: logs}
 	fasthttp.ListenAndServe(":8084", h.handler)

storageCache/garbageCollecor.go

@@ -0,0 +1,69 @@
+package storageCache
+
+import (
+	"fmt"
+	"io/ioutil"
+	"sort"
+	"time"
+
+	"github.com/inhies/go-bytesize"
+)
+
+type File struct {
+	Hash       string
+	Size       uint // in bytes
+	LastAccess time.Time
+}
+
+func (d *DataStore) StorageFileManager() {
+
+	for {
+		time.Sleep(5 * time.Second)
+		d.garbageCollector()
+	}
+}
+
+func (d *DataStore) garbageCollector() {
+	cachePath := d.Conf.StoragePath
+	fileStats := []File{}
+	files, _ := ioutil.ReadDir(cachePath)
+
+	for _, f := range files {
+		f := File{Hash: f.Name(), Size: uint(f.Size()), LastAccess: f.ModTime()}
+		fileStats = append(fileStats, f)
+	}
+
+	cacheSize := calculateCacheSize(fileStats)
+	fmt.Println("cache size Storage:", bytesize.ByteSize(cacheSize).Format("%.5f", "GB", false))
+
+	if cacheSize < uint(d.Conf.UseMaxDiskGb*int(bytesize.GB)) {
+		return
+	}
+
+	// sort files by last access
+	sort.Slice(fileStats, func(i, j int) bool {
+		return fileStats[i].LastAccess.Before(fileStats[j].LastAccess)
+	})
+
+	fileSizeRemoved := uint(0)
+	filesRemoved := 0
+	for _, file := range fileStats {
+		if cacheSize < uint(d.Conf.UseMaxDiskGb*int(bytesize.GB)) {
+			fmt.Println("Removed from Storage:", filesRemoved, "files with a total size of:", bytesize.ByteSize(fileSizeRemoved).Format("%.5f", "GB", false))
+			d.Log.LogCache(time.Now(), "storageGC", cacheSize, uint(d.Conf.UseMaxDiskGb*int(bytesize.GB)))
+			return
+		}
+		d.delete(file.Hash)
+		cacheSize -= file.Size
+		fileSizeRemoved += file.Size
+		filesRemoved++
+	}
+}
+
+func calculateCacheSize(fileStats []File) uint {
+	cacheSize := uint(0)
+	for _, f := range fileStats {
+		cacheSize += f.Size
+	}
+	return cacheSize
+}

storageCache/storageCache.go

@@ -4,21 +4,14 @@ import (
 	"crypto/sha256"
 	"encoding/hex"
 	"fmt"
-	"log"
 	"os"
 	"simple-S3-cache/config"
+	"simple-S3-cache/log"
 	"time"
 )
 
-type File struct {
-	Hash string
-	Size uint // in bytes
-	hits uint
-}
-
 type DataStore struct {
 	Conf config.Config
-	Ch   chan File
 	Log  log.Logger
 }