diff --git a/README.md b/README.md index 382a8b0..1573efc 100644 --- a/README.md +++ b/README.md @@ -97,9 +97,11 @@ Flags: -H, --tips Show tips. -u, --update update indices (without this chkbit will verify files in readonly mode) --show-ignored-only only show ignored files + -m, --show-missing show missing files/directories --algo="blake3" hash algorithm: md5, sha512, blake3 (default: blake3) -f, --force force update of damaged items -s, --skip-symlinks do not follow symlinks + -D, --no-dir-in-index do not track directories in the index -l, --log-file=STRING write to a logfile if specified --log-verbose verbose logging --index-name=".chkbit" filename where chkbit stores its hashes, needs to start with '.' (default: .chkbit) @@ -111,6 +113,27 @@ Flags: -V, --version show version information ``` +``` +$ chkbit -H + +.chkbitignore rules: + each line should contain exactly one name + you may use Unix shell-style wildcards (see README) + lines starting with '#' are skipped + lines starting with '/' are only applied to the current directory + +Status codes: + DMG: error, data damage detected + EIX: error, index damaged + old: warning, file replaced by an older version + new: new file + upd: file updated + ok : check ok + del: file/directory removed + ign: ignored (see .chkbitignore) + EXC: exception/panic +``` + chkbit is set to use only 5 workers by default so it will not slow your system to a crawl. You can specify a higher number to make it a lot faster if the IO throughput can also keep up. diff --git a/cmd/chkbit/help.go b/cmd/chkbit/help.go index 7772640..babcd2c 100644 --- a/cmd/chkbit/help.go +++ b/cmd/chkbit/help.go @@ -19,6 +19,7 @@ Status codes: new: new file upd: file updated ok : check ok + del: file/directory removed ign: ignored (see .chkbitignore) EXC: exception/panic ` diff --git a/cmd/chkbit/main.go b/cmd/chkbit/main.go index 12be65f..d767ad6 100644 --- a/cmd/chkbit/main.go +++ b/cmd/chkbit/main.go @@ -46,9 +46,11 @@ var cli struct { Tips bool `short:"H" help:"Show tips."` Update bool `short:"u" help:"update indices (without this chkbit will verify files in readonly mode)"` ShowIgnoredOnly bool `help:"only show ignored files"` + ShowMissing bool `short:"m" help:"show missing files/directories"` Algo string `default:"blake3" help:"hash algorithm: md5, sha512, blake3 (default: blake3)"` Force bool `short:"f" help:"force update of damaged items"` SkipSymlinks bool `short:"s" help:"do not follow symlinks"` + NoDirInIndex bool `short:"D" help:"do not track directories in the index"` LogFile string `short:"l" help:"write to a logfile if specified"` LogVerbose bool `help:"verbose logging"` IndexName string `default:".chkbit" help:"filename where chkbit stores its hashes, needs to start with '.' (default: .chkbit)"` @@ -63,14 +65,10 @@ var cli struct { type Main struct { dmgList []string errList []string - numIdxUpd int - numNew int - numUpd int verbose bool logger *log.Logger logVerbose bool progress Progress - total int termWidth int fps *util.RateCalc bps *util.RateCalc @@ -82,34 +80,26 @@ func (m *Main) log(text string) { func (m *Main) logStatus(stat chkbit.Status, message string) bool { if stat == chkbit.STATUS_UPDATE_INDEX { - m.numIdxUpd++ - } else { - if stat == chkbit.STATUS_ERR_DMG { - m.total++ - m.dmgList = append(m.dmgList, message) - } else if stat == chkbit.STATUS_PANIC { - m.errList = append(m.errList, message) - } else if stat == chkbit.STATUS_OK || stat == chkbit.STATUS_UPDATE || stat == chkbit.STATUS_NEW || stat == chkbit.STATUS_UP_WARN_OLD { - m.total++ - if stat == chkbit.STATUS_UPDATE || stat == chkbit.STATUS_UP_WARN_OLD { - m.numUpd++ - } else if stat == chkbit.STATUS_NEW { - m.numNew++ - } - } + return false + } - if m.logVerbose || stat != chkbit.STATUS_OK && stat != chkbit.STATUS_IGNORE { - m.log(stat.String() + " " + message) - } + if stat == chkbit.STATUS_ERR_DMG { + m.dmgList = append(m.dmgList, message) + } else if stat == chkbit.STATUS_PANIC { + m.errList = append(m.errList, message) + } - if m.verbose || !stat.IsVerbose() { - col := "" - if stat.IsErrorOrWarning() { - col = termAlertFG - } - lterm.Printline(col, stat.String(), " ", message, lterm.Reset) - return true + if m.logVerbose || !stat.IsVerbose() { + m.log(stat.String() + " " + message) + } + + if m.verbose || !stat.IsVerbose() { + col := "" + if stat.IsErrorOrWarning() { + col = termAlertFG } + lterm.Printline(col, stat.String(), " ", message, lterm.Reset) + return true } return false } @@ -130,7 +120,7 @@ func (m *Main) showStatus(context *chkbit.Context) { if m.progress == Fancy { lterm.Write(termBG, termFG1, stat, lterm.ClearLine(0), lterm.Reset, "\r") } else { - fmt.Print(m.total, "\r") + fmt.Print(context.NumTotal, "\r") } } case perf := <-context.PerfQueue: @@ -147,7 +137,7 @@ func (m *Main) showStatus(context *chkbit.Context) { stat = "RO" } stat = fmt.Sprintf("[%s:%d] %5d files $ %s %-13s $ %s %-13s", - stat, context.NumWorkers, m.total, + stat, context.NumWorkers, context.NumTotal, util.Sparkline(m.fps.Stats), statF, util.Sparkline(m.bps.Stats), statB) stat = util.LeftTruncate(stat, m.termWidth-1) @@ -155,7 +145,7 @@ func (m *Main) showStatus(context *chkbit.Context) { stat = strings.Replace(stat, "$", termSepFG+termSep+termFG3, 1) lterm.Write(termBG, termFG1, stat, lterm.ClearLine(0), lterm.Reset, "\r") } else if m.progress == Plain { - fmt.Print(m.total, "\r") + fmt.Print(context.NumTotal, "\r") } } } @@ -176,7 +166,9 @@ func (m *Main) process() *chkbit.Context { context.ForceUpdateDmg = cli.Force context.UpdateIndex = cli.Update context.ShowIgnoredOnly = cli.ShowIgnoredOnly + context.ShowMissing = cli.ShowMissing context.SkipSymlinks = cli.SkipSymlinks + context.TrackDirectories = !cli.NoDirInIndex var wg sync.WaitGroup wg.Add(1) @@ -216,11 +208,11 @@ func (m *Main) printResult(context *chkbit.Context) { if !context.UpdateIndex { mode = " in readonly mode" } - status := fmt.Sprintf("Processed %s%s.", util.LangNum1MutateSuffix(m.total, "file"), mode) + status := fmt.Sprintf("Processed %s%s.", util.LangNum1MutateSuffix(context.NumTotal, "file"), mode) cprint(termOKFG, status) m.log(status) - if m.progress == Fancy && m.total > 0 { + if m.progress == Fancy && context.NumTotal > 0 { elapsed := time.Since(m.fps.Start) elapsedS := elapsed.Seconds() fmt.Println("-", elapsed.Truncate(time.Second), "elapsed") @@ -228,17 +220,26 @@ func (m *Main) printResult(context *chkbit.Context) { fmt.Printf("- %.2f MB/second\n", (float64(m.bps.Total)+float64(m.bps.Current))/float64(sizeMB)/elapsedS) } + del := "" if context.UpdateIndex { - if m.numIdxUpd > 0 { - cprint(termOKFG, fmt.Sprintf("- %s updated\n- %s added\n- %s updated", - util.LangNum1Choice(m.numIdxUpd, "directory was", "directories were"), - util.LangNum1Choice(m.numNew, "file hash was", "file hashes were"), - util.LangNum1Choice(m.numUpd, "file hash was", "file hashes were"))) + if context.NumIdxUpd > 0 { + if context.NumDel > 0 { + del = fmt.Sprintf("\n- %s been removed", util.LangNum1Choice(context.NumDel, "file/directory has", "files/directories have")) + } + cprint(termOKFG, fmt.Sprintf("- %s updated\n- %s added\n- %s updated%s", + util.LangNum1Choice(context.NumIdxUpd, "directory was", "directories were"), + util.LangNum1Choice(context.NumNew, "file hash was", "file hashes were"), + util.LangNum1Choice(context.NumUpd, "file hash was", "file hashes were"), + del)) + } + } else if context.NumNew+context.NumUpd+context.NumDel > 0 { + if context.NumDel > 0 { + del = fmt.Sprintf("\n- %s would have been removed", util.LangNum1Choice(context.NumDel, "file/directory", "files/directories")) } - } else if m.numNew+m.numUpd > 0 { - cprint(termAlertFG, fmt.Sprintf("No changes were made (specify -u to update):\n- %s would have been added and\n- %s would have been updated.", - util.LangNum1MutateSuffix(m.numNew, "file"), - util.LangNum1MutateSuffix(m.numUpd, "file"))) + cprint(termAlertFG, fmt.Sprintf("No changes were made (specify -u to update):\n- %s would have been added\n- %s would have been updated%s", + util.LangNum1MutateSuffix(context.NumNew, "file"), + util.LangNum1MutateSuffix(context.NumUpd, "file"), + del)) } } diff --git a/context.go b/context.go index cebbbaa..768d7b9 100644 --- a/context.go +++ b/context.go @@ -8,18 +8,26 @@ import ( ) type Context struct { - NumWorkers int - ForceUpdateDmg bool - UpdateIndex bool - ShowIgnoredOnly bool - HashAlgo string - SkipSymlinks bool - IndexFilename string - IgnoreFilename string - WorkQueue chan *WorkItem - LogQueue chan *LogEvent - PerfQueue chan *PerfEvent - wg sync.WaitGroup + NumWorkers int + UpdateIndex bool + ShowIgnoredOnly bool + ShowMissing bool + ForceUpdateDmg bool + HashAlgo string + TrackDirectories bool + SkipSymlinks bool + IndexFilename string + IgnoreFilename string + WorkQueue chan *WorkItem + LogQueue chan *LogEvent + PerfQueue chan *PerfEvent + wg sync.WaitGroup + + NumTotal int + NumIdxUpd int + NumNew int + NumUpd int + NumDel int } func NewContext(numWorkers int, hashAlgo string, indexFilename string, ignoreFilename string) (*Context, error) { @@ -44,6 +52,29 @@ func NewContext(numWorkers int, hashAlgo string, indexFilename string, ignoreFil } func (context *Context) log(stat Status, message string) { + switch stat { + case STATUS_ERR_DMG: + context.NumTotal++ + case STATUS_UPDATE_INDEX: + context.NumIdxUpd++ + case STATUS_UP_WARN_OLD: + context.NumTotal++ + context.NumUpd++ + case STATUS_UPDATE: + context.NumTotal++ + context.NumUpd++ + case STATUS_NEW: + context.NumTotal++ + context.NumNew++ + case STATUS_OK: + context.NumTotal++ + case STATUS_MISSING: + context.NumDel++ + //case STATUS_PANIC: + //case STATUS_ERR_IDX: + //case STATUS_IGNORE: + } + context.LogQueue <- &LogEvent{stat, message} } @@ -59,8 +90,8 @@ func (context *Context) perfMonBytes(numBytes int64) { context.PerfQueue <- &PerfEvent{0, numBytes} } -func (context *Context) addWork(path string, filesToIndex []string, ignore *Ignore) { - context.WorkQueue <- &WorkItem{path, filesToIndex, ignore} +func (context *Context) addWork(path string, filesToIndex []string, dirList []string, ignore *Ignore) { + context.WorkQueue <- &WorkItem{path, filesToIndex, dirList, ignore} } func (context *Context) endWork() { @@ -72,6 +103,12 @@ func (context *Context) isChkbitFile(name string) bool { } func (context *Context) Start(pathList []string) { + context.NumTotal = 0 + context.NumIdxUpd = 0 + context.NumNew = 0 + context.NumUpd = 0 + context.NumDel = 0 + var wg sync.WaitGroup wg.Add(context.NumWorkers) for i := 0; i < context.NumWorkers; i++ { @@ -141,7 +178,7 @@ func (context *Context) scanDir(root string, parentIgnore *Ignore) { } } - context.addWork(root, filesToIndex, ignore) + context.addWork(root, filesToIndex, dirList, ignore) for _, name := range dirList { context.scanDir(filepath.Join(root, name), ignore) diff --git a/index.go b/index.go index f8b4abf..27dd639 100644 --- a/index.go +++ b/index.go @@ -5,6 +5,8 @@ import ( "errors" "os" "path/filepath" + "reflect" + "slices" ) const VERSION = 2 // index version @@ -24,6 +26,8 @@ type indexFile struct { // IdxRaw -> map[string]idxInfo IdxRaw json.RawMessage `json:"idx"` IdxHash string `json:"idx_hash"` + // 2024-08 optional, list of subdirectories + Dir []string `json:"dirlist,omitempty"` } type idxInfo1 struct { @@ -36,23 +40,28 @@ type indexFile1 struct { } type Index struct { - context *Context - path string - files []string - cur map[string]idxInfo - new map[string]idxInfo - modified bool - readonly bool + context *Context + path string + files []string + cur map[string]idxInfo + new map[string]idxInfo + curDirList []string + newDirList []string + modified bool + readonly bool } -func NewIndex(context *Context, path string, files []string, readonly bool) *Index { +func newIndex(context *Context, path string, files []string, dirList []string, readonly bool) *Index { + slices.Sort(dirList) return &Index{ - context: context, - path: path, - files: files, - cur: make(map[string]idxInfo), - new: make(map[string]idxInfo), - readonly: readonly, + context: context, + path: path, + files: files, + cur: make(map[string]idxInfo), + new: make(map[string]idxInfo), + curDirList: make([]string, 0), + newDirList: dirList, + readonly: readonly, } } @@ -72,6 +81,10 @@ func (i *Index) logFile(stat Status, name string) { i.context.log(stat, filepath.Join(i.path, name)) } +func (i *Index) logDir(stat Status, name string) { + i.context.log(stat, filepath.Join(i.path, name)+"/") +} + func (i *Index) calcHashes(ignore *Ignore) { for _, name := range i.files { if ignore != nil && ignore.shouldIgnore(name) { @@ -153,6 +166,26 @@ func (i *Index) checkFix(forceUpdateDmg bool) { } } } + if i.context.ShowMissing { + for name := range i.cur { + if _, ok := i.new[name]; !ok { + i.logFile(STATUS_MISSING, name) + i.setMod(true) + } + } + // dirs + m := make(map[string]bool) + for _, n := range i.newDirList { + m[n] = true + } + for _, name := range i.curDirList { + if !m[name] { + i.logDir(STATUS_MISSING, name+"/") + i.setMod(true) + } + } + + } } func (i *Index) calcFile(name string, a string) (*idxInfo, error) { @@ -186,6 +219,9 @@ func (i *Index) save() (bool, error) { IdxRaw: text, IdxHash: hashMd5(text), } + if i.context.TrackDirectories { + data.Dir = i.newDirList + } file, err := json.Marshal(data) if err != nil { @@ -205,6 +241,8 @@ func (i *Index) save() (bool, error) { func (i *Index) load() error { if _, err := os.Stat(i.getIndexFilepath()); err != nil { if os.IsNotExist(err) { + // todo + i.setMod(true) return nil } return err @@ -248,5 +286,12 @@ func (i *Index) load() error { } } } + if data.Dir != nil { + slices.Sort(data.Dir) + i.curDirList = data.Dir + if i.context.TrackDirectories && !reflect.DeepEqual(i.curDirList, i.newDirList) { + i.setMod(true) + } + } return nil } diff --git a/scripts/run_test.go b/scripts/run_test.go index 6b3dafa..13496e6 100644 --- a/scripts/run_test.go +++ b/scripts/run_test.go @@ -11,6 +11,8 @@ import ( "time" ) +// perform integration test using the compiled binary + var testDir = "/tmp/chkbit" func getCmd() string { @@ -25,6 +27,12 @@ func checkOut(t *testing.T, sout string, expected string) { } } +func checkNotOut(t *testing.T, sout string, notExpected string) { + if strings.Contains(sout, notExpected) { + t.Errorf("Did not expect '%s' in output, got '%s'\n", notExpected, sout) + } +} + // misc files var ( @@ -128,14 +136,59 @@ func TestRoot(t *testing.T) { tool := getCmd() root := filepath.Join(testDir, "root") - cmd := exec.Command(tool, "-u", root) + + // step1: update index + cmd := exec.Command(tool, "-um", root) out, err := cmd.Output() if err != nil { - t.Fatalf("cmd.Output() failed with '%s'\n", err) + t.Fatalf("step1 failed with '%s'\n", err) } sout := string(out) - checkOut(t, sout, "60 directories were updated") + checkOut(t, sout, "67 directories were updated") checkOut(t, sout, "300 file hashes were added") + checkNotOut(t, sout, "removed") + + // step2: delete files, check for missing + os.RemoveAll(filepath.Join(root, "thing/change")) + os.Remove(filepath.Join(root, "time/hour/minute/body-information.csv")) + + cmd = exec.Command(tool, "-m", root) + out, err = cmd.Output() + if err != nil { + t.Fatalf("step2 failed with '%s'\n", err) + } + sout = string(out) + checkOut(t, sout, "del /tmp/chkbit/root/thing/change/") + checkOut(t, sout, "2 files/directories would have been removed") + + // step2a: do not report missing without -m + cmd = exec.Command(tool, root) + out, err = cmd.Output() + if err != nil { + t.Fatalf("step2a failed with '%s'\n", err) + } + sout = string(out) + checkNotOut(t, sout, "del ") + checkNotOut(t, sout, "removed") + + // step3: check for missing and update + cmd = exec.Command(tool, "-um", root) + out, err = cmd.Output() + if err != nil { + t.Fatalf("step3 failed with '%s'\n", err) + } + sout = string(out) + checkOut(t, sout, "del /tmp/chkbit/root/thing/change/") + checkOut(t, sout, "2 files/directories have been removed") + + // step4: check again + cmd = exec.Command(tool, "-u", root) + out, err = cmd.Output() + if err != nil { + t.Fatalf("step4 failed with '%s'\n", err) + } + sout = string(out) + checkOut(t, sout, "Processed 289 files") } func TestDMG(t *testing.T) { diff --git a/scripts/tests b/scripts/tests index 3fff645..5a1b690 100755 --- a/scripts/tests +++ b/scripts/tests @@ -7,5 +7,5 @@ cd $script_dir/.. # prep $script_dir/build -go test -v ./cmd/chkbit/util -go test -v ./scripts +go test -v ./cmd/chkbit/util -count=1 +go test -v ./scripts -count=1 diff --git a/status.go b/status.go index 4fd09b8..49d5cf3 100644 --- a/status.go +++ b/status.go @@ -12,6 +12,7 @@ const ( STATUS_NEW Status = "new" STATUS_OK Status = "ok " STATUS_IGNORE Status = "ign" + STATUS_MISSING Status = "del" ) func (s Status) String() string { diff --git a/worker.go b/worker.go index a2b9f6c..e8921ff 100644 --- a/worker.go +++ b/worker.go @@ -3,6 +3,7 @@ package chkbit type WorkItem struct { path string filesToIndex []string + dirList []string ignore *Ignore } @@ -13,7 +14,7 @@ func (context *Context) runWorker(id int) { break } - index := NewIndex(context, item.path, item.filesToIndex, !context.UpdateIndex) + index := newIndex(context, item.path, item.filesToIndex, item.dirList, !context.UpdateIndex) err := index.load() if err != nil { context.log(STATUS_PANIC, index.getIndexFilepath()+": "+err.Error())