Skip to content

Commit

Permalink
track directories and report missing files/dirs (-m) #16
Browse files Browse the repository at this point in the history
  • Loading branch information
@laktak
laktak committed Aug 20, 2024
1 parent 181b3d8 commit a1327a4
Show file tree
Hide file tree
Showing 9 changed files with 240 additions and 78 deletions.
23 changes: 23 additions & 0 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -97,9 +97,11 @@ Flags:
-H, --tips Show tips.
-u, --update update indices (without this chkbit will verify files in readonly mode)
--show-ignored-only only show ignored files
-m, --show-missing show missing files/directories
--algo="blake3" hash algorithm: md5, sha512, blake3 (default: blake3)
-f, --force force update of damaged items
-s, --skip-symlinks do not follow symlinks
-D, --no-dir-in-index do not track directories in the index
-l, --log-file=STRING write to a logfile if specified
--log-verbose verbose logging
--index-name=".chkbit" filename where chkbit stores its hashes, needs to start with '.' (default: .chkbit)
Expand All @@ -111,6 +113,27 @@ Flags:
-V, --version show version information
```

```
$ chkbit -H
.chkbitignore rules:
each line should contain exactly one name
you may use Unix shell-style wildcards (see README)
lines starting with '#' are skipped
lines starting with '/' are only applied to the current directory
Status codes:
DMG: error, data damage detected
EIX: error, index damaged
old: warning, file replaced by an older version
new: new file
upd: file updated
ok : check ok
del: file/directory removed
ign: ignored (see .chkbitignore)
EXC: exception/panic
```

chkbit is set to use only 5 workers by default so it will not slow your system to a crawl. You can specify a higher number to make it a lot faster if the IO throughput can also keep up.


Expand Down
1 change: 1 addition & 0 deletions cmd/chkbit/help.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ Status codes:
new: new file
upd: file updated
ok : check ok
del: file/directory removed
ign: ignored (see .chkbitignore)
EXC: exception/panic
`
87 changes: 44 additions & 43 deletions cmd/chkbit/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,9 +46,11 @@ var cli struct {
Tips bool `short:"H" help:"Show tips."`
Update bool `short:"u" help:"update indices (without this chkbit will verify files in readonly mode)"`
ShowIgnoredOnly bool `help:"only show ignored files"`
ShowMissing bool `short:"m" help:"show missing files/directories"`
Algo string `default:"blake3" help:"hash algorithm: md5, sha512, blake3 (default: blake3)"`
Force bool `short:"f" help:"force update of damaged items"`
SkipSymlinks bool `short:"s" help:"do not follow symlinks"`
NoDirInIndex bool `short:"D" help:"do not track directories in the index"`
LogFile string `short:"l" help:"write to a logfile if specified"`
LogVerbose bool `help:"verbose logging"`
IndexName string `default:".chkbit" help:"filename where chkbit stores its hashes, needs to start with '.' (default: .chkbit)"`
Expand All @@ -63,14 +65,10 @@ var cli struct {
type Main struct {
dmgList []string
errList []string
numIdxUpd int
numNew int
numUpd int
verbose bool
logger *log.Logger
logVerbose bool
progress Progress
total int
termWidth int
fps *util.RateCalc
bps *util.RateCalc
Expand All @@ -82,34 +80,26 @@ func (m *Main) log(text string) {

func (m *Main) logStatus(stat chkbit.Status, message string) bool {
if stat == chkbit.STATUS_UPDATE_INDEX {
m.numIdxUpd++
} else {
if stat == chkbit.STATUS_ERR_DMG {
m.total++
m.dmgList = append(m.dmgList, message)
} else if stat == chkbit.STATUS_PANIC {
m.errList = append(m.errList, message)
} else if stat == chkbit.STATUS_OK || stat == chkbit.STATUS_UPDATE || stat == chkbit.STATUS_NEW || stat == chkbit.STATUS_UP_WARN_OLD {
m.total++
if stat == chkbit.STATUS_UPDATE || stat == chkbit.STATUS_UP_WARN_OLD {
m.numUpd++
} else if stat == chkbit.STATUS_NEW {
m.numNew++
}
}
return false
}

if m.logVerbose || stat != chkbit.STATUS_OK && stat != chkbit.STATUS_IGNORE {
m.log(stat.String() + " " + message)
}
if stat == chkbit.STATUS_ERR_DMG {
m.dmgList = append(m.dmgList, message)
} else if stat == chkbit.STATUS_PANIC {
m.errList = append(m.errList, message)
}

if m.verbose || !stat.IsVerbose() {
col := ""
if stat.IsErrorOrWarning() {
col = termAlertFG
}
lterm.Printline(col, stat.String(), " ", message, lterm.Reset)
return true
if m.logVerbose || !stat.IsVerbose() {
m.log(stat.String() + " " + message)
}

if m.verbose || !stat.IsVerbose() {
col := ""
if stat.IsErrorOrWarning() {
col = termAlertFG
}
lterm.Printline(col, stat.String(), " ", message, lterm.Reset)
return true
}
return false
}
Expand All @@ -130,7 +120,7 @@ func (m *Main) showStatus(context *chkbit.Context) {
if m.progress == Fancy {
lterm.Write(termBG, termFG1, stat, lterm.ClearLine(0), lterm.Reset, "\r")
} else {
fmt.Print(m.total, "\r")
fmt.Print(context.NumTotal, "\r")
}
}
case perf := <-context.PerfQueue:
Expand All @@ -147,15 +137,15 @@ func (m *Main) showStatus(context *chkbit.Context) {
stat = "RO"
}
stat = fmt.Sprintf("[%s:%d] %5d files $ %s %-13s $ %s %-13s",
stat, context.NumWorkers, m.total,
stat, context.NumWorkers, context.NumTotal,
util.Sparkline(m.fps.Stats), statF,
util.Sparkline(m.bps.Stats), statB)
stat = util.LeftTruncate(stat, m.termWidth-1)
stat = strings.Replace(stat, "$", termSepFG+termSep+termFG2, 1)
stat = strings.Replace(stat, "$", termSepFG+termSep+termFG3, 1)
lterm.Write(termBG, termFG1, stat, lterm.ClearLine(0), lterm.Reset, "\r")
} else if m.progress == Plain {
fmt.Print(m.total, "\r")
fmt.Print(context.NumTotal, "\r")
}
}
}
Expand All @@ -176,7 +166,9 @@ func (m *Main) process() *chkbit.Context {
context.ForceUpdateDmg = cli.Force
context.UpdateIndex = cli.Update
context.ShowIgnoredOnly = cli.ShowIgnoredOnly
context.ShowMissing = cli.ShowMissing
context.SkipSymlinks = cli.SkipSymlinks
context.TrackDirectories = !cli.NoDirInIndex

var wg sync.WaitGroup
wg.Add(1)
Expand Down Expand Up @@ -216,29 +208,38 @@ func (m *Main) printResult(context *chkbit.Context) {
if !context.UpdateIndex {
mode = " in readonly mode"
}
status := fmt.Sprintf("Processed %s%s.", util.LangNum1MutateSuffix(m.total, "file"), mode)
status := fmt.Sprintf("Processed %s%s.", util.LangNum1MutateSuffix(context.NumTotal, "file"), mode)
cprint(termOKFG, status)
m.log(status)

if m.progress == Fancy && m.total > 0 {
if m.progress == Fancy && context.NumTotal > 0 {
elapsed := time.Since(m.fps.Start)
elapsedS := elapsed.Seconds()
fmt.Println("-", elapsed.Truncate(time.Second), "elapsed")
fmt.Printf("- %.2f files/second\n", (float64(m.fps.Total)+float64(m.fps.Current))/elapsedS)
fmt.Printf("- %.2f MB/second\n", (float64(m.bps.Total)+float64(m.bps.Current))/float64(sizeMB)/elapsedS)
}

del := ""
if context.UpdateIndex {
if m.numIdxUpd > 0 {
cprint(termOKFG, fmt.Sprintf("- %s updated\n- %s added\n- %s updated",
util.LangNum1Choice(m.numIdxUpd, "directory was", "directories were"),
util.LangNum1Choice(m.numNew, "file hash was", "file hashes were"),
util.LangNum1Choice(m.numUpd, "file hash was", "file hashes were")))
if context.NumIdxUpd > 0 {
if context.NumDel > 0 {
del = fmt.Sprintf("\n- %s been removed", util.LangNum1Choice(context.NumDel, "file/directory has", "files/directories have"))
}
cprint(termOKFG, fmt.Sprintf("- %s updated\n- %s added\n- %s updated%s",
util.LangNum1Choice(context.NumIdxUpd, "directory was", "directories were"),
util.LangNum1Choice(context.NumNew, "file hash was", "file hashes were"),
util.LangNum1Choice(context.NumUpd, "file hash was", "file hashes were"),
del))
}
} else if context.NumNew+context.NumUpd+context.NumDel > 0 {
if context.NumDel > 0 {
del = fmt.Sprintf("\n- %s would have been removed", util.LangNum1Choice(context.NumDel, "file/directory", "files/directories"))
}
} else if m.numNew+m.numUpd > 0 {
cprint(termAlertFG, fmt.Sprintf("No changes were made (specify -u to update):\n- %s would have been added and\n- %s would have been updated.",
util.LangNum1MutateSuffix(m.numNew, "file"),
util.LangNum1MutateSuffix(m.numUpd, "file")))
cprint(termAlertFG, fmt.Sprintf("No changes were made (specify -u to update):\n- %s would have been added\n- %s would have been updated%s",
util.LangNum1MutateSuffix(context.NumNew, "file"),
util.LangNum1MutateSuffix(context.NumUpd, "file"),
del))
}
}

Expand Down
67 changes: 52 additions & 15 deletions context.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,18 +8,26 @@ import (
)

type Context struct {
NumWorkers int
ForceUpdateDmg bool
UpdateIndex bool
ShowIgnoredOnly bool
HashAlgo string
SkipSymlinks bool
IndexFilename string
IgnoreFilename string
WorkQueue chan *WorkItem
LogQueue chan *LogEvent
PerfQueue chan *PerfEvent
wg sync.WaitGroup
NumWorkers int
UpdateIndex bool
ShowIgnoredOnly bool
ShowMissing bool
ForceUpdateDmg bool
HashAlgo string
TrackDirectories bool
SkipSymlinks bool
IndexFilename string
IgnoreFilename string
WorkQueue chan *WorkItem
LogQueue chan *LogEvent
PerfQueue chan *PerfEvent
wg sync.WaitGroup

NumTotal int
NumIdxUpd int
NumNew int
NumUpd int
NumDel int
}

func NewContext(numWorkers int, hashAlgo string, indexFilename string, ignoreFilename string) (*Context, error) {
Expand All @@ -44,6 +52,29 @@ func NewContext(numWorkers int, hashAlgo string, indexFilename string, ignoreFil
}

func (context *Context) log(stat Status, message string) {
switch stat {
case STATUS_ERR_DMG:
context.NumTotal++
case STATUS_UPDATE_INDEX:
context.NumIdxUpd++
case STATUS_UP_WARN_OLD:
context.NumTotal++
context.NumUpd++
case STATUS_UPDATE:
context.NumTotal++
context.NumUpd++
case STATUS_NEW:
context.NumTotal++
context.NumNew++
case STATUS_OK:
context.NumTotal++
case STATUS_MISSING:
context.NumDel++
//case STATUS_PANIC:
//case STATUS_ERR_IDX:
//case STATUS_IGNORE:
}

context.LogQueue <- &LogEvent{stat, message}
}

Expand All @@ -59,8 +90,8 @@ func (context *Context) perfMonBytes(numBytes int64) {
context.PerfQueue <- &PerfEvent{0, numBytes}
}

func (context *Context) addWork(path string, filesToIndex []string, ignore *Ignore) {
context.WorkQueue <- &WorkItem{path, filesToIndex, ignore}
func (context *Context) addWork(path string, filesToIndex []string, dirList []string, ignore *Ignore) {
context.WorkQueue <- &WorkItem{path, filesToIndex, dirList, ignore}
}

func (context *Context) endWork() {
Expand All @@ -72,6 +103,12 @@ func (context *Context) isChkbitFile(name string) bool {
}

func (context *Context) Start(pathList []string) {
context.NumTotal = 0
context.NumIdxUpd = 0
context.NumNew = 0
context.NumUpd = 0
context.NumDel = 0

var wg sync.WaitGroup
wg.Add(context.NumWorkers)
for i := 0; i < context.NumWorkers; i++ {
Expand Down Expand Up @@ -141,7 +178,7 @@ func (context *Context) scanDir(root string, parentIgnore *Ignore) {
}
}

context.addWork(root, filesToIndex, ignore)
context.addWork(root, filesToIndex, dirList, ignore)

for _, name := range dirList {
context.scanDir(filepath.Join(root, name), ignore)
Expand Down
Loading

0 comments on commit a1327a4

Please sign in to comment.