From 1d97e91341ff3767e74fffdd487488450a48f983 Mon Sep 17 00:00:00 2001 From: Kota Kanbe Date: Thu, 10 Nov 2022 10:19:15 +0900 Subject: [PATCH] fix(libscan): delete map that keeps all file contents detected by FindLock to save memory (#1556) * fix(libscan): delete Map that keeps all files detected by FindLock to save memory * continue analyzing libs if err occurred * FindLockDirs * fix * fix --- config/config.go | 1 + logging/logutil.go | 7 ++ scanner/base.go | 241 +++++++++++++++++++++++--------------------- scanner/debian.go | 2 +- scanner/executil.go | 1 - subcmds/discover.go | 1 + 6 files changed, 134 insertions(+), 119 deletions(-) diff --git a/config/config.go b/config/config.go index e2276b6971..64850c6716 100644 --- a/config/config.go +++ b/config/config.go @@ -240,6 +240,7 @@ type ServerInfo struct { Optional map[string]interface{} `toml:"optional,omitempty" json:"optional,omitempty"` // Optional key-value set that will be outputted to JSON Lockfiles []string `toml:"lockfiles,omitempty" json:"lockfiles,omitempty"` // ie) path/to/package-lock.json FindLock bool `toml:"findLock,omitempty" json:"findLock,omitempty"` + FindLockDirs []string `toml:"findLockDirs,omitempty" json:"findLockDirs,omitempty"` Type string `toml:"type,omitempty" json:"type,omitempty"` // "pseudo" or "" IgnoredJSONKeys []string `toml:"ignoredJSONKeys,omitempty" json:"ignoredJSONKeys,omitempty"` WordPress *WordPressConf `toml:"wordpress,omitempty" json:"wordpress,omitempty"` diff --git a/logging/logutil.go b/logging/logutil.go index 96494abcf7..6198a38aa1 100644 --- a/logging/logutil.go +++ b/logging/logutil.go @@ -45,6 +45,13 @@ func NewNormalLogger() Logger { return Logger{Entry: logrus.Entry{Logger: logrus.New()}} } +// NewNormalLogger creates normal logger +func NewIODiscardLogger() Logger { + l := logrus.New() + l.Out = io.Discard + return Logger{Entry: logrus.Entry{Logger: l}} +} + // NewCustomLogger creates logrus func NewCustomLogger(debug, quiet, logToFile bool, logDir, logMsgAnsiColor, serverName string) Logger { log := logrus.New() diff --git a/scanner/base.go b/scanner/base.go index 4245b34130..9417b4e61b 100644 --- a/scanner/base.go +++ b/scanner/base.go @@ -361,7 +361,6 @@ func (l *base) detectPlatform() { //TODO Azure, GCP... l.setPlatform(models.Platform{Name: "other"}) - return } var dsFingerPrintPrefix = "AgentStatus.agentCertHash: " @@ -582,12 +581,6 @@ func (l *base) parseSystemctlStatus(stdout string) string { return ss[1] } -// LibFile : library file content -type LibFile struct { - Contents []byte - Filemode os.FileMode -} - func (l *base) scanLibraries() (err error) { if len(l.LibraryScanners) != 0 { return nil @@ -598,9 +591,9 @@ func (l *base) scanLibraries() (err error) { return nil } - l.log.Info("Scanning Lockfile...") + l.log.Info("Scanning Language-specific Packages...") - libFilemap := map[string]LibFile{} + found := map[string]bool{} detectFiles := l.ServerInfo.Lockfiles priv := noSudo @@ -615,9 +608,17 @@ func (l *base) scanLibraries() (err error) { findopt += fmt.Sprintf("-name %q -o ", filename) } + dir := "/" + if len(l.ServerInfo.FindLockDirs) != 0 { + dir = strings.Join(l.ServerInfo.FindLockDirs, " ") + } else { + l.log.Infof("It's recommended to specify FindLockDirs in config.toml. If FindLockDirs is not specified, all directories under / will be searched, which may increase CPU load") + } + l.log.Infof("Finding files under %s", dir) + // delete last "-o " // find / -type f -and \( -name "package-lock.json" -o -name "yarn.lock" ... \) 2>&1 | grep -v "find: " - cmd := fmt.Sprintf(`find / -type f -and \( ` + findopt[:len(findopt)-3] + ` \) 2>&1 | grep -v "find: "`) + cmd := fmt.Sprintf(`find %s -type f -and \( `+findopt[:len(findopt)-3]+` \) 2>&1 | grep -v "find: "`, dir) r := exec(l.ServerInfo, cmd, priv) if r.ExitStatus != 0 && r.ExitStatus != 1 { return xerrors.Errorf("Failed to find lock files") @@ -635,116 +636,62 @@ func (l *base) scanLibraries() (err error) { } // skip already exist - if _, ok := libFilemap[path]; ok { + if _, ok := found[path]; ok { continue } - var f LibFile + var contents []byte + var filemode os.FileMode + switch l.Distro.Family { case constant.ServerTypePseudo: fileinfo, err := os.Stat(path) if err != nil { - return xerrors.Errorf("Failed to get target file info. err: %w, filepath: %s", err, path) + l.log.Warnf("Failed to get target file info. err: %s, filepath: %s", err, path) + continue } - f.Filemode = fileinfo.Mode().Perm() - f.Contents, err = os.ReadFile(path) + filemode = fileinfo.Mode().Perm() + contents, err = os.ReadFile(path) if err != nil { - return xerrors.Errorf("Failed to read target file contents. err: %w, filepath: %s", err, path) + l.log.Warnf("Failed to read target file contents. err: %s, filepath: %s", err, path) + continue } default: + l.log.Debugf("Analyzing file: %s", path) cmd := fmt.Sprintf(`stat -c "%%a" %s`, path) - r := exec(l.ServerInfo, cmd, priv) + r := exec(l.ServerInfo, cmd, priv, logging.NewIODiscardLogger()) if !r.isSuccess() { - return xerrors.Errorf("Failed to get target file permission: %s, filepath: %s", r, path) + l.log.Warnf("Failed to get target file permission: %s, filepath: %s", r, path) + continue } permStr := fmt.Sprintf("0%s", strings.ReplaceAll(r.Stdout, "\n", "")) perm, err := strconv.ParseUint(permStr, 8, 32) if err != nil { - return xerrors.Errorf("Failed to parse permission string. err: %w, permission string: %s", err, permStr) + l.log.Warnf("Failed to parse permission string. err: %s, permission string: %s", err, permStr) + continue } - f.Filemode = os.FileMode(perm) + filemode = os.FileMode(perm) cmd = fmt.Sprintf("cat %s", path) - r = exec(l.ServerInfo, cmd, priv) + r = exec(l.ServerInfo, cmd, priv, logging.NewIODiscardLogger()) if !r.isSuccess() { - return xerrors.Errorf("Failed to get target file contents: %s, filepath: %s", r, path) + l.log.Warnf("Failed to get target file contents: %s, filepath: %s", r, path) + continue } - f.Contents = []byte(r.Stdout) + contents = []byte(r.Stdout) } - libFilemap[path] = f - } - - var libraryScanners []models.LibraryScanner - if libraryScanners, err = AnalyzeLibraries(context.Background(), libFilemap, l.ServerInfo.Mode.IsOffline()); err != nil { - return err + found[path] = true + var libraryScanners []models.LibraryScanner + if libraryScanners, err = AnalyzeLibrary(context.Background(), path, contents, filemode, l.ServerInfo.Mode.IsOffline()); err != nil { + return err + } + l.LibraryScanners = append(l.LibraryScanners, libraryScanners...) } - l.LibraryScanners = append(l.LibraryScanners, libraryScanners...) return nil } -// AnalyzeLibraries : detects libs defined in lockfile -func AnalyzeLibraries(ctx context.Context, libFilemap map[string]LibFile, isOffline bool) (libraryScanners []models.LibraryScanner, err error) { - // https://github.com/aquasecurity/trivy/blob/84677903a6fa1b707a32d0e8b2bffc23dde52afa/pkg/fanal/analyzer/const.go - disabledAnalyzers := []analyzer.Type{ - // ====== - // OS - // ====== - analyzer.TypeOSRelease, - analyzer.TypeAlpine, - analyzer.TypeAmazon, - analyzer.TypeCBLMariner, - analyzer.TypeDebian, - analyzer.TypePhoton, - analyzer.TypeCentOS, - analyzer.TypeRocky, - analyzer.TypeAlma, - analyzer.TypeFedora, - analyzer.TypeOracle, - analyzer.TypeRedHatBase, - analyzer.TypeSUSE, - analyzer.TypeUbuntu, - - // OS Package - analyzer.TypeApk, - analyzer.TypeDpkg, - analyzer.TypeDpkgLicense, - analyzer.TypeRpm, - analyzer.TypeRpmqa, - - // OS Package Repository - analyzer.TypeApkRepo, - - // ============ - // Image Config - // ============ - analyzer.TypeApkCommand, - - // ================= - // Structured Config - // ================= - analyzer.TypeYaml, - analyzer.TypeJSON, - analyzer.TypeDockerfile, - analyzer.TypeTerraform, - analyzer.TypeCloudFormation, - analyzer.TypeHelm, - - // ======== - // License - // ======== - analyzer.TypeLicenseFile, - - // ======== - // Secrets - // ======== - analyzer.TypeSecret, - - // ======= - // Red Hat - // ======= - analyzer.TypeRedHatContentManifestType, - analyzer.TypeRedHatDockerfileType, - } +// AnalyzeLibrary : detects library defined in artifact such as lockfile or jar +func AnalyzeLibrary(ctx context.Context, path string, contents []byte, filemode os.FileMode, isOffline bool) (libraryScanners []models.LibraryScanner, err error) { anal, err := analyzer.NewAnalyzerGroup(analyzer.AnalyzerOptions{ Group: analyzer.GroupBuiltin, DisabledAnalyzers: disabledAnalyzers, @@ -753,34 +700,94 @@ func AnalyzeLibraries(ctx context.Context, libFilemap map[string]LibFile, isOffl return nil, xerrors.Errorf("Failed to new analyzer group. err: %w", err) } - for path, f := range libFilemap { - var wg sync.WaitGroup - result := new(analyzer.AnalysisResult) - if err := anal.AnalyzeFile( - ctx, - &wg, - semaphore.NewWeighted(1), - result, - "", - path, - &DummyFileInfo{size: int64(len(f.Contents)), filemode: f.Filemode}, - func() (dio.ReadSeekCloserAt, error) { return dio.NopCloser(bytes.NewReader(f.Contents)), nil }, - nil, - analyzer.AnalysisOptions{Offline: isOffline}, - ); err != nil { - return nil, xerrors.Errorf("Failed to get libs. err: %w", err) - } - wg.Wait() - - libscan, err := convertLibWithScanner(result.Applications) - if err != nil { - return nil, xerrors.Errorf("Failed to convert libs. err: %w", err) - } - libraryScanners = append(libraryScanners, libscan...) + var wg sync.WaitGroup + result := new(analyzer.AnalysisResult) + if err := anal.AnalyzeFile( + ctx, + &wg, + semaphore.NewWeighted(1), + result, + "", + path, + &DummyFileInfo{size: int64(len(contents)), filemode: filemode}, + func() (dio.ReadSeekCloserAt, error) { return dio.NopCloser(bytes.NewReader(contents)), nil }, + nil, + analyzer.AnalysisOptions{Offline: isOffline}, + ); err != nil { + return nil, xerrors.Errorf("Failed to get libs. err: %w", err) + } + wg.Wait() + + libscan, err := convertLibWithScanner(result.Applications) + if err != nil { + return nil, xerrors.Errorf("Failed to convert libs. err: %w", err) } + libraryScanners = append(libraryScanners, libscan...) return libraryScanners, nil } +// https://github.com/aquasecurity/trivy/blob/84677903a6fa1b707a32d0e8b2bffc23dde52afa/pkg/fanal/analyzer/const.go +var disabledAnalyzers = []analyzer.Type{ + // ====== + // OS + // ====== + analyzer.TypeOSRelease, + analyzer.TypeAlpine, + analyzer.TypeAmazon, + analyzer.TypeCBLMariner, + analyzer.TypeDebian, + analyzer.TypePhoton, + analyzer.TypeCentOS, + analyzer.TypeRocky, + analyzer.TypeAlma, + analyzer.TypeFedora, + analyzer.TypeOracle, + analyzer.TypeRedHatBase, + analyzer.TypeSUSE, + analyzer.TypeUbuntu, + + // OS Package + analyzer.TypeApk, + analyzer.TypeDpkg, + analyzer.TypeDpkgLicense, + analyzer.TypeRpm, + analyzer.TypeRpmqa, + + // OS Package Repository + analyzer.TypeApkRepo, + + // ============ + // Image Config + // ============ + analyzer.TypeApkCommand, + + // ================= + // Structured Config + // ================= + analyzer.TypeYaml, + analyzer.TypeJSON, + analyzer.TypeDockerfile, + analyzer.TypeTerraform, + analyzer.TypeCloudFormation, + analyzer.TypeHelm, + + // ======== + // License + // ======== + analyzer.TypeLicenseFile, + + // ======== + // Secrets + // ======== + analyzer.TypeSecret, + + // ======= + // Red Hat + // ======= + analyzer.TypeRedHatContentManifestType, + analyzer.TypeRedHatDockerfileType, +} + // DummyFileInfo is a dummy struct for libscan type DummyFileInfo struct { size int64 diff --git a/scanner/debian.go b/scanner/debian.go index cb8cbe175a..ef293b05ea 100644 --- a/scanner/debian.go +++ b/scanner/debian.go @@ -1155,7 +1155,7 @@ func (o *debian) checkrestart() error { o.Packages[p.Name] = pack for j, proc := range p.NeedRestartProcs { - if proc.HasInit == false { + if !proc.HasInit { continue } packs[i].NeedRestartProcs[j].InitSystem = initName diff --git a/scanner/executil.go b/scanner/executil.go index 6e89fb069f..3d18fcf703 100644 --- a/scanner/executil.go +++ b/scanner/executil.go @@ -128,7 +128,6 @@ func parallelExec(fn func(osTypeInterface) error, timeoutSec ...int) { } } servers = successes - return } func exec(c config.ServerInfo, cmd string, sudo bool, log ...logging.Logger) (result execResult) { diff --git a/subcmds/discover.go b/subcmds/discover.go index 7dc45aee22..06cc6d4c37 100644 --- a/subcmds/discover.go +++ b/subcmds/discover.go @@ -216,6 +216,7 @@ host = "{{$ip}}" #type = "pseudo" #memo = "DB Server" #findLock = true +#findLockDirs = [ "/path/to/prject/lib" ] #lockfiles = ["/path/to/package-lock.json"] #cpeNames = [ "cpe:/a:rubyonrails:ruby_on_rails:4.2.1" ] #owaspDCXMLPath = "/path/to/dependency-check-report.xml"