Merge branch 'main' into dependabot/github_actions/actions/upload-art…

…ifact-4.0.0

.github/workflows/osv-scanner-pr.yml

@@ -16,4 +16,4 @@ permissions:
 
 jobs:
   scan-pr:
-    uses: "google/osv-scanner/.github/workflows/osv-scanner-reusable-pr.yml@main"
+    uses: "google/osv-scanner-action/.github/workflows/osv-scanner-reusable-pr.yml@v1.6.2-beta1"

.github/workflows/osv-scanner-scheduled.yml

@@ -15,4 +15,4 @@ permissions:
 
 jobs:
   scan-scheduled:
-    uses: "google/osv-scanner/.github/workflows/osv-scanner-reusable.yml@main"
+    uses: "google/osv-scanner-action/.github/workflows/osv-scanner-reusable.yml@v1.6.2-beta1"

.github/workflows/test.yml

@@ -15,7 +15,7 @@ jobs:
     - uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v5.0.0
       with:
         go-version: '1.21.0'
-    - uses: actions/setup-node@8f152de45cc393bb48ce5d89d36b731f54556e65 # v4.0.0
+    - uses: actions/setup-node@b39b52d1213e96004bfcb1c61a8a6fa8ab84f3e8 # v4.0.1
       with:
         node-version: 18
     - name: Install libpcap-dev

Makefile

@@ -98,7 +98,7 @@ push_prod_images: push_prod_sandboxes push_analysis_image push_scheduler_image
 # pass '-nopull' to scripts/run_analysis.sh
 #
 sync_%_sandbox:
-	sudo buildah pull docker-daemon:${REGISTRY}/${IMAGE_NAME}:$(TAG)
+	docker save ${REGISTRY}/${IMAGE_NAME}:$(TAG) | sudo podman load
 
 sync_dynamic_analysis_sandbox: IMAGE_NAME=dynamic-analysis
 sync_dynamic_analysis_sandbox: build_dynamic_analysis_sandbox

cmd/analyze/main.go

@@ -152,7 +152,7 @@ func dynamicAnalysis(ctx context.Context, pkg *pkgmanager.Pkg, resultStores *wor
 			"status", string(result.LastStatus))
 	}
 
-	if err := worker.SaveDynamicAnalysisData(ctx, pkg, resultStores, result.AnalysisData); err != nil {
+	if err := worker.SaveDynamicAnalysisData(ctx, pkg, resultStores, result.Data); err != nil {
 		slog.ErrorContext(ctx, "Upload error", "error", err)
 	}
 }

cmd/worker/main.go

@@ -168,7 +168,7 @@ func handleMessage(ctx context.Context, msg *pubsub.Message, packagesBucket *blo
 
 	result, dynamicAnalysisErr := worker.RunDynamicAnalysis(ctx, pkg, dynamicSandboxOpts, "")
 	if dynamicAnalysisErr == nil {
-		dynamicAnalysisErr = worker.SaveDynamicAnalysisData(ctx, pkg, resultStores, result.AnalysisData)
+		dynamicAnalysisErr = worker.SaveDynamicAnalysisData(ctx, pkg, resultStores, result.Data)
 	}
 
 	resultStores.AnalyzedPackageSaved = false

infra/cloudbuild/dynamic_loader/cloudbuild.yaml

@@ -0,0 +1,15 @@
+steps:
+- name: gcr.io/google.com/cloudsdktool/cloud-sdk
+  env:
+  - 'PROJECT_ID=ossf-malware-analysis'
+  - 'LOAD_DATASET=testing'  # TODO: change after testing is completed
+  - 'LOAD_TABLE_PREFIX=merge_'
+  - 'DEST_DATASET=testing'  # TODO: change after testing is completed
+  - 'DEST_TABLE=analysis'
+  - 'RESULT_BUCKET=gs://ossf-malware-analysis-results'
+  - 'SCHEMA_FILE=function/loader/dynamic-analysis-schema.json'
+  entrypoint: '/bin/bash'
+  args: ['scripts/bq_load.sh']
+timeout: 43200s  # 12 hours
+options:
+  logging: CLOUD_LOGGING_ONLY

internal/dynamicanalysis/analysis.go

@@ -21,7 +21,8 @@ const (
 type Result struct {
 	StraceSummary     analysisrun.StraceSummary
 	FileWritesSummary analysisrun.FileWritesSummary
-	// Ids that correlate to the name of the file that saves the actual write buffer contents. We save this separately so we don't need to dig through the FileWritesSummary later on.
+	// IDs that correlate to the name of the file that saves the actual write buffer contents.
+	// We save this separately so that we don't need to dig through the FileWritesSummary later on.
 	FileWriteBufferIds []string
 }
 

internal/staticanalysis/analyze.go

@@ -47,7 +47,7 @@ If staticanalysis.Parsing is not in the list of analysisTasks, jsParserConfig ma
 If an error occurs while traversing the extracted package directory tree, or an invalid
 task is requested, a nil result is returned along with the corresponding error object.
 */
-func AnalyzePackageFiles(ctx context.Context, extractDir string, jsParserConfig parsing.ParserConfig, analysisTasks []Task) (*Result, error) {
+func AnalyzePackageFiles(ctx context.Context, extractDir string, jsParserConfig parsing.ParserConfig, analysisTasks []Task) ([]SingleResult, error) {
 	runTask := map[Task]bool{}
 
 	for _, task := range analysisTasks {
@@ -89,7 +89,7 @@ func AnalyzePackageFiles(ctx context.Context, extractDir string, jsParserConfig
 
 	if runTask[Basic] {
 		slog.InfoContext(ctx, "run basic analysis")
-		basicData, err := basicdata.Analyze(ctx, paths, getPathInArchive)
+		basicData, err := basicdata.Analyze(ctx, paths, basicdata.FormatPaths(getPathInArchive))
 		if err != nil {
 			slog.ErrorContext(ctx, "static analysis basic data error", "error", err)
 		} else if len(basicData) != len(fileResults) {
@@ -133,5 +133,5 @@ func AnalyzePackageFiles(ctx context.Context, extractDir string, jsParserConfig
 		}
 	}
 
-	return &Result{Files: fileResults}, nil
+	return fileResults, nil
 }

internal/staticanalysis/analyze_test.go

@@ -34,12 +34,10 @@ var helloWorldJs = testFile{
 	lineLengths: valuecounts.Count([]int{18}),
 }
 
-func makeDesiredResult(files ...testFile) *Result {
-	result := Result{
-		Files: []SingleResult{},
-	}
-	for _, file := range files {
-		result.Files = append(result.Files, SingleResult{
+func makeDesiredResult(files ...testFile) []SingleResult {
+	result := make([]SingleResult, len(files))
+	for index, file := range files {
+		result[index] = SingleResult{
 			Filename: file.filename,
 			Basic: &basicdata.FileData{
 				DetectedType: file.fileType,
@@ -67,17 +65,17 @@ func makeDesiredResult(files ...testFile) *Result {
 				IPAddresses:           []string{},
 				URLs:                  []string{},
 			},
-		})
+		}
 	}
 
-	return &result
+	return result
 }
 
 func TestAnalyzePackageFiles(t *testing.T) {
 	tests := []struct {
 		name    string
 		files   []testFile
-		want    *Result
+		want    []SingleResult
 		wantErr bool
 	}{
 		{

internal/staticanalysis/basicdata/basic_data.go

@@ -39,32 +39,79 @@ func (bd FileData) String() string {
 	return strings.Join(parts, "\n")
 }
 
+// Option allows controlling the behaviour of Analyze with non-required arguments.
+type Option interface{ set(*analyzeConfig) }
+
+// option implements Option.
+type option func(*analyzeConfig)
+
+func (o option) set(config *analyzeConfig) { o(config) }
+
+// analyzeConfig stores all behaviour configuration for Analyze which is adjustable by Option.
+type analyzeConfig struct {
+	// withLineLengths enables line length analysis
+	withLineLengths bool
+	// formatPathFunc allows providing a custom transformation for file paths
+	// when logging errors. For example, removing a common path prefix.
+	formatPathFunc func(absPath string) string
+}
+
+func getDefaultAnalyzeConfig() analyzeConfig {
+	return analyzeConfig{
+		withLineLengths: true,
+		formatPathFunc:  func(absPath string) string { return absPath },
+	}
+}
+
+// SkipLineLengths disables collecting line length information during analysis, which is
+// useful when the input files are known not to be text files (e.g. a package tarball).
+func SkipLineLengths() Option {
+	return option(func(config *analyzeConfig) {
+		config.withLineLengths = false
+	})
+}
+
+// FormatPaths uses the given function to transform absolute file paths
+// before they are passed to logging.
+func FormatPaths(formatPathFunc func(absPath string) string) Option {
+	return option(func(config *analyzeConfig) {
+		config.formatPathFunc = formatPathFunc
+	})
+}
+
 /*
 Analyze collects basic file information for the specified files. Errors are logged
 rather than returned where possible, to maximise the amount of data collected.
-pathInArchive should return the relative path in the package archive, given an absolute
-path to a file in the package. The relative path is used for the result data.
+Pass instances of Option to control which information is collected.
 */
-func Analyze(ctx context.Context, paths []string, pathInArchive func(absolutePath string) string) ([]FileData, error) {
+func Analyze(ctx context.Context, paths []string, options ...Option) ([]FileData, error) {
 	if len(paths) == 0 {
 		return []FileData{}, nil
 	}
 
-	detectedTypes, err := detectFileTypes(ctx, paths)
-	haveDetectedTypes := true
+	config := getDefaultAnalyzeConfig()
+	for _, o := range options {
+		o.set(&config)
+	}
+
+	var detectedTypes []string
+	var haveDetectedTypes bool
+	types, err := detectFileTypes(ctx, paths)
+	haveDetectedTypes = true
 	if err != nil {
 		slog.ErrorContext(ctx, "failed to run file type detection", "error", err)
 		haveDetectedTypes = false
 	}
-	if len(detectedTypes) != len(paths) {
+	if len(types) != len(paths) {
 		slog.ErrorContext(ctx, fmt.Sprintf("detectFileTypes() returned %d results, expecting %d", len(detectedTypes), len(paths)))
 		haveDetectedTypes = false
 	}
+	detectedTypes = types
 
-	var result []FileData
+	result := make([]FileData, len(paths))
 
 	for index, filePath := range paths {
-		archivePath := pathInArchive(filePath)
+		formattedPath := config.formatPathFunc(filePath)
 		detectedType := ""
 		if haveDetectedTypes {
 			detectedType = detectedTypes[index]
@@ -73,31 +120,33 @@ func Analyze(ctx context.Context, paths []string, pathInArchive func(absolutePat
 		var fileSize int64
 		if fileInfo, err := os.Stat(filePath); err != nil {
 			fileSize = -1 // error value
-			slog.ErrorContext(ctx, "Error during stat file", "path", archivePath, "error", err)
+			slog.ErrorContext(ctx, "Error during stat file", "file", formattedPath, "error", err)
 		} else {
 			fileSize = fileInfo.Size()
 		}
 
 		var sha265Sum string
 		if hash, err := utils.SHA256Hash(filePath); err != nil {
-			slog.ErrorContext(ctx, "Error hashing file", "path", archivePath, "error", err)
+			slog.ErrorContext(ctx, "Error hashing file", "file", formattedPath, "error", err)
 		} else {
 			sha265Sum = hash
 		}
 
 		var lineLengths valuecounts.ValueCounts
-		if ll, err := linelengths.GetLineLengths(filePath, ""); err != nil {
-			slog.ErrorContext(ctx, "Error counting line lengths", "path", archivePath, "error", err)
-		} else {
-			lineLengths = valuecounts.Count(ll)
+		if config.withLineLengths {
+			if ll, err := linelengths.GetLineLengths(filePath, ""); err != nil {
+				slog.ErrorContext(ctx, "Error counting line lengths", "file", formattedPath, "error", err)
+			} else {
+				lineLengths = valuecounts.Count(ll)
+			}
 		}
 
-		result = append(result, FileData{
+		result[index] = FileData{
 			DetectedType: detectedType,
 			Size:         fileSize,
 			SHA256:       sha265Sum,
 			LineLengths:  lineLengths,
-		})
+		}
 	}
 
 	return result, nil

internal/staticanalysis/basicdata/basic_data_test.go

@@ -5,7 +5,6 @@ import (
 	"os"
 	"path/filepath"
 	"reflect"
-	"strings"
 	"testing"
 
 	"github.com/ossf/package-analysis/internal/utils"
@@ -72,11 +71,7 @@ func TestGetBasicData(t *testing.T) {
 				}
 			}
 
-			getArchivePath := func(absolutePath string) string {
-				return strings.TrimPrefix(absolutePath, testDir+string(os.PathSeparator))
-			}
-
-			got, err := Analyze(context.Background(), paths, getArchivePath)
+			got, err := Analyze(context.Background(), paths)
 			if (err != nil) != tt.wantErr {
 				t.Errorf("detectFileTypes() error = %v, wantErr %v", err, tt.wantErr)
 				return

internal/staticanalysis/parsing/package.json

@@ -3,7 +3,7 @@
   "version": "1.0.0",
   "type": "module",
   "dependencies": {
-    "@babel/parser": "^7.23.5",
-    "@babel/traverse": "^7.23.5"
+    "@babel/parser": "^7.23.9",
+    "@babel/traverse": "^7.23.9"
   }
 }