Merge branch 'guacsec:main' into guacdiff

.github/workflows/ci.yaml

@@ -36,7 +36,7 @@ jobs:
     name: CI for integration tests
     steps:
       - name: Checkout code
-        uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # tag=v3
+        uses: actions/checkout@1d96c772d19495a3b5c517cd2bc0cb401ea0529f # tag=v3
       - name: setup-go
         uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # tag=v3.2.1
         with:
@@ -65,7 +65,7 @@ jobs:
     name: CI for unit tests
     steps:
       - name: Checkout code
-        uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # tag=v3
+        uses: actions/checkout@1d96c772d19495a3b5c517cd2bc0cb401ea0529f # tag=v3
       - name: setup-go
         uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # tag=v3.2.1
         with:
@@ -84,7 +84,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout code
-        uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # tag=v3
+        uses: actions/checkout@1d96c772d19495a3b5c517cd2bc0cb401ea0529f # tag=v3
       - name: setup-go
         uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # tag=v3.2.1
         with:
@@ -100,7 +100,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout code
-        uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # tag=v3
+        uses: actions/checkout@1d96c772d19495a3b5c517cd2bc0cb401ea0529f # tag=v3
       - name: setup-go
         uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # tag=v5.0.0
         with:
@@ -119,7 +119,7 @@ jobs:
     name: E2E
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633
+    - uses: actions/checkout@1d96c772d19495a3b5c517cd2bc0cb401ea0529f
     - uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491
       with:
         go-version: '~1.21'
@@ -174,7 +174,7 @@ jobs:
         with:
           install-only: true
       - name: Checkout code
-        uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # tag=v3
+        uses: actions/checkout@1d96c772d19495a3b5c517cd2bc0cb401ea0529f # tag=v3
       - name: setup-go
         uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # tag=v3.2.1
         with:

.github/workflows/db-performance-test.yaml

@@ -50,9 +50,9 @@ jobs:
     name: performance test for backends DBs
     steps:
       - name: Checkout code
-        uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # tag=v3
+        uses: actions/checkout@1d96c772d19495a3b5c517cd2bc0cb401ea0529f # tag=v3
       - name: Checkout guac-data
-        uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
+        uses: actions/checkout@1d96c772d19495a3b5c517cd2bc0cb401ea0529f # v4.1.3
         with:
           repository: 'guacsec/guac-data'
           ref: 'main'

.github/workflows/nightly-release.yaml

@@ -40,7 +40,7 @@ jobs:
     name: trigger nightly build 
     steps:
       - name: Checkout code
-        uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # tag=v3
+        uses: actions/checkout@1d96c772d19495a3b5c517cd2bc0cb401ea0529f # tag=v3
 
       - name: Get GitHub App token
         uses: actions/create-github-app-token@7bfa3a4717ef143a604ee0a99d859b8886a96d00 # v1.9.3

.github/workflows/postmerge.yaml

@@ -26,7 +26,7 @@ jobs:
     name: CI for Integration Merge Test
     steps:
       - name: Checkout code
-        uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # tag=v3
+        uses: actions/checkout@1d96c772d19495a3b5c517cd2bc0cb401ea0529f # tag=v3
       - name: setup-go
         uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # tag=v3.2.1
         with:

.github/workflows/release.yaml

@@ -37,7 +37,7 @@ jobs:
       digest: ${{ steps.hash.outputs.digest }}
     steps:
       - name: Checkout
-        uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
+        uses: actions/checkout@1d96c772d19495a3b5c517cd2bc0cb401ea0529f # v4.1.3
         with:
           fetch-depth: 0
       - name: Login to GitHub Container Registry
@@ -108,7 +108,7 @@ jobs:
     if: startsWith(github.ref, 'refs/tags/')
     steps:
       - name: Checkout code
-        uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # tag=v3
+        uses: actions/checkout@1d96c772d19495a3b5c517cd2bc0cb401ea0529f # tag=v3
       - name: Login to GitHub Container Registry
         uses: docker/login-action@e92390c5fb421da1463c202d546fed0ec5c39f20 # v3.1.0
         with:
@@ -161,7 +161,7 @@ jobs:
     if: startsWith(github.ref, 'refs/tags/')
     steps:
       - name: Checkout code
-        uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # tag=v3
+        uses: actions/checkout@1d96c772d19495a3b5c517cd2bc0cb401ea0529f # tag=v3
       - name: Create and publish compose tarball
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
@@ -189,3 +189,14 @@ jobs:
           gh release upload ${{ github.ref_name }} guac-demo-compose.yaml
           rm guac-demo-compose.yaml
         shell: bash
+      - name: Modify and publish postgres compose yaml
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          #!/usr/bin/env bash
+          set -euo pipefail
+          cp container_files/guac-postgres-compose.yaml .
+          sed -i s/\$GUAC_IMAGE/ghcr.io\\/${{ github.repository_owner }}\\/guac:${{ github.ref_name }}/ guac-postgres-compose.yaml
+          gh release upload ${{ github.ref_name }} guac-postgres-compose.yaml
+          rm guac-postgres-compose.yaml
+        shell: bash

.github/workflows/reusable-local-build.yaml

@@ -28,7 +28,7 @@ jobs:
   build:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
+      - uses: actions/checkout@1d96c772d19495a3b5c517cd2bc0cb401ea0529f # v4.1.3
         with:
           repository: ${{ inputs.repository }}
           ref: ${{ inputs.ref }}

.github/workflows/scorecard.yml

@@ -32,7 +32,7 @@ jobs:
 
     steps:
       - name: "Checkout code"
-        uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
+        uses: actions/checkout@1d96c772d19495a3b5c517cd2bc0cb401ea0529f # v4.1.3
         with:
           persist-credentials: false
 

Makefile

@@ -78,9 +78,6 @@ proto:
 	protoc --go_out=. --go_opt=paths=source_relative \
 		--go-grpc_out=. --go-grpc_opt=paths=source_relative \
 		pkg/collectsub/collectsub/collectsub.proto
-	protoc --go_out=. --go_opt=paths=source_relative \
-	    --go-grpc_out=. --go-grpc_opt=paths=source_relative \
-		pkg/handler/collector/deps_dev/internal/api.proto
 
 # Remove temporary files
 .PHONY: clean

cmd/guaccollect/cmd/deps_dev.go

@@ -82,9 +82,9 @@ you have access to read and write to the respective blob store.`,
 			viper.GetBool("use-csub"),
 			viper.GetBool("service-poll"),
 			viper.GetBool("retrieve-dependencies"),
-			args,
 			viper.GetBool("enable-prometheus"),
 			viper.GetInt("prometheus-port"),
+			args,
 		)
 		if err != nil {
 			fmt.Printf("unable to validate flags: %v\n", err)
@@ -114,8 +114,18 @@ you have access to read and write to the respective blob store.`,
 	},
 }
 
-func validateDepsDevFlags(pubsubAddr string, blobAddr string, csubAddr string, csubTls bool, csubTlsSkipVerify bool, useCsub bool, poll bool, retrieveDependencies bool, args []string,
-	enablePrometheus bool, prometheusPort int,
+func validateDepsDevFlags(
+	pubsubAddr,
+	blobAddr,
+	csubAddr string,
+	csubTls,
+	csubTlsSkipVerify,
+	useCsub,
+	poll,
+	retrieveDependencies,
+	enablePrometheus bool,
+	prometheusPort int,
+	args []string,
 ) (depsDevOptions, error) {
 	var opts depsDevOptions
 	opts.pubsubAddr = pubsubAddr

cmd/guaccollect/cmd/files.go

@@ -26,7 +26,6 @@ import (
 	"time"
 
 	"github.com/guacsec/guac/pkg/blob"
-	"github.com/guacsec/guac/pkg/cli"
 	"github.com/guacsec/guac/pkg/emitter"
 	"github.com/guacsec/guac/pkg/handler/collector"
 	"github.com/guacsec/guac/pkg/handler/collector/file"
@@ -45,8 +44,6 @@ type filesOptions struct {
 	blobAddr string
 	// poll location
 	poll bool
-	// use blob URL for origin instead of source URL (useful if the blob store is persistent and we want to store the blob source location)
-	useBlobURL bool
 }
 
 var filesCmd = &cobra.Command{
@@ -73,7 +70,6 @@ you have access to read and write to the respective blob store.`,
 			viper.GetString("pubsub-addr"),
 			viper.GetString("blob-addr"),
 			viper.GetBool("service-poll"),
-			viper.GetBool("use-blob-url"),
 			args)
 		if err != nil {
 			fmt.Printf("unable to validate flags: %v\n", err)
@@ -85,7 +81,7 @@ you have access to read and write to the respective blob store.`,
 		logger := logging.FromContext(ctx)
 
 		// Register collector
-		fileCollector := file.NewFileCollector(ctx, opts.path, opts.poll, 30*time.Second, opts.useBlobURL)
+		fileCollector := file.NewFileCollector(ctx, opts.path, opts.poll, 30*time.Second)
 		err = collector.RegisterDocumentCollector(fileCollector, file.FileCollector)
 		if err != nil {
 			logger.Fatalf("unable to register file collector: %v", err)
@@ -95,13 +91,12 @@ you have access to read and write to the respective blob store.`,
 	},
 }
 
-func validateFilesFlags(pubsubAddr, blobAddr string, poll, useBlobURL bool, args []string) (filesOptions, error) {
+func validateFilesFlags(pubsubAddr, blobAddr string, poll bool, args []string) (filesOptions, error) {
 	var opts filesOptions
 
 	opts.pubsubAddr = pubsubAddr
 	opts.blobAddr = blobAddr
 	opts.poll = poll
-	opts.useBlobURL = useBlobURL
 
 	if len(args) != 1 {
 		return opts, fmt.Errorf("expected positional argument for file_path")
@@ -193,15 +188,5 @@ func initializeNATsandCollector(ctx context.Context, pubsubAddr string, blobAddr
 }
 
 func init() {
-	set, err := cli.BuildFlags([]string{"use-blob-url"})
-	if err != nil {
-		fmt.Fprintf(os.Stderr, "failed to setup flag: %v", err)
-		os.Exit(1)
-	}
-	filesCmd.PersistentFlags().AddFlagSet(set)
-	if err := viper.BindPFlags(filesCmd.PersistentFlags()); err != nil {
-		fmt.Fprintf(os.Stderr, "failed to bind flags: %v", err)
-		os.Exit(1)
-	}
 	rootCmd.AddCommand(filesCmd)
 }

cmd/guaccollect/cmd/gcs.go

@@ -1,11 +1,12 @@
 package cmd
 
 import (
-	"cloud.google.com/go/storage"
 	"context"
 	"fmt"
+	"os"
+
+	"cloud.google.com/go/storage"
 	"github.com/guacsec/guac/pkg/cli"
-	"github.com/guacsec/guac/pkg/collectsub/client"
 	csub_client "github.com/guacsec/guac/pkg/collectsub/client"
 	"github.com/guacsec/guac/pkg/handler/collector"
 	"github.com/guacsec/guac/pkg/handler/collector/gcs"
@@ -14,14 +15,12 @@ import (
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
 	"google.golang.org/api/option"
-	"os"
 )
 
 type gcsOptions struct {
 	pubSubAddr        string
 	blobAddr          string
-	graphqlEndpoint   string
-	csubClientOptions client.CsubClientOptions
+	csubClientOptions csub_client.CsubClientOptions
 	bucket            string
 }
 
@@ -39,11 +38,10 @@ var gcsCmd = &cobra.Command{
 		opts, err := validateGCSFlags(
 			viper.GetString("pubsub-addr"),
 			viper.GetString("blob-addr"),
-			viper.GetString("gql-addr"),
 			viper.GetString("csub-addr"),
+			viper.GetString(gcsCredentialsPathFlag),
 			viper.GetBool("csub-tls"),
 			viper.GetBool("csub-tls-skip-verify"),
-			viper.GetString(gcsCredentialsPathFlag),
 			args)
 		if err != nil {
 			fmt.Printf("unable to validate flags: %v\n", err)
@@ -66,7 +64,7 @@ var gcsCmd = &cobra.Command{
 			logger.Fatalf("creating client: %v", err)
 		}
 
-		// Register collector by providing a new GCS Client and bucket name
+		// Register collector
 		gcsCollector, err := gcs.NewGCSCollector(gcs.WithBucket(opts.bucket), gcs.WithClient(client))
 		if err != nil {
 			logger.Fatalf("unable to create gcs client: %v", err)
@@ -90,14 +88,21 @@ var gcsCmd = &cobra.Command{
 	},
 }
 
-func validateGCSFlags(pubSubAddr, blobAddr, gqlEndpoint string, csubAddr string, csubTls bool, csubTlsSkipVerify bool, credentialsPath string, args []string) (gcsOptions, error) {
+func validateGCSFlags(
+	pubSubAddr,
+	blobAddr,
+	csubAddr,
+	credentialsPath string,
+	csubTls,
+	csubTlsSkipVerify bool,
+	args []string,
+) (gcsOptions, error) {
 	opts := gcsOptions{
-		pubSubAddr:      pubSubAddr,
-		blobAddr:        blobAddr,
-		graphqlEndpoint: gqlEndpoint,
+		pubSubAddr: pubSubAddr,
+		blobAddr:   blobAddr,
 	}
 
-	csubOpts, err := client.ValidateCsubClientFlags(csubAddr, csubTls, csubTlsSkipVerify)
+	csubOpts, err := csub_client.ValidateCsubClientFlags(csubAddr, csubTls, csubTlsSkipVerify)
 	if err != nil {
 		return opts, fmt.Errorf("unable to validate csub client flags: %w", err)
 	}

cmd/guaccollect/cmd/github.go

@@ -18,13 +18,14 @@ package cmd
 import (
 	"context"
 	"fmt"
-	csubclient "github.com/guacsec/guac/pkg/collectsub/client"
-	"github.com/guacsec/guac/pkg/collectsub/datasource/csubsource"
-	"github.com/guacsec/guac/pkg/collectsub/datasource/inmemsource"
 	"os"
 	"strings"
 	"time"
 
+	csubclient "github.com/guacsec/guac/pkg/collectsub/client"
+	"github.com/guacsec/guac/pkg/collectsub/datasource/csubsource"
+	"github.com/guacsec/guac/pkg/collectsub/datasource/inmemsource"
+
 	"github.com/guacsec/guac/pkg/cli"
 
 	"github.com/guacsec/guac/internal/client/githubclient"
@@ -153,7 +154,19 @@ you have access to read and write to the respective blob store.`,
 	},
 }
 
-func validateGithubFlags(pubsubAddr, blobAddr, csubAddr, githubMode, sbomName, workflowFileName string, csubTls, csubTlsSkipVerify, useCsub, poll bool, args []string) (githubOptions, error) {
+func validateGithubFlags(
+	pubsubAddr,
+	blobAddr,
+	csubAddr,
+	githubMode,
+	sbomName,
+	workflowFileName string,
+	csubTls,
+	csubTlsSkipVerify,
+	useCsub,
+	poll bool,
+	args []string,
+) (githubOptions, error) {
 	var opts githubOptions
 	opts.pubsubAddr = pubsubAddr
 	opts.blobAddr = blobAddr

cmd/guaccollect/cmd/oci.go

@@ -95,7 +95,16 @@ you have access to read and write to the respective blob store.`,
 	},
 }
 
-func validateOCIFlags(pubsubAddr string, blobAddr string, csubAddr string, csubTls bool, csubTlsSkipVerify bool, useCsub bool, poll bool, args []string) (ociOptions, error) {
+func validateOCIFlags(
+	pubsubAddr,
+	blobAddr,
+	csubAddr string,
+	csubTls,
+	csubTlsSkipVerify,
+	useCsub,
+	poll bool,
+	args []string,
+) (ociOptions, error) {
 	var opts ociOptions
 	opts.pubsubAddr = pubsubAddr
 	opts.blobAddr = blobAddr