Merge branch 'guacsec:main' into guacdiff

guacsec · May 16, 2024 · 3473340 · 3473340
2 parents 1897a1a + 761d672
commit 3473340
Show file tree

Hide file tree

Showing 46 changed files with 1,509 additions and 746 deletions.
diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
@@ -36,15 +36,17 @@ jobs:
     name: CI for integration tests
     steps:
       - name: Checkout code
-        uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # tag=v3
+        uses: actions/checkout@44c2b7a8a4ea60a981eaca3cf939b5f4305c123b # tag=v3
       - name: setup-go
-        uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # tag=v3.2.1
+        uses: actions/setup-go@cdcb36043654635271a94b9a6d1392de5bb323a7 # tag=v3.2.1
         with:
           go-version: '1.21'
       - uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4.0.2
         with:
           path: ~/go/pkg/mod
           key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
+      - name: Install atlas
+        uses: ariga/setup-atlas@v0
       - name: Setup the project
         run: go mod download
       - name: Run backends
@@ -65,15 +67,17 @@ jobs:
     name: CI for unit tests
     steps:
       - name: Checkout code
-        uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # tag=v3
+        uses: actions/checkout@44c2b7a8a4ea60a981eaca3cf939b5f4305c123b # tag=v3
       - name: setup-go
-        uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # tag=v3.2.1
+        uses: actions/setup-go@cdcb36043654635271a94b9a6d1392de5bb323a7 # tag=v3.2.1
         with:
           go-version: '1.21'
       - uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4.0.2
         with:
           path: ~/go/pkg/mod
           key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
+      - name: Install atlas
+        uses: ariga/setup-atlas@v0
       - name: Setup the project
         run: go mod download
       - name: Run tests
@@ -84,11 +88,13 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout code
-        uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # tag=v3
+        uses: actions/checkout@44c2b7a8a4ea60a981eaca3cf939b5f4305c123b # tag=v3
       - name: setup-go
-        uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # tag=v3.2.1
+        uses: actions/setup-go@cdcb36043654635271a94b9a6d1392de5bb323a7 # tag=v3.2.1
         with:
           go-version: '1.21'
+      - name: Install atlas
+        uses: ariga/setup-atlas@v0
       - name: Install formatter
         run: go install golang.org/x/tools/cmd/goimports@latest
       - name: Check format
@@ -100,27 +106,31 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout code
-        uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # tag=v3
+        uses: actions/checkout@44c2b7a8a4ea60a981eaca3cf939b5f4305c123b # tag=v3
       - name: setup-go
-        uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # tag=v5.0.0
+        uses: actions/setup-go@cdcb36043654635271a94b9a6d1392de5bb323a7 # tag=v5.0.1
         with:
           go-version: '1.21'
+      - name: Install atlas
+        uses: ariga/setup-atlas@v0
       - name: golangci-lint
-        uses: golangci/golangci-lint-action@9d1e0624a798bb64f6c3cea93db47765312263dc # tag=v3.2.0
+        uses: golangci/golangci-lint-action@a4f60bb28d35aeee14e6880718e0c85ff1882e64 # tag=v3.2.0
         with:
           only-new-issues: true
-          skip-pkg-cache: true
+          skip-cache: true
       - name: Check markdown format
         run: make format
       - name: Check that all linted text is up to date
         run: make generated_up_to_date
+      - name: Run atlas Lint
+        run: make atlas-lint
 
   end-to-end:
     name: E2E
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b
-    - uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491
+    - uses: actions/checkout@44c2b7a8a4ea60a981eaca3cf939b5f4305c123b
+    - uses: actions/setup-go@cdcb36043654635271a94b9a6d1392de5bb323a7
       with:
         go-version: '~1.21'
     - uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d
@@ -174,9 +184,9 @@ jobs:
         with:
           install-only: true
       - name: Checkout code
-        uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # tag=v3
+        uses: actions/checkout@44c2b7a8a4ea60a981eaca3cf939b5f4305c123b # tag=v3
       - name: setup-go
-        uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # tag=v3.2.1
+        uses: actions/setup-go@cdcb36043654635271a94b9a6d1392de5bb323a7 # tag=v3.2.1
         with:
           go-version: '1.21'
       - uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4.0.2

diff --git a/.github/workflows/db-performance-test.yaml b/.github/workflows/db-performance-test.yaml
@@ -50,9 +50,9 @@ jobs:
     name: performance test for backends DBs
     steps:
       - name: Checkout code
-        uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # tag=v3
+        uses: actions/checkout@44c2b7a8a4ea60a981eaca3cf939b5f4305c123b # tag=v3
       - name: Checkout guac-data
-        uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4
+        uses: actions/checkout@44c2b7a8a4ea60a981eaca3cf939b5f4305c123b # v4.1.5
         with:
           repository: 'guacsec/guac-data'
           ref: 'main'

diff --git a/.github/workflows/nightly-release.yaml b/.github/workflows/nightly-release.yaml
@@ -40,10 +40,10 @@ jobs:
     name: trigger nightly build 
     steps:
       - name: Checkout code
-        uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # tag=v3
+        uses: actions/checkout@44c2b7a8a4ea60a981eaca3cf939b5f4305c123b # tag=v3
 
       - name: Get GitHub App token
-        uses: actions/create-github-app-token@7bfa3a4717ef143a604ee0a99d859b8886a96d00 # v1.9.3
+        uses: actions/create-github-app-token@a0de6af83968303c8c955486bf9739a57d23c7f1 # v1.10.0
         id: app-token
         with:
           app_id: ${{ secrets.GH_APP_ID }}

diff --git a/.github/workflows/postmerge.yaml b/.github/workflows/postmerge.yaml
@@ -26,9 +26,9 @@ jobs:
     name: CI for Integration Merge Test
     steps:
       - name: Checkout code
-        uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # tag=v3
+        uses: actions/checkout@44c2b7a8a4ea60a981eaca3cf939b5f4305c123b # tag=v3
       - name: setup-go
-        uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # tag=v3.2.1
+        uses: actions/setup-go@cdcb36043654635271a94b9a6d1392de5bb323a7 # tag=v3.2.1
         with:
           go-version: '1.21'
       - uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4.0.2
@@ -37,6 +37,8 @@ jobs:
           key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
       - name: Setup the project
         run: go mod download
+      - name: Install atlas
+        uses: ariga/setup-atlas@v0
       - name: Run integration merge test
         env:
             GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml
@@ -37,7 +37,7 @@ jobs:
       digest: ${{ steps.hash.outputs.digest }}
     steps:
       - name: Checkout
-        uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4
+        uses: actions/checkout@44c2b7a8a4ea60a981eaca3cf939b5f4305c123b # v4.1.5
         with:
           fetch-depth: 0
       - name: Login to GitHub Container Registry
@@ -47,7 +47,7 @@ jobs:
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
       - name: Set up Go
-        uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v5.0.0
+        uses: actions/setup-go@cdcb36043654635271a94b9a6d1392de5bb323a7 # v5.0.1
         with:
           go-version: '1.21'
       - name: Install cosign
@@ -108,15 +108,15 @@ jobs:
     if: startsWith(github.ref, 'refs/tags/')
     steps:
       - name: Checkout code
-        uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # tag=v3
+        uses: actions/checkout@44c2b7a8a4ea60a981eaca3cf939b5f4305c123b # tag=v3
       - name: Login to GitHub Container Registry
         uses: docker/login-action@e92390c5fb421da1463c202d546fed0ec5c39f20 # v3.1.0
         with:
           registry: ghcr.io
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
       - name: Run Trivy in fs mode to generate SBOM
-        uses: aquasecurity/trivy-action@d710430a6722f083d3b36b8339ff66b32f22ee55 # master
+        uses: aquasecurity/trivy-action@b2933f565dbc598b29947660e66259e3c7bc8561 # master
         with:
           scan-type: 'fs'
           format: 'spdx-json'
@@ -161,7 +161,7 @@ jobs:
     if: startsWith(github.ref, 'refs/tags/')
     steps:
       - name: Checkout code
-        uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # tag=v3
+        uses: actions/checkout@44c2b7a8a4ea60a981eaca3cf939b5f4305c123b # tag=v3
       - name: Create and publish compose tarball
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

diff --git a/.github/workflows/reusable-local-build.yaml b/.github/workflows/reusable-local-build.yaml
@@ -28,12 +28,12 @@ jobs:
   build:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4
+      - uses: actions/checkout@44c2b7a8a4ea60a981eaca3cf939b5f4305c123b # v4.1.5
         with:
           repository: ${{ inputs.repository }}
           ref: ${{ inputs.ref }}
       - name: Set up Go
-        uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v5.0.0
+        uses: actions/setup-go@cdcb36043654635271a94b9a6d1392de5bb323a7 # v5.0.1
         with:
           go-version: 'stable'
       - name: Install GoReleaser

diff --git a/.github/workflows/scorecard.yml b/.github/workflows/scorecard.yml
@@ -32,12 +32,12 @@ jobs:
 
     steps:
       - name: "Checkout code"
-        uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4
+        uses: actions/checkout@44c2b7a8a4ea60a981eaca3cf939b5f4305c123b # v4.1.5
         with:
           persist-credentials: false
 
       - name: "Run analysis"
-        uses: ossf/scorecard-action@0864cf19026789058feabb7e87baa5f140aac736 # v2.3.1
+        uses: ossf/scorecard-action@dc50aa9510b46c811795eb24b2f1ba02a914e534 # v2.3.3
         with:
           results_file: results.sarif
           results_format: sarif

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
@@ -48,7 +48,7 @@ Since GUAC is a complex project, there are 5 topic areas of interest:
 
 - Ingestion: ingest software security metadata. Needs to write parsers for
   documents, maintain ingestion logic, write new collectors, etc.
-- GraphQL: define the GraphQL interface used between ingestion pipeline and
+- API: define the GraphQL/REST interface used between ingestion pipeline and
   backend, and between backend and GUAC-based applications and front-ends.
 - Backends: define efficient code to interface with database backends (Neo4j,
   etc.).
@@ -140,6 +140,14 @@ GUAC contributors are happy to help you advance along the contributor ladder!
 
 ### Reviewers list
 
-| Reviewer                            | Area                                                                                                                       | Vote                                                       |
-|-------------------------------------|----------------------------------------------------------------------------------------------------------------------------|------------------------------------------------------------|
+| Reviewer | Area | Vote |
+|-|-|-|
+| [dejanb](https://github.com/dejanb) | Ingestion | [issues/1885](https://github.com/guacsec/guac/issues/1885) |
+| [mdeicas](https://github.com/mdeicas) | API | [issues/1885](https://github.com/guacsec/guac/issues/1885) |
+
+
+### Owners list
+
+| Owner | Area | Vote |
+|-|-|-|
 | [mrizzi](https://github.com/mrizzi) | [Backends (ent)](https://github.com/guacsec/guac/tree/4012842fab5d738f9bebf03f0cb44fc7ce39438b/pkg/assembler/backends/ent) | [issues/1310](https://github.com/guacsec/guac/issues/1310) |
diff --git a/Makefile b/Makefile
@@ -79,6 +79,23 @@ proto:
 		--go-grpc_out=. --go-grpc_opt=paths=source_relative \
 		pkg/collectsub/collectsub/collectsub.proto
 
+# Run atlas to generate ent migration diff for postgres
+.PHONY: atlas-diff
+atlas-diff: check-atlas-tool-check
+	atlas migrate diff ent_diff \
+  	  --dir "file://pkg/assembler/backends/ent/migrate/migrations" \
+  	  --to "ent://pkg/assembler/backends/ent/schema" \
+	  --dev-url "docker://postgres/15/test?search_path=public"
+
+
+# Run atlas lint to validate and analayze the contents of the migration
+.PHONY: atlas-lint
+atlas-lint: check-atlas-tool-check
+	atlas migrate lint \
+  	  --dir "file://pkg/assembler/backends/ent/migrate/migrations" \
+	  --dev-url "docker://postgres/15/test?search_path=public" \
+	  --latest=1
+
 # Remove temporary files
 .PHONY: clean
 clean:
@@ -100,7 +117,7 @@ fmt-md:
 
 # generate code from autogen tools (gqlgen, genqlclient, mockgen, ent)
 .PHONY: generate
-generate:
+generate: atlas-diff
 	go generate ./...
 
 # build bins for goos/goarch of current host
@@ -283,6 +300,15 @@ check-goreleaser-tool-check:
 		exit 1; \
 	fi
 
+# Check if atlas is installed
+.PHONY: check-atlas-tool-check
+check-atlas-tool-check:
+	@if ! command -v atlas >/dev/null 2>&1; then \
+		echo "atlas is not installed. Please install atlas (https://atlasgo.io/getting-started#installation) and try again."; \
+		exit 1; \
+	fi
+
+
 # Check that all the tools are installed.
 .PHONY: check-tools
-check-tools: check-docker-tool-check check-docker-buildx-tool-check check-docker-compose-tool-check check-protoc-tool-check check-golangci-lint-tool-check check-mockgen-tool-check check-goreleaser-tool-check
+check-tools: check-docker-tool-check check-docker-buildx-tool-check check-docker-compose-tool-check check-protoc-tool-check check-golangci-lint-tool-check check-mockgen-tool-check check-goreleaser-tool-check check-atlas-tool-check
diff --git a/cmd/guaccollect/cmd/s3.go b/cmd/guaccollect/cmd/s3.go
@@ -22,6 +22,7 @@ type s3Options struct {
 	blobAddr          string                        // address for the blob store to connect to
 	s3url             string                        // base url of the s3 to collect from
 	s3bucket          string                        // name of bucket to collect from
+	s3path            string                        // path to s3 folder with documents to collect
 	s3item            string                        // s3 item (only for non-polling behaviour)
 	region            string                        // AWS region, for s3/sqs configuration (defaults to us-east-1)
 	queues            string                        // comma-separated list of queues/topics (only for polling behaviour)
@@ -34,7 +35,9 @@ type s3Options struct {
 var s3Cmd = &cobra.Command{
 	Use:   "s3 [flags]",
 	Short: "takes SBOMs and attestations from S3 compatible bucket and injects them to GUAC graph",
-	Long: `S3 collector can download one item from the storage, the whole bucket or listen to storage events using sqs/kafka (poll) and download the files as they are uploaded.
+	Long: `
+guaccollect S3 collector can download one item from the storage, all items from a folder, a whole bucket
+or listen to storage events using sqs/kafka (poll) and download the files as they are uploaded.
 Make sure that access credentials variables are properly set.`,
 	Example: `Create example bucket:
 
@@ -48,8 +51,14 @@ $ export AWS_SECRET_ACCESS_KEY=zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG
 
 Ingest:
 
-$ guacone collect s3 --s3-url https://play.min.io --s3-bucket guac-test
-$ guacone collect s3 --s3-url play.min.io --s3-bucket guac-test --s3-item alpine-cyclonedx.json
+$ guaccollect s3 --s3-url https://play.min.io --s3-bucket guac-test
+$ guaccollect s3 --s3-url play.min.io --s3-bucket guac-test --s3-item alpine-cyclonedx.json
+
+Ingest from AWS using default url:
+
+$ guaccollect s3 --s3-bucket guac-test --s3-region eu-north-1
+$ guaccollect s3 --s3-bucket guac-test --s3-region eu-north-1 --s3-path sboms/
+
 
 For the polling option, you need to define event bus endpoint for bucket notifications:
 
@@ -66,6 +75,7 @@ $ guacone collect s3 --s3-url http://localhost:9000 --s3-bucket guac-test --poll
 			viper.GetString("csub-addr"),
 			viper.GetString("s3-url"),
 			viper.GetString("s3-bucket"),
+			viper.GetString("s3-path"),
 			viper.GetString("s3-region"),
 			viper.GetString("s3-item"),
 			viper.GetString("s3-mp"),
@@ -88,6 +98,7 @@ $ guacone collect s3 --s3-url http://localhost:9000 --s3-bucket guac-test --poll
 			S3Url:                   s3Opts.s3url,
 			S3Bucket:                s3Opts.s3bucket,
 			S3Region:                s3Opts.region,
+			S3Path:                  s3Opts.s3path,
 			S3Item:                  s3Opts.s3item,
 			MessageProvider:         s3Opts.mp,
 			MessageProviderEndpoint: s3Opts.mpEndpoint,
@@ -117,6 +128,7 @@ func validateS3Opts(
 	csubAddr,
 	s3url,
 	s3bucket,
+	s3path,
 	region,
 	s3item,
 	mp,
@@ -153,6 +165,7 @@ func validateS3Opts(
 		blobAddr:          blobAddr,
 		s3url:             s3url,
 		s3bucket:          s3bucket,
+		s3path:            s3path,
 		s3item:            s3item,
 		region:            region,
 		queues:            queues,
@@ -166,7 +179,7 @@ func validateS3Opts(
 }
 
 func init() {
-	set, err := cli.BuildFlags([]string{"s3-url", "s3-bucket", "s3-item", "s3-region", "s3-queues", "s3-mp", "s3-mp-endpoint"})
+	set, err := cli.BuildFlags([]string{"s3-url", "s3-bucket", "s3-path", "s3-item", "s3-region", "s3-queues", "s3-mp", "s3-mp-endpoint"})
 	if err != nil {
 		fmt.Fprintf(os.Stderr, "failed to setup flag: %v", err)
 		os.Exit(1)