test: run linux package tests nightly against latest contrib main [ne…

…w] (#762)

* previous commits

* release pipeline fixes

* fixes after rebasing

Signed-off-by: Moritz Wiesinger <moritz.wiesinger@dynatrace.com>

* trigger build

* downgrade ubuntu version back to 22.04

Signed-off-by: Moritz Wiesinger <moritz.wiesinger@dynatrace.com>

* try to restore state before rebase

Signed-off-by: Moritz Wiesinger <moritz.wiesinger@dynatrace.com>

* run tests to ensure that everything works

Signed-off-by: Moritz Wiesinger <moritz.wiesinger@dynatrace.com>

* revert after successful testing

Signed-off-by: Moritz Wiesinger <moritz.wiesinger@dynatrace.com>

---------

Signed-off-by: Moritz Wiesinger <moritz.wiesinger@dynatrace.com>

.github/workflows/base-ci-goreleaser.yaml

@@ -12,6 +12,11 @@ on:
       goarch:
         required: true
         type: string
+      nightly:
+        required: false
+        type: boolean
+        default: false
+        description: "Set to true to fetch latest otelcol-contrib main branch version instead of building the version in this repo"
 
 env:
   # renovate: datasource=github-tags depName=goreleaser-pro packageName=goreleaser/goreleaser-pro
@@ -64,9 +69,75 @@ jobs:
           go-version: '1.23'
           check-latest: true
 
-      - name: Generate the sources
+      - name: Create artifacts directory to store build artifacts
+        if: inputs.distribution == 'otelcol-contrib'
+        run: mkdir -p distributions/otelcol-contrib/artifacts
+
+      - name: "[Nightly] Get latest finished run ID from contrib repo build-and-test"
+        id: get-run-id
+        if: inputs.distribution == 'otelcol-contrib' && inputs.nightly == true && matrix.GOARCH == 'amd64' && matrix.GOOS == 'linux'
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          run_id=$(gh run list \
+            --branch main \
+            --workflow build-and-test \
+            --repo open-telemetry/opentelemetry-collector-contrib \
+            --limit 1 \
+            --status success \
+            --json databaseId \
+            --jq '.[0].databaseId' \
+          )
+          echo "Found run ID: $run_id"
+          echo "run_id=$run_id" >> "$GITHUB_OUTPUT"
+
+      - name: "[Nightly] Create sub-directory for otelcol-contrib nightly build"
+        if: inputs.distribution == 'otelcol-contrib' && inputs.nightly == true && matrix.GOARCH == 'amd64' && matrix.GOOS == 'linux'
+        run: mkdir -p distributions/otelcol-contrib/artifacts/otelcol-contrib_linux_amd64_v1
+
+      - name: "[Nightly] Download built otelcol-contrib artifact from contrib repo"
+        if: inputs.distribution == 'otelcol-contrib' && inputs.nightly == true && matrix.GOARCH == 'amd64' && matrix.GOOS == 'linux'
+        uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
+        with:
+          name: collector-binaries-linux-amd64
+          repository: open-telemetry/opentelemetry-collector-contrib
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          run-id: ${{ steps.get-run-id.outputs.run_id }}
+
+      - name: "[Nightly] Move downloaded artifact"
+        if: inputs.distribution == 'otelcol-contrib' && inputs.nightly == true && matrix.GOARCH == 'amd64' && matrix.GOOS == 'linux'
+        run: mv otelcontribcol_linux_amd64 distributions/otelcol-contrib/artifacts/otelcol-contrib_linux_amd64_v1/otelcol-contrib
+
+      - name: Generate the sources for ${{ inputs.distribution }}
+        if: inputs.nightly != true
+        env:
+          DISTRIBUTIONS: ${{ inputs.distribution }}
         run: make generate-sources
 
+        # otelcol-contrib is built in a separate stage
+      - name: Build ${{ inputs.distribution }}
+        if: inputs.distribution == 'otelcol-contrib' && inputs.nightly != true
+        uses: goreleaser/goreleaser-action@9ed2f89a662bf1735a48bc8557fd212fa902bebf # v6.1.0
+        with:
+          distribution: goreleaser-pro
+          version: ${{ env.GORELEASER_PRO_VERSION }}
+          workdir: distributions/otelcol-contrib
+          args: --snapshot --clean --timeout 2h --split --config .goreleaser-build.yaml
+        env:
+          GOOS: ${{ matrix.GOOS }}
+          GOARCH: ${{ matrix.GOARCH }}
+          GOARM: 7 # Default is 6
+          GOAMD64: v1
+          GORELEASER_KEY: ${{ secrets.GORELEASER_KEY }}
+
+      - name: Move built artifacts
+        if: inputs.distribution == 'otelcol-contrib' && inputs.nightly != true
+        run: mv distributions/otelcol-contrib/dist/**/* distributions/otelcol-contrib/artifacts/
+
+      - name: Show built or downloaded content
+        if: inputs.distribution == 'otelcol-contrib'
+        run: ls -laR distributions/otelcol-contrib/artifacts
+
       - name: Run GoReleaser for ${{ inputs.distribution }}
         uses: goreleaser/goreleaser-action@9ed2f89a662bf1735a48bc8557fd212fa902bebf # v6.1.0
         with:

.github/workflows/base-release.yaml

@@ -84,6 +84,36 @@ jobs:
         run: |
           echo "sha_short=$(git rev-parse --short HEAD)" >> $GITHUB_ENV
 
+      - name: Create artifacts directory to store build artifacts
+        if: inputs.distribution == 'otelcol-contrib'
+        run: mkdir -p distributions/otelcol-contrib/artifacts
+
+        # otelcol-contrib is built in a separate stage
+      - name: Build ${{ inputs.distribution }}
+        if: inputs.distribution == 'otelcol-contrib'
+        uses: goreleaser/goreleaser-action@9ed2f89a662bf1735a48bc8557fd212fa902bebf # v6.1.0
+        with:
+          distribution: goreleaser-pro
+          version: ${{ env.GORELEASER_PRO_VERSION }}
+          workdir: distributions/${{ inputs.distribution }}
+          args: release --clean --split --timeout 2h --config .goreleaser-build.yaml --release-header-tmpl=../../.github/release-template.md
+        env:
+          GOOS: ${{ matrix.GOOS }}
+          GOARCH: ${{ matrix.GOARCH }}
+          GOARM: 7 # Default is 6
+          GOAMD64: v1
+          GORELEASER_KEY: ${{ secrets.GORELEASER_KEY }}
+          GITHUB_TOKEN: ${{ secrets.GH_PAT }}
+          COSIGN_YES: true
+
+      - name: Move built artifacts
+        if: inputs.distribution == 'otelcol-contrib'
+        run: mv distributions/otelcol-contrib/dist/**/* distributions/otelcol-contrib/artifacts/
+
+      - name: Show built or downloaded content
+        if: inputs.distribution == 'otelcol-contrib'
+        run: ls -laR distributions/otelcol-contrib/artifacts
+
       - uses: goreleaser/goreleaser-action@9ed2f89a662bf1735a48bc8557fd212fa902bebf # v6.1.0
         with:
           distribution: goreleaser-pro

.github/workflows/ci-goreleaser-contrib.yaml

@@ -36,7 +36,7 @@ jobs:
   package-tests:
      name: Package tests
      needs: check-goreleaser
-     uses: ./.github/workflows/package-tests.yaml
+     uses: ./.github/workflows/base-package-tests.yaml
      with:
        distribution: otelcol-contrib
        type: '[ "deb", "rpm" ]'

.github/workflows/ci-goreleaser-core.yaml

@@ -36,7 +36,7 @@ jobs:
   package-tests:
      name: Package tests
      needs: check-goreleaser
-     uses: ./.github/workflows/package-tests.yaml
+     uses: ./.github/workflows/base-package-tests.yaml
      with:
        distribution: otelcol
        type: '[ "deb", "rpm" ]'

.github/workflows/package-test.yaml

@@ -0,0 +1,24 @@
+name: Package Tests - Contrib
+
+on:
+  schedule:
+    - cron: "0 2 * * *" # every day at 2am UTC
+
+jobs:
+  check-goreleaser:
+    name: Build - Contrib - GoReleaser
+    uses: ./.github/workflows/base-ci-goreleaser.yaml
+    with:
+      distribution: otelcol-contrib
+      goos: '[ "linux" ]'
+      goarch: '[ "amd64" ]'
+      nightly: true
+    secrets: inherit
+
+  package-tests:
+    name: Linux Package tests
+    needs: check-goreleaser
+    uses: ./.github/workflows/base-package-tests.yaml
+    with:
+      distribution: otelcol-contrib
+      type: '[ "deb", "rpm" ]'

cmd/goreleaser/internal/configure.go

@@ -42,22 +42,32 @@ const (
 var (
 	ImagePrefixes      = []string{DockerHub, GHCR}
 	Architectures      = []string{"386", "amd64", "arm", "arm64", "ppc64le", "s390x"}
-	ArmVersions        = []string{"7"}
 	DefaultConfigDists = map[string]bool{CoreDistro: true, ContribDistro: true}
 	MSIWindowsDists    = map[string]bool{CoreDistro: true, ContribDistro: true, OTLPDistro: true}
 	K8sDockerSkipArchs = map[string]bool{"arm": true, "386": true}
 	K8sGoos            = []string{"linux"}
 	K8sArchs           = []string{"amd64", "arm64", "ppc64le", "s390x"}
 )
 
-func Generate(dist string) config.Project {
+func GenerateContribBuildOnly(dist string, buildOrRest bool) config.Project {
+	return config.Project{
+		ProjectName: "opentelemetry-collector-releases",
+		Builds:      Builds(dist, buildOrRest),
+		Version:     2,
+		Monorepo: config.Monorepo{
+			TagPrefix: "v",
+		},
+	}
+}
+
+func Generate(dist string, buildOrRest bool) config.Project {
 	return config.Project{
 		ProjectName: "opentelemetry-collector-releases",
 		Checksum: config.Checksum{
 			NameTemplate: fmt.Sprintf("{{ .ProjectName }}_%v_checksums.txt", dist),
 		},
 		Env:             []string{"COSIGN_YES=true"},
-		Builds:          Builds(dist),
+		Builds:          Builds(dist, buildOrRest),
 		Archives:        Archives(dist),
 		MSI:             WinPackages(dist),
 		NFPMs:           Packages(dist),
@@ -73,37 +83,41 @@ func Generate(dist string) config.Project {
 	}
 }
 
-func Builds(dist string) []config.Build {
+func Builds(dist string, buildOrRest bool) []config.Build {
 	return []config.Build{
-		Build(dist),
+		Build(dist, buildOrRest),
 	}
 }
 
 // Build configures a goreleaser build.
 // https://goreleaser.com/customization/build/
-func Build(dist string) config.Build {
-	var goos []string
-	var archs []string
-	var ignore []config.IgnoredBuild
-	var armVersions []string
+func Build(dist string, buildOrRest bool) config.Build {
+	goos := []string{"darwin", "linux", "windows"}
+	archs := Architectures
+
+	if dist == ContribDistro && !buildOrRest {
+		// only return build config for contrib build file
+		return config.Build{
+			ID:      dist,
+			Builder: "prebuilt",
+			PreBuilt: config.PreBuiltOptions{
+				Path: "artifacts/otelcol-contrib_{{ .Target }}" +
+					"/otelcol-contrib{{- if eq .Os \"windows\" }}.exe{{ end }}",
+			},
+			Goos:   goos,
+			Goarch: archs,
+			Goarm:  ArmVersions(dist),
+			Dir:    "_build",
+			Binary: dist,
+			Ignore: IgnoreBuildCombinations(dist),
+		}
+	}
+
 	if dist == K8sDistro {
 		goos = K8sGoos
 		archs = K8sArchs
-		ignore = make([]config.IgnoredBuild, 0)
-		armVersions = make([]string, 0)
-	} else {
-		goos = []string{"darwin", "linux", "windows"}
-		archs = Architectures
-		ignore = []config.IgnoredBuild{
-			{Goos: "darwin", Goarch: "386"},
-			{Goos: "darwin", Goarch: "arm"},
-			{Goos: "darwin", Goarch: "s390x"},
-			{Goos: "windows", Goarch: "arm"},
-			{Goos: "windows", Goarch: "arm64"},
-			{Goos: "windows", Goarch: "s390x"},
-		}
-		armVersions = ArmVersions
 	}
+
 	return config.Build{
 		ID:     dist,
 		Dir:    "_build",
@@ -115,12 +129,33 @@ func Build(dist string) config.Build {
 		},
 		Goos:   goos,
 		Goarch: archs,
-		Goarm:  armVersions,
-		Ignore: ignore,
+		Goarm:  ArmVersions(dist),
+		Ignore: IgnoreBuildCombinations(dist),
 	}
 }
 
-func Archives(dist string) (r []config.Archive) {
+func IgnoreBuildCombinations(dist string) []config.IgnoredBuild {
+	if dist == K8sDistro {
+		return nil
+	}
+	return []config.IgnoredBuild{
+		{Goos: "darwin", Goarch: "386"},
+		{Goos: "darwin", Goarch: "arm"},
+		{Goos: "darwin", Goarch: "s390x"},
+		{Goos: "windows", Goarch: "arm"},
+		{Goos: "windows", Goarch: "arm64"},
+		{Goos: "windows", Goarch: "s390x"},
+	}
+}
+
+func ArmVersions(dist string) []string {
+	if dist == K8sDistro {
+		return nil
+	}
+	return []string{"7"}
+}
+
+func Archives(dist string) []config.Archive {
 	return []config.Archive{
 		Archive(dist),
 	}
@@ -138,7 +173,7 @@ func Archive(dist string) config.Archive {
 
 func WinPackages(dist string) []config.MSI {
 	if _, ok := MSIWindowsDists[dist]; !ok {
-		return []config.MSI{}
+		return nil
 	}
 	return []config.MSI{
 		WinPackage(dist),
@@ -160,9 +195,9 @@ func WinPackage(dist string) config.MSI {
 	}
 }
 
-func Packages(dist string) (r []config.NFPM) {
+func Packages(dist string) []config.NFPM {
 	if dist == K8sDistro {
-		return []config.NFPM{}
+		return nil
 	}
 	return []config.NFPM{
 		Package(dist),
@@ -191,21 +226,17 @@ func Package(dist string) config.NFPM {
 		})
 	}
 	return config.NFPM{
-		ID:      dist,
-		Builds:  []string{dist},
-		Formats: []string{"deb", "rpm"},
-
+		ID:          dist,
+		Builds:      []string{dist},
+		Formats:     []string{"deb", "rpm"},
 		License:     "Apache 2.0",
 		Description: fmt.Sprintf("OpenTelemetry Collector - %s", dist),
 		Maintainer:  "The OpenTelemetry Collector maintainers <cncf-opentelemetry-maintainers@lists.cncf.io>",
 		Overrides: map[string]config.NFPMOverridables{
 			"rpm": {
-				Dependencies: []string{
-					"/bin/sh",
-				},
+				Dependencies: []string{"/bin/sh"},
 			},
 		},
-
 		NFPMOverridables: config.NFPMOverridables{
 			PackageName: dist,
 			Scripts: config.NFPMScripts{
@@ -219,16 +250,14 @@ func Package(dist string) config.NFPM {
 }
 
 func DockerImages(dist string) []config.Docker {
-	r := make([]config.Docker, 0)
+	var r []config.Docker
 	for _, arch := range Architectures {
-		if dist == K8sDistro {
-			if _, ok := K8sDockerSkipArchs[arch]; ok {
-				continue
-			}
+		if dist == K8sDistro && K8sDockerSkipArchs[arch] {
+			continue
 		}
 		switch arch {
 		case ArmArch:
-			for _, vers := range ArmVersions {
+			for _, vers := range ArmVersions(dist) {
 				r = append(r, DockerImage(dist, arch, vers))
 			}
 		default:
@@ -302,7 +331,7 @@ func DockerManifest(prefix, version, dist string) config.DockerManifest {
 		}
 		switch arch {
 		case ArmArch:
-			for _, armVers := range ArmVersions {
+			for _, armVers := range ArmVersions(dist) {
 				dockerArchTag := strings.ReplaceAll(archName(arch, armVers), "/", "")
 				imageTemplates = append(
 					imageTemplates,