linux-package-test.yml

name: linux-package-test

on:
  push:
    branches:
      - main
  pull_request:
    paths:
      - '.github/workflows/linux-package-test.yml'
      - 'cmd/otelcol/**'
      - 'internal/**'
      - 'Makefile'
      - 'Makefile.Common'
      - '!**.md'
      - '!packaging/technical-addon/**'

concurrency:
  group: linux-package-test-${{ github.event.pull_request.number || github.ref }}
  cancel-in-progress: true

env:
  PYTHON_VERSION: '3.13'
  PIP_VERSION: '24.2'
  REQUIREMENTS_PATH: "packaging/tests/requirements.txt"
  GO_VERSION: 1.22.7

jobs:
  setup-environment:
    # Use 20.04.5 until https://github.com/open-telemetry/opentelemetry-collector-contrib/issues/16450 is resolved
    runs-on: ubuntu-20.04
    steps:
      - name: Check out the codebase.
        uses: actions/checkout@v4

      - name: Set up Go
        uses: actions/setup-go@v5
        with:
          go-version: ${{ env.GO_VERSION }}
          cache-dependency-path: '**/go.sum'

      - name: Installing dependency
        run: |
          make install-tools

  cross-compile:
    # Use 20.04.5 until https://github.com/open-telemetry/opentelemetry-collector-contrib/issues/16450 is resolved
    runs-on: ubuntu-20.04
    needs: [setup-environment]
    strategy:
      matrix:
        SYS_BINARIES: [ "binaries-linux_amd64", "binaries-linux_arm64", "binaries-linux_ppc64le" ]
    steps:
      - name: Check out the codebase.
        uses: actions/checkout@v4

      - name: Set up Go
        uses: actions/setup-go@v5
        with:
          go-version: ${{ env.GO_VERSION }}
          cache-dependency-path: '**/go.sum'

      - name: Build Collector
        run: |
          make ${{ matrix.SYS_BINARIES }}

      - name: Uploading binaries
        uses: actions/upload-artifact@v4
        with:
          name: ${{ matrix.SYS_BINARIES }}
          path: |
            ./bin/*

  agent-bundle-linux:
    runs-on: ${{ fromJSON('["ubuntu-20.04", "otel-arm64"]')[matrix.ARCH == 'arm64'] }}
    strategy:
      matrix:
        ARCH: ["amd64", "arm64"]
      fail-fast: false
    steps:
      - uses: actions/checkout@v4

      - uses: actions/cache@v4
        id: bundle-cache
        with:
          path: .cache/buildx/agent-bundle-${{ matrix.ARCH }}
          key: agent-bundle-buildx-${{ matrix.ARCH }}-${{ hashFiles('packaging/bundle/**') }}
          restore-keys: |
            agent-bundle-buildx-${{ matrix.ARCH }}-

      - run: make -C packaging/bundle agent-bundle-linux ARCH=${{ matrix.ARCH }}
        env:
          BUNDLE_CACHE_HIT: "${{ steps.bundle-cache.outputs.cache-hit }}"

      - uses: actions/upload-artifact@v4
        with:
          name: agent-bundle-linux-${{ matrix.ARCH }}
          path: ./dist/agent-bundle_linux_${{ matrix.ARCH }}.tar.gz

  build-package:
    # Use 20.04.5 until https://github.com/open-telemetry/opentelemetry-collector-contrib/issues/16450 is resolved
    runs-on: ubuntu-20.04
    needs: [cross-compile, agent-bundle-linux]
    strategy:
      matrix:
        SYS_PACKAGE: [ "deb", "rpm", "tar" ]
        ARCH: [ "amd64", "arm64" ]
      fail-fast: false
    steps:
      - name: Check out the codebase.
        uses: actions/checkout@v4
        with:
          fetch-depth: 0

      - name: Set up Go
        uses: actions/setup-go@v5
        with:
          go-version: ${{ env.GO_VERSION }}
          cache-dependency-path: '**/go.sum'

      - name: Downloading binaries-linux_${{ matrix.ARCH }}
        uses: actions/download-artifact@v4
        with:
          name: binaries-linux_${{ matrix.ARCH }}
          path: ./bin

      - uses: actions/download-artifact@v4
        with:
          name: agent-bundle-linux-${{ matrix.ARCH }}
          path: ./dist

      - name: Build ${{ matrix.SYS_PACKAGE }} ${{ matrix.ARCH }} package
        run: make ${{ matrix.SYS_PACKAGE }}-package SKIP_COMPILE=true SKIP_BUNDLE=true VERSION="" ARCH="${{ matrix.ARCH }}"

      - name: Uploading ${{ matrix.SYS_PACKAGE }} ${{ matrix.ARCH }} package artifacts
        uses: actions/upload-artifact@v4
        with:
          name: ${{ matrix.SYS_PACKAGE }}-${{ matrix.ARCH }}-package
          path: ./dist/splunk-otel-collector*

  test-package-matrix:
    runs-on: ubuntu-20.04
    steps:
      - name: Check out code
        uses: actions/checkout@v4

      - name: Get matrix
        id: get-matrix
        run: |
          # create test matrix for distro and arch
          dockerfiles=$(find packaging/tests/images/ -name "Dockerfile.*" | cut -d '.' -f2- | sort -u)
          if [ -z "$dockerfiles" ]; then
            echo "Failed to get dockerfiles from packaging/tests/images!" >&2
            exit 1
          fi
          distro=$(for d in $dockerfiles; do echo -n "\"$d\","; done)
          arch="\"amd64\", \"arm64\""
          matrix="{\"DISTRO\": [${distro%,}], \"ARCH\": [${arch}]}"
          echo "$matrix" | jq
          echo "matrix=${matrix}" >> $GITHUB_OUTPUT
    outputs:
      matrix: ${{ steps.get-matrix.outputs.matrix }}

  test-package:
    # Use 20.04.5 until https://github.com/open-telemetry/opentelemetry-collector-contrib/issues/16450 is resolved
    runs-on: ${{ fromJSON('["ubuntu-20.04", "ubuntu-22.04"]')[matrix.DISTRO == 'amazonlinux-2023'] }}
    timeout-minutes: 30
    needs: [build-package, test-package-matrix]
    strategy:
      matrix: ${{ fromJSON(needs.test-package-matrix.outputs.matrix) }}
      fail-fast: false
    steps:
      - name: Check out the codebase.
        uses: actions/checkout@v4

      - name: Get package type for ${{ matrix.DISTRO }}
        run: |
          for pkg in "deb" "rpm" "tar"; do
            if [[ -f "packaging/tests/images/${pkg}/Dockerfile.${{ matrix.DISTRO }}" ]]; then
              echo "SYS_PACKAGE=${pkg}" >> $GITHUB_ENV
              exit 0
            fi
          done
          echo "Unknown distro '${{ matrix.DISTRO }}'!"
          exit 1

      - uses: actions/download-artifact@v4
        with:
          name: ${{ env.SYS_PACKAGE }}-${{ matrix.ARCH }}-package
          path: ./dist

      - uses: docker/setup-qemu-action@v3
        if: ${{ matrix.ARCH != 'amd64' }}
        with:
          platforms: ${{ matrix.ARCH }}
          image: tonistiigi/binfmt:qemu-v7.0.0

      - name: Setup python
        uses: actions/setup-python@v5
        with:
          python-version: ${{ env.PYTHON_VERSION }}
          cache: 'pip'
          cache-dependency-path: ${{ env.REQUIREMENTS_PATH }}

      - name: Install pytest
        run: |
          if which pip; then
              pip install --upgrade 'pip==${{ env.PIP_VERSION }}'
          else
              curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py
              python get-pip.py 'pip==${{ env.PIP_VERSION }}'
          fi
          pip install -r "${{ env.REQUIREMENTS_PATH }}"

      - name: Test ${{ env.SYS_PACKAGE }} on ${{ matrix.DISTRO }} ${{ matrix.ARCH }}
        id: pytest
        continue-on-error: true
        run: |
          distro="${{ matrix.DISTRO }}"
          if [[ "$distro" = "amazonlinux-2" ]]; then
            # workaround for pytest substring matching
            distro="amazonlinux-2 and not amazonlinux-2023"
          fi
          python3 -u -m pytest -s --verbose -m "${{ env.SYS_PACKAGE }}" \
            -k "$distro and ${{ matrix.ARCH }}" \
            packaging/tests/package_test.py

      # qemu, networking, running systemd in containers, etc., can be flaky
      - name: Re-run failed tests
        if: ${{ steps.pytest.outcome == 'failure' }}
        run: |
          distro="${{ matrix.DISTRO }}"
          if [[ "$distro" = "amazonlinux-2" ]]; then
            # workaround for pytest substring matching
            distro="amazonlinux-2 and not amazonlinux-2023"
          fi
          python3 -u -m pytest -s --verbose -m "${{ env.SYS_PACKAGE }}" \
            -k "$distro and ${{ matrix.ARCH }}" \
            --last-failed \
            packaging/tests/package_test.py

  docker-otelcol:
    # Use 20.04.5 until https://github.com/open-telemetry/opentelemetry-collector-contrib/issues/16450 is resolved
    runs-on: ubuntu-20.04
    needs: [cross-compile, agent-bundle-linux]
    steps:
      # Multiarch images require more disk space
      - uses: jlumbroso/free-disk-space@v1.3.1

      - name: Check out the codebase.
        uses: actions/checkout@v4

      # Required to export a multiarch manifest and images to the local image store
      - name: Set up containerd image store
        uses: crazy-max/ghaction-setup-docker@v3
        with:
          daemon-config: |
            {
              "features": {
                "containerd-snapshotter": true
              }
            }

      - name: Set up Go
        uses: actions/setup-go@v5
        with:
          go-version: ${{ env.GO_VERSION }}
          cache-dependency-path: '**/go.sum'

      - name: Set up QEMU
        uses: docker/setup-qemu-action@v3
        with:
          platforms: arm64,ppc64le
          image: tonistiigi/binfmt:qemu-v7.0.0

      - name: Downloading binaries-linux_amd64
        uses: actions/download-artifact@v4
        with:
          name: binaries-linux_amd64
          path: ./bin

      - name: Downloading binaries-linux_arm64
        uses: actions/download-artifact@v4
        with:
          name: binaries-linux_arm64
          path: ./bin

      - name: Downloading binaries-linux_ppc64le
        uses: actions/download-artifact@v4
        with:
          name: binaries-linux_ppc64le
          path: ./bin

      - uses: actions/download-artifact@v4
        with:
          name: agent-bundle-linux-amd64
          path: ./dist

      - uses: actions/download-artifact@v4
        with:
          name: agent-bundle-linux-arm64
          path: ./dist

      - name: Build the multiarch docker image
        run: make docker-otelcol SKIP_COMPILE=true SKIP_BUNDLE=true ARCH=amd64,arm64,ppc64le

      - name: Save the multiarch image archive to be loaded by downstream jobs
        run: mkdir -p docker-otelcol && docker save -o ./docker-otelcol/image.tar otelcol:latest

      - uses: actions/upload-artifact@v4
        with:
          name: otelcol
          path: ./docker-otelcol

  docker-otelcol-verify:
    # Use 20.04.5 until https://github.com/open-telemetry/opentelemetry-collector-contrib/issues/16450 is resolved
    runs-on: ubuntu-20.04
    needs: [docker-otelcol]
    strategy:
      matrix:
        ARCH: [ "amd64", "arm64", "ppc64le" ]
      fail-fast: false
    steps:
      - name: Check out the codebase.
        uses: actions/checkout@v4

      # Required to load a multiarch archive to the local image store
      - name: Set up containerd image store
        uses: crazy-max/ghaction-setup-docker@v3
        with:
          daemon-config: |
            {
              "features": {
                "containerd-snapshotter": true
              }
            }

      - name: Set up QEMU
        if: ${{ matrix.ARCH != 'amd64' }}
        uses: docker/setup-qemu-action@v3
        with:
          platforms: ${{ matrix.ARCH }}
          image: tonistiigi/binfmt:qemu-v7.0.0

      - uses: actions/download-artifact@v4
        with:
          name: otelcol
          path: ./docker-otelcol

      - name: Load multiarch docker image
        run: docker load -i ./docker-otelcol/image.tar

      - name: Run docker image
        run: |
          # ensure the collector can start with the default config file
          docker run --platform linux/${{ matrix.ARCH }} -d -e SPLUNK_ACCESS_TOKEN=12345 -e SPLUNK_REALM=fake-realm --name otelcol otelcol:latest
          sleep 10
          if [ -z "$( docker ps --filter=status=running --filter=name=otelcol -q )" ]; then
            docker logs otelcol
            echo "Failing job execution: fail to start otelcol docker container with the default config in 10 seconds."
            exit 1
          fi

          # get all provided config files from the container
          tmpdir=$(mktemp -d)
          docker cp otelcol:/etc/otel/collector $tmpdir
          docker rm -f otelcol

          # ensure the collector can start with all provided config files
          configs=$(ls ${tmpdir}/collector/ 2>/dev/null)
          if [ -z "$configs" ]; then
            echo "failed to get config files from otelcol:/etc/otel/collector"
            exit 1
          fi
          for config in $configs; do
            # TODO: set fake-token back to 12345 once https://github.com/open-telemetry/opentelemetry-collector/issues/10937 is resolved
            docker run --platform linux/${{ matrix.ARCH }} -d -e SPLUNK_CONFIG=/etc/otel/collector/${config} -e SPLUNK_ACCESS_TOKEN=fake-token -e SPLUNK_REALM=fake-realm -e ECS_CONTAINER_METADATA_URI_V4=http://localhost --name otelcol otelcol:latest
            sleep 10
            if [ -z "$( docker ps --filter=status=running --filter=name=otelcol -q )" ]; then
              docker logs otelcol
              echo "Failing job execution: fail to start otelcol docker container with ${config} in 10 seconds."
              exit 1
            fi
            docker rm -f otelcol
          done

      - name: Check journalctl
        run: |
          # ensure journalctl can run with the collected libraries
          docker run --platform linux/${{ matrix.ARCH }} -d -e SPLUNK_ACCESS_TOKEN=12345 -e SPLUNK_REALM=fake-realm --name otelcol otelcol:latest
          docker exec otelcol /bin/journalctl
          docker rm -f otelcol

      - name: Check python and java
        if: ${{ matrix.ARCH != 'ppc64le' }}
        run: |
          # ensure python and java can run with the collected libraries
          docker run --platform linux/${{ matrix.ARCH }} -d -e SPLUNK_ACCESS_TOKEN=12345 -e SPLUNK_REALM=fake-realm --name otelcol otelcol:latest
          docker exec otelcol /usr/lib/splunk-otel-collector/agent-bundle/bin/python --version
          docker exec otelcol /usr/lib/splunk-otel-collector/agent-bundle/jre/bin/java -version
          # ensure collectd-python plugins were installed
          docker exec otelcol sh -c 'test -d /usr/lib/splunk-otel-collector/agent-bundle/collectd-python/'
          if [[ "$(docker exec otelcol ls /usr/lib/splunk-otel-collector/agent-bundle/collectd-python/ | wc -l)" -eq 0 ]]; then
            echo "/usr/lib/splunk-otel-collector/agent-bundle/collectd-python/ is empty!" >&2
            exit 1
          fi
          docker rm -f otelcol