performance.yml

name: Performance Regression Tests
# Schedule triggers
on:
  # runs twice a day at 10:05am and 10:05pm
  schedule:
    - cron: "5 10,22 * * *"
  # Allows you to run this workflow manually from the Actions tab
  workflow_dispatch:

jobs:
  # checks fmt of runner code
  # purposefully not a dependency of any other job
  # will block merging, but not prevent developing
  fmt:
    name: Cargo fmt
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v2
      - uses: actions-rs/toolchain@v1
        with:
          profile: minimal
          toolchain: stable
          override: true
      - run: rustup component add rustfmt
      - uses: actions-rs/cargo@v1
        with:
          command: fmt
          args: --manifest-path performance/runner/Cargo.toml --all -- --check

  # runs any tests associated with the runner
  # these tests make sure the runner logic is correct
  test-runner:
    name: Test Runner
    runs-on: ubuntu-latest
    env:
      # turns errors into warnings
      RUSTFLAGS: "-D warnings"
    steps:
      - uses: actions/checkout@v2
      - uses: actions-rs/toolchain@v1
        with:
          profile: minimal
          toolchain: stable
          override: true
      - uses: actions-rs/cargo@v1
        with:
          command: test
          args: --manifest-path performance/runner/Cargo.toml

  # build an optimized binary to be used as the runner in later steps
  build-runner:
    needs: [test-runner]
    name: Build Runner
    runs-on: ubuntu-latest
    env:
      RUSTFLAGS: "-D warnings"
    steps:
      - uses: actions/checkout@v2
      - uses: actions-rs/toolchain@v1
        with:
          profile: minimal
          toolchain: stable
          override: true
      - uses: actions-rs/cargo@v1
        with:
          command: build
          args: --release --manifest-path performance/runner/Cargo.toml
      - uses: actions/upload-artifact@v2
        with:
          name: runner
          path: performance/runner/target/release/runner

  # run the performance measurements on the current or default branch
  measure-dev:
    needs: [build-runner]
    name: Measure Dev Branch
    runs-on: ubuntu-latest
    steps:
      - name: checkout dev
        uses: actions/checkout@v2
      - name: Setup Python
        uses: actions/setup-python@v2.2.2
        with:
          python-version: "3.8"
      - name: install dbt
        run: pip install -r dev-requirements.txt -r editable-requirements.txt
      - name: install hyperfine
        run: wget https://github.com/sharkdp/hyperfine/releases/download/v1.11.0/hyperfine_1.11.0_amd64.deb && sudo dpkg -i hyperfine_1.11.0_amd64.deb
      - uses: actions/download-artifact@v2
        with:
          name: runner
      - name: change permissions
        run: chmod +x ./runner
      - name: run
        run: ./runner measure -b dev -p ${{ github.workspace }}/performance/projects/
      - uses: actions/upload-artifact@v2
        with:
          name: dev-results
          path: performance/results/

  # run the performance measurements on the release branch which we use
  # as a performance baseline. This part takes by far the longest, so
  # we do everything we can first so the job fails fast.
  # -----
  # we need to checkout dbt twice in this job: once for the baseline dbt
  # version, and once to get the latest regression testing projects,
  # metrics, and runner code from the develop or current branch so that
  # the calculations match for both versions of dbt we are comparing.
  measure-baseline:
    needs: [build-runner]
    name: Measure Baseline Branch
    runs-on: ubuntu-latest
    steps:
      - name: checkout latest
        uses: actions/checkout@v2
        with:
          ref: "0.20.latest"
      - name: Setup Python
        uses: actions/setup-python@v2.2.2
        with:
          python-version: "3.8"
      - name: move repo up a level
        run: mkdir ${{ github.workspace }}/../baseline/ && cp -r ${{ github.workspace }} ${{ github.workspace }}/../baseline
      - name: "[debug] ls new dbt location"
        run: ls ${{ github.workspace }}/../baseline/dbt/
      # installation creates egg-links so we have to preserve source
      - name: install dbt from new location
        run: cd ${{ github.workspace }}/../baseline/dbt/ && pip install -r dev-requirements.txt -r editable-requirements.txt
      # checkout the current branch to get all the target projects
      # this deletes the old checked out code which is why we had to copy before
      - name: checkout dev
        uses: actions/checkout@v2
      - name: install hyperfine
        run: wget https://github.com/sharkdp/hyperfine/releases/download/v1.11.0/hyperfine_1.11.0_amd64.deb && sudo dpkg -i hyperfine_1.11.0_amd64.deb
      - uses: actions/download-artifact@v2
        with:
          name: runner
      - name: change permissions
        run: chmod +x ./runner
      - name: run runner
        run: ./runner measure -b baseline -p ${{ github.workspace }}/performance/projects/
      - uses: actions/upload-artifact@v2
        with:
          name: baseline-results
          path: performance/results/

  # detect regressions on the output generated from measuring
  # the two branches. Exits with non-zero code if a regression is detected.
  calculate-regressions:
    needs: [measure-dev, measure-baseline]
    name: Compare Results
    runs-on: ubuntu-latest
    steps:
      - uses: actions/download-artifact@v2
        with:
          name: dev-results
      - uses: actions/download-artifact@v2
        with:
          name: baseline-results
      - name: "[debug] ls result files"
        run: ls
      - uses: actions/download-artifact@v2
        with:
          name: runner
      - name: change permissions
        run: chmod +x ./runner
      - name: make results directory
        run: mkdir ./final-output/
      - name: run calculation
        run: ./runner calculate -r ./ -o ./final-output/
        # always attempt to upload the results even if there were regressions found
      - uses: actions/upload-artifact@v2
        if: ${{ always() }}
        with:
          name: final-calculations
          path: ./final-output/*