benchmark.yml

name: Benchmark

on:
  # FIXME: Remove push trigger
  push:
    branches:
      - akihironitta/bench
  # FIXME: Enable weekly or nightly trigger instead
  # schedule:
  #   - cron: '0 0,8,16 * * *'


# FIXME: For debugging purpose. Adjust this if necessary
concurrency:
  group: benchmark
  cancel-in-progress: true

defaults:
  run:
    shell: bash
    working-directory: benchmarks

jobs:
  benchmark:
    if: github.repository == 'pyg-team/pytorch_geometric'
    runs-on: [self-hosted, nvidia]

    steps:
    - uses: actions/checkout@v3
      with:
        fetch-depth: '0'

    - name: Clean up mamba from previous jobs
      run: |
        ls -al $HOME
        rm -rf $HOME/.mambaforge
        rm -rf $HOME/.conda*
        rm -rf $HOME/micromamba*
        rm /usr/local/bin/micromamba-shell
        ls -al $HOME
        ls -al $HOME/*

    - uses: mamba-org/setup-micromamba@v1
      with:
        environment-name: benchmark
        create-args: 'python=3.10'
        init-shell: bash

    - name: Check runner
      if: always()
      run: |
        echo $HOME
        nvidia-smi
        which python conda mamba micromamba || echo error
        mamba activate base || echo error
        micromamba activate benchmark || echo error

    # - name: Set up asv
    #   run: |
    #     python -m pip install -U pip
    #     pip install "git+https://github.com/airspeed-velocity/asv.git@8f5c8af"
    #     asv machine --machine "pyg-t4" --yes

    # - name: Configure Git
    #   run: |
    #     git config --local user.name "${{ github.actor }}"
    #     git config --local user.email "${{ github.actor }}@users.noreply.github.com"
    #     git config --list

    # - name: Run benchmarks on 2.3.0 and master
    #   timeout-minutes: 10  # FIXME: Adjust timeout
    #   run: |
    #     git branch --all
    #     asv run HASHFILE:HASHFILE.txt \
    #       --launch-method spawn \
    #       --skip-existing-successful \
    #       --verbose \
    #       --show-stderr \
    #       --bench scatter  # FIXME: Run all benchmarks

    # - name: Upload results
    #   uses: actions/upload-artifact@v3
    #   if: always()
    #   with:
    #     name: benchmark-result
    #     path: .asv/result

    # TODO: Enable pulling results
    # - name: Pull asv results
    #   # TODO: `.asv/results/` could be stored in another repo or S3 to lightweight the repo
    #   run: |
    #     git checkout results && git checkout master
    #     git checkout results .asv/results

    # - name: Compare with master on pull request
    #   # Run only a small set of benchmarks for sanity check.
    #   # if: github.event_name == 'pull_request'  # FIXME
    #   # Give 10min for other steps. GitHub Actions allow 360 at the maximum.
    #   timeout-minutes: 350
    #   run: |
    #     git branch --all
    #     # FIXME: Tighten the below ratio 2.0 to 1.1
    #     asv continuous -f 2.0 origin/master HEAD -e
    #   env:
    #     ASV_RUN_OPTIONS: "--skip-existing-successful --parallel -v -e"

    # TODO: Enable uploading results and gh-pages
    # - name: Push results and gh-pages if not on PR
    #   if: always() && github.event_name != 'pull_request'
    #   run: |
    #     # Push new page to `gh-pages` branch
    #     asv publish
    #     git fetch origin gh-pages:gh-pages
    #     asv gh-pages
    #     # Push new results to `results` branch
    #     mv .asv/results new_results
    #     git checkout $GIT_BRANCH_RESULTS
    #     rm -rf .asv/results
    #     mv new_results .asv/results
    #     git add .asv/results
    #     git commit -m "Add results from $(git rev-parse --short $GITHUB_SHA)" && git push origin $GIT_BRANCH_RESULTS || echo "No new results to push :)"
    #   env:
    #     GIT_BRANCH_RESULTS: results