Automatic benchmarking on pull requests

.github/workflows/benchmark_pr.yml

@@ -0,0 +1,78 @@
+name: Benchmark a pull request
+
+on:
+  pull_request_target:
+    branches:
+      - master
+
+permissions:
+  pull-requests: write
+
+jobs:
+    generate_plots:
+        runs-on: ubuntu-latest
+
+        steps:
+            - uses: actions/checkout@v2
+            - uses: julia-actions/setup-julia@v1
+              with:
+                version: "1.8"
+            - uses: julia-actions/cache@v1
+            - name: Extract Package Name from Project.toml
+              id: extract-package-name
+              run: |
+                PACKAGE_NAME=$(grep "^name" Project.toml | sed 's/^name = "\(.*\)"$/\1/')
+                echo "::set-output name=package_name::$PACKAGE_NAME"
+            - name: Build AirspeedVelocity
+              env:
+                JULIA_NUM_THREADS: 2
+              run: |
+                # Lightweight build step, as sometimes the runner runs out of memory:
+                julia -e 'ENV["JULIA_PKG_PRECOMPILE_AUTO"]=0; import Pkg; Pkg.add(;url="https://github.com/MilesCranmer/AirspeedVelocity.jl.git")'
+                julia -e 'ENV["JULIA_PKG_PRECOMPILE_AUTO"]=0; import Pkg; Pkg.build("AirspeedVelocity")'
+            - name: Add ~/.julia/bin to PATH
+              run: |
+                echo "$HOME/.julia/bin" >> $GITHUB_PATH
+            - name: Run benchmarks
+              run: |
+                echo $PATH
+                ls -l ~/.julia/bin
+                mkdir results
+                benchpkg ${{ steps.extract-package-name.outputs.package_name }} --rev="${{github.event.repository.default_branch}},${{github.event.pull_request.head.sha}}" --url=${{ github.event.repository.clone_url }} --bench-on="${{github.event.pull_request.head.sha}}" --output-dir=results/ --tune
+            - name: Create plots from benchmarks
+              run: |
+                mkdir -p plots
+                benchpkgplot ${{ steps.extract-package-name.outputs.package_name }} --rev="${{github.event.repository.default_branch}},${{github.event.pull_request.head.sha}}" --npart=10 --format=png --input-dir=results/ --output-dir=plots/
+            - name: Upload plot as artifact
+              uses: actions/upload-artifact@v2
+              with:
+                name: plots
+                path: plots
+            - name: Create markdown table from benchmarks
+              run: |
+                benchpkgtable ${{ steps.extract-package-name.outputs.package_name }} --rev="${{github.event.repository.default_branch}},${{github.event.pull_request.head.sha}}" --input-dir=results/ --ratio > table.md
+                echo '### Benchmark Results' > body.md
+                echo '' >> body.md
+                echo '' >> body.md
+                cat table.md >> body.md
+                echo '' >> body.md
+                echo '' >> body.md
+                echo '### Benchmark Plots' >> body.md
+                echo 'A plot of the benchmark results have been uploaded as an artifact to the workflow run for this PR.' >> body.md
+                echo 'Go to "Actions"->"Benchmark a pull request"->[the most recent run]->"Artifacts" (at the bottom).' >> body.md
+
+            - name: Find Comment
+              uses: peter-evans/find-comment@v2
+              id: fcbenchmark
+              with:
+                issue-number: ${{ github.event.pull_request.number }}
+                comment-author: 'github-actions[bot]'
+                body-includes: Benchmark Results
+
+            - name: Comment on PR
+              uses: peter-evans/create-or-update-comment@v3
+              with:
+                comment-id: ${{ steps.fcbenchmark.outputs.comment-id }}
+                issue-number: ${{ github.event.pull_request.number }}
+                body-path: body.md
+                edit-mode: replace

benchmark/Project.toml

@@ -0,0 +1,8 @@
+[deps]
+BenchmarkTools = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf"
+BitBasis = "50ba71b6-fa0f-514d-ae9a-0916efc90dcf"
+LuxurySparse = "d05aeea4-b7d4-55ac-b691-9e7fabb07ba2"
+Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"
+PkgBenchmark = "32113eaa-f34f-5b0d-bd6c-c81e245fc73d"
+Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
+StaticArrays = "90137ffa-7385-5640-81b9-e52037218182"

benchmark/benchmarks.jl

@@ -0,0 +1,42 @@
+module BenchmarkUtils
+    """
+    Replace instances of, for example, `using YaoAPI` with `using Yao.YaoAPI`.
+    """
+    const libs = (:YaoAPI, :YaoArrayRegister, :YaoBlocks, :YaoSym)
+    function replace_imports(ex)
+        return _replace_imports(ex)
+    end
+    function _replace_imports(ex)
+        if isa(ex, Expr) && ex.head == :using
+            foreach(ex.args) do e
+                if isa(e, Expr) && first(e.args) in libs
+                    pushfirst!(e.args, :Yao)
+                end
+            end
+        elseif isa(ex, Expr)
+            map!(_replace_imports, ex.args, ex.args)
+        end
+        return ex
+    end
+end
+
+module YaoArrayRegisterBenchmarks
+    using ..BenchmarkUtils: replace_imports
+    using Yao
+    using Yao.YaoBlocks: sprand_hermitian
+    include(replace_imports, "../lib/YaoArrayRegister/benchmark/benchmarks.jl")
+end
+
+module YaoBlocksBenchmarks
+    using ..BenchmarkUtils: replace_imports
+    using Yao
+    include(replace_imports, "../lib/YaoBlocks/benchmark/benchmarks.jl")
+end
+
+using BenchmarkTools
+import .YaoArrayRegisterBenchmarks: SUITE as yarSUITE
+import .YaoBlocksBenchmarks: SUITE as ybSUITE
+
+const SUITE = BenchmarkGroup()
+SUITE["YaoArrayRegister"] = yarSUITE
+SUITE["YaoBlocks"] = ybSUITE