E2ESHARK Test Suite #364
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Copyright 2024 Advanced Micro Devices | |
| # | |
| # Licensed under the Apache License v2.0 with LLVM Exceptions. | |
| # See https://llvm.org/LICENSE.txt for license information. | |
| # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception | |
| name: E2ESHARK Test Suite | |
| on: | |
| workflow_dispatch: | |
| # schedule: | |
| # # Runs at 12:00 PM UTC, which is 5:00 AM PST | |
| # - cron: '0 12 * * *' | |
| jobs: | |
| # e2eshark: | |
| # timeout-minutes: 480 | |
| # name: "Models :: ${{ matrix.backend }} :: ${{ matrix.test-file }}" | |
| # runs-on: ${{ matrix.runs-on }} | |
| # strategy: | |
| # fail-fast: false | |
| # matrix: | |
| # include: | |
| # - name: mi300_gpu1_test | |
| # runs-on: nodai-amdgpu-mi300-x86-64 | |
| # backend: rocm | |
| # device: hip | |
| # target-chip: gfx942 | |
| # test-file: nlp-shard1 | |
| # cache-dir: /home/sai/shark-test-suite-models-cache | |
| # - name: mi300_gpu2_test | |
| # runs-on: nodai-amdgpu-mi300-x86-64 | |
| # backend: rocm | |
| # device: hip | |
| # target-chip: gfx942 | |
| # test-file: nlp-shard2 | |
| # cache-dir: /home/sai/shark-test-suite-models-cache | |
| # - name: mi300_gpu3_test | |
| # runs-on: nodai-amdgpu-mi300-x86-64 | |
| # backend: rocm | |
| # device: hip | |
| # target-chip: gfx942 | |
| # test-file: nlp-shard3 | |
| # cache-dir: /home/sai/shark-test-suite-models-cache | |
| # - name: mi300_gpu4_test | |
| # runs-on: nodai-amdgpu-mi300-x86-64 | |
| # backend: rocm | |
| # device: hip | |
| # target-chip: gfx942 | |
| # test-file: shark-test-suite | |
| # cache-dir: /home/sai/shark-test-suite-models-cache | |
| # - name: mi300_gpu5_test | |
| # runs-on: nodai-amdgpu-mi300-x86-64 | |
| # backend: rocm | |
| # device: hip | |
| # target-chip: gfx942 | |
| # test-file: vai-hf-cnn-fp32-shard1 | |
| # cache-dir: /home/sai/shark-test-suite-models-cache | |
| # - name: mi300_gpu6_test | |
| # runs-on: nodai-amdgpu-mi300-x86-64 | |
| # backend: rocm | |
| # device: hip | |
| # target-chip: gfx942 | |
| # test-file: vai-hf-cnn-fp32-shard2 | |
| # cache-dir: /home/sai/shark-test-suite-models-cache | |
| # - name: mi300_gpu7_test | |
| # runs-on: nodai-amdgpu-mi300-x86-64 | |
| # backend: rocm | |
| # device: hip | |
| # target-chip: gfx942 | |
| # test-file: vai-hf-cnn-fp32-shard3 | |
| # cache-dir: /home/sai/shark-test-suite-models-cache | |
| # - name: mi300_gpu8_test | |
| # runs-on: nodai-amdgpu-mi300-x86-64 | |
| # backend: rocm | |
| # device: hip | |
| # target-chip: gfx942 | |
| # test-file: vai-int8-p0p1-shard1 | |
| # cache-dir: /home/sai/shark-test-suite-models-cache | |
| # - name: mi300_gpu9_test | |
| # runs-on: nodai-amdgpu-mi300-x86-64 | |
| # backend: rocm | |
| # device: hip | |
| # target-chip: gfx942 | |
| # test-file: vai-int8-p0p1-shard2 | |
| # cache-dir: /home/sai/shark-test-suite-models-cache | |
| # - name: mi300_gpu10_test | |
| # runs-on: nodai-amdgpu-mi300-x86-64 | |
| # backend: rocm | |
| # device: hip | |
| # target-chip: gfx942 | |
| # test-file: vai-int8-p0p1-shard3 | |
| # cache-dir: /home/sai/shark-test-suite-models-cache | |
| # - name: mi300_gpu11_test | |
| # runs-on: nodai-amdgpu-mi300-x86-64 | |
| # backend: rocm | |
| # device: hip | |
| # target-chip: gfx942 | |
| # test-file: vai-vision-int8 | |
| # cache-dir: /home/sai/shark-test-suite-models-cache | |
| # - name: mi300_gpu12_test | |
| # runs-on: nodai-amdgpu-mi300-x86-64 | |
| # backend: rocm | |
| # device: hip | |
| # target-chip: gfx942 | |
| # test-file: migraphx | |
| # cache-dir: /home/sai/shark-test-suite-models-cache | |
| # - name: cpu_shard1_test | |
| # runs-on: nodai-amdgpu-mi250-x86-64 | |
| # backend: llvm-cpu | |
| # device: local-task | |
| # target-chip: x86_64-linux-gnu | |
| # test-file: nlp-shard1 | |
| # cache-dir: /groups/aig_sharks/test-suite-ci-cache | |
| # - name: cpu_shard2_test | |
| # runs-on: nodai-amdgpu-mi250-x86-64 | |
| # backend: llvm-cpu | |
| # device: local-task | |
| # target-chip: x86_64-linux-gnu | |
| # test-file: nlp-shard2 | |
| # cache-dir: /groups/aig_sharks/test-suite-ci-cache | |
| # - name: cpu_shard3_test | |
| # runs-on: nodai-amdgpu-mi250-x86-64 | |
| # backend: llvm-cpu | |
| # device: local-task | |
| # target-chip: x86_64-linux-gnu | |
| # test-file: nlp-shard3 | |
| # cache-dir: /groups/aig_sharks/test-suite-ci-cache | |
| # - name: cpu_shard4_test | |
| # runs-on: nodai-amdgpu-mi250-x86-64 | |
| # backend: llvm-cpu | |
| # device: local-task | |
| # target-chip: x86_64-linux-gnu | |
| # test-file: shark-test-suite | |
| # cache-dir: /groups/aig_sharks/test-suite-ci-cache | |
| # - name: cpu_shard5_test | |
| # runs-on: nodai-amdgpu-mi250-x86-64 | |
| # backend: llvm-cpu | |
| # device: local-task | |
| # target-chip: x86_64-linux-gnu | |
| # test-file: vai-hf-cnn-fp32-shard1 | |
| # cache-dir: /groups/aig_sharks/test-suite-ci-cache | |
| # - name: cpu_shard6_test | |
| # runs-on: nodai-amdgpu-mi250-x86-64 | |
| # backend: llvm-cpu | |
| # device: local-task | |
| # target-chip: x86_64-linux-gnu | |
| # test-file: vai-hf-cnn-fp32-shard2 | |
| # cache-dir: /groups/aig_sharks/test-suite-ci-cache | |
| # - name: cpu_shard7_test | |
| # runs-on: nodai-amdgpu-mi250-x86-64 | |
| # backend: llvm-cpu | |
| # device: local-task | |
| # target-chip: x86_64-linux-gnu | |
| # test-file: vai-hf-cnn-fp32-shard3 | |
| # cache-dir: /groups/aig_sharks/test-suite-ci-cache | |
| # - name: cpu_shard8_test | |
| # runs-on: nodai-amdgpu-mi250-x86-64 | |
| # backend: llvm-cpu | |
| # device: local-task | |
| # target-chip: x86_64-linux-gnu | |
| # test-file: vai-int8-p0p1-shard1 | |
| # cache-dir: /groups/aig_sharks/test-suite-ci-cache | |
| # - name: cpu_shard9_test | |
| # runs-on: nodai-amdgpu-mi250-x86-64 | |
| # backend: llvm-cpu | |
| # device: local-task | |
| # target-chip: x86_64-linux-gnu | |
| # test-file: vai-int8-p0p1-shard2 | |
| # cache-dir: /groups/aig_sharks/test-suite-ci-cache | |
| # - name: cpu_shard10_test | |
| # runs-on: nodai-amdgpu-mi250-x86-64 | |
| # backend: llvm-cpu | |
| # device: local-task | |
| # target-chip: x86_64-linux-gnu | |
| # test-file: vai-int8-p0p1-shard3 | |
| # cache-dir: /groups/aig_sharks/test-suite-ci-cache | |
| # - name: cpu_shard11_test | |
| # runs-on: nodai-amdgpu-mi250-x86-64 | |
| # backend: llvm-cpu | |
| # device: local-task | |
| # target-chip: x86_64-linux-gnu | |
| # test-file: vai-vision-int8 | |
| # cache-dir: /groups/aig_sharks/test-suite-ci-cache | |
| # - name: cpu_shard12_test | |
| # runs-on: nodai-amdgpu-mi250-x86-64 | |
| # backend: llvm-cpu | |
| # device: local-task | |
| # target-chip: x86_64-linux-gnu | |
| # test-file: migraphx | |
| # cache-dir: /groups/aig_sharks/test-suite-ci-cache | |
| # env: | |
| # E2E_VENV_DIR: ${{ github.workspace }}/test-suite_venv | |
| # ALT_E2E_VENV_DIR: ${{ github.workspace }}/alt-test-suite_venv | |
| # HF_TOKEN: ${{ secrets.HF_TOKEN }} | |
| # AZ_PRIVATE_CONNECTION: ${{ secrets.ONNXPRIVATESTORAGE_AZ_PRIVATE_CONNECTION }} | |
| # CACHE_DIR: ${{ matrix.cache-dir }} | |
| # steps: | |
| # - name: Checkout Test Suite | |
| # uses: actions/checkout@v2 | |
| # with: | |
| # repository: nod-ai/SHARK-TestSuite | |
| # ref: alt-merge-reports | |
| # path: test-suite | |
| # - name: "Setup alt e2eshark python venv" | |
| # run: | | |
| # rm -rf ${ALT_E2E_VENV_DIR} | |
| # python3.11 -m venv ${ALT_E2E_VENV_DIR} | |
| # source ${ALT_E2E_VENV_DIR}/bin/activate | |
| # pip install --upgrade pip | |
| # pip install -r ./alt_e2eshark/base_requirements.txt | |
| # pip install -r ./alt_e2eshark/iree_requirements.txt | |
| # pip install --no-deps -r ./alt_e2eshark/torch_mlir_requirements.txt | |
| # pip install --find-links https://iree.dev/pip-release-links.html iree-compiler iree-runtime --upgrade | |
| # working-directory: ./test-suite | |
| # - name: Run Onnx Bench Mode | |
| # if: contains(matrix.test-file, 'migraphx') | |
| # run: | | |
| # source ${ALT_E2E_VENV_DIR}/bin/activate | |
| # pip freeze | |
| # cd alt_e2eshark | |
| # free -mh | |
| # python3.11 ./run.py \ | |
| # -r ./test-onnx \ | |
| # --report \ | |
| # --torchtolinalg \ | |
| # --testsfile onnx_tests/models/external_lists/${{ matrix.test-file }}.txt \ | |
| # -b ${{ matrix.backend }} \ | |
| # -d ${{ matrix.device }} \ | |
| # --report-file reports/${{ matrix.test-file }}.md \ | |
| # --mode=cl-onnx-iree \ | |
| # --cleanup=3 \ | |
| # --benchmark \ | |
| # -v | |
| # working-directory: ./test-suite | |
| # - name: Run Onnx Default Mode | |
| # if: ${{ ! contains(matrix.test-file, 'migraphx') }} | |
| # run: | | |
| # source ${ALT_E2E_VENV_DIR}/bin/activate | |
| # pip freeze | |
| # cd alt_e2eshark | |
| # free -mh | |
| # python3.11 ./run.py \ | |
| # -r ./test-onnx \ | |
| # --report \ | |
| # --torchtolinalg \ | |
| # --testsfile onnx_tests/models/external_lists/${{ matrix.test-file }}.txt \ | |
| # -b ${{ matrix.backend }} \ | |
| # -d ${{ matrix.device }} \ | |
| # --report-file reports/${{ matrix.test-file }}.md \ | |
| # --mode=cl-onnx-iree \ | |
| # --cleanup=3 \ | |
| # -v | |
| # working-directory: ./test-suite | |
| # - uses: actions/upload-artifact@master | |
| # with: | |
| # name: ci_reports_${{ matrix.backend }}_${{ matrix.test-file }}_onnx_md | |
| # path: ./test-suite/alt_e2eshark/reports/${{ matrix.test-file }}.md | |
| # - uses: actions/upload-artifact@master | |
| # with: | |
| # name: ci_reports_${{ matrix.backend }}_${{ matrix.test-file }}_onnx_json | |
| # path: ./test-suite/alt_e2eshark/reports/${{ matrix.test-file }}.json | |
| push_artifacts: | |
| # needs: [e2eshark] | |
| runs-on: ${{ matrix.runs-on }} | |
| strategy: | |
| fail-fast: false | |
| max-parallel: 1 | |
| matrix: | |
| include: | |
| - name: merge_rocm_reports | |
| runs-on: nodai-amdgpu-mi300-x86-64 | |
| backend: rocm | |
| - name: merge_cpu_reports | |
| runs-on: nodai-amdgpu-mi250-x86-64 | |
| backend: llvm-cpu | |
| env: | |
| AZ_PUBLIC_KEY: ${{ secrets.SHARKPUBLIC_AZ_PUBLIC_KEY }} | |
| steps: | |
| - name: Checkout Test Suite | |
| uses: actions/checkout@v2 | |
| with: | |
| repository: nod-ai/SHARK-TestSuite | |
| ref: alt-merge-reports | |
| path: test-suite | |
| - name: Checkout repo | |
| uses: actions/checkout@v2 | |
| with: | |
| repository: nod-ai/e2eshark-reports | |
| ref: main | |
| token: ${{ secrets.E2ESHARK_GITHUB_TOKEN }} | |
| path: e2eshark-reports | |
| - name: "Setup alt test suite venv" | |
| run: | | |
| python3.11 -m venv report_venv_alt | |
| source report_venv_alt/bin/activate | |
| pip install --upgrade pip | |
| pip install -r ./test-suite/alt_e2eshark/base_requirements.txt | |
| pip install -r ./test-suite/alt_e2eshark/iree_requirements.txt | |
| pip install --no-deps -r ./test-suite/alt_e2eshark/torch_mlir_requirements.txt | |
| pip install --find-links https://iree.dev/pip-release-links.html iree-compiler iree-runtime --upgrade | |
| - uses: actions/download-artifact@master | |
| with: | |
| name: ci_reports_${{ matrix.backend }}_shark-test-suite_onnx_md | |
| path: ./e2eshark-reports/ci_reports_${{ matrix.backend }}_shark-test-suite_onnx_md | |
| github-token: ${{ secrets.IREE_ARTIFACTS }} | |
| repository: nod-ai/SHARK-TestSuite | |
| run-id: 11323798734 | |
| - uses: actions/download-artifact@master | |
| with: | |
| name: ci_reports_${{ matrix.backend }}_shark-test-suite_onnx_json | |
| path: ./e2eshark-reports/ci_reports_${{ matrix.backend }}_shark-test-suite_onnx_json | |
| github-token: ${{ secrets.IREE_ARTIFACTS }} | |
| repository: nod-ai/SHARK-TestSuite | |
| run-id: 11323798734 | |
| - uses: actions/download-artifact@master | |
| with: | |
| name: ci_reports_${{ matrix.backend }}_vai-hf-cnn-fp32-shard1_onnx_md | |
| path: ./e2eshark-reports/ci_reports_${{ matrix.backend }}_vai-hf-cnn-fp32-shard1_onnx_md | |
| github-token: ${{ secrets.IREE_ARTIFACTS }} | |
| repository: nod-ai/SHARK-TestSuite | |
| run-id: 11323798734 | |
| - uses: actions/download-artifact@master | |
| with: | |
| name: ci_reports_${{ matrix.backend }}_vai-hf-cnn-fp32-shard1_onnx_json | |
| path: ./e2eshark-reports/ci_reports_${{ matrix.backend }}_vai-hf-cnn-fp32-shard1_onnx_json | |
| github-token: ${{ secrets.IREE_ARTIFACTS }} | |
| repository: nod-ai/SHARK-TestSuite | |
| run-id: 11323798734 | |
| - uses: actions/download-artifact@master | |
| with: | |
| name: ci_reports_${{ matrix.backend }}_vai-hf-cnn-fp32-shard2_onnx_md | |
| path: ./e2eshark-reports/ci_reports_${{ matrix.backend }}_vai-hf-cnn-fp32-shard2_onnx_md | |
| github-token: ${{ secrets.IREE_ARTIFACTS }} | |
| repository: nod-ai/SHARK-TestSuite | |
| run-id: 11323798734 | |
| - uses: actions/download-artifact@master | |
| with: | |
| name: ci_reports_${{ matrix.backend }}_vai-hf-cnn-fp32-shard2_onnx_json | |
| path: ./e2eshark-reports/ci_reports_${{ matrix.backend }}_vai-hf-cnn-fp32-shard2_onnx_json | |
| github-token: ${{ secrets.IREE_ARTIFACTS }} | |
| repository: nod-ai/SHARK-TestSuite | |
| run-id: 11323798734 | |
| - uses: actions/download-artifact@master | |
| with: | |
| name: ci_reports_${{ matrix.backend }}_vai-hf-cnn-fp32-shard3_onnx_md | |
| path: ./e2eshark-reports/ci_reports_${{ matrix.backend }}_vai-hf-cnn-fp32-shard3_onnx_md | |
| github-token: ${{ secrets.IREE_ARTIFACTS }} | |
| repository: nod-ai/SHARK-TestSuite | |
| run-id: 11323798734 | |
| - uses: actions/download-artifact@master | |
| with: | |
| name: ci_reports_${{ matrix.backend }}_vai-hf-cnn-fp32-shard3_onnx_json | |
| path: ./e2eshark-reports/ci_reports_${{ matrix.backend }}_vai-hf-cnn-fp32-shard3_onnx_json | |
| github-token: ${{ secrets.IREE_ARTIFACTS }} | |
| repository: nod-ai/SHARK-TestSuite | |
| run-id: 11323798734 | |
| - uses: actions/download-artifact@master | |
| with: | |
| name: ci_reports_${{ matrix.backend }}_vai-int8-p0p1-shard1_onnx_md | |
| path: ./e2eshark-reports/ci_reports_${{ matrix.backend }}_vai-int8-p0p1-shard1_onnx_md | |
| github-token: ${{ secrets.IREE_ARTIFACTS }} | |
| repository: nod-ai/SHARK-TestSuite | |
| run-id: 11323798734 | |
| - uses: actions/download-artifact@master | |
| with: | |
| name: ci_reports_${{ matrix.backend }}_vai-int8-p0p1-shard1_onnx_json | |
| path: ./e2eshark-reports/ci_reports_${{ matrix.backend }}_vai-int8-p0p1-shard1_onnx_json | |
| github-token: ${{ secrets.IREE_ARTIFACTS }} | |
| repository: nod-ai/SHARK-TestSuite | |
| run-id: 11323798734 | |
| - uses: actions/download-artifact@master | |
| with: | |
| name: ci_reports_${{ matrix.backend }}_vai-int8-p0p1-shard2_onnx_md | |
| path: ./e2eshark-reports/ci_reports_${{ matrix.backend }}_vai-int8-p0p1-shard2_onnx_md | |
| github-token: ${{ secrets.IREE_ARTIFACTS }} | |
| repository: nod-ai/SHARK-TestSuite | |
| run-id: 11323798734 | |
| - uses: actions/download-artifact@master | |
| with: | |
| name: ci_reports_${{ matrix.backend }}_vai-int8-p0p1-shard2_onnx_json | |
| path: ./e2eshark-reports/ci_reports_${{ matrix.backend }}_vai-int8-p0p1-shard2_onnx_json | |
| github-token: ${{ secrets.IREE_ARTIFACTS }} | |
| repository: nod-ai/SHARK-TestSuite | |
| run-id: 11323798734 | |
| - uses: actions/download-artifact@master | |
| with: | |
| name: ci_reports_${{ matrix.backend }}_vai-int8-p0p1-shard3_onnx_md | |
| path: ./e2eshark-reports/ci_reports_${{ matrix.backend }}_vai-int8-p0p1-shard3_onnx_md | |
| github-token: ${{ secrets.IREE_ARTIFACTS }} | |
| repository: nod-ai/SHARK-TestSuite | |
| run-id: 11323798734 | |
| - uses: actions/download-artifact@master | |
| with: | |
| name: ci_reports_${{ matrix.backend }}_vai-int8-p0p1-shard3_onnx_json | |
| path: ./e2eshark-reports/ci_reports_${{ matrix.backend }}_vai-int8-p0p1-shard3_onnx_json | |
| github-token: ${{ secrets.IREE_ARTIFACTS }} | |
| repository: nod-ai/SHARK-TestSuite | |
| run-id: 11323798734 | |
| - uses: actions/download-artifact@master | |
| with: | |
| name: ci_reports_${{ matrix.backend }}_vai-vision-int8_onnx_md | |
| path: ./e2eshark-reports/ci_reports_${{ matrix.backend }}_vai-vision-int8_onnx_md | |
| github-token: ${{ secrets.IREE_ARTIFACTS }} | |
| repository: nod-ai/SHARK-TestSuite | |
| run-id: 11323798734 | |
| - uses: actions/download-artifact@master | |
| with: | |
| name: ci_reports_${{ matrix.backend }}_vai-vision-int8_onnx_json | |
| path: ./e2eshark-reports/ci_reports_${{ matrix.backend }}_vai-vision-int8_onnx_json | |
| github-token: ${{ secrets.IREE_ARTIFACTS }} | |
| repository: nod-ai/SHARK-TestSuite | |
| run-id: 11323798734 | |
| - uses: actions/download-artifact@master | |
| with: | |
| name: ci_reports_${{ matrix.backend }}_migraphx_onnx_md | |
| path: ./e2eshark-reports/ci_reports_${{ matrix.backend }}_migraphx_onnx_md | |
| github-token: ${{ secrets.IREE_ARTIFACTS }} | |
| repository: nod-ai/SHARK-TestSuite | |
| run-id: 11323798734 | |
| - uses: actions/download-artifact@master | |
| with: | |
| name: ci_reports_${{ matrix.backend }}_migraphx_onnx_json | |
| path: ./e2eshark-reports/ci_reports_${{ matrix.backend }}_migraphx_onnx_json | |
| github-token: ${{ secrets.IREE_ARTIFACTS }} | |
| repository: nod-ai/SHARK-TestSuite | |
| run-id: 11323798734 | |
| - uses: actions/download-artifact@master | |
| with: | |
| name: ci_reports_${{ matrix.backend }}_nlp-shard1_onnx_md | |
| path: ./e2eshark-reports/ci_reports_${{ matrix.backend }}_nlp-shard1_onnx_md | |
| github-token: ${{ secrets.IREE_ARTIFACTS }} | |
| repository: nod-ai/SHARK-TestSuite | |
| run-id: 11323798734 | |
| - uses: actions/download-artifact@master | |
| with: | |
| name: ci_reports_${{ matrix.backend }}_nlp-shard1_onnx_json | |
| path: ./e2eshark-reports/ci_reports_${{ matrix.backend }}_nlp-shard1_onnx_json | |
| github-token: ${{ secrets.IREE_ARTIFACTS }} | |
| repository: nod-ai/SHARK-TestSuite | |
| run-id: 11323798734 | |
| - uses: actions/download-artifact@master | |
| with: | |
| name: ci_reports_${{ matrix.backend }}_nlp-shard2_onnx_md | |
| path: ./e2eshark-reports/ci_reports_${{ matrix.backend }}_nlp-shard2_onnx_md | |
| github-token: ${{ secrets.IREE_ARTIFACTS }} | |
| repository: nod-ai/SHARK-TestSuite | |
| run-id: 11323798734 | |
| - uses: actions/download-artifact@master | |
| with: | |
| name: ci_reports_${{ matrix.backend }}_nlp-shard2_onnx_json | |
| path: ./e2eshark-reports/ci_reports_${{ matrix.backend }}_nlp-shard2_onnx_json | |
| github-token: ${{ secrets.IREE_ARTIFACTS }} | |
| repository: nod-ai/SHARK-TestSuite | |
| run-id: 11323798734 | |
| - uses: actions/download-artifact@master | |
| with: | |
| name: ci_reports_${{ matrix.backend }}_nlp-shard3_onnx_md | |
| path: ./e2eshark-reports/ci_reports_${{ matrix.backend }}_nlp-shard3_onnx_md | |
| github-token: ${{ secrets.IREE_ARTIFACTS }} | |
| repository: nod-ai/SHARK-TestSuite | |
| run-id: 11323798734 | |
| - uses: actions/download-artifact@master | |
| with: | |
| name: ci_reports_${{ matrix.backend }}_nlp-shard3_onnx_json | |
| path: ./e2eshark-reports/ci_reports_${{ matrix.backend }}_nlp-shard3_onnx_json | |
| github-token: ${{ secrets.IREE_ARTIFACTS }} | |
| repository: nod-ai/SHARK-TestSuite | |
| run-id: 11323798734 | |
| - name: Merge Reports | |
| run: | | |
| source report_venv_alt/bin/activate | |
| python ./test-suite/alt_e2eshark/utils/merge_dicts.py \ | |
| --sources ./e2eshark-reports/ci_reports_${{ matrix.backend }}_vai-hf-cnn-fp32-shard1_onnx_json/vai-hf-cnn-fp32-shard1.json \ | |
| ./e2eshark-reports/ci_reports_${{ matrix.backend }}_vai-hf-cnn-fp32-shard2_onnx_json/vai-hf-cnn-fp32-shard2.json \ | |
| ./e2eshark-reports/ci_reports_${{ matrix.backend }}_vai-hf-cnn-fp32-shard3_onnx_json/vai-hf-cnn-fp32-shard3.json \ | |
| --output ./e2eshark-reports/vai-hf-cnn-fp32.json \ | |
| --report --report-file ./e2eshark-reports/vai-hf-cnn-fp32.md | |
| python ./test-suite/alt_e2eshark/utils/merge_dicts.py \ | |
| --sources ./e2eshark-reports/ci_reports_${{ matrix.backend }}_vai-int8-p0p1-shard1_onnx_json/vai-int8-p0p1-shard1.json \ | |
| ./e2eshark-reports/ci_reports_${{ matrix.backend }}_vai-int8-p0p1-shard2_onnx_json/vai-int8-p0p1-shard2.json \ | |
| ./e2eshark-reports/ci_reports_${{ matrix.backend }}_vai-int8-p0p1-shard3_onnx_json/vai-int8-p0p1-shard3.json \ | |
| --output ./e2eshark-reports/vai-int8-p0p1.json \ | |
| --report --report-file ./e2eshark-reports/vai-int8-p0p1.md | |
| python ./test-suite/alt_e2eshark/utils/merge_dicts.py \ | |
| --sources ./e2eshark-reports/ci_reports_${{ matrix.backend }}_nlp-shard1_onnx_json/nlp-shard1.json \ | |
| ./e2eshark-reports/ci_reports_${{ matrix.backend }}_nlp-shard2_onnx_json/nlp-shard2.json \ | |
| ./e2eshark-reports/ci_reports_${{ matrix.backend }}_nlp-shard3_onnx_json/nlp-shard3.json \ | |
| --output ./e2eshark-reports/nlp.json \ | |
| --report --report-file ./e2eshark-reports/nlp.md | |
| python ./test-suite/alt_e2eshark/utils/merge_dicts.py \ | |
| --sources ./e2eshark-reports/vai-int8-p0p1.json \ | |
| ./e2eshark-reports/vai-hf-cnn-fp32.json \ | |
| ./e2eshark-reports/nlp.json \ | |
| ./e2eshark-reports/ci_reports_${{ matrix.backend }}_shark-test-suite_onnx_json/shark-test-suite.json \ | |
| ./e2eshark-reports/ci_reports_${{ matrix.backend }}_vai-vision-int8_onnx_json/vai-vision-int8.json \ | |
| ./e2eshark-reports/ci_reports_${{ matrix.backend }}_migraphx_onnx_json/migraphx.json \ | |
| --output ./e2eshark-reports/combined_reports.json \ | |
| --report --report-file ./e2eshark-reports/combined_reports.md | |
| # - name: Push status artifacts | |
| # run: | | |
| # git config user.name "GitHub Actions Bot" | |
| # git config user.email "<>" | |
| # git pull | |
| # date=$(date '+%Y-%m-%d') | |
| # mkdir -p ${date}/ci_reports_onnx/${{ matrix.backend }}/vai-hf-cnn-fp32 | |
| # mkdir -p ${date}/ci_reports_onnx/${{ matrix.backend }}/vai-int8-p0p1 | |
| # mkdir -p ${date}/ci_reports_onnx/${{ matrix.backend }}/shark-test-suite | |
| # mkdir -p ${date}/ci_reports_onnx/${{ matrix.backend }}/vai-vision-int8 | |
| # mkdir -p ${date}/ci_reports_onnx/${{ matrix.backend }}/migraphx | |
| # mkdir -p ${date}/ci_reports_onnx/${{ matrix.backend }}/nlp | |
| # mkdir -p ${date}/ci_reports_onnx/${{ matrix.backend }}/combined-reports | |
| # cp vai-hf-cnn-fp32.md ${date}/ci_reports_onnx/${{ matrix.backend }}/vai-hf-cnn-fp32/summary.md | |
| # cp vai-int8-p0p1.md ${date}/ci_reports_onnx/${{ matrix.backend }}/vai-int8-p0p1/summary.md | |
| # cp nlp.md ${date}/ci_reports_onnx/${{ matrix.backend }}/nlp/summary.md | |
| # cp ci_reports_${{ matrix.backend }}_shark-test-suite_onnx_md/shark-test-suite.md ${date}/ci_reports_onnx/${{ matrix.backend }}/shark-test-suite/summary.md | |
| # cp ci_reports_${{ matrix.backend }}_vai-vision-int8_onnx_md/vai-vision-int8.md ${date}/ci_reports_onnx/${{ matrix.backend }}/vai-vision-int8/summary.md | |
| # cp ci_reports_${{ matrix.backend }}_migraphx_onnx_md/migraphx.md ${date}/ci_reports_onnx/${{ matrix.backend }}/migraphx/summary.md | |
| # cp combined_reports.md ${date}/ci_reports_onnx/${{ matrix.backend }}/combined-reports/summary.md | |
| # git add $date | |
| # git commit -m "add CI status reports for e2eshark for ${{ matrix.backend }}" | |
| # git push origin main | |
| # working-directory: ./e2eshark-reports | |
| - name: Regression Reports | |
| run: | | |
| source report_venv_alt/bin/activate | |
| cd test-suite | |
| mkdir latest | |
| mkdir baseline | |
| wget https://sharkpublic.blob.core.windows.net/sharkpublic/latest-test-suite/combined_reports_${{ matrix.backend }}.json -O latest/combined_reports_${{ matrix.backend }}.json | |
| wget https://sharkpublic.blob.core.windows.net/sharkpublic/baseline-test-suite/combined_reports_${{ matrix.backend }}.json -O baseline/combined_reports_${{ matrix.backend }}.json | |
| cd .. | |
| python ./test-suite/alt_e2eshark/utils/regressions.py \ | |
| --new ./e2eshark-reports/combined_reports.json \ | |
| --old ./test-suite/latest/combined_reports_${{ matrix.backend }}.json \ | |
| --report-file ./e2eshark-reports/yesterday_comparison.md \ | |
| --perf_tol_regression=0.05 \ | |
| --perf_tol_progression=0.05 | |
| python ./test-suite/alt_e2eshark/utils/regressions.py \ | |
| --new ./e2eshark-reports/combined_reports.json \ | |
| --old ./test-suite/baseline/combined_reports_${{ matrix.backend }}.json \ | |
| --report-file ./e2eshark-reports/baseline_comparison.md \ | |
| --perf_tol_regression=0.1 \ | |
| --perf_tol_progression=0.1 | |
| az storage blob upload --account-name sharkpublic --container-name sharkpublic \ | |
| --name latest-test-suite/combined_reports_${{ matrix.backend }}.json \ | |
| --file ./e2eshark-reports/combined_reports.json \ | |
| --account-key ${AZ_PUBLIC_KEY} --overwrite | |
| - name: Push regression artifacts | |
| run: | | |
| git config user.name "GitHub Actions Bot" | |
| git config user.email "<>" | |
| git pull | |
| date=$(date '+%Y-%m-%d') | |
| cp yesterday_comparison.md ${date}/ci_reports_onnx/${{ matrix.backend }}/combined-reports/yesterday_comparison.md | |
| cp baseline_comparison.md ${date}/ci_reports_onnx/${{ matrix.backend }}/combined-reports/baseline_comparison.md | |
| git add $date | |
| git commit -m "add CI regression reports for e2eshark for ${{ matrix.backend }}" | |
| git push origin main | |
| working-directory: ./e2eshark-reports |