From 2badfe4e35d36f5bb5f45b035cd11d26b297ed71 Mon Sep 17 00:00:00 2001 From: saienduri Date: Wed, 2 Oct 2024 14:22:24 -0500 Subject: [PATCH] test artifact download --- .github/workflows/test_e2eshark.yml | 813 ++++++++++++++-------------- 1 file changed, 410 insertions(+), 403 deletions(-) diff --git a/.github/workflows/test_e2eshark.yml b/.github/workflows/test_e2eshark.yml index ef3e0de2..e314f5fc 100644 --- a/.github/workflows/test_e2eshark.yml +++ b/.github/workflows/test_e2eshark.yml @@ -27,167 +27,167 @@ jobs: target-chip: gfx942 test-file: nlp-shard1 cache-dir: /home/sai/shark-test-suite-models-cache - - name: mi300_gpu2_test - runs-on: nodai-amdgpu-mi300-x86-64 - backend: rocm - device: hip - target-chip: gfx942 - test-file: nlp-shard2 - cache-dir: /home/sai/shark-test-suite-models-cache - - name: mi300_gpu3_test - runs-on: nodai-amdgpu-mi300-x86-64 - backend: rocm - device: hip - target-chip: gfx942 - test-file: nlp-shard3 - cache-dir: /home/sai/shark-test-suite-models-cache - - name: mi300_gpu4_test - runs-on: nodai-amdgpu-mi300-x86-64 - backend: rocm - device: hip - target-chip: gfx942 - test-file: shark-test-suite - cache-dir: /home/sai/shark-test-suite-models-cache - - name: mi300_gpu5_test - runs-on: nodai-amdgpu-mi300-x86-64 - backend: rocm - device: hip - target-chip: gfx942 - test-file: vai-hf-cnn-fp32-shard1 - cache-dir: /home/sai/shark-test-suite-models-cache - - name: mi300_gpu6_test - runs-on: nodai-amdgpu-mi300-x86-64 - backend: rocm - device: hip - target-chip: gfx942 - test-file: vai-hf-cnn-fp32-shard2 - cache-dir: /home/sai/shark-test-suite-models-cache - - name: mi300_gpu7_test - runs-on: nodai-amdgpu-mi300-x86-64 - backend: rocm - device: hip - target-chip: gfx942 - test-file: vai-hf-cnn-fp32-shard3 - cache-dir: /home/sai/shark-test-suite-models-cache - - name: mi300_gpu8_test - runs-on: nodai-amdgpu-mi300-x86-64 - backend: rocm - device: hip - target-chip: gfx942 - test-file: vai-int8-p0p1-shard1 - cache-dir: /home/sai/shark-test-suite-models-cache - - name: mi300_gpu9_test - runs-on: nodai-amdgpu-mi300-x86-64 - backend: rocm - device: hip - target-chip: gfx942 - test-file: vai-int8-p0p1-shard2 - cache-dir: /home/sai/shark-test-suite-models-cache - - name: mi300_gpu10_test - runs-on: nodai-amdgpu-mi300-x86-64 - backend: rocm - device: hip - target-chip: gfx942 - test-file: vai-int8-p0p1-shard3 - cache-dir: /home/sai/shark-test-suite-models-cache - - name: mi300_gpu11_test - runs-on: nodai-amdgpu-mi300-x86-64 - backend: rocm - device: hip - target-chip: gfx942 - test-file: vai-vision-int8 - cache-dir: /home/sai/shark-test-suite-models-cache - - name: mi300_gpu12_test - runs-on: nodai-amdgpu-mi300-x86-64 - backend: rocm - device: hip - target-chip: gfx942 - test-file: migraphx - cache-dir: /home/sai/shark-test-suite-models-cache - - name: cpu_shard1_test - runs-on: nodai-amdgpu-mi250-x86-64 - backend: llvm-cpu - device: local-task - target-chip: x86_64-linux-gnu - test-file: nlp-shard1 - cache-dir: /groups/aig_sharks/test-suite-ci-cache - - name: cpu_shard2_test - runs-on: nodai-amdgpu-mi250-x86-64 - backend: llvm-cpu - device: local-task - target-chip: x86_64-linux-gnu - test-file: nlp-shard2 - cache-dir: /groups/aig_sharks/test-suite-ci-cache - - name: cpu_shard3_test - runs-on: nodai-amdgpu-mi250-x86-64 - backend: llvm-cpu - device: local-task - target-chip: x86_64-linux-gnu - test-file: nlp-shard3 - cache-dir: /groups/aig_sharks/test-suite-ci-cache - - name: cpu_shard4_test - runs-on: nodai-amdgpu-mi250-x86-64 - backend: llvm-cpu - device: local-task - target-chip: x86_64-linux-gnu - test-file: shark-test-suite - cache-dir: /groups/aig_sharks/test-suite-ci-cache - - name: cpu_shard5_test - runs-on: nodai-amdgpu-mi250-x86-64 - backend: llvm-cpu - device: local-task - target-chip: x86_64-linux-gnu - test-file: vai-hf-cnn-fp32-shard1 - cache-dir: /groups/aig_sharks/test-suite-ci-cache - - name: cpu_shard6_test - runs-on: nodai-amdgpu-mi250-x86-64 - backend: llvm-cpu - device: local-task - target-chip: x86_64-linux-gnu - test-file: vai-hf-cnn-fp32-shard2 - cache-dir: /groups/aig_sharks/test-suite-ci-cache - - name: cpu_shard7_test - runs-on: nodai-amdgpu-mi250-x86-64 - backend: llvm-cpu - device: local-task - target-chip: x86_64-linux-gnu - test-file: vai-hf-cnn-fp32-shard3 - cache-dir: /groups/aig_sharks/test-suite-ci-cache - - name: cpu_shard8_test - runs-on: nodai-amdgpu-mi250-x86-64 - backend: llvm-cpu - device: local-task - target-chip: x86_64-linux-gnu - test-file: vai-int8-p0p1-shard1 - cache-dir: /groups/aig_sharks/test-suite-ci-cache - - name: cpu_shard9_test - runs-on: nodai-amdgpu-mi250-x86-64 - backend: llvm-cpu - device: local-task - target-chip: x86_64-linux-gnu - test-file: vai-int8-p0p1-shard2 - cache-dir: /groups/aig_sharks/test-suite-ci-cache - - name: cpu_shard10_test - runs-on: nodai-amdgpu-mi250-x86-64 - backend: llvm-cpu - device: local-task - target-chip: x86_64-linux-gnu - test-file: vai-int8-p0p1-shard3 - cache-dir: /groups/aig_sharks/test-suite-ci-cache - - name: cpu_shard11_test - runs-on: nodai-amdgpu-mi250-x86-64 - backend: llvm-cpu - device: local-task - target-chip: x86_64-linux-gnu - test-file: vai-vision-int8 - cache-dir: /groups/aig_sharks/test-suite-ci-cache - - name: cpu_shard12_test - runs-on: nodai-amdgpu-mi250-x86-64 - backend: llvm-cpu - device: local-task - target-chip: x86_64-linux-gnu - test-file: migraphx - cache-dir: /groups/aig_sharks/test-suite-ci-cache + # - name: mi300_gpu2_test + # runs-on: nodai-amdgpu-mi300-x86-64 + # backend: rocm + # device: hip + # target-chip: gfx942 + # test-file: nlp-shard2 + # cache-dir: /home/sai/shark-test-suite-models-cache + # - name: mi300_gpu3_test + # runs-on: nodai-amdgpu-mi300-x86-64 + # backend: rocm + # device: hip + # target-chip: gfx942 + # test-file: nlp-shard3 + # cache-dir: /home/sai/shark-test-suite-models-cache + # - name: mi300_gpu4_test + # runs-on: nodai-amdgpu-mi300-x86-64 + # backend: rocm + # device: hip + # target-chip: gfx942 + # test-file: shark-test-suite + # cache-dir: /home/sai/shark-test-suite-models-cache + # - name: mi300_gpu5_test + # runs-on: nodai-amdgpu-mi300-x86-64 + # backend: rocm + # device: hip + # target-chip: gfx942 + # test-file: vai-hf-cnn-fp32-shard1 + # cache-dir: /home/sai/shark-test-suite-models-cache + # - name: mi300_gpu6_test + # runs-on: nodai-amdgpu-mi300-x86-64 + # backend: rocm + # device: hip + # target-chip: gfx942 + # test-file: vai-hf-cnn-fp32-shard2 + # cache-dir: /home/sai/shark-test-suite-models-cache + # - name: mi300_gpu7_test + # runs-on: nodai-amdgpu-mi300-x86-64 + # backend: rocm + # device: hip + # target-chip: gfx942 + # test-file: vai-hf-cnn-fp32-shard3 + # cache-dir: /home/sai/shark-test-suite-models-cache + # - name: mi300_gpu8_test + # runs-on: nodai-amdgpu-mi300-x86-64 + # backend: rocm + # device: hip + # target-chip: gfx942 + # test-file: vai-int8-p0p1-shard1 + # cache-dir: /home/sai/shark-test-suite-models-cache + # - name: mi300_gpu9_test + # runs-on: nodai-amdgpu-mi300-x86-64 + # backend: rocm + # device: hip + # target-chip: gfx942 + # test-file: vai-int8-p0p1-shard2 + # cache-dir: /home/sai/shark-test-suite-models-cache + # - name: mi300_gpu10_test + # runs-on: nodai-amdgpu-mi300-x86-64 + # backend: rocm + # device: hip + # target-chip: gfx942 + # test-file: vai-int8-p0p1-shard3 + # cache-dir: /home/sai/shark-test-suite-models-cache + # - name: mi300_gpu11_test + # runs-on: nodai-amdgpu-mi300-x86-64 + # backend: rocm + # device: hip + # target-chip: gfx942 + # test-file: vai-vision-int8 + # cache-dir: /home/sai/shark-test-suite-models-cache + # - name: mi300_gpu12_test + # runs-on: nodai-amdgpu-mi300-x86-64 + # backend: rocm + # device: hip + # target-chip: gfx942 + # test-file: migraphx + # cache-dir: /home/sai/shark-test-suite-models-cache + # - name: cpu_shard1_test + # runs-on: nodai-amdgpu-mi250-x86-64 + # backend: llvm-cpu + # device: local-task + # target-chip: x86_64-linux-gnu + # test-file: nlp-shard1 + # cache-dir: /groups/aig_sharks/test-suite-ci-cache + # - name: cpu_shard2_test + # runs-on: nodai-amdgpu-mi250-x86-64 + # backend: llvm-cpu + # device: local-task + # target-chip: x86_64-linux-gnu + # test-file: nlp-shard2 + # cache-dir: /groups/aig_sharks/test-suite-ci-cache + # - name: cpu_shard3_test + # runs-on: nodai-amdgpu-mi250-x86-64 + # backend: llvm-cpu + # device: local-task + # target-chip: x86_64-linux-gnu + # test-file: nlp-shard3 + # cache-dir: /groups/aig_sharks/test-suite-ci-cache + # - name: cpu_shard4_test + # runs-on: nodai-amdgpu-mi250-x86-64 + # backend: llvm-cpu + # device: local-task + # target-chip: x86_64-linux-gnu + # test-file: shark-test-suite + # cache-dir: /groups/aig_sharks/test-suite-ci-cache + # - name: cpu_shard5_test + # runs-on: nodai-amdgpu-mi250-x86-64 + # backend: llvm-cpu + # device: local-task + # target-chip: x86_64-linux-gnu + # test-file: vai-hf-cnn-fp32-shard1 + # cache-dir: /groups/aig_sharks/test-suite-ci-cache + # - name: cpu_shard6_test + # runs-on: nodai-amdgpu-mi250-x86-64 + # backend: llvm-cpu + # device: local-task + # target-chip: x86_64-linux-gnu + # test-file: vai-hf-cnn-fp32-shard2 + # cache-dir: /groups/aig_sharks/test-suite-ci-cache + # - name: cpu_shard7_test + # runs-on: nodai-amdgpu-mi250-x86-64 + # backend: llvm-cpu + # device: local-task + # target-chip: x86_64-linux-gnu + # test-file: vai-hf-cnn-fp32-shard3 + # cache-dir: /groups/aig_sharks/test-suite-ci-cache + # - name: cpu_shard8_test + # runs-on: nodai-amdgpu-mi250-x86-64 + # backend: llvm-cpu + # device: local-task + # target-chip: x86_64-linux-gnu + # test-file: vai-int8-p0p1-shard1 + # cache-dir: /groups/aig_sharks/test-suite-ci-cache + # - name: cpu_shard9_test + # runs-on: nodai-amdgpu-mi250-x86-64 + # backend: llvm-cpu + # device: local-task + # target-chip: x86_64-linux-gnu + # test-file: vai-int8-p0p1-shard2 + # cache-dir: /groups/aig_sharks/test-suite-ci-cache + # - name: cpu_shard10_test + # runs-on: nodai-amdgpu-mi250-x86-64 + # backend: llvm-cpu + # device: local-task + # target-chip: x86_64-linux-gnu + # test-file: vai-int8-p0p1-shard3 + # cache-dir: /groups/aig_sharks/test-suite-ci-cache + # - name: cpu_shard11_test + # runs-on: nodai-amdgpu-mi250-x86-64 + # backend: llvm-cpu + # device: local-task + # target-chip: x86_64-linux-gnu + # test-file: vai-vision-int8 + # cache-dir: /groups/aig_sharks/test-suite-ci-cache + # - name: cpu_shard12_test + # runs-on: nodai-amdgpu-mi250-x86-64 + # backend: llvm-cpu + # device: local-task + # target-chip: x86_64-linux-gnu + # test-file: migraphx + # cache-dir: /groups/aig_sharks/test-suite-ci-cache env: E2E_VENV_DIR: ${{ github.workspace }}/test-suite_venv ALT_E2E_VENV_DIR: ${{ github.workspace }}/alt-test-suite_venv @@ -201,256 +201,263 @@ jobs: repository: nod-ai/SHARK-TestSuite ref: alt-merge-reports path: test-suite + + - uses: actions/download-artifact@v4 + with: + name: linux_x86_64_release_packages + github-token: ${{ secrets.IREE_ARTIFACTS }} # token with actions:read permissions on target repo + repository: iree-org/iree + run-id: 11147803917 - - name: "Setup alt e2eshark python venv" - run: | - python3.11 -m venv ${ALT_E2E_VENV_DIR} - source ${ALT_E2E_VENV_DIR}/bin/activate - pip install --upgrade pip - pip install -r ./alt_e2eshark/base_requirements.txt - pip install -r ./alt_e2eshark/iree_requirements.txt - pip install --no-deps -r ./alt_e2eshark/torch_mlir_requirements.txt - working-directory: ./test-suite + # - name: "Setup alt e2eshark python venv" + # run: | + # python3.11 -m venv ${ALT_E2E_VENV_DIR} + # source ${ALT_E2E_VENV_DIR}/bin/activate + # pip install --upgrade pip + # pip install -r ./alt_e2eshark/base_requirements.txt + # pip install -r ./alt_e2eshark/iree_requirements.txt + # pip install --no-deps -r ./alt_e2eshark/torch_mlir_requirements.txt + # working-directory: ./test-suite - - name: Run Onnx Bench Mode - if: contains(matrix.test-file, 'migraphx') - run: | - source ${ALT_E2E_VENV_DIR}/bin/activate - pip freeze - cd alt_e2eshark - free -mh - python3.11 ./run.py \ - -r ./test-onnx \ - --report \ - --torchtolinalg \ - --testsfile onnx_tests/models/external_lists/${{ matrix.test-file }}.txt \ - -b ${{ matrix.backend }} \ - -d ${{ matrix.device }} \ - --report-file reports/${{ matrix.test-file }}.md \ - --mode=cl-onnx-iree \ - --cleanup=3 \ - --benchmark \ - -v - working-directory: ./test-suite + # - name: Run Onnx Bench Mode + # if: contains(matrix.test-file, 'migraphx') + # run: | + # source ${ALT_E2E_VENV_DIR}/bin/activate + # pip freeze + # cd alt_e2eshark + # free -mh + # python3.11 ./run.py \ + # -r ./test-onnx \ + # --report \ + # --torchtolinalg \ + # --testsfile onnx_tests/models/external_lists/${{ matrix.test-file }}.txt \ + # -b ${{ matrix.backend }} \ + # -d ${{ matrix.device }} \ + # --report-file reports/${{ matrix.test-file }}.md \ + # --mode=cl-onnx-iree \ + # --cleanup=3 \ + # --benchmark \ + # -v + # working-directory: ./test-suite - - name: Run Onnx Default Mode - if: ${{ ! contains(matrix.test-file, 'migraphx') }} - run: | - source ${ALT_E2E_VENV_DIR}/bin/activate - pip freeze - cd alt_e2eshark - free -mh - python3.11 ./run.py \ - -r ./test-onnx \ - --report \ - --torchtolinalg \ - --testsfile onnx_tests/models/external_lists/${{ matrix.test-file }}.txt \ - -b ${{ matrix.backend }} \ - -d ${{ matrix.device }} \ - --report-file reports/${{ matrix.test-file }}.md \ - --mode=cl-onnx-iree \ - --cleanup=3 \ - -v - working-directory: ./test-suite + # - name: Run Onnx Default Mode + # if: ${{ ! contains(matrix.test-file, 'migraphx') }} + # run: | + # source ${ALT_E2E_VENV_DIR}/bin/activate + # pip freeze + # cd alt_e2eshark + # free -mh + # python3.11 ./run.py \ + # -r ./test-onnx \ + # --report \ + # --torchtolinalg \ + # --testsfile onnx_tests/models/external_lists/${{ matrix.test-file }}.txt \ + # -b ${{ matrix.backend }} \ + # -d ${{ matrix.device }} \ + # --report-file reports/${{ matrix.test-file }}.md \ + # --mode=cl-onnx-iree \ + # --cleanup=3 \ + # -v + # working-directory: ./test-suite - - uses: actions/upload-artifact@master - with: - name: ci_reports_${{ matrix.backend }}_${{ matrix.test-file }}_onnx_md - path: ./test-suite/alt_e2eshark/reports/${{ matrix.test-file }}.md + # - uses: actions/upload-artifact@master + # with: + # name: ci_reports_${{ matrix.backend }}_${{ matrix.test-file }}_onnx_md + # path: ./test-suite/alt_e2eshark/reports/${{ matrix.test-file }}.md - - uses: actions/upload-artifact@master - with: - name: ci_reports_${{ matrix.backend }}_${{ matrix.test-file }}_onnx_json - path: ./test-suite/alt_e2eshark/reports/${{ matrix.test-file }}.json + # - uses: actions/upload-artifact@master + # with: + # name: ci_reports_${{ matrix.backend }}_${{ matrix.test-file }}_onnx_json + # path: ./test-suite/alt_e2eshark/reports/${{ matrix.test-file }}.json - push_artifacts: - needs: [e2eshark] - runs-on: ${{ matrix.runs-on }} - strategy: - fail-fast: false - max-parallel: 1 - matrix: - include: - - name: merge_rocm_reports - runs-on: nodai-amdgpu-mi300-x86-64 - backend: rocm - - name: merge_cpu_reports - runs-on: nodai-amdgpu-mi250-x86-64 - backend: llvm-cpu - steps: - - name: Checkout Test Suite - uses: actions/checkout@v2 - with: - repository: nod-ai/SHARK-TestSuite - ref: alt-merge-reports - path: test-suite - - name: Checkout repo - uses: actions/checkout@v2 - with: - repository: nod-ai/e2eshark-reports - ref: main - token: ${{ secrets.E2ESHARK_GITHUB_TOKEN }} - path: e2eshark-reports - - name: "Setup alt test suite venv" - run: | - python3.11 -m venv report_venv_alt - source report_venv_alt/bin/activate - pip install --upgrade pip - pip install -r ./test-suite/alt_e2eshark/base_requirements.txt - pip install -r ./test-suite/alt_e2eshark/iree_requirements.txt - pip install --no-deps -r ./test-suite/alt_e2eshark/torch_mlir_requirements.txt - - uses: actions/download-artifact@master - with: - name: ci_reports_${{ matrix.backend }}_shark-test-suite_onnx_md - path: ./e2eshark-reports/ci_reports_${{ matrix.backend }}_shark-test-suite_onnx_md - - uses: actions/download-artifact@master - with: - name: ci_reports_${{ matrix.backend }}_shark-test-suite_onnx_json - path: ./e2eshark-reports/ci_reports_${{ matrix.backend }}_shark-test-suite_onnx_json - - uses: actions/download-artifact@master - with: - name: ci_reports_${{ matrix.backend }}_vai-hf-cnn-fp32-shard1_onnx_md - path: ./e2eshark-reports/ci_reports_${{ matrix.backend }}_vai-hf-cnn-fp32-shard1_onnx_md - - uses: actions/download-artifact@master - with: - name: ci_reports_${{ matrix.backend }}_vai-hf-cnn-fp32-shard1_onnx_json - path: ./e2eshark-reports/ci_reports_${{ matrix.backend }}_vai-hf-cnn-fp32-shard1_onnx_json - - uses: actions/download-artifact@master - with: - name: ci_reports_${{ matrix.backend }}_vai-hf-cnn-fp32-shard2_onnx_md - path: ./e2eshark-reports/ci_reports_${{ matrix.backend }}_vai-hf-cnn-fp32-shard2_onnx_md - - uses: actions/download-artifact@master - with: - name: ci_reports_${{ matrix.backend }}_vai-hf-cnn-fp32-shard2_onnx_json - path: ./e2eshark-reports/ci_reports_${{ matrix.backend }}_vai-hf-cnn-fp32-shard2_onnx_json - - uses: actions/download-artifact@master - with: - name: ci_reports_${{ matrix.backend }}_vai-hf-cnn-fp32-shard3_onnx_md - path: ./e2eshark-reports/ci_reports_${{ matrix.backend }}_vai-hf-cnn-fp32-shard3_onnx_md - - uses: actions/download-artifact@master - with: - name: ci_reports_${{ matrix.backend }}_vai-hf-cnn-fp32-shard3_onnx_json - path: ./e2eshark-reports/ci_reports_${{ matrix.backend }}_vai-hf-cnn-fp32-shard3_onnx_json - - uses: actions/download-artifact@master - with: - name: ci_reports_${{ matrix.backend }}_vai-int8-p0p1-shard1_onnx_md - path: ./e2eshark-reports/ci_reports_${{ matrix.backend }}_vai-int8-p0p1-shard1_onnx_md - - uses: actions/download-artifact@master - with: - name: ci_reports_${{ matrix.backend }}_vai-int8-p0p1-shard1_onnx_json - path: ./e2eshark-reports/ci_reports_${{ matrix.backend }}_vai-int8-p0p1-shard1_onnx_json - - uses: actions/download-artifact@master - with: - name: ci_reports_${{ matrix.backend }}_vai-int8-p0p1-shard2_onnx_md - path: ./e2eshark-reports/ci_reports_${{ matrix.backend }}_vai-int8-p0p1-shard2_onnx_md - - uses: actions/download-artifact@master - with: - name: ci_reports_${{ matrix.backend }}_vai-int8-p0p1-shard2_onnx_json - path: ./e2eshark-reports/ci_reports_${{ matrix.backend }}_vai-int8-p0p1-shard2_onnx_json - - uses: actions/download-artifact@master - with: - name: ci_reports_${{ matrix.backend }}_vai-int8-p0p1-shard3_onnx_md - path: ./e2eshark-reports/ci_reports_${{ matrix.backend }}_vai-int8-p0p1-shard3_onnx_md - - uses: actions/download-artifact@master - with: - name: ci_reports_${{ matrix.backend }}_vai-int8-p0p1-shard3_onnx_json - path: ./e2eshark-reports/ci_reports_${{ matrix.backend }}_vai-int8-p0p1-shard3_onnx_json - - uses: actions/download-artifact@master - with: - name: ci_reports_${{ matrix.backend }}_vai-vision-int8_onnx_md - path: ./e2eshark-reports/ci_reports_${{ matrix.backend }}_vai-vision-int8_onnx_md - - uses: actions/download-artifact@master - with: - name: ci_reports_${{ matrix.backend }}_vai-vision-int8_onnx_json - path: ./e2eshark-reports/ci_reports_${{ matrix.backend }}_vai-vision-int8_onnx_json - - uses: actions/download-artifact@master - with: - name: ci_reports_${{ matrix.backend }}_migraphx_onnx_md - path: ./e2eshark-reports/ci_reports_${{ matrix.backend }}_migraphx_onnx_md - - uses: actions/download-artifact@master - with: - name: ci_reports_${{ matrix.backend }}_migraphx_onnx_json - path: ./e2eshark-reports/ci_reports_${{ matrix.backend }}_migraphx_onnx_json - - uses: actions/download-artifact@master - with: - name: ci_reports_${{ matrix.backend }}_nlp-shard1_onnx_md - path: ./e2eshark-reports/ci_reports_${{ matrix.backend }}_nlp-shard1_onnx_md - - uses: actions/download-artifact@master - with: - name: ci_reports_${{ matrix.backend }}_nlp-shard1_onnx_json - path: ./e2eshark-reports/ci_reports_${{ matrix.backend }}_nlp-shard1_onnx_json - - uses: actions/download-artifact@master - with: - name: ci_reports_${{ matrix.backend }}_nlp-shard2_onnx_md - path: ./e2eshark-reports/ci_reports_${{ matrix.backend }}_nlp-shard2_onnx_md - - uses: actions/download-artifact@master - with: - name: ci_reports_${{ matrix.backend }}_nlp-shard2_onnx_json - path: ./e2eshark-reports/ci_reports_${{ matrix.backend }}_nlp-shard2_onnx_json - - uses: actions/download-artifact@master - with: - name: ci_reports_${{ matrix.backend }}_nlp-shard3_onnx_md - path: ./e2eshark-reports/ci_reports_${{ matrix.backend }}_nlp-shard3_onnx_md - - uses: actions/download-artifact@master - with: - name: ci_reports_${{ matrix.backend }}_nlp-shard3_onnx_json - path: ./e2eshark-reports/ci_reports_${{ matrix.backend }}_nlp-shard3_onnx_json - - name: Merge Reports - run: | - source report_venv_alt/bin/activate + # push_artifacts: + # needs: [e2eshark] + # runs-on: ${{ matrix.runs-on }} + # strategy: + # fail-fast: false + # max-parallel: 1 + # matrix: + # include: + # - name: merge_rocm_reports + # runs-on: nodai-amdgpu-mi300-x86-64 + # backend: rocm + # - name: merge_cpu_reports + # runs-on: nodai-amdgpu-mi250-x86-64 + # backend: llvm-cpu + # steps: + # - name: Checkout Test Suite + # uses: actions/checkout@v2 + # with: + # repository: nod-ai/SHARK-TestSuite + # ref: alt-merge-reports + # path: test-suite + # - name: Checkout repo + # uses: actions/checkout@v2 + # with: + # repository: nod-ai/e2eshark-reports + # ref: main + # token: ${{ secrets.E2ESHARK_GITHUB_TOKEN }} + # path: e2eshark-reports + # - name: "Setup alt test suite venv" + # run: | + # python3.11 -m venv report_venv_alt + # source report_venv_alt/bin/activate + # pip install --upgrade pip + # pip install -r ./test-suite/alt_e2eshark/base_requirements.txt + # pip install -r ./test-suite/alt_e2eshark/iree_requirements.txt + # pip install --no-deps -r ./test-suite/alt_e2eshark/torch_mlir_requirements.txt + # - uses: actions/download-artifact@master + # with: + # name: ci_reports_${{ matrix.backend }}_shark-test-suite_onnx_md + # path: ./e2eshark-reports/ci_reports_${{ matrix.backend }}_shark-test-suite_onnx_md + # - uses: actions/download-artifact@master + # with: + # name: ci_reports_${{ matrix.backend }}_shark-test-suite_onnx_json + # path: ./e2eshark-reports/ci_reports_${{ matrix.backend }}_shark-test-suite_onnx_json + # - uses: actions/download-artifact@master + # with: + # name: ci_reports_${{ matrix.backend }}_vai-hf-cnn-fp32-shard1_onnx_md + # path: ./e2eshark-reports/ci_reports_${{ matrix.backend }}_vai-hf-cnn-fp32-shard1_onnx_md + # - uses: actions/download-artifact@master + # with: + # name: ci_reports_${{ matrix.backend }}_vai-hf-cnn-fp32-shard1_onnx_json + # path: ./e2eshark-reports/ci_reports_${{ matrix.backend }}_vai-hf-cnn-fp32-shard1_onnx_json + # - uses: actions/download-artifact@master + # with: + # name: ci_reports_${{ matrix.backend }}_vai-hf-cnn-fp32-shard2_onnx_md + # path: ./e2eshark-reports/ci_reports_${{ matrix.backend }}_vai-hf-cnn-fp32-shard2_onnx_md + # - uses: actions/download-artifact@master + # with: + # name: ci_reports_${{ matrix.backend }}_vai-hf-cnn-fp32-shard2_onnx_json + # path: ./e2eshark-reports/ci_reports_${{ matrix.backend }}_vai-hf-cnn-fp32-shard2_onnx_json + # - uses: actions/download-artifact@master + # with: + # name: ci_reports_${{ matrix.backend }}_vai-hf-cnn-fp32-shard3_onnx_md + # path: ./e2eshark-reports/ci_reports_${{ matrix.backend }}_vai-hf-cnn-fp32-shard3_onnx_md + # - uses: actions/download-artifact@master + # with: + # name: ci_reports_${{ matrix.backend }}_vai-hf-cnn-fp32-shard3_onnx_json + # path: ./e2eshark-reports/ci_reports_${{ matrix.backend }}_vai-hf-cnn-fp32-shard3_onnx_json + # - uses: actions/download-artifact@master + # with: + # name: ci_reports_${{ matrix.backend }}_vai-int8-p0p1-shard1_onnx_md + # path: ./e2eshark-reports/ci_reports_${{ matrix.backend }}_vai-int8-p0p1-shard1_onnx_md + # - uses: actions/download-artifact@master + # with: + # name: ci_reports_${{ matrix.backend }}_vai-int8-p0p1-shard1_onnx_json + # path: ./e2eshark-reports/ci_reports_${{ matrix.backend }}_vai-int8-p0p1-shard1_onnx_json + # - uses: actions/download-artifact@master + # with: + # name: ci_reports_${{ matrix.backend }}_vai-int8-p0p1-shard2_onnx_md + # path: ./e2eshark-reports/ci_reports_${{ matrix.backend }}_vai-int8-p0p1-shard2_onnx_md + # - uses: actions/download-artifact@master + # with: + # name: ci_reports_${{ matrix.backend }}_vai-int8-p0p1-shard2_onnx_json + # path: ./e2eshark-reports/ci_reports_${{ matrix.backend }}_vai-int8-p0p1-shard2_onnx_json + # - uses: actions/download-artifact@master + # with: + # name: ci_reports_${{ matrix.backend }}_vai-int8-p0p1-shard3_onnx_md + # path: ./e2eshark-reports/ci_reports_${{ matrix.backend }}_vai-int8-p0p1-shard3_onnx_md + # - uses: actions/download-artifact@master + # with: + # name: ci_reports_${{ matrix.backend }}_vai-int8-p0p1-shard3_onnx_json + # path: ./e2eshark-reports/ci_reports_${{ matrix.backend }}_vai-int8-p0p1-shard3_onnx_json + # - uses: actions/download-artifact@master + # with: + # name: ci_reports_${{ matrix.backend }}_vai-vision-int8_onnx_md + # path: ./e2eshark-reports/ci_reports_${{ matrix.backend }}_vai-vision-int8_onnx_md + # - uses: actions/download-artifact@master + # with: + # name: ci_reports_${{ matrix.backend }}_vai-vision-int8_onnx_json + # path: ./e2eshark-reports/ci_reports_${{ matrix.backend }}_vai-vision-int8_onnx_json + # - uses: actions/download-artifact@master + # with: + # name: ci_reports_${{ matrix.backend }}_migraphx_onnx_md + # path: ./e2eshark-reports/ci_reports_${{ matrix.backend }}_migraphx_onnx_md + # - uses: actions/download-artifact@master + # with: + # name: ci_reports_${{ matrix.backend }}_migraphx_onnx_json + # path: ./e2eshark-reports/ci_reports_${{ matrix.backend }}_migraphx_onnx_json + # - uses: actions/download-artifact@master + # with: + # name: ci_reports_${{ matrix.backend }}_nlp-shard1_onnx_md + # path: ./e2eshark-reports/ci_reports_${{ matrix.backend }}_nlp-shard1_onnx_md + # - uses: actions/download-artifact@master + # with: + # name: ci_reports_${{ matrix.backend }}_nlp-shard1_onnx_json + # path: ./e2eshark-reports/ci_reports_${{ matrix.backend }}_nlp-shard1_onnx_json + # - uses: actions/download-artifact@master + # with: + # name: ci_reports_${{ matrix.backend }}_nlp-shard2_onnx_md + # path: ./e2eshark-reports/ci_reports_${{ matrix.backend }}_nlp-shard2_onnx_md + # - uses: actions/download-artifact@master + # with: + # name: ci_reports_${{ matrix.backend }}_nlp-shard2_onnx_json + # path: ./e2eshark-reports/ci_reports_${{ matrix.backend }}_nlp-shard2_onnx_json + # - uses: actions/download-artifact@master + # with: + # name: ci_reports_${{ matrix.backend }}_nlp-shard3_onnx_md + # path: ./e2eshark-reports/ci_reports_${{ matrix.backend }}_nlp-shard3_onnx_md + # - uses: actions/download-artifact@master + # with: + # name: ci_reports_${{ matrix.backend }}_nlp-shard3_onnx_json + # path: ./e2eshark-reports/ci_reports_${{ matrix.backend }}_nlp-shard3_onnx_json + # - name: Merge Reports + # run: | + # source report_venv_alt/bin/activate - python ./test-suite/alt_e2eshark/utils/merge_dicts.py \ - --sources ./e2eshark-reports/ci_reports_${{ matrix.backend }}_vai-hf-cnn-fp32-shard1_onnx_json/vai-hf-cnn-fp32-shard1.json \ - ./e2eshark-reports/ci_reports_${{ matrix.backend }}_vai-hf-cnn-fp32-shard2_onnx_json/vai-hf-cnn-fp32-shard2.json \ - ./e2eshark-reports/ci_reports_${{ matrix.backend }}_vai-hf-cnn-fp32-shard3_onnx_json/vai-hf-cnn-fp32-shard3.json \ - --output ./e2eshark-reports/vai-hf-cnn-fp32.json \ - --report --report-file ./e2eshark-reports/vai-hf-cnn-fp32.md + # python ./test-suite/alt_e2eshark/utils/merge_dicts.py \ + # --sources ./e2eshark-reports/ci_reports_${{ matrix.backend }}_vai-hf-cnn-fp32-shard1_onnx_json/vai-hf-cnn-fp32-shard1.json \ + # ./e2eshark-reports/ci_reports_${{ matrix.backend }}_vai-hf-cnn-fp32-shard2_onnx_json/vai-hf-cnn-fp32-shard2.json \ + # ./e2eshark-reports/ci_reports_${{ matrix.backend }}_vai-hf-cnn-fp32-shard3_onnx_json/vai-hf-cnn-fp32-shard3.json \ + # --output ./e2eshark-reports/vai-hf-cnn-fp32.json \ + # --report --report-file ./e2eshark-reports/vai-hf-cnn-fp32.md - python ./test-suite/alt_e2eshark/utils/merge_dicts.py \ - --sources ./e2eshark-reports/ci_reports_${{ matrix.backend }}_vai-int8-p0p1-shard1_onnx_json/vai-int8-p0p1-shard1.json \ - ./e2eshark-reports/ci_reports_${{ matrix.backend }}_vai-int8-p0p1-shard2_onnx_json/vai-int8-p0p1-shard2.json \ - ./e2eshark-reports/ci_reports_${{ matrix.backend }}_vai-int8-p0p1-shard3_onnx_json/vai-int8-p0p1-shard3.json \ - --output ./e2eshark-reports/vai-int8-p0p1.json \ - --report --report-file ./e2eshark-reports/vai-int8-p0p1.md + # python ./test-suite/alt_e2eshark/utils/merge_dicts.py \ + # --sources ./e2eshark-reports/ci_reports_${{ matrix.backend }}_vai-int8-p0p1-shard1_onnx_json/vai-int8-p0p1-shard1.json \ + # ./e2eshark-reports/ci_reports_${{ matrix.backend }}_vai-int8-p0p1-shard2_onnx_json/vai-int8-p0p1-shard2.json \ + # ./e2eshark-reports/ci_reports_${{ matrix.backend }}_vai-int8-p0p1-shard3_onnx_json/vai-int8-p0p1-shard3.json \ + # --output ./e2eshark-reports/vai-int8-p0p1.json \ + # --report --report-file ./e2eshark-reports/vai-int8-p0p1.md - python ./test-suite/alt_e2eshark/utils/merge_dicts.py \ - --sources ./e2eshark-reports/ci_reports_${{ matrix.backend }}_nlp-shard1_onnx_json/nlp-shard1.json \ - ./e2eshark-reports/ci_reports_${{ matrix.backend }}_nlp-shard2_onnx_json/nlp-shard2.json \ - ./e2eshark-reports/ci_reports_${{ matrix.backend }}_nlp-shard3_onnx_json/nlp-shard3.json \ - --output ./e2eshark-reports/nlp.json \ - --report --report-file ./e2eshark-reports/nlp.md + # python ./test-suite/alt_e2eshark/utils/merge_dicts.py \ + # --sources ./e2eshark-reports/ci_reports_${{ matrix.backend }}_nlp-shard1_onnx_json/nlp-shard1.json \ + # ./e2eshark-reports/ci_reports_${{ matrix.backend }}_nlp-shard2_onnx_json/nlp-shard2.json \ + # ./e2eshark-reports/ci_reports_${{ matrix.backend }}_nlp-shard3_onnx_json/nlp-shard3.json \ + # --output ./e2eshark-reports/nlp.json \ + # --report --report-file ./e2eshark-reports/nlp.md - python ./test-suite/alt_e2eshark/utils/merge_dicts.py \ - --sources ./e2eshark-reports/vai-int8-p0p1.json \ - ./e2eshark-reports/vai-hf-cnn-fp32.json \ - ./e2eshark-reports/nlp.json \ - ./e2eshark-reports/ci_reports_${{ matrix.backend }}_shark-test-suite_onnx_json/shark-test-suite.json \ - ./e2eshark-reports/ci_reports_${{ matrix.backend }}_vai-vision-int8_onnx_json/vai-vision-int8.json \ - ./e2eshark-reports/ci_reports_${{ matrix.backend }}_migraphx_onnx_json/migraphx.json \ - --output ./e2eshark-reports/combined_reports.json \ - --report --report-file ./e2eshark-reports/combined_reports.md + # python ./test-suite/alt_e2eshark/utils/merge_dicts.py \ + # --sources ./e2eshark-reports/vai-int8-p0p1.json \ + # ./e2eshark-reports/vai-hf-cnn-fp32.json \ + # ./e2eshark-reports/nlp.json \ + # ./e2eshark-reports/ci_reports_${{ matrix.backend }}_shark-test-suite_onnx_json/shark-test-suite.json \ + # ./e2eshark-reports/ci_reports_${{ matrix.backend }}_vai-vision-int8_onnx_json/vai-vision-int8.json \ + # ./e2eshark-reports/ci_reports_${{ matrix.backend }}_migraphx_onnx_json/migraphx.json \ + # --output ./e2eshark-reports/combined_reports.json \ + # --report --report-file ./e2eshark-reports/combined_reports.md - - name: Push artifacts - run: | - git config user.name "GitHub Actions Bot" - git config user.email "<>" - git pull - date=$(date '+%Y-%m-%d') - mkdir -p ${date}/ci_reports_onnx/${{ matrix.backend }}/vai-hf-cnn-fp32 - mkdir -p ${date}/ci_reports_onnx/${{ matrix.backend }}/vai-int8-p0p1 - mkdir -p ${date}/ci_reports_onnx/${{ matrix.backend }}/shark-test-suite - mkdir -p ${date}/ci_reports_onnx/${{ matrix.backend }}/vai-vision-int8 - mkdir -p ${date}/ci_reports_onnx/${{ matrix.backend }}/migraphx - mkdir -p ${date}/ci_reports_onnx/${{ matrix.backend }}/nlp - mkdir -p ${date}/ci_reports_onnx/${{ matrix.backend }}/combined-reports - cp vai-hf-cnn-fp32.md ${date}/ci_reports_onnx/${{ matrix.backend }}/vai-hf-cnn-fp32/summary.md - cp vai-int8-p0p1.md ${date}/ci_reports_onnx/${{ matrix.backend }}/vai-int8-p0p1/summary.md - cp nlp.md ${date}/ci_reports_onnx/${{ matrix.backend }}/nlp/summary.md - cp ci_reports_${{ matrix.backend }}_shark-test-suite_onnx_md/shark-test-suite.md ${date}/ci_reports_onnx/${{ matrix.backend }}/shark-test-suite/summary.md - cp ci_reports_${{ matrix.backend }}_vai-vision-int8_onnx_md/vai-vision-int8.md ${date}/ci_reports_onnx/${{ matrix.backend }}/vai-vision-int8/summary.md - cp ci_reports_${{ matrix.backend }}_migraphx_onnx_md/migraphx.md ${date}/ci_reports_onnx/${{ matrix.backend }}/migraphx/summary.md - cp combined_reports.md ${date}/ci_reports_onnx/${{ matrix.backend }}/combined-reports/summary.md - git add $date - git commit -m "add CI reports for e2eshark for ${{ matrix.backend }}" - git push origin main - working-directory: ./e2eshark-reports + # - name: Push artifacts + # run: | + # git config user.name "GitHub Actions Bot" + # git config user.email "<>" + # git pull + # date=$(date '+%Y-%m-%d') + # mkdir -p ${date}/ci_reports_onnx/${{ matrix.backend }}/vai-hf-cnn-fp32 + # mkdir -p ${date}/ci_reports_onnx/${{ matrix.backend }}/vai-int8-p0p1 + # mkdir -p ${date}/ci_reports_onnx/${{ matrix.backend }}/shark-test-suite + # mkdir -p ${date}/ci_reports_onnx/${{ matrix.backend }}/vai-vision-int8 + # mkdir -p ${date}/ci_reports_onnx/${{ matrix.backend }}/migraphx + # mkdir -p ${date}/ci_reports_onnx/${{ matrix.backend }}/nlp + # mkdir -p ${date}/ci_reports_onnx/${{ matrix.backend }}/combined-reports + # cp vai-hf-cnn-fp32.md ${date}/ci_reports_onnx/${{ matrix.backend }}/vai-hf-cnn-fp32/summary.md + # cp vai-int8-p0p1.md ${date}/ci_reports_onnx/${{ matrix.backend }}/vai-int8-p0p1/summary.md + # cp nlp.md ${date}/ci_reports_onnx/${{ matrix.backend }}/nlp/summary.md + # cp ci_reports_${{ matrix.backend }}_shark-test-suite_onnx_md/shark-test-suite.md ${date}/ci_reports_onnx/${{ matrix.backend }}/shark-test-suite/summary.md + # cp ci_reports_${{ matrix.backend }}_vai-vision-int8_onnx_md/vai-vision-int8.md ${date}/ci_reports_onnx/${{ matrix.backend }}/vai-vision-int8/summary.md + # cp ci_reports_${{ matrix.backend }}_migraphx_onnx_md/migraphx.md ${date}/ci_reports_onnx/${{ matrix.backend }}/migraphx/summary.md + # cp combined_reports.md ${date}/ci_reports_onnx/${{ matrix.backend }}/combined-reports/summary.md + # git add $date + # git commit -m "add CI reports for e2eshark for ${{ matrix.backend }}" + # git push origin main + # working-directory: ./e2eshark-reports