test_e2eshark.yml

# Copyright 2024 Advanced Micro Devices
#
# Licensed under the Apache License v2.0 with LLVM Exceptions.
# See https://llvm.org/LICENSE.txt for license information.
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception

name: E2ESHARK Test Suite
on:
  workflow_dispatch:
  # schedule:
  # # Runs at 12:00 PM UTC, which is 5:00 AM PST
  #  - cron: '0 12 * * *'

jobs: 
  # e2eshark:
  #   timeout-minutes: 480
  #   name: "Models :: ${{ matrix.backend }} :: ${{ matrix.test-file }}"
  #   runs-on: ${{ matrix.runs-on }}
  #   strategy:
  #     fail-fast: false
  #     matrix:
  #       include:
  #         - name: mi300_gpu1_test
  #           runs-on: nodai-amdgpu-mi300-x86-64
  #           backend: rocm
  #           device: hip
  #           target-chip: gfx942
  #           test-file: nlp-shard1
  #           cache-dir: /home/sai/shark-test-suite-models-cache
  #         - name: mi300_gpu2_test
  #           runs-on: nodai-amdgpu-mi300-x86-64
  #           backend: rocm
  #           device: hip
  #           target-chip: gfx942
  #           test-file: nlp-shard2
  #           cache-dir: /home/sai/shark-test-suite-models-cache
  #         - name: mi300_gpu3_test
  #           runs-on: nodai-amdgpu-mi300-x86-64
  #           backend: rocm
  #           device: hip
  #           target-chip: gfx942
  #           test-file: nlp-shard3
  #           cache-dir: /home/sai/shark-test-suite-models-cache
  #         - name: mi300_gpu4_test
  #           runs-on: nodai-amdgpu-mi300-x86-64
  #           backend: rocm
  #           device: hip
  #           target-chip: gfx942
  #           test-file: shark-test-suite
  #           cache-dir: /home/sai/shark-test-suite-models-cache
  #         - name: mi300_gpu5_test
  #           runs-on: nodai-amdgpu-mi300-x86-64
  #           backend: rocm
  #           device: hip
  #           target-chip: gfx942
  #           test-file: vai-hf-cnn-fp32-shard1
  #           cache-dir: /home/sai/shark-test-suite-models-cache
  #         - name: mi300_gpu6_test
  #           runs-on: nodai-amdgpu-mi300-x86-64
  #           backend: rocm
  #           device: hip
  #           target-chip: gfx942
  #           test-file: vai-hf-cnn-fp32-shard2
  #           cache-dir: /home/sai/shark-test-suite-models-cache
  #         - name: mi300_gpu7_test
  #           runs-on: nodai-amdgpu-mi300-x86-64
  #           backend: rocm
  #           device: hip
  #           target-chip: gfx942
  #           test-file: vai-hf-cnn-fp32-shard3
  #           cache-dir: /home/sai/shark-test-suite-models-cache
  #         - name: mi300_gpu8_test
  #           runs-on: nodai-amdgpu-mi300-x86-64
  #           backend: rocm
  #           device: hip
  #           target-chip: gfx942
  #           test-file: vai-int8-p0p1-shard1
  #           cache-dir: /home/sai/shark-test-suite-models-cache
  #         - name: mi300_gpu9_test
  #           runs-on: nodai-amdgpu-mi300-x86-64
  #           backend: rocm
  #           device: hip
  #           target-chip: gfx942
  #           test-file: vai-int8-p0p1-shard2
  #           cache-dir: /home/sai/shark-test-suite-models-cache
  #         - name: mi300_gpu10_test
  #           runs-on: nodai-amdgpu-mi300-x86-64
  #           backend: rocm
  #           device: hip
  #           target-chip: gfx942
  #           test-file: vai-int8-p0p1-shard3
  #           cache-dir: /home/sai/shark-test-suite-models-cache
  #         - name: mi300_gpu11_test
  #           runs-on: nodai-amdgpu-mi300-x86-64
  #           backend: rocm
  #           device: hip
  #           target-chip: gfx942
  #           test-file: vai-vision-int8
  #           cache-dir: /home/sai/shark-test-suite-models-cache
  #         - name: mi300_gpu12_test
  #           runs-on: nodai-amdgpu-mi300-x86-64
  #           backend: rocm
  #           device: hip
  #           target-chip: gfx942
  #           test-file: migraphx
  #           cache-dir: /home/sai/shark-test-suite-models-cache
  #         - name: cpu_shard1_test
  #           runs-on: nodai-amdgpu-mi250-x86-64
  #           backend: llvm-cpu
  #           device: local-task
  #           target-chip: x86_64-linux-gnu
  #           test-file: nlp-shard1
  #           cache-dir: /groups/aig_sharks/test-suite-ci-cache
  #         - name: cpu_shard2_test
  #           runs-on: nodai-amdgpu-mi250-x86-64
  #           backend: llvm-cpu
  #           device: local-task
  #           target-chip: x86_64-linux-gnu
  #           test-file: nlp-shard2
  #           cache-dir: /groups/aig_sharks/test-suite-ci-cache
  #         - name: cpu_shard3_test
  #           runs-on: nodai-amdgpu-mi250-x86-64
  #           backend: llvm-cpu
  #           device: local-task
  #           target-chip: x86_64-linux-gnu
  #           test-file: nlp-shard3
  #           cache-dir: /groups/aig_sharks/test-suite-ci-cache
  #         - name: cpu_shard4_test
  #           runs-on: nodai-amdgpu-mi250-x86-64
  #           backend: llvm-cpu
  #           device: local-task
  #           target-chip: x86_64-linux-gnu
  #           test-file: shark-test-suite
  #           cache-dir: /groups/aig_sharks/test-suite-ci-cache
  #         - name: cpu_shard5_test
  #           runs-on: nodai-amdgpu-mi250-x86-64
  #           backend: llvm-cpu
  #           device: local-task
  #           target-chip: x86_64-linux-gnu
  #           test-file: vai-hf-cnn-fp32-shard1
  #           cache-dir: /groups/aig_sharks/test-suite-ci-cache
  #         - name: cpu_shard6_test
  #           runs-on: nodai-amdgpu-mi250-x86-64
  #           backend: llvm-cpu
  #           device: local-task
  #           target-chip: x86_64-linux-gnu
  #           test-file: vai-hf-cnn-fp32-shard2
  #           cache-dir: /groups/aig_sharks/test-suite-ci-cache
  #         - name: cpu_shard7_test
  #           runs-on: nodai-amdgpu-mi250-x86-64
  #           backend: llvm-cpu
  #           device: local-task
  #           target-chip: x86_64-linux-gnu
  #           test-file: vai-hf-cnn-fp32-shard3
  #           cache-dir: /groups/aig_sharks/test-suite-ci-cache
  #         - name: cpu_shard8_test
  #           runs-on: nodai-amdgpu-mi250-x86-64
  #           backend: llvm-cpu
  #           device: local-task
  #           target-chip: x86_64-linux-gnu
  #           test-file: vai-int8-p0p1-shard1
  #           cache-dir: /groups/aig_sharks/test-suite-ci-cache
  #         - name: cpu_shard9_test
  #           runs-on: nodai-amdgpu-mi250-x86-64
  #           backend: llvm-cpu
  #           device: local-task
  #           target-chip: x86_64-linux-gnu
  #           test-file: vai-int8-p0p1-shard2
  #           cache-dir: /groups/aig_sharks/test-suite-ci-cache
  #         - name: cpu_shard10_test
  #           runs-on: nodai-amdgpu-mi250-x86-64
  #           backend: llvm-cpu
  #           device: local-task
  #           target-chip: x86_64-linux-gnu
  #           test-file: vai-int8-p0p1-shard3
  #           cache-dir: /groups/aig_sharks/test-suite-ci-cache
  #         - name: cpu_shard11_test
  #           runs-on: nodai-amdgpu-mi250-x86-64
  #           backend: llvm-cpu
  #           device: local-task
  #           target-chip: x86_64-linux-gnu
  #           test-file: vai-vision-int8
  #           cache-dir: /groups/aig_sharks/test-suite-ci-cache
  #         - name: cpu_shard12_test
  #           runs-on: nodai-amdgpu-mi250-x86-64
  #           backend: llvm-cpu
  #           device: local-task
  #           target-chip: x86_64-linux-gnu
  #           test-file: migraphx
  #           cache-dir: /groups/aig_sharks/test-suite-ci-cache
  #   env:
  #     E2E_VENV_DIR: ${{ github.workspace }}/test-suite_venv
  #     ALT_E2E_VENV_DIR: ${{ github.workspace }}/alt-test-suite_venv
  #     HF_TOKEN: ${{ secrets.HF_TOKEN }}
  #     AZ_PRIVATE_CONNECTION: ${{ secrets.ONNXPRIVATESTORAGE_AZ_PRIVATE_CONNECTION }}
  #     CACHE_DIR: ${{ matrix.cache-dir }}
  #   steps:
  #     - name: Checkout Test Suite
  #       uses: actions/checkout@v2
  #       with:
  #         repository: nod-ai/SHARK-TestSuite
  #         ref: alt-merge-reports
  #         path: test-suite
      
  #     - name: "Setup alt e2eshark python venv"
  #       run: |
  #         rm -rf ${ALT_E2E_VENV_DIR}
  #         python3.11 -m venv ${ALT_E2E_VENV_DIR}
  #         source ${ALT_E2E_VENV_DIR}/bin/activate
  #         pip install --upgrade pip
  #         pip install -r ./alt_e2eshark/base_requirements.txt
  #         pip install -r ./alt_e2eshark/iree_requirements.txt
  #         pip install --no-deps -r ./alt_e2eshark/torch_mlir_requirements.txt
  #         pip install --find-links https://iree.dev/pip-release-links.html iree-compiler iree-runtime --upgrade
  #       working-directory: ./test-suite

  #     - name: Run Onnx Bench Mode
  #       if: contains(matrix.test-file, 'migraphx')
  #       run: |
  #         source ${ALT_E2E_VENV_DIR}/bin/activate
  #         pip freeze
  #         cd alt_e2eshark
  #         free -mh
  #         python3.11 ./run.py \
  #           -r ./test-onnx \
  #           --report \
  #           --torchtolinalg \
  #           --testsfile onnx_tests/models/external_lists/${{ matrix.test-file }}.txt \
  #           -b ${{ matrix.backend }} \
  #           -d ${{ matrix.device }} \
  #           --report-file reports/${{ matrix.test-file }}.md \
  #           --mode=cl-onnx-iree \
  #           --cleanup=3 \
  #           --benchmark \
  #           -v
  #       working-directory: ./test-suite

  #     - name: Run Onnx Default Mode
  #       if: ${{ ! contains(matrix.test-file, 'migraphx') }}
  #       run: |
  #         source ${ALT_E2E_VENV_DIR}/bin/activate
  #         pip freeze
  #         cd alt_e2eshark
  #         free -mh
  #         python3.11 ./run.py \
  #           -r ./test-onnx \
  #           --report \
  #           --torchtolinalg \
  #           --testsfile onnx_tests/models/external_lists/${{ matrix.test-file }}.txt \
  #           -b ${{ matrix.backend }} \
  #           -d ${{ matrix.device }} \
  #           --report-file reports/${{ matrix.test-file }}.md \
  #           --mode=cl-onnx-iree \
  #           --cleanup=3 \
  #           -v
  #       working-directory: ./test-suite
      
  #     - uses: actions/upload-artifact@master
  #       with:
  #         name: ci_reports_${{ matrix.backend }}_${{ matrix.test-file }}_onnx_md
  #         path: ./test-suite/alt_e2eshark/reports/${{ matrix.test-file }}.md
      
  #     - uses: actions/upload-artifact@master
  #       with:
  #         name: ci_reports_${{ matrix.backend }}_${{ matrix.test-file }}_onnx_json
  #         path: ./test-suite/alt_e2eshark/reports/${{ matrix.test-file }}.json

  push_artifacts:
    # needs: [e2eshark]
    runs-on: ${{ matrix.runs-on }}
    strategy:
      fail-fast: false
      max-parallel: 1
      matrix:
        include:
          - name: merge_rocm_reports
            runs-on: nodai-amdgpu-mi300-x86-64
            backend: rocm
          - name: merge_cpu_reports
            runs-on: nodai-amdgpu-mi250-x86-64
            backend: llvm-cpu
    env:
      AZ_PUBLIC_KEY: ${{ secrets.SHARKPUBLIC_AZ_PUBLIC_KEY }}
    steps:
      - name: Checkout Test Suite
        uses: actions/checkout@v2
        with:
          repository: nod-ai/SHARK-TestSuite
          ref: alt-merge-reports
          path: test-suite
      - name: Checkout repo
        uses: actions/checkout@v2
        with:
          repository: nod-ai/e2eshark-reports
          ref: main
          token: ${{ secrets.E2ESHARK_GITHUB_TOKEN }}
          path: e2eshark-reports
      - name: "Setup alt test suite venv"
        run: |
          python3.11 -m venv report_venv_alt
          source report_venv_alt/bin/activate
          pip install --upgrade pip
          pip install -r ./test-suite/alt_e2eshark/base_requirements.txt
          pip install -r ./test-suite/alt_e2eshark/iree_requirements.txt
          pip install --no-deps -r ./test-suite/alt_e2eshark/torch_mlir_requirements.txt
          pip install --find-links https://iree.dev/pip-release-links.html iree-compiler iree-runtime --upgrade
      - uses: actions/download-artifact@master
        with:
          name: ci_reports_${{ matrix.backend }}_shark-test-suite_onnx_md
          path: ./e2eshark-reports/ci_reports_${{ matrix.backend }}_shark-test-suite_onnx_md
          github-token: ${{ secrets.IREE_ARTIFACTS }}
          repository: nod-ai/SHARK-TestSuite
          run-id: 11323798734
      - uses: actions/download-artifact@master
        with:
          name: ci_reports_${{ matrix.backend }}_shark-test-suite_onnx_json
          path: ./e2eshark-reports/ci_reports_${{ matrix.backend }}_shark-test-suite_onnx_json
          github-token: ${{ secrets.IREE_ARTIFACTS }}
          repository: nod-ai/SHARK-TestSuite
          run-id: 11323798734
      - uses: actions/download-artifact@master
        with:
          name: ci_reports_${{ matrix.backend }}_vai-hf-cnn-fp32-shard1_onnx_md
          path: ./e2eshark-reports/ci_reports_${{ matrix.backend }}_vai-hf-cnn-fp32-shard1_onnx_md
          github-token: ${{ secrets.IREE_ARTIFACTS }}
          repository: nod-ai/SHARK-TestSuite
          run-id: 11323798734
      - uses: actions/download-artifact@master
        with:
          name: ci_reports_${{ matrix.backend }}_vai-hf-cnn-fp32-shard1_onnx_json
          path: ./e2eshark-reports/ci_reports_${{ matrix.backend }}_vai-hf-cnn-fp32-shard1_onnx_json
          github-token: ${{ secrets.IREE_ARTIFACTS }}
          repository: nod-ai/SHARK-TestSuite
          run-id: 11323798734
      - uses: actions/download-artifact@master
        with:
          name: ci_reports_${{ matrix.backend }}_vai-hf-cnn-fp32-shard2_onnx_md
          path: ./e2eshark-reports/ci_reports_${{ matrix.backend }}_vai-hf-cnn-fp32-shard2_onnx_md
          github-token: ${{ secrets.IREE_ARTIFACTS }}
          repository: nod-ai/SHARK-TestSuite
          run-id: 11323798734
      - uses: actions/download-artifact@master
        with:
          name: ci_reports_${{ matrix.backend }}_vai-hf-cnn-fp32-shard2_onnx_json
          path: ./e2eshark-reports/ci_reports_${{ matrix.backend }}_vai-hf-cnn-fp32-shard2_onnx_json
          github-token: ${{ secrets.IREE_ARTIFACTS }}
          repository: nod-ai/SHARK-TestSuite
          run-id: 11323798734
      - uses: actions/download-artifact@master
        with:
          name: ci_reports_${{ matrix.backend }}_vai-hf-cnn-fp32-shard3_onnx_md
          path: ./e2eshark-reports/ci_reports_${{ matrix.backend }}_vai-hf-cnn-fp32-shard3_onnx_md
          github-token: ${{ secrets.IREE_ARTIFACTS }}
          repository: nod-ai/SHARK-TestSuite
          run-id: 11323798734
      - uses: actions/download-artifact@master
        with:
          name: ci_reports_${{ matrix.backend }}_vai-hf-cnn-fp32-shard3_onnx_json
          path: ./e2eshark-reports/ci_reports_${{ matrix.backend }}_vai-hf-cnn-fp32-shard3_onnx_json
          github-token: ${{ secrets.IREE_ARTIFACTS }}
          repository: nod-ai/SHARK-TestSuite
          run-id: 11323798734
      - uses: actions/download-artifact@master
        with:
          name: ci_reports_${{ matrix.backend }}_vai-int8-p0p1-shard1_onnx_md
          path: ./e2eshark-reports/ci_reports_${{ matrix.backend }}_vai-int8-p0p1-shard1_onnx_md
          github-token: ${{ secrets.IREE_ARTIFACTS }}
          repository: nod-ai/SHARK-TestSuite
          run-id: 11323798734
      - uses: actions/download-artifact@master
        with:
          name: ci_reports_${{ matrix.backend }}_vai-int8-p0p1-shard1_onnx_json
          path: ./e2eshark-reports/ci_reports_${{ matrix.backend }}_vai-int8-p0p1-shard1_onnx_json
          github-token: ${{ secrets.IREE_ARTIFACTS }}
          repository: nod-ai/SHARK-TestSuite
          run-id: 11323798734
      - uses: actions/download-artifact@master
        with:
          name: ci_reports_${{ matrix.backend }}_vai-int8-p0p1-shard2_onnx_md
          path: ./e2eshark-reports/ci_reports_${{ matrix.backend }}_vai-int8-p0p1-shard2_onnx_md
          github-token: ${{ secrets.IREE_ARTIFACTS }}
          repository: nod-ai/SHARK-TestSuite
          run-id: 11323798734
      - uses: actions/download-artifact@master
        with:
          name: ci_reports_${{ matrix.backend }}_vai-int8-p0p1-shard2_onnx_json
          path: ./e2eshark-reports/ci_reports_${{ matrix.backend }}_vai-int8-p0p1-shard2_onnx_json
          github-token: ${{ secrets.IREE_ARTIFACTS }}
          repository: nod-ai/SHARK-TestSuite
          run-id: 11323798734
      - uses: actions/download-artifact@master
        with:
          name: ci_reports_${{ matrix.backend }}_vai-int8-p0p1-shard3_onnx_md
          path: ./e2eshark-reports/ci_reports_${{ matrix.backend }}_vai-int8-p0p1-shard3_onnx_md
          github-token: ${{ secrets.IREE_ARTIFACTS }}
          repository: nod-ai/SHARK-TestSuite
          run-id: 11323798734
      - uses: actions/download-artifact@master
        with:
          name: ci_reports_${{ matrix.backend }}_vai-int8-p0p1-shard3_onnx_json
          path: ./e2eshark-reports/ci_reports_${{ matrix.backend }}_vai-int8-p0p1-shard3_onnx_json
          github-token: ${{ secrets.IREE_ARTIFACTS }}
          repository: nod-ai/SHARK-TestSuite
          run-id: 11323798734
      - uses: actions/download-artifact@master
        with:
          name: ci_reports_${{ matrix.backend }}_vai-vision-int8_onnx_md
          path: ./e2eshark-reports/ci_reports_${{ matrix.backend }}_vai-vision-int8_onnx_md
          github-token: ${{ secrets.IREE_ARTIFACTS }}
          repository: nod-ai/SHARK-TestSuite
          run-id: 11323798734
      - uses: actions/download-artifact@master
        with:
          name: ci_reports_${{ matrix.backend }}_vai-vision-int8_onnx_json
          path: ./e2eshark-reports/ci_reports_${{ matrix.backend }}_vai-vision-int8_onnx_json
          github-token: ${{ secrets.IREE_ARTIFACTS }}
          repository: nod-ai/SHARK-TestSuite
          run-id: 11323798734
      - uses: actions/download-artifact@master
        with:
          name: ci_reports_${{ matrix.backend }}_migraphx_onnx_md
          path: ./e2eshark-reports/ci_reports_${{ matrix.backend }}_migraphx_onnx_md
          github-token: ${{ secrets.IREE_ARTIFACTS }}
          repository: nod-ai/SHARK-TestSuite
          run-id: 11323798734
      - uses: actions/download-artifact@master
        with:
          name: ci_reports_${{ matrix.backend }}_migraphx_onnx_json
          path: ./e2eshark-reports/ci_reports_${{ matrix.backend }}_migraphx_onnx_json
          github-token: ${{ secrets.IREE_ARTIFACTS }}
          repository: nod-ai/SHARK-TestSuite
          run-id: 11323798734
      - uses: actions/download-artifact@master
        with:
          name: ci_reports_${{ matrix.backend }}_nlp-shard1_onnx_md
          path: ./e2eshark-reports/ci_reports_${{ matrix.backend }}_nlp-shard1_onnx_md
          github-token: ${{ secrets.IREE_ARTIFACTS }}
          repository: nod-ai/SHARK-TestSuite
          run-id: 11323798734
      - uses: actions/download-artifact@master
        with:
          name: ci_reports_${{ matrix.backend }}_nlp-shard1_onnx_json
          path: ./e2eshark-reports/ci_reports_${{ matrix.backend }}_nlp-shard1_onnx_json
          github-token: ${{ secrets.IREE_ARTIFACTS }}
          repository: nod-ai/SHARK-TestSuite
          run-id: 11323798734
      - uses: actions/download-artifact@master
        with:
          name: ci_reports_${{ matrix.backend }}_nlp-shard2_onnx_md
          path: ./e2eshark-reports/ci_reports_${{ matrix.backend }}_nlp-shard2_onnx_md
          github-token: ${{ secrets.IREE_ARTIFACTS }}
          repository: nod-ai/SHARK-TestSuite
          run-id: 11323798734
      - uses: actions/download-artifact@master
        with:
          name: ci_reports_${{ matrix.backend }}_nlp-shard2_onnx_json
          path: ./e2eshark-reports/ci_reports_${{ matrix.backend }}_nlp-shard2_onnx_json
          github-token: ${{ secrets.IREE_ARTIFACTS }}
          repository: nod-ai/SHARK-TestSuite
          run-id: 11323798734
      - uses: actions/download-artifact@master
        with:
          name: ci_reports_${{ matrix.backend }}_nlp-shard3_onnx_md
          path: ./e2eshark-reports/ci_reports_${{ matrix.backend }}_nlp-shard3_onnx_md
          github-token: ${{ secrets.IREE_ARTIFACTS }}
          repository: nod-ai/SHARK-TestSuite
          run-id: 11323798734
      - uses: actions/download-artifact@master
        with:
          name: ci_reports_${{ matrix.backend }}_nlp-shard3_onnx_json
          path: ./e2eshark-reports/ci_reports_${{ matrix.backend }}_nlp-shard3_onnx_json
          github-token: ${{ secrets.IREE_ARTIFACTS }}
          repository: nod-ai/SHARK-TestSuite
          run-id: 11323798734
      - name: Merge Reports
        run: |
          source report_venv_alt/bin/activate

          python ./test-suite/alt_e2eshark/utils/merge_dicts.py \
          --sources ./e2eshark-reports/ci_reports_${{ matrix.backend }}_vai-hf-cnn-fp32-shard1_onnx_json/vai-hf-cnn-fp32-shard1.json \
          ./e2eshark-reports/ci_reports_${{ matrix.backend }}_vai-hf-cnn-fp32-shard2_onnx_json/vai-hf-cnn-fp32-shard2.json \
          ./e2eshark-reports/ci_reports_${{ matrix.backend }}_vai-hf-cnn-fp32-shard3_onnx_json/vai-hf-cnn-fp32-shard3.json \
          --output ./e2eshark-reports/vai-hf-cnn-fp32.json \
          --report --report-file ./e2eshark-reports/vai-hf-cnn-fp32.md

          python ./test-suite/alt_e2eshark/utils/merge_dicts.py \
          --sources ./e2eshark-reports/ci_reports_${{ matrix.backend }}_vai-int8-p0p1-shard1_onnx_json/vai-int8-p0p1-shard1.json \
          ./e2eshark-reports/ci_reports_${{ matrix.backend }}_vai-int8-p0p1-shard2_onnx_json/vai-int8-p0p1-shard2.json \
          ./e2eshark-reports/ci_reports_${{ matrix.backend }}_vai-int8-p0p1-shard3_onnx_json/vai-int8-p0p1-shard3.json \
          --output ./e2eshark-reports/vai-int8-p0p1.json \
          --report --report-file ./e2eshark-reports/vai-int8-p0p1.md

          python ./test-suite/alt_e2eshark/utils/merge_dicts.py \
          --sources ./e2eshark-reports/ci_reports_${{ matrix.backend }}_nlp-shard1_onnx_json/nlp-shard1.json \
          ./e2eshark-reports/ci_reports_${{ matrix.backend }}_nlp-shard2_onnx_json/nlp-shard2.json \
          ./e2eshark-reports/ci_reports_${{ matrix.backend }}_nlp-shard3_onnx_json/nlp-shard3.json \
          --output ./e2eshark-reports/nlp.json \
          --report --report-file ./e2eshark-reports/nlp.md

          python ./test-suite/alt_e2eshark/utils/merge_dicts.py \
          --sources ./e2eshark-reports/vai-int8-p0p1.json \
          ./e2eshark-reports/vai-hf-cnn-fp32.json \
          ./e2eshark-reports/nlp.json \
          ./e2eshark-reports/ci_reports_${{ matrix.backend }}_shark-test-suite_onnx_json/shark-test-suite.json \
          ./e2eshark-reports/ci_reports_${{ matrix.backend }}_vai-vision-int8_onnx_json/vai-vision-int8.json \
          ./e2eshark-reports/ci_reports_${{ matrix.backend }}_migraphx_onnx_json/migraphx.json \
          --output ./e2eshark-reports/combined_reports.json \
          --report --report-file ./e2eshark-reports/combined_reports.md

      # - name: Push status artifacts
      #   run: |
      #     git config user.name "GitHub Actions Bot"
      #     git config user.email "<>"
      #     git pull
      #     date=$(date '+%Y-%m-%d')
      #     mkdir -p ${date}/ci_reports_onnx/${{ matrix.backend }}/vai-hf-cnn-fp32
      #     mkdir -p ${date}/ci_reports_onnx/${{ matrix.backend }}/vai-int8-p0p1
      #     mkdir -p ${date}/ci_reports_onnx/${{ matrix.backend }}/shark-test-suite
      #     mkdir -p ${date}/ci_reports_onnx/${{ matrix.backend }}/vai-vision-int8
      #     mkdir -p ${date}/ci_reports_onnx/${{ matrix.backend }}/migraphx
      #     mkdir -p ${date}/ci_reports_onnx/${{ matrix.backend }}/nlp
      #     mkdir -p ${date}/ci_reports_onnx/${{ matrix.backend }}/combined-reports
      #     cp vai-hf-cnn-fp32.md ${date}/ci_reports_onnx/${{ matrix.backend }}/vai-hf-cnn-fp32/summary.md
      #     cp vai-int8-p0p1.md ${date}/ci_reports_onnx/${{ matrix.backend }}/vai-int8-p0p1/summary.md
      #     cp nlp.md ${date}/ci_reports_onnx/${{ matrix.backend }}/nlp/summary.md
      #     cp ci_reports_${{ matrix.backend }}_shark-test-suite_onnx_md/shark-test-suite.md ${date}/ci_reports_onnx/${{ matrix.backend }}/shark-test-suite/summary.md
      #     cp ci_reports_${{ matrix.backend }}_vai-vision-int8_onnx_md/vai-vision-int8.md ${date}/ci_reports_onnx/${{ matrix.backend }}/vai-vision-int8/summary.md
      #     cp ci_reports_${{ matrix.backend }}_migraphx_onnx_md/migraphx.md ${date}/ci_reports_onnx/${{ matrix.backend }}/migraphx/summary.md
      #     cp combined_reports.md ${date}/ci_reports_onnx/${{ matrix.backend }}/combined-reports/summary.md
      #     git add $date
      #     git commit -m "add CI status reports for e2eshark for ${{ matrix.backend }}"
      #     git push origin main
      #   working-directory: ./e2eshark-reports
      
      - name: Regression Reports
        run: |
          source report_venv_alt/bin/activate
          cd test-suite
          mkdir latest
          mkdir baseline
          wget https://sharkpublic.blob.core.windows.net/sharkpublic/latest-test-suite/combined_reports_${{ matrix.backend }}.json -O latest/combined_reports_${{ matrix.backend }}.json
          wget https://sharkpublic.blob.core.windows.net/sharkpublic/baseline-test-suite/combined_reports_${{ matrix.backend }}.json -O baseline/combined_reports_${{ matrix.backend }}.json
          cd ..

          python ./test-suite/alt_e2eshark/utils/regressions.py \
            --new ./e2eshark-reports/combined_reports.json \
            --old ./test-suite/latest/combined_reports_${{ matrix.backend }}.json \
            --report-file ./e2eshark-reports/yesterday_comparison.md \
            --perf_tol_regression=0.05 \
            --perf_tol_progression=0.05
          
          python ./test-suite/alt_e2eshark/utils/regressions.py \
            --new ./e2eshark-reports/combined_reports.json \
            --old ./test-suite/baseline/combined_reports_${{ matrix.backend }}.json \
            --report-file ./e2eshark-reports/baseline_comparison.md \
            --perf_tol_regression=0.1 \
            --perf_tol_progression=0.1
          
          az storage blob upload --account-name sharkpublic --container-name sharkpublic \
            --name latest-test-suite/combined_reports_${{ matrix.backend }}.json \
            --file ./e2eshark-reports/combined_reports.json \
            --account-key ${AZ_PUBLIC_KEY} --overwrite

      - name: Push regression artifacts
        run: |
          git config user.name "GitHub Actions Bot"
          git config user.email "<>"
          git pull
          date=$(date '+%Y-%m-%d')
          cp yesterday_comparison.md ${date}/ci_reports_onnx/${{ matrix.backend }}/combined-reports/yesterday_comparison.md
          cp baseline_comparison.md ${date}/ci_reports_onnx/${{ matrix.backend }}/combined-reports/baseline_comparison.md
          git add $date
          git commit -m "add CI regression reports for e2eshark for ${{ matrix.backend }}"
          git push origin main
        working-directory: ./e2eshark-reports