E2ESHARK Test Suite #362
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
# Copyright 2024 Advanced Micro Devices | ||
# | ||
# Licensed under the Apache License v2.0 with LLVM Exceptions. | ||
# See https://llvm.org/LICENSE.txt for license information. | ||
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception | ||
name: E2ESHARK Test Suite | ||
on: | ||
workflow_dispatch: | ||
# schedule: | ||
# # Runs at 12:00 PM UTC, which is 5:00 AM PST | ||
# - cron: '0 12 * * *' | ||
jobs: | ||
# e2eshark: | ||
# timeout-minutes: 480 | ||
# name: "Models :: ${{ matrix.backend }} :: ${{ matrix.test-file }}" | ||
# runs-on: ${{ matrix.runs-on }} | ||
# strategy: | ||
# fail-fast: false | ||
# matrix: | ||
# include: | ||
# - name: mi300_gpu1_test | ||
# runs-on: nodai-amdgpu-mi300-x86-64 | ||
# backend: rocm | ||
# device: hip | ||
# target-chip: gfx942 | ||
# test-file: nlp-shard1 | ||
# cache-dir: /home/sai/shark-test-suite-models-cache | ||
# - name: mi300_gpu2_test | ||
# runs-on: nodai-amdgpu-mi300-x86-64 | ||
# backend: rocm | ||
# device: hip | ||
# target-chip: gfx942 | ||
# test-file: nlp-shard2 | ||
# cache-dir: /home/sai/shark-test-suite-models-cache | ||
# - name: mi300_gpu3_test | ||
# runs-on: nodai-amdgpu-mi300-x86-64 | ||
# backend: rocm | ||
# device: hip | ||
# target-chip: gfx942 | ||
# test-file: nlp-shard3 | ||
# cache-dir: /home/sai/shark-test-suite-models-cache | ||
# - name: mi300_gpu4_test | ||
# runs-on: nodai-amdgpu-mi300-x86-64 | ||
# backend: rocm | ||
# device: hip | ||
# target-chip: gfx942 | ||
# test-file: shark-test-suite | ||
# cache-dir: /home/sai/shark-test-suite-models-cache | ||
# - name: mi300_gpu5_test | ||
# runs-on: nodai-amdgpu-mi300-x86-64 | ||
# backend: rocm | ||
# device: hip | ||
# target-chip: gfx942 | ||
# test-file: vai-hf-cnn-fp32-shard1 | ||
# cache-dir: /home/sai/shark-test-suite-models-cache | ||
# - name: mi300_gpu6_test | ||
# runs-on: nodai-amdgpu-mi300-x86-64 | ||
# backend: rocm | ||
# device: hip | ||
# target-chip: gfx942 | ||
# test-file: vai-hf-cnn-fp32-shard2 | ||
# cache-dir: /home/sai/shark-test-suite-models-cache | ||
# - name: mi300_gpu7_test | ||
# runs-on: nodai-amdgpu-mi300-x86-64 | ||
# backend: rocm | ||
# device: hip | ||
# target-chip: gfx942 | ||
# test-file: vai-hf-cnn-fp32-shard3 | ||
# cache-dir: /home/sai/shark-test-suite-models-cache | ||
# - name: mi300_gpu8_test | ||
# runs-on: nodai-amdgpu-mi300-x86-64 | ||
# backend: rocm | ||
# device: hip | ||
# target-chip: gfx942 | ||
# test-file: vai-int8-p0p1-shard1 | ||
# cache-dir: /home/sai/shark-test-suite-models-cache | ||
# - name: mi300_gpu9_test | ||
# runs-on: nodai-amdgpu-mi300-x86-64 | ||
# backend: rocm | ||
# device: hip | ||
# target-chip: gfx942 | ||
# test-file: vai-int8-p0p1-shard2 | ||
# cache-dir: /home/sai/shark-test-suite-models-cache | ||
# - name: mi300_gpu10_test | ||
# runs-on: nodai-amdgpu-mi300-x86-64 | ||
# backend: rocm | ||
# device: hip | ||
# target-chip: gfx942 | ||
# test-file: vai-int8-p0p1-shard3 | ||
# cache-dir: /home/sai/shark-test-suite-models-cache | ||
# - name: mi300_gpu11_test | ||
# runs-on: nodai-amdgpu-mi300-x86-64 | ||
# backend: rocm | ||
# device: hip | ||
# target-chip: gfx942 | ||
# test-file: vai-vision-int8 | ||
# cache-dir: /home/sai/shark-test-suite-models-cache | ||
# - name: mi300_gpu12_test | ||
# runs-on: nodai-amdgpu-mi300-x86-64 | ||
# backend: rocm | ||
# device: hip | ||
# target-chip: gfx942 | ||
# test-file: migraphx | ||
# cache-dir: /home/sai/shark-test-suite-models-cache | ||
# - name: cpu_shard1_test | ||
# runs-on: nodai-amdgpu-mi250-x86-64 | ||
# backend: llvm-cpu | ||
# device: local-task | ||
# target-chip: x86_64-linux-gnu | ||
# test-file: nlp-shard1 | ||
# cache-dir: /groups/aig_sharks/test-suite-ci-cache | ||
# - name: cpu_shard2_test | ||
# runs-on: nodai-amdgpu-mi250-x86-64 | ||
# backend: llvm-cpu | ||
# device: local-task | ||
# target-chip: x86_64-linux-gnu | ||
# test-file: nlp-shard2 | ||
# cache-dir: /groups/aig_sharks/test-suite-ci-cache | ||
# - name: cpu_shard3_test | ||
# runs-on: nodai-amdgpu-mi250-x86-64 | ||
# backend: llvm-cpu | ||
# device: local-task | ||
# target-chip: x86_64-linux-gnu | ||
# test-file: nlp-shard3 | ||
# cache-dir: /groups/aig_sharks/test-suite-ci-cache | ||
# - name: cpu_shard4_test | ||
# runs-on: nodai-amdgpu-mi250-x86-64 | ||
# backend: llvm-cpu | ||
# device: local-task | ||
# target-chip: x86_64-linux-gnu | ||
# test-file: shark-test-suite | ||
# cache-dir: /groups/aig_sharks/test-suite-ci-cache | ||
# - name: cpu_shard5_test | ||
# runs-on: nodai-amdgpu-mi250-x86-64 | ||
# backend: llvm-cpu | ||
# device: local-task | ||
# target-chip: x86_64-linux-gnu | ||
# test-file: vai-hf-cnn-fp32-shard1 | ||
# cache-dir: /groups/aig_sharks/test-suite-ci-cache | ||
# - name: cpu_shard6_test | ||
# runs-on: nodai-amdgpu-mi250-x86-64 | ||
# backend: llvm-cpu | ||
# device: local-task | ||
# target-chip: x86_64-linux-gnu | ||
# test-file: vai-hf-cnn-fp32-shard2 | ||
# cache-dir: /groups/aig_sharks/test-suite-ci-cache | ||
# - name: cpu_shard7_test | ||
# runs-on: nodai-amdgpu-mi250-x86-64 | ||
# backend: llvm-cpu | ||
# device: local-task | ||
# target-chip: x86_64-linux-gnu | ||
# test-file: vai-hf-cnn-fp32-shard3 | ||
# cache-dir: /groups/aig_sharks/test-suite-ci-cache | ||
# - name: cpu_shard8_test | ||
# runs-on: nodai-amdgpu-mi250-x86-64 | ||
# backend: llvm-cpu | ||
# device: local-task | ||
# target-chip: x86_64-linux-gnu | ||
# test-file: vai-int8-p0p1-shard1 | ||
# cache-dir: /groups/aig_sharks/test-suite-ci-cache | ||
# - name: cpu_shard9_test | ||
# runs-on: nodai-amdgpu-mi250-x86-64 | ||
# backend: llvm-cpu | ||
# device: local-task | ||
# target-chip: x86_64-linux-gnu | ||
# test-file: vai-int8-p0p1-shard2 | ||
# cache-dir: /groups/aig_sharks/test-suite-ci-cache | ||
# - name: cpu_shard10_test | ||
# runs-on: nodai-amdgpu-mi250-x86-64 | ||
# backend: llvm-cpu | ||
# device: local-task | ||
# target-chip: x86_64-linux-gnu | ||
# test-file: vai-int8-p0p1-shard3 | ||
# cache-dir: /groups/aig_sharks/test-suite-ci-cache | ||
# - name: cpu_shard11_test | ||
# runs-on: nodai-amdgpu-mi250-x86-64 | ||
# backend: llvm-cpu | ||
# device: local-task | ||
# target-chip: x86_64-linux-gnu | ||
# test-file: vai-vision-int8 | ||
# cache-dir: /groups/aig_sharks/test-suite-ci-cache | ||
# - name: cpu_shard12_test | ||
# runs-on: nodai-amdgpu-mi250-x86-64 | ||
# backend: llvm-cpu | ||
# device: local-task | ||
# target-chip: x86_64-linux-gnu | ||
# test-file: migraphx | ||
# cache-dir: /groups/aig_sharks/test-suite-ci-cache | ||
# env: | ||
# E2E_VENV_DIR: ${{ github.workspace }}/test-suite_venv | ||
# ALT_E2E_VENV_DIR: ${{ github.workspace }}/alt-test-suite_venv | ||
# HF_TOKEN: ${{ secrets.HF_TOKEN }} | ||
# AZ_PRIVATE_CONNECTION: ${{ secrets.ONNXPRIVATESTORAGE_AZ_PRIVATE_CONNECTION }} | ||
# CACHE_DIR: ${{ matrix.cache-dir }} | ||
# steps: | ||
# - name: Checkout Test Suite | ||
# uses: actions/checkout@v2 | ||
# with: | ||
# repository: nod-ai/SHARK-TestSuite | ||
# ref: alt-merge-reports | ||
# path: test-suite | ||
# - name: "Setup alt e2eshark python venv" | ||
# run: | | ||
# rm -rf ${ALT_E2E_VENV_DIR} | ||
# python3.11 -m venv ${ALT_E2E_VENV_DIR} | ||
# source ${ALT_E2E_VENV_DIR}/bin/activate | ||
# pip install --upgrade pip | ||
# pip install -r ./alt_e2eshark/base_requirements.txt | ||
# pip install -r ./alt_e2eshark/iree_requirements.txt | ||
# pip install --no-deps -r ./alt_e2eshark/torch_mlir_requirements.txt | ||
# pip install --find-links https://iree.dev/pip-release-links.html iree-compiler iree-runtime --upgrade | ||
# working-directory: ./test-suite | ||
# - name: Run Onnx Bench Mode | ||
# if: contains(matrix.test-file, 'migraphx') | ||
# run: | | ||
# source ${ALT_E2E_VENV_DIR}/bin/activate | ||
# pip freeze | ||
# cd alt_e2eshark | ||
# free -mh | ||
# python3.11 ./run.py \ | ||
# -r ./test-onnx \ | ||
# --report \ | ||
# --torchtolinalg \ | ||
# --testsfile onnx_tests/models/external_lists/${{ matrix.test-file }}.txt \ | ||
# -b ${{ matrix.backend }} \ | ||
# -d ${{ matrix.device }} \ | ||
# --report-file reports/${{ matrix.test-file }}.md \ | ||
# --mode=cl-onnx-iree \ | ||
# --cleanup=3 \ | ||
# --benchmark \ | ||
# -v | ||
# working-directory: ./test-suite | ||
# - name: Run Onnx Default Mode | ||
# if: ${{ ! contains(matrix.test-file, 'migraphx') }} | ||
# run: | | ||
# source ${ALT_E2E_VENV_DIR}/bin/activate | ||
# pip freeze | ||
# cd alt_e2eshark | ||
# free -mh | ||
# python3.11 ./run.py \ | ||
# -r ./test-onnx \ | ||
# --report \ | ||
# --torchtolinalg \ | ||
# --testsfile onnx_tests/models/external_lists/${{ matrix.test-file }}.txt \ | ||
# -b ${{ matrix.backend }} \ | ||
# -d ${{ matrix.device }} \ | ||
# --report-file reports/${{ matrix.test-file }}.md \ | ||
# --mode=cl-onnx-iree \ | ||
# --cleanup=3 \ | ||
# -v | ||
# working-directory: ./test-suite | ||
# - uses: actions/upload-artifact@master | ||
# with: | ||
# name: ci_reports_${{ matrix.backend }}_${{ matrix.test-file }}_onnx_md | ||
# path: ./test-suite/alt_e2eshark/reports/${{ matrix.test-file }}.md | ||
# - uses: actions/upload-artifact@master | ||
# with: | ||
# name: ci_reports_${{ matrix.backend }}_${{ matrix.test-file }}_onnx_json | ||
# path: ./test-suite/alt_e2eshark/reports/${{ matrix.test-file }}.json | ||
push_artifacts: | ||
Check failure on line 268 in .github/workflows/test_e2eshark.yml GitHub Actions / E2ESHARK Test SuiteInvalid workflow file
|
||
needs: [e2eshark] | ||
runs-on: ${{ matrix.runs-on }} | ||
strategy: | ||
fail-fast: false | ||
max-parallel: 1 | ||
matrix: | ||
include: | ||
- name: merge_rocm_reports | ||
runs-on: nodai-amdgpu-mi300-x86-64 | ||
backend: rocm | ||
- name: merge_cpu_reports | ||
runs-on: nodai-amdgpu-mi250-x86-64 | ||
backend: llvm-cpu | ||
env: | ||
AZ_PUBLIC_KEY: ${{ secrets.SHARKPUBLIC_AZ_PUBLIC_KEY }} | ||
steps: | ||
- name: Checkout Test Suite | ||
uses: actions/checkout@v2 | ||
with: | ||
repository: nod-ai/SHARK-TestSuite | ||
ref: alt-merge-reports | ||
path: test-suite | ||
- name: Checkout repo | ||
uses: actions/checkout@v2 | ||
with: | ||
repository: nod-ai/e2eshark-reports | ||
ref: main | ||
token: ${{ secrets.E2ESHARK_GITHUB_TOKEN }} | ||
path: e2eshark-reports | ||
- name: "Setup alt test suite venv" | ||
run: | | ||
python3.11 -m venv report_venv_alt | ||
source report_venv_alt/bin/activate | ||
pip install --upgrade pip | ||
pip install -r ./test-suite/alt_e2eshark/base_requirements.txt | ||
pip install -r ./test-suite/alt_e2eshark/iree_requirements.txt | ||
pip install --no-deps -r ./test-suite/alt_e2eshark/torch_mlir_requirements.txt | ||
pip install --find-links https://iree.dev/pip-release-links.html iree-compiler iree-runtime --upgrade | ||
- uses: actions/download-artifact@master | ||
with: | ||
name: ci_reports_${{ matrix.backend }}_shark-test-suite_onnx_md | ||
path: ./e2eshark-reports/ci_reports_${{ matrix.backend }}_shark-test-suite_onnx_md | ||
github-token: ${{ secrets.IREE_ARTIFACTS }} | ||
repository: nod-ai/SHARK-TestSuite | ||
run-id: 11323798734 | ||
- uses: actions/download-artifact@master | ||
with: | ||
name: ci_reports_${{ matrix.backend }}_shark-test-suite_onnx_json | ||
path: ./e2eshark-reports/ci_reports_${{ matrix.backend }}_shark-test-suite_onnx_json | ||
github-token: ${{ secrets.IREE_ARTIFACTS }} | ||
repository: nod-ai/SHARK-TestSuite | ||
run-id: 11323798734 | ||
- uses: actions/download-artifact@master | ||
with: | ||
name: ci_reports_${{ matrix.backend }}_vai-hf-cnn-fp32-shard1_onnx_md | ||
path: ./e2eshark-reports/ci_reports_${{ matrix.backend }}_vai-hf-cnn-fp32-shard1_onnx_md | ||
github-token: ${{ secrets.IREE_ARTIFACTS }} | ||
repository: nod-ai/SHARK-TestSuite | ||
run-id: 11323798734 | ||
- uses: actions/download-artifact@master | ||
with: | ||
name: ci_reports_${{ matrix.backend }}_vai-hf-cnn-fp32-shard1_onnx_json | ||
path: ./e2eshark-reports/ci_reports_${{ matrix.backend }}_vai-hf-cnn-fp32-shard1_onnx_json | ||
github-token: ${{ secrets.IREE_ARTIFACTS }} | ||
repository: nod-ai/SHARK-TestSuite | ||
run-id: 11323798734 | ||
- uses: actions/download-artifact@master | ||
with: | ||
name: ci_reports_${{ matrix.backend }}_vai-hf-cnn-fp32-shard2_onnx_md | ||
path: ./e2eshark-reports/ci_reports_${{ matrix.backend }}_vai-hf-cnn-fp32-shard2_onnx_md | ||
github-token: ${{ secrets.IREE_ARTIFACTS }} | ||
repository: nod-ai/SHARK-TestSuite | ||
run-id: 11323798734 | ||
- uses: actions/download-artifact@master | ||
with: | ||
name: ci_reports_${{ matrix.backend }}_vai-hf-cnn-fp32-shard2_onnx_json | ||
path: ./e2eshark-reports/ci_reports_${{ matrix.backend }}_vai-hf-cnn-fp32-shard2_onnx_json | ||
github-token: ${{ secrets.IREE_ARTIFACTS }} | ||
repository: nod-ai/SHARK-TestSuite | ||
run-id: 11323798734 | ||
- uses: actions/download-artifact@master | ||
with: | ||
name: ci_reports_${{ matrix.backend }}_vai-hf-cnn-fp32-shard3_onnx_md | ||
path: ./e2eshark-reports/ci_reports_${{ matrix.backend }}_vai-hf-cnn-fp32-shard3_onnx_md | ||
github-token: ${{ secrets.IREE_ARTIFACTS }} | ||
repository: nod-ai/SHARK-TestSuite | ||
run-id: 11323798734 | ||
- uses: actions/download-artifact@master | ||
with: | ||
name: ci_reports_${{ matrix.backend }}_vai-hf-cnn-fp32-shard3_onnx_json | ||
path: ./e2eshark-reports/ci_reports_${{ matrix.backend }}_vai-hf-cnn-fp32-shard3_onnx_json | ||
github-token: ${{ secrets.IREE_ARTIFACTS }} | ||
repository: nod-ai/SHARK-TestSuite | ||
run-id: 11323798734 | ||
- uses: actions/download-artifact@master | ||
with: | ||
name: ci_reports_${{ matrix.backend }}_vai-int8-p0p1-shard1_onnx_md | ||
path: ./e2eshark-reports/ci_reports_${{ matrix.backend }}_vai-int8-p0p1-shard1_onnx_md | ||
github-token: ${{ secrets.IREE_ARTIFACTS }} | ||
repository: nod-ai/SHARK-TestSuite | ||
run-id: 11323798734 | ||
- uses: actions/download-artifact@master | ||
with: | ||
name: ci_reports_${{ matrix.backend }}_vai-int8-p0p1-shard1_onnx_json | ||
path: ./e2eshark-reports/ci_reports_${{ matrix.backend }}_vai-int8-p0p1-shard1_onnx_json | ||
github-token: ${{ secrets.IREE_ARTIFACTS }} | ||
repository: nod-ai/SHARK-TestSuite | ||
run-id: 11323798734 | ||
- uses: actions/download-artifact@master | ||
with: | ||
name: ci_reports_${{ matrix.backend }}_vai-int8-p0p1-shard2_onnx_md | ||
path: ./e2eshark-reports/ci_reports_${{ matrix.backend }}_vai-int8-p0p1-shard2_onnx_md | ||
github-token: ${{ secrets.IREE_ARTIFACTS }} | ||
repository: nod-ai/SHARK-TestSuite | ||
run-id: 11323798734 | ||
- uses: actions/download-artifact@master | ||
with: | ||
name: ci_reports_${{ matrix.backend }}_vai-int8-p0p1-shard2_onnx_json | ||
path: ./e2eshark-reports/ci_reports_${{ matrix.backend }}_vai-int8-p0p1-shard2_onnx_json | ||
github-token: ${{ secrets.IREE_ARTIFACTS }} | ||
repository: nod-ai/SHARK-TestSuite | ||
run-id: 11323798734 | ||
- uses: actions/download-artifact@master | ||
with: | ||
name: ci_reports_${{ matrix.backend }}_vai-int8-p0p1-shard3_onnx_md | ||
path: ./e2eshark-reports/ci_reports_${{ matrix.backend }}_vai-int8-p0p1-shard3_onnx_md | ||
github-token: ${{ secrets.IREE_ARTIFACTS }} | ||
repository: nod-ai/SHARK-TestSuite | ||
run-id: 11323798734 | ||
- uses: actions/download-artifact@master | ||
with: | ||
name: ci_reports_${{ matrix.backend }}_vai-int8-p0p1-shard3_onnx_json | ||
path: ./e2eshark-reports/ci_reports_${{ matrix.backend }}_vai-int8-p0p1-shard3_onnx_json | ||
github-token: ${{ secrets.IREE_ARTIFACTS }} | ||
repository: nod-ai/SHARK-TestSuite | ||
run-id: 11323798734 | ||
- uses: actions/download-artifact@master | ||
with: | ||
name: ci_reports_${{ matrix.backend }}_vai-vision-int8_onnx_md | ||
path: ./e2eshark-reports/ci_reports_${{ matrix.backend }}_vai-vision-int8_onnx_md | ||
github-token: ${{ secrets.IREE_ARTIFACTS }} | ||
repository: nod-ai/SHARK-TestSuite | ||
run-id: 11323798734 | ||
- uses: actions/download-artifact@master | ||
with: | ||
name: ci_reports_${{ matrix.backend }}_vai-vision-int8_onnx_json | ||
path: ./e2eshark-reports/ci_reports_${{ matrix.backend }}_vai-vision-int8_onnx_json | ||
github-token: ${{ secrets.IREE_ARTIFACTS }} | ||
repository: nod-ai/SHARK-TestSuite | ||
run-id: 11323798734 | ||
- uses: actions/download-artifact@master | ||
with: | ||
name: ci_reports_${{ matrix.backend }}_migraphx_onnx_md | ||
path: ./e2eshark-reports/ci_reports_${{ matrix.backend }}_migraphx_onnx_md | ||
github-token: ${{ secrets.IREE_ARTIFACTS }} | ||
repository: nod-ai/SHARK-TestSuite | ||
run-id: 11323798734 | ||
- uses: actions/download-artifact@master | ||
with: | ||
name: ci_reports_${{ matrix.backend }}_migraphx_onnx_json | ||
path: ./e2eshark-reports/ci_reports_${{ matrix.backend }}_migraphx_onnx_json | ||
github-token: ${{ secrets.IREE_ARTIFACTS }} | ||
repository: nod-ai/SHARK-TestSuite | ||
run-id: 11323798734 | ||
- uses: actions/download-artifact@master | ||
with: | ||
name: ci_reports_${{ matrix.backend }}_nlp-shard1_onnx_md | ||
path: ./e2eshark-reports/ci_reports_${{ matrix.backend }}_nlp-shard1_onnx_md | ||
github-token: ${{ secrets.IREE_ARTIFACTS }} | ||
repository: nod-ai/SHARK-TestSuite | ||
run-id: 11323798734 | ||
- uses: actions/download-artifact@master | ||
with: | ||
name: ci_reports_${{ matrix.backend }}_nlp-shard1_onnx_json | ||
path: ./e2eshark-reports/ci_reports_${{ matrix.backend }}_nlp-shard1_onnx_json | ||
github-token: ${{ secrets.IREE_ARTIFACTS }} | ||
repository: nod-ai/SHARK-TestSuite | ||
run-id: 11323798734 | ||
- uses: actions/download-artifact@master | ||
with: | ||
name: ci_reports_${{ matrix.backend }}_nlp-shard2_onnx_md | ||
path: ./e2eshark-reports/ci_reports_${{ matrix.backend }}_nlp-shard2_onnx_md | ||
github-token: ${{ secrets.IREE_ARTIFACTS }} | ||
repository: nod-ai/SHARK-TestSuite | ||
run-id: 11323798734 | ||
- uses: actions/download-artifact@master | ||
with: | ||
name: ci_reports_${{ matrix.backend }}_nlp-shard2_onnx_json | ||
path: ./e2eshark-reports/ci_reports_${{ matrix.backend }}_nlp-shard2_onnx_json | ||
github-token: ${{ secrets.IREE_ARTIFACTS }} | ||
repository: nod-ai/SHARK-TestSuite | ||
run-id: 11323798734 | ||
- uses: actions/download-artifact@master | ||
with: | ||
name: ci_reports_${{ matrix.backend }}_nlp-shard3_onnx_md | ||
path: ./e2eshark-reports/ci_reports_${{ matrix.backend }}_nlp-shard3_onnx_md | ||
github-token: ${{ secrets.IREE_ARTIFACTS }} | ||
repository: nod-ai/SHARK-TestSuite | ||
run-id: 11323798734 | ||
- uses: actions/download-artifact@master | ||
with: | ||
name: ci_reports_${{ matrix.backend }}_nlp-shard3_onnx_json | ||
path: ./e2eshark-reports/ci_reports_${{ matrix.backend }}_nlp-shard3_onnx_json | ||
github-token: ${{ secrets.IREE_ARTIFACTS }} | ||
repository: nod-ai/SHARK-TestSuite | ||
run-id: 11323798734 | ||
- name: Merge Reports | ||
run: | | ||
source report_venv_alt/bin/activate | ||
python ./test-suite/alt_e2eshark/utils/merge_dicts.py \ | ||
--sources ./e2eshark-reports/ci_reports_${{ matrix.backend }}_vai-hf-cnn-fp32-shard1_onnx_json/vai-hf-cnn-fp32-shard1.json \ | ||
./e2eshark-reports/ci_reports_${{ matrix.backend }}_vai-hf-cnn-fp32-shard2_onnx_json/vai-hf-cnn-fp32-shard2.json \ | ||
./e2eshark-reports/ci_reports_${{ matrix.backend }}_vai-hf-cnn-fp32-shard3_onnx_json/vai-hf-cnn-fp32-shard3.json \ | ||
--output ./e2eshark-reports/vai-hf-cnn-fp32.json \ | ||
--report --report-file ./e2eshark-reports/vai-hf-cnn-fp32.md | ||
python ./test-suite/alt_e2eshark/utils/merge_dicts.py \ | ||
--sources ./e2eshark-reports/ci_reports_${{ matrix.backend }}_vai-int8-p0p1-shard1_onnx_json/vai-int8-p0p1-shard1.json \ | ||
./e2eshark-reports/ci_reports_${{ matrix.backend }}_vai-int8-p0p1-shard2_onnx_json/vai-int8-p0p1-shard2.json \ | ||
./e2eshark-reports/ci_reports_${{ matrix.backend }}_vai-int8-p0p1-shard3_onnx_json/vai-int8-p0p1-shard3.json \ | ||
--output ./e2eshark-reports/vai-int8-p0p1.json \ | ||
--report --report-file ./e2eshark-reports/vai-int8-p0p1.md | ||
python ./test-suite/alt_e2eshark/utils/merge_dicts.py \ | ||
--sources ./e2eshark-reports/ci_reports_${{ matrix.backend }}_nlp-shard1_onnx_json/nlp-shard1.json \ | ||
./e2eshark-reports/ci_reports_${{ matrix.backend }}_nlp-shard2_onnx_json/nlp-shard2.json \ | ||
./e2eshark-reports/ci_reports_${{ matrix.backend }}_nlp-shard3_onnx_json/nlp-shard3.json \ | ||
--output ./e2eshark-reports/nlp.json \ | ||
--report --report-file ./e2eshark-reports/nlp.md | ||
python ./test-suite/alt_e2eshark/utils/merge_dicts.py \ | ||
--sources ./e2eshark-reports/vai-int8-p0p1.json \ | ||
./e2eshark-reports/vai-hf-cnn-fp32.json \ | ||
./e2eshark-reports/nlp.json \ | ||
./e2eshark-reports/ci_reports_${{ matrix.backend }}_shark-test-suite_onnx_json/shark-test-suite.json \ | ||
./e2eshark-reports/ci_reports_${{ matrix.backend }}_vai-vision-int8_onnx_json/vai-vision-int8.json \ | ||
./e2eshark-reports/ci_reports_${{ matrix.backend }}_migraphx_onnx_json/migraphx.json \ | ||
--output ./e2eshark-reports/combined_reports.json \ | ||
--report --report-file ./e2eshark-reports/combined_reports.md | ||
- name: Push status artifacts | ||
run: | | ||
git config user.name "GitHub Actions Bot" | ||
git config user.email "<>" | ||
git pull | ||
date=$(date '+%Y-%m-%d') | ||
mkdir -p ${date}/ci_reports_onnx/${{ matrix.backend }}/vai-hf-cnn-fp32 | ||
mkdir -p ${date}/ci_reports_onnx/${{ matrix.backend }}/vai-int8-p0p1 | ||
mkdir -p ${date}/ci_reports_onnx/${{ matrix.backend }}/shark-test-suite | ||
mkdir -p ${date}/ci_reports_onnx/${{ matrix.backend }}/vai-vision-int8 | ||
mkdir -p ${date}/ci_reports_onnx/${{ matrix.backend }}/migraphx | ||
mkdir -p ${date}/ci_reports_onnx/${{ matrix.backend }}/nlp | ||
mkdir -p ${date}/ci_reports_onnx/${{ matrix.backend }}/combined-reports | ||
cp vai-hf-cnn-fp32.md ${date}/ci_reports_onnx/${{ matrix.backend }}/vai-hf-cnn-fp32/summary.md | ||
cp vai-int8-p0p1.md ${date}/ci_reports_onnx/${{ matrix.backend }}/vai-int8-p0p1/summary.md | ||
cp nlp.md ${date}/ci_reports_onnx/${{ matrix.backend }}/nlp/summary.md | ||
cp ci_reports_${{ matrix.backend }}_shark-test-suite_onnx_md/shark-test-suite.md ${date}/ci_reports_onnx/${{ matrix.backend }}/shark-test-suite/summary.md | ||
cp ci_reports_${{ matrix.backend }}_vai-vision-int8_onnx_md/vai-vision-int8.md ${date}/ci_reports_onnx/${{ matrix.backend }}/vai-vision-int8/summary.md | ||
cp ci_reports_${{ matrix.backend }}_migraphx_onnx_md/migraphx.md ${date}/ci_reports_onnx/${{ matrix.backend }}/migraphx/summary.md | ||
cp combined_reports.md ${date}/ci_reports_onnx/${{ matrix.backend }}/combined-reports/summary.md | ||
git add $date | ||
git commit -m "add CI status reports for e2eshark for ${{ matrix.backend }}" | ||
git push origin main | ||
working-directory: ./e2eshark-reports | ||
- name: Regression Reports | ||
run: | | ||
source report_venv_alt/bin/activate | ||
cd test-suite | ||
mkdir latest | ||
mkdir baseline | ||
wget https://sharkpublic.blob.core.windows.net/sharkpublic/latest-test-suite/combined_reports_${{ matrix.backend }}.json -O latest/combined_reports_${{ matrix.backend }}.json | ||
wget https://sharkpublic.blob.core.windows.net/sharkpublic/baseline-test-suite/combined_reports_${{ matrix.backend }}.json -O baseline/combined_reports_${{ matrix.backend }}.json | ||
cd .. | ||
python ./test-suite/alt_e2eshark/utils/regressions.py \ | ||
--new ./e2eshark-reports/combined_reports.json \ | ||
--old ./test-suite/latest/combined_reports_${{ matrix.backend }}.json \ | ||
--report-file ./e2eshark-reports/yesterday_comparison.md \ | ||
--perf_tol_regression=0.05 \ | ||
--perf_tol_progression=0.05 | ||
python ./test-suite/alt_e2eshark/utils/regressions.py \ | ||
--new ./e2eshark-reports/combined_reports.json \ | ||
--old ./test-suite/baseline/combined_reports_${{ matrix.backend }}.json \ | ||
--report-file ./e2eshark-reports/baseline_comparison.md \ | ||
--perf_tol_regression=0.1 \ | ||
--perf_tol_progression=0.1 | ||
az storage blob upload --account-name sharkpublic --container-name sharkpublic \ | ||
--name latest-test-suite/combined_reports_${{ matrix.backend }}.json \ | ||
--file ./e2eshark-reports/combined_reports.json \ | ||
--account-key ${AZ_PUBLIC_KEY} --overwrite | ||
- name: Push regression artifacts | ||
run: | | ||
git config user.name "GitHub Actions Bot" | ||
git config user.email "<>" | ||
git pull | ||
date=$(date '+%Y-%m-%d') | ||
cp yesterday_comparison.md ${date}/ci_reports_onnx/${{ matrix.backend }}/combined-reports/yesterday_comparison.md | ||
cp baseline_comparison.md ${date}/ci_reports_onnx/${{ matrix.backend }}/combined-reports/baseline_comparison.md | ||
git add $date | ||
git commit -m "add CI regression reports for e2eshark for ${{ matrix.backend }}" | ||
git push origin main | ||
working-directory: ./e2eshark-reports |