E2ESHARK Test Suite #313
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
# Copyright 2024 Advanced Micro Devices | |
# | |
# Licensed under the Apache License v2.0 with LLVM Exceptions. | |
# See https://llvm.org/LICENSE.txt for license information. | |
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception | |
name: E2ESHARK Test Suite | |
on: | |
workflow_dispatch: | |
# schedule: | |
# # Runs at 12:00 PM UTC, which is 5:00 AM PST | |
# - cron: '0 12 * * *' | |
jobs: | |
e2eshark: | |
name: "Models :: ${{ matrix.backend }} :: ${{ matrix.test-file }}" | |
runs-on: ${{ matrix.runs-on }} | |
strategy: | |
fail-fast: false | |
matrix: | |
include: | |
- name: mi300_gpu1_test | |
runs-on: nodai-amdgpu-mi300-x86-64 | |
backend: rocm | |
device: hip | |
target-chip: gfx942 | |
test-file: shark-test-suite | |
visible-device: 0 | |
cache-dir: /home/sai/shark-test-suite-models-cache | |
- name: mi300_gpu2_test | |
runs-on: nodai-amdgpu-mi300-x86-64 | |
backend: rocm | |
device: hip | |
target-chip: gfx942 | |
test-file: vai-hf-cnn-fp32-shard1 | |
visible-device: 1 | |
cache-dir: /home/sai/shark-test-suite-models-cache | |
- name: mi300_gpu3_test | |
runs-on: nodai-amdgpu-mi300-x86-64 | |
backend: rocm | |
device: hip | |
target-chip: gfx942 | |
test-file: vai-hf-cnn-fp32-shard2 | |
visible-device: 2 | |
cache-dir: /home/sai/shark-test-suite-models-cache | |
- name: mi300_gpu4_test | |
runs-on: nodai-amdgpu-mi300-x86-64 | |
backend: rocm | |
device: hip | |
target-chip: gfx942 | |
test-file: vai-hf-cnn-fp32-shard3 | |
visible-device: 3 | |
cache-dir: /home/sai/shark-test-suite-models-cache | |
- name: mi300_gpu5_test | |
runs-on: nodai-amdgpu-mi300-x86-64 | |
backend: rocm | |
device: hip | |
target-chip: gfx942 | |
test-file: vai-int8-p0p1-shard1 | |
visible-device: 4 | |
cache-dir: /home/sai/shark-test-suite-models-cache | |
- name: mi300_gpu6_test | |
runs-on: nodai-amdgpu-mi300-x86-64 | |
backend: rocm | |
device: hip | |
target-chip: gfx942 | |
test-file: vai-int8-p0p1-shard2 | |
visible-device: 5 | |
cache-dir: /home/sai/shark-test-suite-models-cache | |
- name: mi300_gpu7_test | |
runs-on: nodai-amdgpu-mi300-x86-64 | |
backend: rocm | |
device: hip | |
target-chip: gfx942 | |
test-file: vai-int8-p0p1-shard3 | |
visible-device: 6 | |
cache-dir: /home/sai/shark-test-suite-models-cache | |
- name: mi300_gpu8_test | |
runs-on: nodai-amdgpu-mi300-x86-64 | |
backend: rocm | |
device: hip | |
target-chip: gfx942 | |
test-file: vai-vision-int8 | |
visible-device: 7 | |
cache-dir: /home/sai/shark-test-suite-models-cache | |
- name: cpu_shard1_test | |
runs-on: nodai-amdgpu-mi250-x86-64 | |
backend: llvm-cpu | |
device: local-task | |
target-chip: x86_64-linux-gnu | |
test-file: shark-test-suite | |
cache-dir: /groups/aig_sharks/test-suite-ci-cache | |
- name: cpu_shard2_test | |
runs-on: nodai-amdgpu-mi250-x86-64 | |
backend: llvm-cpu | |
device: local-task | |
target-chip: x86_64-linux-gnu | |
test-file: vai-hf-cnn-fp32-shard1 | |
cache-dir: /groups/aig_sharks/test-suite-ci-cache | |
- name: cpu_shard3_test | |
runs-on: nodai-amdgpu-mi250-x86-64 | |
backend: llvm-cpu | |
device: local-task | |
target-chip: x86_64-linux-gnu | |
test-file: vai-hf-cnn-fp32-shard2 | |
cache-dir: /groups/aig_sharks/test-suite-ci-cache | |
- name: cpu_shard4_test | |
runs-on: nodai-amdgpu-mi250-x86-64 | |
backend: llvm-cpu | |
device: local-task | |
target-chip: x86_64-linux-gnu | |
test-file: vai-hf-cnn-fp32-shard3 | |
cache-dir: /groups/aig_sharks/test-suite-ci-cache | |
- name: cpu_shard5_test | |
runs-on: nodai-amdgpu-mi250-x86-64 | |
backend: llvm-cpu | |
device: local-task | |
target-chip: x86_64-linux-gnu | |
test-file: vai-int8-p0p1-shard1 | |
cache-dir: /groups/aig_sharks/test-suite-ci-cache | |
- name: cpu_shard6_test | |
runs-on: nodai-amdgpu-mi250-x86-64 | |
backend: llvm-cpu | |
device: local-task | |
target-chip: x86_64-linux-gnu | |
test-file: vai-int8-p0p1-shard2 | |
cache-dir: /groups/aig_sharks/test-suite-ci-cache | |
- name: cpu_shard7_test | |
runs-on: nodai-amdgpu-mi250-x86-64 | |
backend: llvm-cpu | |
device: local-task | |
target-chip: x86_64-linux-gnu | |
test-file: vai-int8-p0p1-shard3 | |
cache-dir: /groups/aig_sharks/test-suite-ci-cache | |
- name: cpu_shard8_test | |
runs-on: nodai-amdgpu-mi250-x86-64 | |
backend: llvm-cpu | |
device: local-task | |
target-chip: x86_64-linux-gnu | |
test-file: vai-vision-int8 | |
cache-dir: /groups/aig_sharks/test-suite-ci-cache | |
env: | |
E2E_VENV_DIR: ${{ github.workspace }}/test-suite_venv | |
ALT_E2E_VENV_DIR: ${{ github.workspace }}/alt-test-suite_venv | |
HF_TOKEN: ${{ secrets.HF_TOKEN }} | |
AZ_PRIVATE_CONNECTION: ${{ secrets.ONNXPRIVATESTORAGE_AZ_PRIVATE_CONNECTION }} | |
ROCR_VISIBLE_DEVICES: ${{ matrix.visible-device }} | |
CACHE_DIR: ${{ matrix.cache-dir }} | |
steps: | |
- name: Checkout Test Suite | |
uses: actions/checkout@v2 | |
with: | |
repository: nod-ai/SHARK-TestSuite | |
ref: alt-merge-reports | |
path: test-suite | |
- name: Checkout SHARK Turbine | |
uses: actions/checkout@v2 | |
with: | |
repository: nod-ai/SHARK-Turbine | |
path: SHARK-Turbine | |
- name: Checkout iree turbine | |
uses: actions/checkout@v2 | |
with: | |
repository: iree-org/iree-turbine | |
path: iree-turbine | |
- name: "Setup e2eshark python venv" | |
run: | | |
python3.11 -m venv ${E2E_VENV_DIR} | |
source ${E2E_VENV_DIR}/bin/activate | |
pip install --upgrade pip | |
pip install -r ./e2eshark/requirements.txt | |
pip install \ | |
--find-links https://iree.dev/pip-release-links.html \ | |
--upgrade \ | |
iree-compiler \ | |
iree-runtime | |
working-directory: ./test-suite | |
- name: "Setup alt e2eshark python venv" | |
run: | | |
python3.11 -m venv ${ALT_E2E_VENV_DIR} | |
source ${ALT_E2E_VENV_DIR}/bin/activate | |
pip install --upgrade pip | |
pip install -r ./alt_e2eshark/base_requirements.txt | |
pip install -r ./alt_e2eshark/iree_requirements.txt | |
pip install --no-deps -r ./alt_e2eshark/torch_mlir_requirements.txt | |
working-directory: ./test-suite | |
- name: Run Onnx Mode | |
run: | | |
source ${ALT_E2E_VENV_DIR}/bin/activate | |
pip list | |
cd alt_e2eshark | |
free -mh | |
python3.11 ./run.py \ | |
-r ./test-onnx \ | |
--report \ | |
--testsfile onnx_tests/models/external_lists/${{ matrix.test-file }}.txt \ | |
-b ${{ matrix.backend }} \ | |
-d ${{ matrix.device }} \ | |
--report-file reports/${{ matrix.test-file }}.md \ | |
--mode=cl-onnx-iree \ | |
-v | |
working-directory: ./test-suite | |
- name: Setup turbine python venv | |
run: | | |
source ${E2E_VENV_DIR}/bin/activate | |
pip install -f https://iree.dev/pip-release-links.html --upgrade \ | |
-r ../iree-turbine/iree-requirements.txt | |
pip install -e ../iree-turbine[testing] | |
pip install --no-compile --pre --upgrade -e ../SHARK-Turbine/models -r ../SHARK-Turbine/models/requirements.txt | |
working-directory: ./test-suite | |
- name: Run Turbine Mode | |
if: contains(matrix.test-file, 'shark-test-suite') | |
run: | | |
source ${E2E_VENV_DIR}/bin/activate | |
cd e2eshark | |
free -mh | |
python3.11 ./run.py \ | |
-r ./test-turbine \ | |
--report \ | |
--cachedir ${{ matrix.cache-dir }} \ | |
--mode turbine \ | |
-g models \ | |
--targetbackend ${{ matrix.backend }} \ | |
--device ${{ matrix.device }} \ | |
--targetchip ${{ matrix.target-chip }} \ | |
--cleanup \ | |
--postprocess \ | |
--ci \ | |
-v | |
working-directory: ./test-suite | |
- uses: actions/upload-artifact@master | |
with: | |
name: ci_reports_${{ matrix.backend }}_${{ matrix.test-file }}_onnx.md | |
path: ./test-suite/alt_e2eshark/reports/${{ matrix.test-file }}.md | |
- uses: actions/upload-artifact@master | |
with: | |
name: ci_reports_${{ matrix.backend }}_${{ matrix.test-file }}_onnx.json | |
path: ./test-suite/alt_e2eshark/reports/${{ matrix.test-file }}.json | |
- uses: actions/upload-artifact@master | |
if: contains(matrix.test-file, 'shark-test-suite') | |
with: | |
name: ci_reports_${{ matrix.backend }}_turbine | |
path: ./test-suite/e2eshark/ci_reports_${{ matrix.backend }}_turbine | |
push_artifacts: | |
needs: [e2eshark] | |
runs-on: ${{ matrix.runs-on }} | |
strategy: | |
fail-fast: false | |
max-parallel: 1 | |
matrix: | |
include: | |
- name: merge_rocm_reports | |
runs-on: nodai-amdgpu-mi300-x86-64 | |
backend: rocm | |
- name: merge_cpu_reports | |
runs-on: nodai-amdgpu-mi250-x86-64 | |
backend: llvm-cpu | |
steps: | |
- name: Checkout Test Suite | |
uses: actions/checkout@v2 | |
with: | |
repository: nod-ai/SHARK-TestSuite | |
ref: alt-merge-reports | |
path: test-suite | |
- name: Checkout repo | |
uses: actions/checkout@v2 | |
with: | |
repository: nod-ai/e2eshark-reports | |
ref: main | |
token: ${{ secrets.E2ESHARK_GITHUB_TOKEN }} | |
path: e2eshark-reports | |
- name: "Setup alt test suite venv" | |
run: | | |
python3.11 -m venv report_venv_alt | |
source report_venv_alt/bin/activate | |
pip install --upgrade pip | |
pip install -r ./test-suite/alt_e2eshark/base_requirements.txt | |
pip install -r ./test-suite/alt_e2eshark/iree_requirements.txt | |
pip install --no-deps -r ./test-suite/alt_e2eshark/torch_mlir_requirements.txt | |
- uses: actions/download-artifact@master | |
with: | |
name: ci_reports_${{ matrix.backend }}_shark-test-suite_onnx.md | |
path: ./e2eshark-reports/ci_reports_${{ matrix.backend }}_shark-test-suite_onnx.md | |
- uses: actions/download-artifact@master | |
with: | |
name: ci_reports_${{ matrix.backend }}_shark-test-suite_onnx.json | |
path: ./e2eshark-reports/ci_reports_${{ matrix.backend }}_shark-test-suite_onnx.json | |
- uses: actions/download-artifact@master | |
with: | |
name: ci_reports_${{ matrix.backend }}_vai-hf-cnn-fp32-shard1_onnx.md | |
path: ./e2eshark-reports/ci_reports_${{ matrix.backend }}_vai-hf-cnn-fp32-shard1_onnx.md | |
- uses: actions/download-artifact@master | |
with: | |
name: ci_reports_${{ matrix.backend }}_vai-hf-cnn-fp32-shard1_onnx.json | |
path: ./e2eshark-reports/ci_reports_${{ matrix.backend }}_vai-hf-cnn-fp32-shard1_onnx.json | |
- uses: actions/download-artifact@master | |
with: | |
name: ci_reports_${{ matrix.backend }}_vai-hf-cnn-fp32-shard2_onnx.md | |
path: ./e2eshark-reports/ci_reports_${{ matrix.backend }}_vai-hf-cnn-fp32-shard2_onnx.md | |
- uses: actions/download-artifact@master | |
with: | |
name: ci_reports_${{ matrix.backend }}_vai-hf-cnn-fp32-shard2_onnx.json | |
path: ./e2eshark-reports/ci_reports_${{ matrix.backend }}_vai-hf-cnn-fp32-shard2_onnx.json | |
- uses: actions/download-artifact@master | |
with: | |
name: ci_reports_${{ matrix.backend }}_vai-hf-cnn-fp32-shard3_onnx.md | |
path: ./e2eshark-reports/ci_reports_${{ matrix.backend }}_vai-hf-cnn-fp32-shard3_onnx.md | |
- uses: actions/download-artifact@master | |
with: | |
name: ci_reports_${{ matrix.backend }}_vai-hf-cnn-fp32-shard3_onnx.json | |
path: ./e2eshark-reports/ci_reports_${{ matrix.backend }}_vai-hf-cnn-fp32-shard3_onnx.json | |
- uses: actions/download-artifact@master | |
with: | |
name: ci_reports_${{ matrix.backend }}_vai-int8-p0p1-shard1_onnx.md | |
path: ./e2eshark-reports/ci_reports_${{ matrix.backend }}_vai-int8-p0p1-shard1_onnx.md | |
- uses: actions/download-artifact@master | |
with: | |
name: ci_reports_${{ matrix.backend }}_vai-int8-p0p1-shard1_onnx.json | |
path: ./e2eshark-reports/ci_reports_${{ matrix.backend }}_vai-int8-p0p1-shard1_onnx.json | |
- uses: actions/download-artifact@master | |
with: | |
name: ci_reports_${{ matrix.backend }}_vai-int8-p0p1-shard2_onnx.md | |
path: ./e2eshark-reports/ci_reports_${{ matrix.backend }}_vai-int8-p0p1-shard2_onnx.md | |
- uses: actions/download-artifact@master | |
with: | |
name: ci_reports_${{ matrix.backend }}_vai-int8-p0p1-shard2_onnx.json | |
path: ./e2eshark-reports/ci_reports_${{ matrix.backend }}_vai-int8-p0p1-shard2_onnx.json | |
- uses: actions/download-artifact@master | |
with: | |
name: ci_reports_${{ matrix.backend }}_vai-int8-p0p1-shard3_onnx.md | |
path: ./e2eshark-reports/ci_reports_${{ matrix.backend }}_vai-int8-p0p1-shard3_onnx.md | |
- uses: actions/download-artifact@master | |
with: | |
name: ci_reports_${{ matrix.backend }}_vai-int8-p0p1-shard3_onnx.json | |
path: ./e2eshark-reports/ci_reports_${{ matrix.backend }}_vai-int8-p0p1-shard3_onnx.json | |
- uses: actions/download-artifact@master | |
with: | |
name: ci_reports_${{ matrix.backend }}_vai-vision-int8_onnx.md | |
path: ./e2eshark-reports/ci_reports_${{ matrix.backend }}_vai-vision-int8_onnx.md | |
- uses: actions/download-artifact@master | |
with: | |
name: ci_reports_${{ matrix.backend }}_vai-vision-int8_onnx.json | |
path: ./e2eshark-reports/ci_reports_${{ matrix.backend }}_vai-vision-int8_onnx.json | |
- uses: actions/download-artifact@master | |
with: | |
name: ci_reports_${{ matrix.backend }}_turbine | |
path: ./e2eshark-reports/${{ matrix.backend }} | |
- name: Merge Reports | |
run: | | |
source report_venv_alt/bin/activate | |
python ./test-suite/alt_e2eshark/utils/merge_dicts.py \ | |
--sources ./e2eshark-reports/ci_reports_${{ matrix.backend }}_vai-hf-cnn-fp32-shard1_onnx.json/vai-hf-cnn-fp32-shard1.json \ | |
./e2eshark-reports/ci_reports_${{ matrix.backend }}_vai-hf-cnn-fp32-shard2_onnx.json/vai-hf-cnn-fp32-shard2.json \ | |
./e2eshark-reports/ci_reports_${{ matrix.backend }}_vai-hf-cnn-fp32-shard3_onnx.json/vai-hf-cnn-fp32-shard3.json \ | |
--output ./e2eshark-reports/vai-hf-cnn-fp32.json \ | |
--report --report-file ./e2eshark-reports/vai-hf-cnn-fp32.md | |
python ./test-suite/alt_e2eshark/utils/merge_dicts.py \ | |
--sources ./e2eshark-reports/ci_reports_${{ matrix.backend }}_vai-int8-p0p1-shard1_onnx.json/vai-int8-p0p1-shard1.json \ | |
./e2eshark-reports/ci_reports_${{ matrix.backend }}_vai-int8-p0p1-shard2_onnx.json/vai-int8-p0p1-shard2.json \ | |
./e2eshark-reports/ci_reports_${{ matrix.backend }}_vai-int8-p0p1-shard3_onnx.json/vai-int8-p0p1-shard3.json \ | |
--output ./e2eshark-reports/vai-int8-p0p1.json \ | |
--report --report-file ./e2eshark-reports/vai-int8-p0p1.md | |
python ./test-suite/alt_e2eshark/utils/merge_dicts.py \ | |
--sources ./e2eshark-reports/vai-int8-p0p1.json \ | |
./e2eshark-reports/vai-hf-cnn-fp32.json \ | |
./e2eshark-reports/ci_reports_${{ matrix.backend }}_shark-test-suite_onnx.json/shark-test-suite.json \ | |
./e2eshark-reports/ci_reports_${{ matrix.backend }}_vai-vision-int8_onnx.json/vai-vision-int8.json \ | |
--output ./e2eshark-reports/combined_reports.json \ | |
--report --report-file ./e2eshark-reports/combined_reports.md | |
- name: Push artifacts | |
run: | | |
git config user.name "GitHub Actions Bot" | |
git config user.email "<>" | |
git pull | |
date=$(date '+%Y-%m-%d') | |
mkdir -p ${date}/ci_reports_onnx/${{ matrix.backend }}/vai-hf-cnn-fp32 | |
mkdir -p ${date}/ci_reports_onnx/${{ matrix.backend }}/vai-int8-p0p1 | |
mkdir -p ${date}/ci_reports_onnx/${{ matrix.backend }}/shark-test-suite | |
mkdir -p ${date}/ci_reports_onnx/${{ matrix.backend }}/vai-vision-int8 | |
mkdir -p ${date}/ci_reports_onnx/${{ matrix.backend }}/combined-reports | |
mkdir -p ${date}/ci_reports_turbine/${{ matrix.backend }}/turbine_models | |
cp vai-hf-cnn-fp32.md ${date}/ci_reports_onnx/${{ matrix.backend }}/vai-hf-cnn-fp32/summary.md | |
cp vai-int8-p0p1.md ${date}/ci_reports_onnx/${{ matrix.backend }}/vai-int8-p0p1/summary.md | |
cp ci_reports_${{ matrix.backend }}_shark-test-suite_onnx.md/shark-test-suite.md ${date}/ci_reports_onnx/${{ matrix.backend }}/shark-test-suite/summary.md | |
cp ci_reports_${{ matrix.backend }}_vai-vision-int8_onnx.md/vai-vision-int8.md ${date}/ci_reports_onnx/${{ matrix.backend }}/vai-vision-int8/summary.md | |
cp combined_reports.md ${date}/ci_reports_onnx/${{ matrix.backend }}/combined-reports/summary.md | |
cp -r ./${{ matrix.backend }}/turbine_models/*.md ${date}/ci_reports_turbine/${{ matrix.backend }}/turbine_models | |
git add $date | |
git commit -m "add CI reports for e2eshark for ${{ matrix.backend }}" | |
git push origin main | |
working-directory: ./e2eshark-reports |