E2ESHARK Test Suite #269
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Copyright 2024 Advanced Micro Devices | |
| # | |
| # Licensed under the Apache License v2.0 with LLVM Exceptions. | |
| # See https://llvm.org/LICENSE.txt for license information. | |
| # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception | |
| name: E2ESHARK Test Suite | |
| on: | |
| workflow_dispatch: | |
| schedule: | |
| # Runs at 12:00 PM UTC, which is 5:00 AM PST | |
| - cron: '0 12 * * *' | |
| jobs: | |
| e2eshark: | |
| name: "Models :: ${{ matrix.backend }} :: ${{ matrix.test-file }}" | |
| runs-on: ${{ matrix.runs-on }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| - name: mi300_gpu1_test | |
| runs-on: nodai-amdgpu-mi300-x86-64 | |
| backend: rocm | |
| device: hip | |
| target-chip: gfx942 | |
| test-file: shark-test-suite | |
| visible-device: 0 | |
| - name: mi300_gpu1_test | |
| runs-on: nodai-amdgpu-mi300-x86-64 | |
| backend: rocm | |
| device: hip | |
| target-chip: gfx942 | |
| test-file: vai-hf-cnn-fp32-shard1 | |
| visible-device: 1 | |
| - name: mi300_gpu1_test | |
| runs-on: nodai-amdgpu-mi300-x86-64 | |
| backend: rocm | |
| device: hip | |
| target-chip: gfx942 | |
| test-file: vai-hf-cnn-fp32-shard2 | |
| visible-device: 2 | |
| - name: mi300_gpu1_test | |
| runs-on: nodai-amdgpu-mi300-x86-64 | |
| backend: rocm | |
| device: hip | |
| target-chip: gfx942 | |
| test-file: vai-hf-cnn-fp32-shard3 | |
| visible-device: 3 | |
| - name: mi300_gpu1_test | |
| runs-on: nodai-amdgpu-mi300-x86-64 | |
| backend: rocm | |
| device: hip | |
| target-chip: gfx942 | |
| test-file: vai-int8-p0p1-shard1 | |
| visible-device: 4 | |
| - name: mi300_gpu1_test | |
| runs-on: nodai-amdgpu-mi300-x86-64 | |
| backend: rocm | |
| device: hip | |
| target-chip: gfx942 | |
| test-file: vai-int8-p0p1-shard2 | |
| visible-device: 5 | |
| - name: mi300_gpu1_test | |
| runs-on: nodai-amdgpu-mi300-x86-64 | |
| backend: rocm | |
| device: hip | |
| target-chip: gfx942 | |
| test-file: vai-int8-p0p1-shard3 | |
| visible-device: 6 | |
| - name: mi300_gpu1_test | |
| runs-on: nodai-amdgpu-mi300-x86-64 | |
| backend: rocm | |
| device: hip | |
| target-chip: gfx942 | |
| test-file: vai-vision-int8 | |
| visible-device: 7 | |
| env: | |
| E2E_VENV_DIR: ${{ github.workspace }}/test-suite_venv | |
| HF_TOKEN: ${{ secrets.HF_TOKEN }} | |
| AZ_PRIVATE_CONNECTION: ${{ secrets.ONNXPRIVATESTORAGE_AZ_PRIVATE_CONNECTION }} | |
| ROCR_VISIBLE_DEVICES: ${{ matrix.visible-device }} | |
| steps: | |
| - name: Checkout Test Suite | |
| uses: actions/checkout@v2 | |
| with: | |
| repository: nod-ai/SHARK-TestSuite | |
| ref: mi300 | |
| path: test-suite | |
| - name: Checkout SHARK Turbine | |
| uses: actions/checkout@v2 | |
| with: | |
| repository: nod-ai/SHARK-Turbine | |
| path: SHARK-Turbine | |
| - name: Checkout iree turbine | |
| uses: actions/checkout@v2 | |
| with: | |
| repository: iree-org/iree-turbine | |
| path: iree-turbine | |
| - name: "Setup e2eshark python venv" | |
| run: | | |
| python3.11 -m venv ${E2E_VENV_DIR} | |
| source ${E2E_VENV_DIR}/bin/activate | |
| pip install --upgrade pip | |
| pip install -r ./e2eshark/requirements.txt | |
| pip install \ | |
| --find-links https://iree.dev/pip-release-links.html \ | |
| --upgrade \ | |
| iree-compiler \ | |
| iree-runtime | |
| pip install \ | |
| --find-links https://github.com/llvm/torch-mlir-release/releases/expanded_assets/dev-wheels \ | |
| --upgrade \ | |
| torch-mlir | |
| working-directory: ./test-suite | |
| - name: Run Onnx Mode | |
| run: | | |
| source ${E2E_VENV_DIR}/bin/activate | |
| pip list | |
| cd e2eshark | |
| free -mh | |
| python3.11 ./run.py \ | |
| -r ./test-onnx \ | |
| --report \ | |
| --cachedir /home/sai/shark-test-suite-models-cache \ | |
| --testsfile ./${{ matrix.test-file }}.txt \ | |
| --mode onnx \ | |
| -f onnx pytorch \ | |
| --targetbackend ${{ matrix.backend }} \ | |
| --device ${{ matrix.device }} \ | |
| --targetchip ${{ matrix.target-chip }} \ | |
| --tolerance .01 .01 \ | |
| --cleanup \ | |
| --postprocess \ | |
| --ci \ | |
| -v | |
| working-directory: ./test-suite | |
| - name: Setup turbine python venv | |
| run: | | |
| source ${E2E_VENV_DIR}/bin/activate | |
| pip install -f https://iree.dev/pip-release-links.html --upgrade \ | |
| -r ../iree-turbine/iree-requirements.txt | |
| pip install -e ../iree-turbine[testing] | |
| pip install --no-compile --pre --upgrade -e ../SHARK-Turbine/models -r ../SHARK-Turbine/models/requirements.txt | |
| working-directory: ./test-suite | |
| - name: Run Turbine Mode | |
| if: contains(matrix.test-file, 'shark-test-suite') | |
| run: | | |
| source ${E2E_VENV_DIR}/bin/activate | |
| cd e2eshark | |
| free -mh | |
| HF_TOKEN=${{ secrets.HF_TOKEN }} python3.11 ./run.py \ | |
| -r ./test-turbine \ | |
| --report \ | |
| --cachedir /home/sai/shark-test-suite-models-cache \ | |
| --mode turbine \ | |
| --tests pytorch/models/bart-large \ | |
| --targetbackend ${{ matrix.backend }} \ | |
| --device ${{ matrix.device }} \ | |
| --targetchip ${{ matrix.target-chip }} \ | |
| --cleanup \ | |
| --postprocess \ | |
| --ci \ | |
| -v | |
| working-directory: ./test-suite | |
| - name: Checkout repo | |
| uses: actions/checkout@v2 | |
| with: | |
| repository: nod-ai/e2eshark-reports | |
| ref: 'main' | |
| token: ${{ secrets.E2ESHARK_GITHUB_TOKEN }} | |
| path: e2eshark-reports | |
| - name: Push artifacts | |
| run: | | |
| date=$(date '+%Y-%m-%d') | |
| mkdir -p ${date}/ci_reports_${{ matrix.backend }}_onnx/${{ matrix.test-file }} | |
| mkdir -p ${date}/ci_reports_${{ matrix.backend }}_turbine/turbine_models | |
| mkdir -p ../test-suite/e2eshark/ci_reports_${{ matrix.backend }}_turbine/turbine_models | |
| cp -r ../test-suite/e2eshark/ci_reports_${{ matrix.backend }}_onnx/${{ matrix.test-file }} ${date}/ci_reports_${{ matrix.backend }}_onnx/ | |
| cp -r ../test-suite/e2eshark/ci_reports_${{ matrix.backend }}_turbine/turbine_models/. ${date}/ci_reports_${{ matrix.backend }}_turbine/ | |
| git config user.name "GitHub Actions Bot" | |
| git config user.email "<>" | |
| git add $date | |
| git commit -m "add CI reports for e2eshark for ${{ matrix.test-file }}" | |
| git push origin main | |
| working-directory: ./e2eshark-reports |