Skip to content

Parallelize remote file downloads within each iree_tests directory. #462

Parallelize remote file downloads within each iree_tests directory.

Parallelize remote file downloads within each iree_tests directory. #462

Workflow file for this run

# Copyright 2024 Advanced Micro Devices, Inc.
#
# Licensed under the Apache License v2.0 with LLVM Exceptions.
# See https://llvm.org/LICENSE.txt for license information.
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
name: IREE Test Suite
on:
pull_request:
paths:
# This file itself.
- ".github/workflows/test_iree.yml"
# The iree_tests subproject.
- "iree_tests/**"
# The iree_special_models subproject.
- "iree_special_models/**"
workflow_dispatch:
schedule:
# Runs at 3:00 PM UTC, which is 8:00 AM PST
- cron: "0 15 * * *"
concurrency:
# A PR number if a pull request and otherwise the commit hash. This cancels
# queued and in-progress runs for the same PR (presubmit) or commit
# (postsubmit). The workflow name is prepended to avoid conflicts between
# different workflows.
group: ${{ github.workflow }}-${{ github.event.number || github.sha }}
cancel-in-progress: true
jobs:
onnx:
name: "ONNX :: ${{ matrix.name }} :: ${{ matrix.runs-on }}"
runs-on: ${{ matrix.runs-on }}
strategy:
fail-fast: false
matrix:
include:
- name: cpu_llvm_sync
runs-on: ubuntu-24.04
config-file: onnx_cpu_llvm_sync.json
env:
VENV_DIR: ${{ github.workspace }}/.venv
CONFIG_FILE_PATH: iree_tests/configs/${{ matrix.config-file }}
steps:
- name: "Checking out repository"
uses: actions/checkout@v4
with:
submodules: false
lfs: true
# Install Python packages.
- name: "Setting up Python"
uses: actions/setup-python@v5
with:
python-version: "3.11"
- name: "Setup Python venv"
run: python3 -m venv ${VENV_DIR}
- name: "Installing IREE nightly release Python packages"
run: |
source ${VENV_DIR}/bin/activate
python3 -m pip install \
--find-links https://iree.dev/pip-release-links.html \
--upgrade \
iree-compiler \
iree-runtime
- name: "Installing Python test requirements"
run: |
source ${VENV_DIR}/bin/activate
python3 -m pip install -r iree_tests/requirements.txt
pip install --no-compile --pre --upgrade -e common_tools
# Run tests and output new config files as needed.
- name: "Running the generated ONNX test suite"
run: |
source ${VENV_DIR}/bin/activate
pytest iree_tests/onnx/ \
-n auto \
-rpfE \
--timeout=30 \
--durations=10 \
--report-log=/tmp/iree_tests_onnx_${{ matrix.name }}_logs.json \
--config-files=${CONFIG_FILE_PATH}
- name: "Updating config file with latest XFAIL lists"
if: failure()
run: |
source ${VENV_DIR}/bin/activate
python iree_tests/update_config_xfails.py \
--log-file=/tmp/iree_tests_onnx_${{ matrix.name }}_logs.json \
--config-file=${CONFIG_FILE_PATH}
cat ${CONFIG_FILE_PATH}
- name: "Uploading new config file"
if: failure()
uses: actions/upload-artifact@v4
with:
name: ${{ matrix.config-file }}
path: ${{ env.CONFIG_FILE_PATH }}
models:
name: "Models :: ${{ matrix.name }} :: ${{ matrix.runs-on }}"
runs-on: ${{ matrix.runs-on }}
strategy:
fail-fast: false
matrix:
include:
- name: cpu_llvm_task
runs-on: nodai-amdgpu-mi250-x86-64
models-config-file: models_cpu_llvm_task.json
backend: cpu
- name: gpu_mi250_rocm
runs-on: nodai-amdgpu-mi250-x86-64
models-config-file: models_gpu_rocm_gfx90a.json
backend: rocm
env:
VENV_DIR: ${{ github.workspace }}/.venv
ROCM_CHIP: gfx90a
IREE_TEST_FILES: ~/iree_tests_cache
IREE_TEST_PATH_EXTENSION: ${{ github.workspace }}/iree_tests/specs
MODELS_CONFIG_FILE_PATH: iree_tests/configs/${{ matrix.models-config-file }}
TEST_SUITE_REPO_ROOT: ${{ github.workspace }}
steps:
- name: "Checking out repository"
uses: actions/checkout@v4
with:
submodules: false
lfs: true
# Install Python packages.
# Note: skipping actions/setup-python since these self-hosted runners already have Python.
- name: "Setup Python venv"
run: python3 -m venv ${VENV_DIR}
- name: "Installing IREE nightly release Python packages"
run: |
source ${VENV_DIR}/bin/activate
python3 -m pip install \
--find-links https://iree.dev/pip-release-links.html \
--upgrade \
iree-compiler \
iree-runtime
- name: "Installing Python test requirements"
run: |
source ${VENV_DIR}/bin/activate
python3 -m pip install -r iree_tests/requirements.txt
python3 -m pip install -r iree_special_models/requirements.txt
pip install --no-compile --pre --upgrade -e common_tools
# Download remote files.
- name: "Downloading remote files for real weight model tests"
run: |
source ${VENV_DIR}/bin/activate
python3 iree_tests/download_remote_files.py --root-dir iree_special_models
python3 iree_tests/download_remote_files.py --root-dir iree_tests/pytorch/models
python3 iree_tests/download_remote_files.py --root-dir iree_tests/sharktank
- name: "Running real weight model tests"
if: "matrix.models-config-file != '' && !cancelled()"
run: |
source ${VENV_DIR}/bin/activate
pytest \
iree_tests/pytorch/models \
iree_tests/sharktank \
-n 4 \
-rA \
-k real_weights \
--no-skip-tests-missing-files \
--capture=no \
--log-cli-level=info \
--timeout=1200 \
--durations=0 \
--config-files=${MODELS_CONFIG_FILE_PATH}
- name: "Running SDXL special model tests"
id: special_models_sdxl
if: ${{ !cancelled() }}
run: |
source ${VENV_DIR}/bin/activate
pytest iree_special_models/sdxl \
-k ${{ matrix.backend }} \
-rpfE \
--capture=no \
--log-cli-level=info \
--timeout=1200 \
--durations=0
- name: "Running SD3 special model tests"
id: special_models_sd3
if: ${{ !cancelled() }}
run: |
source ${VENV_DIR}/bin/activate
pytest iree_special_models/sd3 \
-k ${{ matrix.backend }} \
-rpfE \
--capture=no \
--log-cli-level=info \
--timeout=1200 \
--durations=0
- name: "Running SDXL ROCm pipeline benchmark"
if: |
contains(matrix.name, 'rocm') &&
steps.special_models_sdxl.outcome == 'success'
run: |
source ${VENV_DIR}/bin/activate
pytest iree_tests/benchmarks/sdxl/benchmark_sdxl_rocm.py \
--goldentime-rocm-e2e-ms 1450.0 \
--goldentime-rocm-unet-ms 370.0 \
--goldentime-rocm-clip-ms 18.5 \
--goldentime-rocm-vae-ms 315.0 \
--goldendispatch-rocm-unet 1714 \
--goldendispatch-rocm-clip 1569 \
--goldendispatch-rocm-vae 248 \
--goldensize-rocm-unet-bytes 2280000 \
--goldensize-rocm-clip-bytes 860000 \
--goldensize-rocm-vae-bytes 840000 \
--gpu-number 6 \
--rocm-chip gfx90a \
--log-cli-level=info
echo "$(<job_summary.md )" >> $GITHUB_STEP_SUMMARY