test_iree.yml

# Copyright 2024 Advanced Micro Devices, Inc.
#
# Licensed under the Apache License v2.0 with LLVM Exceptions.
# See https://llvm.org/LICENSE.txt for license information.
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception

name: IREE Test Suite
on:
  # TODO(scotttodd): run on schedule (nightly), and/or on pushes to `main`
  pull_request:
    paths:
      - "iree_tests/**"
  workflow_dispatch:

concurrency:
  # A PR number if a pull request and otherwise the commit hash. This cancels
  # queued and in-progress runs for the same PR (presubmit) or commit
  # (postsubmit). The workflow name is prepended to avoid conflicts between
  # different workflows.
  group: ${{ github.workflow }}-${{ github.event.number || github.sha }}
  cancel-in-progress: true

jobs:
  linux_x86_64_onnx:
    if: false  # DO NOT SUBMIT (testing just one job)
    name: Linux (x86_64) Onnx
    runs-on: ubuntu-latest
    env:
      VENV_DIR: ${{ github.workspace }}/.venv
    steps:
      - name: "Checking out repository"
        uses: actions/checkout@v4
        with:
          submodules: false
          lfs: true
      - name: "Setting up Python"
        uses: actions/setup-python@v5
        with:
          python-version: '3.11'

      - name: "Setup Python venv"
        run: python3 -m venv ${VENV_DIR}

      # TODO(scotttodd): swap this with workflow artifact downloads
      # TODO(scotttodd): make this configurable with workflow inputs
      # TODO(scotttodd): pin to a version? in this repo we could just test latest
      - name: "Installing IREE nightly release Python packages"
        run: |
          source ${VENV_DIR}/bin/activate
          python3 -m pip install \
            --find-links https://iree.dev/pip-release-links.html \
            --upgrade \
            iree-compiler \
            iree-runtime

      - name: "Installing other Python requirements"
        run: |
          source ${VENV_DIR}/bin/activate
          python3 -m pip install -r iree_tests/requirements.txt

      - name: "Running simple tests"
        run: |
          source ${VENV_DIR}/bin/activate
          pytest iree_tests/simple --durations=0

      - name: "Running the generated ONNX test suite"
        run: |
          source ${VENV_DIR}/bin/activate
          pytest iree_tests/onnx/node/generated -n auto -rpfE --timeout=30 --retries 2 --retry-delay 5 --durations=10

  linux_x86_64_w7900_gpu_onnx:
    name: Linux (x86_64 w7900) Onnx
    runs-on: nodai-amdgpu-w7900-x86-64
    env:
      VENV_DIR: ${{ github.workspace }}/.venv
    steps:
      - name: "Checking out repository"
        uses: actions/checkout@v4
        with:
          submodules: false
          lfs: true
      - name: "Setting up Python"
        uses: actions/setup-python@v5
        with:
          python-version: '3.11'
      - name: "Setup Python venv"
        run: python3 -m venv ${VENV_DIR}
      - name: "Installing IREE nightly release Python packages"
        run: |
          source ${VENV_DIR}/bin/activate
          python3 -m pip install \
            --find-links https://iree.dev/pip-release-links.html \
            --upgrade iree-compiler iree-runtime
      - name: "Installing other Python requirements"
        run: |
          source ${VENV_DIR}/bin/activate
          python3 -m pip install -r iree_tests/requirements.txt
      - name: "Running the generated ONNX test suite"
        env:
            IREE_TEST_CONFIG_FILES: iree_tests/configs/config_onnx_gpu_rocm_rdna3.json
        run: |
          source ${VENV_DIR}/bin/activate
          pytest iree_tests/onnx/node/generated -rpfE --timeout=30 --durations=10

  linux_x86_64_w7900_gpu_models:
    if: false  # DO NOT SUBMIT (testing just one job)
    name: Linux (x86_64 w7900) Models GPU
    runs-on: nodai-amdgpu-w7900-x86-64
    env:
      VENV_DIR: ${{ github.workspace }}/.venv
    steps:
      - name: "Checking out repository"
        uses: actions/checkout@v4
        with:
          submodules: false
          lfs: true
      - name: "Setting up Python"
        uses: actions/setup-python@v5
        with:
          python-version: '3.11'

      - name: "Setup Python venv"
        run: python3 -m venv ${VENV_DIR}

      - name: "Installing IREE nightly release Python packages"
        run: |
          source ${VENV_DIR}/bin/activate
          python3 -m pip install \
            --find-links https://iree.dev/pip-release-links.html \
            --upgrade \
            iree-compiler \
            iree-runtime

      - name: "Installing other Python requirements"
        run: |
          source ${VENV_DIR}/bin/activate
          python3 -m pip install -r iree_tests/requirements.txt
      # TODO(scotttodd): add a local cache for these large files to a persistent runner
      - name: "Downloading remote files for real weight model tests"
        run: |
          source ${VENV_DIR}/bin/activate
          python3 iree_tests/download_remote_files.py --root-dir pytorch/models
      - name: "Running real weight model tests"
        env:
            IREE_TEST_CONFIG_FILES: iree_tests/configs/config_pytorch_models_cpu_llvm_task.json
        run: |
          source ${VENV_DIR}/bin/activate
          pytest iree_tests/pytorch/models -s -n 4 -k real_weights -rpfE --timeout=1200 --retries 2 --retry-delay 5 --durations=0