diff --git a/.github/workflows/test_iree.yml b/.github/workflows/test_iree.yml index d634e082b..34958ca94 100644 --- a/.github/workflows/test_iree.yml +++ b/.github/workflows/test_iree.yml @@ -21,8 +21,8 @@ concurrency: cancel-in-progress: true jobs: - linux_x86_64: - name: Linux (x86_64) + linux_x86_64_onnx: + name: Linux (x86_64) Onnx runs-on: ubuntu-latest env: VENV_DIR: ${{ github.workspace }}/.venv @@ -67,12 +67,46 @@ jobs: source ${VENV_DIR}/bin/activate pytest iree_tests/onnx/node/generated -n auto -rpfE --timeout=30 --retries 2 --retry-delay 5 --durations=10 + linux_x86_64_w7900_gpu_models: + name: Linux (x86_64 w7900) Models GPU + runs-on: nodai-amdgpu-w7900-x86-64 + env: + VENV_DIR: ${{ github.workspace }}/.venv + steps: + - name: "Checking out repository" + uses: actions/checkout@v4 + with: + submodules: false + lfs: true + - name: "Setting up Python" + uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: "Setup Python venv" + run: python3 -m venv ${VENV_DIR} + + - name: "Installing IREE nightly release Python packages" + run: | + source ${VENV_DIR}/bin/activate + python3 -m pip install \ + --find-links https://iree.dev/pip-release-links.html \ + --upgrade \ + iree-compiler \ + iree-runtime + + - name: "Installing other Python requirements" + run: | + source ${VENV_DIR}/bin/activate + python3 -m pip install -r iree_tests/requirements.txt # TODO(scotttodd): add a local cache for these large files to a persistent runner - name: "Downloading remote files for real weight model tests" run: | source ${VENV_DIR}/bin/activate - python3 iree_tests/download_remote_files.py + python3 iree_tests/download_remote_files.py --root-dir pytorch/models - name: "Running real weight model tests" + env: + IREE_TEST_CONFIG_FILES: iree_tests/configs/config_pytorch_models_cpu_llvm_task.json run: | source ${VENV_DIR}/bin/activate - pytest iree_tests -n auto -k real_weights -rpfE --timeout=600 --retries 2 --retry-delay 5 --durations=0 + pytest iree_tests/pytorch/models -s -n 4 -k real_weights -rpfE --timeout=1200 --retries 2 --retry-delay 5 --durations=0 diff --git a/iree_tests/configs/config_pytorch_models_cpu_llvm_task.json b/iree_tests/configs/config_pytorch_models_cpu_llvm_task.json new file mode 100644 index 000000000..048f6d4c1 --- /dev/null +++ b/iree_tests/configs/config_pytorch_models_cpu_llvm_task.json @@ -0,0 +1,15 @@ +{ + "config_name": "sdxl_cpu_llvm_task", + "iree_compile_flags" : [ + "--iree-hal-target-backends=llvm-cpu", + "--iree-llvmcpu-target-cpu-features=host", + "--iree-llvmcpu-distribution-size=32" + ], + "iree_run_module_flags": [ + "--device=local-task" + ], + "skip_compile_tests": [], + "skip_run_tests": [], + "expected_compile_failures": [], + "expected_run_failures": [] + } diff --git a/iree_tests/pytorch/models/llama-tank/real_weights_data_flags.txt b/iree_tests/future-pytorch-models/llama-tank/real_weights_data_flags.txt similarity index 100% rename from iree_tests/pytorch/models/llama-tank/real_weights_data_flags.txt rename to iree_tests/future-pytorch-models/llama-tank/real_weights_data_flags.txt diff --git a/iree_tests/pytorch/models/llama-tank/splat_data_flags.txt b/iree_tests/future-pytorch-models/llama-tank/splat_data_flags.txt similarity index 100% rename from iree_tests/pytorch/models/llama-tank/splat_data_flags.txt rename to iree_tests/future-pytorch-models/llama-tank/splat_data_flags.txt diff --git a/iree_tests/pytorch/models/llama-tank/splats.irpa b/iree_tests/future-pytorch-models/llama-tank/splats.irpa similarity index 100% rename from iree_tests/pytorch/models/llama-tank/splats.irpa rename to iree_tests/future-pytorch-models/llama-tank/splats.irpa diff --git a/iree_tests/pytorch/models/llama-tank/test_cases.json b/iree_tests/future-pytorch-models/llama-tank/test_cases.json similarity index 100% rename from iree_tests/pytorch/models/llama-tank/test_cases.json rename to iree_tests/future-pytorch-models/llama-tank/test_cases.json diff --git a/iree_tests/pytorch/models/sd-clip-tank/real_weights_data_flags.txt b/iree_tests/future-pytorch-models/sd-clip-tank/real_weights_data_flags.txt similarity index 100% rename from iree_tests/pytorch/models/sd-clip-tank/real_weights_data_flags.txt rename to iree_tests/future-pytorch-models/sd-clip-tank/real_weights_data_flags.txt diff --git a/iree_tests/pytorch/models/sd-clip-tank/sd-clip-tank.mlirbc b/iree_tests/future-pytorch-models/sd-clip-tank/sd-clip-tank.mlirbc similarity index 100% rename from iree_tests/pytorch/models/sd-clip-tank/sd-clip-tank.mlirbc rename to iree_tests/future-pytorch-models/sd-clip-tank/sd-clip-tank.mlirbc diff --git a/iree_tests/pytorch/models/sd-clip-tank/splat_data_flags.txt b/iree_tests/future-pytorch-models/sd-clip-tank/splat_data_flags.txt similarity index 100% rename from iree_tests/pytorch/models/sd-clip-tank/splat_data_flags.txt rename to iree_tests/future-pytorch-models/sd-clip-tank/splat_data_flags.txt diff --git a/iree_tests/pytorch/models/sd-clip-tank/splats.irpa b/iree_tests/future-pytorch-models/sd-clip-tank/splats.irpa similarity index 100% rename from iree_tests/pytorch/models/sd-clip-tank/splats.irpa rename to iree_tests/future-pytorch-models/sd-clip-tank/splats.irpa diff --git a/iree_tests/pytorch/models/sd-clip-tank/test_cases.json b/iree_tests/future-pytorch-models/sd-clip-tank/test_cases.json similarity index 100% rename from iree_tests/pytorch/models/sd-clip-tank/test_cases.json rename to iree_tests/future-pytorch-models/sd-clip-tank/test_cases.json diff --git a/iree_tests/pytorch/models/sd-unet-tank/real_weights_data_flags.txt b/iree_tests/future-pytorch-models/sd-unet-tank/real_weights_data_flags.txt similarity index 100% rename from iree_tests/pytorch/models/sd-unet-tank/real_weights_data_flags.txt rename to iree_tests/future-pytorch-models/sd-unet-tank/real_weights_data_flags.txt diff --git a/iree_tests/pytorch/models/sd-unet-tank/sd-unet-tank.mlirbc b/iree_tests/future-pytorch-models/sd-unet-tank/sd-unet-tank.mlirbc similarity index 100% rename from iree_tests/pytorch/models/sd-unet-tank/sd-unet-tank.mlirbc rename to iree_tests/future-pytorch-models/sd-unet-tank/sd-unet-tank.mlirbc diff --git a/iree_tests/pytorch/models/sd-unet-tank/splat_data_flags.txt b/iree_tests/future-pytorch-models/sd-unet-tank/splat_data_flags.txt similarity index 100% rename from iree_tests/pytorch/models/sd-unet-tank/splat_data_flags.txt rename to iree_tests/future-pytorch-models/sd-unet-tank/splat_data_flags.txt diff --git a/iree_tests/pytorch/models/sd-unet-tank/splats.irpa b/iree_tests/future-pytorch-models/sd-unet-tank/splats.irpa similarity index 100% rename from iree_tests/pytorch/models/sd-unet-tank/splats.irpa rename to iree_tests/future-pytorch-models/sd-unet-tank/splats.irpa diff --git a/iree_tests/pytorch/models/sd-unet-tank/test_cases.json b/iree_tests/future-pytorch-models/sd-unet-tank/test_cases.json similarity index 100% rename from iree_tests/pytorch/models/sd-unet-tank/test_cases.json rename to iree_tests/future-pytorch-models/sd-unet-tank/test_cases.json diff --git a/iree_tests/pytorch/models/sd-vae-decode-tank/real_weights_data_flags.txt b/iree_tests/future-pytorch-models/sd-vae-decode-tank/real_weights_data_flags.txt similarity index 100% rename from iree_tests/pytorch/models/sd-vae-decode-tank/real_weights_data_flags.txt rename to iree_tests/future-pytorch-models/sd-vae-decode-tank/real_weights_data_flags.txt diff --git a/iree_tests/pytorch/models/sd-vae-decode-tank/splat_data_flags.txt b/iree_tests/future-pytorch-models/sd-vae-decode-tank/splat_data_flags.txt similarity index 100% rename from iree_tests/pytorch/models/sd-vae-decode-tank/splat_data_flags.txt rename to iree_tests/future-pytorch-models/sd-vae-decode-tank/splat_data_flags.txt diff --git a/iree_tests/pytorch/models/sd-vae-decode-tank/splats.irpa b/iree_tests/future-pytorch-models/sd-vae-decode-tank/splats.irpa similarity index 100% rename from iree_tests/pytorch/models/sd-vae-decode-tank/splats.irpa rename to iree_tests/future-pytorch-models/sd-vae-decode-tank/splats.irpa diff --git a/iree_tests/pytorch/models/sd-vae-decode-tank/test_cases.json b/iree_tests/future-pytorch-models/sd-vae-decode-tank/test_cases.json similarity index 100% rename from iree_tests/pytorch/models/sd-vae-decode-tank/test_cases.json rename to iree_tests/future-pytorch-models/sd-vae-decode-tank/test_cases.json diff --git a/iree_tests/pytorch/models/sdxl-prompt-encoder-tank/model.mlirbc b/iree_tests/pytorch/models/sdxl-prompt-encoder-tank/model.mlirbc new file mode 100644 index 000000000..c1659eb73 --- /dev/null +++ b/iree_tests/pytorch/models/sdxl-prompt-encoder-tank/model.mlirbc @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dda3922975dbedf82864104cced0186bd552393f3af7d2ef02474e697aa8c724 +size 693353 diff --git a/iree_tests/pytorch/models/sdxl-prompt-encoder-tank/real_weights_data_flags.txt b/iree_tests/pytorch/models/sdxl-prompt-encoder-tank/real_weights_data_flags.txt new file mode 100644 index 000000000..fc85e7190 --- /dev/null +++ b/iree_tests/pytorch/models/sdxl-prompt-encoder-tank/real_weights_data_flags.txt @@ -0,0 +1,8 @@ +--parameters=model=real_weights.irpa +--input=1x64xi64=@inference_input.0.bin +--input=1x64xi64=@inference_input.1.bin +--input=1x64xi64=@inference_input.2.bin +--input=1x64xi64=@inference_input.3.bin +--expected_output=2x64x2048xf16=@inference_output.0.bin +--expected_output=2x1280xf16=@inference_output.1.bin +--expected_f16_threshold=1.0f diff --git a/iree_tests/pytorch/models/sdxl-prompt-encoder-tank/splat_data_flags.txt b/iree_tests/pytorch/models/sdxl-prompt-encoder-tank/splat_data_flags.txt new file mode 100644 index 000000000..45a953567 --- /dev/null +++ b/iree_tests/pytorch/models/sdxl-prompt-encoder-tank/splat_data_flags.txt @@ -0,0 +1,5 @@ +--input="1x64xi64" +--input="1x64xi64" +--input="1x64xi64" +--input="1x64xi64" +--parameters=splats.irpa diff --git a/iree_tests/pytorch/models/sdxl-prompt-encoder-tank/splats.irpa b/iree_tests/pytorch/models/sdxl-prompt-encoder-tank/splats.irpa new file mode 100644 index 000000000..48f1a240c --- /dev/null +++ b/iree_tests/pytorch/models/sdxl-prompt-encoder-tank/splats.irpa @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:018b337cbbc5f66d0791b3487f80bf8f5f1c3634f567f89bc160c66477a462ca +size 118784 diff --git a/iree_tests/pytorch/models/sdxl-prompt-encoder-tank/test_cases.json b/iree_tests/pytorch/models/sdxl-prompt-encoder-tank/test_cases.json new file mode 100644 index 000000000..3ee78c84c --- /dev/null +++ b/iree_tests/pytorch/models/sdxl-prompt-encoder-tank/test_cases.json @@ -0,0 +1,29 @@ +{ + "test_cases": [ + { + "name": "splats", + "runtime_flagfile": "splat_data_flags.txt", + "remote_file_groups": [] + }, + { + "name": "real_weights", + "runtime_flagfile": "real_weights_data_flags.txt", + "remote_file_groups": [ + { + "azure_account_url": "https://sharkpublic.blob.core.windows.net", + "azure_container_name": "sharkpublic", + "azure_base_blob_name": "sai/sdxl-prompt-encoder/", + "files": [ + "inference_input.0.bin", + "inference_input.1.bin", + "inference_input.2.bin", + "inference_input.3.bin", + "inference_output.0.bin", + "inference_output.1.bin", + "real_weights.irpa" + ] + } + ] + } + ] +} diff --git a/iree_tests/pytorch/models/sdxl-scheduled-unet-3-tank/model.mlirbc b/iree_tests/pytorch/models/sdxl-scheduled-unet-3-tank/model.mlirbc new file mode 100644 index 000000000..275016b7c --- /dev/null +++ b/iree_tests/pytorch/models/sdxl-scheduled-unet-3-tank/model.mlirbc @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:81e934a304c92495182703ba8f1c3e31d9f5ce2c9a4d1f1e7dfbd63c2e3a804f +size 1004980 diff --git a/iree_tests/pytorch/models/sdxl-scheduled-unet-3-tank/real_weights_data_flags.txt b/iree_tests/pytorch/models/sdxl-scheduled-unet-3-tank/real_weights_data_flags.txt new file mode 100644 index 000000000..e74a0260e --- /dev/null +++ b/iree_tests/pytorch/models/sdxl-scheduled-unet-3-tank/real_weights_data_flags.txt @@ -0,0 +1,8 @@ +--parameters=model=real_weights.irpa +--module=sdxl_scheduled_unet_pipeline_fp16_.vmfb +--input=1x4x128x128xf16=@inference_input.0.bin +--input=2x64x2048xf16=@inference_input.1.bin +--input=2x1280xf16=@inference_input.2.bin +--input=1xf16=@inference_input.3.bin +--expected_output=1x4x128x128xf16=@inference_output.0.bin +--expected_f16_threshold=0.8f \ No newline at end of file diff --git a/iree_tests/pytorch/models/sdxl-scheduled-unet-3-tank/sdxl_scheduled_unet_pipeline_fp16_.vmfb b/iree_tests/pytorch/models/sdxl-scheduled-unet-3-tank/sdxl_scheduled_unet_pipeline_fp16_.vmfb new file mode 100644 index 000000000..317ab61a5 Binary files /dev/null and b/iree_tests/pytorch/models/sdxl-scheduled-unet-3-tank/sdxl_scheduled_unet_pipeline_fp16_.vmfb differ diff --git a/iree_tests/pytorch/models/sdxl-scheduled-unet-3-tank/splat_data_flags.txt b/iree_tests/pytorch/models/sdxl-scheduled-unet-3-tank/splat_data_flags.txt new file mode 100644 index 000000000..0157aed39 --- /dev/null +++ b/iree_tests/pytorch/models/sdxl-scheduled-unet-3-tank/splat_data_flags.txt @@ -0,0 +1,5 @@ +--input="1x4x128x128xf16" +--input="2x64x2048xf16" +--input="2x1280xf16" +--input="1xf16" +--parameters=splats.irpa diff --git a/iree_tests/pytorch/models/sdxl-scheduled-unet-3-tank/splats.irpa b/iree_tests/pytorch/models/sdxl-scheduled-unet-3-tank/splats.irpa new file mode 100644 index 000000000..2d1360b93 --- /dev/null +++ b/iree_tests/pytorch/models/sdxl-scheduled-unet-3-tank/splats.irpa @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d3dd8f1ee797944c3ab3286858b89257dbbb55bb7b894c82437d1a0d94b9f1a9 +size 270336 diff --git a/iree_tests/pytorch/models/sdxl-scheduled-unet-3-tank/test_cases.json b/iree_tests/pytorch/models/sdxl-scheduled-unet-3-tank/test_cases.json new file mode 100644 index 000000000..07a79ea55 --- /dev/null +++ b/iree_tests/pytorch/models/sdxl-scheduled-unet-3-tank/test_cases.json @@ -0,0 +1,28 @@ +{ + "test_cases": [ + { + "name": "splats", + "runtime_flagfile": "splat_data_flags.txt", + "remote_file_groups": [] + }, + { + "name": "real_weights", + "runtime_flagfile": "real_weights_data_flags.txt", + "remote_file_groups": [ + { + "azure_account_url": "https://sharkpublic.blob.core.windows.net", + "azure_container_name": "sharkpublic", + "azure_base_blob_name": "sai/sdxl-scheduled-unet/", + "files": [ + "inference_input.0.bin", + "inference_input.1.bin", + "inference_input.2.bin", + "inference_input.3.bin", + "inference_output.0.bin", + "real_weights.irpa" + ] + } + ] + } + ] +} diff --git a/iree_tests/pytorch/models/sdxl-vae-decode-tank/model.mlirbc b/iree_tests/pytorch/models/sdxl-vae-decode-tank/model.mlirbc new file mode 100644 index 000000000..a5c7f7ef5 --- /dev/null +++ b/iree_tests/pytorch/models/sdxl-vae-decode-tank/model.mlirbc @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d98d2d1a09c6fa6ef784998032589c7cb9b6ce1ad2d3d53596e2585ffeb35d37 +size 99172 diff --git a/iree_tests/pytorch/models/sdxl-vae-decode-tank/real_weights_data_flags.txt b/iree_tests/pytorch/models/sdxl-vae-decode-tank/real_weights_data_flags.txt new file mode 100644 index 000000000..aceb8839c --- /dev/null +++ b/iree_tests/pytorch/models/sdxl-vae-decode-tank/real_weights_data_flags.txt @@ -0,0 +1,4 @@ +--parameters=model=real_weights.irpa +--input=1x4x128x128xf16=@inference_input.0.bin +--expected_output=1x3x1024x1024xf16=@inference_output.0.bin +--expected_f16_threshold=0.02f diff --git a/iree_tests/pytorch/models/sdxl-vae-decode-tank/splat_data_flags.txt b/iree_tests/pytorch/models/sdxl-vae-decode-tank/splat_data_flags.txt new file mode 100644 index 000000000..b560c5161 --- /dev/null +++ b/iree_tests/pytorch/models/sdxl-vae-decode-tank/splat_data_flags.txt @@ -0,0 +1,2 @@ +--input="1x4x128x128xf16" +--parameters=splats.irpa diff --git a/iree_tests/pytorch/models/sdxl-vae-decode-tank/splats.irpa b/iree_tests/pytorch/models/sdxl-vae-decode-tank/splats.irpa new file mode 100644 index 000000000..27c29ef7f --- /dev/null +++ b/iree_tests/pytorch/models/sdxl-vae-decode-tank/splats.irpa @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:01bb22697cfde0ac4ffdb1440b92ab98d6f757dc92b5399febb868fbef6f7641 +size 36864 diff --git a/iree_tests/pytorch/models/sdxl-vae-decode-tank/test_cases.json b/iree_tests/pytorch/models/sdxl-vae-decode-tank/test_cases.json new file mode 100644 index 000000000..7ad2e8f4c --- /dev/null +++ b/iree_tests/pytorch/models/sdxl-vae-decode-tank/test_cases.json @@ -0,0 +1,25 @@ +{ + "test_cases": [ + { + "name": "splats", + "runtime_flagfile": "splat_data_flags.txt", + "remote_file_groups": [] + }, + { + "name": "real_weights", + "runtime_flagfile": "real_weights_data_flags.txt", + "remote_file_groups": [ + { + "azure_account_url": "https://sharkpublic.blob.core.windows.net", + "azure_container_name": "sharkpublic", + "azure_base_blob_name": "sai/sdxl-vae-decode/", + "files": [ + "inference_input.0.bin", + "inference_output.0.bin", + "real_weights.irpa" + ] + } + ] + } + ] +}