diff --git a/.github/workflows/test_iree.yml b/.github/workflows/test_iree.yml index b7dee446b..e4263e0e1 100644 --- a/.github/workflows/test_iree.yml +++ b/.github/workflows/test_iree.yml @@ -75,4 +75,4 @@ jobs: - name: "Running real weight model tests" run: | source ${VENV_DIR}/bin/activate - pytest iree_tests -n auto -k real_weights -rpfE --timeout=60 --retries 2 --retry-delay 5 --durations=0 + pytest iree_tests -n auto -k real_weights -rpfE --timeout=600 --retries 2 --retry-delay 5 --durations=0 diff --git a/iree_tests/pytorch/models/llama-tank/real_weights_data_flags.txt b/iree_tests/pytorch/models/llama-tank/real_weights_data_flags.txt new file mode 100644 index 000000000..327fcb28d --- /dev/null +++ b/iree_tests/pytorch/models/llama-tank/real_weights_data_flags.txt @@ -0,0 +1,67 @@ +--parameters=real_weights.irpa +--input=1x93xi64=@inference_input.0.bin +--expected_output=1x93x32000xf32=@inference_output.0.bin +--expected_output=1x32x93x128xf32=@inference_output.1.bin +--expected_output=1x32x93x128xf32=@inference_output.2.bin +--expected_output=1x32x93x128xf32=@inference_output.3.bin +--expected_output=1x32x93x128xf32=@inference_output.4.bin +--expected_output=1x32x93x128xf32=@inference_output.5.bin +--expected_output=1x32x93x128xf32=@inference_output.6.bin +--expected_output=1x32x93x128xf32=@inference_output.7.bin +--expected_output=1x32x93x128xf32=@inference_output.8.bin +--expected_output=1x32x93x128xf32=@inference_output.9.bin +--expected_output=1x32x93x128xf32=@inference_output.10.bin +--expected_output=1x32x93x128xf32=@inference_output.11.bin +--expected_output=1x32x93x128xf32=@inference_output.12.bin +--expected_output=1x32x93x128xf32=@inference_output.13.bin +--expected_output=1x32x93x128xf32=@inference_output.14.bin +--expected_output=1x32x93x128xf32=@inference_output.15.bin +--expected_output=1x32x93x128xf32=@inference_output.16.bin +--expected_output=1x32x93x128xf32=@inference_output.17.bin +--expected_output=1x32x93x128xf32=@inference_output.18.bin +--expected_output=1x32x93x128xf32=@inference_output.19.bin +--expected_output=1x32x93x128xf32=@inference_output.20.bin +--expected_output=1x32x93x128xf32=@inference_output.21.bin +--expected_output=1x32x93x128xf32=@inference_output.22.bin +--expected_output=1x32x93x128xf32=@inference_output.23.bin +--expected_output=1x32x93x128xf32=@inference_output.24.bin +--expected_output=1x32x93x128xf32=@inference_output.25.bin +--expected_output=1x32x93x128xf32=@inference_output.26.bin +--expected_output=1x32x93x128xf32=@inference_output.27.bin +--expected_output=1x32x93x128xf32=@inference_output.28.bin +--expected_output=1x32x93x128xf32=@inference_output.29.bin +--expected_output=1x32x93x128xf32=@inference_output.30.bin +--expected_output=1x32x93x128xf32=@inference_output.31.bin +--expected_output=1x32x93x128xf32=@inference_output.32.bin +--expected_output=1x32x93x128xf32=@inference_output.33.bin +--expected_output=1x32x93x128xf32=@inference_output.34.bin +--expected_output=1x32x93x128xf32=@inference_output.35.bin +--expected_output=1x32x93x128xf32=@inference_output.36.bin +--expected_output=1x32x93x128xf32=@inference_output.37.bin +--expected_output=1x32x93x128xf32=@inference_output.38.bin +--expected_output=1x32x93x128xf32=@inference_output.39.bin +--expected_output=1x32x93x128xf32=@inference_output.40.bin +--expected_output=1x32x93x128xf32=@inference_output.41.bin +--expected_output=1x32x93x128xf32=@inference_output.42.bin +--expected_output=1x32x93x128xf32=@inference_output.43.bin +--expected_output=1x32x93x128xf32=@inference_output.44.bin +--expected_output=1x32x93x128xf32=@inference_output.45.bin +--expected_output=1x32x93x128xf32=@inference_output.46.bin +--expected_output=1x32x93x128xf32=@inference_output.47.bin +--expected_output=1x32x93x128xf32=@inference_output.48.bin +--expected_output=1x32x93x128xf32=@inference_output.49.bin +--expected_output=1x32x93x128xf32=@inference_output.50.bin +--expected_output=1x32x93x128xf32=@inference_output.51.bin +--expected_output=1x32x93x128xf32=@inference_output.52.bin +--expected_output=1x32x93x128xf32=@inference_output.53.bin +--expected_output=1x32x93x128xf32=@inference_output.54.bin +--expected_output=1x32x93x128xf32=@inference_output.55.bin +--expected_output=1x32x93x128xf32=@inference_output.56.bin +--expected_output=1x32x93x128xf32=@inference_output.57.bin +--expected_output=1x32x93x128xf32=@inference_output.58.bin +--expected_output=1x32x93x128xf32=@inference_output.59.bin +--expected_output=1x32x93x128xf32=@inference_output.60.bin +--expected_output=1x32x93x128xf32=@inference_output.61.bin +--expected_output=1x32x93x128xf32=@inference_output.62.bin +--expected_output=1x32x93x128xf32=@inference_output.63.bin +--expected_output=1x32x93x128xf32=@inference_output.64.bin diff --git a/iree_tests/pytorch/models/llama-tank/splat_data_flags.txt b/iree_tests/pytorch/models/llama-tank/splat_data_flags.txt new file mode 100644 index 000000000..78cbe856c --- /dev/null +++ b/iree_tests/pytorch/models/llama-tank/splat_data_flags.txt @@ -0,0 +1,2 @@ +--input="1x93xi64" +--parameters=splats.irpa diff --git a/iree_tests/pytorch/models/llama-tank/splats.irpa b/iree_tests/pytorch/models/llama-tank/splats.irpa new file mode 100644 index 000000000..0ce69ae3d Binary files /dev/null and b/iree_tests/pytorch/models/llama-tank/splats.irpa differ diff --git a/iree_tests/pytorch/models/llama-tank/test_cases.json b/iree_tests/pytorch/models/llama-tank/test_cases.json new file mode 100644 index 000000000..3deeb9440 --- /dev/null +++ b/iree_tests/pytorch/models/llama-tank/test_cases.json @@ -0,0 +1,18 @@ +{ + "test_cases": [ + { + "name": "splats", + "runtime_flagfile": "splat_data_flags.txt", + "remote_file_groups": [ + { + "azure_account_url": "https://sharkpublic.blob.core.windows.net", + "azure_container_name": "sharkpublic", + "azure_base_blob_name": "sai/llama-tank/", + "files": [ + "llama-tank.mlirbc" + ] + } + ] + } + ] +} diff --git a/iree_tests/pytorch/models/sd-clip-tank/real_weights_data_flags.txt b/iree_tests/pytorch/models/sd-clip-tank/real_weights_data_flags.txt new file mode 100644 index 000000000..b5fe5e578 --- /dev/null +++ b/iree_tests/pytorch/models/sd-clip-tank/real_weights_data_flags.txt @@ -0,0 +1,4 @@ +--parameters=model=real_weights.irpa +--input=1x77xi64=@inference_input.0.bin +--expected_output=1x77x768xf32=@inference_output.0.bin +--expected_output=1x768xf32=@inference_output.1.bin diff --git a/iree_tests/pytorch/models/sd-clip-tank/sd-clip-tank.mlirbc b/iree_tests/pytorch/models/sd-clip-tank/sd-clip-tank.mlirbc new file mode 100644 index 000000000..d7807a0b6 Binary files /dev/null and b/iree_tests/pytorch/models/sd-clip-tank/sd-clip-tank.mlirbc differ diff --git a/iree_tests/pytorch/models/sd-clip-tank/splat_data_flags.txt b/iree_tests/pytorch/models/sd-clip-tank/splat_data_flags.txt new file mode 100644 index 000000000..9f971495b --- /dev/null +++ b/iree_tests/pytorch/models/sd-clip-tank/splat_data_flags.txt @@ -0,0 +1,2 @@ +--input="1x77xi64" +--parameters=splats.irpa diff --git a/iree_tests/pytorch/models/sd-clip-tank/splats.irpa b/iree_tests/pytorch/models/sd-clip-tank/splats.irpa new file mode 100644 index 000000000..7c201b10f Binary files /dev/null and b/iree_tests/pytorch/models/sd-clip-tank/splats.irpa differ diff --git a/iree_tests/pytorch/models/sd-clip-tank/test_cases.json b/iree_tests/pytorch/models/sd-clip-tank/test_cases.json new file mode 100644 index 000000000..37456e3de --- /dev/null +++ b/iree_tests/pytorch/models/sd-clip-tank/test_cases.json @@ -0,0 +1,26 @@ +{ + "test_cases": [ + { + "name": "splats", + "runtime_flagfile": "splat_data_flags.txt", + "remote_file_groups": [] + }, + { + "name": "real_weights", + "runtime_flagfile": "real_weights_data_flags.txt", + "remote_file_groups": [ + { + "azure_account_url": "https://sharkpublic.blob.core.windows.net", + "azure_container_name": "sharkpublic", + "azure_base_blob_name": "sai/sd-clip-tank/", + "files": [ + "inference_input.0.bin", + "inference_output.0.bin", + "inference_output.1.bin", + "real_weights.irpa" + ] + } + ] + } + ] +} diff --git a/iree_tests/pytorch/models/sd-unet-tank/real_weights_data_flags.txt b/iree_tests/pytorch/models/sd-unet-tank/real_weights_data_flags.txt new file mode 100644 index 000000000..cb3c458ca --- /dev/null +++ b/iree_tests/pytorch/models/sd-unet-tank/real_weights_data_flags.txt @@ -0,0 +1,5 @@ +--parameters=model=real_weights.irpa +--input=1x4x64x64xf32=@inference_input.0.bin +--input=1xf32=@inference_input.1.bin +--input=2x77x768xf32=@inference_input.2.bin +--expected_output=1x4x64x64xf32=@inference_output.0.bin diff --git a/iree_tests/pytorch/models/sd-unet-tank/sd-unet-tank.mlirbc b/iree_tests/pytorch/models/sd-unet-tank/sd-unet-tank.mlirbc new file mode 100644 index 000000000..18ac2faee Binary files /dev/null and b/iree_tests/pytorch/models/sd-unet-tank/sd-unet-tank.mlirbc differ diff --git a/iree_tests/pytorch/models/sd-unet-tank/splat_data_flags.txt b/iree_tests/pytorch/models/sd-unet-tank/splat_data_flags.txt new file mode 100644 index 000000000..04abda720 --- /dev/null +++ b/iree_tests/pytorch/models/sd-unet-tank/splat_data_flags.txt @@ -0,0 +1,4 @@ +--input="1x4x64x64xf32" +--input="1xf32" +--input="2x77x768xf32" +--parameters=splats.irpa diff --git a/iree_tests/pytorch/models/sd-unet-tank/splats.irpa b/iree_tests/pytorch/models/sd-unet-tank/splats.irpa new file mode 100644 index 000000000..0ffdb3f04 Binary files /dev/null and b/iree_tests/pytorch/models/sd-unet-tank/splats.irpa differ diff --git a/iree_tests/pytorch/models/sd-unet-tank/test_cases.json b/iree_tests/pytorch/models/sd-unet-tank/test_cases.json new file mode 100644 index 000000000..d1d758c3f --- /dev/null +++ b/iree_tests/pytorch/models/sd-unet-tank/test_cases.json @@ -0,0 +1,27 @@ +{ + "test_cases": [ + { + "name": "splats", + "runtime_flagfile": "splat_data_flags.txt", + "remote_file_groups": [] + }, + { + "name": "real_weights", + "runtime_flagfile": "real_weights_data_flags.txt", + "remote_file_groups": [ + { + "azure_account_url": "https://sharkpublic.blob.core.windows.net", + "azure_container_name": "sharkpublic", + "azure_base_blob_name": "sai/sd-unet-tank/", + "files": [ + "inference_input.0.bin", + "inference_input.1.bin", + "inference_input.2.bin", + "inference_output.0.bin", + "real_weights.irpa" + ] + } + ] + } + ] +} diff --git a/iree_tests/pytorch/models/sd-vae-decode-tank/real_weights_data_flags.txt b/iree_tests/pytorch/models/sd-vae-decode-tank/real_weights_data_flags.txt new file mode 100644 index 000000000..6b90771ec --- /dev/null +++ b/iree_tests/pytorch/models/sd-vae-decode-tank/real_weights_data_flags.txt @@ -0,0 +1,3 @@ +--parameters=model=real_weights.irpa +--input=1x4x64x64xf32=@inference_input.0.bin +--expected_output=1x3x512x512xf32=@inference_output.0.bin diff --git a/iree_tests/pytorch/models/sd-vae-decode-tank/splat_data_flags.txt b/iree_tests/pytorch/models/sd-vae-decode-tank/splat_data_flags.txt new file mode 100644 index 000000000..0f9d6c31f --- /dev/null +++ b/iree_tests/pytorch/models/sd-vae-decode-tank/splat_data_flags.txt @@ -0,0 +1,2 @@ +--input="1x4x64x64xf32" +--parameters=splats.irpa diff --git a/iree_tests/pytorch/models/sd-vae-decode-tank/splats.irpa b/iree_tests/pytorch/models/sd-vae-decode-tank/splats.irpa new file mode 100644 index 000000000..05d3328a5 Binary files /dev/null and b/iree_tests/pytorch/models/sd-vae-decode-tank/splats.irpa differ diff --git a/iree_tests/pytorch/models/sd-vae-decode-tank/test_cases.json b/iree_tests/pytorch/models/sd-vae-decode-tank/test_cases.json new file mode 100644 index 000000000..80b057d67 --- /dev/null +++ b/iree_tests/pytorch/models/sd-vae-decode-tank/test_cases.json @@ -0,0 +1,35 @@ +{ + "test_cases": [ + { + "name": "splats", + "runtime_flagfile": "splat_data_flags.txt", + "remote_file_groups": [ + { + "azure_account_url": "https://sharkpublic.blob.core.windows.net", + "azure_container_name": "sharkpublic", + "azure_base_blob_name": "sai/sd-vae-decode-tank/", + "files": [ + "sd-vae-decode-tank.mlirbc" + ] + } + ] + }, + { + "name": "real_weights", + "runtime_flagfile": "real_weights_data_flags.txt", + "remote_file_groups": [ + { + "azure_account_url": "https://sharkpublic.blob.core.windows.net", + "azure_container_name": "sharkpublic", + "azure_base_blob_name": "sai/sd-vae-decode-tank/", + "files": [ + "inference_input.0.bin", + "inference_output.0.bin", + "real_weights.irpa", + "sd-vae-decode-tank.mlirbc" + ] + } + ] + } + ] +}