Add options for bypassing xfails and runs from configs. (#123)

These let you use options from config files but ignore xfail and skip
run settings to easily get logs with all test failures, like these:

*
[pytest_compile_only_2024_03_21.txt](https://gist.github.com/ScottTodd/ecc9c57c01bfc5e996a15cdd38df6a9c)
*
[pytest_all_failures_2024_03_21.txt](https://gist.github.com/ScottTodd/1a02531cc76a3b8566428207e39d1870)

Both of those are using `IREE compiler version 20240321.838 @
5f2743baaa79160a3883854e60e5188822dceeb1`

(This was previously possible by editing the config .json file, but now
it's easier)

So minimal repro instructions for latest nightly IREE ONNX test suite
failures:

```
git clone https://github.com/nod-ai/SHARK-TestSuite.git
cd SHARK-TestSuite
python -m venv .venv
source .venv/bin/activate
python -m pip install -r iree_tests/requirements.txt
python -m pip install --find-links https://iree.dev/pip-release-links.html iree-compiler iree-runtime --upgrade
pytest iree_tests/onnx -n auto --ignore-xfails --config-files ./iree_tests/configs/config_cpu_llvm_sync.json
```

See the README for other common scenarios (like using a source build of
IREE)

iree_tests/README.md

@@ -91,13 +91,29 @@ $ pytest iree_tests -n auto
 Run tests using custom config files:
 
 ```bash
-$ pytest iree_tests --config-files ./configs/config_gpu_vulkan.json
+$ pytest iree_tests --config-files ./iree_tests/configs/config_gpu_vulkan.json
 
 # OR set an environment variable
 $ export IREE_TEST_CONFIG_FILES=/iree/config_cpu_llvm_sync.json;/iree/config_gpu_vulkan.json
 $ pytest iree_tests
 ```
 
+Run ONNX tests on CPU and print all errors:
+
+```bash
+$ pytest iree_tests/onnx -n auto \
+    --ignore-xfails \
+    --config-files ./iree_tests/configs/config_cpu_llvm_sync.json
+```
+
+Run ONNX compilation tests only and print all errors:
+
+```bash
+$ pytest iree_tests/onnx -n auto \
+    --ignore-xfails --skip-all-runs \
+    --config-files ./iree_tests/configs/config_cpu_llvm_sync.json
+```
+
 ### Advanced pytest usage tips
 
 Collect tests (but do not run them):

iree_tests/configs/config_cpu_llvm_sync.json

@@ -794,9 +794,11 @@
     "test_clip_default_int8_min",
     "test_clip_default_int8_min_expanded",
     "test_constant_pad",
+    "test_constantofshape_float_ones",
     "test_constantofshape_int_shape_zero",
     "test_constantofshape_int_zeros",
     "test_div_uint8",
+    "test_dropout_default_mask_ratio",
     "test_elu_default",
     "test_gather_0",
     "test_gather_1",
@@ -834,13 +836,14 @@
     "test_pow_types_float32_uint64",
     "test_qlinearconv",
     "test_qlinearmatmul_2D_int8_float16",
+    "test_qlinearmatmul_2D_int8_float32",
     "test_qlinearmatmul_3D_int8_float16",
     "test_qlinearmatmul_3D_int8_float32",
     "test_qlinearmatmul_3D_uint8_float16",
-    "test_qlinearmatmul_2D_int8_float32",
     "test_qlinearmatmul_3D_uint8_float32",
     "test_quantizelinear",
     "test_range_int32_type_negative_delta",
+    "test_reduce_min_empty_set",
     "test_scatter_elements_with_negative_indices",
     "test_selu_default",
     "test_shape",

iree_tests/conftest.py

@@ -66,6 +66,20 @@ def pytest_addoption(parser):
         help="List of config JSON files used to build test cases",
     )
 
+    parser.addoption(
+        "--ignore-xfails",
+        action="store_true",
+        default=False,
+        help="Ignores expected compile/run failures from configs, to print all error output",
+    )
+
+    parser.addoption(
+        "--skip-all-runs",
+        action="store_true",
+        default=False,
+        help="Skips all 'run' tests, overriding 'skip_run_tests' in configs",
+    )
+
 
 def pytest_sessionstart(session):
     session.config.iree_test_configs = []
@@ -91,6 +105,7 @@ def pytest_collect_file(parent, file_path):
 
 # --------------------------------------------------------------------------- #
 
+
 @dataclass(frozen=True)
 class IreeCompileAndRunTestSpec:
     """Specification for an IREE "compile and run" test."""
@@ -224,10 +239,17 @@ def collect(self):
                 continue
 
             expect_compile_success = (
-                test_name not in config["expected_compile_failures"]
+                self.config.getoption("ignore_xfails")
+                or test_name not in config["expected_compile_failures"]
+            )
+            expect_run_success = (
+                self.config.getoption("ignore_xfails")
+                or test_name not in config["expected_run_failures"]
+            )
+            skip_run = (
+                self.config.getoption("skip_all_runs")
+                or test_name in config["skip_run_tests"]
             )
-            expect_run_success = test_name not in config["expected_run_failures"]
-            skip_run = test_name in config["skip_run_tests"]
             config_name = config["config_name"]
 
             # TODO(scotttodd): don't compile once per test case?