refactor to decrease number of workflows

.github/workflows/test_cli_cpu_ipex.yaml

@@ -48,7 +48,9 @@ jobs:
           pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cpu
           pip install -e .[testing,ipex,diffusers,timm]
 
-      - name: Run tests
-        run: |
-          # pytest tests/test_cli.py -s -k "cli and cpu and ipex"
-          pytest tests/test_examples.py -s -k "examples and cpu and ipex"
+      # - name: Run tests
+      #   run: |
+      #     pytest tests/test_cli.py -s -k "cli and cpu and ipex"
+
+      - name: Run tests from example folder
+          pytest tests/test_examples.py -s -k "cpu and ipex"

.github/workflows/test_cli_cpu_llama_cpp.yaml

@@ -48,7 +48,10 @@ jobs:
           pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cpu
           pip install -e .[testing,llama-cpp]
 
-      - name: Run tests
+      # - name: Run tests
+      #   run: |
+      #     pytest tests/test_cli.py -s -k "llama_cpp"
+
+      - name: Run tests from example folder
         run: |
-          # pytest tests/test_cli.py -s -k "llama_cpp"
-          pytest tests/test_examples.py -s -k "examples and cpu and llama_cpp"
+          pytest tests/test_examples.py -s -k "cpu and llama_cpp"

.github/workflows/test_cli_cpu_neural_compressor.yaml

@@ -48,7 +48,10 @@ jobs:
           pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cpu
           pip install -e .[testing,neural-compressor,diffusers,timm]
 
-      - name: Run tests
+      # - name: Run tests
+      #   run: |
+      #     pytest tests/test_cli.py -s -k "cli and cpu and neural_compressor"
+
+      - name: Run tests from example folder
         run: |
-          # pytest tests/test_cli.py -s -k "cli and cpu and neural_compressor"
-          pytest tests/test_examples.py -s -k "examples and cpu and neural_compressor"
+          pytest tests/test_examples.py -s -k "cpu and neural_compressor"

.github/workflows/test_cli_cpu_onnxruntime.yaml

@@ -48,7 +48,10 @@ jobs:
           pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cpu
           pip install -e .[testing,onnxruntime,diffusers,timm]
 
-      - name: Run tests
+      # - name: Run tests
+      #   run: |
+      #     pytest tests/test_cli.py -s -k "cli and cpu and onnxruntime"
+
+      - name: Run tests from example folder
         run: |
-          # pytest tests/test_cli.py -s -k "cli and cpu and onnxruntime"
-          pytest tests/test_examples.py -s -k "examples and cpu and onnxruntime"
+          pytest tests/test_examples.py -s -k "cpu and onnxruntime"

.github/workflows/test_cli_cpu_openvino.yaml

@@ -48,7 +48,10 @@ jobs:
           pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cpu
           pip install -e .[testing,openvino,diffusers,timm]
 
-      - name: Run tests
+      # - name: Run tests
+      #   run: |
+      #     pytest tests/test_cli.py -s -k "cli and cpu and openvino"
+
+      - name: Run tests from example folder
         run: |
-          # pytest tests/test_cli.py -s -k "cli and cpu and openvino"
-          pytest tests/test_examples.py -s -k "examples and cpu and openvino"
+          pytest tests/test_examples.py -s -k "cpu and openvino"

.github/workflows/test_cli_cpu_py_txi.yaml

@@ -48,7 +48,10 @@ jobs:
           pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cpu
           pip install -e .[testing,py-txi]
 
-      - name: Run tests
+      # - name: Run tests
+      #   run: |
+      #     pytest tests/test_cli.py -s -k "cli and cpu and py_txi"
+
+      - name: Run tests from example folder
         run: |
-          # pytest tests/test_cli.py -s -k "cli and cpu and py_txi"
           pytest tests/test_examples.py -s -k "examples and cpu and py_txi"

.github/workflows/test_cli_cpu_pytorch.yaml

@@ -48,8 +48,11 @@ jobs:
           pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cpu
           pip install -e .[testing,diffusers,timm,peft]
 
-      - name: Run tests
+      # - name: Run tests
+      #   run: |
+      #     pytest tests/test_cli.py -s -k "cli and cpu and pytorch"
+
+      - name: Run tests from example folder
         run: |
-          # pytest tests/test_cli.py -s -k "cli and cpu and pytorch"
-          pytest tests/test_examples.py -s -k "examples and cpu and pytorch"
+          pytest tests/test_examples.py -s -k "cpu and pytorch"
 

.github/workflows/test_cli_cuda_onnxruntime.yaml

@@ -46,7 +46,10 @@ jobs:
         run: |
           pip install -e .[testing,onnxruntime-gpu,diffusers,timm]
 
-      - name: Run tests
+      # - name: Run tests
+      #   run: |
+      #     pytest tests/test_cli.py -x -s -k "cli and cuda and onnxruntime"
+
+      - name: Run tests from example folder
         run: |
-          # pytest tests/test_cli.py -x -s -k "cli and cuda and onnxruntime"
-          pytest tests/test_examples.py -x -s -k "cli and cuda and onnxruntime"
+          pytest tests/test_examples.py -x -s -k "cuda and onnxruntime"

.github/workflows/test_cli_cuda_py_txi.yaml

@@ -48,8 +48,11 @@ jobs:
           pip install --upgrade pip
           pip install -e .[testing,py-txi]
 
-      - name: Run tests
+      # - name: Run tests
+      #   run: |
+      #     pytest tests/test_cli.py -x -s -k "cli and cuda and py_txi"
+
+      - name: Run tests from example folder
         run: |
-          pytest tests/test_cli.py -x -s -k "cli and cuda and py_txi"
-          pytest tests/test_examples_folder.py -x -s -k "examples and cuda and py_txi"
+          pytest tests/test_examples.py -x -s -k "cuda and py_txi"
 

.github/workflows/test_cli_cuda_pytorch.yaml

@@ -77,11 +77,14 @@ jobs:
         run: |
           pip install -e .[testing,diffusers,timm,peft,deepspeed]
 
-      - name: Run tests (parallel)
-        run: |
-          pytest tests/test_cli.py -x -s -k "cli and cuda and pytorch and (dp or ddp or device_map)"
+      # - name: Run tests (parallel)
+      #   run: |
+      #     pytest tests/test_cli.py -x -s -k "cli and cuda and pytorch and (dp or ddp or device_map)"
+
+      # - name: Run tests (sequential)
+      #   run: |
+      #     FORCE_SEQUENTIAL=1 pytest tests/test_cli.py -x -s -k "cli and cuda and pytorch and (deepspeed_inference)"
 
-      - name: Run tests (sequential)
+      - name: Run test from example folder
         run: |
-          # FORCE_SEQUENTIAL=1 pytest tests/test_cli.py -x -s -k "cli and cuda and pytorch and (deepspeed_inference)"
-          FORCE_SEQUENTIAL=1 pytest tests/test_examples_folder.py -x -s -k "examples and cuda and pytorch and (deepspeed_inference)"
+          pytest tests/test_examples.py -x -s -k "cuda and pytorch"

.github/workflows/test_cli_cuda_tensorrt_llm.yaml

@@ -77,7 +77,10 @@ jobs:
         run: |
           pip install -e .[testing]
 
-      - name: Run tests (sequential)
+      # - name: Run tests (sequential)
+      #   run: |
+      #     FORCE_SEQUENTIAL=1 pytest tests/test_cli.py -x -s -k "cli and cuda and tensorrt_llm and (tp or pp)"
+
+      - name: Run tests from example folder
         run: |
-          # FORCE_SEQUENTIAL=1 pytest tests/test_cli.py -x -s -k "cli and cuda and tensorrt_llm and (tp or pp)"
-          FORCE_SEQUENTIAL=1 pytest tests/test_examples_folder.py -x -s -k "examples and cuda and tensorrt_llm and (tp or pp)"
+          pytest tests/test_examples.py -x -s -k "cuda and tensorrt_llm"

.github/workflows/test_cli_cuda_torch_ort.yaml

@@ -79,7 +79,10 @@ jobs:
           pip install -e .[testing,torch-ort,peft] 
           pip install optimum@git+https://github.com/huggingface/optimum.git
 
-      - name: Run tests
+      # - name: Run tests
+      #   run: |
+      #     pytest tests/test_cli.py -x -s -k "cli and cuda and torch_ort and (dp or ddp or device_map)"
+
+      - name: Run tests from example folder
         run: |
-          # pytest tests/test_cli.py -x -s -k "cli and cuda and torch_ort and (dp or ddp or device_map)"
-          pytest tests/test_examples_folder.py -x -s -k "examples and cuda and torch_ort and (dp or ddp or device_map)"
+          pytest tests/test_examples.py -x -s -k "cuda and torch_ort"

.github/workflows/test_cli_cuda_vllm.yaml

@@ -56,6 +56,7 @@ jobs:
       (github.event_name == 'push') ||
       (github.event_name == 'workflow_dispatch') ||
       contains( github.event.pull_request.labels.*.name, 'cli') ||
+      contains( github.event.pull_request.labels.*.name, 'examples') ||
       contains( github.event.pull_request.labels.*.name, 'cuda') ||
       contains( github.event.pull_request.labels.*.name, 'vllm') ||
       contains( github.event.pull_request.labels.*.name, 'multi_gpu') ||
@@ -77,7 +78,10 @@ jobs:
         run: |
           pip install -e .[testing]
 
-      - name: Run tests (sequential)
+      # - name: Run tests (sequential)
+      #   run: |
+      #     FORCE_SEQUENTIAL=1 pytest tests/test_cli.py -x -s -k "cli and cuda and vllm and (tp or pp)"
+
+      - name: Run tests from example folder
         run: |
-          # FORCE_SEQUENTIAL=1 pytest tests/test_cli.py -x -s -k "cli and cuda and vllm and (tp or pp)"
-          FORCE_SEQUENTIAL=1 pytest tests/test_examples_folder.py -x -s -k "examples and cuda and vllm and (tp or pp)"
+          pytest tests/test_examples.py -x -s -k "cuda and vllm"

.github/workflows/test_cli_misc.yaml

@@ -61,7 +61,9 @@ jobs:
           pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cpu
           pip install -e .[testing]
 
-      - name: Run tests
-        run: |
-          # pytest tests/test_cli.py -s -k "cli and not (cpu or cuda or rocm or mps)"
+      # - name: Run tests
+      #   run: |
+      #     pytest tests/test_cli.py -s -k "cli and not (cpu or cuda or rocm or mps)"
+
+      - name: Run tests from example folder
           pytest tests/test_examples_folder.py -s -k "examples and not (cpu or cuda or rocm or mps)"

.github/workflows/test_cli_rocm_pytorch.yaml

@@ -59,6 +59,7 @@ jobs:
       (github.event_name == 'push') ||
       (github.event_name == 'workflow_dispatch') ||
       contains( github.event.pull_request.labels.*.name, 'cli') ||
+      contains( github.event.pull_request.labels.*.name, 'examples') ||
       contains( github.event.pull_request.labels.*.name, 'rocm') ||
       contains( github.event.pull_request.labels.*.name, 'pytorch') ||
       contains( github.event.pull_request.labels.*.name, 'multi_gpu') ||
@@ -83,11 +84,14 @@ jobs:
         run: |
           pip install -e .[testing,diffusers,timm,deepspeed,peft,autoawq,auto-gptq] "deepspeed<0.15"
 
-      - name: Run tests (parallel)
-        run: |
-          pytest tests/test_cli.py -x -s -k "cli and cuda and pytorch and (dp or ddp or device_map)"
+      # - name: Run tests (parallel)
+      #   run: |
+      #     pytest tests/test_cli.py -x -s -k "cli and cuda and pytorch and (dp or ddp or device_map)"
+
+      # - name: Run tests (sequential)
+      #   run: |
+      #     FORCE_SEQUENTIAL=1 pytest tests/test_cli.py -x -s -k "cli and cuda and pytorch and (deepspeed_inference)"
 
-      - name: Run tests (sequential)
+      - name: Run tests from example folder
         run: |
-          # FORCE_SEQUENTIAL=1 pytest tests/test_cli.py -x -s -k "cli and cuda and pytorch and (deepspeed_inference)"
-          FORCE_SEQUENTIAL=1 pytest tests/test_examples_folder.py -x -s -k "examples and cuda and pytorch and (deepspeed_inference)"
+          pytest tests/test_examples.py -x -s -k "cuda and pytorch"

tests/test_examples_folder.py

@@ -39,7 +39,7 @@
 ]
 
 # can be run with pytest tests/test_example.py -s -k "cpu and txi"
-CPU_TXI_EXAMPLE_CONFIGS = [
+CPU_PY_TXI_EXAMPLE_CONFIGS = [
     "tei_bge.yaml",
 ]
 
@@ -51,7 +51,7 @@
 ]
 
 # can be run with pytest tests/test_example.py -s -k "cuda and txi"
-CUDA_TXI_EXAMPLE_CONFIGS = [
+CUDA_PY_TXI_EXAMPLE_CONFIGS = [
     "tgi_llama.yaml",
 ]
 
@@ -78,8 +78,8 @@
     + CPU_NEURAL_COMPRESSOR_EXAMPLE_CONFIGS
     + CPU_ONNXRUNTIME_EXAMPLE_CONFIGS
     + CPU_OPENVINO_EXAMPLE_CONFIGS
-    + CPU_TXI_EXAMPLE_CONFIGS
-    + CUDA_TXI_EXAMPLE_CONFIGS
+    + CPU_PY_TXI_EXAMPLE_CONFIGS
+    + CUDA_PY_TXI_EXAMPLE_CONFIGS
     + CUDA_TRT_EXAMPLE_CONFIGS
     + CUDA_VLLM_EXAMPLE_CONFIGS
 )
@@ -143,13 +143,13 @@ def test_cpu_openvino_examples(config_name):
     test_example_configs(config_name)
 
 
-@pytest.mark.parametrize("config_name", CPU_TXI_EXAMPLE_CONFIGS)
-def test_cpu_txi_examples(config_name):
+@pytest.mark.parametrize("config_name", CPU_PY_TXI_EXAMPLE_CONFIGS)
+def test_cpu_py_txi_examples(config_name):
     test_example_configs(config_name)
 
 
-@pytest.mark.parametrize("config_name", CUDA_TXI_EXAMPLE_CONFIGS)
-def test_cuda_txi_examples(config_name):
+@pytest.mark.parametrize("config_name", CUDA_PY_TXI_EXAMPLE_CONFIGS)
+def test_cuda_py_txi_examples(config_name):
     test_example_configs(config_name)