From 4e791ee9843f87dd35a3c026bb7b63f5e20d6aef Mon Sep 17 00:00:00 2001
From: dbogunowicz <97082108+dbogunowicz@users.noreply.github.com>
Date: Thu, 7 Mar 2024 16:43:48 +0100
Subject: [PATCH] Pass arguments for pipeline creation properly to
 deepsparse.evaluate (#1624)

* initial commit

* fix tests

* Update src/deepsparse/evaluation/utils.py

* quality
---
 src/deepsparse/evaluation/evaluator.py    |  7 ++++---
 src/deepsparse/evaluation/utils.py        | 19 +++++++++++--------
 tests/deepsparse/evaluation/test_utils.py |  4 ++--
 3 files changed, 17 insertions(+), 13 deletions(-)

diff --git a/src/deepsparse/evaluation/evaluator.py b/src/deepsparse/evaluation/evaluator.py
index 3d18f8489f..6d24c8f6da 100644
--- a/src/deepsparse/evaluation/evaluator.py
+++ b/src/deepsparse/evaluation/evaluator.py
@@ -56,9 +56,10 @@ def evaluate(
 
     # if target is a string, turn it into an appropriate pipeline
     # otherwise assume it is a pipeline
-    pipeline = (
-        create_pipeline(model, engine_type) if isinstance(model, (Path, str)) else model
-    )
+    if isinstance(model, (Path, str)):
+        pipeline, kwargs = create_pipeline(model, engine_type, **kwargs)
+    else:
+        pipeline = model
 
     eval_integration = EvaluationRegistry.resolve(pipeline, datasets, integration)
 
diff --git a/src/deepsparse/evaluation/utils.py b/src/deepsparse/evaluation/utils.py
index 15503f0553..e4ea74e5a8 100644
--- a/src/deepsparse/evaluation/utils.py
+++ b/src/deepsparse/evaluation/utils.py
@@ -199,14 +199,17 @@ def create_pipeline(
         text generation model from. This can be a local
         or remote path to the model or a sparsezoo stub
     :param engine_type: The engine type to initialize the model with.
-    :return: The initialized pipeline
+    :return: The initialized pipeline and the mutated
+        (potentially reduced number of) kwargs
     """
     engine_type = engine_type or DEEPSPARSE_ENGINE
-    return Pipeline.create(
-        task=kwargs.pop("task", "text-generation"),
-        model_path=model_path,
-        sequence_length=kwargs.pop("sequence_length", 2048),
-        engine_type=engine_type,
-        batch_size=kwargs.pop("batch_size", 1),
-        **kwargs,
+    return (
+        Pipeline.create(
+            task=kwargs.pop("task", "text-generation"),
+            model_path=model_path,
+            sequence_length=kwargs.pop("sequence_length", 2048),
+            engine_type=engine_type,
+            batch_size=kwargs.pop("batch_size", 1),
+        ),
+        kwargs,
     )
diff --git a/tests/deepsparse/evaluation/test_utils.py b/tests/deepsparse/evaluation/test_utils.py
index f8f3c731a8..4e4360719d 100644
--- a/tests/deepsparse/evaluation/test_utils.py
+++ b/tests/deepsparse/evaluation/test_utils.py
@@ -62,10 +62,10 @@ def pipeline_target():
 
 
 def test_initialize_model_from_target_pipeline_onnx(pipeline_target):
-    model = create_pipeline(pipeline_target, "onnxruntime")
+    model, _ = create_pipeline(pipeline_target, "onnxruntime")
     assert model.ops.get("single_engine")._engine_type == "onnxruntime"
 
 
 def test_initialize_model_from_target_pipeline_with_kwargs(pipeline_target):
-    model = create_pipeline(pipeline_target, "deepsparse", sequence_length=64)
+    model, _ = create_pipeline(pipeline_target, "deepsparse", sequence_length=64)
     assert model.ops.get("process_input").sequence_length == 64