From 14b59a8a15e283a6c2b8c8353099259543d2d970 Mon Sep 17 00:00:00 2001 From: dbogunowicz <97082108+dbogunowicz@users.noreply.github.com> Date: Wed, 6 Mar 2024 14:40:37 +0100 Subject: [PATCH] [Fix] Raise ValueError when llm-eval gets an unknown dataset to evaluate on (instead of failing silently) (#1625) * initial commit * fixing tests --- .../integrations/lm_evaluation_harness.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/src/deepsparse/evaluation/integrations/lm_evaluation_harness.py b/src/deepsparse/evaluation/integrations/lm_evaluation_harness.py index 6e0711cb3c..654aedf5d8 100644 --- a/src/deepsparse/evaluation/integrations/lm_evaluation_harness.py +++ b/src/deepsparse/evaluation/integrations/lm_evaluation_harness.py @@ -67,8 +67,17 @@ def integration_eval( """ pipeline = DeepSparseLM(pipeline=pipeline, batch_size=batch_size) - datasets = (",").join(datasets) if isinstance(datasets, list) else datasets - task_names = utils.pattern_match(datasets.split(","), tasks.ALL_TASKS) + datasets = datasets if isinstance(datasets, list) else [datasets] + task_names = [] + for idx, dataset in enumerate(datasets): + task_name = utils.pattern_match(dataset, tasks.ALL_TASKS) + if not task_name: + raise ValueError( + f"could recognize the dataset: {datasets[idx]}. Make sure " + "that the requested dataset is compatible with the " + "llm-evaluation-harness" + ) + task_names.extend(task_name) _LOGGER.info(f"Selected Tasks: {task_names}")