PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
refine eval, add more option (#2114)
Weiweife7676ded728
diff --git a/auto_round/cli/main.py b/auto_round/cli/main.pyindex 09b0b8263..859435c71 100644--- a/auto_round/cli/main.py+++ b/auto_round/cli/main.py@@ -448,6 +448,9 @@ def run_eval(argv=None):trust_remote_code=not args.disable_trust_remote_code,eval_model_dtype=args.eval_model_dtype,add_bos_token=args.add_bos_token,+ num_fewshot=args.num_fewshot,+ gen_kwargs=args.eval_gen_kwargs,+ fewshot_as_multiturn=args.fewshot_as_multiturn,)else:eval(args)diff --git a/auto_round/cli/parser.py b/auto_round/cli/parser.pyindex 81ae5d3eb..7175a1f8c 100644--- a/auto_round/cli/parser.py+++ b/auto_round/cli/parser.py@@ -221,6 +221,16 @@ def build_quantize_parser(*, prog: str = "auto_round quantize") -> argparse.Arguev.add_argument("--limit", type=float, default=None, metavar="N|0<N<1", help="Evaluation example limit as a count or fraction.")+ ev.add_argument("--num_fewshot", "--num-fewshot", default=None, type=int, help="Number of few-shot examples.")+ ev.add_argument(+ "--eval_gen_kwargs", "--eval-gen-kwargs", default=None, type=str, help="Generation kwargs for LM-Eval."+ )+ ev.add_argument(+ "--fewshot_as_multiturn",+ "--fewshot-as-multiturn",+ action="store_true",+ help="Use multi-turn format for few-shot examples in LM-Eval.",+ )ev.add_argument("--eval_task_by_task", action="store_true", help="Evaluate tasks sequentially instead of batching.")ev.add_argument("--eval_backend", default="hf", type=str, choices=["hf", "vllm"], help="Backend to use for evaluation."diff --git a/auto_round/eval/eval_cli.py b/auto_round/eval/eval_cli.pyindex 674e2085b..02151b6fc 100644--- a/auto_round/eval/eval_cli.py+++ b/auto_round/eval/eval_cli.py@@ -16,7 +16,7 @@ import importlib.utilimport osimport time-import torch.nn+import torchfrom transformers.utils.versions import require_versionfrom auto_round.utils import (@@ -101,6 +101,16 @@ class EvalArgumentParser(argparse.ArgumentParser):"Integer: exact number of examples (e.g., 1000). ""Float between 0-1: fraction of total examples.",)+ self.add_argument("--num_fewshot", "--num-fewshot", default=None, type=int, help="Number of few-shot examples.")+ self.add_argument(+ "--eval_gen_kwargs", "--eval-gen-kwargs", default=None, type=str, help="Generation kwargs for LM-Eval."+ )+ self.add_argument(+ "--fewshot_as_multiturn",+ "--fewshot-as-multiturn",+ action="store_true",+ help="Use multi-turn format for few-shot examples in LM-Eval.",+ )self.add_argument("--eval_backend",default="hf",@@ -225,6 +235,9 @@ def eval(args):batch_size=batch_size,device=device_str,limit=args.limit,+ num_fewshot=getattr(args, "num_fewshot", None),+ gen_kwargs=getattr(args, "eval_gen_kwargs", None),+ fewshot_as_multiturn=getattr(args, "fewshot_as_multiturn", False),add_bos_token=args.add_bos_token,)print(make_table(res))@@ -242,6 +255,9 @@ def eval(args):device=device_str,batch_size=batch_size,limit=args.limit,+ num_fewshot=getattr(args, "num_fewshot", None),+ gen_kwargs=getattr(args, "eval_gen_kwargs", None),+ fewshot_as_multiturn=getattr(args, "fewshot_as_multiturn", False),)from lm_eval.utils import make_table # pylint: disable=E0401@@ -323,6 +339,9 @@ def eval_with_vllm(args):model=vllm_lm,tasks=tasks,limit=args.limit,+ num_fewshot=getattr(args, "num_fewshot", None),+ gen_kwargs=getattr(args, "eval_gen_kwargs", None),+ fewshot_as_multiturn=getattr(args, "fewshot_as_multiturn", False),)print(make_table(res))@@ -342,6 +361,9 @@ def eval_task_by_task(retry_times=3,mllm=False,add_bos_token=False,+ num_fewshot=None,+ gen_kwargs=None,+ fewshot_as_multiturn=False,):require_version("lm_eval>=0.4.2", "lm-eval is required for evaluation, please install it with `pip install 'lm-eval>=0.4.2'`"@@ -404,7 +426,17 @@ def eval_task_by_task(add_bos_token=add_bos_token,)- _evaluate_tasks_with_retry(tasks, hflm, device_str, batch_size, limit, retry_times)+ _evaluate_tasks_with_retry(+ tasks,+ hflm,+ device_str,+ batch_size,+ limit,+ retry_times,+ num_fewshot=num_fewshot,+ gen_kwargs=gen_kwargs,+ fewshot_as_multiturn=fewshot_as_multiturn,+ )def _load_gguf_model_if_needed(model_path, eval_model_dtype=None):@@ -459,7 +491,39 @@ def _load_gguf_model_if_needed(model_path, eval_model_dtype=None):return model, tokenizer, is_gguf_file, gguf_file-def _evaluate_tasks_with_retry(tasks, hflm, device_str, batch_size, limit, retry_times):+def _get_lm_eval_task_manager(tasks: list[str]) -> object | None:+ """Use exact installed task dirs when possible to avoid scanning all lm-eval tasks."""+ try:+ import lm_eval # pylint: disable=E0401+ from lm_eval.tasks import TaskManager # pylint: disable=E0401+ except ImportError:+ return None++ tasks_root = os.path.join(os.path.dirname(lm_eval.__file__), "tasks")+ task_paths = []+ for task in tasks:+ task_path = os.path.join(tasks_root, task)+ if not os.path.isdir(task_path):+ return None+ task_paths.append(task_path)++ try:+ return TaskManager(include_defaults=False, include_path=task_paths)+ except TypeError:+ return None+++def _evaluate_tasks_with_retry(+ tasks,+ hflm,+ device_str,+ batch_size,+ limit,+ retry_times,+ num_fewshot=None,+ gen_kwargs=None,+ fewshot_as_multiturn=False,+):"""Evaluate tasks with automatic retry on OOM errors.Args:@@ -487,41 +551,59 @@ def _evaluate_tasks_with_retry(tasks, hflm, device_str, batch_size, limit, retryres_all = {}res_keys = ["results", "versions", "n-shot", "higher_is_better"]st = time.time()+ task_manager = _get_lm_eval_task_manager(tasks)for task in tasks:current_retry_times = retry_timesres = None+ last_error = Nonewhile current_retry_times:try:res = lm_eval.simple_evaluate(- model=hflm, model_args=None, device=device_str, tasks=task, batch_size=batch_size, limit=limit+ model=hflm,+ model_args=None,+ device=device_str,+ tasks=task,+ batch_size=batch_size,+ limit=limit,+ num_fewshot=num_fewshot,+ gen_kwargs=gen_kwargs,+ task_manager=task_manager,+ fewshot_as_multiturn=fewshot_as_multiturn,)break- except Exception as e:+ except torch.OutOfMemoryError as e:+ last_error = ecuda_error_msg = traceback.format_exc()+ ori_batch_sizes = getattr(hflm, "batch_sizes", None) or {"0": 64}+ if not getattr(hflm, "batch_sizes", None):+ hflm.batch_sizes = ori_batch_sizes.copy()try:- ori_batch_sizes = hflm.batch_sizes or {"0": 64}- if not hflm.batch_sizes:- hflm.batch_sizes = ori_batch_sizes.copy()- try:- for k, v in hflm.batch_sizes.items():- hflm.batch_sizes[k] = max(v // 2, 1)- logger.warning(f"Out of memory, reset batch_size to {hflm.batch_sizes} and re-try.")- res = lm_eval.simple_evaluate(- model=hflm, model_args=None, device=device_str, tasks=task, batch_size=1, limit=limit- )- hflm.batch_sizes = ori_batch_sizes- except Exception as e:- traceback.print_exc()- res = None- except Exception as e:- logger.error(cuda_error_msg)+ for k, v in hflm.batch_sizes.items():+ hflm.batch_sizes[k] = max(v // 2, 1)+ logger.warning(f"Out of memory, reset batch_size to {hflm.batch_sizes} and re-try.")+ res = lm_eval.simple_evaluate(+ model=hflm,+ model_args=None,+ device=device_str,+ tasks=task,+ batch_size=1,+ limit=limit,+ num_fewshot=num_fewshot,+ gen_kwargs=gen_kwargs,+ task_manager=task_manager,+ fewshot_as_multiturn=fewshot_as_multiturn,+ )+ except torch.OutOfMemoryError as e:+ last_error = etraceback.print_exc()res = None+ finally:+ hflm.batch_sizes = ori_batch_sizescurrent_retry_times -= 1if res is None:- raise RuntimeError(f"Failed to evaluate task '{task}' after {retry_times} attempts")+ raise RuntimeError(f"Failed to evaluate task '{task}' after {retry_times} attempts") from last_errorif not res_all:res_all = reselse:diff --git a/auto_round/eval/evaluation.py b/auto_round/eval/evaluation.pyindex 760c3acca..c01e5feb7 100644--- a/auto_round/eval/evaluation.py+++ b/auto_round/eval/evaluation.py@@ -310,6 +310,9 @@ def evaluate_with_model_instance(model, tokenizer, device_str, args):batch_size=args.eval_bs,eval_model_dtype=get_model_dtype(args.eval_model_dtype, "auto"),add_bos_token=args.add_bos_token,+ num_fewshot=getattr(args, "num_fewshot", None),+ gen_kwargs=getattr(args, "eval_gen_kwargs", None),+ fewshot_as_multiturn=getattr(args, "fewshot_as_multiturn", False),)else:# Batch evaluation@@ -328,6 +331,9 @@ def evaluate_with_model_instance(model, tokenizer, device_str, args):device=device_str,eval_model_dtype=get_model_dtype(args.eval_model_dtype, "auto"),add_bos_token=args.add_bos_token,+ num_fewshot=getattr(args, "num_fewshot", None),+ gen_kwargs=getattr(args, "eval_gen_kwargs", None),+ fewshot_as_multiturn=getattr(args, "fewshot_as_multiturn", False),)print(make_table(res))print("evaluation running time=%ds" % (time.time() - st))@@ -366,6 +372,9 @@ def evaluate_with_model_path(eval_folder, device_str, autoround, args):eval_model_dtype=get_model_dtype(args.eval_model_dtype, "auto"),mllm=getattr(autoround, "mllm", False),add_bos_token=args.add_bos_token,+ num_fewshot=getattr(args, "num_fewshot", None),+ gen_kwargs=getattr(args, "eval_gen_kwargs", None),+ fewshot_as_multiturn=getattr(args, "fewshot_as_multiturn", False),)else:# Batch evaluation@@ -398,6 +407,9 @@ def evaluate_with_model_path(eval_folder, device_str, autoround, args):device=device_str,batch_size=eval_bs,limit=args.limit,+ num_fewshot=getattr(args, "num_fewshot", None),+ gen_kwargs=getattr(args, "eval_gen_kwargs", None),+ fewshot_as_multiturn=getattr(args, "fewshot_as_multiturn", False),)print(make_table(res))print("evaluation running time=%ds" % (time.time() - st))@@ -468,6 +480,9 @@ def run_model_evaluation(model, tokenizer, autoround, folders, formats, args):vllm_args.disable_trust_remote_code = getattr(args, "disable_trust_remote_code", False)vllm_args.add_bos_token = getattr(args, "add_bos_token", False)vllm_args.seed = getattr(args, "seed", 42)+ vllm_args.num_fewshot = getattr(args, "num_fewshot", None)+ vllm_args.eval_gen_kwargs = getattr(args, "eval_gen_kwargs", None)+ vllm_args.fewshot_as_multiturn = getattr(args, "fewshot_as_multiturn", False)# VLLM-specific parametersvllm_args.vllm_args = getattr(args, "vllm_args", None)eval_with_vllm(vllm_args)diff --git a/docs/step_by_step.md b/docs/step_by_step.mdindex f10466382..f5fea9bda 100644--- a/docs/step_by_step.md+++ b/docs/step_by_step.md@@ -1093,6 +1093,7 @@ CUDA_VISIBLE_DEVICES=0,1 auto-round "your_model_path" --eval --tasks lambada_ope- Use the `--eval` flag to evaluate models directly. This supports both original and quantized models.- The `--eval_task_by_task` option helps handle task failures by evaluating tasks sequentially. This only applies to the HF backend.+- Use `--num_fewshot`, `--eval_gen_kwargs`, and `--fewshot_as_multiturn` to pass few-shot and generation options through to lm-eval.- When multiple formats are exported, the last format in the list will be used for evaluation.- For vLLM backend, you can use `--device 0,1,2` to specify GPU devices. This will automatically set `CUDA_VISIBLE_DEVICES` and configure `tensor_parallel_size` based on the number of devices. Alternatively, you can manually set these via environment variables and `--vllm_args`.diff --git a/docs/step_by_step_CN.md b/docs/step_by_step_CN.mdindex 77b5eedfa..e553e3b6e 100644--- a/docs/step_by_step_CN.md+++ b/docs/step_by_step_CN.md@@ -1060,6 +1060,7 @@ CUDA_VISIBLE_DEVICES=0,1 auto-round "your_model_path" --eval --tasks lambada_ope- 对于原始模型和量化后的模型,都支持用 `--eval` 参数直接评估。- 为应对部分任务运行失败的情况,可使用 `--eval_task_by_task` 参数,按顺序执行评测任务(该参数目前只适用于 HF 后端)。+- 可使用 `--num_fewshot`、`--eval_gen_kwargs` 和 `--fewshot_as_multiturn` 将 few-shot 与生成参数传递给 lm-eval。- 若导出了多种格式,会自动选用列表中的**最后一种格式**的模型评估。- 对于 vLLM 后端,可通过 `--device 0,1,2` 指定 GPU 设备。该参数会自动设置 `CUDA_VISIBLE_DEVICES`,并根据设备数量配置 `tensor_parallel_size` 。此外,也支持通过环境变量和 `--vllm_args` 参数进行手动设置。diff --git a/test/unit/test_cpu/advanced/test_evaluation_functions.py b/test/unit/test_cpu/advanced/test_evaluation_functions.pyindex d6476ec21..e195ada39 100644--- a/test/unit/test_cpu/advanced/test_evaluation_functions.py+++ b/test/unit/test_cpu/advanced/test_evaluation_functions.py@@ -16,14 +16,14 @@CPU tests for evaluation utility functions.Lightweight tests focusing on key utility functions without heavy model loading.-Run with: pytest test/test_cpu/advanced/test_evaluation_functions.py+Run with: pytest test/unit/test_cpu/advanced/test_evaluation_functions.py"""-import os+import argparse+import sys+from types import ModuleType, SimpleNamespacefrom unittest.mock import MagicMock, patch-import pytest-class TestSelectGgufEvalFile:"""Test GGUF text file selection for evaluation."""@@ -138,11 +138,178 @@ class TestLoadGgufModelIfNeeded:assert is_gguf is Falseassert gguf_file is None- def test_load_gguf_model_non_string_model(self, tiny_opt_model_path):+ def test_load_gguf_model_non_string_model(self):"""Test with model object (not a string path)."""from auto_round.eval.eval_cli import _load_gguf_model_if_needed- model, tokenizer, is_gguf, gguf_file = _load_gguf_model_if_needed(tiny_opt_model_path)+ model_obj = object()+ model, tokenizer, is_gguf, gguf_file = _load_gguf_model_if_needed(model_obj)+ assert model is model_objassert tokenizer is Noneassert is_gguf is Falseassert gguf_file is None+++def _make_eval_args(**overrides):+ defaults = dict(+ model="test-model",+ model_name="test-model",+ mllm=False,+ device_map="0",+ tasks="lambada_openai",+ disable_trust_remote_code=False,+ seed=42,+ eval_bs=2,+ eval_task_by_task=False,+ eval_model_dtype=None,+ limit=5,+ num_fewshot=3,+ eval_gen_kwargs="temperature=0.1,top_p=0.9",+ fewshot_as_multiturn=True,+ eval_backend="hf",+ add_bos_token=False,+ vllm_args=None,+ )+ defaults.update(overrides)+ return argparse.Namespace(**defaults)+++def _fake_lm_eval_modules(simple_evaluate_impl):+ root_module = ModuleType("lm_eval")+ utils_module = ModuleType("lm_eval.utils")+ utils_module.make_table = lambda result: "table"++ evaluator_module = ModuleType("lm_eval.evaluator")+ evaluator_module.simple_evaluate = simple_evaluate_impl++ models_module = ModuleType("lm_eval.models")+ vllm_module = ModuleType("lm_eval.models.vllm_causallms")+ vllm_module.VLLM = MagicMock(side_effect=lambda **kwargs: SimpleNamespace(kwargs=kwargs))++ vllm_vlm_module = ModuleType("lm_eval.models.vllm_vlms")+ vllm_vlm_module.VLLM_VLM = MagicMock(side_effect=lambda **kwargs: SimpleNamespace(kwargs=kwargs))++ root_module.evaluator = evaluator_module+ root_module.utils = utils_module+ root_module.models = models_module+ models_module.vllm_causallms = vllm_module+ models_module.vllm_vlms = vllm_vlm_module++ return {+ "lm_eval": root_module,+ "lm_eval.utils": utils_module,+ "lm_eval.models": models_module,+ "lm_eval.evaluator": evaluator_module,+ "lm_eval.models.vllm_causallms": vllm_module,+ "lm_eval.models.vllm_vlms": vllm_vlm_module,+ }+++class TestEvalArgumentForwarding:+ def test_run_eval_task_by_task_forwards_eval_generation_arguments(self):+ from auto_round.cli.main import run_eval++ args = _make_eval_args(eval_task_by_task=True)++ with patch("auto_round.cli.main.setup_eval_parser", return_value=args), patch(+ "auto_round.utils.is_gguf_model", return_value=False+ ), patch("auto_round.utils.is_mllm_model", return_value=False), patch(+ "auto_round.eval.eval_cli.eval_task_by_task"+ ) as mock_eval_task_by_task:+ run_eval([])++ _, kwargs = mock_eval_task_by_task.call_args+ assert kwargs["num_fewshot"] == args.num_fewshot+ assert kwargs["gen_kwargs"] == args.eval_gen_kwargs+ assert kwargs["fewshot_as_multiturn"] == args.fewshot_as_multiturn++ def test_eval_hf_batch_forwards_eval_generation_arguments(self):+ from auto_round.eval.eval_cli import eval++ args = _make_eval_args()+ result = {"results": {"lambada_openai": {}}, "versions": {}, "n-shot": {}, "higher_is_better": {}}++ with patch("auto_round.eval.eval_cli.require_version"), patch(+ "auto_round.eval.eval_cli.is_diffusion_model", return_value=False+ ), patch(+ "auto_round.eval.eval_cli._eval_init", return_value=(["lambada_openai"], "pretrained=test-model", "cpu")+ ), patch(+ "auto_round.eval.eval_cli._load_gguf_model_if_needed", return_value=(None, None, False, None)+ ), patch(+ "auto_round.eval.evaluation.simple_evaluate", return_value=result+ ) as mock_simple_evaluate, patch(+ "builtins.print"+ ), patch.dict(+ sys.modules, _fake_lm_eval_modules(MagicMock(return_value=result))+ ):+ eval(args)++ _, kwargs = mock_simple_evaluate.call_args+ assert kwargs["num_fewshot"] == args.num_fewshot+ assert kwargs["gen_kwargs"] == args.eval_gen_kwargs+ assert kwargs["fewshot_as_multiturn"] == args.fewshot_as_multiturn++ def test_eval_gguf_batch_forwards_eval_generation_arguments(self):+ from auto_round.eval.eval_cli import eval++ args = _make_eval_args()+ result = {"results": {"lambada_openai": {}}, "versions": {}, "n-shot": {}, "higher_is_better": {}}++ with patch("auto_round.eval.eval_cli.require_version"), patch(+ "auto_round.eval.eval_cli.is_diffusion_model", return_value=False+ ), patch(+ "auto_round.eval.eval_cli._eval_init", return_value=(["lambada_openai"], "pretrained=test-model", "cpu")+ ), patch(+ "auto_round.eval.eval_cli._load_gguf_model_if_needed",+ return_value=(object(), object(), True, "model.gguf"),+ ), patch(+ "auto_round.eval.evaluation.simple_evaluate_user_model", return_value=result+ ) as mock_simple_evaluate_user_model, patch(+ "builtins.print"+ ), patch.dict(+ sys.modules, _fake_lm_eval_modules(MagicMock(return_value=result))+ ):+ eval(args)++ _, kwargs = mock_simple_evaluate_user_model.call_args+ assert kwargs["num_fewshot"] == args.num_fewshot+ assert kwargs["gen_kwargs"] == args.eval_gen_kwargs+ assert kwargs["fewshot_as_multiturn"] == args.fewshot_as_multiturn++ def test_eval_with_vllm_forwards_eval_generation_arguments(self):+ from auto_round.eval.eval_cli import eval_with_vllm++ args = _make_eval_args()+ evaluator_simple_evaluate = MagicMock(+ return_value={"results": {"lambada_openai": {}}, "versions": {}, "n-shot": {}, "higher_is_better": {}}+ )++ with patch("auto_round.eval.eval_cli.get_device_and_parallelism", return_value=("cuda:0", False)), patch(+ "auto_round.eval.eval_cli.get_major_device", return_value="cuda"+ ), patch("auto_round.eval.eval_cli.get_model_dtype", return_value="float16"), patch(+ "builtins.print"+ ), patch.dict(+ sys.modules, _fake_lm_eval_modules(evaluator_simple_evaluate)+ ):+ eval_with_vllm(args)++ _, kwargs = evaluator_simple_evaluate.call_args+ assert kwargs["num_fewshot"] == args.num_fewshot+ assert kwargs["gen_kwargs"] == args.eval_gen_kwargs+ assert kwargs["fewshot_as_multiturn"] == args.fewshot_as_multiturn++ def test_run_model_evaluation_vllm_forwards_eval_generation_arguments(self, tmp_path):+ from auto_round.eval.evaluation import run_model_evaluation++ args = _make_eval_args(eval_backend="vllm")+ autoround = SimpleNamespace()++ with patch("auto_round.utils.model.detect_model_type", return_value="llm"), patch(+ "auto_round.utils.device_manager.get_device_and_parallelism", return_value=("cpu", False)+ ), patch("auto_round.eval.eval_cli.eval_with_vllm") as mock_eval_with_vllm:+ run_model_evaluation(None, None, autoround, str(tmp_path), ["auto_round"], args)++ forwarded_args = mock_eval_with_vllm.call_args.args[0]+ assert forwarded_args.num_fewshot == args.num_fewshot+ assert forwarded_args.eval_gen_kwargs == args.eval_gen_kwargs+ assert forwarded_args.fewshot_as_multiturn == args.fewshot_as_multiturndiff --git a/test/unit/test_cpu/eval/test_eval_cli.py b/test/unit/test_cpu/eval/test_eval_cli.pyindex f21a0fd37..c3d678742 100644--- a/test/unit/test_cpu/eval/test_eval_cli.py+++ b/test/unit/test_cpu/eval/test_eval_cli.py@@ -19,6 +19,7 @@ from types import SimpleNamespacefrom unittest.mock import MagicMock, patchimport pytest+import torchimport transformersfrom auto_round.eval import eval_cli@@ -131,6 +132,21 @@ class TestEvalArgumentParser:args = parser.parse_args(["--vllm_args", "tensor_parallel_size=2,gpu_memory_utilization=0.9"])assert args.vllm_args == "tensor_parallel_size=2,gpu_memory_utilization=0.9"+ def test_fewshot_and_generation_args_are_accepted(self):+ parser = eval_cli.EvalArgumentParser()+ args = parser.parse_args(+ [+ "--num-fewshot",+ "3",+ "--eval-gen-kwargs",+ "temperature=0.1",+ "--fewshot-as-multiturn",+ ]+ )+ assert args.num_fewshot == 3+ assert args.eval_gen_kwargs == "temperature=0.1"+ assert args.fewshot_as_multiturn is True+class TestEvalInit:"""Tests for `_eval_init` task normalization, device resolution, and dtype."""@@ -584,7 +600,7 @@ class TestEvaluateTasksWithRetry:def fake_simple_evaluate(**kwargs):calls.append(kwargs.get("batch_size"))if len(calls) <= 2 and calls[-1] == 8:- raise RuntimeError("oom")+ raise torch.OutOfMemoryError("CUDA out of memory")return fake_resmonkeypatch.setattr("lm_eval.simple_evaluate", fake_simple_evaluate)@@ -601,13 +617,19 @@ class TestEvaluateTasksWithRetry:assert calls == [8, 1]- def test_exhausted_retries_raises_runtime_error(self, monkeypatch):+ def test_non_oom_error_is_raised_without_retry(self, monkeypatch):+ calls = []++ def fake_simple_evaluate(**kwargs):+ calls.append(kwargs)+ raise RuntimeError("permanent failure")+monkeypatch.setattr("lm_eval.simple_evaluate",- lambda **kwargs: (_ for _ in ()).throw(RuntimeError("permanent failure")),+ fake_simple_evaluate,)- with pytest.raises(RuntimeError, match="Failed to evaluate task 'bad-task'"):+ with pytest.raises(RuntimeError, match="permanent failure"):eval_cli._evaluate_tasks_with_retry(tasks=["bad-task"],hflm=object(),@@ -617,6 +639,8 @@ class TestEvaluateTasksWithRetry:retry_times=2,)+ assert len(calls) == 1+def test_multiple_tasks_are_aggregated(self, monkeypatch):fake_hflm = object()res_a = {