Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

refine eval, add more option (#2114)

Weiweife7676ded728
diff --git a/auto_round/cli/main.py b/auto_round/cli/main.py
index 09b0b8263..859435c71 100644
--- a/auto_round/cli/main.py
+++ b/auto_round/cli/main.py
@@ -448,6 +448,9 @@ def run_eval(argv=None):
trust_remote_code=not args.disable_trust_remote_code,
eval_model_dtype=args.eval_model_dtype,
add_bos_token=args.add_bos_token,
+ num_fewshot=args.num_fewshot,
+ gen_kwargs=args.eval_gen_kwargs,
+ fewshot_as_multiturn=args.fewshot_as_multiturn,
)
else:
eval(args)
diff --git a/auto_round/cli/parser.py b/auto_round/cli/parser.py
index 81ae5d3eb..7175a1f8c 100644
--- a/auto_round/cli/parser.py
+++ b/auto_round/cli/parser.py
@@ -221,6 +221,16 @@ def build_quantize_parser(*, prog: str = "auto_round quantize") -> argparse.Argu
ev.add_argument(
"--limit", type=float, default=None, metavar="N|0<N<1", help="Evaluation example limit as a count or fraction."
)
+ ev.add_argument("--num_fewshot", "--num-fewshot", default=None, type=int, help="Number of few-shot examples.")
+ ev.add_argument(
+ "--eval_gen_kwargs", "--eval-gen-kwargs", default=None, type=str, help="Generation kwargs for LM-Eval."
+ )
+ ev.add_argument(
+ "--fewshot_as_multiturn",
+ "--fewshot-as-multiturn",
+ action="store_true",
+ help="Use multi-turn format for few-shot examples in LM-Eval.",
+ )
ev.add_argument("--eval_task_by_task", action="store_true", help="Evaluate tasks sequentially instead of batching.")
ev.add_argument(
"--eval_backend", default="hf", type=str, choices=["hf", "vllm"], help="Backend to use for evaluation."
diff --git a/auto_round/eval/eval_cli.py b/auto_round/eval/eval_cli.py
index 674e2085b..02151b6fc 100644
--- a/auto_round/eval/eval_cli.py
+++ b/auto_round/eval/eval_cli.py
@@ -16,7 +16,7 @@ import importlib.util
import os
import time
-import torch.nn
+import torch
from transformers.utils.versions import require_version
from auto_round.utils import (
@@ -101,6 +101,16 @@ class EvalArgumentParser(argparse.ArgumentParser):
"Integer: exact number of examples (e.g., 1000). "
"Float between 0-1: fraction of total examples.",
)
+ self.add_argument("--num_fewshot", "--num-fewshot", default=None, type=int, help="Number of few-shot examples.")
+ self.add_argument(
+ "--eval_gen_kwargs", "--eval-gen-kwargs", default=None, type=str, help="Generation kwargs for LM-Eval."
+ )
+ self.add_argument(
+ "--fewshot_as_multiturn",
+ "--fewshot-as-multiturn",
+ action="store_true",
+ help="Use multi-turn format for few-shot examples in LM-Eval.",
+ )
self.add_argument(
"--eval_backend",
default="hf",
@@ -225,6 +235,9 @@ def eval(args):
batch_size=batch_size,
device=device_str,
limit=args.limit,
+ num_fewshot=getattr(args, "num_fewshot", None),
+ gen_kwargs=getattr(args, "eval_gen_kwargs", None),
+ fewshot_as_multiturn=getattr(args, "fewshot_as_multiturn", False),
add_bos_token=args.add_bos_token,
)
print(make_table(res))
@@ -242,6 +255,9 @@ def eval(args):
device=device_str,
batch_size=batch_size,
limit=args.limit,
+ num_fewshot=getattr(args, "num_fewshot", None),
+ gen_kwargs=getattr(args, "eval_gen_kwargs", None),
+ fewshot_as_multiturn=getattr(args, "fewshot_as_multiturn", False),
)
from lm_eval.utils import make_table # pylint: disable=E0401
@@ -323,6 +339,9 @@ def eval_with_vllm(args):
model=vllm_lm,
tasks=tasks,
limit=args.limit,
+ num_fewshot=getattr(args, "num_fewshot", None),
+ gen_kwargs=getattr(args, "eval_gen_kwargs", None),
+ fewshot_as_multiturn=getattr(args, "fewshot_as_multiturn", False),
)
print(make_table(res))
@@ -342,6 +361,9 @@ def eval_task_by_task(
retry_times=3,
mllm=False,
add_bos_token=False,
+ num_fewshot=None,
+ gen_kwargs=None,
+ fewshot_as_multiturn=False,
):
require_version(
"lm_eval>=0.4.2", "lm-eval is required for evaluation, please install it with `pip install 'lm-eval>=0.4.2'`"
@@ -404,7 +426,17 @@ def eval_task_by_task(
add_bos_token=add_bos_token,
)
- _evaluate_tasks_with_retry(tasks, hflm, device_str, batch_size, limit, retry_times)
+ _evaluate_tasks_with_retry(
+ tasks,
+ hflm,
+ device_str,
+ batch_size,
+ limit,
+ retry_times,
+ num_fewshot=num_fewshot,
+ gen_kwargs=gen_kwargs,
+ fewshot_as_multiturn=fewshot_as_multiturn,
+ )
def _load_gguf_model_if_needed(model_path, eval_model_dtype=None):
@@ -459,7 +491,39 @@ def _load_gguf_model_if_needed(model_path, eval_model_dtype=None):
return model, tokenizer, is_gguf_file, gguf_file
-def _evaluate_tasks_with_retry(tasks, hflm, device_str, batch_size, limit, retry_times):
+def _get_lm_eval_task_manager(tasks: list[str]) -> object | None:
+ """Use exact installed task dirs when possible to avoid scanning all lm-eval tasks."""
+ try:
+ import lm_eval # pylint: disable=E0401
+ from lm_eval.tasks import TaskManager # pylint: disable=E0401
+ except ImportError:
+ return None
+
+ tasks_root = os.path.join(os.path.dirname(lm_eval.__file__), "tasks")
+ task_paths = []
+ for task in tasks:
+ task_path = os.path.join(tasks_root, task)
+ if not os.path.isdir(task_path):
+ return None
+ task_paths.append(task_path)
+
+ try:
+ return TaskManager(include_defaults=False, include_path=task_paths)
+ except TypeError:
+ return None
+
+
+def _evaluate_tasks_with_retry(
+ tasks,
+ hflm,
+ device_str,
+ batch_size,
+ limit,
+ retry_times,
+ num_fewshot=None,
+ gen_kwargs=None,
+ fewshot_as_multiturn=False,
+):
"""Evaluate tasks with automatic retry on OOM errors.
Args:
@@ -487,41 +551,59 @@ def _evaluate_tasks_with_retry(tasks, hflm, device_str, batch_size, limit, retry
res_all = {}
res_keys = ["results", "versions", "n-shot", "higher_is_better"]
st = time.time()
+ task_manager = _get_lm_eval_task_manager(tasks)
for task in tasks:
current_retry_times = retry_times
res = None
+ last_error = None
while current_retry_times:
try:
res = lm_eval.simple_evaluate(
- model=hflm, model_args=None, device=device_str, tasks=task, batch_size=batch_size, limit=limit
+ model=hflm,
+ model_args=None,
+ device=device_str,
+ tasks=task,
+ batch_size=batch_size,
+ limit=limit,
+ num_fewshot=num_fewshot,
+ gen_kwargs=gen_kwargs,
+ task_manager=task_manager,
+ fewshot_as_multiturn=fewshot_as_multiturn,
)
break
- except Exception as e:
+ except torch.OutOfMemoryError as e:
+ last_error = e
cuda_error_msg = traceback.format_exc()
+ ori_batch_sizes = getattr(hflm, "batch_sizes", None) or {"0": 64}
+ if not getattr(hflm, "batch_sizes", None):
+ hflm.batch_sizes = ori_batch_sizes.copy()
try:
- ori_batch_sizes = hflm.batch_sizes or {"0": 64}
- if not hflm.batch_sizes:
- hflm.batch_sizes = ori_batch_sizes.copy()
- try:
- for k, v in hflm.batch_sizes.items():
- hflm.batch_sizes[k] = max(v // 2, 1)
- logger.warning(f"Out of memory, reset batch_size to {hflm.batch_sizes} and re-try.")
- res = lm_eval.simple_evaluate(
- model=hflm, model_args=None, device=device_str, tasks=task, batch_size=1, limit=limit
- )
- hflm.batch_sizes = ori_batch_sizes
- except Exception as e:
- traceback.print_exc()
- res = None
- except Exception as e:
- logger.error(cuda_error_msg)
+ for k, v in hflm.batch_sizes.items():
+ hflm.batch_sizes[k] = max(v // 2, 1)
+ logger.warning(f"Out of memory, reset batch_size to {hflm.batch_sizes} and re-try.")
+ res = lm_eval.simple_evaluate(
+ model=hflm,
+ model_args=None,
+ device=device_str,
+ tasks=task,
+ batch_size=1,
+ limit=limit,
+ num_fewshot=num_fewshot,
+ gen_kwargs=gen_kwargs,
+ task_manager=task_manager,
+ fewshot_as_multiturn=fewshot_as_multiturn,
+ )
+ except torch.OutOfMemoryError as e:
+ last_error = e
traceback.print_exc()
res = None
+ finally:
+ hflm.batch_sizes = ori_batch_sizes
current_retry_times -= 1
if res is None:
- raise RuntimeError(f"Failed to evaluate task '{task}' after {retry_times} attempts")
+ raise RuntimeError(f"Failed to evaluate task '{task}' after {retry_times} attempts") from last_error
if not res_all:
res_all = res
else:
diff --git a/auto_round/eval/evaluation.py b/auto_round/eval/evaluation.py
index 760c3acca..c01e5feb7 100644
--- a/auto_round/eval/evaluation.py
+++ b/auto_round/eval/evaluation.py
@@ -310,6 +310,9 @@ def evaluate_with_model_instance(model, tokenizer, device_str, args):
batch_size=args.eval_bs,
eval_model_dtype=get_model_dtype(args.eval_model_dtype, "auto"),
add_bos_token=args.add_bos_token,
+ num_fewshot=getattr(args, "num_fewshot", None),
+ gen_kwargs=getattr(args, "eval_gen_kwargs", None),
+ fewshot_as_multiturn=getattr(args, "fewshot_as_multiturn", False),
)
else:
# Batch evaluation
@@ -328,6 +331,9 @@ def evaluate_with_model_instance(model, tokenizer, device_str, args):
device=device_str,
eval_model_dtype=get_model_dtype(args.eval_model_dtype, "auto"),
add_bos_token=args.add_bos_token,
+ num_fewshot=getattr(args, "num_fewshot", None),
+ gen_kwargs=getattr(args, "eval_gen_kwargs", None),
+ fewshot_as_multiturn=getattr(args, "fewshot_as_multiturn", False),
)
print(make_table(res))
print("evaluation running time=%ds" % (time.time() - st))
@@ -366,6 +372,9 @@ def evaluate_with_model_path(eval_folder, device_str, autoround, args):
eval_model_dtype=get_model_dtype(args.eval_model_dtype, "auto"),
mllm=getattr(autoround, "mllm", False),
add_bos_token=args.add_bos_token,
+ num_fewshot=getattr(args, "num_fewshot", None),
+ gen_kwargs=getattr(args, "eval_gen_kwargs", None),
+ fewshot_as_multiturn=getattr(args, "fewshot_as_multiturn", False),
)
else:
# Batch evaluation
@@ -398,6 +407,9 @@ def evaluate_with_model_path(eval_folder, device_str, autoround, args):
device=device_str,
batch_size=eval_bs,
limit=args.limit,
+ num_fewshot=getattr(args, "num_fewshot", None),
+ gen_kwargs=getattr(args, "eval_gen_kwargs", None),
+ fewshot_as_multiturn=getattr(args, "fewshot_as_multiturn", False),
)
print(make_table(res))
print("evaluation running time=%ds" % (time.time() - st))
@@ -468,6 +480,9 @@ def run_model_evaluation(model, tokenizer, autoround, folders, formats, args):
vllm_args.disable_trust_remote_code = getattr(args, "disable_trust_remote_code", False)
vllm_args.add_bos_token = getattr(args, "add_bos_token", False)
vllm_args.seed = getattr(args, "seed", 42)
+ vllm_args.num_fewshot = getattr(args, "num_fewshot", None)
+ vllm_args.eval_gen_kwargs = getattr(args, "eval_gen_kwargs", None)
+ vllm_args.fewshot_as_multiturn = getattr(args, "fewshot_as_multiturn", False)
# VLLM-specific parameters
vllm_args.vllm_args = getattr(args, "vllm_args", None)
eval_with_vllm(vllm_args)
diff --git a/docs/step_by_step.md b/docs/step_by_step.md
index f10466382..f5fea9bda 100644
--- a/docs/step_by_step.md
+++ b/docs/step_by_step.md
@@ -1093,6 +1093,7 @@ CUDA_VISIBLE_DEVICES=0,1 auto-round "your_model_path" --eval --tasks lambada_ope
- Use the `--eval` flag to evaluate models directly. This supports both original and quantized models.
- The `--eval_task_by_task` option helps handle task failures by evaluating tasks sequentially. This only applies to the HF backend.
+- Use `--num_fewshot`, `--eval_gen_kwargs`, and `--fewshot_as_multiturn` to pass few-shot and generation options through to lm-eval.
- When multiple formats are exported, the last format in the list will be used for evaluation.
- For vLLM backend, you can use `--device 0,1,2` to specify GPU devices. This will automatically set `CUDA_VISIBLE_DEVICES` and configure `tensor_parallel_size` based on the number of devices. Alternatively, you can manually set these via environment variables and `--vllm_args`.
diff --git a/docs/step_by_step_CN.md b/docs/step_by_step_CN.md
index 77b5eedfa..e553e3b6e 100644
--- a/docs/step_by_step_CN.md
+++ b/docs/step_by_step_CN.md
@@ -1060,6 +1060,7 @@ CUDA_VISIBLE_DEVICES=0,1 auto-round "your_model_path" --eval --tasks lambada_ope
- 对于原始模型和量化后的模型,都支持用 `--eval` 参数直接评估。
- 为应对部分任务运行失败的情况,可使用 `--eval_task_by_task` 参数,按顺序执行评测任务(该参数目前只适用于 HF 后端)。
+- 可使用 `--num_fewshot`、`--eval_gen_kwargs` 和 `--fewshot_as_multiturn` 将 few-shot 与生成参数传递给 lm-eval。
- 若导出了多种格式,会自动选用列表中的**最后一种格式**的模型评估。
- 对于 vLLM 后端,可通过 `--device 0,1,2` 指定 GPU 设备。该参数会自动设置 `CUDA_VISIBLE_DEVICES`,并根据设备数量配置 `tensor_parallel_size` 。此外,也支持通过环境变量和 `--vllm_args` 参数进行手动设置。
diff --git a/test/unit/test_cpu/advanced/test_evaluation_functions.py b/test/unit/test_cpu/advanced/test_evaluation_functions.py
index d6476ec21..e195ada39 100644
--- a/test/unit/test_cpu/advanced/test_evaluation_functions.py
+++ b/test/unit/test_cpu/advanced/test_evaluation_functions.py
@@ -16,14 +16,14 @@
CPU tests for evaluation utility functions.
Lightweight tests focusing on key utility functions without heavy model loading.
-Run with: pytest test/test_cpu/advanced/test_evaluation_functions.py
+Run with: pytest test/unit/test_cpu/advanced/test_evaluation_functions.py
"""
-import os
+import argparse
+import sys
+from types import ModuleType, SimpleNamespace
from unittest.mock import MagicMock, patch
-import pytest
-
class TestSelectGgufEvalFile:
"""Test GGUF text file selection for evaluation."""
@@ -138,11 +138,178 @@ class TestLoadGgufModelIfNeeded:
assert is_gguf is False
assert gguf_file is None
- def test_load_gguf_model_non_string_model(self, tiny_opt_model_path):
+ def test_load_gguf_model_non_string_model(self):
"""Test with model object (not a string path)."""
from auto_round.eval.eval_cli import _load_gguf_model_if_needed
- model, tokenizer, is_gguf, gguf_file = _load_gguf_model_if_needed(tiny_opt_model_path)
+ model_obj = object()
+ model, tokenizer, is_gguf, gguf_file = _load_gguf_model_if_needed(model_obj)
+ assert model is model_obj
assert tokenizer is None
assert is_gguf is False
assert gguf_file is None
+
+
+def _make_eval_args(**overrides):
+ defaults = dict(
+ model="test-model",
+ model_name="test-model",
+ mllm=False,
+ device_map="0",
+ tasks="lambada_openai",
+ disable_trust_remote_code=False,
+ seed=42,
+ eval_bs=2,
+ eval_task_by_task=False,
+ eval_model_dtype=None,
+ limit=5,
+ num_fewshot=3,
+ eval_gen_kwargs="temperature=0.1,top_p=0.9",
+ fewshot_as_multiturn=True,
+ eval_backend="hf",
+ add_bos_token=False,
+ vllm_args=None,
+ )
+ defaults.update(overrides)
+ return argparse.Namespace(**defaults)
+
+
+def _fake_lm_eval_modules(simple_evaluate_impl):
+ root_module = ModuleType("lm_eval")
+ utils_module = ModuleType("lm_eval.utils")
+ utils_module.make_table = lambda result: "table"
+
+ evaluator_module = ModuleType("lm_eval.evaluator")
+ evaluator_module.simple_evaluate = simple_evaluate_impl
+
+ models_module = ModuleType("lm_eval.models")
+ vllm_module = ModuleType("lm_eval.models.vllm_causallms")
+ vllm_module.VLLM = MagicMock(side_effect=lambda **kwargs: SimpleNamespace(kwargs=kwargs))
+
+ vllm_vlm_module = ModuleType("lm_eval.models.vllm_vlms")
+ vllm_vlm_module.VLLM_VLM = MagicMock(side_effect=lambda **kwargs: SimpleNamespace(kwargs=kwargs))
+
+ root_module.evaluator = evaluator_module
+ root_module.utils = utils_module
+ root_module.models = models_module
+ models_module.vllm_causallms = vllm_module
+ models_module.vllm_vlms = vllm_vlm_module
+
+ return {
+ "lm_eval": root_module,
+ "lm_eval.utils": utils_module,
+ "lm_eval.models": models_module,
+ "lm_eval.evaluator": evaluator_module,
+ "lm_eval.models.vllm_causallms": vllm_module,
+ "lm_eval.models.vllm_vlms": vllm_vlm_module,
+ }
+
+
+class TestEvalArgumentForwarding:
+ def test_run_eval_task_by_task_forwards_eval_generation_arguments(self):
+ from auto_round.cli.main import run_eval
+
+ args = _make_eval_args(eval_task_by_task=True)
+
+ with patch("auto_round.cli.main.setup_eval_parser", return_value=args), patch(
+ "auto_round.utils.is_gguf_model", return_value=False
+ ), patch("auto_round.utils.is_mllm_model", return_value=False), patch(
+ "auto_round.eval.eval_cli.eval_task_by_task"
+ ) as mock_eval_task_by_task:
+ run_eval([])
+
+ _, kwargs = mock_eval_task_by_task.call_args
+ assert kwargs["num_fewshot"] == args.num_fewshot
+ assert kwargs["gen_kwargs"] == args.eval_gen_kwargs
+ assert kwargs["fewshot_as_multiturn"] == args.fewshot_as_multiturn
+
+ def test_eval_hf_batch_forwards_eval_generation_arguments(self):
+ from auto_round.eval.eval_cli import eval
+
+ args = _make_eval_args()
+ result = {"results": {"lambada_openai": {}}, "versions": {}, "n-shot": {}, "higher_is_better": {}}
+
+ with patch("auto_round.eval.eval_cli.require_version"), patch(
+ "auto_round.eval.eval_cli.is_diffusion_model", return_value=False
+ ), patch(
+ "auto_round.eval.eval_cli._eval_init", return_value=(["lambada_openai"], "pretrained=test-model", "cpu")
+ ), patch(
+ "auto_round.eval.eval_cli._load_gguf_model_if_needed", return_value=(None, None, False, None)
+ ), patch(
+ "auto_round.eval.evaluation.simple_evaluate", return_value=result
+ ) as mock_simple_evaluate, patch(
+ "builtins.print"
+ ), patch.dict(
+ sys.modules, _fake_lm_eval_modules(MagicMock(return_value=result))
+ ):
+ eval(args)
+
+ _, kwargs = mock_simple_evaluate.call_args
+ assert kwargs["num_fewshot"] == args.num_fewshot
+ assert kwargs["gen_kwargs"] == args.eval_gen_kwargs
+ assert kwargs["fewshot_as_multiturn"] == args.fewshot_as_multiturn
+
+ def test_eval_gguf_batch_forwards_eval_generation_arguments(self):
+ from auto_round.eval.eval_cli import eval
+
+ args = _make_eval_args()
+ result = {"results": {"lambada_openai": {}}, "versions": {}, "n-shot": {}, "higher_is_better": {}}
+
+ with patch("auto_round.eval.eval_cli.require_version"), patch(
+ "auto_round.eval.eval_cli.is_diffusion_model", return_value=False
+ ), patch(
+ "auto_round.eval.eval_cli._eval_init", return_value=(["lambada_openai"], "pretrained=test-model", "cpu")
+ ), patch(
+ "auto_round.eval.eval_cli._load_gguf_model_if_needed",
+ return_value=(object(), object(), True, "model.gguf"),
+ ), patch(
+ "auto_round.eval.evaluation.simple_evaluate_user_model", return_value=result
+ ) as mock_simple_evaluate_user_model, patch(
+ "builtins.print"
+ ), patch.dict(
+ sys.modules, _fake_lm_eval_modules(MagicMock(return_value=result))
+ ):
+ eval(args)
+
+ _, kwargs = mock_simple_evaluate_user_model.call_args
+ assert kwargs["num_fewshot"] == args.num_fewshot
+ assert kwargs["gen_kwargs"] == args.eval_gen_kwargs
+ assert kwargs["fewshot_as_multiturn"] == args.fewshot_as_multiturn
+
+ def test_eval_with_vllm_forwards_eval_generation_arguments(self):
+ from auto_round.eval.eval_cli import eval_with_vllm
+
+ args = _make_eval_args()
+ evaluator_simple_evaluate = MagicMock(
+ return_value={"results": {"lambada_openai": {}}, "versions": {}, "n-shot": {}, "higher_is_better": {}}
+ )
+
+ with patch("auto_round.eval.eval_cli.get_device_and_parallelism", return_value=("cuda:0", False)), patch(
+ "auto_round.eval.eval_cli.get_major_device", return_value="cuda"
+ ), patch("auto_round.eval.eval_cli.get_model_dtype", return_value="float16"), patch(
+ "builtins.print"
+ ), patch.dict(
+ sys.modules, _fake_lm_eval_modules(evaluator_simple_evaluate)
+ ):
+ eval_with_vllm(args)
+
+ _, kwargs = evaluator_simple_evaluate.call_args
+ assert kwargs["num_fewshot"] == args.num_fewshot
+ assert kwargs["gen_kwargs"] == args.eval_gen_kwargs
+ assert kwargs["fewshot_as_multiturn"] == args.fewshot_as_multiturn
+
+ def test_run_model_evaluation_vllm_forwards_eval_generation_arguments(self, tmp_path):
+ from auto_round.eval.evaluation import run_model_evaluation
+
+ args = _make_eval_args(eval_backend="vllm")
+ autoround = SimpleNamespace()
+
+ with patch("auto_round.utils.model.detect_model_type", return_value="llm"), patch(
+ "auto_round.utils.device_manager.get_device_and_parallelism", return_value=("cpu", False)
+ ), patch("auto_round.eval.eval_cli.eval_with_vllm") as mock_eval_with_vllm:
+ run_model_evaluation(None, None, autoround, str(tmp_path), ["auto_round"], args)
+
+ forwarded_args = mock_eval_with_vllm.call_args.args[0]
+ assert forwarded_args.num_fewshot == args.num_fewshot
+ assert forwarded_args.eval_gen_kwargs == args.eval_gen_kwargs
+ assert forwarded_args.fewshot_as_multiturn == args.fewshot_as_multiturn
diff --git a/test/unit/test_cpu/eval/test_eval_cli.py b/test/unit/test_cpu/eval/test_eval_cli.py
index f21a0fd37..c3d678742 100644
--- a/test/unit/test_cpu/eval/test_eval_cli.py
+++ b/test/unit/test_cpu/eval/test_eval_cli.py
@@ -19,6 +19,7 @@ from types import SimpleNamespace
from unittest.mock import MagicMock, patch
import pytest
+import torch
import transformers
from auto_round.eval import eval_cli
@@ -131,6 +132,21 @@ class TestEvalArgumentParser:
args = parser.parse_args(["--vllm_args", "tensor_parallel_size=2,gpu_memory_utilization=0.9"])
assert args.vllm_args == "tensor_parallel_size=2,gpu_memory_utilization=0.9"
+ def test_fewshot_and_generation_args_are_accepted(self):
+ parser = eval_cli.EvalArgumentParser()
+ args = parser.parse_args(
+ [
+ "--num-fewshot",
+ "3",
+ "--eval-gen-kwargs",
+ "temperature=0.1",
+ "--fewshot-as-multiturn",
+ ]
+ )
+ assert args.num_fewshot == 3
+ assert args.eval_gen_kwargs == "temperature=0.1"
+ assert args.fewshot_as_multiturn is True
+
class TestEvalInit:
"""Tests for `_eval_init` task normalization, device resolution, and dtype."""
@@ -584,7 +600,7 @@ class TestEvaluateTasksWithRetry:
def fake_simple_evaluate(**kwargs):
calls.append(kwargs.get("batch_size"))
if len(calls) <= 2 and calls[-1] == 8:
- raise RuntimeError("oom")
+ raise torch.OutOfMemoryError("CUDA out of memory")
return fake_res
monkeypatch.setattr("lm_eval.simple_evaluate", fake_simple_evaluate)
@@ -601,13 +617,19 @@ class TestEvaluateTasksWithRetry:
assert calls == [8, 1]
- def test_exhausted_retries_raises_runtime_error(self, monkeypatch):
+ def test_non_oom_error_is_raised_without_retry(self, monkeypatch):
+ calls = []
+
+ def fake_simple_evaluate(**kwargs):
+ calls.append(kwargs)
+ raise RuntimeError("permanent failure")
+
monkeypatch.setattr(
"lm_eval.simple_evaluate",
- lambda **kwargs: (_ for _ in ()).throw(RuntimeError("permanent failure")),
+ fake_simple_evaluate,
)
- with pytest.raises(RuntimeError, match="Failed to evaluate task 'bad-task'"):
+ with pytest.raises(RuntimeError, match="permanent failure"):
eval_cli._evaluate_tasks_with_retry(
tasks=["bad-task"],
hflm=object(),
@@ -617,6 +639,8 @@ class TestEvaluateTasksWithRetry:
retry_times=2,
)
+ assert len(calls) == 1
+
def test_multiple_tasks_are_aggregated(self, monkeypatch):
fake_hflm = object()
res_a = {