PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
refactor: Simplify vllm evaluation by setting multiprocessing method directly (#2534)
Xin Hee2e3f17f685d
diff --git a/auto_round/eval/eval_cli.py b/auto_round/eval/eval_cli.pyindex 0a08f0ccf..b40c552a3 100644--- a/auto_round/eval/eval_cli.py+++ b/auto_round/eval/eval_cli.py@@ -15,6 +15,7 @@ import argparseimport importlib.utilimport osimport time+from pathlib import Pathimport torchfrom transformers.utils.versions import require_version@@ -266,19 +267,10 @@ def eval(args):def eval_with_vllm(args):- import multiprocessing as mp- import time+ # set vllm worker method+ os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"- if mp.get_start_method(allow_none=True) != "spawn":- try:- mp.set_start_method("spawn", force=True)- except RuntimeError as e:- if mp.get_start_method(allow_none=True) != "spawn":- raise RuntimeError(- f"Failed to set multiprocessing start method to 'spawn' for vLLM evaluation "- f"(current method: '{mp.get_start_method(allow_none=True)}'). "- f"vLLM requires the 'spawn' start method to prevent CUDA re-initialization errors."- ) from e+ import timefrom lm_eval import evaluator # pylint: disable=E0401from lm_eval.models.vllm_causallms import VLLM # pylint: disable=E0401@@ -351,6 +343,7 @@ def eval_with_vllm(args):vllm_lm = VLLM_VLM(**vllm_kwargs)else:vllm_lm = VLLM(**vllm_kwargs)+ task_manager = _get_lm_eval_task_manager(tasks)res = evaluator.simple_evaluate(model=vllm_lm,tasks=tasks,@@ -358,6 +351,7 @@ def eval_with_vllm(args):num_fewshot=getattr(args, "num_fewshot", None),gen_kwargs=getattr(args, "eval_gen_kwargs", None),fewshot_as_multiturn=getattr(args, "fewshot_as_multiturn", False),+ task_manager=task_manager,)print(make_table(res))@@ -515,16 +509,21 @@ def _get_lm_eval_task_manager(tasks: list[str]) -> object | None:except ImportError:return None- tasks_root = os.path.join(os.path.dirname(lm_eval.__file__), "tasks")+ tasks_root = Path(lm_eval.__file__).parent / "tasks"task_paths = []for task in tasks:- task_path = os.path.join(tasks_root, task)- if not os.path.isdir(task_path):+ task_path = tasks_root / task+ if task_path.is_dir():+ task_paths.append(str(task_path))+ continue++ task_configs = list(tasks_root.rglob(f"{task}.yaml"))+ if not task_configs:return None- task_paths.append(task_path)+ task_paths.extend(str(task_config.parent) for task_config in task_configs)try:- return TaskManager(include_defaults=False, include_path=task_paths)+ return TaskManager(include_defaults=False, include_path=list(dict.fromkeys(task_paths)))except TypeError:return Nonediff --git a/test/unit/common/eval/test_eval_cli.py b/test/unit/common/eval/test_eval_cli.pyindex ffa8bd275..54e3dfa70 100644--- a/test/unit/common/eval/test_eval_cli.py+++ b/test/unit/common/eval/test_eval_cli.py@@ -13,7 +13,6 @@# limitations under the License."""CPU-only pytest coverage for `auto_round.eval.eval_cli`."""-import multiprocessing as mpimport osimport sysfrom types import SimpleNamespace@@ -355,6 +354,8 @@ class TestEvalWithVllm:)captured_kwargs = {}+ captured_evaluate_kwargs = {}+ task_manager = object()fake_res = {"results": {"mmlu": {}}, "versions": {}, "n-shot": {}, "higher_is_better": {}}class FakeVLLM:@@ -372,10 +373,13 @@ class TestEvalWithVllm:monkeypatch.setattr(eval_cli, "get_major_device", lambda: "cuda")monkeypatch.setattr(eval_cli, "get_device_and_parallelism", lambda device: ("cuda", False))monkeypatch.setattr(eval_cli, "get_model_dtype", lambda dtype, default="auto": "auto")- monkeypatch.setattr(- "lm_eval.evaluator.simple_evaluate",- lambda **kwargs: fake_res,- )+ monkeypatch.setattr(eval_cli, "_get_lm_eval_task_manager", lambda tasks: task_manager)++ def fake_simple_evaluate(**kwargs):+ captured_evaluate_kwargs.update(kwargs)+ return fake_res++ monkeypatch.setattr("lm_eval.evaluator.simple_evaluate", fake_simple_evaluate)monkeypatch.setenv("CUDA_VISIBLE_DEVICES", "")monkeypatch.setenv("TOKENIZERS_PARALLELISM", "false")@@ -384,6 +388,32 @@ class TestEvalWithVllm:assert captured_kwargs["pretrained"] == "model-id"assert captured_kwargs["tensor_parallel_size"] == 2+ assert captured_evaluate_kwargs["task_manager"] is task_manager++ def test_task_manager_finds_nested_yaml_task(self, monkeypatch, tmp_path):+ tasks_root = tmp_path / "lm_eval" / "tasks"+ task_dir = tasks_root / "lambada"+ task_dir.mkdir(parents=True)+ (task_dir / "lambada_openai.yaml").write_text("task: lambada_openai\n")++ fake_lm_eval = type(sys)("lm_eval")+ fake_lm_eval.__file__ = str(tmp_path / "lm_eval" / "__init__.py")+ captured = {}++ class FakeTaskManager:+ def __init__(self, **kwargs):+ captured.update(kwargs)++ fake_tasks = type(sys)("lm_eval.tasks")+ fake_tasks.TaskManager = FakeTaskManager+ monkeypatch.setitem(sys.modules, "lm_eval", fake_lm_eval)+ monkeypatch.setitem(sys.modules, "lm_eval.tasks", fake_tasks)++ task_manager = eval_cli._get_lm_eval_task_manager(["lambada_openai"])++ assert isinstance(task_manager, FakeTaskManager)+ assert captured["include_defaults"] is False+ assert captured["include_path"] == [str(task_dir)]def test_mllm_uses_vllm_vlm(self, monkeypatch):args = SimpleNamespace(@@ -463,123 +493,6 @@ class TestEvalWithVllm:assert os.environ.get("CUDA_VISIBLE_DEVICES") == "fake-env"- def test_eval_with_vllm_configures_spawn_start_method(self, monkeypatch):- called_set = []-- def mock_get_start_method(allow_none=True):- return "fork"-- def mock_set_start_method(method, force=False):- called_set.append((method, force))-- monkeypatch.setattr(mp, "get_start_method", mock_get_start_method)- monkeypatch.setattr(mp, "set_start_method", mock_set_start_method)-- fake_vllm_causallms = type(sys)("lm_eval.models.vllm_causallms")- fake_vllm_causallms.VLLM = type("VLLM", (), {"__init__": lambda *args, **kwargs: None})- monkeypatch.setitem(sys.modules, "lm_eval.models.vllm_causallms", fake_vllm_causallms)-- monkeypatch.setattr(eval_cli, "get_major_device", lambda: "cuda")- monkeypatch.setattr(eval_cli, "get_device_and_parallelism", lambda device: ("cuda", False))- monkeypatch.setattr(eval_cli, "get_model_dtype", lambda dtype, default="auto": "auto")- monkeypatch.setattr(- "lm_eval.evaluator.simple_evaluate",- lambda **kwargs: {"results": {"mmlu": {}}, "versions": {}, "n-shot": {}, "higher_is_better": {}},- )-- args = SimpleNamespace(- model_name="mock-model",- tasks="mmlu",- device_map="0",- eval_model_dtype="auto",- eval_bs=None,- disable_trust_remote_code=True,- add_bos_token=False,- mllm=False,- limit=1,- vllm_args=None,- )-- with patch("auto_round.utils.DEVICE_ENVIRON_VARIABLE_MAPPING", {"cuda": "CUDA_VISIBLE_DEVICES"}):- eval_cli.eval_with_vllm(args)-- assert ("spawn", True) in called_set-- def test_eval_with_vllm_already_spawn_noop(self, monkeypatch):- called_set = []-- def mock_get_start_method(allow_none=True):- return "spawn"-- def mock_set_start_method(method, force=False):- called_set.append((method, force))-- monkeypatch.setattr(mp, "get_start_method", mock_get_start_method)- monkeypatch.setattr(mp, "set_start_method", mock_set_start_method)-- fake_vllm_causallms = type(sys)("lm_eval.models.vllm_causallms")- fake_vllm_causallms.VLLM = type("VLLM", (), {"__init__": lambda *args, **kwargs: None})- monkeypatch.setitem(sys.modules, "lm_eval.models.vllm_causallms", fake_vllm_causallms)-- monkeypatch.setattr(eval_cli, "get_major_device", lambda: "cuda")- monkeypatch.setattr(eval_cli, "get_device_and_parallelism", lambda device: ("cuda", False))- monkeypatch.setattr(eval_cli, "get_model_dtype", lambda dtype, default="auto": "auto")- monkeypatch.setattr(- "lm_eval.evaluator.simple_evaluate",- lambda **kwargs: {"results": {"mmlu": {}}, "versions": {}, "n-shot": {}, "higher_is_better": {}},- )-- args = SimpleNamespace(- model_name="mock-model",- tasks="mmlu",- device_map="0",- eval_model_dtype="auto",- eval_bs=None,- disable_trust_remote_code=True,- add_bos_token=False,- mllm=False,- limit=1,- vllm_args=None,- )-- with patch("auto_round.utils.DEVICE_ENVIRON_VARIABLE_MAPPING", {"cuda": "CUDA_VISIBLE_DEVICES"}):- eval_cli.eval_with_vllm(args)-- assert called_set == []-- def test_eval_with_vllm_spawn_failure_raises(self, monkeypatch):- def mock_get_start_method(allow_none=True):- return "fork"-- def mock_set_start_method(method, force=False):- raise RuntimeError("context has already been set")-- monkeypatch.setattr(mp, "get_start_method", mock_get_start_method)- monkeypatch.setattr(mp, "set_start_method", mock_set_start_method)-- monkeypatch.setattr(eval_cli, "get_major_device", lambda: "cuda")- monkeypatch.setattr(eval_cli, "get_device_and_parallelism", lambda device: ("cuda", False))- monkeypatch.setattr(eval_cli, "get_model_dtype", lambda dtype, default="auto": "auto")-- args = SimpleNamespace(- model_name="mock-model",- tasks="mmlu",- device_map="0",- eval_model_dtype="auto",- eval_bs=None,- disable_trust_remote_code=True,- add_bos_token=False,- mllm=False,- limit=1,- vllm_args=None,- )-- with (- patch("auto_round.utils.DEVICE_ENVIRON_VARIABLE_MAPPING", {"cuda": "CUDA_VISIBLE_DEVICES"}),- pytest.raises(RuntimeError, match="Failed to set multiprocessing start method to 'spawn'"),- ):- eval_cli.eval_with_vllm(args)-class TestEvalTaskByTask:"""Tests for `eval_task_by_task`, focusing on CPU-safe branches."""