Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

refactor: Simplify vllm evaluation by setting multiprocessing method directly (#2534)

Xin Hee2e3f17f685d
diff --git a/auto_round/eval/eval_cli.py b/auto_round/eval/eval_cli.py
index 0a08f0ccf..b40c552a3 100644
--- a/auto_round/eval/eval_cli.py
+++ b/auto_round/eval/eval_cli.py
@@ -15,6 +15,7 @@ import argparse
import importlib.util
import os
import time
+from pathlib import Path
import torch
from transformers.utils.versions import require_version
@@ -266,19 +267,10 @@ def eval(args):
def eval_with_vllm(args):
- import multiprocessing as mp
- import time
+ # set vllm worker method
+ os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
- if mp.get_start_method(allow_none=True) != "spawn":
- try:
- mp.set_start_method("spawn", force=True)
- except RuntimeError as e:
- if mp.get_start_method(allow_none=True) != "spawn":
- raise RuntimeError(
- f"Failed to set multiprocessing start method to 'spawn' for vLLM evaluation "
- f"(current method: '{mp.get_start_method(allow_none=True)}'). "
- f"vLLM requires the 'spawn' start method to prevent CUDA re-initialization errors."
- ) from e
+ import time
from lm_eval import evaluator # pylint: disable=E0401
from lm_eval.models.vllm_causallms import VLLM # pylint: disable=E0401
@@ -351,6 +343,7 @@ def eval_with_vllm(args):
vllm_lm = VLLM_VLM(**vllm_kwargs)
else:
vllm_lm = VLLM(**vllm_kwargs)
+ task_manager = _get_lm_eval_task_manager(tasks)
res = evaluator.simple_evaluate(
model=vllm_lm,
tasks=tasks,
@@ -358,6 +351,7 @@ def eval_with_vllm(args):
num_fewshot=getattr(args, "num_fewshot", None),
gen_kwargs=getattr(args, "eval_gen_kwargs", None),
fewshot_as_multiturn=getattr(args, "fewshot_as_multiturn", False),
+ task_manager=task_manager,
)
print(make_table(res))
@@ -515,16 +509,21 @@ def _get_lm_eval_task_manager(tasks: list[str]) -> object | None:
except ImportError:
return None
- tasks_root = os.path.join(os.path.dirname(lm_eval.__file__), "tasks")
+ tasks_root = Path(lm_eval.__file__).parent / "tasks"
task_paths = []
for task in tasks:
- task_path = os.path.join(tasks_root, task)
- if not os.path.isdir(task_path):
+ task_path = tasks_root / task
+ if task_path.is_dir():
+ task_paths.append(str(task_path))
+ continue
+
+ task_configs = list(tasks_root.rglob(f"{task}.yaml"))
+ if not task_configs:
return None
- task_paths.append(task_path)
+ task_paths.extend(str(task_config.parent) for task_config in task_configs)
try:
- return TaskManager(include_defaults=False, include_path=task_paths)
+ return TaskManager(include_defaults=False, include_path=list(dict.fromkeys(task_paths)))
except TypeError:
return None
diff --git a/test/unit/common/eval/test_eval_cli.py b/test/unit/common/eval/test_eval_cli.py
index ffa8bd275..54e3dfa70 100644
--- a/test/unit/common/eval/test_eval_cli.py
+++ b/test/unit/common/eval/test_eval_cli.py
@@ -13,7 +13,6 @@
# limitations under the License.
"""CPU-only pytest coverage for `auto_round.eval.eval_cli`."""
-import multiprocessing as mp
import os
import sys
from types import SimpleNamespace
@@ -355,6 +354,8 @@ class TestEvalWithVllm:
)
captured_kwargs = {}
+ captured_evaluate_kwargs = {}
+ task_manager = object()
fake_res = {"results": {"mmlu": {}}, "versions": {}, "n-shot": {}, "higher_is_better": {}}
class FakeVLLM:
@@ -372,10 +373,13 @@ class TestEvalWithVllm:
monkeypatch.setattr(eval_cli, "get_major_device", lambda: "cuda")
monkeypatch.setattr(eval_cli, "get_device_and_parallelism", lambda device: ("cuda", False))
monkeypatch.setattr(eval_cli, "get_model_dtype", lambda dtype, default="auto": "auto")
- monkeypatch.setattr(
- "lm_eval.evaluator.simple_evaluate",
- lambda **kwargs: fake_res,
- )
+ monkeypatch.setattr(eval_cli, "_get_lm_eval_task_manager", lambda tasks: task_manager)
+
+ def fake_simple_evaluate(**kwargs):
+ captured_evaluate_kwargs.update(kwargs)
+ return fake_res
+
+ monkeypatch.setattr("lm_eval.evaluator.simple_evaluate", fake_simple_evaluate)
monkeypatch.setenv("CUDA_VISIBLE_DEVICES", "")
monkeypatch.setenv("TOKENIZERS_PARALLELISM", "false")
@@ -384,6 +388,32 @@ class TestEvalWithVllm:
assert captured_kwargs["pretrained"] == "model-id"
assert captured_kwargs["tensor_parallel_size"] == 2
+ assert captured_evaluate_kwargs["task_manager"] is task_manager
+
+ def test_task_manager_finds_nested_yaml_task(self, monkeypatch, tmp_path):
+ tasks_root = tmp_path / "lm_eval" / "tasks"
+ task_dir = tasks_root / "lambada"
+ task_dir.mkdir(parents=True)
+ (task_dir / "lambada_openai.yaml").write_text("task: lambada_openai\n")
+
+ fake_lm_eval = type(sys)("lm_eval")
+ fake_lm_eval.__file__ = str(tmp_path / "lm_eval" / "__init__.py")
+ captured = {}
+
+ class FakeTaskManager:
+ def __init__(self, **kwargs):
+ captured.update(kwargs)
+
+ fake_tasks = type(sys)("lm_eval.tasks")
+ fake_tasks.TaskManager = FakeTaskManager
+ monkeypatch.setitem(sys.modules, "lm_eval", fake_lm_eval)
+ monkeypatch.setitem(sys.modules, "lm_eval.tasks", fake_tasks)
+
+ task_manager = eval_cli._get_lm_eval_task_manager(["lambada_openai"])
+
+ assert isinstance(task_manager, FakeTaskManager)
+ assert captured["include_defaults"] is False
+ assert captured["include_path"] == [str(task_dir)]
def test_mllm_uses_vllm_vlm(self, monkeypatch):
args = SimpleNamespace(
@@ -463,123 +493,6 @@ class TestEvalWithVllm:
assert os.environ.get("CUDA_VISIBLE_DEVICES") == "fake-env"
- def test_eval_with_vllm_configures_spawn_start_method(self, monkeypatch):
- called_set = []
-
- def mock_get_start_method(allow_none=True):
- return "fork"
-
- def mock_set_start_method(method, force=False):
- called_set.append((method, force))
-
- monkeypatch.setattr(mp, "get_start_method", mock_get_start_method)
- monkeypatch.setattr(mp, "set_start_method", mock_set_start_method)
-
- fake_vllm_causallms = type(sys)("lm_eval.models.vllm_causallms")
- fake_vllm_causallms.VLLM = type("VLLM", (), {"__init__": lambda *args, **kwargs: None})
- monkeypatch.setitem(sys.modules, "lm_eval.models.vllm_causallms", fake_vllm_causallms)
-
- monkeypatch.setattr(eval_cli, "get_major_device", lambda: "cuda")
- monkeypatch.setattr(eval_cli, "get_device_and_parallelism", lambda device: ("cuda", False))
- monkeypatch.setattr(eval_cli, "get_model_dtype", lambda dtype, default="auto": "auto")
- monkeypatch.setattr(
- "lm_eval.evaluator.simple_evaluate",
- lambda **kwargs: {"results": {"mmlu": {}}, "versions": {}, "n-shot": {}, "higher_is_better": {}},
- )
-
- args = SimpleNamespace(
- model_name="mock-model",
- tasks="mmlu",
- device_map="0",
- eval_model_dtype="auto",
- eval_bs=None,
- disable_trust_remote_code=True,
- add_bos_token=False,
- mllm=False,
- limit=1,
- vllm_args=None,
- )
-
- with patch("auto_round.utils.DEVICE_ENVIRON_VARIABLE_MAPPING", {"cuda": "CUDA_VISIBLE_DEVICES"}):
- eval_cli.eval_with_vllm(args)
-
- assert ("spawn", True) in called_set
-
- def test_eval_with_vllm_already_spawn_noop(self, monkeypatch):
- called_set = []
-
- def mock_get_start_method(allow_none=True):
- return "spawn"
-
- def mock_set_start_method(method, force=False):
- called_set.append((method, force))
-
- monkeypatch.setattr(mp, "get_start_method", mock_get_start_method)
- monkeypatch.setattr(mp, "set_start_method", mock_set_start_method)
-
- fake_vllm_causallms = type(sys)("lm_eval.models.vllm_causallms")
- fake_vllm_causallms.VLLM = type("VLLM", (), {"__init__": lambda *args, **kwargs: None})
- monkeypatch.setitem(sys.modules, "lm_eval.models.vllm_causallms", fake_vllm_causallms)
-
- monkeypatch.setattr(eval_cli, "get_major_device", lambda: "cuda")
- monkeypatch.setattr(eval_cli, "get_device_and_parallelism", lambda device: ("cuda", False))
- monkeypatch.setattr(eval_cli, "get_model_dtype", lambda dtype, default="auto": "auto")
- monkeypatch.setattr(
- "lm_eval.evaluator.simple_evaluate",
- lambda **kwargs: {"results": {"mmlu": {}}, "versions": {}, "n-shot": {}, "higher_is_better": {}},
- )
-
- args = SimpleNamespace(
- model_name="mock-model",
- tasks="mmlu",
- device_map="0",
- eval_model_dtype="auto",
- eval_bs=None,
- disable_trust_remote_code=True,
- add_bos_token=False,
- mllm=False,
- limit=1,
- vllm_args=None,
- )
-
- with patch("auto_round.utils.DEVICE_ENVIRON_VARIABLE_MAPPING", {"cuda": "CUDA_VISIBLE_DEVICES"}):
- eval_cli.eval_with_vllm(args)
-
- assert called_set == []
-
- def test_eval_with_vllm_spawn_failure_raises(self, monkeypatch):
- def mock_get_start_method(allow_none=True):
- return "fork"
-
- def mock_set_start_method(method, force=False):
- raise RuntimeError("context has already been set")
-
- monkeypatch.setattr(mp, "get_start_method", mock_get_start_method)
- monkeypatch.setattr(mp, "set_start_method", mock_set_start_method)
-
- monkeypatch.setattr(eval_cli, "get_major_device", lambda: "cuda")
- monkeypatch.setattr(eval_cli, "get_device_and_parallelism", lambda device: ("cuda", False))
- monkeypatch.setattr(eval_cli, "get_model_dtype", lambda dtype, default="auto": "auto")
-
- args = SimpleNamespace(
- model_name="mock-model",
- tasks="mmlu",
- device_map="0",
- eval_model_dtype="auto",
- eval_bs=None,
- disable_trust_remote_code=True,
- add_bos_token=False,
- mllm=False,
- limit=1,
- vllm_args=None,
- )
-
- with (
- patch("auto_round.utils.DEVICE_ENVIRON_VARIABLE_MAPPING", {"cuda": "CUDA_VISIBLE_DEVICES"}),
- pytest.raises(RuntimeError, match="Failed to set multiprocessing start method to 'spawn'"),
- ):
- eval_cli.eval_with_vllm(args)
-
class TestEvalTaskByTask:
"""Tests for `eval_task_by_task`, focusing on CPU-safe branches."""