Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

fix: use spawn start method for vLLM evaluation to avoid CUDA re-initialization error (#2421)

Reginald Alfret V234ac7af40c3
diff --git a/auto_round/eval/eval_cli.py b/auto_round/eval/eval_cli.py
index 4433e6c29..0a08f0ccf 100644
--- a/auto_round/eval/eval_cli.py
+++ b/auto_round/eval/eval_cli.py
@@ -266,8 +266,20 @@ def eval(args):
def eval_with_vllm(args):
+ import multiprocessing as mp
import time
+ if mp.get_start_method(allow_none=True) != "spawn":
+ try:
+ mp.set_start_method("spawn", force=True)
+ except RuntimeError as e:
+ if mp.get_start_method(allow_none=True) != "spawn":
+ raise RuntimeError(
+ f"Failed to set multiprocessing start method to 'spawn' for vLLM evaluation "
+ f"(current method: '{mp.get_start_method(allow_none=True)}'). "
+ f"vLLM requires the 'spawn' start method to prevent CUDA re-initialization errors."
+ ) from e
+
from lm_eval import evaluator # pylint: disable=E0401
from lm_eval.models.vllm_causallms import VLLM # pylint: disable=E0401
from lm_eval.utils import make_table # pylint: disable=E0401
diff --git a/test/unit/common/eval/test_eval_cli.py b/test/unit/common/eval/test_eval_cli.py
index 3cc4a51f8..35c58892f 100644
--- a/test/unit/common/eval/test_eval_cli.py
+++ b/test/unit/common/eval/test_eval_cli.py
@@ -13,6 +13,7 @@
# limitations under the License.
"""CPU-only pytest coverage for `auto_round.eval.eval_cli`."""
+import multiprocessing as mp
import os
import sys
from types import SimpleNamespace
@@ -462,6 +463,121 @@ class TestEvalWithVllm:
assert os.environ.get("CUDA_VISIBLE_DEVICES") == "fake-env"
+ def test_eval_with_vllm_configures_spawn_start_method(self, monkeypatch):
+ called_set = []
+
+ def mock_get_start_method(allow_none=True):
+ return "fork"
+
+ def mock_set_start_method(method, force=False):
+ called_set.append((method, force))
+
+ monkeypatch.setattr(mp, "get_start_method", mock_get_start_method)
+ monkeypatch.setattr(mp, "set_start_method", mock_set_start_method)
+
+ fake_vllm_causallms = type(sys)("lm_eval.models.vllm_causallms")
+ fake_vllm_causallms.VLLM = type("VLLM", (), {"__init__": lambda *args, **kwargs: None})
+ monkeypatch.setitem(sys.modules, "lm_eval.models.vllm_causallms", fake_vllm_causallms)
+
+ monkeypatch.setattr(eval_cli, "get_major_device", lambda: "cuda")
+ monkeypatch.setattr(eval_cli, "get_device_and_parallelism", lambda device: ("cuda", False))
+ monkeypatch.setattr(eval_cli, "get_model_dtype", lambda dtype, default="auto": "auto")
+ monkeypatch.setattr(
+ "lm_eval.evaluator.simple_evaluate",
+ lambda **kwargs: {"results": {"mmlu": {}}, "versions": {}, "n-shot": {}, "higher_is_better": {}},
+ )
+
+ args = SimpleNamespace(
+ model_name="mock-model",
+ tasks="mmlu",
+ device_map="0",
+ eval_model_dtype="auto",
+ eval_bs=None,
+ disable_trust_remote_code=True,
+ add_bos_token=False,
+ mllm=False,
+ limit=1,
+ vllm_args=None,
+ )
+
+ with patch("auto_round.utils.DEVICE_ENVIRON_VARIABLE_MAPPING", {"cuda": "CUDA_VISIBLE_DEVICES"}):
+ eval_cli.eval_with_vllm(args)
+
+ assert ("spawn", True) in called_set
+
+ def test_eval_with_vllm_already_spawn_noop(self, monkeypatch):
+ called_set = []
+
+ def mock_get_start_method(allow_none=True):
+ return "spawn"
+
+ def mock_set_start_method(method, force=False):
+ called_set.append((method, force))
+
+ monkeypatch.setattr(mp, "get_start_method", mock_get_start_method)
+ monkeypatch.setattr(mp, "set_start_method", mock_set_start_method)
+
+ fake_vllm_causallms = type(sys)("lm_eval.models.vllm_causallms")
+ fake_vllm_causallms.VLLM = type("VLLM", (), {"__init__": lambda *args, **kwargs: None})
+ monkeypatch.setitem(sys.modules, "lm_eval.models.vllm_causallms", fake_vllm_causallms)
+
+ monkeypatch.setattr(eval_cli, "get_major_device", lambda: "cuda")
+ monkeypatch.setattr(eval_cli, "get_device_and_parallelism", lambda device: ("cuda", False))
+ monkeypatch.setattr(eval_cli, "get_model_dtype", lambda dtype, default="auto": "auto")
+ monkeypatch.setattr(
+ "lm_eval.evaluator.simple_evaluate",
+ lambda **kwargs: {"results": {"mmlu": {}}, "versions": {}, "n-shot": {}, "higher_is_better": {}},
+ )
+
+ args = SimpleNamespace(
+ model_name="mock-model",
+ tasks="mmlu",
+ device_map="0",
+ eval_model_dtype="auto",
+ eval_bs=None,
+ disable_trust_remote_code=True,
+ add_bos_token=False,
+ mllm=False,
+ limit=1,
+ vllm_args=None,
+ )
+
+ with patch("auto_round.utils.DEVICE_ENVIRON_VARIABLE_MAPPING", {"cuda": "CUDA_VISIBLE_DEVICES"}):
+ eval_cli.eval_with_vllm(args)
+
+ assert called_set == []
+
+ def test_eval_with_vllm_spawn_failure_raises(self, monkeypatch):
+ def mock_get_start_method(allow_none=True):
+ return "fork"
+
+ def mock_set_start_method(method, force=False):
+ raise RuntimeError("context has already been set")
+
+ monkeypatch.setattr(mp, "get_start_method", mock_get_start_method)
+ monkeypatch.setattr(mp, "set_start_method", mock_set_start_method)
+
+ monkeypatch.setattr(eval_cli, "get_major_device", lambda: "cuda")
+ monkeypatch.setattr(eval_cli, "get_device_and_parallelism", lambda device: ("cuda", False))
+ monkeypatch.setattr(eval_cli, "get_model_dtype", lambda dtype, default="auto": "auto")
+
+ args = SimpleNamespace(
+ model_name="mock-model",
+ tasks="mmlu",
+ device_map="0",
+ eval_model_dtype="auto",
+ eval_bs=None,
+ disable_trust_remote_code=True,
+ add_bos_token=False,
+ mllm=False,
+ limit=1,
+ vllm_args=None,
+ )
+
+ with patch("auto_round.utils.DEVICE_ENVIRON_VARIABLE_MAPPING", {"cuda": "CUDA_VISIBLE_DEVICES"}):
+ with pytest.raises(RuntimeError, match="Failed to set multiprocessing start method to 'spawn'"):
+ eval_cli.eval_with_vllm(args)
+
class TestEvalTaskByTask:
"""Tests for `eval_task_by_task`, focusing on CPU-safe branches."""