PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
fix: use spawn instead of fork for dataset subprocess on macOS (#1980)
Ramakrishnan Sathyavageeswaran1a7ed4953ec3
diff --git a/auto_round/calib_dataset.py b/auto_round/calib_dataset.pyindex 7d2e3a2f8..e46e95032 100644--- a/auto_round/calib_dataset.py+++ b/auto_round/calib_dataset.py@@ -980,7 +980,11 @@ def get_dataset(tokenizer, seqlen, dataset_name="NeelNanda/pile-10k", seed=42, nif os.name == "nt":raise OSError("fork is not available on Windows")- ctx = multiprocessing.get_context("fork")+ # macOS crashes with SIGSEGV (exit code -11) when using "fork" after threads+ # have been started (PyTorch, tokenizers, and HuggingFace datasets all use+ # threads). Use "spawn" on macOS, which is safe but requires pickling args.+ mp_context = "spawn" if sys.platform == "darwin" else "fork"+ ctx = multiprocessing.get_context(mp_context)p = ctx.Process(target=_get_dataset_impl,args=(tokenizer, seqlen, dataset_name, seed, nsamples),diff --git a/test/test_cpu/core/test_calib_dataset_subprocess.py b/test/test_cpu/core/test_calib_dataset_subprocess.pynew file mode 100644index 000000000..b28a3c5e5--- /dev/null+++ b/test/test_cpu/core/test_calib_dataset_subprocess.py@@ -0,0 +1,83 @@+"""Regression tests for subprocess dataset preprocessing (#1890).++On macOS, ``multiprocessing.get_context("fork")`` triggers SIGSEGV (exit -11)+because PyTorch and tokenizers start threads before the subprocess is spawned.+The fix selects ``"spawn"`` on macOS and ``"fork"`` on Linux.+"""++import os+++class _FakeProcess:+ """Minimal subprocess stub: starts, joins, and exits cleanly."""++ def __init__(self, **kwargs):+ pass++ def start(self):+ pass++ def join(self):+ pass++ exitcode = 0+++def _fake_get_context(captured):+ """Return a factory that records the requested multiprocessing context name."""++ def get_context(method):+ captured["method"] = method++ class _FakeCtx:+ Process = _FakeProcess++ return _FakeCtx()++ return get_context+++def test_mac_uses_spawn_context(monkeypatch):+ """On macOS, get_dataset must request the ``spawn`` multiprocessing context."""+ import auto_round.calib_dataset as cd++ captured = {}+ monkeypatch.setattr(cd.multiprocessing, "get_context", _fake_get_context(captured))+ monkeypatch.setattr(cd.sys, "platform", "darwin")+ monkeypatch.setattr(cd.os, "name", "posix")+ monkeypatch.setattr(cd, "_get_dataset_impl", lambda *a, **kw: None)+ monkeypatch.setattr(cd.envs, "AR_DISABLE_DATASET_SUBPROCESS", False)++ cd.get_dataset(tokenizer=None, seqlen=512)++ assert captured.get("method") == "spawn", f"expected 'spawn' on macOS, got {captured.get('method')!r}"+++def test_linux_uses_fork_context(monkeypatch):+ """On Linux, get_dataset must request the ``fork`` multiprocessing context."""+ import auto_round.calib_dataset as cd++ captured = {}+ monkeypatch.setattr(cd.multiprocessing, "get_context", _fake_get_context(captured))+ monkeypatch.setattr(cd.sys, "platform", "linux")+ monkeypatch.setattr(cd.os, "name", "posix")+ monkeypatch.setattr(cd, "_get_dataset_impl", lambda *a, **kw: None)+ monkeypatch.setattr(cd.envs, "AR_DISABLE_DATASET_SUBPROCESS", False)++ cd.get_dataset(tokenizer=None, seqlen=512)++ assert captured.get("method") == "fork", f"expected 'fork' on Linux, got {captured.get('method')!r}"+++def test_windows_falls_back_to_inprocess(monkeypatch):+ """On Windows (os.name == 'nt'), subprocess is skipped and in-process runs."""+ import auto_round.calib_dataset as cd++ inprocess_called = []+ monkeypatch.setattr(cd.os, "name", "nt")+ monkeypatch.setattr(cd, "_get_dataset_impl", lambda *a, **kw: inprocess_called.append(True))+ monkeypatch.setattr(cd.envs, "AR_DISABLE_DATASET_SUBPROCESS", False)++ cd.get_dataset(tokenizer=None, seqlen=512)++ assert inprocess_called, "in-process fallback should have been called on Windows"