Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
tests: give each xdist worker one CPU share for torch (#13494)
Daniel Handc1860c40e36
diff --git a/tests/_shared/xdist_cpu_share.py b/tests/_shared/xdist_cpu_share.pynew file mode 100644index 00000000000..64bcc887f81--- /dev/null+++ b/tests/_shared/xdist_cpu_share.py@@ -0,0 +1,123 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""One CPU share per pytest-xdist worker for the torch / OpenMP / BLAS thread pools.++Each xdist worker otherwise starts a pool as wide as the machine, so `-n 4` on a 4 vCPU+runner runs 16 spinning threads on 4 cores and CPU training tests slow down by an order+of magnitude. Loaded by tests/conftest.py before torch is imported. Stdlib only.+"""++from __future__ import annotations++import os+from typing import MutableMapping, Optional++THREAD_VARS = ("OMP_NUM_THREADS", "MKL_NUM_THREADS", "OPENBLAS_NUM_THREADS")+# Set by apply(): the share it gave each worker, or "explicit" when the caller had already+# chosen a thread count. tests/test_xdist_torch_thread_cap.py reads it.+MARKER = "UNSLOTH_TEST_XDIST_CPU_SHARE"+CGROUP_ROOT = "/sys/fs/cgroup"+++def _first_line(path: str) -> Optional[str]:+ try:+ with open(path, "r", encoding = "utf-8") as f:+ return f.readline().strip()+ except OSError:+ return None+++def _dirs(root: str, rel: Optional[str]) -> list:+ """``root/rel`` and every ancestor up to *root*, innermost first: a parent's limit binds too."""+ dirs = []+ if rel and rel != "/":+ current = os.path.normpath(os.path.join(root, rel.lstrip("/")))+ while current.startswith(root + os.sep):+ dirs.append(current)+ current = os.path.dirname(current)+ dirs.append(root)+ return dirs+++def _ratio(quota: Optional[str], period: Optional[str]) -> Optional[float]:+ try:+ q, p = int(quota), int(period)+ except (TypeError, ValueError):+ return None # "max", missing or unreadable means unlimited+ return q / p if q > 0 and p > 0 else None+++def cgroup_cpu_quota(+ root: str = CGROUP_ROOT, proc_cgroup: str = "/proc/self/cgroup"+) -> Optional[float]:+ """The tightest CFS CPU quota over this process's cgroup and its ancestors, in cores.++ None when no quota applies. sched_getaffinity() does not see a quota, so a container+ limited to 4 CPUs on a 64 CPU host would otherwise hand each of 4 workers 16 threads.+ """+ try:+ with open(proc_cgroup, "r", encoding = "utf-8") as f:+ lines = [line.strip() for line in f if line.strip()]+ except OSError:+ lines = []+ quotas = []+ if os.path.isfile(os.path.join(root, "cgroup.controllers")): # v2+ rel = next((line[3:] for line in lines if line.startswith("0::")), None)+ for directory in _dirs(root, rel):+ raw = (_first_line(os.path.join(directory, "cpu.max")) or "").split()+ if len(raw) == 2:+ quotas.append(_ratio(raw[0], raw[1]))+ else: # v1+ rel = None+ for line in lines:+ parts = line.split(":", 2)+ if len(parts) == 3 and "cpu" in parts[1].split(","):+ rel = parts[2]+ break+ for mount in ("cpu", "cpu,cpuacct", "cpuacct,cpu"):+ base = os.path.join(root, mount)+ if not os.path.isdir(base):+ continue+ for directory in _dirs(base, rel):+ quotas.append(+ _ratio(+ _first_line(os.path.join(directory, "cpu.cfs_quota_us")),+ _first_line(os.path.join(directory, "cpu.cfs_period_us")),+ )+ )+ break+ quotas = [q for q in quotas if q is not None]+ return min(quotas) if quotas else None+++def usable_cpus() -> int:+ """CPUs this process may actually use: the affinity mask, capped by any cgroup quota."""+ try:+ cpus = len(os.sched_getaffinity(0))+ except (AttributeError, OSError):+ cpus = os.cpu_count() or 1+ quota = cgroup_cpu_quota()+ if quota is not None:+ cpus = min(cpus, max(1, int(quota)))+ return max(1, cpus)+++def apply(environ: MutableMapping[str, str], cpus: Optional[int] = None) -> Optional[str]:+ """Give each xdist worker ``cpus // workers`` threads. Returns what it recorded in MARKER.++ Leaves every variable alone when any of them is already set: MKL_NUM_THREADS takes+ precedence over OMP_NUM_THREADS in torch, so filling in the others beside an explicit+ OMP_NUM_THREADS would override the caller's choice.+ """+ workers = int(environ.get("PYTEST_XDIST_WORKER_COUNT") or 0)+ if workers <= 1:+ return None+ if any(environ.get(var) for var in THREAD_VARS):+ environ[MARKER] = "explicit"+ return "explicit"+ share = str(max(1, (cpus if cpus is not None else usable_cpus()) // workers))+ for var in THREAD_VARS:+ environ[var] = share+ environ[MARKER] = share+ return sharediff --git a/tests/conftest.py b/tests/conftest.pyindex e6d4f84fba2..356d6dd5e6d 100644--- a/tests/conftest.py+++ b/tests/conftest.py@@ -28,6 +28,21 @@ for _up in _iso.parents:_spec.loader.exec_module(_mod) # sets the env vars on importbreak+# --- one CPU share per xdist worker -----------------------------------------------+# Also before torch is imported. Each xdist worker otherwise starts a torch / OpenMP pool+# as wide as the machine, so `-n 4` on a 4 vCPU runner runs 16 spinning threads on 4 cores+# and CPU training tests slow down by an order of magnitude: test_decision_model's Clef+# training took >330 s there against ~50 s alone. An explicit thread setting is kept.+# Loaded by path: tests/_shared is put on sys.path only below.+import os as _os # noqa: E402++_share_path = _iso.parent / "_shared" / "xdist_cpu_share.py"+if _share_path.is_file():+ _spec = _ilu.spec_from_file_location("xdist_cpu_share", _share_path)+ _mod = _ilu.module_from_spec(_spec)+ _spec.loader.exec_module(_mod)+ _mod.apply(_os.environ)+# --- shared test helpers on sys.path -----------------------------------------------# tests/_shared holds no package marker and pytest only puts a *test file's* own# directory on sys.path, so tests/python/, tests/studio/install/ and tests/security/diff --git a/tests/test_xdist_torch_thread_cap.py b/tests/test_xdist_torch_thread_cap.pynew file mode 100644index 00000000000..4f117b0d47b--- /dev/null+++ b/tests/test_xdist_torch_thread_cap.py@@ -0,0 +1,106 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""tests/conftest.py gives each xdist worker one share of the CPUs for torch.++Without it every worker's torch pool is as wide as the machine, and CPU training tests+thrash each other on CI's 4 vCPU runners (test_decision_model's Clef training went past+the 330 s timeout). The end-to-end check only means something under xdist; the rules it+relies on (explicit settings win, cgroup quotas count) are checked directly below.+"""++import importlib.util+import os+from pathlib import Path++import pytest++_spec = importlib.util.spec_from_file_location(+ "xdist_cpu_share_under_test", Path(__file__).resolve().parent / "_shared" / "xdist_cpu_share.py"+)+share_mod = importlib.util.module_from_spec(_spec)+_spec.loader.exec_module(share_mod)+++def test_each_xdist_worker_gets_one_cpu_share():+ workers = int(os.environ.get("PYTEST_XDIST_WORKER_COUNT") or 0)+ if workers <= 1:+ pytest.skip("not running under pytest-xdist")+ recorded = os.environ.get(share_mod.MARKER)+ assert recorded, "conftest did not apply a per-worker CPU share"+ if recorded == "explicit":+ pytest.skip("the caller set a thread count; conftest left it alone")+ import torch++ assert recorded == str(max(1, share_mod.usable_cpus() // workers))+ for var in share_mod.THREAD_VARS:+ assert os.environ[var] == recorded, var+ assert torch.get_num_threads() == int(recorded)+++def test_share_divides_the_cpus_between_workers():+ env = {"PYTEST_XDIST_WORKER_COUNT": "4"}+ assert share_mod.apply(env, cpus = 16) == "4"+ assert all(env[var] == "4" for var in share_mod.THREAD_VARS)+ env = {"PYTEST_XDIST_WORKER_COUNT": "8"}+ assert share_mod.apply(env, cpus = 4) == "1"+++def test_serial_runs_are_left_alone():+ for env in ({}, {"PYTEST_XDIST_WORKER_COUNT": "1"}):+ assert share_mod.apply(env, cpus = 16) is None+ assert not any(var in env for var in share_mod.THREAD_VARS)++[email protected]("var", share_mod.THREAD_VARS)+def test_one_explicit_thread_setting_keeps_all_of_them_unset(var):+ # MKL_NUM_THREADS outranks OMP_NUM_THREADS in torch, so filling the others in beside an+ # explicit one would silently override it.+ env = {"PYTEST_XDIST_WORKER_COUNT": "4", var: "3"}+ assert share_mod.apply(env, cpus = 16) == "explicit"+ assert env[var] == "3"+ assert [v for v in share_mod.THREAD_VARS if v in env] == [var]+++def _write(path: Path, text: str) -> None:+ path.parent.mkdir(parents = True, exist_ok = True)+ path.write_text(text, encoding = "utf-8")+++def test_cgroup_v2_quota_takes_the_tightest_ancestor(tmp_path):+ root = tmp_path / "cg"+ _write(root / "cgroup.controllers", "cpu memory\n")+ _write(root / "cpu.max", "max 100000\n")+ _write(root / "job" / "cpu.max", "400000 100000\n")+ _write(root / "job" / "leaf" / "cpu.max", "max 100000\n")+ proc = tmp_path / "proc_cgroup"+ _write(proc, "0::/job/leaf\n")+ assert share_mod.cgroup_cpu_quota(str(root), str(proc)) == 4.0+++def test_cgroup_v2_without_a_quota_is_unlimited(tmp_path):+ root = tmp_path / "cg"+ _write(root / "cgroup.controllers", "cpu\n")+ _write(root / "cpu.max", "max 100000\n")+ proc = tmp_path / "proc_cgroup"+ _write(proc, "0::/\n")+ assert share_mod.cgroup_cpu_quota(str(root), str(proc)) is None+++def test_cgroup_v1_quota_is_read(tmp_path):+ root = tmp_path / "cg"+ _write(root / "cpu,cpuacct" / "cpu.cfs_quota_us", "-1\n")+ _write(root / "cpu,cpuacct" / "cpu.cfs_period_us", "100000\n")+ _write(root / "cpu,cpuacct" / "docker" / "abc" / "cpu.cfs_quota_us", "200000\n")+ _write(root / "cpu,cpuacct" / "docker" / "abc" / "cpu.cfs_period_us", "100000\n")+ proc = tmp_path / "proc_cgroup"+ _write(proc, "4:cpu,cpuacct:/docker/abc\n3:memory:/docker/abc\n")+ assert share_mod.cgroup_cpu_quota(str(root), str(proc)) == 2.0+++def test_usable_cpus_is_capped_by_the_quota(monkeypatch):+ monkeypatch.setattr(share_mod, "cgroup_cpu_quota", lambda: 2.5)+ monkeypatch.setattr(os, "sched_getaffinity", lambda pid: set(range(64)), raising = False)+ assert share_mod.usable_cpus() == 2+ monkeypatch.setattr(share_mod, "cgroup_cpu_quota", lambda: None)+ assert share_mod.usable_cpus() == 64