Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

tests: give each xdist worker one CPU share for torch (#13494)

Daniel Handc1860c40e36
diff --git a/tests/_shared/xdist_cpu_share.py b/tests/_shared/xdist_cpu_share.py
new file mode 100644
index 00000000000..64bcc887f81
--- /dev/null
+++ b/tests/_shared/xdist_cpu_share.py
@@ -0,0 +1,123 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""One CPU share per pytest-xdist worker for the torch / OpenMP / BLAS thread pools.
+
+Each xdist worker otherwise starts a pool as wide as the machine, so `-n 4` on a 4 vCPU
+runner runs 16 spinning threads on 4 cores and CPU training tests slow down by an order
+of magnitude. Loaded by tests/conftest.py before torch is imported. Stdlib only.
+"""
+
+from __future__ import annotations
+
+import os
+from typing import MutableMapping, Optional
+
+THREAD_VARS = ("OMP_NUM_THREADS", "MKL_NUM_THREADS", "OPENBLAS_NUM_THREADS")
+# Set by apply(): the share it gave each worker, or "explicit" when the caller had already
+# chosen a thread count. tests/test_xdist_torch_thread_cap.py reads it.
+MARKER = "UNSLOTH_TEST_XDIST_CPU_SHARE"
+CGROUP_ROOT = "/sys/fs/cgroup"
+
+
+def _first_line(path: str) -> Optional[str]:
+ try:
+ with open(path, "r", encoding = "utf-8") as f:
+ return f.readline().strip()
+ except OSError:
+ return None
+
+
+def _dirs(root: str, rel: Optional[str]) -> list:
+ """``root/rel`` and every ancestor up to *root*, innermost first: a parent's limit binds too."""
+ dirs = []
+ if rel and rel != "/":
+ current = os.path.normpath(os.path.join(root, rel.lstrip("/")))
+ while current.startswith(root + os.sep):
+ dirs.append(current)
+ current = os.path.dirname(current)
+ dirs.append(root)
+ return dirs
+
+
+def _ratio(quota: Optional[str], period: Optional[str]) -> Optional[float]:
+ try:
+ q, p = int(quota), int(period)
+ except (TypeError, ValueError):
+ return None # "max", missing or unreadable means unlimited
+ return q / p if q > 0 and p > 0 else None
+
+
+def cgroup_cpu_quota(
+ root: str = CGROUP_ROOT, proc_cgroup: str = "/proc/self/cgroup"
+) -> Optional[float]:
+ """The tightest CFS CPU quota over this process's cgroup and its ancestors, in cores.
+
+ None when no quota applies. sched_getaffinity() does not see a quota, so a container
+ limited to 4 CPUs on a 64 CPU host would otherwise hand each of 4 workers 16 threads.
+ """
+ try:
+ with open(proc_cgroup, "r", encoding = "utf-8") as f:
+ lines = [line.strip() for line in f if line.strip()]
+ except OSError:
+ lines = []
+ quotas = []
+ if os.path.isfile(os.path.join(root, "cgroup.controllers")): # v2
+ rel = next((line[3:] for line in lines if line.startswith("0::")), None)
+ for directory in _dirs(root, rel):
+ raw = (_first_line(os.path.join(directory, "cpu.max")) or "").split()
+ if len(raw) == 2:
+ quotas.append(_ratio(raw[0], raw[1]))
+ else: # v1
+ rel = None
+ for line in lines:
+ parts = line.split(":", 2)
+ if len(parts) == 3 and "cpu" in parts[1].split(","):
+ rel = parts[2]
+ break
+ for mount in ("cpu", "cpu,cpuacct", "cpuacct,cpu"):
+ base = os.path.join(root, mount)
+ if not os.path.isdir(base):
+ continue
+ for directory in _dirs(base, rel):
+ quotas.append(
+ _ratio(
+ _first_line(os.path.join(directory, "cpu.cfs_quota_us")),
+ _first_line(os.path.join(directory, "cpu.cfs_period_us")),
+ )
+ )
+ break
+ quotas = [q for q in quotas if q is not None]
+ return min(quotas) if quotas else None
+
+
+def usable_cpus() -> int:
+ """CPUs this process may actually use: the affinity mask, capped by any cgroup quota."""
+ try:
+ cpus = len(os.sched_getaffinity(0))
+ except (AttributeError, OSError):
+ cpus = os.cpu_count() or 1
+ quota = cgroup_cpu_quota()
+ if quota is not None:
+ cpus = min(cpus, max(1, int(quota)))
+ return max(1, cpus)
+
+
+def apply(environ: MutableMapping[str, str], cpus: Optional[int] = None) -> Optional[str]:
+ """Give each xdist worker ``cpus // workers`` threads. Returns what it recorded in MARKER.
+
+ Leaves every variable alone when any of them is already set: MKL_NUM_THREADS takes
+ precedence over OMP_NUM_THREADS in torch, so filling in the others beside an explicit
+ OMP_NUM_THREADS would override the caller's choice.
+ """
+ workers = int(environ.get("PYTEST_XDIST_WORKER_COUNT") or 0)
+ if workers <= 1:
+ return None
+ if any(environ.get(var) for var in THREAD_VARS):
+ environ[MARKER] = "explicit"
+ return "explicit"
+ share = str(max(1, (cpus if cpus is not None else usable_cpus()) // workers))
+ for var in THREAD_VARS:
+ environ[var] = share
+ environ[MARKER] = share
+ return share
diff --git a/tests/conftest.py b/tests/conftest.py
index e6d4f84fba2..356d6dd5e6d 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -28,6 +28,21 @@ for _up in _iso.parents:
_spec.loader.exec_module(_mod) # sets the env vars on import
break
+# --- one CPU share per xdist worker -----------------------------------------------
+# Also before torch is imported. Each xdist worker otherwise starts a torch / OpenMP pool
+# as wide as the machine, so `-n 4` on a 4 vCPU runner runs 16 spinning threads on 4 cores
+# and CPU training tests slow down by an order of magnitude: test_decision_model's Clef
+# training took >330 s there against ~50 s alone. An explicit thread setting is kept.
+# Loaded by path: tests/_shared is put on sys.path only below.
+import os as _os # noqa: E402
+
+_share_path = _iso.parent / "_shared" / "xdist_cpu_share.py"
+if _share_path.is_file():
+ _spec = _ilu.spec_from_file_location("xdist_cpu_share", _share_path)
+ _mod = _ilu.module_from_spec(_spec)
+ _spec.loader.exec_module(_mod)
+ _mod.apply(_os.environ)
+
# --- shared test helpers on sys.path -----------------------------------------------
# tests/_shared holds no package marker and pytest only puts a *test file's* own
# directory on sys.path, so tests/python/, tests/studio/install/ and tests/security/
diff --git a/tests/test_xdist_torch_thread_cap.py b/tests/test_xdist_torch_thread_cap.py
new file mode 100644
index 00000000000..4f117b0d47b
--- /dev/null
+++ b/tests/test_xdist_torch_thread_cap.py
@@ -0,0 +1,106 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""tests/conftest.py gives each xdist worker one share of the CPUs for torch.
+
+Without it every worker's torch pool is as wide as the machine, and CPU training tests
+thrash each other on CI's 4 vCPU runners (test_decision_model's Clef training went past
+the 330 s timeout). The end-to-end check only means something under xdist; the rules it
+relies on (explicit settings win, cgroup quotas count) are checked directly below.
+"""
+
+import importlib.util
+import os
+from pathlib import Path
+
+import pytest
+
+_spec = importlib.util.spec_from_file_location(
+ "xdist_cpu_share_under_test", Path(__file__).resolve().parent / "_shared" / "xdist_cpu_share.py"
+)
+share_mod = importlib.util.module_from_spec(_spec)
+_spec.loader.exec_module(share_mod)
+
+
+def test_each_xdist_worker_gets_one_cpu_share():
+ workers = int(os.environ.get("PYTEST_XDIST_WORKER_COUNT") or 0)
+ if workers <= 1:
+ pytest.skip("not running under pytest-xdist")
+ recorded = os.environ.get(share_mod.MARKER)
+ assert recorded, "conftest did not apply a per-worker CPU share"
+ if recorded == "explicit":
+ pytest.skip("the caller set a thread count; conftest left it alone")
+ import torch
+
+ assert recorded == str(max(1, share_mod.usable_cpus() // workers))
+ for var in share_mod.THREAD_VARS:
+ assert os.environ[var] == recorded, var
+ assert torch.get_num_threads() == int(recorded)
+
+
+def test_share_divides_the_cpus_between_workers():
+ env = {"PYTEST_XDIST_WORKER_COUNT": "4"}
+ assert share_mod.apply(env, cpus = 16) == "4"
+ assert all(env[var] == "4" for var in share_mod.THREAD_VARS)
+ env = {"PYTEST_XDIST_WORKER_COUNT": "8"}
+ assert share_mod.apply(env, cpus = 4) == "1"
+
+
+def test_serial_runs_are_left_alone():
+ for env in ({}, {"PYTEST_XDIST_WORKER_COUNT": "1"}):
+ assert share_mod.apply(env, cpus = 16) is None
+ assert not any(var in env for var in share_mod.THREAD_VARS)
+
+
[email protected]("var", share_mod.THREAD_VARS)
+def test_one_explicit_thread_setting_keeps_all_of_them_unset(var):
+ # MKL_NUM_THREADS outranks OMP_NUM_THREADS in torch, so filling the others in beside an
+ # explicit one would silently override it.
+ env = {"PYTEST_XDIST_WORKER_COUNT": "4", var: "3"}
+ assert share_mod.apply(env, cpus = 16) == "explicit"
+ assert env[var] == "3"
+ assert [v for v in share_mod.THREAD_VARS if v in env] == [var]
+
+
+def _write(path: Path, text: str) -> None:
+ path.parent.mkdir(parents = True, exist_ok = True)
+ path.write_text(text, encoding = "utf-8")
+
+
+def test_cgroup_v2_quota_takes_the_tightest_ancestor(tmp_path):
+ root = tmp_path / "cg"
+ _write(root / "cgroup.controllers", "cpu memory\n")
+ _write(root / "cpu.max", "max 100000\n")
+ _write(root / "job" / "cpu.max", "400000 100000\n")
+ _write(root / "job" / "leaf" / "cpu.max", "max 100000\n")
+ proc = tmp_path / "proc_cgroup"
+ _write(proc, "0::/job/leaf\n")
+ assert share_mod.cgroup_cpu_quota(str(root), str(proc)) == 4.0
+
+
+def test_cgroup_v2_without_a_quota_is_unlimited(tmp_path):
+ root = tmp_path / "cg"
+ _write(root / "cgroup.controllers", "cpu\n")
+ _write(root / "cpu.max", "max 100000\n")
+ proc = tmp_path / "proc_cgroup"
+ _write(proc, "0::/\n")
+ assert share_mod.cgroup_cpu_quota(str(root), str(proc)) is None
+
+
+def test_cgroup_v1_quota_is_read(tmp_path):
+ root = tmp_path / "cg"
+ _write(root / "cpu,cpuacct" / "cpu.cfs_quota_us", "-1\n")
+ _write(root / "cpu,cpuacct" / "cpu.cfs_period_us", "100000\n")
+ _write(root / "cpu,cpuacct" / "docker" / "abc" / "cpu.cfs_quota_us", "200000\n")
+ _write(root / "cpu,cpuacct" / "docker" / "abc" / "cpu.cfs_period_us", "100000\n")
+ proc = tmp_path / "proc_cgroup"
+ _write(proc, "4:cpu,cpuacct:/docker/abc\n3:memory:/docker/abc\n")
+ assert share_mod.cgroup_cpu_quota(str(root), str(proc)) == 2.0
+
+
+def test_usable_cpus_is_capped_by_the_quota(monkeypatch):
+ monkeypatch.setattr(share_mod, "cgroup_cpu_quota", lambda: 2.5)
+ monkeypatch.setattr(os, "sched_getaffinity", lambda pid: set(range(64)), raising = False)
+ assert share_mod.usable_cpus() == 2
+ monkeypatch.setattr(share_mod, "cgroup_cpu_quota", lambda: None)
+ assert share_mod.usable_cpus() == 64