Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio CPU diffusion: one thread per logical CPU on small SMT hosts, for sd.cpp and torch (1.03x to 1.26x) (#16774)

Daniel Han4db7518f240b
diff --git a/studio/backend/core/inference/diffusion.py b/studio/backend/core/inference/diffusion.py
index 7ff5c699ab5..88322b86a3b 100644
--- a/studio/backend/core/inference/diffusion.py
+++ b/studio/backend/core/inference/diffusion.py
@@ -104,6 +104,7 @@ from .diffusion_compat import (
speech_pick_refusal,
)
from .diffusion_device import (
+ apply_cpu_render_threads,
DiffusionDeviceTarget,
apply_diffusion_device_ordinal,
diffusion_device_scope,
@@ -14134,6 +14135,7 @@ class DiffusionBackend:
restore_eager_gate: Optional[Callable[[], None]] = None
try:
self._state_device_target(state)
+ apply_cpu_render_threads(state.device)
# Local `state` keeps the pipe alive past unload() nulling _state; resolve (prompt, seed) jobs first.
jobs, seed = resolve_batch_jobs(
prompt = prompt,
diff --git a/studio/backend/core/inference/diffusion_device.py b/studio/backend/core/inference/diffusion_device.py
index f95deda96c5..884eeeb8ac9 100644
--- a/studio/backend/core/inference/diffusion_device.py
+++ b/studio/backend/core/inference/diffusion_device.py
@@ -973,6 +973,27 @@ def _mps_or_cpu_target(torch: Any) -> DiffusionDeviceTarget:
return _cpu_target(torch)
+def apply_cpu_render_threads(device: Any) -> Optional[int]:
+ """Match torch CPU threads to sd.cpp's ``_default_threads``; per thread, so call from the render thread. An explicit
+ OMP_NUM_THREADS / MKL_NUM_THREADS (torch sizes its intra-op pool from them) is kept, as configure_cpu_threads does."""
+ if str(device).split(":")[0] != "cpu":
+ return None
+ if any(os.environ.get(name, "").strip() for name in ("OMP_NUM_THREADS", "MKL_NUM_THREADS")):
+ return None
+ try:
+ import torch
+
+ from .sd_cpp_backend import _default_threads
+
+ threads = int(_default_threads())
+ if threads > 0 and torch.get_num_threads() != threads:
+ torch.set_num_threads(threads)
+ return threads
+ except Exception: # noqa: BLE001 -- a thread count is an optimisation only
+ return None
+ return None
+
+
def _cpu_target(torch: Any, dtype: Any = None) -> DiffusionDeviceTarget:
# torch is None on the no-torch CPU fallback; leave dtype=None rather than crash.
if dtype is None and torch is not None:
diff --git a/studio/backend/core/inference/sd_cpp_backend.py b/studio/backend/core/inference/sd_cpp_backend.py
index a1b316c0930..e80f57bf2d7 100644
--- a/studio/backend/core/inference/sd_cpp_backend.py
+++ b/studio/backend/core/inference/sd_cpp_backend.py
@@ -364,10 +364,86 @@ def _linux_physical_cores(sysfs: str = "/sys/devices/system/cpu") -> Optional[in
return len(cores) or None
-def _default_threads() -> int:
- """``UNSLOTH_CPU_THREADS``, else macOS P-cores, else (Linux) the physical cores in the affinity mask, else
- ``os.cpu_count() // 2``; capped by the cgroup quota on Linux. Half the logical CPUs is one thread per core only on
- a 2-way SMT host that may use all of them: it halves a host without SMT and oversubscribes a cpuset."""
+# 0 keeps one sd.cpp thread per physical core on an SMT host (the default runs one per logical CPU there).
+SD_CPP_CPU_SMT_ENV = "UNSLOTH_DIFFUSION_CPU_SMT"
+# Many-core servers keep one thread per core: ggml's per-op barrier across many threads outweighs SMT gains.
+SD_CPP_CPU_SMT_MAX_CORES = 16
+
+
+def _linux_hybrid_cpu(sysfs: str = "/sys/devices") -> bool:
+ """Intel hybrid (P + E) hosts register a ``cpu_atom`` PMU."""
+ try:
+ with open(f"{sysfs}/cpu_atom/cpus", encoding = "utf-8") as f:
+ return bool(f.read().strip())
+ except OSError:
+ return False
+
+
+def _smt_threads_enabled() -> bool:
+ return os.environ.get(SD_CPP_CPU_SMT_ENV, "").strip().lower() not in ("0", "false", "no", "off")
+
+
+# GetLogicalProcessorInformationEx: RelationProcessorCore records (one per physical core).
+_RELATION_PROCESSOR_CORE = 0
+
+
+def _parse_windows_cores(buf: bytes) -> Optional[tuple[int, bool]]:
+ """(cores, hybrid); byte 9 of each record is EfficiencyClass, more than one class = P + E cores."""
+ import struct
+
+ offset, cores, classes = 0, 0, set()
+ while offset + 10 <= len(buf):
+ relationship, size = struct.unpack_from("<II", buf, offset)
+ if size <= 0:
+ return None
+ if relationship == _RELATION_PROCESSOR_CORE:
+ cores += 1
+ classes.add(buf[offset + 9])
+ offset += size
+ return (cores, len(classes) > 1) if cores else None
+
+
+def _windows_cores() -> Optional[tuple[int, bool]]:
+ if sys.platform != "win32":
+ return None
+ try:
+ import ctypes
+ from ctypes import wintypes
+
+ kernel32 = ctypes.WinDLL("kernel32", use_last_error = True)
+ size = wintypes.DWORD(0)
+ kernel32.GetLogicalProcessorInformationEx(
+ _RELATION_PROCESSOR_CORE, None, ctypes.byref(size)
+ )
+ if not size.value:
+ return None
+ buf = ctypes.create_string_buffer(size.value)
+ if not kernel32.GetLogicalProcessorInformationEx(
+ _RELATION_PROCESSOR_CORE, buf, ctypes.byref(size)
+ ):
+ return None
+ return _parse_windows_cores(buf.raw[: size.value])
+ except Exception: # noqa: BLE001
+ return None
+
+
+def _smt_threads(physical: int, allowed: Optional[int], hybrid: bool, smt: bool) -> int:
+ if (
+ smt
+ and allowed
+ and physical < allowed
+ and physical <= SD_CPP_CPU_SMT_MAX_CORES
+ and _smt_threads_enabled()
+ and not hybrid
+ ):
+ return allowed
+ return physical
+
+
+def _default_threads(smt: bool = True) -> int:
+ """``UNSLOTH_CPU_THREADS``, else macOS P-cores, else logical CPUs on a small non-hybrid SMT host, else physical
+ cores, else ``os.cpu_count() // 2``; capped by the cgroup quota. SMT measured 1.06-1.26x faster, pixel-identical,
+ on 2-4 core VMs; hybrid hosts keep the core count (ggml's even split would wait on E cores)."""
try:
configured = int(os.environ.get("UNSLOTH_CPU_THREADS") or 0)
except ValueError:
@@ -379,8 +455,12 @@ def _default_threads() -> int:
return performance_cores
linux = sys.platform.startswith("linux")
physical = _linux_physical_cores() if linux else None
+ windows = None if linux else _windows_cores()
if physical:
- threads = physical
+ threads = _smt_threads(physical, _allowed_cpus(), _linux_hybrid_cpu(), smt)
+ elif windows is not None and smt:
+ cores, hybrid = windows
+ threads = _smt_threads(cores, os.cpu_count(), hybrid, smt)
else:
cpu = os.cpu_count()
threads = cpu // 2 if cpu else 8
@@ -427,7 +507,7 @@ def _explicit_threads() -> Optional[int]:
return performance_cores
if sys.platform.startswith("linux"):
cpu = os.cpu_count()
- threads = _default_threads()
+ threads = _default_threads(smt = False)
sd_default = _sd_cpp_default_threads(cpu)
if threads == sd_default:
return None
diff --git a/studio/backend/core/inference/video.py b/studio/backend/core/inference/video.py
index 99bf09523f3..1d481e4fcce 100644
--- a/studio/backend/core/inference/video.py
+++ b/studio/backend/core/inference/video.py
@@ -103,6 +103,7 @@ from .diffusion_step_skip import (
uninstall_static_step_skip,
)
from .diffusion_device import (
+ apply_cpu_render_threads,
DiffusionDeviceTarget,
apply_diffusion_device_ordinal,
diffusion_device_scope,
@@ -12472,6 +12473,7 @@ class VideoBackend:
raise RuntimeError(VIDEO_CANCELLED_MSG)
self._active_generate_cancel = cancel
begin_attention_run(state)
+ apply_cpu_render_threads(state.device)
# Bound below, once the request is resolved. None means the failure beat the resolution, and there is
# nothing truthful to report.
request_shape: Optional[dict[str, Any]] = None
diff --git a/studio/backend/tests/test_sd_cpp_cpu_defaults.py b/studio/backend/tests/test_sd_cpp_cpu_defaults.py
index 43e8bfa6da3..f5ac198d1dd 100644
--- a/studio/backend/tests/test_sd_cpp_cpu_defaults.py
+++ b/studio/backend/tests/test_sd_cpp_cpu_defaults.py
@@ -63,6 +63,8 @@ def _host(
platform = "linux",
physical = None,
allowed = None,
+ hybrid = False,
+ windows = None,
):
monkeypatch.delenv("UNSLOTH_CPU_THREADS", raising = False)
monkeypatch.setattr(bk.sys, "platform", platform)
@@ -70,6 +72,9 @@ def _host(
monkeypatch.setattr(bk, "_cgroup_cpu_limit", lambda: limit)
monkeypatch.setattr(bk, "_linux_physical_cores", lambda *a, **k: physical)
monkeypatch.setattr(bk, "_allowed_cpus", lambda: cpus if allowed is None else allowed)
+ monkeypatch.setattr(bk, "_linux_hybrid_cpu", lambda *a, **k: hybrid)
+ monkeypatch.setattr(bk, "_windows_cores", lambda: windows)
+ monkeypatch.delenv(bk.SD_CPP_CPU_SMT_ENV, raising = False)
def test_without_a_quota_the_default_is_unchanged(monkeypatch):
@@ -337,14 +342,16 @@ def test_physical_cores_is_none_without_topology_or_affinity(monkeypatch, tmp_pa
assert bk._linux_physical_cores(_sysfs(tmp_path / "y", _NO_SMT8)) is None
-def test_the_linux_default_is_the_physical_core_count(monkeypatch):
+def test_the_linux_default_counts_cores_then_smt_siblings(monkeypatch):
# No SMT: every core, not half of them
_host(monkeypatch, cpus = 8, physical = 8)
assert bk._default_threads() == 8
- # A 4-CPU cpuset (2 cores) on a 192-CPU host: 2, not 96
- _host(monkeypatch, cpus = 192, physical = 2)
+ # A 4-CPU cpuset (2 cores x 2 SMT) on a 192-CPU host: its 4 CPUs, not 96
+ _host(monkeypatch, cpus = 192, allowed = 4, physical = 2)
+ assert bk._default_threads() == 4
+ _host(monkeypatch, cpus = 192, allowed = 2, physical = 2)
assert bk._default_threads() == 2
- # 2-way SMT with the whole machine: unchanged
+ # 2-way SMT with the whole machine: a many-core server keeps one thread per core
_host(monkeypatch, cpus = 192, physical = 96)
assert bk._default_threads() == 96
# The quota still caps it, and an unreadable topology keeps half the CPUs
@@ -396,3 +403,131 @@ def test_the_real_host_topology_reads(monkeypatch):
if count is None:
pytest.skip("no sysfs topology here")
assert 1 <= count <= len(bk.os.sched_getaffinity(0))
+
+
+def test_a_small_smt_host_runs_one_thread_per_logical_cpu(monkeypatch):
+ # 4 cores x 2 SMT (desktop / laptop / cloud VM): all 8 logical CPUs
+ _host(monkeypatch, cpus = 8, physical = 4)
+ assert bk._default_threads() == 8
+ _host(monkeypatch, cpus = 32, physical = 16)
+ assert bk._default_threads() == 32
+ _host(monkeypatch, cpus = 64, physical = 32)
+ assert bk._default_threads() == 32
+ # A cpuset holding 3 whole cores plus one lone sibling: the 7 CPUs it may use
+ _host(monkeypatch, cpus = 192, allowed = 7, physical = 4)
+ assert bk._default_threads() == 7
+ _host(monkeypatch, cpus = 8, physical = 4, limit = 6)
+ assert bk._default_threads() == 6
+
+
+def test_smt_threads_skip_hybrid_hosts_the_opt_out_and_h3_video(monkeypatch):
+ _host(monkeypatch, cpus = 20, physical = 14, hybrid = True) # Intel 6P + 8E: one per core
+ assert bk._default_threads() == 14
+ _host(monkeypatch, cpus = 8, physical = 4)
+ monkeypatch.setenv(bk.SD_CPP_CPU_SMT_ENV, "0")
+ assert bk._default_threads() == 4
+ monkeypatch.delenv(bk.SD_CPP_CPU_SMT_ENV)
+ assert bk._default_threads(smt = False) == 4
+ monkeypatch.setenv("UNSLOTH_CPU_THREADS", "3")
+ assert bk._default_threads() == 3
+
+
+def test_linux_hybrid_cpu_reads_the_cpu_atom_pmu(tmp_path):
+ assert bk._linux_hybrid_cpu(str(tmp_path)) is False
+ (tmp_path / "cpu_atom").mkdir()
+ (tmp_path / "cpu_atom" / "cpus").write_text("\n")
+ assert bk._linux_hybrid_cpu(str(tmp_path)) is False
+ (tmp_path / "cpu_atom" / "cpus").write_text("12-19\n")
+ assert bk._linux_hybrid_cpu(str(tmp_path)) is True
+
+
+def _win_record(
+ relationship,
+ efficiency = 0,
+ smt = True,
+):
+ import struct
+
+ # SYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX: Relationship, Size, Flags, EfficiencyClass, ...
+ body = struct.pack("<BB20xH", 1 if smt else 0, efficiency, 1) + struct.pack("<QH6x", 0b11, 0)
+ return struct.pack("<II", relationship, 8 + len(body)) + body
+
+
+def test_parse_windows_cores_counts_cores_and_efficiency_classes():
+ cache = _win_record(2) # RelationCache records are skipped
+ four = b"".join(_win_record(0) for _ in range(4)) + cache
+ assert bk._parse_windows_cores(four) == (4, False)
+ hybrid = b"".join(_win_record(0, 1) for _ in range(6)) + b"".join(
+ _win_record(0, 0, smt = False) for _ in range(8)
+ )
+ assert bk._parse_windows_cores(hybrid) == (14, True)
+ assert bk._parse_windows_cores(b"") is None
+ assert bk._parse_windows_cores(cache) is None
+ assert bk._parse_windows_cores(b"\0" * 16) is None # a zero Size never loops
+
+
+def test_windows_runs_the_logical_cpus_on_a_small_non_hybrid_smt_host(monkeypatch):
+ _host(monkeypatch, cpus = 4, platform = "win32", windows = (2, False))
+ assert bk._default_threads() == 4
+ _host(monkeypatch, cpus = 16, platform = "win32", windows = (8, False))
+ assert bk._default_threads() == 16
+ # hybrid (P + E cores), no SMT, or many cores: one per core
+ _host(monkeypatch, cpus = 20, platform = "win32", windows = (14, True))
+ assert bk._default_threads() == 14
+ _host(monkeypatch, cpus = 8, platform = "win32", windows = (8, False))
+ assert bk._default_threads() == 8
+ _host(monkeypatch, cpus = 64, platform = "win32", windows = (32, False))
+ assert bk._default_threads() == 32
+ # unreadable topology: half the logical CPUs, as before
+ _host(monkeypatch, cpus = 16, platform = "win32", windows = None)
+ assert bk._default_threads() == 8
+ _host(monkeypatch, cpus = 4, platform = "win32", windows = (2, False))
+ monkeypatch.setenv(bk.SD_CPP_CPU_SMT_ENV, "off")
+ assert bk._default_threads() == 2
+ monkeypatch.delenv(bk.SD_CPP_CPU_SMT_ENV)
+ assert bk._default_threads(smt = False) == 2
+
+
+def test_windows_cores_is_none_off_windows(monkeypatch):
+ monkeypatch.setattr(bk.sys, "platform", "linux")
+ assert bk._windows_cores() is None
+
+
+def test_cpu_renders_size_torch_threads_like_sd_cpp(monkeypatch):
+ torch = pytest.importorskip("torch")
+ from core.inference import diffusion_device as dd
+
+ for name in ("OMP_NUM_THREADS", "MKL_NUM_THREADS"):
+ monkeypatch.delenv(name, raising = False)
+ before = torch.get_num_threads()
+ calls = []
+ monkeypatch.setattr(torch, "set_num_threads", lambda n: calls.append(n))
+ monkeypatch.setattr(bk, "_default_threads", lambda *a, **k: before + 3)
+ assert dd.apply_cpu_render_threads("cuda:0") is None
+ assert dd.apply_cpu_render_threads("mps") is None
+ assert calls == []
+ assert dd.apply_cpu_render_threads("cpu") == before + 3
+ assert dd.apply_cpu_render_threads(torch.device("cpu")) == before + 3
+ assert calls == [before + 3, before + 3]
+ monkeypatch.setattr(bk, "_default_threads", lambda *a, **k: torch.get_num_threads())
+ assert dd.apply_cpu_render_threads("cpu") is None
+ monkeypatch.setattr(bk, "_default_threads", lambda *a, **k: 1 / 0)
+ assert dd.apply_cpu_render_threads("cpu") is None
+
+
[email protected]("name", ["OMP_NUM_THREADS", "MKL_NUM_THREADS"])
+def test_cpu_renders_keep_an_explicit_torch_thread_pool(monkeypatch, name):
+ torch = pytest.importorskip("torch")
+ from core.inference import diffusion_device as dd
+
+ for other in ("OMP_NUM_THREADS", "MKL_NUM_THREADS"):
+ monkeypatch.delenv(other, raising = False)
+ monkeypatch.setenv(name, "2")
+ monkeypatch.setenv("UNSLOTH_CPU_THREADS", "8")
+ calls = []
+ monkeypatch.setattr(torch, "set_num_threads", lambda n: calls.append(n))
+ monkeypatch.setattr(bk, "_default_threads", lambda *a, **k: torch.get_num_threads() + 3)
+ assert dd.apply_cpu_render_threads("cpu") is None
+ assert calls == []
+ monkeypatch.setenv(name, " ")
+ assert dd.apply_cpu_render_threads("cpu") == torch.get_num_threads() + 3