Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio CPU diffusion: one thread per logical CPU on small SMT hosts, for sd.cpp and torch (1.03x to 1.26x) (#16774)
Daniel Han4db7518f240b
diff --git a/studio/backend/core/inference/diffusion.py b/studio/backend/core/inference/diffusion.pyindex 7ff5c699ab5..88322b86a3b 100644--- a/studio/backend/core/inference/diffusion.py+++ b/studio/backend/core/inference/diffusion.py@@ -104,6 +104,7 @@ from .diffusion_compat import (speech_pick_refusal,)from .diffusion_device import (+ apply_cpu_render_threads,DiffusionDeviceTarget,apply_diffusion_device_ordinal,diffusion_device_scope,@@ -14134,6 +14135,7 @@ class DiffusionBackend:restore_eager_gate: Optional[Callable[[], None]] = Nonetry:self._state_device_target(state)+ apply_cpu_render_threads(state.device)# Local `state` keeps the pipe alive past unload() nulling _state; resolve (prompt, seed) jobs first.jobs, seed = resolve_batch_jobs(prompt = prompt,diff --git a/studio/backend/core/inference/diffusion_device.py b/studio/backend/core/inference/diffusion_device.pyindex f95deda96c5..884eeeb8ac9 100644--- a/studio/backend/core/inference/diffusion_device.py+++ b/studio/backend/core/inference/diffusion_device.py@@ -973,6 +973,27 @@ def _mps_or_cpu_target(torch: Any) -> DiffusionDeviceTarget:return _cpu_target(torch)+def apply_cpu_render_threads(device: Any) -> Optional[int]:+ """Match torch CPU threads to sd.cpp's ``_default_threads``; per thread, so call from the render thread. An explicit+ OMP_NUM_THREADS / MKL_NUM_THREADS (torch sizes its intra-op pool from them) is kept, as configure_cpu_threads does."""+ if str(device).split(":")[0] != "cpu":+ return None+ if any(os.environ.get(name, "").strip() for name in ("OMP_NUM_THREADS", "MKL_NUM_THREADS")):+ return None+ try:+ import torch++ from .sd_cpp_backend import _default_threads++ threads = int(_default_threads())+ if threads > 0 and torch.get_num_threads() != threads:+ torch.set_num_threads(threads)+ return threads+ except Exception: # noqa: BLE001 -- a thread count is an optimisation only+ return None+ return None++def _cpu_target(torch: Any, dtype: Any = None) -> DiffusionDeviceTarget:# torch is None on the no-torch CPU fallback; leave dtype=None rather than crash.if dtype is None and torch is not None:diff --git a/studio/backend/core/inference/sd_cpp_backend.py b/studio/backend/core/inference/sd_cpp_backend.pyindex a1b316c0930..e80f57bf2d7 100644--- a/studio/backend/core/inference/sd_cpp_backend.py+++ b/studio/backend/core/inference/sd_cpp_backend.py@@ -364,10 +364,86 @@ def _linux_physical_cores(sysfs: str = "/sys/devices/system/cpu") -> Optional[inreturn len(cores) or None-def _default_threads() -> int:- """``UNSLOTH_CPU_THREADS``, else macOS P-cores, else (Linux) the physical cores in the affinity mask, else- ``os.cpu_count() // 2``; capped by the cgroup quota on Linux. Half the logical CPUs is one thread per core only on- a 2-way SMT host that may use all of them: it halves a host without SMT and oversubscribes a cpuset."""+# 0 keeps one sd.cpp thread per physical core on an SMT host (the default runs one per logical CPU there).+SD_CPP_CPU_SMT_ENV = "UNSLOTH_DIFFUSION_CPU_SMT"+# Many-core servers keep one thread per core: ggml's per-op barrier across many threads outweighs SMT gains.+SD_CPP_CPU_SMT_MAX_CORES = 16+++def _linux_hybrid_cpu(sysfs: str = "/sys/devices") -> bool:+ """Intel hybrid (P + E) hosts register a ``cpu_atom`` PMU."""+ try:+ with open(f"{sysfs}/cpu_atom/cpus", encoding = "utf-8") as f:+ return bool(f.read().strip())+ except OSError:+ return False+++def _smt_threads_enabled() -> bool:+ return os.environ.get(SD_CPP_CPU_SMT_ENV, "").strip().lower() not in ("0", "false", "no", "off")+++# GetLogicalProcessorInformationEx: RelationProcessorCore records (one per physical core).+_RELATION_PROCESSOR_CORE = 0+++def _parse_windows_cores(buf: bytes) -> Optional[tuple[int, bool]]:+ """(cores, hybrid); byte 9 of each record is EfficiencyClass, more than one class = P + E cores."""+ import struct++ offset, cores, classes = 0, 0, set()+ while offset + 10 <= len(buf):+ relationship, size = struct.unpack_from("<II", buf, offset)+ if size <= 0:+ return None+ if relationship == _RELATION_PROCESSOR_CORE:+ cores += 1+ classes.add(buf[offset + 9])+ offset += size+ return (cores, len(classes) > 1) if cores else None+++def _windows_cores() -> Optional[tuple[int, bool]]:+ if sys.platform != "win32":+ return None+ try:+ import ctypes+ from ctypes import wintypes++ kernel32 = ctypes.WinDLL("kernel32", use_last_error = True)+ size = wintypes.DWORD(0)+ kernel32.GetLogicalProcessorInformationEx(+ _RELATION_PROCESSOR_CORE, None, ctypes.byref(size)+ )+ if not size.value:+ return None+ buf = ctypes.create_string_buffer(size.value)+ if not kernel32.GetLogicalProcessorInformationEx(+ _RELATION_PROCESSOR_CORE, buf, ctypes.byref(size)+ ):+ return None+ return _parse_windows_cores(buf.raw[: size.value])+ except Exception: # noqa: BLE001+ return None+++def _smt_threads(physical: int, allowed: Optional[int], hybrid: bool, smt: bool) -> int:+ if (+ smt+ and allowed+ and physical < allowed+ and physical <= SD_CPP_CPU_SMT_MAX_CORES+ and _smt_threads_enabled()+ and not hybrid+ ):+ return allowed+ return physical+++def _default_threads(smt: bool = True) -> int:+ """``UNSLOTH_CPU_THREADS``, else macOS P-cores, else logical CPUs on a small non-hybrid SMT host, else physical+ cores, else ``os.cpu_count() // 2``; capped by the cgroup quota. SMT measured 1.06-1.26x faster, pixel-identical,+ on 2-4 core VMs; hybrid hosts keep the core count (ggml's even split would wait on E cores)."""try:configured = int(os.environ.get("UNSLOTH_CPU_THREADS") or 0)except ValueError:@@ -379,8 +455,12 @@ def _default_threads() -> int:return performance_coreslinux = sys.platform.startswith("linux")physical = _linux_physical_cores() if linux else None+ windows = None if linux else _windows_cores()if physical:- threads = physical+ threads = _smt_threads(physical, _allowed_cpus(), _linux_hybrid_cpu(), smt)+ elif windows is not None and smt:+ cores, hybrid = windows+ threads = _smt_threads(cores, os.cpu_count(), hybrid, smt)else:cpu = os.cpu_count()threads = cpu // 2 if cpu else 8@@ -427,7 +507,7 @@ def _explicit_threads() -> Optional[int]:return performance_coresif sys.platform.startswith("linux"):cpu = os.cpu_count()- threads = _default_threads()+ threads = _default_threads(smt = False)sd_default = _sd_cpp_default_threads(cpu)if threads == sd_default:return Nonediff --git a/studio/backend/core/inference/video.py b/studio/backend/core/inference/video.pyindex 99bf09523f3..1d481e4fcce 100644--- a/studio/backend/core/inference/video.py+++ b/studio/backend/core/inference/video.py@@ -103,6 +103,7 @@ from .diffusion_step_skip import (uninstall_static_step_skip,)from .diffusion_device import (+ apply_cpu_render_threads,DiffusionDeviceTarget,apply_diffusion_device_ordinal,diffusion_device_scope,@@ -12472,6 +12473,7 @@ class VideoBackend:raise RuntimeError(VIDEO_CANCELLED_MSG)self._active_generate_cancel = cancelbegin_attention_run(state)+ apply_cpu_render_threads(state.device)# Bound below, once the request is resolved. None means the failure beat the resolution, and there is# nothing truthful to report.request_shape: Optional[dict[str, Any]] = Nonediff --git a/studio/backend/tests/test_sd_cpp_cpu_defaults.py b/studio/backend/tests/test_sd_cpp_cpu_defaults.pyindex 43e8bfa6da3..f5ac198d1dd 100644--- a/studio/backend/tests/test_sd_cpp_cpu_defaults.py+++ b/studio/backend/tests/test_sd_cpp_cpu_defaults.py@@ -63,6 +63,8 @@ def _host(platform = "linux",physical = None,allowed = None,+ hybrid = False,+ windows = None,):monkeypatch.delenv("UNSLOTH_CPU_THREADS", raising = False)monkeypatch.setattr(bk.sys, "platform", platform)@@ -70,6 +72,9 @@ def _host(monkeypatch.setattr(bk, "_cgroup_cpu_limit", lambda: limit)monkeypatch.setattr(bk, "_linux_physical_cores", lambda *a, **k: physical)monkeypatch.setattr(bk, "_allowed_cpus", lambda: cpus if allowed is None else allowed)+ monkeypatch.setattr(bk, "_linux_hybrid_cpu", lambda *a, **k: hybrid)+ monkeypatch.setattr(bk, "_windows_cores", lambda: windows)+ monkeypatch.delenv(bk.SD_CPP_CPU_SMT_ENV, raising = False)def test_without_a_quota_the_default_is_unchanged(monkeypatch):@@ -337,14 +342,16 @@ def test_physical_cores_is_none_without_topology_or_affinity(monkeypatch, tmp_paassert bk._linux_physical_cores(_sysfs(tmp_path / "y", _NO_SMT8)) is None-def test_the_linux_default_is_the_physical_core_count(monkeypatch):+def test_the_linux_default_counts_cores_then_smt_siblings(monkeypatch):# No SMT: every core, not half of them_host(monkeypatch, cpus = 8, physical = 8)assert bk._default_threads() == 8- # A 4-CPU cpuset (2 cores) on a 192-CPU host: 2, not 96- _host(monkeypatch, cpus = 192, physical = 2)+ # A 4-CPU cpuset (2 cores x 2 SMT) on a 192-CPU host: its 4 CPUs, not 96+ _host(monkeypatch, cpus = 192, allowed = 4, physical = 2)+ assert bk._default_threads() == 4+ _host(monkeypatch, cpus = 192, allowed = 2, physical = 2)assert bk._default_threads() == 2- # 2-way SMT with the whole machine: unchanged+ # 2-way SMT with the whole machine: a many-core server keeps one thread per core_host(monkeypatch, cpus = 192, physical = 96)assert bk._default_threads() == 96# The quota still caps it, and an unreadable topology keeps half the CPUs@@ -396,3 +403,131 @@ def test_the_real_host_topology_reads(monkeypatch):if count is None:pytest.skip("no sysfs topology here")assert 1 <= count <= len(bk.os.sched_getaffinity(0))+++def test_a_small_smt_host_runs_one_thread_per_logical_cpu(monkeypatch):+ # 4 cores x 2 SMT (desktop / laptop / cloud VM): all 8 logical CPUs+ _host(monkeypatch, cpus = 8, physical = 4)+ assert bk._default_threads() == 8+ _host(monkeypatch, cpus = 32, physical = 16)+ assert bk._default_threads() == 32+ _host(monkeypatch, cpus = 64, physical = 32)+ assert bk._default_threads() == 32+ # A cpuset holding 3 whole cores plus one lone sibling: the 7 CPUs it may use+ _host(monkeypatch, cpus = 192, allowed = 7, physical = 4)+ assert bk._default_threads() == 7+ _host(monkeypatch, cpus = 8, physical = 4, limit = 6)+ assert bk._default_threads() == 6+++def test_smt_threads_skip_hybrid_hosts_the_opt_out_and_h3_video(monkeypatch):+ _host(monkeypatch, cpus = 20, physical = 14, hybrid = True) # Intel 6P + 8E: one per core+ assert bk._default_threads() == 14+ _host(monkeypatch, cpus = 8, physical = 4)+ monkeypatch.setenv(bk.SD_CPP_CPU_SMT_ENV, "0")+ assert bk._default_threads() == 4+ monkeypatch.delenv(bk.SD_CPP_CPU_SMT_ENV)+ assert bk._default_threads(smt = False) == 4+ monkeypatch.setenv("UNSLOTH_CPU_THREADS", "3")+ assert bk._default_threads() == 3+++def test_linux_hybrid_cpu_reads_the_cpu_atom_pmu(tmp_path):+ assert bk._linux_hybrid_cpu(str(tmp_path)) is False+ (tmp_path / "cpu_atom").mkdir()+ (tmp_path / "cpu_atom" / "cpus").write_text("\n")+ assert bk._linux_hybrid_cpu(str(tmp_path)) is False+ (tmp_path / "cpu_atom" / "cpus").write_text("12-19\n")+ assert bk._linux_hybrid_cpu(str(tmp_path)) is True+++def _win_record(+ relationship,+ efficiency = 0,+ smt = True,+):+ import struct++ # SYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX: Relationship, Size, Flags, EfficiencyClass, ...+ body = struct.pack("<BB20xH", 1 if smt else 0, efficiency, 1) + struct.pack("<QH6x", 0b11, 0)+ return struct.pack("<II", relationship, 8 + len(body)) + body+++def test_parse_windows_cores_counts_cores_and_efficiency_classes():+ cache = _win_record(2) # RelationCache records are skipped+ four = b"".join(_win_record(0) for _ in range(4)) + cache+ assert bk._parse_windows_cores(four) == (4, False)+ hybrid = b"".join(_win_record(0, 1) for _ in range(6)) + b"".join(+ _win_record(0, 0, smt = False) for _ in range(8)+ )+ assert bk._parse_windows_cores(hybrid) == (14, True)+ assert bk._parse_windows_cores(b"") is None+ assert bk._parse_windows_cores(cache) is None+ assert bk._parse_windows_cores(b"\0" * 16) is None # a zero Size never loops+++def test_windows_runs_the_logical_cpus_on_a_small_non_hybrid_smt_host(monkeypatch):+ _host(monkeypatch, cpus = 4, platform = "win32", windows = (2, False))+ assert bk._default_threads() == 4+ _host(monkeypatch, cpus = 16, platform = "win32", windows = (8, False))+ assert bk._default_threads() == 16+ # hybrid (P + E cores), no SMT, or many cores: one per core+ _host(monkeypatch, cpus = 20, platform = "win32", windows = (14, True))+ assert bk._default_threads() == 14+ _host(monkeypatch, cpus = 8, platform = "win32", windows = (8, False))+ assert bk._default_threads() == 8+ _host(monkeypatch, cpus = 64, platform = "win32", windows = (32, False))+ assert bk._default_threads() == 32+ # unreadable topology: half the logical CPUs, as before+ _host(monkeypatch, cpus = 16, platform = "win32", windows = None)+ assert bk._default_threads() == 8+ _host(monkeypatch, cpus = 4, platform = "win32", windows = (2, False))+ monkeypatch.setenv(bk.SD_CPP_CPU_SMT_ENV, "off")+ assert bk._default_threads() == 2+ monkeypatch.delenv(bk.SD_CPP_CPU_SMT_ENV)+ assert bk._default_threads(smt = False) == 2+++def test_windows_cores_is_none_off_windows(monkeypatch):+ monkeypatch.setattr(bk.sys, "platform", "linux")+ assert bk._windows_cores() is None+++def test_cpu_renders_size_torch_threads_like_sd_cpp(monkeypatch):+ torch = pytest.importorskip("torch")+ from core.inference import diffusion_device as dd++ for name in ("OMP_NUM_THREADS", "MKL_NUM_THREADS"):+ monkeypatch.delenv(name, raising = False)+ before = torch.get_num_threads()+ calls = []+ monkeypatch.setattr(torch, "set_num_threads", lambda n: calls.append(n))+ monkeypatch.setattr(bk, "_default_threads", lambda *a, **k: before + 3)+ assert dd.apply_cpu_render_threads("cuda:0") is None+ assert dd.apply_cpu_render_threads("mps") is None+ assert calls == []+ assert dd.apply_cpu_render_threads("cpu") == before + 3+ assert dd.apply_cpu_render_threads(torch.device("cpu")) == before + 3+ assert calls == [before + 3, before + 3]+ monkeypatch.setattr(bk, "_default_threads", lambda *a, **k: torch.get_num_threads())+ assert dd.apply_cpu_render_threads("cpu") is None+ monkeypatch.setattr(bk, "_default_threads", lambda *a, **k: 1 / 0)+ assert dd.apply_cpu_render_threads("cpu") is None++[email protected]("name", ["OMP_NUM_THREADS", "MKL_NUM_THREADS"])+def test_cpu_renders_keep_an_explicit_torch_thread_pool(monkeypatch, name):+ torch = pytest.importorskip("torch")+ from core.inference import diffusion_device as dd++ for other in ("OMP_NUM_THREADS", "MKL_NUM_THREADS"):+ monkeypatch.delenv(other, raising = False)+ monkeypatch.setenv(name, "2")+ monkeypatch.setenv("UNSLOTH_CPU_THREADS", "8")+ calls = []+ monkeypatch.setattr(torch, "set_num_threads", lambda n: calls.append(n))+ monkeypatch.setattr(bk, "_default_threads", lambda *a, **k: torch.get_num_threads() + 3)+ assert dd.apply_cpu_render_threads("cpu") is None+ assert calls == []+ monkeypatch.setenv(name, " ")+ assert dd.apply_cpu_render_threads("cpu") == torch.get_num_threads() + 3