Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: size sd.cpp's thread pool from the Apple fast cores (#13349)

Daniel Han429b2bb6d2db
diff --git a/studio/backend/core/inference/sd_cpp_args.py b/studio/backend/core/inference/sd_cpp_args.py
index e55f9d65562..7fb711d8b54 100644
--- a/studio/backend/core/inference/sd_cpp_args.py
+++ b/studio/backend/core/inference/sd_cpp_args.py
@@ -583,6 +583,7 @@ def build_sd_cpp_video_command(
*,
output_path: str,
offload: Optional[list[str]] = None,
+ threads: Optional[int] = None,
verbose: bool = False,
extra_args: Optional[list[str]] = None,
) -> list[str]:
@@ -657,6 +658,8 @@ def build_sd_cpp_video_command(
if params.seed is not None:
cmd += ["--seed", str(int(params.seed))]
cmd += ["--output", output_path]
+ if threads is not None:
+ cmd += ["--threads", str(int(threads))]
offload = list(offload or [])
if offload:
cmd += offload
diff --git a/studio/backend/core/inference/sd_cpp_backend.py b/studio/backend/core/inference/sd_cpp_backend.py
index 7dda3573a91..4729e7be51d 100644
--- a/studio/backend/core/inference/sd_cpp_backend.py
+++ b/studio/backend/core/inference/sd_cpp_backend.py
@@ -22,6 +22,7 @@ import base64
import contextlib
import json
import logging
+import functools
import os
import re
import shutil
@@ -297,20 +298,99 @@ def _cgroup_cpu_limit(
path = os.path.dirname(path)
[email protected]_cache(maxsize = 1)
+def _libc() -> Any:
+ """Cached: ``find_library`` walks the filesystem."""
+ import ctypes
+ import ctypes.util
+
+ libc = ctypes.CDLL(ctypes.util.find_library("c") or "libc.dylib")
+ libc.sysctlbyname.argtypes = [
+ ctypes.c_char_p,
+ ctypes.c_void_p,
+ ctypes.POINTER(ctypes.c_size_t),
+ ctypes.c_void_p,
+ ctypes.c_size_t,
+ ]
+ libc.sysctlbyname.restype = ctypes.c_int
+ return libc
+
+
+def _sysctl(name: str, as_str: bool = False) -> Any:
+ try:
+ import ctypes
+
+ libc = _libc()
+ if as_str:
+ buf = ctypes.create_string_buffer(64)
+ size = ctypes.c_size_t(len(buf))
+ ok = libc.sysctlbyname(name.encode(), buf, ctypes.byref(size), None, 0) == 0
+ return buf.value.decode(errors = "replace") if ok else None
+ value = ctypes.c_int(0)
+ size = ctypes.c_size_t(ctypes.sizeof(value))
+ ok = libc.sysctlbyname(name.encode(), ctypes.byref(value), ctypes.byref(size), None, 0) == 0
+ return int(value.value) if ok and value.value > 0 else None
+ except Exception: # noqa: BLE001
+ return None
+
+
+# M5 Pro / Max: perflevel0 "Super", perflevel1 "Performance".
+_FAST_PERF_LEVELS = ("super", "performance")
+
+
+def _apple_performance_cores() -> Optional[int]:
+ """macOS fast cores: perflevel0 plus lower tiers named Super/Performance (sd.cpp reads perflevel0 only).
+ No SMT, so ``cpu_count() // 2`` undercounts."""
+ if sys.platform != "darwin":
+ return None
+ levels = _sysctl("hw.nperflevels") or 0
+ total = 0
+ for level in range(levels):
+ if level > 0:
+ name = _sysctl(f"hw.perflevel{level}.name", as_str = True)
+ if name is None or name.strip().lower() not in _FAST_PERF_LEVELS:
+ continue
+ total += _sysctl(f"hw.perflevel{level}.physicalcpu") or 0
+ return total or _sysctl("hw.perflevel0.physicalcpu") or _sysctl("hw.physicalcpu")
+
+
def _default_threads() -> int:
- """``UNSLOTH_CPU_THREADS``, else ``os.cpu_count() // 2`` capped by the cgroup quota ``os.cpu_count()`` ignores."""
+ """``UNSLOTH_CPU_THREADS``, else macOS P-cores, else ``os.cpu_count() // 2`` capped by the cgroup quota."""
try:
configured = int(os.environ.get("UNSLOTH_CPU_THREADS") or 0)
except ValueError:
configured = 0
if configured > 0:
return configured
+ performance_cores = _apple_performance_cores()
+ if performance_cores:
+ return performance_cores
cpu = os.cpu_count()
threads = cpu // 2 if cpu else 8
limit = _cgroup_cpu_limit() if sys.platform.startswith("linux") else None
return max(1, min(threads, limit) if limit else threads)
+def _explicit_threads() -> Optional[int]:
+ """``UNSLOTH_CPU_THREADS``, the macOS fast cores, or a cgroup quota below sd.cpp's physical-core default; None
+ keeps that default."""
+ try:
+ configured = int(os.environ.get("UNSLOTH_CPU_THREADS") or 0)
+ except ValueError:
+ configured = 0
+ if configured > 0:
+ return configured
+ performance_cores = _apple_performance_cores()
+ if performance_cores:
+ return performance_cores
+ if sys.platform.startswith("linux"):
+ limit = _cgroup_cpu_limit()
+ cpu = os.cpu_count()
+ if limit and cpu and limit < cpu // 2:
+ return max(1, limit)
+ return None
+
+
def _server_binary_runnable(binary: str) -> bool:
"""Best-effort probe that ``binary`` can actually execute (not just exist). Runs ``<binary>
--help`` with the same runtime env the server will use, so a present but unrunnable build
diff --git a/studio/backend/core/inference/sd_cpp_engine.py b/studio/backend/core/inference/sd_cpp_engine.py
index ecfdd23e1a8..c58b218d1b4 100644
--- a/studio/backend/core/inference/sd_cpp_engine.py
+++ b/studio/backend/core/inference/sd_cpp_engine.py
@@ -890,6 +890,7 @@ class SdCppEngine:
*,
output_path: str,
offload: Optional[list[str]] = None,
+ threads: Optional[int] = None,
verbose: bool = False,
extra_args: Optional[list[str]] = None,
timeout: Optional[float] = NATIVE_GENERATION_TIMEOUT_S,
@@ -904,6 +905,7 @@ class SdCppEngine:
params,
output_path = str(self._prepare_out(output_path)),
offload = offload,
+ threads = threads,
verbose = verbose,
extra_args = extra_args,
)
diff --git a/studio/backend/core/inference/video.py b/studio/backend/core/inference/video.py
index e1e28a247df..93b9bc25b22 100644
--- a/studio/backend/core/inference/video.py
+++ b/studio/backend/core/inference/video.py
@@ -12563,11 +12563,14 @@ class VideoBackend:
reuse_live_server = False,
compute_bytes = compute_bytes,
)
+ from .sd_cpp_backend import _explicit_threads
generated = runtime.engine.generate_video(
runtime.files,
video_params,
output_path = str(output_path),
offload = render_flags,
+ # sd.cpp's default undercounts M5 Pro / Max and ignores env / cgroup quota
+ threads = _explicit_threads(),
env = dict(runtime.env) or None,
on_log = on_log,
cancel_event = cancel,
diff --git a/studio/backend/core/inference/video_minimax_h3.py b/studio/backend/core/inference/video_minimax_h3.py
index 2ecf441d093..447cc48f69c 100644
--- a/studio/backend/core/inference/video_minimax_h3.py
+++ b/studio/backend/core/inference/video_minimax_h3.py
@@ -2008,9 +2008,11 @@ class H3NativeServerSlot:
target = abort_on_cancel, daemon = True, name = "h3-sd-server-start-cancel"
).start()
try:
+ from .sd_cpp_backend import _explicit_threads
server.start(
self.files,
offload = list(flag_tuple),
+ threads = _explicit_threads(),
env = dict(env_pairs) or None,
# A context flag on sd-server (per-run on sd-cli); same RNG keeps seeds identical across paths.
extra_args = ["--rng", "cpu"],
diff --git a/studio/backend/tests/test_h3_native_memory_fit.py b/studio/backend/tests/test_h3_native_memory_fit.py
index fefbbb26124..83ac44db8cc 100644
--- a/studio/backend/tests/test_h3_native_memory_fit.py
+++ b/studio/backend/tests/test_h3_native_memory_fit.py
@@ -757,6 +757,7 @@ class _FakeServer:
files,
*,
offload,
+ threads = None,
env = None,
extra_args = None,
):
diff --git a/studio/backend/tests/test_h3_native_server.py b/studio/backend/tests/test_h3_native_server.py
index b116c5d9e79..2371167cf90 100644
--- a/studio/backend/tests/test_h3_native_server.py
+++ b/studio/backend/tests/test_h3_native_server.py
@@ -411,6 +411,7 @@ def test_slot_holds_the_managed_tree_while_its_server_lives(monkeypatch):
files,
*,
offload = None,
+ threads = None,
env = None,
extra_args = None,
):
@@ -452,6 +453,7 @@ def test_slot_respawns_a_dead_server(monkeypatch):
files,
*,
offload = None,
+ threads = None,
env = None,
extra_args = None,
):
@@ -490,6 +492,7 @@ class _LifeSrv:
files,
*,
offload = None,
+ threads = None,
env = None,
extra_args = None,
):
@@ -867,6 +870,7 @@ def test_cancel_aborts_a_server_start_in_progress(monkeypatch):
files,
*,
offload = None,
+ threads = None,
env = None,
extra_args = None,
):
@@ -941,6 +945,7 @@ def test_the_tree_stays_held_until_the_server_has_exited(monkeypatch):
files,
*,
offload = None,
+ threads = None,
env = None,
extra_args = None,
):
diff --git a/studio/backend/tests/test_sd_cpp_cpu_defaults.py b/studio/backend/tests/test_sd_cpp_cpu_defaults.py
index 022fe73f138..6662f8dbb2d 100644
--- a/studio/backend/tests/test_sd_cpp_cpu_defaults.py
+++ b/studio/backend/tests/test_sd_cpp_cpu_defaults.py
@@ -34,7 +34,6 @@ def _cgroups(
({"/user.slice/app.scope": "max 100000"}, None),
({"/user.slice/app.scope": "400000 100000"}, 4),
({"/user.slice/app.scope": "250000 100000"}, 3),
- # A parent's quota binds too, and the tightest one wins.
({"/user.slice": "200000 100000", "/user.slice/app.scope": "800000 100000"}, 2),
({"/": "600000 100000"}, 6),
({"/user.slice/app.scope": "garbage"}, None),
@@ -85,12 +84,197 @@ def test_a_quota_caps_the_default(monkeypatch):
def test_other_platforms_never_read_cgroups(monkeypatch):
_host(monkeypatch, cpus = 10, platform = "darwin")
+ monkeypatch.setattr(bk, "_apple_performance_cores", lambda: None)
monkeypatch.setattr(bk, "_cgroup_cpu_limit", lambda: pytest.fail("no cgroups off Linux"))
assert bk._default_threads() == 5
[email protected]("cpus, p_cores", [(3, 3), (10, 8), (16, 12), (10, 4)])
+def test_macos_uses_the_performance_core_count_not_half_the_cores(monkeypatch, cpus, p_cores):
+ _host(monkeypatch, cpus = cpus, platform = "darwin")
+ monkeypatch.setattr(bk, "_apple_performance_cores", lambda: p_cores)
+ assert bk._default_threads() == p_cores
+ monkeypatch.setenv("UNSLOTH_CPU_THREADS", "2")
+ assert bk._default_threads() == 2
+
+
+def test_apple_performance_cores_is_none_off_macos(monkeypatch):
+ monkeypatch.setattr(bk.sys, "platform", "linux")
+ assert bk._apple_performance_cores() is None
+
+
@pytest.mark.parametrize("value, expected", [("12", 12), ("0", 4), ("x", 4), ("", 4), ("-3", 4)])
def test_unsloth_cpu_threads_wins_over_the_default_and_the_quota(monkeypatch, value, expected):
_host(monkeypatch, limit = 4)
monkeypatch.setenv("UNSLOTH_CPU_THREADS", value)
assert bk._default_threads() == expected
+
+
+_TOPOLOGIES = {
+ "m1pro": {
+ "hw.nperflevels": 2,
+ "hw.perflevel0.name": "Performance",
+ "hw.perflevel0.physicalcpu": 8,
+ "hw.perflevel1.name": "Efficiency",
+ "hw.perflevel1.physicalcpu": 2,
+ "hw.physicalcpu": 10,
+ },
+ # ggml-org/llama.cpp#29782
+ "m5max": {
+ "hw.nperflevels": 2,
+ "hw.perflevel0.name": "Super",
+ "hw.perflevel0.physicalcpu": 6,
+ "hw.perflevel1.name": "Performance",
+ "hw.perflevel1.physicalcpu": 12,
+ "hw.physicalcpu": 18,
+ },
+ "vm": {
+ "hw.nperflevels": 1,
+ "hw.perflevel0.name": "Performance",
+ "hw.perflevel0.physicalcpu": 3,
+ "hw.physicalcpu": 3,
+ },
+ "intel": {"hw.physicalcpu": 6},
+ "unnamed": {
+ "hw.nperflevels": 2,
+ "hw.perflevel0.physicalcpu": 4,
+ "hw.perflevel1.physicalcpu": 6,
+ "hw.physicalcpu": 10,
+ },
+ # a lower tier with a name this code does not know is left out, as an unnamed one is
+ "unknown_tier": {
+ "hw.nperflevels": 3,
+ "hw.perflevel0.name": "Super",
+ "hw.perflevel0.physicalcpu": 4,
+ "hw.perflevel1.name": "Performance",
+ "hw.perflevel1.physicalcpu": 6,
+ "hw.perflevel2.name": "Background",
+ "hw.perflevel2.physicalcpu": 4,
+ "hw.physicalcpu": 14,
+ },
+}
+
+
+ "topology, expected",
+ [("m1pro", 8), ("m5max", 18), ("vm", 3), ("intel", 6), ("unnamed", 4), ("unknown_tier", 10)],
+)
+def test_apple_cores_count_perflevel0_and_the_named_fast_tiers(monkeypatch, topology, expected):
+ table = _TOPOLOGIES[topology]
+ monkeypatch.setattr(bk.sys, "platform", "darwin")
+ monkeypatch.setattr(bk, "_sysctl", lambda name, as_str = False: table.get(name))
+ assert bk._apple_performance_cores() == expected
+
+
+def test_the_libc_handle_is_looked_up_once_and_typed(monkeypatch):
+ import ctypes
+ import ctypes.util
+
+ class _Fn:
+ def __call__(self, *args):
+ return 0
+
+ class _Libc:
+ def __init__(self, path):
+ self.sysctlbyname = _Fn()
+
+ calls = []
+ monkeypatch.setattr(
+ ctypes.util, "find_library", lambda name: calls.append(name) or "libc.dylib"
+ )
+ monkeypatch.setattr(ctypes, "CDLL", _Libc)
+ bk._libc.cache_clear()
+ try:
+ for _ in range(3):
+ bk._sysctl("hw.physicalcpu")
+ assert calls == ["c"]
+ fn = bk._libc().sysctlbyname
+ assert fn.restype is ctypes.c_int and len(fn.argtypes) == 5
+ finally:
+ bk._libc.cache_clear()
+
+
+def test_minimax_h3_video_cli_and_server_get_the_default_thread_count(monkeypatch, tmp_path):
+ """H3 runs its text encoder on the Mac CPU, so both native video paths pass Studio's count."""
+ from core.inference import sd_cpp_engine as eng
+ from core.inference import sd_cpp_server as srv
+ from core.inference import video_minimax_h3 as h3
+ from core.inference.sd_cpp_args import (
+ SdCppModelFiles,
+ SdCppVideoGenParams,
+ build_sd_cpp_video_command,
+ )
+
+ files = SdCppModelFiles(
+ diffusion_model = "/m/h3.gguf", vae = "/m/vae.safetensors", llm = "/m/te.gguf"
+ )
+ params = SdCppVideoGenParams(prompt = "a dog", width = 448, height = 256, num_frames = 17, fps = 24)
+ cmd = build_sd_cpp_video_command("/bin/sd-cli", files, params, output_path = "/o.mp4", threads = 7)
+ assert cmd[cmd.index("--threads") + 1] == "7"
+ assert "--threads" not in build_sd_cpp_video_command(
+ "/bin/sd-cli", files, params, output_path = "/o.mp4"
+ )
+
+ seen = {}
+ engine = eng.SdCppEngine.__new__(eng.SdCppEngine)
+ monkeypatch.setattr(engine, "_require_binary", lambda: "/bin/sd-cli", raising = False)
+ monkeypatch.setattr(
+ engine, "_run", lambda cmd, out, **kw: seen.setdefault("cmd", cmd), raising = False
+ )
+ engine.generate_video(files, params, output_path = str(tmp_path / "o.mp4"), threads = 5)
+ assert seen["cmd"][seen["cmd"].index("--threads") + 1] == "5"
+
+ monkeypatch.setattr(bk, "_explicit_threads", lambda: 11)
+ started = {}
+
+ class _Srv:
+ def __init__(self, binary):
+ self.alive = False
+
+ def start(
+ self,
+ files,
+ *,
+ offload = None,
+ threads = None,
+ env = None,
+ extra_args = None,
+ ):
+ started["threads"] = threads
+ self.alive = True
+
+ def is_alive(self):
+ return self.alive
+
+ def stop(self):
+ self.alive = False
+
+ monkeypatch.setattr(srv, "SdCppServer", _Srv)
+ monkeypatch.setattr("core.inference.sd_cpp_engine.is_managed_binary", lambda b: False)
+ slot = h3.H3NativeServerSlot("/bin/sd-server", files, ("--offload-to-cpu",))
+ slot.get()
+ slot.stop()
+ assert started["threads"] == 11
+
+
+def test_h3_video_keeps_sd_cpps_default_threads_unless_studio_knows_better(monkeypatch):
+ # Linux / Windows with no override and no tighter quota: no --threads, as on main (sd.cpp counts physical cores)
+ monkeypatch.delenv("UNSLOTH_CPU_THREADS", raising = False)
+ monkeypatch.setattr(bk, "_apple_performance_cores", lambda: None)
+ monkeypatch.setattr(bk.os, "cpu_count", lambda: 32)
+ monkeypatch.setattr(bk.sys, "platform", "linux")
+ monkeypatch.setattr(bk, "_cgroup_cpu_limit", lambda *a, **k: None)
+ assert bk._explicit_threads() is None
+ monkeypatch.setattr(
+ bk, "_cgroup_cpu_limit", lambda *a, **k: 16
+ ) # equal to the default: still sd.cpp's own
+ assert bk._explicit_threads() is None
+ monkeypatch.setattr(bk, "_cgroup_cpu_limit", lambda *a, **k: 6) # a tighter container quota
+ assert bk._explicit_threads() == 6
+ monkeypatch.setattr(bk.sys, "platform", "win32")
+ assert bk._explicit_threads() is None
+ monkeypatch.setenv("UNSLOTH_CPU_THREADS", "12")
+ assert bk._explicit_threads() == 12
+ monkeypatch.delenv("UNSLOTH_CPU_THREADS")
+ monkeypatch.setattr(bk, "_apple_performance_cores", lambda: 10)
+ assert bk._explicit_threads() == 10