Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: size sd.cpp's thread pool from the Apple fast cores (#13349)
Daniel Han429b2bb6d2db
diff --git a/studio/backend/core/inference/sd_cpp_args.py b/studio/backend/core/inference/sd_cpp_args.pyindex e55f9d65562..7fb711d8b54 100644--- a/studio/backend/core/inference/sd_cpp_args.py+++ b/studio/backend/core/inference/sd_cpp_args.py@@ -583,6 +583,7 @@ def build_sd_cpp_video_command(*,output_path: str,offload: Optional[list[str]] = None,+ threads: Optional[int] = None,verbose: bool = False,extra_args: Optional[list[str]] = None,) -> list[str]:@@ -657,6 +658,8 @@ def build_sd_cpp_video_command(if params.seed is not None:cmd += ["--seed", str(int(params.seed))]cmd += ["--output", output_path]+ if threads is not None:+ cmd += ["--threads", str(int(threads))]offload = list(offload or [])if offload:cmd += offloaddiff --git a/studio/backend/core/inference/sd_cpp_backend.py b/studio/backend/core/inference/sd_cpp_backend.pyindex 7dda3573a91..4729e7be51d 100644--- a/studio/backend/core/inference/sd_cpp_backend.py+++ b/studio/backend/core/inference/sd_cpp_backend.py@@ -22,6 +22,7 @@ import base64import contextlibimport jsonimport logging+import functoolsimport osimport reimport shutil@@ -297,20 +298,99 @@ def _cgroup_cpu_limit(path = os.path.dirname(path)[email protected]_cache(maxsize = 1)+def _libc() -> Any:+ """Cached: ``find_library`` walks the filesystem."""+ import ctypes+ import ctypes.util++ libc = ctypes.CDLL(ctypes.util.find_library("c") or "libc.dylib")+ libc.sysctlbyname.argtypes = [+ ctypes.c_char_p,+ ctypes.c_void_p,+ ctypes.POINTER(ctypes.c_size_t),+ ctypes.c_void_p,+ ctypes.c_size_t,+ ]+ libc.sysctlbyname.restype = ctypes.c_int+ return libc+++def _sysctl(name: str, as_str: bool = False) -> Any:+ try:+ import ctypes++ libc = _libc()+ if as_str:+ buf = ctypes.create_string_buffer(64)+ size = ctypes.c_size_t(len(buf))+ ok = libc.sysctlbyname(name.encode(), buf, ctypes.byref(size), None, 0) == 0+ return buf.value.decode(errors = "replace") if ok else None+ value = ctypes.c_int(0)+ size = ctypes.c_size_t(ctypes.sizeof(value))+ ok = libc.sysctlbyname(name.encode(), ctypes.byref(value), ctypes.byref(size), None, 0) == 0+ return int(value.value) if ok and value.value > 0 else None+ except Exception: # noqa: BLE001+ return None+++# M5 Pro / Max: perflevel0 "Super", perflevel1 "Performance".+_FAST_PERF_LEVELS = ("super", "performance")+++def _apple_performance_cores() -> Optional[int]:+ """macOS fast cores: perflevel0 plus lower tiers named Super/Performance (sd.cpp reads perflevel0 only).+ No SMT, so ``cpu_count() // 2`` undercounts."""+ if sys.platform != "darwin":+ return None+ levels = _sysctl("hw.nperflevels") or 0+ total = 0+ for level in range(levels):+ if level > 0:+ name = _sysctl(f"hw.perflevel{level}.name", as_str = True)+ if name is None or name.strip().lower() not in _FAST_PERF_LEVELS:+ continue+ total += _sysctl(f"hw.perflevel{level}.physicalcpu") or 0+ return total or _sysctl("hw.perflevel0.physicalcpu") or _sysctl("hw.physicalcpu")++def _default_threads() -> int:- """``UNSLOTH_CPU_THREADS``, else ``os.cpu_count() // 2`` capped by the cgroup quota ``os.cpu_count()`` ignores."""+ """``UNSLOTH_CPU_THREADS``, else macOS P-cores, else ``os.cpu_count() // 2`` capped by the cgroup quota."""try:configured = int(os.environ.get("UNSLOTH_CPU_THREADS") or 0)except ValueError:configured = 0if configured > 0:return configured+ performance_cores = _apple_performance_cores()+ if performance_cores:+ return performance_corescpu = os.cpu_count()threads = cpu // 2 if cpu else 8limit = _cgroup_cpu_limit() if sys.platform.startswith("linux") else Nonereturn max(1, min(threads, limit) if limit else threads)+def _explicit_threads() -> Optional[int]:+ """``UNSLOTH_CPU_THREADS``, the macOS fast cores, or a cgroup quota below sd.cpp's physical-core default; None+ keeps that default."""+ try:+ configured = int(os.environ.get("UNSLOTH_CPU_THREADS") or 0)+ except ValueError:+ configured = 0+ if configured > 0:+ return configured+ performance_cores = _apple_performance_cores()+ if performance_cores:+ return performance_cores+ if sys.platform.startswith("linux"):+ limit = _cgroup_cpu_limit()+ cpu = os.cpu_count()+ if limit and cpu and limit < cpu // 2:+ return max(1, limit)+ return None++def _server_binary_runnable(binary: str) -> bool:"""Best-effort probe that ``binary`` can actually execute (not just exist). Runs ``<binary>--help`` with the same runtime env the server will use, so a present but unrunnable builddiff --git a/studio/backend/core/inference/sd_cpp_engine.py b/studio/backend/core/inference/sd_cpp_engine.pyindex ecfdd23e1a8..c58b218d1b4 100644--- a/studio/backend/core/inference/sd_cpp_engine.py+++ b/studio/backend/core/inference/sd_cpp_engine.py@@ -890,6 +890,7 @@ class SdCppEngine:*,output_path: str,offload: Optional[list[str]] = None,+ threads: Optional[int] = None,verbose: bool = False,extra_args: Optional[list[str]] = None,timeout: Optional[float] = NATIVE_GENERATION_TIMEOUT_S,@@ -904,6 +905,7 @@ class SdCppEngine:params,output_path = str(self._prepare_out(output_path)),offload = offload,+ threads = threads,verbose = verbose,extra_args = extra_args,)diff --git a/studio/backend/core/inference/video.py b/studio/backend/core/inference/video.pyindex e1e28a247df..93b9bc25b22 100644--- a/studio/backend/core/inference/video.py+++ b/studio/backend/core/inference/video.py@@ -12563,11 +12563,14 @@ class VideoBackend:reuse_live_server = False,compute_bytes = compute_bytes,)+ from .sd_cpp_backend import _explicit_threadsgenerated = runtime.engine.generate_video(runtime.files,video_params,output_path = str(output_path),offload = render_flags,+ # sd.cpp's default undercounts M5 Pro / Max and ignores env / cgroup quota+ threads = _explicit_threads(),env = dict(runtime.env) or None,on_log = on_log,cancel_event = cancel,diff --git a/studio/backend/core/inference/video_minimax_h3.py b/studio/backend/core/inference/video_minimax_h3.pyindex 2ecf441d093..447cc48f69c 100644--- a/studio/backend/core/inference/video_minimax_h3.py+++ b/studio/backend/core/inference/video_minimax_h3.py@@ -2008,9 +2008,11 @@ class H3NativeServerSlot:target = abort_on_cancel, daemon = True, name = "h3-sd-server-start-cancel").start()try:+ from .sd_cpp_backend import _explicit_threadsserver.start(self.files,offload = list(flag_tuple),+ threads = _explicit_threads(),env = dict(env_pairs) or None,# A context flag on sd-server (per-run on sd-cli); same RNG keeps seeds identical across paths.extra_args = ["--rng", "cpu"],diff --git a/studio/backend/tests/test_h3_native_memory_fit.py b/studio/backend/tests/test_h3_native_memory_fit.pyindex fefbbb26124..83ac44db8cc 100644--- a/studio/backend/tests/test_h3_native_memory_fit.py+++ b/studio/backend/tests/test_h3_native_memory_fit.py@@ -757,6 +757,7 @@ class _FakeServer:files,*,offload,+ threads = None,env = None,extra_args = None,):diff --git a/studio/backend/tests/test_h3_native_server.py b/studio/backend/tests/test_h3_native_server.pyindex b116c5d9e79..2371167cf90 100644--- a/studio/backend/tests/test_h3_native_server.py+++ b/studio/backend/tests/test_h3_native_server.py@@ -411,6 +411,7 @@ def test_slot_holds_the_managed_tree_while_its_server_lives(monkeypatch):files,*,offload = None,+ threads = None,env = None,extra_args = None,):@@ -452,6 +453,7 @@ def test_slot_respawns_a_dead_server(monkeypatch):files,*,offload = None,+ threads = None,env = None,extra_args = None,):@@ -490,6 +492,7 @@ class _LifeSrv:files,*,offload = None,+ threads = None,env = None,extra_args = None,):@@ -867,6 +870,7 @@ def test_cancel_aborts_a_server_start_in_progress(monkeypatch):files,*,offload = None,+ threads = None,env = None,extra_args = None,):@@ -941,6 +945,7 @@ def test_the_tree_stays_held_until_the_server_has_exited(monkeypatch):files,*,offload = None,+ threads = None,env = None,extra_args = None,):diff --git a/studio/backend/tests/test_sd_cpp_cpu_defaults.py b/studio/backend/tests/test_sd_cpp_cpu_defaults.pyindex 022fe73f138..6662f8dbb2d 100644--- a/studio/backend/tests/test_sd_cpp_cpu_defaults.py+++ b/studio/backend/tests/test_sd_cpp_cpu_defaults.py@@ -34,7 +34,6 @@ def _cgroups(({"/user.slice/app.scope": "max 100000"}, None),({"/user.slice/app.scope": "400000 100000"}, 4),({"/user.slice/app.scope": "250000 100000"}, 3),- # A parent's quota binds too, and the tightest one wins.({"/user.slice": "200000 100000", "/user.slice/app.scope": "800000 100000"}, 2),({"/": "600000 100000"}, 6),({"/user.slice/app.scope": "garbage"}, None),@@ -85,12 +84,197 @@ def test_a_quota_caps_the_default(monkeypatch):def test_other_platforms_never_read_cgroups(monkeypatch):_host(monkeypatch, cpus = 10, platform = "darwin")+ monkeypatch.setattr(bk, "_apple_performance_cores", lambda: None)monkeypatch.setattr(bk, "_cgroup_cpu_limit", lambda: pytest.fail("no cgroups off Linux"))assert bk._default_threads() == 5[email protected]("cpus, p_cores", [(3, 3), (10, 8), (16, 12), (10, 4)])+def test_macos_uses_the_performance_core_count_not_half_the_cores(monkeypatch, cpus, p_cores):+ _host(monkeypatch, cpus = cpus, platform = "darwin")+ monkeypatch.setattr(bk, "_apple_performance_cores", lambda: p_cores)+ assert bk._default_threads() == p_cores+ monkeypatch.setenv("UNSLOTH_CPU_THREADS", "2")+ assert bk._default_threads() == 2+++def test_apple_performance_cores_is_none_off_macos(monkeypatch):+ monkeypatch.setattr(bk.sys, "platform", "linux")+ assert bk._apple_performance_cores() is None++@pytest.mark.parametrize("value, expected", [("12", 12), ("0", 4), ("x", 4), ("", 4), ("-3", 4)])def test_unsloth_cpu_threads_wins_over_the_default_and_the_quota(monkeypatch, value, expected):_host(monkeypatch, limit = 4)monkeypatch.setenv("UNSLOTH_CPU_THREADS", value)assert bk._default_threads() == expected+++_TOPOLOGIES = {+ "m1pro": {+ "hw.nperflevels": 2,+ "hw.perflevel0.name": "Performance",+ "hw.perflevel0.physicalcpu": 8,+ "hw.perflevel1.name": "Efficiency",+ "hw.perflevel1.physicalcpu": 2,+ "hw.physicalcpu": 10,+ },+ # ggml-org/llama.cpp#29782+ "m5max": {+ "hw.nperflevels": 2,+ "hw.perflevel0.name": "Super",+ "hw.perflevel0.physicalcpu": 6,+ "hw.perflevel1.name": "Performance",+ "hw.perflevel1.physicalcpu": 12,+ "hw.physicalcpu": 18,+ },+ "vm": {+ "hw.nperflevels": 1,+ "hw.perflevel0.name": "Performance",+ "hw.perflevel0.physicalcpu": 3,+ "hw.physicalcpu": 3,+ },+ "intel": {"hw.physicalcpu": 6},+ "unnamed": {+ "hw.nperflevels": 2,+ "hw.perflevel0.physicalcpu": 4,+ "hw.perflevel1.physicalcpu": 6,+ "hw.physicalcpu": 10,+ },+ # a lower tier with a name this code does not know is left out, as an unnamed one is+ "unknown_tier": {+ "hw.nperflevels": 3,+ "hw.perflevel0.name": "Super",+ "hw.perflevel0.physicalcpu": 4,+ "hw.perflevel1.name": "Performance",+ "hw.perflevel1.physicalcpu": 6,+ "hw.perflevel2.name": "Background",+ "hw.perflevel2.physicalcpu": 4,+ "hw.physicalcpu": 14,+ },+}+++ "topology, expected",+ [("m1pro", 8), ("m5max", 18), ("vm", 3), ("intel", 6), ("unnamed", 4), ("unknown_tier", 10)],+)+def test_apple_cores_count_perflevel0_and_the_named_fast_tiers(monkeypatch, topology, expected):+ table = _TOPOLOGIES[topology]+ monkeypatch.setattr(bk.sys, "platform", "darwin")+ monkeypatch.setattr(bk, "_sysctl", lambda name, as_str = False: table.get(name))+ assert bk._apple_performance_cores() == expected+++def test_the_libc_handle_is_looked_up_once_and_typed(monkeypatch):+ import ctypes+ import ctypes.util++ class _Fn:+ def __call__(self, *args):+ return 0++ class _Libc:+ def __init__(self, path):+ self.sysctlbyname = _Fn()++ calls = []+ monkeypatch.setattr(+ ctypes.util, "find_library", lambda name: calls.append(name) or "libc.dylib"+ )+ monkeypatch.setattr(ctypes, "CDLL", _Libc)+ bk._libc.cache_clear()+ try:+ for _ in range(3):+ bk._sysctl("hw.physicalcpu")+ assert calls == ["c"]+ fn = bk._libc().sysctlbyname+ assert fn.restype is ctypes.c_int and len(fn.argtypes) == 5+ finally:+ bk._libc.cache_clear()+++def test_minimax_h3_video_cli_and_server_get_the_default_thread_count(monkeypatch, tmp_path):+ """H3 runs its text encoder on the Mac CPU, so both native video paths pass Studio's count."""+ from core.inference import sd_cpp_engine as eng+ from core.inference import sd_cpp_server as srv+ from core.inference import video_minimax_h3 as h3+ from core.inference.sd_cpp_args import (+ SdCppModelFiles,+ SdCppVideoGenParams,+ build_sd_cpp_video_command,+ )++ files = SdCppModelFiles(+ diffusion_model = "/m/h3.gguf", vae = "/m/vae.safetensors", llm = "/m/te.gguf"+ )+ params = SdCppVideoGenParams(prompt = "a dog", width = 448, height = 256, num_frames = 17, fps = 24)+ cmd = build_sd_cpp_video_command("/bin/sd-cli", files, params, output_path = "/o.mp4", threads = 7)+ assert cmd[cmd.index("--threads") + 1] == "7"+ assert "--threads" not in build_sd_cpp_video_command(+ "/bin/sd-cli", files, params, output_path = "/o.mp4"+ )++ seen = {}+ engine = eng.SdCppEngine.__new__(eng.SdCppEngine)+ monkeypatch.setattr(engine, "_require_binary", lambda: "/bin/sd-cli", raising = False)+ monkeypatch.setattr(+ engine, "_run", lambda cmd, out, **kw: seen.setdefault("cmd", cmd), raising = False+ )+ engine.generate_video(files, params, output_path = str(tmp_path / "o.mp4"), threads = 5)+ assert seen["cmd"][seen["cmd"].index("--threads") + 1] == "5"++ monkeypatch.setattr(bk, "_explicit_threads", lambda: 11)+ started = {}++ class _Srv:+ def __init__(self, binary):+ self.alive = False++ def start(+ self,+ files,+ *,+ offload = None,+ threads = None,+ env = None,+ extra_args = None,+ ):+ started["threads"] = threads+ self.alive = True++ def is_alive(self):+ return self.alive++ def stop(self):+ self.alive = False++ monkeypatch.setattr(srv, "SdCppServer", _Srv)+ monkeypatch.setattr("core.inference.sd_cpp_engine.is_managed_binary", lambda b: False)+ slot = h3.H3NativeServerSlot("/bin/sd-server", files, ("--offload-to-cpu",))+ slot.get()+ slot.stop()+ assert started["threads"] == 11+++def test_h3_video_keeps_sd_cpps_default_threads_unless_studio_knows_better(monkeypatch):+ # Linux / Windows with no override and no tighter quota: no --threads, as on main (sd.cpp counts physical cores)+ monkeypatch.delenv("UNSLOTH_CPU_THREADS", raising = False)+ monkeypatch.setattr(bk, "_apple_performance_cores", lambda: None)+ monkeypatch.setattr(bk.os, "cpu_count", lambda: 32)+ monkeypatch.setattr(bk.sys, "platform", "linux")+ monkeypatch.setattr(bk, "_cgroup_cpu_limit", lambda *a, **k: None)+ assert bk._explicit_threads() is None+ monkeypatch.setattr(+ bk, "_cgroup_cpu_limit", lambda *a, **k: 16+ ) # equal to the default: still sd.cpp's own+ assert bk._explicit_threads() is None+ monkeypatch.setattr(bk, "_cgroup_cpu_limit", lambda *a, **k: 6) # a tighter container quota+ assert bk._explicit_threads() == 6+ monkeypatch.setattr(bk.sys, "platform", "win32")+ assert bk._explicit_threads() is None+ monkeypatch.setenv("UNSLOTH_CPU_THREADS", "12")+ assert bk._explicit_threads() == 12+ monkeypatch.delenv("UNSLOTH_CPU_THREADS")+ monkeypatch.setattr(bk, "_apple_performance_cores", lambda: 10)+ assert bk._explicit_threads() == 10