Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio diffusion: test-launch sd.cpp binaries without the Windows missing-DLL box (#16031)

oobabooga982ab34fda66
diff --git a/studio/backend/core/inference/sd_cpp_backend.py b/studio/backend/core/inference/sd_cpp_backend.py
index bf05e7bad63..9cc7f868491 100644
--- a/studio/backend/core/inference/sd_cpp_backend.py
+++ b/studio/backend/core/inference/sd_cpp_backend.py
@@ -107,7 +107,10 @@ from core.inference.sd_cpp_server import SdCppServer
from loggers import get_logger
from utils.gpu_memory_events import invalidates_gpu_memory as _invalidates_gpu_memory
from utils.account_context import account_thread, current_account_id
-from utils.subprocess_compat import windows_hidden_subprocess_kwargs
+from utils.subprocess_compat import (
+ windows_hidden_subprocess_kwargs,
+ windows_loader_error_boxes_suppressed,
+)
logger = get_logger(__name__)
@@ -440,13 +443,14 @@ def _server_binary_runnable(binary: str) -> bool:
import subprocess
try:
- proc = subprocess.run(
- [binary, "--help"],
- capture_output = True,
- timeout = 20,
- env = runtime_env(binary),
- **windows_hidden_subprocess_kwargs(),
- )
+ with windows_loader_error_boxes_suppressed():
+ proc = subprocess.run(
+ [binary, "--help"],
+ capture_output = True,
+ timeout = 20,
+ env = runtime_env(binary),
+ **windows_hidden_subprocess_kwargs(),
+ )
except OSError:
return False # cannot exec at all (wrong arch / no execute bit / missing loader)
except Exception: # noqa: BLE001 -- don't block on a flaky probe
@@ -484,16 +488,17 @@ def _sd_cpp_probe_output(binary: str, *args: str) -> Optional[str]:
import subprocess
try:
- proc = subprocess.run(
- [binary, *args],
- capture_output = True,
- text = True,
- encoding = "utf-8",
- errors = "replace",
- timeout = 20,
- env = runtime_env(binary),
- **windows_hidden_subprocess_kwargs(),
- )
+ with windows_loader_error_boxes_suppressed():
+ proc = subprocess.run(
+ [binary, *args],
+ capture_output = True,
+ text = True,
+ encoding = "utf-8",
+ errors = "replace",
+ timeout = 20,
+ env = runtime_env(binary),
+ **windows_hidden_subprocess_kwargs(),
+ )
except Exception: # noqa: BLE001 -- cannot exec / timeout: "cannot tell"
return None
if proc.returncode != 0:
diff --git a/studio/backend/core/inference/sd_cpp_engine.py b/studio/backend/core/inference/sd_cpp_engine.py
index bd9edb5849d..40ba44151c4 100644
--- a/studio/backend/core/inference/sd_cpp_engine.py
+++ b/studio/backend/core/inference/sd_cpp_engine.py
@@ -28,7 +28,10 @@ from utils.process_lifetime import (
is_process_shutting_down,
)
from utils.native_path_leases import child_env_without_native_path_secret
-from utils.subprocess_compat import windows_hidden_subprocess_kwargs
+from utils.subprocess_compat import (
+ windows_hidden_subprocess_kwargs,
+ windows_loader_error_boxes_suppressed,
+)
from core.inference.sd_cpp_args import (
SdCppGenParams,
SdCppModelFiles,
@@ -447,17 +450,18 @@ def sd_cpp_binary_identifies(binary: str) -> bool:
return cached[0]
returncode = None
try:
- result = subprocess.run(
- [binary, "--help"],
- capture_output = True,
- text = True,
- encoding = "utf-8",
- errors = "replace",
- timeout = 10,
- check = False,
- env = runtime_env(binary),
- **windows_hidden_subprocess_kwargs(),
- )
+ with windows_loader_error_boxes_suppressed():
+ result = subprocess.run(
+ [binary, "--help"],
+ capture_output = True,
+ text = True,
+ encoding = "utf-8",
+ errors = "replace",
+ timeout = 10,
+ check = False,
+ env = runtime_env(binary),
+ **windows_hidden_subprocess_kwargs(),
+ )
help_text = (result.stdout or "") + "\n" + (result.stderr or "")
returncode = result.returncode
except (OSError, subprocess.SubprocessError):
@@ -652,16 +656,18 @@ class SdCppEngine:
if self._version is not None:
return self._version
try:
- res = subprocess.run(
- [self.binary, "--version"],
- capture_output = True,
- text = True,
- encoding = "utf-8",
- errors = "replace",
- timeout = timeout,
- check = False,
- env = runtime_env(self.binary),
- )
+ with windows_loader_error_boxes_suppressed():
+ res = subprocess.run(
+ [self.binary, "--version"],
+ capture_output = True,
+ text = True,
+ encoding = "utf-8",
+ errors = "replace",
+ timeout = timeout,
+ check = False,
+ env = runtime_env(self.binary),
+ **windows_hidden_subprocess_kwargs(),
+ )
except (OSError, subprocess.SubprocessError):
return None
if res.returncode != 0:
diff --git a/studio/backend/tests/test_sd_cpp_probe_error_mode.py b/studio/backend/tests/test_sd_cpp_probe_error_mode.py
new file mode 100644
index 00000000000..d3f9d9208f7
--- /dev/null
+++ b/studio/backend/tests/test_sd_cpp_probe_error_mode.py
@@ -0,0 +1,230 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""sd.cpp test-launches run with SEM_FAILCRITICALERRORS, so a missing DLL exits instead of opening a box."""
+
+from __future__ import annotations
+
+import ctypes
+import subprocess
+import sys
+import threading
+import types
+
+import pytest
+
+from core.inference import sd_cpp_backend as bk
+from core.inference import sd_cpp_engine as eng
+from utils import subprocess_compat
+
+_STATUS_DLL_NOT_FOUND = 0xC0000135
+
+
+def error_mode(monkeypatch):
+ """A fake kernel32 on win32; ``state["value"]`` is the process error mode."""
+ state = {"value": 0x8000}
+
+ class _Kernel32:
+ def GetErrorMode(self):
+ return state["value"]
+
+ def SetErrorMode(self, value):
+ previous, state["value"] = state["value"], value
+ return previous
+
+ monkeypatch.setattr(sys, "platform", "win32")
+ monkeypatch.setattr(ctypes, "WinDLL", lambda _name: _Kernel32(), raising = False)
+ yield state
+ # A failed test can leave a holder behind; the record is process-wide.
+ subprocess_compat._ERROR_MODE.held.clear()
+
+
+def _record_mode(
+ monkeypatch,
+ state,
+ returncode = _STATUS_DLL_NOT_FOUND,
+):
+ seen = []
+
+ def fake_run(*_args, **_kwargs):
+ seen.append(state["value"])
+ return types.SimpleNamespace(returncode = returncode, stdout = "", stderr = "")
+
+ # The backend probes import subprocess inside the function; the engine at module level.
+ monkeypatch.setattr(subprocess, "run", fake_run)
+ return seen
+
+
+def test_the_bit_is_or_ed_in_and_the_previous_mode_restored(error_mode):
+ with subprocess_compat.windows_loader_error_boxes_suppressed():
+ assert error_mode["value"] == 0x8001
+ assert error_mode["value"] == 0x8000
+
+
+def test_nested_holders_keep_the_bit_until_the_last_one_leaves(error_mode):
+ outer = subprocess_compat.windows_loader_error_boxes_suppressed()
+ inner = subprocess_compat.windows_loader_error_boxes_suppressed()
+ outer.__enter__()
+ inner.__enter__()
+ outer.__exit__(None, None, None)
+ # A plain save/restore would clear the bit here while the inner probe is still spawning.
+ assert error_mode["value"] == 0x8001
+ inner.__exit__(None, None, None)
+ assert error_mode["value"] == 0x8000
+
+
+def test_concurrent_holders_never_see_the_bit_cleared(error_mode):
+ barrier = threading.Barrier(8)
+ cleared = []
+
+ def hold():
+ with subprocess_compat.windows_loader_error_boxes_suppressed():
+ barrier.wait()
+ cleared.append(not error_mode["value"] & 1)
+
+ threads = [threading.Thread(target = hold) for _ in range(8)]
+ for t in threads:
+ t.start()
+ for t in threads:
+ t.join()
+ assert cleared == [False] * 8
+ assert error_mode["value"] == 0x8000
+
+
+def test_the_mode_is_restored_when_the_launch_raises(error_mode):
+ with pytest.raises(subprocess.TimeoutExpired):
+ with subprocess_compat.windows_loader_error_boxes_suppressed():
+ raise subprocess.TimeoutExpired("sd-server", 20)
+ assert error_mode["value"] == 0x8000
+
+
+def test_off_windows_the_mode_is_untouched(monkeypatch):
+ monkeypatch.setattr(sys, "platform", "linux")
+ monkeypatch.setattr(
+ ctypes, "WinDLL", lambda _name: pytest.fail("kernel32 loaded off Windows"), raising = False
+ )
+ with subprocess_compat.windows_loader_error_boxes_suppressed():
+ pass
+
+
+def _installer_and_backend_helpers():
+ installer = bk._installer_module()
+ # The installer loads its own copy of this module; the record must still be one per process.
+ assert sys.modules["backend.utils.subprocess_compat"] is not subprocess_compat
+ return (
+ installer._windows_error_mode_suppressed,
+ subprocess_compat.windows_loader_error_boxes_suppressed,
+ )
+
+
+def test_installer_leaving_first_keeps_the_backend_probes_bit(error_mode):
+ installer_scope, backend_scope = _installer_and_backend_helpers()
+ install = installer_scope()
+ probe = backend_scope()
+ install.__enter__()
+ probe.__enter__()
+ assert error_mode["value"] == 0x8003
+ install.__exit__(None, None, None)
+ # The install's crash-box flags go; the probe still holds SEM_FAILCRITICALERRORS.
+ assert error_mode["value"] == 0x8001
+ with backend_scope():
+ assert error_mode["value"] == 0x8001
+ probe.__exit__(None, None, None)
+ assert error_mode["value"] == 0x8000
+
+
+def test_backend_probe_leaving_first_keeps_the_installers_flags(error_mode):
+ installer_scope, backend_scope = _installer_and_backend_helpers()
+ probe = backend_scope()
+ install = installer_scope()
+ probe.__enter__()
+ install.__enter__()
+ probe.__exit__(None, None, None)
+ assert error_mode["value"] == 0x8003
+ install.__exit__(None, None, None)
+ # Nothing left set for good, SEM_NOGPFAULTERRORBOX included.
+ assert error_mode["value"] == 0x8000
+
+
+def test_server_runnable_probe_launches_with_the_bit_and_reads_dll_not_found(
+ error_mode, monkeypatch
+):
+ seen = _record_mode(monkeypatch, error_mode)
+ assert bk._server_binary_runnable("sd-server.exe") is False
+ assert seen == [0x8001]
+ assert error_mode["value"] == 0x8000
+
+
+def test_probe_output_launches_with_the_bit(error_mode, monkeypatch):
+ seen = _record_mode(monkeypatch, error_mode)
+ assert bk._sd_cpp_probe_output("sd-cli.exe", "--help") is None
+ assert seen == [0x8001]
+ assert error_mode["value"] == 0x8000
+
+
+def test_engine_identity_probe_launches_with_the_bit(error_mode, monkeypatch, tmp_path):
+ binary = tmp_path / "sd-cli.exe"
+ binary.write_bytes(b"stub")
+ seen = _record_mode(monkeypatch, error_mode)
+ assert eng.sd_cpp_binary_identifies(str(binary)) is False
+ assert seen == [0x8001]
+ assert error_mode["value"] == 0x8000
+
+
+def test_engine_version_probe_launches_with_the_bit(error_mode, monkeypatch, tmp_path):
+ binary = tmp_path / "sd-cli.exe"
+ binary.write_bytes(b"stub")
+ seen = _record_mode(monkeypatch, error_mode)
+ engine = eng.SdCppEngine(binary = str(binary))
+ monkeypatch.setattr(engine, "is_available", lambda: True)
+ assert engine.version() is None
+ assert seen == [0x8001]
+ assert error_mode["value"] == 0x8000
+
+
+_windows_only = pytest.mark.skipif(sys.platform != "win32", reason = "real kernel32")
+
+
+def _child_error_mode() -> int:
+ out = subprocess.run(
+ [sys.executable, "-c", "import ctypes; print(ctypes.WinDLL('kernel32').GetErrorMode())"],
+ capture_output = True,
+ text = True,
+ check = True,
+ timeout = 60,
+ )
+ return int(out.stdout.strip())
+
+
+@_windows_only
+def test_a_real_child_inherits_the_bit_only_while_held():
+ kernel32 = ctypes.WinDLL("kernel32")
+ original = kernel32.GetErrorMode()
+ kernel32.SetErrorMode(original & ~1)
+ try:
+ assert not _child_error_mode() & 1
+ with subprocess_compat.windows_loader_error_boxes_suppressed():
+ assert _child_error_mode() & 1
+ assert not _child_error_mode() & 1
+ finally:
+ kernel32.SetErrorMode(original)
+
+
+@_windows_only
+def test_a_real_binary_missing_its_dll_is_reported_unrunnable(tmp_path, monkeypatch):
+ # python.exe copied away from pythonXY.dll, PATH cut to System32: a loader failure like ROCm without HIP.
+ import os
+ import shutil
+
+ binary = tmp_path / "sd-server.exe"
+ shutil.copy(getattr(sys, "_base_executable", sys.executable), binary)
+ monkeypatch.setenv("PATH", os.path.join(os.environ["SystemRoot"], "System32"))
+ with subprocess_compat.windows_loader_error_boxes_suppressed():
+ rc = subprocess.run([str(binary), "--help"], capture_output = True, timeout = 60).returncode
+ assert rc & 0xFFFFFFFF == _STATUS_DLL_NOT_FOUND
+ assert bk._server_binary_runnable(str(binary)) is False
+
+
+if __name__ == "__main__":
+ raise SystemExit(pytest.main([__file__, "-v"]))
diff --git a/studio/backend/utils/subprocess_compat.py b/studio/backend/utils/subprocess_compat.py
index ea49d315d16..c2443e8d400 100644
--- a/studio/backend/utils/subprocess_compat.py
+++ b/studio/backend/utils/subprocess_compat.py
@@ -3,8 +3,30 @@
"""Cross-platform subprocess helpers for the Unsloth Studio backend."""
+import contextlib
import subprocess
import sys
+import threading
+import types
+from typing import Iterator
+
+_SEM_FAILCRITICALERRORS = 0x0001
+
+
+def _process_error_mode_state() -> types.ModuleType:
+ """One per process: the installer imports this file as backend.utils.subprocess_compat, the backend as utils.subprocess_compat."""
+ name = "_unsloth_windows_error_mode"
+ state = sys.modules.get(name)
+ if state is None:
+ fresh = types.ModuleType(name)
+ fresh.lock = threading.Lock()
+ fresh.held = {} # token -> flags
+ fresh.before = 0
+ state = sys.modules.setdefault(name, fresh)
+ return state
+
+
+_ERROR_MODE = _process_error_mode_state()
def windows_hidden_subprocess_kwargs() -> dict[str, object]:
@@ -27,3 +49,37 @@ def windows_hidden_subprocess_kwargs() -> dict[str, object]:
kwargs["startupinfo"] = startupinfo
return kwargs
+
+
+def _apply_held_error_mode(kernel32) -> None:
+ flags = 0
+ for held in _ERROR_MODE.held.values():
+ flags |= held
+ kernel32.SetErrorMode(_ERROR_MODE.before | flags)
+
+
+def windows_loader_error_boxes_suppressed(flags: int = _SEM_FAILCRITICALERRORS) -> Iterator[None]:
+ """The child inherits SEM_FAILCRITICALERRORS, so a missing DLL exits 0xC0000135 instead of a modal box.
+ Process-wide: mode = the one before the first holder | every holder's flags; the last out restores it."""
+ if sys.platform != "win32":
+ yield
+ return
+ try:
+ import ctypes
+ kernel32 = ctypes.WinDLL("kernel32")
+ except Exception: # noqa: BLE001 -- best effort; the launch still runs
+ yield
+ return
+ token = object()
+ with _ERROR_MODE.lock:
+ if not _ERROR_MODE.held:
+ _ERROR_MODE.before = kernel32.GetErrorMode()
+ _ERROR_MODE.held[token] = flags
+ _apply_held_error_mode(kernel32)
+ try:
+ yield
+ finally:
+ with _ERROR_MODE.lock:
+ del _ERROR_MODE.held[token]
+ _apply_held_error_mode(kernel32)
diff --git a/studio/install_sd_cpp_prebuilt.py b/studio/install_sd_cpp_prebuilt.py
index 96b0bdf02b7..5e3b821dcc5 100644
--- a/studio/install_sd_cpp_prebuilt.py
+++ b/studio/install_sd_cpp_prebuilt.py
@@ -24,11 +24,13 @@ from typing import Optional, Sequence
if __package__:
from .backend.utils.auth_safe import auth_safe_open
+ from .backend.utils.subprocess_compat import windows_loader_error_boxes_suppressed
else:
_STUDIO_DIR = os.path.dirname(os.path.abspath(__file__))
if _STUDIO_DIR not in sys.path:
sys.path.insert(0, _STUDIO_DIR)
from backend.utils.auth_safe import auth_safe_open
+ from backend.utils.subprocess_compat import windows_loader_error_boxes_suppressed
# GPU hosts run diffusers, so only CPU/Apple assets are needed.
DEFAULT_REPO = "unslothai/stable-diffusion.cpp"
@@ -784,31 +786,10 @@ def _probe_env(binary: Path) -> dict:
def _windows_error_mode_suppressed():
- """On Windows, suppress the missing-DLL dialog so a refusal returns an exit status."""
- import contextlib
-
- @contextlib.contextmanager
- def _cm():
- if sys.platform != "win32":
- yield
- return
- try:
- import ctypes
- kernel32 = ctypes.windll.kernel32 # type: ignore[attr-defined]
- # SEM_FAILCRITICALERRORS | SEM_NOGPFAULTERRORBOX | SEM_NOOPENFILEERRORBOX
- previous = kernel32.SetErrorMode(0x0001 | 0x0002 | 0x8000)
- except Exception: # noqa: BLE001 -- best effort; the probe still runs
- yield
- return
- try:
- yield
- finally:
- try:
- kernel32.SetErrorMode(previous)
- except Exception: # noqa: BLE001
- pass
-
- return _cm()
+ """On Windows, suppress the missing-DLL dialog so a refusal returns an exit status. Shares the
+ backend's process-wide record, since the backend's own probes can run during an install."""
+ # SEM_FAILCRITICALERRORS | SEM_NOGPFAULTERRORBOX | SEM_NOOPENFILEERRORBOX
+ return windows_loader_error_boxes_suppressed(0x0001 | 0x0002 | 0x8000)
def probe_binary_starts(binary: Path, *, timeout: float = 30.0) -> Optional[str]: