Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: initialise the CUDA driver off the GIL so load-progress polls stay responsive (#14452)

Daniel Hanfe8aab30b013
diff --git a/studio/backend/tests/test_cuda_prime_off_gil.py b/studio/backend/tests/test_cuda_prime_off_gil.py
new file mode 100644
index 00000000000..8858e8815a1
--- /dev/null
+++ b/studio/backend/tests/test_cuda_prime_off_gil.py
@@ -0,0 +1,317 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+"""Hardware detection must not hold the GIL across the CUDA driver init.
+
+torch.cuda.is_available() reaches cuInit inside cudaGetDeviceCount with the GIL held, for seconds on a
+multi-GPU host, so every other thread (the event loop answering /images/load-progress included) froze
+for that long during the first load. Detection now runs cuInit through ctypes first, which releases
+the GIL, and torch's own call then returns at once."""
+
+import os
+import subprocess
+import sys
+import textwrap
+import threading
+import time
+from pathlib import Path
+from types import SimpleNamespace
+
+import pytest
+
+from utils import cuda_driver_prime as prime
+from utils.hardware import hardware as hw
+
+BACKEND = Path(__file__).resolve().parents[1]
+
+
[email protected](autouse = True)
+def isolated(monkeypatch):
+ monkeypatch.setattr(os, "environ", os.environ.copy())
+ for name in ("PYTORCH_NVML_BASED_CUDA_CHECK", "UNSLOTH_STUDIO_DISABLE_CUDA_PRIME"):
+ monkeypatch.delenv(name, raising = False)
+ monkeypatch.setenv("CUDA_VISIBLE_DEVICES", "0")
+ prime._reset_for_tests()
+ yield
+ prime._reset_for_tests()
+
+
+class _FakeCuInit:
+ """Stands in for a ctypes foreign function (argtypes / restype are set on it)."""
+
+ def __init__(
+ self,
+ calls,
+ gate = None,
+ ):
+ self.calls = calls
+ self.gate = gate
+
+ def __call__(self, flags):
+ if self.gate is not None:
+ self.gate.wait(5)
+ self.calls.append(("cuInit", flags))
+ return 0
+
+
+class _FakeLib:
+ def __init__(
+ self,
+ calls,
+ gate = None,
+ ):
+ self.cuInit = _FakeCuInit(calls, gate)
+
+
+def _fake_torch(
+ calls,
+ *,
+ cuda = "13.0",
+ hip = None,
+ initialized = False,
+):
+ def is_available():
+ calls.append(("is_available",))
+ return True
+
+ return SimpleNamespace(
+ version = SimpleNamespace(cuda = cuda, hip = hip),
+ cuda = SimpleNamespace(is_available = is_available, is_initialized = lambda: initialized),
+ )
+
+
+def fake_cdll(monkeypatch):
+ import ctypes
+
+ calls, opened, gate = [], [], []
+
+ def cdll(name):
+ opened.append(name)
+ return _FakeLib(calls, gate[0] if gate else None)
+
+ monkeypatch.setattr(ctypes, "CDLL", cdll)
+ monkeypatch.setattr(prime.sys, "platform", "linux")
+ return calls, opened, gate
+
+
+def test_prime_runs_cuinit_once(fake_cdll):
+ calls, opened, _gate = fake_cdll
+ torch = _fake_torch(calls)
+ assert prime.prime_cuda_driver(torch) is True
+ assert calls == [("cuInit", 0)]
+ assert opened == ["libcuda.so.1"]
+ # Once per process.
+ assert prime.prime_cuda_driver(torch) is False
+ assert calls == [("cuInit", 0)]
+
+
+ "torch_kw, env",
+ [
+ ({"cuda": None}, {}), # CPU build
+ ({"hip": "7.1"}, {}), # ROCm build
+ ({"initialized": True}, {}), # torch already initialised CUDA
+ ({}, {"PYTORCH_NVML_BASED_CUDA_CHECK": "1"}), # user opted out of cuInit here
+ ({}, {"UNSLOTH_STUDIO_DISABLE_CUDA_PRIME": "1"}), # kill switch
+ ({}, {"CUDA_VISIBLE_DEVICES": ""}), # CUDA hidden
+ ({}, {"CUDA_VISIBLE_DEVICES": "-1"}),
+ ],
+)
+def test_prime_skips(fake_cdll, monkeypatch, torch_kw, env):
+ calls, opened, _gate = fake_cdll
+ for key, value in env.items():
+ monkeypatch.setenv(key, value)
+ assert prime.prime_cuda_driver(_fake_torch(calls, **torch_kw)) is False
+ assert calls == [] and opened == []
+
+
+def test_prime_skips_macos(fake_cdll, monkeypatch):
+ calls, opened, _gate = fake_cdll
+ monkeypatch.setattr(prime.sys, "platform", "darwin")
+ assert prime.prime_cuda_driver(_fake_torch(calls)) is False
+ assert opened == []
+
+
+def test_prime_never_imports_torch(fake_cdll, monkeypatch):
+ calls, opened, _gate = fake_cdll
+ monkeypatch.delitem(sys.modules, "torch", raising = False)
+ assert prime.prime_cuda_driver() is False
+ assert opened == [] and "torch" not in sys.modules
+
+
+def test_prime_never_raises_without_a_driver(monkeypatch):
+ import ctypes
+
+ def cdll(name):
+ raise OSError(f"{name}: cannot open shared object file")
+
+ monkeypatch.setattr(ctypes, "CDLL", cdll)
+ monkeypatch.setattr(prime.sys, "platform", "linux")
+ assert prime.prime_cuda_driver(_fake_torch([])) is False
+
+
+def test_concurrent_caller_waits_for_the_init_in_flight(fake_cdll):
+ """A second thread must not fall through to torch's GIL-holding probe while the init runs."""
+ calls, _opened, gate = fake_cdll
+ release = threading.Event()
+ gate.append(release)
+ torch = _fake_torch(calls)
+ first = threading.Thread(target = prime.prime_cuda_driver, args = (torch,))
+ first.start()
+ while not prime._claimed:
+ time.sleep(0.001)
+ second_done = threading.Event()
+
+ def second():
+ prime.prime_cuda_driver(torch)
+ torch.cuda.is_available()
+ second_done.set()
+
+ t2 = threading.Thread(target = second)
+ t2.start()
+ time.sleep(0.1)
+ assert not second_done.is_set() and calls == []
+ release.set()
+ first.join(5)
+ t2.join(5)
+ assert calls == [("cuInit", 0), ("is_available",)]
+
+
[email protected](not hasattr(os, "fork"), reason = "needs os.fork")
+def test_a_child_forked_during_the_init_does_not_wait_for_it(fake_cdll):
+ calls, _opened, gate = fake_cdll
+ release = threading.Event()
+ gate.append(release)
+ torch = _fake_torch(calls)
+ first = threading.Thread(target = prime.prime_cuda_driver, args = (torch,))
+ first.start()
+ while not prime._claimed:
+ time.sleep(0.001)
+ try:
+ pid = os.fork()
+ if pid == 0: # child: the parent's init thread is gone; this must return at once
+ started = time.monotonic()
+ ran = prime.prime_cuda_driver(torch)
+ os._exit(0 if (not ran and time.monotonic() - started < 2.0) else 1)
+ deadline = time.monotonic() + 10
+ status = None
+ while time.monotonic() < deadline:
+ done, status = os.waitpid(pid, os.WNOHANG)
+ if done:
+ break
+ time.sleep(0.05)
+ else:
+ os.kill(pid, 9)
+ os.waitpid(pid, 0)
+ pytest.fail("the forked child waited on the parent's cuInit")
+ assert os.waitstatus_to_exitcode(status) == 0
+ finally:
+ release.set()
+ first.join(5)
+
+
+def test_detection_primes_before_torch_probe(fake_cdll, monkeypatch):
+ calls, _opened, _gate = fake_cdll
+ torch = _fake_torch(calls)
+ monkeypatch.setitem(sys.modules, "torch", torch)
+ monkeypatch.setattr(hw, "_has_torch", lambda: True)
+ monkeypatch.setattr(hw, "_ze_affinity_mask", lambda: None)
+ seen = []
+
+ def is_available():
+ seen.append(list(calls))
+ raise RuntimeError("stop") # only the order up to the probe matters
+
+ torch.cuda.is_available = is_available
+ with hw._DETECT_LOCK:
+ try:
+ hw._detect_hardware_locked()
+ except Exception:
+ pass
+ assert seen and seen[0] == [("cuInit", 0)]
+
+
+def test_diffusers_import_window_primes(fake_cdll, monkeypatch):
+ from utils import torch_warmup
+
+ calls, _opened, _gate = fake_cdll
+ monkeypatch.setitem(sys.modules, "torch", _fake_torch(calls))
+ monkeypatch.setattr(torch_warmup, "ensure_dynamo_imported", lambda **_k: True)
+ monkeypatch.setattr(torch_warmup, "claim_media_import_window", lambda _log: None)
+ assert torch_warmup.close_dynamo_import_window(torch_warmup.logger) is True
+ assert calls == [("cuInit", 0)]
+
+
+def test_torch_stack_gate_primes(fake_cdll, monkeypatch):
+ from utils import torch_warmup
+
+ calls, _opened, _gate = fake_cdll
+ monkeypatch.setitem(sys.modules, "torch", _fake_torch(calls))
+ monkeypatch.setattr(torch_warmup, "_dynamo_done", True)
+ assert torch_warmup.gate_torch_stack_import("test") is True
+ assert calls == [("cuInit", 0)]
+
+
+def test_torch_stack_gate_kill_switch_still_primes(fake_cdll, monkeypatch):
+ from utils import torch_warmup
+
+ calls, _opened, _gate = fake_cdll
+ monkeypatch.setitem(sys.modules, "torch", _fake_torch(calls))
+ monkeypatch.setattr(torch_warmup, "_dynamo_done", False)
+ monkeypatch.setenv(torch_warmup.DYNAMO_GATE_DISABLE_ENV_VAR, "1")
+ assert torch_warmup.gate_torch_stack_import("test") is False
+ assert calls == [("cuInit", 0)]
+
+
+_GIL_PROBE = textwrap.dedent(
+ """
+ import sys, threading, time
+ sys.path.insert(0, sys.argv[1])
+ import torch
+ from utils.hardware import hardware as hw
+ gaps, stop = [], threading.Event()
+ def tick():
+ last = time.perf_counter()
+ while not stop.is_set():
+ time.sleep(0.005)
+ now = time.perf_counter(); gaps.append(now - last); last = now
+ t = threading.Thread(target = tick, daemon = True); t.start(); time.sleep(0.2)
+ with hw._DETECT_LOCK:
+ hw._detect_hardware_locked()
+ torch.cuda.is_available()
+ stop.set(); t.join()
+ print(f"MAXGAP {max(gaps):.3f} {torch.cuda.is_initialized()}")
+ """
+)
+
+
+def _real_cuda_host() -> bool:
+ if os.environ.get("CUDA_VISIBLE_DEVICES", "x").strip() in ("", "-1"):
+ return False
+ try:
+ import ctypes
+
+ import torch
+
+ if not torch.version.cuda or getattr(torch.version, "hip", None):
+ return False
+ ctypes.CDLL("libcuda.so.1")
+ except Exception:
+ return False
+ return sys.platform.startswith("linux")
+
+
[email protected](not _real_cuda_host(), reason = "needs an NVIDIA driver and a CUDA torch")
+def test_real_detection_keeps_other_threads_running():
+ """A fresh process (cold driver) runs detection while a 5 ms ticker thread measures its largest gap.
+ Before: the gap is the whole cuInit (0.4-6.8 s measured on B200 hosts). After: under 20 ms."""
+ out = subprocess.run(
+ [sys.executable, "-c", _GIL_PROBE, str(BACKEND)],
+ capture_output = True,
+ text = True,
+ timeout = 300,
+ env = {**os.environ, "UNSLOTH_STUDIO_DISABLE_CUDA_PRIME": ""},
+ )
+ line = [l for l in out.stdout.splitlines() if l.startswith("MAXGAP")]
+ assert line, out.stdout[-2000:] + out.stderr[-2000:]
+ gap = float(line[-1].split()[1])
+ assert gap < 0.25, f"another thread was frozen {gap:.2f}s during hardware detection"
diff --git a/studio/backend/utils/cuda_driver_prime.py b/studio/backend/utils/cuda_driver_prime.py
new file mode 100644
index 00000000000..d12a687b1f1
--- /dev/null
+++ b/studio/backend/utils/cuda_driver_prime.py
@@ -0,0 +1,111 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Run ``cuInit`` via ctypes (releases the GIL) before torch's first CUDA probe, which holds the GIL for the
+whole multi-second driver init and freezes the event loop. Idempotent; never raises. Concurrent callers wait on an
+Event rather than entering torch's GIL-holding call. ``UNSLOTH_STUDIO_DISABLE_CUDA_PRIME=1`` disables it.
+"""
+
+from __future__ import annotations
+
+import os
+import sys
+import threading
+
+DISABLE_ENV_VAR = "UNSLOTH_STUDIO_DISABLE_CUDA_PRIME"
+
+_claim_lock = threading.Lock()
+_claimed = False
+_done = threading.Event()
+
+
+def _driver_names() -> tuple[str, ...]:
+ if sys.platform.startswith("linux"):
+ return ("libcuda.so.1", "libcuda.so")
+ if sys.platform == "win32":
+ return ("nvcuda.dll",)
+ return ()
+
+
+def _should_prime(torch_module) -> bool:
+ if os.environ.get(DISABLE_ENV_VAR) == "1":
+ return False
+ # The user asked torch to answer from NVML so the process is not cuInit'ed here (fork safety).
+ if os.environ.get("PYTORCH_NVML_BASED_CUDA_CHECK") == "1":
+ return False
+ cvd = os.environ.get("CUDA_VISIBLE_DEVICES")
+ if cvd is not None and cvd.strip() in ("", "-1"):
+ return False
+ version = getattr(torch_module, "version", None)
+ if not getattr(version, "cuda", None) or getattr(version, "hip", None):
+ return False
+ try:
+ return not torch_module.cuda.is_initialized()
+ except Exception:
+ return False
+
+
+def _cu_init() -> bool:
+ import ctypes
+ for name in _driver_names():
+ try:
+ lib = ctypes.CDLL(name)
+ except OSError:
+ continue
+ try:
+ fn = lib.cuInit
+ fn.argtypes = [ctypes.c_uint]
+ fn.restype = ctypes.c_int
+ fn(0)
+ except Exception:
+ return False
+ return True
+ return False
+
+
+def prime_cuda_driver(torch_module = None) -> bool:
+ """Run ``cuInit`` off the GIL once per process, before torch's first CUDA probe. True iff this call ran it."""
+ global _claimed
+ if _done.is_set():
+ return False
+ with _claim_lock:
+ first = not _claimed
+ _claimed = True
+ if not first:
+ _done.wait()
+ return False
+ try:
+ if not _driver_names():
+ return False
+ if torch_module is None:
+ # Never imports torch: a caller still inside `import torch` would deadlock its waiters.
+ torch_module = sys.modules.get("torch")
+ if torch_module is None:
+ return False
+ if not _should_prime(torch_module):
+ return False
+ return _cu_init()
+ except Exception:
+ return False
+ finally:
+ _done.set()
+
+
+def _after_fork_in_child() -> None:
+ # The thread that would set _done exists only in the parent, and cuInit is not fork-safe: the child never primes.
+ global _claim_lock, _claimed, _done
+ _claim_lock = threading.Lock()
+ _claimed = True
+ _done = threading.Event()
+ _done.set()
+
+
+if hasattr(os, "register_at_fork"):
+ os.register_at_fork(after_in_child = _after_fork_in_child)
+
+
+def _reset_for_tests() -> None:
+ global _claimed
+ with _claim_lock:
+ _claimed = False
+ _done.clear()
diff --git a/studio/backend/utils/hardware/hardware.py b/studio/backend/utils/hardware/hardware.py
index 406469665aa..0d20775edff 100644
--- a/studio/backend/utils/hardware/hardware.py
+++ b/studio/backend/utils/hardware/hardware.py
@@ -1563,6 +1563,12 @@ def _detect_hardware_locked() -> DeviceType:
cvd = os.environ.get("CUDA_VISIBLE_DEVICES")
cuda_hidden = cvd is not None and cvd.strip() in ("", "-1")
force_xpu = os.environ.get("UNSLOTH_FORCE_XPU") == "1"
+ # cuInit off the GIL first: torch's probe below holds it for the whole driver init.
+ try:
+ from utils.cuda_driver_prime import prime_cuda_driver
+ prime_cuda_driver(torch)
+ except Exception: # noqa: BLE001 - never fatal (also loaded standalone, without the package)
+ pass
try:
cuda_unavailable = not torch.cuda.is_available()
except Exception:
diff --git a/studio/backend/utils/torch_warmup.py b/studio/backend/utils/torch_warmup.py
index a1004ec36e2..f8dcc6dd0b6 100644
--- a/studio/backend/utils/torch_warmup.py
+++ b/studio/backend/utils/torch_warmup.py
@@ -267,10 +267,25 @@ def gate_torch_stack_import(reason: str, log = None) -> bool:
``torch._inductor``; racing the warm inside ``import torch._dynamo`` takes the two package locks
in opposite order, and CPython's deadlock detector leaves a half-built module in sys.modules."""
if _dynamo_done:
+ _prime_cuda_driver()
return True
if os.environ.get(DYNAMO_GATE_DISABLE_ENV_VAR) == "1":
+ _prime_cuda_driver() # the kill switch skips only the dynamo wait
return False
- return ensure_dynamo_imported(log = log or logger, reason = reason, timeout = _dynamo_gate_timeout())
+ imported = ensure_dynamo_imported(
+ log = log or logger, reason = reason, timeout = _dynamo_gate_timeout()
+ )
+ _prime_cuda_driver()
+ return imported
+
+
+def _prime_cuda_driver() -> None:
+ """``cuInit`` off the GIL before the gated importer probes CUDA. See utils/cuda_driver_prime.py."""
+ try:
+ from utils.cuda_driver_prime import prime_cuda_driver # noqa: PLC0415
+ prime_cuda_driver()
+ except Exception as exc: # noqa: BLE001 -- never fatal; torch initialises as before
+ logger.debug("cuda driver prime skipped: %r", exc)
def ensure_dynamo_imported(
@@ -341,6 +356,8 @@ def close_dynamo_import_window(log) -> bool:
front of its first one. A warning, not a retry: a process that lost the race does not
recover. Wrap the IMPORT of this module too, since it reaches a private CPython name."""
imported = ensure_dynamo_imported(log = log, reason = "diffusers import")
+ # `import diffusers` probes CUDA at module level (diffusers.utils.torch_utils) with the GIL held.
+ _prime_cuda_driver()
# Not nested in the dynamo gate: the background import it waits on reaches torch._dynamo.
claim_media_import_window(log)
if imported: