Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: initialise the CUDA driver off the GIL so load-progress polls stay responsive (#14452)
Daniel Hanfe8aab30b013
diff --git a/studio/backend/tests/test_cuda_prime_off_gil.py b/studio/backend/tests/test_cuda_prime_off_gil.pynew file mode 100644index 00000000000..8858e8815a1--- /dev/null+++ b/studio/backend/tests/test_cuda_prime_off_gil.py@@ -0,0 +1,317 @@+# SPDX-License-Identifier: AGPL-3.0-only+"""Hardware detection must not hold the GIL across the CUDA driver init.++torch.cuda.is_available() reaches cuInit inside cudaGetDeviceCount with the GIL held, for seconds on a+multi-GPU host, so every other thread (the event loop answering /images/load-progress included) froze+for that long during the first load. Detection now runs cuInit through ctypes first, which releases+the GIL, and torch's own call then returns at once."""++import os+import subprocess+import sys+import textwrap+import threading+import time+from pathlib import Path+from types import SimpleNamespace++import pytest++from utils import cuda_driver_prime as prime+from utils.hardware import hardware as hw++BACKEND = Path(__file__).resolve().parents[1]++[email protected](autouse = True)+def isolated(monkeypatch):+ monkeypatch.setattr(os, "environ", os.environ.copy())+ for name in ("PYTORCH_NVML_BASED_CUDA_CHECK", "UNSLOTH_STUDIO_DISABLE_CUDA_PRIME"):+ monkeypatch.delenv(name, raising = False)+ monkeypatch.setenv("CUDA_VISIBLE_DEVICES", "0")+ prime._reset_for_tests()+ yield+ prime._reset_for_tests()+++class _FakeCuInit:+ """Stands in for a ctypes foreign function (argtypes / restype are set on it)."""++ def __init__(+ self,+ calls,+ gate = None,+ ):+ self.calls = calls+ self.gate = gate++ def __call__(self, flags):+ if self.gate is not None:+ self.gate.wait(5)+ self.calls.append(("cuInit", flags))+ return 0+++class _FakeLib:+ def __init__(+ self,+ calls,+ gate = None,+ ):+ self.cuInit = _FakeCuInit(calls, gate)+++def _fake_torch(+ calls,+ *,+ cuda = "13.0",+ hip = None,+ initialized = False,+):+ def is_available():+ calls.append(("is_available",))+ return True++ return SimpleNamespace(+ version = SimpleNamespace(cuda = cuda, hip = hip),+ cuda = SimpleNamespace(is_available = is_available, is_initialized = lambda: initialized),+ )+++def fake_cdll(monkeypatch):+ import ctypes++ calls, opened, gate = [], [], []++ def cdll(name):+ opened.append(name)+ return _FakeLib(calls, gate[0] if gate else None)++ monkeypatch.setattr(ctypes, "CDLL", cdll)+ monkeypatch.setattr(prime.sys, "platform", "linux")+ return calls, opened, gate+++def test_prime_runs_cuinit_once(fake_cdll):+ calls, opened, _gate = fake_cdll+ torch = _fake_torch(calls)+ assert prime.prime_cuda_driver(torch) is True+ assert calls == [("cuInit", 0)]+ assert opened == ["libcuda.so.1"]+ # Once per process.+ assert prime.prime_cuda_driver(torch) is False+ assert calls == [("cuInit", 0)]+++ "torch_kw, env",+ [+ ({"cuda": None}, {}), # CPU build+ ({"hip": "7.1"}, {}), # ROCm build+ ({"initialized": True}, {}), # torch already initialised CUDA+ ({}, {"PYTORCH_NVML_BASED_CUDA_CHECK": "1"}), # user opted out of cuInit here+ ({}, {"UNSLOTH_STUDIO_DISABLE_CUDA_PRIME": "1"}), # kill switch+ ({}, {"CUDA_VISIBLE_DEVICES": ""}), # CUDA hidden+ ({}, {"CUDA_VISIBLE_DEVICES": "-1"}),+ ],+)+def test_prime_skips(fake_cdll, monkeypatch, torch_kw, env):+ calls, opened, _gate = fake_cdll+ for key, value in env.items():+ monkeypatch.setenv(key, value)+ assert prime.prime_cuda_driver(_fake_torch(calls, **torch_kw)) is False+ assert calls == [] and opened == []+++def test_prime_skips_macos(fake_cdll, monkeypatch):+ calls, opened, _gate = fake_cdll+ monkeypatch.setattr(prime.sys, "platform", "darwin")+ assert prime.prime_cuda_driver(_fake_torch(calls)) is False+ assert opened == []+++def test_prime_never_imports_torch(fake_cdll, monkeypatch):+ calls, opened, _gate = fake_cdll+ monkeypatch.delitem(sys.modules, "torch", raising = False)+ assert prime.prime_cuda_driver() is False+ assert opened == [] and "torch" not in sys.modules+++def test_prime_never_raises_without_a_driver(monkeypatch):+ import ctypes++ def cdll(name):+ raise OSError(f"{name}: cannot open shared object file")++ monkeypatch.setattr(ctypes, "CDLL", cdll)+ monkeypatch.setattr(prime.sys, "platform", "linux")+ assert prime.prime_cuda_driver(_fake_torch([])) is False+++def test_concurrent_caller_waits_for_the_init_in_flight(fake_cdll):+ """A second thread must not fall through to torch's GIL-holding probe while the init runs."""+ calls, _opened, gate = fake_cdll+ release = threading.Event()+ gate.append(release)+ torch = _fake_torch(calls)+ first = threading.Thread(target = prime.prime_cuda_driver, args = (torch,))+ first.start()+ while not prime._claimed:+ time.sleep(0.001)+ second_done = threading.Event()++ def second():+ prime.prime_cuda_driver(torch)+ torch.cuda.is_available()+ second_done.set()++ t2 = threading.Thread(target = second)+ t2.start()+ time.sleep(0.1)+ assert not second_done.is_set() and calls == []+ release.set()+ first.join(5)+ t2.join(5)+ assert calls == [("cuInit", 0), ("is_available",)]++[email protected](not hasattr(os, "fork"), reason = "needs os.fork")+def test_a_child_forked_during_the_init_does_not_wait_for_it(fake_cdll):+ calls, _opened, gate = fake_cdll+ release = threading.Event()+ gate.append(release)+ torch = _fake_torch(calls)+ first = threading.Thread(target = prime.prime_cuda_driver, args = (torch,))+ first.start()+ while not prime._claimed:+ time.sleep(0.001)+ try:+ pid = os.fork()+ if pid == 0: # child: the parent's init thread is gone; this must return at once+ started = time.monotonic()+ ran = prime.prime_cuda_driver(torch)+ os._exit(0 if (not ran and time.monotonic() - started < 2.0) else 1)+ deadline = time.monotonic() + 10+ status = None+ while time.monotonic() < deadline:+ done, status = os.waitpid(pid, os.WNOHANG)+ if done:+ break+ time.sleep(0.05)+ else:+ os.kill(pid, 9)+ os.waitpid(pid, 0)+ pytest.fail("the forked child waited on the parent's cuInit")+ assert os.waitstatus_to_exitcode(status) == 0+ finally:+ release.set()+ first.join(5)+++def test_detection_primes_before_torch_probe(fake_cdll, monkeypatch):+ calls, _opened, _gate = fake_cdll+ torch = _fake_torch(calls)+ monkeypatch.setitem(sys.modules, "torch", torch)+ monkeypatch.setattr(hw, "_has_torch", lambda: True)+ monkeypatch.setattr(hw, "_ze_affinity_mask", lambda: None)+ seen = []++ def is_available():+ seen.append(list(calls))+ raise RuntimeError("stop") # only the order up to the probe matters++ torch.cuda.is_available = is_available+ with hw._DETECT_LOCK:+ try:+ hw._detect_hardware_locked()+ except Exception:+ pass+ assert seen and seen[0] == [("cuInit", 0)]+++def test_diffusers_import_window_primes(fake_cdll, monkeypatch):+ from utils import torch_warmup++ calls, _opened, _gate = fake_cdll+ monkeypatch.setitem(sys.modules, "torch", _fake_torch(calls))+ monkeypatch.setattr(torch_warmup, "ensure_dynamo_imported", lambda **_k: True)+ monkeypatch.setattr(torch_warmup, "claim_media_import_window", lambda _log: None)+ assert torch_warmup.close_dynamo_import_window(torch_warmup.logger) is True+ assert calls == [("cuInit", 0)]+++def test_torch_stack_gate_primes(fake_cdll, monkeypatch):+ from utils import torch_warmup++ calls, _opened, _gate = fake_cdll+ monkeypatch.setitem(sys.modules, "torch", _fake_torch(calls))+ monkeypatch.setattr(torch_warmup, "_dynamo_done", True)+ assert torch_warmup.gate_torch_stack_import("test") is True+ assert calls == [("cuInit", 0)]+++def test_torch_stack_gate_kill_switch_still_primes(fake_cdll, monkeypatch):+ from utils import torch_warmup++ calls, _opened, _gate = fake_cdll+ monkeypatch.setitem(sys.modules, "torch", _fake_torch(calls))+ monkeypatch.setattr(torch_warmup, "_dynamo_done", False)+ monkeypatch.setenv(torch_warmup.DYNAMO_GATE_DISABLE_ENV_VAR, "1")+ assert torch_warmup.gate_torch_stack_import("test") is False+ assert calls == [("cuInit", 0)]+++_GIL_PROBE = textwrap.dedent(+ """+ import sys, threading, time+ sys.path.insert(0, sys.argv[1])+ import torch+ from utils.hardware import hardware as hw+ gaps, stop = [], threading.Event()+ def tick():+ last = time.perf_counter()+ while not stop.is_set():+ time.sleep(0.005)+ now = time.perf_counter(); gaps.append(now - last); last = now+ t = threading.Thread(target = tick, daemon = True); t.start(); time.sleep(0.2)+ with hw._DETECT_LOCK:+ hw._detect_hardware_locked()+ torch.cuda.is_available()+ stop.set(); t.join()+ print(f"MAXGAP {max(gaps):.3f} {torch.cuda.is_initialized()}")+ """+)+++def _real_cuda_host() -> bool:+ if os.environ.get("CUDA_VISIBLE_DEVICES", "x").strip() in ("", "-1"):+ return False+ try:+ import ctypes++ import torch++ if not torch.version.cuda or getattr(torch.version, "hip", None):+ return False+ ctypes.CDLL("libcuda.so.1")+ except Exception:+ return False+ return sys.platform.startswith("linux")++[email protected](not _real_cuda_host(), reason = "needs an NVIDIA driver and a CUDA torch")+def test_real_detection_keeps_other_threads_running():+ """A fresh process (cold driver) runs detection while a 5 ms ticker thread measures its largest gap.+ Before: the gap is the whole cuInit (0.4-6.8 s measured on B200 hosts). After: under 20 ms."""+ out = subprocess.run(+ [sys.executable, "-c", _GIL_PROBE, str(BACKEND)],+ capture_output = True,+ text = True,+ timeout = 300,+ env = {**os.environ, "UNSLOTH_STUDIO_DISABLE_CUDA_PRIME": ""},+ )+ line = [l for l in out.stdout.splitlines() if l.startswith("MAXGAP")]+ assert line, out.stdout[-2000:] + out.stderr[-2000:]+ gap = float(line[-1].split()[1])+ assert gap < 0.25, f"another thread was frozen {gap:.2f}s during hardware detection"diff --git a/studio/backend/utils/cuda_driver_prime.py b/studio/backend/utils/cuda_driver_prime.pynew file mode 100644index 00000000000..d12a687b1f1--- /dev/null+++ b/studio/backend/utils/cuda_driver_prime.py@@ -0,0 +1,111 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""Run ``cuInit`` via ctypes (releases the GIL) before torch's first CUDA probe, which holds the GIL for the+whole multi-second driver init and freezes the event loop. Idempotent; never raises. Concurrent callers wait on an+Event rather than entering torch's GIL-holding call. ``UNSLOTH_STUDIO_DISABLE_CUDA_PRIME=1`` disables it.+"""++from __future__ import annotations++import os+import sys+import threading++DISABLE_ENV_VAR = "UNSLOTH_STUDIO_DISABLE_CUDA_PRIME"++_claim_lock = threading.Lock()+_claimed = False+_done = threading.Event()+++def _driver_names() -> tuple[str, ...]:+ if sys.platform.startswith("linux"):+ return ("libcuda.so.1", "libcuda.so")+ if sys.platform == "win32":+ return ("nvcuda.dll",)+ return ()+++def _should_prime(torch_module) -> bool:+ if os.environ.get(DISABLE_ENV_VAR) == "1":+ return False+ # The user asked torch to answer from NVML so the process is not cuInit'ed here (fork safety).+ if os.environ.get("PYTORCH_NVML_BASED_CUDA_CHECK") == "1":+ return False+ cvd = os.environ.get("CUDA_VISIBLE_DEVICES")+ if cvd is not None and cvd.strip() in ("", "-1"):+ return False+ version = getattr(torch_module, "version", None)+ if not getattr(version, "cuda", None) or getattr(version, "hip", None):+ return False+ try:+ return not torch_module.cuda.is_initialized()+ except Exception:+ return False+++def _cu_init() -> bool:+ import ctypes+ for name in _driver_names():+ try:+ lib = ctypes.CDLL(name)+ except OSError:+ continue+ try:+ fn = lib.cuInit+ fn.argtypes = [ctypes.c_uint]+ fn.restype = ctypes.c_int+ fn(0)+ except Exception:+ return False+ return True+ return False+++def prime_cuda_driver(torch_module = None) -> bool:+ """Run ``cuInit`` off the GIL once per process, before torch's first CUDA probe. True iff this call ran it."""+ global _claimed+ if _done.is_set():+ return False+ with _claim_lock:+ first = not _claimed+ _claimed = True+ if not first:+ _done.wait()+ return False+ try:+ if not _driver_names():+ return False+ if torch_module is None:+ # Never imports torch: a caller still inside `import torch` would deadlock its waiters.+ torch_module = sys.modules.get("torch")+ if torch_module is None:+ return False+ if not _should_prime(torch_module):+ return False+ return _cu_init()+ except Exception:+ return False+ finally:+ _done.set()+++def _after_fork_in_child() -> None:+ # The thread that would set _done exists only in the parent, and cuInit is not fork-safe: the child never primes.+ global _claim_lock, _claimed, _done+ _claim_lock = threading.Lock()+ _claimed = True+ _done = threading.Event()+ _done.set()+++if hasattr(os, "register_at_fork"):+ os.register_at_fork(after_in_child = _after_fork_in_child)+++def _reset_for_tests() -> None:+ global _claimed+ with _claim_lock:+ _claimed = False+ _done.clear()diff --git a/studio/backend/utils/hardware/hardware.py b/studio/backend/utils/hardware/hardware.pyindex 406469665aa..0d20775edff 100644--- a/studio/backend/utils/hardware/hardware.py+++ b/studio/backend/utils/hardware/hardware.py@@ -1563,6 +1563,12 @@ def _detect_hardware_locked() -> DeviceType:cvd = os.environ.get("CUDA_VISIBLE_DEVICES")cuda_hidden = cvd is not None and cvd.strip() in ("", "-1")force_xpu = os.environ.get("UNSLOTH_FORCE_XPU") == "1"+ # cuInit off the GIL first: torch's probe below holds it for the whole driver init.+ try:+ from utils.cuda_driver_prime import prime_cuda_driver+ prime_cuda_driver(torch)+ except Exception: # noqa: BLE001 - never fatal (also loaded standalone, without the package)+ passtry:cuda_unavailable = not torch.cuda.is_available()except Exception:diff --git a/studio/backend/utils/torch_warmup.py b/studio/backend/utils/torch_warmup.pyindex a1004ec36e2..f8dcc6dd0b6 100644--- a/studio/backend/utils/torch_warmup.py+++ b/studio/backend/utils/torch_warmup.py@@ -267,10 +267,25 @@ def gate_torch_stack_import(reason: str, log = None) -> bool:``torch._inductor``; racing the warm inside ``import torch._dynamo`` takes the two package locksin opposite order, and CPython's deadlock detector leaves a half-built module in sys.modules."""if _dynamo_done:+ _prime_cuda_driver()return Trueif os.environ.get(DYNAMO_GATE_DISABLE_ENV_VAR) == "1":+ _prime_cuda_driver() # the kill switch skips only the dynamo waitreturn False- return ensure_dynamo_imported(log = log or logger, reason = reason, timeout = _dynamo_gate_timeout())+ imported = ensure_dynamo_imported(+ log = log or logger, reason = reason, timeout = _dynamo_gate_timeout()+ )+ _prime_cuda_driver()+ return imported+++def _prime_cuda_driver() -> None:+ """``cuInit`` off the GIL before the gated importer probes CUDA. See utils/cuda_driver_prime.py."""+ try:+ from utils.cuda_driver_prime import prime_cuda_driver # noqa: PLC0415+ prime_cuda_driver()+ except Exception as exc: # noqa: BLE001 -- never fatal; torch initialises as before+ logger.debug("cuda driver prime skipped: %r", exc)def ensure_dynamo_imported(@@ -341,6 +356,8 @@ def close_dynamo_import_window(log) -> bool:front of its first one. A warning, not a retry: a process that lost the race does notrecover. Wrap the IMPORT of this module too, since it reaches a private CPython name."""imported = ensure_dynamo_imported(log = log, reason = "diffusers import")+ # `import diffusers` probes CUDA at module level (diffusers.utils.torch_utils) with the GIL held.+ _prime_cuda_driver()# Not nested in the dynamo gate: the background import it waits on reaches torch._dynamo.claim_media_import_window(log)if imported: