Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: wait for the torch warm's dynamo import before a load's first download (#12472)
Daniel Han923b8f464516
diff --git a/studio/backend/core/inference/diffusion.py b/studio/backend/core/inference/diffusion.pyindex 73976163611..041169f24fd 100644--- a/studio/backend/core/inference/diffusion.py+++ b/studio/backend/core/inference/diffusion.py@@ -2988,6 +2988,12 @@ class DiffusionBackend:# load_pipeline and nothing before it, so the prefetch that moves the bytes ran unrestricted. READ, not# popped: load_pipeline takes it too.local_files_only = bool(kwargs.get("local_files_only"))+ # Before any download: unsloth_zoo enters the dynamo cycle from the inductor side, racing the warm.+ try:+ from utils.torch_warmup import gate_torch_stack_import+ gate_torch_stack_import("image load", logger)+ except Exception as exc: # noqa: BLE001 - a safety net, never a new failure+ logger.debug("dynamo import gate skipped: %r", exc)try:# Resolve the base repo and estimate sizes here (both network) so begin_load returns instantly.fam = detect_family_for_pick(@@ -5118,8 +5124,8 @@ class DiffusionBackend:apply_diffusion_device_ordinal(target)device, dtype = target.device, target.dtype- # Before the first `import diffusers` below, which is the earliest dynamo consumer on this- # path and therefore the only position that dominates the rest of them. Importing+ # Before the first `import diffusers` below, the earliest dynamo consumer in load_pipeline+ # (_run_load's downloads are gated earlier). Importing# diffusers alone pulls in torch._dynamo (every module in diffusers.hooks evaluates# @torch.compiler.disable() at class-body time), and so do the hook-based paths that# follow: the FP8 text-encoder cast (diffusion_precision), the step cachediff --git a/studio/backend/core/inference/video.py b/studio/backend/core/inference/video.pyindex 3982160b4f2..c48135d6780 100644--- a/studio/backend/core/inference/video.py+++ b/studio/backend/core/inference/video.py@@ -2753,6 +2753,12 @@ class VideoBackend:# reaches the Hub just as a weight pull does. READ, not popped: load_pipeline takes it too (it is in this# thread's kwargs by contract).local_files_only = bool(kwargs.get("local_files_only"))+ # Before any download: unsloth_zoo enters the dynamo cycle from the inductor side, racing the warm.+ try:+ from utils.torch_warmup import gate_torch_stack_import+ gate_torch_stack_import("video load", logger)+ except Exception as exc: # noqa: BLE001 - a safety net, never a new failure+ logger.debug("dynamo import gate skipped: %r", exc)try:fam = _detect_load_family(kwargs["repo_id"],diff --git a/studio/backend/tests/test_dynamo_import_race.py b/studio/backend/tests/test_dynamo_import_race.pynew file mode 100644index 00000000000..f264d2a2d30--- /dev/null+++ b/studio/backend/tests/test_dynamo_import_race.py@@ -0,0 +1,205 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""A download racing the torch warm's ``import torch._dynamo`` must not poison the process.++The warm enters the torch._dynamo / torch._inductor import cycle at ``torch._dynamo``; a load+thread's first download imports ``unsloth_zoo`` (via ``utils.hf_xet_fallback``), which enters+the same cycle at ``torch._inductor``. The two threads then take the two package locks in+opposite order, CPython's import deadlock detector hands one of them a half-built module, and+the process keeps failing with ``partially initialized module 'torch._dynamo' ... has no+attribute 'utils'`` until a restart.++The race is made deterministic with real torch in a fresh subprocess: the warm thread is+paused INSIDE ``import torch._dynamo`` (holding its module lock) until the download thread is+either blocked on that import or waiting at Studio's gate, then released. ``unsloth_zoo`` is a+stand-in whose ``hf_xet_fallback`` imports ``torch._inductor.utils``, as the real one does.+"""++from __future__ import annotations++import json+import os+import subprocess+import sys+import textwrap+from pathlib import Path++import pytest++_BACKEND = Path(__file__).resolve().parent.parent++pytest.importorskip("torch")++_CHILD = textwrap.dedent(+ r"""+ import importlib, importlib._bootstrap as bootstrap, json, sys, threading, time+ from importlib.machinery import PathFinder++ backend, fake_root = sys.argv[1], sys.argv[2]+ sys.path[:0] = [backend, fake_root]+ import torch+ assert "torch._dynamo" not in sys.modules, "torch imported dynamo eagerly; race untestable"++ from utils import hf_xet_fallback, torch_warmup++ hf_xet_fallback._gpu_present = lambda: True # one import attempt, no CPU retry path+ reader = {}++ def reader_is_parked():+ tid = reader.get("tid")+ if tid is None:+ return False+ if reader["thread"].is_alive() is False:+ return True+ blocking = getattr(bootstrap, "_blocking_on", {}).get(tid)+ if blocking:+ return True # waiting on a module lock the warm holds+ frame = sys._current_frames().get(tid)+ while frame is not None:+ if frame.f_code.co_name == "ensure_dynamo_imported":+ return True # waiting at Studio's gate+ frame = frame.f_back+ return False++ class _PausingLoader:+ def __init__(self, inner):+ self.inner = inner++ def create_module(self, spec):+ return self.inner.create_module(spec)++ def exec_module(self, module):+ reader_started.set()+ deadline = time.monotonic() + 120+ while not reader_is_parked() and time.monotonic() < deadline:+ time.sleep(0.01)+ time.sleep(0.2)+ self.inner.exec_module(module)++ class _PauseWarmInsideDynamo:+ def find_spec(self, name, path = None, target = None):+ if name != "torch._dynamo.aot_compile" or threading.current_thread().name != "warm":+ return None+ spec = PathFinder.find_spec(name, path, target)+ spec.loader = _PausingLoader(spec.loader)+ return spec++ sys.meta_path.insert(0, _PauseWarmInsideDynamo())+ reader_started = threading.Event()+ result = {}++ def warm():+ result["warm"] = torch_warmup.ensure_dynamo_imported()++ def download():+ try:+ result["download"] = hf_xet_fallback._load_shared()+ except BaseException as exc:+ result["download"] = repr(exc)+ result["download_error"] = repr(hf_xet_fallback._shared_import_error)++ w = threading.Thread(target = warm, name = "warm")+ w.start()+ assert reader_started.wait(120), "warm never reached torch._dynamo"+ r = threading.Thread(target = download, name = "download")+ reader["thread"] = r+ r.start()+ reader["tid"] = r.ident+ w.join(300)+ r.join(300)++ # What the next image load would see: the process is either whole or poisoned.+ try:+ import torch._dynamo.utils+ import torch._inductor.utils+ result["after"] = bool(getattr(torch._dynamo, "utils", None)) and hasattr(+ torch._inductor.utils, "IndentedBuffer"+ )+ except BaseException as exc:+ result["after"] = repr(exc)+ print("RESULT " + json.dumps(result), flush = True)+ """+)+++def _run(tmp_path: Path, extra_env: dict | None = None) -> dict:+ fake = tmp_path / "fake_site"+ (fake / "unsloth_zoo").mkdir(parents = True)+ (fake / "unsloth_zoo" / "__init__.py").write_text("")+ (fake / "unsloth_zoo" / "hf_xet_fallback.py").write_text(+ "import torch._inductor.utils # the real module enters the cycle the same way\n"+ )+ script = tmp_path / "race_child.py"+ script.write_text(_CHILD)+ env = dict(os.environ)+ env.pop("UNSLOTH_STUDIO_DISABLE_DYNAMO_IMPORT_GATE", None)+ env.update(extra_env or {})+ proc = subprocess.run(+ [sys.executable, str(script), str(_BACKEND), str(fake)],+ capture_output = True,+ text = True,+ timeout = 900,+ env = env,+ cwd = str(tmp_path),+ )+ lines = [l for l in proc.stdout.splitlines() if l.startswith("RESULT ")]+ assert lines, f"child produced no result (rc={proc.returncode}):\n{proc.stdout}\n{proc.stderr}"+ return json.loads(lines[-1][len("RESULT ") :])+++def test_download_racing_the_warm_dynamo_import_leaves_torch_whole(tmp_path):+ result = _run(tmp_path)+ assert result == {+ "warm": True,+ "download": True,+ "download_error": "None",+ "after": True,+ }, result+++def test_kill_switch_restores_the_ungated_download(tmp_path, monkeypatch):+ """With the gate disabled the download does not wait on the warm; it races as before."""+ from utils import torch_warmup++ monkeypatch.setenv(torch_warmup.DYNAMO_GATE_DISABLE_ENV_VAR, "1")+ monkeypatch.setattr(torch_warmup, "_dynamo_done", False, raising = False)+ assert torch_warmup.gate_torch_stack_import("test") is False+++def test_gate_gives_up_after_its_timeout(monkeypatch):+ """A wedged importer must not hang a load forever: the wait is bounded and logged."""+ from utils import torch_warmup++ monkeypatch.setattr(torch_warmup, "_dynamo_done", False, raising = False)+ monkeypatch.delenv(torch_warmup.DYNAMO_GATE_DISABLE_ENV_VAR, raising = False)+ monkeypatch.setenv(torch_warmup.DYNAMO_GATE_TIMEOUT_ENV_VAR, "0.2")+ lines = []++ class _Log:+ def info(self, msg, *args):+ lines.append(("info", msg % args))++ def warning(self, msg, *args):+ lines.append(("warning", msg % args))++ assert torch_warmup._dynamo_lock.acquire(timeout = 5)+ try:+ assert torch_warmup.gate_torch_stack_import("image load", _Log()) is False+ finally:+ torch_warmup._dynamo_lock.release()+ assert [level for level, _ in lines] == ["info", "warning"], lines+ assert "image load: waiting for another thread" in lines[0][1]+ assert "gave up waiting" in lines[1][1]+++def test_load_threads_gate_before_their_first_download():+ """Both media load threads take the gate before anything that can import unsloth_zoo."""+ for rel, first_download in (+ ("core/inference/diffusion.py", "self._prefetch_files("),+ ("core/inference/video.py", "_assert_pick_is_not_speech("),+ ):+ src = (_BACKEND / rel).read_text(encoding = "utf-8")+ start = src.index(" def _run_load(self, **kwargs: Any) -> None:")+ gate = src.index("gate_torch_stack_import(", start)+ assert gate < src.index(first_download, start), reldiff --git a/studio/backend/utils/hf_xet_fallback.py b/studio/backend/utils/hf_xet_fallback.pyindex 3688a08b828..56664524bf9 100644--- a/studio/backend/utils/hf_xet_fallback.py+++ b/studio/backend/utils/hf_xet_fallback.py@@ -74,6 +74,15 @@ def _gpu_present() -> bool:return False+def _gate_torch_stack(reason: str) -> None:+ """Let the torch warm finish ``import torch._dynamo`` before an ``unsloth_zoo`` import (never fatal)."""+ try:+ from utils.torch_warmup import gate_torch_stack_import+ gate_torch_stack_import(reason)+ except Exception: # noqa: BLE001, S110 - the gate is a safety net, never a new failure+ pass++def _load_shared() -> bool:"""Import ``unsloth_zoo.hf_xet_fallback`` on demand; return True if available. Deferred soimporting this module at worker startup does not pull transformers in before the sidecar is@@ -81,6 +90,8 @@ def _load_shared() -> bool:global _shared, _shared_available, _shared_import_errorif _shared_available is not None:return _shared_available+ # Outside _load_lock: a thread waiting on the warm must not hold up the env-var bookkeeping.+ _gate_torch_stack("unsloth_zoo.hf_xet_fallback import")with _load_lock:if _shared_available is not None:return _shared_available@@ -176,6 +187,7 @@ def _load_optional(module_name: str) -> Any:if cached is not _UNTRIED:return cached+ _gate_torch_stack(f"{module_name} import")try:module = importlib.import_module(module_name)_optional_modules[module_name] = modulediff --git a/studio/backend/utils/torch_warmup.py b/studio/backend/utils/torch_warmup.pyindex b7bd5131268..81149dbea19 100644--- a/studio/backend/utils/torch_warmup.py+++ b/studio/backend/utils/torch_warmup.py@@ -246,19 +246,73 @@ def _prime_nvlink_topology() -> Optional[threading.Thread]:_dynamo_lock = threading.Lock()_dynamo_done = False+# Kill switch for the request-side gates only; the gates in front of `import diffusers` stay on.+DYNAMO_GATE_DISABLE_ENV_VAR = "UNSLOTH_STUDIO_DISABLE_DYNAMO_IMPORT_GATE"+DYNAMO_GATE_TIMEOUT_ENV_VAR = "UNSLOTH_STUDIO_DYNAMO_IMPORT_GATE_TIMEOUT"+_DEFAULT_DYNAMO_GATE_TIMEOUT_S = 600.0-def ensure_dynamo_imported() -> bool:++def _dynamo_gate_timeout() -> float:+ try:+ value = float(os.environ.get(DYNAMO_GATE_TIMEOUT_ENV_VAR, _DEFAULT_DYNAMO_GATE_TIMEOUT_S))+ except ValueError:+ return _DEFAULT_DYNAMO_GATE_TIMEOUT_S+ return value if value > 0 else _DEFAULT_DYNAMO_GATE_TIMEOUT_S+++def gate_torch_stack_import(reason: str, log = None) -> bool:+ """Finish ``import torch._dynamo`` before ``reason`` imports anything that reaches it. True iff imported.++ ``unsloth_zoo``, ``torchao`` and ``diffusers`` enter the dynamo / inductor import cycle at+ ``torch._inductor``; racing the warm inside ``import torch._dynamo`` takes the two package locks+ in opposite order, and CPython's deadlock detector leaves a half-built module in sys.modules."""+ if _dynamo_done:+ return True+ if os.environ.get(DYNAMO_GATE_DISABLE_ENV_VAR) == "1":+ return False+ return ensure_dynamo_imported(log = log or logger, reason = reason, timeout = _dynamo_gate_timeout())+++def ensure_dynamo_imported(+ log = None,+ reason: Optional[str] = None,+ timeout: Optional[float] = None,+) -> bool:"""Finish ``import torch._dynamo`` on ONE thread. True iff dynamo is importable.``_dynamo`` is a LAZY submodule, so ``torch._dynamo.X`` hands back a still-initialisingmodule: ``.config`` binds early and ``.utils`` late, and a read in between raises``partially initialized module ... has no attribute 'utils'`` (#10350, #10963). Ordinaryloads open that window, not torch.compile: ``diffusers.hooks`` evaluates- ``@torch.compiler.disable()`` at class-body time. Wins only by getting there first."""+ ``@torch.compiler.disable()`` at class-body time. Wins only by getting there first.++ ``timeout`` (seconds, None = forever) bounds a wait on another importer; False on expiry."""global _dynamo_doneif _dynamo_done:return True- with _dynamo_lock:+ if not _dynamo_lock.acquire(blocking = False):+ waited = time.perf_counter()+ if log is not None:+ log.info(+ "%s: waiting for another thread to finish importing torch._dynamo",+ reason or "torch import",+ )+ if not _dynamo_lock.acquire(timeout = -1 if timeout is None else timeout):+ (log or logger).warning(+ "%s: gave up waiting for the torch._dynamo import after %.0fs; continuing "+ "without it (set %s=1 to skip this wait)",+ reason or "torch import",+ time.perf_counter() - waited,+ DYNAMO_GATE_DISABLE_ENV_VAR,+ )+ return False+ if log is not None:+ log.info(+ "%s: torch._dynamo ready after waiting %.1fs",+ reason or "torch import",+ time.perf_counter() - waited,+ )+ try:if _dynamo_done:return Truetry:@@ -276,6 +330,8 @@ def ensure_dynamo_imported() -> bool:return False_dynamo_done = Truereturn True+ finally:+ _dynamo_lock.release()def close_dynamo_import_window(log) -> bool:@@ -284,7 +340,7 @@ def close_dynamo_import_window(log) -> bool:`import diffusers` is itself a dynamo importer, so every media load path owes this call infront of its first one. A warning, not a retry: a process that lost the race does notrecover. Wrap the IMPORT of this module too, since it reaches a private CPython name."""- if ensure_dynamo_imported():+ if ensure_dynamo_imported(log = log, reason = "diffusers import"):return Truelog.warning("torch._dynamo is not importable in this process; "