Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Close the partially initialized torch._dynamo window on the diffusion load path (#10981)

Daniel Han5fbbd61e6f90
diff --git a/studio/backend/core/inference/diffusion.py b/studio/backend/core/inference/diffusion.py
index 26aaa526503..09f09ef83af 100644
--- a/studio/backend/core/inference/diffusion.py
+++ b/studio/backend/core/inference/diffusion.py
@@ -256,6 +256,37 @@ def _hf_token_in_play(hf_token: Optional[str]) -> bool:
return False
+_DYNAMO_PARTIAL_RE = re.compile(
+ r"partially initialized module 'torch\._dynamo'|"
+ r"module 'torch\._dynamo' has no attribute 'utils'"
+)
+
+
+def dynamo_partial_init_message(exc: BaseException) -> Optional[str]:
+ """Rewrite the half-initialised ``torch._dynamo`` failure into the step that unblocks the
+ user, else None so an unrelated load error keeps its own text. Same contract as
+ ``hub_access_message``: only the toast changes, the raw exception still reaches the log.
+
+ Worth special-casing because the raw text names a private torch module and reads as a bug in
+ the model, while the actual remedy is a restart and nothing else. Measured on torch 2.10:
+ once a process loses this import race the state does not recover, so retrying the load in
+ the same process fails the same way (0 of 14 retries resolved)."""
+ seen: set[int] = set()
+ while exc is not None and id(exc) not in seen:
+ seen.add(id(exc))
+ if _DYNAMO_PARTIAL_RE.search(str(exc)):
+ return (
+ "PyTorch's compiler module (torch._dynamo) ended up half-initialised in this "
+ "process, so the image model could not finish loading. Restart Unsloth and load "
+ "it again; this state does not clear on its own."
+ )
+ # Same walk as _gated_in_chain: `raise ... from None` means the raiser deliberately hid
+ # the inner error, so following __context__ past it would answer a visible, unrelated
+ # failure (corrupt weights, say) with restart advice that does not apply to it.
+ exc = exc.__cause__ or (None if exc.__suppress_context__ else exc.__context__)
+ return None
+
+
def hub_access_message(exc: BaseException, *, had_token: bool) -> Optional[str]:
"""Rewrite a gated-repo failure into the step that actually unblocks the user, else None so an
unrelated load error keeps its own text. Only the toast is affected; the raw exception,
@@ -2183,7 +2214,10 @@ class DiffusionBackend:
# A cancelled/superseded load raised below; don't log/stamp it onto the current load.
if self._load_token != token:
return
- logger.error("diffusion.load_failed: %s", exc)
+ # exc_info: the client only ever gets str(exc) (see below), so without the traceback
+ # here a one-line failure is unattributable to any call site. #10350 and #10963 both
+ # sat unreproducible for want of the frames this logs.
+ logger.error("diffusion.load_failed: %s", exc, exc_info = True)
if self._state is not None:
from .gpu_arbiter import DIFFUSION, restore_owner_account
from hub.services.models.account_access import restore_resident_metadata
@@ -2200,9 +2234,11 @@ class DiffusionBackend:
from utils.native_path_leases import redact_native_paths
try:
- text = hub_access_message(
- exc, had_token = _hf_token_in_play(kwargs.get("hf_token"))
- ) or str(exc)
+ text = (
+ hub_access_message(exc, had_token = _hf_token_in_play(kwargs.get("hf_token")))
+ or dynamo_partial_init_message(exc)
+ or str(exc)
+ )
except Exception: # noqa: BLE001
text = str(exc)
with self._lock:
@@ -3373,6 +3409,22 @@ class DiffusionBackend:
apply_diffusion_device_ordinal(target)
device, dtype = target.device, target.dtype
+ # Before the first `import diffusers` below, which is the earliest dynamo consumer on this
+ # path and therefore the only position that dominates the rest of them. Importing
+ # diffusers alone pulls in torch._dynamo (every module in diffusers.hooks evaluates
+ # @torch.compiler.disable() at class-body time), and so do the hook-based paths that
+ # follow: the FP8 text-encoder cast (diffusion_precision), the step cache
+ # (diffusion_cache), the compile cache, apply_speed_optims, and apply_memory_plan's
+ # offload. Whichever gets there first is the one that can lose the concurrent import
+ # race, and several of them swallow their own failure by design, so placing this after
+ # any of them would only observe an already-poisoned module (#10350, #10963).
+ # Normally a no-op: the background torch warm already did it at boot.
+ try:
+ from utils.torch_warmup import close_dynamo_import_window
+ close_dynamo_import_window(logger)
+ except Exception as exc: # noqa: BLE001 - optimisation only
+ logger.debug("dynamo pre-import skipped: %r", exc)
+
import diffusers
# diffusers hard-codes _tqdm_active = True at import and honours no env var, so setup_logging cannot reach it.
diff --git a/studio/backend/core/inference/diffusion_families.py b/studio/backend/core/inference/diffusion_families.py
index 52311badc05..24716d8c7db 100644
--- a/studio/backend/core/inference/diffusion_families.py
+++ b/studio/backend/core/inference/diffusion_families.py
@@ -1227,6 +1227,21 @@ def assert_pipeline_class_available(
escaped ``/images/download-plan``, which catches only (ValueError, FileNotFoundError), as a bare
500 with the message lost.
"""
+ # Here rather than at the call sites, because this is where the import is. Every caller runs
+ # on a request thread that can be racing the background torch warm: image validation, video
+ # validation, and the training preflight via _assert_family_pipeline_available. `import
+ # diffusers` pulls torch._dynamo in by itself (diffusers.hooks evaluates
+ # @torch.compiler.disable() at class-body time), and so does the hasattr below, which is what
+ # actually imports the pipeline's submodule (#10350, #10963). Guarded around the IMPORT as
+ # well as the call: utils.torch_warmup reaches importlib._bootstrap._ModuleLockManager, a
+ # private CPython name, and a build lacking it must not turn this check into a failure.
+ try:
+ from loggers import get_logger
+ from utils.torch_warmup import close_dynamo_import_window
+ close_dynamo_import_window(get_logger(__name__))
+ except Exception: # noqa: BLE001, S110 - optimisation only, and this module has no logger
+ pass
+
try:
import diffusers
present = hasattr(diffusers, pipeline_class)
diff --git a/studio/backend/core/inference/video.py b/studio/backend/core/inference/video.py
index da966adfd69..c6b174092b4 100644
--- a/studio/backend/core/inference/video.py
+++ b/studio/backend/core/inference/video.py
@@ -1208,6 +1208,8 @@ class VideoBackend:
f"load asked for {h3_task}. Pick the matching checkpoint."
)
else:
+ # assert_pipeline_class_available closes the dynamo window itself, ahead of its own
+ # `import diffusers`, which covers this validation and the training preflight too.
# Refuse a too-old diffusers here rather than deep in the load.
from .diffusion_families import assert_pipeline_class_available
assert_pipeline_class_available(fam.pipeline_class, fam.name)
@@ -3455,6 +3457,15 @@ class VideoBackend:
)
return self.status()
+ # Below the H3 native return, which is the one video path that never imports diffusers,
+ # and above every consumer that does. Same window as diffusion.py: `import diffusers`
+ # is itself a torch._dynamo importer, and so is the offload step further down.
+ try:
+ from utils.torch_warmup import close_dynamo_import_window
+ close_dynamo_import_window(logger)
+ except Exception as exc: # noqa: BLE001 - optimisation only
+ logger.debug("dynamo pre-import skipped: %r", exc)
+
import diffusers
import torch
diff --git a/studio/backend/core/training/trainer.py b/studio/backend/core/training/trainer.py
index d85735030fc..ad534ff7ab4 100644
--- a/studio/backend/core/training/trainer.py
+++ b/studio/backend/core/training/trainer.py
@@ -36,8 +36,13 @@ from utils.hardware import (
)
# recompile_limit was removed in some ROCm torch builds; guard for older wheels.
-if hasattr(torch._dynamo.config, "recompile_limit"):
- torch._dynamo.config.recompile_limit = 64
+# getattr, not `torch._dynamo.config` directly: _dynamo is a LAZY torch submodule, so the bare
+# attribute triggers its import and returns a half-built module if one is already in flight
+# elsewhere in the process, raising at module-import time on a path with no handler. The two
+# inference call sites already read it this way; this one is the last that did not.
+_dynamo_config = getattr(getattr(torch, "_dynamo", None), "config", None)
+if _dynamo_config is not None and hasattr(_dynamo_config, "recompile_limit"):
+ _dynamo_config.recompile_limit = 64
# Drop any unsloth/unsloth_zoo namespace-package shadow before importing them.
diff --git a/studio/backend/tests/test_dynamo_import_window.py b/studio/backend/tests/test_dynamo_import_window.py
new file mode 100644
index 00000000000..dd15a1cdaae
--- /dev/null
+++ b/studio/backend/tests/test_dynamo_import_window.py
@@ -0,0 +1,423 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""The partially-initialised ``torch._dynamo`` window (#10350, #10963).
+
+``torch/__init__.py`` makes ``_dynamo`` a lazy submodule, so ``torch._dynamo.X`` imports it
+on demand and returns whatever ``sys.modules`` holds, a still-initialising module included.
+``diffusers.hooks`` opens that window on every diffusion load with an offload policy, because
+it evaluates ``@torch.compiler.disable()`` at class-body time and that is ``import
+torch._dynamo``. Nothing on the offload path is guarded, so a read from another thread in
+that window fails the whole load with a bare one-line error.
+
+These tests pin the three things the fix depends on, none of which need a real torch:
+
+ 1. ``ensure_dynamo_imported`` resolves ``.utils`` BY ATTRIBUTE, so the state an ``import``
+ cannot see (submodule in sys.modules, never bound on the parent) is still caught.
+ 2. It is idempotent and single-flight, which is the property that closes the window.
+ 3. The load path calls it BEFORE ``apply_memory_plan``, and the load failure handler logs
+ with ``exc_info``. Both are asserted against the source, since reaching them for real
+ needs a GPU and a model.
+"""
+
+from __future__ import annotations
+
+import ast
+import builtins
+import sys
+import threading
+import types
+from pathlib import Path
+
+import pytest
+
+_BACKEND = Path(__file__).resolve().parent.parent
+
+
+def warm(monkeypatch):
+ """``utils.torch_warmup`` with its dynamo latch reset, so each test starts cold."""
+ from utils import torch_warmup
+
+ monkeypatch.setattr(torch_warmup, "_dynamo_done", False, raising = False)
+ return torch_warmup
+
+
+def _fake_torch(monkeypatch, *, bind_utils: bool):
+ """A ``torch`` whose ``_dynamo`` may or may not have ``.utils`` bound on it.
+
+ ``bind_utils=False`` is the reported state: ``torch._dynamo`` and ``torch._dynamo.utils``
+ are both in ``sys.modules``, so both imports succeed, but the parent never got the
+ attribute -- which is how the compile stack reads it.
+ """
+ torch = types.ModuleType("torch")
+ dynamo = types.ModuleType("torch._dynamo")
+ utils = types.ModuleType("torch._dynamo.utils")
+ if bind_utils:
+ dynamo.utils = utils
+ torch._dynamo = dynamo
+ monkeypatch.setitem(sys.modules, "torch", torch)
+ monkeypatch.setitem(sys.modules, "torch._dynamo", dynamo)
+ monkeypatch.setitem(sys.modules, "torch._dynamo.utils", utils)
+ return torch
+
+
+def test_reports_false_when_utils_is_not_bound_on_the_parent(warm, monkeypatch):
+ """The failure an ``import`` cannot observe. Both imports succeed here; only the
+ attribute read distinguishes the broken state, so a probe that merely imported would
+ report success in exactly the case that matters."""
+ _fake_torch(monkeypatch, bind_utils = False)
+ assert warm.ensure_dynamo_imported() is False
+
+
+def test_reports_true_and_latches_once_dynamo_is_whole(warm, monkeypatch):
+ _fake_torch(monkeypatch, bind_utils = True)
+ assert warm.ensure_dynamo_imported() is True
+ assert warm._dynamo_done is True
+
+
+def test_absent_torch_is_not_fatal(warm, monkeypatch):
+ """A --no-torch host reports False rather than raising: callers keep their eager paths."""
+ monkeypatch.setitem(sys.modules, "torch", None)
+ assert warm.ensure_dynamo_imported() is False
+
+
+def test_concurrent_callers_import_once(warm, monkeypatch):
+ """Single-flight is the whole mechanism: the window closes because exactly ONE thread
+ performs the first import while the rest wait on our lock rather than racing CPython's.
+
+ Asserted by counting the IMPORTS, not the lock entries. Threads that arrive before the
+ first one latches legitimately queue on the lock, so a bound on lock entries is a race
+ against thread scheduling; the double-checked flag inside the lock is what guarantees
+ the import body runs once, and that is the property worth pinning."""
+ _fake_torch(monkeypatch, bind_utils = True)
+
+ real_import = builtins.__import__
+ imports = []
+ lock = threading.Lock()
+
+ def _counting_import(name, *args, **kwargs):
+ if name.startswith("torch._dynamo"):
+ with lock:
+ imports.append(name)
+ return real_import(name, *args, **kwargs)
+
+ monkeypatch.setattr(builtins, "__import__", _counting_import)
+
+ results = []
+ threads = [
+ threading.Thread(target = lambda: results.append(warm.ensure_dynamo_imported()))
+ for _ in range(12)
+ ]
+ for t in threads:
+ t.start()
+ for t in threads:
+ t.join()
+
+ assert results == [True] * 12
+ # `import torch._dynamo` plus `import torch._dynamo.utils`, from one thread only.
+ assert len(imports) == 2, f"the import body ran more than once: {imports}"
+
+
+def test_the_warm_closes_the_window_before_it_starts_a_thread():
+ """The warm is what actually fixes this: it gets the first import done while the process
+ is still single-threaded. _prime_nvlink_topology starts the first thread this module
+ spawns, so warming after it would forfeit that.
+
+ Asserted on source order rather than by running the stage, which would need real hardware
+ detection."""
+ import inspect
+ from utils import torch_warmup
+
+ body = inspect.getsource(torch_warmup._warm_inference_backend)
+ assert "ensure_dynamo_imported()" in body, "the warm no longer closes the dynamo window"
+ assert body.index("ensure_dynamo_imported()") < body.index(
+ "_prime_nvlink_topology()"
+ ), "dynamo must be imported before the warm starts any thread"
+
+
+def test_the_stage_list_and_its_purge_contract_are_untouched():
+ """The window is closed INSIDE an existing stage, not by adding one. _STAGES carries a
+ purge-on-failure mapping whose only plausible entry for a dynamo stage would be torch,
+ and purging torch is never right. This pins that decision."""
+ from utils import torch_warmup
+ assert [name for name, _ in torch_warmup._STAGES] == [
+ "hardware",
+ "inference_backend",
+ "transformers",
+ "datasets",
+ ]
+
+
+def _load_pipeline_body():
+ tree = ast.parse((_BACKEND / "core/inference/diffusion.py").read_text(encoding = "utf-8"))
+ return next(
+ node
+ for node in ast.walk(tree)
+ if isinstance(node, ast.FunctionDef) and node.name == "load_pipeline"
+ )
+
+
+def test_load_path_closes_the_window_before_every_dynamo_consumer():
+ """Order is the point, not presence, and there is more than one consumer.
+
+ ``apply_memory_plan`` imports ``diffusers.hooks``, but the speed path gets there FIRST on the
+ default GGUF profile: ``apply_speed_optims`` reads ``torch._dynamo.config`` and
+ ``compile_cache.begin`` enters the compile stack. Whichever runs first is the one that can
+ lose the race, and the speed path's own best-effort handler would swallow it, quietly
+ disabling compile while leaving the module poisoned for the offload below. So the pre-import
+ has to precede all three, not just the offload."""
+ body = _load_pipeline_body()
+ # Keyed on lineno, not on ast.walk order: walk is breadth-first, so wrapping a call in a
+ # try/except would silently reorder it and make this assertion meaningless.
+ lines = {}
+ for node in ast.walk(body):
+ if isinstance(node, ast.Call) and isinstance(node.func, ast.Name):
+ lines.setdefault(node.func.id, node.lineno)
+ elif isinstance(node, ast.Call) and isinstance(node.func, ast.Attribute):
+ lines.setdefault(node.func.attr, node.lineno)
+ assert "close_dynamo_import_window" in lines, "the load path never closes the dynamo window"
+ for consumer in (
+ "hidream_te4_kwargs", # FP8 text-encoder cast -> diffusion_precision -> diffusers.hooks
+ "apply_step_cache", # diffusion_cache -> diffusers.hooks
+ "begin", # compile_cache.begin -> the compile stack
+ "apply_speed_optims", # reads torch._dynamo.config
+ "apply_memory_plan", # offload -> diffusers.hooks
+ ):
+ assert consumer in lines, f"{consumer} is no longer on this path; re-check the ordering"
+ assert (
+ lines["close_dynamo_import_window"] < lines[consumer]
+ ), f"the dynamo pre-import runs after {consumer}, which can reach dynamo first"
+
+ # And ahead of the plain `import diffusers` too, which pulls dynamo in by itself: every
+ # module in diffusers.hooks evaluates @torch.compiler.disable() at class-body time.
+ src = (_BACKEND / "core/inference/diffusion.py").read_text(encoding = "utf-8").splitlines()
+ first_diffusers = next(
+ n for n, line in enumerate(src, 1) if line.strip() == "import diffusers" and n > body.lineno
+ )
+ assert (
+ lines["close_dynamo_import_window"] < first_diffusers
+ ), "the pre-import runs after `import diffusers`, which triggers the dynamo import itself"
+
+
+def test_the_video_path_closes_the_window_before_each_of_its_diffusers_imports():
+ """The image path is not the only one that reaches diffusers.
+
+ ``core/inference/video.py`` imports it twice: once during modular validation, which runs on
+ the REQUEST thread, and once during pipeline assembly. The server accepts requests as soon as
+ the socket binds, while the background warm may still be inside ``import torch._dynamo``, so
+ a video load issued right after startup can recreate exactly the race this PR closes for
+ images. Every ``import diffusers`` in this file must be preceded by the guard.
+ """
+ src = (_BACKEND / "core/inference/video.py").read_text(encoding = "utf-8").splitlines()
+ # A call to assert_pipeline_class_available counts: it closes the window itself, ahead of its
+ # own `import diffusers`, so an import below one is already protected.
+ guards = [
+ n
+ for n, line in enumerate(src, 1)
+ if "close_dynamo_import_window(" in line
+ or ("assert_pipeline_class_available(" in line and "import" not in line)
+ ]
+ imports = [n for n, line in enumerate(src, 1) if line.strip() == "import diffusers"]
+ assert imports, "video.py no longer imports diffusers; re-check this test"
+ assert guards, "the video path never closes the dynamo window"
+ for imp in imports:
+ assert any(
+ g < imp for g in guards
+ ), f"`import diffusers` at video.py:{imp} has no dynamo guard above it"
+ # Above it in the same block, not merely somewhere earlier in a 6000-line file.
+ assert (
+ imp - max(g for g in guards if g < imp) < 40
+ ), f"the guard for video.py:{imp} is too far above it to be the one protecting it"
+
+
+def test_the_pipeline_class_probe_closes_the_window_itself():
+ """The guard belongs where the import is, not at each call site.
+
+ ``assert_pipeline_class_available`` does its own ``import diffusers`` and then a ``hasattr``
+ that imports the pipeline's submodule, and all three request-thread entry points reach it:
+ image validation (diffusion.validate_load_request), video validation, and the training
+ preflight (_assert_family_pipeline_available -> DiffusionLoraConfig.normalized). Guarding it
+ once here covers every caller, including any added later, which chasing call sites does not.
+ """
+ tree = ast.parse(
+ (_BACKEND / "core/inference/diffusion_families.py").read_text(encoding = "utf-8")
+ )
+ fn = next(
+ n
+ for n in ast.walk(tree)
+ if isinstance(n, ast.FunctionDef) and n.name == "assert_pipeline_class_available"
+ )
+ guard = next(
+ (
+ n.lineno
+ for n in ast.walk(fn)
+ if isinstance(n, ast.Call)
+ and isinstance(n.func, ast.Name)
+ and n.func.id == "close_dynamo_import_window"
+ ),
+ None,
+ )
+ assert guard is not None, "the pipeline-class probe never closes the dynamo window"
+
+ imports = [
+ n.lineno
+ for n in ast.walk(fn)
+ if isinstance(n, ast.Import) and any(a.name == "diffusers" for a in n.names)
+ ]
+ assert imports, "assert_pipeline_class_available no longer imports diffusers"
+ assert guard < min(imports), "the guard runs after the import it is supposed to precede"
+
+
+def test_every_request_thread_entry_point_reaches_that_probe():
+ """Pins the three callers the guard is there for, so a path that stops routing through
+ assert_pipeline_class_available and imports diffusers directly fails here."""
+ for rel, caller in (
+ ("core/inference/diffusion.py", "image validation"),
+ ("core/inference/video.py", "video validation"),
+ ("core/training/diffusion_train_common.py", "training preflight"),
+ ):
+ src = (_BACKEND / rel).read_text(encoding = "utf-8")
+ assert (
+ "assert_pipeline_class_available(" in src
+ ), f"{caller} ({rel}) no longer routes through the guarded probe; it needs its own guard"
+
+
+def test_load_failure_is_logged_with_a_traceback():
+ """The client only ever receives str(exc), so without exc_info here a one-line failure
+ cannot be attributed to any call site. Both issues stalled for exactly this reason."""
+ source = (_BACKEND / "core/inference/diffusion.py").read_text(encoding = "utf-8")
+ tree = ast.parse(source)
+ for node in ast.walk(tree):
+ if not isinstance(node, ast.Call):
+ continue
+ args = [a for a in node.args if isinstance(a, ast.Constant)]
+ if not any(str(a.value).startswith("diffusion.load_failed") for a in args):
+ continue
+ assert any(
+ kw.arg == "exc_info" for kw in node.keywords
+ ), "diffusion.load_failed must log exc_info, or no reporter can supply a traceback"
+ return
+ pytest.fail("no diffusion.load_failed log call found")
+
+
+def test_the_dynamo_failure_is_rewritten_into_something_actionable():
+ """The raw text names a private torch module and reads as a bug in the model, while the
+ only remedy is a restart. Measured on torch 2.10: a process that has lost this import race
+ does not recover, so "try again" in the same process is wrong advice."""
+ from core.inference.diffusion import dynamo_partial_init_message
+
+ exc = AttributeError(
+ "partially initialized module 'torch._dynamo' has no attribute 'utils' "
+ "(most likely due to a circular import)"
+ )
+ msg = dynamo_partial_init_message(exc)
+ assert msg and "Restart Unsloth" in msg
+ assert "torch._dynamo" in msg, "name the module so the log and the toast can be tied together"
+
+
+def test_the_rewrite_finds_the_failure_through_an_exception_chain():
+ """It surfaces from inside diffusers, so it arrives wrapped."""
+ from core.inference.diffusion import dynamo_partial_init_message
+ try:
+ try:
+ raise AttributeError("module 'torch._dynamo' has no attribute 'utils'")
+ except AttributeError as inner:
+ raise RuntimeError("Failed to import diffusers.hooks") from inner
+ except RuntimeError as outer:
+ assert dynamo_partial_init_message(outer) is not None
+
+
+def test_a_suppressed_context_is_not_followed():
+ """``raise ... from None`` means the raiser deliberately hid the inner error, so following
+ __context__ past it would answer a visible, unrelated failure with restart advice that does
+ not apply. Same walk as _gated_in_chain, which is what this file's neighbour already does."""
+ from core.inference.diffusion import dynamo_partial_init_message
+ try:
+ try:
+ raise AttributeError("module 'torch._dynamo' has no attribute 'utils'")
+ except AttributeError:
+ raise RuntimeError("Model weights are corrupt: checksum mismatch") from None
+ except RuntimeError as visible:
+ assert visible.__suppress_context__ is True
+ assert (
+ dynamo_partial_init_message(visible) is None
+ ), "a hidden dynamo error replaced an unrelated visible failure"
+
+
+def test_an_explicit_cause_is_still_followed():
+ """``raise ... from inner`` sets __suppress_context__ too, but the cause is explicit: the
+ raiser is pointing AT the inner error, so the rewrite must still find it."""
+ from core.inference.diffusion import dynamo_partial_init_message
+ try:
+ try:
+ raise AttributeError("module 'torch._dynamo' has no attribute 'utils'")
+ except AttributeError as inner:
+ raise RuntimeError("Failed to import diffusers.hooks") from inner
+ except RuntimeError as outer:
+ assert outer.__suppress_context__ is True
+ assert dynamo_partial_init_message(outer) is not None
+
+
+def test_a_cyclic_exception_chain_terminates():
+ """The walk is guarded by identity, not a depth counter, so a self-referential chain
+ cannot spin."""
+ from core.inference.diffusion import dynamo_partial_init_message
+
+ a = RuntimeError("a")
+ b = RuntimeError("b")
+ a.__cause__ = b
+ b.__cause__ = a
+ assert dynamo_partial_init_message(a) is None
+
+
+def test_an_unrelated_load_failure_keeps_its_own_text():
+ """Same contract as hub_access_message: rewrite only what it recognises, or a real error
+ would be replaced by advice that does not apply to it."""
+ from core.inference.diffusion import dynamo_partial_init_message
+
+ assert dynamo_partial_init_message(RuntimeError("CUDA out of memory")) is None
+ assert dynamo_partial_init_message(FileNotFoundError("no such file")) is None
+
+
+def test_the_rewrite_is_wired_into_the_load_failure_handler():
+ """Asserted on source, since reaching the handler needs a GPU and a model."""
+ body = ast.unparse(_load_pipeline_failure_handler())
+ assert (
+ "dynamo_partial_init_message" in body
+ ), "the load failure handler no longer rewrites the dynamo error"
+ assert body.index("hub_access_message") < body.index(
+ "dynamo_partial_init_message"
+ ), "a gated-repo message must keep priority; it is the more specific diagnosis"
+
+
+def _load_pipeline_failure_handler():
+ tree = ast.parse((_BACKEND / "core/inference/diffusion.py").read_text(encoding = "utf-8"))
+ for node in ast.walk(tree):
+ if isinstance(node, ast.Try):
+ seg = ast.unparse(node)
+ if "hub_access_message" in seg and "redact_native_paths" not in seg:
+ return node
+ raise AssertionError("could not find the load failure handler")
+
+
+def test_trainer_reads_dynamo_config_defensively():
+ """A bare ``torch._dynamo.config`` at module scope triggers the lazy import and can bind a
+ half-built module, raising at import time where nothing handles it."""
+ # AST, not a substring search: the prose explaining why this form is wrong necessarily
+ # contains the wrong form, so a text match would fail on its own comment.
+ tree = ast.parse((_BACKEND / "core/training/trainer.py").read_text(encoding = "utf-8"))
+ bare = [
+ node
+ for node in ast.walk(tree)
+ if isinstance(node, ast.Attribute)
+ and node.attr == "config"
+ and isinstance(node.value, ast.Attribute)
+ and node.value.attr == "_dynamo"
+ and isinstance(node.value.value, ast.Name)
+ and node.value.value.id == "torch"
+ ]
+ assert not bare, (
+ f"trainer.py reads torch._dynamo.config directly at line(s) "
+ f"{[n.lineno for n in bare]}; use the getattr form"
+ )
diff --git a/studio/backend/utils/torch_warmup.py b/studio/backend/utils/torch_warmup.py
index 6e9f84e4e25..573615b3c80 100644
--- a/studio/backend/utils/torch_warmup.py
+++ b/studio/backend/utils/torch_warmup.py
@@ -206,7 +206,15 @@ def _warm_datasets() -> None:
def _warm_inference_backend() -> None:
# Its constructor reaches hw.get_device(), so whoever builds it first pays for detection, which lazily is some request, and sync helpers call the getter inline from async handlers. Building it here makes the getter a dict read. After hardware, to reuse it.
from core.inference import get_inference_backend
+
get_inference_backend()
+ # Before _prime_nvlink_topology, which is the first thread this module starts: once that
+ # exists, "the first torch._dynamo import is single-threaded" stops being true. Folded into
+ # this stage rather than added as a fifth one on purpose -- _STAGES carries a purge-on-
+ # failure contract (_STAGE_PACKAGE) whose only sensible entry here would be torch, and
+ # purging torch is never right. ensure_dynamo_imported never raises, so this stage's
+ # failure semantics are unchanged.
+ ensure_dynamo_imported()
_prime_nvlink_topology()
@@ -245,6 +253,81 @@ def _prime_nvlink_topology() -> Optional[threading.Thread]:
return worker
+_dynamo_lock = threading.Lock()
+_dynamo_done = False
+
+
+def ensure_dynamo_imported() -> bool:
+ """Finish ``import torch._dynamo`` on ONE thread. True iff dynamo is importable.
+
+ ``torch/__init__.py`` makes ``_dynamo`` a LAZY submodule, so any ``torch._dynamo.X``
+ calls ``importlib.import_module``, which hands back whatever is in ``sys.modules`` -- a
+ still-initialising module included, since ``_lock_unlock_module`` swallows ``_DeadlockError``
+ to avoid deadlocking. The import takes ~0.6-3.9s and binds ``.config`` at ~0.007s but
+ ``.utils`` only at ~0.284s, so for most of it the module is present and ``.utils`` is not.
+ Anything reading it in that window raises ``partially initialized module 'torch._dynamo'
+ has no attribute 'utils'`` (#10350, #10963).
+
+ The window is opened by ordinary loads, not by torch.compile: ``diffusers.hooks``
+ evaluates ``@torch.compiler.disable()`` at class-body time and ``torch/compiler/__init__.py``
+ is ``import torch._dynamo``, so the CPU-offload step of any diffusion load opens it well
+ outside every compile gate. Accelerate hit the same thing and wrapped it lazily; diffusers
+ does not.
+
+ Doing the first import once, early and single-threaded is what closes it: measured on torch
+ 2.10.0 (the version both reports carry), 4-8 threads entering the dynamo/inductor cycle
+ together failed 19/20 runs without this and 0/20 with it.
+
+ Honest limit: this makes the first import single-threaded IN PRACTICE by getting there
+ first. It cannot make a genuinely concurrent first import safe -- CPython's per-module lock
+ is the thing being contended, and third-party code reaches dynamo without passing through
+ here. Never fatal: a host with no torch, or a dynamo that legitimately fails to import,
+ reports False and leaves callers on their existing eager fallbacks."""
+ global _dynamo_done
+ if _dynamo_done:
+ return True
+ with _dynamo_lock:
+ if _dynamo_done:
+ return True
+ try:
+ import torch # noqa: PLC0415
+ import torch._dynamo # noqa: PLC0415
+ import torch._dynamo.utils # noqa: F401, PLC0415
+
+ # By ATTRIBUTE, not just by import: a submodule already in sys.modules is returned
+ # by `import` without being bound on its parent, which is the broken state itself.
+ # torch's own compile stack reads it this way (_functorch/aot_autograd.py).
+ if getattr(torch._dynamo, "utils", None) is None:
+ return False
+ except Exception as exc: # noqa: BLE001 -- no torch, or a dynamo that cannot import
+ logger.debug("torch._dynamo warm skipped: %r", exc)
+ return False
+ _dynamo_done = True
+ return True
+
+
+def close_dynamo_import_window(log) -> bool:
+ """``ensure_dynamo_imported()`` plus the breadcrumb, for a caller about to import diffusers.
+
+ Every media load path reaches diffusers, and `import diffusers` is itself a dynamo importer,
+ so each one owes this call in front of its first such import. The warning is deliberately not
+ a retry: measured on torch 2.10, a process that has lost this race does not recover (0 of 14
+ retries resolved), and evicting the half-built package to re-import is the C-extension purge
+ that ``purge_partial_import`` already refuses. What is useful is a breadcrumb, so that if
+ anything below dies on dynamo the log already says the condition predates the load.
+
+ Call sites still wrap the IMPORT of this module as well as the call: this file reaches
+ ``importlib._bootstrap._ModuleLockManager``, a private CPython name, and a build lacking it
+ must not take a load down. Best-effort throughout, never a new failure."""
+ if ensure_dynamo_imported():
+ return True
+ log.warning(
+ "torch._dynamo is not importable in this process; "
+ "if this load fails on a dynamo import, restart Unsloth"
+ )
+ return False
+
+
_STAGES = (
("hardware", _warm_hardware),
("inference_backend", _warm_inference_backend),