Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: keep Qwen-Image-2.1 int8 / fp8 compiling on torch 2.12 (CantSplit) (#11677)

Daniel Han69c154d56e39
diff --git a/studio/backend/core/inference/diffusion.py b/studio/backend/core/inference/diffusion.py
index bf9381ca7c9..ba6ad5e9062 100644
--- a/studio/backend/core/inference/diffusion.py
+++ b/studio/backend/core/inference/diffusion.py
@@ -124,11 +124,15 @@ from .diffusion_speed import (
SPEED_MAX,
SPEED_OFF,
apply_speed_optims,
+ auto_dynamic_active,
+ compile_dynamic,
compile_eligible,
compiled_shapes_are_static,
+ dynamo_graph_count,
normalize_speed_mode,
resolve_speed_mode,
restore_backend_flags,
+ settle_compile_fallback,
snapshot_backend_flags,
)
from .diffusion_attention import (
@@ -5562,7 +5566,11 @@ class DiffusionBackend:
"fullgraph": cache_engaged is None
and not cache_may_toggle
and plan.offload_policy == OFFLOAD_NONE,
- "dynamic": effective_speed != SPEED_MAX,
+ # The value apply_speed_optims resolves (automatic for torchao weights), so a bundle
+ # from an explicit-dynamic build is not reused for an automatic-dynamic one.
+ "dynamic": compile_dynamic(
+ getattr(pipe, "transformer", None), effective_speed != SPEED_MAX
+ ),
"mode": "max-autotune-no-cudagraphs"
if effective_speed == SPEED_MAX
else "default",
@@ -6907,7 +6915,7 @@ class DiffusionBackend:
"fullgraph": state.transformer_cache is None
and not state.cache_auto
and state.offload_policy == OFFLOAD_NONE,
- "dynamic": True,
+ "dynamic": compile_dynamic(getattr(state.pipe, "transformer", None), True),
"mode": "default",
},
logger = logger,
@@ -7416,6 +7424,7 @@ class DiffusionBackend:
per_image_seeds: list[int] = []
chunk_shapes: list[int] = []
pending = list(chunks)
+ graphs_before = dynamo_graph_count()
while pending:
chunk = pending.pop(0)
chunk_kwargs = dict(kwargs)
@@ -7479,6 +7488,11 @@ class DiffusionBackend:
len(second_half),
)
continue
+ finally:
+ # A guarded block that fell back to eager (compile failed at its first forward) no longer runs
+ # compiled: report it on every exit, cancel and error included, so status, LoRA gating and the
+ # compile-cache shape registry stop treating it as compiled.
+ settle_compile_fallback(state, state.pipe, logger)
if cancel.is_set():
raise RuntimeError(DIFFUSION_CANCELLED_MSG)
images.extend(out)
@@ -7505,6 +7519,10 @@ class DiffusionBackend:
(reg_width, reg_height, int(chunk_batch)),
static = static_shapes,
)
+ if auto_dynamic_active(state.pipe) and dynamo_graph_count() > graphs_before:
+ # Automatic dynamic recompiles on the first new text length at an already-registered
+ # (width, height, batch): persist those graphs too, or every fresh process pays them again.
+ compile_cache.mark_recompiled(state.compile_cache_ctx)
compile_cache.save_async(state.compile_cache_ctx, logger = logger)
except Exception: # noqa: BLE001 - cache persistence is best-effort
pass
diff --git a/studio/backend/core/inference/diffusion_batched.py b/studio/backend/core/inference/diffusion_batched.py
index a1cf1909cb3..dea83231e7b 100644
--- a/studio/backend/core/inference/diffusion_batched.py
+++ b/studio/backend/core/inference/diffusion_batched.py
@@ -132,9 +132,18 @@ def uniform_prompt(chunk: list[tuple[str, int]]) -> Optional[str]:
def is_oom_error(exc: BaseException) -> bool:
"""Whether an exception is a CUDA/accelerator out-of-memory, worth a smaller
- retry. Matched structurally (class name across torch versions / devices) and
- by message, so the caller needn't import torch to classify."""
- for klass in type(exc).__mro__:
- if klass.__name__ == "OutOfMemoryError":
+ retry. Matched structurally (class name across torch versions / devices, and
+ backend subclasses such as ``OutOfMemoryError_``) and by message, so the caller
+ needn't import torch to classify. Walks ``__cause__`` / ``__context__``: torch.compile
+ wraps an OOM hit while autotuning in a compiler error whose own message is generic,
+ and that allocation failure still deserves the smaller retry."""
+ seen = set()
+ cur: Optional[BaseException] = exc
+ while cur is not None and id(cur) not in seen:
+ seen.add(id(cur))
+ if any(klass.__name__.startswith("OutOfMemory") for klass in type(cur).__mro__):
return True
- return "out of memory" in str(exc).lower()
+ if "out of memory" in str(cur).lower():
+ return True
+ cur = cur.__cause__ or cur.__context__
+ return False
diff --git a/studio/backend/core/inference/diffusion_cache.py b/studio/backend/core/inference/diffusion_cache.py
index 42372482b36..902665e81c2 100644
--- a/studio/backend/core/inference/diffusion_cache.py
+++ b/studio/backend/core/inference/diffusion_cache.py
@@ -179,7 +179,20 @@ def _compile_hooked_block_inners(transformer: Any, logger: Any = None) -> int:
continue # not a plain bound method; arming would miss the block
# fullgraph=False / dynamic=True: a cache is active (its decision graph-breaks) and this matches the
# default tier. Dynamo caches per code object, so re-arming after a toggle is ~free.
- fn_ref.original_forward = torch.compile(orig, fullgraph = False, dynamic = True)
+ # Automatic dynamic when the speed layer chose it (torchao weights): see _compile_repeated_blocks.
+ dynamic = None if getattr(transformer, "_unsloth_auto_dynamic", False) else True
+ compiled = torch.compile(orig, fullgraph = False, dynamic = dynamic)
+ # Same runtime fallback as the block's own compile: a lowering failure on the first computed step
+ # restores the eager inner instead of failing the render.
+ guard = getattr(transformer, "_unsloth_compile_guard", None)
+ if guard is not None:
+
+ def restore(ref: Any = fn_ref, inner: Any = orig) -> None:
+ ref.original_forward = inner
+
+ guard.restores.append(restore)
+ compiled = guard.wrap(compiled, orig, transformer)
+ fn_ref.original_forward = compiled
hook._unsloth_orig_inner = orig
armed += 1
except Exception as exc: # noqa: BLE001 -- best-effort: the cache still works eager
diff --git a/studio/backend/core/inference/diffusion_compile_cache.py b/studio/backend/core/inference/diffusion_compile_cache.py
index 56d25624c1c..9ad178db357 100644
--- a/studio/backend/core/inference/diffusion_compile_cache.py
+++ b/studio/backend/core/inference/diffusion_compile_cache.py
@@ -525,6 +525,20 @@ def register_shape(ctx: Optional[CacheContext], shape: Any, *, static: bool) ->
pass
+def mark_recompiled(ctx: Optional[CacheContext]) -> None:
+ """A render compiled new graphs for a shape the bundle already covers (automatic dynamic re-specialises on the
+ first new text length, keyed by nothing ``register_shape`` sees), so the bundle on disk lacks them: clear
+ ``saved`` so the next ``save`` rewrites it. Never raises."""
+ if ctx is None:
+ return
+ try:
+ with _dirty_lock:
+ ctx.saved = False
+ ctx.dirty_seq += 1
+ except Exception: # noqa: BLE001 - bookkeeping only
+ pass
+
+
def _try_load(
ctx: CacheContext,
logger: Any,
diff --git a/studio/backend/core/inference/diffusion_speed.py b/studio/backend/core/inference/diffusion_speed.py
index 2658925399e..b5c8475e559 100644
--- a/studio/backend/core/inference/diffusion_speed.py
+++ b/studio/backend/core/inference/diffusion_speed.py
@@ -388,7 +388,12 @@ def compiled_shapes_are_static(pipe: Any, speed_mode: Optional[str]) -> bool:
mode = normalize_speed_mode(speed_mode)
if mode == SPEED_MAX:
return True
- return mode == SPEED_DEFAULT and _denoiser_unet(pipe) is not None
+ if mode != SPEED_DEFAULT:
+ return False
+ # A torchao-quantised DiT compiles with automatic dynamic: its first shapes get their own artifacts.
+ return _denoiser_unet(pipe) is not None or any(
+ getattr(t, "_unsloth_auto_dynamic", False) for t in _denoiser_dits(pipe)
+ )
def _denoiser_dits(pipe: Any) -> list:
@@ -465,12 +470,19 @@ def _compile_repeated_blocks(
# Compile every denoiser DiT (dual-DiT families run both); a per-DiT failure degrades only that one to eager.
engaged = False
for transformer in dits:
+ dit_kwargs = dict(kwargs)
+ dit_kwargs["dynamic"] = compile_dynamic(transformer, kwargs["dynamic"])
+ if dit_kwargs["dynamic"] is None:
+ transformer._unsloth_auto_dynamic = True
try:
- transformer.compile_repeated_blocks(**kwargs)
+ transformer.compile_repeated_blocks(**dit_kwargs)
engaged = True
except Exception as exc: # noqa: BLE001 - optimisation only
_warn(logger, "compile_repeated_blocks", exc)
continue
+ # compile_repeated_blocks is lazy: inductor only runs on the first forward, inside generate(), where a lowering
+ # bug would fail the render. Guard every compiled block so such a failure drops this DiT to eager instead.
+ guard_compiled_blocks(transformer, logger)
# A step cache engaged BEFORE this compile already wrapped each block forward in a disabled hook, so the compute
# branch would run eager and forfeit the regional compile. Re-point the hooks' inner forward at compiled
# wrappers (no-op without them).
@@ -482,6 +494,216 @@ def _compile_repeated_blocks(
return engaged
+def compile_dynamic(transformer: Any, dynamic: Optional[bool]) -> Optional[bool]:
+ """The ``dynamic`` a DiT is actually compiled with, so compile-cache fingerprints key on the same value.
+
+ dynamic=True makes even the constant segment starts symbolic, and on Qwen-Image-2.1 the attention output cat
+ (text + target, length s87 - s89) then fuses into torchao's per-row activation-quant reduction, which inductor
+ cannot split (CantSplit, every render failed). Automatic dynamic (None) compiles the first shapes static and only
+ generalises what actually varies: stable after ~3 recompiles, same numerics."""
+ if dynamic and transformer is not None and _carries_torchao_weights(transformer):
+ return None
+ return dynamic
+
+
+def _carries_torchao_weights(module: Any) -> bool:
+ """Whether any parameter of ``module`` is a torchao tensor subclass (the int8 / fp8 dense fast path).
+
+ Checks ``.data`` too: some torchao builds keep ``Linear.weight`` a plain ``nn.Parameter`` wrapping the subclass,
+ the same two representations ``transformer_is_quantised`` covers. Only torchao's own types count, so a GGUF or
+ bitsandbytes parameter subclass keeps its existing compile policy."""
+
+ def _torchao(t: Any) -> bool:
+ return t is not None and type(t).__module__.startswith("torchao")
+
+ try:
+ return any(_torchao(p) or _torchao(getattr(p, "data", None)) for p in module.parameters())
+ except Exception: # noqa: BLE001 - not a torch module (tests/fakes)
+ return False
+
+
+def is_compile_failure(exc: BaseException) -> bool:
+ """True for an error torch.compile raises while BUILDING a graph (dynamo tracing, inductor lowering / codegen), as
+ opposed to one raised by kernels that already built. Only the former is safe to answer with an eager retry: nothing
+ ran yet. An OOM is never one, even when inductor wraps it (autotune ran out), so the OOM backoff still sees it."""
+ try:
+ import torch
+ except Exception: # noqa: BLE001 - no torch, nothing compiled
+ return False
+ from .diffusion_batched import is_oom_error
+
+ # The batch backoff's own classifier walks the whole cause chain, so an OOM anywhere in it goes to the backoff
+ # (which recognises the same outer exception) instead of the eager retry.
+ if is_oom_error(exc):
+ return False
+ kinds: list = []
+ dynamo_exc = getattr(getattr(torch, "_dynamo", None), "exc", None)
+ for name in (
+ "BackendCompilerFailed",
+ "Unsupported",
+ "TorchRuntimeError",
+ "InternalTorchDynamoError",
+ ):
+ kind = getattr(dynamo_exc, name, None) if dynamo_exc is not None else None
+ if isinstance(kind, type):
+ kinds.append(kind)
+ try:
+ from torch._inductor.exc import (
+ InductorError,
+ ) # torch 2.7+; older torch wraps it in BackendCompilerFailed
+ kinds.append(InductorError)
+ except Exception: # noqa: BLE001
+ pass
+ return bool(kinds) and isinstance(exc, tuple(kinds))
+
+
+class _CompileGuard:
+ """Per-DiT record of a compile that failed at runtime. The first failure flips every guarded block of that DiT to
+ eager (one shared decision: a half-compiled stack would retry the same broken lowering per block)."""
+
+ def __init__(self, logger: Any) -> None:
+ self.logger = logger
+ self.error: Optional[str] = None
+ self.restores: list = []
+
+ def fail(self, exc: BaseException, owner: Any) -> None:
+ if self.error is None:
+ self.error = f"{type(exc).__name__}: {str(exc).splitlines()[0] if str(exc) else ''}"[
+ :300
+ ]
+ for restore in self.restores:
+ try:
+ restore()
+ except Exception: # noqa: BLE001 - per-block best-effort; the guard still routes to eager
+ pass
+ if self.logger is not None:
+ self.logger.warning(
+ "diffusion.speed: torch.compile failed on %s at the first forward (%s); this load runs eager",
+ type(owner).__name__,
+ self.error,
+ )
+
+ def wrap(self, compiled: Any, eager: Any, owner: Any) -> Any:
+ guard = self
+
+ def guarded(*args: Any, **kwargs: Any) -> Any:
+ if guard.error is None:
+ try:
+ return compiled(*args, **kwargs)
+ except Exception as exc: # noqa: BLE001 - reraised unless a compile-time failure
+ if not is_compile_failure(exc):
+ raise
+ guard.fail(exc, owner)
+ # The handled exception pins inductor's frames (and the traced fake tensors) through the retry.
+ exc.__traceback__ = None
+ return eager(*args, **kwargs)
+
+ guarded._unsloth_compile_guard = guard
+ return guarded
+
+
+def guard_compiled_blocks(transformer: Any, logger: Any = None) -> int:
+ """Wrap each ``Module.compile``d submodule's ``_compiled_call_impl`` so a compile-time failure on the first forward
+ (e.g. inductor ``CantSplit`` on a torchao-quantised Qwen-Image-2.1 block) falls back to that module's eager
+ ``_call_impl`` for this and every later call. Idempotent. Returns the number of modules guarded."""
+ guard = getattr(transformer, "_unsloth_compile_guard", None)
+ if guard is None:
+ guard = _CompileGuard(logger)
+ try:
+ transformer._unsloth_compile_guard = guard
+ except Exception: # noqa: BLE001 - not a settable module (tests/fakes)
+ return 0
+ count = 0
+ try:
+ modules = list(transformer.modules())
+ except Exception: # noqa: BLE001 - not a torch module
+ return 0
+ for module in modules:
+ compiled = getattr(module, "_compiled_call_impl", None)
+ if compiled is None or getattr(compiled, "_unsloth_compile_guard", None) is not None:
+ continue
+ eager = getattr(module, "_call_impl", None)
+ if eager is None:
+ continue
+
+ def restore(m: Any = module) -> None:
+ m._compiled_call_impl = None
+
+ guard.restores.append(restore)
+ module._compiled_call_impl = guard.wrap(compiled, eager, transformer)
+ count += 1
+ return count
+
+
+def _guarded_dits(pipe: Any) -> list:
+ """Every DiT a compile guard can sit on: the standard denoisers plus a modular workflow's named partition
+ (MiniMax-H3's ``transformer_ref`` compiles through a view, so the bare pipe's ``_denoiser_dits`` misses it)."""
+ dits = _denoiser_dits(pipe)
+ extra = getattr(pipe, "transformer_ref", None)
+ if extra is not None and extra not in dits:
+ dits.append(extra)
+ return dits
+
+
+def compiled_dits_active(pipe: Any) -> bool:
+ """True while at least one guarded DiT still runs its compiled blocks (dual-DiT pipelines fall back per DiT)."""
+ for transformer in _guarded_dits(pipe):
+ guard = getattr(transformer, "_unsloth_compile_guard", None)
+ if guard is not None and not guard.error:
+ return True
+ return False
+
+
+def settle_compile_fallback(
+ state: Any,
+ pipe: Any,
+ logger: Any = None,
+) -> Optional[str]:
+ """After a render, fold a runtime compile fallback into ``state.speed_optims`` (image and video backends share it).
+
+ Adds ``compile_fallback_eager`` once, and drops ``compiled`` only when NO guarded DiT still runs compiled, so a
+ dual-DiT load whose second expert still compiles keeps the LoRA gate and the compile-cache shape registry. Returns
+ the recorded failure, or None when nothing fell back."""
+ fallback = compile_fallback_error(pipe)
+ if not fallback:
+ return None
+ optims = tuple(getattr(state, "speed_optims", None) or ())
+ updated = list(optims)
+ if "compile_fallback_eager" not in updated:
+ updated.append("compile_fallback_eager")
+ if logger is not None:
+ logger.warning("diffusion.speed: regional compile fell back to eager: %s", fallback)
+ if "compiled" in updated and not compiled_dits_active(pipe):
+ updated.remove("compiled")
+ if tuple(updated) != optims:
+ # The load states are frozen dataclasses; the status must still reflect what actually runs.
+ object.__setattr__(state, "speed_optims", tuple(updated))
+ return fallback
+
+
+def auto_dynamic_active(pipe: Any) -> bool:
+ """Whether any denoiser DiT compiled with automatic dynamic (torchao weights on the default tier)."""
+ return any(getattr(t, "_unsloth_auto_dynamic", False) for t in _guarded_dits(pipe))
+
+
+def dynamo_graph_count() -> int:
+ """Graphs dynamo has compiled in this process (0 when unavailable); a delta across a render means it compiled."""
+ try:
+ from torch._dynamo.utils import counters
+ return int(counters["stats"]["unique_graphs"])
+ except Exception: # noqa: BLE001
+ return 0
+
+
+def compile_fallback_error(pipe: Any) -> Optional[str]:
+ """The compile failure a guarded DiT fell back from, or None while every compiled DiT still runs compiled."""
+ for transformer in _guarded_dits(pipe):
+ guard = getattr(transformer, "_unsloth_compile_guard", None)
+ if guard is not None and guard.error:
+ return guard.error
+ return None
+
+
def _compile_vae_decode(pipe: Any, logger: Any) -> bool:
"""torch.compile the VAE ``decode`` bound method in place (U-Net families; caller gates).
Instance-level assignment: the pipe owns it and the module object is untouched."""
diff --git a/studio/backend/core/inference/video.py b/studio/backend/core/inference/video.py
index c553e08e3ba..b338ace4008 100644
--- a/studio/backend/core/inference/video.py
+++ b/studio/backend/core/inference/video.py
@@ -92,6 +92,7 @@ from .diffusion_speed import (
apply_speed_optims,
resolve_speed_mode,
restore_backend_flags,
+ settle_compile_fallback,
snapshot_backend_flags,
)
from .diffusion_auto_policy import (
@@ -6280,6 +6281,11 @@ class VideoBackend:
except Exception: # noqa: BLE001 -- cleanup is best-effort
pass
raise RuntimeError(VIDEO_CANCELLED_MSG) from None
+ finally:
+ # A guarded compiled block that failed to build at its first forward now runs eager; the status
+ # must not keep reporting it compiled (a forced-compile quantised load runs ~30x slower eager),
+ # whether this render finished, was cancelled or failed.
+ settle_compile_fallback(state, pipe, logger)
if cancel.is_set():
raise RuntimeError(VIDEO_CANCELLED_MSG)
diff --git a/studio/backend/tests/test_diffusion_batched.py b/studio/backend/tests/test_diffusion_batched.py
index 26df099b083..6cded8f84d0 100644
--- a/studio/backend/tests/test_diffusion_batched.py
+++ b/studio/backend/tests/test_diffusion_batched.py
@@ -151,3 +151,22 @@ def test_is_oom_error_matches_class_name_and_message():
assert is_oom_error(RuntimeError("CUDA out of memory. Tried to allocate 2 GiB"))
assert not is_oom_error(RuntimeError("shape mismatch"))
assert not is_oom_error(ValueError("bad prompt"))
+
+
+def test_is_oom_error_sees_an_oom_wrapped_by_a_compiler_error():
+ # torch.compile re-raises an autotune OOM as a compiler error with a generic message; the backoff must still split.
+ class OutOfMemoryError_(RuntimeError):
+ pass
+
+ class BackendCompilerFailed(RuntimeError):
+ pass
+
+ for inner in (OutOfMemoryError_("alloc"), RuntimeError("HIP out of memory")):
+ try:
+ try:
+ raise inner
+ except RuntimeError as oom:
+ raise BackendCompilerFailed("autotune failed") from oom
+ except BackendCompilerFailed as outer:
+ assert is_oom_error(outer)
+ assert not is_oom_error(BackendCompilerFailed("autotune failed"))
diff --git a/studio/backend/tests/test_diffusion_compile_cache.py b/studio/backend/tests/test_diffusion_compile_cache.py
index 00eccb92b26..b7abd546942 100644
--- a/studio/backend/tests/test_diffusion_compile_cache.py
+++ b/studio/backend/tests/test_diffusion_compile_cache.py
@@ -1133,3 +1133,25 @@ def test_a_temp_file_left_by_a_killed_save_is_collected(monkeypatch, tmp_path, m
assert cc._collect_superseded(ctx.dir, None) == [stranded.name]
assert not stranded.exists()
assert ctx.bundle.exists()
+
+
+def test_mark_recompiled_dirties_a_context_whose_shape_is_already_registered(tmp_path):
+ # Automatic dynamic recompiles on the first new text length at an unchanged (width, height, batch); register_shape
+ # sees no new key, so the recompiled graphs only reach disk if the context is re-dirtied explicitly.
+ ctx = cc.CacheContext(
+ key = "abc",
+ dir = tmp_path / "abc",
+ bundle = tmp_path / "abc" / "cache.bin",
+ manifest_path = tmp_path / "abc" / "manifest.json",
+ env_fp = "e",
+ model_fp = "m",
+ mode = "auto",
+ )
+ cc.register_shape(ctx, (1024, 1024, 1), static = True)
+ ctx.saved = True
+ seq = ctx.dirty_seq
+ cc.register_shape(ctx, (1024, 1024, 1), static = True)
+ assert ctx.saved is True and ctx.dirty_seq == seq
+ cc.mark_recompiled(ctx)
+ assert ctx.saved is False and ctx.dirty_seq == seq + 1
+ cc.mark_recompiled(None) # no context, no error
diff --git a/studio/backend/tests/test_diffusion_speed.py b/studio/backend/tests/test_diffusion_speed.py
index 40d9d6f6065..76befa7d08c 100644
--- a/studio/backend/tests/test_diffusion_speed.py
+++ b/studio/backend/tests/test_diffusion_speed.py
@@ -1117,3 +1117,264 @@ def test_cuda_graph_install_failure_leaves_the_load_usable(monkeypatch):
)
assert applied["cuda_graph"] is False and calls["installs"] == 1
assert applied["compiled"] is True # the rest of the tier still engaged
+
+
+# ── runtime compile failure falls back to eager ──────────────────────────────
+# compile_repeated_blocks is lazy: inductor runs on the first forward, inside generate(). A lowering bug there
+# (Qwen-Image-2.1 int8 / fp8: inductor CantSplit) used to fail every render; the guard drops that DiT to eager.
+
+
+class _BackendCompilerFailed(Exception):
+ pass
+
+
+class _OutOfMemory(RuntimeError):
+ pass
+
+
+def _stub_torch_compile_errors(monkeypatch):
+ torch = _stub_torch(monkeypatch)
+ torch._dynamo = types.SimpleNamespace(
+ exc = types.SimpleNamespace(BackendCompilerFailed = _BackendCompilerFailed)
+ )
+ torch.OutOfMemoryError = _OutOfMemory
+ return torch
+
+
+class _Block:
+ """A Module.compile'd block: __call__ routes through _compiled_call_impl when set, like nn.Module."""
+
+ def __init__(self, compiled) -> None:
+ self.eager_calls = 0
+ self._compiled_call_impl = compiled
+
+ def _call_impl(self, x):
+ self.eager_calls += 1
+ return x + 1
+
+ def __call__(self, x):
+ if self._compiled_call_impl is not None:
+ return self._compiled_call_impl(x)
+ return self._call_impl(x)
+
+
+class _Dit:
+ def __init__(self, blocks) -> None:
+ self.blocks = blocks
+
+ def modules(self):
+ return [self, *self.blocks]
+
+
+def test_compile_failure_at_first_forward_falls_back_to_eager(monkeypatch):
+ _stub_torch_compile_errors(monkeypatch)
+ calls = {"compiled": 0}
+
+ def broken(x):
+ calls["compiled"] += 1
+ raise _BackendCompilerFailed("CantSplit: 4096*s87 - 4096*s89 not divisible by s87 - s89")
+
+ blocks = [_Block(broken), _Block(broken)]
+ dit = _Dit(blocks)
+ assert ds_mod.guard_compiled_blocks(dit) == 2
+ assert blocks[0](1) == 2 # the failed call itself is answered eagerly
+ assert blocks[1](1) == 2
+ assert blocks[0](5) == 6
+ # One failure flips the whole DiT: the broken lowering is attempted once, not per block or per call.
+ assert calls["compiled"] == 1
+ assert all(b._compiled_call_impl is None for b in blocks)
+ pipe = types.SimpleNamespace(transformer = dit)
+ assert "CantSplit" in ds_mod.compile_fallback_error(pipe)
+
+
+def test_compiled_block_that_works_is_untouched(monkeypatch):
+ _stub_torch_compile_errors(monkeypatch)
+ block = _Block(lambda x: x * 10)
+ dit = _Dit([block])
+ ds_mod.guard_compiled_blocks(dit)
+ ds_mod.guard_compiled_blocks(dit) # idempotent: no double wrap
+ assert block(3) == 30
+ assert block.eager_calls == 0
+ assert ds_mod.compile_fallback_error(types.SimpleNamespace(transformer = dit)) is None
+
+
+def test_settle_fallback_keeps_compiled_while_another_dit_still_compiles(monkeypatch):
+ # Dual-DiT loads fall back per DiT. While the second expert still runs compiled, status keeps "compiled" (the LoRA
+ # gate and the compile-cache shape registry read it) and only gains the fallback marker; once both fell back,
+ # "compiled" goes.
+ _stub_torch_compile_errors(monkeypatch)
+
+ def broken(x):
+ raise _BackendCompilerFailed("CantSplit")
+
+ first = _Dit([_Block(broken)])
+ second_block = _Block(broken)
+ second_block_ok = _Block(lambda x: x * 2)
+ second = _Dit([second_block_ok])
+ ds_mod.guard_compiled_blocks(first)
+ ds_mod.guard_compiled_blocks(second)
+ pipe = types.SimpleNamespace(transformer = first, transformer_2 = second)
+ state = types.SimpleNamespace(speed_optims = ("compiled", "cuda_graph"))
+ first.blocks[0](1)
+ assert "CantSplit" in ds_mod.settle_compile_fallback(state, pipe)
+ assert state.speed_optims == ("compiled", "cuda_graph", "compile_fallback_eager")
+ ds_mod.settle_compile_fallback(state, pipe) # idempotent
+ assert state.speed_optims == ("compiled", "cuda_graph", "compile_fallback_eager")
+
+ third = _Dit([second_block])
+ ds_mod.guard_compiled_blocks(third)
+ pipe.transformer_2 = third
+ third.blocks[0](1)
+ ds_mod.settle_compile_fallback(state, pipe)
+ assert state.speed_optims == ("cuda_graph", "compile_fallback_eager")
+
+
+def test_settle_fallback_sees_a_modular_workflows_named_partition(monkeypatch):
+ # MiniMax-H3's reference workflow compiles transformer_ref through a view; settlement runs on the bare pipe.
+ _stub_torch_compile_errors(monkeypatch)
+
+ def broken(x):
+ raise _BackendCompilerFailed("CantSplit")
+
+ ref = _Dit([_Block(broken)])
+ ds_mod.guard_compiled_blocks(ref)
+ pipe = types.SimpleNamespace(transformer_ref = ref)
+ state = types.SimpleNamespace(speed_optims = ("compiled",))
+ ref.blocks[0](1)
+ assert "CantSplit" in ds_mod.settle_compile_fallback(state, pipe)
+ assert state.speed_optims == ("compile_fallback_eager",)
+
+
+def test_settle_fallback_is_a_noop_without_a_failure(monkeypatch):
+ _stub_torch_compile_errors(monkeypatch)
+ dit = _Dit([_Block(lambda x: x)])
+ ds_mod.guard_compiled_blocks(dit)
+ state = types.SimpleNamespace(speed_optims = ("compiled",))
+ assert ds_mod.settle_compile_fallback(state, types.SimpleNamespace(transformer = dit)) is None
+ assert state.speed_optims == ("compiled",)
+
+
+class _BackendOutOfMemory(RuntimeError):
+ pass
+
+
+_BackendOutOfMemory.__name__ = "OutOfMemoryError_"
+
+
[email protected]("inner", ["message", "backend_class"])
+def test_noncanonical_oom_under_a_compile_error_is_not_swallowed(monkeypatch, inner):
+ # A compile-time allocation failure can surface as a plain RuntimeError("... out of memory ...") or a
+ # backend-specific OutOfMemoryError_ under BackendCompilerFailed; the batch backoff must still see it.
+ _stub_torch_compile_errors(monkeypatch)
+
+ def fails(x):
+ cause = (
+ RuntimeError("HIP out of memory. Tried to allocate 2.00 GiB")
+ if inner == "message"
+ else (_BackendOutOfMemory("allocation failed"))
+ )
+ try:
+ raise cause
+ except RuntimeError as err:
+ raise _BackendCompilerFailed("autotune failed") from err
+
+ block = _Block(fails)
+ dit = _Dit([block])
+ ds_mod.guard_compiled_blocks(dit)
+ with pytest.raises(_BackendCompilerFailed) as raised:
+ block(1)
+ assert block.eager_calls == 0
+ assert ds_mod.compile_fallback_error(types.SimpleNamespace(transformer = dit)) is None
+ # What the image / video handlers receive is the outer compiler error; their backoff must classify it as OOM.
+ from core.inference.diffusion_batched import is_oom_error
+
+ assert is_oom_error(raised.value)
+
+
[email protected]("kind", ["runtime", "oom"])
+def test_non_compile_errors_are_not_swallowed(monkeypatch, kind):
+ # Only a failure while BUILDING the graph is safe to retry eagerly. A kernel error, and an OOM even when inductor
+ # wrapped it, must reach the caller (the OOM backoff splits the batch on it).
+ _stub_torch_compile_errors(monkeypatch)
+
+ def fails(x):
+ if kind == "runtime":
+ raise RuntimeError("CUDA error: an illegal memory access was encountered")
+ try:
+ raise _OutOfMemory("CUDA out of memory")
+ except _OutOfMemory as oom:
+ raise _BackendCompilerFailed("autotune ran out") from oom
+
+ block = _Block(fails)
+ dit = _Dit([block])
+ ds_mod.guard_compiled_blocks(dit)
+ with pytest.raises((RuntimeError, _BackendCompilerFailed)):
+ block(1)
+ assert block.eager_calls == 0
+ assert ds_mod.compile_fallback_error(types.SimpleNamespace(transformer = dit)) is None
+
+
+class _TorchaoWeight:
+ pass
+
+
+_TorchaoWeight.__module__ = "torchao.quantization.quantize_.workflows.int8.int8_tensor"
+
+
+def test_torchao_dit_compiles_with_automatic_dynamic(monkeypatch):
+ # dynamic=True made Qwen-Image-2.1's attention-output cat fuse into torchao's per-row activation-quant reduction,
+ # which inductor cannot split; automatic dynamic (None) compiles it. A bf16 DiT keeps dynamic=True.
+ _stub_torch(monkeypatch)
+ _stub_gguf_accel(monkeypatch)
+ quant = _Pipe(with_compile = True)
+ quant.transformer.parameters = lambda: iter([_TorchaoWeight()])
+ apply_speed_optims(quant, _target(), is_gguf = False, family = _family(), speed_mode = SPEED_DEFAULT)
+ assert quant.compile_kwargs["dynamic"] is None
+ assert ds_mod.compiled_shapes_are_static(quant, SPEED_DEFAULT) is True
+
+ dense = _Pipe(with_compile = True)
+ dense.transformer.parameters = lambda: iter([object()])
+ apply_speed_optims(dense, _target(), is_gguf = False, family = _family(), speed_mode = SPEED_DEFAULT)
+ assert dense.compile_kwargs["dynamic"] is True
+ assert ds_mod.compiled_shapes_are_static(dense, SPEED_DEFAULT) is False
+
+
+def test_torchao_payload_wrapped_in_a_plain_parameter_still_counts():
+ # Some torchao builds keep Linear.weight an nn.Parameter whose .data is the subclass.
+ wrapped = types.SimpleNamespace(data = _TorchaoWeight())
+ assert ds_mod._carries_torchao_weights(
+ types.SimpleNamespace(parameters = lambda: iter([wrapped]))
+ )
+ other = types.SimpleNamespace(data = object())
+ assert not ds_mod._carries_torchao_weights(
+ types.SimpleNamespace(parameters = lambda: iter([other]))
+ )
+
+
+def test_compile_dynamic_is_the_value_the_cache_fingerprint_keys_on():
+ # diffusion.py fingerprints compile-cache bundles with compile_dynamic, so an automatic-dynamic torchao build
+ # never reuses a bundle written by the old explicit-dynamic path.
+ quant = types.SimpleNamespace(parameters = lambda: iter([_TorchaoWeight()]))
+ dense = types.SimpleNamespace(parameters = lambda: iter([object()]))
+ assert ds_mod.compile_dynamic(quant, True) is None
+ assert ds_mod.compile_dynamic(quant, False) is False
+ assert ds_mod.compile_dynamic(dense, True) is True
+ assert ds_mod.compile_dynamic(None, True) is True
+
+
+def test_max_tier_keeps_static_compile_for_torchao_dit(monkeypatch):
+ _stub_torch(monkeypatch)
+ _stub_gguf_accel(monkeypatch)
+ pipe = _Pipe(with_compile = True)
+ pipe.transformer.parameters = lambda: iter([_TorchaoWeight()])
+ apply_speed_optims(pipe, _target(), is_gguf = False, family = _family(), speed_mode = SPEED_MAX)
+ assert pipe.compile_kwargs["dynamic"] is False
+
+
+def test_auto_dynamic_active_follows_the_torchao_marker():
+ dit = types.SimpleNamespace()
+ pipe = types.SimpleNamespace(transformer = dit)
+ assert ds_mod.auto_dynamic_active(pipe) is False
+ dit._unsloth_auto_dynamic = True
+ assert ds_mod.auto_dynamic_active(pipe) is True
+ assert isinstance(ds_mod.dynamo_graph_count(), int)