Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: keep Qwen-Image-2.1 int8 / fp8 compiling on torch 2.12 (CantSplit) (#11677)
Daniel Han69c154d56e39
diff --git a/studio/backend/core/inference/diffusion.py b/studio/backend/core/inference/diffusion.pyindex bf9381ca7c9..ba6ad5e9062 100644--- a/studio/backend/core/inference/diffusion.py+++ b/studio/backend/core/inference/diffusion.py@@ -124,11 +124,15 @@ from .diffusion_speed import (SPEED_MAX,SPEED_OFF,apply_speed_optims,+ auto_dynamic_active,+ compile_dynamic,compile_eligible,compiled_shapes_are_static,+ dynamo_graph_count,normalize_speed_mode,resolve_speed_mode,restore_backend_flags,+ settle_compile_fallback,snapshot_backend_flags,)from .diffusion_attention import (@@ -5562,7 +5566,11 @@ class DiffusionBackend:"fullgraph": cache_engaged is Noneand not cache_may_toggleand plan.offload_policy == OFFLOAD_NONE,- "dynamic": effective_speed != SPEED_MAX,+ # The value apply_speed_optims resolves (automatic for torchao weights), so a bundle+ # from an explicit-dynamic build is not reused for an automatic-dynamic one.+ "dynamic": compile_dynamic(+ getattr(pipe, "transformer", None), effective_speed != SPEED_MAX+ ),"mode": "max-autotune-no-cudagraphs"if effective_speed == SPEED_MAXelse "default",@@ -6907,7 +6915,7 @@ class DiffusionBackend:"fullgraph": state.transformer_cache is Noneand not state.cache_autoand state.offload_policy == OFFLOAD_NONE,- "dynamic": True,+ "dynamic": compile_dynamic(getattr(state.pipe, "transformer", None), True),"mode": "default",},logger = logger,@@ -7416,6 +7424,7 @@ class DiffusionBackend:per_image_seeds: list[int] = []chunk_shapes: list[int] = []pending = list(chunks)+ graphs_before = dynamo_graph_count()while pending:chunk = pending.pop(0)chunk_kwargs = dict(kwargs)@@ -7479,6 +7488,11 @@ class DiffusionBackend:len(second_half),)continue+ finally:+ # A guarded block that fell back to eager (compile failed at its first forward) no longer runs+ # compiled: report it on every exit, cancel and error included, so status, LoRA gating and the+ # compile-cache shape registry stop treating it as compiled.+ settle_compile_fallback(state, state.pipe, logger)if cancel.is_set():raise RuntimeError(DIFFUSION_CANCELLED_MSG)images.extend(out)@@ -7505,6 +7519,10 @@ class DiffusionBackend:(reg_width, reg_height, int(chunk_batch)),static = static_shapes,)+ if auto_dynamic_active(state.pipe) and dynamo_graph_count() > graphs_before:+ # Automatic dynamic recompiles on the first new text length at an already-registered+ # (width, height, batch): persist those graphs too, or every fresh process pays them again.+ compile_cache.mark_recompiled(state.compile_cache_ctx)compile_cache.save_async(state.compile_cache_ctx, logger = logger)except Exception: # noqa: BLE001 - cache persistence is best-effortpassdiff --git a/studio/backend/core/inference/diffusion_batched.py b/studio/backend/core/inference/diffusion_batched.pyindex a1cf1909cb3..dea83231e7b 100644--- a/studio/backend/core/inference/diffusion_batched.py+++ b/studio/backend/core/inference/diffusion_batched.py@@ -132,9 +132,18 @@ def uniform_prompt(chunk: list[tuple[str, int]]) -> Optional[str]:def is_oom_error(exc: BaseException) -> bool:"""Whether an exception is a CUDA/accelerator out-of-memory, worth a smaller- retry. Matched structurally (class name across torch versions / devices) and- by message, so the caller needn't import torch to classify."""- for klass in type(exc).__mro__:- if klass.__name__ == "OutOfMemoryError":+ retry. Matched structurally (class name across torch versions / devices, and+ backend subclasses such as ``OutOfMemoryError_``) and by message, so the caller+ needn't import torch to classify. Walks ``__cause__`` / ``__context__``: torch.compile+ wraps an OOM hit while autotuning in a compiler error whose own message is generic,+ and that allocation failure still deserves the smaller retry."""+ seen = set()+ cur: Optional[BaseException] = exc+ while cur is not None and id(cur) not in seen:+ seen.add(id(cur))+ if any(klass.__name__.startswith("OutOfMemory") for klass in type(cur).__mro__):return True- return "out of memory" in str(exc).lower()+ if "out of memory" in str(cur).lower():+ return True+ cur = cur.__cause__ or cur.__context__+ return Falsediff --git a/studio/backend/core/inference/diffusion_cache.py b/studio/backend/core/inference/diffusion_cache.pyindex 42372482b36..902665e81c2 100644--- a/studio/backend/core/inference/diffusion_cache.py+++ b/studio/backend/core/inference/diffusion_cache.py@@ -179,7 +179,20 @@ def _compile_hooked_block_inners(transformer: Any, logger: Any = None) -> int:continue # not a plain bound method; arming would miss the block# fullgraph=False / dynamic=True: a cache is active (its decision graph-breaks) and this matches the# default tier. Dynamo caches per code object, so re-arming after a toggle is ~free.- fn_ref.original_forward = torch.compile(orig, fullgraph = False, dynamic = True)+ # Automatic dynamic when the speed layer chose it (torchao weights): see _compile_repeated_blocks.+ dynamic = None if getattr(transformer, "_unsloth_auto_dynamic", False) else True+ compiled = torch.compile(orig, fullgraph = False, dynamic = dynamic)+ # Same runtime fallback as the block's own compile: a lowering failure on the first computed step+ # restores the eager inner instead of failing the render.+ guard = getattr(transformer, "_unsloth_compile_guard", None)+ if guard is not None:++ def restore(ref: Any = fn_ref, inner: Any = orig) -> None:+ ref.original_forward = inner++ guard.restores.append(restore)+ compiled = guard.wrap(compiled, orig, transformer)+ fn_ref.original_forward = compiledhook._unsloth_orig_inner = origarmed += 1except Exception as exc: # noqa: BLE001 -- best-effort: the cache still works eagerdiff --git a/studio/backend/core/inference/diffusion_compile_cache.py b/studio/backend/core/inference/diffusion_compile_cache.pyindex 56d25624c1c..9ad178db357 100644--- a/studio/backend/core/inference/diffusion_compile_cache.py+++ b/studio/backend/core/inference/diffusion_compile_cache.py@@ -525,6 +525,20 @@ def register_shape(ctx: Optional[CacheContext], shape: Any, *, static: bool) ->pass+def mark_recompiled(ctx: Optional[CacheContext]) -> None:+ """A render compiled new graphs for a shape the bundle already covers (automatic dynamic re-specialises on the+ first new text length, keyed by nothing ``register_shape`` sees), so the bundle on disk lacks them: clear+ ``saved`` so the next ``save`` rewrites it. Never raises."""+ if ctx is None:+ return+ try:+ with _dirty_lock:+ ctx.saved = False+ ctx.dirty_seq += 1+ except Exception: # noqa: BLE001 - bookkeeping only+ pass++def _try_load(ctx: CacheContext,logger: Any,diff --git a/studio/backend/core/inference/diffusion_speed.py b/studio/backend/core/inference/diffusion_speed.pyindex 2658925399e..b5c8475e559 100644--- a/studio/backend/core/inference/diffusion_speed.py+++ b/studio/backend/core/inference/diffusion_speed.py@@ -388,7 +388,12 @@ def compiled_shapes_are_static(pipe: Any, speed_mode: Optional[str]) -> bool:mode = normalize_speed_mode(speed_mode)if mode == SPEED_MAX:return True- return mode == SPEED_DEFAULT and _denoiser_unet(pipe) is not None+ if mode != SPEED_DEFAULT:+ return False+ # A torchao-quantised DiT compiles with automatic dynamic: its first shapes get their own artifacts.+ return _denoiser_unet(pipe) is not None or any(+ getattr(t, "_unsloth_auto_dynamic", False) for t in _denoiser_dits(pipe)+ )def _denoiser_dits(pipe: Any) -> list:@@ -465,12 +470,19 @@ def _compile_repeated_blocks(# Compile every denoiser DiT (dual-DiT families run both); a per-DiT failure degrades only that one to eager.engaged = Falsefor transformer in dits:+ dit_kwargs = dict(kwargs)+ dit_kwargs["dynamic"] = compile_dynamic(transformer, kwargs["dynamic"])+ if dit_kwargs["dynamic"] is None:+ transformer._unsloth_auto_dynamic = Truetry:- transformer.compile_repeated_blocks(**kwargs)+ transformer.compile_repeated_blocks(**dit_kwargs)engaged = Trueexcept Exception as exc: # noqa: BLE001 - optimisation only_warn(logger, "compile_repeated_blocks", exc)continue+ # compile_repeated_blocks is lazy: inductor only runs on the first forward, inside generate(), where a lowering+ # bug would fail the render. Guard every compiled block so such a failure drops this DiT to eager instead.+ guard_compiled_blocks(transformer, logger)# A step cache engaged BEFORE this compile already wrapped each block forward in a disabled hook, so the compute# branch would run eager and forfeit the regional compile. Re-point the hooks' inner forward at compiled# wrappers (no-op without them).@@ -482,6 +494,216 @@ def _compile_repeated_blocks(return engaged+def compile_dynamic(transformer: Any, dynamic: Optional[bool]) -> Optional[bool]:+ """The ``dynamic`` a DiT is actually compiled with, so compile-cache fingerprints key on the same value.++ dynamic=True makes even the constant segment starts symbolic, and on Qwen-Image-2.1 the attention output cat+ (text + target, length s87 - s89) then fuses into torchao's per-row activation-quant reduction, which inductor+ cannot split (CantSplit, every render failed). Automatic dynamic (None) compiles the first shapes static and only+ generalises what actually varies: stable after ~3 recompiles, same numerics."""+ if dynamic and transformer is not None and _carries_torchao_weights(transformer):+ return None+ return dynamic+++def _carries_torchao_weights(module: Any) -> bool:+ """Whether any parameter of ``module`` is a torchao tensor subclass (the int8 / fp8 dense fast path).++ Checks ``.data`` too: some torchao builds keep ``Linear.weight`` a plain ``nn.Parameter`` wrapping the subclass,+ the same two representations ``transformer_is_quantised`` covers. Only torchao's own types count, so a GGUF or+ bitsandbytes parameter subclass keeps its existing compile policy."""++ def _torchao(t: Any) -> bool:+ return t is not None and type(t).__module__.startswith("torchao")++ try:+ return any(_torchao(p) or _torchao(getattr(p, "data", None)) for p in module.parameters())+ except Exception: # noqa: BLE001 - not a torch module (tests/fakes)+ return False+++def is_compile_failure(exc: BaseException) -> bool:+ """True for an error torch.compile raises while BUILDING a graph (dynamo tracing, inductor lowering / codegen), as+ opposed to one raised by kernels that already built. Only the former is safe to answer with an eager retry: nothing+ ran yet. An OOM is never one, even when inductor wraps it (autotune ran out), so the OOM backoff still sees it."""+ try:+ import torch+ except Exception: # noqa: BLE001 - no torch, nothing compiled+ return False+ from .diffusion_batched import is_oom_error++ # The batch backoff's own classifier walks the whole cause chain, so an OOM anywhere in it goes to the backoff+ # (which recognises the same outer exception) instead of the eager retry.+ if is_oom_error(exc):+ return False+ kinds: list = []+ dynamo_exc = getattr(getattr(torch, "_dynamo", None), "exc", None)+ for name in (+ "BackendCompilerFailed",+ "Unsupported",+ "TorchRuntimeError",+ "InternalTorchDynamoError",+ ):+ kind = getattr(dynamo_exc, name, None) if dynamo_exc is not None else None+ if isinstance(kind, type):+ kinds.append(kind)+ try:+ from torch._inductor.exc import (+ InductorError,+ ) # torch 2.7+; older torch wraps it in BackendCompilerFailed+ kinds.append(InductorError)+ except Exception: # noqa: BLE001+ pass+ return bool(kinds) and isinstance(exc, tuple(kinds))+++class _CompileGuard:+ """Per-DiT record of a compile that failed at runtime. The first failure flips every guarded block of that DiT to+ eager (one shared decision: a half-compiled stack would retry the same broken lowering per block)."""++ def __init__(self, logger: Any) -> None:+ self.logger = logger+ self.error: Optional[str] = None+ self.restores: list = []++ def fail(self, exc: BaseException, owner: Any) -> None:+ if self.error is None:+ self.error = f"{type(exc).__name__}: {str(exc).splitlines()[0] if str(exc) else ''}"[+ :300+ ]+ for restore in self.restores:+ try:+ restore()+ except Exception: # noqa: BLE001 - per-block best-effort; the guard still routes to eager+ pass+ if self.logger is not None:+ self.logger.warning(+ "diffusion.speed: torch.compile failed on %s at the first forward (%s); this load runs eager",+ type(owner).__name__,+ self.error,+ )++ def wrap(self, compiled: Any, eager: Any, owner: Any) -> Any:+ guard = self++ def guarded(*args: Any, **kwargs: Any) -> Any:+ if guard.error is None:+ try:+ return compiled(*args, **kwargs)+ except Exception as exc: # noqa: BLE001 - reraised unless a compile-time failure+ if not is_compile_failure(exc):+ raise+ guard.fail(exc, owner)+ # The handled exception pins inductor's frames (and the traced fake tensors) through the retry.+ exc.__traceback__ = None+ return eager(*args, **kwargs)++ guarded._unsloth_compile_guard = guard+ return guarded+++def guard_compiled_blocks(transformer: Any, logger: Any = None) -> int:+ """Wrap each ``Module.compile``d submodule's ``_compiled_call_impl`` so a compile-time failure on the first forward+ (e.g. inductor ``CantSplit`` on a torchao-quantised Qwen-Image-2.1 block) falls back to that module's eager+ ``_call_impl`` for this and every later call. Idempotent. Returns the number of modules guarded."""+ guard = getattr(transformer, "_unsloth_compile_guard", None)+ if guard is None:+ guard = _CompileGuard(logger)+ try:+ transformer._unsloth_compile_guard = guard+ except Exception: # noqa: BLE001 - not a settable module (tests/fakes)+ return 0+ count = 0+ try:+ modules = list(transformer.modules())+ except Exception: # noqa: BLE001 - not a torch module+ return 0+ for module in modules:+ compiled = getattr(module, "_compiled_call_impl", None)+ if compiled is None or getattr(compiled, "_unsloth_compile_guard", None) is not None:+ continue+ eager = getattr(module, "_call_impl", None)+ if eager is None:+ continue++ def restore(m: Any = module) -> None:+ m._compiled_call_impl = None++ guard.restores.append(restore)+ module._compiled_call_impl = guard.wrap(compiled, eager, transformer)+ count += 1+ return count+++def _guarded_dits(pipe: Any) -> list:+ """Every DiT a compile guard can sit on: the standard denoisers plus a modular workflow's named partition+ (MiniMax-H3's ``transformer_ref`` compiles through a view, so the bare pipe's ``_denoiser_dits`` misses it)."""+ dits = _denoiser_dits(pipe)+ extra = getattr(pipe, "transformer_ref", None)+ if extra is not None and extra not in dits:+ dits.append(extra)+ return dits+++def compiled_dits_active(pipe: Any) -> bool:+ """True while at least one guarded DiT still runs its compiled blocks (dual-DiT pipelines fall back per DiT)."""+ for transformer in _guarded_dits(pipe):+ guard = getattr(transformer, "_unsloth_compile_guard", None)+ if guard is not None and not guard.error:+ return True+ return False+++def settle_compile_fallback(+ state: Any,+ pipe: Any,+ logger: Any = None,+) -> Optional[str]:+ """After a render, fold a runtime compile fallback into ``state.speed_optims`` (image and video backends share it).++ Adds ``compile_fallback_eager`` once, and drops ``compiled`` only when NO guarded DiT still runs compiled, so a+ dual-DiT load whose second expert still compiles keeps the LoRA gate and the compile-cache shape registry. Returns+ the recorded failure, or None when nothing fell back."""+ fallback = compile_fallback_error(pipe)+ if not fallback:+ return None+ optims = tuple(getattr(state, "speed_optims", None) or ())+ updated = list(optims)+ if "compile_fallback_eager" not in updated:+ updated.append("compile_fallback_eager")+ if logger is not None:+ logger.warning("diffusion.speed: regional compile fell back to eager: %s", fallback)+ if "compiled" in updated and not compiled_dits_active(pipe):+ updated.remove("compiled")+ if tuple(updated) != optims:+ # The load states are frozen dataclasses; the status must still reflect what actually runs.+ object.__setattr__(state, "speed_optims", tuple(updated))+ return fallback+++def auto_dynamic_active(pipe: Any) -> bool:+ """Whether any denoiser DiT compiled with automatic dynamic (torchao weights on the default tier)."""+ return any(getattr(t, "_unsloth_auto_dynamic", False) for t in _guarded_dits(pipe))+++def dynamo_graph_count() -> int:+ """Graphs dynamo has compiled in this process (0 when unavailable); a delta across a render means it compiled."""+ try:+ from torch._dynamo.utils import counters+ return int(counters["stats"]["unique_graphs"])+ except Exception: # noqa: BLE001+ return 0+++def compile_fallback_error(pipe: Any) -> Optional[str]:+ """The compile failure a guarded DiT fell back from, or None while every compiled DiT still runs compiled."""+ for transformer in _guarded_dits(pipe):+ guard = getattr(transformer, "_unsloth_compile_guard", None)+ if guard is not None and guard.error:+ return guard.error+ return None++def _compile_vae_decode(pipe: Any, logger: Any) -> bool:"""torch.compile the VAE ``decode`` bound method in place (U-Net families; caller gates).Instance-level assignment: the pipe owns it and the module object is untouched."""diff --git a/studio/backend/core/inference/video.py b/studio/backend/core/inference/video.pyindex c553e08e3ba..b338ace4008 100644--- a/studio/backend/core/inference/video.py+++ b/studio/backend/core/inference/video.py@@ -92,6 +92,7 @@ from .diffusion_speed import (apply_speed_optims,resolve_speed_mode,restore_backend_flags,+ settle_compile_fallback,snapshot_backend_flags,)from .diffusion_auto_policy import (@@ -6280,6 +6281,11 @@ class VideoBackend:except Exception: # noqa: BLE001 -- cleanup is best-effortpassraise RuntimeError(VIDEO_CANCELLED_MSG) from None+ finally:+ # A guarded compiled block that failed to build at its first forward now runs eager; the status+ # must not keep reporting it compiled (a forced-compile quantised load runs ~30x slower eager),+ # whether this render finished, was cancelled or failed.+ settle_compile_fallback(state, pipe, logger)if cancel.is_set():raise RuntimeError(VIDEO_CANCELLED_MSG)diff --git a/studio/backend/tests/test_diffusion_batched.py b/studio/backend/tests/test_diffusion_batched.pyindex 26df099b083..6cded8f84d0 100644--- a/studio/backend/tests/test_diffusion_batched.py+++ b/studio/backend/tests/test_diffusion_batched.py@@ -151,3 +151,22 @@ def test_is_oom_error_matches_class_name_and_message():assert is_oom_error(RuntimeError("CUDA out of memory. Tried to allocate 2 GiB"))assert not is_oom_error(RuntimeError("shape mismatch"))assert not is_oom_error(ValueError("bad prompt"))+++def test_is_oom_error_sees_an_oom_wrapped_by_a_compiler_error():+ # torch.compile re-raises an autotune OOM as a compiler error with a generic message; the backoff must still split.+ class OutOfMemoryError_(RuntimeError):+ pass++ class BackendCompilerFailed(RuntimeError):+ pass++ for inner in (OutOfMemoryError_("alloc"), RuntimeError("HIP out of memory")):+ try:+ try:+ raise inner+ except RuntimeError as oom:+ raise BackendCompilerFailed("autotune failed") from oom+ except BackendCompilerFailed as outer:+ assert is_oom_error(outer)+ assert not is_oom_error(BackendCompilerFailed("autotune failed"))diff --git a/studio/backend/tests/test_diffusion_compile_cache.py b/studio/backend/tests/test_diffusion_compile_cache.pyindex 00eccb92b26..b7abd546942 100644--- a/studio/backend/tests/test_diffusion_compile_cache.py+++ b/studio/backend/tests/test_diffusion_compile_cache.py@@ -1133,3 +1133,25 @@ def test_a_temp_file_left_by_a_killed_save_is_collected(monkeypatch, tmp_path, massert cc._collect_superseded(ctx.dir, None) == [stranded.name]assert not stranded.exists()assert ctx.bundle.exists()+++def test_mark_recompiled_dirties_a_context_whose_shape_is_already_registered(tmp_path):+ # Automatic dynamic recompiles on the first new text length at an unchanged (width, height, batch); register_shape+ # sees no new key, so the recompiled graphs only reach disk if the context is re-dirtied explicitly.+ ctx = cc.CacheContext(+ key = "abc",+ dir = tmp_path / "abc",+ bundle = tmp_path / "abc" / "cache.bin",+ manifest_path = tmp_path / "abc" / "manifest.json",+ env_fp = "e",+ model_fp = "m",+ mode = "auto",+ )+ cc.register_shape(ctx, (1024, 1024, 1), static = True)+ ctx.saved = True+ seq = ctx.dirty_seq+ cc.register_shape(ctx, (1024, 1024, 1), static = True)+ assert ctx.saved is True and ctx.dirty_seq == seq+ cc.mark_recompiled(ctx)+ assert ctx.saved is False and ctx.dirty_seq == seq + 1+ cc.mark_recompiled(None) # no context, no errordiff --git a/studio/backend/tests/test_diffusion_speed.py b/studio/backend/tests/test_diffusion_speed.pyindex 40d9d6f6065..76befa7d08c 100644--- a/studio/backend/tests/test_diffusion_speed.py+++ b/studio/backend/tests/test_diffusion_speed.py@@ -1117,3 +1117,264 @@ def test_cuda_graph_install_failure_leaves_the_load_usable(monkeypatch):)assert applied["cuda_graph"] is False and calls["installs"] == 1assert applied["compiled"] is True # the rest of the tier still engaged+++# ── runtime compile failure falls back to eager ──────────────────────────────+# compile_repeated_blocks is lazy: inductor runs on the first forward, inside generate(). A lowering bug there+# (Qwen-Image-2.1 int8 / fp8: inductor CantSplit) used to fail every render; the guard drops that DiT to eager.+++class _BackendCompilerFailed(Exception):+ pass+++class _OutOfMemory(RuntimeError):+ pass+++def _stub_torch_compile_errors(monkeypatch):+ torch = _stub_torch(monkeypatch)+ torch._dynamo = types.SimpleNamespace(+ exc = types.SimpleNamespace(BackendCompilerFailed = _BackendCompilerFailed)+ )+ torch.OutOfMemoryError = _OutOfMemory+ return torch+++class _Block:+ """A Module.compile'd block: __call__ routes through _compiled_call_impl when set, like nn.Module."""++ def __init__(self, compiled) -> None:+ self.eager_calls = 0+ self._compiled_call_impl = compiled++ def _call_impl(self, x):+ self.eager_calls += 1+ return x + 1++ def __call__(self, x):+ if self._compiled_call_impl is not None:+ return self._compiled_call_impl(x)+ return self._call_impl(x)+++class _Dit:+ def __init__(self, blocks) -> None:+ self.blocks = blocks++ def modules(self):+ return [self, *self.blocks]+++def test_compile_failure_at_first_forward_falls_back_to_eager(monkeypatch):+ _stub_torch_compile_errors(monkeypatch)+ calls = {"compiled": 0}++ def broken(x):+ calls["compiled"] += 1+ raise _BackendCompilerFailed("CantSplit: 4096*s87 - 4096*s89 not divisible by s87 - s89")++ blocks = [_Block(broken), _Block(broken)]+ dit = _Dit(blocks)+ assert ds_mod.guard_compiled_blocks(dit) == 2+ assert blocks[0](1) == 2 # the failed call itself is answered eagerly+ assert blocks[1](1) == 2+ assert blocks[0](5) == 6+ # One failure flips the whole DiT: the broken lowering is attempted once, not per block or per call.+ assert calls["compiled"] == 1+ assert all(b._compiled_call_impl is None for b in blocks)+ pipe = types.SimpleNamespace(transformer = dit)+ assert "CantSplit" in ds_mod.compile_fallback_error(pipe)+++def test_compiled_block_that_works_is_untouched(monkeypatch):+ _stub_torch_compile_errors(monkeypatch)+ block = _Block(lambda x: x * 10)+ dit = _Dit([block])+ ds_mod.guard_compiled_blocks(dit)+ ds_mod.guard_compiled_blocks(dit) # idempotent: no double wrap+ assert block(3) == 30+ assert block.eager_calls == 0+ assert ds_mod.compile_fallback_error(types.SimpleNamespace(transformer = dit)) is None+++def test_settle_fallback_keeps_compiled_while_another_dit_still_compiles(monkeypatch):+ # Dual-DiT loads fall back per DiT. While the second expert still runs compiled, status keeps "compiled" (the LoRA+ # gate and the compile-cache shape registry read it) and only gains the fallback marker; once both fell back,+ # "compiled" goes.+ _stub_torch_compile_errors(monkeypatch)++ def broken(x):+ raise _BackendCompilerFailed("CantSplit")++ first = _Dit([_Block(broken)])+ second_block = _Block(broken)+ second_block_ok = _Block(lambda x: x * 2)+ second = _Dit([second_block_ok])+ ds_mod.guard_compiled_blocks(first)+ ds_mod.guard_compiled_blocks(second)+ pipe = types.SimpleNamespace(transformer = first, transformer_2 = second)+ state = types.SimpleNamespace(speed_optims = ("compiled", "cuda_graph"))+ first.blocks[0](1)+ assert "CantSplit" in ds_mod.settle_compile_fallback(state, pipe)+ assert state.speed_optims == ("compiled", "cuda_graph", "compile_fallback_eager")+ ds_mod.settle_compile_fallback(state, pipe) # idempotent+ assert state.speed_optims == ("compiled", "cuda_graph", "compile_fallback_eager")++ third = _Dit([second_block])+ ds_mod.guard_compiled_blocks(third)+ pipe.transformer_2 = third+ third.blocks[0](1)+ ds_mod.settle_compile_fallback(state, pipe)+ assert state.speed_optims == ("cuda_graph", "compile_fallback_eager")+++def test_settle_fallback_sees_a_modular_workflows_named_partition(monkeypatch):+ # MiniMax-H3's reference workflow compiles transformer_ref through a view; settlement runs on the bare pipe.+ _stub_torch_compile_errors(monkeypatch)++ def broken(x):+ raise _BackendCompilerFailed("CantSplit")++ ref = _Dit([_Block(broken)])+ ds_mod.guard_compiled_blocks(ref)+ pipe = types.SimpleNamespace(transformer_ref = ref)+ state = types.SimpleNamespace(speed_optims = ("compiled",))+ ref.blocks[0](1)+ assert "CantSplit" in ds_mod.settle_compile_fallback(state, pipe)+ assert state.speed_optims == ("compile_fallback_eager",)+++def test_settle_fallback_is_a_noop_without_a_failure(monkeypatch):+ _stub_torch_compile_errors(monkeypatch)+ dit = _Dit([_Block(lambda x: x)])+ ds_mod.guard_compiled_blocks(dit)+ state = types.SimpleNamespace(speed_optims = ("compiled",))+ assert ds_mod.settle_compile_fallback(state, types.SimpleNamespace(transformer = dit)) is None+ assert state.speed_optims == ("compiled",)+++class _BackendOutOfMemory(RuntimeError):+ pass+++_BackendOutOfMemory.__name__ = "OutOfMemoryError_"++[email protected]("inner", ["message", "backend_class"])+def test_noncanonical_oom_under_a_compile_error_is_not_swallowed(monkeypatch, inner):+ # A compile-time allocation failure can surface as a plain RuntimeError("... out of memory ...") or a+ # backend-specific OutOfMemoryError_ under BackendCompilerFailed; the batch backoff must still see it.+ _stub_torch_compile_errors(monkeypatch)++ def fails(x):+ cause = (+ RuntimeError("HIP out of memory. Tried to allocate 2.00 GiB")+ if inner == "message"+ else (_BackendOutOfMemory("allocation failed"))+ )+ try:+ raise cause+ except RuntimeError as err:+ raise _BackendCompilerFailed("autotune failed") from err++ block = _Block(fails)+ dit = _Dit([block])+ ds_mod.guard_compiled_blocks(dit)+ with pytest.raises(_BackendCompilerFailed) as raised:+ block(1)+ assert block.eager_calls == 0+ assert ds_mod.compile_fallback_error(types.SimpleNamespace(transformer = dit)) is None+ # What the image / video handlers receive is the outer compiler error; their backoff must classify it as OOM.+ from core.inference.diffusion_batched import is_oom_error++ assert is_oom_error(raised.value)++[email protected]("kind", ["runtime", "oom"])+def test_non_compile_errors_are_not_swallowed(monkeypatch, kind):+ # Only a failure while BUILDING the graph is safe to retry eagerly. A kernel error, and an OOM even when inductor+ # wrapped it, must reach the caller (the OOM backoff splits the batch on it).+ _stub_torch_compile_errors(monkeypatch)++ def fails(x):+ if kind == "runtime":+ raise RuntimeError("CUDA error: an illegal memory access was encountered")+ try:+ raise _OutOfMemory("CUDA out of memory")+ except _OutOfMemory as oom:+ raise _BackendCompilerFailed("autotune ran out") from oom++ block = _Block(fails)+ dit = _Dit([block])+ ds_mod.guard_compiled_blocks(dit)+ with pytest.raises((RuntimeError, _BackendCompilerFailed)):+ block(1)+ assert block.eager_calls == 0+ assert ds_mod.compile_fallback_error(types.SimpleNamespace(transformer = dit)) is None+++class _TorchaoWeight:+ pass+++_TorchaoWeight.__module__ = "torchao.quantization.quantize_.workflows.int8.int8_tensor"+++def test_torchao_dit_compiles_with_automatic_dynamic(monkeypatch):+ # dynamic=True made Qwen-Image-2.1's attention-output cat fuse into torchao's per-row activation-quant reduction,+ # which inductor cannot split; automatic dynamic (None) compiles it. A bf16 DiT keeps dynamic=True.+ _stub_torch(monkeypatch)+ _stub_gguf_accel(monkeypatch)+ quant = _Pipe(with_compile = True)+ quant.transformer.parameters = lambda: iter([_TorchaoWeight()])+ apply_speed_optims(quant, _target(), is_gguf = False, family = _family(), speed_mode = SPEED_DEFAULT)+ assert quant.compile_kwargs["dynamic"] is None+ assert ds_mod.compiled_shapes_are_static(quant, SPEED_DEFAULT) is True++ dense = _Pipe(with_compile = True)+ dense.transformer.parameters = lambda: iter([object()])+ apply_speed_optims(dense, _target(), is_gguf = False, family = _family(), speed_mode = SPEED_DEFAULT)+ assert dense.compile_kwargs["dynamic"] is True+ assert ds_mod.compiled_shapes_are_static(dense, SPEED_DEFAULT) is False+++def test_torchao_payload_wrapped_in_a_plain_parameter_still_counts():+ # Some torchao builds keep Linear.weight an nn.Parameter whose .data is the subclass.+ wrapped = types.SimpleNamespace(data = _TorchaoWeight())+ assert ds_mod._carries_torchao_weights(+ types.SimpleNamespace(parameters = lambda: iter([wrapped]))+ )+ other = types.SimpleNamespace(data = object())+ assert not ds_mod._carries_torchao_weights(+ types.SimpleNamespace(parameters = lambda: iter([other]))+ )+++def test_compile_dynamic_is_the_value_the_cache_fingerprint_keys_on():+ # diffusion.py fingerprints compile-cache bundles with compile_dynamic, so an automatic-dynamic torchao build+ # never reuses a bundle written by the old explicit-dynamic path.+ quant = types.SimpleNamespace(parameters = lambda: iter([_TorchaoWeight()]))+ dense = types.SimpleNamespace(parameters = lambda: iter([object()]))+ assert ds_mod.compile_dynamic(quant, True) is None+ assert ds_mod.compile_dynamic(quant, False) is False+ assert ds_mod.compile_dynamic(dense, True) is True+ assert ds_mod.compile_dynamic(None, True) is True+++def test_max_tier_keeps_static_compile_for_torchao_dit(monkeypatch):+ _stub_torch(monkeypatch)+ _stub_gguf_accel(monkeypatch)+ pipe = _Pipe(with_compile = True)+ pipe.transformer.parameters = lambda: iter([_TorchaoWeight()])+ apply_speed_optims(pipe, _target(), is_gguf = False, family = _family(), speed_mode = SPEED_MAX)+ assert pipe.compile_kwargs["dynamic"] is False+++def test_auto_dynamic_active_follows_the_torchao_marker():+ dit = types.SimpleNamespace()+ pipe = types.SimpleNamespace(transformer = dit)+ assert ds_mod.auto_dynamic_active(pipe) is False+ dit._unsloth_auto_dynamic = True+ assert ds_mod.auto_dynamic_active(pipe) is True+ assert isinstance(ds_mod.dynamo_graph_count(), int)