Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: offload-first retry for an OOM VAE decode, fuse GroupNorm+SiLU on uncovered VAEs by structure (#13802)
Daniel Hanc57c21f04719
diff --git a/studio/backend/core/inference/decode_layers.py b/studio/backend/core/inference/decode_layers.pynew file mode 100644index 00000000000..ed2a78d6a36--- /dev/null+++ b/studio/backend/core/inference/decode_layers.py@@ -0,0 +1,115 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""Install and remove one wrapper layer on an object's method slot (a VAE's ``decode`` / ``_decode``), exactly.++``wrap`` records what the slot held on that object (an instance-dict entry, possibly an instance-bound override, or+nothing, so the class method showed through) and puts the layer in the instance dict. ``unwrap`` puts back exactly+that when the layer is still outermost. When later wrappers captured it (they hold it in their closures), the layer+cannot be cut out: it turns into a pass-through to what it wrapped, and the ``__wrapped__`` chain skips it.+"""++from __future__ import annotations++import functools+from typing import Any, Callable, Optional++LAYERS_ATTR = "_unsloth_decode_layers"+_MISSING = object()+++class Layer:+ def __init__(self, owner: Any, attr: str, name: str, inner: Callable, original: Any) -> None:+ self.owner = owner+ self.attr = attr+ self.name = name+ self.inner = inner+ self.original = original # the owner's own instance-dict entry before wrapping, or _MISSING+ self.active = True+ self.fn: Optional[Callable] = None++ @property+ def had_own(self) -> bool:+ return self.original is not _MISSING+++def wrap(owner: Any, attr: str, name: str, make: Callable[[Callable, "Layer"], Callable]) -> Layer:+ """Install ``make(inner, layer)`` as the outermost ``attr`` layer of ``owner``."""+ original = owner.__dict__.get(attr, _MISSING)+ inner = getattr(owner, attr)+ layer = Layer(owner, attr, name, inner, original)+ fn = make(inner, layer)++ def dispatch(*args: Any, **kwargs: Any) -> Any:+ if not layer.active:+ return layer.inner(*args, **kwargs)+ return fn(*args, **kwargs)++ functools.update_wrapper(dispatch, fn)+ dispatch.__wrapped__ = inner+ dispatch._unsloth_layer = layer+ layer.fn = dispatch+ setattr(owner, attr, dispatch)+ owner.__dict__.setdefault(LAYERS_ATTR, []).append(layer)+ return layer+++def find(owner: Any, attr: str, name: str) -> Optional[Layer]:+ for layer in reversed(getattr(owner, "__dict__", {}).get(LAYERS_ATTR, ()) or ()):+ if layer.attr == attr and layer.name == name and layer.active:+ return layer+ return None+++def unwrap(layer: Optional[Layer]) -> bool:+ """Remove ``layer``. True when it was outermost and the slot holds exactly what it held before wrapping; False+ when later wrappers sit over it (it is now a pass-through) or it was already removed."""+ if layer is None or not layer.active:+ return False+ owner, attr = layer.owner, layer.attr+ layer.active = False+ layers = owner.__dict__.get(LAYERS_ATTR, [])+ if layer in layers:+ layers.remove(layer)+ if not layers:+ owner.__dict__.pop(LAYERS_ATTR, None)+ current = owner.__dict__.get(attr, _MISSING)+ if current is layer.fn:+ original = layer.original+ for _ in range(16):+ inner_layer = getattr(original, "_unsloth_layer", None)+ if inner_layer is None or inner_layer.active or original is not inner_layer.fn:+ break+ original = inner_layer.original+ if original is not _MISSING:+ owner.__dict__[attr] = original+ else:+ owner.__dict__.pop(attr, None)+ return True+ fn = None if current is _MISSING else current+ for _ in range(16):+ if fn is None:+ break+ if getattr(fn, "__wrapped__", None) is layer.fn:+ try:+ fn.__wrapped__ = layer.inner+ except Exception: # noqa: BLE001 - read-only callable: the pass-through still holds+ pass+ break+ fn = next_layer(fn)+ return False+++def next_layer(fn: Any) -> Any:+ """What the decode wrapper ``fn`` calls through to: the decode in its mutable slot (an eager outer wrapper that+ keeps what it wraps rewritable: the fp16 non-finite check, the ROCm bf16 decode) when it has one, otherwise its+ ``__wrapped__``. Every walker of a decode chain uses this, so a slot is never skipped."""+ slot = getattr(fn, "_unsloth_decode_slot", None)+ if slot is not None:+ inner = getattr(slot, "__dict__", {}).get("decode")+ if inner is not None:+ return inner+ return getattr(fn, "__wrapped__", None)+++__all__ = ["LAYERS_ATTR", "Layer", "find", "next_layer", "unwrap", "wrap"]diff --git a/studio/backend/core/inference/diffusion.py b/studio/backend/core/inference/diffusion.pyindex 04af24d9f34..c8c35d6f365 100644--- a/studio/backend/core/inference/diffusion.py+++ b/studio/backend/core/inference/diffusion.py@@ -9267,6 +9267,12 @@ class DiffusionBackend:)if vae_fp16:speed_applied["vae_fp16_decode"] = True+ # After the speed layer, so it wraps the fused / single-frame _decode the layer installed.+ try:+ from .diffusion_vae_oom import install as install_vae_oom_retry+ install_vae_oom_retry(getattr(pipe, "vae", None), logger, pipe = pipe)+ except Exception as exc: # noqa: BLE001 - a decode OOM then fails as before+ logger.warning("diffusion.vae_oom: not installed: %s", exc)self._raise_if_load_cancelled(_load_token)if (transformer_quant_engaged is not None@@ -11768,6 +11774,14 @@ class DiffusionBackend:if _denoiser_hooked(state.pipe):compile_pipe_below_offload_hooks(state.pipe, logger)arm_graphs_after_placement(state.pipe, speed_applied, logger)+ # the speed layer just rewrapped vae.decode (a compiled decode guard): the OOM retry's scope goes back on top+ try:+ from .diffusion_vae_oom import ensure_outermost+ ensure_outermost(getattr(state.pipe, "vae", None), logger)+ except Exception as exc: # noqa: BLE001 - a decode OOM then fails as before+ logger.warning(+ "diffusion.vae_oom: scope not restored after the deferred profile: %s", exc+ )cuda_graph.arm_block_graphs(state.pipe,speed_applied,diff --git a/studio/backend/core/inference/diffusion_cuda_graph.py b/studio/backend/core/inference/diffusion_cuda_graph.pyindex 3383e9be46a..00ac17cbd71 100644--- a/studio/backend/core/inference/diffusion_cuda_graph.py+++ b/studio/backend/core/inference/diffusion_cuda_graph.py@@ -2120,18 +2120,33 @@ def set_bypass(handles: Any, on: bool) -> None:pass-def reset_all(handles: Any) -> None:+def _holds_graphs(handle: Any) -> bool:+ """True while ``handle`` (or its fallback handle) still holds a captured graph."""+ if getattr(handle, "cache", None):+ return True+ fallback = getattr(handle, "fallback_handle", None)+ return fallback is not None and fallback is not handle and _holds_graphs(fallback)+++def reset_all(handles: Any, strict: bool = False) -> None:"""Drop every captured graph, for every handle (the weights changed).Drops the pool token with the last graph: a stale token dies on the allocator's"use_count > 0 INTERNAL ASSERT FAILED", raised after ``torch.cuda.graph`` entered its side- stream, so the thread is left off the default stream too."""+ stream, so the thread is left off the default stream too. ``strict``: raise when a handle failed+ to reset or still holds a graph (a caller about to move weights must not leave one that could replay)."""+ failed: list = []for handle in handles or ():try:handle.reset()- except Exception: # noqa: BLE001 - optimisation only- pass+ except Exception as exc: # noqa: BLE001 - optimisation only, unless strict+ failed.append(f"{type(handle).__name__}: {type(exc).__name__}: {exc}")+ continue+ if strict and _holds_graphs(handle):+ failed.append(f"{type(handle).__name__}: still holds a captured graph")_drop_pool_if_unused()+ if strict and failed:+ raise RuntimeError("could not drop every captured CUDA graph: " + "; ".join(failed))def uninstall_all(handles: Any, *, logger: Any = None) -> None:diff --git a/studio/backend/core/inference/diffusion_device.py b/studio/backend/core/inference/diffusion_device.pyindex 664d6eb8215..2cda790b7cb 100644--- a/studio/backend/core/inference/diffusion_device.py+++ b/studio/backend/core/inference/diffusion_device.py@@ -12,6 +12,7 @@ the capability flags optimisation paths key off.from __future__ import annotationsimport os+import typesfrom contextlib import contextmanagerfrom contextvars import ContextVarfrom dataclasses import dataclass@@ -295,6 +296,9 @@ def install_rocm_vae_bf16_decode(if mode == "weights" and not parts:mode = "autocast" # no separable decode half: cast nothing+ # The wrapped decode lives in a mutable slot (as the fp16 decode's does): a layer that must sit under this cast (a+ # compiled decode guard under the OOM scope) can be found and rewrapped in place.+ slot = types.SimpleNamespace(decode = decode)if mode == "weights":for part in parts:part.to(torch.bfloat16)@@ -304,16 +308,17 @@ def install_rocm_vae_bf16_decode(def _bf16_decode(z: Any, *args: Any, **kwargs: Any) -> Any:if isinstance(z, torch.Tensor) and z.is_floating_point():z = z.to(torch.bfloat16)- return _as_float32(decode(z, *args, **kwargs), torch)+ return _as_float32(slot.decode(z, *args, **kwargs), torch)else:def _bf16_decode(*args: Any, **kwargs: Any) -> Any:with torch.autocast(device_type = "cuda", dtype = torch.bfloat16):- out = decode(*args, **kwargs)+ out = slot.decode(*args, **kwargs)return _as_float32(out, torch)_bf16_decode._unsloth_bf16_decode = True # type: ignore[attr-defined]+ _bf16_decode._unsloth_decode_slot = slot # type: ignore[attr-defined]_bf16_decode.__wrapped__ = decode # type: ignore[attr-defined]vae.decode = _bf16_decodevae._unsloth_bf16_decode_mode = modediff --git a/studio/backend/core/inference/diffusion_gguf_compile.py b/studio/backend/core/inference/diffusion_gguf_compile.pyindex 3eda3c1b8a8..5635966fc0d 100644--- a/studio/backend/core/inference/diffusion_gguf_compile.py+++ b/studio/backend/core/inference/diffusion_gguf_compile.py@@ -104,7 +104,8 @@ def _guard(compiled: Any, eager: Any, logger: Any) -> Any:@functools.wraps(eager)def guarded(*args: Any, **kwargs: Any) -> Any:- if not failed:+ from .diffusion_bg_compile import eager_forced # noqa: PLC0415+ if not failed and not eager_forced():# The lazy compile runs in the caller's context; on torch 2.12+ the load's knob writes may not be visible.diffusion_compile_config.apply()try:diff --git a/studio/backend/core/inference/diffusion_memory.py b/studio/backend/core/inference/diffusion_memory.pyindex 831700df95b..f38e17dc6a1 100644--- a/studio/backend/core/inference/diffusion_memory.py+++ b/studio/backend/core/inference/diffusion_memory.py@@ -3409,6 +3409,31 @@ def release_resident_groups(blocks from the last); returns a callable pinning exactly those groups again, or None when nothing was resident.A request larger than the measured placement reserved (reference images, a bigger canvas, a batch) then runs asthe flat plan. ``denoisers_only`` leaves the text encoders' groups alone (the prompt encode needs them)."""+ released: list = []+ streamed: set = set()++ def restore(strict: bool = False) -> None:+ """Pin the released groups again. ``strict``: raise when any of them is still not resident afterwards (the+ re-pin catches its own failures and leaves those groups streamed, which callers that must retry need to+ know)."""+ for module in released:+ _keep_groups_resident(+ module,+ module._unsloth_resident_room,+ module._unsloth_resident_device,+ logger,+ only = streamed,+ )+ if strict:+ left = [+ g+ for module in released+ for g in (_offload_groups(module) or [])+ if id(g) in streamed and not getattr(g, "_unsloth_resident", False)+ ]+ if left:+ raise RuntimeError(f"re-pinning resident offload groups left {len(left)} streamed")+try:import torch@@ -3422,8 +3447,6 @@ def release_resident_groups(return Nonemodules.sort(key = lambda m: 0 if _is_text_encoder_module(pipe, m) else 1)left = int(need_mib) * 1024 * 1024- released: list = []- streamed: set = set()for module in modules:for group in reversed(_offload_groups(module) or []):if left <= 0:@@ -3434,15 +3457,16 @@ def release_resident_groups(state = getattr(module, "_unsloth_stream_state", None)if isinstance(state, dict):state["streamed"] = 1+ if module not in released:+ released.append(module)# tied partners stream together (a resident partner would keep reading the host copy)for member in [group, *(getattr(group, "_unsloth_tied_groups", None) or [])]:if not getattr(member, "_unsloth_resident", False):continue- _release_group(member)+ # Recorded before demoting: a release that fails part way is still re-pinned by ``restore``.streamed.add(id(member))+ _release_group(member)left -= int(getattr(member, "_unsloth_resident_bytes", 0))- if module not in released:- released.append(module)if not released:return Nonefor module in released:@@ -3459,21 +3483,12 @@ def release_resident_groups(reason,)- def restore() -> None:- for module in released:- _keep_groups_resident(- module,- module._unsloth_resident_room,- module._unsloth_resident_device,- logger,- only = streamed,- )-return restoreexcept Exception as exc: # noqa: BLE001 - the guard still refuses what cannot fitif logger is not None:logger.warning("diffusion.memory: releasing resident groups failed (%s)", exc)- return None+ # groups released before the failure stay streamed until the caller re-pins them: hand back their restore+ return restore if released else Nonedef resident_group_mib(pipe: Any) -> int:@@ -3492,10 +3507,14 @@ def hook_resident_denoiser(pipe: Any,device: Any,logger: Any = None,+ resident: bool = True,) -> bool:"""Give a hookless resident torchao transformer block-level offload hooks, every group kept resident, sorelease_resident_groups can stream part of it. On demand, not at load: hooks at load recompile every load's second- render. True when installed."""+ render. True when installed. ``resident=False``: every group streams (the load's flat plan), for a transformer+ that cannot come back to the card whole. A failure whose own return to the card fails raises, the transformer+ split between host and card with no hooks (return_torchao_denoiser / hook_resident_denoiser(resident=False) are+ the ways out)."""transformer = getattr(pipe, "transformer", None)if (transformer is None@@ -3546,19 +3565,17 @@ def hook_resident_denoiser(_skip_top_level_copy_back(transformer, logger)if gkwargs.get("record_stream"):install_group_prefetch(transformer, onload, logger)- kept = _keep_groups_resident(transformer, dit_mib + 1, onload, logger)- # the helper reports its own failure as 0 MiB kept: anything still streamed is a failed install- if not all(getattr(g, "_unsloth_resident", False) for g in _offload_groups(transformer)):- raise RuntimeError("not every offload group stayed resident")+ kept = 0+ if resident:+ kept = _keep_groups_resident(transformer, dit_mib + 1, onload, logger)+ if not all(+ getattr(g, "_unsloth_resident", False) for g in _offload_groups(transformer)+ ):+ raise RuntimeError("not every offload group stayed resident")except Exception as exc: # noqa: BLE001 - the guard still refuses what cannot fit- # the apply already moved the weights to their host copies: back to the card, as before the call- _remove_group_offload_hooks(transformer)- _drop_module_forward_hooks_since(transformer, torch_hooks)- from .diffusion_offload_prefetch import PREFETCHER_ATTR-- for attr in (PREFETCHER_ATTR, "_unsloth_stream_state"):- transformer.__dict__.pop(attr, None)- _return_module_tensors(transformer, onload)+ # the apply already moved the weights to their host copies: back to the card, as before the call (a return+ # that itself fails raises: the caller owns the way back)+ _unhook_and_return(transformer, onload, torch_hooks)if logger is not None:logger.warning("diffusion.memory: resident transformer left without hooks (%s)", exc)return False@@ -3584,6 +3601,44 @@ def _module_forward_hook_ids(module: Any) -> dict[str, set]:return {name: set(getattr(module, name, None) or ()) for name in _FORWARD_HOOK_DICTS}+def _unhook_and_return(transformer: Any, onload: Any, torch_hooks: Optional[dict]) -> None:+ _remove_group_offload_hooks(transformer)+ if torch_hooks is not None:+ _drop_module_forward_hooks_since(transformer, torch_hooks)+ from .diffusion_offload_prefetch import PREFETCHER_ATTR++ for attr in (PREFETCHER_ATTR, "_unsloth_stream_state"):+ transformer.__dict__.pop(attr, None)+ _return_module_tensors(transformer, onload)+++def return_torchao_denoiser(+ pipe: Any,+ device: Any,+ logger: Any = None,+ torch_hooks: Optional[dict] = None,+) -> None:+ """The way back to where a hookless resident torchao transformer started, from wherever a failed+ hook_resident_denoiser left it: streamed under offload hooks (settled) -> every group resident again, the state+ a successful hook_resident_denoiser leaves; split with no hooks -> whole on the card. Raises if it is not there."""+ import torch++ transformer = getattr(pipe, "transformer", None)+ if transformer is None:+ return+ onload = torch.device(device)+ if _offload_groups(transformer):+ _keep_groups_resident(transformer, _module_host_mib(transformer) + 1, onload, logger)+ if not all(getattr(g, "_unsloth_resident", False) for g in _offload_groups(transformer)):+ raise RuntimeError("the transformer's offload groups are not all resident again")+ return+ _unhook_and_return(transformer, onload, torch_hooks)+++def module_forward_hook_ids(module: Any) -> dict[str, set]:+ return _module_forward_hook_ids(module)++def _drop_module_forward_hooks_since(module: Any, before: dict[str, set]) -> None:"""Remove the torch forward hooks registered on ``module`` after ``before`` was taken (the group prefetcher's)."""for name, ids in before.items():diff --git a/studio/backend/core/inference/diffusion_speed.py b/studio/backend/core/inference/diffusion_speed.pyindex 32ac6a7b658..a0af8b82ce2 100644--- a/studio/backend/core/inference/diffusion_speed.py+++ b/studio/backend/core/inference/diffusion_speed.py@@ -896,6 +896,7 @@ def _vae_channels_last_3d(vae: Any, logger: Any, *, fused: bool) -> bool:logger.info("diffusion.speed: channels_last skipped for %s: %s", name, reason)return Falserelaid = 0+ touched: list = []for part in (getattr(vae, "post_quant_conv", None), getattr(vae, "decoder", None)):if part is None or not callable(getattr(part, "modules", None)):continue@@ -908,7 +909,12 @@ def _vae_channels_last_3d(vae: Any, logger: Any, *, fused: bool) -> bool:fmt = torch.channels_last if weight.dim() == 4 else torch.channels_last_3dif not weight.is_contiguous(memory_format = fmt):weight.data = weight.data.contiguous(memory_format = fmt)+ touched.append(module)relaid += 1+ if touched:+ # Relaid because fusion is planned; fusion matched by structure may still be rejected at its first decode,+ # which restores these weights' layout (diffusion_vae_fused._restore_contiguous_layout).+ vae.__dict__.setdefault("_unsloth_channels_last_modules", []).extend(touched)if logger is not None:logger.info("diffusion.speed: channels_last(_3d) decode weights on %s (%d convs)", name, relaid@@ -1412,6 +1418,11 @@ def is_compile_failure(exc: BaseException) -> bool:return bool(kinds) and isinstance(exc, tuple(kinds))+def _drop_tracebacks(exc: BaseException) -> None:+ from .diffusion_vae_oom import drop_tracebacks # noqa: PLC0415+ drop_tracebacks(exc)++class _CompileGuard:"""Per-DiT record of a compile that failed at runtime. The first failure flips every guarded block of that DiT toeager (one shared decision: a half-compiled stack would retry the same broken lowering per block)."""@@ -1452,7 +1463,7 @@ class _CompileGuard:raiseguard.fail(exc, owner)# The handled exception pins inductor's frames (and the traced fake tensors) through the retry.- exc.__traceback__ = None+ _drop_tracebacks(exc)return eager(*args, **kwargs)guarded._unsloth_compile_guard = guard@@ -1707,6 +1718,9 @@ def _guard_compiled_decode(def guarded(*args: Any, **kwargs: Any) -> Any:if eager_when_tiled and getattr(vae, "use_tiling", False):return eager(*args, **kwargs)+ # the one force-eager signal (a background compile in flight, the OOM scope's eager retry)+ if _bg_eager_forced():+ return eager(*args, **kwargs)if not failed:try:return compiled(*args, **kwargs)@@ -1720,10 +1734,13 @@ def _guard_compiled_decode(try:vae._unsloth_compile_decode_error = errorvae._unsloth_compiled_decode = False- if had_own:- owner.decode = eager- else:- del owner.decode+ # Only while the slot still holds this guard: a layer installed over it since (the OOM scope)+ # stays, and this guard (``failed``) runs eager under it from now on.+ if getattr(owner, "__dict__", {}).get("decode") is guarded:+ if had_own:+ owner.decode = eager+ else:+ del owner.decodeexcept Exception: # noqa: BLE001 - `failed` still routes this wrapper to eagerpassif getattr(vae, "_unsloth_eager_contiguous", False):@@ -1738,12 +1755,30 @@ def _guard_compiled_decode(error,)# The handled exception pins inductor's frames (and the traced fake tensors) through the retry.- exc.__traceback__ = None+ _drop_tracebacks(exc)return eager(*args, **kwargs)+ # diffusion_vae_oom reads these to retry a runtime OOM of the compiled decode on the eager path+ guarded._unsloth_compiled_decode = True+ guarded._unsloth_eager_decode = eagerreturn guarded+def _decode_slot_owner(fn: Any) -> Any:+ """The mutable slot an eager outer wrapper (fp16 non-finite check, ROCm bf16 cast) calls through, found under+ any layer installed over it since (the OOM scope), or None."""+ from .decode_layers import next_layer # noqa: PLC0415++ for _ in range(32):+ if fn is None:+ return None+ slot = getattr(fn, "_unsloth_decode_slot", None)+ if slot is not None:+ return slot+ fn = next_layer(fn)+ return None++def _compile_vae_decode(pipe: Any,logger: Any,@@ -1754,7 +1789,7 @@ def _compile_vae_decode(vae = getattr(pipe, "vae", None)# An outer wrapper that must stay eager (fp16 non-finite check) exposes the slot it calls through.outer = getattr(vae, "__dict__", {}).get("decode") if vae is not None else None- owner = getattr(outer, "_unsloth_decode_slot", None) or vae+ owner = _decode_slot_owner(outer) or vaedecode = getattr(owner, "decode", None) if vae is not None else Noneif not callable(decode):return Falsediff --git a/studio/backend/core/inference/diffusion_vae_fused.py b/studio/backend/core/inference/diffusion_vae_fused.pyindex a55f47150a9..25aa99df815 100644--- a/studio/backend/core/inference/diffusion_vae_fused.py+++ b/studio/backend/core/inference/diffusion_vae_fused.py@@ -12,6 +12,7 @@ from __future__ import annotationsimport osimport sys+import threadingimport typesfrom functools import lru_cachefrom typing import Any, Optional@@ -504,17 +505,29 @@ def _torch():return torch+def _stock_in(t: Any) -> Any:+ """A kernel input as the kernel may read it."""+ cls = globals().get("_FusedActCls")+ if cls is None or type(t) is not cls:+ return t+ if t.__dict__.get("_unsloth_is_stock", False):+ return _plain(t)+ t._unsloth_abort()++def group_norm_act(x: Any,norm: Any,act: bool = True,in_bias: Any = None,+ on_kernel: Any = None,) -> Any:- """``silu?(group_norm(x + in_bias))`` for 3/4/5-D ``x``; channels-last output; 5-D stats span T like nn.GroupNorm."""+ """``silu?(group_norm(x + in_bias))`` for 3/4/5-D ``x``; channels-last output; 5-D stats span T like nn.GroupNorm.+ ``on_kernel``: called once the Triton kernels completed for this call (not on a stock fallback)."""torch = _torch()+ x, in_bias = _stock_in(x), _stock_in(in_bias)if not _gn_fusable(x, norm):return group_norm_reference(x, norm, act, in_bias)- k = _kernels()ndim = x.dim()x5, _ = _as5d(x)b, c, t, h, w = x5.shape@@ -527,6 +540,7 @@ def group_norm_act(b, c, t, h, w = x5.shapeelse:return group_norm_reference(x, norm, act, in_bias)+ k = _kernels()g = int(norm.num_groups)cpg = c // ghw = h * w@@ -556,6 +570,8 @@ def group_norm_act(CPG = cpg, ACT = bool(act), HAS_IN_BIAS = in_bias is not None, BLOCK_P = block_p2, BLOCK_C = block_c,num_warps = 4,) # fmt: skip+ if on_kernel is not None and not torch.compiler.is_compiling():+ on_kernel() # read by the structural calibration: only a completed Triton launch countsout = out.permute(0, 4, 1, 2, 3) # B, C, T', H', W'shape = x.shapeif ndim == 5:@@ -637,6 +653,7 @@ def rms_norm_act(``replicate_pad=(ph, pw)``: HunyuanVideo-1.5 replicate padding instead (``cache`` must be None); ``back`` repeats thelast frame (LTX-2)."""+ x, in_bias = _stock_in(x), _stock_in(in_bias)torch = _torch()if replicate_pad is not None and cache is not None:raise ValueError("replicate padding takes no cache")@@ -988,6 +1005,8 @@ def _guard(try:return fast(*args, **kwargs)except Exception as exc: # noqa: BLE001+ if isinstance(exc, FusedActivationRead):+ raisefor a, saved in zip(lists, snap):a[:] = savedif not _is_oom(exc):@@ -1105,7 +1124,21 @@ def install_group_norm_vae(vae: Any, logger: Any = None) -> int:return n+_STRUCTURAL_LAYER = "vae_fused_structural"+++def _restore_structural_decode(vae: Any) -> None:+ """Remove the structural calibrator's ``_decode`` layer: exactly what it replaced comes back when it is outermost+ (an instance-bound override included); under a later layer (the OOM retry) it becomes a pass-through."""+ from . import decode_layers # noqa: PLC0415+ decode_layers.unwrap(decode_layers.find(vae, "_decode", _STRUCTURAL_LAYER))++def uninstall(vae: Any) -> None:+ _restore_structural_decode(vae)+ _restore_contiguous_layout(vae)+ vae.__dict__.pop("_unsloth_vae_structural", None)+ vae.__dict__.pop("_unsloth_vae_fused_structural", None)vae.__dict__.pop("_unsloth_vae_fused_installed", None)vae.__dict__.pop("_unsloth_vae_fused_cl_weights", None)for m in vae.modules():@@ -2062,11 +2095,17 @@ SUPPORTED_VAES = frozenset(def will_install(vae: Any) -> bool:- """Whether :func:`install` would engage on ``vae`` here (class covered, NVIDIA CUDA, usable Triton, not disabled)."""+ """Whether :func:`install` would engage on ``vae`` here (class covered or matched by structure, NVIDIA CUDA,+ usable Triton, not disabled)."""return (vae is not None- and type(vae).__name__ in SUPPORTED_VAES- and (type(vae).__name__ != "AutoencoderKLLTX2Video" or _ltx2_convs_fusable(vae))+ and (+ (+ type(vae).__name__ in SUPPORTED_VAES+ and (type(vae).__name__ != "AutoencoderKLLTX2Video" or _ltx2_convs_fusable(vae))+ )+ or structural_candidate(vae)+ )and runtime_ok())@@ -2083,6 +2122,10 @@ def install(if done:return done # idempotent: a dual-DiT family runs the speed layer once per expert over the same VAEn = _install(vae, logger)+ if not n and type(vae).__name__ not in SUPPORTED_VAES:+ n = install_structural(vae, logger)+ if n:+ vae._unsloth_vae_fused_structural = True# Only where measured faster, and only when a fused pass feeds a conv (relaid weights alone slow LTX-2's decode).if (n@@ -2116,3 +2159,781 @@ def _install(vae: Any, logger: Any = None) -> int:n = install_ltx2_vae(vae, logger)return n + (install_vectorised_blend(vae) if n else 0)return 0+++# Structural path for unmeasured VAEs: a GroupNorm pairs with SiLU only if every output fed an nn.SiLU (hook trace of+# one tiny decode); the fused decode must match stock within tolerance or the patch is removed.+# Kill switch ``UNSLOTH_VAE_FUSED_STRUCTURAL=0``.+STRUCTURAL_ENV = "UNSLOTH_VAE_FUSED_STRUCTURAL"+# The calibration crop. The trace marks only the norm outputs (cheap at any size), so the crop is sized for the+# kernels instead: 16 latents keeps every spatial extent a multiple of 16, the same Triton specialisations a full+# decode uses, so calibrating compiles nothing the real decode would not (an 8-latent crop measured +1.5 to 2 s+# of extra compiles on a cold Triton cache).+_PROBE_LATENTS = 16+_PROBE_FRAMES = 2+# Calibration verdicts are kept on disk per (class, config, decoder layout, torch, triton, device capability), so a+# later process skips the trace and the checks. Off: UNSLOTH_VAE_FUSED_STRUCTURAL_CACHE=0.+STRUCTURAL_CACHE_ENV = "UNSLOTH_VAE_FUSED_STRUCTURAL_CACHE"+_STRUCTURAL_CACHE_VERSION = 3+_PROBE_MAX_ABS = 0.125+_PROBE_MEAN_ABS = 0.01+++def structural_disabled() -> bool:+ return os.environ.get(STRUCTURAL_ENV, "").strip().lower() in _FALSE+++def _plain_group_norms(part: Any) -> list:+ torch = _torch()+ return [+ m+ for m in part.modules()+ if isinstance(m, torch.nn.GroupNorm)+ and type(m).forward is torch.nn.GroupNorm.forward+ and "forward" not in m.__dict__+ ]+++def _plain_silus(part: Any) -> list:+ torch = _torch()+ return [+ m+ for m in part.modules()+ if isinstance(m, torch.nn.SiLU)+ and type(m).forward is torch.nn.SiLU.forward+ and "forward" not in m.__dict__+ ]+++def structural_candidate(vae: Any) -> bool:+ """A VAE outside the measured set whose decoder holds plain nn.GroupNorm modules, reached through ``_decode``."""+ if vae is None or structural_disabled() or type(vae).__name__ in SUPPORTED_VAES:+ return False+ if getattr(vae, "_unsloth_decode_blocks_bypassed", False):+ return False+ decoder = getattr(vae, "decoder", None)+ if decoder is None or not callable(getattr(vae, "_decode", None)):+ return False+ try:+ return bool(_plain_group_norms(decoder))+ except Exception: # noqa: BLE001+ return False+++def _decoded_tensor(out: Any) -> Any:+ if isinstance(out, (tuple, list)):+ return out[0]+ return getattr(out, "sample", out)+++def _probe_latent(z: Any) -> Any:+ sl = [slice(None)] * z.dim()+ sl[-1] = slice(0, min(_PROBE_LATENTS, z.shape[-1]))+ sl[-2] = slice(0, min(_PROBE_LATENTS, z.shape[-2]))+ if z.dim() == 5:+ sl[2] = slice(0, min(_PROBE_FRAMES, z.shape[2]))+ return z[tuple(sl)].contiguous()+++def structural_cache_disabled() -> bool:+ return (os.environ.get(STRUCTURAL_CACHE_ENV) or "").strip().lower() in (+ "0",+ "off",+ "false",+ "no",+ )+++def _structural_cache_dir() -> Any:+ from pathlib import Path+ try:+ from .diffusion_compile_cache import cache_root+ return Path(cache_root()) / "vae_structural"+ except Exception: # noqa: BLE001+ return None+++def _vae_config_json(vae: Any) -> str:+ import json++ cfg = getattr(vae, "config", None)+ try:+ data = dict(cfg) if cfg is not None else {}+ except Exception: # noqa: BLE001+ data = dict(getattr(cfg, "__dict__", {}) or {})+ return json.dumps(+ {k: data[k] for k in sorted(data) if not str(k).startswith("_")},+ sort_keys = True,+ default = str,+ )+++_FINGERPRINT_ELEMS = 64+++def _decoder_fingerprint(decoder: Any) -> Optional[str]:+ """The checkpoint's identity, cheaply: every decoder tensor's name, shape and dtype, plus its first+ ``_FINGERPRINT_ELEMS`` values (one device-to-host copy). Another checkpoint of the same config (a fine-tune, a+ different dtype) is another fingerprint. None when the values cannot be read (meta / offloaded weights)."""+ import hashlib++ torch = _torch()+ try:+ named = [*decoder.named_parameters(), *decoder.named_buffers()]+ if any(getattr(t, "is_meta", False) for _, t in named):+ return None+ h = hashlib.sha256(+ repr([(n, tuple(t.shape), str(t.dtype), str(t.device.type)) for n, t in named]).encode()+ )+ with torch.no_grad():+ heads = [+ t.detach().reshape(-1)[:_FINGERPRINT_ELEMS].float() for _, t in named if t.numel()+ ]+ if heads:+ by_device: dict = {}+ for t in heads:+ by_device.setdefault(t.device, []).append(t)+ for dev in sorted(by_device, key = str):+ h.update(torch.cat(by_device[dev]).cpu().numpy().tobytes())+ return h.hexdigest()[:32]+ except Exception: # noqa: BLE001 - no identity: the caller does not use the cache+ return None+++def structural_cache_key(vae: Any, norms: list, silus: list, device: Any) -> Optional[str]:+ """Key of a calibration verdict: everything the pairing and the kernels' numerics depend on. Any change (another+ class, config, decoder layout, checkpoint weights or dtype, hooks, torch, triton or device capability) is a new+ key, so it is traced and checked again. None (no cache) when the weights' identity cannot be read."""+ import hashlib++ torch = _torch()+ try:+ import triton+ triton_version = getattr(triton, "__version__", "?")+ except Exception: # noqa: BLE001+ triton_version = None+ try:+ cap = (+ tuple(torch.cuda.get_device_capability(device))+ if getattr(device, "type", None) == "cuda"+ else None+ )+ except Exception: # noqa: BLE001+ cap = None+ fingerprint = _decoder_fingerprint(vae.decoder)+ if fingerprint is None:+ return None # no cheap identity for these weights: no cached verdict, trace and check every process+ names = {id(m): n for n, m in vae.decoder.named_modules()}+ layout = [+ (+ names.get(id(m), "?"),+ type(m).__name__,+ int(getattr(m, "num_groups", 0)),+ int(getattr(m, "num_channels", 0)),+ )+ for m in norms+ ] + [(names.get(id(m), "?"), type(m).__name__) for m in silus]+ blob = repr((+ _STRUCTURAL_CACHE_VERSION, f"{type(vae).__module__}.{type(vae).__qualname__}", _vae_config_json(vae), layout,+ str(torch.__version__), triton_version, cap, _PROBE_LATENTS, _PROBE_MAX_ABS, _PROBE_MEAN_ABS, fingerprint,+ _global_module_hooks(), [_has_user_hooks(m) for m in (*norms, *silus)],+ )) # fmt: skip+ return hashlib.sha256(blob.encode()).hexdigest()[:32]+++def _load_pairing(key: Optional[str], n_norms: int) -> Optional[dict]:+ import json++ root = _structural_cache_dir()+ if key is None or root is None or structural_cache_disabled():+ return None+ try:+ data = json.loads((root / f"{key}.json").read_text(encoding = "utf-8"))+ except Exception: # noqa: BLE001 - absent or unreadable: calibrate+ return None+ paired = data.get("paired")+ if data.get("key") != key or data.get("status") not in ("ok", "rejected", "no_match"):+ return None+ if (+ not isinstance(paired, list)+ or len(paired) != n_norms+ or any(p not in (None, True, False) for p in paired)+ ):+ return None+ consumers = data.get("consumers")+ if data["status"] == "ok" and (+ not isinstance(consumers, list) or len(consumers) != n_norms+ or any(not isinstance(c, list) or any(not isinstance(j, int) for j in c) for c in consumers)+ ): # fmt: skip+ return None+ return data+++def _save_pairing(key: Optional[str], record: dict) -> None:+ import json++ root = _structural_cache_dir()+ if key is None or root is None or structural_cache_disabled():+ return+ try:+ root.mkdir(parents = True, exist_ok = True)+ tmp = root / f".{key}.{os.getpid()}.tmp"+ tmp.write_text(json.dumps({**record, "key": key}, default = str), encoding = "utf-8")+ os.replace(tmp, root / f"{key}.json")+ except Exception: # noqa: BLE001 - a cache that cannot be written only costs the next process a calibration+ pass+++def _has_user_hooks(module: Any) -> bool:+ for name in ("_forward_hooks", "_forward_pre_hooks", "_backward_hooks", "_backward_pre_hooks"):+ if getattr(module, name, None):+ return True+ return False+++def _global_module_hooks() -> bool:+ try:+ from torch.nn.modules import module as mm+ except Exception: # noqa: BLE001+ return True+ return any(+ getattr(mm, name, None)+ for name in ("_global_forward_hooks", "_global_forward_pre_hooks", "_global_backward_hooks")+ )+++# Tensor methods that only read metadata: not a reader of the values.+_META_FUNCS = frozenset((+ "__get__", "dim", "size", "stride", "numel", "nelement", "ndimension", "element_size", "is_contiguous",+ "storage_offset", "is_floating_point", "is_complex", "get_device", "__len__", "data_ptr",+)) # fmt: skip+++# The reader callback of the structural trace is per thread, and traces run one at a time (once per VAE per process):+# two first decodes on two threads can neither clear nor overwrite each other's callback.+_TRACE_LOCAL = threading.local()+_TRACE_LOCK = threading.RLock()+++def _tracked_tensor_class() -> Any:+ """A tensor subclass marking a norm's output: every torch function that touches it reports in, and its results+ come back as plain tensors, so only the direct readers of a norm output are seen (cheap: no per-op hook on the+ rest of the decoder)."""+ torch = _torch()+ cls = globals().get("_TrackedCls")+ if cls is not None:+ return cls+ from torch.utils._pytree import tree_leaves++ class _Tracked(torch.Tensor):+ @classmethod+ def __torch_function__(+ cls,+ func,+ types_,+ args = (),+ kwargs = None,+ ):+ kwargs = kwargs or {}+ # this thread's trace only: a concurrent trace (another VAE, another thread) has its own callback+ trace = getattr(_TRACE_LOCAL, "read", None)+ if trace is not None and getattr(func, "__name__", "") not in _META_FUNCS:+ for leaf in tree_leaves((args, kwargs)):+ if isinstance(leaf, _Tracked):+ trace(leaf)+ with torch._C.DisableTorchFunctionSubclass():+ return func(*args, **kwargs)++ globals()["_TrackedCls"] = _Tracked+ return _Tracked+++def _trace_group_norm_silu(norms: list, silus: list, run: Any) -> tuple:+ """(calls per norm, outputs of each norm consumed ONLY by an nn.SiLU, run output, info) over one ``run()``.++ A norm's output counts as consumed by the SiLU only when the trace proves the SiLU is its sole reader: the output+ is marked (a tensor subclass), every torch function reading it (a view, an in-place update, a residual add) is+ recorded, and any such read outside the SiLU call marks it shared. ``info["shared"]``: per norm, reads elsewhere.+ Norms or SiLUs carrying hooks of their own, global module hooks, or no subclass support: nothing counts as+ exclusively consumed."""+ n = len(norms)+ calls, consumed, shared = [0] * n, [0] * n, [0] * n+ produced: dict = {}+ keep: list = []+ handles = []+ in_silu: list = []+ hooked = _global_module_hooks() or any(_has_user_hooks(m) for m in (*norms, *silus))+ try:+ tracked = _tracked_tensor_class()+ except Exception: # noqa: BLE001 - no way to see every reader: fuse GroupNorm alone+ tracked = None++ def norm_hook(idx: int):+ def hook(_m, _inp, out):+ calls[idx] += 1+ if tracked is None or type(out) is not _torch().Tensor:+ return None+ marked = out.as_subclass(tracked)+ produced[id(marked)] = idx+ keep.append(marked)+ return marked++ return hook++ silu_index = {id(m): j for j, m in enumerate(silus)}+ readers: list = [set() for _ in range(n)]++ def silu_pre(m, inp):+ idx = produced.get(id(inp[0])) if inp else None+ if idx is not None:+ consumed[idx] += 1+ readers[idx].add(silu_index.get(id(m), -1))+ in_silu.append(idx)++ def silu_post(_m, _inp, _out):+ if in_silu:+ in_silu.pop()++ def read(t: Any) -> None:+ idx = produced.get(id(t))+ if idx is not None and not (in_silu and in_silu[-1] == idx):+ shared[idx] += 1++ with _TRACE_LOCK:+ previous = getattr(_TRACE_LOCAL, "read", None)+ try:+ for i, m in enumerate(norms):+ handles.append(m.register_forward_hook(norm_hook(i)))+ for m in silus:+ handles.append(m.register_forward_pre_hook(silu_pre))+ handles.append(m.register_forward_hook(silu_post))+ if tracked is not None:+ _TRACE_LOCAL.read = read+ out = run()+ finally:+ _TRACE_LOCAL.read = previous+ for h in handles:+ h.remove()+ keep.clear()+ if hooked or tracked is None:+ consumed = [0] * n+ else:+ consumed = [k if sh == 0 else 0 for k, sh in zip(consumed, shared)]+ if out is not None and tracked is not None and isinstance(out, tracked):+ out = out.as_subclass(_torch().Tensor)+ return calls, consumed, out, {"shared": shared, "consumers": [sorted(r) for r in readers]}+++def _site_hooks(norms: list, paired: list) -> tuple:+ """Forward hooks comparing each fused site with stock on the input it is given, during the crop decode itself+ (one pass: no extra kernel specialisations). Returns (handles, worst) where worst is [max abs, mean abs, sites]."""+ torch = _torch()+ worst = [0.0, 0.0, 0]+ handles = []+ for norm, act in zip(norms, paired):+ if act is None:+ continue++ def hook(+ m,+ inp,+ out,+ act = act,+ ):+ out = _plain(out)+ ref = _stock_forward(m)(inp[0])+ if act:+ ref = torch.nn.functional.silu(ref)+ err = (out.float() - ref.float()).abs()+ worst[0] = max(worst[0], float(err.max()))+ worst[1] = max(worst[1], float(err.mean()))+ worst[2] += 1++ handles.append(norm.register_forward_hook(hook))+ return handles, worst+++# Tensor properties that are metadata only (a read of these is not a read of the values).+_META_PROPS = frozenset((+ "shape", "dtype", "device", "is_cuda", "layout", "ndim", "requires_grad", "is_leaf", "grad_fn", "names",+ "is_sparse", "is_quantized", "is_meta", "is_mkldnn", "is_nested", "itemsize", "nbytes",+)) # fmt: skip+++_RAW_READS = frozenset(("data_ptr", "untyped_storage", "storage", "_typed_storage"))+++def _is_meta_read(func: Any) -> bool:+ name = getattr(func, "__name__", "")+ if name == "__get__":+ return getattr(getattr(func, "__self__", None), "__name__", "") in _META_PROPS+ return name in _META_FUNCS and name not in _RAW_READS+++class FusedActivationRead(RuntimeError):+ """A fused GroupNorm+SiLU output was read by something other than its paired SiLU. It holds only the activated+ values: the stock GroupNorm value such a reader needs was never materialised, and recomputing it later from the+ norm's input is not exact (the decoder may have written to that input in place since, and inference tensors,+ which Studio renders with, keep no version counter to tell). So that decode stops here; the structural decode+ layer turns activation fusion off for this VAE (GroupNorm stays fused), records that, and decodes again."""+++def _fused_act_class() -> Any:+ """The output of a fused GroupNorm+SiLU: it holds the ACTIVATED values, which only the SiLU module(s) the trace+ paired with it may read (once: the first consumes the mark). Any other read of the values (a residual, another+ norm, a view, an in-place write, a raw pointer, a SiLU after the mark was consumed), whatever the shape or+ dataflow of the decode that does it, raises FusedActivationRead: the decode is redone without activation fusion,+ so a wrong value is never returned and nothing aliases the norm's (mutable) input. After an in-place paired SiLU+ the storage IS the stock tensor's (silu_ overwrote it), so reads then see it as a plain tensor."""+ cls = globals().get("_FusedActCls")+ if cls is not None:+ return cls+ torch = _torch()+ from torch.utils._pytree import tree_leaves++ class _FusedAct(torch.Tensor):+ @classmethod+ def __torch_function__(+ cls,+ func,+ types_,+ args = (),+ kwargs = None,+ ):+ kwargs = kwargs or {}+ if not _is_meta_read(func):+ for leaf in tree_leaves((args, kwargs)):+ if isinstance(leaf, _FusedAct) and not leaf.__dict__.get(+ "_unsloth_is_stock", False+ ):+ leaf._unsloth_abort(func)+ with torch._C.DisableTorchFunctionSubclass():+ return func(*args, **kwargs)++ def _unsloth_abort(self, func: Any = None) -> None:+ state = self.__dict__.get("_unsloth_state")+ if state is not None:+ state.foreign_reads = getattr(state, "foreign_reads", 0) + 1+ raise FusedActivationRead(+ f"fused GroupNorm+SiLU output read by {getattr(func, '__name__', 'a kernel')} before / instead of its "+ "paired SiLU"+ )++ globals()["_FusedActCls"] = _FusedAct+ return _FusedAct+++def _plain(t: Any) -> Any:+ """``t`` as a plain tensor: a fused GroupNorm+SiLU output's activated values (no recompute)."""+ cls = globals().get("_FusedActCls")+ if cls is None or type(t) is not cls:+ return t+ torch = _torch()+ with torch._C.DisableTorchFunctionSubclass():+ return t.as_subclass(torch.Tensor)+++def _compiling() -> bool:+ try:+ return bool(_torch().compiler.is_compiling())+ except Exception: # noqa: BLE001+ return False+++def _fast_gn_maybe_act(+ norm: Any,+ act: bool,+ state: Any,+ silus: Any = None,+) -> Any:+ """The fused GroupNorm(+SiLU) forward. A fused activation's output is a _FusedAct bound to exactly the SiLU+ module(s) the trace saw consume it (``silus``; None: any one SiLU): the first of those to read it gets the+ activated values and consumes the mark; every other reader gets the stock GroupNorm output. Under torch.compile+ (no tensor subclass in the graph) the activation is left to the SiLU, which inductor fuses itself."""+ token = "any" if silus is None else frozenset(id(m) for m in silus)+ fused_cls = _fused_act_class() if act else None+ torch = _torch()++ def fast(x: Any) -> Any:+ # any patched norm (fused activation or GroupNorm only) reading a fused output reads its stock value: the+ # kernel reads raw memory+ x = _stock_in(x)+ fuse = act and not _compiling()+ ran: list = []+ y = group_norm_act(x, norm, act = fuse, on_kernel = lambda: ran.append(1))+ if ran:+ state.kernel_calls += 1+ probe = getattr(getattr(state, "local", None), "kernel_runs", None)+ if probe is not None:+ probe.append(1)+ state.calls += 1+ if fuse:+ with torch._C.DisableTorchFunctionSubclass():+ y = y.as_subclass(fused_cls)+ y._unsloth_state, y._unsloth_silu_for = state, token+ return y++ return fast+++def _silu_passthrough(silu: Any) -> Any:+ torch = _torch()+ fused_cls = _fused_act_class()+ inplace = bool(getattr(silu, "inplace", False))++ def forward(x: Any) -> Any:+ token = getattr(x, "_unsloth_silu_for", None) if type(x) is fused_cls else None+ if token is not None and (token == "any" or id(silu) in token):+ x._unsloth_silu_for = None+ if inplace:+ x._unsloth_is_stock = (+ True # stock silu_ overwrote the norm output: its storage is now exactly that+ )+ return _plain(x)+ return torch.nn.functional.silu(x, inplace = inplace)++ forward._unsloth_vae_fused = True+ return forward+++def _structural_patch(+ norms: list,+ paired: list,+ silus: list,+ state: Any,+ logger: Any,+ consumers: Optional[list] = None,+) -> int:+ """``consumers``: per norm, the indices into ``silus`` of the SiLU module(s) the trace saw read its output; only+ those SiLUs consume its fused-activation mark (None: any SiLU, once)."""+ n = 0+ patched_silus: list = []+ for i, (norm, act) in enumerate(zip(norms, paired)):+ if act is None:+ continue+ mine = None+ if act and consumers is not None:+ mine = [silus[j] for j in consumers[i] if 0 <= j < len(silus)]+ patched_silus += mine+ _guard(+ norm,+ _fast_gn_maybe_act(norm, act, state, mine),+ _stock_forward(norm),+ "GroupNorm+SiLU" if act else "GroupNorm",+ logger,+ )+ n += 1+ if any(paired):+ targets = silus if consumers is None else patched_silus+ for silu in {id(m): m for m in targets}.values():+ silu.forward = _silu_passthrough(silu)+ return n+++def _structural_unpatch(+ norms: list,+ silus: list,+ vae: Any = None,+) -> None:+ for m in (*norms, *silus):+ m.__dict__.pop("forward", None)+ m.__dict__.pop("_unsloth_vae_fused_failed", None)+ if vae is not None:+ _restore_contiguous_layout(vae)+++def _restore_contiguous_layout(vae: Any) -> int:+ """Undo the channels_last(_3d) relayout the speed layer applied because fusion was planned: without the fused+ passes that layout decodes slower. Only the conv weights it recorded are touched."""+ modules = vae.__dict__.pop("_unsloth_channels_last_modules", None) or ()+ n = 0+ for module in modules:+ weight = getattr(module, "weight", None)+ if weight is not None and not weight.is_contiguous():+ weight.data = weight.data.contiguous()+ n += 1+ return n+++def _clear_decoder_cache(vae: Any) -> None:+ """Reset a causal decoder's frame / feature cache (Wan, Qwen-Image, LTX-2, HunyuanVideo expose clear_cache)."""+ clear = getattr(vae, "clear_cache", None)+ if callable(clear):+ try:+ clear()+ except Exception: # noqa: BLE001 - a VAE without a usable cache has nothing to reset+ pass+++def install_structural(vae: Any, logger: Any = None) -> int:+ """Arm the dataflow-matched fused GroupNorm(+SiLU) path on ``vae``'s decoder; it is calibrated (paired, checked+ against stock) on the first CUDA decode and removed if the check fails. Returns the plain GroupNorm count."""+ if not structural_candidate(vae) or not runtime_ok():+ return 0+ if getattr(vae, "_unsloth_vae_structural", None) is not None:+ return vae._unsloth_vae_structural.norms+ decoder = vae.decoder+ norms = _plain_group_norms(decoder)+ silus = _plain_silus(decoder)+ state = types.SimpleNamespace(+ status = "pending", norms = len(norms), gn_silu = 0, gn = 0, calls = 0, kernel_calls = 0,+ max_abs = None, mean_abs = None, site_max_abs = None, site_mean_abs = None, local = threading.local(),+ paired = None, consumers = None, cached = False, key = None, act_off = False, foreign_reads = 0,+ ) # fmt: skip+ inner = vae._decode+ torch = _torch()++ def _fresh(fn: Any) -> Any:+ def call(*a: Any, **k: Any) -> Any:+ _clear_decoder_cache(vae)+ return fn(*a, **k)++ return call++ def calibrate(z: Any, args: tuple, kwargs: dict) -> None:+ key = state.key = structural_cache_key(vae, norms, silus, getattr(z, "device", None))+ cached = _load_pairing(key, len(norms))+ if cached is not None:+ # a verdict for this exact class / config / layout / torch / triton / device: no trace, no checks+ state.cached = True+ if cached["status"] != "ok":+ state.status = cached["status"]+ _structural_unpatch(norms, silus, vae)+ return+ paired = state.paired = cached["paired"]+ state.consumers = cached["consumers"]+ _structural_patch(norms, paired, silus, state, logger, cached["consumers"])+ state.status = "ok"+ state.gn_silu = sum(1 for p in paired if p)+ state.gn = sum(1 for p in paired if p is False)+ state.max_abs, state.mean_abs = cached.get("max_abs"), cached.get("mean_abs")+ state.site_max_abs, state.site_mean_abs = (+ cached.get("site_max_abs"),+ cached.get("site_mean_abs"),+ )+ state.calls = 0+ return+ _calibrate(z, args, kwargs)+ if state.status in ("ok", "rejected", "no_match"):+ _save_pairing(key, {+ "status": state.status, "paired": state.paired, "consumers": state.consumers,+ "max_abs": state.max_abs, "mean_abs": state.mean_abs,+ "site_max_abs": state.site_max_abs, "site_mean_abs": state.site_mean_abs,+ }) # fmt: skip++ def _calibrate(z: Any, args: tuple, kwargs: dict) -> None:+ tiny = _probe_latent(z)+ with torch.no_grad():+ calls, consumed, ref, info = _trace_group_norm_silu(+ norms, silus, lambda: _decoded_tensor(_fresh(inner)(tiny, *args, **kwargs))+ )+ # fused with its SiLU only when that SiLU is the output's sole reader; otherwise GroupNorm alone+ paired = [None if c == 0 else (c == k) for c, k in zip(calls, consumed)]+ state.paired, state.consumers = paired, info["consumers"]+ n = _structural_patch(norms, paired, silus, state, logger, info["consumers"])+ if n == 0:+ state.status = "no_match"+ _structural_unpatch(norms, silus, vae)+ return+ state.local.kernel_runs = []+ handles, worst = _site_hooks(norms, paired)+ try:+ out = _decoded_tensor(_fresh(inner)(tiny, *args, **kwargs))+ fused_ran = bool(state.local.kernel_runs)+ finally:+ state.local.kernel_runs = None+ for h in handles:+ h.remove()+ site = (worst[0], worst[1]) if worst[2] else None+ err = (out.float() - ref.float()).abs()+ state.max_abs, state.mean_abs = float(err.max()), float(err.mean())+ state.site_max_abs, state.site_mean_abs = site if site is not None else (None, None)+ sites_ok = site is not None and site[0] <= _PROBE_MAX_ABS and site[1] <= _PROBE_MEAN_ABS+ if (+ fused_ran+ and sites_ok+ and state.max_abs <= _PROBE_MAX_ABS+ and state.mean_abs <= _PROBE_MEAN_ABS+ ):+ state.status = "ok"+ state.gn_silu = sum(1 for p in paired if p)+ state.gn = sum(1 for p in paired if p is False)+ state.calls = 0+ if logger is not None:+ logger.info(+ "diffusion.vae_fused: %s matched by structure: %d GroupNorm+SiLU, %d GroupNorm fused "+ "(probe max |d| %.4f, mean |d| %.5f)",+ type(vae).__name__, state.gn_silu, state.gn, state.max_abs, state.mean_abs,+ ) # fmt: skip+ else:+ state.status = "rejected"+ _structural_unpatch(norms, silus, vae)+ if logger is not None:+ logger.warning(+ "diffusion.vae_fused: %s structural fusion rejected (probe max |d| %s, mean |d| %s)",+ type(vae).__name__, state.max_abs, state.mean_abs,+ ) # fmt: skip++ def _decode(z: Any, *args: Any, **kwargs: Any) -> Any:+ if state.status == "pending" and getattr(z, "is_cuda", False):+ with _TRACE_LOCK:+ if state.status == "pending":+ try:+ calibrate(z, args, kwargs)+ except Exception as exc: # noqa: BLE001 - never fail a decode over the probe+ state.status = "rejected"+ _structural_unpatch(norms, silus, vae)+ if logger is not None:+ logger.warning("diffusion.vae_fused: structural probe failed: %s", exc)+ _clear_decoder_cache(+ vae+ ) # the real decode must not see the probe's causal cache+ aborted = False+ try:+ return inner(z, *args, **kwargs)+ except FusedActivationRead:+ if state.status != "ok":+ raise+ aborted = True+ if aborted:+ _activation_fusion_off()+ _clear_decoder_cache(vae)+ return inner(z, *args, **kwargs)++ def _activation_fusion_off() -> None:+ """A fused activation was read out of turn at this decode's shape / dataflow: from now on (and in later+ processes: the cached verdict is downgraded) this VAE fuses GroupNorm only and every SiLU runs stock."""+ with _TRACE_LOCK:+ if not any(state.paired or ()):+ return+ paired = [False if p else p for p in state.paired]+ _structural_unpatch(norms, silus)+ _structural_patch(norms, paired, silus, state, logger)+ state.paired, state.consumers = paired, [[] for _ in norms]+ state.gn += state.gn_silu+ state.gn_silu = 0+ state.act_off = True+ if logger is not None:+ logger.warning(+ "diffusion.vae_fused: %s read a fused GroupNorm+SiLU output out of turn; decoding again with "+ "GroupNorm fused alone", type(vae).__name__,+ ) # fmt: skip+ _save_pairing(getattr(state, "key", None), {+ "status": "ok", "paired": paired, "consumers": state.consumers,+ "max_abs": state.max_abs, "mean_abs": state.mean_abs,+ "site_max_abs": state.site_max_abs, "site_mean_abs": state.site_mean_abs,+ }) # fmt: skip++ _decode._unsloth_vae_fused = True+ vae._unsloth_vae_structural = state+ from . import decode_layers # noqa: PLC0415++ decode_layers.wrap(vae, "_decode", _STRUCTURAL_LAYER, lambda _inner, _layer: _decode)+ if logger is not None:+ logger.info(+ "diffusion.vae_fused: %s is outside the measured set; %d GroupNorms will be matched by structure "+ "on the first decode",+ type(vae).__name__,+ len(norms),+ )+ return len(norms)diff --git a/studio/backend/core/inference/diffusion_vae_oom.py b/studio/backend/core/inference/diffusion_vae_oom.pynew file mode 100644index 00000000000..3326b539e29--- /dev/null+++ b/studio/backend/core/inference/diffusion_vae_oom.py@@ -0,0 +1,1183 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""Recover a VAE decode that runs out of memory by offloading, never by shrinking tiles.++Wraps the instance ``_decode`` (under the decode-level wrappers, over the tiled decode). On an OOM, each step runs only+if the previous one still runs out of memory:++a. Free the decode's device without changing the math: Studio's resident offload groups (text encoders, denoiser+ blocks) stream again, captured CUDA graphs are dropped, and every other pipeline component still whole on the card+ moves to the host. The SAME decode is retried, so an untiled decode stays untiled and the output is unchanged.+b. An untiled decode retries tiled at the VAE's stock tile sizes: ComfyUI's single tiled retry.+c. The VAE's own weights stream from the host per layer (diffusers leaf-level group offload), still at stock tiles.++Tile sizes are never touched: tiles under the stock size bring back visible seams. Everything moved goes back where+it was once the decode returns or fails. Kill switch ``UNSLOTH_VAE_OOM_RETRY=0``.+"""++from __future__ import annotations++import functools+import gc+import os+import time+import weakref+from typing import Any, Callable, Optional++from . import decode_layers++OOM_RETRY_ENV = "UNSLOTH_VAE_OOM_RETRY"+HOST_MARGIN_MIB = 2048+_SESSION_ATTR = "_unsloth_vae_oom_session"+_PIPE_ATTR = "_unsloth_vae_oom_pipe"+++def oom_retry_disabled() -> bool:+ return os.environ.get(OOM_RETRY_ENV, "").strip().lower() in ("0", "false", "no", "off")+++def _is_oom_one(exc: BaseException) -> bool:+ if any(cls.__name__ in ("OutOfMemoryError", "OutOfMemoryError_") for cls in type(exc).__mro__):+ return True+ text = str(exc).lower()+ return isinstance(exc, RuntimeError) and (+ "out of memory" in text or "tried to allocate" in text+ )+++def is_oom(exc: BaseException) -> bool:+ """An allocator failure, also when wrapped: torch.compile re-raises an OOM inside a compiler error whose own+ message is generic, and a compiled or parallel decode can report it as an ``ExceptionGroup`` member, so+ ``__cause__``, ``__context__`` and group members are walked (bounded)."""+ seen: set = set()+ pending: list = [exc]+ while pending and len(seen) < 32:+ cur = pending.pop(0)+ if cur is None or id(cur) in seen:+ continue+ seen.add(id(cur))+ if _is_oom_one(cur):+ return True+ members = getattr(cur, "exceptions", None)+ if isinstance(members, (tuple, list)):+ pending.extend(m for m in members if isinstance(m, BaseException))+ pending.extend((cur.__cause__, cur.__context__))+ return False+++def tile_attrs(vae: Any) -> dict:+ """The tile geometry a VAE exposes; the retry never changes it."""+ out = {}+ for axis in ("size", "height", "width"):+ for kind in (+ "tile_sample_min",+ "tile_latent_min",+ "tile_sample_stride",+ "tile_sample_min_overlap",+ ):+ name = f"{kind}_{axis}"+ value = vae.__dict__.get(name, getattr(type(vae), name, None))+ if isinstance(value, int) and not isinstance(value, bool) and value > 0:+ out[name] = value+ return out+++def _empty_cache(device: Any) -> None:+ try:+ import torch+ if getattr(device, "type", None) == "cuda":+ torch.cuda.empty_cache()+ except Exception: # noqa: BLE001+ pass+++def _release_cache(z: Any, vae: Any = None) -> None:+ try:+ clear = getattr(vae, "clear_cache", None)+ if callable(clear):+ clear()+ except Exception: # noqa: BLE001+ pass+ gc.collect()+ _empty_cache(getattr(z, "device", None))+++def drop_tracebacks(exc: Optional[BaseException]) -> Optional[BaseException]:+ """Clear the traceback of ``exc`` and of every exception chained to it (``__cause__`` / ``__context__``, cycles+ guarded): a kept exception (to re-raise if nothing recovers) must not pin the failed attempt's frames and the+ tensors they hold through the retry. Every place the scope or the ladder keeps an exception calls this first."""+ seen: set = set()+ todo = [exc]+ while todo:+ e = todo.pop()+ if e is None or id(e) in seen:+ continue+ seen.add(id(e))+ e.__traceback__ = None+ todo += [e.__cause__, e.__context__]+ for sub in getattr(e, "exceptions", ()) or ():+ todo.append(sub)+ return exc+++def clear_partial_decode(vae: Any, z: Any = None) -> None:+ """Before ANY retry of a decode that failed part way: drop a causal decoder's frame / feature cache (Wan,+ Qwen-Image, LTX-2 and HunyuanVideo expose ``clear_cache``), which still holds the failed attempt's state and+ tensors, and return the freed blocks to the device. Every fallback (wide tiles -> stock tiles, LTX-2 tiles ->+ the floor tile, untiled -> tiled, the ladder's own steps) calls this first."""+ _release_cache(z, vae)+++def _compiling() -> bool:+ try:+ import torch+ return bool(torch.compiler.is_compiling())+ except Exception: # noqa: BLE001+ return False+++def _outer_gate_owns_untiled(vae: Any) -> bool:+ """video_vae_untiled's gate turns tiling off for its own attempt and retries tiled itself. Outer wrappers (the ROCm+ bf16 decode) expose it only through ``__wrapped__``."""+ fn = vae.__dict__.get("decode")+ for _ in range(8):+ if fn is None:+ return False+ if getattr(fn, "_unsloth_untiled_decode", False):+ return True+ fn = decode_layers.next_layer(fn)+ return False+++def _module_tensors(module: Any) -> list:+ return [*module.parameters(), *module.buffers()]+++def _group_hooked(module: Any) -> bool:+ """Under diffusers group offloading hooks (it streams where it is)."""+ hooks = getattr(getattr(module, "_diffusers_hook", None), "hooks", None) or {}+ return any("group_offloading" in str(name) for name in hooks)+++def _hooked(module: Any) -> bool:+ """Placed by an offload mechanism (accelerate model / sequential offload, diffusers group offload): that mechanism+ owns where its weights live."""+ for m in module.modules():+ if getattr(m, "_hf_hook", None) is not None:+ return True+ hooks = getattr(getattr(m, "_diffusers_hook", None), "hooks", None) or {}+ if any("group_offloading" in str(name) for name in hooks):+ return True+ return False+++def _whole_on(module: Any, device: Any) -> bool:+ tensors = _module_tensors(module)+ return bool(tensors) and all(t.device == device for t in tensors)+++def _nbytes(module: Any) -> int:+ seen: set = set()+ total = 0+ for t in _module_tensors(module):+ if id(t) in seen:+ continue+ seen.add(id(t))+ total += int(t.numel()) * int(t.element_size())+ return total+++def _host_free_mib() -> Optional[int]:+ try:+ from .diffusion_memory import _available_system_memory_mib+ return _available_system_memory_mib()+ except Exception: # noqa: BLE001+ return None+++def _holds_torchao(module: Any) -> bool:+ try:+ from .diffusion_memory import _holds_torchao_weights+ return bool(_holds_torchao_weights(module))+ except Exception: # noqa: BLE001+ return False+++def _return_to(module: Any, device: Any) -> None:+ try:+ module.to(device)+ except Exception: # noqa: BLE001 - a weight Module.to cannot move: tensor by tensor, as the offload path does+ from .diffusion_memory import _return_module_tensors+ _return_module_tensors(module, device)+++def _to_host(module: Any) -> None:+ module.to("cpu")+++def _placement(module: Any, device: Any) -> str:+ """How a component sits on the card: "plain" (whole there, no hooks), "torchao_resident" (whole there, torchao+ weights, no hooks), "groups" (resident offload groups), or "managed" (an offload mechanism places it)."""+ if getattr(module, "_unsloth_resident_room", None):+ return "groups"+ if _hooked(module):+ return "managed"+ if not _whole_on(module, device):+ return "managed"+ return "torchao_resident" if _holds_torchao(module) else "plain"+++def _call_restore(restore: Callable) -> None:+ try:+ restore(strict = True)+ except TypeError as exc:+ if "strict" not in str(exc):+ raise+ restore()+++_DEFERRED_ATTR = "_unsloth_vae_oom_deferred"+_RECORD_ATTR = "_unsloth_vae_oom_record"+++class DeferredReturn:+ """The way back for state a decode's restore could not undo (the card filled up meanwhile).++ Registered on the module that has to come back: a forward pre-hook retries every pending undo before the module+ runs, and the module carries an accelerate-shaped ``_hf_hook`` whose ``execution_device`` is the card, so+ ``DiffusionPipeline._execution_device`` still resolves the card while the module waits on the host (diffusers+ reads that attribute for offloaded components; ``apply_forward_hook`` decode methods call ``pre_forward``). Each+ pending undo leaves only once it has succeeded; when none is left the hooks go. ``required``: the module cannot run+ where it is (weights on the host, no offload hooks), so a retry that still runs out of memory raises; otherwise+ (the VAE streams through its offload hooks, resident groups stream) it runs as it is and retries next time."""++ no_grad = False++ def __init__(self, module: Any, device: Any, required: Any) -> None:+ self.module_ref = weakref.ref(module)+ self.execution_device = device+ self.required_checks: list = [required]+ self.pending: list = []+ self.handle = module.register_forward_pre_hook(self._pre_hook)+ self.owns_hf_hook = "_hf_hook" not in module.__dict__+ if self.owns_hf_hook:+ module._hf_hook = self+ module.__dict__[_DEFERRED_ATTR] = self++ def init_hook(self, module: Any) -> Any:+ return module++ def pre_forward(self, module: Any, *args: Any, **kwargs: Any) -> tuple:+ self.attempt()+ return args, kwargs++ def post_forward(self, module: Any, output: Any) -> Any:+ return output++ def detach_hook(self, module: Any) -> Any:+ return module++ def _pre_hook(self, module: Any, args: Any) -> None:+ self.attempt()++ @property+ def required(self) -> bool:+ """Whether the module cannot run where it is, CHECKED now (each pending way back registers a bool or a+ predicate on the module's current state): a settle that later succeeded makes it optional again."""+ for check in self.required_checks:+ try:+ if check() if callable(check) else check:+ return True+ except Exception: # noqa: BLE001 - its state cannot be read: treat it as not runnable+ return True+ return False++ def add(self, label: str, undo: Callable[[], None]) -> None:+ self.pending.append((label, undo))++ def attempt(self) -> bool:+ """Retry every pending undo in order; True once none is left."""+ while self.pending:+ label, undo = self.pending[0]+ try:+ undo()+ except Exception as exc: # noqa: BLE001+ if not self.required:+ return False+ # its weights are on the host and nothing would move them: running it would fail on the wrong+ # device, so say why (an out-of-memory keeps "out of memory" in the text for the OOM handlers)+ module = self.module_ref()+ raise RuntimeError(+ f"{type(module).__name__} ({label.split(':', 1)[-1]}) is still on the host after a VAE decode "+ f"ran out of memory and cannot return to {self.execution_device}: "+ + (+ "out of memory; free GPU memory and retry"+ if is_oom(exc)+ else f"{type(exc).__name__}: {exc}"+ )+ ) from exc+ self.pending.pop(0)+ self.detach()+ return True++ def detach(self) -> None:+ module = self.module_ref()+ try:+ self.handle.remove()+ except Exception: # noqa: BLE001+ pass+ if module is None:+ return+ if self.owns_hf_hook and module.__dict__.get("_hf_hook") is self:+ del module.__dict__["_hf_hook"]+ if module.__dict__.get(_DEFERRED_ATTR) is self:+ del module.__dict__[_DEFERRED_ATTR]+++def _defer(+ module: Any, device: Any, label: str, undo: Callable[[], None], required: Any, owner: Any+) -> None:+ """Hand ``undo`` to ``module``'s deferred return (one per module); ``owner`` (the VAE) records the module so+ uninstall can settle it."""+ deferred = module.__dict__.get(_DEFERRED_ATTR)+ if deferred is None:+ deferred = DeferredReturn(module, device, required)+ else:+ deferred.required_checks.append(required)+ deferred.add(label, undo)+ if owner is not None:+ record = owner.__dict__.setdefault(_RECORD_ATTR, {})+ refs = record.setdefault("deferred", [])+ if not any(r() is module for r in refs):+ refs.append(weakref.ref(module))+++_FAULT: Optional[Callable[[str], None]] = None+++def _fault(point: str) -> None:+ if _FAULT is not None:+ _FAULT(point)+++class Change:+ """One state change of a recovery. ``undo`` puts it back; when ``undo`` fails part way, ``settle`` leaves the+ module in a consistent working state and ``defer`` hands the way back to the module's DeferredReturn, whose+ retry is ``retry`` (undo, falling back to settle)."""++ def __init__(self, label: str, undo: Callable[[], None], settle: Callable[[], None] = lambda: None,+ defer: Optional[Callable[[Callable[[], None], BaseException], None]] = None,) -> None: # fmt: skip+ self.label = label+ self.undo = undo+ self.settle = settle+ self.defer = defer++ def retry(self) -> None:+ try:+ _fault(self.label + ":undo")+ self.undo()+ except Exception:+ _fault(self.label + ":settle")+ self.settle()+ raise+++class DecodeSession:+ """One OOM recovery as a transaction: every state change goes through ``apply`` (do, then record its Change),+ and ``restore`` rolls the record back in reverse order. A Change leaves the record only once its own undo+ succeeded or its module was settled into a consistent state with a deferred way back."""++ def __init__(+ self,+ vae: Any,+ device: Any,+ logger: Any = None,+ ) -> None:+ self.vae = vae+ self.device = device+ self.logger = logger+ self.freed = False+ self.vae_offloaded = False+ self.log: list = [] # Change, in the order applied++ @property+ def moved(self) -> list:+ return [c.label.split(":", 1)[1] for c in self.log if c.label.startswith("moved:")]++ def _note(self, msg: str, *args: Any) -> None:+ if self.logger is not None:+ self.logger.warning(msg, *args)++ def apply(self, do: Callable[[], None], change: Change) -> bool:+ """Run ``do``; record ``change`` on success. A ``do`` that fails (part way, possibly) is rolled back now."""+ try:+ _fault(change.label + ":do")+ do()+ except Exception as exc: # noqa: BLE001+ self._note("diffusion.vae_oom: %s failed (%s); rolled back", change.label, exc)+ self.rollback(change)+ return False+ self.log.append(change)+ return True++ def rollback(self, change: Change) -> bool:+ try:+ _fault(change.label + ":undo")+ change.undo()+ return True+ except Exception as exc: # noqa: BLE001+ try:+ _fault(change.label + ":settle")+ change.settle()+ except Exception as exc2: # noqa: BLE001 - the deferred return below retries from wherever it is+ self._note("diffusion.vae_oom: settling %s failed (%s)", change.label, exc2)+ if change.defer is not None:+ try:+ change.defer(change.retry, exc)+ except Exception as exc3: # noqa: BLE001+ self._note("diffusion.vae_oom: deferring %s failed (%s)", change.label, exc3)+ return False++ def restore(self) -> None:+ while self.log:+ self.rollback(self.log.pop())+ self.vae_offloaded = False++ def free_others(self, pipe: Any) -> int:+ """Step (a). Returns how many things were released (0: nothing on the card to free)."""+ if self.freed or pipe is None or getattr(self.device, "type", None) != "cuda":+ return 0+ self.freed = True+ released = 0+ # Captured graphs pin their pool and the weights' addresses: drop them before any weight moves (the image and+ # video backends drop them on any OOM too). Nothing to undo: the next step re-captures. If dropping them+ # fails, no weight may move under a graph that could still replay: step (a) stops here.+ graphs = getattr(pipe, "_unsloth_cuda_graphs", ()) or ()+ if graphs:++ def reset() -> None:+ from . import diffusion_cuda_graph++ # strict: a handle that failed to reset (or still holds a graph) aborts step (a) before any move+ diffusion_cuda_graph.reset_all(graphs, strict = True)++ if not self.apply(reset, Change("graphs_reset", lambda: None)):+ return 0+ released += 1+ self._hook_torchao_resident(pipe)+ released += self._release_groups(pipe)+ released += self._move_components(pipe)+ if self.moved:+ self._note(+ "diffusion.vae_oom: moved %s to the host for the decode", ", ".join(self.moved)+ )+ return released++ def _hook_torchao_resident(self, pipe: Any) -> bool:+ """A hookless resident torchao transformer cannot move whole (Module.to skips or breaks its weights): give it+ block offload hooks with every group resident (diffusion_memory.hook_resident_denoiser, the path an oversized+ request takes), so the group release below can stream it. The hooks stay afterwards, every group re-pinned,+ exactly as after an oversized request."""+ transformer = getattr(pipe, "transformer", None)+ if transformer is None or _placement(transformer, self.device) != "torchao_resident":+ return False+ from . import diffusion_memory++ hooked: list = []+ torch_hooks = diffusion_memory.module_forward_hook_ids(transformer)+ device, logger = self.device, self.logger++ def do() -> None:+ hooked.append(bool(diffusion_memory.hook_resident_denoiser(pipe, device, logger)))++ def undo() -> None:+ # Installed: the hooks stay (every group re-pinned by the release's restore, as after an oversized+ # request). Otherwise back to where it started from wherever it is: split -> whole on the card; settled+ # (streamed under hooks) -> every group resident again. Idempotent: a clean refusal moves nothing.+ if not (hooked and hooked[0]):+ diffusion_memory.return_torchao_denoiser(pipe, device, logger, torch_hooks)++ def settle() -> None:+ # cannot come back whole: stream it under block offload hooks (the load's flat plan), so it still runs+ # (already under hooks: a re-pin that failed part way leaves some groups streamed, which runs)+ if not _group_hooked(transformer) and not diffusion_memory.hook_resident_denoiser(+ pipe, device, logger, resident = False+ ):+ raise RuntimeError("the transformer could not be streamed under offload hooks")++ def defer(retry: Callable[[], None], exc: BaseException) -> None:+ # required until it is checked to stream under offload hooks (split with no hooks it cannot run)+ _defer(transformer, device, "torchao_hooks:transformer", retry, lambda: not _group_hooked(transformer),+ self.vae) # fmt: skip++ if not self.apply(do, Change("torchao_hooks", undo, settle, defer)):+ return False+ return bool(hooked and hooked[0])++ def _release_groups(self, pipe: Any) -> int:+ """Resident offload groups (text encoders first, then the denoiser's blocks) stream again."""+ from . import diffusion_memory++ got: list = []+ anchor = next(+ (m for m in (getattr(pipe, "components", {}) or {}).values() if getattr(m, "_unsloth_resident_room", None)),+ None,+ ) # fmt: skip+ if anchor is None:+ return 0++ def do() -> None:+ restore = diffusion_memory.release_resident_groups(+ pipe, 1 << 30, self.logger, reason = "a VAE decode that ran out of memory"+ )+ if restore is not None:+ got.append(restore)++ def undo() -> None:+ while got:+ # strict: an incomplete re-pin raises (the release path's re-pin otherwise swallows it), so the+ # settle / defer below runs and the restore is kept until every group is resident again+ _call_restore(got[-1])+ got.pop()++ def settle() -> None:+ restore = diffusion_memory.release_resident_groups(+ pipe, 1 << 30, self.logger, reason = "a failed re-pin"+ )+ if restore is not None:+ got.append(restore)++ def defer(retry: Callable[[], None], exc: BaseException) -> None:+ # Optional only while every offload group is checked to keep its hooks: a group, resident or streamed,+ # then runs where it is (a re-pin and a flat-plan settle that both stopped part way leave a mix of+ # the two, which still runs); the re-pin is retried before the next forward.+ _defer(+ anchor,+ self.device,+ "resident_groups",+ retry,+ lambda: not _groups_hooked(pipe),+ self.vae,+ )++ change = Change("resident_groups", undo, settle, defer)+ if not self.apply(do, change):+ return 0+ if not got:+ self.log.remove(change)+ return 0+ return 1++ def _move_components(self, pipe: Any) -> int:+ """Components whole on the card with no offload hooks: to the host, largest first, while host RAM allows."""+ import torch++ candidates = []+ for name, comp in (getattr(pipe, "components", {}) or {}).items():+ if comp is self.vae or not isinstance(comp, torch.nn.Module):+ continue+ if _placement(comp, self.device) != "plain":+ continue+ candidates.append((_nbytes(comp), name, comp))+ candidates.sort(key = lambda c: -c[0])+ host = _host_free_mib()+ released = 0+ for size, name, comp in candidates:+ mib = size >> 20+ if host is not None and mib + HOST_MARGIN_MIB > host:+ self._note(+ "diffusion.vae_oom: %s (%d MiB) stays on the card: host RAM is short", name, mib+ )+ continue+ if self.apply(lambda comp = comp: _to_host(comp), self._component_change(name, comp)):+ released += 1+ if host is not None:+ host -= mib+ return released++ def _component_change(self, name: str, comp: Any) -> Change:+ device = self.device++ def settle() -> None:+ _to_host(comp)+ _empty_cache(device)++ def defer(retry: Callable[[], None], exc: BaseException) -> None:+ self._note(+ "diffusion.vae_oom: %s could not return to the card after the decode (%s); it returns before its "+ "next forward", name, exc,+ ) # fmt: skip+ _defer(comp, device, f"moved:{name}", retry, True, self.vae)++ return Change(f"moved:{name}", lambda: _return_to(comp, device), settle, defer)++ def _component_undo(self, name: str, comp: Any) -> tuple:+ """(undo, defer) for a component moved to the host; kept for callers recording a move by hand."""+ change = self._component_change(name, comp)+ return change.retry, lambda exc: change.defer(change.retry, exc)++ def offload_vae(self) -> bool:+ """Step (c): the VAE's weights stream per layer from the host (diffusers leaf-level group offload)."""+ if self.vae_offloaded or getattr(self.device, "type", None) != "cuda" or _hooked(self.vae):+ return False+ vae, device = self.vae, self.device+ host, mib = _host_free_mib(), _nbytes(vae) >> 20+ if host is not None and mib + HOST_MARGIN_MIB > host:+ # the streamed copy lives in host RAM: never trade a decode OOM for a host OOM+ self._note(+ "diffusion.vae_oom: %s (%d MiB) stays on the card: host RAM is short",+ type(vae).__name__,+ mib,+ )+ return False++ settled: list = []++ def settle() -> None:+ settled.clear()+ _park_vae(vae, device)+ settled.append(True)++ def defer(retry: Callable[[], None], exc: BaseException) -> None:+ # optional only when the VAE verifiably streams whole from the host under its offload hooks; otherwise+ # (the park failed too: split, maybe hookless) the next decode must not run before it is repaired+ streams = bool(settled) and _vae_streams(vae)+ settled.clear()+ self._note("diffusion.vae_oom: %s %s (%s); it returns at its next decode", type(vae).__name__,+ "stays streamed from the host" if streams else "is split between host and card", exc) # fmt: skip+ _defer(vae, device, "vae_offload", retry, lambda: not _vae_streams(vae), vae)++ change = Change("vae_offload", lambda: _remove_vae_offload(vae, device), settle, defer)+ if not self.apply(lambda: _apply_vae_offload(vae, device), change):+ return False+ self.vae_offloaded = True+ self._note(+ "diffusion.vae_oom: %s weights stream from the host for this decode", type(vae).__name__+ )+ return True+++def _apply_vae_offload(vae: Any, device: Any) -> None:+ import torch+ from diffusers.hooks import apply_group_offloading+ apply_group_offloading(+ vae, onload_device = device, offload_device = torch.device("cpu"), offload_type = "leaf_level"+ )+++def _remove_vae_offload(vae: Any, device: Any) -> None:+ from .diffusion_memory import _remove_group_offload_hooks+ _remove_group_offload_hooks(vae)+ _return_to(vae, device)+++def _groups_hooked(pipe: Any) -> bool:+ """Every component placed by resident offload groups still carries its group offloading hooks."""+ comps = [+ m+ for m in (getattr(pipe, "components", {}) or {}).values()+ if getattr(m, "_unsloth_resident_room", None)+ ]+ return all(_group_hooked(m) for m in comps)+++def _whole_on_host(module: Any) -> bool:+ tensors = [*module.parameters(), *module.buffers()]+ return all(getattr(t.device, "type", None) == "cpu" for t in tensors)+++def _vae_hooked(vae: Any) -> bool:+ """The VAE carries its own leaf-level group offloading hooks (not merely a deferred return's ``_hf_hook``)."""+ return _group_hooked(vae)+++def _vae_streams(vae: Any) -> bool:+ """Checked, not assumed: the VAE's offload hooks are on and every weight is on the host, so it decodes+ (streamed) as it is."""+ return _vae_hooked(vae) and _whole_on_host(vae)+++def _park_vae(vae: Any, device: Any) -> None:+ """Whole on the host with the offload hooks on: the VAE still decodes (streamed)."""+ from .diffusion_memory import _remove_group_offload_hooks++ _remove_group_offload_hooks(vae)+ _to_host(vae)+ _empty_cache(device)+ _apply_vae_offload(vae, device)+++def _undo_vae_offload(vae: Any, device: Any) -> None:+ """Hooks off and weights back on the card; if that fails, the VAE is parked streamed (``_park_vae``) and the+ error raised, so the caller can retry later."""+ try:+ _remove_vae_offload(vae, device)+ except Exception:+ _park_vae(vae, device)+ raise+++def settle_deferred(module: Any, strict: bool = False) -> bool:+ """Run ``module``'s pending deferred returns now; True when nothing is left pending. ``strict``: a REQUIRED+ repair that still fails raises (its clear error), so the caller never runs the module in an inconsistent state;+ an optional one (it runs as it is) returns False."""+ deferred = getattr(module, "__dict__", {}).get(_DEFERRED_ATTR)+ if deferred is None:+ return True+ try:+ return deferred.attempt()+ except Exception: # noqa: BLE001 - still out of memory: the deferred return stays+ if strict:+ raise+ return False+++def _pipe_of(vae: Any) -> Any:+ ref = vae.__dict__.get(_PIPE_ATTR)+ return ref() if callable(ref) else None+++def _device_of(z: Any) -> Any:+ return getattr(z, "device", None)+++def free_first(vae: Any, z: Any = None) -> bool:+ """Step (a) for a Studio decode wrapper about to fall back to a cheaper decode on an OOM (wide tiles -> stock+ tiles, LTX-2 tiles -> the floor tile, untiled -> tiled): free the card inside the current decode's recovery+ session. True when something was freed, so the wrapper retries its own attempt first. Outside a decode scope, or+ with the retry switched off, or once this decode already freed the card: False (fall back as before)."""+ holder = getattr(vae, "__dict__", {}).get(_SESSION_ATTR)+ if holder is None or oom_retry_disabled() or _compiling():+ return False+ session = holder.get("session")+ if session is None:+ session = holder["session"] = DecodeSession(+ vae, _device_of(z), vae.__dict__.get("_unsloth_vae_oom_logger")+ )+ if session.freed:+ return False+ _release_cache(z, vae)+ freed = bool(session.free_others(_pipe_of(vae)))+ stats = vae.__dict__.get("_unsloth_vae_oom_stats")+ if isinstance(stats, dict):+ stats["freed_first"] = stats.get("freed_first", 0) + int(freed)+ return freed+++def _ladder_steps(+ tiled_now: bool,+ can_tile: bool,+ eager_rung: bool = False,+) -> list:+ """The ladder's rungs, the one list both the ``_decode`` retry and the outer decode scope climb, in order:+ (name, tiled, offload_vae, eager). "freed": the rest of the pipeline off the card, the same decode again;+ "eager" (scope only, a compiled guard below): every compiled guard on its eager path, where the ``_decode``+ ladder runs too; "tiled": stock tile sizes; "offload": the VAE's weights streamed from the host."""+ steps: list = [("freed", tiled_now, False, False)]+ if eager_rung:+ steps.append(("eager", tiled_now, False, True))+ if can_tile and not tiled_now:+ steps.append(("tiled", True, False, eager_rung))+ steps.append(("offload", tiled_now or can_tile, True, eager_rung))+ return steps+++_STEP_TEXT = {"freed": "with the rest of the pipeline off the card", "eager": "on the eager path",+ "tiled": "tiled at stock tile sizes", "offload": "with the VAE weights streamed from the host"} # fmt: skip+++def _climb(vae: Any, session: "DecodeSession", steps: list, attempt: Callable, stats: dict, z: Any,+ logger: Any) -> tuple: # fmt: skip+ """Run ``steps`` in order, each rung's state change through ``session`` (applied once per logical decode: a+ rung a nested climb already took is skipped), then ``attempt(tiled, eager)``. (True, out, rung) on the first+ attempt that returns; (False, None, None) when none does."""+ for name, tiled, offload_vae, eager in steps:+ _release_cache(z, vae)+ if name == "freed":+ t0 = time.perf_counter()+ freed = session.free_others(_pipe_of(vae))+ stats["free_s"] = round(time.perf_counter() - t0, 3)+ if not freed:+ continue # nothing (more) was freed: the same decode would fail the same way+ elif offload_vae and not session.offload_vae():+ continue+ if logger is not None:+ logger.warning("diffusion.vae_oom: %s decode ran out of memory; retrying %s", type(vae).__name__,+ _STEP_TEXT[name]) # fmt: skip+ stats[f"{name}_retry"] = stats.get(f"{name}_retry", 0) + 1+ failed = False+ try:+ out = attempt(tiled, eager)+ except Exception as exc: # noqa: BLE001+ if not is_oom(exc) and not offload_vae:+ raise+ if not is_oom(exc) and logger is not None:+ logger.warning(+ "diffusion.vae_oom: streamed %s decode failed (%s)", type(vae).__name__, exc+ )+ drop_tracebacks(exc)+ failed = True+ if not failed:+ return True, out, name+ stats["oom"] = stats.get("oom", 0) + 1+ return False, None, None+++_RETRY_LAYER = "vae_oom_retry"+_SCOPE_LAYER = "vae_oom_scope"+++def _scope_session(vae: Any) -> Optional[dict]:+ return vae.__dict__.get(_SESSION_ATTR)+++def _close_scope(vae: Any, holder: dict, z: Any) -> None:+ vae.__dict__.pop(_SESSION_ATTR, None)+ session = holder.get("session")+ if session is None:+ return+ t0 = time.perf_counter()+ session.restore()+ stats = vae.__dict__.get("_unsloth_vae_oom_stats")+ if isinstance(stats, dict):+ stats["restore_s"] = round(time.perf_counter() - t0, 3)+ _empty_cache(_device_of(z))+++def install(+ vae: Any,+ logger: Any = None,+ pipe: Any = None,+) -> bool:+ """Wrap ``vae._decode`` with the offload-first OOM ladder, and the public ``decode`` with the scope one logical+ decode's recovery spans (a compiled decode's guard included). Idempotent. ``pipe``: the pipeline whose other+ components step (a) may move off the card (kept as a weak reference; a reinstall keeps the last one)."""+ if vae is None or oom_retry_disabled():+ return False+ if pipe is not None:+ try:+ vae.__dict__[_PIPE_ATTR] = weakref.ref(pipe)+ except TypeError:+ pass+ if not callable(getattr(vae, "_decode", None)):+ return False+ if decode_layers.find(vae, "_decode", _RETRY_LAYER) is not None:+ return True+ can_tile = callable(getattr(vae, "enable_tiling", None)) and hasattr(vae, "use_tiling")+ stats = {"calls": 0, "oom": 0, "freed_retry": 0, "tiled_retry": 0, "offload_retry": 0, "recovered": 0,+ "recovered_by": None, "failed": 0} # fmt: skip++ def make_retry(inner: Callable, layer: Any) -> Callable:+ depth = [0]++ def _attempt(z: Any, args: tuple, kwargs: dict, tiled: bool) -> Any:+ prev = bool(getattr(vae, "use_tiling", False))+ vae._unsloth_vae_oom_active = True+ try:+ if can_tile:+ vae.use_tiling = tiled+ return inner(z, *args, **kwargs)+ finally:+ if can_tile:+ vae.use_tiling = prev+ vae._unsloth_vae_oom_active = False++ def _decode(z: Any, *args: Any, **kwargs: Any) -> Any:+ # Traced inside a compiled decode: a pass-through (no try/except in the graph; the compiled guard+ # retries). Nested (a wide-tile override decoding one tile through _decode): the outermost call retries.+ if _compiling() or oom_retry_disabled() or depth[0]:+ return inner(z, *args, **kwargs)+ depth[0] += 1+ holder = _scope_session(vae)+ owns = holder is None+ if owns:+ holder = {"session": None}+ vae.__dict__[_SESSION_ATTR] = holder+ try:+ if owns:+ settle_deferred(+ vae, strict = True+ ) # a split VAE never decodes before it is repaired+ return _retrying(z, args, kwargs, holder)+ finally:+ depth[0] -= 1+ if owns:+ _close_scope(vae, holder, z)+ # reached without the scope: a later rewrap (a compile fallback restoring its eager decode, a+ # speed layer run after load) dropped or covered it; put it back on top for the next decode+ ensure_outermost(vae, logger)++ def _retrying(z: Any, args: tuple, kwargs: dict, holder: dict) -> Any:+ stats["calls"] += 1+ tiled_now = bool(getattr(vae, "use_tiling", False))+ steps = _ladder_steps(tiled_now, can_tile)+ recovered = holder.get("recovered")+ if recovered is not None:+ names = [step[0] for step in steps]+ start = names.index(recovered) if recovered in names else 0+ name, tiled = steps[start][:2]+ try:+ return _attempt(z, args, kwargs, tiled)+ except Exception as exc: # noqa: BLE001+ if not is_oom(exc):+ raise+ drop_tracebacks(exc)+ holder.setdefault("first_oom", exc)+ stats["oom"] += 1+ steps = steps[start + 1 :]+ else:+ gate_owned = False+ try:+ return inner(z, *args, **kwargs)+ except Exception as exc: # noqa: BLE001+ if not is_oom(exc):+ raise+ gate_owned = not tiled_now and _outer_gate_owns_untiled(vae)+ drop_tracebacks(exc)+ holder.setdefault("first_oom", exc)+ stats["oom"] += 1+ if gate_owned:+ # The untiled-when-fits gate retries tiled itself; the ladder still sees the OOM first: free the+ # card (step a) and retry untiled once, then hand back to the gate.+ if free_first(vae, z):+ stats["freed_retry"] += 1+ try:+ out = inner(z, *args, **kwargs)+ stats["recovered"] += 1+ stats["recovered_by"] = "freed"+ return out+ except Exception as exc: # noqa: BLE001+ if not is_oom(exc):+ raise+ drop_tracebacks(exc)+ raise holder["first_oom"]+ session = holder.get("session")+ if session is None:+ session = holder["session"] = DecodeSession(vae, _device_of(z), logger)+ ok, out, name = _climb(vae, session, steps, lambda tiled, eager: _attempt(z, args, kwargs, tiled),+ stats, z, logger) # fmt: skip+ if ok:+ stats["recovered"] += 1+ stats["recovered_by"] = name+ holder["recovered"] = name+ return out+ holder["exhausted"] = True+ stats["failed"] += 1+ _release_cache(z, vae)+ first = holder.get("first_oom")+ note = ("VAE decode ran out of memory even with the rest of the pipeline and the VAE weights off the "+ "card; free GPU memory or lower the resolution") # fmt: skip+ if first is None:+ raise RuntimeError(note)+ if callable(getattr(first, "add_note", None)) and note not in (+ getattr(first, "__notes__", None) or ()+ ):+ first.add_note(note)+ raise first++ _decode._unsloth_vae_oom_retry = True+ _decode._unsloth_vae_oom_stats = stats+ return _decode++ decode_layers.wrap(vae, "_decode", _RETRY_LAYER, make_retry)+ vae._unsloth_vae_oom_stats = stats+ vae.__dict__["_unsloth_vae_oom_logger"] = logger+ _install_scope(vae, logger)+ if logger is not None:+ logger.info(+ "diffusion.vae_oom: %s decode recovers an out-of-memory by offloading, then tiling at stock sizes",+ type(vae).__name__,+ )+ return True+++def _compiled_below(vae: Any, top: Any) -> bool:+ """A compiled decode guard sits somewhere under ``top`` (walked through every wrapper kind), or the speed layer+ compiled this VAE's decode (a guard behind a closure the walk cannot see)."""+ fn, seen = top, 0+ while fn is not None and seen < 64:+ seen += 1+ if getattr(fn, "_unsloth_compiled_decode", False) and callable(+ getattr(fn, "_unsloth_eager_decode", None)+ ):+ return True+ fn = decode_layers.next_layer(fn)+ return bool(vae.__dict__.get("_unsloth_compiled_decode", False))+++# Decode wrappers that must sit BELOW the scope: a compiled decode guard (the graph would trace the scope as a+# pass-through and its guard re-raises the OOM), the untiled-when-fits gate (it retries tiled itself) and the dtype+# wrappers (their own allocations: the fp16 isfinite check, the bf16 -> fp32 output cast).+_BELOW_SCOPE_MARKERS = ("_unsloth_compiled_decode", "_unsloth_untiled_decode", "_unsloth_bf16_decode",+ "_unsloth_decode_slot") # fmt: skip+++def _scope_intact(vae: Any) -> bool:+ """The scope layer is active, reachable from the current public decode, and none of the wrappers that must sit+ below it is above it."""+ layer = vae.__dict__.get(_RECORD_ATTR, {}).get("scope")+ if layer is None or not layer.active or layer.owner is not vae:+ return False+ fn = vae.__dict__.get("decode")+ for _ in range(32):+ if fn is None:+ break+ if fn is layer.fn:+ return True+ if any(getattr(fn, m, None) for m in _BELOW_SCOPE_MARKERS) and not getattr(+ fn, "_unsloth_vae_oom_scope", False+ ):+ return False+ fn = decode_layers.next_layer(fn)+ return False+++def ensure_outermost(vae: Any, logger: Any = None) -> bool:+ """(Re)assert the OOM scope as the outermost decode layer after anything rewrapped ``vae.decode`` (a deferred+ speed profile, a VAE compile, a compile fallback, a dtype wrapper). Called after every rewrap site and when a+ decode closes; idempotent and cheap. True when the retry is installed and its scope is in place."""+ if (+ vae is None+ or decode_layers.find(vae, "_decode", _RETRY_LAYER) is None+ or oom_retry_disabled()+ ):+ return False+ if _scope_intact(vae):+ return True+ record = vae.__dict__.setdefault(_RECORD_ATTR, {})+ decode_layers.unwrap(record.pop("scope", None))+ return _install_scope(vae, logger)+++def _install_scope(vae: Any, logger: Any = None) -> bool:+ """Wrap the public ``decode`` as its outermost layer: the outermost call opens the recovery scope every inner+ ``_decode`` (batch slices, frames, tiles) shares, and closes it, restoring what the recovery moved, once the+ whole decode returns. An out-of-memory that reaches the scope (a compiled graph, which never enters the+ ``_decode`` ladder, or a wrapper's own allocation) frees the card and retries the whole decode as it was (a+ compiled one with the same kernels, same output); then, with a compiled guard below, the whole decode once more+ with every guard on its eager path, where the ``_decode`` ladder continues in the same session."""+ if decode_layers.find(vae, "decode", _SCOPE_LAYER) is not None:+ return True+ if not callable(getattr(vae, "decode", None)):+ return False++ def make_scope(inner: Callable, layer: Any) -> Callable:+ def _try(args: tuple, kwargs: dict, failures: list) -> tuple:+ try:+ return True, inner(*args, **kwargs)+ except Exception as exc: # noqa: BLE001+ if not is_oom(exc):+ raise+ drop_tracebacks(exc)+ failures.append(exc)+ return False, None++ def decode(*args: Any, **kwargs: Any) -> Any:+ if _compiling() or oom_retry_disabled() or _scope_session(vae) is not None:+ return inner(*args, **kwargs)+ holder: dict = {"session": None}+ vae.__dict__[_SESSION_ATTR] = holder+ z = args[0] if args else kwargs.get("z")+ failures: list = []+ try:+ # a VAE left streamed by an earlier restore that ran out of memory comes back first, if it fits now;+ # one left split (a required repair) never decodes before it is repaired: its clear error surfaces+ settle_deferred(vae, strict = True)+ ok, out = _try(args, kwargs, failures)+ if ok:+ return out+ compiled = _compiled_below(vae, inner)+ if holder.get("exhausted") and not compiled:+ raise failures[0] # the _decode ladder already climbed every rung for this OOM+ stats = vae.__dict__.get("_unsloth_vae_oom_stats")+ if not isinstance(stats, dict):+ stats = {}+ stats["outer_oom"] = stats.get("outer_oom", 0) + 1+ if compiled:+ stats["compiled_oom"] = stats.get("compiled_oom", 0) + 1+ session = holder.get("session")+ if session is None:+ session = holder["session"] = DecodeSession(vae, _device_of(z), logger)+ # The same rungs as the _decode ladder, for an OOM it never saw (a compiled graph, a dtype wrapper's+ # own allocation): each attempt is the whole decode, wrappers included.+ can_tile = callable(getattr(vae, "enable_tiling", None)) and hasattr(+ vae, "use_tiling"+ )+ tiled_now = bool(getattr(vae, "use_tiling", False))++ def attempt(tiled: bool, eager: bool) -> Any:+ from . import diffusion_bg_compile # noqa: PLC0415+ prev = bool(getattr(vae, "use_tiling", False))+ try:+ if can_tile:+ vae.use_tiling = tiled+ if eager:+ with diffusion_bg_compile.force_eager():+ return inner(*args, **kwargs)+ return inner(*args, **kwargs)+ finally:+ if can_tile:+ vae.use_tiling = prev++ ok, out, name = _climb(vae, session, _ladder_steps(tiled_now, can_tile, compiled), attempt, stats,+ z, logger) # fmt: skip+ if not ok:+ raise failures[0]+ stats["recovered"] = stats.get("recovered", 0) + 1+ if name == "freed" and compiled:+ stats["recovered_by"] = "freed_compiled"+ elif name != "eager" or holder.get("recovered") is None:+ stats["recovered_by"] = name+ else:+ stats["recovered_by"] = holder["recovered"]+ return out+ finally:+ failures.clear()+ _close_scope(vae, holder, z)+ ensure_outermost(vae, logger)++ decode._unsloth_vae_oom_guard = True+ decode._unsloth_vae_oom_scope = True+ return decode++ layer = decode_layers.wrap(vae, "decode", _SCOPE_LAYER, make_scope)+ vae.__dict__.setdefault(_RECORD_ATTR, {})["scope"] = layer+ return True+++def uninstall(vae: Any) -> None:+ """Remove everything ``install`` and its retries put in place: the ``_decode`` layer, the ``decode`` scope /+ compiled guard (cut out exactly when outermost, a pass-through under later wrappers), and every deferred return.+ A deferred return is settled first (its pending undo runs); one that still cannot run (the card is full) stays,+ since it is that module's only way back to the card."""+ record = vae.__dict__.get(_RECORD_ATTR, {})+ decode_layers.unwrap(record.pop("scope", None))+ for ref in record.pop("deferred", []):+ module = ref()+ if module is not None and not settle_deferred(module):+ record.setdefault("deferred", []).append(ref)+ if not record:+ vae.__dict__.pop(_RECORD_ATTR, None)+ layer = decode_layers.find(vae, "_decode", _RETRY_LAYER)+ if layer is None:+ return+ decode_layers.unwrap(layer)+ vae.__dict__.pop("_unsloth_vae_oom_stats", None)+++__all__ = [+ "DecodeSession",+ "DeferredReturn",+ "clear_partial_decode",+ "drop_tracebacks",+ "ensure_outermost",+ "free_first",+ "settle_deferred",+ "OOM_RETRY_ENV",+ "install",+ "is_oom",+ "oom_retry_disabled",+ "tile_attrs",+ "uninstall",+]diff --git a/studio/backend/core/inference/diffusion_vae_tiling.py b/studio/backend/core/inference/diffusion_vae_tiling.pyindex ce9b6d9f4b5..7a60d808faa 100644--- a/studio/backend/core/inference/diffusion_vae_tiling.py+++ b/studio/backend/core/inference/diffusion_vae_tiling.py@@ -19,6 +19,8 @@ import typesfrom typing import Any, Optionalfrom .diffusion_device import float64_device+from .diffusion_vae_oom import clear_partial_decode+from .diffusion_vae_oom import free_first as oom_free_firstWIDE_TILES_ENV = "UNSLOTH_DIFFUSION_VAE_WIDE_TILES"@@ -267,8 +269,10 @@ def _free_mib(def _fused(vae: Any) -> bool:- return bool(getattr(vae, "_unsloth_vae_fused_installed", 0)) and not getattr(- vae, "_unsloth_vae_fused_failed", False+ return (+ bool(getattr(vae, "_unsloth_vae_fused_installed", 0))+ and not getattr(vae, "_unsloth_vae_fused_failed", False)+ and not getattr(vae, "_unsloth_vae_fused_structural", False))@@ -536,6 +540,10 @@ def _release_cache(z: Any) -> None:pass+def _clear_partial(vae: Any, z: Any) -> None:+ clear_partial_decode(vae, z)++def _log_stock_fallback(vae: Any, reason: str) -> None:logger = vae.__dict__.get("_unsloth_wide_logger")seen = vae.__dict__.setdefault("_unsloth_wide_fallback_logged", set())@@ -576,23 +584,27 @@ def install(vae: Any, logger: Any = None) -> bool:) -> Any:if wide_tiles_disabled() or stock_layout_ok(geometry, z.shape[-2], z.shape[-1]):return _stock(z, return_dict)- # Never trade an OOM for quality: floor tile cannot fit (or OOMs) -> the smaller stock tiles.- reason = floor_shortfall(self, z)- if reason is None:- for max_area in ("auto", 0):- try:- return tiled_decode(self, z, return_dict = return_dict, max_area = max_area)- except Exception as exc: # noqa: BLE001- if not _is_oom(exc):- raise- reason = f"the wide tiles ran out of memory ({type(exc).__name__})"- tried = (self.__dict__.get("_unsloth_last_decode_tile") or (0, 0))[:2]- # outside the except block, so the failed attempt's tensors are freed- _release_cache(z)- ratio, tile, _ = _geometry(self)- if tuple(tried) == (min(tile, z.shape[-2]), min(tile, z.shape[-1])):- break+ # Never trade an OOM for quality: floor tile cannot fit (or OOMs) -> the smaller stock tiles. Before+ # settling for them, the OOM retry frees the card (rest of the pipeline off it) and the wide tiles go again.+ for _round in range(2):+ reason = floor_shortfall(self, z)+ if reason is None:+ for max_area in ("auto", 0):+ try:+ return tiled_decode(self, z, return_dict = return_dict, max_area = max_area)+ except Exception as exc: # noqa: BLE001+ if not _is_oom(exc):+ raise+ reason = f"the wide tiles ran out of memory ({type(exc).__name__})"+ tried = (self.__dict__.get("_unsloth_last_decode_tile") or (0, 0))[:2]+ _clear_partial(self, z)+ ratio, tile, _ = _geometry(self)+ if tuple(tried) == (min(tile, z.shape[-2]), min(tile, z.shape[-1])):+ break+ if _round or not oom_free_first(self, z):+ break_log_stock_fallback(self, reason)+ _clear_partial(self, z)return _stock(z, return_dict)def _tiled_encode(self: Any, x: Any) -> Any:diff --git a/studio/backend/core/inference/media_decode_phase.py b/studio/backend/core/inference/media_decode_phase.pyindex b8ce152f87f..cc9fc6fc292 100644--- a/studio/backend/core/inference/media_decode_phase.py+++ b/studio/backend/core/inference/media_decode_phase.py@@ -34,6 +34,9 @@ def decode_phase(on_decode()return original(*args, **kwargs)+ _decode.__wrapped__ = (+ original # the decode stack stays walkable (diffusion_vae_oom checks its scope)+ )return _decodefor name in attrs:diff --git a/studio/backend/core/inference/video.py b/studio/backend/core/inference/video.pyindex bba15a4094d..eb7f2d6182e 100644--- a/studio/backend/core/inference/video.py+++ b/studio/backend/core/inference/video.py@@ -9428,6 +9428,14 @@ class VideoBackend:vae_bf16_mode = install_rocm_vae_bf16_decode(pipe, target, logger = logger)if vae_bf16_mode:speed_optims += ("vae_bf16_decode", f"vae_bf16_{vae_bf16_mode}")+ # Offload-first retry on a decode OOM, installed last: its decode scope is the outermost decode layer+ # (one recovery session per logical decode) and every inner wrapper that falls back on an OOM (wide+ # tiles, untiled-when-fits) frees the card through it first (diffusion_vae_oom.free_first).+ try:+ from .diffusion_vae_oom import install as install_vae_oom_retry+ install_vae_oom_retry(getattr(pipe, "vae", None), logger, pipe = pipe)+ except Exception as exc: # noqa: BLE001 - a decode OOM then fails as before+ logger.warning("video.vae_oom: not installed: %s", exc)resolved = build_resolved_record({@@ -10649,6 +10657,12 @@ class VideoBackend:speed_optims = speed_optims + tuple(f"h3_vae_{name}" for name in vae_levers)except Exception as exc: # noqa: BLE001 -- optimisation only, never fail a loadlogger.warning("video.h3_vae_fast failed, keeping the stock VAE: %s", exc)+ # the H3 VAE speedups rewrap vae.decode: the OOM retry's scope (when installed) goes back on top+ try:+ from .diffusion_vae_oom import ensure_outermost+ ensure_outermost(getattr(pipe, "vae", None), logger)+ except Exception as exc: # noqa: BLE001 - a decode OOM then fails as before+ logger.warning("video.vae_oom: scope not restored after the H3 VAE speedups: %s", exc)if "cudnn_benchmark" in speed_optims:try:from .video_minimax_h3_vae import install_audio_vae_without_cudnn_benchmarkdiff --git a/studio/backend/core/inference/video_ltx2_vae_tiles.py b/studio/backend/core/inference/video_ltx2_vae_tiles.pyindex fa7ffe946e8..927c0fe43e2 100644--- a/studio/backend/core/inference/video_ltx2_vae_tiles.py+++ b/studio/backend/core/inference/video_ltx2_vae_tiles.py@@ -14,6 +14,8 @@ import osimport typesfrom typing import Any, Optional+from .diffusion_vae_oom import clear_partial_decode+from .diffusion_vae_oom import free_first as oom_free_firstfrom .diffusion_vae_tiling import axis_weights as _axis_weightsWIDE_TILES_ENV = "UNSLOTH_VIDEO_VAE_WIDE_TILES"@@ -248,11 +250,22 @@ def tiled_decode(if (th, tw) == floor or not is_oom_error(exc):raisefailed = True+ if failed and oom_free_first(vae, z):+ # the OOM retry freed the card (rest of the pipeline off it): the same tiles again before the floor tile+ failed = False+ clear_partial_decode(vae, z) # the failed attempt's causal state goes before any retry+ try:+ dec = _decode_tiles(vae, z, temb, causal, th, tw, budgeted)+ except Exception as exc: # noqa: BLE001+ if not is_oom_error(exc):+ raise+ failed = Trueif failed:# Retried outside the handler so the failed attempt's tensors (held by the traceback) are gone first.import torchstats["oom_fallback"] += 1+ clear_partial_decode(vae, z)torch.cuda.empty_cache()th, tw = floorvae._unsloth_last_decode_tile = floordiff --git a/studio/backend/core/inference/video_vae_untiled.py b/studio/backend/core/inference/video_vae_untiled.pyindex 2522f469663..83d8498fe8f 100644--- a/studio/backend/core/inference/video_vae_untiled.py+++ b/studio/backend/core/inference/video_vae_untiled.py@@ -84,6 +84,18 @@ def _free_bytes(device: Any) -> Optional[int]:return None+def _in_chain(fn: Any, target: Any) -> bool:+ for _ in range(16):+ if fn is None:+ return False+ if fn is target:+ return True+ from .decode_layers import next_layer # noqa: PLC0415++ fn = next_layer(fn)+ return False++def _decode_device(vae: Any, z: Any) -> Any:"""Where the decode allocates: the latent's card, unless a multi-GPU split put the VAE on another CUDA card (itsboundary moves the latent there inside this wrapper). An offloaded VAE (weights on the host) keeps the latent's."""@@ -162,6 +174,11 @@ def install_untiled_decode(# Retried outside the handler so the failed attempt's tensors (held by the traceback) are gone first.stats["oom_fallback"] += 1oom_need[:] = [min([need, *oom_need])]+ from .diffusion_vae_oom import clear_partial_decode # noqa: PLC0415++ clear_partial_decode(+ vae, z+ ) # the untiled attempt's causal state goes before the tiled decodetorch.cuda.empty_cache()if logger is not None:logger.warning("video.vae_untiled: untiled decode ran out of memory; decoding tiled")@@ -172,7 +189,8 @@ def install_untiled_decode(return gated(*args, **kwargs)finally:# A failed VAE compile restores its eager decode into this slot; the guarded inner call already runs eager.- if vae.__dict__.get("decode") is not untiled_when_fits:+ # (a layer installed over this one, e.g. the OOM retry's decode scope, keeps it in its __wrapped__ chain)+ if not _in_chain(vae.__dict__.get("decode"), untiled_when_fits):vae.decode = untiled_when_fitsuntiled_when_fits._unsloth_untiled_decode = Truediff --git a/studio/backend/tests/conftest.py b/studio/backend/tests/conftest.pyindex d5486f433e0..46e0764c1e2 100644--- a/studio/backend/tests/conftest.py+++ b/studio/backend/tests/conftest.py@@ -121,6 +121,13 @@ def _skills_home_root(tmp_path_factory):_skills_home_counter = itertools.count()[email protected](autouse = True)+def _no_vae_structural_verdict_cache(monkeypatch):+ """Structural VAE fusion verdicts persist under Studio's cache root; tests must not read or write the real one+ (the cache tests turn it on against a temp dir)."""+ monkeypatch.setenv("UNSLOTH_VAE_FUSED_STRUCTURAL_CACHE", "0")++@pytest.fixture(autouse = True)def _zoo_without_gdn_decode_kernel(monkeypatch):# Whether Qwen3.5 / Qwen3-Next ask for causal-conv1d depends on the installed unsloth_zoo; pin thediff --git a/studio/backend/tests/test_diffusion_measured_placement.py b/studio/backend/tests/test_diffusion_measured_placement.pyindex 1b81a742128..ddc38d41a31 100644--- a/studio/backend/tests/test_diffusion_measured_placement.py+++ b/studio/backend/tests/test_diffusion_measured_placement.py@@ -932,7 +932,7 @@ def test_failed_release_keeps_the_copy_stream_wait(monkeypatch):monkeypatch.setattr(dm, "_release_group", boom)pipe = types.SimpleNamespace(components = {"transformer": module})- assert dm.release_resident_groups(pipe, 4) is None+ assert callable(dm.release_resident_groups(pipe, 4))assert module._unsloth_stream_state["streamed"] >= 1diff --git a/studio/backend/tests/test_diffusion_vae_oom.py b/studio/backend/tests/test_diffusion_vae_oom.pynew file mode 100644index 00000000000..78adf8e9fc7--- /dev/null+++ b/studio/backend/tests/test_diffusion_vae_oom.py@@ -0,0 +1,2643 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""diffusion_vae_oom: the offload-first OOM ladder (CPU, fake OOMs and a fake card) and the structural fused+GroupNorm(+SiLU) match."""++import types++import pytest++torch = pytest.importorskip("torch")++from core.inference import diffusion_vae_oom as O # noqa: E402+from core.inference import diffusion_vae_fused as F # noqa: E402+from core.inference import decode_layers as L # noqa: E402+++_REAL_RETURN_TO = O._return_to+++class _FakeOOM(RuntimeError):+ def __init__(self):+ super().__init__("CUDA out of memory. Tried to allocate 2.00 GiB")+++_CARD = torch.float64+++class _Budget:+ """A fake card: what is resident there (pipeline modules, the VAE weights) plus the decode's need must fit."""++ def __init__(self, cap):+ self.cap = cap++ def resident(self, modules):+ return sum(+ int(t.numel()) * t.element_size()+ for m in modules+ for t in [*m.parameters(), *m.buffers()]+ if t.dtype == _CARD+ )+++class _VaeLike(torch.nn.Module):+ """Stock-diffusers-shaped tiling attrs; a decode needs ``untiled_need`` bytes untiled, ``tiled_need`` tiled, on top+ of everything resident on the fake card (the VAE's own weights included)."""++ spatial_compression_ratio = 8++ def __init__(+ self,+ budget,+ pipe_modules,+ untiled_need,+ tiled_need,+ weight_bytes = 4096,+ ):+ super().__init__()+ self.use_tiling = False+ self.tile_sample_min_height = 256+ self.tile_sample_min_width = 256+ self.tile_sample_stride_height = 192+ self.tile_sample_stride_width = 192+ self.conv = torch.nn.Linear(weight_bytes // 8, 1, bias = False)+ self.budget, self.pipe_modules = budget, pipe_modules+ self.untiled_need, self.tiled_need = untiled_need, tiled_need+ self.seen = []+ self.cleared = 0++ def enable_tiling(self):+ self.use_tiling = True++ def clear_cache(self):+ self.cleared += 1++ def _decode(+ self,+ z,+ return_dict = True,+ ):+ self.seen.append(+ (self.use_tiling, dict(O.tile_attrs(self)), self.conv.weight.dtype == _CARD)+ )+ need = self.tiled_need if self.use_tiling else self.untiled_need+ if self.budget.resident([*self.pipe_modules, self]) + need > self.budget.cap:+ raise _FakeOOM()+ return (+ z.repeat_interleave(8, -1).repeat_interleave(8, -2) + (0.5 if self.use_tiling else 0.0),+ )+++class _Pipe:+ def __init__(self, **components):+ self.components = components+ for k, v in components.items():+ setattr(self, k, v)+++def _on_card(nbytes):+ m = torch.nn.Linear(nbytes // 8, 1, bias = False)+ return m.to(_CARD)++[email protected](autouse = True)+def _fake_card(monkeypatch):+ """The ladder only frees a CUDA card: let the meta device stand in for one, and host RAM be plentiful."""+ monkeypatch.setattr(O, "_device_of", lambda z: _FakeDevice())+ monkeypatch.setattr(O, "_host_free_mib", lambda: 1 << 20)+ monkeypatch.setattr(O, "_empty_cache", lambda device: None)+ yield+++class _FakeDevice:+ type = "cuda"+++def _whole_on(module, device):+ tensors = [*module.parameters(), *module.buffers()]+ return bool(tensors) and all(t.dtype == _CARD for t in tensors)++[email protected](autouse = True)+def _fake_moves(monkeypatch):+ monkeypatch.setattr(O, "_whole_on", _whole_on)+ monkeypatch.setattr(O, "_return_to", lambda module, device: module.to(_CARD))+ monkeypatch.setattr(O, "_to_host", lambda module: module.to(torch.float32))+ monkeypatch.setattr(O, "_whole_on_host", lambda module: all(+ t.dtype == torch.float32 for t in [*module.parameters(), *module.buffers()]), raising = False) # fmt: skip+ yield+++def _setup(+ cap,+ untiled_need,+ tiled_need,+ other = 8192,+ tiled = False,+ can_free = True,+):+ budget = _Budget(cap)+ transformer = _on_card(other) if can_free else torch.nn.Linear(4, 1)+ text_encoder = _on_card(other // 2) if can_free else torch.nn.Linear(4, 1)+ vae = _VaeLike(budget, [transformer, text_encoder], untiled_need, tiled_need)+ vae.to(_CARD)+ vae.use_tiling = tiled+ pipe = _Pipe(transformer = transformer, text_encoder = text_encoder, vae = vae, tokenizer = object())+ assert O.install(vae, pipe = pipe)+ return vae, pipe+++def _z():+ return torch.ones(1, 4, 1, 8, 8)+++def _devices(pipe):+ return {+ n: next(m.parameters()).dtype+ for n, m in pipe.components.items()+ if isinstance(m, torch.nn.Module)+ }+++def test_decode_that_fits_is_untouched():+ vae, pipe = _setup(cap = 1 << 20, untiled_need = 100, tiled_need = 10)+ before = _devices(pipe)+ out = vae._decode(_z())+ assert vae.seen == [(False, O.tile_attrs(vae), True)]+ assert torch.equal(out[0], _z().repeat_interleave(8, -1).repeat_interleave(8, -2))+ assert _devices(pipe) == before+ assert vae._unsloth_vae_oom_stats["oom"] == 0+++def test_step_a_frees_the_pipeline_and_retries_the_same_untiled_decode():+ """Fail-before: untiled needs more than is free beside the transformer and text encoder; with them on the host it+ fits. The retry is the SAME untiled decode (bit-identical), not a tiled one, and both come back."""+ # resident: 8192 + 4096 + 4096 (vae) = 16384; untiled 10000 fits only once 12288 is freed+ vae, pipe = _setup(cap = 16384 + 1000, untiled_need = 10000, tiled_need = 500)+ before = _devices(pipe)+ out = vae._decode(_z())+ assert [s[0] for s in vae.seen] == [False, False]+ assert torch.equal(out[0], _z().repeat_interleave(8, -1).repeat_interleave(8, -2))+ stats = vae._unsloth_vae_oom_stats+ assert (+ stats["recovered_by"] == "freed"+ and stats["tiled_retry"] == 0+ and stats["offload_retry"] == 0+ )+ assert _devices(pipe) == before+ assert O._SESSION_ATTR not in vae.__dict__+++def test_step_b_stock_tiles_when_freeing_is_not_enough():+ vae, pipe = _setup(cap = 16384 + 1000, untiled_need = 10**9, tiled_need = 2000)+ stock = O.tile_attrs(vae)+ out = vae._decode(_z())+ assert [s[0] for s in vae.seen] == [False, False, True]+ assert all(s[1] == stock for s in vae.seen) # never below the stock tiles+ assert vae._unsloth_vae_oom_stats["recovered_by"] == "tiled"+ assert torch.equal(out[0], _z().repeat_interleave(8, -1).repeat_interleave(8, -2) + 0.5)+ assert vae.use_tiling is False and _devices(pipe)["transformer"] == _CARD+++def test_step_c_offloads_the_vae_at_stock_tiles(monkeypatch):+ """Fail-before: even tiled, the VAE's own weights leave no room; streaming them from the host does."""+ offloads = []++ monkeypatch.setattr(+ O,+ "_apply_vae_offload",+ lambda vae, device: (offloads.append(1), vae.conv.to(torch.float32)),+ )+ monkeypatch.setattr(O, "_undo_vae_offload", lambda vae, device: vae.conv.to(_CARD))+ vae, pipe = _setup(cap = 4096 + 100, untiled_need = 10**9, tiled_need = 3000, tiled = True)+ stock = O.tile_attrs(vae)+ vae._decode(_z())+ assert vae.seen == [(True, stock, True), (True, stock, True), (True, stock, False)]+ assert vae._unsloth_vae_oom_stats["recovered_by"] == "offload" and offloads == [1]+ assert vae.conv.weight.dtype == _CARD+ assert _devices(pipe)["text_encoder"] == _CARD+++def test_streamed_decode_that_breaks_fails_on_the_oom(monkeypatch):+ """A non-OOM error inside the streamed (step c) decode, e.g. an offload hook the environment cannot run, must not+ replace the out-of-memory the render already had; the VAE still comes back."""+ monkeypatch.setattr(O, "_apply_vae_offload", lambda vae, device: None)+ monkeypatch.setattr(O, "_undo_vae_offload", lambda vae, device: None)+ vae, pipe = _setup(cap = 1, untiled_need = 10**9, tiled_need = 10**9, tiled = True)+ real = L.find(vae, "_decode", O._RETRY_LAYER).inner++ def inner(z, *a, **k):+ holder = vae.__dict__.get(O._SESSION_ATTR)+ if holder is not None and holder["session"] is not None and holder["session"].vae_offloaded:+ raise NameError("send_to_device")+ return real(z, *a, **k)++ O.uninstall(vae)+ vae._decode = inner+ O.install(vae)+ with pytest.raises(RuntimeError, match = "even with the rest of the pipeline"):+ vae._decode(_z())+++def test_every_step_fails_raises_and_restores(monkeypatch):+ monkeypatch.setattr(O.DecodeSession, "offload_vae", lambda self: False)+ vae, pipe = _setup(cap = 1, untiled_need = 10**9, tiled_need = 10**9)+ before = _devices(pipe)+ with pytest.raises(RuntimeError, match = "even with the rest of the pipeline"):+ vae._decode(_z())+ assert _devices(pipe) == before+ assert all(s[1] == O.tile_attrs(vae) for s in vae.seen)+ assert vae.use_tiling is False and vae.cleared >= 2+++def test_nothing_to_free_skips_the_identical_retry(monkeypatch):+ monkeypatch.setattr(O.DecodeSession, "offload_vae", lambda self: False)+ vae, pipe = _setup(cap = 4096 + 600, untiled_need = 10**9, tiled_need = 500, can_free = False)+ vae._decode(_z())+ assert [s[0] for s in vae.seen] == [False, True]+ assert vae._unsloth_vae_oom_stats["freed_retry"] == 0+++def test_offload_hooked_and_shared_components_stay(monkeypatch):+ """Components an offload mechanism places (accelerate / group offload) are left to it; the VAE never moves in (a)."""+ vae, pipe = _setup(cap = 16384 + 1000, untiled_need = 10**9, tiled_need = 10**9)+ pipe.components["transformer"]._hf_hook = object()+ monkeypatch.setattr(O.DecodeSession, "offload_vae", lambda self: False)+ moved = []+ real = torch.nn.Module.to++ def spy(self, *a, **k):+ if a and a[0] == torch.float32:+ moved.append(self)+ return real(self, *a, **k)++ monkeypatch.setattr(torch.nn.Module, "to", spy)+ with pytest.raises(RuntimeError):+ vae._decode(_z())+ assert moved == [pipe.components["text_encoder"]]+++def test_host_ram_short_keeps_component(monkeypatch):+ monkeypatch.setattr(O, "_host_free_mib", lambda: 1)+ session = O.DecodeSession(None, _FakeDevice())+ pipe = _Pipe(transformer = _on_card(1 << 20))+ session.free_others(pipe)+ assert session.moved == []+++def test_restore_that_runs_out_of_memory_returns_on_next_forward(monkeypatch):+ m = _on_card(64)+ session = O.DecodeSession(None, _FakeDevice())+ session.log.append(session._component_change("transformer", m))+ m.to(torch.float32)+ calls = []++ def flaky(module, device):+ calls.append(1)+ module.to(torch.float16)+ if len(calls) <= 2:+ raise _FakeOOM()+ module.to(_CARD)++ monkeypatch.setattr(O, "_return_to", flaky)+ session.restore()+ assert session.log == []+ assert next(m.parameters()).dtype == torch.float32+ # Item 4282775926: the card is still full at the next forward: the OOM surfaces and the way back STAYS+ with pytest.raises(RuntimeError, match = "still on the host.*out of memory"):+ m(torch.zeros(1, 8, dtype = _CARD))+ assert next(m.parameters()).dtype == torch.float32 and len(m._forward_pre_hooks) == 1+ assert isinstance(m.__dict__.get("_hf_hook"), O.DeferredReturn)+ m(torch.zeros(1, 8, dtype = _CARD))+ assert next(m.parameters()).dtype == _CARD and len(m._forward_pre_hooks) == 0+ assert "_hf_hook" not in m.__dict__ and O._DEFERRED_ATTR not in m.__dict__+++def _cpu_group_offload_ok() -> bool:+ """diffusers' group offloading builds on a host with no accelerator (older releases ask torch.accelerator for one+ and raise); the CPU onload below stands in for the card only where it does."""+ try:+ from diffusers.hooks import apply_group_offloading+ apply_group_offloading(+ torch.nn.Linear(2, 2), onload_device = torch.device("cpu"), offload_type = "leaf_level"+ )+ return True+ except Exception: # noqa: BLE001+ return False+++needs_cpu_group_offload = pytest.mark.skipif(+ not _cpu_group_offload_ok(), reason = "this diffusers' group offloading needs an accelerator"+)+++def _tiny_vae():+ diffusers = pytest.importorskip("diffusers")+ torch.manual_seed(0)+ return diffusers.AutoencoderKL(+ block_out_channels = (8, 16), down_block_types = ("DownEncoderBlock2D",) * 2,+ up_block_types = ("UpDecoderBlock2D",) * 2, latent_channels = 4, norm_num_groups = 4, layers_per_block = 1,+ ).eval() # fmt: skip+++@needs_cpu_group_offload+def test_vae_restore_oom_keeps_offload_hooks_and_retries(monkeypatch):+ """Item 4282775917: the VAE's weights cannot all return after a streamed decode. Before: hooks removed, weights+ stranded on the host. After: weights whole on the host WITH the offload hooks back (it still decodes, streamed),+ and its next decode brings it back once memory allows."""+ vae = _tiny_vae()+ z = torch.randn(1, 4, 8, 8)+ with torch.no_grad():+ ref = vae.decode(z).sample+ cpu = torch.device("cpu")+ monkeypatch.setattr(O, "_to_host", lambda m: m.to("cpu", torch.float32))+ full = [True]++ def ret(module, device):+ if full[0]:+ module.decoder.conv_in.to(torch.float64)+ raise _FakeOOM()+ _REAL_RETURN_TO(module, device)++ monkeypatch.setattr(O, "_return_to", ret)+ session = O.DecodeSession(vae, cpu)+ O._apply_vae_offload(vae, cpu)+ session.log.append(+ O.Change(+ "vae_offload",+ lambda: O._remove_vae_offload(vae, cpu),+ lambda: O._park_vae(vae, cpu),+ lambda retry, exc: O._defer(vae, cpu, "vae_offload", retry, False, vae),+ )+ )+ session.restore()+ monkeypatch.undo()+ assert session.log == []+ assert O._hooked(vae), "the weights are on the host: the offload hooks must stay"+ assert all(p.dtype == torch.float32 for p in vae.parameters())+ with torch.no_grad():+ assert torch.equal(vae.decode(z).sample, ref)+ assert O.settle_deferred(vae)+ assert (+ not O._hooked(vae)+ and O._DEFERRED_ATTR not in vae.__dict__+ and "_hf_hook" not in vae.__dict__+ )+ with torch.no_grad():+ assert torch.equal(vae.decode(z).sample, ref)+++def test_execution_device_resolves_the_card_while_a_component_waits_on_the_host(monkeypatch):+ """A real diffusers pipeline whose first component could not return: ``_execution_device`` must still be the+ card, and the next call must run (the component comes back before its forward). The meta device stands in for+ the host, the CPU for the card."""+ diffusers = pytest.importorskip("diffusers")++ class TinyPipe(diffusers.DiffusionPipeline):+ def __init__(self, unet, vae):+ super().__init__()+ self.register_modules(unet = unet, vae = vae)++ def __call__(self):+ x = torch.ones(1, 4, 8, 8, device = self._execution_device)+ return self.vae.decode(self.unet(x, 1).sample).sample++ torch.manual_seed(0)+ unet = diffusers.UNet2DModel(+ sample_size = 8, in_channels = 4, out_channels = 4, block_out_channels = (8, 16), layers_per_block = 1,+ norm_num_groups = 4, down_block_types = ("DownBlock2D", "DownBlock2D"), up_block_types = ("UpBlock2D",) * 2,+ ).eval() # fmt: skip+ pipe = TinyPipe(unet, _tiny_vae())+ with torch.no_grad():+ ref = pipe()+ saved = {k: v.clone() for k, v in unet.state_dict().items()}+ card = torch.device("cpu")+ monkeypatch.setattr(O, "_to_host", lambda m: m.to("meta"))+ oom = [True]++ def ret(module, device):+ if oom[0]:+ raise _FakeOOM()+ module.to_empty(device = device)+ module.load_state_dict(saved)++ monkeypatch.setattr(O, "_return_to", ret)+ session = O.DecodeSession(pipe.vae, card)+ unet.to("meta")+ session.log.append(session._component_change("unet", unet))+ session.restore()+ assert next(unet.parameters()).device.type == "meta"+ assert pipe._execution_device == card # fail-before: meta+ oom[0] = False+ with torch.no_grad():+ out = pipe()+ assert torch.equal(out, ref) and next(unet.parameters()).device == card+ assert "_hf_hook" not in unet.__dict__ and len(unet._forward_pre_hooks) == 0+++def test_uninstall_removes_compiled_guard_and_settles_deferred():+ """Item 4282775938: uninstall puts the compiled decode back (before: the guard stayed and kept retrying) and+ settles every deferred return the wrapper registered."""+ vae, pipe = _setup(cap = 1 << 20, untiled_need = 1, tiled_need = 1)+ O.uninstall(vae)++ def compiled(z, return_dict = True):+ return ("compiled",)++ compiled._unsloth_compiled_decode = True+ compiled._unsloth_eager_decode = lambda *a, **k: ("eager",)+ vae.decode = compiled+ assert O.install(vae, pipe = pipe)+ assert vae.__dict__["decode"] is not compiled and vae.__dict__["decode"]._unsloth_vae_oom_guard+ m = pipe.components["text_encoder"]+ O._defer(m, _FakeDevice(), "moved:text_encoder", lambda: None, True, vae)+ assert O._DEFERRED_ATTR in m.__dict__+ O.uninstall(vae)+ assert vae.__dict__["decode"] is compiled+ assert O._DEFERRED_ATTR not in m.__dict__ and "_hf_hook" not in m.__dict__+ assert O._RECORD_ATTR not in vae.__dict__ and "_decode" not in vae.__dict__+ assert O.install(vae, pipe = pipe) and vae.__dict__["decode"]._unsloth_vae_oom_guard+++def test_uninstall_keeps_a_deferred_return_that_still_cannot_run():+ vae, pipe = _setup(cap = 1 << 20, untiled_need = 1, tiled_need = 1)+ m = pipe.components["text_encoder"]+ O._defer(+ m, _FakeDevice(), "moved:text_encoder", lambda: (_ for _ in ()).throw(_FakeOOM()), True, vae+ )+ O.uninstall(vae)+ assert O._DEFERRED_ATTR in m.__dict__ # its only way back to the card+++def test_step_a_drops_graphs_and_repins_resident_groups(monkeypatch):+ from core.inference import diffusion_cuda_graph as G+ from core.inference import diffusion_memory as M++ events = []+ monkeypatch.setattr(+ G, "reset_all", lambda handles, strict = False: events.append(("graphs", tuple(handles)))+ )+ monkeypatch.setattr(+ M, "release_resident_groups",+ lambda pipe, need, logger = None, reason = "": (events.append(("release", need)), lambda: events.append(("repin",)))[1],+ ) # fmt: skip+ vae, pipe = _setup(cap = 16384 + 1000, untiled_need = 10000, tiled_need = 500)+ pipe._unsloth_cuda_graphs = ("h",)+ pipe.components["text_encoder"]._unsloth_resident_room = 1+ vae._decode(_z())+ assert (+ events[0] == ("graphs", ("h",)) and events[1][0] == "release" and events[-1] == ("repin",)+ )+++def test_untiled_gate_keeps_its_own_fallback():+ vae, pipe = _setup(cap = 16384 + 1000, untiled_need = 10000, tiled_need = 500)++ def gate(*a, **k):+ return None++ gate._unsloth_untiled_decode = True++ def bf16_decode(*a, **k): # the ROCm bf16 decode exposes the gate only through __wrapped__+ return gate(*a, **k)++ bf16_decode.__wrapped__ = gate+ for outer in (gate, bf16_decode):+ vae.seen.clear()+ vae.decode = outer+ # Item 4283572976: the ladder sees the gate's untiled OOM first: the card is freed and untiled retried+ vae._decode(_z())+ assert [s[0] for s in vae.seen] == [False, False]+ assert vae._unsloth_vae_oom_stats["recovered_by"] == "freed"+ # nothing to free (or still out of memory): the gate's own tiled fallback gets the original OOM+ vae, pipe = _setup(cap = 4096 + 600, untiled_need = 10**9, tiled_need = 500, can_free = False)+ vae.decode = gate+ with pytest.raises(_FakeOOM):+ vae._decode(_z())+ assert [s[0] for s in vae.seen] == [False]+++def test_kill_switch(monkeypatch):+ vae, pipe = _setup(cap = 16384 + 1000, untiled_need = 10000, tiled_need = 500)+ monkeypatch.setenv(O.OOM_RETRY_ENV, "0")+ with pytest.raises(RuntimeError, match = "out of memory"):+ vae._decode(_z()) # installed, switched off at call time: the OOM surfaces as on main+ assert len(vae.seen) == 1+ other = _VaeLike(_Budget(1), [], 1, 1)+ assert O.install(other) is False and "_decode" not in other.__dict__+++def test_non_oom_propagates():+ vae, pipe = _setup(cap = 1 << 20, untiled_need = 1, tiled_need = 1)+ O.uninstall(vae)+ vae._decode = types.MethodType(lambda self, z: (_ for _ in ()).throw(ValueError("x")), vae)+ O.install(vae)+ with pytest.raises(ValueError):+ vae._decode(_z())+++def test_uninstall_restores_class_decode_and_reinstall_keeps_pipe():+ vae, pipe = _setup(cap = 1 << 20, untiled_need = 1, tiled_need = 1)+ assert "_decode" in vae.__dict__+ O.uninstall(vae)+ assert "_decode" not in vae.__dict__+ assert O.install(vae) and O._pipe_of(vae) is pipe+++def test_no_halving_api_left():+ for name in ("halved", "MIN_TILE_LATENTS", "MIN_PSNR_DB", "oom_halving_active"):+ assert not hasattr(O, name)+++@needs_cpu_group_offload+def test_real_vae_offload_hooks_round_trip(monkeypatch):+ """Step (c)'s real mechanism on a tiny diffusers VAE (CPU onload stands in for the card): leaf-level hooks decode+ bit-identically and are fully removed afterwards, weights back where they were."""+ diffusers = pytest.importorskip("diffusers")+ torch.manual_seed(0)+ vae = diffusers.AutoencoderKL(+ block_out_channels = (8, 16), down_block_types = ("DownEncoderBlock2D",) * 2,+ up_block_types = ("UpDecoderBlock2D",) * 2, latent_channels = 4, norm_num_groups = 4, layers_per_block = 1,+ ).eval() # fmt: skip+ z = torch.randn(1, 4, 8, 8)+ monkeypatch.setattr(O, "_return_to", _REAL_RETURN_TO)+ with torch.no_grad():+ ref = vae.decode(z).sample+ cpu = torch.device("cpu")+ O._apply_vae_offload(vae, cpu)+ assert O._hooked(vae)+ with torch.no_grad():+ out = vae.decode(z).sample+ assert torch.equal(out, ref)+ O._undo_vae_offload(vae, cpu)+ assert not O._hooked(vae)+ with torch.no_grad():+ assert torch.equal(vae.decode(z).sample, ref)+++def test_compiled_decode_oom_frees_then_retries_compiled():+ """Step (a) under a compiled decode retries the compiled callable itself (same kernels), not the eager path."""+ vae, pipe = _setup(cap = 16384 + 1000, untiled_need = 10000, tiled_need = 500)+ calls = []++ def eager(z, return_dict = True):+ calls.append("eager")+ return vae._decode(z, return_dict)++ def compiled(z, return_dict = True):+ calls.append("compiled")+ if vae.budget.resident([*vae.pipe_modules, vae]) + vae.untiled_need > vae.budget.cap:+ try:+ raise _FakeOOM()+ except _FakeOOM as oom:+ raise RuntimeError("compiled graph failed") from oom+ return ("ok",)++ from core.inference import diffusion_speed as S++ O.uninstall(vae)+ vae.decode = S._guard_compiled_decode(vae, compiled, eager, None)+ assert O.install(vae)+ before = _devices(pipe)+ assert vae.decode(_z()) == ("ok",)+ assert calls == ["compiled", "compiled"]+ assert (+ vae._unsloth_vae_oom_stats["recovered_by"] == "freed_compiled" and _devices(pipe) == before+ )+++def test_compiled_decode_oom_falls_to_the_eager_ladder_sharing_the_session():+ vae, pipe = _setup(cap = 16384 + 1000, untiled_need = 10**9, tiled_need = 500)+ calls = []++ def eager(z, return_dict = True):+ calls.append("eager")+ assert O._SESSION_ATTR in vae.__dict__+ return vae._decode(z, return_dict)++ def compiled(z, return_dict = True):+ calls.append("compiled")+ raise _FakeOOM()++ from core.inference import diffusion_speed as S++ O.uninstall(vae)+ vae.decode = S._guard_compiled_decode(vae, compiled, eager, None)+ assert O.install(vae)+ before = _devices(pipe)+ out = vae.decode(_z())+ assert calls == ["compiled", "compiled", "eager"]+ assert vae._unsloth_vae_oom_stats["recovered_by"] == "tiled" and out[0].shape[-1] == 64+ assert _devices(pipe) == before and O._SESSION_ATTR not in vae.__dict__+++def test_compiled_guard_absent_without_compile():+ vae, pipe = _setup(cap = 1 << 20, untiled_need = 1, tiled_need = 1)+ assert "decode" not in vae.__dict__+++def test_is_oom_walks_the_cause_chain():+ class OutOfMemoryError(RuntimeError):+ pass++ try:+ try:+ raise OutOfMemoryError("CUDA out of memory")+ except OutOfMemoryError as inner:+ raise RuntimeError("backend compiler failed") from inner+ except RuntimeError as exc:+ wrapped = exc+ assert O.is_oom(wrapped)+ assert not O.is_oom(RuntimeError("backend compiler failed"))+++def test_is_oom_walks_exception_group_members():+ import builtins++ class Group(Exception):+ def __init__(self, msg, exceptions):+ super().__init__(msg)+ self.exceptions = tuple(exceptions)++ oom = RuntimeError("CUDA out of memory. Tried to allocate 2.00 GiB")+ groups = [Group("decode failed", [ValueError("cancelled"), oom])]+ if hasattr(builtins, "ExceptionGroup"):+ groups.append(builtins.ExceptionGroup("decode failed", [ValueError("cancelled"), oom]))+ for group in groups:+ assert O.is_oom(group)+ assert O.is_oom(Group("outer", [group]))+ assert not O.is_oom(Group("decode failed", [ValueError("cancelled"), KeyError("x")]))+ cyclic = Group("cycle", [])+ cyclic.exceptions = (cyclic,)+ assert not O.is_oom(cyclic)+++def test_speed_layer_compiled_guard_exposes_eager():+ from core.inference import diffusion_speed as S++ eager = lambda *a, **k: "eager" # noqa: E731+ guarded = S._guard_compiled_decode(+ types.SimpleNamespace(), lambda *a, **k: "compiled", eager, None+ )+ assert guarded._unsloth_compiled_decode is True and guarded._unsloth_eager_decode is eager+++class _Block(torch.nn.Module):+ def __init__(self, c):+ super().__init__()+ self.act = torch.nn.SiLU()+ self.norm1 = torch.nn.GroupNorm(4, c)+ self.conv = torch.nn.Conv2d(c, c, 3, padding = 1)+ self.norm2 = torch.nn.GroupNorm(4, c)++ def forward(self, x):+ h = self.act(self.norm1(x))+ h = self.conv(h)+ h2 = self.norm2(h)+ return x + self.act(h2) + h2+++class _Dec(torch.nn.Module):+ def __init__(self, c):+ super().__init__()+ self.block = _Block(c)+ self.norm_out = torch.nn.GroupNorm(4, c)+ self.out_act = torch.nn.SiLU()+ self.attn_norm = torch.nn.GroupNorm(4, c)++ def forward(self, x):+ x = self.block(x)+ x = x + self.attn_norm(x)+ return self.out_act(self.norm_out(x))+++class AutoencoderKLSomethingNew(torch.nn.Module):+ def __init__(self, c = 8):+ super().__init__()+ self.decoder = _Dec(c)++ def _decode(+ self,+ z,+ return_dict = True,+ ):+ return (self.decoder(z),)+++def test_dataflow_trace_pairs_only_exclusive_group_norm_silu():+ vae = AutoencoderKLSomethingNew()+ norms = F._plain_group_norms(vae.decoder)+ silus = F._plain_silus(vae.decoder)+ calls, consumed, _, info = F._trace_group_norm_silu(+ norms, silus, lambda: vae._decode(torch.randn(1, 8, 6, 6))+ )+ names = {id(m): n for n, m in vae.decoder.named_modules()}+ got = {names[id(m)]: (c, k) for m, c, k in zip(norms, calls, consumed)}+ assert got["block.norm1"] == (1, 1)+ assert got["norm_out"] == (1, 1)+ assert got["attn_norm"] == (1, 0)+ # Item 4283572932: norm2's output reaches the SiLU AND the residual: not exclusively consumed (before: (1, 1),+ # fused with its SiLU, which changed what the residual read)+ assert got["block.norm2"] == (1, 0)+ shared = {names[id(m)]: k for m, k in zip(norms, info["shared"])}+ assert shared["block.norm2"] >= 1 and shared["block.norm1"] == 0 and shared["norm_out"] == 0+++def _install_on_cpu(monkeypatch, vae):+ monkeypatch.setattr(F, "runtime_ok", lambda: True)+ monkeypatch.delenv(F.STRUCTURAL_ENV, raising = False)+ n = F.install_structural(vae)+ assert n == 4+ state = vae._unsloth_vae_structural+ # calibrate as on the first CUDA decode (CPU tensors take the reference GroupNorm, so it is exact here)+ z = torch.randn(1, 8, 6, 6)+ inner = L.find(vae, "_decode", F._STRUCTURAL_LAYER).inner+ with torch.no_grad():+ calls, consumed, ref, _ = F._trace_group_norm_silu(+ F._plain_group_norms(vae.decoder), F._plain_silus(vae.decoder), lambda: inner(z)[0]+ )+ return state, z, ref+++def test_structural_patch_is_exact_when_pairing_is_right(monkeypatch):+ vae = AutoencoderKLSomethingNew()+ state, z, ref = _install_on_cpu(monkeypatch, vae)+ norms = F._plain_group_norms(vae.decoder)+ silus = F._plain_silus(vae.decoder)+ names = {id(m): n for n, m in vae.decoder.named_modules()}+ paired = [+ names[id(m)] in ("block.norm1", "norm_out") or (False if names[id(m)] else None)+ for m in norms+ ]+ F._structural_patch(norms, paired, silus, state, None)+ with torch.no_grad():+ out = L.find(vae, "_decode", F._STRUCTURAL_LAYER).inner(z)[0]+ assert torch.allclose(out, ref, atol = 1e-6)+ assert state.calls == 4+ F._structural_unpatch(norms, silus)+ assert not any("forward" in m.__dict__ for m in (*norms, *silus))+++def test_structural_probe_rejects_a_wrong_pairing(monkeypatch):+ """norm2 looks paired to the trace but its output also feeds a residual: the probe check rejects the patch."""+ vae = AutoencoderKLSomethingNew()+ monkeypatch.setattr(F, "runtime_ok", lambda: True)+ F.install_structural(vae)+ state = vae._unsloth_vae_structural+ z = torch.randn(1, 8, 6, 6) * 3+ monkeypatch.setattr(torch.Tensor, "is_cuda", property(lambda self: True))+ monkeypatch.setattr(F, "_gn_fusable", lambda x, norm: False)+ with torch.no_grad():+ vae._decode(z)+ assert state.status == "rejected"+ assert not any("forward" in m.__dict__ for m in vae.decoder.modules())+++def test_structural_skips_measured_classes_and_kill_switch(monkeypatch):+ class AutoencoderKL(AutoencoderKLSomethingNew):+ pass++ assert F.structural_candidate(AutoencoderKL()) is False+ assert F.structural_candidate(AutoencoderKLSomethingNew()) is True+ monkeypatch.setenv(F.STRUCTURAL_ENV, "0")+ assert F.structural_candidate(AutoencoderKLSomethingNew()) is False+ monkeypatch.delenv(F.STRUCTURAL_ENV)+ vae = AutoencoderKLSomethingNew()+ vae._unsloth_decode_blocks_bypassed = True+ assert F.structural_candidate(vae) is False+++def _cuda_triton() -> bool:+ return torch.cuda.is_available() and not getattr(torch.version, "hip", None) and F.runtime_ok()+++needs_cuda = pytest.mark.skipif(not _cuda_triton(), reason = "needs NVIDIA CUDA + Triton")+++def _psnr(a, b):+ mse = (a.float() - b.float()).pow(2).mean().item()+ return float("inf") if mse == 0 else 10 * torch.log10(torch.tensor(4.0 / mse)).item()+++def _tiny(name):+ diffusers = pytest.importorskip("diffusers")+ torch.manual_seed(0)+ if name == "AutoencoderKLHunyuanImage":+ return diffusers.AutoencoderKLHunyuanImage(+ in_channels = 3, out_channels = 3, latent_channels = 8, block_out_channels = (32, 64, 64),+ layers_per_block = 1, spatial_compression_ratio = 4, sample_size = 64,+ ), (1, 8, 12, 10) # fmt: skip+ if name == "AutoencoderKLHunyuanVideo":+ return diffusers.AutoencoderKLHunyuanVideo(+ in_channels = 3, out_channels = 3, latent_channels = 4,+ down_block_types = ("HunyuanVideoDownBlock3D",) * 3, up_block_types = ("HunyuanVideoUpBlock3D",) * 3,+ block_out_channels = (32, 32, 32), layers_per_block = 1, norm_num_groups = 8,+ spatial_compression_ratio = 4, temporal_compression_ratio = 4,+ ), (1, 4, 2, 10, 12) # fmt: skip+ raise AssertionError(name)+++@needs_cuda[email protected]("name", ["AutoencoderKLHunyuanImage", "AutoencoderKLHunyuanVideo"])+def test_structural_engages_on_uncovered_class(name):+ vae, shape = _tiny(name)+ vae = vae.cuda().to(torch.bfloat16).eval()+ z = torch.randn(*shape, device = "cuda", dtype = torch.bfloat16)+ with torch.no_grad():+ ref = vae.decode(z, return_dict = False)[0].float()+ assert F.will_install(vae) and F.install(vae) > 0+ out = vae.decode(z, return_dict = False)[0].float()+ state = vae._unsloth_vae_structural+ assert state.status == "ok", vars(state)+ assert state.gn_silu > 0 and state.calls > 0+ assert _psnr(out, ref) > 35, _psnr(out, ref)+ F.uninstall(vae)+ assert "_decode" not in vae.__dict__+ assert torch.equal(vae.decode(z, return_dict = False)[0].float(), ref)+++class AutoencoderKLTileableNew(AutoencoderKLSomethingNew):+ spatial_compression_ratio = 8++ def __init__(self, c = 8):+ super().__init__(c)+ self.use_tiling = False+ self.tile_sample_min_height = 256+ self.tile_sample_min_width = 256+ self.tile_sample_stride_height = 192+ self.tile_sample_stride_width = 192+ self.decoder.proj3d = torch.nn.Conv3d(c, c, 3)++ def enable_tiling(self):+ self.use_tiling = True+++def _layers(vae):+ fn, out = vae.__dict__.get("_decode"), []+ while fn is not None and len(out) < 8:+ out.append(fn)+ fn = getattr(fn, "__wrapped__", None)+ return out+++def test_uninstall_unwraps_structural_layer_under_the_oom_wrapper(monkeypatch):+ """Item 4: with the OOM retry on top, uninstall must still remove the structural calibrator underneath (and keep+ the retry). Before the fix the calibrator stayed and its saved state was dropped."""+ monkeypatch.setattr(F, "runtime_ok", lambda: True)+ vae = AutoencoderKLTileableNew()+ assert F.install_structural(vae) > 0+ assert O.install(vae)+ assert any(getattr(f, "_unsloth_vae_fused", False) for f in _layers(vae))+ F.uninstall(vae)+ layers = _layers(vae)+ assert not any(getattr(f, "_unsloth_vae_fused", False) for f in layers), layers+ assert getattr(vae.__dict__.get("_decode"), "_unsloth_vae_oom_retry", False)+ assert L.find(vae, "_decode", F._STRUCTURAL_LAYER) is None+ with torch.no_grad():+ out = vae._decode(torch.randn(1, 8, 6, 6))+ assert out[0].shape == (1, 8, 6, 6)+++def test_rejected_structural_fusion_restores_contiguous_layout(monkeypatch):+ """Item 5: the speed layer relays 3-D decode convs to channels_last_3d because fusion is planned; a rejected+ calibration must put them back. Before the fix the weights stayed channels_last_3d."""+ from core.inference import diffusion_speed as S++ monkeypatch.delenv(S.VAE_CHANNELS_LAST_3D_ENV, raising = False)+ monkeypatch.setattr(F, "runtime_ok", lambda: True)+ vae = AutoencoderKLTileableNew()+ w = vae.decoder.proj3d.weight+ assert S._vae_channels_last_3d(vae, None, fused = True)+ assert w.is_contiguous(memory_format = torch.channels_last_3d) and not w.is_contiguous()+ F.install_structural(vae)+ monkeypatch.setattr(torch.Tensor, "is_cuda", property(lambda self: True))+ monkeypatch.setattr(F, "_gn_fusable", lambda x, norm: False)+ with torch.no_grad():+ vae._decode(torch.randn(1, 8, 6, 6) * 3)+ assert vae._unsloth_vae_structural.status == "rejected"+ assert vae.decoder.proj3d.weight.is_contiguous()+++def test_uninstall_restores_contiguous_layout(monkeypatch):+ """Uninstall undoes the channels_last_3d relayout recorded for the fused passes."""+ from core.inference import diffusion_speed as S++ monkeypatch.delenv(S.VAE_CHANNELS_LAST_3D_ENV, raising = False)+ monkeypatch.setattr(F, "runtime_ok", lambda: True)+ vae = AutoencoderKLTileableNew()+ assert S._vae_channels_last_3d(vae, None, fused = True)+ F.install_structural(vae)+ assert not vae.decoder.proj3d.weight.is_contiguous()+ F.uninstall(vae)+ assert vae.decoder.proj3d.weight.is_contiguous()+ assert "_unsloth_channels_last_modules" not in vae.__dict__+++class _CleanDec(torch.nn.Module):+ def __init__(self, c):+ super().__init__()+ self.norm_out = torch.nn.GroupNorm(4, c)+ self.act = torch.nn.SiLU()++ def forward(self, x):+ return self.act(self.norm_out(x))+++class AutoencoderKLCleanNew(torch.nn.Module):+ def __init__(self, c = 8):+ super().__init__()+ self.decoder = _CleanDec(c)++ def _decode(+ self,+ z,+ return_dict = True,+ ):+ return (self.decoder(z),)+++def test_structural_rejected_when_every_norm_falls_back(monkeypatch):+ """A correct pairing whose norms all take group_norm_act's stock fallback fused nothing, so the+ calibration must reject it (before: counted as fused and accepted)."""+ monkeypatch.setattr(F, "runtime_ok", lambda: True)+ vae = AutoencoderKLCleanNew()+ F.install_structural(vae)+ monkeypatch.setattr(torch.Tensor, "is_cuda", property(lambda self: True))+ monkeypatch.setattr(F, "_gn_fusable", lambda x, norm: False)+ with torch.no_grad():+ vae._decode(torch.randn(1, 8, 6, 6))+ state = vae._unsloth_vae_structural+ assert state.status == "rejected" and state.kernel_calls == 0+ assert not any("forward" in m.__dict__ for m in vae.decoder.modules())+++def test_kernel_calls_skip_the_temporal_stride_fallback(monkeypatch):+ """A 5-D norm that passes _gn_fusable but whose T axis cannot be flattened takes the stock reference inside+ group_norm_act; calibration must not count it as a kernel run."""+ monkeypatch.setattr(F, "_gn_fusable", lambda x, norm: True)+ norm = torch.nn.GroupNorm(4, 8)+ state = types.SimpleNamespace(kernel_calls = 0, calls = 0)+ fast = F._fast_gn_maybe_act(norm, True, state)+ x = torch.randn(1, 8, 3, 4, 5).transpose(2, 3).contiguous().transpose(2, 3)+ assert x.stride(2) != x.shape[3] * x.stride(3)+ with torch.no_grad():+ y = fast(x)+ assert state.calls == 1 and state.kernel_calls == 0+ assert torch.allclose(F._plain(y), torch.nn.functional.silu(norm(x)), atol = 1e-5)+ with pytest.raises(F.FusedActivationRead):+ y + 0+++class _CausalNew(AutoencoderKLCleanNew):+ """A causal decoder whose output depends on a persistent cache filled by earlier decodes."""++ def __init__(self, c = 8):+ super().__init__(c)+ self.cache = None++ def clear_cache(self):+ self.cache = None++ def _decode(+ self,+ z,+ return_dict = True,+ ):+ out = self.decoder(z)+ if self.cache is not None:+ out = out + self.cache.mean() # stale frames leak into the output+ self.cache = z.detach()+ return (out,)+++def test_structural_probe_does_not_leak_cache_into_the_real_decode(monkeypatch):+ """The calibration's probe decodes must not leave cache entries the real decode reads."""+ monkeypatch.setattr(F, "runtime_ok", lambda: True)+ ref_vae = _CausalNew()+ torch.manual_seed(0)+ z = torch.randn(1, 8, 6, 6) + 2.0+ with torch.no_grad():+ ref = ref_vae._decode(z)[0]+ vae = _CausalNew()+ vae.load_state_dict(ref_vae.state_dict())+ F.install_structural(vae)+ monkeypatch.setattr(torch.Tensor, "is_cuda", property(lambda self: True))+ with torch.no_grad():+ out = vae._decode(z)[0]+ assert torch.allclose(out, ref, atol = 1e-6), (out - ref).abs().max()+++def test_required_deferred_return_raises_on_any_failure():+ """Item 4283031561: a required return that fails with a non-OOM error raised nothing before, so the forward ran+ with its weights on the host. Now it raises a clear error, and the way back stays for a later retry."""+ m = torch.nn.Linear(4, 4)+ O._defer(m, _FakeDevice(), "moved:transformer", lambda: (_ for _ in ()).throw(ValueError("no such device")),+ True, None) # fmt: skip+ with pytest.raises(+ RuntimeError, match = "transformer.*still on the host.*ValueError: no such device"+ ):+ m(torch.zeros(1, 4))+ assert O._DEFERRED_ATTR in m.__dict__ and len(m._forward_pre_hooks) == 1+ v = torch.nn.Linear(4, 4)+ O._defer(+ v, _FakeDevice(), "vae_offload", lambda: (_ for _ in ()).throw(ValueError("x")), False, None+ )+ v(torch.zeros(1, 4))+ assert O._DEFERRED_ATTR in v.__dict__+++def _compiled_vae():+ vae, pipe = _setup(cap = 1 << 20, untiled_need = 1, tiled_need = 1)+ O.uninstall(vae)+ calls = []++ def compiled(z, return_dict = True):+ calls.append("compiled")+ raise _FakeOOM()++ compiled._unsloth_compiled_decode = True+ compiled._unsloth_eager_decode = lambda *a, **k: ("eager",)+ vae.decode = compiled+ assert O.install(vae, pipe = pipe)+ return vae, pipe, compiled, calls+++def test_uninstall_neutralises_a_compiled_guard_under_a_later_wrapper():+ """Item 4283031573: video.py wraps vae.decode (untiled gate, ROCm bf16) after the guard, capturing it. Before:+ uninstall saw a different outermost decode, dropped its record, and the guard kept retrying. Now the buried guard+ is a pass-through: an OOM out of the compiled decode surfaces, with no freeing and no eager retry."""+ vae, pipe, compiled, calls = _compiled_vae()+ guard = vae.__dict__["decode"]++ def later(*a, **k):+ return guard(*a, **k)++ later.__wrapped__ = guard+ vae.decode = later+ O.uninstall(vae)+ assert vae.__dict__["decode"] is later and later.__wrapped__ is compiled+ with pytest.raises(_FakeOOM):+ vae.decode(_z())+ assert calls == ["compiled"] and _devices(pipe)["transformer"] == _CARD+++def test_uninstall_restores_compiled_decode_when_outermost():+ vae, pipe, compiled, calls = _compiled_vae()+ O.uninstall(vae)+ assert vae.__dict__["decode"] is compiled++[email protected]("which", ["oom", "fused"])+def test_uninstall_keeps_an_instance_bound_decode_override(monkeypatch, which):+ """Item 4283031579: a VAE whose ``_decode`` was already an instance-bound override (types.MethodType) got the+ class method back on uninstall before (the override was popped). Now exactly the override comes back."""+ if which == "fused":+ monkeypatch.setattr(F, "runtime_ok", lambda: True)+ monkeypatch.delenv(F.STRUCTURAL_ENV, raising = False)+ vae = AutoencoderKLSomethingNew()+ else:+ vae, pipe = _setup(cap = 1 << 20, untiled_need = 1, tiled_need = 1)+ O.uninstall(vae)++ def override(self, z, *a, **k):+ return type(self)._decode(self, z, *a, **k)++ bound = types.MethodType(override, vae)+ vae._decode = bound+ if which == "fused":+ assert F.install_structural(vae) > 0+ F.uninstall(vae)+ else:+ assert O.install(vae)+ O.uninstall(vae)+ assert vae.__dict__.get("_decode") is bound+++def test_uninstall_restores_the_class_decode_when_there_was_no_override():+ vae, pipe = _setup(cap = 1 << 20, untiled_need = 1, tiled_need = 1)+ O.uninstall(vae)+ assert (+ "_decode" not in vae.__dict__+ and "decode" not in vae.__dict__+ and L.LAYERS_ATTR not in vae.__dict__+ )+++class _SlicedVae(_VaeLike):+ def decode(+ self,+ z,+ return_dict = True,+ ):+ return (torch.cat([self._decode(z[i : i + 1])[0] for i in range(z.shape[0])]),)+++def test_one_session_spans_a_sliced_batch_decode(monkeypatch):+ """Item 4283031587: with slicing, every slice's _decode used to recover on its own: move the pipeline off,+ restore it, then move it off again for the next slice. Now the public decode opens one session for the batch:+ the pipeline moves off once and comes back once, after the last slice."""+ moves = []+ monkeypatch.setattr(O, "_to_host", lambda m: (moves.append("off"), m.to(torch.float32)))+ monkeypatch.setattr(O, "_return_to", lambda m, d: (moves.append("on"), m.to(_CARD)))+ budget = _Budget(16384 + 1000)+ transformer, text_encoder = _on_card(8192), _on_card(4096)+ vae = _SlicedVae(budget, [transformer, text_encoder], 10000, 500)+ vae.to(_CARD)+ pipe = _Pipe(transformer = transformer, text_encoder = text_encoder, vae = vae)+ assert O.install(vae, pipe = pipe)+ out = vae.decode(torch.ones(3, 4, 1, 8, 8))[0]+ assert out.shape[0] == 3+ assert moves == ["off", "off", "on", "on"]+ assert [s[0] for s in vae.seen] == [False, False, False, False]+ assert _devices(pipe)["transformer"] == _CARD and O._SESSION_ATTR not in vae.__dict__+++def test_real_release_resident_groups_keeps_the_restore_of_a_partial_release(monkeypatch):+ """Item 4283219211: diffusion_memory.release_resident_groups caught a failure part way and returned None, so the+ groups it had already demoted were never re-pinned. It now returns the restore for them (the failing one too)."""+ from core.inference import diffusion_memory as M++ class G:+ def __init__(self):+ self._unsloth_resident = True+ self._unsloth_resident_bytes = 1 << 20++ groups = [G(), G(), G()]+ module = torch.nn.Linear(2, 2)+ module._unsloth_resident_room = 1+ module._unsloth_resident_device = "cpu"+ pipe = types.SimpleNamespace(components = {"transformer": module})+ demoted = []++ def release(group):+ if len(demoted) == 1:+ raise RuntimeError("copy stream failed")+ group._unsloth_resident = False+ demoted.append(group)++ repinned = []+ monkeypatch.setattr(M, "_offload_groups", lambda m: groups)+ monkeypatch.setattr(M, "_release_group", release)+ monkeypatch.setattr(+ M, "_keep_groups_resident", lambda m, room, dev, logger, only: repinned.append(set(only))+ )+ restore = M.release_resident_groups(pipe, 1 << 30)+ assert restore is not None and len(demoted) == 1+ restore()+ assert repinned == [{id(groups[2]), id(groups[1])}]+++def test_unwrapping_an_outer_layer_collapses_a_removed_inner_layer():+ """Item 4283219222: inner layer removed while the outer one wraps it, then the outer removed: the slot must go+ back to what the inner one had replaced (here: nothing, the class method), not to the dead inner dispatch."""++ class C:+ def f(self):+ return "class"++ c = C()+ inner = L.wrap(c, "f", "inner", lambda fn, layer: lambda: "inner:" + fn())+ outer = L.wrap(c, "f", "outer", lambda fn, layer: lambda: "outer:" + fn())+ assert c.f() == "outer:inner:class"+ assert L.unwrap(inner) is False and c.f() == "outer:class"+ assert L.unwrap(outer) is True+ assert "f" not in c.__dict__ and c.f() == "class"+ bound = types.MethodType(lambda self: "bound", c)+ c.f = bound+ inner = L.wrap(c, "f", "inner", lambda fn, layer: lambda: "inner:" + fn())+ outer = L.wrap(c, "f", "outer", lambda fn, layer: lambda: "outer:" + fn())+ L.unwrap(inner)+ L.unwrap(outer)+ assert c.__dict__["f"] is bound+++def test_component_whose_move_and_rollback_both_fail_gets_a_deferred_return(monkeypatch):+ """Item 4283219189: the move to the host failed part way and moving it back failed too; the component was left+ split across devices with nothing recorded. Now it is settled whole on the host with a deferred return."""+ m = _on_card(64)+ calls = []++ def to_host(module):+ calls.append("host")+ if len(calls) == 1:+ next(module.parameters()).data = next(module.parameters()).data.to(torch.float32)+ raise _FakeOOM()+ module.to(torch.float32)++ def back(module, device):+ raise _FakeOOM()++ monkeypatch.setattr(O, "_to_host", to_host)+ monkeypatch.setattr(O, "_return_to", back)+ session = O.DecodeSession(None, _FakeDevice())+ assert (+ session.apply(lambda: O._to_host(m), session._component_change("transformer", m)) is False+ )+ assert session.log == [] and all(p.dtype == torch.float32 for p in m.parameters())+ assert isinstance(m.__dict__.get(O._DEFERRED_ATTR), O.DeferredReturn)+++def test_trace_refuses_activation_fusion_for_hooked_or_inplace_reused_norms():+ """Item 4283572932, class: exclusivity is proven on the trace. A hook on the norm, or an in-place reuse of its+ output outside the SiLU, keeps the norm unpaired (GroupNorm-only fusion)."""++ class Blk(torch.nn.Module):+ def __init__(self):+ super().__init__()+ self.norm = torch.nn.GroupNorm(4, 8)+ self.act = torch.nn.SiLU()++ def forward(self, x):+ h = self.norm(x)+ y = self.act(h)+ h.mul_(2) # in-place reuse after the SiLU read it+ return y + h++ blk = Blk()+ calls, consumed, _, info = F._trace_group_norm_silu(+ [blk.norm], [blk.act], lambda: blk(torch.randn(1, 8, 4, 4))+ )+ assert calls == [1] and consumed == [0] and info["shared"][0] >= 1+ clean = torch.nn.Sequential(torch.nn.GroupNorm(4, 8), torch.nn.SiLU())+ calls, consumed, _, _ = F._trace_group_norm_silu(+ [clean[0]], [clean[1]], lambda: clean(torch.randn(1, 8, 4, 4))+ )+ assert consumed == [1]+ h = clean[0].register_forward_hook(lambda m, i, o: None)+ calls, consumed, _, _ = F._trace_group_norm_silu(+ [clean[0]], [clean[1]], lambda: clean(torch.randn(1, 8, 4, 4))+ )+ h.remove()+ assert consumed == [0]+++def _fake_kernel(+ monkeypatch,+ wrong_site = None,+ kernel_for = None,+ during = None,+):+ """group_norm_act on CPU: the reference, flagged as a kernel run; ``wrong_site`` gets a local error."""++ def fake(+ x,+ norm,+ act = True,+ in_bias = None,+ on_kernel = None,+ **_,+ ):+ y = F.group_norm_reference(x, norm, act, in_bias)+ if norm is wrong_site:+ y = y.clone()+ y[..., 0, 0] += 0.5+ if during is not None and kernel_for is not None and norm not in kernel_for:+ during()+ if kernel_for is None or norm in kernel_for:+ if on_kernel is not None:+ on_kernel()+ legacy = F.__dict__.get("_GN_KERNEL_RUNS")+ if legacy is not None:+ legacy[0] += 1+ return y++ monkeypatch.setattr(F, "group_norm_act", fake)+ monkeypatch.setattr(F, "runtime_ok", lambda: True)+ monkeypatch.delenv(F.STRUCTURAL_ENV, raising = False)+ monkeypatch.setattr(torch.Tensor, "is_cuda", property(lambda self: True))+++def test_structural_calibration_checks_every_fused_site(monkeypatch):+ """Class ask: the calibration compares each fused site against stock on the input it saw, not only the final crop.+ A site that is wrong in one corner is rejected even when the crop error stays small."""+ vae = AutoencoderKLCleanNew()+ norms = F._plain_group_norms(vae.decoder)+ _fake_kernel(monkeypatch, wrong_site = norms[0])+ F.install_structural(vae)+ with torch.no_grad():+ vae._decode(torch.randn(1, 8, 6, 6))+ state = vae._unsloth_vae_structural+ assert state.status == "rejected" and state.site_max_abs >= 0.4+ ok = AutoencoderKLCleanNew()+ _fake_kernel(monkeypatch)+ F.install_structural(ok)+ with torch.no_grad():+ ok._decode(torch.randn(1, 8, 6, 6))+ assert (+ ok._unsloth_vae_structural.status == "ok" and ok._unsloth_vae_structural.site_max_abs == 0.0+ )+++def test_structural_kernel_accounting_is_local_to_the_calibration(monkeypatch):+ """Item 4283572986: another VAE's kernel run (another thread) during this VAE's calibration used to bump the+ shared counter and make an all-fallback calibration look fused. Now only this calibration's own runs count."""+ vae = AutoencoderKLCleanNew()+ other_norm = torch.nn.GroupNorm(4, 8)+ other = F._fast_gn_maybe_act(other_norm, True, types.SimpleNamespace(kernel_calls = 0, calls = 0))+ import threading++ def concurrent():+ t = threading.Thread(target = lambda: other(torch.randn(1, 8, 4, 4)))+ t.start()+ t.join()++ _fake_kernel(monkeypatch, kernel_for = {other_norm}, during = concurrent)+ F.install_structural(vae)+ with torch.no_grad():+ vae._decode(torch.randn(1, 8, 6, 6))+ assert vae._unsloth_vae_structural.status == "rejected"+++def test_strict_restore_reports_an_incomplete_repin(monkeypatch):+ """Item 4283572942: the re-pin swallows its own failures; the VAE path restores strictly and gets an error while+ any released group is still streamed, so its settle / defer path runs and the restore is kept."""+ from core.inference import diffusion_memory as M++ class Grp:+ def __init__(self):+ self._unsloth_resident = True+ self._unsloth_resident_bytes = 1 << 20++ groups = [Grp(), Grp()]+ module = torch.nn.Linear(2, 2)+ module._unsloth_resident_room = 1+ module._unsloth_resident_device = "cpu"+ pipe = types.SimpleNamespace(components = {"transformer": module})+ monkeypatch.setattr(M, "_offload_groups", lambda m: groups)+ monkeypatch.setattr(M, "_release_group", lambda g: setattr(g, "_unsloth_resident", False))+ pinned = [0]++ def keep(m, room, dev, logger, only):+ groups[0]._unsloth_resident = True+ pinned[0] += 1++ monkeypatch.setattr(M, "_keep_groups_resident", keep)+ restore = M.release_resident_groups(pipe, 1 << 30)+ restore() # the lenient form other callers use: unchanged+ with pytest.raises(RuntimeError, match = "left 1 streamed"):+ restore(strict = True)+ got = [restore]+ session = O.DecodeSession(None, _FakeDevice())+ released = []+ monkeypatch.setattr(+ M, "release_resident_groups", lambda *a, **k: (released.append(1), restore)[1]+ )+ anchor = module+ pipe2 = types.SimpleNamespace(components = {"transformer": anchor})+ assert session._release_groups(pipe2) == 1+ session.restore()+ assert session.log == [] and O._DEFERRED_ATTR in anchor.__dict__+ del got+++def test_step_a_streams_a_resident_torchao_transformer(monkeypatch):+ """Item 4283572958: a hookless resident torchao transformer was skipped by step (a)."""+ from core.inference import diffusion_memory as M++ vae, pipe = _setup(cap = 16384 + 1000, untiled_need = 10000, tiled_need = 500)+ transformer = pipe.components["transformer"]+ pipe.transformer = transformer+ monkeypatch.setattr(O, "_holds_torchao", lambda m: m is transformer)+ events = []++ def hook(+ p,+ device,+ logger = None,+ ):+ events.append("hook")+ transformer._unsloth_resident_room = 1+ return True++ def release(+ p,+ need,+ logger = None,+ reason = "",+ ):+ events.append("release")+ transformer.to(torch.float32)+ return lambda strict = False: (events.append("repin"), transformer.to(_CARD))++ monkeypatch.setattr(M, "hook_resident_denoiser", hook)+ monkeypatch.setattr(M, "release_resident_groups", release)+ vae._decode(_z())+ assert events == ["hook", "release", "repin"]+ assert vae._unsloth_vae_oom_stats["recovered_by"] == "freed"+++def test_vae_offload_respects_host_ram(monkeypatch):+ """Item 4283572966: step (c) creates host copies of the VAE weights; with too little host RAM it is skipped."""+ applied = []+ monkeypatch.setattr(O, "_apply_vae_offload", lambda v, d: applied.append(1))+ monkeypatch.setattr(O, "_host_free_mib", lambda: 1)+ session = O.DecodeSession(_VaeLike(_Budget(1), [], 1, 1), _FakeDevice())+ assert session.offload_vae() is False and applied == []+ monkeypatch.setattr(O, "_host_free_mib", lambda: 1 << 20)+ assert session.offload_vae() is True and applied == [1]+++def test_wide_tiles_free_the_card_before_falling_back_to_stock_tiles(monkeypatch):+ """Item 4283572976: Studio's wide tiles caught their own OOM and fell back to the smaller stock tiles before the+ OOM retry could offload anything. Now the retry frees the card first and the wide tiles decode."""+ diffusers = pytest.importorskip("diffusers")+ from core.inference import diffusion_vae_tiling as vt++ monkeypatch.delenv(vt.WIDE_TILES_ENV, raising = False)+ monkeypatch.setenv(vt.MAX_TILE_ENV, "1")+ monkeypatch.setattr(vt, "floor_shortfall", lambda vae, z: None)+ torch.manual_seed(0)+ vae = diffusers.AutoencoderKLQwenImage(+ base_dim = 8, z_dim = 4, dim_mult = [1, 2, 4, 4], num_res_blocks = 1, attn_scales = [],+ temperal_downsample = [False, True, True], latents_mean = [0.0] * 4, latents_std = [1.0] * 4,+ ).eval() # fmt: skip+ vae.enable_tiling()+ assert vt.install(vae)+ z = torch.randn(1, 4, 1, 48, 48, generator = torch.Generator().manual_seed(1))+ freed, fallbacks = [], []+ monkeypatch.setattr(vt, "_log_stock_fallback", lambda vae, reason: fallbacks.append(reason))+ monkeypatch.setattr(O, "_device_of", lambda z: _FakeDevice())+ monkeypatch.setattr(+ O.DecodeSession,+ "free_others",+ lambda self, pipe: (freed.append(1), setattr(self, "freed", True), 1)[2],+ )+ real = vae.decoder.forward++ def decoder(x, *a, **k):+ if not freed:+ raise _FakeOOM()+ return real(x, *a, **k)++ monkeypatch.setattr(vae.decoder, "forward", decoder)+ assert O.install(vae)+ with torch.no_grad():+ vae.decode(z)+ assert freed == [1]+ assert (+ fallbacks == []+ ) # the wide tiles decoded after the card was freed; no fallback to the stock tiles+++def test_ltx2_tiles_free_the_card_before_the_floor_tile(monkeypatch):+ pytest.importorskip("diffusers") # tiled_decode builds a diffusers DecoderOutput+ from core.inference import video_ltx2_vae_tiles as lt++ tried = []+ freed = []++ def tiles(vae, z, temb, causal, th, tw, budgeted):+ tried.append((th, tw))+ if (th, tw) != (16, 16) and not freed:+ raise _FakeOOM()+ return torch.zeros(1)++ monkeypatch.setattr(lt, "_decode_tiles", tiles)+ monkeypatch.setattr(lt, "_plan", lambda vae, z: ((32, 32), True))+ monkeypatch.setattr(lt, "oom_free_first", lambda vae, z: (freed.append(1), True)[1])+ vae = types.SimpleNamespace()+ lt.tiled_decode(vae, torch.zeros(1, 4, 2, 64, 64), return_dict = False)+ assert tried == [(32, 32), (32, 32)] and freed == [1]+++def test_ladder_is_the_outermost_decode_layer_in_both_pipelines():+ """Class ask (c): in the load order of diffusion.py and video.py the OOM retry is installed after every decode+ wrapper that can fall back on an OOM, so its scope is the outermost decode layer; inner fallbacks (wide tiles,+ LTX-2 tiles, untiled gate) call free_first before going cheaper."""+ import inspect+ import re++ from core.inference import diffusion as D+ from core.inference import video as V+ from core.inference import diffusion_vae_tiling as vt+ from core.inference import video_ltx2_vae_tiles as lt++ image = inspect.getsource(D)+ assert image.index("install_vae_oom_retry(") > image.index(+ "install_wide_vae_tiles(getattr(pipe"+ )+ video = inspect.getsource(V)+ oom = video.index("install_vae_oom_retry(")+ for wrapper in (+ "install_ltx2_vae_tiles(",+ "install_untiled_decode(",+ "install_rocm_vae_bf16_decode(pipe",+ ):+ assert video.index(wrapper) < oom, wrapper+ assert re.search(r"oom_free_first\(self, z\)", inspect.getsource(vt))+ assert "oom_free_first(vae, z)" in inspect.getsource(lt.tiled_decode)+++def test_scope_stays_outermost_over_the_untiled_gate(monkeypatch):+ """The untiled gate re-asserts its slot after each call; with the OOM scope installed over it, it must keep the+ scope (its __wrapped__ chain reaches the gate) instead of dropping it."""+ from core.inference import video_vae_untiled as U++ vae, pipe = _setup(cap = 1 << 20, untiled_need = 1, tiled_need = 1)+ O.uninstall(vae)+ vae.use_tiling = True+ vae.decode = lambda z, return_dict = True: vae._decode(z, return_dict)+ monkeypatch.setattr(U, "_enabled", lambda: True)+ family = next(iter(U._BYTES_PER_LATENT_PIXEL))+ monkeypatch.setattr(U, "_free_bytes", lambda d: 0)+ assert U.install_untiled_decode(types.SimpleNamespace(vae = vae), family)+ gate = vae.__dict__["decode"]+ assert O.install(vae, pipe = pipe)+ scope = vae.__dict__["decode"]+ assert getattr(scope, "_unsloth_vae_oom_scope", False) and scope.__wrapped__ is gate+ vae.decode(_z())+ assert vae.__dict__["decode"] is scope # still outermost after a call+++def test_tiny_crop_pairs_like_the_full_crop():+ """The pairing traced on the smallest calibration crop equals the one traced on a full-size latent."""+ vae = AutoencoderKLSomethingNew()+ norms, silus = F._plain_group_norms(vae.decoder), F._plain_silus(vae.decoder)+ z = torch.randn(1, 8, 40, 40)+ with torch.no_grad():+ full = F._trace_group_norm_silu(norms, silus, lambda: vae._decode(z))+ for side in (F._PROBE_LATENTS, 8, 2):+ tiny = F._trace_group_norm_silu(norms, silus, lambda: vae._decode(z[..., :side, :side]))+ assert full[0] == tiny[0] and full[1] == tiny[1], side+ assert [s > 0 for s in full[3]["shared"]] == [s > 0 for s in tiny[3]["shared"]], side+ assert F._probe_latent(z).shape[-1] == F._PROBE_LATENTS+++def _calibrated(monkeypatch, tmp_path, vae, **kw):+ monkeypatch.setenv(F.STRUCTURAL_CACHE_ENV, "1")+ monkeypatch.setenv("UNSLOTH_DIFFUSION_COMPILE_CACHE_DIR", str(tmp_path))+ _fake_kernel(monkeypatch, **kw)+ traces = []+ real = F._trace_group_norm_silu+ monkeypatch.setattr(+ F, "_trace_group_norm_silu", lambda *a, **k: (traces.append(1), real(*a, **k))[1]+ )+ F.install_structural(vae)+ with torch.no_grad():+ vae._decode(torch.randn(1, 8, 6, 6))+ return traces+++def test_structural_verdict_is_cached_and_reused(monkeypatch, tmp_path):+ first = AutoencoderKLCleanNew()+ assert _calibrated(monkeypatch, tmp_path, first) == [1]+ assert first._unsloth_vae_structural.status == "ok" and not first._unsloth_vae_structural.cached+ assert len(list((tmp_path / "vae_structural").glob("*.json"))) == 1+ second = AutoencoderKLCleanNew()+ assert _calibrated(monkeypatch, tmp_path, second) == [] # no trace, no checks+ st = second._unsloth_vae_structural+ assert st.status == "ok" and st.cached and st.gn_silu == first._unsloth_vae_structural.gn_silu+ assert any(getattr(m.forward, "_unsloth_vae_fused", False) for m in F._plain_group_norms(second.decoder)+ if "forward" in m.__dict__) or any("forward" in m.__dict__ for m in second.decoder.modules()) # fmt: skip+++def test_stale_cache_key_traces_again(monkeypatch, tmp_path):+ """Another torch / triton / config is another key: the old verdict is not used and the VAE is traced again."""+ assert _calibrated(monkeypatch, tmp_path, AutoencoderKLCleanNew()) == [1]+ monkeypatch.setattr(torch, "__version__", "0.0.0-other")+ assert _calibrated(monkeypatch, tmp_path, AutoencoderKLCleanNew()) == [1]+ assert len(list((tmp_path / "vae_structural").glob("*.json"))) == 2+++def test_corrupt_or_mismatched_cache_entry_traces_again(monkeypatch, tmp_path):+ assert _calibrated(monkeypatch, tmp_path, AutoencoderKLCleanNew()) == [1]+ (entry,) = (tmp_path / "vae_structural").glob("*.json")+ entry.write_text('{"status": "ok", "paired": [true]}') # wrong key / length+ assert _calibrated(monkeypatch, tmp_path, AutoencoderKLCleanNew()) == [1]+++def test_cached_rejection_skips_fusion(monkeypatch, tmp_path):+ bad = AutoencoderKLCleanNew()+ norms = F._plain_group_norms(bad.decoder)+ assert _calibrated(monkeypatch, tmp_path, bad, wrong_site = norms[0]) == [1]+ assert bad._unsloth_vae_structural.status == "rejected"+ again = AutoencoderKLCleanNew()+ assert _calibrated(monkeypatch, tmp_path, again) == []+ assert again._unsloth_vae_structural.status == "rejected" and not any(+ "forward" in m.__dict__ for m in again.decoder.modules()+ )+++def test_structural_cache_kill_switch(monkeypatch, tmp_path):+ assert _calibrated(monkeypatch, tmp_path, AutoencoderKLCleanNew()) == [1]+ monkeypatch.setenv(F.STRUCTURAL_CACHE_ENV, "0")+ traces = []+ real = F._trace_group_norm_silu+ monkeypatch.setattr(+ F, "_trace_group_norm_silu", lambda *a, **k: (traces.append(1), real(*a, **k))[1]+ )+ vae = AutoencoderKLCleanNew()+ F.install_structural(vae)+ with torch.no_grad():+ vae._decode(torch.randn(1, 8, 6, 6))+ assert traces == [1]++[email protected]("reuse", [False, True])+def test_fused_activation_mark_is_consumed_by_the_matched_silu(reuse):+ """Item 4285009704: the mark on a fused GroupNorm+SiLU output used to survive the SiLU that skipped, so a+ following SiLU (or the same module applied again) skipped too. It is consumed by the matched SiLU now."""+ norm = torch.nn.GroupNorm(4, 8)+ first, second = torch.nn.SiLU(), torch.nn.SiLU()+ if reuse:+ second = first+ state = types.SimpleNamespace(kernel_calls = 0, calls = 0)+ fast = F._fast_gn_maybe_act(norm, True, state)+ p1, p2 = F._silu_passthrough(first), F._silu_passthrough(second)+ x = torch.randn(1, 8, 4, 4)+ with torch.no_grad():+ ref = torch.nn.functional.silu(torch.nn.functional.silu(norm(x)))+ got = p2(p1(fast(x)))+ assert torch.allclose(got, ref, atol = 1e-6)+++def test_fused_activation_mark_only_for_the_traced_silu():+ """The mark names the SiLU module(s) the trace saw read the norm; another SiLU never consumes it (its read is a+ foreign read: the decode is redone without activation fusion)."""+ norm = torch.nn.GroupNorm(4, 8)+ matched, other = torch.nn.SiLU(), torch.nn.SiLU()+ state = types.SimpleNamespace(kernel_calls = 0, calls = 0)+ fast = F._fast_gn_maybe_act(norm, True, state, [matched])+ x = torch.randn(1, 8, 4, 4)+ with torch.no_grad():+ with pytest.raises(F.FusedActivationRead):+ F._silu_passthrough(other)(fast(x))+ y = fast(x)+ assert torch.allclose(+ F._silu_passthrough(matched)(y), torch.nn.functional.silu(norm(x)), atol = 1e-6+ )+++class _DoubleSiluDec(torch.nn.Module):+ def __init__(self, reuse):+ super().__init__()+ self.norm = torch.nn.GroupNorm(4, 8)+ self.act = torch.nn.SiLU()+ self.act2 = self.act if reuse else torch.nn.SiLU()+ self.conv = torch.nn.Conv2d(8, 8, 1)++ def forward(self, x):+ return self.conv(self.act2(self.act(self.norm(x))))+++class AutoencoderKLDoubleSilu(torch.nn.Module):+ def __init__(self, reuse):+ super().__init__()+ self.decoder = _DoubleSiluDec(reuse)++ def _decode(+ self,+ z,+ return_dict = True,+ ):+ return (self.decoder(z),)++[email protected]("reuse", [False, True])+def test_structural_double_silu_matches_stock(monkeypatch, reuse):+ vae = AutoencoderKLDoubleSilu(reuse)+ z = torch.randn(1, 8, 6, 6)+ with torch.no_grad():+ ref = vae._decode(z)[0]+ _fake_kernel(monkeypatch)+ assert F.install_structural(vae) == 1+ with torch.no_grad():+ vae._decode(z)+ out = vae._decode(z)[0]+ assert vae._unsloth_vae_structural.status == "ok" and vae._unsloth_vae_structural.gn_silu == 1+ assert torch.allclose(out, ref, atol = 1e-6)+++def test_every_decode_rewrap_site_restores_the_oom_scope():+ """Item 4285009695, class: every function in the image / video backends that (directly or through any helper)+ reassigns a VAE's ``decode`` must re-assert the OOM scope afterwards (install it, or ensure_outermost). The+ rewrap helpers are found by AST: any function assigning ``.decode`` or calling such a function, to a fixpoint."""+ import ast+ import pathlib++ root = pathlib.Path(O.__file__).parent+ skip = {"diffusion_vae_oom.py", "decode_layers.py", "media_decode_phase.py"}+ ensure = {"install_vae_oom_retry", "ensure_outermost"}+ funcs = {}+ for f in root.glob("*.py"):+ if f.name in skip:+ continue+ tree = ast.parse(f.read_text())+ for node in tree.body:+ if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)):+ funcs[(f.name, node.name)] = node+ if isinstance(node, ast.ClassDef):+ for b in node.body:+ if isinstance(b, (ast.FunctionDef, ast.AsyncFunctionDef)):+ funcs[(f.name, f"{node.name}.{b.name}")] = b++ def called(c):+ fn = c.func+ return (+ fn.id+ if isinstance(fn, ast.Name)+ else fn.attr+ if isinstance(fn, ast.Attribute)+ else None+ )++ dirty = set()+ for k, node in funcs.items():+ for n in ast.walk(node):+ if isinstance(n, ast.Assign) and any(+ isinstance(t, ast.Attribute) and t.attr == "decode" for t in n.targets+ ):+ dirty.add(k)+ assert ("diffusion_speed.py", "_compile_vae_decode") in dirty or any(+ k[0] == "diffusion_speed.py" for k in dirty+ )+ changed = True+ while changed:+ changed = False+ names = {k[1].split(".")[-1] for k in dirty}+ for k, node in funcs.items():+ if k in dirty:+ continue+ last = ens = 0+ for n in ast.walk(node):+ if isinstance(n, ast.Call):+ c = called(n)+ if c in names and c != k[1].split(".")[-1]:+ last = max(last, n.lineno)+ if c in ensure:+ ens = max(ens, n.lineno)+ if last and ens <= last:+ dirty.add(k)+ changed = True+ leaks = sorted(k for k in dirty if k[0] in ("diffusion.py", "video.py"))+ assert leaks == [], leaks+++class _PublicVae(_VaeLike):+ def decode(+ self,+ z,+ return_dict = True,+ ):+ return self._decode(z, return_dict)+++def _setup_public(cap, untiled_need, tiled_need):+ budget = _Budget(cap)+ transformer, text_encoder = _on_card(8192), _on_card(4096)+ vae = _PublicVae(budget, [transformer, text_encoder], untiled_need, tiled_need)+ vae.to(_CARD)+ pipe = _Pipe(transformer = transformer, text_encoder = text_encoder, vae = vae)+ assert O.install(vae, pipe = pipe)+ assert getattr(vae.__dict__.get("decode"), "_unsloth_vae_oom_scope", False)+ return vae, pipe+++def test_ensure_outermost_puts_the_scope_over_a_later_compiled_guard():+ """Item 4285009695: a deferred speed profile compiles vae.decode after load; its guard wraps the scope (which the+ graph traces as a pass-through) and re-raises OOMs. ensure_outermost puts a fresh scope back on top, which runs+ the ladder on a compiled OOM."""+ vae, pipe = _setup_public(cap = 16384 + 1000, untiled_need = 10000, tiled_need = 500)+ scoped = vae.decode+ calls = []++ def compiled(z, return_dict = True):+ calls.append("compiled")+ if vae.budget.resident([*vae.pipe_modules, vae]) + vae.untiled_need > vae.budget.cap:+ raise _FakeOOM()+ return ("ok",)++ def guard(*a, **k): # the speed layer's guard: re-raises OOMs+ return compiled(*a, **k)++ guard._unsloth_compiled_decode = True+ guard._unsloth_eager_decode = scoped+ vae.decode = guard+ with pytest.raises(_FakeOOM):+ vae.decode(_z()) # before: the compiled OOM escapes the ladder+ assert O.ensure_outermost(vae)+ top = vae.__dict__["decode"]+ assert getattr(top, "_unsloth_vae_oom_scope", False) and top.__wrapped__ is guard+ calls.clear()+ assert vae.decode(_z()) == ("ok",)+ assert calls == ["compiled", "compiled"]+ assert O.ensure_outermost(vae) and vae.__dict__["decode"] is top+++def test_scope_heals_itself_after_a_rewrap_drops_it():+ """A compile fallback restoring its eager decode (or any rewrap) can drop the scope at run time; the next+ _decode reached without it reinstalls it on top."""+ vae, pipe = _setup_public(cap = 1 << 20, untiled_need = 1, tiled_need = 1)+ del vae.decode+ vae.decode(_z())+ assert getattr(vae.__dict__.get("decode"), "_unsloth_vae_oom_scope", False)+++def test_structural_traces_on_two_threads_keep_their_own_reader():+ """Item 4285966457: the reader callback was one class attribute, so a second thread's trace finishing while the+ first was mid-run cleared it, and the first trace missed a residual read of its norm output (a false exclusive+ pairing). The callback is per thread and traces run one at a time."""+ import threading++ shared_norm, shared_act = torch.nn.GroupNorm(4, 8), torch.nn.SiLU()+ other_norm, other_act = torch.nn.GroupNorm(4, 8), torch.nn.SiLU()+ paused, resume = threading.Event(), threading.Event()++ def run_shared():+ x = torch.randn(1, 8, 4, 4)+ h = shared_norm(x)+ paused.set()+ resume.wait(+ timeout = 2+ ) # the other thread's trace runs to completion here (before: unlocked)+ return shared_act(h) + h++ result = {}++ def a():+ result["a"] = F._trace_group_norm_silu([shared_norm], [shared_act], run_shared)++ def b():+ paused.wait(timeout = 2)+ result["b"] = F._trace_group_norm_silu(+ [other_norm], [other_act], lambda: other_act(other_norm(torch.randn(1, 8, 4, 4)))+ )+ resume.set()++ ta, tb = threading.Thread(target = a), threading.Thread(target = b)+ ta.start()+ tb.start()+ ta.join(10)+ tb.join(10)+ calls_a, consumed_a, _, info_a = result["a"]+ assert calls_a == [1] and consumed_a == [0] and info_a["shared"] == [1]+ calls_b, consumed_b, _, _ = result["b"]+ assert calls_b == [1] and consumed_b == [1]+++class _WideResidualDec(torch.nn.Module):+ """Reads its norm output only through the SiLU on small inputs (the calibration crop); above a width threshold it+ also adds the raw norm output back (a residual that only appears at full shape)."""++ def __init__(self, threshold = 16):+ super().__init__()+ self.threshold = threshold+ self.norm = torch.nn.GroupNorm(4, 8)+ self.act = torch.nn.SiLU()+ self.norm2 = torch.nn.GroupNorm(4, 8)+ self.act2 = torch.nn.SiLU()+ self.conv = torch.nn.Conv2d(8, 8, 3, padding = 1)++ def forward(self, x):+ h = self.norm(x)+ out = self.conv(self.act(h))+ if x.shape[-1] > self.threshold:+ out = out + h # the raw norm output, at full shape only+ g = self.norm2(h)+ else:+ g = self.norm2(out)+ return self.act2(g) + out+++class AutoencoderKLWideResidual(torch.nn.Module):+ def __init__(self):+ super().__init__()+ self.decoder = _WideResidualDec()++ def _decode(+ self,+ z,+ return_dict = True,+ ):+ return (self.decoder(z),)+++def test_fused_activation_is_exact_when_full_shape_dataflow_differs_from_the_crop(monkeypatch):+ """Item 4285966466: the crop trace proves the SiLU is the norm's only reader, but at full width the decoder also+ adds the raw norm output back and feeds it to another norm. The fused output gives only the paired SiLU the+ activated values; every other reader gets the stock norm output, so the decode matches stock at any shape."""+ vae = AutoencoderKLWideResidual()+ z_small, z_wide = torch.randn(1, 8, 12, 12), torch.randn(1, 8, 12, 40)+ with torch.no_grad():+ ref_wide = vae._decode(z_wide)[0]+ ref_small = vae._decode(z_small)[0]+ _fake_kernel(monkeypatch)+ F.install_structural(vae)+ with torch.no_grad():+ first = vae._decode(z_wide)[0]+ state = vae._unsloth_vae_structural+ assert state.status == "ok" and state.act_off and state.gn_silu == 0 and state.gn == 2+ assert state.foreign_reads >= 1+ out_wide = vae._decode(z_wide)[0]+ out_small = vae._decode(z_small)[0]+ assert torch.allclose(first, ref_wide, atol = 1e-6)+ assert torch.allclose(out_wide, ref_wide, atol = 1e-6)+ assert torch.allclose(out_small, ref_small, atol = 1e-6)+ small_first = (+ AutoencoderKLWideResidual()+ ) # a VAE only ever decoded at the crop's dataflow keeps both fused+ small_first.load_state_dict(vae.state_dict())+ F.install_structural(small_first)+ with torch.no_grad():+ small_first._decode(z_small)+ assert small_first._unsloth_vae_structural.gn_silu == 2+++def test_fused_output_read_by_another_norm_aborts():+ norm, norm2 = torch.nn.GroupNorm(4, 8), torch.nn.GroupNorm(4, 8)+ state = types.SimpleNamespace(kernel_calls = 0, calls = 0)+ fast, fast2 = F._fast_gn_maybe_act(norm, True, state), F._fast_gn_maybe_act(norm2, False, state)+ x = torch.randn(1, 8, 4, 4)+ with torch.no_grad():+ with pytest.raises(F.FusedActivationRead):+ fast2(fast(x))+ y = fast(x)+ with pytest.raises(F.FusedActivationRead):+ torch.cat([y, y])+ assert y.shape == norm(x).shape and y.dtype == x.dtype+++def test_inplace_paired_silu_leaves_the_activated_values_for_later_readers():+ norm, act = torch.nn.GroupNorm(4, 8), torch.nn.SiLU(inplace = True)+ fast = F._fast_gn_maybe_act(norm, True, types.SimpleNamespace(kernel_calls = 0, calls = 0), [act])+ x = torch.randn(1, 8, 4, 4)+ with torch.no_grad():+ n = norm(x)+ a = torch.nn.functional.silu(n, inplace = True)+ ref = a + n+ y = fast(x)+ got = F._silu_passthrough(act)(y) + y+ assert torch.allclose(got, ref, atol = 1e-6)+++def test_structural_cache_key_follows_the_checkpoint_dtype_and_hooks():+ """Item 4285966472: the key ignored the weights, so one checkpoint's verdict authorised another of the same+ class / config. It now hashes every decoder tensor's name / shape / dtype and leading values, plus hook state."""+ torch.manual_seed(0)+ a = AutoencoderKLCleanNew()+ b = AutoencoderKLCleanNew()+ with torch.no_grad():+ b.decoder.norm_out.weight.copy_(a.decoder.norm_out.weight)+ b.decoder.norm_out.bias.copy_(a.decoder.norm_out.bias)++ def key(v):+ return F.structural_cache_key(+ v, F._plain_group_norms(v.decoder), F._plain_silus(v.decoder), None+ )++ assert key(a) is not None and key(a) == key(b)+ with torch.no_grad():+ b.decoder.norm_out.weight.add_(0.01)+ assert key(a) != key(b)+ c = AutoencoderKLCleanNew()+ c.load_state_dict(a.state_dict())+ k_a = key(a)+ c.to(torch.float64)+ assert key(c) != k_a+ c = AutoencoderKLCleanNew()+ c.load_state_dict(a.state_dict())+ assert key(c) == k_a+ h = c.decoder.norm_out.register_forward_hook(lambda *args: None)+ assert key(c) != k_a # hooks change what the trace may prove+ h.remove()+ meta = AutoencoderKLCleanNew().to("meta")+ assert key(meta) is None+++def test_strict_graph_reset_reports_a_handle_that_kept_its_graph():+ from core.inference import diffusion_cuda_graph as G++ class Handle:+ def __init__(+ self,+ fail = False,+ keep = False,+ ):+ self.cache, self.fail, self.keep = {"shape": "graph"}, fail, keep++ def reset(self):+ if self.fail:+ raise RuntimeError("reset failed")+ if not self.keep:+ self.cache.clear()+ return self++ G.reset_all([Handle(fail = True)])+ G.reset_all([Handle(), Handle()], strict = True)+ with pytest.raises(RuntimeError, match = "captured CUDA graph"):+ G.reset_all([Handle(), Handle(fail = True)], strict = True)+ with pytest.raises(RuntimeError, match = "captured CUDA graph"):+ G.reset_all([Handle(keep = True)], strict = True)+++def test_step_a_moves_no_weight_when_a_graph_could_not_be_dropped(monkeypatch):+ """Item 4285966487: reset_all swallowed a handle's failure and step (a) then moved weights under a graph that+ could still replay old addresses. A strict reset failure now stops step (a) before any move; the ladder goes on+ (stock tiles here)."""+ from core.inference import diffusion_cuda_graph as G++ class Stuck:+ cache = {"k": "graph"}++ def reset(self):+ raise RuntimeError("cannot drop")++ moves = []+ monkeypatch.setattr(O, "_to_host", lambda m: (moves.append(m), m.to(torch.float32)))+ vae, pipe = _setup(cap = 16384 + 1000 + 4096, untiled_need = 10000, tiled_need = 500)+ pipe._unsloth_cuda_graphs = (Stuck(),)+ out = vae.decode(_z())[0] if "decode" in vae.__dict__ else vae._decode(_z())[0]+ assert moves == []+ assert _devices(pipe)["transformer"] == _CARD and _devices(pipe)["text_encoder"] == _CARD+ assert vae._unsloth_vae_oom_stats.get("recovered_by") == "tiled"+ assert out is not None+++def test_scope_finds_a_compiled_guard_under_the_rocm_bf16_decode(monkeypatch):+ """Items 4285966482 / 4286353651: ROCm order is compile, then the bf16 decode wraps the compiled guard, then the+ OOM retry. The scope is the outermost layer (above the bf16 cast, whose own fp32 output cast may run out of+ memory too) and reaches the compiled guard under it: a compiled OOM frees the card and retries the whole decode,+ compiled."""+ from core.inference import diffusion_device as D+ from core.inference import diffusion_speed as S++ budget = _Budget(16384 + 1000)+ transformer, text_encoder = _on_card(8192), _on_card(4096)+ vae = _PublicVae(budget, [transformer, text_encoder], 10000, 500)+ vae.to(_CARD)+ vae.dtype = torch.float32+ pipe = _Pipe(transformer = transformer, text_encoder = text_encoder, vae = vae)+ calls = []++ def compiled(z, return_dict = True):+ calls.append("compiled")+ if budget.resident([transformer, text_encoder, vae]) + 10000 > budget.cap:+ raise _FakeOOM()+ return (z,)++ vae.decode = S._guard_compiled_decode(vae, compiled, vae.decode, None)+ guard = vae.__dict__["decode"]+ monkeypatch.setattr(D, "_rocm_bf16_decode_arch", lambda torch_, target: "gfx1151")+ monkeypatch.setenv(D.VAE_BF16_DECODE_ENV, "autocast")+ target = types.SimpleNamespace(device = "cuda", backend = "rocm")+ assert D.install_rocm_vae_bf16_decode(pipe, target) == "autocast"+ bf16 = vae.__dict__["decode"]+ assert O.install(vae, pipe = pipe)+ top = vae.__dict__["decode"]+ assert getattr(top, "_unsloth_vae_oom_scope", False) and top.__wrapped__ is bf16+ assert bf16._unsloth_decode_slot.decode is guard+ assert O.ensure_outermost(vae) and vae.__dict__["decode"] is top+ import warnings++ with warnings.catch_warnings():+ warnings.simplefilter("ignore")+ out = vae.decode(_z())+ assert calls == ["compiled", "compiled"] and out[0].dtype == torch.float32+++class _CompileFailed(RuntimeError):+ pass+++def _fp16_shaped(+ vae,+ budget,+ own_need,+ oom_once = None,+):+ """An fp16-decode-shaped dtype wrapper: calls through a mutable slot, then allocates for its own non-finite check+ (``own_need`` bytes on top of what is resident), which can run out of memory itself."""+ slot = types.SimpleNamespace(decode = vae.decode)++ def fp16_decode(z, *a, **k):+ out = slot.decode(z, *a, **k)+ if budget.resident([*vae.pipe_modules, vae]) + own_need > budget.cap:+ raise _FakeOOM()+ return out++ fp16_decode._unsloth_decode_slot = slot+ vae.decode = fp16_decode+ return fp16_decode, slot+++def test_dtype_wrapper_own_allocation_oom_reaches_the_scope():+ """Item 4286353651: with the scope inside the dtype wrapper's slot, an OOM in the wrapper's own work (the fp16+ isfinite check, the ROCm fp32 output cast) escaped it. The scope is outermost now: it frees the card and the+ whole decode, wrapper included, succeeds."""+ budget = _Budget(16384 + 1000)+ transformer, text_encoder = _on_card(8192), _on_card(4096)+ vae = _PublicVae(budget, [transformer, text_encoder], 1, 1)+ vae.to(_CARD)+ pipe = _Pipe(transformer = transformer, text_encoder = text_encoder, vae = vae)+ wrapper, slot = _fp16_shaped(vae, budget, own_need = 6000)+ assert O.install(vae, pipe = pipe)+ top = vae.__dict__["decode"]+ assert getattr(top, "_unsloth_vae_oom_scope", False) and top.__wrapped__ is wrapper+ before = _devices(pipe)+ out = vae.decode(_z())+ assert out[0].shape[-1] == 64+ assert vae._unsloth_vae_oom_stats["recovered_by"] == "freed" and _devices(pipe) == before++[email protected]("slot_kind", ["vae", "fp16"])+def test_compile_fallback_keeps_the_scope(monkeypatch, slot_kind):+ """Item 4286353643: a lazy compile that fails replaced its owner slot with the eager decode. When the scope sat in+ that slot (fp16 / bf16 orders) or on top of the guard, the fallback dropped it and later sliced decodes ran+ without one shared session. The fallback now leaves a slot it no longer holds alone, and the scope re-asserts+ itself when a decode closes."""+ from core.inference import diffusion_speed as S++ monkeypatch.setattr(S, "is_compile_failure", lambda exc: isinstance(exc, _CompileFailed))+ budget = _Budget(16384 + 1000)+ transformer, text_encoder = _on_card(8192), _on_card(4096)+ vae = _SlicedVae(budget, [transformer, text_encoder], 10000, 500)+ vae.to(_CARD)+ pipe = _Pipe(transformer = transformer, text_encoder = text_encoder, vae = vae)++ def compiled(z, *a, **k):+ raise _CompileFailed("inductor lowering failed")++ if slot_kind == "fp16":+ _, slot = _fp16_shaped(vae, budget, own_need = 0)+ slot.decode = S._guard_compiled_decode(vae, compiled, slot.decode, None, owner = slot)+ else:+ vae.decode = S._guard_compiled_decode(vae, compiled, vae.decode, None)+ assert O.install(vae, pipe = pipe)+ moves = []+ monkeypatch.setattr(O, "_to_host", lambda m: (moves.append("off"), m.to(torch.float32)))+ monkeypatch.setattr(O, "_return_to", lambda m, d: (moves.append("on"), m.to(_CARD)))+ out = vae.decode(torch.ones(3, 4, 1, 8, 8))[+ 0+ ] # compile fails, eager decode, slice OOMs recover+ assert out.shape[0] == 3+ top = vae.__dict__["decode"]+ assert getattr(top, "_unsloth_vae_oom_scope", False) # still outermost after the fallback+ moves.clear()+ out = vae.decode(torch.ones(3, 4, 1, 8, 8))[0]+ assert out.shape[0] == 3 and moves == ["off", "off", "on", "on"]+++def test_compiled_guard_behind_the_untiled_gate_reaches_the_ladder(monkeypatch):+ """The video untiled gate calls what it wraps through a closure (no slot), so a compiled+ guard under it was invisible and its tiled-attempt OOM escaped. Under the single placement rule the scope is+ outermost; it frees the card, retries the whole decode compiled, then with every guard eager."""+ from core.inference import diffusion_speed as S+ from core.inference import video_vae_untiled as U++ budget = _Budget(16384 + 1000)+ transformer, text_encoder = _on_card(8192), _on_card(4096)+ vae = _PublicVae(budget, [transformer, text_encoder], 10**9, 10**9)+ vae.to(_CARD)+ vae.use_tiling = True+ pipe = _Pipe(transformer = transformer, text_encoder = text_encoder, vae = vae)+ calls = []++ def compiled(z, *a, **k):+ calls.append(("compiled", vae.use_tiling))+ raise _FakeOOM()++ vae.decode = S._guard_compiled_decode(vae, compiled, vae.decode, None)+ vae._unsloth_compiled_decode = True+ monkeypatch.setattr(+ U, "_free_bytes", lambda device: 0+ ) # untiled never fits: the gate decodes tiled+ family = next(iter(U._BYTES_PER_LATENT_PIXEL))+ assert U.install_untiled_decode(pipe, family)+ assert O.install(vae, pipe = pipe)+ vae.tiled_need = 500+ out = vae.decode(_z())+ assert calls == [("compiled", True), ("compiled", True)]+ assert out[0].shape[-1] == 64 and vae.seen[-1][0] is True+ assert getattr(vae.__dict__["decode"], "_unsloth_vae_oom_scope", False)+++def _raw_read(t):+ """What a Triton kernel sees: the values at ``t.data_ptr()`` indexed with ``t``'s own strides (CPU, float32)."""+ import ctypes++ n = t.storage_offset() + sum((s - 1) * st for s, st in zip(t.shape, t.stride())) + 1+ buf = (ctypes.c_float * n).from_address(t.data_ptr())+ flat = torch.frombuffer(buf, dtype = torch.float32, count = n).clone()+ return torch.as_strided(flat, t.shape, t.stride(), t.storage_offset())+++def test_patched_norms_never_hand_a_fused_input_to_the_kernel(monkeypatch):+ """Item 4286353624: a GroupNorm-only site (act False) handed a fused GroupNorm+SiLU output to its kernel as is,+ and a kernel reads raw memory (activated values). Every patched entry checks its input first, and a raw pointer+ read of a fused output from any other module's kernel is caught too: the decode is redone without activation+ fusion instead (the structural tests check the redone output against stock)."""+ seen = []++ def kernel(+ x,+ norm,+ act = True,+ in_bias = None,+ on_kernel = None,+ **_,+ ):+ raw = _raw_read(x)+ seen.append(raw)+ y = torch.nn.functional.group_norm(raw, norm.num_groups, norm.weight, norm.bias, norm.eps)+ return torch.nn.functional.silu(y) if act else y++ monkeypatch.setattr(F, "group_norm_act", kernel)+ norm, norm2, norm3 = (+ torch.nn.GroupNorm(4, 8),+ torch.nn.GroupNorm(4, 8),+ torch.nn.GroupNorm(4, 8),+ )+ state = types.SimpleNamespace(kernel_calls = 0, calls = 0)+ fused = F._fast_gn_maybe_act(norm, True, state)+ gn_only = F._fast_gn_maybe_act(norm2, False, state)+ fused2 = F._fast_gn_maybe_act(norm3, True, state)+ x = torch.randn(1, 8, 4, 4)+ with torch.no_grad():+ for reader in (gn_only, fused2, _raw_read):+ seen.clear()+ y = fused(x)+ assert len(seen) == 1 and torch.equal(seen[0], x)+ with pytest.raises(F.FusedActivationRead):+ reader(y)+ assert len(seen) == 1 # the kernel never saw the activated values+++def test_after_an_inplace_paired_silu_the_tensor_reads_as_stock():+ """stock silu_ overwrites the norm output: after the paired in-place SiLU the fused tensor's storage is exactly+ that, so later reads (raw pointers included) are plain reads, not aborts."""+ norm, act = torch.nn.GroupNorm(4, 8), torch.nn.SiLU(inplace = True)+ fast = F._fast_gn_maybe_act(norm, True, types.SimpleNamespace(kernel_calls = 0, calls = 0), [act])+ x = torch.randn(1, 8, 4, 6)+ with torch.no_grad():+ y = fast(x)+ F._silu_passthrough(act)(y)+ assert torch.allclose(+ _raw_read(y.contiguous()), torch.nn.functional.silu(norm(x)), atol = 1e-6+ )+++class _CausalLtx:+ """An LTX-2-shaped VAE whose decoder keeps causal state across calls (reset by clear_cache); OOMs on chosen+ decoder calls."""++ spatial_compression_ratio = 32++ def __init__(self, oom_calls):+ self.state = []+ self.calls = 0+ self.oom_calls = set(oom_calls)+ self.use_tiling = True++ def clear_cache(self):+ self.state = []++ def decoder(+ self,+ z,+ temb = None,+ causal = None,+ ):+ self.calls += 1+ if self.calls in self.oom_calls:+ raise _FakeOOM()+ out = z.repeat_interleave(32, -1).repeat_interleave(32, -2) + 0.001 * len(self.state)+ self.state.append(1)+ return out++[email protected]("k", [1, 2, 3])+def test_ltx2_floor_decode_starts_clean_after_a_freed_retry_fails_part_way(monkeypatch, k):+ """Item 4286353660: the freed retry of the wide LTX-2 tiles ran k tiles, then ran out of memory; the floor-tile+ decode started on that partial causal state. Every retry after a partial causal decode clears it first."""+ pytest.importorskip("diffusers") # tiled_decode builds a diffusers DecoderOutput+ from core.inference import video_ltx2_vae_tiles as T++ z = torch.randn(1, 4, 1, 48, 48)+ clean = _CausalLtx(())+ ref = T._decode_tiles(clean, z, None, None, 16, 16, False)+ # the wide attempt (32-latent tiles: 4 of them) OOMs at once; its freed retry decodes k tiles, then OOMs+ vae = _CausalLtx({1, 1 + k + 1})+ retry_calls = []++ def free_first(v, zz):+ v.clear_cache() # what the real step (a) does before the freed retry+ retry_calls.append(v.calls)+ return True++ monkeypatch.setattr(T, "oom_free_first", free_first)+ monkeypatch.setattr(T, "_plan", lambda vae, z: ((32, 32), True))+ out = T.tiled_decode(vae, z, return_dict = False)[0]+ assert retry_calls == [1] and vae.calls > 1 + k + 1+ assert torch.allclose(out, ref)+++def test_untiled_gate_tiled_retry_starts_clean(monkeypatch):+ """Same class as 4286353660 in the untiled-when-fits gate: an untiled attempt that ran out of memory part way+ left causal state for the tiled decode."""+ from core.inference import video_vae_untiled as U++ class Causal(torch.nn.Module):+ def __init__(self):+ super().__init__()+ self.use_tiling = True+ self.state = []+ self.decoder = torch.nn.Linear(2, 2)++ def clear_cache(self):+ self.state = []++ def decode(+ self,+ z,+ return_dict = True,+ ):+ out = z + 0.001 * len(self.state)+ self.state.append(1)+ if not self.use_tiling:+ raise _FakeOOM() # the untiled attempt fails after a causal step+ return (out,)++ vae = Causal()+ monkeypatch.setattr(U, "_free_bytes", lambda device: 1 << 50)+ family = next(iter(U._BYTES_PER_LATENT_PIXEL))+ assert U.install_untiled_decode(types.SimpleNamespace(vae = vae), family)+ z = torch.zeros(1, 4, 1, 8, 8)+ out = vae.decode(z)[0]+ assert torch.equal(out, z)+++def test_regional_compiled_blocks_take_the_eager_retry():+ """Item 4286789089: a regionally compiled VAE (compile_repeated_blocks + guard_compiled_blocks) promised the+ scope an eager retry, but the block guard ignored the VAE-level flag, so the retry ran the same compiled blocks.+ Every compiled guard now reads one signal (diffusion_bg_compile.force_eager)."""+ from core.inference import diffusion_speed as S++ budget = _Budget(1 << 40)+ transformer, text_encoder = _on_card(8192), _on_card(4096)++ class Block(torch.nn.Module):+ def forward(self, z):+ calls.append("eager")+ return z++ class Vae(_VaeLike):+ def __init__(self):+ super().__init__(budget, [transformer, text_encoder], 1, 1)+ self.block = Block()++ def _decode(+ self,+ z,+ return_dict = True,+ ):+ return (self.block(z).repeat_interleave(8, -1).repeat_interleave(8, -2),)++ def decode(+ self,+ z,+ return_dict = True,+ ):+ return self._decode(z, return_dict)++ calls = []+ vae = Vae().to(_CARD)+ pipe = _Pipe(transformer = transformer, text_encoder = text_encoder, vae = vae)++ def compiled_block(*a, **k):+ calls.append("compiled")+ raise _FakeOOM()++ vae.block._compiled_call_impl = compiled_block+ assert S.guard_compiled_blocks(vae) >= 1+ vae._unsloth_compiled_decode = True+ assert O.install(vae, pipe = pipe)+ out = vae.decode(_z())+ assert out[0].shape[-1] == 64+ assert calls[-1] == "eager" and "compiled" in calls+++def _exact_kernel(monkeypatch):+ """group_norm_act on CPU with exactly stock's ops (F.group_norm, F.silu), flagged as a kernel run: a fused decode+ then matches stock bit for bit wherever it computes the same values."""++ def fake(+ x,+ norm,+ act = True,+ in_bias = None,+ on_kernel = None,+ **_,+ ):+ y = torch.nn.functional.group_norm(x if in_bias is None else x + in_bias, norm.num_groups, norm.weight,+ norm.bias, norm.eps) # fmt: skip+ if on_kernel is not None:+ on_kernel()+ return torch.nn.functional.silu(y) if act else y++ monkeypatch.setattr(F, "group_norm_act", fake)+ monkeypatch.setattr(F, "runtime_ok", lambda: True)+ monkeypatch.delenv(F.STRUCTURAL_ENV, raising = False)+ monkeypatch.setattr(torch.Tensor, "is_cuda", property(lambda self: True))+++def _mutate(t, op, b):+ if op == "add_":+ t.add_(b)+ elif op == "mul_":+ t.mul_(2.0)+ elif op == "setitem":+ t[:, 0] = 3.0+ elif op == "copy_":+ t.copy_(t * 0.5 + b)+ elif op == "out":+ torch.add(t, b, out = t)+ elif op == "view_add_":+ t[:, 1:3].add_(1.0)+ elif op == "iadd":+ t += b+ elif op == "input_add_":+ pass+ return t+++class _MutatingDec(torch.nn.Module):+ """At full width (beyond the calibration crop) it writes to the norm output, or to the norm's INPUT, before+ the paired SiLU / a residual read; on the crop it only feeds the SiLU."""++ def __init__(self, op):+ super().__init__()+ self.op = op+ self.norm = torch.nn.GroupNorm(4, 8)+ self.act = torch.nn.SiLU()+ self.b = torch.nn.Parameter(torch.randn(1, 8, 1, 1))++ def forward(self, x):+ x = x * 1.0+ h = self.norm(x)+ if x.shape[-1] <= 16:+ return self.act(h)+ if self.op == "input_add_":+ x.add_(+ 1.0+ ) # the norm's input changes after the norm, before a residual read of its output+ return self.act(h) + h * 2 + x+ h = _mutate(h, self.op, self.b)+ return self.act(h)+++class AutoencoderKLMutating(torch.nn.Module):+ def __init__(self, op):+ super().__init__()+ self.decoder = _MutatingDec(op)++ def _decode(+ self,+ z,+ return_dict = True,+ ):+ return (self.decoder(z),)++[email protected]("mode", ["no_grad", "inference_mode"])+ "op", ["add_", "mul_", "setitem", "copy_", "out", "view_add_", "iadd", "input_add_"]+)+def test_writes_around_a_fused_norm_match_stock_exactly(monkeypatch, op, mode):+ """Items 4286789097 / 4287220388: a write to the fused output before its paired SiLU (add_, mul_, setitem,+ copy_, out=, through a view, +=), or a write to the norm's INPUT before a residual read of its output, gave a+ stale or recomputed-from-changed-input value. Now any such read aborts that decode, which is redone with+ GroupNorm fused alone: the output matches stock bit for bit, under no_grad and under inference_mode (whose+ tensors keep no version counter)."""+ torch.manual_seed(0)+ vae = AutoencoderKLMutating(op)+ z = torch.randn(1, 8, 6, 40)+ ctx = torch.inference_mode if mode == "inference_mode" else torch.no_grad+ with ctx():+ ref = vae._decode(z.clone())[0]+ _exact_kernel(monkeypatch)+ F.install_structural(vae)+ with ctx():+ first = vae._decode(z.clone())[0]+ again = vae._decode(z.clone())[0]+ assert torch.equal(first, ref) and torch.equal(again, ref)+ state = vae._unsloth_vae_structural+ assert state.status == "ok" and state.gn == 1+ if op != "iadd":+ assert state.act_off and state.foreign_reads >= 1+++def test_kept_oom_drops_the_tracebacks_of_its_whole_chain():+ """Item 4286789111: only the outer exception's traceback was cleared, so a wrapped OOM's inner frame (and the+ tensor it held) stayed alive through the retry via the kept exception. Every chained traceback goes now."""+ import weakref++ budget = _Budget(16384 + 1000)+ transformer, text_encoder = _on_card(8192), _on_card(4096)+ seen = {}++ class Vae(_PublicVae):+ def _decode(+ self,+ z,+ return_dict = True,+ ):+ if "ref" not in seen:++ def inner_frame():+ big = torch.empty(1 << 20)+ seen["ref"] = weakref.ref(big)+ raise _FakeOOM()++ try:+ inner_frame()+ except _FakeOOM as oom:+ raise RuntimeError(+ "backend compiler failed"+ ) from oom # a wrapped OOM (is_oom walks the chain)+ seen["alive_at_retry"] = seen["ref"]() is not None+ return (z.repeat_interleave(8, -1).repeat_interleave(8, -2),)++ vae = Vae(budget, [transformer, text_encoder], 1, 1).to(_CARD)+ pipe = _Pipe(transformer = transformer, text_encoder = text_encoder, vae = vae)+ assert O.install(vae, pipe = pipe)+ vae.decode(_z())+ assert seen["alive_at_retry"] is False+++def test_drop_tracebacks_walks_cause_context_and_cycles():+ def raise_chain():+ try:+ try:+ raise ValueError("a")+ except ValueError as a:+ raise KeyError("b") from a+ except KeyError as b:+ raise RuntimeError("c")++ try:+ raise_chain()+ except RuntimeError as c:+ exc = c+ exc.__context__.__cause__.__context__ = exc+ O.drop_tracebacks(exc)+ e, seen = exc, 0+ for e in (exc, exc.__context__, exc.__context__.__cause__):+ assert e.__traceback__ is None+++def test_dtype_wrapper_oom_climbs_to_the_vae_offload(monkeypatch):+ """Item 4287727804: the inner _decode succeeded but the fp16-shaped wrapper's own check ran out of memory, and+ freeing the rest of the pipeline was not enough; the scope re-raised, since only the _decode ladder (which never+ saw this OOM) knew the later rungs. The scope now climbs the one ladder (same rungs, same session): freed, stock+ tiles, then the VAE's weights streamed from the host, which is what fits here."""+ offloads = []+ monkeypatch.setattr(+ O,+ "_apply_vae_offload",+ lambda vae, device: (offloads.append(1), vae.conv.to(torch.float32)),+ )+ budget = _Budget(4096 + 100)+ transformer, text_encoder = _on_card(8192), _on_card(4096)+ vae = _PublicVae(budget, [transformer, text_encoder], 1, 1)+ vae.to(_CARD)+ pipe = _Pipe(transformer = transformer, text_encoder = text_encoder, vae = vae)+ _fp16_shaped(vae, budget, own_need = 3000) # fits only once the VAE's 4096 bytes leave the card+ assert O.install(vae, pipe = pipe)+ before = _devices(pipe)+ out = vae.decode(_z())+ assert out[0].shape[-1] == 64 and offloads == [1]+ stats = vae._unsloth_vae_oom_stats+ assert stats["recovered_by"] == "offload" and stats["outer_oom"] == 1+ assert _devices(pipe) == before # everything came back after the decode+++def test_split_vae_never_decodes_before_its_required_repair(monkeypatch):+ """Item 4287727812: undoing the VAE offload failed and parking it failed too, so the VAE was split with no+ working streamed path, yet its deferred return was optional: the next decode ran on the inconsistent VAE. The+ return is required unless the VAE is CHECKED to stream whole from the host, and a decode whose required repair+ still fails raises that clear error instead of decoding."""+ vae, pipe = _setup(cap = 1 << 20, untiled_need = 1, tiled_need = 1)+ session = O.DecodeSession(vae, _FakeDevice())+ calls = []++ def fail_back():+ vae.conv.to(torch.float32)+ raise _FakeOOM()++ def fail_park():+ raise _FakeOOM()++ change = O.Change("vae_offload", fail_back, fail_park,+ lambda retry, exc: O._defer(vae, _FakeDevice(), "vae_offload", retry,+ lambda: not O._vae_streams(vae), vae)) # fmt: skip+ session.rollback(change)+ deferred = vae.__dict__[O._DEFERRED_ATTR]+ assert deferred.required+ seen_before = len(vae.seen)+ with pytest.raises(RuntimeError, match = "cannot return"):+ vae.decode(_z()) if "decode" in vae.__dict__ else vae._decode(_z())+ assert len(vae.seen) == seen_before+ deferred.pending[:] = [("vae_offload", lambda: vae.conv.to(_CARD))]+ vae._decode(_z())+ assert O._DEFERRED_ATTR not in vae.__dict__ and len(vae.seen) == seen_before + 1diff --git a/studio/backend/tests/test_diffusion_vae_oom_faults.py b/studio/backend/tests/test_diffusion_vae_oom_faults.pynew file mode 100644index 00000000000..2e61acd1614--- /dev/null+++ b/studio/backend/tests/test_diffusion_vae_oom_faults.py@@ -0,0 +1,542 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""Fault injection over the VAE OOM ladder: an OOM or a generic error at EVERY mutation point of the recovery and of+its restore, on a fake pipeline (a resident component, resident offload groups, CUDA graphs, a VAE with an+instance-bound ``_decode`` override and a later ``decode`` wrapper, a sliced batch decode). After each decode+(returned or raised) the system must be consistent: every module on its card or with a working deferred return that+resolves the card; resident groups all re-pinned or all streamed with a deferred re-pin; the wrapper stack exact after+uninstall; later slices on the recovered path; the surfaced error the decode's own OOM when nothing recovered.++CPU only: a float64 tensor stands in for one on the card, float32 for one on the host."""++import itertools+import types++import pytest++torch = pytest.importorskip("torch")++from core.inference import decode_layers as L # noqa: E402+from core.inference import diffusion_cuda_graph as G # noqa: E402+from core.inference import diffusion_memory as M # noqa: E402+from core.inference import diffusion_vae_oom as O # noqa: E402++CARD, HOST = torch.float64, torch.float32+++class FakeOOM(RuntimeError):+ def __init__(self, where = ""):+ super().__init__(f"CUDA out of memory. Tried to allocate 2.00 GiB ({where})")+++class Injected(ValueError):+ pass+++class Device:+ type = "cuda"+++DEV = Device()+++class World:+ """The injector: counts mutation points; raises at the ``at``-th one (``kind`` "oom" or "error")."""++ def __init__(+ self,+ at = None,+ kind = "oom",+ ):+ self.at, self.kind, self.n, self.points, self.fired = at, kind, 0, [], None++ def __call__(self, point):+ self.points.append(point)+ self.n += 1+ if self.at is not None and self.n in (+ self.at if isinstance(self.at, tuple) else (self.at,)+ ):+ self.fired = point+ raise FakeOOM(point) if self.kind == "oom" else Injected(point)+++def _move(module, dtype, world, tag):+ """Parameter by parameter, a mutation point before each: a fault leaves the module part moved."""+ for i, p in enumerate(list(module.parameters())):+ world(f"{tag}:param{i}")+ p.data = p.data.to(dtype)+++class Group:+ def __init__(self):+ self._unsloth_resident = True+++class Vae(torch.nn.Module):+ spatial_compression_ratio = 8++ def __init__(self, budget):+ super().__init__()+ self.use_tiling = False+ self.tile_sample_min_height = 256+ self.tile_sample_stride_height = 192+ self.a = torch.nn.Linear(64, 8, bias = False)+ self.b = torch.nn.Linear(64, 8, bias = False)+ self.budget = budget+ self.fake_hooks = False+ self.slices = []++ def enable_tiling(self):+ self.use_tiling = True++ def _decode(+ self,+ z,+ return_dict = True,+ ):+ streamed = self.fake_hooks+ need = {False: 9, True: 5}[bool(self.use_tiling)] - (3 if streamed else 0)+ self.budget.world(f"decode:slice{self.budget.slice}:{'tiled' if self.use_tiling else 'untiled'}"+ f"{':streamed' if streamed else ''}") # fmt: skip+ if self.budget.used() + need > self.budget.cap:+ raise FakeOOM("decode")+ self.slices.append((self.budget.slice, bool(self.use_tiling), streamed))+ return (z + (0.5 if self.use_tiling else 0.0),)++ def decode(+ self,+ z,+ return_dict = True,+ ):+ out = []+ for i in range(z.shape[0]):+ self.budget.slice = i+ out.append(self._decode(z[i : i + 1])[0])+ return (torch.cat(out),)+++class Pipe:+ pass+++class Budget:+ """Units of card memory: the transformer 4, each resident group 1, the VAE weights 3, plus the decode's need."""++ def __init__(self, cap, world):+ self.cap, self.world, self.slice = cap, world, 0+ self.transformer = self.groups = self.vae = None++ def used(self):+ own = getattr(self.transformer, "_fake_groups", None)+ if own is not None:+ u = sum(2 for g in own if g._unsloth_resident)+ else:+ u = 4 if all(p.dtype == CARD for p in self.transformer.parameters()) else 0+ u += sum(1 for g in self.groups if g._unsloth_resident and g not in (own or ()))+ u += 0 if self.vae.fake_hooks else 3+ return u+++def build(+ cap,+ world,+ monkeypatch,+ tiled = False,+ variant = "base",+):+ budget = Budget(cap, world)+ transformer = torch.nn.Linear(8, 8).to(CARD)+ text_encoder = torch.nn.Linear(8, 8, bias = False).to(CARD)+ text_encoder._diffusers_hook = types.SimpleNamespace(hooks = {"group_offloading": 1})+ text_encoder._unsloth_resident_room = 1+ groups = [Group() for _ in range(3)]+ vae = Vae(budget).to(CARD)+ vae.use_tiling = tiled+ budget.transformer, budget.groups, budget.vae = transformer, groups, vae+ pipe = Pipe()+ pipe.components = {"transformer": transformer, "text_encoder": text_encoder, "vae": vae}+ pipe.vae, pipe.transformer, pipe._unsloth_cuda_graphs = vae, transformer, ("graph",)++ def release(+ pipe_,+ need,+ logger = None,+ reason = "",+ ):+ # diffusion_memory.release_resident_groups's contract: bookkeeping before each demotion; a failure part way+ # is caught and the restore for what was released is returned. Its re-pin swallows its own failures (as+ # _keep_groups_resident does); ``strict`` reports groups still streamed.+ released = []++ def restore(strict = False):+ try:+ for g in released:+ world("groups:repin")+ g._unsloth_resident = True+ except Exception: # noqa: BLE001+ pass+ if strict and any(not g._unsloth_resident for g in released):+ raise RuntimeError("re-pinning resident offload groups left some streamed")++ try:+ for g in reversed(groups):+ if g._unsloth_resident:+ released.append(g)+ world("groups:release")+ g._unsloth_resident = False+ except Exception: # noqa: BLE001+ return restore if released else None+ return restore if released else None++ def apply_offload(v, device):+ world("vae_hooks:add")+ v.fake_hooks = True+ _move(v, HOST, world, "vae_out")++ def remove_offload(v, device):+ world("vae_hooks:remove")+ v.fake_hooks = False+ _move(v, CARD, world, "vae_back")++ def park(v, device):+ v.fake_hooks = False+ _move(v, HOST, world, "vae_park")+ world("vae_hooks:readd")+ v.fake_hooks = True++ def unhook():+ for g in transformer.__dict__.pop("_fake_groups", None) or ():+ groups.remove(g)+ transformer.__dict__.pop("_diffusers_hook", None)+ transformer.__dict__.pop("_unsloth_resident_room", None)++ def hook_resident(+ pipe_,+ device,+ logger = None,+ resident = True,+ ):+ if getattr(transformer, "_fake_groups", None) is not None:+ return False+ world("torchao:hook")+ own = [Group(), Group()]+ for g in own:+ g._unsloth_resident = False+ transformer._fake_groups = own+ groups.extend(own)+ transformer._diffusers_hook = types.SimpleNamespace(hooks = {"group_offloading": 1})+ try:+ _move(transformer, HOST, world, "torchao_out")+ if resident:+ _move(transformer, CARD, world, "torchao_keep")+ for g in own:+ g._unsloth_resident = True+ transformer._unsloth_resident_room = 1+ except Exception: # noqa: BLE001+ unhook()+ _move(transformer, CARD, world, "torchao_back")+ return False+ return True++ def return_torchao(+ pipe_,+ device,+ logger = None,+ torch_hooks = None,+ ):+ own = getattr(transformer, "_fake_groups", None)+ if own is not None:+ _move(transformer, CARD, world, "torchao_repin")+ for g in own:+ g._unsloth_resident = True+ transformer._unsloth_resident_room = 1+ return+ unhook()+ _move(transformer, CARD, world, "torchao_back")++ monkeypatch.setattr(M, "release_resident_groups", release)+ monkeypatch.setattr(M, "hook_resident_denoiser", hook_resident)+ monkeypatch.setattr(M, "return_torchao_denoiser", return_torchao, raising = False)+ monkeypatch.setattr(M, "module_forward_hook_ids", lambda m: {}, raising = False)+ monkeypatch.setattr(O, "_holds_torchao", lambda m: variant == "torchao" and m is transformer)+ monkeypatch.setattr(G, "reset_all", lambda handles, strict = False: world("graphs:reset"))+ monkeypatch.setattr(O, "_to_host", lambda m: _move(m, HOST, world, "out"))+ monkeypatch.setattr(O, "_return_to", lambda m, d: _move(m, CARD, world, "back"))+ monkeypatch.setattr(O, "_apply_vae_offload", apply_offload)+ monkeypatch.setattr(O, "_remove_vae_offload", remove_offload)+ monkeypatch.setattr(O, "_park_vae", park)+ monkeypatch.setattr(O, "_whole_on", lambda m, d: all(p.dtype == CARD for p in m.parameters()))+ monkeypatch.setattr(+ O, "_whole_on_host", lambda m: all(p.dtype == HOST for p in m.parameters()), raising = False+ )+ monkeypatch.setattr(+ O, "_vae_hooked", lambda m: bool(getattr(m, "fake_hooks", False)), raising = False+ )+ monkeypatch.setattr(O, "_hooked", lambda m: bool(getattr(m, "fake_hooks", False)) or (+ getattr(m, "_diffusers_hook", None) is not None)) # fmt: skip+ monkeypatch.setattr(O, "_host_free_mib", lambda: 1 if variant == "host_short" else 1 << 20)+ monkeypatch.setattr(O, "_empty_cache", lambda d: None)+ monkeypatch.setattr(O, "_device_of", lambda z: DEV)+ monkeypatch.setattr(O, "_release_cache", lambda z, vae = None: None)++ override = types.MethodType(lambda self, z, *a, **k: Vae._decode(self, z, *a, **k), vae)+ vae._decode = override+ before = {k: vae.__dict__.get(k, "MISSING") for k in ("_decode", "decode")}+ assert O.install(vae, pipe = pipe)+ scope = vae.__dict__["decode"]++ def later(*a, **k):+ return scope(*a, **k)++ later.__wrapped__ = scope+ vae.decode = later+ return pipe, budget, before, override, later+++STEPS = {(False, False): 0, (True, False): 1, (True, True): 2}+++def check(pipe, budget, before, override, later, world, outcome):+ vae, transformer, text_encoder = (+ pipe.vae,+ pipe.components["transformer"],+ pipe.components["text_encoder"],+ )+ ctx = f"fault {world.kind} at {world.fired!r}, cap {budget.cap}, outcome {outcome!r}"+ # surfaced error: the decode's own OOM when nothing recovered; an injected error only from a decode attempt+ if isinstance(outcome, BaseException):+ if isinstance(outcome, Injected):+ assert world.fired and world.fired.startswith("decode:"), ctx+ else:+ assert isinstance(outcome, FakeOOM), ctx+ # later slices stay on the recovered path (never back to a cheaper mode than an earlier slice used)+ modes = [STEPS[(t, s)] for _, t, s in vae.slices]+ assert modes == sorted(modes), (ctx, vae.slices)+ # a VAE left split (its repair is REQUIRED) never decodes before it is repaired: while the repair still fails,+ # the next decode raises that clear error instead of decoding with the inconsistent VAE+ vdef = vae.__dict__.get(O._DEFERRED_ATTR)+ if vdef is not None:+ # optional only when it is checked to stream whole from the host; anything else is a required repair+ streams = vae.fake_hooks and all(p.dtype == HOST for p in vae.parameters())+ assert vdef.required == (not streams), (ctx, vdef.required, streams)+ if vdef is not None and vdef.required:+ world.at, world.kind = (world.n + 1, world.n + 2), "oom"+ vae.slices.clear()+ with pytest.raises(RuntimeError, match = "cannot return"):+ vae.decode(torch.zeros(1, 4, 1, 2, 2))+ assert vae.slices == [], ctx+ world.at = None # memory frees up: everything deferred must now find its way back+ budget.cap = 10**6+ deferred = transformer.__dict__.get(O._DEFERRED_ATTR)+ if getattr(transformer, "_fake_groups", None) is not None and deferred is None:+ assert all(p.dtype == (CARD if all(g._unsloth_resident for g in transformer._fake_groups) else HOST)+ for p in transformer.parameters()), ctx # fmt: skip+ elif deferred is None:+ assert all(p.dtype == CARD for p in transformer.parameters()), ctx+ else:+ assert (+ deferred.execution_device is DEV and transformer.__dict__.get("_hf_hook") is deferred+ ), ctx+ transformer(torch.zeros(1, 8, dtype = CARD))+ assert all(p.dtype == CARD for p in transformer.parameters()), ctx+ assert (+ O._DEFERRED_ATTR not in transformer.__dict__ and not transformer._forward_pre_hooks+ ), ctx+ resident = [g._unsloth_resident for g in budget.groups]+ if not all(resident):+ anchors = [m for m in (transformer, text_encoder) if O._DEFERRED_ATTR in m.__dict__]+ assert anchors, ctx+ for m in anchors:+ m(torch.zeros(1, 8, dtype = CARD))+ assert all(g._unsloth_resident for g in budget.groups), ctx+ vdef = vae.__dict__.get(O._DEFERRED_ATTR)+ if vdef is None:+ assert not vae.fake_hooks and all(p.dtype == CARD for p in vae.parameters()), ctx+ else:+ assert O.settle_deferred(vae), ctx+ assert not vae.fake_hooks and all(p.dtype == CARD for p in vae.parameters()), ctx+ assert O._SESSION_ATTR not in vae.__dict__, ctx+ # the wrapper stack: uninstall leaves exactly what was there plus the later wrapper (a pass-through below it)+ O.uninstall(vae)+ assert vae.__dict__.get("_decode") is override and vae.__dict__.get("decode") is later, ctx+ assert L.LAYERS_ATTR not in vae.__dict__ and O._RECORD_ATTR not in vae.__dict__, ctx+ vae.slices.clear()+ assert vae.decode(torch.zeros(2, 4, 1, 2, 2))[0].shape[0] == 2 and len(vae.slices) == 2, ctx+++def _run(+ cap,+ at,+ kind,+ monkeypatch,+ tiled = False,+ verify = True,+ variant = "base",+):+ world = World(at, kind)+ with monkeypatch.context() as mp:+ pipe, budget, before, override, later = build(cap, world, mp, tiled, variant)+ mp.setattr(O, "_FAULT", world)+ try:+ out = pipe.vae.decode(torch.zeros(3, 4, 1, 2, 2))+ outcome = out[0].shape+ except (FakeOOM, Injected) as exc:+ outcome = exc+ world.decode_points = world.n+ if verify:+ check(pipe, budget, before, override, later, world, outcome)+ return world+++CAPS = (20, 13, 8, 2, 1)+VARIANTS = ("base", "torchao", "host_short")+++def _cases():+ import pytest as _pytest++ mp = _pytest.MonkeyPatch()+ cases = []+ try:+ for variant, cap, tiled in itertools.product(VARIANTS, CAPS, (False, True)):+ n = _run(cap, None, "oom", mp, tiled, verify = False, variant = variant).decode_points+ cases += [+ (variant, cap, tiled, at, kind)+ for at, kind in itertools.product(range(1, n + 1), ("oom", "error"))+ ]+ finally:+ mp.undo()+ return cases+++CASES = _cases()+++def test_fault_injection_covers_every_mutation_point():+ assert len(CASES) >= 900, len(CASES)+ kinds = {p.split(":")[0] for v, cap, tiled in itertools.product(VARIANTS, CAPS, (False, True))+ for p in _points(cap, tiled, v)} # fmt: skip+ for point in ("graphs", "groups", "out", "back", "vae_hooks", "vae_out", "vae_back", "decode",+ "graphs_reset", "resident_groups", "moved", "vae_offload", "torchao", "torchao_hooks",+ "torchao_out", "torchao_keep"): # fmt: skip+ assert point in kinds, (point, kinds)+++def _points(+ cap,+ tiled,+ variant = "base",+):+ mp = pytest.MonkeyPatch()+ try:+ world = _run(cap, None, "oom", mp, tiled, verify = False, variant = variant)+ finally:+ mp.undo()+ return world.points[: world.decode_points]++[email protected]("variant,cap,tiled,at,kind", CASES)+def test_fault_injected_recovery_stays_consistent(variant, cap, tiled, at, kind, monkeypatch):+ _run(cap, at, kind, monkeypatch, tiled, variant = variant)++[email protected]("variant", VARIANTS)[email protected]("cap", CAPS)+def test_unfaulted_ladder_reaches_its_step(cap, variant, monkeypatch):+ world = World()+ with monkeypatch.context() as mp:+ pipe, budget, *_ = build(cap, world, mp, variant = variant)+ try:+ pipe.vae.decode(torch.zeros(3, 4, 1, 2, 2))+ raised = None+ except FakeOOM as exc:+ raised = exc+ stats = pipe.vae._unsloth_vae_oom_stats+ expect = {20: None, 13: "freed", 8: "tiled", 2: "offload", 1: None}[cap]+ if variant == "host_short":+ expect = {20: None, 13: "tiled", 8: None, 2: None, 1: None}[cap]+ if cap in (8, 2):+ assert raised is not None+ return+ assert stats.get("recovered_by") == expect, stats+ assert (raised is not None) == (cap == 1)+ if cap != 20:+ assert len({(t, s) for _, t, s in pipe.vae.slices}) <= 1+++def _torchao_double_cases():+ """Item 4285966491: the torchao hook conversion fails (first fault, inside hook_resident_denoiser) AND its own+ return to the card fails (second fault, any later point): the transformer is left split between host and card+ with no hooks, and the session must still leave it on the card, streamed under hooks, or with a deferred return."""+ mp = pytest.MonkeyPatch()+ cases = []+ try:+ for cap, tiled in itertools.product((13, 8, 2), (False, True)):+ pts = _run(cap, None, "oom", mp, tiled, verify = False, variant = "torchao").points+ firsts = [+ i + 1 for i, p in enumerate(pts) if p.startswith(("torchao_out:", "torchao_keep:"))+ ]+ for a1 in firsts:+ n = _run(cap, a1, "oom", mp, tiled, verify = False, variant = "torchao").decode_points+ cases += [+ (cap, tiled, a1, a2, kind)+ for a2 in range(a1 + 1, n + 1)+ for kind in ("oom", "error")+ ]+ finally:+ mp.undo()+ return cases+++TORCHAO_DOUBLE = _torchao_double_cases()+++def test_torchao_double_faults_reach_the_split_state():+ assert len(TORCHAO_DOUBLE) >= 50, len(TORCHAO_DOUBLE)+ mp = pytest.MonkeyPatch()+ try:+ cap, tiled, a1, a2, kind = next(c for c in TORCHAO_DOUBLE if c[4] == "oom")+ world = _run(cap, (a1, a1 + 1), "oom", mp, tiled, verify = False, variant = "torchao")+ finally:+ mp.undo()+ assert world.points[a1].startswith("torchao_back:"), world.points[a1 - 1 : a1 + 2]++[email protected]("cap,tiled,a1,a2,kind", TORCHAO_DOUBLE)+def test_torchao_conversion_and_its_return_both_fail(cap, tiled, a1, a2, kind, monkeypatch):+ _run(cap, (a1, a2), kind, monkeypatch, tiled, variant = "torchao")+++_RESTORE_POINTS = (":undo", ":settle", "back:", "vae_back:", "vae_park:", "vae_hooks:remove", "vae_hooks:readd",+ "groups:repin", "torchao_back:", "torchao_repin:", "groups:release") # fmt: skip+++def _restore_double_cases():+ """Item 4287727812: a restore (undo) that fails, then a second fault anywhere after it (in the settle / park /+ deferred path): every Change's settle + defer pairing must leave a coherent module or a REQUIRED repair."""+ mp = pytest.MonkeyPatch()+ cases = []+ try:+ for variant, cap, tiled in itertools.product(VARIANTS, (13, 8, 2), (False, True)):+ pts = _run(cap, None, "oom", mp, tiled, verify = False, variant = variant).points+ firsts = [i + 1 for i, p in enumerate(pts) if any(k in p for k in _RESTORE_POINTS)]+ for a1 in firsts:+ n = _run(cap, a1, "oom", mp, tiled, verify = False, variant = variant).decode_points+ cases += [(variant, cap, tiled, a1, a2) for a2 in range(a1 + 1, min(n, a1 + 8) + 1)]+ finally:+ mp.undo()+ return cases+++RESTORE_DOUBLE = _restore_double_cases()+++def test_restore_double_faults_cover_the_park_path():+ assert len(RESTORE_DOUBLE) >= 200, len(RESTORE_DOUBLE)+ assert any(c[0] == "base" and c[2] in (False, True) for c in RESTORE_DOUBLE)++[email protected]("variant,cap,tiled,a1,a2", RESTORE_DOUBLE)+def test_restore_and_its_settle_both_fail(variant, cap, tiled, a1, a2, monkeypatch):+ _run(cap, (a1, a2), "oom", monkeypatch, tiled, variant = variant)