Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: offload-first retry for an OOM VAE decode, fuse GroupNorm+SiLU on uncovered VAEs by structure (#13802)

Daniel Hanc57c21f04719
diff --git a/studio/backend/core/inference/decode_layers.py b/studio/backend/core/inference/decode_layers.py
new file mode 100644
index 00000000000..ed2a78d6a36
--- /dev/null
+++ b/studio/backend/core/inference/decode_layers.py
@@ -0,0 +1,115 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Install and remove one wrapper layer on an object's method slot (a VAE's ``decode`` / ``_decode``), exactly.
+
+``wrap`` records what the slot held on that object (an instance-dict entry, possibly an instance-bound override, or
+nothing, so the class method showed through) and puts the layer in the instance dict. ``unwrap`` puts back exactly
+that when the layer is still outermost. When later wrappers captured it (they hold it in their closures), the layer
+cannot be cut out: it turns into a pass-through to what it wrapped, and the ``__wrapped__`` chain skips it.
+"""
+
+from __future__ import annotations
+
+import functools
+from typing import Any, Callable, Optional
+
+LAYERS_ATTR = "_unsloth_decode_layers"
+_MISSING = object()
+
+
+class Layer:
+ def __init__(self, owner: Any, attr: str, name: str, inner: Callable, original: Any) -> None:
+ self.owner = owner
+ self.attr = attr
+ self.name = name
+ self.inner = inner
+ self.original = original # the owner's own instance-dict entry before wrapping, or _MISSING
+ self.active = True
+ self.fn: Optional[Callable] = None
+
+ @property
+ def had_own(self) -> bool:
+ return self.original is not _MISSING
+
+
+def wrap(owner: Any, attr: str, name: str, make: Callable[[Callable, "Layer"], Callable]) -> Layer:
+ """Install ``make(inner, layer)`` as the outermost ``attr`` layer of ``owner``."""
+ original = owner.__dict__.get(attr, _MISSING)
+ inner = getattr(owner, attr)
+ layer = Layer(owner, attr, name, inner, original)
+ fn = make(inner, layer)
+
+ def dispatch(*args: Any, **kwargs: Any) -> Any:
+ if not layer.active:
+ return layer.inner(*args, **kwargs)
+ return fn(*args, **kwargs)
+
+ functools.update_wrapper(dispatch, fn)
+ dispatch.__wrapped__ = inner
+ dispatch._unsloth_layer = layer
+ layer.fn = dispatch
+ setattr(owner, attr, dispatch)
+ owner.__dict__.setdefault(LAYERS_ATTR, []).append(layer)
+ return layer
+
+
+def find(owner: Any, attr: str, name: str) -> Optional[Layer]:
+ for layer in reversed(getattr(owner, "__dict__", {}).get(LAYERS_ATTR, ()) or ()):
+ if layer.attr == attr and layer.name == name and layer.active:
+ return layer
+ return None
+
+
+def unwrap(layer: Optional[Layer]) -> bool:
+ """Remove ``layer``. True when it was outermost and the slot holds exactly what it held before wrapping; False
+ when later wrappers sit over it (it is now a pass-through) or it was already removed."""
+ if layer is None or not layer.active:
+ return False
+ owner, attr = layer.owner, layer.attr
+ layer.active = False
+ layers = owner.__dict__.get(LAYERS_ATTR, [])
+ if layer in layers:
+ layers.remove(layer)
+ if not layers:
+ owner.__dict__.pop(LAYERS_ATTR, None)
+ current = owner.__dict__.get(attr, _MISSING)
+ if current is layer.fn:
+ original = layer.original
+ for _ in range(16):
+ inner_layer = getattr(original, "_unsloth_layer", None)
+ if inner_layer is None or inner_layer.active or original is not inner_layer.fn:
+ break
+ original = inner_layer.original
+ if original is not _MISSING:
+ owner.__dict__[attr] = original
+ else:
+ owner.__dict__.pop(attr, None)
+ return True
+ fn = None if current is _MISSING else current
+ for _ in range(16):
+ if fn is None:
+ break
+ if getattr(fn, "__wrapped__", None) is layer.fn:
+ try:
+ fn.__wrapped__ = layer.inner
+ except Exception: # noqa: BLE001 - read-only callable: the pass-through still holds
+ pass
+ break
+ fn = next_layer(fn)
+ return False
+
+
+def next_layer(fn: Any) -> Any:
+ """What the decode wrapper ``fn`` calls through to: the decode in its mutable slot (an eager outer wrapper that
+ keeps what it wraps rewritable: the fp16 non-finite check, the ROCm bf16 decode) when it has one, otherwise its
+ ``__wrapped__``. Every walker of a decode chain uses this, so a slot is never skipped."""
+ slot = getattr(fn, "_unsloth_decode_slot", None)
+ if slot is not None:
+ inner = getattr(slot, "__dict__", {}).get("decode")
+ if inner is not None:
+ return inner
+ return getattr(fn, "__wrapped__", None)
+
+
+__all__ = ["LAYERS_ATTR", "Layer", "find", "next_layer", "unwrap", "wrap"]
diff --git a/studio/backend/core/inference/diffusion.py b/studio/backend/core/inference/diffusion.py
index 04af24d9f34..c8c35d6f365 100644
--- a/studio/backend/core/inference/diffusion.py
+++ b/studio/backend/core/inference/diffusion.py
@@ -9267,6 +9267,12 @@ class DiffusionBackend:
)
if vae_fp16:
speed_applied["vae_fp16_decode"] = True
+ # After the speed layer, so it wraps the fused / single-frame _decode the layer installed.
+ try:
+ from .diffusion_vae_oom import install as install_vae_oom_retry
+ install_vae_oom_retry(getattr(pipe, "vae", None), logger, pipe = pipe)
+ except Exception as exc: # noqa: BLE001 - a decode OOM then fails as before
+ logger.warning("diffusion.vae_oom: not installed: %s", exc)
self._raise_if_load_cancelled(_load_token)
if (
transformer_quant_engaged is not None
@@ -11768,6 +11774,14 @@ class DiffusionBackend:
if _denoiser_hooked(state.pipe):
compile_pipe_below_offload_hooks(state.pipe, logger)
arm_graphs_after_placement(state.pipe, speed_applied, logger)
+ # the speed layer just rewrapped vae.decode (a compiled decode guard): the OOM retry's scope goes back on top
+ try:
+ from .diffusion_vae_oom import ensure_outermost
+ ensure_outermost(getattr(state.pipe, "vae", None), logger)
+ except Exception as exc: # noqa: BLE001 - a decode OOM then fails as before
+ logger.warning(
+ "diffusion.vae_oom: scope not restored after the deferred profile: %s", exc
+ )
cuda_graph.arm_block_graphs(
state.pipe,
speed_applied,
diff --git a/studio/backend/core/inference/diffusion_cuda_graph.py b/studio/backend/core/inference/diffusion_cuda_graph.py
index 3383e9be46a..00ac17cbd71 100644
--- a/studio/backend/core/inference/diffusion_cuda_graph.py
+++ b/studio/backend/core/inference/diffusion_cuda_graph.py
@@ -2120,18 +2120,33 @@ def set_bypass(handles: Any, on: bool) -> None:
pass
-def reset_all(handles: Any) -> None:
+def _holds_graphs(handle: Any) -> bool:
+ """True while ``handle`` (or its fallback handle) still holds a captured graph."""
+ if getattr(handle, "cache", None):
+ return True
+ fallback = getattr(handle, "fallback_handle", None)
+ return fallback is not None and fallback is not handle and _holds_graphs(fallback)
+
+
+def reset_all(handles: Any, strict: bool = False) -> None:
"""Drop every captured graph, for every handle (the weights changed).
Drops the pool token with the last graph: a stale token dies on the allocator's
"use_count > 0 INTERNAL ASSERT FAILED", raised after ``torch.cuda.graph`` entered its side
- stream, so the thread is left off the default stream too."""
+ stream, so the thread is left off the default stream too. ``strict``: raise when a handle failed
+ to reset or still holds a graph (a caller about to move weights must not leave one that could replay)."""
+ failed: list = []
for handle in handles or ():
try:
handle.reset()
- except Exception: # noqa: BLE001 - optimisation only
- pass
+ except Exception as exc: # noqa: BLE001 - optimisation only, unless strict
+ failed.append(f"{type(handle).__name__}: {type(exc).__name__}: {exc}")
+ continue
+ if strict and _holds_graphs(handle):
+ failed.append(f"{type(handle).__name__}: still holds a captured graph")
_drop_pool_if_unused()
+ if strict and failed:
+ raise RuntimeError("could not drop every captured CUDA graph: " + "; ".join(failed))
def uninstall_all(handles: Any, *, logger: Any = None) -> None:
diff --git a/studio/backend/core/inference/diffusion_device.py b/studio/backend/core/inference/diffusion_device.py
index 664d6eb8215..2cda790b7cb 100644
--- a/studio/backend/core/inference/diffusion_device.py
+++ b/studio/backend/core/inference/diffusion_device.py
@@ -12,6 +12,7 @@ the capability flags optimisation paths key off.
from __future__ import annotations
import os
+import types
from contextlib import contextmanager
from contextvars import ContextVar
from dataclasses import dataclass
@@ -295,6 +296,9 @@ def install_rocm_vae_bf16_decode(
if mode == "weights" and not parts:
mode = "autocast" # no separable decode half: cast nothing
+ # The wrapped decode lives in a mutable slot (as the fp16 decode's does): a layer that must sit under this cast (a
+ # compiled decode guard under the OOM scope) can be found and rewrapped in place.
+ slot = types.SimpleNamespace(decode = decode)
if mode == "weights":
for part in parts:
part.to(torch.bfloat16)
@@ -304,16 +308,17 @@ def install_rocm_vae_bf16_decode(
def _bf16_decode(z: Any, *args: Any, **kwargs: Any) -> Any:
if isinstance(z, torch.Tensor) and z.is_floating_point():
z = z.to(torch.bfloat16)
- return _as_float32(decode(z, *args, **kwargs), torch)
+ return _as_float32(slot.decode(z, *args, **kwargs), torch)
else:
def _bf16_decode(*args: Any, **kwargs: Any) -> Any:
with torch.autocast(device_type = "cuda", dtype = torch.bfloat16):
- out = decode(*args, **kwargs)
+ out = slot.decode(*args, **kwargs)
return _as_float32(out, torch)
_bf16_decode._unsloth_bf16_decode = True # type: ignore[attr-defined]
+ _bf16_decode._unsloth_decode_slot = slot # type: ignore[attr-defined]
_bf16_decode.__wrapped__ = decode # type: ignore[attr-defined]
vae.decode = _bf16_decode
vae._unsloth_bf16_decode_mode = mode
diff --git a/studio/backend/core/inference/diffusion_gguf_compile.py b/studio/backend/core/inference/diffusion_gguf_compile.py
index 3eda3c1b8a8..5635966fc0d 100644
--- a/studio/backend/core/inference/diffusion_gguf_compile.py
+++ b/studio/backend/core/inference/diffusion_gguf_compile.py
@@ -104,7 +104,8 @@ def _guard(compiled: Any, eager: Any, logger: Any) -> Any:
@functools.wraps(eager)
def guarded(*args: Any, **kwargs: Any) -> Any:
- if not failed:
+ from .diffusion_bg_compile import eager_forced # noqa: PLC0415
+ if not failed and not eager_forced():
# The lazy compile runs in the caller's context; on torch 2.12+ the load's knob writes may not be visible.
diffusion_compile_config.apply()
try:
diff --git a/studio/backend/core/inference/diffusion_memory.py b/studio/backend/core/inference/diffusion_memory.py
index 831700df95b..f38e17dc6a1 100644
--- a/studio/backend/core/inference/diffusion_memory.py
+++ b/studio/backend/core/inference/diffusion_memory.py
@@ -3409,6 +3409,31 @@ def release_resident_groups(
blocks from the last); returns a callable pinning exactly those groups again, or None when nothing was resident.
A request larger than the measured placement reserved (reference images, a bigger canvas, a batch) then runs as
the flat plan. ``denoisers_only`` leaves the text encoders' groups alone (the prompt encode needs them)."""
+ released: list = []
+ streamed: set = set()
+
+ def restore(strict: bool = False) -> None:
+ """Pin the released groups again. ``strict``: raise when any of them is still not resident afterwards (the
+ re-pin catches its own failures and leaves those groups streamed, which callers that must retry need to
+ know)."""
+ for module in released:
+ _keep_groups_resident(
+ module,
+ module._unsloth_resident_room,
+ module._unsloth_resident_device,
+ logger,
+ only = streamed,
+ )
+ if strict:
+ left = [
+ g
+ for module in released
+ for g in (_offload_groups(module) or [])
+ if id(g) in streamed and not getattr(g, "_unsloth_resident", False)
+ ]
+ if left:
+ raise RuntimeError(f"re-pinning resident offload groups left {len(left)} streamed")
+
try:
import torch
@@ -3422,8 +3447,6 @@ def release_resident_groups(
return None
modules.sort(key = lambda m: 0 if _is_text_encoder_module(pipe, m) else 1)
left = int(need_mib) * 1024 * 1024
- released: list = []
- streamed: set = set()
for module in modules:
for group in reversed(_offload_groups(module) or []):
if left <= 0:
@@ -3434,15 +3457,16 @@ def release_resident_groups(
state = getattr(module, "_unsloth_stream_state", None)
if isinstance(state, dict):
state["streamed"] = 1
+ if module not in released:
+ released.append(module)
# tied partners stream together (a resident partner would keep reading the host copy)
for member in [group, *(getattr(group, "_unsloth_tied_groups", None) or [])]:
if not getattr(member, "_unsloth_resident", False):
continue
- _release_group(member)
+ # Recorded before demoting: a release that fails part way is still re-pinned by ``restore``.
streamed.add(id(member))
+ _release_group(member)
left -= int(getattr(member, "_unsloth_resident_bytes", 0))
- if module not in released:
- released.append(module)
if not released:
return None
for module in released:
@@ -3459,21 +3483,12 @@ def release_resident_groups(
reason,
)
- def restore() -> None:
- for module in released:
- _keep_groups_resident(
- module,
- module._unsloth_resident_room,
- module._unsloth_resident_device,
- logger,
- only = streamed,
- )
-
return restore
except Exception as exc: # noqa: BLE001 - the guard still refuses what cannot fit
if logger is not None:
logger.warning("diffusion.memory: releasing resident groups failed (%s)", exc)
- return None
+ # groups released before the failure stay streamed until the caller re-pins them: hand back their restore
+ return restore if released else None
def resident_group_mib(pipe: Any) -> int:
@@ -3492,10 +3507,14 @@ def hook_resident_denoiser(
pipe: Any,
device: Any,
logger: Any = None,
+ resident: bool = True,
) -> bool:
"""Give a hookless resident torchao transformer block-level offload hooks, every group kept resident, so
release_resident_groups can stream part of it. On demand, not at load: hooks at load recompile every load's second
- render. True when installed."""
+ render. True when installed. ``resident=False``: every group streams (the load's flat plan), for a transformer
+ that cannot come back to the card whole. A failure whose own return to the card fails raises, the transformer
+ split between host and card with no hooks (return_torchao_denoiser / hook_resident_denoiser(resident=False) are
+ the ways out)."""
transformer = getattr(pipe, "transformer", None)
if (
transformer is None
@@ -3546,19 +3565,17 @@ def hook_resident_denoiser(
_skip_top_level_copy_back(transformer, logger)
if gkwargs.get("record_stream"):
install_group_prefetch(transformer, onload, logger)
- kept = _keep_groups_resident(transformer, dit_mib + 1, onload, logger)
- # the helper reports its own failure as 0 MiB kept: anything still streamed is a failed install
- if not all(getattr(g, "_unsloth_resident", False) for g in _offload_groups(transformer)):
- raise RuntimeError("not every offload group stayed resident")
+ kept = 0
+ if resident:
+ kept = _keep_groups_resident(transformer, dit_mib + 1, onload, logger)
+ if not all(
+ getattr(g, "_unsloth_resident", False) for g in _offload_groups(transformer)
+ ):
+ raise RuntimeError("not every offload group stayed resident")
except Exception as exc: # noqa: BLE001 - the guard still refuses what cannot fit
- # the apply already moved the weights to their host copies: back to the card, as before the call
- _remove_group_offload_hooks(transformer)
- _drop_module_forward_hooks_since(transformer, torch_hooks)
- from .diffusion_offload_prefetch import PREFETCHER_ATTR
-
- for attr in (PREFETCHER_ATTR, "_unsloth_stream_state"):
- transformer.__dict__.pop(attr, None)
- _return_module_tensors(transformer, onload)
+ # the apply already moved the weights to their host copies: back to the card, as before the call (a return
+ # that itself fails raises: the caller owns the way back)
+ _unhook_and_return(transformer, onload, torch_hooks)
if logger is not None:
logger.warning("diffusion.memory: resident transformer left without hooks (%s)", exc)
return False
@@ -3584,6 +3601,44 @@ def _module_forward_hook_ids(module: Any) -> dict[str, set]:
return {name: set(getattr(module, name, None) or ()) for name in _FORWARD_HOOK_DICTS}
+def _unhook_and_return(transformer: Any, onload: Any, torch_hooks: Optional[dict]) -> None:
+ _remove_group_offload_hooks(transformer)
+ if torch_hooks is not None:
+ _drop_module_forward_hooks_since(transformer, torch_hooks)
+ from .diffusion_offload_prefetch import PREFETCHER_ATTR
+
+ for attr in (PREFETCHER_ATTR, "_unsloth_stream_state"):
+ transformer.__dict__.pop(attr, None)
+ _return_module_tensors(transformer, onload)
+
+
+def return_torchao_denoiser(
+ pipe: Any,
+ device: Any,
+ logger: Any = None,
+ torch_hooks: Optional[dict] = None,
+) -> None:
+ """The way back to where a hookless resident torchao transformer started, from wherever a failed
+ hook_resident_denoiser left it: streamed under offload hooks (settled) -> every group resident again, the state
+ a successful hook_resident_denoiser leaves; split with no hooks -> whole on the card. Raises if it is not there."""
+ import torch
+
+ transformer = getattr(pipe, "transformer", None)
+ if transformer is None:
+ return
+ onload = torch.device(device)
+ if _offload_groups(transformer):
+ _keep_groups_resident(transformer, _module_host_mib(transformer) + 1, onload, logger)
+ if not all(getattr(g, "_unsloth_resident", False) for g in _offload_groups(transformer)):
+ raise RuntimeError("the transformer's offload groups are not all resident again")
+ return
+ _unhook_and_return(transformer, onload, torch_hooks)
+
+
+def module_forward_hook_ids(module: Any) -> dict[str, set]:
+ return _module_forward_hook_ids(module)
+
+
def _drop_module_forward_hooks_since(module: Any, before: dict[str, set]) -> None:
"""Remove the torch forward hooks registered on ``module`` after ``before`` was taken (the group prefetcher's)."""
for name, ids in before.items():
diff --git a/studio/backend/core/inference/diffusion_speed.py b/studio/backend/core/inference/diffusion_speed.py
index 32ac6a7b658..a0af8b82ce2 100644
--- a/studio/backend/core/inference/diffusion_speed.py
+++ b/studio/backend/core/inference/diffusion_speed.py
@@ -896,6 +896,7 @@ def _vae_channels_last_3d(vae: Any, logger: Any, *, fused: bool) -> bool:
logger.info("diffusion.speed: channels_last skipped for %s: %s", name, reason)
return False
relaid = 0
+ touched: list = []
for part in (getattr(vae, "post_quant_conv", None), getattr(vae, "decoder", None)):
if part is None or not callable(getattr(part, "modules", None)):
continue
@@ -908,7 +909,12 @@ def _vae_channels_last_3d(vae: Any, logger: Any, *, fused: bool) -> bool:
fmt = torch.channels_last if weight.dim() == 4 else torch.channels_last_3d
if not weight.is_contiguous(memory_format = fmt):
weight.data = weight.data.contiguous(memory_format = fmt)
+ touched.append(module)
relaid += 1
+ if touched:
+ # Relaid because fusion is planned; fusion matched by structure may still be rejected at its first decode,
+ # which restores these weights' layout (diffusion_vae_fused._restore_contiguous_layout).
+ vae.__dict__.setdefault("_unsloth_channels_last_modules", []).extend(touched)
if logger is not None:
logger.info(
"diffusion.speed: channels_last(_3d) decode weights on %s (%d convs)", name, relaid
@@ -1412,6 +1418,11 @@ def is_compile_failure(exc: BaseException) -> bool:
return bool(kinds) and isinstance(exc, tuple(kinds))
+def _drop_tracebacks(exc: BaseException) -> None:
+ from .diffusion_vae_oom import drop_tracebacks # noqa: PLC0415
+ drop_tracebacks(exc)
+
+
class _CompileGuard:
"""Per-DiT record of a compile that failed at runtime. The first failure flips every guarded block of that DiT to
eager (one shared decision: a half-compiled stack would retry the same broken lowering per block)."""
@@ -1452,7 +1463,7 @@ class _CompileGuard:
raise
guard.fail(exc, owner)
# The handled exception pins inductor's frames (and the traced fake tensors) through the retry.
- exc.__traceback__ = None
+ _drop_tracebacks(exc)
return eager(*args, **kwargs)
guarded._unsloth_compile_guard = guard
@@ -1707,6 +1718,9 @@ def _guard_compiled_decode(
def guarded(*args: Any, **kwargs: Any) -> Any:
if eager_when_tiled and getattr(vae, "use_tiling", False):
return eager(*args, **kwargs)
+ # the one force-eager signal (a background compile in flight, the OOM scope's eager retry)
+ if _bg_eager_forced():
+ return eager(*args, **kwargs)
if not failed:
try:
return compiled(*args, **kwargs)
@@ -1720,10 +1734,13 @@ def _guard_compiled_decode(
try:
vae._unsloth_compile_decode_error = error
vae._unsloth_compiled_decode = False
- if had_own:
- owner.decode = eager
- else:
- del owner.decode
+ # Only while the slot still holds this guard: a layer installed over it since (the OOM scope)
+ # stays, and this guard (``failed``) runs eager under it from now on.
+ if getattr(owner, "__dict__", {}).get("decode") is guarded:
+ if had_own:
+ owner.decode = eager
+ else:
+ del owner.decode
except Exception: # noqa: BLE001 - `failed` still routes this wrapper to eager
pass
if getattr(vae, "_unsloth_eager_contiguous", False):
@@ -1738,12 +1755,30 @@ def _guard_compiled_decode(
error,
)
# The handled exception pins inductor's frames (and the traced fake tensors) through the retry.
- exc.__traceback__ = None
+ _drop_tracebacks(exc)
return eager(*args, **kwargs)
+ # diffusion_vae_oom reads these to retry a runtime OOM of the compiled decode on the eager path
+ guarded._unsloth_compiled_decode = True
+ guarded._unsloth_eager_decode = eager
return guarded
+def _decode_slot_owner(fn: Any) -> Any:
+ """The mutable slot an eager outer wrapper (fp16 non-finite check, ROCm bf16 cast) calls through, found under
+ any layer installed over it since (the OOM scope), or None."""
+ from .decode_layers import next_layer # noqa: PLC0415
+
+ for _ in range(32):
+ if fn is None:
+ return None
+ slot = getattr(fn, "_unsloth_decode_slot", None)
+ if slot is not None:
+ return slot
+ fn = next_layer(fn)
+ return None
+
+
def _compile_vae_decode(
pipe: Any,
logger: Any,
@@ -1754,7 +1789,7 @@ def _compile_vae_decode(
vae = getattr(pipe, "vae", None)
# An outer wrapper that must stay eager (fp16 non-finite check) exposes the slot it calls through.
outer = getattr(vae, "__dict__", {}).get("decode") if vae is not None else None
- owner = getattr(outer, "_unsloth_decode_slot", None) or vae
+ owner = _decode_slot_owner(outer) or vae
decode = getattr(owner, "decode", None) if vae is not None else None
if not callable(decode):
return False
diff --git a/studio/backend/core/inference/diffusion_vae_fused.py b/studio/backend/core/inference/diffusion_vae_fused.py
index a55f47150a9..25aa99df815 100644
--- a/studio/backend/core/inference/diffusion_vae_fused.py
+++ b/studio/backend/core/inference/diffusion_vae_fused.py
@@ -12,6 +12,7 @@ from __future__ import annotations
import os
import sys
+import threading
import types
from functools import lru_cache
from typing import Any, Optional
@@ -504,17 +505,29 @@ def _torch():
return torch
+def _stock_in(t: Any) -> Any:
+ """A kernel input as the kernel may read it."""
+ cls = globals().get("_FusedActCls")
+ if cls is None or type(t) is not cls:
+ return t
+ if t.__dict__.get("_unsloth_is_stock", False):
+ return _plain(t)
+ t._unsloth_abort()
+
+
def group_norm_act(
x: Any,
norm: Any,
act: bool = True,
in_bias: Any = None,
+ on_kernel: Any = None,
) -> Any:
- """``silu?(group_norm(x + in_bias))`` for 3/4/5-D ``x``; channels-last output; 5-D stats span T like nn.GroupNorm."""
+ """``silu?(group_norm(x + in_bias))`` for 3/4/5-D ``x``; channels-last output; 5-D stats span T like nn.GroupNorm.
+ ``on_kernel``: called once the Triton kernels completed for this call (not on a stock fallback)."""
torch = _torch()
+ x, in_bias = _stock_in(x), _stock_in(in_bias)
if not _gn_fusable(x, norm):
return group_norm_reference(x, norm, act, in_bias)
- k = _kernels()
ndim = x.dim()
x5, _ = _as5d(x)
b, c, t, h, w = x5.shape
@@ -527,6 +540,7 @@ def group_norm_act(
b, c, t, h, w = x5.shape
else:
return group_norm_reference(x, norm, act, in_bias)
+ k = _kernels()
g = int(norm.num_groups)
cpg = c // g
hw = h * w
@@ -556,6 +570,8 @@ def group_norm_act(
CPG = cpg, ACT = bool(act), HAS_IN_BIAS = in_bias is not None, BLOCK_P = block_p2, BLOCK_C = block_c,
num_warps = 4,
) # fmt: skip
+ if on_kernel is not None and not torch.compiler.is_compiling():
+ on_kernel() # read by the structural calibration: only a completed Triton launch counts
out = out.permute(0, 4, 1, 2, 3) # B, C, T', H', W'
shape = x.shape
if ndim == 5:
@@ -637,6 +653,7 @@ def rms_norm_act(
``replicate_pad=(ph, pw)``: HunyuanVideo-1.5 replicate padding instead (``cache`` must be None); ``back`` repeats the
last frame (LTX-2)."""
+ x, in_bias = _stock_in(x), _stock_in(in_bias)
torch = _torch()
if replicate_pad is not None and cache is not None:
raise ValueError("replicate padding takes no cache")
@@ -988,6 +1005,8 @@ def _guard(
try:
return fast(*args, **kwargs)
except Exception as exc: # noqa: BLE001
+ if isinstance(exc, FusedActivationRead):
+ raise
for a, saved in zip(lists, snap):
a[:] = saved
if not _is_oom(exc):
@@ -1105,7 +1124,21 @@ def install_group_norm_vae(vae: Any, logger: Any = None) -> int:
return n
+_STRUCTURAL_LAYER = "vae_fused_structural"
+
+
+def _restore_structural_decode(vae: Any) -> None:
+ """Remove the structural calibrator's ``_decode`` layer: exactly what it replaced comes back when it is outermost
+ (an instance-bound override included); under a later layer (the OOM retry) it becomes a pass-through."""
+ from . import decode_layers # noqa: PLC0415
+ decode_layers.unwrap(decode_layers.find(vae, "_decode", _STRUCTURAL_LAYER))
+
+
def uninstall(vae: Any) -> None:
+ _restore_structural_decode(vae)
+ _restore_contiguous_layout(vae)
+ vae.__dict__.pop("_unsloth_vae_structural", None)
+ vae.__dict__.pop("_unsloth_vae_fused_structural", None)
vae.__dict__.pop("_unsloth_vae_fused_installed", None)
vae.__dict__.pop("_unsloth_vae_fused_cl_weights", None)
for m in vae.modules():
@@ -2062,11 +2095,17 @@ SUPPORTED_VAES = frozenset(
def will_install(vae: Any) -> bool:
- """Whether :func:`install` would engage on ``vae`` here (class covered, NVIDIA CUDA, usable Triton, not disabled)."""
+ """Whether :func:`install` would engage on ``vae`` here (class covered or matched by structure, NVIDIA CUDA,
+ usable Triton, not disabled)."""
return (
vae is not None
- and type(vae).__name__ in SUPPORTED_VAES
- and (type(vae).__name__ != "AutoencoderKLLTX2Video" or _ltx2_convs_fusable(vae))
+ and (
+ (
+ type(vae).__name__ in SUPPORTED_VAES
+ and (type(vae).__name__ != "AutoencoderKLLTX2Video" or _ltx2_convs_fusable(vae))
+ )
+ or structural_candidate(vae)
+ )
and runtime_ok()
)
@@ -2083,6 +2122,10 @@ def install(
if done:
return done # idempotent: a dual-DiT family runs the speed layer once per expert over the same VAE
n = _install(vae, logger)
+ if not n and type(vae).__name__ not in SUPPORTED_VAES:
+ n = install_structural(vae, logger)
+ if n:
+ vae._unsloth_vae_fused_structural = True
# Only where measured faster, and only when a fused pass feeds a conv (relaid weights alone slow LTX-2's decode).
if (
n
@@ -2116,3 +2159,781 @@ def _install(vae: Any, logger: Any = None) -> int:
n = install_ltx2_vae(vae, logger)
return n + (install_vectorised_blend(vae) if n else 0)
return 0
+
+
+# Structural path for unmeasured VAEs: a GroupNorm pairs with SiLU only if every output fed an nn.SiLU (hook trace of
+# one tiny decode); the fused decode must match stock within tolerance or the patch is removed.
+# Kill switch ``UNSLOTH_VAE_FUSED_STRUCTURAL=0``.
+STRUCTURAL_ENV = "UNSLOTH_VAE_FUSED_STRUCTURAL"
+# The calibration crop. The trace marks only the norm outputs (cheap at any size), so the crop is sized for the
+# kernels instead: 16 latents keeps every spatial extent a multiple of 16, the same Triton specialisations a full
+# decode uses, so calibrating compiles nothing the real decode would not (an 8-latent crop measured +1.5 to 2 s
+# of extra compiles on a cold Triton cache).
+_PROBE_LATENTS = 16
+_PROBE_FRAMES = 2
+# Calibration verdicts are kept on disk per (class, config, decoder layout, torch, triton, device capability), so a
+# later process skips the trace and the checks. Off: UNSLOTH_VAE_FUSED_STRUCTURAL_CACHE=0.
+STRUCTURAL_CACHE_ENV = "UNSLOTH_VAE_FUSED_STRUCTURAL_CACHE"
+_STRUCTURAL_CACHE_VERSION = 3
+_PROBE_MAX_ABS = 0.125
+_PROBE_MEAN_ABS = 0.01
+
+
+def structural_disabled() -> bool:
+ return os.environ.get(STRUCTURAL_ENV, "").strip().lower() in _FALSE
+
+
+def _plain_group_norms(part: Any) -> list:
+ torch = _torch()
+ return [
+ m
+ for m in part.modules()
+ if isinstance(m, torch.nn.GroupNorm)
+ and type(m).forward is torch.nn.GroupNorm.forward
+ and "forward" not in m.__dict__
+ ]
+
+
+def _plain_silus(part: Any) -> list:
+ torch = _torch()
+ return [
+ m
+ for m in part.modules()
+ if isinstance(m, torch.nn.SiLU)
+ and type(m).forward is torch.nn.SiLU.forward
+ and "forward" not in m.__dict__
+ ]
+
+
+def structural_candidate(vae: Any) -> bool:
+ """A VAE outside the measured set whose decoder holds plain nn.GroupNorm modules, reached through ``_decode``."""
+ if vae is None or structural_disabled() or type(vae).__name__ in SUPPORTED_VAES:
+ return False
+ if getattr(vae, "_unsloth_decode_blocks_bypassed", False):
+ return False
+ decoder = getattr(vae, "decoder", None)
+ if decoder is None or not callable(getattr(vae, "_decode", None)):
+ return False
+ try:
+ return bool(_plain_group_norms(decoder))
+ except Exception: # noqa: BLE001
+ return False
+
+
+def _decoded_tensor(out: Any) -> Any:
+ if isinstance(out, (tuple, list)):
+ return out[0]
+ return getattr(out, "sample", out)
+
+
+def _probe_latent(z: Any) -> Any:
+ sl = [slice(None)] * z.dim()
+ sl[-1] = slice(0, min(_PROBE_LATENTS, z.shape[-1]))
+ sl[-2] = slice(0, min(_PROBE_LATENTS, z.shape[-2]))
+ if z.dim() == 5:
+ sl[2] = slice(0, min(_PROBE_FRAMES, z.shape[2]))
+ return z[tuple(sl)].contiguous()
+
+
+def structural_cache_disabled() -> bool:
+ return (os.environ.get(STRUCTURAL_CACHE_ENV) or "").strip().lower() in (
+ "0",
+ "off",
+ "false",
+ "no",
+ )
+
+
+def _structural_cache_dir() -> Any:
+ from pathlib import Path
+ try:
+ from .diffusion_compile_cache import cache_root
+ return Path(cache_root()) / "vae_structural"
+ except Exception: # noqa: BLE001
+ return None
+
+
+def _vae_config_json(vae: Any) -> str:
+ import json
+
+ cfg = getattr(vae, "config", None)
+ try:
+ data = dict(cfg) if cfg is not None else {}
+ except Exception: # noqa: BLE001
+ data = dict(getattr(cfg, "__dict__", {}) or {})
+ return json.dumps(
+ {k: data[k] for k in sorted(data) if not str(k).startswith("_")},
+ sort_keys = True,
+ default = str,
+ )
+
+
+_FINGERPRINT_ELEMS = 64
+
+
+def _decoder_fingerprint(decoder: Any) -> Optional[str]:
+ """The checkpoint's identity, cheaply: every decoder tensor's name, shape and dtype, plus its first
+ ``_FINGERPRINT_ELEMS`` values (one device-to-host copy). Another checkpoint of the same config (a fine-tune, a
+ different dtype) is another fingerprint. None when the values cannot be read (meta / offloaded weights)."""
+ import hashlib
+
+ torch = _torch()
+ try:
+ named = [*decoder.named_parameters(), *decoder.named_buffers()]
+ if any(getattr(t, "is_meta", False) for _, t in named):
+ return None
+ h = hashlib.sha256(
+ repr([(n, tuple(t.shape), str(t.dtype), str(t.device.type)) for n, t in named]).encode()
+ )
+ with torch.no_grad():
+ heads = [
+ t.detach().reshape(-1)[:_FINGERPRINT_ELEMS].float() for _, t in named if t.numel()
+ ]
+ if heads:
+ by_device: dict = {}
+ for t in heads:
+ by_device.setdefault(t.device, []).append(t)
+ for dev in sorted(by_device, key = str):
+ h.update(torch.cat(by_device[dev]).cpu().numpy().tobytes())
+ return h.hexdigest()[:32]
+ except Exception: # noqa: BLE001 - no identity: the caller does not use the cache
+ return None
+
+
+def structural_cache_key(vae: Any, norms: list, silus: list, device: Any) -> Optional[str]:
+ """Key of a calibration verdict: everything the pairing and the kernels' numerics depend on. Any change (another
+ class, config, decoder layout, checkpoint weights or dtype, hooks, torch, triton or device capability) is a new
+ key, so it is traced and checked again. None (no cache) when the weights' identity cannot be read."""
+ import hashlib
+
+ torch = _torch()
+ try:
+ import triton
+ triton_version = getattr(triton, "__version__", "?")
+ except Exception: # noqa: BLE001
+ triton_version = None
+ try:
+ cap = (
+ tuple(torch.cuda.get_device_capability(device))
+ if getattr(device, "type", None) == "cuda"
+ else None
+ )
+ except Exception: # noqa: BLE001
+ cap = None
+ fingerprint = _decoder_fingerprint(vae.decoder)
+ if fingerprint is None:
+ return None # no cheap identity for these weights: no cached verdict, trace and check every process
+ names = {id(m): n for n, m in vae.decoder.named_modules()}
+ layout = [
+ (
+ names.get(id(m), "?"),
+ type(m).__name__,
+ int(getattr(m, "num_groups", 0)),
+ int(getattr(m, "num_channels", 0)),
+ )
+ for m in norms
+ ] + [(names.get(id(m), "?"), type(m).__name__) for m in silus]
+ blob = repr((
+ _STRUCTURAL_CACHE_VERSION, f"{type(vae).__module__}.{type(vae).__qualname__}", _vae_config_json(vae), layout,
+ str(torch.__version__), triton_version, cap, _PROBE_LATENTS, _PROBE_MAX_ABS, _PROBE_MEAN_ABS, fingerprint,
+ _global_module_hooks(), [_has_user_hooks(m) for m in (*norms, *silus)],
+ )) # fmt: skip
+ return hashlib.sha256(blob.encode()).hexdigest()[:32]
+
+
+def _load_pairing(key: Optional[str], n_norms: int) -> Optional[dict]:
+ import json
+
+ root = _structural_cache_dir()
+ if key is None or root is None or structural_cache_disabled():
+ return None
+ try:
+ data = json.loads((root / f"{key}.json").read_text(encoding = "utf-8"))
+ except Exception: # noqa: BLE001 - absent or unreadable: calibrate
+ return None
+ paired = data.get("paired")
+ if data.get("key") != key or data.get("status") not in ("ok", "rejected", "no_match"):
+ return None
+ if (
+ not isinstance(paired, list)
+ or len(paired) != n_norms
+ or any(p not in (None, True, False) for p in paired)
+ ):
+ return None
+ consumers = data.get("consumers")
+ if data["status"] == "ok" and (
+ not isinstance(consumers, list) or len(consumers) != n_norms
+ or any(not isinstance(c, list) or any(not isinstance(j, int) for j in c) for c in consumers)
+ ): # fmt: skip
+ return None
+ return data
+
+
+def _save_pairing(key: Optional[str], record: dict) -> None:
+ import json
+
+ root = _structural_cache_dir()
+ if key is None or root is None or structural_cache_disabled():
+ return
+ try:
+ root.mkdir(parents = True, exist_ok = True)
+ tmp = root / f".{key}.{os.getpid()}.tmp"
+ tmp.write_text(json.dumps({**record, "key": key}, default = str), encoding = "utf-8")
+ os.replace(tmp, root / f"{key}.json")
+ except Exception: # noqa: BLE001 - a cache that cannot be written only costs the next process a calibration
+ pass
+
+
+def _has_user_hooks(module: Any) -> bool:
+ for name in ("_forward_hooks", "_forward_pre_hooks", "_backward_hooks", "_backward_pre_hooks"):
+ if getattr(module, name, None):
+ return True
+ return False
+
+
+def _global_module_hooks() -> bool:
+ try:
+ from torch.nn.modules import module as mm
+ except Exception: # noqa: BLE001
+ return True
+ return any(
+ getattr(mm, name, None)
+ for name in ("_global_forward_hooks", "_global_forward_pre_hooks", "_global_backward_hooks")
+ )
+
+
+# Tensor methods that only read metadata: not a reader of the values.
+_META_FUNCS = frozenset((
+ "__get__", "dim", "size", "stride", "numel", "nelement", "ndimension", "element_size", "is_contiguous",
+ "storage_offset", "is_floating_point", "is_complex", "get_device", "__len__", "data_ptr",
+)) # fmt: skip
+
+
+# The reader callback of the structural trace is per thread, and traces run one at a time (once per VAE per process):
+# two first decodes on two threads can neither clear nor overwrite each other's callback.
+_TRACE_LOCAL = threading.local()
+_TRACE_LOCK = threading.RLock()
+
+
+def _tracked_tensor_class() -> Any:
+ """A tensor subclass marking a norm's output: every torch function that touches it reports in, and its results
+ come back as plain tensors, so only the direct readers of a norm output are seen (cheap: no per-op hook on the
+ rest of the decoder)."""
+ torch = _torch()
+ cls = globals().get("_TrackedCls")
+ if cls is not None:
+ return cls
+ from torch.utils._pytree import tree_leaves
+
+ class _Tracked(torch.Tensor):
+ @classmethod
+ def __torch_function__(
+ cls,
+ func,
+ types_,
+ args = (),
+ kwargs = None,
+ ):
+ kwargs = kwargs or {}
+ # this thread's trace only: a concurrent trace (another VAE, another thread) has its own callback
+ trace = getattr(_TRACE_LOCAL, "read", None)
+ if trace is not None and getattr(func, "__name__", "") not in _META_FUNCS:
+ for leaf in tree_leaves((args, kwargs)):
+ if isinstance(leaf, _Tracked):
+ trace(leaf)
+ with torch._C.DisableTorchFunctionSubclass():
+ return func(*args, **kwargs)
+
+ globals()["_TrackedCls"] = _Tracked
+ return _Tracked
+
+
+def _trace_group_norm_silu(norms: list, silus: list, run: Any) -> tuple:
+ """(calls per norm, outputs of each norm consumed ONLY by an nn.SiLU, run output, info) over one ``run()``.
+
+ A norm's output counts as consumed by the SiLU only when the trace proves the SiLU is its sole reader: the output
+ is marked (a tensor subclass), every torch function reading it (a view, an in-place update, a residual add) is
+ recorded, and any such read outside the SiLU call marks it shared. ``info["shared"]``: per norm, reads elsewhere.
+ Norms or SiLUs carrying hooks of their own, global module hooks, or no subclass support: nothing counts as
+ exclusively consumed."""
+ n = len(norms)
+ calls, consumed, shared = [0] * n, [0] * n, [0] * n
+ produced: dict = {}
+ keep: list = []
+ handles = []
+ in_silu: list = []
+ hooked = _global_module_hooks() or any(_has_user_hooks(m) for m in (*norms, *silus))
+ try:
+ tracked = _tracked_tensor_class()
+ except Exception: # noqa: BLE001 - no way to see every reader: fuse GroupNorm alone
+ tracked = None
+
+ def norm_hook(idx: int):
+ def hook(_m, _inp, out):
+ calls[idx] += 1
+ if tracked is None or type(out) is not _torch().Tensor:
+ return None
+ marked = out.as_subclass(tracked)
+ produced[id(marked)] = idx
+ keep.append(marked)
+ return marked
+
+ return hook
+
+ silu_index = {id(m): j for j, m in enumerate(silus)}
+ readers: list = [set() for _ in range(n)]
+
+ def silu_pre(m, inp):
+ idx = produced.get(id(inp[0])) if inp else None
+ if idx is not None:
+ consumed[idx] += 1
+ readers[idx].add(silu_index.get(id(m), -1))
+ in_silu.append(idx)
+
+ def silu_post(_m, _inp, _out):
+ if in_silu:
+ in_silu.pop()
+
+ def read(t: Any) -> None:
+ idx = produced.get(id(t))
+ if idx is not None and not (in_silu and in_silu[-1] == idx):
+ shared[idx] += 1
+
+ with _TRACE_LOCK:
+ previous = getattr(_TRACE_LOCAL, "read", None)
+ try:
+ for i, m in enumerate(norms):
+ handles.append(m.register_forward_hook(norm_hook(i)))
+ for m in silus:
+ handles.append(m.register_forward_pre_hook(silu_pre))
+ handles.append(m.register_forward_hook(silu_post))
+ if tracked is not None:
+ _TRACE_LOCAL.read = read
+ out = run()
+ finally:
+ _TRACE_LOCAL.read = previous
+ for h in handles:
+ h.remove()
+ keep.clear()
+ if hooked or tracked is None:
+ consumed = [0] * n
+ else:
+ consumed = [k if sh == 0 else 0 for k, sh in zip(consumed, shared)]
+ if out is not None and tracked is not None and isinstance(out, tracked):
+ out = out.as_subclass(_torch().Tensor)
+ return calls, consumed, out, {"shared": shared, "consumers": [sorted(r) for r in readers]}
+
+
+def _site_hooks(norms: list, paired: list) -> tuple:
+ """Forward hooks comparing each fused site with stock on the input it is given, during the crop decode itself
+ (one pass: no extra kernel specialisations). Returns (handles, worst) where worst is [max abs, mean abs, sites]."""
+ torch = _torch()
+ worst = [0.0, 0.0, 0]
+ handles = []
+ for norm, act in zip(norms, paired):
+ if act is None:
+ continue
+
+ def hook(
+ m,
+ inp,
+ out,
+ act = act,
+ ):
+ out = _plain(out)
+ ref = _stock_forward(m)(inp[0])
+ if act:
+ ref = torch.nn.functional.silu(ref)
+ err = (out.float() - ref.float()).abs()
+ worst[0] = max(worst[0], float(err.max()))
+ worst[1] = max(worst[1], float(err.mean()))
+ worst[2] += 1
+
+ handles.append(norm.register_forward_hook(hook))
+ return handles, worst
+
+
+# Tensor properties that are metadata only (a read of these is not a read of the values).
+_META_PROPS = frozenset((
+ "shape", "dtype", "device", "is_cuda", "layout", "ndim", "requires_grad", "is_leaf", "grad_fn", "names",
+ "is_sparse", "is_quantized", "is_meta", "is_mkldnn", "is_nested", "itemsize", "nbytes",
+)) # fmt: skip
+
+
+_RAW_READS = frozenset(("data_ptr", "untyped_storage", "storage", "_typed_storage"))
+
+
+def _is_meta_read(func: Any) -> bool:
+ name = getattr(func, "__name__", "")
+ if name == "__get__":
+ return getattr(getattr(func, "__self__", None), "__name__", "") in _META_PROPS
+ return name in _META_FUNCS and name not in _RAW_READS
+
+
+class FusedActivationRead(RuntimeError):
+ """A fused GroupNorm+SiLU output was read by something other than its paired SiLU. It holds only the activated
+ values: the stock GroupNorm value such a reader needs was never materialised, and recomputing it later from the
+ norm's input is not exact (the decoder may have written to that input in place since, and inference tensors,
+ which Studio renders with, keep no version counter to tell). So that decode stops here; the structural decode
+ layer turns activation fusion off for this VAE (GroupNorm stays fused), records that, and decodes again."""
+
+
+def _fused_act_class() -> Any:
+ """The output of a fused GroupNorm+SiLU: it holds the ACTIVATED values, which only the SiLU module(s) the trace
+ paired with it may read (once: the first consumes the mark). Any other read of the values (a residual, another
+ norm, a view, an in-place write, a raw pointer, a SiLU after the mark was consumed), whatever the shape or
+ dataflow of the decode that does it, raises FusedActivationRead: the decode is redone without activation fusion,
+ so a wrong value is never returned and nothing aliases the norm's (mutable) input. After an in-place paired SiLU
+ the storage IS the stock tensor's (silu_ overwrote it), so reads then see it as a plain tensor."""
+ cls = globals().get("_FusedActCls")
+ if cls is not None:
+ return cls
+ torch = _torch()
+ from torch.utils._pytree import tree_leaves
+
+ class _FusedAct(torch.Tensor):
+ @classmethod
+ def __torch_function__(
+ cls,
+ func,
+ types_,
+ args = (),
+ kwargs = None,
+ ):
+ kwargs = kwargs or {}
+ if not _is_meta_read(func):
+ for leaf in tree_leaves((args, kwargs)):
+ if isinstance(leaf, _FusedAct) and not leaf.__dict__.get(
+ "_unsloth_is_stock", False
+ ):
+ leaf._unsloth_abort(func)
+ with torch._C.DisableTorchFunctionSubclass():
+ return func(*args, **kwargs)
+
+ def _unsloth_abort(self, func: Any = None) -> None:
+ state = self.__dict__.get("_unsloth_state")
+ if state is not None:
+ state.foreign_reads = getattr(state, "foreign_reads", 0) + 1
+ raise FusedActivationRead(
+ f"fused GroupNorm+SiLU output read by {getattr(func, '__name__', 'a kernel')} before / instead of its "
+ "paired SiLU"
+ )
+
+ globals()["_FusedActCls"] = _FusedAct
+ return _FusedAct
+
+
+def _plain(t: Any) -> Any:
+ """``t`` as a plain tensor: a fused GroupNorm+SiLU output's activated values (no recompute)."""
+ cls = globals().get("_FusedActCls")
+ if cls is None or type(t) is not cls:
+ return t
+ torch = _torch()
+ with torch._C.DisableTorchFunctionSubclass():
+ return t.as_subclass(torch.Tensor)
+
+
+def _compiling() -> bool:
+ try:
+ return bool(_torch().compiler.is_compiling())
+ except Exception: # noqa: BLE001
+ return False
+
+
+def _fast_gn_maybe_act(
+ norm: Any,
+ act: bool,
+ state: Any,
+ silus: Any = None,
+) -> Any:
+ """The fused GroupNorm(+SiLU) forward. A fused activation's output is a _FusedAct bound to exactly the SiLU
+ module(s) the trace saw consume it (``silus``; None: any one SiLU): the first of those to read it gets the
+ activated values and consumes the mark; every other reader gets the stock GroupNorm output. Under torch.compile
+ (no tensor subclass in the graph) the activation is left to the SiLU, which inductor fuses itself."""
+ token = "any" if silus is None else frozenset(id(m) for m in silus)
+ fused_cls = _fused_act_class() if act else None
+ torch = _torch()
+
+ def fast(x: Any) -> Any:
+ # any patched norm (fused activation or GroupNorm only) reading a fused output reads its stock value: the
+ # kernel reads raw memory
+ x = _stock_in(x)
+ fuse = act and not _compiling()
+ ran: list = []
+ y = group_norm_act(x, norm, act = fuse, on_kernel = lambda: ran.append(1))
+ if ran:
+ state.kernel_calls += 1
+ probe = getattr(getattr(state, "local", None), "kernel_runs", None)
+ if probe is not None:
+ probe.append(1)
+ state.calls += 1
+ if fuse:
+ with torch._C.DisableTorchFunctionSubclass():
+ y = y.as_subclass(fused_cls)
+ y._unsloth_state, y._unsloth_silu_for = state, token
+ return y
+
+ return fast
+
+
+def _silu_passthrough(silu: Any) -> Any:
+ torch = _torch()
+ fused_cls = _fused_act_class()
+ inplace = bool(getattr(silu, "inplace", False))
+
+ def forward(x: Any) -> Any:
+ token = getattr(x, "_unsloth_silu_for", None) if type(x) is fused_cls else None
+ if token is not None and (token == "any" or id(silu) in token):
+ x._unsloth_silu_for = None
+ if inplace:
+ x._unsloth_is_stock = (
+ True # stock silu_ overwrote the norm output: its storage is now exactly that
+ )
+ return _plain(x)
+ return torch.nn.functional.silu(x, inplace = inplace)
+
+ forward._unsloth_vae_fused = True
+ return forward
+
+
+def _structural_patch(
+ norms: list,
+ paired: list,
+ silus: list,
+ state: Any,
+ logger: Any,
+ consumers: Optional[list] = None,
+) -> int:
+ """``consumers``: per norm, the indices into ``silus`` of the SiLU module(s) the trace saw read its output; only
+ those SiLUs consume its fused-activation mark (None: any SiLU, once)."""
+ n = 0
+ patched_silus: list = []
+ for i, (norm, act) in enumerate(zip(norms, paired)):
+ if act is None:
+ continue
+ mine = None
+ if act and consumers is not None:
+ mine = [silus[j] for j in consumers[i] if 0 <= j < len(silus)]
+ patched_silus += mine
+ _guard(
+ norm,
+ _fast_gn_maybe_act(norm, act, state, mine),
+ _stock_forward(norm),
+ "GroupNorm+SiLU" if act else "GroupNorm",
+ logger,
+ )
+ n += 1
+ if any(paired):
+ targets = silus if consumers is None else patched_silus
+ for silu in {id(m): m for m in targets}.values():
+ silu.forward = _silu_passthrough(silu)
+ return n
+
+
+def _structural_unpatch(
+ norms: list,
+ silus: list,
+ vae: Any = None,
+) -> None:
+ for m in (*norms, *silus):
+ m.__dict__.pop("forward", None)
+ m.__dict__.pop("_unsloth_vae_fused_failed", None)
+ if vae is not None:
+ _restore_contiguous_layout(vae)
+
+
+def _restore_contiguous_layout(vae: Any) -> int:
+ """Undo the channels_last(_3d) relayout the speed layer applied because fusion was planned: without the fused
+ passes that layout decodes slower. Only the conv weights it recorded are touched."""
+ modules = vae.__dict__.pop("_unsloth_channels_last_modules", None) or ()
+ n = 0
+ for module in modules:
+ weight = getattr(module, "weight", None)
+ if weight is not None and not weight.is_contiguous():
+ weight.data = weight.data.contiguous()
+ n += 1
+ return n
+
+
+def _clear_decoder_cache(vae: Any) -> None:
+ """Reset a causal decoder's frame / feature cache (Wan, Qwen-Image, LTX-2, HunyuanVideo expose clear_cache)."""
+ clear = getattr(vae, "clear_cache", None)
+ if callable(clear):
+ try:
+ clear()
+ except Exception: # noqa: BLE001 - a VAE without a usable cache has nothing to reset
+ pass
+
+
+def install_structural(vae: Any, logger: Any = None) -> int:
+ """Arm the dataflow-matched fused GroupNorm(+SiLU) path on ``vae``'s decoder; it is calibrated (paired, checked
+ against stock) on the first CUDA decode and removed if the check fails. Returns the plain GroupNorm count."""
+ if not structural_candidate(vae) or not runtime_ok():
+ return 0
+ if getattr(vae, "_unsloth_vae_structural", None) is not None:
+ return vae._unsloth_vae_structural.norms
+ decoder = vae.decoder
+ norms = _plain_group_norms(decoder)
+ silus = _plain_silus(decoder)
+ state = types.SimpleNamespace(
+ status = "pending", norms = len(norms), gn_silu = 0, gn = 0, calls = 0, kernel_calls = 0,
+ max_abs = None, mean_abs = None, site_max_abs = None, site_mean_abs = None, local = threading.local(),
+ paired = None, consumers = None, cached = False, key = None, act_off = False, foreign_reads = 0,
+ ) # fmt: skip
+ inner = vae._decode
+ torch = _torch()
+
+ def _fresh(fn: Any) -> Any:
+ def call(*a: Any, **k: Any) -> Any:
+ _clear_decoder_cache(vae)
+ return fn(*a, **k)
+
+ return call
+
+ def calibrate(z: Any, args: tuple, kwargs: dict) -> None:
+ key = state.key = structural_cache_key(vae, norms, silus, getattr(z, "device", None))
+ cached = _load_pairing(key, len(norms))
+ if cached is not None:
+ # a verdict for this exact class / config / layout / torch / triton / device: no trace, no checks
+ state.cached = True
+ if cached["status"] != "ok":
+ state.status = cached["status"]
+ _structural_unpatch(norms, silus, vae)
+ return
+ paired = state.paired = cached["paired"]
+ state.consumers = cached["consumers"]
+ _structural_patch(norms, paired, silus, state, logger, cached["consumers"])
+ state.status = "ok"
+ state.gn_silu = sum(1 for p in paired if p)
+ state.gn = sum(1 for p in paired if p is False)
+ state.max_abs, state.mean_abs = cached.get("max_abs"), cached.get("mean_abs")
+ state.site_max_abs, state.site_mean_abs = (
+ cached.get("site_max_abs"),
+ cached.get("site_mean_abs"),
+ )
+ state.calls = 0
+ return
+ _calibrate(z, args, kwargs)
+ if state.status in ("ok", "rejected", "no_match"):
+ _save_pairing(key, {
+ "status": state.status, "paired": state.paired, "consumers": state.consumers,
+ "max_abs": state.max_abs, "mean_abs": state.mean_abs,
+ "site_max_abs": state.site_max_abs, "site_mean_abs": state.site_mean_abs,
+ }) # fmt: skip
+
+ def _calibrate(z: Any, args: tuple, kwargs: dict) -> None:
+ tiny = _probe_latent(z)
+ with torch.no_grad():
+ calls, consumed, ref, info = _trace_group_norm_silu(
+ norms, silus, lambda: _decoded_tensor(_fresh(inner)(tiny, *args, **kwargs))
+ )
+ # fused with its SiLU only when that SiLU is the output's sole reader; otherwise GroupNorm alone
+ paired = [None if c == 0 else (c == k) for c, k in zip(calls, consumed)]
+ state.paired, state.consumers = paired, info["consumers"]
+ n = _structural_patch(norms, paired, silus, state, logger, info["consumers"])
+ if n == 0:
+ state.status = "no_match"
+ _structural_unpatch(norms, silus, vae)
+ return
+ state.local.kernel_runs = []
+ handles, worst = _site_hooks(norms, paired)
+ try:
+ out = _decoded_tensor(_fresh(inner)(tiny, *args, **kwargs))
+ fused_ran = bool(state.local.kernel_runs)
+ finally:
+ state.local.kernel_runs = None
+ for h in handles:
+ h.remove()
+ site = (worst[0], worst[1]) if worst[2] else None
+ err = (out.float() - ref.float()).abs()
+ state.max_abs, state.mean_abs = float(err.max()), float(err.mean())
+ state.site_max_abs, state.site_mean_abs = site if site is not None else (None, None)
+ sites_ok = site is not None and site[0] <= _PROBE_MAX_ABS and site[1] <= _PROBE_MEAN_ABS
+ if (
+ fused_ran
+ and sites_ok
+ and state.max_abs <= _PROBE_MAX_ABS
+ and state.mean_abs <= _PROBE_MEAN_ABS
+ ):
+ state.status = "ok"
+ state.gn_silu = sum(1 for p in paired if p)
+ state.gn = sum(1 for p in paired if p is False)
+ state.calls = 0
+ if logger is not None:
+ logger.info(
+ "diffusion.vae_fused: %s matched by structure: %d GroupNorm+SiLU, %d GroupNorm fused "
+ "(probe max |d| %.4f, mean |d| %.5f)",
+ type(vae).__name__, state.gn_silu, state.gn, state.max_abs, state.mean_abs,
+ ) # fmt: skip
+ else:
+ state.status = "rejected"
+ _structural_unpatch(norms, silus, vae)
+ if logger is not None:
+ logger.warning(
+ "diffusion.vae_fused: %s structural fusion rejected (probe max |d| %s, mean |d| %s)",
+ type(vae).__name__, state.max_abs, state.mean_abs,
+ ) # fmt: skip
+
+ def _decode(z: Any, *args: Any, **kwargs: Any) -> Any:
+ if state.status == "pending" and getattr(z, "is_cuda", False):
+ with _TRACE_LOCK:
+ if state.status == "pending":
+ try:
+ calibrate(z, args, kwargs)
+ except Exception as exc: # noqa: BLE001 - never fail a decode over the probe
+ state.status = "rejected"
+ _structural_unpatch(norms, silus, vae)
+ if logger is not None:
+ logger.warning("diffusion.vae_fused: structural probe failed: %s", exc)
+ _clear_decoder_cache(
+ vae
+ ) # the real decode must not see the probe's causal cache
+ aborted = False
+ try:
+ return inner(z, *args, **kwargs)
+ except FusedActivationRead:
+ if state.status != "ok":
+ raise
+ aborted = True
+ if aborted:
+ _activation_fusion_off()
+ _clear_decoder_cache(vae)
+ return inner(z, *args, **kwargs)
+
+ def _activation_fusion_off() -> None:
+ """A fused activation was read out of turn at this decode's shape / dataflow: from now on (and in later
+ processes: the cached verdict is downgraded) this VAE fuses GroupNorm only and every SiLU runs stock."""
+ with _TRACE_LOCK:
+ if not any(state.paired or ()):
+ return
+ paired = [False if p else p for p in state.paired]
+ _structural_unpatch(norms, silus)
+ _structural_patch(norms, paired, silus, state, logger)
+ state.paired, state.consumers = paired, [[] for _ in norms]
+ state.gn += state.gn_silu
+ state.gn_silu = 0
+ state.act_off = True
+ if logger is not None:
+ logger.warning(
+ "diffusion.vae_fused: %s read a fused GroupNorm+SiLU output out of turn; decoding again with "
+ "GroupNorm fused alone", type(vae).__name__,
+ ) # fmt: skip
+ _save_pairing(getattr(state, "key", None), {
+ "status": "ok", "paired": paired, "consumers": state.consumers,
+ "max_abs": state.max_abs, "mean_abs": state.mean_abs,
+ "site_max_abs": state.site_max_abs, "site_mean_abs": state.site_mean_abs,
+ }) # fmt: skip
+
+ _decode._unsloth_vae_fused = True
+ vae._unsloth_vae_structural = state
+ from . import decode_layers # noqa: PLC0415
+
+ decode_layers.wrap(vae, "_decode", _STRUCTURAL_LAYER, lambda _inner, _layer: _decode)
+ if logger is not None:
+ logger.info(
+ "diffusion.vae_fused: %s is outside the measured set; %d GroupNorms will be matched by structure "
+ "on the first decode",
+ type(vae).__name__,
+ len(norms),
+ )
+ return len(norms)
diff --git a/studio/backend/core/inference/diffusion_vae_oom.py b/studio/backend/core/inference/diffusion_vae_oom.py
new file mode 100644
index 00000000000..3326b539e29
--- /dev/null
+++ b/studio/backend/core/inference/diffusion_vae_oom.py
@@ -0,0 +1,1183 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Recover a VAE decode that runs out of memory by offloading, never by shrinking tiles.
+
+Wraps the instance ``_decode`` (under the decode-level wrappers, over the tiled decode). On an OOM, each step runs only
+if the previous one still runs out of memory:
+
+a. Free the decode's device without changing the math: Studio's resident offload groups (text encoders, denoiser
+ blocks) stream again, captured CUDA graphs are dropped, and every other pipeline component still whole on the card
+ moves to the host. The SAME decode is retried, so an untiled decode stays untiled and the output is unchanged.
+b. An untiled decode retries tiled at the VAE's stock tile sizes: ComfyUI's single tiled retry.
+c. The VAE's own weights stream from the host per layer (diffusers leaf-level group offload), still at stock tiles.
+
+Tile sizes are never touched: tiles under the stock size bring back visible seams. Everything moved goes back where
+it was once the decode returns or fails. Kill switch ``UNSLOTH_VAE_OOM_RETRY=0``.
+"""
+
+from __future__ import annotations
+
+import functools
+import gc
+import os
+import time
+import weakref
+from typing import Any, Callable, Optional
+
+from . import decode_layers
+
+OOM_RETRY_ENV = "UNSLOTH_VAE_OOM_RETRY"
+HOST_MARGIN_MIB = 2048
+_SESSION_ATTR = "_unsloth_vae_oom_session"
+_PIPE_ATTR = "_unsloth_vae_oom_pipe"
+
+
+def oom_retry_disabled() -> bool:
+ return os.environ.get(OOM_RETRY_ENV, "").strip().lower() in ("0", "false", "no", "off")
+
+
+def _is_oom_one(exc: BaseException) -> bool:
+ if any(cls.__name__ in ("OutOfMemoryError", "OutOfMemoryError_") for cls in type(exc).__mro__):
+ return True
+ text = str(exc).lower()
+ return isinstance(exc, RuntimeError) and (
+ "out of memory" in text or "tried to allocate" in text
+ )
+
+
+def is_oom(exc: BaseException) -> bool:
+ """An allocator failure, also when wrapped: torch.compile re-raises an OOM inside a compiler error whose own
+ message is generic, and a compiled or parallel decode can report it as an ``ExceptionGroup`` member, so
+ ``__cause__``, ``__context__`` and group members are walked (bounded)."""
+ seen: set = set()
+ pending: list = [exc]
+ while pending and len(seen) < 32:
+ cur = pending.pop(0)
+ if cur is None or id(cur) in seen:
+ continue
+ seen.add(id(cur))
+ if _is_oom_one(cur):
+ return True
+ members = getattr(cur, "exceptions", None)
+ if isinstance(members, (tuple, list)):
+ pending.extend(m for m in members if isinstance(m, BaseException))
+ pending.extend((cur.__cause__, cur.__context__))
+ return False
+
+
+def tile_attrs(vae: Any) -> dict:
+ """The tile geometry a VAE exposes; the retry never changes it."""
+ out = {}
+ for axis in ("size", "height", "width"):
+ for kind in (
+ "tile_sample_min",
+ "tile_latent_min",
+ "tile_sample_stride",
+ "tile_sample_min_overlap",
+ ):
+ name = f"{kind}_{axis}"
+ value = vae.__dict__.get(name, getattr(type(vae), name, None))
+ if isinstance(value, int) and not isinstance(value, bool) and value > 0:
+ out[name] = value
+ return out
+
+
+def _empty_cache(device: Any) -> None:
+ try:
+ import torch
+ if getattr(device, "type", None) == "cuda":
+ torch.cuda.empty_cache()
+ except Exception: # noqa: BLE001
+ pass
+
+
+def _release_cache(z: Any, vae: Any = None) -> None:
+ try:
+ clear = getattr(vae, "clear_cache", None)
+ if callable(clear):
+ clear()
+ except Exception: # noqa: BLE001
+ pass
+ gc.collect()
+ _empty_cache(getattr(z, "device", None))
+
+
+def drop_tracebacks(exc: Optional[BaseException]) -> Optional[BaseException]:
+ """Clear the traceback of ``exc`` and of every exception chained to it (``__cause__`` / ``__context__``, cycles
+ guarded): a kept exception (to re-raise if nothing recovers) must not pin the failed attempt's frames and the
+ tensors they hold through the retry. Every place the scope or the ladder keeps an exception calls this first."""
+ seen: set = set()
+ todo = [exc]
+ while todo:
+ e = todo.pop()
+ if e is None or id(e) in seen:
+ continue
+ seen.add(id(e))
+ e.__traceback__ = None
+ todo += [e.__cause__, e.__context__]
+ for sub in getattr(e, "exceptions", ()) or ():
+ todo.append(sub)
+ return exc
+
+
+def clear_partial_decode(vae: Any, z: Any = None) -> None:
+ """Before ANY retry of a decode that failed part way: drop a causal decoder's frame / feature cache (Wan,
+ Qwen-Image, LTX-2 and HunyuanVideo expose ``clear_cache``), which still holds the failed attempt's state and
+ tensors, and return the freed blocks to the device. Every fallback (wide tiles -> stock tiles, LTX-2 tiles ->
+ the floor tile, untiled -> tiled, the ladder's own steps) calls this first."""
+ _release_cache(z, vae)
+
+
+def _compiling() -> bool:
+ try:
+ import torch
+ return bool(torch.compiler.is_compiling())
+ except Exception: # noqa: BLE001
+ return False
+
+
+def _outer_gate_owns_untiled(vae: Any) -> bool:
+ """video_vae_untiled's gate turns tiling off for its own attempt and retries tiled itself. Outer wrappers (the ROCm
+ bf16 decode) expose it only through ``__wrapped__``."""
+ fn = vae.__dict__.get("decode")
+ for _ in range(8):
+ if fn is None:
+ return False
+ if getattr(fn, "_unsloth_untiled_decode", False):
+ return True
+ fn = decode_layers.next_layer(fn)
+ return False
+
+
+def _module_tensors(module: Any) -> list:
+ return [*module.parameters(), *module.buffers()]
+
+
+def _group_hooked(module: Any) -> bool:
+ """Under diffusers group offloading hooks (it streams where it is)."""
+ hooks = getattr(getattr(module, "_diffusers_hook", None), "hooks", None) or {}
+ return any("group_offloading" in str(name) for name in hooks)
+
+
+def _hooked(module: Any) -> bool:
+ """Placed by an offload mechanism (accelerate model / sequential offload, diffusers group offload): that mechanism
+ owns where its weights live."""
+ for m in module.modules():
+ if getattr(m, "_hf_hook", None) is not None:
+ return True
+ hooks = getattr(getattr(m, "_diffusers_hook", None), "hooks", None) or {}
+ if any("group_offloading" in str(name) for name in hooks):
+ return True
+ return False
+
+
+def _whole_on(module: Any, device: Any) -> bool:
+ tensors = _module_tensors(module)
+ return bool(tensors) and all(t.device == device for t in tensors)
+
+
+def _nbytes(module: Any) -> int:
+ seen: set = set()
+ total = 0
+ for t in _module_tensors(module):
+ if id(t) in seen:
+ continue
+ seen.add(id(t))
+ total += int(t.numel()) * int(t.element_size())
+ return total
+
+
+def _host_free_mib() -> Optional[int]:
+ try:
+ from .diffusion_memory import _available_system_memory_mib
+ return _available_system_memory_mib()
+ except Exception: # noqa: BLE001
+ return None
+
+
+def _holds_torchao(module: Any) -> bool:
+ try:
+ from .diffusion_memory import _holds_torchao_weights
+ return bool(_holds_torchao_weights(module))
+ except Exception: # noqa: BLE001
+ return False
+
+
+def _return_to(module: Any, device: Any) -> None:
+ try:
+ module.to(device)
+ except Exception: # noqa: BLE001 - a weight Module.to cannot move: tensor by tensor, as the offload path does
+ from .diffusion_memory import _return_module_tensors
+ _return_module_tensors(module, device)
+
+
+def _to_host(module: Any) -> None:
+ module.to("cpu")
+
+
+def _placement(module: Any, device: Any) -> str:
+ """How a component sits on the card: "plain" (whole there, no hooks), "torchao_resident" (whole there, torchao
+ weights, no hooks), "groups" (resident offload groups), or "managed" (an offload mechanism places it)."""
+ if getattr(module, "_unsloth_resident_room", None):
+ return "groups"
+ if _hooked(module):
+ return "managed"
+ if not _whole_on(module, device):
+ return "managed"
+ return "torchao_resident" if _holds_torchao(module) else "plain"
+
+
+def _call_restore(restore: Callable) -> None:
+ try:
+ restore(strict = True)
+ except TypeError as exc:
+ if "strict" not in str(exc):
+ raise
+ restore()
+
+
+_DEFERRED_ATTR = "_unsloth_vae_oom_deferred"
+_RECORD_ATTR = "_unsloth_vae_oom_record"
+
+
+class DeferredReturn:
+ """The way back for state a decode's restore could not undo (the card filled up meanwhile).
+
+ Registered on the module that has to come back: a forward pre-hook retries every pending undo before the module
+ runs, and the module carries an accelerate-shaped ``_hf_hook`` whose ``execution_device`` is the card, so
+ ``DiffusionPipeline._execution_device`` still resolves the card while the module waits on the host (diffusers
+ reads that attribute for offloaded components; ``apply_forward_hook`` decode methods call ``pre_forward``). Each
+ pending undo leaves only once it has succeeded; when none is left the hooks go. ``required``: the module cannot run
+ where it is (weights on the host, no offload hooks), so a retry that still runs out of memory raises; otherwise
+ (the VAE streams through its offload hooks, resident groups stream) it runs as it is and retries next time."""
+
+ no_grad = False
+
+ def __init__(self, module: Any, device: Any, required: Any) -> None:
+ self.module_ref = weakref.ref(module)
+ self.execution_device = device
+ self.required_checks: list = [required]
+ self.pending: list = []
+ self.handle = module.register_forward_pre_hook(self._pre_hook)
+ self.owns_hf_hook = "_hf_hook" not in module.__dict__
+ if self.owns_hf_hook:
+ module._hf_hook = self
+ module.__dict__[_DEFERRED_ATTR] = self
+
+ def init_hook(self, module: Any) -> Any:
+ return module
+
+ def pre_forward(self, module: Any, *args: Any, **kwargs: Any) -> tuple:
+ self.attempt()
+ return args, kwargs
+
+ def post_forward(self, module: Any, output: Any) -> Any:
+ return output
+
+ def detach_hook(self, module: Any) -> Any:
+ return module
+
+ def _pre_hook(self, module: Any, args: Any) -> None:
+ self.attempt()
+
+ @property
+ def required(self) -> bool:
+ """Whether the module cannot run where it is, CHECKED now (each pending way back registers a bool or a
+ predicate on the module's current state): a settle that later succeeded makes it optional again."""
+ for check in self.required_checks:
+ try:
+ if check() if callable(check) else check:
+ return True
+ except Exception: # noqa: BLE001 - its state cannot be read: treat it as not runnable
+ return True
+ return False
+
+ def add(self, label: str, undo: Callable[[], None]) -> None:
+ self.pending.append((label, undo))
+
+ def attempt(self) -> bool:
+ """Retry every pending undo in order; True once none is left."""
+ while self.pending:
+ label, undo = self.pending[0]
+ try:
+ undo()
+ except Exception as exc: # noqa: BLE001
+ if not self.required:
+ return False
+ # its weights are on the host and nothing would move them: running it would fail on the wrong
+ # device, so say why (an out-of-memory keeps "out of memory" in the text for the OOM handlers)
+ module = self.module_ref()
+ raise RuntimeError(
+ f"{type(module).__name__} ({label.split(':', 1)[-1]}) is still on the host after a VAE decode "
+ f"ran out of memory and cannot return to {self.execution_device}: "
+ + (
+ "out of memory; free GPU memory and retry"
+ if is_oom(exc)
+ else f"{type(exc).__name__}: {exc}"
+ )
+ ) from exc
+ self.pending.pop(0)
+ self.detach()
+ return True
+
+ def detach(self) -> None:
+ module = self.module_ref()
+ try:
+ self.handle.remove()
+ except Exception: # noqa: BLE001
+ pass
+ if module is None:
+ return
+ if self.owns_hf_hook and module.__dict__.get("_hf_hook") is self:
+ del module.__dict__["_hf_hook"]
+ if module.__dict__.get(_DEFERRED_ATTR) is self:
+ del module.__dict__[_DEFERRED_ATTR]
+
+
+def _defer(
+ module: Any, device: Any, label: str, undo: Callable[[], None], required: Any, owner: Any
+) -> None:
+ """Hand ``undo`` to ``module``'s deferred return (one per module); ``owner`` (the VAE) records the module so
+ uninstall can settle it."""
+ deferred = module.__dict__.get(_DEFERRED_ATTR)
+ if deferred is None:
+ deferred = DeferredReturn(module, device, required)
+ else:
+ deferred.required_checks.append(required)
+ deferred.add(label, undo)
+ if owner is not None:
+ record = owner.__dict__.setdefault(_RECORD_ATTR, {})
+ refs = record.setdefault("deferred", [])
+ if not any(r() is module for r in refs):
+ refs.append(weakref.ref(module))
+
+
+_FAULT: Optional[Callable[[str], None]] = None
+
+
+def _fault(point: str) -> None:
+ if _FAULT is not None:
+ _FAULT(point)
+
+
+class Change:
+ """One state change of a recovery. ``undo`` puts it back; when ``undo`` fails part way, ``settle`` leaves the
+ module in a consistent working state and ``defer`` hands the way back to the module's DeferredReturn, whose
+ retry is ``retry`` (undo, falling back to settle)."""
+
+ def __init__(self, label: str, undo: Callable[[], None], settle: Callable[[], None] = lambda: None,
+ defer: Optional[Callable[[Callable[[], None], BaseException], None]] = None,) -> None: # fmt: skip
+ self.label = label
+ self.undo = undo
+ self.settle = settle
+ self.defer = defer
+
+ def retry(self) -> None:
+ try:
+ _fault(self.label + ":undo")
+ self.undo()
+ except Exception:
+ _fault(self.label + ":settle")
+ self.settle()
+ raise
+
+
+class DecodeSession:
+ """One OOM recovery as a transaction: every state change goes through ``apply`` (do, then record its Change),
+ and ``restore`` rolls the record back in reverse order. A Change leaves the record only once its own undo
+ succeeded or its module was settled into a consistent state with a deferred way back."""
+
+ def __init__(
+ self,
+ vae: Any,
+ device: Any,
+ logger: Any = None,
+ ) -> None:
+ self.vae = vae
+ self.device = device
+ self.logger = logger
+ self.freed = False
+ self.vae_offloaded = False
+ self.log: list = [] # Change, in the order applied
+
+ @property
+ def moved(self) -> list:
+ return [c.label.split(":", 1)[1] for c in self.log if c.label.startswith("moved:")]
+
+ def _note(self, msg: str, *args: Any) -> None:
+ if self.logger is not None:
+ self.logger.warning(msg, *args)
+
+ def apply(self, do: Callable[[], None], change: Change) -> bool:
+ """Run ``do``; record ``change`` on success. A ``do`` that fails (part way, possibly) is rolled back now."""
+ try:
+ _fault(change.label + ":do")
+ do()
+ except Exception as exc: # noqa: BLE001
+ self._note("diffusion.vae_oom: %s failed (%s); rolled back", change.label, exc)
+ self.rollback(change)
+ return False
+ self.log.append(change)
+ return True
+
+ def rollback(self, change: Change) -> bool:
+ try:
+ _fault(change.label + ":undo")
+ change.undo()
+ return True
+ except Exception as exc: # noqa: BLE001
+ try:
+ _fault(change.label + ":settle")
+ change.settle()
+ except Exception as exc2: # noqa: BLE001 - the deferred return below retries from wherever it is
+ self._note("diffusion.vae_oom: settling %s failed (%s)", change.label, exc2)
+ if change.defer is not None:
+ try:
+ change.defer(change.retry, exc)
+ except Exception as exc3: # noqa: BLE001
+ self._note("diffusion.vae_oom: deferring %s failed (%s)", change.label, exc3)
+ return False
+
+ def restore(self) -> None:
+ while self.log:
+ self.rollback(self.log.pop())
+ self.vae_offloaded = False
+
+ def free_others(self, pipe: Any) -> int:
+ """Step (a). Returns how many things were released (0: nothing on the card to free)."""
+ if self.freed or pipe is None or getattr(self.device, "type", None) != "cuda":
+ return 0
+ self.freed = True
+ released = 0
+ # Captured graphs pin their pool and the weights' addresses: drop them before any weight moves (the image and
+ # video backends drop them on any OOM too). Nothing to undo: the next step re-captures. If dropping them
+ # fails, no weight may move under a graph that could still replay: step (a) stops here.
+ graphs = getattr(pipe, "_unsloth_cuda_graphs", ()) or ()
+ if graphs:
+
+ def reset() -> None:
+ from . import diffusion_cuda_graph
+
+ # strict: a handle that failed to reset (or still holds a graph) aborts step (a) before any move
+ diffusion_cuda_graph.reset_all(graphs, strict = True)
+
+ if not self.apply(reset, Change("graphs_reset", lambda: None)):
+ return 0
+ released += 1
+ self._hook_torchao_resident(pipe)
+ released += self._release_groups(pipe)
+ released += self._move_components(pipe)
+ if self.moved:
+ self._note(
+ "diffusion.vae_oom: moved %s to the host for the decode", ", ".join(self.moved)
+ )
+ return released
+
+ def _hook_torchao_resident(self, pipe: Any) -> bool:
+ """A hookless resident torchao transformer cannot move whole (Module.to skips or breaks its weights): give it
+ block offload hooks with every group resident (diffusion_memory.hook_resident_denoiser, the path an oversized
+ request takes), so the group release below can stream it. The hooks stay afterwards, every group re-pinned,
+ exactly as after an oversized request."""
+ transformer = getattr(pipe, "transformer", None)
+ if transformer is None or _placement(transformer, self.device) != "torchao_resident":
+ return False
+ from . import diffusion_memory
+
+ hooked: list = []
+ torch_hooks = diffusion_memory.module_forward_hook_ids(transformer)
+ device, logger = self.device, self.logger
+
+ def do() -> None:
+ hooked.append(bool(diffusion_memory.hook_resident_denoiser(pipe, device, logger)))
+
+ def undo() -> None:
+ # Installed: the hooks stay (every group re-pinned by the release's restore, as after an oversized
+ # request). Otherwise back to where it started from wherever it is: split -> whole on the card; settled
+ # (streamed under hooks) -> every group resident again. Idempotent: a clean refusal moves nothing.
+ if not (hooked and hooked[0]):
+ diffusion_memory.return_torchao_denoiser(pipe, device, logger, torch_hooks)
+
+ def settle() -> None:
+ # cannot come back whole: stream it under block offload hooks (the load's flat plan), so it still runs
+ # (already under hooks: a re-pin that failed part way leaves some groups streamed, which runs)
+ if not _group_hooked(transformer) and not diffusion_memory.hook_resident_denoiser(
+ pipe, device, logger, resident = False
+ ):
+ raise RuntimeError("the transformer could not be streamed under offload hooks")
+
+ def defer(retry: Callable[[], None], exc: BaseException) -> None:
+ # required until it is checked to stream under offload hooks (split with no hooks it cannot run)
+ _defer(transformer, device, "torchao_hooks:transformer", retry, lambda: not _group_hooked(transformer),
+ self.vae) # fmt: skip
+
+ if not self.apply(do, Change("torchao_hooks", undo, settle, defer)):
+ return False
+ return bool(hooked and hooked[0])
+
+ def _release_groups(self, pipe: Any) -> int:
+ """Resident offload groups (text encoders first, then the denoiser's blocks) stream again."""
+ from . import diffusion_memory
+
+ got: list = []
+ anchor = next(
+ (m for m in (getattr(pipe, "components", {}) or {}).values() if getattr(m, "_unsloth_resident_room", None)),
+ None,
+ ) # fmt: skip
+ if anchor is None:
+ return 0
+
+ def do() -> None:
+ restore = diffusion_memory.release_resident_groups(
+ pipe, 1 << 30, self.logger, reason = "a VAE decode that ran out of memory"
+ )
+ if restore is not None:
+ got.append(restore)
+
+ def undo() -> None:
+ while got:
+ # strict: an incomplete re-pin raises (the release path's re-pin otherwise swallows it), so the
+ # settle / defer below runs and the restore is kept until every group is resident again
+ _call_restore(got[-1])
+ got.pop()
+
+ def settle() -> None:
+ restore = diffusion_memory.release_resident_groups(
+ pipe, 1 << 30, self.logger, reason = "a failed re-pin"
+ )
+ if restore is not None:
+ got.append(restore)
+
+ def defer(retry: Callable[[], None], exc: BaseException) -> None:
+ # Optional only while every offload group is checked to keep its hooks: a group, resident or streamed,
+ # then runs where it is (a re-pin and a flat-plan settle that both stopped part way leave a mix of
+ # the two, which still runs); the re-pin is retried before the next forward.
+ _defer(
+ anchor,
+ self.device,
+ "resident_groups",
+ retry,
+ lambda: not _groups_hooked(pipe),
+ self.vae,
+ )
+
+ change = Change("resident_groups", undo, settle, defer)
+ if not self.apply(do, change):
+ return 0
+ if not got:
+ self.log.remove(change)
+ return 0
+ return 1
+
+ def _move_components(self, pipe: Any) -> int:
+ """Components whole on the card with no offload hooks: to the host, largest first, while host RAM allows."""
+ import torch
+
+ candidates = []
+ for name, comp in (getattr(pipe, "components", {}) or {}).items():
+ if comp is self.vae or not isinstance(comp, torch.nn.Module):
+ continue
+ if _placement(comp, self.device) != "plain":
+ continue
+ candidates.append((_nbytes(comp), name, comp))
+ candidates.sort(key = lambda c: -c[0])
+ host = _host_free_mib()
+ released = 0
+ for size, name, comp in candidates:
+ mib = size >> 20
+ if host is not None and mib + HOST_MARGIN_MIB > host:
+ self._note(
+ "diffusion.vae_oom: %s (%d MiB) stays on the card: host RAM is short", name, mib
+ )
+ continue
+ if self.apply(lambda comp = comp: _to_host(comp), self._component_change(name, comp)):
+ released += 1
+ if host is not None:
+ host -= mib
+ return released
+
+ def _component_change(self, name: str, comp: Any) -> Change:
+ device = self.device
+
+ def settle() -> None:
+ _to_host(comp)
+ _empty_cache(device)
+
+ def defer(retry: Callable[[], None], exc: BaseException) -> None:
+ self._note(
+ "diffusion.vae_oom: %s could not return to the card after the decode (%s); it returns before its "
+ "next forward", name, exc,
+ ) # fmt: skip
+ _defer(comp, device, f"moved:{name}", retry, True, self.vae)
+
+ return Change(f"moved:{name}", lambda: _return_to(comp, device), settle, defer)
+
+ def _component_undo(self, name: str, comp: Any) -> tuple:
+ """(undo, defer) for a component moved to the host; kept for callers recording a move by hand."""
+ change = self._component_change(name, comp)
+ return change.retry, lambda exc: change.defer(change.retry, exc)
+
+ def offload_vae(self) -> bool:
+ """Step (c): the VAE's weights stream per layer from the host (diffusers leaf-level group offload)."""
+ if self.vae_offloaded or getattr(self.device, "type", None) != "cuda" or _hooked(self.vae):
+ return False
+ vae, device = self.vae, self.device
+ host, mib = _host_free_mib(), _nbytes(vae) >> 20
+ if host is not None and mib + HOST_MARGIN_MIB > host:
+ # the streamed copy lives in host RAM: never trade a decode OOM for a host OOM
+ self._note(
+ "diffusion.vae_oom: %s (%d MiB) stays on the card: host RAM is short",
+ type(vae).__name__,
+ mib,
+ )
+ return False
+
+ settled: list = []
+
+ def settle() -> None:
+ settled.clear()
+ _park_vae(vae, device)
+ settled.append(True)
+
+ def defer(retry: Callable[[], None], exc: BaseException) -> None:
+ # optional only when the VAE verifiably streams whole from the host under its offload hooks; otherwise
+ # (the park failed too: split, maybe hookless) the next decode must not run before it is repaired
+ streams = bool(settled) and _vae_streams(vae)
+ settled.clear()
+ self._note("diffusion.vae_oom: %s %s (%s); it returns at its next decode", type(vae).__name__,
+ "stays streamed from the host" if streams else "is split between host and card", exc) # fmt: skip
+ _defer(vae, device, "vae_offload", retry, lambda: not _vae_streams(vae), vae)
+
+ change = Change("vae_offload", lambda: _remove_vae_offload(vae, device), settle, defer)
+ if not self.apply(lambda: _apply_vae_offload(vae, device), change):
+ return False
+ self.vae_offloaded = True
+ self._note(
+ "diffusion.vae_oom: %s weights stream from the host for this decode", type(vae).__name__
+ )
+ return True
+
+
+def _apply_vae_offload(vae: Any, device: Any) -> None:
+ import torch
+ from diffusers.hooks import apply_group_offloading
+ apply_group_offloading(
+ vae, onload_device = device, offload_device = torch.device("cpu"), offload_type = "leaf_level"
+ )
+
+
+def _remove_vae_offload(vae: Any, device: Any) -> None:
+ from .diffusion_memory import _remove_group_offload_hooks
+ _remove_group_offload_hooks(vae)
+ _return_to(vae, device)
+
+
+def _groups_hooked(pipe: Any) -> bool:
+ """Every component placed by resident offload groups still carries its group offloading hooks."""
+ comps = [
+ m
+ for m in (getattr(pipe, "components", {}) or {}).values()
+ if getattr(m, "_unsloth_resident_room", None)
+ ]
+ return all(_group_hooked(m) for m in comps)
+
+
+def _whole_on_host(module: Any) -> bool:
+ tensors = [*module.parameters(), *module.buffers()]
+ return all(getattr(t.device, "type", None) == "cpu" for t in tensors)
+
+
+def _vae_hooked(vae: Any) -> bool:
+ """The VAE carries its own leaf-level group offloading hooks (not merely a deferred return's ``_hf_hook``)."""
+ return _group_hooked(vae)
+
+
+def _vae_streams(vae: Any) -> bool:
+ """Checked, not assumed: the VAE's offload hooks are on and every weight is on the host, so it decodes
+ (streamed) as it is."""
+ return _vae_hooked(vae) and _whole_on_host(vae)
+
+
+def _park_vae(vae: Any, device: Any) -> None:
+ """Whole on the host with the offload hooks on: the VAE still decodes (streamed)."""
+ from .diffusion_memory import _remove_group_offload_hooks
+
+ _remove_group_offload_hooks(vae)
+ _to_host(vae)
+ _empty_cache(device)
+ _apply_vae_offload(vae, device)
+
+
+def _undo_vae_offload(vae: Any, device: Any) -> None:
+ """Hooks off and weights back on the card; if that fails, the VAE is parked streamed (``_park_vae``) and the
+ error raised, so the caller can retry later."""
+ try:
+ _remove_vae_offload(vae, device)
+ except Exception:
+ _park_vae(vae, device)
+ raise
+
+
+def settle_deferred(module: Any, strict: bool = False) -> bool:
+ """Run ``module``'s pending deferred returns now; True when nothing is left pending. ``strict``: a REQUIRED
+ repair that still fails raises (its clear error), so the caller never runs the module in an inconsistent state;
+ an optional one (it runs as it is) returns False."""
+ deferred = getattr(module, "__dict__", {}).get(_DEFERRED_ATTR)
+ if deferred is None:
+ return True
+ try:
+ return deferred.attempt()
+ except Exception: # noqa: BLE001 - still out of memory: the deferred return stays
+ if strict:
+ raise
+ return False
+
+
+def _pipe_of(vae: Any) -> Any:
+ ref = vae.__dict__.get(_PIPE_ATTR)
+ return ref() if callable(ref) else None
+
+
+def _device_of(z: Any) -> Any:
+ return getattr(z, "device", None)
+
+
+def free_first(vae: Any, z: Any = None) -> bool:
+ """Step (a) for a Studio decode wrapper about to fall back to a cheaper decode on an OOM (wide tiles -> stock
+ tiles, LTX-2 tiles -> the floor tile, untiled -> tiled): free the card inside the current decode's recovery
+ session. True when something was freed, so the wrapper retries its own attempt first. Outside a decode scope, or
+ with the retry switched off, or once this decode already freed the card: False (fall back as before)."""
+ holder = getattr(vae, "__dict__", {}).get(_SESSION_ATTR)
+ if holder is None or oom_retry_disabled() or _compiling():
+ return False
+ session = holder.get("session")
+ if session is None:
+ session = holder["session"] = DecodeSession(
+ vae, _device_of(z), vae.__dict__.get("_unsloth_vae_oom_logger")
+ )
+ if session.freed:
+ return False
+ _release_cache(z, vae)
+ freed = bool(session.free_others(_pipe_of(vae)))
+ stats = vae.__dict__.get("_unsloth_vae_oom_stats")
+ if isinstance(stats, dict):
+ stats["freed_first"] = stats.get("freed_first", 0) + int(freed)
+ return freed
+
+
+def _ladder_steps(
+ tiled_now: bool,
+ can_tile: bool,
+ eager_rung: bool = False,
+) -> list:
+ """The ladder's rungs, the one list both the ``_decode`` retry and the outer decode scope climb, in order:
+ (name, tiled, offload_vae, eager). "freed": the rest of the pipeline off the card, the same decode again;
+ "eager" (scope only, a compiled guard below): every compiled guard on its eager path, where the ``_decode``
+ ladder runs too; "tiled": stock tile sizes; "offload": the VAE's weights streamed from the host."""
+ steps: list = [("freed", tiled_now, False, False)]
+ if eager_rung:
+ steps.append(("eager", tiled_now, False, True))
+ if can_tile and not tiled_now:
+ steps.append(("tiled", True, False, eager_rung))
+ steps.append(("offload", tiled_now or can_tile, True, eager_rung))
+ return steps
+
+
+_STEP_TEXT = {"freed": "with the rest of the pipeline off the card", "eager": "on the eager path",
+ "tiled": "tiled at stock tile sizes", "offload": "with the VAE weights streamed from the host"} # fmt: skip
+
+
+def _climb(vae: Any, session: "DecodeSession", steps: list, attempt: Callable, stats: dict, z: Any,
+ logger: Any) -> tuple: # fmt: skip
+ """Run ``steps`` in order, each rung's state change through ``session`` (applied once per logical decode: a
+ rung a nested climb already took is skipped), then ``attempt(tiled, eager)``. (True, out, rung) on the first
+ attempt that returns; (False, None, None) when none does."""
+ for name, tiled, offload_vae, eager in steps:
+ _release_cache(z, vae)
+ if name == "freed":
+ t0 = time.perf_counter()
+ freed = session.free_others(_pipe_of(vae))
+ stats["free_s"] = round(time.perf_counter() - t0, 3)
+ if not freed:
+ continue # nothing (more) was freed: the same decode would fail the same way
+ elif offload_vae and not session.offload_vae():
+ continue
+ if logger is not None:
+ logger.warning("diffusion.vae_oom: %s decode ran out of memory; retrying %s", type(vae).__name__,
+ _STEP_TEXT[name]) # fmt: skip
+ stats[f"{name}_retry"] = stats.get(f"{name}_retry", 0) + 1
+ failed = False
+ try:
+ out = attempt(tiled, eager)
+ except Exception as exc: # noqa: BLE001
+ if not is_oom(exc) and not offload_vae:
+ raise
+ if not is_oom(exc) and logger is not None:
+ logger.warning(
+ "diffusion.vae_oom: streamed %s decode failed (%s)", type(vae).__name__, exc
+ )
+ drop_tracebacks(exc)
+ failed = True
+ if not failed:
+ return True, out, name
+ stats["oom"] = stats.get("oom", 0) + 1
+ return False, None, None
+
+
+_RETRY_LAYER = "vae_oom_retry"
+_SCOPE_LAYER = "vae_oom_scope"
+
+
+def _scope_session(vae: Any) -> Optional[dict]:
+ return vae.__dict__.get(_SESSION_ATTR)
+
+
+def _close_scope(vae: Any, holder: dict, z: Any) -> None:
+ vae.__dict__.pop(_SESSION_ATTR, None)
+ session = holder.get("session")
+ if session is None:
+ return
+ t0 = time.perf_counter()
+ session.restore()
+ stats = vae.__dict__.get("_unsloth_vae_oom_stats")
+ if isinstance(stats, dict):
+ stats["restore_s"] = round(time.perf_counter() - t0, 3)
+ _empty_cache(_device_of(z))
+
+
+def install(
+ vae: Any,
+ logger: Any = None,
+ pipe: Any = None,
+) -> bool:
+ """Wrap ``vae._decode`` with the offload-first OOM ladder, and the public ``decode`` with the scope one logical
+ decode's recovery spans (a compiled decode's guard included). Idempotent. ``pipe``: the pipeline whose other
+ components step (a) may move off the card (kept as a weak reference; a reinstall keeps the last one)."""
+ if vae is None or oom_retry_disabled():
+ return False
+ if pipe is not None:
+ try:
+ vae.__dict__[_PIPE_ATTR] = weakref.ref(pipe)
+ except TypeError:
+ pass
+ if not callable(getattr(vae, "_decode", None)):
+ return False
+ if decode_layers.find(vae, "_decode", _RETRY_LAYER) is not None:
+ return True
+ can_tile = callable(getattr(vae, "enable_tiling", None)) and hasattr(vae, "use_tiling")
+ stats = {"calls": 0, "oom": 0, "freed_retry": 0, "tiled_retry": 0, "offload_retry": 0, "recovered": 0,
+ "recovered_by": None, "failed": 0} # fmt: skip
+
+ def make_retry(inner: Callable, layer: Any) -> Callable:
+ depth = [0]
+
+ def _attempt(z: Any, args: tuple, kwargs: dict, tiled: bool) -> Any:
+ prev = bool(getattr(vae, "use_tiling", False))
+ vae._unsloth_vae_oom_active = True
+ try:
+ if can_tile:
+ vae.use_tiling = tiled
+ return inner(z, *args, **kwargs)
+ finally:
+ if can_tile:
+ vae.use_tiling = prev
+ vae._unsloth_vae_oom_active = False
+
+ def _decode(z: Any, *args: Any, **kwargs: Any) -> Any:
+ # Traced inside a compiled decode: a pass-through (no try/except in the graph; the compiled guard
+ # retries). Nested (a wide-tile override decoding one tile through _decode): the outermost call retries.
+ if _compiling() or oom_retry_disabled() or depth[0]:
+ return inner(z, *args, **kwargs)
+ depth[0] += 1
+ holder = _scope_session(vae)
+ owns = holder is None
+ if owns:
+ holder = {"session": None}
+ vae.__dict__[_SESSION_ATTR] = holder
+ try:
+ if owns:
+ settle_deferred(
+ vae, strict = True
+ ) # a split VAE never decodes before it is repaired
+ return _retrying(z, args, kwargs, holder)
+ finally:
+ depth[0] -= 1
+ if owns:
+ _close_scope(vae, holder, z)
+ # reached without the scope: a later rewrap (a compile fallback restoring its eager decode, a
+ # speed layer run after load) dropped or covered it; put it back on top for the next decode
+ ensure_outermost(vae, logger)
+
+ def _retrying(z: Any, args: tuple, kwargs: dict, holder: dict) -> Any:
+ stats["calls"] += 1
+ tiled_now = bool(getattr(vae, "use_tiling", False))
+ steps = _ladder_steps(tiled_now, can_tile)
+ recovered = holder.get("recovered")
+ if recovered is not None:
+ names = [step[0] for step in steps]
+ start = names.index(recovered) if recovered in names else 0
+ name, tiled = steps[start][:2]
+ try:
+ return _attempt(z, args, kwargs, tiled)
+ except Exception as exc: # noqa: BLE001
+ if not is_oom(exc):
+ raise
+ drop_tracebacks(exc)
+ holder.setdefault("first_oom", exc)
+ stats["oom"] += 1
+ steps = steps[start + 1 :]
+ else:
+ gate_owned = False
+ try:
+ return inner(z, *args, **kwargs)
+ except Exception as exc: # noqa: BLE001
+ if not is_oom(exc):
+ raise
+ gate_owned = not tiled_now and _outer_gate_owns_untiled(vae)
+ drop_tracebacks(exc)
+ holder.setdefault("first_oom", exc)
+ stats["oom"] += 1
+ if gate_owned:
+ # The untiled-when-fits gate retries tiled itself; the ladder still sees the OOM first: free the
+ # card (step a) and retry untiled once, then hand back to the gate.
+ if free_first(vae, z):
+ stats["freed_retry"] += 1
+ try:
+ out = inner(z, *args, **kwargs)
+ stats["recovered"] += 1
+ stats["recovered_by"] = "freed"
+ return out
+ except Exception as exc: # noqa: BLE001
+ if not is_oom(exc):
+ raise
+ drop_tracebacks(exc)
+ raise holder["first_oom"]
+ session = holder.get("session")
+ if session is None:
+ session = holder["session"] = DecodeSession(vae, _device_of(z), logger)
+ ok, out, name = _climb(vae, session, steps, lambda tiled, eager: _attempt(z, args, kwargs, tiled),
+ stats, z, logger) # fmt: skip
+ if ok:
+ stats["recovered"] += 1
+ stats["recovered_by"] = name
+ holder["recovered"] = name
+ return out
+ holder["exhausted"] = True
+ stats["failed"] += 1
+ _release_cache(z, vae)
+ first = holder.get("first_oom")
+ note = ("VAE decode ran out of memory even with the rest of the pipeline and the VAE weights off the "
+ "card; free GPU memory or lower the resolution") # fmt: skip
+ if first is None:
+ raise RuntimeError(note)
+ if callable(getattr(first, "add_note", None)) and note not in (
+ getattr(first, "__notes__", None) or ()
+ ):
+ first.add_note(note)
+ raise first
+
+ _decode._unsloth_vae_oom_retry = True
+ _decode._unsloth_vae_oom_stats = stats
+ return _decode
+
+ decode_layers.wrap(vae, "_decode", _RETRY_LAYER, make_retry)
+ vae._unsloth_vae_oom_stats = stats
+ vae.__dict__["_unsloth_vae_oom_logger"] = logger
+ _install_scope(vae, logger)
+ if logger is not None:
+ logger.info(
+ "diffusion.vae_oom: %s decode recovers an out-of-memory by offloading, then tiling at stock sizes",
+ type(vae).__name__,
+ )
+ return True
+
+
+def _compiled_below(vae: Any, top: Any) -> bool:
+ """A compiled decode guard sits somewhere under ``top`` (walked through every wrapper kind), or the speed layer
+ compiled this VAE's decode (a guard behind a closure the walk cannot see)."""
+ fn, seen = top, 0
+ while fn is not None and seen < 64:
+ seen += 1
+ if getattr(fn, "_unsloth_compiled_decode", False) and callable(
+ getattr(fn, "_unsloth_eager_decode", None)
+ ):
+ return True
+ fn = decode_layers.next_layer(fn)
+ return bool(vae.__dict__.get("_unsloth_compiled_decode", False))
+
+
+# Decode wrappers that must sit BELOW the scope: a compiled decode guard (the graph would trace the scope as a
+# pass-through and its guard re-raises the OOM), the untiled-when-fits gate (it retries tiled itself) and the dtype
+# wrappers (their own allocations: the fp16 isfinite check, the bf16 -> fp32 output cast).
+_BELOW_SCOPE_MARKERS = ("_unsloth_compiled_decode", "_unsloth_untiled_decode", "_unsloth_bf16_decode",
+ "_unsloth_decode_slot") # fmt: skip
+
+
+def _scope_intact(vae: Any) -> bool:
+ """The scope layer is active, reachable from the current public decode, and none of the wrappers that must sit
+ below it is above it."""
+ layer = vae.__dict__.get(_RECORD_ATTR, {}).get("scope")
+ if layer is None or not layer.active or layer.owner is not vae:
+ return False
+ fn = vae.__dict__.get("decode")
+ for _ in range(32):
+ if fn is None:
+ break
+ if fn is layer.fn:
+ return True
+ if any(getattr(fn, m, None) for m in _BELOW_SCOPE_MARKERS) and not getattr(
+ fn, "_unsloth_vae_oom_scope", False
+ ):
+ return False
+ fn = decode_layers.next_layer(fn)
+ return False
+
+
+def ensure_outermost(vae: Any, logger: Any = None) -> bool:
+ """(Re)assert the OOM scope as the outermost decode layer after anything rewrapped ``vae.decode`` (a deferred
+ speed profile, a VAE compile, a compile fallback, a dtype wrapper). Called after every rewrap site and when a
+ decode closes; idempotent and cheap. True when the retry is installed and its scope is in place."""
+ if (
+ vae is None
+ or decode_layers.find(vae, "_decode", _RETRY_LAYER) is None
+ or oom_retry_disabled()
+ ):
+ return False
+ if _scope_intact(vae):
+ return True
+ record = vae.__dict__.setdefault(_RECORD_ATTR, {})
+ decode_layers.unwrap(record.pop("scope", None))
+ return _install_scope(vae, logger)
+
+
+def _install_scope(vae: Any, logger: Any = None) -> bool:
+ """Wrap the public ``decode`` as its outermost layer: the outermost call opens the recovery scope every inner
+ ``_decode`` (batch slices, frames, tiles) shares, and closes it, restoring what the recovery moved, once the
+ whole decode returns. An out-of-memory that reaches the scope (a compiled graph, which never enters the
+ ``_decode`` ladder, or a wrapper's own allocation) frees the card and retries the whole decode as it was (a
+ compiled one with the same kernels, same output); then, with a compiled guard below, the whole decode once more
+ with every guard on its eager path, where the ``_decode`` ladder continues in the same session."""
+ if decode_layers.find(vae, "decode", _SCOPE_LAYER) is not None:
+ return True
+ if not callable(getattr(vae, "decode", None)):
+ return False
+
+ def make_scope(inner: Callable, layer: Any) -> Callable:
+ def _try(args: tuple, kwargs: dict, failures: list) -> tuple:
+ try:
+ return True, inner(*args, **kwargs)
+ except Exception as exc: # noqa: BLE001
+ if not is_oom(exc):
+ raise
+ drop_tracebacks(exc)
+ failures.append(exc)
+ return False, None
+
+ def decode(*args: Any, **kwargs: Any) -> Any:
+ if _compiling() or oom_retry_disabled() or _scope_session(vae) is not None:
+ return inner(*args, **kwargs)
+ holder: dict = {"session": None}
+ vae.__dict__[_SESSION_ATTR] = holder
+ z = args[0] if args else kwargs.get("z")
+ failures: list = []
+ try:
+ # a VAE left streamed by an earlier restore that ran out of memory comes back first, if it fits now;
+ # one left split (a required repair) never decodes before it is repaired: its clear error surfaces
+ settle_deferred(vae, strict = True)
+ ok, out = _try(args, kwargs, failures)
+ if ok:
+ return out
+ compiled = _compiled_below(vae, inner)
+ if holder.get("exhausted") and not compiled:
+ raise failures[0] # the _decode ladder already climbed every rung for this OOM
+ stats = vae.__dict__.get("_unsloth_vae_oom_stats")
+ if not isinstance(stats, dict):
+ stats = {}
+ stats["outer_oom"] = stats.get("outer_oom", 0) + 1
+ if compiled:
+ stats["compiled_oom"] = stats.get("compiled_oom", 0) + 1
+ session = holder.get("session")
+ if session is None:
+ session = holder["session"] = DecodeSession(vae, _device_of(z), logger)
+ # The same rungs as the _decode ladder, for an OOM it never saw (a compiled graph, a dtype wrapper's
+ # own allocation): each attempt is the whole decode, wrappers included.
+ can_tile = callable(getattr(vae, "enable_tiling", None)) and hasattr(
+ vae, "use_tiling"
+ )
+ tiled_now = bool(getattr(vae, "use_tiling", False))
+
+ def attempt(tiled: bool, eager: bool) -> Any:
+ from . import diffusion_bg_compile # noqa: PLC0415
+ prev = bool(getattr(vae, "use_tiling", False))
+ try:
+ if can_tile:
+ vae.use_tiling = tiled
+ if eager:
+ with diffusion_bg_compile.force_eager():
+ return inner(*args, **kwargs)
+ return inner(*args, **kwargs)
+ finally:
+ if can_tile:
+ vae.use_tiling = prev
+
+ ok, out, name = _climb(vae, session, _ladder_steps(tiled_now, can_tile, compiled), attempt, stats,
+ z, logger) # fmt: skip
+ if not ok:
+ raise failures[0]
+ stats["recovered"] = stats.get("recovered", 0) + 1
+ if name == "freed" and compiled:
+ stats["recovered_by"] = "freed_compiled"
+ elif name != "eager" or holder.get("recovered") is None:
+ stats["recovered_by"] = name
+ else:
+ stats["recovered_by"] = holder["recovered"]
+ return out
+ finally:
+ failures.clear()
+ _close_scope(vae, holder, z)
+ ensure_outermost(vae, logger)
+
+ decode._unsloth_vae_oom_guard = True
+ decode._unsloth_vae_oom_scope = True
+ return decode
+
+ layer = decode_layers.wrap(vae, "decode", _SCOPE_LAYER, make_scope)
+ vae.__dict__.setdefault(_RECORD_ATTR, {})["scope"] = layer
+ return True
+
+
+def uninstall(vae: Any) -> None:
+ """Remove everything ``install`` and its retries put in place: the ``_decode`` layer, the ``decode`` scope /
+ compiled guard (cut out exactly when outermost, a pass-through under later wrappers), and every deferred return.
+ A deferred return is settled first (its pending undo runs); one that still cannot run (the card is full) stays,
+ since it is that module's only way back to the card."""
+ record = vae.__dict__.get(_RECORD_ATTR, {})
+ decode_layers.unwrap(record.pop("scope", None))
+ for ref in record.pop("deferred", []):
+ module = ref()
+ if module is not None and not settle_deferred(module):
+ record.setdefault("deferred", []).append(ref)
+ if not record:
+ vae.__dict__.pop(_RECORD_ATTR, None)
+ layer = decode_layers.find(vae, "_decode", _RETRY_LAYER)
+ if layer is None:
+ return
+ decode_layers.unwrap(layer)
+ vae.__dict__.pop("_unsloth_vae_oom_stats", None)
+
+
+__all__ = [
+ "DecodeSession",
+ "DeferredReturn",
+ "clear_partial_decode",
+ "drop_tracebacks",
+ "ensure_outermost",
+ "free_first",
+ "settle_deferred",
+ "OOM_RETRY_ENV",
+ "install",
+ "is_oom",
+ "oom_retry_disabled",
+ "tile_attrs",
+ "uninstall",
+]
diff --git a/studio/backend/core/inference/diffusion_vae_tiling.py b/studio/backend/core/inference/diffusion_vae_tiling.py
index ce9b6d9f4b5..7a60d808faa 100644
--- a/studio/backend/core/inference/diffusion_vae_tiling.py
+++ b/studio/backend/core/inference/diffusion_vae_tiling.py
@@ -19,6 +19,8 @@ import types
from typing import Any, Optional
from .diffusion_device import float64_device
+from .diffusion_vae_oom import clear_partial_decode
+from .diffusion_vae_oom import free_first as oom_free_first
WIDE_TILES_ENV = "UNSLOTH_DIFFUSION_VAE_WIDE_TILES"
@@ -267,8 +269,10 @@ def _free_mib(
def _fused(vae: Any) -> bool:
- return bool(getattr(vae, "_unsloth_vae_fused_installed", 0)) and not getattr(
- vae, "_unsloth_vae_fused_failed", False
+ return (
+ bool(getattr(vae, "_unsloth_vae_fused_installed", 0))
+ and not getattr(vae, "_unsloth_vae_fused_failed", False)
+ and not getattr(vae, "_unsloth_vae_fused_structural", False)
)
@@ -536,6 +540,10 @@ def _release_cache(z: Any) -> None:
pass
+def _clear_partial(vae: Any, z: Any) -> None:
+ clear_partial_decode(vae, z)
+
+
def _log_stock_fallback(vae: Any, reason: str) -> None:
logger = vae.__dict__.get("_unsloth_wide_logger")
seen = vae.__dict__.setdefault("_unsloth_wide_fallback_logged", set())
@@ -576,23 +584,27 @@ def install(vae: Any, logger: Any = None) -> bool:
) -> Any:
if wide_tiles_disabled() or stock_layout_ok(geometry, z.shape[-2], z.shape[-1]):
return _stock(z, return_dict)
- # Never trade an OOM for quality: floor tile cannot fit (or OOMs) -> the smaller stock tiles.
- reason = floor_shortfall(self, z)
- if reason is None:
- for max_area in ("auto", 0):
- try:
- return tiled_decode(self, z, return_dict = return_dict, max_area = max_area)
- except Exception as exc: # noqa: BLE001
- if not _is_oom(exc):
- raise
- reason = f"the wide tiles ran out of memory ({type(exc).__name__})"
- tried = (self.__dict__.get("_unsloth_last_decode_tile") or (0, 0))[:2]
- # outside the except block, so the failed attempt's tensors are freed
- _release_cache(z)
- ratio, tile, _ = _geometry(self)
- if tuple(tried) == (min(tile, z.shape[-2]), min(tile, z.shape[-1])):
- break
+ # Never trade an OOM for quality: floor tile cannot fit (or OOMs) -> the smaller stock tiles. Before
+ # settling for them, the OOM retry frees the card (rest of the pipeline off it) and the wide tiles go again.
+ for _round in range(2):
+ reason = floor_shortfall(self, z)
+ if reason is None:
+ for max_area in ("auto", 0):
+ try:
+ return tiled_decode(self, z, return_dict = return_dict, max_area = max_area)
+ except Exception as exc: # noqa: BLE001
+ if not _is_oom(exc):
+ raise
+ reason = f"the wide tiles ran out of memory ({type(exc).__name__})"
+ tried = (self.__dict__.get("_unsloth_last_decode_tile") or (0, 0))[:2]
+ _clear_partial(self, z)
+ ratio, tile, _ = _geometry(self)
+ if tuple(tried) == (min(tile, z.shape[-2]), min(tile, z.shape[-1])):
+ break
+ if _round or not oom_free_first(self, z):
+ break
_log_stock_fallback(self, reason)
+ _clear_partial(self, z)
return _stock(z, return_dict)
def _tiled_encode(self: Any, x: Any) -> Any:
diff --git a/studio/backend/core/inference/media_decode_phase.py b/studio/backend/core/inference/media_decode_phase.py
index b8ce152f87f..cc9fc6fc292 100644
--- a/studio/backend/core/inference/media_decode_phase.py
+++ b/studio/backend/core/inference/media_decode_phase.py
@@ -34,6 +34,9 @@ def decode_phase(
on_decode()
return original(*args, **kwargs)
+ _decode.__wrapped__ = (
+ original # the decode stack stays walkable (diffusion_vae_oom checks its scope)
+ )
return _decode
for name in attrs:
diff --git a/studio/backend/core/inference/video.py b/studio/backend/core/inference/video.py
index bba15a4094d..eb7f2d6182e 100644
--- a/studio/backend/core/inference/video.py
+++ b/studio/backend/core/inference/video.py
@@ -9428,6 +9428,14 @@ class VideoBackend:
vae_bf16_mode = install_rocm_vae_bf16_decode(pipe, target, logger = logger)
if vae_bf16_mode:
speed_optims += ("vae_bf16_decode", f"vae_bf16_{vae_bf16_mode}")
+ # Offload-first retry on a decode OOM, installed last: its decode scope is the outermost decode layer
+ # (one recovery session per logical decode) and every inner wrapper that falls back on an OOM (wide
+ # tiles, untiled-when-fits) frees the card through it first (diffusion_vae_oom.free_first).
+ try:
+ from .diffusion_vae_oom import install as install_vae_oom_retry
+ install_vae_oom_retry(getattr(pipe, "vae", None), logger, pipe = pipe)
+ except Exception as exc: # noqa: BLE001 - a decode OOM then fails as before
+ logger.warning("video.vae_oom: not installed: %s", exc)
resolved = build_resolved_record(
{
@@ -10649,6 +10657,12 @@ class VideoBackend:
speed_optims = speed_optims + tuple(f"h3_vae_{name}" for name in vae_levers)
except Exception as exc: # noqa: BLE001 -- optimisation only, never fail a load
logger.warning("video.h3_vae_fast failed, keeping the stock VAE: %s", exc)
+ # the H3 VAE speedups rewrap vae.decode: the OOM retry's scope (when installed) goes back on top
+ try:
+ from .diffusion_vae_oom import ensure_outermost
+ ensure_outermost(getattr(pipe, "vae", None), logger)
+ except Exception as exc: # noqa: BLE001 - a decode OOM then fails as before
+ logger.warning("video.vae_oom: scope not restored after the H3 VAE speedups: %s", exc)
if "cudnn_benchmark" in speed_optims:
try:
from .video_minimax_h3_vae import install_audio_vae_without_cudnn_benchmark
diff --git a/studio/backend/core/inference/video_ltx2_vae_tiles.py b/studio/backend/core/inference/video_ltx2_vae_tiles.py
index fa7ffe946e8..927c0fe43e2 100644
--- a/studio/backend/core/inference/video_ltx2_vae_tiles.py
+++ b/studio/backend/core/inference/video_ltx2_vae_tiles.py
@@ -14,6 +14,8 @@ import os
import types
from typing import Any, Optional
+from .diffusion_vae_oom import clear_partial_decode
+from .diffusion_vae_oom import free_first as oom_free_first
from .diffusion_vae_tiling import axis_weights as _axis_weights
WIDE_TILES_ENV = "UNSLOTH_VIDEO_VAE_WIDE_TILES"
@@ -248,11 +250,22 @@ def tiled_decode(
if (th, tw) == floor or not is_oom_error(exc):
raise
failed = True
+ if failed and oom_free_first(vae, z):
+ # the OOM retry freed the card (rest of the pipeline off it): the same tiles again before the floor tile
+ failed = False
+ clear_partial_decode(vae, z) # the failed attempt's causal state goes before any retry
+ try:
+ dec = _decode_tiles(vae, z, temb, causal, th, tw, budgeted)
+ except Exception as exc: # noqa: BLE001
+ if not is_oom_error(exc):
+ raise
+ failed = True
if failed:
# Retried outside the handler so the failed attempt's tensors (held by the traceback) are gone first.
import torch
stats["oom_fallback"] += 1
+ clear_partial_decode(vae, z)
torch.cuda.empty_cache()
th, tw = floor
vae._unsloth_last_decode_tile = floor
diff --git a/studio/backend/core/inference/video_vae_untiled.py b/studio/backend/core/inference/video_vae_untiled.py
index 2522f469663..83d8498fe8f 100644
--- a/studio/backend/core/inference/video_vae_untiled.py
+++ b/studio/backend/core/inference/video_vae_untiled.py
@@ -84,6 +84,18 @@ def _free_bytes(device: Any) -> Optional[int]:
return None
+def _in_chain(fn: Any, target: Any) -> bool:
+ for _ in range(16):
+ if fn is None:
+ return False
+ if fn is target:
+ return True
+ from .decode_layers import next_layer # noqa: PLC0415
+
+ fn = next_layer(fn)
+ return False
+
+
def _decode_device(vae: Any, z: Any) -> Any:
"""Where the decode allocates: the latent's card, unless a multi-GPU split put the VAE on another CUDA card (its
boundary moves the latent there inside this wrapper). An offloaded VAE (weights on the host) keeps the latent's."""
@@ -162,6 +174,11 @@ def install_untiled_decode(
# Retried outside the handler so the failed attempt's tensors (held by the traceback) are gone first.
stats["oom_fallback"] += 1
oom_need[:] = [min([need, *oom_need])]
+ from .diffusion_vae_oom import clear_partial_decode # noqa: PLC0415
+
+ clear_partial_decode(
+ vae, z
+ ) # the untiled attempt's causal state goes before the tiled decode
torch.cuda.empty_cache()
if logger is not None:
logger.warning("video.vae_untiled: untiled decode ran out of memory; decoding tiled")
@@ -172,7 +189,8 @@ def install_untiled_decode(
return gated(*args, **kwargs)
finally:
# A failed VAE compile restores its eager decode into this slot; the guarded inner call already runs eager.
- if vae.__dict__.get("decode") is not untiled_when_fits:
+ # (a layer installed over this one, e.g. the OOM retry's decode scope, keeps it in its __wrapped__ chain)
+ if not _in_chain(vae.__dict__.get("decode"), untiled_when_fits):
vae.decode = untiled_when_fits
untiled_when_fits._unsloth_untiled_decode = True
diff --git a/studio/backend/tests/conftest.py b/studio/backend/tests/conftest.py
index d5486f433e0..46e0764c1e2 100644
--- a/studio/backend/tests/conftest.py
+++ b/studio/backend/tests/conftest.py
@@ -121,6 +121,13 @@ def _skills_home_root(tmp_path_factory):
_skills_home_counter = itertools.count()
[email protected](autouse = True)
+def _no_vae_structural_verdict_cache(monkeypatch):
+ """Structural VAE fusion verdicts persist under Studio's cache root; tests must not read or write the real one
+ (the cache tests turn it on against a temp dir)."""
+ monkeypatch.setenv("UNSLOTH_VAE_FUSED_STRUCTURAL_CACHE", "0")
+
+
@pytest.fixture(autouse = True)
def _zoo_without_gdn_decode_kernel(monkeypatch):
# Whether Qwen3.5 / Qwen3-Next ask for causal-conv1d depends on the installed unsloth_zoo; pin the
diff --git a/studio/backend/tests/test_diffusion_measured_placement.py b/studio/backend/tests/test_diffusion_measured_placement.py
index 1b81a742128..ddc38d41a31 100644
--- a/studio/backend/tests/test_diffusion_measured_placement.py
+++ b/studio/backend/tests/test_diffusion_measured_placement.py
@@ -932,7 +932,7 @@ def test_failed_release_keeps_the_copy_stream_wait(monkeypatch):
monkeypatch.setattr(dm, "_release_group", boom)
pipe = types.SimpleNamespace(components = {"transformer": module})
- assert dm.release_resident_groups(pipe, 4) is None
+ assert callable(dm.release_resident_groups(pipe, 4))
assert module._unsloth_stream_state["streamed"] >= 1
diff --git a/studio/backend/tests/test_diffusion_vae_oom.py b/studio/backend/tests/test_diffusion_vae_oom.py
new file mode 100644
index 00000000000..78adf8e9fc7
--- /dev/null
+++ b/studio/backend/tests/test_diffusion_vae_oom.py
@@ -0,0 +1,2643 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""diffusion_vae_oom: the offload-first OOM ladder (CPU, fake OOMs and a fake card) and the structural fused
+GroupNorm(+SiLU) match."""
+
+import types
+
+import pytest
+
+torch = pytest.importorskip("torch")
+
+from core.inference import diffusion_vae_oom as O # noqa: E402
+from core.inference import diffusion_vae_fused as F # noqa: E402
+from core.inference import decode_layers as L # noqa: E402
+
+
+_REAL_RETURN_TO = O._return_to
+
+
+class _FakeOOM(RuntimeError):
+ def __init__(self):
+ super().__init__("CUDA out of memory. Tried to allocate 2.00 GiB")
+
+
+_CARD = torch.float64
+
+
+class _Budget:
+ """A fake card: what is resident there (pipeline modules, the VAE weights) plus the decode's need must fit."""
+
+ def __init__(self, cap):
+ self.cap = cap
+
+ def resident(self, modules):
+ return sum(
+ int(t.numel()) * t.element_size()
+ for m in modules
+ for t in [*m.parameters(), *m.buffers()]
+ if t.dtype == _CARD
+ )
+
+
+class _VaeLike(torch.nn.Module):
+ """Stock-diffusers-shaped tiling attrs; a decode needs ``untiled_need`` bytes untiled, ``tiled_need`` tiled, on top
+ of everything resident on the fake card (the VAE's own weights included)."""
+
+ spatial_compression_ratio = 8
+
+ def __init__(
+ self,
+ budget,
+ pipe_modules,
+ untiled_need,
+ tiled_need,
+ weight_bytes = 4096,
+ ):
+ super().__init__()
+ self.use_tiling = False
+ self.tile_sample_min_height = 256
+ self.tile_sample_min_width = 256
+ self.tile_sample_stride_height = 192
+ self.tile_sample_stride_width = 192
+ self.conv = torch.nn.Linear(weight_bytes // 8, 1, bias = False)
+ self.budget, self.pipe_modules = budget, pipe_modules
+ self.untiled_need, self.tiled_need = untiled_need, tiled_need
+ self.seen = []
+ self.cleared = 0
+
+ def enable_tiling(self):
+ self.use_tiling = True
+
+ def clear_cache(self):
+ self.cleared += 1
+
+ def _decode(
+ self,
+ z,
+ return_dict = True,
+ ):
+ self.seen.append(
+ (self.use_tiling, dict(O.tile_attrs(self)), self.conv.weight.dtype == _CARD)
+ )
+ need = self.tiled_need if self.use_tiling else self.untiled_need
+ if self.budget.resident([*self.pipe_modules, self]) + need > self.budget.cap:
+ raise _FakeOOM()
+ return (
+ z.repeat_interleave(8, -1).repeat_interleave(8, -2) + (0.5 if self.use_tiling else 0.0),
+ )
+
+
+class _Pipe:
+ def __init__(self, **components):
+ self.components = components
+ for k, v in components.items():
+ setattr(self, k, v)
+
+
+def _on_card(nbytes):
+ m = torch.nn.Linear(nbytes // 8, 1, bias = False)
+ return m.to(_CARD)
+
+
[email protected](autouse = True)
+def _fake_card(monkeypatch):
+ """The ladder only frees a CUDA card: let the meta device stand in for one, and host RAM be plentiful."""
+ monkeypatch.setattr(O, "_device_of", lambda z: _FakeDevice())
+ monkeypatch.setattr(O, "_host_free_mib", lambda: 1 << 20)
+ monkeypatch.setattr(O, "_empty_cache", lambda device: None)
+ yield
+
+
+class _FakeDevice:
+ type = "cuda"
+
+
+def _whole_on(module, device):
+ tensors = [*module.parameters(), *module.buffers()]
+ return bool(tensors) and all(t.dtype == _CARD for t in tensors)
+
+
[email protected](autouse = True)
+def _fake_moves(monkeypatch):
+ monkeypatch.setattr(O, "_whole_on", _whole_on)
+ monkeypatch.setattr(O, "_return_to", lambda module, device: module.to(_CARD))
+ monkeypatch.setattr(O, "_to_host", lambda module: module.to(torch.float32))
+ monkeypatch.setattr(O, "_whole_on_host", lambda module: all(
+ t.dtype == torch.float32 for t in [*module.parameters(), *module.buffers()]), raising = False) # fmt: skip
+ yield
+
+
+def _setup(
+ cap,
+ untiled_need,
+ tiled_need,
+ other = 8192,
+ tiled = False,
+ can_free = True,
+):
+ budget = _Budget(cap)
+ transformer = _on_card(other) if can_free else torch.nn.Linear(4, 1)
+ text_encoder = _on_card(other // 2) if can_free else torch.nn.Linear(4, 1)
+ vae = _VaeLike(budget, [transformer, text_encoder], untiled_need, tiled_need)
+ vae.to(_CARD)
+ vae.use_tiling = tiled
+ pipe = _Pipe(transformer = transformer, text_encoder = text_encoder, vae = vae, tokenizer = object())
+ assert O.install(vae, pipe = pipe)
+ return vae, pipe
+
+
+def _z():
+ return torch.ones(1, 4, 1, 8, 8)
+
+
+def _devices(pipe):
+ return {
+ n: next(m.parameters()).dtype
+ for n, m in pipe.components.items()
+ if isinstance(m, torch.nn.Module)
+ }
+
+
+def test_decode_that_fits_is_untouched():
+ vae, pipe = _setup(cap = 1 << 20, untiled_need = 100, tiled_need = 10)
+ before = _devices(pipe)
+ out = vae._decode(_z())
+ assert vae.seen == [(False, O.tile_attrs(vae), True)]
+ assert torch.equal(out[0], _z().repeat_interleave(8, -1).repeat_interleave(8, -2))
+ assert _devices(pipe) == before
+ assert vae._unsloth_vae_oom_stats["oom"] == 0
+
+
+def test_step_a_frees_the_pipeline_and_retries_the_same_untiled_decode():
+ """Fail-before: untiled needs more than is free beside the transformer and text encoder; with them on the host it
+ fits. The retry is the SAME untiled decode (bit-identical), not a tiled one, and both come back."""
+ # resident: 8192 + 4096 + 4096 (vae) = 16384; untiled 10000 fits only once 12288 is freed
+ vae, pipe = _setup(cap = 16384 + 1000, untiled_need = 10000, tiled_need = 500)
+ before = _devices(pipe)
+ out = vae._decode(_z())
+ assert [s[0] for s in vae.seen] == [False, False]
+ assert torch.equal(out[0], _z().repeat_interleave(8, -1).repeat_interleave(8, -2))
+ stats = vae._unsloth_vae_oom_stats
+ assert (
+ stats["recovered_by"] == "freed"
+ and stats["tiled_retry"] == 0
+ and stats["offload_retry"] == 0
+ )
+ assert _devices(pipe) == before
+ assert O._SESSION_ATTR not in vae.__dict__
+
+
+def test_step_b_stock_tiles_when_freeing_is_not_enough():
+ vae, pipe = _setup(cap = 16384 + 1000, untiled_need = 10**9, tiled_need = 2000)
+ stock = O.tile_attrs(vae)
+ out = vae._decode(_z())
+ assert [s[0] for s in vae.seen] == [False, False, True]
+ assert all(s[1] == stock for s in vae.seen) # never below the stock tiles
+ assert vae._unsloth_vae_oom_stats["recovered_by"] == "tiled"
+ assert torch.equal(out[0], _z().repeat_interleave(8, -1).repeat_interleave(8, -2) + 0.5)
+ assert vae.use_tiling is False and _devices(pipe)["transformer"] == _CARD
+
+
+def test_step_c_offloads_the_vae_at_stock_tiles(monkeypatch):
+ """Fail-before: even tiled, the VAE's own weights leave no room; streaming them from the host does."""
+ offloads = []
+
+ monkeypatch.setattr(
+ O,
+ "_apply_vae_offload",
+ lambda vae, device: (offloads.append(1), vae.conv.to(torch.float32)),
+ )
+ monkeypatch.setattr(O, "_undo_vae_offload", lambda vae, device: vae.conv.to(_CARD))
+ vae, pipe = _setup(cap = 4096 + 100, untiled_need = 10**9, tiled_need = 3000, tiled = True)
+ stock = O.tile_attrs(vae)
+ vae._decode(_z())
+ assert vae.seen == [(True, stock, True), (True, stock, True), (True, stock, False)]
+ assert vae._unsloth_vae_oom_stats["recovered_by"] == "offload" and offloads == [1]
+ assert vae.conv.weight.dtype == _CARD
+ assert _devices(pipe)["text_encoder"] == _CARD
+
+
+def test_streamed_decode_that_breaks_fails_on_the_oom(monkeypatch):
+ """A non-OOM error inside the streamed (step c) decode, e.g. an offload hook the environment cannot run, must not
+ replace the out-of-memory the render already had; the VAE still comes back."""
+ monkeypatch.setattr(O, "_apply_vae_offload", lambda vae, device: None)
+ monkeypatch.setattr(O, "_undo_vae_offload", lambda vae, device: None)
+ vae, pipe = _setup(cap = 1, untiled_need = 10**9, tiled_need = 10**9, tiled = True)
+ real = L.find(vae, "_decode", O._RETRY_LAYER).inner
+
+ def inner(z, *a, **k):
+ holder = vae.__dict__.get(O._SESSION_ATTR)
+ if holder is not None and holder["session"] is not None and holder["session"].vae_offloaded:
+ raise NameError("send_to_device")
+ return real(z, *a, **k)
+
+ O.uninstall(vae)
+ vae._decode = inner
+ O.install(vae)
+ with pytest.raises(RuntimeError, match = "even with the rest of the pipeline"):
+ vae._decode(_z())
+
+
+def test_every_step_fails_raises_and_restores(monkeypatch):
+ monkeypatch.setattr(O.DecodeSession, "offload_vae", lambda self: False)
+ vae, pipe = _setup(cap = 1, untiled_need = 10**9, tiled_need = 10**9)
+ before = _devices(pipe)
+ with pytest.raises(RuntimeError, match = "even with the rest of the pipeline"):
+ vae._decode(_z())
+ assert _devices(pipe) == before
+ assert all(s[1] == O.tile_attrs(vae) for s in vae.seen)
+ assert vae.use_tiling is False and vae.cleared >= 2
+
+
+def test_nothing_to_free_skips_the_identical_retry(monkeypatch):
+ monkeypatch.setattr(O.DecodeSession, "offload_vae", lambda self: False)
+ vae, pipe = _setup(cap = 4096 + 600, untiled_need = 10**9, tiled_need = 500, can_free = False)
+ vae._decode(_z())
+ assert [s[0] for s in vae.seen] == [False, True]
+ assert vae._unsloth_vae_oom_stats["freed_retry"] == 0
+
+
+def test_offload_hooked_and_shared_components_stay(monkeypatch):
+ """Components an offload mechanism places (accelerate / group offload) are left to it; the VAE never moves in (a)."""
+ vae, pipe = _setup(cap = 16384 + 1000, untiled_need = 10**9, tiled_need = 10**9)
+ pipe.components["transformer"]._hf_hook = object()
+ monkeypatch.setattr(O.DecodeSession, "offload_vae", lambda self: False)
+ moved = []
+ real = torch.nn.Module.to
+
+ def spy(self, *a, **k):
+ if a and a[0] == torch.float32:
+ moved.append(self)
+ return real(self, *a, **k)
+
+ monkeypatch.setattr(torch.nn.Module, "to", spy)
+ with pytest.raises(RuntimeError):
+ vae._decode(_z())
+ assert moved == [pipe.components["text_encoder"]]
+
+
+def test_host_ram_short_keeps_component(monkeypatch):
+ monkeypatch.setattr(O, "_host_free_mib", lambda: 1)
+ session = O.DecodeSession(None, _FakeDevice())
+ pipe = _Pipe(transformer = _on_card(1 << 20))
+ session.free_others(pipe)
+ assert session.moved == []
+
+
+def test_restore_that_runs_out_of_memory_returns_on_next_forward(monkeypatch):
+ m = _on_card(64)
+ session = O.DecodeSession(None, _FakeDevice())
+ session.log.append(session._component_change("transformer", m))
+ m.to(torch.float32)
+ calls = []
+
+ def flaky(module, device):
+ calls.append(1)
+ module.to(torch.float16)
+ if len(calls) <= 2:
+ raise _FakeOOM()
+ module.to(_CARD)
+
+ monkeypatch.setattr(O, "_return_to", flaky)
+ session.restore()
+ assert session.log == []
+ assert next(m.parameters()).dtype == torch.float32
+ # Item 4282775926: the card is still full at the next forward: the OOM surfaces and the way back STAYS
+ with pytest.raises(RuntimeError, match = "still on the host.*out of memory"):
+ m(torch.zeros(1, 8, dtype = _CARD))
+ assert next(m.parameters()).dtype == torch.float32 and len(m._forward_pre_hooks) == 1
+ assert isinstance(m.__dict__.get("_hf_hook"), O.DeferredReturn)
+ m(torch.zeros(1, 8, dtype = _CARD))
+ assert next(m.parameters()).dtype == _CARD and len(m._forward_pre_hooks) == 0
+ assert "_hf_hook" not in m.__dict__ and O._DEFERRED_ATTR not in m.__dict__
+
+
+def _cpu_group_offload_ok() -> bool:
+ """diffusers' group offloading builds on a host with no accelerator (older releases ask torch.accelerator for one
+ and raise); the CPU onload below stands in for the card only where it does."""
+ try:
+ from diffusers.hooks import apply_group_offloading
+ apply_group_offloading(
+ torch.nn.Linear(2, 2), onload_device = torch.device("cpu"), offload_type = "leaf_level"
+ )
+ return True
+ except Exception: # noqa: BLE001
+ return False
+
+
+needs_cpu_group_offload = pytest.mark.skipif(
+ not _cpu_group_offload_ok(), reason = "this diffusers' group offloading needs an accelerator"
+)
+
+
+def _tiny_vae():
+ diffusers = pytest.importorskip("diffusers")
+ torch.manual_seed(0)
+ return diffusers.AutoencoderKL(
+ block_out_channels = (8, 16), down_block_types = ("DownEncoderBlock2D",) * 2,
+ up_block_types = ("UpDecoderBlock2D",) * 2, latent_channels = 4, norm_num_groups = 4, layers_per_block = 1,
+ ).eval() # fmt: skip
+
+
+@needs_cpu_group_offload
+def test_vae_restore_oom_keeps_offload_hooks_and_retries(monkeypatch):
+ """Item 4282775917: the VAE's weights cannot all return after a streamed decode. Before: hooks removed, weights
+ stranded on the host. After: weights whole on the host WITH the offload hooks back (it still decodes, streamed),
+ and its next decode brings it back once memory allows."""
+ vae = _tiny_vae()
+ z = torch.randn(1, 4, 8, 8)
+ with torch.no_grad():
+ ref = vae.decode(z).sample
+ cpu = torch.device("cpu")
+ monkeypatch.setattr(O, "_to_host", lambda m: m.to("cpu", torch.float32))
+ full = [True]
+
+ def ret(module, device):
+ if full[0]:
+ module.decoder.conv_in.to(torch.float64)
+ raise _FakeOOM()
+ _REAL_RETURN_TO(module, device)
+
+ monkeypatch.setattr(O, "_return_to", ret)
+ session = O.DecodeSession(vae, cpu)
+ O._apply_vae_offload(vae, cpu)
+ session.log.append(
+ O.Change(
+ "vae_offload",
+ lambda: O._remove_vae_offload(vae, cpu),
+ lambda: O._park_vae(vae, cpu),
+ lambda retry, exc: O._defer(vae, cpu, "vae_offload", retry, False, vae),
+ )
+ )
+ session.restore()
+ monkeypatch.undo()
+ assert session.log == []
+ assert O._hooked(vae), "the weights are on the host: the offload hooks must stay"
+ assert all(p.dtype == torch.float32 for p in vae.parameters())
+ with torch.no_grad():
+ assert torch.equal(vae.decode(z).sample, ref)
+ assert O.settle_deferred(vae)
+ assert (
+ not O._hooked(vae)
+ and O._DEFERRED_ATTR not in vae.__dict__
+ and "_hf_hook" not in vae.__dict__
+ )
+ with torch.no_grad():
+ assert torch.equal(vae.decode(z).sample, ref)
+
+
+def test_execution_device_resolves_the_card_while_a_component_waits_on_the_host(monkeypatch):
+ """A real diffusers pipeline whose first component could not return: ``_execution_device`` must still be the
+ card, and the next call must run (the component comes back before its forward). The meta device stands in for
+ the host, the CPU for the card."""
+ diffusers = pytest.importorskip("diffusers")
+
+ class TinyPipe(diffusers.DiffusionPipeline):
+ def __init__(self, unet, vae):
+ super().__init__()
+ self.register_modules(unet = unet, vae = vae)
+
+ def __call__(self):
+ x = torch.ones(1, 4, 8, 8, device = self._execution_device)
+ return self.vae.decode(self.unet(x, 1).sample).sample
+
+ torch.manual_seed(0)
+ unet = diffusers.UNet2DModel(
+ sample_size = 8, in_channels = 4, out_channels = 4, block_out_channels = (8, 16), layers_per_block = 1,
+ norm_num_groups = 4, down_block_types = ("DownBlock2D", "DownBlock2D"), up_block_types = ("UpBlock2D",) * 2,
+ ).eval() # fmt: skip
+ pipe = TinyPipe(unet, _tiny_vae())
+ with torch.no_grad():
+ ref = pipe()
+ saved = {k: v.clone() for k, v in unet.state_dict().items()}
+ card = torch.device("cpu")
+ monkeypatch.setattr(O, "_to_host", lambda m: m.to("meta"))
+ oom = [True]
+
+ def ret(module, device):
+ if oom[0]:
+ raise _FakeOOM()
+ module.to_empty(device = device)
+ module.load_state_dict(saved)
+
+ monkeypatch.setattr(O, "_return_to", ret)
+ session = O.DecodeSession(pipe.vae, card)
+ unet.to("meta")
+ session.log.append(session._component_change("unet", unet))
+ session.restore()
+ assert next(unet.parameters()).device.type == "meta"
+ assert pipe._execution_device == card # fail-before: meta
+ oom[0] = False
+ with torch.no_grad():
+ out = pipe()
+ assert torch.equal(out, ref) and next(unet.parameters()).device == card
+ assert "_hf_hook" not in unet.__dict__ and len(unet._forward_pre_hooks) == 0
+
+
+def test_uninstall_removes_compiled_guard_and_settles_deferred():
+ """Item 4282775938: uninstall puts the compiled decode back (before: the guard stayed and kept retrying) and
+ settles every deferred return the wrapper registered."""
+ vae, pipe = _setup(cap = 1 << 20, untiled_need = 1, tiled_need = 1)
+ O.uninstall(vae)
+
+ def compiled(z, return_dict = True):
+ return ("compiled",)
+
+ compiled._unsloth_compiled_decode = True
+ compiled._unsloth_eager_decode = lambda *a, **k: ("eager",)
+ vae.decode = compiled
+ assert O.install(vae, pipe = pipe)
+ assert vae.__dict__["decode"] is not compiled and vae.__dict__["decode"]._unsloth_vae_oom_guard
+ m = pipe.components["text_encoder"]
+ O._defer(m, _FakeDevice(), "moved:text_encoder", lambda: None, True, vae)
+ assert O._DEFERRED_ATTR in m.__dict__
+ O.uninstall(vae)
+ assert vae.__dict__["decode"] is compiled
+ assert O._DEFERRED_ATTR not in m.__dict__ and "_hf_hook" not in m.__dict__
+ assert O._RECORD_ATTR not in vae.__dict__ and "_decode" not in vae.__dict__
+ assert O.install(vae, pipe = pipe) and vae.__dict__["decode"]._unsloth_vae_oom_guard
+
+
+def test_uninstall_keeps_a_deferred_return_that_still_cannot_run():
+ vae, pipe = _setup(cap = 1 << 20, untiled_need = 1, tiled_need = 1)
+ m = pipe.components["text_encoder"]
+ O._defer(
+ m, _FakeDevice(), "moved:text_encoder", lambda: (_ for _ in ()).throw(_FakeOOM()), True, vae
+ )
+ O.uninstall(vae)
+ assert O._DEFERRED_ATTR in m.__dict__ # its only way back to the card
+
+
+def test_step_a_drops_graphs_and_repins_resident_groups(monkeypatch):
+ from core.inference import diffusion_cuda_graph as G
+ from core.inference import diffusion_memory as M
+
+ events = []
+ monkeypatch.setattr(
+ G, "reset_all", lambda handles, strict = False: events.append(("graphs", tuple(handles)))
+ )
+ monkeypatch.setattr(
+ M, "release_resident_groups",
+ lambda pipe, need, logger = None, reason = "": (events.append(("release", need)), lambda: events.append(("repin",)))[1],
+ ) # fmt: skip
+ vae, pipe = _setup(cap = 16384 + 1000, untiled_need = 10000, tiled_need = 500)
+ pipe._unsloth_cuda_graphs = ("h",)
+ pipe.components["text_encoder"]._unsloth_resident_room = 1
+ vae._decode(_z())
+ assert (
+ events[0] == ("graphs", ("h",)) and events[1][0] == "release" and events[-1] == ("repin",)
+ )
+
+
+def test_untiled_gate_keeps_its_own_fallback():
+ vae, pipe = _setup(cap = 16384 + 1000, untiled_need = 10000, tiled_need = 500)
+
+ def gate(*a, **k):
+ return None
+
+ gate._unsloth_untiled_decode = True
+
+ def bf16_decode(*a, **k): # the ROCm bf16 decode exposes the gate only through __wrapped__
+ return gate(*a, **k)
+
+ bf16_decode.__wrapped__ = gate
+ for outer in (gate, bf16_decode):
+ vae.seen.clear()
+ vae.decode = outer
+ # Item 4283572976: the ladder sees the gate's untiled OOM first: the card is freed and untiled retried
+ vae._decode(_z())
+ assert [s[0] for s in vae.seen] == [False, False]
+ assert vae._unsloth_vae_oom_stats["recovered_by"] == "freed"
+ # nothing to free (or still out of memory): the gate's own tiled fallback gets the original OOM
+ vae, pipe = _setup(cap = 4096 + 600, untiled_need = 10**9, tiled_need = 500, can_free = False)
+ vae.decode = gate
+ with pytest.raises(_FakeOOM):
+ vae._decode(_z())
+ assert [s[0] for s in vae.seen] == [False]
+
+
+def test_kill_switch(monkeypatch):
+ vae, pipe = _setup(cap = 16384 + 1000, untiled_need = 10000, tiled_need = 500)
+ monkeypatch.setenv(O.OOM_RETRY_ENV, "0")
+ with pytest.raises(RuntimeError, match = "out of memory"):
+ vae._decode(_z()) # installed, switched off at call time: the OOM surfaces as on main
+ assert len(vae.seen) == 1
+ other = _VaeLike(_Budget(1), [], 1, 1)
+ assert O.install(other) is False and "_decode" not in other.__dict__
+
+
+def test_non_oom_propagates():
+ vae, pipe = _setup(cap = 1 << 20, untiled_need = 1, tiled_need = 1)
+ O.uninstall(vae)
+ vae._decode = types.MethodType(lambda self, z: (_ for _ in ()).throw(ValueError("x")), vae)
+ O.install(vae)
+ with pytest.raises(ValueError):
+ vae._decode(_z())
+
+
+def test_uninstall_restores_class_decode_and_reinstall_keeps_pipe():
+ vae, pipe = _setup(cap = 1 << 20, untiled_need = 1, tiled_need = 1)
+ assert "_decode" in vae.__dict__
+ O.uninstall(vae)
+ assert "_decode" not in vae.__dict__
+ assert O.install(vae) and O._pipe_of(vae) is pipe
+
+
+def test_no_halving_api_left():
+ for name in ("halved", "MIN_TILE_LATENTS", "MIN_PSNR_DB", "oom_halving_active"):
+ assert not hasattr(O, name)
+
+
+@needs_cpu_group_offload
+def test_real_vae_offload_hooks_round_trip(monkeypatch):
+ """Step (c)'s real mechanism on a tiny diffusers VAE (CPU onload stands in for the card): leaf-level hooks decode
+ bit-identically and are fully removed afterwards, weights back where they were."""
+ diffusers = pytest.importorskip("diffusers")
+ torch.manual_seed(0)
+ vae = diffusers.AutoencoderKL(
+ block_out_channels = (8, 16), down_block_types = ("DownEncoderBlock2D",) * 2,
+ up_block_types = ("UpDecoderBlock2D",) * 2, latent_channels = 4, norm_num_groups = 4, layers_per_block = 1,
+ ).eval() # fmt: skip
+ z = torch.randn(1, 4, 8, 8)
+ monkeypatch.setattr(O, "_return_to", _REAL_RETURN_TO)
+ with torch.no_grad():
+ ref = vae.decode(z).sample
+ cpu = torch.device("cpu")
+ O._apply_vae_offload(vae, cpu)
+ assert O._hooked(vae)
+ with torch.no_grad():
+ out = vae.decode(z).sample
+ assert torch.equal(out, ref)
+ O._undo_vae_offload(vae, cpu)
+ assert not O._hooked(vae)
+ with torch.no_grad():
+ assert torch.equal(vae.decode(z).sample, ref)
+
+
+def test_compiled_decode_oom_frees_then_retries_compiled():
+ """Step (a) under a compiled decode retries the compiled callable itself (same kernels), not the eager path."""
+ vae, pipe = _setup(cap = 16384 + 1000, untiled_need = 10000, tiled_need = 500)
+ calls = []
+
+ def eager(z, return_dict = True):
+ calls.append("eager")
+ return vae._decode(z, return_dict)
+
+ def compiled(z, return_dict = True):
+ calls.append("compiled")
+ if vae.budget.resident([*vae.pipe_modules, vae]) + vae.untiled_need > vae.budget.cap:
+ try:
+ raise _FakeOOM()
+ except _FakeOOM as oom:
+ raise RuntimeError("compiled graph failed") from oom
+ return ("ok",)
+
+ from core.inference import diffusion_speed as S
+
+ O.uninstall(vae)
+ vae.decode = S._guard_compiled_decode(vae, compiled, eager, None)
+ assert O.install(vae)
+ before = _devices(pipe)
+ assert vae.decode(_z()) == ("ok",)
+ assert calls == ["compiled", "compiled"]
+ assert (
+ vae._unsloth_vae_oom_stats["recovered_by"] == "freed_compiled" and _devices(pipe) == before
+ )
+
+
+def test_compiled_decode_oom_falls_to_the_eager_ladder_sharing_the_session():
+ vae, pipe = _setup(cap = 16384 + 1000, untiled_need = 10**9, tiled_need = 500)
+ calls = []
+
+ def eager(z, return_dict = True):
+ calls.append("eager")
+ assert O._SESSION_ATTR in vae.__dict__
+ return vae._decode(z, return_dict)
+
+ def compiled(z, return_dict = True):
+ calls.append("compiled")
+ raise _FakeOOM()
+
+ from core.inference import diffusion_speed as S
+
+ O.uninstall(vae)
+ vae.decode = S._guard_compiled_decode(vae, compiled, eager, None)
+ assert O.install(vae)
+ before = _devices(pipe)
+ out = vae.decode(_z())
+ assert calls == ["compiled", "compiled", "eager"]
+ assert vae._unsloth_vae_oom_stats["recovered_by"] == "tiled" and out[0].shape[-1] == 64
+ assert _devices(pipe) == before and O._SESSION_ATTR not in vae.__dict__
+
+
+def test_compiled_guard_absent_without_compile():
+ vae, pipe = _setup(cap = 1 << 20, untiled_need = 1, tiled_need = 1)
+ assert "decode" not in vae.__dict__
+
+
+def test_is_oom_walks_the_cause_chain():
+ class OutOfMemoryError(RuntimeError):
+ pass
+
+ try:
+ try:
+ raise OutOfMemoryError("CUDA out of memory")
+ except OutOfMemoryError as inner:
+ raise RuntimeError("backend compiler failed") from inner
+ except RuntimeError as exc:
+ wrapped = exc
+ assert O.is_oom(wrapped)
+ assert not O.is_oom(RuntimeError("backend compiler failed"))
+
+
+def test_is_oom_walks_exception_group_members():
+ import builtins
+
+ class Group(Exception):
+ def __init__(self, msg, exceptions):
+ super().__init__(msg)
+ self.exceptions = tuple(exceptions)
+
+ oom = RuntimeError("CUDA out of memory. Tried to allocate 2.00 GiB")
+ groups = [Group("decode failed", [ValueError("cancelled"), oom])]
+ if hasattr(builtins, "ExceptionGroup"):
+ groups.append(builtins.ExceptionGroup("decode failed", [ValueError("cancelled"), oom]))
+ for group in groups:
+ assert O.is_oom(group)
+ assert O.is_oom(Group("outer", [group]))
+ assert not O.is_oom(Group("decode failed", [ValueError("cancelled"), KeyError("x")]))
+ cyclic = Group("cycle", [])
+ cyclic.exceptions = (cyclic,)
+ assert not O.is_oom(cyclic)
+
+
+def test_speed_layer_compiled_guard_exposes_eager():
+ from core.inference import diffusion_speed as S
+
+ eager = lambda *a, **k: "eager" # noqa: E731
+ guarded = S._guard_compiled_decode(
+ types.SimpleNamespace(), lambda *a, **k: "compiled", eager, None
+ )
+ assert guarded._unsloth_compiled_decode is True and guarded._unsloth_eager_decode is eager
+
+
+class _Block(torch.nn.Module):
+ def __init__(self, c):
+ super().__init__()
+ self.act = torch.nn.SiLU()
+ self.norm1 = torch.nn.GroupNorm(4, c)
+ self.conv = torch.nn.Conv2d(c, c, 3, padding = 1)
+ self.norm2 = torch.nn.GroupNorm(4, c)
+
+ def forward(self, x):
+ h = self.act(self.norm1(x))
+ h = self.conv(h)
+ h2 = self.norm2(h)
+ return x + self.act(h2) + h2
+
+
+class _Dec(torch.nn.Module):
+ def __init__(self, c):
+ super().__init__()
+ self.block = _Block(c)
+ self.norm_out = torch.nn.GroupNorm(4, c)
+ self.out_act = torch.nn.SiLU()
+ self.attn_norm = torch.nn.GroupNorm(4, c)
+
+ def forward(self, x):
+ x = self.block(x)
+ x = x + self.attn_norm(x)
+ return self.out_act(self.norm_out(x))
+
+
+class AutoencoderKLSomethingNew(torch.nn.Module):
+ def __init__(self, c = 8):
+ super().__init__()
+ self.decoder = _Dec(c)
+
+ def _decode(
+ self,
+ z,
+ return_dict = True,
+ ):
+ return (self.decoder(z),)
+
+
+def test_dataflow_trace_pairs_only_exclusive_group_norm_silu():
+ vae = AutoencoderKLSomethingNew()
+ norms = F._plain_group_norms(vae.decoder)
+ silus = F._plain_silus(vae.decoder)
+ calls, consumed, _, info = F._trace_group_norm_silu(
+ norms, silus, lambda: vae._decode(torch.randn(1, 8, 6, 6))
+ )
+ names = {id(m): n for n, m in vae.decoder.named_modules()}
+ got = {names[id(m)]: (c, k) for m, c, k in zip(norms, calls, consumed)}
+ assert got["block.norm1"] == (1, 1)
+ assert got["norm_out"] == (1, 1)
+ assert got["attn_norm"] == (1, 0)
+ # Item 4283572932: norm2's output reaches the SiLU AND the residual: not exclusively consumed (before: (1, 1),
+ # fused with its SiLU, which changed what the residual read)
+ assert got["block.norm2"] == (1, 0)
+ shared = {names[id(m)]: k for m, k in zip(norms, info["shared"])}
+ assert shared["block.norm2"] >= 1 and shared["block.norm1"] == 0 and shared["norm_out"] == 0
+
+
+def _install_on_cpu(monkeypatch, vae):
+ monkeypatch.setattr(F, "runtime_ok", lambda: True)
+ monkeypatch.delenv(F.STRUCTURAL_ENV, raising = False)
+ n = F.install_structural(vae)
+ assert n == 4
+ state = vae._unsloth_vae_structural
+ # calibrate as on the first CUDA decode (CPU tensors take the reference GroupNorm, so it is exact here)
+ z = torch.randn(1, 8, 6, 6)
+ inner = L.find(vae, "_decode", F._STRUCTURAL_LAYER).inner
+ with torch.no_grad():
+ calls, consumed, ref, _ = F._trace_group_norm_silu(
+ F._plain_group_norms(vae.decoder), F._plain_silus(vae.decoder), lambda: inner(z)[0]
+ )
+ return state, z, ref
+
+
+def test_structural_patch_is_exact_when_pairing_is_right(monkeypatch):
+ vae = AutoencoderKLSomethingNew()
+ state, z, ref = _install_on_cpu(monkeypatch, vae)
+ norms = F._plain_group_norms(vae.decoder)
+ silus = F._plain_silus(vae.decoder)
+ names = {id(m): n for n, m in vae.decoder.named_modules()}
+ paired = [
+ names[id(m)] in ("block.norm1", "norm_out") or (False if names[id(m)] else None)
+ for m in norms
+ ]
+ F._structural_patch(norms, paired, silus, state, None)
+ with torch.no_grad():
+ out = L.find(vae, "_decode", F._STRUCTURAL_LAYER).inner(z)[0]
+ assert torch.allclose(out, ref, atol = 1e-6)
+ assert state.calls == 4
+ F._structural_unpatch(norms, silus)
+ assert not any("forward" in m.__dict__ for m in (*norms, *silus))
+
+
+def test_structural_probe_rejects_a_wrong_pairing(monkeypatch):
+ """norm2 looks paired to the trace but its output also feeds a residual: the probe check rejects the patch."""
+ vae = AutoencoderKLSomethingNew()
+ monkeypatch.setattr(F, "runtime_ok", lambda: True)
+ F.install_structural(vae)
+ state = vae._unsloth_vae_structural
+ z = torch.randn(1, 8, 6, 6) * 3
+ monkeypatch.setattr(torch.Tensor, "is_cuda", property(lambda self: True))
+ monkeypatch.setattr(F, "_gn_fusable", lambda x, norm: False)
+ with torch.no_grad():
+ vae._decode(z)
+ assert state.status == "rejected"
+ assert not any("forward" in m.__dict__ for m in vae.decoder.modules())
+
+
+def test_structural_skips_measured_classes_and_kill_switch(monkeypatch):
+ class AutoencoderKL(AutoencoderKLSomethingNew):
+ pass
+
+ assert F.structural_candidate(AutoencoderKL()) is False
+ assert F.structural_candidate(AutoencoderKLSomethingNew()) is True
+ monkeypatch.setenv(F.STRUCTURAL_ENV, "0")
+ assert F.structural_candidate(AutoencoderKLSomethingNew()) is False
+ monkeypatch.delenv(F.STRUCTURAL_ENV)
+ vae = AutoencoderKLSomethingNew()
+ vae._unsloth_decode_blocks_bypassed = True
+ assert F.structural_candidate(vae) is False
+
+
+def _cuda_triton() -> bool:
+ return torch.cuda.is_available() and not getattr(torch.version, "hip", None) and F.runtime_ok()
+
+
+needs_cuda = pytest.mark.skipif(not _cuda_triton(), reason = "needs NVIDIA CUDA + Triton")
+
+
+def _psnr(a, b):
+ mse = (a.float() - b.float()).pow(2).mean().item()
+ return float("inf") if mse == 0 else 10 * torch.log10(torch.tensor(4.0 / mse)).item()
+
+
+def _tiny(name):
+ diffusers = pytest.importorskip("diffusers")
+ torch.manual_seed(0)
+ if name == "AutoencoderKLHunyuanImage":
+ return diffusers.AutoencoderKLHunyuanImage(
+ in_channels = 3, out_channels = 3, latent_channels = 8, block_out_channels = (32, 64, 64),
+ layers_per_block = 1, spatial_compression_ratio = 4, sample_size = 64,
+ ), (1, 8, 12, 10) # fmt: skip
+ if name == "AutoencoderKLHunyuanVideo":
+ return diffusers.AutoencoderKLHunyuanVideo(
+ in_channels = 3, out_channels = 3, latent_channels = 4,
+ down_block_types = ("HunyuanVideoDownBlock3D",) * 3, up_block_types = ("HunyuanVideoUpBlock3D",) * 3,
+ block_out_channels = (32, 32, 32), layers_per_block = 1, norm_num_groups = 8,
+ spatial_compression_ratio = 4, temporal_compression_ratio = 4,
+ ), (1, 4, 2, 10, 12) # fmt: skip
+ raise AssertionError(name)
+
+
+@needs_cuda
[email protected]("name", ["AutoencoderKLHunyuanImage", "AutoencoderKLHunyuanVideo"])
+def test_structural_engages_on_uncovered_class(name):
+ vae, shape = _tiny(name)
+ vae = vae.cuda().to(torch.bfloat16).eval()
+ z = torch.randn(*shape, device = "cuda", dtype = torch.bfloat16)
+ with torch.no_grad():
+ ref = vae.decode(z, return_dict = False)[0].float()
+ assert F.will_install(vae) and F.install(vae) > 0
+ out = vae.decode(z, return_dict = False)[0].float()
+ state = vae._unsloth_vae_structural
+ assert state.status == "ok", vars(state)
+ assert state.gn_silu > 0 and state.calls > 0
+ assert _psnr(out, ref) > 35, _psnr(out, ref)
+ F.uninstall(vae)
+ assert "_decode" not in vae.__dict__
+ assert torch.equal(vae.decode(z, return_dict = False)[0].float(), ref)
+
+
+class AutoencoderKLTileableNew(AutoencoderKLSomethingNew):
+ spatial_compression_ratio = 8
+
+ def __init__(self, c = 8):
+ super().__init__(c)
+ self.use_tiling = False
+ self.tile_sample_min_height = 256
+ self.tile_sample_min_width = 256
+ self.tile_sample_stride_height = 192
+ self.tile_sample_stride_width = 192
+ self.decoder.proj3d = torch.nn.Conv3d(c, c, 3)
+
+ def enable_tiling(self):
+ self.use_tiling = True
+
+
+def _layers(vae):
+ fn, out = vae.__dict__.get("_decode"), []
+ while fn is not None and len(out) < 8:
+ out.append(fn)
+ fn = getattr(fn, "__wrapped__", None)
+ return out
+
+
+def test_uninstall_unwraps_structural_layer_under_the_oom_wrapper(monkeypatch):
+ """Item 4: with the OOM retry on top, uninstall must still remove the structural calibrator underneath (and keep
+ the retry). Before the fix the calibrator stayed and its saved state was dropped."""
+ monkeypatch.setattr(F, "runtime_ok", lambda: True)
+ vae = AutoencoderKLTileableNew()
+ assert F.install_structural(vae) > 0
+ assert O.install(vae)
+ assert any(getattr(f, "_unsloth_vae_fused", False) for f in _layers(vae))
+ F.uninstall(vae)
+ layers = _layers(vae)
+ assert not any(getattr(f, "_unsloth_vae_fused", False) for f in layers), layers
+ assert getattr(vae.__dict__.get("_decode"), "_unsloth_vae_oom_retry", False)
+ assert L.find(vae, "_decode", F._STRUCTURAL_LAYER) is None
+ with torch.no_grad():
+ out = vae._decode(torch.randn(1, 8, 6, 6))
+ assert out[0].shape == (1, 8, 6, 6)
+
+
+def test_rejected_structural_fusion_restores_contiguous_layout(monkeypatch):
+ """Item 5: the speed layer relays 3-D decode convs to channels_last_3d because fusion is planned; a rejected
+ calibration must put them back. Before the fix the weights stayed channels_last_3d."""
+ from core.inference import diffusion_speed as S
+
+ monkeypatch.delenv(S.VAE_CHANNELS_LAST_3D_ENV, raising = False)
+ monkeypatch.setattr(F, "runtime_ok", lambda: True)
+ vae = AutoencoderKLTileableNew()
+ w = vae.decoder.proj3d.weight
+ assert S._vae_channels_last_3d(vae, None, fused = True)
+ assert w.is_contiguous(memory_format = torch.channels_last_3d) and not w.is_contiguous()
+ F.install_structural(vae)
+ monkeypatch.setattr(torch.Tensor, "is_cuda", property(lambda self: True))
+ monkeypatch.setattr(F, "_gn_fusable", lambda x, norm: False)
+ with torch.no_grad():
+ vae._decode(torch.randn(1, 8, 6, 6) * 3)
+ assert vae._unsloth_vae_structural.status == "rejected"
+ assert vae.decoder.proj3d.weight.is_contiguous()
+
+
+def test_uninstall_restores_contiguous_layout(monkeypatch):
+ """Uninstall undoes the channels_last_3d relayout recorded for the fused passes."""
+ from core.inference import diffusion_speed as S
+
+ monkeypatch.delenv(S.VAE_CHANNELS_LAST_3D_ENV, raising = False)
+ monkeypatch.setattr(F, "runtime_ok", lambda: True)
+ vae = AutoencoderKLTileableNew()
+ assert S._vae_channels_last_3d(vae, None, fused = True)
+ F.install_structural(vae)
+ assert not vae.decoder.proj3d.weight.is_contiguous()
+ F.uninstall(vae)
+ assert vae.decoder.proj3d.weight.is_contiguous()
+ assert "_unsloth_channels_last_modules" not in vae.__dict__
+
+
+class _CleanDec(torch.nn.Module):
+ def __init__(self, c):
+ super().__init__()
+ self.norm_out = torch.nn.GroupNorm(4, c)
+ self.act = torch.nn.SiLU()
+
+ def forward(self, x):
+ return self.act(self.norm_out(x))
+
+
+class AutoencoderKLCleanNew(torch.nn.Module):
+ def __init__(self, c = 8):
+ super().__init__()
+ self.decoder = _CleanDec(c)
+
+ def _decode(
+ self,
+ z,
+ return_dict = True,
+ ):
+ return (self.decoder(z),)
+
+
+def test_structural_rejected_when_every_norm_falls_back(monkeypatch):
+ """A correct pairing whose norms all take group_norm_act's stock fallback fused nothing, so the
+ calibration must reject it (before: counted as fused and accepted)."""
+ monkeypatch.setattr(F, "runtime_ok", lambda: True)
+ vae = AutoencoderKLCleanNew()
+ F.install_structural(vae)
+ monkeypatch.setattr(torch.Tensor, "is_cuda", property(lambda self: True))
+ monkeypatch.setattr(F, "_gn_fusable", lambda x, norm: False)
+ with torch.no_grad():
+ vae._decode(torch.randn(1, 8, 6, 6))
+ state = vae._unsloth_vae_structural
+ assert state.status == "rejected" and state.kernel_calls == 0
+ assert not any("forward" in m.__dict__ for m in vae.decoder.modules())
+
+
+def test_kernel_calls_skip_the_temporal_stride_fallback(monkeypatch):
+ """A 5-D norm that passes _gn_fusable but whose T axis cannot be flattened takes the stock reference inside
+ group_norm_act; calibration must not count it as a kernel run."""
+ monkeypatch.setattr(F, "_gn_fusable", lambda x, norm: True)
+ norm = torch.nn.GroupNorm(4, 8)
+ state = types.SimpleNamespace(kernel_calls = 0, calls = 0)
+ fast = F._fast_gn_maybe_act(norm, True, state)
+ x = torch.randn(1, 8, 3, 4, 5).transpose(2, 3).contiguous().transpose(2, 3)
+ assert x.stride(2) != x.shape[3] * x.stride(3)
+ with torch.no_grad():
+ y = fast(x)
+ assert state.calls == 1 and state.kernel_calls == 0
+ assert torch.allclose(F._plain(y), torch.nn.functional.silu(norm(x)), atol = 1e-5)
+ with pytest.raises(F.FusedActivationRead):
+ y + 0
+
+
+class _CausalNew(AutoencoderKLCleanNew):
+ """A causal decoder whose output depends on a persistent cache filled by earlier decodes."""
+
+ def __init__(self, c = 8):
+ super().__init__(c)
+ self.cache = None
+
+ def clear_cache(self):
+ self.cache = None
+
+ def _decode(
+ self,
+ z,
+ return_dict = True,
+ ):
+ out = self.decoder(z)
+ if self.cache is not None:
+ out = out + self.cache.mean() # stale frames leak into the output
+ self.cache = z.detach()
+ return (out,)
+
+
+def test_structural_probe_does_not_leak_cache_into_the_real_decode(monkeypatch):
+ """The calibration's probe decodes must not leave cache entries the real decode reads."""
+ monkeypatch.setattr(F, "runtime_ok", lambda: True)
+ ref_vae = _CausalNew()
+ torch.manual_seed(0)
+ z = torch.randn(1, 8, 6, 6) + 2.0
+ with torch.no_grad():
+ ref = ref_vae._decode(z)[0]
+ vae = _CausalNew()
+ vae.load_state_dict(ref_vae.state_dict())
+ F.install_structural(vae)
+ monkeypatch.setattr(torch.Tensor, "is_cuda", property(lambda self: True))
+ with torch.no_grad():
+ out = vae._decode(z)[0]
+ assert torch.allclose(out, ref, atol = 1e-6), (out - ref).abs().max()
+
+
+def test_required_deferred_return_raises_on_any_failure():
+ """Item 4283031561: a required return that fails with a non-OOM error raised nothing before, so the forward ran
+ with its weights on the host. Now it raises a clear error, and the way back stays for a later retry."""
+ m = torch.nn.Linear(4, 4)
+ O._defer(m, _FakeDevice(), "moved:transformer", lambda: (_ for _ in ()).throw(ValueError("no such device")),
+ True, None) # fmt: skip
+ with pytest.raises(
+ RuntimeError, match = "transformer.*still on the host.*ValueError: no such device"
+ ):
+ m(torch.zeros(1, 4))
+ assert O._DEFERRED_ATTR in m.__dict__ and len(m._forward_pre_hooks) == 1
+ v = torch.nn.Linear(4, 4)
+ O._defer(
+ v, _FakeDevice(), "vae_offload", lambda: (_ for _ in ()).throw(ValueError("x")), False, None
+ )
+ v(torch.zeros(1, 4))
+ assert O._DEFERRED_ATTR in v.__dict__
+
+
+def _compiled_vae():
+ vae, pipe = _setup(cap = 1 << 20, untiled_need = 1, tiled_need = 1)
+ O.uninstall(vae)
+ calls = []
+
+ def compiled(z, return_dict = True):
+ calls.append("compiled")
+ raise _FakeOOM()
+
+ compiled._unsloth_compiled_decode = True
+ compiled._unsloth_eager_decode = lambda *a, **k: ("eager",)
+ vae.decode = compiled
+ assert O.install(vae, pipe = pipe)
+ return vae, pipe, compiled, calls
+
+
+def test_uninstall_neutralises_a_compiled_guard_under_a_later_wrapper():
+ """Item 4283031573: video.py wraps vae.decode (untiled gate, ROCm bf16) after the guard, capturing it. Before:
+ uninstall saw a different outermost decode, dropped its record, and the guard kept retrying. Now the buried guard
+ is a pass-through: an OOM out of the compiled decode surfaces, with no freeing and no eager retry."""
+ vae, pipe, compiled, calls = _compiled_vae()
+ guard = vae.__dict__["decode"]
+
+ def later(*a, **k):
+ return guard(*a, **k)
+
+ later.__wrapped__ = guard
+ vae.decode = later
+ O.uninstall(vae)
+ assert vae.__dict__["decode"] is later and later.__wrapped__ is compiled
+ with pytest.raises(_FakeOOM):
+ vae.decode(_z())
+ assert calls == ["compiled"] and _devices(pipe)["transformer"] == _CARD
+
+
+def test_uninstall_restores_compiled_decode_when_outermost():
+ vae, pipe, compiled, calls = _compiled_vae()
+ O.uninstall(vae)
+ assert vae.__dict__["decode"] is compiled
+
+
[email protected]("which", ["oom", "fused"])
+def test_uninstall_keeps_an_instance_bound_decode_override(monkeypatch, which):
+ """Item 4283031579: a VAE whose ``_decode`` was already an instance-bound override (types.MethodType) got the
+ class method back on uninstall before (the override was popped). Now exactly the override comes back."""
+ if which == "fused":
+ monkeypatch.setattr(F, "runtime_ok", lambda: True)
+ monkeypatch.delenv(F.STRUCTURAL_ENV, raising = False)
+ vae = AutoencoderKLSomethingNew()
+ else:
+ vae, pipe = _setup(cap = 1 << 20, untiled_need = 1, tiled_need = 1)
+ O.uninstall(vae)
+
+ def override(self, z, *a, **k):
+ return type(self)._decode(self, z, *a, **k)
+
+ bound = types.MethodType(override, vae)
+ vae._decode = bound
+ if which == "fused":
+ assert F.install_structural(vae) > 0
+ F.uninstall(vae)
+ else:
+ assert O.install(vae)
+ O.uninstall(vae)
+ assert vae.__dict__.get("_decode") is bound
+
+
+def test_uninstall_restores_the_class_decode_when_there_was_no_override():
+ vae, pipe = _setup(cap = 1 << 20, untiled_need = 1, tiled_need = 1)
+ O.uninstall(vae)
+ assert (
+ "_decode" not in vae.__dict__
+ and "decode" not in vae.__dict__
+ and L.LAYERS_ATTR not in vae.__dict__
+ )
+
+
+class _SlicedVae(_VaeLike):
+ def decode(
+ self,
+ z,
+ return_dict = True,
+ ):
+ return (torch.cat([self._decode(z[i : i + 1])[0] for i in range(z.shape[0])]),)
+
+
+def test_one_session_spans_a_sliced_batch_decode(monkeypatch):
+ """Item 4283031587: with slicing, every slice's _decode used to recover on its own: move the pipeline off,
+ restore it, then move it off again for the next slice. Now the public decode opens one session for the batch:
+ the pipeline moves off once and comes back once, after the last slice."""
+ moves = []
+ monkeypatch.setattr(O, "_to_host", lambda m: (moves.append("off"), m.to(torch.float32)))
+ monkeypatch.setattr(O, "_return_to", lambda m, d: (moves.append("on"), m.to(_CARD)))
+ budget = _Budget(16384 + 1000)
+ transformer, text_encoder = _on_card(8192), _on_card(4096)
+ vae = _SlicedVae(budget, [transformer, text_encoder], 10000, 500)
+ vae.to(_CARD)
+ pipe = _Pipe(transformer = transformer, text_encoder = text_encoder, vae = vae)
+ assert O.install(vae, pipe = pipe)
+ out = vae.decode(torch.ones(3, 4, 1, 8, 8))[0]
+ assert out.shape[0] == 3
+ assert moves == ["off", "off", "on", "on"]
+ assert [s[0] for s in vae.seen] == [False, False, False, False]
+ assert _devices(pipe)["transformer"] == _CARD and O._SESSION_ATTR not in vae.__dict__
+
+
+def test_real_release_resident_groups_keeps_the_restore_of_a_partial_release(monkeypatch):
+ """Item 4283219211: diffusion_memory.release_resident_groups caught a failure part way and returned None, so the
+ groups it had already demoted were never re-pinned. It now returns the restore for them (the failing one too)."""
+ from core.inference import diffusion_memory as M
+
+ class G:
+ def __init__(self):
+ self._unsloth_resident = True
+ self._unsloth_resident_bytes = 1 << 20
+
+ groups = [G(), G(), G()]
+ module = torch.nn.Linear(2, 2)
+ module._unsloth_resident_room = 1
+ module._unsloth_resident_device = "cpu"
+ pipe = types.SimpleNamespace(components = {"transformer": module})
+ demoted = []
+
+ def release(group):
+ if len(demoted) == 1:
+ raise RuntimeError("copy stream failed")
+ group._unsloth_resident = False
+ demoted.append(group)
+
+ repinned = []
+ monkeypatch.setattr(M, "_offload_groups", lambda m: groups)
+ monkeypatch.setattr(M, "_release_group", release)
+ monkeypatch.setattr(
+ M, "_keep_groups_resident", lambda m, room, dev, logger, only: repinned.append(set(only))
+ )
+ restore = M.release_resident_groups(pipe, 1 << 30)
+ assert restore is not None and len(demoted) == 1
+ restore()
+ assert repinned == [{id(groups[2]), id(groups[1])}]
+
+
+def test_unwrapping_an_outer_layer_collapses_a_removed_inner_layer():
+ """Item 4283219222: inner layer removed while the outer one wraps it, then the outer removed: the slot must go
+ back to what the inner one had replaced (here: nothing, the class method), not to the dead inner dispatch."""
+
+ class C:
+ def f(self):
+ return "class"
+
+ c = C()
+ inner = L.wrap(c, "f", "inner", lambda fn, layer: lambda: "inner:" + fn())
+ outer = L.wrap(c, "f", "outer", lambda fn, layer: lambda: "outer:" + fn())
+ assert c.f() == "outer:inner:class"
+ assert L.unwrap(inner) is False and c.f() == "outer:class"
+ assert L.unwrap(outer) is True
+ assert "f" not in c.__dict__ and c.f() == "class"
+ bound = types.MethodType(lambda self: "bound", c)
+ c.f = bound
+ inner = L.wrap(c, "f", "inner", lambda fn, layer: lambda: "inner:" + fn())
+ outer = L.wrap(c, "f", "outer", lambda fn, layer: lambda: "outer:" + fn())
+ L.unwrap(inner)
+ L.unwrap(outer)
+ assert c.__dict__["f"] is bound
+
+
+def test_component_whose_move_and_rollback_both_fail_gets_a_deferred_return(monkeypatch):
+ """Item 4283219189: the move to the host failed part way and moving it back failed too; the component was left
+ split across devices with nothing recorded. Now it is settled whole on the host with a deferred return."""
+ m = _on_card(64)
+ calls = []
+
+ def to_host(module):
+ calls.append("host")
+ if len(calls) == 1:
+ next(module.parameters()).data = next(module.parameters()).data.to(torch.float32)
+ raise _FakeOOM()
+ module.to(torch.float32)
+
+ def back(module, device):
+ raise _FakeOOM()
+
+ monkeypatch.setattr(O, "_to_host", to_host)
+ monkeypatch.setattr(O, "_return_to", back)
+ session = O.DecodeSession(None, _FakeDevice())
+ assert (
+ session.apply(lambda: O._to_host(m), session._component_change("transformer", m)) is False
+ )
+ assert session.log == [] and all(p.dtype == torch.float32 for p in m.parameters())
+ assert isinstance(m.__dict__.get(O._DEFERRED_ATTR), O.DeferredReturn)
+
+
+def test_trace_refuses_activation_fusion_for_hooked_or_inplace_reused_norms():
+ """Item 4283572932, class: exclusivity is proven on the trace. A hook on the norm, or an in-place reuse of its
+ output outside the SiLU, keeps the norm unpaired (GroupNorm-only fusion)."""
+
+ class Blk(torch.nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.norm = torch.nn.GroupNorm(4, 8)
+ self.act = torch.nn.SiLU()
+
+ def forward(self, x):
+ h = self.norm(x)
+ y = self.act(h)
+ h.mul_(2) # in-place reuse after the SiLU read it
+ return y + h
+
+ blk = Blk()
+ calls, consumed, _, info = F._trace_group_norm_silu(
+ [blk.norm], [blk.act], lambda: blk(torch.randn(1, 8, 4, 4))
+ )
+ assert calls == [1] and consumed == [0] and info["shared"][0] >= 1
+ clean = torch.nn.Sequential(torch.nn.GroupNorm(4, 8), torch.nn.SiLU())
+ calls, consumed, _, _ = F._trace_group_norm_silu(
+ [clean[0]], [clean[1]], lambda: clean(torch.randn(1, 8, 4, 4))
+ )
+ assert consumed == [1]
+ h = clean[0].register_forward_hook(lambda m, i, o: None)
+ calls, consumed, _, _ = F._trace_group_norm_silu(
+ [clean[0]], [clean[1]], lambda: clean(torch.randn(1, 8, 4, 4))
+ )
+ h.remove()
+ assert consumed == [0]
+
+
+def _fake_kernel(
+ monkeypatch,
+ wrong_site = None,
+ kernel_for = None,
+ during = None,
+):
+ """group_norm_act on CPU: the reference, flagged as a kernel run; ``wrong_site`` gets a local error."""
+
+ def fake(
+ x,
+ norm,
+ act = True,
+ in_bias = None,
+ on_kernel = None,
+ **_,
+ ):
+ y = F.group_norm_reference(x, norm, act, in_bias)
+ if norm is wrong_site:
+ y = y.clone()
+ y[..., 0, 0] += 0.5
+ if during is not None and kernel_for is not None and norm not in kernel_for:
+ during()
+ if kernel_for is None or norm in kernel_for:
+ if on_kernel is not None:
+ on_kernel()
+ legacy = F.__dict__.get("_GN_KERNEL_RUNS")
+ if legacy is not None:
+ legacy[0] += 1
+ return y
+
+ monkeypatch.setattr(F, "group_norm_act", fake)
+ monkeypatch.setattr(F, "runtime_ok", lambda: True)
+ monkeypatch.delenv(F.STRUCTURAL_ENV, raising = False)
+ monkeypatch.setattr(torch.Tensor, "is_cuda", property(lambda self: True))
+
+
+def test_structural_calibration_checks_every_fused_site(monkeypatch):
+ """Class ask: the calibration compares each fused site against stock on the input it saw, not only the final crop.
+ A site that is wrong in one corner is rejected even when the crop error stays small."""
+ vae = AutoencoderKLCleanNew()
+ norms = F._plain_group_norms(vae.decoder)
+ _fake_kernel(monkeypatch, wrong_site = norms[0])
+ F.install_structural(vae)
+ with torch.no_grad():
+ vae._decode(torch.randn(1, 8, 6, 6))
+ state = vae._unsloth_vae_structural
+ assert state.status == "rejected" and state.site_max_abs >= 0.4
+ ok = AutoencoderKLCleanNew()
+ _fake_kernel(monkeypatch)
+ F.install_structural(ok)
+ with torch.no_grad():
+ ok._decode(torch.randn(1, 8, 6, 6))
+ assert (
+ ok._unsloth_vae_structural.status == "ok" and ok._unsloth_vae_structural.site_max_abs == 0.0
+ )
+
+
+def test_structural_kernel_accounting_is_local_to_the_calibration(monkeypatch):
+ """Item 4283572986: another VAE's kernel run (another thread) during this VAE's calibration used to bump the
+ shared counter and make an all-fallback calibration look fused. Now only this calibration's own runs count."""
+ vae = AutoencoderKLCleanNew()
+ other_norm = torch.nn.GroupNorm(4, 8)
+ other = F._fast_gn_maybe_act(other_norm, True, types.SimpleNamespace(kernel_calls = 0, calls = 0))
+ import threading
+
+ def concurrent():
+ t = threading.Thread(target = lambda: other(torch.randn(1, 8, 4, 4)))
+ t.start()
+ t.join()
+
+ _fake_kernel(monkeypatch, kernel_for = {other_norm}, during = concurrent)
+ F.install_structural(vae)
+ with torch.no_grad():
+ vae._decode(torch.randn(1, 8, 6, 6))
+ assert vae._unsloth_vae_structural.status == "rejected"
+
+
+def test_strict_restore_reports_an_incomplete_repin(monkeypatch):
+ """Item 4283572942: the re-pin swallows its own failures; the VAE path restores strictly and gets an error while
+ any released group is still streamed, so its settle / defer path runs and the restore is kept."""
+ from core.inference import diffusion_memory as M
+
+ class Grp:
+ def __init__(self):
+ self._unsloth_resident = True
+ self._unsloth_resident_bytes = 1 << 20
+
+ groups = [Grp(), Grp()]
+ module = torch.nn.Linear(2, 2)
+ module._unsloth_resident_room = 1
+ module._unsloth_resident_device = "cpu"
+ pipe = types.SimpleNamespace(components = {"transformer": module})
+ monkeypatch.setattr(M, "_offload_groups", lambda m: groups)
+ monkeypatch.setattr(M, "_release_group", lambda g: setattr(g, "_unsloth_resident", False))
+ pinned = [0]
+
+ def keep(m, room, dev, logger, only):
+ groups[0]._unsloth_resident = True
+ pinned[0] += 1
+
+ monkeypatch.setattr(M, "_keep_groups_resident", keep)
+ restore = M.release_resident_groups(pipe, 1 << 30)
+ restore() # the lenient form other callers use: unchanged
+ with pytest.raises(RuntimeError, match = "left 1 streamed"):
+ restore(strict = True)
+ got = [restore]
+ session = O.DecodeSession(None, _FakeDevice())
+ released = []
+ monkeypatch.setattr(
+ M, "release_resident_groups", lambda *a, **k: (released.append(1), restore)[1]
+ )
+ anchor = module
+ pipe2 = types.SimpleNamespace(components = {"transformer": anchor})
+ assert session._release_groups(pipe2) == 1
+ session.restore()
+ assert session.log == [] and O._DEFERRED_ATTR in anchor.__dict__
+ del got
+
+
+def test_step_a_streams_a_resident_torchao_transformer(monkeypatch):
+ """Item 4283572958: a hookless resident torchao transformer was skipped by step (a)."""
+ from core.inference import diffusion_memory as M
+
+ vae, pipe = _setup(cap = 16384 + 1000, untiled_need = 10000, tiled_need = 500)
+ transformer = pipe.components["transformer"]
+ pipe.transformer = transformer
+ monkeypatch.setattr(O, "_holds_torchao", lambda m: m is transformer)
+ events = []
+
+ def hook(
+ p,
+ device,
+ logger = None,
+ ):
+ events.append("hook")
+ transformer._unsloth_resident_room = 1
+ return True
+
+ def release(
+ p,
+ need,
+ logger = None,
+ reason = "",
+ ):
+ events.append("release")
+ transformer.to(torch.float32)
+ return lambda strict = False: (events.append("repin"), transformer.to(_CARD))
+
+ monkeypatch.setattr(M, "hook_resident_denoiser", hook)
+ monkeypatch.setattr(M, "release_resident_groups", release)
+ vae._decode(_z())
+ assert events == ["hook", "release", "repin"]
+ assert vae._unsloth_vae_oom_stats["recovered_by"] == "freed"
+
+
+def test_vae_offload_respects_host_ram(monkeypatch):
+ """Item 4283572966: step (c) creates host copies of the VAE weights; with too little host RAM it is skipped."""
+ applied = []
+ monkeypatch.setattr(O, "_apply_vae_offload", lambda v, d: applied.append(1))
+ monkeypatch.setattr(O, "_host_free_mib", lambda: 1)
+ session = O.DecodeSession(_VaeLike(_Budget(1), [], 1, 1), _FakeDevice())
+ assert session.offload_vae() is False and applied == []
+ monkeypatch.setattr(O, "_host_free_mib", lambda: 1 << 20)
+ assert session.offload_vae() is True and applied == [1]
+
+
+def test_wide_tiles_free_the_card_before_falling_back_to_stock_tiles(monkeypatch):
+ """Item 4283572976: Studio's wide tiles caught their own OOM and fell back to the smaller stock tiles before the
+ OOM retry could offload anything. Now the retry frees the card first and the wide tiles decode."""
+ diffusers = pytest.importorskip("diffusers")
+ from core.inference import diffusion_vae_tiling as vt
+
+ monkeypatch.delenv(vt.WIDE_TILES_ENV, raising = False)
+ monkeypatch.setenv(vt.MAX_TILE_ENV, "1")
+ monkeypatch.setattr(vt, "floor_shortfall", lambda vae, z: None)
+ torch.manual_seed(0)
+ vae = diffusers.AutoencoderKLQwenImage(
+ base_dim = 8, z_dim = 4, dim_mult = [1, 2, 4, 4], num_res_blocks = 1, attn_scales = [],
+ temperal_downsample = [False, True, True], latents_mean = [0.0] * 4, latents_std = [1.0] * 4,
+ ).eval() # fmt: skip
+ vae.enable_tiling()
+ assert vt.install(vae)
+ z = torch.randn(1, 4, 1, 48, 48, generator = torch.Generator().manual_seed(1))
+ freed, fallbacks = [], []
+ monkeypatch.setattr(vt, "_log_stock_fallback", lambda vae, reason: fallbacks.append(reason))
+ monkeypatch.setattr(O, "_device_of", lambda z: _FakeDevice())
+ monkeypatch.setattr(
+ O.DecodeSession,
+ "free_others",
+ lambda self, pipe: (freed.append(1), setattr(self, "freed", True), 1)[2],
+ )
+ real = vae.decoder.forward
+
+ def decoder(x, *a, **k):
+ if not freed:
+ raise _FakeOOM()
+ return real(x, *a, **k)
+
+ monkeypatch.setattr(vae.decoder, "forward", decoder)
+ assert O.install(vae)
+ with torch.no_grad():
+ vae.decode(z)
+ assert freed == [1]
+ assert (
+ fallbacks == []
+ ) # the wide tiles decoded after the card was freed; no fallback to the stock tiles
+
+
+def test_ltx2_tiles_free_the_card_before_the_floor_tile(monkeypatch):
+ pytest.importorskip("diffusers") # tiled_decode builds a diffusers DecoderOutput
+ from core.inference import video_ltx2_vae_tiles as lt
+
+ tried = []
+ freed = []
+
+ def tiles(vae, z, temb, causal, th, tw, budgeted):
+ tried.append((th, tw))
+ if (th, tw) != (16, 16) and not freed:
+ raise _FakeOOM()
+ return torch.zeros(1)
+
+ monkeypatch.setattr(lt, "_decode_tiles", tiles)
+ monkeypatch.setattr(lt, "_plan", lambda vae, z: ((32, 32), True))
+ monkeypatch.setattr(lt, "oom_free_first", lambda vae, z: (freed.append(1), True)[1])
+ vae = types.SimpleNamespace()
+ lt.tiled_decode(vae, torch.zeros(1, 4, 2, 64, 64), return_dict = False)
+ assert tried == [(32, 32), (32, 32)] and freed == [1]
+
+
+def test_ladder_is_the_outermost_decode_layer_in_both_pipelines():
+ """Class ask (c): in the load order of diffusion.py and video.py the OOM retry is installed after every decode
+ wrapper that can fall back on an OOM, so its scope is the outermost decode layer; inner fallbacks (wide tiles,
+ LTX-2 tiles, untiled gate) call free_first before going cheaper."""
+ import inspect
+ import re
+
+ from core.inference import diffusion as D
+ from core.inference import video as V
+ from core.inference import diffusion_vae_tiling as vt
+ from core.inference import video_ltx2_vae_tiles as lt
+
+ image = inspect.getsource(D)
+ assert image.index("install_vae_oom_retry(") > image.index(
+ "install_wide_vae_tiles(getattr(pipe"
+ )
+ video = inspect.getsource(V)
+ oom = video.index("install_vae_oom_retry(")
+ for wrapper in (
+ "install_ltx2_vae_tiles(",
+ "install_untiled_decode(",
+ "install_rocm_vae_bf16_decode(pipe",
+ ):
+ assert video.index(wrapper) < oom, wrapper
+ assert re.search(r"oom_free_first\(self, z\)", inspect.getsource(vt))
+ assert "oom_free_first(vae, z)" in inspect.getsource(lt.tiled_decode)
+
+
+def test_scope_stays_outermost_over_the_untiled_gate(monkeypatch):
+ """The untiled gate re-asserts its slot after each call; with the OOM scope installed over it, it must keep the
+ scope (its __wrapped__ chain reaches the gate) instead of dropping it."""
+ from core.inference import video_vae_untiled as U
+
+ vae, pipe = _setup(cap = 1 << 20, untiled_need = 1, tiled_need = 1)
+ O.uninstall(vae)
+ vae.use_tiling = True
+ vae.decode = lambda z, return_dict = True: vae._decode(z, return_dict)
+ monkeypatch.setattr(U, "_enabled", lambda: True)
+ family = next(iter(U._BYTES_PER_LATENT_PIXEL))
+ monkeypatch.setattr(U, "_free_bytes", lambda d: 0)
+ assert U.install_untiled_decode(types.SimpleNamespace(vae = vae), family)
+ gate = vae.__dict__["decode"]
+ assert O.install(vae, pipe = pipe)
+ scope = vae.__dict__["decode"]
+ assert getattr(scope, "_unsloth_vae_oom_scope", False) and scope.__wrapped__ is gate
+ vae.decode(_z())
+ assert vae.__dict__["decode"] is scope # still outermost after a call
+
+
+def test_tiny_crop_pairs_like_the_full_crop():
+ """The pairing traced on the smallest calibration crop equals the one traced on a full-size latent."""
+ vae = AutoencoderKLSomethingNew()
+ norms, silus = F._plain_group_norms(vae.decoder), F._plain_silus(vae.decoder)
+ z = torch.randn(1, 8, 40, 40)
+ with torch.no_grad():
+ full = F._trace_group_norm_silu(norms, silus, lambda: vae._decode(z))
+ for side in (F._PROBE_LATENTS, 8, 2):
+ tiny = F._trace_group_norm_silu(norms, silus, lambda: vae._decode(z[..., :side, :side]))
+ assert full[0] == tiny[0] and full[1] == tiny[1], side
+ assert [s > 0 for s in full[3]["shared"]] == [s > 0 for s in tiny[3]["shared"]], side
+ assert F._probe_latent(z).shape[-1] == F._PROBE_LATENTS
+
+
+def _calibrated(monkeypatch, tmp_path, vae, **kw):
+ monkeypatch.setenv(F.STRUCTURAL_CACHE_ENV, "1")
+ monkeypatch.setenv("UNSLOTH_DIFFUSION_COMPILE_CACHE_DIR", str(tmp_path))
+ _fake_kernel(monkeypatch, **kw)
+ traces = []
+ real = F._trace_group_norm_silu
+ monkeypatch.setattr(
+ F, "_trace_group_norm_silu", lambda *a, **k: (traces.append(1), real(*a, **k))[1]
+ )
+ F.install_structural(vae)
+ with torch.no_grad():
+ vae._decode(torch.randn(1, 8, 6, 6))
+ return traces
+
+
+def test_structural_verdict_is_cached_and_reused(monkeypatch, tmp_path):
+ first = AutoencoderKLCleanNew()
+ assert _calibrated(monkeypatch, tmp_path, first) == [1]
+ assert first._unsloth_vae_structural.status == "ok" and not first._unsloth_vae_structural.cached
+ assert len(list((tmp_path / "vae_structural").glob("*.json"))) == 1
+ second = AutoencoderKLCleanNew()
+ assert _calibrated(monkeypatch, tmp_path, second) == [] # no trace, no checks
+ st = second._unsloth_vae_structural
+ assert st.status == "ok" and st.cached and st.gn_silu == first._unsloth_vae_structural.gn_silu
+ assert any(getattr(m.forward, "_unsloth_vae_fused", False) for m in F._plain_group_norms(second.decoder)
+ if "forward" in m.__dict__) or any("forward" in m.__dict__ for m in second.decoder.modules()) # fmt: skip
+
+
+def test_stale_cache_key_traces_again(monkeypatch, tmp_path):
+ """Another torch / triton / config is another key: the old verdict is not used and the VAE is traced again."""
+ assert _calibrated(monkeypatch, tmp_path, AutoencoderKLCleanNew()) == [1]
+ monkeypatch.setattr(torch, "__version__", "0.0.0-other")
+ assert _calibrated(monkeypatch, tmp_path, AutoencoderKLCleanNew()) == [1]
+ assert len(list((tmp_path / "vae_structural").glob("*.json"))) == 2
+
+
+def test_corrupt_or_mismatched_cache_entry_traces_again(monkeypatch, tmp_path):
+ assert _calibrated(monkeypatch, tmp_path, AutoencoderKLCleanNew()) == [1]
+ (entry,) = (tmp_path / "vae_structural").glob("*.json")
+ entry.write_text('{"status": "ok", "paired": [true]}') # wrong key / length
+ assert _calibrated(monkeypatch, tmp_path, AutoencoderKLCleanNew()) == [1]
+
+
+def test_cached_rejection_skips_fusion(monkeypatch, tmp_path):
+ bad = AutoencoderKLCleanNew()
+ norms = F._plain_group_norms(bad.decoder)
+ assert _calibrated(monkeypatch, tmp_path, bad, wrong_site = norms[0]) == [1]
+ assert bad._unsloth_vae_structural.status == "rejected"
+ again = AutoencoderKLCleanNew()
+ assert _calibrated(monkeypatch, tmp_path, again) == []
+ assert again._unsloth_vae_structural.status == "rejected" and not any(
+ "forward" in m.__dict__ for m in again.decoder.modules()
+ )
+
+
+def test_structural_cache_kill_switch(monkeypatch, tmp_path):
+ assert _calibrated(monkeypatch, tmp_path, AutoencoderKLCleanNew()) == [1]
+ monkeypatch.setenv(F.STRUCTURAL_CACHE_ENV, "0")
+ traces = []
+ real = F._trace_group_norm_silu
+ monkeypatch.setattr(
+ F, "_trace_group_norm_silu", lambda *a, **k: (traces.append(1), real(*a, **k))[1]
+ )
+ vae = AutoencoderKLCleanNew()
+ F.install_structural(vae)
+ with torch.no_grad():
+ vae._decode(torch.randn(1, 8, 6, 6))
+ assert traces == [1]
+
+
[email protected]("reuse", [False, True])
+def test_fused_activation_mark_is_consumed_by_the_matched_silu(reuse):
+ """Item 4285009704: the mark on a fused GroupNorm+SiLU output used to survive the SiLU that skipped, so a
+ following SiLU (or the same module applied again) skipped too. It is consumed by the matched SiLU now."""
+ norm = torch.nn.GroupNorm(4, 8)
+ first, second = torch.nn.SiLU(), torch.nn.SiLU()
+ if reuse:
+ second = first
+ state = types.SimpleNamespace(kernel_calls = 0, calls = 0)
+ fast = F._fast_gn_maybe_act(norm, True, state)
+ p1, p2 = F._silu_passthrough(first), F._silu_passthrough(second)
+ x = torch.randn(1, 8, 4, 4)
+ with torch.no_grad():
+ ref = torch.nn.functional.silu(torch.nn.functional.silu(norm(x)))
+ got = p2(p1(fast(x)))
+ assert torch.allclose(got, ref, atol = 1e-6)
+
+
+def test_fused_activation_mark_only_for_the_traced_silu():
+ """The mark names the SiLU module(s) the trace saw read the norm; another SiLU never consumes it (its read is a
+ foreign read: the decode is redone without activation fusion)."""
+ norm = torch.nn.GroupNorm(4, 8)
+ matched, other = torch.nn.SiLU(), torch.nn.SiLU()
+ state = types.SimpleNamespace(kernel_calls = 0, calls = 0)
+ fast = F._fast_gn_maybe_act(norm, True, state, [matched])
+ x = torch.randn(1, 8, 4, 4)
+ with torch.no_grad():
+ with pytest.raises(F.FusedActivationRead):
+ F._silu_passthrough(other)(fast(x))
+ y = fast(x)
+ assert torch.allclose(
+ F._silu_passthrough(matched)(y), torch.nn.functional.silu(norm(x)), atol = 1e-6
+ )
+
+
+class _DoubleSiluDec(torch.nn.Module):
+ def __init__(self, reuse):
+ super().__init__()
+ self.norm = torch.nn.GroupNorm(4, 8)
+ self.act = torch.nn.SiLU()
+ self.act2 = self.act if reuse else torch.nn.SiLU()
+ self.conv = torch.nn.Conv2d(8, 8, 1)
+
+ def forward(self, x):
+ return self.conv(self.act2(self.act(self.norm(x))))
+
+
+class AutoencoderKLDoubleSilu(torch.nn.Module):
+ def __init__(self, reuse):
+ super().__init__()
+ self.decoder = _DoubleSiluDec(reuse)
+
+ def _decode(
+ self,
+ z,
+ return_dict = True,
+ ):
+ return (self.decoder(z),)
+
+
[email protected]("reuse", [False, True])
+def test_structural_double_silu_matches_stock(monkeypatch, reuse):
+ vae = AutoencoderKLDoubleSilu(reuse)
+ z = torch.randn(1, 8, 6, 6)
+ with torch.no_grad():
+ ref = vae._decode(z)[0]
+ _fake_kernel(monkeypatch)
+ assert F.install_structural(vae) == 1
+ with torch.no_grad():
+ vae._decode(z)
+ out = vae._decode(z)[0]
+ assert vae._unsloth_vae_structural.status == "ok" and vae._unsloth_vae_structural.gn_silu == 1
+ assert torch.allclose(out, ref, atol = 1e-6)
+
+
+def test_every_decode_rewrap_site_restores_the_oom_scope():
+ """Item 4285009695, class: every function in the image / video backends that (directly or through any helper)
+ reassigns a VAE's ``decode`` must re-assert the OOM scope afterwards (install it, or ensure_outermost). The
+ rewrap helpers are found by AST: any function assigning ``.decode`` or calling such a function, to a fixpoint."""
+ import ast
+ import pathlib
+
+ root = pathlib.Path(O.__file__).parent
+ skip = {"diffusion_vae_oom.py", "decode_layers.py", "media_decode_phase.py"}
+ ensure = {"install_vae_oom_retry", "ensure_outermost"}
+ funcs = {}
+ for f in root.glob("*.py"):
+ if f.name in skip:
+ continue
+ tree = ast.parse(f.read_text())
+ for node in tree.body:
+ if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)):
+ funcs[(f.name, node.name)] = node
+ if isinstance(node, ast.ClassDef):
+ for b in node.body:
+ if isinstance(b, (ast.FunctionDef, ast.AsyncFunctionDef)):
+ funcs[(f.name, f"{node.name}.{b.name}")] = b
+
+ def called(c):
+ fn = c.func
+ return (
+ fn.id
+ if isinstance(fn, ast.Name)
+ else fn.attr
+ if isinstance(fn, ast.Attribute)
+ else None
+ )
+
+ dirty = set()
+ for k, node in funcs.items():
+ for n in ast.walk(node):
+ if isinstance(n, ast.Assign) and any(
+ isinstance(t, ast.Attribute) and t.attr == "decode" for t in n.targets
+ ):
+ dirty.add(k)
+ assert ("diffusion_speed.py", "_compile_vae_decode") in dirty or any(
+ k[0] == "diffusion_speed.py" for k in dirty
+ )
+ changed = True
+ while changed:
+ changed = False
+ names = {k[1].split(".")[-1] for k in dirty}
+ for k, node in funcs.items():
+ if k in dirty:
+ continue
+ last = ens = 0
+ for n in ast.walk(node):
+ if isinstance(n, ast.Call):
+ c = called(n)
+ if c in names and c != k[1].split(".")[-1]:
+ last = max(last, n.lineno)
+ if c in ensure:
+ ens = max(ens, n.lineno)
+ if last and ens <= last:
+ dirty.add(k)
+ changed = True
+ leaks = sorted(k for k in dirty if k[0] in ("diffusion.py", "video.py"))
+ assert leaks == [], leaks
+
+
+class _PublicVae(_VaeLike):
+ def decode(
+ self,
+ z,
+ return_dict = True,
+ ):
+ return self._decode(z, return_dict)
+
+
+def _setup_public(cap, untiled_need, tiled_need):
+ budget = _Budget(cap)
+ transformer, text_encoder = _on_card(8192), _on_card(4096)
+ vae = _PublicVae(budget, [transformer, text_encoder], untiled_need, tiled_need)
+ vae.to(_CARD)
+ pipe = _Pipe(transformer = transformer, text_encoder = text_encoder, vae = vae)
+ assert O.install(vae, pipe = pipe)
+ assert getattr(vae.__dict__.get("decode"), "_unsloth_vae_oom_scope", False)
+ return vae, pipe
+
+
+def test_ensure_outermost_puts_the_scope_over_a_later_compiled_guard():
+ """Item 4285009695: a deferred speed profile compiles vae.decode after load; its guard wraps the scope (which the
+ graph traces as a pass-through) and re-raises OOMs. ensure_outermost puts a fresh scope back on top, which runs
+ the ladder on a compiled OOM."""
+ vae, pipe = _setup_public(cap = 16384 + 1000, untiled_need = 10000, tiled_need = 500)
+ scoped = vae.decode
+ calls = []
+
+ def compiled(z, return_dict = True):
+ calls.append("compiled")
+ if vae.budget.resident([*vae.pipe_modules, vae]) + vae.untiled_need > vae.budget.cap:
+ raise _FakeOOM()
+ return ("ok",)
+
+ def guard(*a, **k): # the speed layer's guard: re-raises OOMs
+ return compiled(*a, **k)
+
+ guard._unsloth_compiled_decode = True
+ guard._unsloth_eager_decode = scoped
+ vae.decode = guard
+ with pytest.raises(_FakeOOM):
+ vae.decode(_z()) # before: the compiled OOM escapes the ladder
+ assert O.ensure_outermost(vae)
+ top = vae.__dict__["decode"]
+ assert getattr(top, "_unsloth_vae_oom_scope", False) and top.__wrapped__ is guard
+ calls.clear()
+ assert vae.decode(_z()) == ("ok",)
+ assert calls == ["compiled", "compiled"]
+ assert O.ensure_outermost(vae) and vae.__dict__["decode"] is top
+
+
+def test_scope_heals_itself_after_a_rewrap_drops_it():
+ """A compile fallback restoring its eager decode (or any rewrap) can drop the scope at run time; the next
+ _decode reached without it reinstalls it on top."""
+ vae, pipe = _setup_public(cap = 1 << 20, untiled_need = 1, tiled_need = 1)
+ del vae.decode
+ vae.decode(_z())
+ assert getattr(vae.__dict__.get("decode"), "_unsloth_vae_oom_scope", False)
+
+
+def test_structural_traces_on_two_threads_keep_their_own_reader():
+ """Item 4285966457: the reader callback was one class attribute, so a second thread's trace finishing while the
+ first was mid-run cleared it, and the first trace missed a residual read of its norm output (a false exclusive
+ pairing). The callback is per thread and traces run one at a time."""
+ import threading
+
+ shared_norm, shared_act = torch.nn.GroupNorm(4, 8), torch.nn.SiLU()
+ other_norm, other_act = torch.nn.GroupNorm(4, 8), torch.nn.SiLU()
+ paused, resume = threading.Event(), threading.Event()
+
+ def run_shared():
+ x = torch.randn(1, 8, 4, 4)
+ h = shared_norm(x)
+ paused.set()
+ resume.wait(
+ timeout = 2
+ ) # the other thread's trace runs to completion here (before: unlocked)
+ return shared_act(h) + h
+
+ result = {}
+
+ def a():
+ result["a"] = F._trace_group_norm_silu([shared_norm], [shared_act], run_shared)
+
+ def b():
+ paused.wait(timeout = 2)
+ result["b"] = F._trace_group_norm_silu(
+ [other_norm], [other_act], lambda: other_act(other_norm(torch.randn(1, 8, 4, 4)))
+ )
+ resume.set()
+
+ ta, tb = threading.Thread(target = a), threading.Thread(target = b)
+ ta.start()
+ tb.start()
+ ta.join(10)
+ tb.join(10)
+ calls_a, consumed_a, _, info_a = result["a"]
+ assert calls_a == [1] and consumed_a == [0] and info_a["shared"] == [1]
+ calls_b, consumed_b, _, _ = result["b"]
+ assert calls_b == [1] and consumed_b == [1]
+
+
+class _WideResidualDec(torch.nn.Module):
+ """Reads its norm output only through the SiLU on small inputs (the calibration crop); above a width threshold it
+ also adds the raw norm output back (a residual that only appears at full shape)."""
+
+ def __init__(self, threshold = 16):
+ super().__init__()
+ self.threshold = threshold
+ self.norm = torch.nn.GroupNorm(4, 8)
+ self.act = torch.nn.SiLU()
+ self.norm2 = torch.nn.GroupNorm(4, 8)
+ self.act2 = torch.nn.SiLU()
+ self.conv = torch.nn.Conv2d(8, 8, 3, padding = 1)
+
+ def forward(self, x):
+ h = self.norm(x)
+ out = self.conv(self.act(h))
+ if x.shape[-1] > self.threshold:
+ out = out + h # the raw norm output, at full shape only
+ g = self.norm2(h)
+ else:
+ g = self.norm2(out)
+ return self.act2(g) + out
+
+
+class AutoencoderKLWideResidual(torch.nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.decoder = _WideResidualDec()
+
+ def _decode(
+ self,
+ z,
+ return_dict = True,
+ ):
+ return (self.decoder(z),)
+
+
+def test_fused_activation_is_exact_when_full_shape_dataflow_differs_from_the_crop(monkeypatch):
+ """Item 4285966466: the crop trace proves the SiLU is the norm's only reader, but at full width the decoder also
+ adds the raw norm output back and feeds it to another norm. The fused output gives only the paired SiLU the
+ activated values; every other reader gets the stock norm output, so the decode matches stock at any shape."""
+ vae = AutoencoderKLWideResidual()
+ z_small, z_wide = torch.randn(1, 8, 12, 12), torch.randn(1, 8, 12, 40)
+ with torch.no_grad():
+ ref_wide = vae._decode(z_wide)[0]
+ ref_small = vae._decode(z_small)[0]
+ _fake_kernel(monkeypatch)
+ F.install_structural(vae)
+ with torch.no_grad():
+ first = vae._decode(z_wide)[0]
+ state = vae._unsloth_vae_structural
+ assert state.status == "ok" and state.act_off and state.gn_silu == 0 and state.gn == 2
+ assert state.foreign_reads >= 1
+ out_wide = vae._decode(z_wide)[0]
+ out_small = vae._decode(z_small)[0]
+ assert torch.allclose(first, ref_wide, atol = 1e-6)
+ assert torch.allclose(out_wide, ref_wide, atol = 1e-6)
+ assert torch.allclose(out_small, ref_small, atol = 1e-6)
+ small_first = (
+ AutoencoderKLWideResidual()
+ ) # a VAE only ever decoded at the crop's dataflow keeps both fused
+ small_first.load_state_dict(vae.state_dict())
+ F.install_structural(small_first)
+ with torch.no_grad():
+ small_first._decode(z_small)
+ assert small_first._unsloth_vae_structural.gn_silu == 2
+
+
+def test_fused_output_read_by_another_norm_aborts():
+ norm, norm2 = torch.nn.GroupNorm(4, 8), torch.nn.GroupNorm(4, 8)
+ state = types.SimpleNamespace(kernel_calls = 0, calls = 0)
+ fast, fast2 = F._fast_gn_maybe_act(norm, True, state), F._fast_gn_maybe_act(norm2, False, state)
+ x = torch.randn(1, 8, 4, 4)
+ with torch.no_grad():
+ with pytest.raises(F.FusedActivationRead):
+ fast2(fast(x))
+ y = fast(x)
+ with pytest.raises(F.FusedActivationRead):
+ torch.cat([y, y])
+ assert y.shape == norm(x).shape and y.dtype == x.dtype
+
+
+def test_inplace_paired_silu_leaves_the_activated_values_for_later_readers():
+ norm, act = torch.nn.GroupNorm(4, 8), torch.nn.SiLU(inplace = True)
+ fast = F._fast_gn_maybe_act(norm, True, types.SimpleNamespace(kernel_calls = 0, calls = 0), [act])
+ x = torch.randn(1, 8, 4, 4)
+ with torch.no_grad():
+ n = norm(x)
+ a = torch.nn.functional.silu(n, inplace = True)
+ ref = a + n
+ y = fast(x)
+ got = F._silu_passthrough(act)(y) + y
+ assert torch.allclose(got, ref, atol = 1e-6)
+
+
+def test_structural_cache_key_follows_the_checkpoint_dtype_and_hooks():
+ """Item 4285966472: the key ignored the weights, so one checkpoint's verdict authorised another of the same
+ class / config. It now hashes every decoder tensor's name / shape / dtype and leading values, plus hook state."""
+ torch.manual_seed(0)
+ a = AutoencoderKLCleanNew()
+ b = AutoencoderKLCleanNew()
+ with torch.no_grad():
+ b.decoder.norm_out.weight.copy_(a.decoder.norm_out.weight)
+ b.decoder.norm_out.bias.copy_(a.decoder.norm_out.bias)
+
+ def key(v):
+ return F.structural_cache_key(
+ v, F._plain_group_norms(v.decoder), F._plain_silus(v.decoder), None
+ )
+
+ assert key(a) is not None and key(a) == key(b)
+ with torch.no_grad():
+ b.decoder.norm_out.weight.add_(0.01)
+ assert key(a) != key(b)
+ c = AutoencoderKLCleanNew()
+ c.load_state_dict(a.state_dict())
+ k_a = key(a)
+ c.to(torch.float64)
+ assert key(c) != k_a
+ c = AutoencoderKLCleanNew()
+ c.load_state_dict(a.state_dict())
+ assert key(c) == k_a
+ h = c.decoder.norm_out.register_forward_hook(lambda *args: None)
+ assert key(c) != k_a # hooks change what the trace may prove
+ h.remove()
+ meta = AutoencoderKLCleanNew().to("meta")
+ assert key(meta) is None
+
+
+def test_strict_graph_reset_reports_a_handle_that_kept_its_graph():
+ from core.inference import diffusion_cuda_graph as G
+
+ class Handle:
+ def __init__(
+ self,
+ fail = False,
+ keep = False,
+ ):
+ self.cache, self.fail, self.keep = {"shape": "graph"}, fail, keep
+
+ def reset(self):
+ if self.fail:
+ raise RuntimeError("reset failed")
+ if not self.keep:
+ self.cache.clear()
+ return self
+
+ G.reset_all([Handle(fail = True)])
+ G.reset_all([Handle(), Handle()], strict = True)
+ with pytest.raises(RuntimeError, match = "captured CUDA graph"):
+ G.reset_all([Handle(), Handle(fail = True)], strict = True)
+ with pytest.raises(RuntimeError, match = "captured CUDA graph"):
+ G.reset_all([Handle(keep = True)], strict = True)
+
+
+def test_step_a_moves_no_weight_when_a_graph_could_not_be_dropped(monkeypatch):
+ """Item 4285966487: reset_all swallowed a handle's failure and step (a) then moved weights under a graph that
+ could still replay old addresses. A strict reset failure now stops step (a) before any move; the ladder goes on
+ (stock tiles here)."""
+ from core.inference import diffusion_cuda_graph as G
+
+ class Stuck:
+ cache = {"k": "graph"}
+
+ def reset(self):
+ raise RuntimeError("cannot drop")
+
+ moves = []
+ monkeypatch.setattr(O, "_to_host", lambda m: (moves.append(m), m.to(torch.float32)))
+ vae, pipe = _setup(cap = 16384 + 1000 + 4096, untiled_need = 10000, tiled_need = 500)
+ pipe._unsloth_cuda_graphs = (Stuck(),)
+ out = vae.decode(_z())[0] if "decode" in vae.__dict__ else vae._decode(_z())[0]
+ assert moves == []
+ assert _devices(pipe)["transformer"] == _CARD and _devices(pipe)["text_encoder"] == _CARD
+ assert vae._unsloth_vae_oom_stats.get("recovered_by") == "tiled"
+ assert out is not None
+
+
+def test_scope_finds_a_compiled_guard_under_the_rocm_bf16_decode(monkeypatch):
+ """Items 4285966482 / 4286353651: ROCm order is compile, then the bf16 decode wraps the compiled guard, then the
+ OOM retry. The scope is the outermost layer (above the bf16 cast, whose own fp32 output cast may run out of
+ memory too) and reaches the compiled guard under it: a compiled OOM frees the card and retries the whole decode,
+ compiled."""
+ from core.inference import diffusion_device as D
+ from core.inference import diffusion_speed as S
+
+ budget = _Budget(16384 + 1000)
+ transformer, text_encoder = _on_card(8192), _on_card(4096)
+ vae = _PublicVae(budget, [transformer, text_encoder], 10000, 500)
+ vae.to(_CARD)
+ vae.dtype = torch.float32
+ pipe = _Pipe(transformer = transformer, text_encoder = text_encoder, vae = vae)
+ calls = []
+
+ def compiled(z, return_dict = True):
+ calls.append("compiled")
+ if budget.resident([transformer, text_encoder, vae]) + 10000 > budget.cap:
+ raise _FakeOOM()
+ return (z,)
+
+ vae.decode = S._guard_compiled_decode(vae, compiled, vae.decode, None)
+ guard = vae.__dict__["decode"]
+ monkeypatch.setattr(D, "_rocm_bf16_decode_arch", lambda torch_, target: "gfx1151")
+ monkeypatch.setenv(D.VAE_BF16_DECODE_ENV, "autocast")
+ target = types.SimpleNamespace(device = "cuda", backend = "rocm")
+ assert D.install_rocm_vae_bf16_decode(pipe, target) == "autocast"
+ bf16 = vae.__dict__["decode"]
+ assert O.install(vae, pipe = pipe)
+ top = vae.__dict__["decode"]
+ assert getattr(top, "_unsloth_vae_oom_scope", False) and top.__wrapped__ is bf16
+ assert bf16._unsloth_decode_slot.decode is guard
+ assert O.ensure_outermost(vae) and vae.__dict__["decode"] is top
+ import warnings
+
+ with warnings.catch_warnings():
+ warnings.simplefilter("ignore")
+ out = vae.decode(_z())
+ assert calls == ["compiled", "compiled"] and out[0].dtype == torch.float32
+
+
+class _CompileFailed(RuntimeError):
+ pass
+
+
+def _fp16_shaped(
+ vae,
+ budget,
+ own_need,
+ oom_once = None,
+):
+ """An fp16-decode-shaped dtype wrapper: calls through a mutable slot, then allocates for its own non-finite check
+ (``own_need`` bytes on top of what is resident), which can run out of memory itself."""
+ slot = types.SimpleNamespace(decode = vae.decode)
+
+ def fp16_decode(z, *a, **k):
+ out = slot.decode(z, *a, **k)
+ if budget.resident([*vae.pipe_modules, vae]) + own_need > budget.cap:
+ raise _FakeOOM()
+ return out
+
+ fp16_decode._unsloth_decode_slot = slot
+ vae.decode = fp16_decode
+ return fp16_decode, slot
+
+
+def test_dtype_wrapper_own_allocation_oom_reaches_the_scope():
+ """Item 4286353651: with the scope inside the dtype wrapper's slot, an OOM in the wrapper's own work (the fp16
+ isfinite check, the ROCm fp32 output cast) escaped it. The scope is outermost now: it frees the card and the
+ whole decode, wrapper included, succeeds."""
+ budget = _Budget(16384 + 1000)
+ transformer, text_encoder = _on_card(8192), _on_card(4096)
+ vae = _PublicVae(budget, [transformer, text_encoder], 1, 1)
+ vae.to(_CARD)
+ pipe = _Pipe(transformer = transformer, text_encoder = text_encoder, vae = vae)
+ wrapper, slot = _fp16_shaped(vae, budget, own_need = 6000)
+ assert O.install(vae, pipe = pipe)
+ top = vae.__dict__["decode"]
+ assert getattr(top, "_unsloth_vae_oom_scope", False) and top.__wrapped__ is wrapper
+ before = _devices(pipe)
+ out = vae.decode(_z())
+ assert out[0].shape[-1] == 64
+ assert vae._unsloth_vae_oom_stats["recovered_by"] == "freed" and _devices(pipe) == before
+
+
[email protected]("slot_kind", ["vae", "fp16"])
+def test_compile_fallback_keeps_the_scope(monkeypatch, slot_kind):
+ """Item 4286353643: a lazy compile that fails replaced its owner slot with the eager decode. When the scope sat in
+ that slot (fp16 / bf16 orders) or on top of the guard, the fallback dropped it and later sliced decodes ran
+ without one shared session. The fallback now leaves a slot it no longer holds alone, and the scope re-asserts
+ itself when a decode closes."""
+ from core.inference import diffusion_speed as S
+
+ monkeypatch.setattr(S, "is_compile_failure", lambda exc: isinstance(exc, _CompileFailed))
+ budget = _Budget(16384 + 1000)
+ transformer, text_encoder = _on_card(8192), _on_card(4096)
+ vae = _SlicedVae(budget, [transformer, text_encoder], 10000, 500)
+ vae.to(_CARD)
+ pipe = _Pipe(transformer = transformer, text_encoder = text_encoder, vae = vae)
+
+ def compiled(z, *a, **k):
+ raise _CompileFailed("inductor lowering failed")
+
+ if slot_kind == "fp16":
+ _, slot = _fp16_shaped(vae, budget, own_need = 0)
+ slot.decode = S._guard_compiled_decode(vae, compiled, slot.decode, None, owner = slot)
+ else:
+ vae.decode = S._guard_compiled_decode(vae, compiled, vae.decode, None)
+ assert O.install(vae, pipe = pipe)
+ moves = []
+ monkeypatch.setattr(O, "_to_host", lambda m: (moves.append("off"), m.to(torch.float32)))
+ monkeypatch.setattr(O, "_return_to", lambda m, d: (moves.append("on"), m.to(_CARD)))
+ out = vae.decode(torch.ones(3, 4, 1, 8, 8))[
+ 0
+ ] # compile fails, eager decode, slice OOMs recover
+ assert out.shape[0] == 3
+ top = vae.__dict__["decode"]
+ assert getattr(top, "_unsloth_vae_oom_scope", False) # still outermost after the fallback
+ moves.clear()
+ out = vae.decode(torch.ones(3, 4, 1, 8, 8))[0]
+ assert out.shape[0] == 3 and moves == ["off", "off", "on", "on"]
+
+
+def test_compiled_guard_behind_the_untiled_gate_reaches_the_ladder(monkeypatch):
+ """The video untiled gate calls what it wraps through a closure (no slot), so a compiled
+ guard under it was invisible and its tiled-attempt OOM escaped. Under the single placement rule the scope is
+ outermost; it frees the card, retries the whole decode compiled, then with every guard eager."""
+ from core.inference import diffusion_speed as S
+ from core.inference import video_vae_untiled as U
+
+ budget = _Budget(16384 + 1000)
+ transformer, text_encoder = _on_card(8192), _on_card(4096)
+ vae = _PublicVae(budget, [transformer, text_encoder], 10**9, 10**9)
+ vae.to(_CARD)
+ vae.use_tiling = True
+ pipe = _Pipe(transformer = transformer, text_encoder = text_encoder, vae = vae)
+ calls = []
+
+ def compiled(z, *a, **k):
+ calls.append(("compiled", vae.use_tiling))
+ raise _FakeOOM()
+
+ vae.decode = S._guard_compiled_decode(vae, compiled, vae.decode, None)
+ vae._unsloth_compiled_decode = True
+ monkeypatch.setattr(
+ U, "_free_bytes", lambda device: 0
+ ) # untiled never fits: the gate decodes tiled
+ family = next(iter(U._BYTES_PER_LATENT_PIXEL))
+ assert U.install_untiled_decode(pipe, family)
+ assert O.install(vae, pipe = pipe)
+ vae.tiled_need = 500
+ out = vae.decode(_z())
+ assert calls == [("compiled", True), ("compiled", True)]
+ assert out[0].shape[-1] == 64 and vae.seen[-1][0] is True
+ assert getattr(vae.__dict__["decode"], "_unsloth_vae_oom_scope", False)
+
+
+def _raw_read(t):
+ """What a Triton kernel sees: the values at ``t.data_ptr()`` indexed with ``t``'s own strides (CPU, float32)."""
+ import ctypes
+
+ n = t.storage_offset() + sum((s - 1) * st for s, st in zip(t.shape, t.stride())) + 1
+ buf = (ctypes.c_float * n).from_address(t.data_ptr())
+ flat = torch.frombuffer(buf, dtype = torch.float32, count = n).clone()
+ return torch.as_strided(flat, t.shape, t.stride(), t.storage_offset())
+
+
+def test_patched_norms_never_hand_a_fused_input_to_the_kernel(monkeypatch):
+ """Item 4286353624: a GroupNorm-only site (act False) handed a fused GroupNorm+SiLU output to its kernel as is,
+ and a kernel reads raw memory (activated values). Every patched entry checks its input first, and a raw pointer
+ read of a fused output from any other module's kernel is caught too: the decode is redone without activation
+ fusion instead (the structural tests check the redone output against stock)."""
+ seen = []
+
+ def kernel(
+ x,
+ norm,
+ act = True,
+ in_bias = None,
+ on_kernel = None,
+ **_,
+ ):
+ raw = _raw_read(x)
+ seen.append(raw)
+ y = torch.nn.functional.group_norm(raw, norm.num_groups, norm.weight, norm.bias, norm.eps)
+ return torch.nn.functional.silu(y) if act else y
+
+ monkeypatch.setattr(F, "group_norm_act", kernel)
+ norm, norm2, norm3 = (
+ torch.nn.GroupNorm(4, 8),
+ torch.nn.GroupNorm(4, 8),
+ torch.nn.GroupNorm(4, 8),
+ )
+ state = types.SimpleNamespace(kernel_calls = 0, calls = 0)
+ fused = F._fast_gn_maybe_act(norm, True, state)
+ gn_only = F._fast_gn_maybe_act(norm2, False, state)
+ fused2 = F._fast_gn_maybe_act(norm3, True, state)
+ x = torch.randn(1, 8, 4, 4)
+ with torch.no_grad():
+ for reader in (gn_only, fused2, _raw_read):
+ seen.clear()
+ y = fused(x)
+ assert len(seen) == 1 and torch.equal(seen[0], x)
+ with pytest.raises(F.FusedActivationRead):
+ reader(y)
+ assert len(seen) == 1 # the kernel never saw the activated values
+
+
+def test_after_an_inplace_paired_silu_the_tensor_reads_as_stock():
+ """stock silu_ overwrites the norm output: after the paired in-place SiLU the fused tensor's storage is exactly
+ that, so later reads (raw pointers included) are plain reads, not aborts."""
+ norm, act = torch.nn.GroupNorm(4, 8), torch.nn.SiLU(inplace = True)
+ fast = F._fast_gn_maybe_act(norm, True, types.SimpleNamespace(kernel_calls = 0, calls = 0), [act])
+ x = torch.randn(1, 8, 4, 6)
+ with torch.no_grad():
+ y = fast(x)
+ F._silu_passthrough(act)(y)
+ assert torch.allclose(
+ _raw_read(y.contiguous()), torch.nn.functional.silu(norm(x)), atol = 1e-6
+ )
+
+
+class _CausalLtx:
+ """An LTX-2-shaped VAE whose decoder keeps causal state across calls (reset by clear_cache); OOMs on chosen
+ decoder calls."""
+
+ spatial_compression_ratio = 32
+
+ def __init__(self, oom_calls):
+ self.state = []
+ self.calls = 0
+ self.oom_calls = set(oom_calls)
+ self.use_tiling = True
+
+ def clear_cache(self):
+ self.state = []
+
+ def decoder(
+ self,
+ z,
+ temb = None,
+ causal = None,
+ ):
+ self.calls += 1
+ if self.calls in self.oom_calls:
+ raise _FakeOOM()
+ out = z.repeat_interleave(32, -1).repeat_interleave(32, -2) + 0.001 * len(self.state)
+ self.state.append(1)
+ return out
+
+
[email protected]("k", [1, 2, 3])
+def test_ltx2_floor_decode_starts_clean_after_a_freed_retry_fails_part_way(monkeypatch, k):
+ """Item 4286353660: the freed retry of the wide LTX-2 tiles ran k tiles, then ran out of memory; the floor-tile
+ decode started on that partial causal state. Every retry after a partial causal decode clears it first."""
+ pytest.importorskip("diffusers") # tiled_decode builds a diffusers DecoderOutput
+ from core.inference import video_ltx2_vae_tiles as T
+
+ z = torch.randn(1, 4, 1, 48, 48)
+ clean = _CausalLtx(())
+ ref = T._decode_tiles(clean, z, None, None, 16, 16, False)
+ # the wide attempt (32-latent tiles: 4 of them) OOMs at once; its freed retry decodes k tiles, then OOMs
+ vae = _CausalLtx({1, 1 + k + 1})
+ retry_calls = []
+
+ def free_first(v, zz):
+ v.clear_cache() # what the real step (a) does before the freed retry
+ retry_calls.append(v.calls)
+ return True
+
+ monkeypatch.setattr(T, "oom_free_first", free_first)
+ monkeypatch.setattr(T, "_plan", lambda vae, z: ((32, 32), True))
+ out = T.tiled_decode(vae, z, return_dict = False)[0]
+ assert retry_calls == [1] and vae.calls > 1 + k + 1
+ assert torch.allclose(out, ref)
+
+
+def test_untiled_gate_tiled_retry_starts_clean(monkeypatch):
+ """Same class as 4286353660 in the untiled-when-fits gate: an untiled attempt that ran out of memory part way
+ left causal state for the tiled decode."""
+ from core.inference import video_vae_untiled as U
+
+ class Causal(torch.nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.use_tiling = True
+ self.state = []
+ self.decoder = torch.nn.Linear(2, 2)
+
+ def clear_cache(self):
+ self.state = []
+
+ def decode(
+ self,
+ z,
+ return_dict = True,
+ ):
+ out = z + 0.001 * len(self.state)
+ self.state.append(1)
+ if not self.use_tiling:
+ raise _FakeOOM() # the untiled attempt fails after a causal step
+ return (out,)
+
+ vae = Causal()
+ monkeypatch.setattr(U, "_free_bytes", lambda device: 1 << 50)
+ family = next(iter(U._BYTES_PER_LATENT_PIXEL))
+ assert U.install_untiled_decode(types.SimpleNamespace(vae = vae), family)
+ z = torch.zeros(1, 4, 1, 8, 8)
+ out = vae.decode(z)[0]
+ assert torch.equal(out, z)
+
+
+def test_regional_compiled_blocks_take_the_eager_retry():
+ """Item 4286789089: a regionally compiled VAE (compile_repeated_blocks + guard_compiled_blocks) promised the
+ scope an eager retry, but the block guard ignored the VAE-level flag, so the retry ran the same compiled blocks.
+ Every compiled guard now reads one signal (diffusion_bg_compile.force_eager)."""
+ from core.inference import diffusion_speed as S
+
+ budget = _Budget(1 << 40)
+ transformer, text_encoder = _on_card(8192), _on_card(4096)
+
+ class Block(torch.nn.Module):
+ def forward(self, z):
+ calls.append("eager")
+ return z
+
+ class Vae(_VaeLike):
+ def __init__(self):
+ super().__init__(budget, [transformer, text_encoder], 1, 1)
+ self.block = Block()
+
+ def _decode(
+ self,
+ z,
+ return_dict = True,
+ ):
+ return (self.block(z).repeat_interleave(8, -1).repeat_interleave(8, -2),)
+
+ def decode(
+ self,
+ z,
+ return_dict = True,
+ ):
+ return self._decode(z, return_dict)
+
+ calls = []
+ vae = Vae().to(_CARD)
+ pipe = _Pipe(transformer = transformer, text_encoder = text_encoder, vae = vae)
+
+ def compiled_block(*a, **k):
+ calls.append("compiled")
+ raise _FakeOOM()
+
+ vae.block._compiled_call_impl = compiled_block
+ assert S.guard_compiled_blocks(vae) >= 1
+ vae._unsloth_compiled_decode = True
+ assert O.install(vae, pipe = pipe)
+ out = vae.decode(_z())
+ assert out[0].shape[-1] == 64
+ assert calls[-1] == "eager" and "compiled" in calls
+
+
+def _exact_kernel(monkeypatch):
+ """group_norm_act on CPU with exactly stock's ops (F.group_norm, F.silu), flagged as a kernel run: a fused decode
+ then matches stock bit for bit wherever it computes the same values."""
+
+ def fake(
+ x,
+ norm,
+ act = True,
+ in_bias = None,
+ on_kernel = None,
+ **_,
+ ):
+ y = torch.nn.functional.group_norm(x if in_bias is None else x + in_bias, norm.num_groups, norm.weight,
+ norm.bias, norm.eps) # fmt: skip
+ if on_kernel is not None:
+ on_kernel()
+ return torch.nn.functional.silu(y) if act else y
+
+ monkeypatch.setattr(F, "group_norm_act", fake)
+ monkeypatch.setattr(F, "runtime_ok", lambda: True)
+ monkeypatch.delenv(F.STRUCTURAL_ENV, raising = False)
+ monkeypatch.setattr(torch.Tensor, "is_cuda", property(lambda self: True))
+
+
+def _mutate(t, op, b):
+ if op == "add_":
+ t.add_(b)
+ elif op == "mul_":
+ t.mul_(2.0)
+ elif op == "setitem":
+ t[:, 0] = 3.0
+ elif op == "copy_":
+ t.copy_(t * 0.5 + b)
+ elif op == "out":
+ torch.add(t, b, out = t)
+ elif op == "view_add_":
+ t[:, 1:3].add_(1.0)
+ elif op == "iadd":
+ t += b
+ elif op == "input_add_":
+ pass
+ return t
+
+
+class _MutatingDec(torch.nn.Module):
+ """At full width (beyond the calibration crop) it writes to the norm output, or to the norm's INPUT, before
+ the paired SiLU / a residual read; on the crop it only feeds the SiLU."""
+
+ def __init__(self, op):
+ super().__init__()
+ self.op = op
+ self.norm = torch.nn.GroupNorm(4, 8)
+ self.act = torch.nn.SiLU()
+ self.b = torch.nn.Parameter(torch.randn(1, 8, 1, 1))
+
+ def forward(self, x):
+ x = x * 1.0
+ h = self.norm(x)
+ if x.shape[-1] <= 16:
+ return self.act(h)
+ if self.op == "input_add_":
+ x.add_(
+ 1.0
+ ) # the norm's input changes after the norm, before a residual read of its output
+ return self.act(h) + h * 2 + x
+ h = _mutate(h, self.op, self.b)
+ return self.act(h)
+
+
+class AutoencoderKLMutating(torch.nn.Module):
+ def __init__(self, op):
+ super().__init__()
+ self.decoder = _MutatingDec(op)
+
+ def _decode(
+ self,
+ z,
+ return_dict = True,
+ ):
+ return (self.decoder(z),)
+
+
[email protected]("mode", ["no_grad", "inference_mode"])
+ "op", ["add_", "mul_", "setitem", "copy_", "out", "view_add_", "iadd", "input_add_"]
+)
+def test_writes_around_a_fused_norm_match_stock_exactly(monkeypatch, op, mode):
+ """Items 4286789097 / 4287220388: a write to the fused output before its paired SiLU (add_, mul_, setitem,
+ copy_, out=, through a view, +=), or a write to the norm's INPUT before a residual read of its output, gave a
+ stale or recomputed-from-changed-input value. Now any such read aborts that decode, which is redone with
+ GroupNorm fused alone: the output matches stock bit for bit, under no_grad and under inference_mode (whose
+ tensors keep no version counter)."""
+ torch.manual_seed(0)
+ vae = AutoencoderKLMutating(op)
+ z = torch.randn(1, 8, 6, 40)
+ ctx = torch.inference_mode if mode == "inference_mode" else torch.no_grad
+ with ctx():
+ ref = vae._decode(z.clone())[0]
+ _exact_kernel(monkeypatch)
+ F.install_structural(vae)
+ with ctx():
+ first = vae._decode(z.clone())[0]
+ again = vae._decode(z.clone())[0]
+ assert torch.equal(first, ref) and torch.equal(again, ref)
+ state = vae._unsloth_vae_structural
+ assert state.status == "ok" and state.gn == 1
+ if op != "iadd":
+ assert state.act_off and state.foreign_reads >= 1
+
+
+def test_kept_oom_drops_the_tracebacks_of_its_whole_chain():
+ """Item 4286789111: only the outer exception's traceback was cleared, so a wrapped OOM's inner frame (and the
+ tensor it held) stayed alive through the retry via the kept exception. Every chained traceback goes now."""
+ import weakref
+
+ budget = _Budget(16384 + 1000)
+ transformer, text_encoder = _on_card(8192), _on_card(4096)
+ seen = {}
+
+ class Vae(_PublicVae):
+ def _decode(
+ self,
+ z,
+ return_dict = True,
+ ):
+ if "ref" not in seen:
+
+ def inner_frame():
+ big = torch.empty(1 << 20)
+ seen["ref"] = weakref.ref(big)
+ raise _FakeOOM()
+
+ try:
+ inner_frame()
+ except _FakeOOM as oom:
+ raise RuntimeError(
+ "backend compiler failed"
+ ) from oom # a wrapped OOM (is_oom walks the chain)
+ seen["alive_at_retry"] = seen["ref"]() is not None
+ return (z.repeat_interleave(8, -1).repeat_interleave(8, -2),)
+
+ vae = Vae(budget, [transformer, text_encoder], 1, 1).to(_CARD)
+ pipe = _Pipe(transformer = transformer, text_encoder = text_encoder, vae = vae)
+ assert O.install(vae, pipe = pipe)
+ vae.decode(_z())
+ assert seen["alive_at_retry"] is False
+
+
+def test_drop_tracebacks_walks_cause_context_and_cycles():
+ def raise_chain():
+ try:
+ try:
+ raise ValueError("a")
+ except ValueError as a:
+ raise KeyError("b") from a
+ except KeyError as b:
+ raise RuntimeError("c")
+
+ try:
+ raise_chain()
+ except RuntimeError as c:
+ exc = c
+ exc.__context__.__cause__.__context__ = exc
+ O.drop_tracebacks(exc)
+ e, seen = exc, 0
+ for e in (exc, exc.__context__, exc.__context__.__cause__):
+ assert e.__traceback__ is None
+
+
+def test_dtype_wrapper_oom_climbs_to_the_vae_offload(monkeypatch):
+ """Item 4287727804: the inner _decode succeeded but the fp16-shaped wrapper's own check ran out of memory, and
+ freeing the rest of the pipeline was not enough; the scope re-raised, since only the _decode ladder (which never
+ saw this OOM) knew the later rungs. The scope now climbs the one ladder (same rungs, same session): freed, stock
+ tiles, then the VAE's weights streamed from the host, which is what fits here."""
+ offloads = []
+ monkeypatch.setattr(
+ O,
+ "_apply_vae_offload",
+ lambda vae, device: (offloads.append(1), vae.conv.to(torch.float32)),
+ )
+ budget = _Budget(4096 + 100)
+ transformer, text_encoder = _on_card(8192), _on_card(4096)
+ vae = _PublicVae(budget, [transformer, text_encoder], 1, 1)
+ vae.to(_CARD)
+ pipe = _Pipe(transformer = transformer, text_encoder = text_encoder, vae = vae)
+ _fp16_shaped(vae, budget, own_need = 3000) # fits only once the VAE's 4096 bytes leave the card
+ assert O.install(vae, pipe = pipe)
+ before = _devices(pipe)
+ out = vae.decode(_z())
+ assert out[0].shape[-1] == 64 and offloads == [1]
+ stats = vae._unsloth_vae_oom_stats
+ assert stats["recovered_by"] == "offload" and stats["outer_oom"] == 1
+ assert _devices(pipe) == before # everything came back after the decode
+
+
+def test_split_vae_never_decodes_before_its_required_repair(monkeypatch):
+ """Item 4287727812: undoing the VAE offload failed and parking it failed too, so the VAE was split with no
+ working streamed path, yet its deferred return was optional: the next decode ran on the inconsistent VAE. The
+ return is required unless the VAE is CHECKED to stream whole from the host, and a decode whose required repair
+ still fails raises that clear error instead of decoding."""
+ vae, pipe = _setup(cap = 1 << 20, untiled_need = 1, tiled_need = 1)
+ session = O.DecodeSession(vae, _FakeDevice())
+ calls = []
+
+ def fail_back():
+ vae.conv.to(torch.float32)
+ raise _FakeOOM()
+
+ def fail_park():
+ raise _FakeOOM()
+
+ change = O.Change("vae_offload", fail_back, fail_park,
+ lambda retry, exc: O._defer(vae, _FakeDevice(), "vae_offload", retry,
+ lambda: not O._vae_streams(vae), vae)) # fmt: skip
+ session.rollback(change)
+ deferred = vae.__dict__[O._DEFERRED_ATTR]
+ assert deferred.required
+ seen_before = len(vae.seen)
+ with pytest.raises(RuntimeError, match = "cannot return"):
+ vae.decode(_z()) if "decode" in vae.__dict__ else vae._decode(_z())
+ assert len(vae.seen) == seen_before
+ deferred.pending[:] = [("vae_offload", lambda: vae.conv.to(_CARD))]
+ vae._decode(_z())
+ assert O._DEFERRED_ATTR not in vae.__dict__ and len(vae.seen) == seen_before + 1
diff --git a/studio/backend/tests/test_diffusion_vae_oom_faults.py b/studio/backend/tests/test_diffusion_vae_oom_faults.py
new file mode 100644
index 00000000000..2e61acd1614
--- /dev/null
+++ b/studio/backend/tests/test_diffusion_vae_oom_faults.py
@@ -0,0 +1,542 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Fault injection over the VAE OOM ladder: an OOM or a generic error at EVERY mutation point of the recovery and of
+its restore, on a fake pipeline (a resident component, resident offload groups, CUDA graphs, a VAE with an
+instance-bound ``_decode`` override and a later ``decode`` wrapper, a sliced batch decode). After each decode
+(returned or raised) the system must be consistent: every module on its card or with a working deferred return that
+resolves the card; resident groups all re-pinned or all streamed with a deferred re-pin; the wrapper stack exact after
+uninstall; later slices on the recovered path; the surfaced error the decode's own OOM when nothing recovered.
+
+CPU only: a float64 tensor stands in for one on the card, float32 for one on the host."""
+
+import itertools
+import types
+
+import pytest
+
+torch = pytest.importorskip("torch")
+
+from core.inference import decode_layers as L # noqa: E402
+from core.inference import diffusion_cuda_graph as G # noqa: E402
+from core.inference import diffusion_memory as M # noqa: E402
+from core.inference import diffusion_vae_oom as O # noqa: E402
+
+CARD, HOST = torch.float64, torch.float32
+
+
+class FakeOOM(RuntimeError):
+ def __init__(self, where = ""):
+ super().__init__(f"CUDA out of memory. Tried to allocate 2.00 GiB ({where})")
+
+
+class Injected(ValueError):
+ pass
+
+
+class Device:
+ type = "cuda"
+
+
+DEV = Device()
+
+
+class World:
+ """The injector: counts mutation points; raises at the ``at``-th one (``kind`` "oom" or "error")."""
+
+ def __init__(
+ self,
+ at = None,
+ kind = "oom",
+ ):
+ self.at, self.kind, self.n, self.points, self.fired = at, kind, 0, [], None
+
+ def __call__(self, point):
+ self.points.append(point)
+ self.n += 1
+ if self.at is not None and self.n in (
+ self.at if isinstance(self.at, tuple) else (self.at,)
+ ):
+ self.fired = point
+ raise FakeOOM(point) if self.kind == "oom" else Injected(point)
+
+
+def _move(module, dtype, world, tag):
+ """Parameter by parameter, a mutation point before each: a fault leaves the module part moved."""
+ for i, p in enumerate(list(module.parameters())):
+ world(f"{tag}:param{i}")
+ p.data = p.data.to(dtype)
+
+
+class Group:
+ def __init__(self):
+ self._unsloth_resident = True
+
+
+class Vae(torch.nn.Module):
+ spatial_compression_ratio = 8
+
+ def __init__(self, budget):
+ super().__init__()
+ self.use_tiling = False
+ self.tile_sample_min_height = 256
+ self.tile_sample_stride_height = 192
+ self.a = torch.nn.Linear(64, 8, bias = False)
+ self.b = torch.nn.Linear(64, 8, bias = False)
+ self.budget = budget
+ self.fake_hooks = False
+ self.slices = []
+
+ def enable_tiling(self):
+ self.use_tiling = True
+
+ def _decode(
+ self,
+ z,
+ return_dict = True,
+ ):
+ streamed = self.fake_hooks
+ need = {False: 9, True: 5}[bool(self.use_tiling)] - (3 if streamed else 0)
+ self.budget.world(f"decode:slice{self.budget.slice}:{'tiled' if self.use_tiling else 'untiled'}"
+ f"{':streamed' if streamed else ''}") # fmt: skip
+ if self.budget.used() + need > self.budget.cap:
+ raise FakeOOM("decode")
+ self.slices.append((self.budget.slice, bool(self.use_tiling), streamed))
+ return (z + (0.5 if self.use_tiling else 0.0),)
+
+ def decode(
+ self,
+ z,
+ return_dict = True,
+ ):
+ out = []
+ for i in range(z.shape[0]):
+ self.budget.slice = i
+ out.append(self._decode(z[i : i + 1])[0])
+ return (torch.cat(out),)
+
+
+class Pipe:
+ pass
+
+
+class Budget:
+ """Units of card memory: the transformer 4, each resident group 1, the VAE weights 3, plus the decode's need."""
+
+ def __init__(self, cap, world):
+ self.cap, self.world, self.slice = cap, world, 0
+ self.transformer = self.groups = self.vae = None
+
+ def used(self):
+ own = getattr(self.transformer, "_fake_groups", None)
+ if own is not None:
+ u = sum(2 for g in own if g._unsloth_resident)
+ else:
+ u = 4 if all(p.dtype == CARD for p in self.transformer.parameters()) else 0
+ u += sum(1 for g in self.groups if g._unsloth_resident and g not in (own or ()))
+ u += 0 if self.vae.fake_hooks else 3
+ return u
+
+
+def build(
+ cap,
+ world,
+ monkeypatch,
+ tiled = False,
+ variant = "base",
+):
+ budget = Budget(cap, world)
+ transformer = torch.nn.Linear(8, 8).to(CARD)
+ text_encoder = torch.nn.Linear(8, 8, bias = False).to(CARD)
+ text_encoder._diffusers_hook = types.SimpleNamespace(hooks = {"group_offloading": 1})
+ text_encoder._unsloth_resident_room = 1
+ groups = [Group() for _ in range(3)]
+ vae = Vae(budget).to(CARD)
+ vae.use_tiling = tiled
+ budget.transformer, budget.groups, budget.vae = transformer, groups, vae
+ pipe = Pipe()
+ pipe.components = {"transformer": transformer, "text_encoder": text_encoder, "vae": vae}
+ pipe.vae, pipe.transformer, pipe._unsloth_cuda_graphs = vae, transformer, ("graph",)
+
+ def release(
+ pipe_,
+ need,
+ logger = None,
+ reason = "",
+ ):
+ # diffusion_memory.release_resident_groups's contract: bookkeeping before each demotion; a failure part way
+ # is caught and the restore for what was released is returned. Its re-pin swallows its own failures (as
+ # _keep_groups_resident does); ``strict`` reports groups still streamed.
+ released = []
+
+ def restore(strict = False):
+ try:
+ for g in released:
+ world("groups:repin")
+ g._unsloth_resident = True
+ except Exception: # noqa: BLE001
+ pass
+ if strict and any(not g._unsloth_resident for g in released):
+ raise RuntimeError("re-pinning resident offload groups left some streamed")
+
+ try:
+ for g in reversed(groups):
+ if g._unsloth_resident:
+ released.append(g)
+ world("groups:release")
+ g._unsloth_resident = False
+ except Exception: # noqa: BLE001
+ return restore if released else None
+ return restore if released else None
+
+ def apply_offload(v, device):
+ world("vae_hooks:add")
+ v.fake_hooks = True
+ _move(v, HOST, world, "vae_out")
+
+ def remove_offload(v, device):
+ world("vae_hooks:remove")
+ v.fake_hooks = False
+ _move(v, CARD, world, "vae_back")
+
+ def park(v, device):
+ v.fake_hooks = False
+ _move(v, HOST, world, "vae_park")
+ world("vae_hooks:readd")
+ v.fake_hooks = True
+
+ def unhook():
+ for g in transformer.__dict__.pop("_fake_groups", None) or ():
+ groups.remove(g)
+ transformer.__dict__.pop("_diffusers_hook", None)
+ transformer.__dict__.pop("_unsloth_resident_room", None)
+
+ def hook_resident(
+ pipe_,
+ device,
+ logger = None,
+ resident = True,
+ ):
+ if getattr(transformer, "_fake_groups", None) is not None:
+ return False
+ world("torchao:hook")
+ own = [Group(), Group()]
+ for g in own:
+ g._unsloth_resident = False
+ transformer._fake_groups = own
+ groups.extend(own)
+ transformer._diffusers_hook = types.SimpleNamespace(hooks = {"group_offloading": 1})
+ try:
+ _move(transformer, HOST, world, "torchao_out")
+ if resident:
+ _move(transformer, CARD, world, "torchao_keep")
+ for g in own:
+ g._unsloth_resident = True
+ transformer._unsloth_resident_room = 1
+ except Exception: # noqa: BLE001
+ unhook()
+ _move(transformer, CARD, world, "torchao_back")
+ return False
+ return True
+
+ def return_torchao(
+ pipe_,
+ device,
+ logger = None,
+ torch_hooks = None,
+ ):
+ own = getattr(transformer, "_fake_groups", None)
+ if own is not None:
+ _move(transformer, CARD, world, "torchao_repin")
+ for g in own:
+ g._unsloth_resident = True
+ transformer._unsloth_resident_room = 1
+ return
+ unhook()
+ _move(transformer, CARD, world, "torchao_back")
+
+ monkeypatch.setattr(M, "release_resident_groups", release)
+ monkeypatch.setattr(M, "hook_resident_denoiser", hook_resident)
+ monkeypatch.setattr(M, "return_torchao_denoiser", return_torchao, raising = False)
+ monkeypatch.setattr(M, "module_forward_hook_ids", lambda m: {}, raising = False)
+ monkeypatch.setattr(O, "_holds_torchao", lambda m: variant == "torchao" and m is transformer)
+ monkeypatch.setattr(G, "reset_all", lambda handles, strict = False: world("graphs:reset"))
+ monkeypatch.setattr(O, "_to_host", lambda m: _move(m, HOST, world, "out"))
+ monkeypatch.setattr(O, "_return_to", lambda m, d: _move(m, CARD, world, "back"))
+ monkeypatch.setattr(O, "_apply_vae_offload", apply_offload)
+ monkeypatch.setattr(O, "_remove_vae_offload", remove_offload)
+ monkeypatch.setattr(O, "_park_vae", park)
+ monkeypatch.setattr(O, "_whole_on", lambda m, d: all(p.dtype == CARD for p in m.parameters()))
+ monkeypatch.setattr(
+ O, "_whole_on_host", lambda m: all(p.dtype == HOST for p in m.parameters()), raising = False
+ )
+ monkeypatch.setattr(
+ O, "_vae_hooked", lambda m: bool(getattr(m, "fake_hooks", False)), raising = False
+ )
+ monkeypatch.setattr(O, "_hooked", lambda m: bool(getattr(m, "fake_hooks", False)) or (
+ getattr(m, "_diffusers_hook", None) is not None)) # fmt: skip
+ monkeypatch.setattr(O, "_host_free_mib", lambda: 1 if variant == "host_short" else 1 << 20)
+ monkeypatch.setattr(O, "_empty_cache", lambda d: None)
+ monkeypatch.setattr(O, "_device_of", lambda z: DEV)
+ monkeypatch.setattr(O, "_release_cache", lambda z, vae = None: None)
+
+ override = types.MethodType(lambda self, z, *a, **k: Vae._decode(self, z, *a, **k), vae)
+ vae._decode = override
+ before = {k: vae.__dict__.get(k, "MISSING") for k in ("_decode", "decode")}
+ assert O.install(vae, pipe = pipe)
+ scope = vae.__dict__["decode"]
+
+ def later(*a, **k):
+ return scope(*a, **k)
+
+ later.__wrapped__ = scope
+ vae.decode = later
+ return pipe, budget, before, override, later
+
+
+STEPS = {(False, False): 0, (True, False): 1, (True, True): 2}
+
+
+def check(pipe, budget, before, override, later, world, outcome):
+ vae, transformer, text_encoder = (
+ pipe.vae,
+ pipe.components["transformer"],
+ pipe.components["text_encoder"],
+ )
+ ctx = f"fault {world.kind} at {world.fired!r}, cap {budget.cap}, outcome {outcome!r}"
+ # surfaced error: the decode's own OOM when nothing recovered; an injected error only from a decode attempt
+ if isinstance(outcome, BaseException):
+ if isinstance(outcome, Injected):
+ assert world.fired and world.fired.startswith("decode:"), ctx
+ else:
+ assert isinstance(outcome, FakeOOM), ctx
+ # later slices stay on the recovered path (never back to a cheaper mode than an earlier slice used)
+ modes = [STEPS[(t, s)] for _, t, s in vae.slices]
+ assert modes == sorted(modes), (ctx, vae.slices)
+ # a VAE left split (its repair is REQUIRED) never decodes before it is repaired: while the repair still fails,
+ # the next decode raises that clear error instead of decoding with the inconsistent VAE
+ vdef = vae.__dict__.get(O._DEFERRED_ATTR)
+ if vdef is not None:
+ # optional only when it is checked to stream whole from the host; anything else is a required repair
+ streams = vae.fake_hooks and all(p.dtype == HOST for p in vae.parameters())
+ assert vdef.required == (not streams), (ctx, vdef.required, streams)
+ if vdef is not None and vdef.required:
+ world.at, world.kind = (world.n + 1, world.n + 2), "oom"
+ vae.slices.clear()
+ with pytest.raises(RuntimeError, match = "cannot return"):
+ vae.decode(torch.zeros(1, 4, 1, 2, 2))
+ assert vae.slices == [], ctx
+ world.at = None # memory frees up: everything deferred must now find its way back
+ budget.cap = 10**6
+ deferred = transformer.__dict__.get(O._DEFERRED_ATTR)
+ if getattr(transformer, "_fake_groups", None) is not None and deferred is None:
+ assert all(p.dtype == (CARD if all(g._unsloth_resident for g in transformer._fake_groups) else HOST)
+ for p in transformer.parameters()), ctx # fmt: skip
+ elif deferred is None:
+ assert all(p.dtype == CARD for p in transformer.parameters()), ctx
+ else:
+ assert (
+ deferred.execution_device is DEV and transformer.__dict__.get("_hf_hook") is deferred
+ ), ctx
+ transformer(torch.zeros(1, 8, dtype = CARD))
+ assert all(p.dtype == CARD for p in transformer.parameters()), ctx
+ assert (
+ O._DEFERRED_ATTR not in transformer.__dict__ and not transformer._forward_pre_hooks
+ ), ctx
+ resident = [g._unsloth_resident for g in budget.groups]
+ if not all(resident):
+ anchors = [m for m in (transformer, text_encoder) if O._DEFERRED_ATTR in m.__dict__]
+ assert anchors, ctx
+ for m in anchors:
+ m(torch.zeros(1, 8, dtype = CARD))
+ assert all(g._unsloth_resident for g in budget.groups), ctx
+ vdef = vae.__dict__.get(O._DEFERRED_ATTR)
+ if vdef is None:
+ assert not vae.fake_hooks and all(p.dtype == CARD for p in vae.parameters()), ctx
+ else:
+ assert O.settle_deferred(vae), ctx
+ assert not vae.fake_hooks and all(p.dtype == CARD for p in vae.parameters()), ctx
+ assert O._SESSION_ATTR not in vae.__dict__, ctx
+ # the wrapper stack: uninstall leaves exactly what was there plus the later wrapper (a pass-through below it)
+ O.uninstall(vae)
+ assert vae.__dict__.get("_decode") is override and vae.__dict__.get("decode") is later, ctx
+ assert L.LAYERS_ATTR not in vae.__dict__ and O._RECORD_ATTR not in vae.__dict__, ctx
+ vae.slices.clear()
+ assert vae.decode(torch.zeros(2, 4, 1, 2, 2))[0].shape[0] == 2 and len(vae.slices) == 2, ctx
+
+
+def _run(
+ cap,
+ at,
+ kind,
+ monkeypatch,
+ tiled = False,
+ verify = True,
+ variant = "base",
+):
+ world = World(at, kind)
+ with monkeypatch.context() as mp:
+ pipe, budget, before, override, later = build(cap, world, mp, tiled, variant)
+ mp.setattr(O, "_FAULT", world)
+ try:
+ out = pipe.vae.decode(torch.zeros(3, 4, 1, 2, 2))
+ outcome = out[0].shape
+ except (FakeOOM, Injected) as exc:
+ outcome = exc
+ world.decode_points = world.n
+ if verify:
+ check(pipe, budget, before, override, later, world, outcome)
+ return world
+
+
+CAPS = (20, 13, 8, 2, 1)
+VARIANTS = ("base", "torchao", "host_short")
+
+
+def _cases():
+ import pytest as _pytest
+
+ mp = _pytest.MonkeyPatch()
+ cases = []
+ try:
+ for variant, cap, tiled in itertools.product(VARIANTS, CAPS, (False, True)):
+ n = _run(cap, None, "oom", mp, tiled, verify = False, variant = variant).decode_points
+ cases += [
+ (variant, cap, tiled, at, kind)
+ for at, kind in itertools.product(range(1, n + 1), ("oom", "error"))
+ ]
+ finally:
+ mp.undo()
+ return cases
+
+
+CASES = _cases()
+
+
+def test_fault_injection_covers_every_mutation_point():
+ assert len(CASES) >= 900, len(CASES)
+ kinds = {p.split(":")[0] for v, cap, tiled in itertools.product(VARIANTS, CAPS, (False, True))
+ for p in _points(cap, tiled, v)} # fmt: skip
+ for point in ("graphs", "groups", "out", "back", "vae_hooks", "vae_out", "vae_back", "decode",
+ "graphs_reset", "resident_groups", "moved", "vae_offload", "torchao", "torchao_hooks",
+ "torchao_out", "torchao_keep"): # fmt: skip
+ assert point in kinds, (point, kinds)
+
+
+def _points(
+ cap,
+ tiled,
+ variant = "base",
+):
+ mp = pytest.MonkeyPatch()
+ try:
+ world = _run(cap, None, "oom", mp, tiled, verify = False, variant = variant)
+ finally:
+ mp.undo()
+ return world.points[: world.decode_points]
+
+
[email protected]("variant,cap,tiled,at,kind", CASES)
+def test_fault_injected_recovery_stays_consistent(variant, cap, tiled, at, kind, monkeypatch):
+ _run(cap, at, kind, monkeypatch, tiled, variant = variant)
+
+
[email protected]("variant", VARIANTS)
[email protected]("cap", CAPS)
+def test_unfaulted_ladder_reaches_its_step(cap, variant, monkeypatch):
+ world = World()
+ with monkeypatch.context() as mp:
+ pipe, budget, *_ = build(cap, world, mp, variant = variant)
+ try:
+ pipe.vae.decode(torch.zeros(3, 4, 1, 2, 2))
+ raised = None
+ except FakeOOM as exc:
+ raised = exc
+ stats = pipe.vae._unsloth_vae_oom_stats
+ expect = {20: None, 13: "freed", 8: "tiled", 2: "offload", 1: None}[cap]
+ if variant == "host_short":
+ expect = {20: None, 13: "tiled", 8: None, 2: None, 1: None}[cap]
+ if cap in (8, 2):
+ assert raised is not None
+ return
+ assert stats.get("recovered_by") == expect, stats
+ assert (raised is not None) == (cap == 1)
+ if cap != 20:
+ assert len({(t, s) for _, t, s in pipe.vae.slices}) <= 1
+
+
+def _torchao_double_cases():
+ """Item 4285966491: the torchao hook conversion fails (first fault, inside hook_resident_denoiser) AND its own
+ return to the card fails (second fault, any later point): the transformer is left split between host and card
+ with no hooks, and the session must still leave it on the card, streamed under hooks, or with a deferred return."""
+ mp = pytest.MonkeyPatch()
+ cases = []
+ try:
+ for cap, tiled in itertools.product((13, 8, 2), (False, True)):
+ pts = _run(cap, None, "oom", mp, tiled, verify = False, variant = "torchao").points
+ firsts = [
+ i + 1 for i, p in enumerate(pts) if p.startswith(("torchao_out:", "torchao_keep:"))
+ ]
+ for a1 in firsts:
+ n = _run(cap, a1, "oom", mp, tiled, verify = False, variant = "torchao").decode_points
+ cases += [
+ (cap, tiled, a1, a2, kind)
+ for a2 in range(a1 + 1, n + 1)
+ for kind in ("oom", "error")
+ ]
+ finally:
+ mp.undo()
+ return cases
+
+
+TORCHAO_DOUBLE = _torchao_double_cases()
+
+
+def test_torchao_double_faults_reach_the_split_state():
+ assert len(TORCHAO_DOUBLE) >= 50, len(TORCHAO_DOUBLE)
+ mp = pytest.MonkeyPatch()
+ try:
+ cap, tiled, a1, a2, kind = next(c for c in TORCHAO_DOUBLE if c[4] == "oom")
+ world = _run(cap, (a1, a1 + 1), "oom", mp, tiled, verify = False, variant = "torchao")
+ finally:
+ mp.undo()
+ assert world.points[a1].startswith("torchao_back:"), world.points[a1 - 1 : a1 + 2]
+
+
[email protected]("cap,tiled,a1,a2,kind", TORCHAO_DOUBLE)
+def test_torchao_conversion_and_its_return_both_fail(cap, tiled, a1, a2, kind, monkeypatch):
+ _run(cap, (a1, a2), kind, monkeypatch, tiled, variant = "torchao")
+
+
+_RESTORE_POINTS = (":undo", ":settle", "back:", "vae_back:", "vae_park:", "vae_hooks:remove", "vae_hooks:readd",
+ "groups:repin", "torchao_back:", "torchao_repin:", "groups:release") # fmt: skip
+
+
+def _restore_double_cases():
+ """Item 4287727812: a restore (undo) that fails, then a second fault anywhere after it (in the settle / park /
+ deferred path): every Change's settle + defer pairing must leave a coherent module or a REQUIRED repair."""
+ mp = pytest.MonkeyPatch()
+ cases = []
+ try:
+ for variant, cap, tiled in itertools.product(VARIANTS, (13, 8, 2), (False, True)):
+ pts = _run(cap, None, "oom", mp, tiled, verify = False, variant = variant).points
+ firsts = [i + 1 for i, p in enumerate(pts) if any(k in p for k in _RESTORE_POINTS)]
+ for a1 in firsts:
+ n = _run(cap, a1, "oom", mp, tiled, verify = False, variant = variant).decode_points
+ cases += [(variant, cap, tiled, a1, a2) for a2 in range(a1 + 1, min(n, a1 + 8) + 1)]
+ finally:
+ mp.undo()
+ return cases
+
+
+RESTORE_DOUBLE = _restore_double_cases()
+
+
+def test_restore_double_faults_cover_the_park_path():
+ assert len(RESTORE_DOUBLE) >= 200, len(RESTORE_DOUBLE)
+ assert any(c[0] == "base" and c[2] in (False, True) for c in RESTORE_DOUBLE)
+
+
[email protected]("variant,cap,tiled,a1,a2", RESTORE_DOUBLE)
+def test_restore_and_its_settle_both_fail(variant, cap, tiled, a1, a2, monkeypatch):
+ _run(cap, (a1, a2), "oom", monkeypatch, tiled, variant = variant)