Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: Triton-fused VAE norms, caches and attention for every image and video VAE (1.7x to 6.3x decode) (#12078)

Daniel Hane07672bd2d19
diff --git a/studio/backend/core/inference/diffusion_speed.py b/studio/backend/core/inference/diffusion_speed.py
index 8fe300e6a74..07f913d1e57 100644
--- a/studio/backend/core/inference/diffusion_speed.py
+++ b/studio/backend/core/inference/diffusion_speed.py
@@ -437,6 +437,7 @@ def apply_speed_optims(
applied = {
"channels_last": False,
"vae_fp16_decode": False,
+ "vae_fused": False,
"cudnn_benchmark": False,
"tf32": False,
"fp16_accum": False,
@@ -459,6 +460,8 @@ def apply_speed_optims(
applied["channels_last"] = _vae_channels_last(pipe, logger)
# Near-lossless, not bit-identical, so never on "off" (returned above).
applied["vae_fp16_decode"] = _video_vae_half_decode(pipe, target, family, logger)
+ if on_cuda:
+ applied["vae_fused"] = _install_fused_vae(pipe, logger)
if on_cuda:
applied["cudnn_benchmark"] = _enable_cudnn_benchmark(logger)
@@ -511,8 +514,14 @@ def apply_speed_optims(
eager_when_tiled = _vae_eager_when_tiled(pipe),
)
- if applied["channels_last"] and not _channels_last_decode_wins(
- pipe, target, applied["compiled_vae_decode"], offload_active
+ # Fused passes feed cuDNN channels-last activations; contiguous weights would be relaid out per call.
+ fused_cl = bool(getattr(getattr(pipe, "vae", None), "_unsloth_vae_fused_cl_weights", False))
+ if (
+ applied["channels_last"]
+ and not fused_cl
+ and not _channels_last_decode_wins(
+ pipe, target, applied["compiled_vae_decode"], offload_active
+ )
):
applied["channels_last"] = not _vae_contiguous(pipe, logger)
elif applied["compiled_vae_decode"] and not _channels_last_decode_wins(
@@ -1108,8 +1117,34 @@ def vae_decode_compile_allowed(pipe: Any, speed_mode: str) -> bool:
return _denoiser_unet(pipe) is None and _vae_decode_compile_allowed(pipe, speed_mode)
+def _install_fused_vae(pipe: Any, logger: Any) -> bool:
+ """Triton-fused VAE norm passes (diffusion_vae_fused), unless the env forces the VAE decode compile instead."""
+ if os.environ.get(COMPILE_VAE_ENV, "").strip().lower() in _VAE_TRUE_TOKENS:
+ return False
+ try:
+ from . import diffusion_vae_fused # noqa: PLC0415 - Triton import only on CUDA loads
+ return diffusion_vae_fused.install(getattr(pipe, "vae", None), logger) > 0
+ except Exception as exc: # noqa: BLE001 - optimisation only
+ _warn(logger, "fused vae", exc)
+ return False
+
+
+def _fused_vae_planned(pipe: Any) -> bool:
+ if os.environ.get(COMPILE_VAE_ENV, "").strip().lower() in _VAE_TRUE_TOKENS:
+ return False
+ try:
+ from . import diffusion_vae_fused # noqa: PLC0415
+ return diffusion_vae_fused.will_install(getattr(pipe, "vae", None))
+ except Exception: # noqa: BLE001
+ return False
+
+
def _vae_decode_compile_allowed(pipe: Any, speed_mode: str) -> bool:
- """U-Nets always; a DiT only on ``max`` (it costs a 60-70 s slower first render) or with the env forced on."""
+ """U-Nets always; a DiT only on ``max`` (it costs a 60-70 s slower first render) or with the env forced on.
+
+ Never when the fused eager VAE path engages: it is faster than the compiled decode, with no cold compile."""
+ if _fused_vae_planned(pipe):
+ return False
if _denoiser_unet(pipe) is not None:
return True
raw = os.environ.get(COMPILE_VAE_ENV, "").strip().lower()
@@ -1354,7 +1389,17 @@ def _fuse_qkv(pipe: Any, logger: Any) -> bool:
fn = getattr(pipe, "fuse_qkv_projections", None)
if callable(fn):
try:
+ vae = getattr(pipe, "vae", None)
+ fused_vae_attn = callable(getattr(vae, "modules", None)) and any(
+ type(getattr(m, "processor", None)).__name__ == "FusedSingleHeadProcessor"
+ for m in vae.modules()
+ )
fn()
+ if (
+ fused_vae_attn
+ ): # the pipe-level fuse resets every VAE processor to FusedAttnProcessor2_0
+ from . import diffusion_vae_fused # noqa: PLC0415
+ diffusion_vae_fused.install_attention_processors(vae)
return True
except Exception as exc: # noqa: BLE001 - optimisation only
_warn(logger, "fuse_qkv_projections", exc)
diff --git a/studio/backend/core/inference/diffusion_vae_fused.py b/studio/backend/core/inference/diffusion_vae_fused.py
new file mode 100644
index 00000000000..6437a431cf4
--- /dev/null
+++ b/studio/backend/core/inference/diffusion_vae_fused.py
@@ -0,0 +1,2113 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Triton-fused GroupNorm / channel RMS norm for eagerly decoded image and video VAEs.
+
+CUDA GroupNorm has no channels-last kernel, so every norm copies NHWC <-> NCHW; these kernels read any layout,
+write channels-last with fp32 statistics. ``install(vae)`` patches instances (never classes), each falling back
+to stock on a new exception. NVIDIA CUDA + Triton >= 3.3 only; ``UNSLOTH_VAE_FUSED=0`` disables.
+"""
+
+from __future__ import annotations
+
+import os
+import sys
+import types
+from functools import lru_cache
+from typing import Any, Optional
+
+VAE_FUSED_ENV = "UNSLOTH_VAE_FUSED"
+_FALSE = ("0", "false", "no", "off")
+_MIN_TRITON = (3, 3)
+_MAX_C = 4096
+
+
+def fused_vae_disabled() -> bool:
+ return os.environ.get(VAE_FUSED_ENV, "").strip().lower() in _FALSE
+
+
+@lru_cache(maxsize = 1)
+def _triton_version_ok() -> bool:
+ try:
+ import re
+
+ import triton
+
+ m = re.match(r"(\d+)\.(\d+)", str(triton.__version__))
+ return bool(m) and (int(m.group(1)), int(m.group(2))) >= _MIN_TRITON
+ except Exception: # noqa: BLE001
+ return False
+
+
+@lru_cache(maxsize = 1)
+def _toolchain_ok() -> bool:
+ if sys.platform != "win32":
+ return True
+ try:
+ from .._msvc_env import crt_headers_reachable
+ return bool(crt_headers_reachable())
+ except Exception: # noqa: BLE001
+ return True
+
+
+# Loop-bound-only args are not specialized (else ~70 JIT variants on a tiled Wan decode); strides and C stay
+# specialized so channel-contiguous loads vectorize.
+_SHAPE_ARGS = {
+ "_rms_act": ("T", "H", "W", "Ho", "Wo", "ph", "pw", "To", "front", "n_cache"),
+ "_bias_residual": ("P", "T", "HW", "W"),
+ "_dup_up_add": ("T_o", "H_o", "W_o", "t_off"),
+ "_up_nearest2x": ("T_o", "H", "W"),
+}
+
+
+@lru_cache(maxsize = 1)
+def _kernels() -> Optional[types.SimpleNamespace]:
+ try:
+ import triton
+ import triton.language as tl
+ except Exception: # noqa: BLE001
+ return None
+ globals().update(triton = triton, tl = tl)
+
+ @triton.jit
+ def _gn_partials(
+ x_ptr, bias_ptr, pn_ptr, pmean_ptr, pm2_ptr,
+ T, HW, W, n_chunks, sb, sc, st, sh, sw,
+ C: tl.constexpr, G: tl.constexpr, CPG: tl.constexpr, BLOCK_P: tl.constexpr, HAS_BIAS: tl.constexpr
+ ): # fmt: skip
+ chunk = tl.program_id(0)
+ bt = tl.program_id(1)
+ t = bt % T
+ b = bt // T
+ p = chunk * BLOCK_P + tl.arange(0, BLOCK_P)
+ pmask = p < HW
+ hh = p // W
+ ww = p - hh * W
+ c = tl.arange(0, C)
+ off = (
+ b.to(tl.int64) * sb
+ + t.to(tl.int64) * st
+ + hh.to(tl.int64)[:, None] * sh
+ + ww.to(tl.int64)[:, None] * sw
+ + c.to(tl.int64)[None, :] * sc
+ )
+ v = tl.load(x_ptr + off, mask = pmask[:, None], other = 0.0).to(tl.float32)
+ if HAS_BIAS:
+ v = v + tl.load(bias_ptr + c).to(tl.float32)[None, :]
+ v = tl.where(pmask[:, None], v, 0.0)
+ n_pix = tl.sum(pmask.to(tl.float32), axis = 0)
+ v3 = tl.reshape(v, (BLOCK_P, G, CPG))
+ mean = tl.sum(tl.sum(v3, axis = 2), axis = 0) / (n_pix * CPG)
+ d = tl.where(pmask[:, None, None], v3 - mean[None, :, None], 0.0)
+ m2 = tl.sum(tl.sum(d * d, axis = 2), axis = 0)
+ slot = bt.to(tl.int64) * n_chunks + chunk
+ g = tl.arange(0, G)
+ tl.store(pmean_ptr + slot * G + g, mean)
+ tl.store(pm2_ptr + slot * G + g, m2)
+ tl.store(pn_ptr + slot, n_pix * CPG)
+
+ @triton.jit
+ def _gn_combine(
+ pn_ptr, pmean_ptr, pm2_ptr, mean_ptr, rstd_ptr, n_chunks, G, eps, BLOCK: tl.constexpr
+ ):
+ row = tl.program_id(0)
+ g = row % G
+ bt = (row // G).to(tl.int64)
+ zero = tl.sum(tl.zeros([BLOCK], dtype = tl.float32), axis = 0)
+ acc_n = zero
+ acc_s = zero
+ for k0 in range(0, n_chunks, BLOCK):
+ k = k0 + tl.arange(0, BLOCK)
+ km = k < n_chunks
+ n = tl.load(pn_ptr + bt * n_chunks + k, mask = km, other = 0.0)
+ mu = tl.load(pmean_ptr + (bt * n_chunks + k) * G + g, mask = km, other = 0.0)
+ acc_n += tl.sum(n, axis = 0)
+ acc_s += tl.sum(n * mu, axis = 0)
+ mean = acc_s / acc_n
+ acc_m2 = zero
+ for k0 in range(0, n_chunks, BLOCK):
+ k = k0 + tl.arange(0, BLOCK)
+ km = k < n_chunks
+ n = tl.load(pn_ptr + bt * n_chunks + k, mask = km, other = 0.0)
+ mu = tl.load(pmean_ptr + (bt * n_chunks + k) * G + g, mask = km, other = 0.0)
+ m2 = tl.load(pm2_ptr + (bt * n_chunks + k) * G + g, mask = km, other = 0.0)
+ dm = mu - mean
+ acc_m2 += tl.sum(m2 + n * dm * dm, axis = 0)
+ tl.store(mean_ptr + row, mean)
+ tl.store(rstd_ptr + row, 1.0 / tl.sqrt(acc_m2 / acc_n + eps))
+
+ @triton.jit
+ def _gn_apply(
+ x_ptr, out_ptr, mean_ptr, rstd_ptr, w_ptr, b_ptr, ib_ptr,
+ C, T, HW, W, G, sb, sc, st, sh, sw,
+ CPG: tl.constexpr, ACT: tl.constexpr, HAS_IN_BIAS: tl.constexpr, BLOCK_P: tl.constexpr, BLOCK_C: tl.constexpr
+ ): # fmt: skip
+ pid_p = tl.program_id(0)
+ bt = tl.program_id(1)
+ pid_c = tl.program_id(2)
+ t = bt % T
+ b = bt // T
+ p = pid_p * BLOCK_P + tl.arange(0, BLOCK_P)
+ pmask = p < HW
+ hh = p // W
+ ww = p - hh * W
+ c = pid_c * BLOCK_C + tl.arange(0, BLOCK_C)
+ cmask = c < C
+ m = pmask[:, None] & cmask[None, :]
+ off = (
+ b.to(tl.int64) * sb
+ + t.to(tl.int64) * st
+ + hh.to(tl.int64)[:, None] * sh
+ + ww.to(tl.int64)[:, None] * sw
+ + c.to(tl.int64)[None, :] * sc
+ )
+ v = tl.load(x_ptr + off, mask = m, other = 0.0).to(tl.float32)
+ if HAS_IN_BIAS:
+ v = v + tl.load(ib_ptr + c, mask = cmask, other = 0.0).to(tl.float32)[None, :]
+ gidx = bt * G + c // CPG
+ mu = tl.load(mean_ptr + gidx, mask = cmask, other = 0.0)
+ rs = tl.load(rstd_ptr + gidx, mask = cmask, other = 0.0)
+ gw = tl.load(w_ptr + c, mask = cmask, other = 0.0).to(tl.float32)
+ gb = tl.load(b_ptr + c, mask = cmask, other = 0.0).to(tl.float32)
+ a = rs * gw
+ v = v * a[None, :] + (gb - mu * a)[None, :]
+ if ACT:
+ v = v / (1.0 + tl.exp(-v))
+ out_off = (bt.to(tl.int64) * HW + p.to(tl.int64))[:, None] * C + c[None, :]
+ tl.store(out_ptr + out_off, v.to(out_ptr.dtype.element_ty), mask = m)
+
+ @triton.jit(do_not_specialize = _SHAPE_ARGS["_rms_act"])
+ def _rms_act(
+ x_ptr, out_ptr, w_ptr, b_ptr, ib_ptr, cache_ptr,
+ C, T, H, W, Ho, Wo, ph, pw, To, front, n_cache, scale, eps,
+ sb, sc, st, sh, sw, cb, cc, ct, ch, cw,
+ NORM: tl.constexpr, ACT: tl.constexpr, HAS_BIAS: tl.constexpr, HAS_IN_BIAS: tl.constexpr,
+ REPLICATE: tl.constexpr, MEAN_SQ: tl.constexpr, BLOCK_P: tl.constexpr, BLOCK_C: tl.constexpr
+ ): # fmt: skip
+ pid_p = tl.program_id(0)
+ bt = tl.program_id(1)
+ t_o = bt % To
+ b = bt // To
+ p = pid_p * BLOCK_P + tl.arange(0, BLOCK_P)
+ pmask = p < Ho * Wo
+ ho = p // Wo
+ wo = p - ho * Wo
+ if REPLICATE:
+ hh = tl.minimum(tl.maximum(ho - ph, 0), H - 1)
+ ww = tl.minimum(tl.maximum(wo - pw, 0), W - 1)
+ else:
+ hh = ho
+ ww = wo
+ c = tl.arange(0, BLOCK_C)
+ cmask = c < C
+ m = pmask[:, None] & cmask[None, :]
+ out_off = ((b.to(tl.int64) * To + t_o) * (Ho * Wo) + p.to(tl.int64))[:, None] * C + c[
+ None, :
+ ]
+ if (not REPLICATE) and t_o < front:
+ tc = t_o - (front - n_cache)
+ if tc >= 0:
+ coff = (
+ b.to(tl.int64) * cb
+ + tc.to(tl.int64) * ct
+ + hh.to(tl.int64)[:, None] * ch
+ + ww.to(tl.int64)[:, None] * cw
+ + c.to(tl.int64)[None, :] * cc
+ )
+ v = tl.load(cache_ptr + coff, mask = m, other = 0.0)
+ tl.store(out_ptr + out_off, v.to(out_ptr.dtype.element_ty), mask = m)
+ else:
+ tl.store(
+ out_ptr + out_off,
+ tl.zeros([BLOCK_P, BLOCK_C], out_ptr.dtype.element_ty),
+ mask = m,
+ )
+ else:
+ t = tl.minimum(tl.maximum(t_o - front, 0), T - 1)
+ off = (
+ b.to(tl.int64) * sb
+ + t.to(tl.int64) * st
+ + hh.to(tl.int64)[:, None] * sh
+ + ww.to(tl.int64)[:, None] * sw
+ + c.to(tl.int64)[None, :] * sc
+ )
+ v = tl.load(x_ptr + off, mask = m, other = 0.0).to(tl.float32)
+ if HAS_IN_BIAS:
+ v = v + tl.load(ib_ptr + c, mask = cmask, other = 0.0).to(tl.float32)[None, :]
+ v = tl.where(m, v, 0.0)
+ if NORM:
+ ss = tl.sum(v * v, axis = 1)
+ if MEAN_SQ:
+ # LTX PerChannelRMSNorm: x / sqrt(mean(x^2) + eps), no affine
+ v = v / tl.sqrt(ss / C + eps)[:, None]
+ else:
+ # Wan-lineage RMS_norm: F.normalize(x) * sqrt(C) * gamma (+ bias)
+ inv = 1.0 / tl.maximum(tl.sqrt(ss), 1e-12)
+ gw = tl.load(w_ptr + c, mask = cmask, other = 0.0).to(tl.float32) * scale
+ v = v * inv[:, None] * gw[None, :]
+ if HAS_BIAS:
+ v = v + tl.load(b_ptr + c, mask = cmask, other = 0.0).to(tl.float32)[None, :]
+ if ACT:
+ v = v / (1.0 + tl.exp(-v))
+ tl.store(out_ptr + out_off, v.to(out_ptr.dtype.element_ty), mask = m)
+
+ @triton.jit(do_not_specialize = _SHAPE_ARGS["_bias_residual"])
+ def _bias_residual(
+ o_ptr, ob_ptr, r_ptr, rb_ptr, P, C, T, HW, W, sb, sc, st, sh, sw, inv_scale,
+ HAS_OB: tl.constexpr, HAS_RB: tl.constexpr, SCALE: tl.constexpr, BLOCK_P: tl.constexpr, BLOCK_C: tl.constexpr
+ ): # fmt: skip
+ p = tl.program_id(0) * BLOCK_P + tl.arange(0, BLOCK_P)
+ c = tl.program_id(1) * BLOCK_C + tl.arange(0, BLOCK_C)
+ pm = p < P
+ cm = c < C
+ m = pm[:, None] & cm[None, :]
+ o_off = p.to(tl.int64)[:, None] * C + c[None, :]
+ o = tl.load(o_ptr + o_off, mask = m, other = 0.0).to(tl.float32)
+ if HAS_OB:
+ o = o + tl.load(ob_ptr + c, mask = cm, other = 0.0).to(tl.float32)[None, :]
+ ww = p % W
+ rest = p // W
+ hh = rest % (HW // W)
+ bt = rest // (HW // W)
+ t = bt % T
+ b = bt // T
+ r_off = (
+ b.to(tl.int64) * sb + t.to(tl.int64) * st + hh.to(tl.int64) * sh + ww.to(tl.int64) * sw
+ )[:, None] + c.to(tl.int64)[None, :] * sc
+ r = tl.load(r_ptr + r_off, mask = m, other = 0.0).to(tl.float32)
+ if HAS_RB:
+ r = r + tl.load(rb_ptr + c, mask = cm, other = 0.0).to(tl.float32)[None, :]
+ o = o + r
+ if SCALE:
+ o = o * inv_scale
+ tl.store(o_ptr + o_off, o.to(o_ptr.dtype.element_ty), mask = m)
+
+ @triton.jit
+ def _dcae_up_add(
+ h_ptr, hb_ptr, x_ptr, out_ptr,
+ c, Cin, F, H, W, Fo, rep,
+ hsb, hsc, hst, hsh, hsw, xsb, xsc, xst, xsh, xsw,
+ TEMPORAL: tl.constexpr, HAS_HB: tl.constexpr, BLOCK_P: tl.constexpr, BLOCK_C: tl.constexpr
+ ): # fmt: skip
+ pid_p = tl.program_id(0)
+ bf = tl.program_id(1)
+ pid_c = tl.program_id(2)
+ fo = bf % Fo
+ b = bf // Fo
+ Wo = 2 * W
+ p = pid_p * BLOCK_P + tl.arange(0, BLOCK_P)
+ pm = p < 4 * H * W
+ ho = p // Wo
+ wo = p - ho * Wo
+ hh = ho // 2
+ ww = wo // 2
+ ph = (ho - hh * 2) * 2 + (wo - ww * 2) # r2i * 2 + r3i
+ ci = pid_c * BLOCK_C + tl.arange(0, BLOCK_C)
+ cm = ci < c
+ m = pm[:, None] & cm[None, :]
+ if TEMPORAL:
+ if fo == 0:
+ f = fo * 0
+ pk = ph[:, None] * (2 * c) + ci[None, :]
+ xk = ph[:, None] * (Cin // 4) + ci[None, :] // (rep // 2)
+ else:
+ f = 1 + (fo - 1) // 2
+ r1 = (fo - 1) % 2
+ pk = (r1 * 4 + ph)[:, None] * c + ci[None, :]
+ xk = pk // rep
+ else:
+ f = fo
+ pk = ph[:, None] * c + ci[None, :]
+ xk = pk // rep
+ pix_h = (
+ b.to(tl.int64) * hsb
+ + f.to(tl.int64) * hst
+ + (hh.to(tl.int64) * hsh + ww.to(tl.int64) * hsw)[:, None]
+ )
+ v = tl.load(h_ptr + pix_h + pk.to(tl.int64) * hsc, mask = m, other = 0.0).to(tl.float32)
+ if HAS_HB:
+ v = v + tl.load(hb_ptr + pk, mask = m, other = 0.0).to(tl.float32)
+ v = v.to(out_ptr.dtype.element_ty).to(
+ tl.float32
+ ) # the stock conv output is rounded before the add
+ pix_x = (
+ b.to(tl.int64) * xsb
+ + f.to(tl.int64) * xst
+ + (hh.to(tl.int64) * xsh + ww.to(tl.int64) * xsw)[:, None]
+ )
+ xv = tl.load(x_ptr + pix_x + xk.to(tl.int64) * xsc, mask = m, other = 0.0).to(tl.float32)
+ o_off = (bf.to(tl.int64) * (4 * H * W) + p.to(tl.int64))[:, None] * c + ci[None, :]
+ tl.store(out_ptr + o_off, (v + xv).to(out_ptr.dtype.element_ty), mask = m)
+
+ @triton.jit
+ def _softmax_rows(s_ptr, p_ptr, S, P_STRIDE, scale, BLOCK: tl.constexpr):
+ row = tl.program_id(0).to(tl.int64)
+ base = s_ptr + row * P_STRIDE
+ m = tl.full([BLOCK], float("-inf"), tl.float32)
+ l = tl.zeros([BLOCK], tl.float32)
+ for k0 in range(0, S, BLOCK):
+ cols = k0 + tl.arange(0, BLOCK)
+ x = tl.load(base + cols, mask = cols < S, other = float("-inf")) * scale
+ m_new = tl.maximum(m, x)
+ alpha = tl.where(m_new == float("-inf"), 0.0, tl.exp(m - m_new))
+ l = l * alpha + tl.where(x == float("-inf"), 0.0, tl.exp(x - m_new))
+ m = m_new
+ m_row = tl.max(m, axis = 0)
+ l_row = tl.sum(l * tl.where(m == float("-inf"), 0.0, tl.exp(m - m_row)), axis = 0)
+ inv = 1.0 / l_row
+ for k0 in range(0, P_STRIDE, BLOCK):
+ cols = k0 + tl.arange(0, BLOCK)
+ x = tl.load(base + cols, mask = cols < S, other = float("-inf")) * scale
+ p = tl.where(cols < S, tl.exp(x - m_row) * inv, 0.0)
+ tl.store(
+ p_ptr + row * P_STRIDE + cols, p.to(p_ptr.dtype.element_ty), mask = cols < P_STRIDE
+ )
+
+ @triton.jit(do_not_specialize = _SHAPE_ARGS["_dup_up_add"])
+ def _dup_up_add(
+ o_ptr, x_ptr, C, T_o, H_o, W_o, t_off, repeats,
+ osb, osc, ost, osh, osw, xsb, xsc, xst, xsh, xsw,
+ FT: tl.constexpr, FS: tl.constexpr, BLOCK_P: tl.constexpr, BLOCK_C: tl.constexpr
+ ): # fmt: skip
+ pid_p = tl.program_id(0)
+ bt = tl.program_id(1)
+ pid_c = tl.program_id(2)
+ t = bt % T_o
+ b = bt // T_o
+ p = pid_p * BLOCK_P + tl.arange(0, BLOCK_P)
+ pm = p < H_o * W_o
+ ho = p // W_o
+ wo = p - ho * W_o
+ c = pid_c * BLOCK_C + tl.arange(0, BLOCK_C)
+ cm = c < C
+ m = pm[:, None] & cm[None, :]
+ tt = t + t_off
+ ti = tt // FT
+ a = tt - ti * FT
+ hi = ho // FS
+ i = ho - hi * FS
+ wi = wo // FS
+ j = wo - wi * FS
+ k = ((c[None, :] * FT + a) * FS + i[:, None]) * FS + j[:, None]
+ src = k // repeats
+ x_off = (b.to(tl.int64) * xsb + ti.to(tl.int64) * xst + (hi.to(tl.int64) * xsh + wi.to(tl.int64) * xsw)[:, None]
+ + src.to(tl.int64) * xsc) # fmt: skip
+ o_off = (b.to(tl.int64) * osb + t.to(tl.int64) * ost + (ho.to(tl.int64) * osh + wo.to(tl.int64) * osw)[:, None]
+ + c.to(tl.int64)[None, :] * osc) # fmt: skip
+ xv = tl.load(x_ptr + x_off, mask = m, other = 0.0).to(tl.float32)
+ ov = tl.load(o_ptr + o_off, mask = m, other = 0.0).to(tl.float32)
+ tl.store(o_ptr + o_off, (ov + xv).to(o_ptr.dtype.element_ty), mask = m)
+
+ @triton.jit(do_not_specialize = _SHAPE_ARGS["_up_nearest2x"])
+ def _up_nearest2x(
+ x_ptr, out_ptr, C, T_o, H, W, xsb, xsc, xst, xsh, xsw,
+ INTERLEAVE: tl.constexpr, BLOCK_P: tl.constexpr, BLOCK_C: tl.constexpr
+ ): # fmt: skip
+ # INTERLEAVE: x is Wan time_conv output (B, 2C, T_o/2, H, W); frame 2t+j takes channels [jC, (j+1)C).
+ pid_p = tl.program_id(0)
+ bt = tl.program_id(1)
+ pid_c = tl.program_id(2)
+ t = bt % T_o
+ b = bt // T_o
+ p = pid_p * BLOCK_P + tl.arange(0, BLOCK_P)
+ pm = p < 4 * H * W
+ ho = p // (2 * W)
+ wo = p - ho * (2 * W)
+ c = pid_c * BLOCK_C + tl.arange(0, BLOCK_C)
+ cm = c < C
+ m = pm[:, None] & cm[None, :]
+ if INTERLEAVE:
+ ti = t // 2
+ ch = (t - ti * 2) * C + c
+ else:
+ ti = t
+ ch = c
+ x_off = (b.to(tl.int64) * xsb + ti.to(tl.int64) * xst
+ + ((ho // 2).to(tl.int64) * xsh + (wo // 2).to(tl.int64) * xsw)[:, None] + ch.to(tl.int64)[None, :] * xsc) # fmt: skip
+ v = tl.load(x_ptr + x_off, mask = m, other = 0.0)
+ o_off = (bt.to(tl.int64) * (4 * H * W) + p.to(tl.int64))[:, None] * C + c[None, :]
+ tl.store(out_ptr + o_off, v, mask = m)
+
+ return types.SimpleNamespace(
+ gn_partials = _gn_partials,
+ gn_combine = _gn_combine,
+ gn_apply = _gn_apply,
+ rms_act = _rms_act,
+ bias_residual = _bias_residual,
+ dcae_up_add = _dcae_up_add,
+ softmax_rows = _softmax_rows,
+ dup_up_add = _dup_up_add,
+ up_nearest2x = _up_nearest2x,
+ )
+
+
+def _next_pow2(n: int) -> int:
+ return 1 << max(0, int(n) - 1).bit_length()
+
+
+def runtime_ok() -> bool:
+ """NVIDIA CUDA + a usable Triton; everything else keeps the stock path."""
+ if fused_vae_disabled():
+ return False
+ try:
+ import torch
+ except Exception: # noqa: BLE001
+ return False
+ if not torch.cuda.is_available() or getattr(torch.version, "hip", None):
+ return False
+ return _triton_version_ok() and _toolchain_ok() and _kernels() is not None
+
+
+def _as5d(x: Any) -> tuple:
+ """(tensor viewed as B,C,T,H,W, was_4d)."""
+ if x.dim() == 4:
+ return x.unsqueeze(2), True
+ if x.dim() == 3: # (B, C, L): attention's GroupNorm input
+ return x.unsqueeze(2).unsqueeze(2), True
+ return x, False
+
+
+def group_norm_reference(
+ x: Any,
+ norm: Any,
+ act: bool,
+ in_bias: Any = None,
+) -> Any:
+ import torch.nn.functional as F
+
+ if in_bias is not None:
+ x = x + in_bias.view(1, -1, *([1] * (x.dim() - 2))).to(x.dtype)
+ y = F.group_norm(x, norm.num_groups, norm.weight, norm.bias, norm.eps)
+ return F.silu(y) if act else y
+
+
+def _gn_fusable(x: Any, norm: Any) -> bool:
+ c = x.shape[1] if x.dim() >= 2 else 0
+ g = int(getattr(norm, "num_groups", 0) or 0)
+ return (
+ x.is_cuda
+ and x.dim() in (3, 4, 5)
+ and g > 0
+ and c % g == 0
+ and c & (c - 1) == 0
+ and g & (g - 1) == 0
+ and c <= 1024
+ and getattr(norm, "affine", True)
+ and norm.weight is not None
+ and x.dtype in (_torch().float16, _torch().bfloat16, _torch().float32)
+ and x.numel() > 0
+ )
+
+
+def _torch():
+ import torch
+ return torch
+
+
+def group_norm_act(
+ x: Any,
+ norm: Any,
+ act: bool = True,
+ in_bias: Any = None,
+) -> Any:
+ """``silu?(group_norm(x + in_bias))`` for 3/4/5-D ``x``; channels-last output; 5-D stats span T like nn.GroupNorm."""
+ torch = _torch()
+ if not _gn_fusable(x, norm):
+ return group_norm_reference(x, norm, act, in_bias)
+ k = _kernels()
+ ndim = x.dim()
+ x5, _ = _as5d(x)
+ b, c, t, h, w = x5.shape
+ # nn.GroupNorm pools the whole (T, H, W) extent: flatten T into H when strides allow, else fall back
+ if t > 1:
+ if x5.stride(2) == h * x5.stride(3):
+ x5 = x5.as_strided(
+ (b, c, 1, t * h, w), (x5.stride(0), x5.stride(1), 0, x5.stride(3), x5.stride(4))
+ )
+ b, c, t, h, w = x5.shape
+ else:
+ return group_norm_reference(x, norm, act, in_bias)
+ g = int(norm.num_groups)
+ cpg = c // g
+ hw = h * w
+ sb, sc, st, sh, sw = x5.stride()
+ ib = in_bias if in_bias is not None else x5
+ budget = 4096 if x5.element_size() == 4 else 8192
+ block_p = max(1, min(256, budget // c))
+ n_chunks = (hw + block_p - 1) // block_p
+ rows = b * t
+ pmean, pm2 = torch.empty((2, rows * n_chunks * g), dtype = torch.float32, device = x.device)
+ pn = torch.empty(rows * n_chunks, dtype = torch.float32, device = x.device)
+ k.gn_partials[(n_chunks, rows)](
+ x5, ib, pn, pmean, pm2, t, hw, w, n_chunks, sb, sc, st, sh, sw,
+ C = c, G = g, CPG = cpg, BLOCK_P = block_p, HAS_BIAS = in_bias is not None, num_warps = 4,
+ ) # fmt: skip
+ mean = torch.empty(rows * g, dtype = torch.float32, device = x.device)
+ rstd = torch.empty(rows * g, dtype = torch.float32, device = x.device)
+ k.gn_combine[(rows * g,)](
+ pn, pmean, pm2, mean, rstd, n_chunks, g, float(norm.eps), BLOCK = 1024, num_warps = 4
+ )
+ out = torch.empty((b, t, h, w, c), dtype = x.dtype, device = x.device)
+ block_c = min(128, _next_pow2(c))
+ block_p2 = max(1, 4096 // block_c)
+ grid = ((hw + block_p2 - 1) // block_p2, rows, (c + block_c - 1) // block_c)
+ k.gn_apply[grid](
+ x5, out, mean, rstd, norm.weight, norm.bias, ib, c, t, hw, w, g, sb, sc, st, sh, sw,
+ CPG = cpg, ACT = bool(act), HAS_IN_BIAS = in_bias is not None, BLOCK_P = block_p2, BLOCK_C = block_c,
+ num_warps = 4,
+ ) # fmt: skip
+ out = out.permute(0, 4, 1, 2, 3) # B, C, T', H', W'
+ shape = x.shape
+ if ndim == 5:
+ return out.reshape(b, c, *shape[2:]) if out.shape != shape else out
+ if ndim == 4:
+ return out[:, :, 0]
+ return out[:, :, 0].reshape(shape) # (B, C, L) view over the (B, L, C) buffer
+
+
+def rms_norm_reference(
+ x: Any,
+ norm: Any,
+ act: bool,
+ in_bias: Any = None,
+) -> Any:
+ import torch.nn.functional as F
+
+ if in_bias is not None:
+ x = x + in_bias.view(1, -1, *([1] * (x.dim() - 2))).to(x.dtype)
+ y = x if norm is None else norm(x)
+ return F.silu(y) if act else y
+
+
+def _is_pixel_norm(norm: Any) -> bool:
+ return type(norm).__name__ == "PerChannelRMSNorm" and getattr(norm, "channel_dim", 1) == 1
+
+
+def _rms_params(norm: Any) -> Optional[tuple]:
+ torch = _torch()
+ if _is_pixel_norm(norm):
+ return None, None, False, 1.0
+ gamma = norm.gamma.reshape(-1)
+ bias = norm.bias
+ has_bias = isinstance(bias, torch.Tensor)
+ if not has_bias and float(bias) != 0.0:
+ return None
+ return gamma, (bias.reshape(-1) if has_bias else gamma), has_bias, float(norm.scale)
+
+
+def _rms_fusable(x: Any, norm: Any, cache: Any) -> bool:
+ torch = _torch()
+ c = x.shape[1]
+ return (
+ x.is_cuda
+ and x.dim() == 5
+ and c <= _MAX_C
+ and x.dtype in (torch.float16, torch.bfloat16, torch.float32)
+ and x.numel() > 0
+ and (
+ cache is None
+ or (
+ torch.is_tensor(cache)
+ and cache.dim() == 5
+ and cache.shape[:2] == x.shape[:2]
+ and cache.shape[3:] == x.shape[3:]
+ and cache.device == x.device
+ )
+ )
+ and (
+ norm is None
+ or _is_pixel_norm(norm)
+ or (getattr(norm, "channel_first", False) and _rms_params(norm) is not None)
+ )
+ )
+
+
+def rms_norm_act(
+ x: Any,
+ norm: Any,
+ act: bool = True,
+ *,
+ front: int = 0,
+ cache: Any = None,
+ in_bias: Any = None,
+ replicate_pad: Optional[tuple] = None,
+ back: int = 0,
+) -> Any:
+ """``cat([cache or zeros] -> front frames, silu?(rms_norm(x + in_bias)))`` along T, one channels-last_3d tensor.
+
+ ``replicate_pad=(ph, pw)``: HunyuanVideo-1.5 replicate padding instead (``cache`` must be None); ``back`` repeats the
+ last frame (LTX-2)."""
+ torch = _torch()
+ if replicate_pad is not None and cache is not None:
+ raise ValueError("replicate padding takes no cache")
+ if back and replicate_pad is None:
+ raise ValueError("back frames need replicate padding")
+ if not _rms_fusable(x, norm, cache):
+ y = rms_norm_reference(x, norm, act, in_bias)
+ if replicate_pad is not None:
+ import torch.nn.functional as F
+
+ ph, pw = replicate_pad
+ y = F.pad(y.float(), (pw, pw, ph, ph, front, back), mode = "replicate").to(y.dtype)
+ return y.contiguous(memory_format = torch.channels_last_3d)
+ if front:
+ import torch.nn.functional as F
+
+ n_cache = 0
+ if cache is not None:
+ cache = cache[:, :, -front:].to(y.dtype)
+ n_cache = cache.shape[2]
+ y = torch.cat([cache, y], dim = 2)
+ y = F.pad(y, (0, 0, 0, 0, front - n_cache, 0))
+ return y.contiguous(memory_format = torch.channels_last_3d)
+ k = _kernels()
+ mean_sq = norm is not None and _is_pixel_norm(norm)
+ if norm is not None and not mean_sq:
+ gamma, bias, has_bias, scale = _rms_params(norm)
+ else:
+ gamma, bias, has_bias, scale = x, x, False, 1.0
+ eps = float(getattr(norm, "eps", 0.0) or 0.0) if mean_sq else 0.0
+ b, c, t, h, w = x.shape
+ ph, pw = replicate_pad if replicate_pad is not None else (0, 0)
+ ho, wo = h + 2 * ph, w + 2 * pw
+ to = t + front + back
+ n_cache = 0 if cache is None else min(front, cache.shape[2])
+ cache_t = x if cache is None else cache[:, :, cache.shape[2] - n_cache :]
+ out = torch.empty((b, to, ho, wo, c), dtype = x.dtype, device = x.device)
+ block_c = _next_pow2(c)
+ block_p = max(1, min(128, 8192 // block_c))
+ grid = ((ho * wo + block_p - 1) // block_p, b * to)
+ k.rms_act[grid](
+ x, out, gamma, bias, in_bias if in_bias is not None else x, cache_t,
+ c, t, h, w, ho, wo, ph, pw, to, front, n_cache, scale, eps,
+ *x.stride(), *cache_t.stride(),
+ NORM = norm is not None, ACT = bool(act), HAS_BIAS = has_bias, HAS_IN_BIAS = in_bias is not None,
+ REPLICATE = replicate_pad is not None, MEAN_SQ = mean_sq, BLOCK_P = block_p, BLOCK_C = block_c,
+ num_warps = 4 if block_c * block_p <= 4096 else 8,
+ ) # fmt: skip
+ return out.permute(0, 4, 1, 2, 3)
+
+
+def _conv_kind(conv: Any) -> Optional[str]:
+ torch = _torch()
+ pad = getattr(conv, "_padding", None)
+ if pad is None or getattr(conv, "padding_mode", "zeros") != "zeros" or conv.groups != 1:
+ return None
+ if (
+ isinstance(conv, torch.nn.Conv3d)
+ and len(pad) == 6
+ and pad[5] == 0
+ and pad[0] == pad[1]
+ and pad[2] == pad[3]
+ ):
+ return "3d"
+ if (
+ isinstance(conv, torch.nn.Conv2d)
+ and len(pad) == 4
+ and pad[0] == pad[1]
+ and pad[2] == pad[3]
+ ):
+ return "2d"
+ return None
+
+
+def causal_conv(
+ conv: Any,
+ x: Any,
+ cache: Any = None,
+ *,
+ norm: Any = None,
+ act: bool = False,
+ in_bias: Any = None,
+ with_bias: bool = True,
+) -> tuple:
+ """Wan-lineage causal conv of ``silu?(rms?(x + in_bias))``; returns (out, new_cache).
+
+ ``new_cache`` is a compact copy so the cache never pins the whole input."""
+ import torch.nn.functional as F
+
+ kind = _conv_kind(conv)
+ bias = conv.bias if with_bias else None
+ if kind == "2d": # Qwen-Image-2.1: the one-frame specialisation, never cached
+ if cache is not None:
+ raise ValueError("2D causal conv takes no cache")
+ y = (
+ x
+ if (norm is None and not act and in_bias is None)
+ else rms_norm_act(x, norm, act, in_bias = in_bias)
+ )
+ pw, _, ph, _ = conv._padding
+ out = F.conv2d(y[:, :, 0], conv.weight, bias, conv.stride, (ph, pw), conv.dilation)
+ return out.unsqueeze(2), None
+ if kind != "3d":
+ raise ValueError("unsupported causal conv " + type(conv).__name__)
+ pw, _, ph, _, front, _ = conv._padding
+ kt = conv.kernel_size[0]
+ spatial = (0, ph, pw)
+ t = x.shape[2]
+ if front == 0:
+ y = (
+ x
+ if (norm is None and not act and in_bias is None)
+ else rms_norm_act(x, norm, act, in_bias = in_bias)
+ )
+ return F.conv3d(y, conv.weight, bias, conv.stride, spatial, conv.dilation), None
+ if (
+ cache is None
+ and t == 1
+ and front == kt - 1
+ and conv.stride[0] == 1
+ and conv.dilation[0] == 1
+ ):
+ y = rms_norm_act(x, norm, act, in_bias = in_bias)
+ weight = conv.weight[:, :, -1:]
+ return F.conv3d(y, weight, bias, conv.stride, spatial, conv.dilation), y
+ p = rms_norm_act(x, norm, act, front = front, cache = cache, in_bias = in_bias)
+ out = F.conv3d(p, conv.weight, bias, conv.stride, spatial, conv.dilation)
+ new_cache = p[:, :, -2:]
+ return out, (new_cache if p.shape[2] <= 2 else new_cache.clone())
+
+
+# Single-head VAE attention: head_dim > 256 has no flash / cuDNN kernel, so two GEMMs + fp32 softmax beat SDPA.
+
+_ATTN_SCORE_BYTES = 256 * 2**20
+
+
+def single_head_attention(q: Any, k: Any, v: Any) -> Any:
+ """softmax(q k^T / sqrt(d)) v for (B, L, D) half-precision q/k/v, query-chunked to bound the fp32 scores."""
+ torch = _torch()
+ b, length, d = q.shape
+ s_len = k.shape[1]
+ scale = d**-0.5
+ # keys padded to a multiple of 64 (cuBLAS slow path on odd extents); padded keys get probability exactly 0
+ s_pad = (s_len + 63) // 64 * 64
+ if s_pad != s_len:
+ kp = torch.zeros((b, s_pad, d), dtype = k.dtype, device = k.device)
+ vp = torch.zeros((b, s_pad, v.shape[-1]), dtype = v.dtype, device = v.device)
+ kp[:, :s_len] = k
+ vp[:, :s_len] = v
+ k, v = kp, vp
+ rows = int(max(64, min(length, _ATTN_SCORE_BYTES // max(1, b * s_pad * 4))) // 64 * 64)
+ kt = k.transpose(1, 2)
+ out = torch.empty((b, length, v.shape[-1]), dtype = v.dtype, device = v.device)
+ kern = _kernels()
+ for i in range(0, length, rows):
+ scores = torch.bmm(q[:, i : i + rows], kt, out_dtype = torch.float32)
+ r = scores.shape[1]
+ probs = torch.empty((b, r, s_pad), dtype = v.dtype, device = v.device)
+ kern.softmax_rows[(b * r,)](scores, probs, s_len, s_pad, scale, BLOCK = 2048, num_warps = 8)
+ del scores
+ torch.bmm(probs, v, out = out[:, i : i + rows])
+ return out
+
+
+def _mm_attention_ok(q: Any) -> bool:
+ torch = _torch()
+ return q.is_cuda and q.dtype in (torch.float16, torch.bfloat16) and q.shape[-1] > 256
+
+
+class FusedSingleHeadProcessor:
+ """``AttnProcessor2_0`` using :func:`single_head_attention`; anything else goes to ``fallback``."""
+
+ def __init__(self, fallback: Any):
+ self.fallback = fallback
+
+ def __call__(self, attn: Any, hidden_states: Any, encoder_hidden_states: Any = None, attention_mask: Any = None,
+ temb: Any = None, *args: Any, **kwargs: Any,) -> Any: # fmt: skip
+ if (
+ args
+ or kwargs
+ or encoder_hidden_states is not None
+ or attention_mask is not None
+ or attn.heads != 1
+ or attn.spatial_norm is not None
+ or attn.norm_q is not None
+ or attn.norm_k is not None
+ or hidden_states.ndim != 4
+ or getattr(attn, "_unsloth_vae_fused_failed", False)
+ or not _attention_input_ok(attn, hidden_states)
+ ):
+ return self.fallback(
+ attn, hidden_states, encoder_hidden_states, attention_mask, temb, *args, **kwargs
+ )
+ try:
+ return self._fused(attn, hidden_states)
+ except Exception as exc: # noqa: BLE001
+ # OOM falls back for this call only; anything else for good. Outside the handler so failed tensors are freed.
+ if not _is_oom(exc):
+ attn._unsloth_vae_fused_failed = True
+ return self.fallback(attn, hidden_states, encoder_hidden_states, attention_mask, temb)
+
+ @staticmethod
+ def _fused(attn: Any, hidden_states: Any) -> Any:
+ residual = hidden_states
+ b, c, h, w = hidden_states.shape
+ x = hidden_states.view(b, c, h * w).transpose(1, 2)
+ if attn.group_norm is not None:
+ x = attn.group_norm(x.transpose(1, 2)).transpose(1, 2)
+ q, k, v = attn.to_q(x), attn.to_k(x), attn.to_v(x)
+ o = single_head_attention(q, k, v)
+ o = attn.to_out[1](attn.to_out[0](o))
+ o = o.transpose(-1, -2).reshape(b, c, h, w)
+ if attn.residual_connection:
+ o = o + residual
+ return o / attn.rescale_output_factor
+
+
+def _attention_input_ok(attn: Any, hidden_states: Any) -> bool:
+ """Checked before any work, so an fp32 (upcast) or CPU VAE never pays for projections it then discards."""
+ torch = _torch()
+ width = getattr(getattr(attn, "to_q", None), "out_features", 0)
+ return (
+ hidden_states.is_cuda
+ and hidden_states.dtype in (torch.float16, torch.bfloat16)
+ and width > 256
+ )
+
+
+def install_attention_processors(part: Any) -> int:
+ n = 0
+ for m in part.modules():
+ if (
+ type(m).__name__ == "Attention"
+ and getattr(m, "heads", None) == 1
+ and callable(getattr(m, "processor", None))
+ ):
+ if not isinstance(m.processor, FusedSingleHeadProcessor):
+ m.processor = FusedSingleHeadProcessor(m.processor)
+ n += 1
+ return n
+
+
+def _fast_wan_attention(block: Any) -> Any:
+ import torch.nn.functional as F
+ def fast(x: Any) -> Any:
+ identity = x
+ b, c, t, h, w = x.shape
+ y = rms_norm_act(
+ x, block.norm, False
+ ) # (b, c, t, h, w) channels-last_3d: (b, t, h, w, c) in memory
+ y = y.permute(0, 2, 1, 3, 4).reshape(b * t, c, h, w) # a channels-last view, no copy
+ qkv = F.conv2d(y, block.to_qkv.weight, block.to_qkv.bias)
+ qkv = qkv.permute(0, 2, 3, 1).reshape(b * t, h * w, 3 * c)
+ q, k, v = qkv[..., :c], qkv[..., c : 2 * c], qkv[..., 2 * c :]
+ if not _mm_attention_ok(q):
+ return _stock_forward(block)(x)
+ o = single_head_attention(q, k, v) # (b*t, hw, c)
+ o = F.linear(o, block.proj.weight.view(c, c), block.proj.bias)
+ o = o.view(b, t, h, w, c).permute(0, 4, 1, 2, 3)
+ return add_bias_residual(
+ o.contiguous(memory_format = _torch().channels_last_3d), None, identity
+ )
+
+ return fast
+
+
+def conv_nobias(conv: Any, x: Any) -> Any:
+ """``conv(x)`` minus the bias; the caller adds ``conv.bias`` in a later fused pass."""
+ import torch.nn.functional as F
+
+ if getattr(conv, "padding_mode", "zeros") != "zeros":
+ raise ValueError("conv_nobias: padding_mode " + str(conv.padding_mode))
+ fn = F.conv2d if x.dim() == 4 else F.conv3d
+ return fn(x, conv.weight, None, conv.stride, conv.padding, conv.dilation, conv.groups)
+
+
+def _cl_contig(t: Any) -> bool:
+ torch = _torch()
+ fmt = torch.channels_last if t.dim() == 4 else torch.channels_last_3d
+ return t.is_contiguous(memory_format = fmt)
+
+
+def add_bias_residual(
+ out: Any,
+ out_bias: Any,
+ res: Any,
+ res_bias: Any = None,
+ scale: float = 1.0,
+) -> Any:
+ """``(out + out_bias + res + res_bias) / scale`` written into ``out`` (a fresh channels-last conv output)."""
+ torch = _torch()
+ k = _kernels()
+ if (
+ k is None
+ or not out.is_cuda
+ or out.dim() not in (4, 5)
+ or not _cl_contig(out)
+ or res.shape != out.shape
+ ):
+ if out_bias is not None:
+ out.add_(out_bias.view(1, -1, *([1] * (out.dim() - 2))).to(out.dtype))
+ if res_bias is not None:
+ res = res + res_bias.view(1, -1, *([1] * (res.dim() - 2))).to(res.dtype)
+ out.add_(res)
+ return out if scale == 1.0 else out.div_(scale)
+ r5, _ = _as5d(res)
+ o5, _ = _as5d(out)
+ b, c, t, h, w = o5.shape
+ pixels = b * t * h * w
+ block_c = min(128, _next_pow2(c))
+ block_p = max(1, 4096 // block_c)
+ grid = ((pixels + block_p - 1) // block_p, (c + block_c - 1) // block_c)
+ k.bias_residual[grid](
+ out, out_bias if out_bias is not None else out, res, res_bias if res_bias is not None else res,
+ pixels, c, t, h * w, w, *r5.stride(), 1.0 / float(scale),
+ HAS_OB = out_bias is not None, HAS_RB = res_bias is not None, SCALE = float(scale) != 1.0,
+ BLOCK_P = block_p, BLOCK_C = block_c, num_warps = 4,
+ ) # fmt: skip
+ return out
+
+
+def _is_oom(exc: BaseException) -> bool:
+ try:
+ if isinstance(exc, _torch().cuda.OutOfMemoryError):
+ return True
+ except Exception: # noqa: BLE001
+ pass
+ return "out of memory" in str(exc).lower()
+
+
+def _guard(
+ module: Any,
+ fast: Any,
+ stock: Any,
+ label: str,
+ logger: Any,
+ oom_stock: bool = False,
+) -> None:
+ """``module.forward = fast`` until it raises something ``stock`` does not; then stock for good.
+
+ Causal-cache state is snapshotted and restored before the stock retry so a half-failed fast path cannot desync
+ the decode. ``oom_stock``: an OOM retries stock for that call only."""
+
+ def forward(*args, **kwargs):
+ if getattr(module, "_unsloth_vae_fused_failed", False):
+ return stock(*args, **kwargs)
+ lists = [a for a in (*args, *kwargs.values()) if isinstance(a, list)]
+ snap = [a[:] for a in lists]
+ try:
+ return fast(*args, **kwargs)
+ except Exception as exc: # noqa: BLE001
+ for a, saved in zip(lists, snap):
+ a[:] = saved
+ if not _is_oom(exc):
+ out = stock(*args, **kwargs)
+ module._unsloth_vae_fused_failed = True
+ if logger is not None:
+ logger.warning(
+ "diffusion.vae_fused: fused %s failed, using the stock path: %s", label, exc
+ )
+ return out
+ if not oom_stock:
+ raise
+ return stock(
+ *args, **kwargs
+ ) # outside the handler, so the failed call's tensors are freed first
+
+ forward._unsloth_vae_fused = True
+ module.forward = forward
+
+
+def _stock_forward(module: Any) -> Any:
+ return type(module).forward.__get__(module)
+
+
+def _resnet2d_fusable(block: Any) -> bool:
+ torch = _torch()
+ return (
+ isinstance(getattr(block, "norm1", None), torch.nn.GroupNorm)
+ and isinstance(getattr(block, "norm2", None), torch.nn.GroupNorm)
+ and isinstance(getattr(block, "nonlinearity", None), torch.nn.SiLU)
+ and getattr(block, "upsample", None) is None
+ and getattr(block, "downsample", None) is None
+ and getattr(block, "time_emb_proj", None) is None
+ # "group" / "spatial" name the temb norm; without temb (a VAE) both run plain norm2 like "default"
+ and getattr(block, "time_embedding_norm", "default") in ("default", "group")
+ and isinstance(getattr(block, "conv1", None), torch.nn.Conv2d)
+ and isinstance(getattr(block, "conv2", None), torch.nn.Conv2d)
+ )
+
+
+def _fast_resnet2d(block: Any) -> Any:
+ def fast(
+ input_tensor: Any,
+ temb: Any = None,
+ *args: Any,
+ **kwargs: Any,
+ ) -> Any:
+ if temb is not None or args or kwargs or block.training:
+ return _stock_forward(block)(input_tensor, temb, *args, **kwargs)
+ h = group_norm_act(input_tensor, block.norm1, act = True)
+ h = conv_nobias(block.conv1, h)
+ h = group_norm_act(h, block.norm2, act = True, in_bias = block.conv1.bias)
+ h = conv_nobias(block.conv2, h)
+ shortcut = (
+ block.conv_shortcut(input_tensor) if block.conv_shortcut is not None else input_tensor
+ )
+ return add_bias_residual(
+ h, block.conv2.bias, shortcut, None, float(block.output_scale_factor)
+ )
+
+ return fast
+
+
+def _fast_norm(norm: Any, act: bool) -> Any:
+ def fast(x: Any) -> Any:
+ return group_norm_act(x, norm, act = act)
+
+ return fast
+
+
+def install_group_norm_vae(vae: Any, logger: Any = None) -> int:
+ """Patch every eligible ResnetBlock2D / GroupNorm of a 2-D (AutoencoderKL-family) VAE. Returns patch count."""
+ torch = _torch()
+ if vae is None or not runtime_ok():
+ return 0
+ n = 0
+ fused_act_norms = set()
+ for part_name in ("encoder", "decoder"):
+ part = getattr(vae, part_name, None)
+ if part is None:
+ continue
+ n += install_attention_processors(part)
+ norm_out = getattr(part, "conv_norm_out", None)
+ act = getattr(part, "conv_act", None)
+ if isinstance(norm_out, torch.nn.GroupNorm) and isinstance(act, torch.nn.SiLU):
+ fused_act_norms.add(id(norm_out))
+ _guard(
+ norm_out,
+ _fast_norm(norm_out, True),
+ lambda x, _n = norm_out: torch.nn.functional.silu(torch.nn.GroupNorm.forward(_n, x)),
+ "conv_norm_out",
+ logger,
+ )
+ act.forward = lambda x: x
+ act._unsloth_vae_fused = True
+ n += 1
+ for module in part.modules():
+ if getattr(module.forward, "_unsloth_vae_fused", False):
+ continue
+ if type(module).__name__ == "ResnetBlock2D" and _resnet2d_fusable(module):
+ _guard(
+ module, _fast_resnet2d(module), _stock_forward(module), "ResnetBlock2D", logger
+ )
+ n += 1
+ for module in part.modules():
+ if (
+ isinstance(module, torch.nn.GroupNorm)
+ and id(module) not in fused_act_norms
+ and not getattr(module.forward, "_unsloth_vae_fused", False)
+ ):
+ _guard(
+ module, _fast_norm(module, False), _stock_forward(module), "GroupNorm", logger
+ )
+ n += 1
+ return n
+
+
+def uninstall(vae: Any) -> None:
+ vae.__dict__.pop("_unsloth_vae_fused_installed", None)
+ vae.__dict__.pop("_unsloth_vae_fused_cl_weights", None)
+ for m in vae.modules():
+ if isinstance(getattr(m, "processor", None), FusedSingleHeadProcessor):
+ m.processor = m.processor.fallback
+ for attr in ("tiled_decode", "blend_v", "blend_h", "blend_t"):
+ if getattr(vae.__dict__.get(attr), "_unsloth_vae_fused", False):
+ delattr(vae, attr)
+ for module in vae.modules():
+ if getattr(getattr(module, "forward", None), "_unsloth_vae_fused", False) or getattr(
+ module, "_unsloth_vae_fused", False
+ ):
+ try:
+ del module.forward
+ except AttributeError:
+ pass
+ module.__dict__.pop("_unsloth_vae_fused", None)
+ module.__dict__.pop("_unsloth_vae_fused_failed", None)
+ module.__dict__.pop("prepare_causal_attention_mask", None)
+
+
+_WAN_VAES = frozenset({"AutoencoderKLWan", "AutoencoderKLQwenImage", "AutoencoderKLQwenImage21"})
+_CACHE_T = 2
+
+
+def _is_rms(norm: Any) -> bool:
+ return (
+ type(norm).__name__.endswith("RMS_norm")
+ and hasattr(norm, "gamma")
+ and hasattr(norm, "scale")
+ )
+
+
+def _wan_resblock_fusable(block: Any) -> bool:
+ torch = _torch()
+ return (
+ _is_rms(getattr(block, "norm1", None))
+ and _is_rms(getattr(block, "norm2", None))
+ and isinstance(getattr(block, "nonlinearity", None), torch.nn.SiLU)
+ and _conv_kind(getattr(block, "conv1", None)) is not None
+ and _conv_kind(getattr(block, "conv2", None)) is not None
+ )
+
+
+def _fast_wan_resblock(block: Any) -> Any:
+ import torch.nn.functional as F
+ def fast(
+ x: Any,
+ feat_cache: Any = None,
+ feat_idx: Any = None,
+ *args: Any,
+ **kwargs: Any,
+ ) -> Any:
+ if args or kwargs or block.training or (block.dropout.p and block.dropout.training):
+ return _stock_forward(block)(
+ x, feat_cache, feat_idx if feat_idx is not None else [0], *args, **kwargs
+ )
+ sc = block.conv_shortcut
+ if isinstance(sc, _torch().nn.Identity):
+ h = x
+ elif _conv_kind(sc) == "2d":
+ h = F.conv2d(x[:, :, 0], sc.weight, sc.bias, sc.stride, 0).unsqueeze(2)
+ else:
+ h = F.conv3d(x, sc.weight, sc.bias, sc.stride)
+ if feat_cache is not None:
+ idx = feat_idx[0]
+ out, new = causal_conv(
+ block.conv1, x, feat_cache[idx], norm = block.norm1, act = True, with_bias = False
+ )
+ feat_cache[idx] = new
+ feat_idx[0] += 1
+ idx = feat_idx[0]
+ out, new = causal_conv(
+ block.conv2, out, feat_cache[idx], norm = block.norm2, act = True, in_bias = block.conv1.bias,
+ with_bias = False,
+ ) # fmt: skip
+ feat_cache[idx] = new
+ feat_idx[0] += 1
+ else:
+ out, _ = causal_conv(block.conv1, x, None, norm = block.norm1, act = True, with_bias = False)
+ out, _ = causal_conv(
+ block.conv2,
+ out,
+ None,
+ norm = block.norm2,
+ act = True,
+ in_bias = block.conv1.bias,
+ with_bias = False,
+ )
+ return add_bias_residual(out, block.conv2.bias, h)
+
+ return fast
+
+
+def _fast_causal_conv(conv: Any) -> Any:
+ def fast(x: Any, cache_x: Any = None) -> Any:
+ return causal_conv(conv, x, cache_x)[0]
+
+ return fast
+
+
+def _fast_rms_act(norm: Any) -> Any:
+ def fast(x: Any) -> Any:
+ if x.dim() != 5:
+ return _torch().nn.functional.silu(type(norm).forward(norm, x))
+ return rms_norm_act(x, norm, True)
+
+ return fast
+
+
+def dup_up_add(out: Any, x: Any, dup: Any, first_chunk: bool) -> Any:
+ """``out + DupUp3D(x, first_chunk)`` added into ``out`` (a fresh tensor) without materialising the shortcut."""
+ k = _kernels()
+ ft, fs, rep = int(dup.factor_t), int(dup.factor_s), int(dup.repeats)
+ b, c, t_o, h_o, w_o = out.shape
+ t_off = ft - 1 if first_chunk else 0
+ if (x.shape[2] * ft - t_off, x.shape[3] * fs, x.shape[4] * fs) != (
+ t_o,
+ h_o,
+ w_o,
+ ) or c != dup.out_channels:
+ raise ValueError("dup_up_add: shape mismatch")
+ block_c = min(128, _next_pow2(c))
+ block_p = max(1, 4096 // block_c)
+ grid = ((h_o * w_o + block_p - 1) // block_p, b * t_o, (c + block_c - 1) // block_c)
+ k.dup_up_add[grid](
+ out, x, c, t_o, h_o, w_o, t_off, rep, *out.stride(), *x.stride(),
+ FT = ft, FS = fs, BLOCK_P = block_p, BLOCK_C = block_c, num_warps = 4,
+ ) # fmt: skip
+ return out
+
+
+def up_nearest2x(x: Any, interleave: bool) -> Any:
+ """Channels-last nearest 2x upsample; ``interleave`` also does Wan's time_conv frame interleave."""
+ torch = _torch()
+ k = _kernels()
+ b, c2, t, h, w = x.shape
+ c = c2 // 2 if interleave else c2
+ t_o = 2 * t if interleave else t
+ out = torch.empty((b * t_o, 2 * h, 2 * w, c), dtype = x.dtype, device = x.device)
+ block_c = min(128, _next_pow2(c))
+ block_p = max(1, 4096 // block_c)
+ grid = ((4 * h * w + block_p - 1) // block_p, b * t_o, (c + block_c - 1) // block_c)
+ k.up_nearest2x[grid](
+ x,
+ out,
+ c,
+ t_o,
+ h,
+ w,
+ *x.stride(),
+ INTERLEAVE = interleave,
+ BLOCK_P = block_p,
+ BLOCK_C = block_c,
+ num_warps = 4,
+ )
+ return out.permute(0, 3, 1, 2), t_o
+
+
+def _fast_wan_resample(mod: Any) -> Any:
+ import torch.nn.functional as F
+
+ conv = mod.resample[1]
+
+ def fast(
+ x: Any,
+ feat_cache: Any = None,
+ feat_idx: Any = None,
+ ) -> Any:
+ if feat_idx is None:
+ feat_idx = [0]
+ b = x.shape[0]
+ interleave = False
+ y = x
+ if mod.mode == "upsample3d" and feat_cache is not None:
+ idx = feat_idx[0]
+ if feat_cache[idx] is None:
+ feat_cache[idx] = "Rep"
+ else:
+ prev = None if isinstance(feat_cache[idx], str) else feat_cache[idx]
+ y, new = causal_conv(mod.time_conv, x, prev)
+ feat_cache[idx] = new
+ interleave = True
+ feat_idx[0] += 1
+ up, t_o = up_nearest2x(y, interleave)
+ out = F.conv2d(up, conv.weight, conv.bias, conv.stride, conv.padding)
+ return out.view(b, t_o, out.shape[1], out.shape[2], out.shape[3]).permute(0, 2, 1, 3, 4)
+
+ return fast
+
+
+def _wan_resample_ok(mod: Any) -> bool:
+ torch = _torch()
+ seq = getattr(mod, "resample", None)
+ return (
+ getattr(mod, "mode", None) in ("upsample2d", "upsample3d")
+ and isinstance(seq, torch.nn.Sequential)
+ and len(seq) == 2
+ and isinstance(seq[0], torch.nn.Upsample)
+ and seq[0].mode in ("nearest", "nearest-exact")
+ and tuple(float(f) for f in (seq[0].scale_factor or ())) == (2.0, 2.0)
+ and isinstance(seq[1], torch.nn.Conv2d)
+ and (mod.mode == "upsample2d" or _conv_kind(getattr(mod, "time_conv", None)) == "3d")
+ )
+
+
+def _fast_wan_residual_up(block: Any) -> Any:
+ def fast(
+ x: Any,
+ feat_cache: Any = None,
+ feat_idx: Any = None,
+ first_chunk: bool = False,
+ ) -> Any:
+ if feat_idx is None:
+ feat_idx = [0]
+ x_in = x # the stock ``x.clone()`` guards nothing: no block below writes its input
+ for resnet in block.resnets:
+ x = (
+ resnet(x, feat_cache = feat_cache, feat_idx = feat_idx)
+ if feat_cache is not None
+ else resnet(x)
+ )
+ if block.upsampler is not None:
+ if feat_cache is not None:
+ x = block.upsampler(x, feat_cache = feat_cache, feat_idx = feat_idx)
+ else:
+ x = block.upsampler(x)
+ if block.avg_shortcut is not None:
+ if x.data_ptr() == x_in.data_ptr():
+ x = x.clone()
+ x = dup_up_add(x, x_in, block.avg_shortcut, first_chunk)
+ return x
+
+ return fast
+
+
+def install_wan_vae(vae: Any, logger: Any = None) -> int:
+ """Patch the residual blocks, causal convs and output heads of a Wan-lineage VAE. Returns patch count."""
+ torch = _torch()
+ if vae is None or not runtime_ok():
+ return 0
+ n = 0
+ for part_name in ("encoder", "decoder"):
+ part = getattr(vae, part_name, None)
+ if part is None:
+ continue
+ norm_out = getattr(part, "norm_out", None)
+ act = getattr(part, "nonlinearity", None)
+ if _is_rms(norm_out) and isinstance(act, torch.nn.SiLU):
+ _guard(
+ norm_out,
+ _fast_rms_act(norm_out),
+ lambda x, _n = norm_out: torch.nn.functional.silu(type(_n).forward(_n, x)),
+ "norm_out",
+ logger,
+ )
+ act.forward = lambda x: x
+ act._unsloth_vae_fused = True
+ n += 1
+ for module in part.modules():
+ if "forward" in module.__dict__:
+ continue # already patched (by us or another speed path)
+ name = type(module).__name__
+ if name.endswith("ResidualBlock") and _wan_resblock_fusable(module):
+ _guard(
+ module,
+ _fast_wan_resblock(module),
+ _stock_forward(module),
+ "residual block",
+ logger,
+ )
+ n += 1
+ elif (
+ name.endswith("ResidualUpBlock")
+ and hasattr(module, "avg_shortcut")
+ and hasattr(module, "resnets")
+ ):
+ _guard(
+ module,
+ _fast_wan_residual_up(module),
+ _stock_forward(module),
+ "residual up block",
+ logger,
+ )
+ n += 1
+ elif name.endswith("Resample") and _wan_resample_ok(module):
+ _guard(
+ module, _fast_wan_resample(module), _stock_forward(module), "resample", logger
+ )
+ n += 1
+ elif (
+ name.endswith("AttentionBlock")
+ and _is_rms(getattr(module, "norm", None))
+ and isinstance(getattr(module, "to_qkv", None), torch.nn.Conv2d)
+ and isinstance(getattr(module, "proj", None), torch.nn.Conv2d)
+ ):
+ _guard(
+ module, _fast_wan_attention(module), _stock_forward(module), "attention", logger,
+ oom_stock = True,
+ ) # fmt: skip
+ n += 1
+ for module in part.modules():
+ if "forward" in module.__dict__:
+ continue
+ if type(module).__name__.endswith("CausalConv3d") and _conv_kind(module) is not None:
+ _guard(
+ module, _fast_causal_conv(module), _stock_forward(module), "causal conv", logger
+ )
+ n += 1
+ elif (
+ isinstance(module, torch.nn.Upsample)
+ and type(module).__name__.endswith("Upsample")
+ and module.mode in ("nearest", "nearest-exact")
+ ):
+ _guard(module, _fast_upsample(module), _stock_forward(module), "upsample", logger)
+ n += 1
+ if install_wan_tile_batch(vae, logger):
+ n += 1
+ return n
+
+
+def _fast_upsample(mod: Any) -> Any:
+ """Wan's nearest upsample in the input dtype: bit-identical to its fp32 round trip, without two copies."""
+
+ def fast(x: Any) -> Any:
+ import torch.nn.functional as F
+ return F.interpolate(
+ x,
+ mod.size,
+ mod.scale_factor,
+ mod.mode,
+ mod.align_corners,
+ recompute_scale_factor = mod.recompute_scale_factor,
+ )
+
+ return fast
+
+
+TILE_BATCH_ENV = "UNSLOTH_VAE_TILE_BATCH"
+_TILE_BATCH_MAX = 4
+# measured decode peak per 256 px Wan-2.1 tile (fp16, 4 frames, 96 full-res channels) ~= 1 GiB = 24x one activation
+_TILE_PEAK_FACTOR = 24
+
+
+def _tile_batch_cap(vae: Any, z: Any) -> int:
+ """Tiles per decoder call: env override, else what half the free VRAM holds, at most 4."""
+ raw = os.environ.get(TILE_BATCH_ENV, "").strip()
+ if raw.isdigit() and int(raw) > 0:
+ return int(raw)
+ try:
+ free, _ = _torch().cuda.mem_get_info(z.device)
+ norm_out = getattr(getattr(vae, "decoder", None), "norm_out", None)
+ c_last = int(norm_out.gamma.numel()) if norm_out is not None else 128
+ px = int(vae.tile_sample_min_height) * int(vae.tile_sample_min_width)
+ frames = int(getattr(vae, "temporal_compression_ratio", 4) or 4)
+ elem = next(vae.decoder.parameters()).element_size()
+ per_tile = _TILE_PEAK_FACTOR * px * frames * c_last * elem
+ return int(max(1, min(_TILE_BATCH_MAX, (free // 2) // max(1, per_tile))))
+ except Exception: # noqa: BLE001
+ return 1
+
+
+def _blend_weights(extent: int, device: Any) -> tuple:
+ torch = _torch()
+ # stock python-scalar blend weights as fp32 opmath, built on device: a host list is a sync H2D copy per seam
+ y = torch.arange(extent, dtype = torch.float64, device = device) / extent
+ wb = y.float()
+ wa = (1 - y).float()
+ return wa, wb
+
+
+def blend_seam(a: Any, b: Any, extent: int, dim: int) -> Any:
+ """Vectorised, bit-identical ``blend_v`` (dim=-2) / ``blend_h`` (dim=-1): one pass instead of ``extent`` x 4."""
+ torch = _torch()
+ extent = min(a.shape[dim], b.shape[dim], extent)
+ if extent <= 0:
+ return b
+ wa, wb = _blend_weights(extent, b.device)
+ shape = [1] * b.dim()
+ shape[dim] = extent
+ wa, wb = wa.view(shape), wb.view(shape)
+ sa = a.narrow(dim, a.shape[dim] - extent, extent)
+ sb = b.narrow(dim, 0, extent)
+ dt = b.dtype
+ pa = (sa.float() * wa).to(dt)
+ pb = (sb.float() * wb).to(dt)
+ sb.copy_((pa.float() + pb.float()).to(dt))
+ return b
+
+
+def _accepts_first_chunk(decoder: Any) -> bool:
+ import inspect
+ try:
+ return "first_chunk" in inspect.signature(decoder.forward).parameters
+ except (TypeError, ValueError):
+ return False
+
+
+def _wan_batched_tiled_decode(
+ self: Any,
+ z: Any,
+ return_dict: bool = True,
+ clamp: bool = True,
+) -> Any:
+ """``tiled_decode`` with same-shaped tiles decoded as one batch and vectorised seam blending."""
+ torch = _torch()
+ from diffusers.models.autoencoders.vae import DecoderOutput
+
+ _, _, num_frames, height, width = z.shape
+ ratio = self.spatial_compression_ratio
+ sample_height, sample_width = height * ratio, width * ratio
+ tmh, tmw = self.tile_sample_min_height // ratio, self.tile_sample_min_width // ratio
+ tsh, tsw = self.tile_sample_stride_height // ratio, self.tile_sample_stride_width // ratio
+ out_sh, out_sw = self.tile_sample_stride_height, self.tile_sample_stride_width
+ patch = getattr(self.config, "patch_size", None)
+ if patch is not None:
+ sample_height, sample_width = sample_height // patch, sample_width // patch
+ out_sh, out_sw = out_sh // patch, out_sw // patch
+ blend_h = self.tile_sample_min_height // patch - out_sh
+ blend_w = self.tile_sample_min_width // patch - out_sw
+ else:
+ blend_h = self.tile_sample_min_height - out_sh
+ blend_w = self.tile_sample_min_width - out_sw
+ first_chunk = _accepts_first_chunk(self.decoder)
+ rows_i = list(range(0, height, tsh))
+ cols_j = list(range(0, width, tsw))
+ groups: dict = {}
+ for i in rows_i:
+ for j in cols_j:
+ groups.setdefault((min(tmh, height - i), min(tmw, width - j)), []).append((i, j))
+ cap = _tile_batch_cap(self, z)
+ tiles: dict = {}
+
+ def decode_tiles(chunk: list) -> Any:
+ zt = torch.cat([z[:, :, :, i : i + tmh, j : j + tmw] for i, j in chunk], 0)
+ self.clear_cache()
+ frames = []
+ for k in range(num_frames):
+ self._conv_idx = [0]
+ tile = self.post_quant_conv(zt[:, :, k : k + 1])
+ kw = {"first_chunk": k == 0} if first_chunk else {}
+ frames.append(
+ self.decoder(tile, feat_cache = self._feat_map, feat_idx = self._conv_idx, **kw)
+ )
+ return torch.cat(frames, dim = 2)
+
+ for members in groups.values():
+ s = 0
+ while s < len(members):
+ chunk = members[s : s + cap]
+ try:
+ out = decode_tiles(chunk)
+ except Exception as exc: # noqa: BLE001
+ if len(chunk) == 1 or not _is_oom(exc):
+ raise
+ self.clear_cache()
+ torch.cuda.empty_cache()
+ cap = 1
+ continue
+ for n, key in enumerate(chunk):
+ tiles[key] = out[n : n + 1]
+ s += len(chunk)
+ self.clear_cache()
+ rows = [[tiles[(i, j)] for j in cols_j] for i in rows_i]
+ result_rows = []
+ for ri, row in enumerate(rows):
+ result_row = []
+ for ci, tile in enumerate(row):
+ if ri > 0:
+ tile = blend_seam(rows[ri - 1][ci], tile, blend_h, -2)
+ if ci > 0:
+ tile = blend_seam(row[ci - 1], tile, blend_w, -1)
+ result_row.append(tile[:, :, :, :out_sh, :out_sw])
+ result_rows.append(torch.cat(result_row, dim = -1))
+ dec = torch.cat(result_rows, dim = 3)[:, :, :, :sample_height, :sample_width]
+ if patch is not None:
+ from diffusers.models.autoencoders.autoencoder_kl_wan import unpatchify
+ dec = unpatchify(dec, patch_size = patch)
+ if (
+ clamp
+ ): # as the class's stock tiled_decode: Wan and Qwen-Image-2.1 clamp, Qwen-Image does not
+ dec = torch.clamp(dec, min = -1.0, max = 1.0)
+ if not return_dict:
+ return (dec,)
+ return DecoderOutput(sample = dec)
+
+
+@lru_cache(maxsize = None)
+def _stock_tiled_decode_clamps(cls: type) -> Optional[bool]:
+ """Whether ``cls.tiled_decode`` clamps to [-1, 1], read from the diffusers source; None (keep stock) if unreadable."""
+ import inspect
+ import re
+
+ try:
+ src = inspect.getsource(cls.tiled_decode)
+ except (OSError, TypeError, AttributeError):
+ return None
+ return re.search(r"\bclamp\(\s*dec\b", src) is not None
+
+
+def install_wan_tile_batch(vae: Any, logger: Any = None) -> bool:
+ if (
+ vae is None
+ or "tiled_decode" in vae.__dict__
+ or not callable(getattr(vae, "tiled_decode", None))
+ ):
+ return False
+ clamp = _stock_tiled_decode_clamps(type(vae))
+ if clamp is None:
+ return False
+ stock = _stock_forward_attr(vae, "tiled_decode")
+
+ def tiled_decode(z: Any, return_dict: bool = True) -> Any:
+ if z.shape[0] != 1 or getattr(vae, "_unsloth_vae_fused_failed", False):
+ return stock(z, return_dict = return_dict)
+ try:
+ return _wan_batched_tiled_decode(vae, z, return_dict, clamp)
+ except Exception as exc: # noqa: BLE001
+ if _is_oom(exc):
+ raise
+ vae._unsloth_vae_fused_failed = True
+ if logger is not None:
+ logger.warning(
+ "diffusion.vae_fused: batched tiled decode failed, using the stock loop: %s",
+ exc,
+ )
+ return stock(z, return_dict = return_dict)
+
+ tiled_decode._unsloth_vae_fused = True
+ vae.tiled_decode = tiled_decode
+ return True
+
+
+def _stock_forward_attr(obj: Any, name: str) -> Any:
+ return getattr(type(obj), name).__get__(obj)
+
+
+def _hv_conv_ok(conv: Any) -> bool:
+ torch = _torch()
+ inner = getattr(conv, "conv", None)
+ pad = getattr(conv, "time_causal_padding", None)
+ return (
+ isinstance(inner, torch.nn.Conv3d)
+ and getattr(conv, "pad_mode", None) == "replicate"
+ and pad is not None
+ and len(pad) == 6
+ and pad[0] == pad[1]
+ and pad[2] == pad[3]
+ and pad[5] == 0
+ and inner.padding == (0, 0, 0)
+ and inner.groups == 1
+ and inner.padding_mode == "zeros"
+ )
+
+
+def hv_conv(
+ conv: Any,
+ x: Any,
+ *,
+ norm: Any = None,
+ act: bool = False,
+ in_bias: Any = None,
+ with_bias: bool = True,
+) -> Any:
+ """``conv(pad_replicate(silu?(rms?(x + in_bias))))`` with the pad, norm and act in one channels-last pass."""
+ import torch.nn.functional as F
+
+ pw, _, ph, _, front, _ = conv.time_causal_padding
+ inner = conv.conv
+ y = rms_norm_act(x, norm, act, front = front, in_bias = in_bias, replicate_pad = (ph, pw))
+ return F.conv3d(
+ y, inner.weight, inner.bias if with_bias else None, inner.stride, 0, inner.dilation
+ )
+
+
+def _hv_resnet_ok(block: Any) -> bool:
+ torch = _torch()
+ return (
+ _is_rms(getattr(block, "norm1", None))
+ and _is_rms(getattr(block, "norm2", None))
+ and isinstance(getattr(block, "nonlinearity", None), torch.nn.SiLU)
+ and _hv_conv_ok(getattr(block, "conv1", None))
+ and _hv_conv_ok(getattr(block, "conv2", None))
+ )
+
+
+def _fast_hv_resnet(block: Any) -> Any:
+ import torch.nn.functional as F
+ def fast(x: Any) -> Any:
+ h = hv_conv(block.conv1, x, norm = block.norm1, act = True, with_bias = False)
+ h = hv_conv(
+ block.conv2,
+ h,
+ norm = block.norm2,
+ act = True,
+ in_bias = block.conv1.conv.bias,
+ with_bias = False,
+ )
+ sc = block.conv_shortcut
+ res = x if sc is None else F.conv3d(x, sc.weight, sc.bias, sc.stride, sc.padding)
+ return add_bias_residual(h, block.conv2.conv.bias, res)
+
+ return fast
+
+
+def _fast_hv_conv(conv: Any) -> Any:
+ def fast(x: Any) -> Any:
+ return hv_conv(conv, x)
+
+ return fast
+
+
+def _fast_hv_norm_act(norm: Any) -> Any:
+ def fast(x: Any) -> Any:
+ if x.dim() != 5:
+ return _torch().nn.functional.silu(type(norm).forward(norm, x))
+ return rms_norm_act(x, norm, True)
+
+ return fast
+
+
+def hv_upsample(up: Any, x: Any) -> Any:
+ """``HunyuanVideo15Upsample.forward``: conv, then depth-to-space + repeat_interleave shortcut + add in one pass."""
+ torch = _torch()
+ k = _kernels()
+ conv = up.conv
+ h = hv_conv(conv, x, with_bias = False)
+ b, cin, f, hh, ww = x.shape
+ cp = h.shape[1]
+ temporal = bool(up.add_temporal_upsample)
+ factor = 8 if temporal else 4
+ c = cp // factor
+ rep = int(up.repeats)
+ if cp % factor or (temporal and (f < 1 or rep % 2 or cin % 4)) or rep * cin != factor * c:
+ raise ValueError("hv_upsample: unexpected shapes")
+ fo = 1 + (f - 1) * 2 if temporal else f
+ out = torch.empty((b, fo, 2 * hh, 2 * ww, c), dtype = x.dtype, device = x.device)
+ block_c = min(128, _next_pow2(c))
+ block_p = max(1, 4096 // block_c)
+ grid = ((4 * hh * ww + block_p - 1) // block_p, b * fo, (c + block_c - 1) // block_c)
+ bias = conv.conv.bias
+ k.dcae_up_add[grid](
+ h, bias if bias is not None else h, x, out, c, cin, f, hh, ww, fo, rep, *h.stride(), *x.stride(),
+ TEMPORAL = temporal, HAS_HB = bias is not None, BLOCK_P = block_p, BLOCK_C = block_c, num_warps = 4,
+ ) # fmt: skip
+ return out.permute(0, 4, 1, 2, 3)
+
+
+def _hv_causal_mask(
+ n_frame: int,
+ n_hw: int,
+ dtype: Any,
+ device: Any,
+ batch_size: Any = None,
+) -> Any:
+ """Vectorised ``prepare_causal_attention_mask``: 0 where key frame <= query frame, else -inf."""
+ torch = _torch()
+ seq_len = n_frame * n_hw
+ mask = torch.zeros((seq_len, seq_len), dtype = dtype, device = device)
+ later = torch.ones((n_frame, n_frame), dtype = torch.bool, device = device).triu_(1)
+ mask.view(n_frame, n_hw, n_frame, n_hw).masked_fill_(later[:, None, :, None], float("-inf"))
+ if batch_size is not None:
+ mask = mask.unsqueeze(0).expand(batch_size, -1, -1)
+ return mask
+
+
+def install_hv15_vae(vae: Any, logger: Any = None) -> int:
+ torch = _torch()
+ if vae is None or not runtime_ok():
+ return 0
+ n = 0
+ for part_name in ("encoder", "decoder"):
+ part = getattr(vae, part_name, None)
+ if part is None:
+ continue
+ norm_out, act = getattr(part, "norm_out", None), getattr(part, "conv_act", None)
+ if (
+ _is_rms(norm_out)
+ and isinstance(act, torch.nn.SiLU)
+ and "forward" not in norm_out.__dict__
+ ):
+ _guard(
+ norm_out,
+ _fast_hv_norm_act(norm_out),
+ lambda x, _n = norm_out: torch.nn.functional.silu(type(_n).forward(_n, x)),
+ "norm_out",
+ logger,
+ )
+ act.forward = lambda x: x
+ act._unsloth_vae_fused = True
+ n += 1
+ for module in part.modules():
+ if "forward" in module.__dict__:
+ continue
+ name = type(module).__name__
+ if name.endswith("ResnetBlock") and _hv_resnet_ok(module):
+ _guard(module, _fast_hv_resnet(module), _stock_forward(module), "resnet", logger)
+ n += 1
+ for module in part.modules():
+ if "forward" in module.__dict__:
+ continue
+ name = type(module).__name__
+ if (
+ name.endswith("Upsample")
+ and _hv_conv_ok(getattr(module, "conv", None))
+ and hasattr(module, "repeats")
+ ):
+ _guard(
+ module,
+ lambda x, _m = module: hv_upsample(_m, x),
+ _stock_forward(module),
+ "upsample",
+ logger,
+ )
+ n += 1
+ elif name.endswith("AttnBlock") and callable(
+ getattr(module, "prepare_causal_attention_mask", None)
+ ):
+ module.prepare_causal_attention_mask = _hv_causal_mask
+ n += 1
+ for module in part.modules():
+ if "forward" in module.__dict__:
+ continue
+ if type(module).__name__.endswith("CausalConv3d") and _hv_conv_ok(module):
+ _guard(module, _fast_hv_conv(module), _stock_forward(module), "causal conv", logger)
+ n += 1
+ return n
+
+
+def _ltx_conv_ok(conv: Any) -> bool:
+ torch = _torch()
+ inner = getattr(conv, "conv", None)
+ return (
+ isinstance(inner, torch.nn.Conv3d)
+ and inner.padding_mode == "zeros"
+ and inner.padding[0] == 0
+ and inner.stride[0] == 1
+ and inner.dilation[0] == 1
+ and len(getattr(conv, "kernel_size", ())) == 3
+ and conv.kernel_size[0] % 2 == 1
+ )
+
+
+def ltx_conv(
+ conv: Any,
+ x: Any,
+ causal: bool,
+ *,
+ norm: Any = None,
+ act: bool = False,
+ in_bias: Any = None,
+ with_bias: bool = True,
+) -> Any:
+ """``LTX2VideoCausalConv3d.forward`` over ``silu?(pixel_norm?(x + in_bias))``, pad + norm + act in one pass."""
+ import torch.nn.functional as F
+
+ kt = conv.kernel_size[0]
+ front, back = (kt - 1, 0) if causal else ((kt - 1) // 2, (kt - 1) // 2)
+ inner = conv.conv
+ if kt == 1 and norm is None and not act and in_bias is None:
+ y = x
+ else:
+ y = rms_norm_act(
+ x, norm, act, front = front, back = back, in_bias = in_bias, replicate_pad = (0, 0)
+ )
+ return F.conv3d(
+ y,
+ inner.weight,
+ inner.bias if with_bias else None,
+ inner.stride,
+ inner.padding,
+ inner.dilation,
+ )
+
+
+def _ltx_resnet_ok(block: Any) -> bool:
+ torch = _torch()
+ return (
+ _is_pixel_norm(getattr(block, "norm1", None))
+ and _is_pixel_norm(getattr(block, "norm2", None))
+ and isinstance(getattr(block, "nonlinearity", None), torch.nn.SiLU)
+ and _ltx_conv_ok(getattr(block, "conv1", None))
+ and _ltx_conv_ok(getattr(block, "conv2", None))
+ and getattr(block, "per_channel_scale1", None) is None
+ and getattr(block, "per_channel_scale2", None) is None
+ and getattr(block, "scale_shift_table", None) is None
+ )
+
+
+def _fast_ltx_resnet(block: Any) -> Any:
+ import torch.nn.functional as F
+ def fast(
+ inputs: Any,
+ temb: Any = None,
+ generator: Any = None,
+ causal: bool = True,
+ ) -> Any:
+ if block.training and block.dropout.p:
+ return _stock_forward(block)(inputs, temb, generator, causal = causal)
+ h = ltx_conv(block.conv1, inputs, causal, norm = block.norm1, act = True, with_bias = False)
+ h = ltx_conv(
+ block.conv2,
+ h,
+ causal,
+ norm = block.norm2,
+ act = True,
+ in_bias = block.conv1.conv.bias,
+ with_bias = False,
+ )
+ res = inputs
+ if block.norm3 is not None:
+ res = block.norm3(res.movedim(1, -1)).movedim(-1, 1)
+ if block.conv_shortcut is not None:
+ sc = block.conv_shortcut
+ res = F.conv3d(res, sc.weight, sc.bias, sc.stride, sc.padding)
+ return add_bias_residual(h, block.conv2.conv.bias, res)
+
+ return fast
+
+
+def _fast_ltx_conv(conv: Any) -> Any:
+ def fast(hidden_states: Any, causal: bool = True) -> Any:
+ return ltx_conv(conv, hidden_states, causal)
+
+ return fast
+
+
+def _fast_pixel_norm_act(norm: Any) -> Any:
+ def fast(x: Any, channel_dim: Any = None) -> Any:
+ if x.dim() != 5 or channel_dim not in (None, 1):
+ return _torch().nn.functional.silu(type(norm).forward(norm, x, channel_dim))
+ return rms_norm_act(x, norm, True)
+
+ return fast
+
+
+def _ltx2_convs_fusable(vae: Any) -> bool:
+ """LTX-2's reflect-padded decoder fuses no conv: norm_out alone decoded 0.97-0.98x (L4 / A100 / RTX PRO 6000)."""
+ decoder = getattr(vae, "decoder", None)
+ return decoder is not None and any(
+ type(m).__name__ == "LTX2VideoCausalConv3d" and _ltx_conv_ok(m) for m in decoder.modules()
+ )
+
+
+def install_ltx2_vae(vae: Any, logger: Any = None) -> int:
+ torch = _torch()
+ if vae is None or not runtime_ok():
+ return 0
+ n = 0
+ # decoder only: the causal one-frame encode (i2v conditioning) measured 33 -> 35 ms fused, so it stays stock
+ for part_name in ("decoder",):
+ part = getattr(vae, part_name, None)
+ if part is None:
+ continue
+ norm_out, act = getattr(part, "norm_out", None), getattr(part, "conv_act", None)
+ if (
+ _is_pixel_norm(norm_out)
+ and isinstance(act, torch.nn.SiLU)
+ and getattr(part, "time_embedder", None) is None
+ and "forward" not in norm_out.__dict__
+ ):
+ _guard(
+ norm_out,
+ _fast_pixel_norm_act(norm_out),
+ lambda x, channel_dim = None, _n = norm_out: torch.nn.functional.silu(
+ type(_n).forward(_n, x, channel_dim)
+ ),
+ "norm_out",
+ logger,
+ )
+ act.forward = lambda x: x
+ act._unsloth_vae_fused = True
+ n += 1
+ for module in part.modules():
+ if "forward" in module.__dict__:
+ continue
+ if type(module).__name__.endswith("ResnetBlock3d") and _ltx_resnet_ok(module):
+ _guard(module, _fast_ltx_resnet(module), _stock_forward(module), "resnet", logger)
+ n += 1
+ for module in part.modules():
+ if "forward" in module.__dict__:
+ continue
+ if type(module).__name__ == "LTX2VideoCausalConv3d" and _ltx_conv_ok(module):
+ _guard(
+ module, _fast_ltx_conv(module), _stock_forward(module), "causal conv", logger
+ )
+ n += 1
+ return n
+
+
+def install_vectorised_blend(vae: Any) -> int:
+ """Replace per-row ``blend_v`` / ``blend_h`` / ``blend_t`` loops with the bit-identical :func:`blend_seam`."""
+ n = 0
+ for name, dim in (("blend_v", -2), ("blend_h", -1), ("blend_t", -3)):
+ if callable(getattr(vae, name, None)) and name not in vae.__dict__:
+ fn = lambda a, b, blend_extent, _d = dim: blend_seam(a, b, blend_extent, _d) # noqa: E731
+ fn._unsloth_vae_fused = True
+ setattr(vae, name, fn)
+ n += 1
+ return n
+
+
+_CL_WEIGHT_VAES = frozenset(
+ {"AutoencoderKL", "AutoencoderKLFlux2", "AutoencoderKLHunyuanVideo15", "AutoencoderKLLTX2Video"}
+)
+
+
+def channels_last_weights(vae: Any) -> int:
+ """The fused passes hand cuDNN channels-last activations; contiguous weights would be re-laid-out on every call."""
+ torch = _torch()
+ n = 0
+ for part in (getattr(vae, "encoder", None), getattr(vae, "decoder", None)):
+ if part is None:
+ continue
+ for m in part.modules():
+ w = getattr(m, "weight", None)
+ if (
+ isinstance(m, (torch.nn.Conv2d, torch.nn.Conv3d))
+ and isinstance(w, torch.Tensor)
+ and w.dim() in (4, 5)
+ ):
+ fmt = torch.channels_last if w.dim() == 4 else torch.channels_last_3d
+ if not w.is_contiguous(memory_format = fmt):
+ w.data = w.data.contiguous(memory_format = fmt)
+ n += 1
+ return n
+
+
+def _fuses_a_conv(vae: Any) -> bool:
+ torch = _torch()
+ return any(
+ getattr(m.__dict__.get("forward"), "_unsloth_vae_fused", False)
+ and any(isinstance(c, (torch.nn.Conv2d, torch.nn.Conv3d)) for c in m.modules())
+ for m in vae.modules()
+ )
+
+
+SUPPORTED_VAES = frozenset(
+ {
+ "AutoencoderKL",
+ "AutoencoderKLFlux2",
+ "AutoencoderKLWan",
+ "AutoencoderKLQwenImage",
+ "AutoencoderKLQwenImage21",
+ "AutoencoderKLHunyuanVideo15",
+ "AutoencoderKLLTX2Video",
+ }
+)
+
+
+def will_install(vae: Any) -> bool:
+ """Whether :func:`install` would engage on ``vae`` here (class covered, NVIDIA CUDA, usable Triton, not disabled)."""
+ return (
+ vae is not None
+ and type(vae).__name__ in SUPPORTED_VAES
+ and (type(vae).__name__ != "AutoencoderKLLTX2Video" or _ltx2_convs_fusable(vae))
+ and runtime_ok()
+ )
+
+
+def install(
+ vae: Any,
+ logger: Any = None,
+ level: str = "fused",
+) -> int:
+ """Install every fused path that applies to ``vae``'s class. Returns the number of patched modules."""
+ if vae is None or not will_install(vae):
+ return 0
+ done = getattr(vae, "_unsloth_vae_fused_installed", 0)
+ if done:
+ return done # idempotent: a dual-DiT family runs the speed layer once per expert over the same VAE
+ n = _install(vae, logger)
+ # Only where measured faster, and only when a fused pass feeds a conv (relaid weights alone slow LTX-2's decode).
+ if (
+ n
+ and type(vae).__name__ in _CL_WEIGHT_VAES
+ and os.environ.get("UNSLOTH_VAE_FUSED_CL_WEIGHTS", "1") != "0"
+ and _fuses_a_conv(vae)
+ ):
+ channels_last_weights(vae)
+ vae._unsloth_vae_fused_cl_weights = True
+ if n:
+ vae._unsloth_vae_fused_installed = n
+ if logger is not None:
+ logger.info("diffusion.vae_fused: fused %d VAE modules on %s", n, type(vae).__name__)
+ return n
+
+
+def _install(vae: Any, logger: Any = None) -> int:
+ name = type(vae).__name__
+ if name in ("AutoencoderKL", "AutoencoderKLFlux2"):
+ n = install_group_norm_vae(vae, logger)
+ return n + (install_vectorised_blend(vae) if n else 0)
+ if name in _WAN_VAES:
+ n = install_wan_vae(vae, logger)
+ return n + (install_vectorised_blend(vae) if n else 0)
+ if name == "AutoencoderKLHunyuanVideo15":
+ n = install_hv15_vae(vae, logger)
+ return n + (install_vectorised_blend(vae) if n else 0)
+ if name == "AutoencoderKLLTX2Video":
+ if not _ltx2_convs_fusable(vae):
+ return 0
+ n = install_ltx2_vae(vae, logger)
+ return n + (install_vectorised_blend(vae) if n else 0)
+ return 0
diff --git a/studio/backend/tests/test_diffusion_speed.py b/studio/backend/tests/test_diffusion_speed.py
index 67048d863f9..c7ffd5e980b 100644
--- a/studio/backend/tests/test_diffusion_speed.py
+++ b/studio/backend/tests/test_diffusion_speed.py
@@ -427,6 +427,7 @@ def test_speed_off_applies_nothing(monkeypatch):
assert applied == {
"channels_last": False,
"vae_fp16_decode": False,
+ "vae_fused": False,
"cudnn_benchmark": False,
"tf32": False,
"fused_qkv": False,
@@ -682,6 +683,22 @@ def test_eager_vae_decode_keeps_contiguous_weights(monkeypatch, tier):
assert applied["channels_last"] is False and pipe.vae.mem_format == torch.contiguous_format
[email protected]("tier", [SPEED_EAGER, SPEED_DEFAULT])
+def test_fused_vae_keeps_its_channels_last_weights(monkeypatch, tier):
+ torch = _stub_torch(monkeypatch)
+ monkeypatch.delenv(ds_mod.COMPILE_VAE_ENV, raising = False)
+ pipe = _Pipe(with_compile = True)
+
+ def fused(p, logger):
+ p.vae._unsloth_vae_fused_cl_weights = True
+ return True
+
+ monkeypatch.setattr(ds_mod, "_install_fused_vae", fused)
+ applied = apply_speed_optims(pipe, _target(), is_gguf = False, family = _family(), speed_mode = tier)
+ assert applied["vae_fused"] is True
+ assert applied["channels_last"] is True and pipe.vae.mem_format == torch.channels_last
+
+
@pytest.mark.parametrize(
"backend, device", [("rocm", "cuda"), ("mps", "mps"), ("xpu", "xpu"), ("cpu", "cpu")]
)
diff --git a/studio/backend/tests/test_diffusion_vae_fused.py b/studio/backend/tests/test_diffusion_vae_fused.py
new file mode 100644
index 00000000000..e6c8e403113
--- /dev/null
+++ b/studio/backend/tests/test_diffusion_vae_fused.py
@@ -0,0 +1,633 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""diffusion_vae_fused: bit-exact helpers (CPU), gating, and fused-vs-stock decode/encode on tiny random VAEs (CUDA)."""
+
+import pytest
+
+torch = pytest.importorskip("torch")
+
+from core.inference import diffusion_vae_fused as F # noqa: E402
+
+
+def _cuda_triton() -> bool:
+ return torch.cuda.is_available() and not getattr(torch.version, "hip", None) and F.runtime_ok()
+
+
+needs_cuda = pytest.mark.skipif(not _cuda_triton(), reason = "needs NVIDIA CUDA + Triton")
+
+
[email protected]("dtype", [torch.float16, torch.bfloat16, torch.float32])
[email protected]("extent", [1, 7, 32])
+def test_blend_seam_bit_identical_to_stock_loops(dtype, extent):
+ diffusers = pytest.importorskip("diffusers")
+ wan = diffusers.AutoencoderKLWan
+ g = torch.Generator().manual_seed(0)
+ a = torch.randn(1, 3, 2, 40, 44, generator = g).to(dtype)
+ b = torch.randn(1, 3, 2, 40, 44, generator = g).to(dtype)
+ for stock, dim in ((wan.blend_v, -2), (wan.blend_h, -1)):
+ assert torch.equal(
+ stock(None, a, b.clone(), extent), F.blend_seam(a, b.clone(), extent, dim)
+ )
+
+
+def test_hv_causal_mask_matches_stock():
+ mod = pytest.importorskip("diffusers.models.autoencoders.autoencoder_kl_hunyuanvideo15")
+ stock = mod.HunyuanVideo15AttnBlock.prepare_causal_attention_mask(
+ 3, 5, torch.float32, "cpu", batch_size = 2
+ )
+ assert torch.equal(stock, F._hv_causal_mask(3, 5, torch.float32, "cpu", batch_size = 2))
+
+
+def test_disabled_by_env(monkeypatch):
+ monkeypatch.setenv(F.VAE_FUSED_ENV, "0")
+ assert F.runtime_ok() is False
+ assert F.will_install(object()) is False
+
+
+def test_rocm_and_old_triton_keep_stock(monkeypatch):
+ # the #11801 kernels are gated off on ROCm (missing add_rn / rint, wrong fp16 GroupNorm on gfx1151): same here
+ monkeypatch.delenv(F.VAE_FUSED_ENV, raising = False)
+ monkeypatch.setattr(torch.cuda, "is_available", lambda: True)
+ monkeypatch.setattr(torch.version, "hip", "6.4.0", raising = False)
+ assert F.runtime_ok() is False
+ monkeypatch.setattr(torch.version, "hip", None, raising = False)
+ import sys
+ import types
+
+ F._triton_version_ok.cache_clear()
+ monkeypatch.setitem(sys.modules, "triton", types.SimpleNamespace(__version__ = "3.2.0"))
+ try:
+ assert F.runtime_ok() is False
+ finally:
+ F._triton_version_ok.cache_clear()
+
+
+def test_missing_triton_keeps_stock(monkeypatch):
+ import sys
+
+ monkeypatch.delenv(F.VAE_FUSED_ENV, raising = False)
+ monkeypatch.setattr(torch.cuda, "is_available", lambda: True)
+ monkeypatch.setattr(torch.version, "hip", None, raising = False)
+ F._triton_version_ok.cache_clear()
+ monkeypatch.setitem(sys.modules, "triton", None) # `import triton` raises ImportError
+ try:
+ assert F.runtime_ok() is False
+ assert F.will_install(type("AutoencoderKL", (), {})()) is False
+ finally:
+ F._triton_version_ok.cache_clear()
+
+
+def test_windows_without_msvc_keeps_stock(monkeypatch):
+ monkeypatch.setattr(F.sys, "platform", "win32")
+ F._toolchain_ok.cache_clear()
+ import core._msvc_env as msvc
+
+ monkeypatch.setattr(msvc, "crt_headers_reachable", lambda: False)
+ try:
+ assert F._toolchain_ok() is False
+ finally:
+ F._toolchain_ok.cache_clear()
+
+
[email protected]("dtype", [torch.bfloat16, torch.float32])
+def test_blend_seam_4d_matches_autoencoder_kl(dtype):
+ diffusers = pytest.importorskip("diffusers")
+ akl = diffusers.AutoencoderKL
+ g = torch.Generator().manual_seed(1)
+ a = torch.randn(1, 3, 40, 44, generator = g).to(dtype)
+ b = torch.randn(1, 3, 40, 44, generator = g).to(dtype)
+ assert torch.equal(akl.blend_v(None, a, b.clone(), 16), F.blend_seam(a, b.clone(), 16, -2))
+ assert torch.equal(akl.blend_h(None, a, b.clone(), 16), F.blend_seam(a, b.clone(), 16, -1))
+
+
+def test_install_is_noop_without_runtime(monkeypatch):
+ monkeypatch.setattr(F, "runtime_ok", lambda: False)
+
+ class AutoencoderKL(torch.nn.Module):
+ pass
+
+ assert F.install(AutoencoderKL()) == 0
+
+
+def test_uncovered_class_not_planned():
+ class AutoencoderKLMiniMaxH3(torch.nn.Module): # has its own fast path
+ pass
+
+ assert F.will_install(AutoencoderKLMiniMaxH3()) is False
+
+
+def test_speed_layer_skips_vae_compile_when_fused(monkeypatch):
+ from core.inference import diffusion_speed as S
+
+ monkeypatch.delenv(S.COMPILE_VAE_ENV, raising = False)
+ monkeypatch.setattr(F, "will_install", lambda vae: True)
+
+ class Pipe:
+ vae = object()
+ unet = None
+
+ assert S._fused_vae_planned(Pipe()) is True
+ assert S._vae_decode_compile_allowed(Pipe(), "max") is False
+ # forcing the compile keeps the old behaviour and skips the fused path
+ monkeypatch.setenv(S.COMPILE_VAE_ENV, "1")
+ assert S._fused_vae_planned(Pipe()) is False
+ assert S._install_fused_vae(Pipe(), None) is False
+
+
+def _psnr(a, b):
+ mse = (a.float() - b.float()).pow(2).mean().item()
+ return float("inf") if mse == 0 else 10 * torch.log10(torch.tensor(4.0 / mse)).item()
+
+
+def _decode(vae, z):
+ return vae.decode(z, return_dict = False)[0].float()
+
+
+def _check(
+ vae,
+ z,
+ *,
+ min_psnr,
+ encode_x = None,
+):
+ with torch.inference_mode():
+ ref = _decode(vae, z)
+ n = F.install(vae)
+ assert n > 0
+ out = _decode(vae, z)
+ assert not any(
+ getattr(m, "_unsloth_vae_fused_failed", False) for m in vae.modules()
+ ), "fused path fell back"
+ assert out.shape == ref.shape
+ assert _psnr(out, ref) > min_psnr, _psnr(out, ref)
+ if encode_x is not None:
+ F.uninstall(vae)
+ lat_ref = vae.encode(encode_x).latent_dist.mode().float()
+ F.install(vae)
+ lat = vae.encode(encode_x).latent_dist.mode().float()
+ assert (lat - lat_ref).norm() / lat_ref.norm() < 2e-2
+ F.uninstall(vae)
+ assert not any(
+ getattr(m.__dict__.get("forward"), "_unsloth_vae_fused", False) for m in vae.modules()
+ )
+ # the channels-last weights stay, and cuDNN may pick another algorithm for them (torch 2.7: 71 dB)
+ assert torch.equal(_decode(vae, z), ref) or _psnr(_decode(vae, z), ref) > 60
+
+
+@needs_cuda
[email protected]("dtype", [torch.bfloat16, torch.float16, torch.float32])
+def test_autoencoder_kl_fused_matches_stock(dtype):
+ from diffusers import AutoencoderKL
+
+ torch.manual_seed(0)
+ vae = AutoencoderKL(
+ block_out_channels = (32, 64), down_block_types = ("DownEncoderBlock2D",) * 2,
+ up_block_types = ("UpDecoderBlock2D",) * 2, latent_channels = 4, norm_num_groups = 32, layers_per_block = 1,
+ ).cuda().to(dtype).eval() # fmt: skip
+ vae.to(memory_format = torch.channels_last)
+ z = torch.randn(1, 4, 24, 20, device = "cuda", dtype = dtype)
+ x = torch.rand(1, 3, 48, 40, device = "cuda", dtype = dtype) * 2 - 1
+ _check(vae, z, min_psnr = 40 if dtype != torch.float32 else 70, encode_x = x)
+
+
+@needs_cuda
[email protected]("tiled", [False, True])
+def test_wan_fused_matches_stock_across_chunks(tiled, monkeypatch):
+ from diffusers import AutoencoderKLWan
+
+ torch.manual_seed(0)
+ vae = (
+ AutoencoderKLWan(base_dim = 32, z_dim = 4, dim_mult = [1, 2, 2, 2], num_res_blocks = 1)
+ .cuda()
+ .half()
+ .eval()
+ )
+ if tiled:
+ vae.enable_tiling(tile_sample_min_height = 64, tile_sample_min_width = 64, tile_sample_stride_height = 48,
+ tile_sample_stride_width = 48) # fmt: skip
+ monkeypatch.setenv(F.TILE_BATCH_ENV, "3")
+ z = torch.randn(1, 4, 4, 14, 18, device = "cuda", dtype = torch.float16)
+ x = torch.rand(1, 3, 5, 48, 64, device = "cuda", dtype = torch.float16) * 2 - 1
+ _check(vae, z, min_psnr = 45, encode_x = None if tiled else x)
+
+
+@needs_cuda
+def test_wan22_residual_decoder_fused_matches_stock():
+ from diffusers import AutoencoderKLWan
+
+ torch.manual_seed(0)
+ vae = AutoencoderKLWan(
+ base_dim = 32, decoder_base_dim = 32, z_dim = 8, dim_mult = [1, 2, 2, 2], num_res_blocks = 1, in_channels = 12,
+ out_channels = 12, is_residual = True, patch_size = 2, scale_factor_spatial = 16,
+ latents_mean = [0.0] * 8, latents_std = [1.0] * 8,
+ ).cuda().half().eval() # fmt: skip
+ z = torch.randn(1, 8, 3, 6, 8, device = "cuda", dtype = torch.float16)
+ _check(vae, z, min_psnr = 45)
+
+
+@needs_cuda
+def test_qwenimage_single_frame_fused_matches_stock():
+ from diffusers import AutoencoderKLQwenImage
+
+ torch.manual_seed(0)
+ vae = (
+ AutoencoderKLQwenImage(base_dim = 32, z_dim = 4, dim_mult = [1, 2, 2, 2], num_res_blocks = 1)
+ .cuda()
+ .bfloat16()
+ .eval()
+ )
+ z = torch.randn(1, 4, 1, 16, 12, device = "cuda", dtype = torch.bfloat16)
+ x = torch.rand(1, 3, 1, 64, 48, device = "cuda", dtype = torch.bfloat16) * 2 - 1
+ _check(vae, z, min_psnr = 40, encode_x = x)
+
+
+@needs_cuda
+def test_hunyuanvideo15_fused_matches_stock():
+ from diffusers import AutoencoderKLHunyuanVideo15
+
+ torch.manual_seed(0)
+ vae = AutoencoderKLHunyuanVideo15(
+ in_channels = 3, out_channels = 3, latent_channels = 8, block_out_channels = (32, 64, 64, 64, 64),
+ layers_per_block = 1, spatial_compression_ratio = 16, temporal_compression_ratio = 4,
+ ).cuda().bfloat16().eval() # fmt: skip
+ z = torch.randn(1, 8, 3, 4, 5, device = "cuda", dtype = torch.bfloat16)
+ _check(vae, z, min_psnr = 40)
+
+
+@needs_cuda
+def test_ltx2_fused_matches_stock():
+ from diffusers import AutoencoderKLLTX2Video
+
+ from core.inference.video_ltx2 import _VIDEO_VAE_CONFIG
+
+ torch.manual_seed(0)
+ cfg = dict(_VIDEO_VAE_CONFIG)
+ cfg.update(
+ latent_channels = 16, block_out_channels = (32, 64, 128, 128), decoder_block_out_channels = (32, 64, 64, 128),
+ layers_per_block = (1, 1, 1, 1, 1), decoder_layers_per_block = (1, 1, 1, 1, 1),
+ ) # fmt: skip
+ vae = AutoencoderKLLTX2Video(**cfg).cuda().bfloat16().eval()
+ z = torch.randn(1, 16, 3, 3, 4, device = "cuda", dtype = torch.bfloat16)
+ _check(vae, z, min_psnr = 40)
+
+
+@needs_cuda
+def test_ltx2_reflect_decoder_stays_stock(monkeypatch):
+ from diffusers import AutoencoderKLLTX2Video
+
+ from core.inference.video_ltx2 import _VIDEO_VAE_CONFIG
+
+ monkeypatch.delenv("UNSLOTH_VAE_FUSED_CL_WEIGHTS", raising = False)
+ cfg = dict(_VIDEO_VAE_CONFIG)
+ cfg.update(
+ latent_channels = 16, block_out_channels = (32, 64, 128, 128), decoder_block_out_channels = (32, 64, 64, 128),
+ layers_per_block = (1, 1, 1, 1, 1), decoder_layers_per_block = (1, 1, 1, 1, 1),
+ ) # fmt: skip
+ convs = lambda v: [m.weight for m in v.decoder.modules() if isinstance(m, torch.nn.Conv3d)] # noqa: E731
+ reflect = (
+ AutoencoderKLLTX2Video(**{**cfg, "decoder_spatial_padding_mode": "reflect"})
+ .cuda()
+ .bfloat16()
+ .eval()
+ )
+ assert F.will_install(reflect) is False
+ assert F.install(reflect) == 0
+ assert all(w.is_contiguous() for w in convs(reflect))
+ assert not any("forward" in m.__dict__ for m in reflect.modules())
+ zeros = AutoencoderKLLTX2Video(**cfg).cuda().bfloat16().eval()
+ assert F.install(zeros) > 0
+ assert all(w.is_contiguous(memory_format = torch.channels_last_3d) for w in convs(zeros))
+
+
+@needs_cuda
+def test_guard_falls_back_to_stock_for_good(monkeypatch):
+ from diffusers import AutoencoderKL
+
+ torch.manual_seed(0)
+ vae = AutoencoderKL(
+ block_out_channels = (32,), down_block_types = ("DownEncoderBlock2D",), up_block_types = ("UpDecoderBlock2D",),
+ latent_channels = 4, norm_num_groups = 32, layers_per_block = 1,
+ ).cuda().eval() # fmt: skip
+ z = torch.randn(1, 4, 8, 8, device = "cuda")
+ with torch.inference_mode():
+ ref = _decode(vae, z)
+ F.install(vae)
+
+ def boom(*a, **k):
+ raise RuntimeError("kernel exploded")
+
+ monkeypatch.setattr(F, "group_norm_act", boom)
+ out = _decode(vae, z)
+ assert torch.allclose(
+ out, ref, atol = 5e-3
+ ) # channels-last weights: another (TF32) cuDNN algorithm
+ assert any(getattr(m, "_unsloth_vae_fused_failed", False) for m in vae.modules())
+
+
+@needs_cuda
+def test_guard_restores_causal_cache_state_on_fallback(monkeypatch):
+ # a fast residual block that dies AFTER writing its first cache slot must not desync the stock retry
+ from diffusers import AutoencoderKLWan
+
+ torch.manual_seed(0)
+ vae = (
+ AutoencoderKLWan(base_dim = 32, z_dim = 4, dim_mult = [1, 2, 2, 2], num_res_blocks = 1)
+ .cuda()
+ .half()
+ .eval()
+ )
+ z = torch.randn(1, 4, 3, 8, 8, device = "cuda", dtype = torch.float16)
+ with torch.inference_mode():
+ ref = _decode(vae, z)
+ F.install(vae)
+ real = F.causal_conv
+ calls = {"n": 0}
+
+ def flaky(
+ conv,
+ x,
+ cache = None,
+ **kw,
+ ):
+ calls["n"] += 1
+ if (
+ calls["n"] == 2
+ ): # conv2 of the first block: conv1 already advanced feat_idx and wrote a slot
+ raise RuntimeError("boom")
+ return real(conv, x, cache, **kw)
+
+ monkeypatch.setattr(F, "causal_conv", flaky)
+ out = _decode(vae, z)
+ assert _psnr(out, ref) > 45
+
+
+def _single_head_attention(dim, device, dtype):
+ from diffusers.models.attention_processor import Attention, AttnProcessor2_0
+
+ attn = Attention(
+ dim, heads = 1, dim_head = dim, rescale_output_factor = 1.0, eps = 1e-6, norm_num_groups = 32, bias = True,
+ upcast_softmax = True, residual_connection = True, _from_deprecated_attn_block = True,
+ processor = AttnProcessor2_0(),
+ ).to(device, dtype).eval() # fmt: skip
+ attn.processor = F.FusedSingleHeadProcessor(attn.processor)
+ return attn
+
+
+def test_attention_fp32_or_cpu_input_goes_straight_to_stock(monkeypatch):
+ pytest.importorskip("diffusers")
+ attn = _single_head_attention(320, "cpu", torch.float32)
+ monkeypatch.setattr(
+ F.FusedSingleHeadProcessor, "_fused", staticmethod(lambda *a: pytest.fail("fused path ran"))
+ )
+ x = torch.randn(1, 320, 4, 4)
+ with torch.inference_mode():
+ out = attn(x)
+ assert out.shape == x.shape
+ assert not getattr(attn, "_unsloth_vae_fused_failed", False)
+
+
+@needs_cuda
[email protected]("oom", [True, False])
+def test_attention_oom_falls_back_for_that_call_only(monkeypatch, oom):
+ pytest.importorskip("diffusers")
+ attn = _single_head_attention(512, "cuda", torch.bfloat16)
+ x = torch.randn(1, 512, 16, 16, device = "cuda", dtype = torch.bfloat16)
+ with torch.inference_mode():
+ ref = attn.processor.fallback(attn, x)
+
+ def boom(*a, **k):
+ raise (
+ torch.cuda.OutOfMemoryError("CUDA out of memory")
+ if oom
+ else RuntimeError("kernel exploded")
+ )
+
+ monkeypatch.setattr(F, "single_head_attention", boom)
+ out = attn(x)
+ assert torch.equal(out, ref)
+ # an OOM keeps the fused path for the next call; a real failure retires it
+ assert getattr(attn, "_unsloth_vae_fused_failed", False) is (not oom)
+
+
+@needs_cuda
+def test_qwenimage_tiled_decode_is_not_clamped_like_stock(monkeypatch):
+ # diffusers' AutoencoderKLQwenImage.tiled_decode returns the raw decoder output (Wan's and Qwen-Image-2.1's clamp)
+ from diffusers import AutoencoderKLQwenImage
+
+ torch.manual_seed(0)
+ vae = (
+ AutoencoderKLQwenImage(base_dim = 32, z_dim = 4, dim_mult = [1, 2, 2, 2], num_res_blocks = 1)
+ .cuda()
+ .eval()
+ )
+ vae = vae.to(torch.bfloat16)
+ vae.enable_tiling(tile_sample_min_height = 64, tile_sample_min_width = 64, tile_sample_stride_height = 48,
+ tile_sample_stride_width = 48) # fmt: skip
+ monkeypatch.setenv(F.TILE_BATCH_ENV, "3")
+ with torch.no_grad():
+ vae.decoder.conv_out.weight.mul_(64) # the decode lands far outside [-1, 1]
+ z = torch.randn(1, 4, 1, 14, 18, device = "cuda", dtype = torch.bfloat16)
+ with torch.inference_mode():
+ ref = _decode(vae, z)
+ assert ref.abs().max() > 1.5 # the case under test exists
+ F.install(vae)
+ out = _decode(vae, z)
+ assert out.shape == ref.shape
+ assert out.abs().max() > 1.5
+ assert ((out - ref).norm() / ref.norm()).item() < 2e-2 # same values, unclamped
+
+
+def test_hv_causal_mask_allocates_nothing_quadratic_besides_the_output():
+ # the stock row loop allocates only the seq_len x seq_len mask (6.4 GB extra at 80k tokens otherwise)
+ from torch.utils._python_dispatch import TorchDispatchMode
+ from torch.utils._pytree import tree_leaves
+
+ class _Allocs(TorchDispatchMode):
+ def __init__(self):
+ super().__init__()
+ self.storages = []
+
+ def __torch_dispatch__(
+ self,
+ func,
+ types,
+ args = (),
+ kwargs = None,
+ ):
+ out = func(*args, **(kwargs or {}))
+ for t in tree_leaves(out):
+ if isinstance(t, torch.Tensor):
+ self.storages.append(
+ (t.untyped_storage().data_ptr(), t.untyped_storage().nbytes())
+ )
+ return out
+
+ mod = pytest.importorskip("diffusers.models.autoencoders.autoencoder_kl_hunyuanvideo15")
+ n_frame, n_hw = 6, 32
+ seq_len = n_frame * n_hw
+ rec = _Allocs()
+ with rec:
+ mask = F._hv_causal_mask(n_frame, n_hw, torch.bfloat16, "cpu", batch_size = 2)
+ own = mask.untyped_storage().data_ptr()
+ assert [nb for ptr, nb in rec.storages if ptr != own and nb >= seq_len * seq_len] == []
+ stock = mod.HunyuanVideo15AttnBlock.prepare_causal_attention_mask(
+ n_frame, n_hw, torch.bfloat16, "cpu", batch_size = 2
+ )
+ assert torch.equal(mask, stock) and mask.stride() == stock.stride()
+
+
+@needs_cuda
[email protected]("dtype", [torch.float16, torch.bfloat16, torch.float32])
+def test_blend_seam_on_cuda_is_bit_identical_and_host_free(dtype):
+ # a seam blend must not pull host tensors onto the device (a synchronous copy per seam)
+ from diffusers import AutoencoderKLWan
+ from torch.utils._python_dispatch import TorchDispatchMode
+ from torch.utils._pytree import tree_leaves
+
+ class _HostArgs(TorchDispatchMode):
+ def __init__(self):
+ super().__init__()
+ self.ops = []
+
+ def __torch_dispatch__(
+ self,
+ func,
+ types,
+ args = (),
+ kwargs = None,
+ ):
+ if any(
+ isinstance(t, torch.Tensor) and t.device.type == "cpu"
+ for t in tree_leaves((args, kwargs))
+ ):
+ self.ops.append(str(func))
+ return func(*args, **(kwargs or {}))
+
+ g = torch.Generator("cuda").manual_seed(0)
+ a = torch.randn(1, 3, 2, 40, 44, generator = g, device = "cuda").to(dtype)
+ b = torch.randn(1, 3, 2, 40, 44, generator = g, device = "cuda").to(dtype)
+ rec = _HostArgs()
+ with rec:
+ out = F.blend_seam(a, b.clone(), 17, -1)
+ assert rec.ops == []
+ assert torch.equal(AutoencoderKLWan.blend_h(None, a, b.clone(), 17), out)
+
+
+def test_wan_attention_oom_retries_stock_for_that_call_only(monkeypatch):
+ # the fused attention holds fp32 score chunks the stock SDPA never allocates, so its OOM must not abort the decode
+ diffusers = pytest.importorskip("diffusers")
+ monkeypatch.setattr(F, "runtime_ok", lambda: True)
+ torch.manual_seed(0)
+ vae = diffusers.AutoencoderKLWan(
+ base_dim = 32, z_dim = 4, dim_mult = [1, 2, 2, 2], num_res_blocks = 1
+ ).eval()
+ F.install_wan_vae(vae)
+ block = next(m for m in vae.decoder.modules() if type(m).__name__.endswith("AttentionBlock"))
+ x = torch.randn(1, block.norm.gamma.shape[0], 1, 4, 4)
+ with torch.inference_mode():
+ ref = type(block).forward(block, x)
+ monkeypatch.setattr(F, "_mm_attention_ok", lambda q: True)
+
+ def norm_5d(x, norm, act): # the fused norm is CUDA-only; per frame like the stock block
+ y = norm(x.permute(0, 2, 1, 3, 4).flatten(0, 1)).unflatten(0, x.shape[::2][:2])
+ return y.permute(0, 2, 1, 3, 4).contiguous(memory_format = torch.channels_last_3d)
+
+ monkeypatch.setattr(F, "rms_norm_act", norm_5d)
+
+ def oom(*a, **k):
+ raise torch.cuda.OutOfMemoryError("CUDA out of memory")
+
+ monkeypatch.setattr(F, "single_head_attention", oom)
+ out = block(x)
+ assert torch.equal(out, ref)
+ assert not getattr(block, "_unsloth_vae_fused_failed", False)
+
+
[email protected]("frames", [1, 2, 4])
+def test_causal_cache_is_a_compact_two_frame_copy(frames):
+ # a view would pin the whole (frames + 2)-frame conv input per cache slot until the next chunk
+ mod = pytest.importorskip("diffusers.models.autoencoders.autoencoder_kl_wan")
+ conv = mod.WanCausalConv3d(8, 8, 3, padding = 1).eval()
+ x = torch.randn(1, 8, frames, 4, 4)
+ cache = torch.randn(1, 8, 2, 4, 4)
+ with torch.inference_mode():
+ out, new = F.causal_conv(conv, x, cache)
+ ref = conv(x, cache)
+ stock = torch.cat([cache, x], dim = 2)[:, :, -2:]
+ assert torch.allclose(out, ref, atol = 1e-5)
+ assert torch.equal(new, stock)
+ assert new.untyped_storage().nbytes() == new.numel() * new.element_size()
+
+
+def test_pipeline_qkv_fuse_keeps_the_fused_vae_attention():
+ # SDXL's pipe-level fuse_qkv_projections() resets every VAE processor; the fused one must be re-wrapped around it
+ diffusers = pytest.importorskip("diffusers")
+ from diffusers.models.attention_processor import FusedAttnProcessor2_0
+ from diffusers.pipelines.pipeline_utils import StableDiffusionMixin
+
+ from core.inference import diffusion_speed as S
+
+ torch.manual_seed(0)
+ vae = diffusers.AutoencoderKL(
+ block_out_channels = (32,), down_block_types = ("DownEncoderBlock2D",), up_block_types = ("UpDecoderBlock2D",),
+ latent_channels = 4, norm_num_groups = 32, layers_per_block = 1,
+ ).eval() # fmt: skip
+ unet = diffusers.UNet2DConditionModel(
+ block_out_channels = (32, 64), layers_per_block = 1, sample_size = 8, in_channels = 4, out_channels = 4,
+ down_block_types = ("DownBlock2D", "CrossAttnDownBlock2D"), up_block_types = ("CrossAttnUpBlock2D", "UpBlock2D"),
+ cross_attention_dim = 32, norm_num_groups = 32,
+ ).eval() # fmt: skip
+
+ class Pipe(StableDiffusionMixin):
+ pass
+
+ pipe = Pipe()
+ pipe.vae, pipe.unet = vae, unet
+ assert F.install_attention_processors(vae) > 0
+ assert S._fuse_qkv(pipe, None) is True
+ procs = [m.processor for m in vae.modules() if type(m).__name__ == "Attention"]
+ assert procs and all(isinstance(p, F.FusedSingleHeadProcessor) for p in procs)
+ assert all(isinstance(p.fallback, FusedAttnProcessor2_0) for p in procs)
+ assert all(isinstance(p, FusedAttnProcessor2_0) for p in unet.attn_processors.values())
+
+
+def _variants(kernel) -> int:
+ caches = getattr(kernel, "device_caches", None)
+ if caches is not None: # Triton >= 3.2: {device: (kernel_cache, ...)}
+ return sum(len(c[0]) for c in caches.values())
+ return sum(len(c) for c in getattr(kernel, "cache", {}).values())
+
+
+@needs_cuda
+def test_shape_args_do_not_multiply_jit_variants():
+ # sizes must reuse the compiled kernel (else ~90 JIT variants on a tiled Wan decode); strides stay specialized
+ k = F._kernels()
+ for name, args in F._SHAPE_ARGS.items():
+ params = {p.name: p for p in getattr(k, name.lstrip("_")).params}
+ assert set(args) <= set(params), (name, set(args) - set(params))
+ assert all(params[a].do_not_specialize for a in args), name
+ cl = torch.channels_last_3d
+ seen = None
+ for t, h, w, front in (
+ (4, 16, 16, 2),
+ (1, 16, 16, 2),
+ (3, 13, 17, 1),
+ (1, 7, 9, 2),
+ (5, 1, 33, 0),
+ ):
+ x = torch.randn(1, 64, t, h, w, device = "cuda", dtype = torch.float16).contiguous(
+ memory_format = cl
+ )
+ cache = torch.randn(1, 64, 1, h, w, device = "cuda", dtype = torch.float16).contiguous(
+ memory_format = cl
+ )
+ out = F.rms_norm_act(x, None, act = True, front = front, cache = cache if front else None)
+ ref = torch.nn.functional.silu(x)
+ if front:
+ ref = torch.cat([torch.zeros_like(cache[:, :, : front - 1]), cache, ref], 2)
+ torch.testing.assert_close(
+ out, ref, atol = 2e-3, rtol = 2e-3
+ ) # fp32 silu vs fp16 opmath: last-ulp
+ seen = seen or _variants(k.rms_act)
+ assert _variants(k.rms_act) == seen, (t, h, w, front)