Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: Triton-fused VAE norms, caches and attention for every image and video VAE (1.7x to 6.3x decode) (#12078)
Daniel Hane07672bd2d19
diff --git a/studio/backend/core/inference/diffusion_speed.py b/studio/backend/core/inference/diffusion_speed.pyindex 8fe300e6a74..07f913d1e57 100644--- a/studio/backend/core/inference/diffusion_speed.py+++ b/studio/backend/core/inference/diffusion_speed.py@@ -437,6 +437,7 @@ def apply_speed_optims(applied = {"channels_last": False,"vae_fp16_decode": False,+ "vae_fused": False,"cudnn_benchmark": False,"tf32": False,"fp16_accum": False,@@ -459,6 +460,8 @@ def apply_speed_optims(applied["channels_last"] = _vae_channels_last(pipe, logger)# Near-lossless, not bit-identical, so never on "off" (returned above).applied["vae_fp16_decode"] = _video_vae_half_decode(pipe, target, family, logger)+ if on_cuda:+ applied["vae_fused"] = _install_fused_vae(pipe, logger)if on_cuda:applied["cudnn_benchmark"] = _enable_cudnn_benchmark(logger)@@ -511,8 +514,14 @@ def apply_speed_optims(eager_when_tiled = _vae_eager_when_tiled(pipe),)- if applied["channels_last"] and not _channels_last_decode_wins(- pipe, target, applied["compiled_vae_decode"], offload_active+ # Fused passes feed cuDNN channels-last activations; contiguous weights would be relaid out per call.+ fused_cl = bool(getattr(getattr(pipe, "vae", None), "_unsloth_vae_fused_cl_weights", False))+ if (+ applied["channels_last"]+ and not fused_cl+ and not _channels_last_decode_wins(+ pipe, target, applied["compiled_vae_decode"], offload_active+ )):applied["channels_last"] = not _vae_contiguous(pipe, logger)elif applied["compiled_vae_decode"] and not _channels_last_decode_wins(@@ -1108,8 +1117,34 @@ def vae_decode_compile_allowed(pipe: Any, speed_mode: str) -> bool:return _denoiser_unet(pipe) is None and _vae_decode_compile_allowed(pipe, speed_mode)+def _install_fused_vae(pipe: Any, logger: Any) -> bool:+ """Triton-fused VAE norm passes (diffusion_vae_fused), unless the env forces the VAE decode compile instead."""+ if os.environ.get(COMPILE_VAE_ENV, "").strip().lower() in _VAE_TRUE_TOKENS:+ return False+ try:+ from . import diffusion_vae_fused # noqa: PLC0415 - Triton import only on CUDA loads+ return diffusion_vae_fused.install(getattr(pipe, "vae", None), logger) > 0+ except Exception as exc: # noqa: BLE001 - optimisation only+ _warn(logger, "fused vae", exc)+ return False+++def _fused_vae_planned(pipe: Any) -> bool:+ if os.environ.get(COMPILE_VAE_ENV, "").strip().lower() in _VAE_TRUE_TOKENS:+ return False+ try:+ from . import diffusion_vae_fused # noqa: PLC0415+ return diffusion_vae_fused.will_install(getattr(pipe, "vae", None))+ except Exception: # noqa: BLE001+ return False++def _vae_decode_compile_allowed(pipe: Any, speed_mode: str) -> bool:- """U-Nets always; a DiT only on ``max`` (it costs a 60-70 s slower first render) or with the env forced on."""+ """U-Nets always; a DiT only on ``max`` (it costs a 60-70 s slower first render) or with the env forced on.++ Never when the fused eager VAE path engages: it is faster than the compiled decode, with no cold compile."""+ if _fused_vae_planned(pipe):+ return Falseif _denoiser_unet(pipe) is not None:return Trueraw = os.environ.get(COMPILE_VAE_ENV, "").strip().lower()@@ -1354,7 +1389,17 @@ def _fuse_qkv(pipe: Any, logger: Any) -> bool:fn = getattr(pipe, "fuse_qkv_projections", None)if callable(fn):try:+ vae = getattr(pipe, "vae", None)+ fused_vae_attn = callable(getattr(vae, "modules", None)) and any(+ type(getattr(m, "processor", None)).__name__ == "FusedSingleHeadProcessor"+ for m in vae.modules()+ )fn()+ if (+ fused_vae_attn+ ): # the pipe-level fuse resets every VAE processor to FusedAttnProcessor2_0+ from . import diffusion_vae_fused # noqa: PLC0415+ diffusion_vae_fused.install_attention_processors(vae)return Trueexcept Exception as exc: # noqa: BLE001 - optimisation only_warn(logger, "fuse_qkv_projections", exc)diff --git a/studio/backend/core/inference/diffusion_vae_fused.py b/studio/backend/core/inference/diffusion_vae_fused.pynew file mode 100644index 00000000000..6437a431cf4--- /dev/null+++ b/studio/backend/core/inference/diffusion_vae_fused.py@@ -0,0 +1,2113 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""Triton-fused GroupNorm / channel RMS norm for eagerly decoded image and video VAEs.++CUDA GroupNorm has no channels-last kernel, so every norm copies NHWC <-> NCHW; these kernels read any layout,+write channels-last with fp32 statistics. ``install(vae)`` patches instances (never classes), each falling back+to stock on a new exception. NVIDIA CUDA + Triton >= 3.3 only; ``UNSLOTH_VAE_FUSED=0`` disables.+"""++from __future__ import annotations++import os+import sys+import types+from functools import lru_cache+from typing import Any, Optional++VAE_FUSED_ENV = "UNSLOTH_VAE_FUSED"+_FALSE = ("0", "false", "no", "off")+_MIN_TRITON = (3, 3)+_MAX_C = 4096+++def fused_vae_disabled() -> bool:+ return os.environ.get(VAE_FUSED_ENV, "").strip().lower() in _FALSE+++@lru_cache(maxsize = 1)+def _triton_version_ok() -> bool:+ try:+ import re++ import triton++ m = re.match(r"(\d+)\.(\d+)", str(triton.__version__))+ return bool(m) and (int(m.group(1)), int(m.group(2))) >= _MIN_TRITON+ except Exception: # noqa: BLE001+ return False+++@lru_cache(maxsize = 1)+def _toolchain_ok() -> bool:+ if sys.platform != "win32":+ return True+ try:+ from .._msvc_env import crt_headers_reachable+ return bool(crt_headers_reachable())+ except Exception: # noqa: BLE001+ return True+++# Loop-bound-only args are not specialized (else ~70 JIT variants on a tiled Wan decode); strides and C stay+# specialized so channel-contiguous loads vectorize.+_SHAPE_ARGS = {+ "_rms_act": ("T", "H", "W", "Ho", "Wo", "ph", "pw", "To", "front", "n_cache"),+ "_bias_residual": ("P", "T", "HW", "W"),+ "_dup_up_add": ("T_o", "H_o", "W_o", "t_off"),+ "_up_nearest2x": ("T_o", "H", "W"),+}+++@lru_cache(maxsize = 1)+def _kernels() -> Optional[types.SimpleNamespace]:+ try:+ import triton+ import triton.language as tl+ except Exception: # noqa: BLE001+ return None+ globals().update(triton = triton, tl = tl)++ @triton.jit+ def _gn_partials(+ x_ptr, bias_ptr, pn_ptr, pmean_ptr, pm2_ptr,+ T, HW, W, n_chunks, sb, sc, st, sh, sw,+ C: tl.constexpr, G: tl.constexpr, CPG: tl.constexpr, BLOCK_P: tl.constexpr, HAS_BIAS: tl.constexpr+ ): # fmt: skip+ chunk = tl.program_id(0)+ bt = tl.program_id(1)+ t = bt % T+ b = bt // T+ p = chunk * BLOCK_P + tl.arange(0, BLOCK_P)+ pmask = p < HW+ hh = p // W+ ww = p - hh * W+ c = tl.arange(0, C)+ off = (+ b.to(tl.int64) * sb+ + t.to(tl.int64) * st+ + hh.to(tl.int64)[:, None] * sh+ + ww.to(tl.int64)[:, None] * sw+ + c.to(tl.int64)[None, :] * sc+ )+ v = tl.load(x_ptr + off, mask = pmask[:, None], other = 0.0).to(tl.float32)+ if HAS_BIAS:+ v = v + tl.load(bias_ptr + c).to(tl.float32)[None, :]+ v = tl.where(pmask[:, None], v, 0.0)+ n_pix = tl.sum(pmask.to(tl.float32), axis = 0)+ v3 = tl.reshape(v, (BLOCK_P, G, CPG))+ mean = tl.sum(tl.sum(v3, axis = 2), axis = 0) / (n_pix * CPG)+ d = tl.where(pmask[:, None, None], v3 - mean[None, :, None], 0.0)+ m2 = tl.sum(tl.sum(d * d, axis = 2), axis = 0)+ slot = bt.to(tl.int64) * n_chunks + chunk+ g = tl.arange(0, G)+ tl.store(pmean_ptr + slot * G + g, mean)+ tl.store(pm2_ptr + slot * G + g, m2)+ tl.store(pn_ptr + slot, n_pix * CPG)++ @triton.jit+ def _gn_combine(+ pn_ptr, pmean_ptr, pm2_ptr, mean_ptr, rstd_ptr, n_chunks, G, eps, BLOCK: tl.constexpr+ ):+ row = tl.program_id(0)+ g = row % G+ bt = (row // G).to(tl.int64)+ zero = tl.sum(tl.zeros([BLOCK], dtype = tl.float32), axis = 0)+ acc_n = zero+ acc_s = zero+ for k0 in range(0, n_chunks, BLOCK):+ k = k0 + tl.arange(0, BLOCK)+ km = k < n_chunks+ n = tl.load(pn_ptr + bt * n_chunks + k, mask = km, other = 0.0)+ mu = tl.load(pmean_ptr + (bt * n_chunks + k) * G + g, mask = km, other = 0.0)+ acc_n += tl.sum(n, axis = 0)+ acc_s += tl.sum(n * mu, axis = 0)+ mean = acc_s / acc_n+ acc_m2 = zero+ for k0 in range(0, n_chunks, BLOCK):+ k = k0 + tl.arange(0, BLOCK)+ km = k < n_chunks+ n = tl.load(pn_ptr + bt * n_chunks + k, mask = km, other = 0.0)+ mu = tl.load(pmean_ptr + (bt * n_chunks + k) * G + g, mask = km, other = 0.0)+ m2 = tl.load(pm2_ptr + (bt * n_chunks + k) * G + g, mask = km, other = 0.0)+ dm = mu - mean+ acc_m2 += tl.sum(m2 + n * dm * dm, axis = 0)+ tl.store(mean_ptr + row, mean)+ tl.store(rstd_ptr + row, 1.0 / tl.sqrt(acc_m2 / acc_n + eps))++ @triton.jit+ def _gn_apply(+ x_ptr, out_ptr, mean_ptr, rstd_ptr, w_ptr, b_ptr, ib_ptr,+ C, T, HW, W, G, sb, sc, st, sh, sw,+ CPG: tl.constexpr, ACT: tl.constexpr, HAS_IN_BIAS: tl.constexpr, BLOCK_P: tl.constexpr, BLOCK_C: tl.constexpr+ ): # fmt: skip+ pid_p = tl.program_id(0)+ bt = tl.program_id(1)+ pid_c = tl.program_id(2)+ t = bt % T+ b = bt // T+ p = pid_p * BLOCK_P + tl.arange(0, BLOCK_P)+ pmask = p < HW+ hh = p // W+ ww = p - hh * W+ c = pid_c * BLOCK_C + tl.arange(0, BLOCK_C)+ cmask = c < C+ m = pmask[:, None] & cmask[None, :]+ off = (+ b.to(tl.int64) * sb+ + t.to(tl.int64) * st+ + hh.to(tl.int64)[:, None] * sh+ + ww.to(tl.int64)[:, None] * sw+ + c.to(tl.int64)[None, :] * sc+ )+ v = tl.load(x_ptr + off, mask = m, other = 0.0).to(tl.float32)+ if HAS_IN_BIAS:+ v = v + tl.load(ib_ptr + c, mask = cmask, other = 0.0).to(tl.float32)[None, :]+ gidx = bt * G + c // CPG+ mu = tl.load(mean_ptr + gidx, mask = cmask, other = 0.0)+ rs = tl.load(rstd_ptr + gidx, mask = cmask, other = 0.0)+ gw = tl.load(w_ptr + c, mask = cmask, other = 0.0).to(tl.float32)+ gb = tl.load(b_ptr + c, mask = cmask, other = 0.0).to(tl.float32)+ a = rs * gw+ v = v * a[None, :] + (gb - mu * a)[None, :]+ if ACT:+ v = v / (1.0 + tl.exp(-v))+ out_off = (bt.to(tl.int64) * HW + p.to(tl.int64))[:, None] * C + c[None, :]+ tl.store(out_ptr + out_off, v.to(out_ptr.dtype.element_ty), mask = m)++ @triton.jit(do_not_specialize = _SHAPE_ARGS["_rms_act"])+ def _rms_act(+ x_ptr, out_ptr, w_ptr, b_ptr, ib_ptr, cache_ptr,+ C, T, H, W, Ho, Wo, ph, pw, To, front, n_cache, scale, eps,+ sb, sc, st, sh, sw, cb, cc, ct, ch, cw,+ NORM: tl.constexpr, ACT: tl.constexpr, HAS_BIAS: tl.constexpr, HAS_IN_BIAS: tl.constexpr,+ REPLICATE: tl.constexpr, MEAN_SQ: tl.constexpr, BLOCK_P: tl.constexpr, BLOCK_C: tl.constexpr+ ): # fmt: skip+ pid_p = tl.program_id(0)+ bt = tl.program_id(1)+ t_o = bt % To+ b = bt // To+ p = pid_p * BLOCK_P + tl.arange(0, BLOCK_P)+ pmask = p < Ho * Wo+ ho = p // Wo+ wo = p - ho * Wo+ if REPLICATE:+ hh = tl.minimum(tl.maximum(ho - ph, 0), H - 1)+ ww = tl.minimum(tl.maximum(wo - pw, 0), W - 1)+ else:+ hh = ho+ ww = wo+ c = tl.arange(0, BLOCK_C)+ cmask = c < C+ m = pmask[:, None] & cmask[None, :]+ out_off = ((b.to(tl.int64) * To + t_o) * (Ho * Wo) + p.to(tl.int64))[:, None] * C + c[+ None, :+ ]+ if (not REPLICATE) and t_o < front:+ tc = t_o - (front - n_cache)+ if tc >= 0:+ coff = (+ b.to(tl.int64) * cb+ + tc.to(tl.int64) * ct+ + hh.to(tl.int64)[:, None] * ch+ + ww.to(tl.int64)[:, None] * cw+ + c.to(tl.int64)[None, :] * cc+ )+ v = tl.load(cache_ptr + coff, mask = m, other = 0.0)+ tl.store(out_ptr + out_off, v.to(out_ptr.dtype.element_ty), mask = m)+ else:+ tl.store(+ out_ptr + out_off,+ tl.zeros([BLOCK_P, BLOCK_C], out_ptr.dtype.element_ty),+ mask = m,+ )+ else:+ t = tl.minimum(tl.maximum(t_o - front, 0), T - 1)+ off = (+ b.to(tl.int64) * sb+ + t.to(tl.int64) * st+ + hh.to(tl.int64)[:, None] * sh+ + ww.to(tl.int64)[:, None] * sw+ + c.to(tl.int64)[None, :] * sc+ )+ v = tl.load(x_ptr + off, mask = m, other = 0.0).to(tl.float32)+ if HAS_IN_BIAS:+ v = v + tl.load(ib_ptr + c, mask = cmask, other = 0.0).to(tl.float32)[None, :]+ v = tl.where(m, v, 0.0)+ if NORM:+ ss = tl.sum(v * v, axis = 1)+ if MEAN_SQ:+ # LTX PerChannelRMSNorm: x / sqrt(mean(x^2) + eps), no affine+ v = v / tl.sqrt(ss / C + eps)[:, None]+ else:+ # Wan-lineage RMS_norm: F.normalize(x) * sqrt(C) * gamma (+ bias)+ inv = 1.0 / tl.maximum(tl.sqrt(ss), 1e-12)+ gw = tl.load(w_ptr + c, mask = cmask, other = 0.0).to(tl.float32) * scale+ v = v * inv[:, None] * gw[None, :]+ if HAS_BIAS:+ v = v + tl.load(b_ptr + c, mask = cmask, other = 0.0).to(tl.float32)[None, :]+ if ACT:+ v = v / (1.0 + tl.exp(-v))+ tl.store(out_ptr + out_off, v.to(out_ptr.dtype.element_ty), mask = m)++ @triton.jit(do_not_specialize = _SHAPE_ARGS["_bias_residual"])+ def _bias_residual(+ o_ptr, ob_ptr, r_ptr, rb_ptr, P, C, T, HW, W, sb, sc, st, sh, sw, inv_scale,+ HAS_OB: tl.constexpr, HAS_RB: tl.constexpr, SCALE: tl.constexpr, BLOCK_P: tl.constexpr, BLOCK_C: tl.constexpr+ ): # fmt: skip+ p = tl.program_id(0) * BLOCK_P + tl.arange(0, BLOCK_P)+ c = tl.program_id(1) * BLOCK_C + tl.arange(0, BLOCK_C)+ pm = p < P+ cm = c < C+ m = pm[:, None] & cm[None, :]+ o_off = p.to(tl.int64)[:, None] * C + c[None, :]+ o = tl.load(o_ptr + o_off, mask = m, other = 0.0).to(tl.float32)+ if HAS_OB:+ o = o + tl.load(ob_ptr + c, mask = cm, other = 0.0).to(tl.float32)[None, :]+ ww = p % W+ rest = p // W+ hh = rest % (HW // W)+ bt = rest // (HW // W)+ t = bt % T+ b = bt // T+ r_off = (+ b.to(tl.int64) * sb + t.to(tl.int64) * st + hh.to(tl.int64) * sh + ww.to(tl.int64) * sw+ )[:, None] + c.to(tl.int64)[None, :] * sc+ r = tl.load(r_ptr + r_off, mask = m, other = 0.0).to(tl.float32)+ if HAS_RB:+ r = r + tl.load(rb_ptr + c, mask = cm, other = 0.0).to(tl.float32)[None, :]+ o = o + r+ if SCALE:+ o = o * inv_scale+ tl.store(o_ptr + o_off, o.to(o_ptr.dtype.element_ty), mask = m)++ @triton.jit+ def _dcae_up_add(+ h_ptr, hb_ptr, x_ptr, out_ptr,+ c, Cin, F, H, W, Fo, rep,+ hsb, hsc, hst, hsh, hsw, xsb, xsc, xst, xsh, xsw,+ TEMPORAL: tl.constexpr, HAS_HB: tl.constexpr, BLOCK_P: tl.constexpr, BLOCK_C: tl.constexpr+ ): # fmt: skip+ pid_p = tl.program_id(0)+ bf = tl.program_id(1)+ pid_c = tl.program_id(2)+ fo = bf % Fo+ b = bf // Fo+ Wo = 2 * W+ p = pid_p * BLOCK_P + tl.arange(0, BLOCK_P)+ pm = p < 4 * H * W+ ho = p // Wo+ wo = p - ho * Wo+ hh = ho // 2+ ww = wo // 2+ ph = (ho - hh * 2) * 2 + (wo - ww * 2) # r2i * 2 + r3i+ ci = pid_c * BLOCK_C + tl.arange(0, BLOCK_C)+ cm = ci < c+ m = pm[:, None] & cm[None, :]+ if TEMPORAL:+ if fo == 0:+ f = fo * 0+ pk = ph[:, None] * (2 * c) + ci[None, :]+ xk = ph[:, None] * (Cin // 4) + ci[None, :] // (rep // 2)+ else:+ f = 1 + (fo - 1) // 2+ r1 = (fo - 1) % 2+ pk = (r1 * 4 + ph)[:, None] * c + ci[None, :]+ xk = pk // rep+ else:+ f = fo+ pk = ph[:, None] * c + ci[None, :]+ xk = pk // rep+ pix_h = (+ b.to(tl.int64) * hsb+ + f.to(tl.int64) * hst+ + (hh.to(tl.int64) * hsh + ww.to(tl.int64) * hsw)[:, None]+ )+ v = tl.load(h_ptr + pix_h + pk.to(tl.int64) * hsc, mask = m, other = 0.0).to(tl.float32)+ if HAS_HB:+ v = v + tl.load(hb_ptr + pk, mask = m, other = 0.0).to(tl.float32)+ v = v.to(out_ptr.dtype.element_ty).to(+ tl.float32+ ) # the stock conv output is rounded before the add+ pix_x = (+ b.to(tl.int64) * xsb+ + f.to(tl.int64) * xst+ + (hh.to(tl.int64) * xsh + ww.to(tl.int64) * xsw)[:, None]+ )+ xv = tl.load(x_ptr + pix_x + xk.to(tl.int64) * xsc, mask = m, other = 0.0).to(tl.float32)+ o_off = (bf.to(tl.int64) * (4 * H * W) + p.to(tl.int64))[:, None] * c + ci[None, :]+ tl.store(out_ptr + o_off, (v + xv).to(out_ptr.dtype.element_ty), mask = m)++ @triton.jit+ def _softmax_rows(s_ptr, p_ptr, S, P_STRIDE, scale, BLOCK: tl.constexpr):+ row = tl.program_id(0).to(tl.int64)+ base = s_ptr + row * P_STRIDE+ m = tl.full([BLOCK], float("-inf"), tl.float32)+ l = tl.zeros([BLOCK], tl.float32)+ for k0 in range(0, S, BLOCK):+ cols = k0 + tl.arange(0, BLOCK)+ x = tl.load(base + cols, mask = cols < S, other = float("-inf")) * scale+ m_new = tl.maximum(m, x)+ alpha = tl.where(m_new == float("-inf"), 0.0, tl.exp(m - m_new))+ l = l * alpha + tl.where(x == float("-inf"), 0.0, tl.exp(x - m_new))+ m = m_new+ m_row = tl.max(m, axis = 0)+ l_row = tl.sum(l * tl.where(m == float("-inf"), 0.0, tl.exp(m - m_row)), axis = 0)+ inv = 1.0 / l_row+ for k0 in range(0, P_STRIDE, BLOCK):+ cols = k0 + tl.arange(0, BLOCK)+ x = tl.load(base + cols, mask = cols < S, other = float("-inf")) * scale+ p = tl.where(cols < S, tl.exp(x - m_row) * inv, 0.0)+ tl.store(+ p_ptr + row * P_STRIDE + cols, p.to(p_ptr.dtype.element_ty), mask = cols < P_STRIDE+ )++ @triton.jit(do_not_specialize = _SHAPE_ARGS["_dup_up_add"])+ def _dup_up_add(+ o_ptr, x_ptr, C, T_o, H_o, W_o, t_off, repeats,+ osb, osc, ost, osh, osw, xsb, xsc, xst, xsh, xsw,+ FT: tl.constexpr, FS: tl.constexpr, BLOCK_P: tl.constexpr, BLOCK_C: tl.constexpr+ ): # fmt: skip+ pid_p = tl.program_id(0)+ bt = tl.program_id(1)+ pid_c = tl.program_id(2)+ t = bt % T_o+ b = bt // T_o+ p = pid_p * BLOCK_P + tl.arange(0, BLOCK_P)+ pm = p < H_o * W_o+ ho = p // W_o+ wo = p - ho * W_o+ c = pid_c * BLOCK_C + tl.arange(0, BLOCK_C)+ cm = c < C+ m = pm[:, None] & cm[None, :]+ tt = t + t_off+ ti = tt // FT+ a = tt - ti * FT+ hi = ho // FS+ i = ho - hi * FS+ wi = wo // FS+ j = wo - wi * FS+ k = ((c[None, :] * FT + a) * FS + i[:, None]) * FS + j[:, None]+ src = k // repeats+ x_off = (b.to(tl.int64) * xsb + ti.to(tl.int64) * xst + (hi.to(tl.int64) * xsh + wi.to(tl.int64) * xsw)[:, None]+ + src.to(tl.int64) * xsc) # fmt: skip+ o_off = (b.to(tl.int64) * osb + t.to(tl.int64) * ost + (ho.to(tl.int64) * osh + wo.to(tl.int64) * osw)[:, None]+ + c.to(tl.int64)[None, :] * osc) # fmt: skip+ xv = tl.load(x_ptr + x_off, mask = m, other = 0.0).to(tl.float32)+ ov = tl.load(o_ptr + o_off, mask = m, other = 0.0).to(tl.float32)+ tl.store(o_ptr + o_off, (ov + xv).to(o_ptr.dtype.element_ty), mask = m)++ @triton.jit(do_not_specialize = _SHAPE_ARGS["_up_nearest2x"])+ def _up_nearest2x(+ x_ptr, out_ptr, C, T_o, H, W, xsb, xsc, xst, xsh, xsw,+ INTERLEAVE: tl.constexpr, BLOCK_P: tl.constexpr, BLOCK_C: tl.constexpr+ ): # fmt: skip+ # INTERLEAVE: x is Wan time_conv output (B, 2C, T_o/2, H, W); frame 2t+j takes channels [jC, (j+1)C).+ pid_p = tl.program_id(0)+ bt = tl.program_id(1)+ pid_c = tl.program_id(2)+ t = bt % T_o+ b = bt // T_o+ p = pid_p * BLOCK_P + tl.arange(0, BLOCK_P)+ pm = p < 4 * H * W+ ho = p // (2 * W)+ wo = p - ho * (2 * W)+ c = pid_c * BLOCK_C + tl.arange(0, BLOCK_C)+ cm = c < C+ m = pm[:, None] & cm[None, :]+ if INTERLEAVE:+ ti = t // 2+ ch = (t - ti * 2) * C + c+ else:+ ti = t+ ch = c+ x_off = (b.to(tl.int64) * xsb + ti.to(tl.int64) * xst+ + ((ho // 2).to(tl.int64) * xsh + (wo // 2).to(tl.int64) * xsw)[:, None] + ch.to(tl.int64)[None, :] * xsc) # fmt: skip+ v = tl.load(x_ptr + x_off, mask = m, other = 0.0)+ o_off = (bt.to(tl.int64) * (4 * H * W) + p.to(tl.int64))[:, None] * C + c[None, :]+ tl.store(out_ptr + o_off, v, mask = m)++ return types.SimpleNamespace(+ gn_partials = _gn_partials,+ gn_combine = _gn_combine,+ gn_apply = _gn_apply,+ rms_act = _rms_act,+ bias_residual = _bias_residual,+ dcae_up_add = _dcae_up_add,+ softmax_rows = _softmax_rows,+ dup_up_add = _dup_up_add,+ up_nearest2x = _up_nearest2x,+ )+++def _next_pow2(n: int) -> int:+ return 1 << max(0, int(n) - 1).bit_length()+++def runtime_ok() -> bool:+ """NVIDIA CUDA + a usable Triton; everything else keeps the stock path."""+ if fused_vae_disabled():+ return False+ try:+ import torch+ except Exception: # noqa: BLE001+ return False+ if not torch.cuda.is_available() or getattr(torch.version, "hip", None):+ return False+ return _triton_version_ok() and _toolchain_ok() and _kernels() is not None+++def _as5d(x: Any) -> tuple:+ """(tensor viewed as B,C,T,H,W, was_4d)."""+ if x.dim() == 4:+ return x.unsqueeze(2), True+ if x.dim() == 3: # (B, C, L): attention's GroupNorm input+ return x.unsqueeze(2).unsqueeze(2), True+ return x, False+++def group_norm_reference(+ x: Any,+ norm: Any,+ act: bool,+ in_bias: Any = None,+) -> Any:+ import torch.nn.functional as F++ if in_bias is not None:+ x = x + in_bias.view(1, -1, *([1] * (x.dim() - 2))).to(x.dtype)+ y = F.group_norm(x, norm.num_groups, norm.weight, norm.bias, norm.eps)+ return F.silu(y) if act else y+++def _gn_fusable(x: Any, norm: Any) -> bool:+ c = x.shape[1] if x.dim() >= 2 else 0+ g = int(getattr(norm, "num_groups", 0) or 0)+ return (+ x.is_cuda+ and x.dim() in (3, 4, 5)+ and g > 0+ and c % g == 0+ and c & (c - 1) == 0+ and g & (g - 1) == 0+ and c <= 1024+ and getattr(norm, "affine", True)+ and norm.weight is not None+ and x.dtype in (_torch().float16, _torch().bfloat16, _torch().float32)+ and x.numel() > 0+ )+++def _torch():+ import torch+ return torch+++def group_norm_act(+ x: Any,+ norm: Any,+ act: bool = True,+ in_bias: Any = None,+) -> Any:+ """``silu?(group_norm(x + in_bias))`` for 3/4/5-D ``x``; channels-last output; 5-D stats span T like nn.GroupNorm."""+ torch = _torch()+ if not _gn_fusable(x, norm):+ return group_norm_reference(x, norm, act, in_bias)+ k = _kernels()+ ndim = x.dim()+ x5, _ = _as5d(x)+ b, c, t, h, w = x5.shape+ # nn.GroupNorm pools the whole (T, H, W) extent: flatten T into H when strides allow, else fall back+ if t > 1:+ if x5.stride(2) == h * x5.stride(3):+ x5 = x5.as_strided(+ (b, c, 1, t * h, w), (x5.stride(0), x5.stride(1), 0, x5.stride(3), x5.stride(4))+ )+ b, c, t, h, w = x5.shape+ else:+ return group_norm_reference(x, norm, act, in_bias)+ g = int(norm.num_groups)+ cpg = c // g+ hw = h * w+ sb, sc, st, sh, sw = x5.stride()+ ib = in_bias if in_bias is not None else x5+ budget = 4096 if x5.element_size() == 4 else 8192+ block_p = max(1, min(256, budget // c))+ n_chunks = (hw + block_p - 1) // block_p+ rows = b * t+ pmean, pm2 = torch.empty((2, rows * n_chunks * g), dtype = torch.float32, device = x.device)+ pn = torch.empty(rows * n_chunks, dtype = torch.float32, device = x.device)+ k.gn_partials[(n_chunks, rows)](+ x5, ib, pn, pmean, pm2, t, hw, w, n_chunks, sb, sc, st, sh, sw,+ C = c, G = g, CPG = cpg, BLOCK_P = block_p, HAS_BIAS = in_bias is not None, num_warps = 4,+ ) # fmt: skip+ mean = torch.empty(rows * g, dtype = torch.float32, device = x.device)+ rstd = torch.empty(rows * g, dtype = torch.float32, device = x.device)+ k.gn_combine[(rows * g,)](+ pn, pmean, pm2, mean, rstd, n_chunks, g, float(norm.eps), BLOCK = 1024, num_warps = 4+ )+ out = torch.empty((b, t, h, w, c), dtype = x.dtype, device = x.device)+ block_c = min(128, _next_pow2(c))+ block_p2 = max(1, 4096 // block_c)+ grid = ((hw + block_p2 - 1) // block_p2, rows, (c + block_c - 1) // block_c)+ k.gn_apply[grid](+ x5, out, mean, rstd, norm.weight, norm.bias, ib, c, t, hw, w, g, sb, sc, st, sh, sw,+ CPG = cpg, ACT = bool(act), HAS_IN_BIAS = in_bias is not None, BLOCK_P = block_p2, BLOCK_C = block_c,+ num_warps = 4,+ ) # fmt: skip+ out = out.permute(0, 4, 1, 2, 3) # B, C, T', H', W'+ shape = x.shape+ if ndim == 5:+ return out.reshape(b, c, *shape[2:]) if out.shape != shape else out+ if ndim == 4:+ return out[:, :, 0]+ return out[:, :, 0].reshape(shape) # (B, C, L) view over the (B, L, C) buffer+++def rms_norm_reference(+ x: Any,+ norm: Any,+ act: bool,+ in_bias: Any = None,+) -> Any:+ import torch.nn.functional as F++ if in_bias is not None:+ x = x + in_bias.view(1, -1, *([1] * (x.dim() - 2))).to(x.dtype)+ y = x if norm is None else norm(x)+ return F.silu(y) if act else y+++def _is_pixel_norm(norm: Any) -> bool:+ return type(norm).__name__ == "PerChannelRMSNorm" and getattr(norm, "channel_dim", 1) == 1+++def _rms_params(norm: Any) -> Optional[tuple]:+ torch = _torch()+ if _is_pixel_norm(norm):+ return None, None, False, 1.0+ gamma = norm.gamma.reshape(-1)+ bias = norm.bias+ has_bias = isinstance(bias, torch.Tensor)+ if not has_bias and float(bias) != 0.0:+ return None+ return gamma, (bias.reshape(-1) if has_bias else gamma), has_bias, float(norm.scale)+++def _rms_fusable(x: Any, norm: Any, cache: Any) -> bool:+ torch = _torch()+ c = x.shape[1]+ return (+ x.is_cuda+ and x.dim() == 5+ and c <= _MAX_C+ and x.dtype in (torch.float16, torch.bfloat16, torch.float32)+ and x.numel() > 0+ and (+ cache is None+ or (+ torch.is_tensor(cache)+ and cache.dim() == 5+ and cache.shape[:2] == x.shape[:2]+ and cache.shape[3:] == x.shape[3:]+ and cache.device == x.device+ )+ )+ and (+ norm is None+ or _is_pixel_norm(norm)+ or (getattr(norm, "channel_first", False) and _rms_params(norm) is not None)+ )+ )+++def rms_norm_act(+ x: Any,+ norm: Any,+ act: bool = True,+ *,+ front: int = 0,+ cache: Any = None,+ in_bias: Any = None,+ replicate_pad: Optional[tuple] = None,+ back: int = 0,+) -> Any:+ """``cat([cache or zeros] -> front frames, silu?(rms_norm(x + in_bias)))`` along T, one channels-last_3d tensor.++ ``replicate_pad=(ph, pw)``: HunyuanVideo-1.5 replicate padding instead (``cache`` must be None); ``back`` repeats the+ last frame (LTX-2)."""+ torch = _torch()+ if replicate_pad is not None and cache is not None:+ raise ValueError("replicate padding takes no cache")+ if back and replicate_pad is None:+ raise ValueError("back frames need replicate padding")+ if not _rms_fusable(x, norm, cache):+ y = rms_norm_reference(x, norm, act, in_bias)+ if replicate_pad is not None:+ import torch.nn.functional as F++ ph, pw = replicate_pad+ y = F.pad(y.float(), (pw, pw, ph, ph, front, back), mode = "replicate").to(y.dtype)+ return y.contiguous(memory_format = torch.channels_last_3d)+ if front:+ import torch.nn.functional as F++ n_cache = 0+ if cache is not None:+ cache = cache[:, :, -front:].to(y.dtype)+ n_cache = cache.shape[2]+ y = torch.cat([cache, y], dim = 2)+ y = F.pad(y, (0, 0, 0, 0, front - n_cache, 0))+ return y.contiguous(memory_format = torch.channels_last_3d)+ k = _kernels()+ mean_sq = norm is not None and _is_pixel_norm(norm)+ if norm is not None and not mean_sq:+ gamma, bias, has_bias, scale = _rms_params(norm)+ else:+ gamma, bias, has_bias, scale = x, x, False, 1.0+ eps = float(getattr(norm, "eps", 0.0) or 0.0) if mean_sq else 0.0+ b, c, t, h, w = x.shape+ ph, pw = replicate_pad if replicate_pad is not None else (0, 0)+ ho, wo = h + 2 * ph, w + 2 * pw+ to = t + front + back+ n_cache = 0 if cache is None else min(front, cache.shape[2])+ cache_t = x if cache is None else cache[:, :, cache.shape[2] - n_cache :]+ out = torch.empty((b, to, ho, wo, c), dtype = x.dtype, device = x.device)+ block_c = _next_pow2(c)+ block_p = max(1, min(128, 8192 // block_c))+ grid = ((ho * wo + block_p - 1) // block_p, b * to)+ k.rms_act[grid](+ x, out, gamma, bias, in_bias if in_bias is not None else x, cache_t,+ c, t, h, w, ho, wo, ph, pw, to, front, n_cache, scale, eps,+ *x.stride(), *cache_t.stride(),+ NORM = norm is not None, ACT = bool(act), HAS_BIAS = has_bias, HAS_IN_BIAS = in_bias is not None,+ REPLICATE = replicate_pad is not None, MEAN_SQ = mean_sq, BLOCK_P = block_p, BLOCK_C = block_c,+ num_warps = 4 if block_c * block_p <= 4096 else 8,+ ) # fmt: skip+ return out.permute(0, 4, 1, 2, 3)+++def _conv_kind(conv: Any) -> Optional[str]:+ torch = _torch()+ pad = getattr(conv, "_padding", None)+ if pad is None or getattr(conv, "padding_mode", "zeros") != "zeros" or conv.groups != 1:+ return None+ if (+ isinstance(conv, torch.nn.Conv3d)+ and len(pad) == 6+ and pad[5] == 0+ and pad[0] == pad[1]+ and pad[2] == pad[3]+ ):+ return "3d"+ if (+ isinstance(conv, torch.nn.Conv2d)+ and len(pad) == 4+ and pad[0] == pad[1]+ and pad[2] == pad[3]+ ):+ return "2d"+ return None+++def causal_conv(+ conv: Any,+ x: Any,+ cache: Any = None,+ *,+ norm: Any = None,+ act: bool = False,+ in_bias: Any = None,+ with_bias: bool = True,+) -> tuple:+ """Wan-lineage causal conv of ``silu?(rms?(x + in_bias))``; returns (out, new_cache).++ ``new_cache`` is a compact copy so the cache never pins the whole input."""+ import torch.nn.functional as F++ kind = _conv_kind(conv)+ bias = conv.bias if with_bias else None+ if kind == "2d": # Qwen-Image-2.1: the one-frame specialisation, never cached+ if cache is not None:+ raise ValueError("2D causal conv takes no cache")+ y = (+ x+ if (norm is None and not act and in_bias is None)+ else rms_norm_act(x, norm, act, in_bias = in_bias)+ )+ pw, _, ph, _ = conv._padding+ out = F.conv2d(y[:, :, 0], conv.weight, bias, conv.stride, (ph, pw), conv.dilation)+ return out.unsqueeze(2), None+ if kind != "3d":+ raise ValueError("unsupported causal conv " + type(conv).__name__)+ pw, _, ph, _, front, _ = conv._padding+ kt = conv.kernel_size[0]+ spatial = (0, ph, pw)+ t = x.shape[2]+ if front == 0:+ y = (+ x+ if (norm is None and not act and in_bias is None)+ else rms_norm_act(x, norm, act, in_bias = in_bias)+ )+ return F.conv3d(y, conv.weight, bias, conv.stride, spatial, conv.dilation), None+ if (+ cache is None+ and t == 1+ and front == kt - 1+ and conv.stride[0] == 1+ and conv.dilation[0] == 1+ ):+ y = rms_norm_act(x, norm, act, in_bias = in_bias)+ weight = conv.weight[:, :, -1:]+ return F.conv3d(y, weight, bias, conv.stride, spatial, conv.dilation), y+ p = rms_norm_act(x, norm, act, front = front, cache = cache, in_bias = in_bias)+ out = F.conv3d(p, conv.weight, bias, conv.stride, spatial, conv.dilation)+ new_cache = p[:, :, -2:]+ return out, (new_cache if p.shape[2] <= 2 else new_cache.clone())+++# Single-head VAE attention: head_dim > 256 has no flash / cuDNN kernel, so two GEMMs + fp32 softmax beat SDPA.++_ATTN_SCORE_BYTES = 256 * 2**20+++def single_head_attention(q: Any, k: Any, v: Any) -> Any:+ """softmax(q k^T / sqrt(d)) v for (B, L, D) half-precision q/k/v, query-chunked to bound the fp32 scores."""+ torch = _torch()+ b, length, d = q.shape+ s_len = k.shape[1]+ scale = d**-0.5+ # keys padded to a multiple of 64 (cuBLAS slow path on odd extents); padded keys get probability exactly 0+ s_pad = (s_len + 63) // 64 * 64+ if s_pad != s_len:+ kp = torch.zeros((b, s_pad, d), dtype = k.dtype, device = k.device)+ vp = torch.zeros((b, s_pad, v.shape[-1]), dtype = v.dtype, device = v.device)+ kp[:, :s_len] = k+ vp[:, :s_len] = v+ k, v = kp, vp+ rows = int(max(64, min(length, _ATTN_SCORE_BYTES // max(1, b * s_pad * 4))) // 64 * 64)+ kt = k.transpose(1, 2)+ out = torch.empty((b, length, v.shape[-1]), dtype = v.dtype, device = v.device)+ kern = _kernels()+ for i in range(0, length, rows):+ scores = torch.bmm(q[:, i : i + rows], kt, out_dtype = torch.float32)+ r = scores.shape[1]+ probs = torch.empty((b, r, s_pad), dtype = v.dtype, device = v.device)+ kern.softmax_rows[(b * r,)](scores, probs, s_len, s_pad, scale, BLOCK = 2048, num_warps = 8)+ del scores+ torch.bmm(probs, v, out = out[:, i : i + rows])+ return out+++def _mm_attention_ok(q: Any) -> bool:+ torch = _torch()+ return q.is_cuda and q.dtype in (torch.float16, torch.bfloat16) and q.shape[-1] > 256+++class FusedSingleHeadProcessor:+ """``AttnProcessor2_0`` using :func:`single_head_attention`; anything else goes to ``fallback``."""++ def __init__(self, fallback: Any):+ self.fallback = fallback++ def __call__(self, attn: Any, hidden_states: Any, encoder_hidden_states: Any = None, attention_mask: Any = None,+ temb: Any = None, *args: Any, **kwargs: Any,) -> Any: # fmt: skip+ if (+ args+ or kwargs+ or encoder_hidden_states is not None+ or attention_mask is not None+ or attn.heads != 1+ or attn.spatial_norm is not None+ or attn.norm_q is not None+ or attn.norm_k is not None+ or hidden_states.ndim != 4+ or getattr(attn, "_unsloth_vae_fused_failed", False)+ or not _attention_input_ok(attn, hidden_states)+ ):+ return self.fallback(+ attn, hidden_states, encoder_hidden_states, attention_mask, temb, *args, **kwargs+ )+ try:+ return self._fused(attn, hidden_states)+ except Exception as exc: # noqa: BLE001+ # OOM falls back for this call only; anything else for good. Outside the handler so failed tensors are freed.+ if not _is_oom(exc):+ attn._unsloth_vae_fused_failed = True+ return self.fallback(attn, hidden_states, encoder_hidden_states, attention_mask, temb)++ @staticmethod+ def _fused(attn: Any, hidden_states: Any) -> Any:+ residual = hidden_states+ b, c, h, w = hidden_states.shape+ x = hidden_states.view(b, c, h * w).transpose(1, 2)+ if attn.group_norm is not None:+ x = attn.group_norm(x.transpose(1, 2)).transpose(1, 2)+ q, k, v = attn.to_q(x), attn.to_k(x), attn.to_v(x)+ o = single_head_attention(q, k, v)+ o = attn.to_out[1](attn.to_out[0](o))+ o = o.transpose(-1, -2).reshape(b, c, h, w)+ if attn.residual_connection:+ o = o + residual+ return o / attn.rescale_output_factor+++def _attention_input_ok(attn: Any, hidden_states: Any) -> bool:+ """Checked before any work, so an fp32 (upcast) or CPU VAE never pays for projections it then discards."""+ torch = _torch()+ width = getattr(getattr(attn, "to_q", None), "out_features", 0)+ return (+ hidden_states.is_cuda+ and hidden_states.dtype in (torch.float16, torch.bfloat16)+ and width > 256+ )+++def install_attention_processors(part: Any) -> int:+ n = 0+ for m in part.modules():+ if (+ type(m).__name__ == "Attention"+ and getattr(m, "heads", None) == 1+ and callable(getattr(m, "processor", None))+ ):+ if not isinstance(m.processor, FusedSingleHeadProcessor):+ m.processor = FusedSingleHeadProcessor(m.processor)+ n += 1+ return n+++def _fast_wan_attention(block: Any) -> Any:+ import torch.nn.functional as F+ def fast(x: Any) -> Any:+ identity = x+ b, c, t, h, w = x.shape+ y = rms_norm_act(+ x, block.norm, False+ ) # (b, c, t, h, w) channels-last_3d: (b, t, h, w, c) in memory+ y = y.permute(0, 2, 1, 3, 4).reshape(b * t, c, h, w) # a channels-last view, no copy+ qkv = F.conv2d(y, block.to_qkv.weight, block.to_qkv.bias)+ qkv = qkv.permute(0, 2, 3, 1).reshape(b * t, h * w, 3 * c)+ q, k, v = qkv[..., :c], qkv[..., c : 2 * c], qkv[..., 2 * c :]+ if not _mm_attention_ok(q):+ return _stock_forward(block)(x)+ o = single_head_attention(q, k, v) # (b*t, hw, c)+ o = F.linear(o, block.proj.weight.view(c, c), block.proj.bias)+ o = o.view(b, t, h, w, c).permute(0, 4, 1, 2, 3)+ return add_bias_residual(+ o.contiguous(memory_format = _torch().channels_last_3d), None, identity+ )++ return fast+++def conv_nobias(conv: Any, x: Any) -> Any:+ """``conv(x)`` minus the bias; the caller adds ``conv.bias`` in a later fused pass."""+ import torch.nn.functional as F++ if getattr(conv, "padding_mode", "zeros") != "zeros":+ raise ValueError("conv_nobias: padding_mode " + str(conv.padding_mode))+ fn = F.conv2d if x.dim() == 4 else F.conv3d+ return fn(x, conv.weight, None, conv.stride, conv.padding, conv.dilation, conv.groups)+++def _cl_contig(t: Any) -> bool:+ torch = _torch()+ fmt = torch.channels_last if t.dim() == 4 else torch.channels_last_3d+ return t.is_contiguous(memory_format = fmt)+++def add_bias_residual(+ out: Any,+ out_bias: Any,+ res: Any,+ res_bias: Any = None,+ scale: float = 1.0,+) -> Any:+ """``(out + out_bias + res + res_bias) / scale`` written into ``out`` (a fresh channels-last conv output)."""+ torch = _torch()+ k = _kernels()+ if (+ k is None+ or not out.is_cuda+ or out.dim() not in (4, 5)+ or not _cl_contig(out)+ or res.shape != out.shape+ ):+ if out_bias is not None:+ out.add_(out_bias.view(1, -1, *([1] * (out.dim() - 2))).to(out.dtype))+ if res_bias is not None:+ res = res + res_bias.view(1, -1, *([1] * (res.dim() - 2))).to(res.dtype)+ out.add_(res)+ return out if scale == 1.0 else out.div_(scale)+ r5, _ = _as5d(res)+ o5, _ = _as5d(out)+ b, c, t, h, w = o5.shape+ pixels = b * t * h * w+ block_c = min(128, _next_pow2(c))+ block_p = max(1, 4096 // block_c)+ grid = ((pixels + block_p - 1) // block_p, (c + block_c - 1) // block_c)+ k.bias_residual[grid](+ out, out_bias if out_bias is not None else out, res, res_bias if res_bias is not None else res,+ pixels, c, t, h * w, w, *r5.stride(), 1.0 / float(scale),+ HAS_OB = out_bias is not None, HAS_RB = res_bias is not None, SCALE = float(scale) != 1.0,+ BLOCK_P = block_p, BLOCK_C = block_c, num_warps = 4,+ ) # fmt: skip+ return out+++def _is_oom(exc: BaseException) -> bool:+ try:+ if isinstance(exc, _torch().cuda.OutOfMemoryError):+ return True+ except Exception: # noqa: BLE001+ pass+ return "out of memory" in str(exc).lower()+++def _guard(+ module: Any,+ fast: Any,+ stock: Any,+ label: str,+ logger: Any,+ oom_stock: bool = False,+) -> None:+ """``module.forward = fast`` until it raises something ``stock`` does not; then stock for good.++ Causal-cache state is snapshotted and restored before the stock retry so a half-failed fast path cannot desync+ the decode. ``oom_stock``: an OOM retries stock for that call only."""++ def forward(*args, **kwargs):+ if getattr(module, "_unsloth_vae_fused_failed", False):+ return stock(*args, **kwargs)+ lists = [a for a in (*args, *kwargs.values()) if isinstance(a, list)]+ snap = [a[:] for a in lists]+ try:+ return fast(*args, **kwargs)+ except Exception as exc: # noqa: BLE001+ for a, saved in zip(lists, snap):+ a[:] = saved+ if not _is_oom(exc):+ out = stock(*args, **kwargs)+ module._unsloth_vae_fused_failed = True+ if logger is not None:+ logger.warning(+ "diffusion.vae_fused: fused %s failed, using the stock path: %s", label, exc+ )+ return out+ if not oom_stock:+ raise+ return stock(+ *args, **kwargs+ ) # outside the handler, so the failed call's tensors are freed first++ forward._unsloth_vae_fused = True+ module.forward = forward+++def _stock_forward(module: Any) -> Any:+ return type(module).forward.__get__(module)+++def _resnet2d_fusable(block: Any) -> bool:+ torch = _torch()+ return (+ isinstance(getattr(block, "norm1", None), torch.nn.GroupNorm)+ and isinstance(getattr(block, "norm2", None), torch.nn.GroupNorm)+ and isinstance(getattr(block, "nonlinearity", None), torch.nn.SiLU)+ and getattr(block, "upsample", None) is None+ and getattr(block, "downsample", None) is None+ and getattr(block, "time_emb_proj", None) is None+ # "group" / "spatial" name the temb norm; without temb (a VAE) both run plain norm2 like "default"+ and getattr(block, "time_embedding_norm", "default") in ("default", "group")+ and isinstance(getattr(block, "conv1", None), torch.nn.Conv2d)+ and isinstance(getattr(block, "conv2", None), torch.nn.Conv2d)+ )+++def _fast_resnet2d(block: Any) -> Any:+ def fast(+ input_tensor: Any,+ temb: Any = None,+ *args: Any,+ **kwargs: Any,+ ) -> Any:+ if temb is not None or args or kwargs or block.training:+ return _stock_forward(block)(input_tensor, temb, *args, **kwargs)+ h = group_norm_act(input_tensor, block.norm1, act = True)+ h = conv_nobias(block.conv1, h)+ h = group_norm_act(h, block.norm2, act = True, in_bias = block.conv1.bias)+ h = conv_nobias(block.conv2, h)+ shortcut = (+ block.conv_shortcut(input_tensor) if block.conv_shortcut is not None else input_tensor+ )+ return add_bias_residual(+ h, block.conv2.bias, shortcut, None, float(block.output_scale_factor)+ )++ return fast+++def _fast_norm(norm: Any, act: bool) -> Any:+ def fast(x: Any) -> Any:+ return group_norm_act(x, norm, act = act)++ return fast+++def install_group_norm_vae(vae: Any, logger: Any = None) -> int:+ """Patch every eligible ResnetBlock2D / GroupNorm of a 2-D (AutoencoderKL-family) VAE. Returns patch count."""+ torch = _torch()+ if vae is None or not runtime_ok():+ return 0+ n = 0+ fused_act_norms = set()+ for part_name in ("encoder", "decoder"):+ part = getattr(vae, part_name, None)+ if part is None:+ continue+ n += install_attention_processors(part)+ norm_out = getattr(part, "conv_norm_out", None)+ act = getattr(part, "conv_act", None)+ if isinstance(norm_out, torch.nn.GroupNorm) and isinstance(act, torch.nn.SiLU):+ fused_act_norms.add(id(norm_out))+ _guard(+ norm_out,+ _fast_norm(norm_out, True),+ lambda x, _n = norm_out: torch.nn.functional.silu(torch.nn.GroupNorm.forward(_n, x)),+ "conv_norm_out",+ logger,+ )+ act.forward = lambda x: x+ act._unsloth_vae_fused = True+ n += 1+ for module in part.modules():+ if getattr(module.forward, "_unsloth_vae_fused", False):+ continue+ if type(module).__name__ == "ResnetBlock2D" and _resnet2d_fusable(module):+ _guard(+ module, _fast_resnet2d(module), _stock_forward(module), "ResnetBlock2D", logger+ )+ n += 1+ for module in part.modules():+ if (+ isinstance(module, torch.nn.GroupNorm)+ and id(module) not in fused_act_norms+ and not getattr(module.forward, "_unsloth_vae_fused", False)+ ):+ _guard(+ module, _fast_norm(module, False), _stock_forward(module), "GroupNorm", logger+ )+ n += 1+ return n+++def uninstall(vae: Any) -> None:+ vae.__dict__.pop("_unsloth_vae_fused_installed", None)+ vae.__dict__.pop("_unsloth_vae_fused_cl_weights", None)+ for m in vae.modules():+ if isinstance(getattr(m, "processor", None), FusedSingleHeadProcessor):+ m.processor = m.processor.fallback+ for attr in ("tiled_decode", "blend_v", "blend_h", "blend_t"):+ if getattr(vae.__dict__.get(attr), "_unsloth_vae_fused", False):+ delattr(vae, attr)+ for module in vae.modules():+ if getattr(getattr(module, "forward", None), "_unsloth_vae_fused", False) or getattr(+ module, "_unsloth_vae_fused", False+ ):+ try:+ del module.forward+ except AttributeError:+ pass+ module.__dict__.pop("_unsloth_vae_fused", None)+ module.__dict__.pop("_unsloth_vae_fused_failed", None)+ module.__dict__.pop("prepare_causal_attention_mask", None)+++_WAN_VAES = frozenset({"AutoencoderKLWan", "AutoencoderKLQwenImage", "AutoencoderKLQwenImage21"})+_CACHE_T = 2+++def _is_rms(norm: Any) -> bool:+ return (+ type(norm).__name__.endswith("RMS_norm")+ and hasattr(norm, "gamma")+ and hasattr(norm, "scale")+ )+++def _wan_resblock_fusable(block: Any) -> bool:+ torch = _torch()+ return (+ _is_rms(getattr(block, "norm1", None))+ and _is_rms(getattr(block, "norm2", None))+ and isinstance(getattr(block, "nonlinearity", None), torch.nn.SiLU)+ and _conv_kind(getattr(block, "conv1", None)) is not None+ and _conv_kind(getattr(block, "conv2", None)) is not None+ )+++def _fast_wan_resblock(block: Any) -> Any:+ import torch.nn.functional as F+ def fast(+ x: Any,+ feat_cache: Any = None,+ feat_idx: Any = None,+ *args: Any,+ **kwargs: Any,+ ) -> Any:+ if args or kwargs or block.training or (block.dropout.p and block.dropout.training):+ return _stock_forward(block)(+ x, feat_cache, feat_idx if feat_idx is not None else [0], *args, **kwargs+ )+ sc = block.conv_shortcut+ if isinstance(sc, _torch().nn.Identity):+ h = x+ elif _conv_kind(sc) == "2d":+ h = F.conv2d(x[:, :, 0], sc.weight, sc.bias, sc.stride, 0).unsqueeze(2)+ else:+ h = F.conv3d(x, sc.weight, sc.bias, sc.stride)+ if feat_cache is not None:+ idx = feat_idx[0]+ out, new = causal_conv(+ block.conv1, x, feat_cache[idx], norm = block.norm1, act = True, with_bias = False+ )+ feat_cache[idx] = new+ feat_idx[0] += 1+ idx = feat_idx[0]+ out, new = causal_conv(+ block.conv2, out, feat_cache[idx], norm = block.norm2, act = True, in_bias = block.conv1.bias,+ with_bias = False,+ ) # fmt: skip+ feat_cache[idx] = new+ feat_idx[0] += 1+ else:+ out, _ = causal_conv(block.conv1, x, None, norm = block.norm1, act = True, with_bias = False)+ out, _ = causal_conv(+ block.conv2,+ out,+ None,+ norm = block.norm2,+ act = True,+ in_bias = block.conv1.bias,+ with_bias = False,+ )+ return add_bias_residual(out, block.conv2.bias, h)++ return fast+++def _fast_causal_conv(conv: Any) -> Any:+ def fast(x: Any, cache_x: Any = None) -> Any:+ return causal_conv(conv, x, cache_x)[0]++ return fast+++def _fast_rms_act(norm: Any) -> Any:+ def fast(x: Any) -> Any:+ if x.dim() != 5:+ return _torch().nn.functional.silu(type(norm).forward(norm, x))+ return rms_norm_act(x, norm, True)++ return fast+++def dup_up_add(out: Any, x: Any, dup: Any, first_chunk: bool) -> Any:+ """``out + DupUp3D(x, first_chunk)`` added into ``out`` (a fresh tensor) without materialising the shortcut."""+ k = _kernels()+ ft, fs, rep = int(dup.factor_t), int(dup.factor_s), int(dup.repeats)+ b, c, t_o, h_o, w_o = out.shape+ t_off = ft - 1 if first_chunk else 0+ if (x.shape[2] * ft - t_off, x.shape[3] * fs, x.shape[4] * fs) != (+ t_o,+ h_o,+ w_o,+ ) or c != dup.out_channels:+ raise ValueError("dup_up_add: shape mismatch")+ block_c = min(128, _next_pow2(c))+ block_p = max(1, 4096 // block_c)+ grid = ((h_o * w_o + block_p - 1) // block_p, b * t_o, (c + block_c - 1) // block_c)+ k.dup_up_add[grid](+ out, x, c, t_o, h_o, w_o, t_off, rep, *out.stride(), *x.stride(),+ FT = ft, FS = fs, BLOCK_P = block_p, BLOCK_C = block_c, num_warps = 4,+ ) # fmt: skip+ return out+++def up_nearest2x(x: Any, interleave: bool) -> Any:+ """Channels-last nearest 2x upsample; ``interleave`` also does Wan's time_conv frame interleave."""+ torch = _torch()+ k = _kernels()+ b, c2, t, h, w = x.shape+ c = c2 // 2 if interleave else c2+ t_o = 2 * t if interleave else t+ out = torch.empty((b * t_o, 2 * h, 2 * w, c), dtype = x.dtype, device = x.device)+ block_c = min(128, _next_pow2(c))+ block_p = max(1, 4096 // block_c)+ grid = ((4 * h * w + block_p - 1) // block_p, b * t_o, (c + block_c - 1) // block_c)+ k.up_nearest2x[grid](+ x,+ out,+ c,+ t_o,+ h,+ w,+ *x.stride(),+ INTERLEAVE = interleave,+ BLOCK_P = block_p,+ BLOCK_C = block_c,+ num_warps = 4,+ )+ return out.permute(0, 3, 1, 2), t_o+++def _fast_wan_resample(mod: Any) -> Any:+ import torch.nn.functional as F++ conv = mod.resample[1]++ def fast(+ x: Any,+ feat_cache: Any = None,+ feat_idx: Any = None,+ ) -> Any:+ if feat_idx is None:+ feat_idx = [0]+ b = x.shape[0]+ interleave = False+ y = x+ if mod.mode == "upsample3d" and feat_cache is not None:+ idx = feat_idx[0]+ if feat_cache[idx] is None:+ feat_cache[idx] = "Rep"+ else:+ prev = None if isinstance(feat_cache[idx], str) else feat_cache[idx]+ y, new = causal_conv(mod.time_conv, x, prev)+ feat_cache[idx] = new+ interleave = True+ feat_idx[0] += 1+ up, t_o = up_nearest2x(y, interleave)+ out = F.conv2d(up, conv.weight, conv.bias, conv.stride, conv.padding)+ return out.view(b, t_o, out.shape[1], out.shape[2], out.shape[3]).permute(0, 2, 1, 3, 4)++ return fast+++def _wan_resample_ok(mod: Any) -> bool:+ torch = _torch()+ seq = getattr(mod, "resample", None)+ return (+ getattr(mod, "mode", None) in ("upsample2d", "upsample3d")+ and isinstance(seq, torch.nn.Sequential)+ and len(seq) == 2+ and isinstance(seq[0], torch.nn.Upsample)+ and seq[0].mode in ("nearest", "nearest-exact")+ and tuple(float(f) for f in (seq[0].scale_factor or ())) == (2.0, 2.0)+ and isinstance(seq[1], torch.nn.Conv2d)+ and (mod.mode == "upsample2d" or _conv_kind(getattr(mod, "time_conv", None)) == "3d")+ )+++def _fast_wan_residual_up(block: Any) -> Any:+ def fast(+ x: Any,+ feat_cache: Any = None,+ feat_idx: Any = None,+ first_chunk: bool = False,+ ) -> Any:+ if feat_idx is None:+ feat_idx = [0]+ x_in = x # the stock ``x.clone()`` guards nothing: no block below writes its input+ for resnet in block.resnets:+ x = (+ resnet(x, feat_cache = feat_cache, feat_idx = feat_idx)+ if feat_cache is not None+ else resnet(x)+ )+ if block.upsampler is not None:+ if feat_cache is not None:+ x = block.upsampler(x, feat_cache = feat_cache, feat_idx = feat_idx)+ else:+ x = block.upsampler(x)+ if block.avg_shortcut is not None:+ if x.data_ptr() == x_in.data_ptr():+ x = x.clone()+ x = dup_up_add(x, x_in, block.avg_shortcut, first_chunk)+ return x++ return fast+++def install_wan_vae(vae: Any, logger: Any = None) -> int:+ """Patch the residual blocks, causal convs and output heads of a Wan-lineage VAE. Returns patch count."""+ torch = _torch()+ if vae is None or not runtime_ok():+ return 0+ n = 0+ for part_name in ("encoder", "decoder"):+ part = getattr(vae, part_name, None)+ if part is None:+ continue+ norm_out = getattr(part, "norm_out", None)+ act = getattr(part, "nonlinearity", None)+ if _is_rms(norm_out) and isinstance(act, torch.nn.SiLU):+ _guard(+ norm_out,+ _fast_rms_act(norm_out),+ lambda x, _n = norm_out: torch.nn.functional.silu(type(_n).forward(_n, x)),+ "norm_out",+ logger,+ )+ act.forward = lambda x: x+ act._unsloth_vae_fused = True+ n += 1+ for module in part.modules():+ if "forward" in module.__dict__:+ continue # already patched (by us or another speed path)+ name = type(module).__name__+ if name.endswith("ResidualBlock") and _wan_resblock_fusable(module):+ _guard(+ module,+ _fast_wan_resblock(module),+ _stock_forward(module),+ "residual block",+ logger,+ )+ n += 1+ elif (+ name.endswith("ResidualUpBlock")+ and hasattr(module, "avg_shortcut")+ and hasattr(module, "resnets")+ ):+ _guard(+ module,+ _fast_wan_residual_up(module),+ _stock_forward(module),+ "residual up block",+ logger,+ )+ n += 1+ elif name.endswith("Resample") and _wan_resample_ok(module):+ _guard(+ module, _fast_wan_resample(module), _stock_forward(module), "resample", logger+ )+ n += 1+ elif (+ name.endswith("AttentionBlock")+ and _is_rms(getattr(module, "norm", None))+ and isinstance(getattr(module, "to_qkv", None), torch.nn.Conv2d)+ and isinstance(getattr(module, "proj", None), torch.nn.Conv2d)+ ):+ _guard(+ module, _fast_wan_attention(module), _stock_forward(module), "attention", logger,+ oom_stock = True,+ ) # fmt: skip+ n += 1+ for module in part.modules():+ if "forward" in module.__dict__:+ continue+ if type(module).__name__.endswith("CausalConv3d") and _conv_kind(module) is not None:+ _guard(+ module, _fast_causal_conv(module), _stock_forward(module), "causal conv", logger+ )+ n += 1+ elif (+ isinstance(module, torch.nn.Upsample)+ and type(module).__name__.endswith("Upsample")+ and module.mode in ("nearest", "nearest-exact")+ ):+ _guard(module, _fast_upsample(module), _stock_forward(module), "upsample", logger)+ n += 1+ if install_wan_tile_batch(vae, logger):+ n += 1+ return n+++def _fast_upsample(mod: Any) -> Any:+ """Wan's nearest upsample in the input dtype: bit-identical to its fp32 round trip, without two copies."""++ def fast(x: Any) -> Any:+ import torch.nn.functional as F+ return F.interpolate(+ x,+ mod.size,+ mod.scale_factor,+ mod.mode,+ mod.align_corners,+ recompute_scale_factor = mod.recompute_scale_factor,+ )++ return fast+++TILE_BATCH_ENV = "UNSLOTH_VAE_TILE_BATCH"+_TILE_BATCH_MAX = 4+# measured decode peak per 256 px Wan-2.1 tile (fp16, 4 frames, 96 full-res channels) ~= 1 GiB = 24x one activation+_TILE_PEAK_FACTOR = 24+++def _tile_batch_cap(vae: Any, z: Any) -> int:+ """Tiles per decoder call: env override, else what half the free VRAM holds, at most 4."""+ raw = os.environ.get(TILE_BATCH_ENV, "").strip()+ if raw.isdigit() and int(raw) > 0:+ return int(raw)+ try:+ free, _ = _torch().cuda.mem_get_info(z.device)+ norm_out = getattr(getattr(vae, "decoder", None), "norm_out", None)+ c_last = int(norm_out.gamma.numel()) if norm_out is not None else 128+ px = int(vae.tile_sample_min_height) * int(vae.tile_sample_min_width)+ frames = int(getattr(vae, "temporal_compression_ratio", 4) or 4)+ elem = next(vae.decoder.parameters()).element_size()+ per_tile = _TILE_PEAK_FACTOR * px * frames * c_last * elem+ return int(max(1, min(_TILE_BATCH_MAX, (free // 2) // max(1, per_tile))))+ except Exception: # noqa: BLE001+ return 1+++def _blend_weights(extent: int, device: Any) -> tuple:+ torch = _torch()+ # stock python-scalar blend weights as fp32 opmath, built on device: a host list is a sync H2D copy per seam+ y = torch.arange(extent, dtype = torch.float64, device = device) / extent+ wb = y.float()+ wa = (1 - y).float()+ return wa, wb+++def blend_seam(a: Any, b: Any, extent: int, dim: int) -> Any:+ """Vectorised, bit-identical ``blend_v`` (dim=-2) / ``blend_h`` (dim=-1): one pass instead of ``extent`` x 4."""+ torch = _torch()+ extent = min(a.shape[dim], b.shape[dim], extent)+ if extent <= 0:+ return b+ wa, wb = _blend_weights(extent, b.device)+ shape = [1] * b.dim()+ shape[dim] = extent+ wa, wb = wa.view(shape), wb.view(shape)+ sa = a.narrow(dim, a.shape[dim] - extent, extent)+ sb = b.narrow(dim, 0, extent)+ dt = b.dtype+ pa = (sa.float() * wa).to(dt)+ pb = (sb.float() * wb).to(dt)+ sb.copy_((pa.float() + pb.float()).to(dt))+ return b+++def _accepts_first_chunk(decoder: Any) -> bool:+ import inspect+ try:+ return "first_chunk" in inspect.signature(decoder.forward).parameters+ except (TypeError, ValueError):+ return False+++def _wan_batched_tiled_decode(+ self: Any,+ z: Any,+ return_dict: bool = True,+ clamp: bool = True,+) -> Any:+ """``tiled_decode`` with same-shaped tiles decoded as one batch and vectorised seam blending."""+ torch = _torch()+ from diffusers.models.autoencoders.vae import DecoderOutput++ _, _, num_frames, height, width = z.shape+ ratio = self.spatial_compression_ratio+ sample_height, sample_width = height * ratio, width * ratio+ tmh, tmw = self.tile_sample_min_height // ratio, self.tile_sample_min_width // ratio+ tsh, tsw = self.tile_sample_stride_height // ratio, self.tile_sample_stride_width // ratio+ out_sh, out_sw = self.tile_sample_stride_height, self.tile_sample_stride_width+ patch = getattr(self.config, "patch_size", None)+ if patch is not None:+ sample_height, sample_width = sample_height // patch, sample_width // patch+ out_sh, out_sw = out_sh // patch, out_sw // patch+ blend_h = self.tile_sample_min_height // patch - out_sh+ blend_w = self.tile_sample_min_width // patch - out_sw+ else:+ blend_h = self.tile_sample_min_height - out_sh+ blend_w = self.tile_sample_min_width - out_sw+ first_chunk = _accepts_first_chunk(self.decoder)+ rows_i = list(range(0, height, tsh))+ cols_j = list(range(0, width, tsw))+ groups: dict = {}+ for i in rows_i:+ for j in cols_j:+ groups.setdefault((min(tmh, height - i), min(tmw, width - j)), []).append((i, j))+ cap = _tile_batch_cap(self, z)+ tiles: dict = {}++ def decode_tiles(chunk: list) -> Any:+ zt = torch.cat([z[:, :, :, i : i + tmh, j : j + tmw] for i, j in chunk], 0)+ self.clear_cache()+ frames = []+ for k in range(num_frames):+ self._conv_idx = [0]+ tile = self.post_quant_conv(zt[:, :, k : k + 1])+ kw = {"first_chunk": k == 0} if first_chunk else {}+ frames.append(+ self.decoder(tile, feat_cache = self._feat_map, feat_idx = self._conv_idx, **kw)+ )+ return torch.cat(frames, dim = 2)++ for members in groups.values():+ s = 0+ while s < len(members):+ chunk = members[s : s + cap]+ try:+ out = decode_tiles(chunk)+ except Exception as exc: # noqa: BLE001+ if len(chunk) == 1 or not _is_oom(exc):+ raise+ self.clear_cache()+ torch.cuda.empty_cache()+ cap = 1+ continue+ for n, key in enumerate(chunk):+ tiles[key] = out[n : n + 1]+ s += len(chunk)+ self.clear_cache()+ rows = [[tiles[(i, j)] for j in cols_j] for i in rows_i]+ result_rows = []+ for ri, row in enumerate(rows):+ result_row = []+ for ci, tile in enumerate(row):+ if ri > 0:+ tile = blend_seam(rows[ri - 1][ci], tile, blend_h, -2)+ if ci > 0:+ tile = blend_seam(row[ci - 1], tile, blend_w, -1)+ result_row.append(tile[:, :, :, :out_sh, :out_sw])+ result_rows.append(torch.cat(result_row, dim = -1))+ dec = torch.cat(result_rows, dim = 3)[:, :, :, :sample_height, :sample_width]+ if patch is not None:+ from diffusers.models.autoencoders.autoencoder_kl_wan import unpatchify+ dec = unpatchify(dec, patch_size = patch)+ if (+ clamp+ ): # as the class's stock tiled_decode: Wan and Qwen-Image-2.1 clamp, Qwen-Image does not+ dec = torch.clamp(dec, min = -1.0, max = 1.0)+ if not return_dict:+ return (dec,)+ return DecoderOutput(sample = dec)+++@lru_cache(maxsize = None)+def _stock_tiled_decode_clamps(cls: type) -> Optional[bool]:+ """Whether ``cls.tiled_decode`` clamps to [-1, 1], read from the diffusers source; None (keep stock) if unreadable."""+ import inspect+ import re++ try:+ src = inspect.getsource(cls.tiled_decode)+ except (OSError, TypeError, AttributeError):+ return None+ return re.search(r"\bclamp\(\s*dec\b", src) is not None+++def install_wan_tile_batch(vae: Any, logger: Any = None) -> bool:+ if (+ vae is None+ or "tiled_decode" in vae.__dict__+ or not callable(getattr(vae, "tiled_decode", None))+ ):+ return False+ clamp = _stock_tiled_decode_clamps(type(vae))+ if clamp is None:+ return False+ stock = _stock_forward_attr(vae, "tiled_decode")++ def tiled_decode(z: Any, return_dict: bool = True) -> Any:+ if z.shape[0] != 1 or getattr(vae, "_unsloth_vae_fused_failed", False):+ return stock(z, return_dict = return_dict)+ try:+ return _wan_batched_tiled_decode(vae, z, return_dict, clamp)+ except Exception as exc: # noqa: BLE001+ if _is_oom(exc):+ raise+ vae._unsloth_vae_fused_failed = True+ if logger is not None:+ logger.warning(+ "diffusion.vae_fused: batched tiled decode failed, using the stock loop: %s",+ exc,+ )+ return stock(z, return_dict = return_dict)++ tiled_decode._unsloth_vae_fused = True+ vae.tiled_decode = tiled_decode+ return True+++def _stock_forward_attr(obj: Any, name: str) -> Any:+ return getattr(type(obj), name).__get__(obj)+++def _hv_conv_ok(conv: Any) -> bool:+ torch = _torch()+ inner = getattr(conv, "conv", None)+ pad = getattr(conv, "time_causal_padding", None)+ return (+ isinstance(inner, torch.nn.Conv3d)+ and getattr(conv, "pad_mode", None) == "replicate"+ and pad is not None+ and len(pad) == 6+ and pad[0] == pad[1]+ and pad[2] == pad[3]+ and pad[5] == 0+ and inner.padding == (0, 0, 0)+ and inner.groups == 1+ and inner.padding_mode == "zeros"+ )+++def hv_conv(+ conv: Any,+ x: Any,+ *,+ norm: Any = None,+ act: bool = False,+ in_bias: Any = None,+ with_bias: bool = True,+) -> Any:+ """``conv(pad_replicate(silu?(rms?(x + in_bias))))`` with the pad, norm and act in one channels-last pass."""+ import torch.nn.functional as F++ pw, _, ph, _, front, _ = conv.time_causal_padding+ inner = conv.conv+ y = rms_norm_act(x, norm, act, front = front, in_bias = in_bias, replicate_pad = (ph, pw))+ return F.conv3d(+ y, inner.weight, inner.bias if with_bias else None, inner.stride, 0, inner.dilation+ )+++def _hv_resnet_ok(block: Any) -> bool:+ torch = _torch()+ return (+ _is_rms(getattr(block, "norm1", None))+ and _is_rms(getattr(block, "norm2", None))+ and isinstance(getattr(block, "nonlinearity", None), torch.nn.SiLU)+ and _hv_conv_ok(getattr(block, "conv1", None))+ and _hv_conv_ok(getattr(block, "conv2", None))+ )+++def _fast_hv_resnet(block: Any) -> Any:+ import torch.nn.functional as F+ def fast(x: Any) -> Any:+ h = hv_conv(block.conv1, x, norm = block.norm1, act = True, with_bias = False)+ h = hv_conv(+ block.conv2,+ h,+ norm = block.norm2,+ act = True,+ in_bias = block.conv1.conv.bias,+ with_bias = False,+ )+ sc = block.conv_shortcut+ res = x if sc is None else F.conv3d(x, sc.weight, sc.bias, sc.stride, sc.padding)+ return add_bias_residual(h, block.conv2.conv.bias, res)++ return fast+++def _fast_hv_conv(conv: Any) -> Any:+ def fast(x: Any) -> Any:+ return hv_conv(conv, x)++ return fast+++def _fast_hv_norm_act(norm: Any) -> Any:+ def fast(x: Any) -> Any:+ if x.dim() != 5:+ return _torch().nn.functional.silu(type(norm).forward(norm, x))+ return rms_norm_act(x, norm, True)++ return fast+++def hv_upsample(up: Any, x: Any) -> Any:+ """``HunyuanVideo15Upsample.forward``: conv, then depth-to-space + repeat_interleave shortcut + add in one pass."""+ torch = _torch()+ k = _kernels()+ conv = up.conv+ h = hv_conv(conv, x, with_bias = False)+ b, cin, f, hh, ww = x.shape+ cp = h.shape[1]+ temporal = bool(up.add_temporal_upsample)+ factor = 8 if temporal else 4+ c = cp // factor+ rep = int(up.repeats)+ if cp % factor or (temporal and (f < 1 or rep % 2 or cin % 4)) or rep * cin != factor * c:+ raise ValueError("hv_upsample: unexpected shapes")+ fo = 1 + (f - 1) * 2 if temporal else f+ out = torch.empty((b, fo, 2 * hh, 2 * ww, c), dtype = x.dtype, device = x.device)+ block_c = min(128, _next_pow2(c))+ block_p = max(1, 4096 // block_c)+ grid = ((4 * hh * ww + block_p - 1) // block_p, b * fo, (c + block_c - 1) // block_c)+ bias = conv.conv.bias+ k.dcae_up_add[grid](+ h, bias if bias is not None else h, x, out, c, cin, f, hh, ww, fo, rep, *h.stride(), *x.stride(),+ TEMPORAL = temporal, HAS_HB = bias is not None, BLOCK_P = block_p, BLOCK_C = block_c, num_warps = 4,+ ) # fmt: skip+ return out.permute(0, 4, 1, 2, 3)+++def _hv_causal_mask(+ n_frame: int,+ n_hw: int,+ dtype: Any,+ device: Any,+ batch_size: Any = None,+) -> Any:+ """Vectorised ``prepare_causal_attention_mask``: 0 where key frame <= query frame, else -inf."""+ torch = _torch()+ seq_len = n_frame * n_hw+ mask = torch.zeros((seq_len, seq_len), dtype = dtype, device = device)+ later = torch.ones((n_frame, n_frame), dtype = torch.bool, device = device).triu_(1)+ mask.view(n_frame, n_hw, n_frame, n_hw).masked_fill_(later[:, None, :, None], float("-inf"))+ if batch_size is not None:+ mask = mask.unsqueeze(0).expand(batch_size, -1, -1)+ return mask+++def install_hv15_vae(vae: Any, logger: Any = None) -> int:+ torch = _torch()+ if vae is None or not runtime_ok():+ return 0+ n = 0+ for part_name in ("encoder", "decoder"):+ part = getattr(vae, part_name, None)+ if part is None:+ continue+ norm_out, act = getattr(part, "norm_out", None), getattr(part, "conv_act", None)+ if (+ _is_rms(norm_out)+ and isinstance(act, torch.nn.SiLU)+ and "forward" not in norm_out.__dict__+ ):+ _guard(+ norm_out,+ _fast_hv_norm_act(norm_out),+ lambda x, _n = norm_out: torch.nn.functional.silu(type(_n).forward(_n, x)),+ "norm_out",+ logger,+ )+ act.forward = lambda x: x+ act._unsloth_vae_fused = True+ n += 1+ for module in part.modules():+ if "forward" in module.__dict__:+ continue+ name = type(module).__name__+ if name.endswith("ResnetBlock") and _hv_resnet_ok(module):+ _guard(module, _fast_hv_resnet(module), _stock_forward(module), "resnet", logger)+ n += 1+ for module in part.modules():+ if "forward" in module.__dict__:+ continue+ name = type(module).__name__+ if (+ name.endswith("Upsample")+ and _hv_conv_ok(getattr(module, "conv", None))+ and hasattr(module, "repeats")+ ):+ _guard(+ module,+ lambda x, _m = module: hv_upsample(_m, x),+ _stock_forward(module),+ "upsample",+ logger,+ )+ n += 1+ elif name.endswith("AttnBlock") and callable(+ getattr(module, "prepare_causal_attention_mask", None)+ ):+ module.prepare_causal_attention_mask = _hv_causal_mask+ n += 1+ for module in part.modules():+ if "forward" in module.__dict__:+ continue+ if type(module).__name__.endswith("CausalConv3d") and _hv_conv_ok(module):+ _guard(module, _fast_hv_conv(module), _stock_forward(module), "causal conv", logger)+ n += 1+ return n+++def _ltx_conv_ok(conv: Any) -> bool:+ torch = _torch()+ inner = getattr(conv, "conv", None)+ return (+ isinstance(inner, torch.nn.Conv3d)+ and inner.padding_mode == "zeros"+ and inner.padding[0] == 0+ and inner.stride[0] == 1+ and inner.dilation[0] == 1+ and len(getattr(conv, "kernel_size", ())) == 3+ and conv.kernel_size[0] % 2 == 1+ )+++def ltx_conv(+ conv: Any,+ x: Any,+ causal: bool,+ *,+ norm: Any = None,+ act: bool = False,+ in_bias: Any = None,+ with_bias: bool = True,+) -> Any:+ """``LTX2VideoCausalConv3d.forward`` over ``silu?(pixel_norm?(x + in_bias))``, pad + norm + act in one pass."""+ import torch.nn.functional as F++ kt = conv.kernel_size[0]+ front, back = (kt - 1, 0) if causal else ((kt - 1) // 2, (kt - 1) // 2)+ inner = conv.conv+ if kt == 1 and norm is None and not act and in_bias is None:+ y = x+ else:+ y = rms_norm_act(+ x, norm, act, front = front, back = back, in_bias = in_bias, replicate_pad = (0, 0)+ )+ return F.conv3d(+ y,+ inner.weight,+ inner.bias if with_bias else None,+ inner.stride,+ inner.padding,+ inner.dilation,+ )+++def _ltx_resnet_ok(block: Any) -> bool:+ torch = _torch()+ return (+ _is_pixel_norm(getattr(block, "norm1", None))+ and _is_pixel_norm(getattr(block, "norm2", None))+ and isinstance(getattr(block, "nonlinearity", None), torch.nn.SiLU)+ and _ltx_conv_ok(getattr(block, "conv1", None))+ and _ltx_conv_ok(getattr(block, "conv2", None))+ and getattr(block, "per_channel_scale1", None) is None+ and getattr(block, "per_channel_scale2", None) is None+ and getattr(block, "scale_shift_table", None) is None+ )+++def _fast_ltx_resnet(block: Any) -> Any:+ import torch.nn.functional as F+ def fast(+ inputs: Any,+ temb: Any = None,+ generator: Any = None,+ causal: bool = True,+ ) -> Any:+ if block.training and block.dropout.p:+ return _stock_forward(block)(inputs, temb, generator, causal = causal)+ h = ltx_conv(block.conv1, inputs, causal, norm = block.norm1, act = True, with_bias = False)+ h = ltx_conv(+ block.conv2,+ h,+ causal,+ norm = block.norm2,+ act = True,+ in_bias = block.conv1.conv.bias,+ with_bias = False,+ )+ res = inputs+ if block.norm3 is not None:+ res = block.norm3(res.movedim(1, -1)).movedim(-1, 1)+ if block.conv_shortcut is not None:+ sc = block.conv_shortcut+ res = F.conv3d(res, sc.weight, sc.bias, sc.stride, sc.padding)+ return add_bias_residual(h, block.conv2.conv.bias, res)++ return fast+++def _fast_ltx_conv(conv: Any) -> Any:+ def fast(hidden_states: Any, causal: bool = True) -> Any:+ return ltx_conv(conv, hidden_states, causal)++ return fast+++def _fast_pixel_norm_act(norm: Any) -> Any:+ def fast(x: Any, channel_dim: Any = None) -> Any:+ if x.dim() != 5 or channel_dim not in (None, 1):+ return _torch().nn.functional.silu(type(norm).forward(norm, x, channel_dim))+ return rms_norm_act(x, norm, True)++ return fast+++def _ltx2_convs_fusable(vae: Any) -> bool:+ """LTX-2's reflect-padded decoder fuses no conv: norm_out alone decoded 0.97-0.98x (L4 / A100 / RTX PRO 6000)."""+ decoder = getattr(vae, "decoder", None)+ return decoder is not None and any(+ type(m).__name__ == "LTX2VideoCausalConv3d" and _ltx_conv_ok(m) for m in decoder.modules()+ )+++def install_ltx2_vae(vae: Any, logger: Any = None) -> int:+ torch = _torch()+ if vae is None or not runtime_ok():+ return 0+ n = 0+ # decoder only: the causal one-frame encode (i2v conditioning) measured 33 -> 35 ms fused, so it stays stock+ for part_name in ("decoder",):+ part = getattr(vae, part_name, None)+ if part is None:+ continue+ norm_out, act = getattr(part, "norm_out", None), getattr(part, "conv_act", None)+ if (+ _is_pixel_norm(norm_out)+ and isinstance(act, torch.nn.SiLU)+ and getattr(part, "time_embedder", None) is None+ and "forward" not in norm_out.__dict__+ ):+ _guard(+ norm_out,+ _fast_pixel_norm_act(norm_out),+ lambda x, channel_dim = None, _n = norm_out: torch.nn.functional.silu(+ type(_n).forward(_n, x, channel_dim)+ ),+ "norm_out",+ logger,+ )+ act.forward = lambda x: x+ act._unsloth_vae_fused = True+ n += 1+ for module in part.modules():+ if "forward" in module.__dict__:+ continue+ if type(module).__name__.endswith("ResnetBlock3d") and _ltx_resnet_ok(module):+ _guard(module, _fast_ltx_resnet(module), _stock_forward(module), "resnet", logger)+ n += 1+ for module in part.modules():+ if "forward" in module.__dict__:+ continue+ if type(module).__name__ == "LTX2VideoCausalConv3d" and _ltx_conv_ok(module):+ _guard(+ module, _fast_ltx_conv(module), _stock_forward(module), "causal conv", logger+ )+ n += 1+ return n+++def install_vectorised_blend(vae: Any) -> int:+ """Replace per-row ``blend_v`` / ``blend_h`` / ``blend_t`` loops with the bit-identical :func:`blend_seam`."""+ n = 0+ for name, dim in (("blend_v", -2), ("blend_h", -1), ("blend_t", -3)):+ if callable(getattr(vae, name, None)) and name not in vae.__dict__:+ fn = lambda a, b, blend_extent, _d = dim: blend_seam(a, b, blend_extent, _d) # noqa: E731+ fn._unsloth_vae_fused = True+ setattr(vae, name, fn)+ n += 1+ return n+++_CL_WEIGHT_VAES = frozenset(+ {"AutoencoderKL", "AutoencoderKLFlux2", "AutoencoderKLHunyuanVideo15", "AutoencoderKLLTX2Video"}+)+++def channels_last_weights(vae: Any) -> int:+ """The fused passes hand cuDNN channels-last activations; contiguous weights would be re-laid-out on every call."""+ torch = _torch()+ n = 0+ for part in (getattr(vae, "encoder", None), getattr(vae, "decoder", None)):+ if part is None:+ continue+ for m in part.modules():+ w = getattr(m, "weight", None)+ if (+ isinstance(m, (torch.nn.Conv2d, torch.nn.Conv3d))+ and isinstance(w, torch.Tensor)+ and w.dim() in (4, 5)+ ):+ fmt = torch.channels_last if w.dim() == 4 else torch.channels_last_3d+ if not w.is_contiguous(memory_format = fmt):+ w.data = w.data.contiguous(memory_format = fmt)+ n += 1+ return n+++def _fuses_a_conv(vae: Any) -> bool:+ torch = _torch()+ return any(+ getattr(m.__dict__.get("forward"), "_unsloth_vae_fused", False)+ and any(isinstance(c, (torch.nn.Conv2d, torch.nn.Conv3d)) for c in m.modules())+ for m in vae.modules()+ )+++SUPPORTED_VAES = frozenset(+ {+ "AutoencoderKL",+ "AutoencoderKLFlux2",+ "AutoencoderKLWan",+ "AutoencoderKLQwenImage",+ "AutoencoderKLQwenImage21",+ "AutoencoderKLHunyuanVideo15",+ "AutoencoderKLLTX2Video",+ }+)+++def will_install(vae: Any) -> bool:+ """Whether :func:`install` would engage on ``vae`` here (class covered, NVIDIA CUDA, usable Triton, not disabled)."""+ return (+ vae is not None+ and type(vae).__name__ in SUPPORTED_VAES+ and (type(vae).__name__ != "AutoencoderKLLTX2Video" or _ltx2_convs_fusable(vae))+ and runtime_ok()+ )+++def install(+ vae: Any,+ logger: Any = None,+ level: str = "fused",+) -> int:+ """Install every fused path that applies to ``vae``'s class. Returns the number of patched modules."""+ if vae is None or not will_install(vae):+ return 0+ done = getattr(vae, "_unsloth_vae_fused_installed", 0)+ if done:+ return done # idempotent: a dual-DiT family runs the speed layer once per expert over the same VAE+ n = _install(vae, logger)+ # Only where measured faster, and only when a fused pass feeds a conv (relaid weights alone slow LTX-2's decode).+ if (+ n+ and type(vae).__name__ in _CL_WEIGHT_VAES+ and os.environ.get("UNSLOTH_VAE_FUSED_CL_WEIGHTS", "1") != "0"+ and _fuses_a_conv(vae)+ ):+ channels_last_weights(vae)+ vae._unsloth_vae_fused_cl_weights = True+ if n:+ vae._unsloth_vae_fused_installed = n+ if logger is not None:+ logger.info("diffusion.vae_fused: fused %d VAE modules on %s", n, type(vae).__name__)+ return n+++def _install(vae: Any, logger: Any = None) -> int:+ name = type(vae).__name__+ if name in ("AutoencoderKL", "AutoencoderKLFlux2"):+ n = install_group_norm_vae(vae, logger)+ return n + (install_vectorised_blend(vae) if n else 0)+ if name in _WAN_VAES:+ n = install_wan_vae(vae, logger)+ return n + (install_vectorised_blend(vae) if n else 0)+ if name == "AutoencoderKLHunyuanVideo15":+ n = install_hv15_vae(vae, logger)+ return n + (install_vectorised_blend(vae) if n else 0)+ if name == "AutoencoderKLLTX2Video":+ if not _ltx2_convs_fusable(vae):+ return 0+ n = install_ltx2_vae(vae, logger)+ return n + (install_vectorised_blend(vae) if n else 0)+ return 0diff --git a/studio/backend/tests/test_diffusion_speed.py b/studio/backend/tests/test_diffusion_speed.pyindex 67048d863f9..c7ffd5e980b 100644--- a/studio/backend/tests/test_diffusion_speed.py+++ b/studio/backend/tests/test_diffusion_speed.py@@ -427,6 +427,7 @@ def test_speed_off_applies_nothing(monkeypatch):assert applied == {"channels_last": False,"vae_fp16_decode": False,+ "vae_fused": False,"cudnn_benchmark": False,"tf32": False,"fused_qkv": False,@@ -682,6 +683,22 @@ def test_eager_vae_decode_keeps_contiguous_weights(monkeypatch, tier):assert applied["channels_last"] is False and pipe.vae.mem_format == torch.contiguous_format[email protected]("tier", [SPEED_EAGER, SPEED_DEFAULT])+def test_fused_vae_keeps_its_channels_last_weights(monkeypatch, tier):+ torch = _stub_torch(monkeypatch)+ monkeypatch.delenv(ds_mod.COMPILE_VAE_ENV, raising = False)+ pipe = _Pipe(with_compile = True)++ def fused(p, logger):+ p.vae._unsloth_vae_fused_cl_weights = True+ return True++ monkeypatch.setattr(ds_mod, "_install_fused_vae", fused)+ applied = apply_speed_optims(pipe, _target(), is_gguf = False, family = _family(), speed_mode = tier)+ assert applied["vae_fused"] is True+ assert applied["channels_last"] is True and pipe.vae.mem_format == torch.channels_last++@pytest.mark.parametrize("backend, device", [("rocm", "cuda"), ("mps", "mps"), ("xpu", "xpu"), ("cpu", "cpu")])diff --git a/studio/backend/tests/test_diffusion_vae_fused.py b/studio/backend/tests/test_diffusion_vae_fused.pynew file mode 100644index 00000000000..e6c8e403113--- /dev/null+++ b/studio/backend/tests/test_diffusion_vae_fused.py@@ -0,0 +1,633 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""diffusion_vae_fused: bit-exact helpers (CPU), gating, and fused-vs-stock decode/encode on tiny random VAEs (CUDA)."""++import pytest++torch = pytest.importorskip("torch")++from core.inference import diffusion_vae_fused as F # noqa: E402+++def _cuda_triton() -> bool:+ return torch.cuda.is_available() and not getattr(torch.version, "hip", None) and F.runtime_ok()+++needs_cuda = pytest.mark.skipif(not _cuda_triton(), reason = "needs NVIDIA CUDA + Triton")++[email protected]("dtype", [torch.float16, torch.bfloat16, torch.float32])[email protected]("extent", [1, 7, 32])+def test_blend_seam_bit_identical_to_stock_loops(dtype, extent):+ diffusers = pytest.importorskip("diffusers")+ wan = diffusers.AutoencoderKLWan+ g = torch.Generator().manual_seed(0)+ a = torch.randn(1, 3, 2, 40, 44, generator = g).to(dtype)+ b = torch.randn(1, 3, 2, 40, 44, generator = g).to(dtype)+ for stock, dim in ((wan.blend_v, -2), (wan.blend_h, -1)):+ assert torch.equal(+ stock(None, a, b.clone(), extent), F.blend_seam(a, b.clone(), extent, dim)+ )+++def test_hv_causal_mask_matches_stock():+ mod = pytest.importorskip("diffusers.models.autoencoders.autoencoder_kl_hunyuanvideo15")+ stock = mod.HunyuanVideo15AttnBlock.prepare_causal_attention_mask(+ 3, 5, torch.float32, "cpu", batch_size = 2+ )+ assert torch.equal(stock, F._hv_causal_mask(3, 5, torch.float32, "cpu", batch_size = 2))+++def test_disabled_by_env(monkeypatch):+ monkeypatch.setenv(F.VAE_FUSED_ENV, "0")+ assert F.runtime_ok() is False+ assert F.will_install(object()) is False+++def test_rocm_and_old_triton_keep_stock(monkeypatch):+ # the #11801 kernels are gated off on ROCm (missing add_rn / rint, wrong fp16 GroupNorm on gfx1151): same here+ monkeypatch.delenv(F.VAE_FUSED_ENV, raising = False)+ monkeypatch.setattr(torch.cuda, "is_available", lambda: True)+ monkeypatch.setattr(torch.version, "hip", "6.4.0", raising = False)+ assert F.runtime_ok() is False+ monkeypatch.setattr(torch.version, "hip", None, raising = False)+ import sys+ import types++ F._triton_version_ok.cache_clear()+ monkeypatch.setitem(sys.modules, "triton", types.SimpleNamespace(__version__ = "3.2.0"))+ try:+ assert F.runtime_ok() is False+ finally:+ F._triton_version_ok.cache_clear()+++def test_missing_triton_keeps_stock(monkeypatch):+ import sys++ monkeypatch.delenv(F.VAE_FUSED_ENV, raising = False)+ monkeypatch.setattr(torch.cuda, "is_available", lambda: True)+ monkeypatch.setattr(torch.version, "hip", None, raising = False)+ F._triton_version_ok.cache_clear()+ monkeypatch.setitem(sys.modules, "triton", None) # `import triton` raises ImportError+ try:+ assert F.runtime_ok() is False+ assert F.will_install(type("AutoencoderKL", (), {})()) is False+ finally:+ F._triton_version_ok.cache_clear()+++def test_windows_without_msvc_keeps_stock(monkeypatch):+ monkeypatch.setattr(F.sys, "platform", "win32")+ F._toolchain_ok.cache_clear()+ import core._msvc_env as msvc++ monkeypatch.setattr(msvc, "crt_headers_reachable", lambda: False)+ try:+ assert F._toolchain_ok() is False+ finally:+ F._toolchain_ok.cache_clear()++[email protected]("dtype", [torch.bfloat16, torch.float32])+def test_blend_seam_4d_matches_autoencoder_kl(dtype):+ diffusers = pytest.importorskip("diffusers")+ akl = diffusers.AutoencoderKL+ g = torch.Generator().manual_seed(1)+ a = torch.randn(1, 3, 40, 44, generator = g).to(dtype)+ b = torch.randn(1, 3, 40, 44, generator = g).to(dtype)+ assert torch.equal(akl.blend_v(None, a, b.clone(), 16), F.blend_seam(a, b.clone(), 16, -2))+ assert torch.equal(akl.blend_h(None, a, b.clone(), 16), F.blend_seam(a, b.clone(), 16, -1))+++def test_install_is_noop_without_runtime(monkeypatch):+ monkeypatch.setattr(F, "runtime_ok", lambda: False)++ class AutoencoderKL(torch.nn.Module):+ pass++ assert F.install(AutoencoderKL()) == 0+++def test_uncovered_class_not_planned():+ class AutoencoderKLMiniMaxH3(torch.nn.Module): # has its own fast path+ pass++ assert F.will_install(AutoencoderKLMiniMaxH3()) is False+++def test_speed_layer_skips_vae_compile_when_fused(monkeypatch):+ from core.inference import diffusion_speed as S++ monkeypatch.delenv(S.COMPILE_VAE_ENV, raising = False)+ monkeypatch.setattr(F, "will_install", lambda vae: True)++ class Pipe:+ vae = object()+ unet = None++ assert S._fused_vae_planned(Pipe()) is True+ assert S._vae_decode_compile_allowed(Pipe(), "max") is False+ # forcing the compile keeps the old behaviour and skips the fused path+ monkeypatch.setenv(S.COMPILE_VAE_ENV, "1")+ assert S._fused_vae_planned(Pipe()) is False+ assert S._install_fused_vae(Pipe(), None) is False+++def _psnr(a, b):+ mse = (a.float() - b.float()).pow(2).mean().item()+ return float("inf") if mse == 0 else 10 * torch.log10(torch.tensor(4.0 / mse)).item()+++def _decode(vae, z):+ return vae.decode(z, return_dict = False)[0].float()+++def _check(+ vae,+ z,+ *,+ min_psnr,+ encode_x = None,+):+ with torch.inference_mode():+ ref = _decode(vae, z)+ n = F.install(vae)+ assert n > 0+ out = _decode(vae, z)+ assert not any(+ getattr(m, "_unsloth_vae_fused_failed", False) for m in vae.modules()+ ), "fused path fell back"+ assert out.shape == ref.shape+ assert _psnr(out, ref) > min_psnr, _psnr(out, ref)+ if encode_x is not None:+ F.uninstall(vae)+ lat_ref = vae.encode(encode_x).latent_dist.mode().float()+ F.install(vae)+ lat = vae.encode(encode_x).latent_dist.mode().float()+ assert (lat - lat_ref).norm() / lat_ref.norm() < 2e-2+ F.uninstall(vae)+ assert not any(+ getattr(m.__dict__.get("forward"), "_unsloth_vae_fused", False) for m in vae.modules()+ )+ # the channels-last weights stay, and cuDNN may pick another algorithm for them (torch 2.7: 71 dB)+ assert torch.equal(_decode(vae, z), ref) or _psnr(_decode(vae, z), ref) > 60+++@needs_cuda[email protected]("dtype", [torch.bfloat16, torch.float16, torch.float32])+def test_autoencoder_kl_fused_matches_stock(dtype):+ from diffusers import AutoencoderKL++ torch.manual_seed(0)+ vae = AutoencoderKL(+ block_out_channels = (32, 64), down_block_types = ("DownEncoderBlock2D",) * 2,+ up_block_types = ("UpDecoderBlock2D",) * 2, latent_channels = 4, norm_num_groups = 32, layers_per_block = 1,+ ).cuda().to(dtype).eval() # fmt: skip+ vae.to(memory_format = torch.channels_last)+ z = torch.randn(1, 4, 24, 20, device = "cuda", dtype = dtype)+ x = torch.rand(1, 3, 48, 40, device = "cuda", dtype = dtype) * 2 - 1+ _check(vae, z, min_psnr = 40 if dtype != torch.float32 else 70, encode_x = x)+++@needs_cuda[email protected]("tiled", [False, True])+def test_wan_fused_matches_stock_across_chunks(tiled, monkeypatch):+ from diffusers import AutoencoderKLWan++ torch.manual_seed(0)+ vae = (+ AutoencoderKLWan(base_dim = 32, z_dim = 4, dim_mult = [1, 2, 2, 2], num_res_blocks = 1)+ .cuda()+ .half()+ .eval()+ )+ if tiled:+ vae.enable_tiling(tile_sample_min_height = 64, tile_sample_min_width = 64, tile_sample_stride_height = 48,+ tile_sample_stride_width = 48) # fmt: skip+ monkeypatch.setenv(F.TILE_BATCH_ENV, "3")+ z = torch.randn(1, 4, 4, 14, 18, device = "cuda", dtype = torch.float16)+ x = torch.rand(1, 3, 5, 48, 64, device = "cuda", dtype = torch.float16) * 2 - 1+ _check(vae, z, min_psnr = 45, encode_x = None if tiled else x)+++@needs_cuda+def test_wan22_residual_decoder_fused_matches_stock():+ from diffusers import AutoencoderKLWan++ torch.manual_seed(0)+ vae = AutoencoderKLWan(+ base_dim = 32, decoder_base_dim = 32, z_dim = 8, dim_mult = [1, 2, 2, 2], num_res_blocks = 1, in_channels = 12,+ out_channels = 12, is_residual = True, patch_size = 2, scale_factor_spatial = 16,+ latents_mean = [0.0] * 8, latents_std = [1.0] * 8,+ ).cuda().half().eval() # fmt: skip+ z = torch.randn(1, 8, 3, 6, 8, device = "cuda", dtype = torch.float16)+ _check(vae, z, min_psnr = 45)+++@needs_cuda+def test_qwenimage_single_frame_fused_matches_stock():+ from diffusers import AutoencoderKLQwenImage++ torch.manual_seed(0)+ vae = (+ AutoencoderKLQwenImage(base_dim = 32, z_dim = 4, dim_mult = [1, 2, 2, 2], num_res_blocks = 1)+ .cuda()+ .bfloat16()+ .eval()+ )+ z = torch.randn(1, 4, 1, 16, 12, device = "cuda", dtype = torch.bfloat16)+ x = torch.rand(1, 3, 1, 64, 48, device = "cuda", dtype = torch.bfloat16) * 2 - 1+ _check(vae, z, min_psnr = 40, encode_x = x)+++@needs_cuda+def test_hunyuanvideo15_fused_matches_stock():+ from diffusers import AutoencoderKLHunyuanVideo15++ torch.manual_seed(0)+ vae = AutoencoderKLHunyuanVideo15(+ in_channels = 3, out_channels = 3, latent_channels = 8, block_out_channels = (32, 64, 64, 64, 64),+ layers_per_block = 1, spatial_compression_ratio = 16, temporal_compression_ratio = 4,+ ).cuda().bfloat16().eval() # fmt: skip+ z = torch.randn(1, 8, 3, 4, 5, device = "cuda", dtype = torch.bfloat16)+ _check(vae, z, min_psnr = 40)+++@needs_cuda+def test_ltx2_fused_matches_stock():+ from diffusers import AutoencoderKLLTX2Video++ from core.inference.video_ltx2 import _VIDEO_VAE_CONFIG++ torch.manual_seed(0)+ cfg = dict(_VIDEO_VAE_CONFIG)+ cfg.update(+ latent_channels = 16, block_out_channels = (32, 64, 128, 128), decoder_block_out_channels = (32, 64, 64, 128),+ layers_per_block = (1, 1, 1, 1, 1), decoder_layers_per_block = (1, 1, 1, 1, 1),+ ) # fmt: skip+ vae = AutoencoderKLLTX2Video(**cfg).cuda().bfloat16().eval()+ z = torch.randn(1, 16, 3, 3, 4, device = "cuda", dtype = torch.bfloat16)+ _check(vae, z, min_psnr = 40)+++@needs_cuda+def test_ltx2_reflect_decoder_stays_stock(monkeypatch):+ from diffusers import AutoencoderKLLTX2Video++ from core.inference.video_ltx2 import _VIDEO_VAE_CONFIG++ monkeypatch.delenv("UNSLOTH_VAE_FUSED_CL_WEIGHTS", raising = False)+ cfg = dict(_VIDEO_VAE_CONFIG)+ cfg.update(+ latent_channels = 16, block_out_channels = (32, 64, 128, 128), decoder_block_out_channels = (32, 64, 64, 128),+ layers_per_block = (1, 1, 1, 1, 1), decoder_layers_per_block = (1, 1, 1, 1, 1),+ ) # fmt: skip+ convs = lambda v: [m.weight for m in v.decoder.modules() if isinstance(m, torch.nn.Conv3d)] # noqa: E731+ reflect = (+ AutoencoderKLLTX2Video(**{**cfg, "decoder_spatial_padding_mode": "reflect"})+ .cuda()+ .bfloat16()+ .eval()+ )+ assert F.will_install(reflect) is False+ assert F.install(reflect) == 0+ assert all(w.is_contiguous() for w in convs(reflect))+ assert not any("forward" in m.__dict__ for m in reflect.modules())+ zeros = AutoencoderKLLTX2Video(**cfg).cuda().bfloat16().eval()+ assert F.install(zeros) > 0+ assert all(w.is_contiguous(memory_format = torch.channels_last_3d) for w in convs(zeros))+++@needs_cuda+def test_guard_falls_back_to_stock_for_good(monkeypatch):+ from diffusers import AutoencoderKL++ torch.manual_seed(0)+ vae = AutoencoderKL(+ block_out_channels = (32,), down_block_types = ("DownEncoderBlock2D",), up_block_types = ("UpDecoderBlock2D",),+ latent_channels = 4, norm_num_groups = 32, layers_per_block = 1,+ ).cuda().eval() # fmt: skip+ z = torch.randn(1, 4, 8, 8, device = "cuda")+ with torch.inference_mode():+ ref = _decode(vae, z)+ F.install(vae)++ def boom(*a, **k):+ raise RuntimeError("kernel exploded")++ monkeypatch.setattr(F, "group_norm_act", boom)+ out = _decode(vae, z)+ assert torch.allclose(+ out, ref, atol = 5e-3+ ) # channels-last weights: another (TF32) cuDNN algorithm+ assert any(getattr(m, "_unsloth_vae_fused_failed", False) for m in vae.modules())+++@needs_cuda+def test_guard_restores_causal_cache_state_on_fallback(monkeypatch):+ # a fast residual block that dies AFTER writing its first cache slot must not desync the stock retry+ from diffusers import AutoencoderKLWan++ torch.manual_seed(0)+ vae = (+ AutoencoderKLWan(base_dim = 32, z_dim = 4, dim_mult = [1, 2, 2, 2], num_res_blocks = 1)+ .cuda()+ .half()+ .eval()+ )+ z = torch.randn(1, 4, 3, 8, 8, device = "cuda", dtype = torch.float16)+ with torch.inference_mode():+ ref = _decode(vae, z)+ F.install(vae)+ real = F.causal_conv+ calls = {"n": 0}++ def flaky(+ conv,+ x,+ cache = None,+ **kw,+ ):+ calls["n"] += 1+ if (+ calls["n"] == 2+ ): # conv2 of the first block: conv1 already advanced feat_idx and wrote a slot+ raise RuntimeError("boom")+ return real(conv, x, cache, **kw)++ monkeypatch.setattr(F, "causal_conv", flaky)+ out = _decode(vae, z)+ assert _psnr(out, ref) > 45+++def _single_head_attention(dim, device, dtype):+ from diffusers.models.attention_processor import Attention, AttnProcessor2_0++ attn = Attention(+ dim, heads = 1, dim_head = dim, rescale_output_factor = 1.0, eps = 1e-6, norm_num_groups = 32, bias = True,+ upcast_softmax = True, residual_connection = True, _from_deprecated_attn_block = True,+ processor = AttnProcessor2_0(),+ ).to(device, dtype).eval() # fmt: skip+ attn.processor = F.FusedSingleHeadProcessor(attn.processor)+ return attn+++def test_attention_fp32_or_cpu_input_goes_straight_to_stock(monkeypatch):+ pytest.importorskip("diffusers")+ attn = _single_head_attention(320, "cpu", torch.float32)+ monkeypatch.setattr(+ F.FusedSingleHeadProcessor, "_fused", staticmethod(lambda *a: pytest.fail("fused path ran"))+ )+ x = torch.randn(1, 320, 4, 4)+ with torch.inference_mode():+ out = attn(x)+ assert out.shape == x.shape+ assert not getattr(attn, "_unsloth_vae_fused_failed", False)+++@needs_cuda[email protected]("oom", [True, False])+def test_attention_oom_falls_back_for_that_call_only(monkeypatch, oom):+ pytest.importorskip("diffusers")+ attn = _single_head_attention(512, "cuda", torch.bfloat16)+ x = torch.randn(1, 512, 16, 16, device = "cuda", dtype = torch.bfloat16)+ with torch.inference_mode():+ ref = attn.processor.fallback(attn, x)++ def boom(*a, **k):+ raise (+ torch.cuda.OutOfMemoryError("CUDA out of memory")+ if oom+ else RuntimeError("kernel exploded")+ )++ monkeypatch.setattr(F, "single_head_attention", boom)+ out = attn(x)+ assert torch.equal(out, ref)+ # an OOM keeps the fused path for the next call; a real failure retires it+ assert getattr(attn, "_unsloth_vae_fused_failed", False) is (not oom)+++@needs_cuda+def test_qwenimage_tiled_decode_is_not_clamped_like_stock(monkeypatch):+ # diffusers' AutoencoderKLQwenImage.tiled_decode returns the raw decoder output (Wan's and Qwen-Image-2.1's clamp)+ from diffusers import AutoencoderKLQwenImage++ torch.manual_seed(0)+ vae = (+ AutoencoderKLQwenImage(base_dim = 32, z_dim = 4, dim_mult = [1, 2, 2, 2], num_res_blocks = 1)+ .cuda()+ .eval()+ )+ vae = vae.to(torch.bfloat16)+ vae.enable_tiling(tile_sample_min_height = 64, tile_sample_min_width = 64, tile_sample_stride_height = 48,+ tile_sample_stride_width = 48) # fmt: skip+ monkeypatch.setenv(F.TILE_BATCH_ENV, "3")+ with torch.no_grad():+ vae.decoder.conv_out.weight.mul_(64) # the decode lands far outside [-1, 1]+ z = torch.randn(1, 4, 1, 14, 18, device = "cuda", dtype = torch.bfloat16)+ with torch.inference_mode():+ ref = _decode(vae, z)+ assert ref.abs().max() > 1.5 # the case under test exists+ F.install(vae)+ out = _decode(vae, z)+ assert out.shape == ref.shape+ assert out.abs().max() > 1.5+ assert ((out - ref).norm() / ref.norm()).item() < 2e-2 # same values, unclamped+++def test_hv_causal_mask_allocates_nothing_quadratic_besides_the_output():+ # the stock row loop allocates only the seq_len x seq_len mask (6.4 GB extra at 80k tokens otherwise)+ from torch.utils._python_dispatch import TorchDispatchMode+ from torch.utils._pytree import tree_leaves++ class _Allocs(TorchDispatchMode):+ def __init__(self):+ super().__init__()+ self.storages = []++ def __torch_dispatch__(+ self,+ func,+ types,+ args = (),+ kwargs = None,+ ):+ out = func(*args, **(kwargs or {}))+ for t in tree_leaves(out):+ if isinstance(t, torch.Tensor):+ self.storages.append(+ (t.untyped_storage().data_ptr(), t.untyped_storage().nbytes())+ )+ return out++ mod = pytest.importorskip("diffusers.models.autoencoders.autoencoder_kl_hunyuanvideo15")+ n_frame, n_hw = 6, 32+ seq_len = n_frame * n_hw+ rec = _Allocs()+ with rec:+ mask = F._hv_causal_mask(n_frame, n_hw, torch.bfloat16, "cpu", batch_size = 2)+ own = mask.untyped_storage().data_ptr()+ assert [nb for ptr, nb in rec.storages if ptr != own and nb >= seq_len * seq_len] == []+ stock = mod.HunyuanVideo15AttnBlock.prepare_causal_attention_mask(+ n_frame, n_hw, torch.bfloat16, "cpu", batch_size = 2+ )+ assert torch.equal(mask, stock) and mask.stride() == stock.stride()+++@needs_cuda[email protected]("dtype", [torch.float16, torch.bfloat16, torch.float32])+def test_blend_seam_on_cuda_is_bit_identical_and_host_free(dtype):+ # a seam blend must not pull host tensors onto the device (a synchronous copy per seam)+ from diffusers import AutoencoderKLWan+ from torch.utils._python_dispatch import TorchDispatchMode+ from torch.utils._pytree import tree_leaves++ class _HostArgs(TorchDispatchMode):+ def __init__(self):+ super().__init__()+ self.ops = []++ def __torch_dispatch__(+ self,+ func,+ types,+ args = (),+ kwargs = None,+ ):+ if any(+ isinstance(t, torch.Tensor) and t.device.type == "cpu"+ for t in tree_leaves((args, kwargs))+ ):+ self.ops.append(str(func))+ return func(*args, **(kwargs or {}))++ g = torch.Generator("cuda").manual_seed(0)+ a = torch.randn(1, 3, 2, 40, 44, generator = g, device = "cuda").to(dtype)+ b = torch.randn(1, 3, 2, 40, 44, generator = g, device = "cuda").to(dtype)+ rec = _HostArgs()+ with rec:+ out = F.blend_seam(a, b.clone(), 17, -1)+ assert rec.ops == []+ assert torch.equal(AutoencoderKLWan.blend_h(None, a, b.clone(), 17), out)+++def test_wan_attention_oom_retries_stock_for_that_call_only(monkeypatch):+ # the fused attention holds fp32 score chunks the stock SDPA never allocates, so its OOM must not abort the decode+ diffusers = pytest.importorskip("diffusers")+ monkeypatch.setattr(F, "runtime_ok", lambda: True)+ torch.manual_seed(0)+ vae = diffusers.AutoencoderKLWan(+ base_dim = 32, z_dim = 4, dim_mult = [1, 2, 2, 2], num_res_blocks = 1+ ).eval()+ F.install_wan_vae(vae)+ block = next(m for m in vae.decoder.modules() if type(m).__name__.endswith("AttentionBlock"))+ x = torch.randn(1, block.norm.gamma.shape[0], 1, 4, 4)+ with torch.inference_mode():+ ref = type(block).forward(block, x)+ monkeypatch.setattr(F, "_mm_attention_ok", lambda q: True)++ def norm_5d(x, norm, act): # the fused norm is CUDA-only; per frame like the stock block+ y = norm(x.permute(0, 2, 1, 3, 4).flatten(0, 1)).unflatten(0, x.shape[::2][:2])+ return y.permute(0, 2, 1, 3, 4).contiguous(memory_format = torch.channels_last_3d)++ monkeypatch.setattr(F, "rms_norm_act", norm_5d)++ def oom(*a, **k):+ raise torch.cuda.OutOfMemoryError("CUDA out of memory")++ monkeypatch.setattr(F, "single_head_attention", oom)+ out = block(x)+ assert torch.equal(out, ref)+ assert not getattr(block, "_unsloth_vae_fused_failed", False)++[email protected]("frames", [1, 2, 4])+def test_causal_cache_is_a_compact_two_frame_copy(frames):+ # a view would pin the whole (frames + 2)-frame conv input per cache slot until the next chunk+ mod = pytest.importorskip("diffusers.models.autoencoders.autoencoder_kl_wan")+ conv = mod.WanCausalConv3d(8, 8, 3, padding = 1).eval()+ x = torch.randn(1, 8, frames, 4, 4)+ cache = torch.randn(1, 8, 2, 4, 4)+ with torch.inference_mode():+ out, new = F.causal_conv(conv, x, cache)+ ref = conv(x, cache)+ stock = torch.cat([cache, x], dim = 2)[:, :, -2:]+ assert torch.allclose(out, ref, atol = 1e-5)+ assert torch.equal(new, stock)+ assert new.untyped_storage().nbytes() == new.numel() * new.element_size()+++def test_pipeline_qkv_fuse_keeps_the_fused_vae_attention():+ # SDXL's pipe-level fuse_qkv_projections() resets every VAE processor; the fused one must be re-wrapped around it+ diffusers = pytest.importorskip("diffusers")+ from diffusers.models.attention_processor import FusedAttnProcessor2_0+ from diffusers.pipelines.pipeline_utils import StableDiffusionMixin++ from core.inference import diffusion_speed as S++ torch.manual_seed(0)+ vae = diffusers.AutoencoderKL(+ block_out_channels = (32,), down_block_types = ("DownEncoderBlock2D",), up_block_types = ("UpDecoderBlock2D",),+ latent_channels = 4, norm_num_groups = 32, layers_per_block = 1,+ ).eval() # fmt: skip+ unet = diffusers.UNet2DConditionModel(+ block_out_channels = (32, 64), layers_per_block = 1, sample_size = 8, in_channels = 4, out_channels = 4,+ down_block_types = ("DownBlock2D", "CrossAttnDownBlock2D"), up_block_types = ("CrossAttnUpBlock2D", "UpBlock2D"),+ cross_attention_dim = 32, norm_num_groups = 32,+ ).eval() # fmt: skip++ class Pipe(StableDiffusionMixin):+ pass++ pipe = Pipe()+ pipe.vae, pipe.unet = vae, unet+ assert F.install_attention_processors(vae) > 0+ assert S._fuse_qkv(pipe, None) is True+ procs = [m.processor for m in vae.modules() if type(m).__name__ == "Attention"]+ assert procs and all(isinstance(p, F.FusedSingleHeadProcessor) for p in procs)+ assert all(isinstance(p.fallback, FusedAttnProcessor2_0) for p in procs)+ assert all(isinstance(p, FusedAttnProcessor2_0) for p in unet.attn_processors.values())+++def _variants(kernel) -> int:+ caches = getattr(kernel, "device_caches", None)+ if caches is not None: # Triton >= 3.2: {device: (kernel_cache, ...)}+ return sum(len(c[0]) for c in caches.values())+ return sum(len(c) for c in getattr(kernel, "cache", {}).values())+++@needs_cuda+def test_shape_args_do_not_multiply_jit_variants():+ # sizes must reuse the compiled kernel (else ~90 JIT variants on a tiled Wan decode); strides stay specialized+ k = F._kernels()+ for name, args in F._SHAPE_ARGS.items():+ params = {p.name: p for p in getattr(k, name.lstrip("_")).params}+ assert set(args) <= set(params), (name, set(args) - set(params))+ assert all(params[a].do_not_specialize for a in args), name+ cl = torch.channels_last_3d+ seen = None+ for t, h, w, front in (+ (4, 16, 16, 2),+ (1, 16, 16, 2),+ (3, 13, 17, 1),+ (1, 7, 9, 2),+ (5, 1, 33, 0),+ ):+ x = torch.randn(1, 64, t, h, w, device = "cuda", dtype = torch.float16).contiguous(+ memory_format = cl+ )+ cache = torch.randn(1, 64, 1, h, w, device = "cuda", dtype = torch.float16).contiguous(+ memory_format = cl+ )+ out = F.rms_norm_act(x, None, act = True, front = front, cache = cache if front else None)+ ref = torch.nn.functional.silu(x)+ if front:+ ref = torch.cat([torch.zeros_like(cache[:, :, : front - 1]), cache, ref], 2)+ torch.testing.assert_close(+ out, ref, atol = 2e-3, rtol = 2e-3+ ) # fp32 silu vs fp16 opmath: last-ulp+ seen = seen or _variants(k.rms_act)+ assert _variants(k.rms_act) == seen, (t, h, w, front)