Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
"""Triton GroupNorm/RMS norm for eager VAE decode (NVIDIA CUDA, Triton>=3.3); patches instances
never classes; falls back to stock on new exception; UNSLOTH_VAE_FUSED=0 off"""
from __future__ import annotations
import contextlib
import os
import sys
import threading
import types
from functools import lru_cache
from typing import Any, Optional
from .rocm_bf16 import is_rocm_torch
VAE_FUSED_ENV = "UNSLOTH_VAE_FUSED"
DEVICE_SCOPE_ENV = "UNSLOTH_DIFFUSION_TRITON_DEVICE_SCOPE"
_FALSE = ("0", "false", "no", "off")
_MIN_TRITON = (3, 3)
_MAX_C = 4096
def fused_vae_disabled() -> bool:
return os.environ.get(VAE_FUSED_ENV, "").strip().lower() in _FALSE
@lru_cache(maxsize = 1)
def _triton_version_ok() -> bool:
try:
import re
import triton
m = re.match(r"(\d+)\.(\d+)", str(triton.__version__))
return bool(m) and (int(m.group(1)), int(m.group(2))) >= _MIN_TRITON
except Exception: # noqa: BLE001
return False
@lru_cache(maxsize = 1)
def _toolchain_ok() -> bool:
if sys.platform != "win32":
return True
try:
from .._msvc_env import crt_headers_reachable
return bool(crt_headers_reachable())
except Exception: # noqa: BLE001
return True
# Loop-bound args not specialized (avoids ~70 JIT variants); strides and C stay specialized so channel-contiguous loads vectorize.
_SHAPE_ARGS = {
"_rms_act": ("T", "H", "W", "Ho", "Wo", "ph", "pw", "To", "front", "n_cache"),
"_bias_residual": ("P", "T", "HW", "W"),
"_dup_up_add": ("T_o", "H_o", "W_o", "t_off"),
"_up_nearest2x": ("T_o", "H", "W"),
"_gn_partials": ("T", "HW", "W", "n_chunks"),
"_gn_combine": ("n_chunks",),
"_gn_apply": ("T", "HW", "W"),
"_softmax_rows": ("S",),
"_dcae_up_add": ("F", "H", "W", "Fo"),
}
@lru_cache(maxsize = 1)
def _kernels() -> Optional[types.SimpleNamespace]:
try:
import triton
import triton.language as tl
except Exception: # noqa: BLE001
return None
globals().update(triton = triton, tl = tl)
@triton.jit(do_not_specialize = _SHAPE_ARGS["_gn_partials"])
def _gn_partials(
x_ptr, bias_ptr, pn_ptr, pmean_ptr, pm2_ptr,
T, HW, W, n_chunks, sb, sc, st, sh, sw,
C: tl.constexpr, G: tl.constexpr, CPG: tl.constexpr, BLOCK_P: tl.constexpr, HAS_BIAS: tl.constexpr # C channels, G groupnorm groups, CPG channels per group, BLOCK_P pixels per program, HAS_BIAS bias present
): # fmt: skip
chunk = tl.program_id(0)
bt = tl.program_id(1)
t = bt % T
b = bt // T
p = chunk * BLOCK_P + tl.arange(0, BLOCK_P)
pmask = p < HW
hh = p // W
ww = p - hh * W
c = tl.arange(0, C)
off = (
b.to(tl.int64) * sb
+ t.to(tl.int64) * st
+ hh.to(tl.int64)[:, None] * sh
+ ww.to(tl.int64)[:, None] * sw
+ c.to(tl.int64)[None, :] * sc
)
v = tl.load(x_ptr + off, mask = pmask[:, None], other = 0.0).to(tl.float32)
if HAS_BIAS:
v = v + tl.load(bias_ptr + c).to(tl.float32)[None, :]
v = tl.where(pmask[:, None], v, 0.0)
n_pix = tl.sum(pmask.to(tl.float32), axis = 0)
v3 = tl.reshape(v, (BLOCK_P, G, CPG))
mean = tl.sum(tl.sum(v3, axis = 2), axis = 0) / (n_pix * CPG)
d = tl.where(pmask[:, None, None], v3 - mean[None, :, None], 0.0)
m2 = tl.sum(tl.sum(d * d, axis = 2), axis = 0)
slot = bt.to(tl.int64) * n_chunks + chunk
g = tl.arange(0, G)
tl.store(pmean_ptr + slot * G + g, mean)
tl.store(pm2_ptr + slot * G + g, m2)
tl.store(pn_ptr + slot, n_pix * CPG)
@triton.jit(do_not_specialize = _SHAPE_ARGS["_gn_combine"])
def _gn_combine(
pn_ptr,
pmean_ptr,
pm2_ptr,
mean_ptr,
rstd_ptr,
n_chunks,
G,
eps,
BLOCK: tl.constexpr, # block size
):
row = tl.program_id(0)
g = row % G
bt = (row // G).to(tl.int64)
zero = tl.sum(tl.zeros([BLOCK], dtype = tl.float32), axis = 0)
acc_n = zero
acc_s = zero
for k0 in range(0, n_chunks, BLOCK):
k = k0 + tl.arange(0, BLOCK)
km = k < n_chunks
n = tl.load(pn_ptr + bt * n_chunks + k, mask = km, other = 0.0)
mu = tl.load(pmean_ptr + (bt * n_chunks + k) * G + g, mask = km, other = 0.0)
acc_n += tl.sum(n, axis = 0)
acc_s += tl.sum(n * mu, axis = 0)
mean = acc_s / acc_n
acc_m2 = zero
for k0 in range(0, n_chunks, BLOCK):
k = k0 + tl.arange(0, BLOCK)
km = k < n_chunks
n = tl.load(pn_ptr + bt * n_chunks + k, mask = km, other = 0.0)
mu = tl.load(pmean_ptr + (bt * n_chunks + k) * G + g, mask = km, other = 0.0)
m2 = tl.load(pm2_ptr + (bt * n_chunks + k) * G + g, mask = km, other = 0.0)
dm = mu - mean
acc_m2 += tl.sum(m2 + n * dm * dm, axis = 0)
tl.store(mean_ptr + row, mean)
tl.store(rstd_ptr + row, 1.0 / tl.sqrt(acc_m2 / acc_n + eps))
@triton.jit(do_not_specialize = _SHAPE_ARGS["_gn_apply"])
def _gn_apply(
x_ptr, out_ptr, mean_ptr, rstd_ptr, w_ptr, b_ptr, ib_ptr,
C, T, HW, W, G, sb, sc, st, sh, sw,
CPG: tl.constexpr, ACT: tl.constexpr, HAS_IN_BIAS: tl.constexpr, BLOCK_P: tl.constexpr, BLOCK_C: tl.constexpr # CPG channels per group, ACT apply silu, HAS_IN_BIAS input bias present, BLOCK_P pixels per program, BLOCK_C channels per program
): # fmt: skip
pid_p = tl.program_id(0)
bt = tl.program_id(1)
pid_c = tl.program_id(2)
t = bt % T
b = bt // T
p = pid_p * BLOCK_P + tl.arange(0, BLOCK_P)
pmask = p < HW
hh = p // W
ww = p - hh * W
c = pid_c * BLOCK_C + tl.arange(0, BLOCK_C)
cmask = c < C
m = pmask[:, None] & cmask[None, :]
off = (
b.to(tl.int64) * sb
+ t.to(tl.int64) * st
+ hh.to(tl.int64)[:, None] * sh
+ ww.to(tl.int64)[:, None] * sw
+ c.to(tl.int64)[None, :] * sc
)
v = tl.load(x_ptr + off, mask = m, other = 0.0).to(tl.float32)
if HAS_IN_BIAS:
v = v + tl.load(ib_ptr + c, mask = cmask, other = 0.0).to(tl.float32)[None, :]
gidx = bt * G + c // CPG
mu = tl.load(mean_ptr + gidx, mask = cmask, other = 0.0)
rs = tl.load(rstd_ptr + gidx, mask = cmask, other = 0.0)
gw = tl.load(w_ptr + c, mask = cmask, other = 0.0).to(tl.float32)
gb = tl.load(b_ptr + c, mask = cmask, other = 0.0).to(tl.float32)
a = rs * gw
v = v * a[None, :] + (gb - mu * a)[None, :]
if ACT:
v = v / (1.0 + tl.exp(-v))
out_off = (bt.to(tl.int64) * HW + p.to(tl.int64))[:, None] * C + c[None, :]
tl.store(out_ptr + out_off, v.to(out_ptr.dtype.element_ty), mask = m)
@triton.jit(do_not_specialize = _SHAPE_ARGS["_rms_act"])
def _rms_act(
x_ptr, out_ptr, w_ptr, b_ptr, ib_ptr, cache_ptr,
C, T, H, W, Ho, Wo, ph, pw, To, front, n_cache, scale, eps,
sb, sc, st, sh, sw, cb, cc, ct, ch, cw,
NORM: tl.constexpr, ACT: tl.constexpr, HAS_BIAS: tl.constexpr, HAS_IN_BIAS: tl.constexpr, # NORM apply rms norm, ACT apply silu, HAS_BIAS bias present, HAS_IN_BIAS input bias present
REPLICATE: tl.constexpr, MEAN_SQ: tl.constexpr, BLOCK_P: tl.constexpr, BLOCK_C: tl.constexpr # REPLICATE replicate temporal padding, MEAN_SQ mean-square rms form, BLOCK_P pixels per program, BLOCK_C channels per program
): # fmt: skip
pid_p = tl.program_id(0)
bt = tl.program_id(1)
t_o = bt % To
b = bt // To
p = pid_p * BLOCK_P + tl.arange(0, BLOCK_P)
pmask = p < Ho * Wo
ho = p // Wo
wo = p - ho * Wo
if REPLICATE:
hh = tl.minimum(tl.maximum(ho - ph, 0), H - 1)
ww = tl.minimum(tl.maximum(wo - pw, 0), W - 1)
else:
hh = ho
ww = wo
c = tl.arange(0, BLOCK_C)
cmask = c < C
m = pmask[:, None] & cmask[None, :]
out_off = ((b.to(tl.int64) * To + t_o) * (Ho * Wo) + p.to(tl.int64))[:, None] * C + c[
None, :
]
if (not REPLICATE) and t_o < front:
tc = t_o - (front - n_cache)
if tc >= 0:
coff = (
b.to(tl.int64) * cb
+ tc.to(tl.int64) * ct
+ hh.to(tl.int64)[:, None] * ch
+ ww.to(tl.int64)[:, None] * cw
+ c.to(tl.int64)[None, :] * cc
)
v = tl.load(cache_ptr + coff, mask = m, other = 0.0)
tl.store(out_ptr + out_off, v.to(out_ptr.dtype.element_ty), mask = m)
else:
tl.store(
out_ptr + out_off,
tl.zeros([BLOCK_P, BLOCK_C], out_ptr.dtype.element_ty),
mask = m,
)
else:
t = tl.minimum(tl.maximum(t_o - front, 0), T - 1)
off = (
b.to(tl.int64) * sb
+ t.to(tl.int64) * st
+ hh.to(tl.int64)[:, None] * sh
+ ww.to(tl.int64)[:, None] * sw
+ c.to(tl.int64)[None, :] * sc
)
v = tl.load(x_ptr + off, mask = m, other = 0.0).to(tl.float32)
if HAS_IN_BIAS:
v = v + tl.load(ib_ptr + c, mask = cmask, other = 0.0).to(tl.float32)[None, :]
v = tl.where(m, v, 0.0)
if NORM:
ss = tl.sum(v * v, axis = 1)
if MEAN_SQ:
# LTX PerChannelRMSNorm: x / sqrt(mean(x^2) + eps), no affine
v = v / tl.sqrt(ss / C + eps)[:, None]
else:
# Wan-lineage RMS_norm: F.normalize(x) * sqrt(C) * gamma (+ bias)
inv = 1.0 / tl.maximum(tl.sqrt(ss), 1e-12)
gw = tl.load(w_ptr + c, mask = cmask, other = 0.0).to(tl.float32) * scale
v = v * inv[:, None] * gw[None, :]
if HAS_BIAS:
v = v + tl.load(b_ptr + c, mask = cmask, other = 0.0).to(tl.float32)[None, :]
if ACT:
v = v / (1.0 + tl.exp(-v))
tl.store(out_ptr + out_off, v.to(out_ptr.dtype.element_ty), mask = m)
@triton.jit(do_not_specialize = _SHAPE_ARGS["_bias_residual"])
def _bias_residual(
o_ptr, ob_ptr, r_ptr, rb_ptr, P, C, T, HW, W, sb, sc, st, sh, sw, inv_scale,
HAS_OB: tl.constexpr, HAS_RB: tl.constexpr, SCALE: tl.constexpr, BLOCK_P: tl.constexpr, BLOCK_C: tl.constexpr # HAS_OB output bias present, HAS_RB residual bias present, SCALE output scale != 1, BLOCK_P pixels per program, BLOCK_C channels per program
): # fmt: skip
p = tl.program_id(0) * BLOCK_P + tl.arange(0, BLOCK_P)
c = tl.program_id(1) * BLOCK_C + tl.arange(0, BLOCK_C)
pm = p < P
cm = c < C
m = pm[:, None] & cm[None, :]
o_off = p.to(tl.int64)[:, None] * C + c[None, :]
o = tl.load(o_ptr + o_off, mask = m, other = 0.0).to(tl.float32)
if HAS_OB:
o = o + tl.load(ob_ptr + c, mask = cm, other = 0.0).to(tl.float32)[None, :]
ww = p % W
rest = p // W
hh = rest % (HW // W)
bt = rest // (HW // W)
t = bt % T
b = bt // T
r_off = (
b.to(tl.int64) * sb + t.to(tl.int64) * st + hh.to(tl.int64) * sh + ww.to(tl.int64) * sw
)[:, None] + c.to(tl.int64)[None, :] * sc
r = tl.load(r_ptr + r_off, mask = m, other = 0.0).to(tl.float32)
if HAS_RB:
r = r + tl.load(rb_ptr + c, mask = cm, other = 0.0).to(tl.float32)[None, :]
o = o + r
if SCALE:
o = o * inv_scale
tl.store(o_ptr + o_off, o.to(o_ptr.dtype.element_ty), mask = m)
@triton.jit(do_not_specialize = _SHAPE_ARGS["_dcae_up_add"])
def _dcae_up_add(
h_ptr, hb_ptr, x_ptr, out_ptr,
c, Cin, F, H, W, Fo, rep,
hsb, hsc, hst, hsh, hsw, xsb, xsc, xst, xsh, xsw,
TEMPORAL: tl.constexpr, HAS_HB: tl.constexpr, BLOCK_P: tl.constexpr, BLOCK_C: tl.constexpr # TEMPORAL temporal upsample, HAS_HB conv bias present, BLOCK_P pixels per program, BLOCK_C channels per program
): # fmt: skip
pid_p = tl.program_id(0)
bf = tl.program_id(1)
pid_c = tl.program_id(2)
fo = bf % Fo
b = bf // Fo
Wo = 2 * W
p = pid_p * BLOCK_P + tl.arange(0, BLOCK_P)
pm = p < 4 * H * W
ho = p // Wo
wo = p - ho * Wo
hh = ho // 2
ww = wo // 2
ph = (ho - hh * 2) * 2 + (wo - ww * 2) # r2i * 2 + r3i
ci = pid_c * BLOCK_C + tl.arange(0, BLOCK_C)
cm = ci < c
m = pm[:, None] & cm[None, :]
if TEMPORAL:
if fo == 0:
f = fo * 0
pk = ph[:, None] * (2 * c) + ci[None, :]
xk = ph[:, None] * (Cin // 4) + ci[None, :] // (rep // 2)
else:
f = 1 + (fo - 1) // 2
r1 = (fo - 1) % 2
pk = (r1 * 4 + ph)[:, None] * c + ci[None, :]
xk = pk // rep
else:
f = fo
pk = ph[:, None] * c + ci[None, :]
xk = pk // rep
pix_h = (
b.to(tl.int64) * hsb
+ f.to(tl.int64) * hst
+ (hh.to(tl.int64) * hsh + ww.to(tl.int64) * hsw)[:, None]
)
v = tl.load(h_ptr + pix_h + pk.to(tl.int64) * hsc, mask = m, other = 0.0).to(tl.float32)
if HAS_HB:
v = v + tl.load(hb_ptr + pk, mask = m, other = 0.0).to(tl.float32)
v = v.to(out_ptr.dtype.element_ty).to(
tl.float32
) # the stock conv output is rounded before the add
pix_x = (
b.to(tl.int64) * xsb
+ f.to(tl.int64) * xst
+ (hh.to(tl.int64) * xsh + ww.to(tl.int64) * xsw)[:, None]
)
xv = tl.load(x_ptr + pix_x + xk.to(tl.int64) * xsc, mask = m, other = 0.0).to(tl.float32)
o_off = (bf.to(tl.int64) * (4 * H * W) + p.to(tl.int64))[:, None] * c + ci[None, :]
tl.store(out_ptr + o_off, (v + xv).to(out_ptr.dtype.element_ty), mask = m)
@triton.jit(do_not_specialize = _SHAPE_ARGS["_softmax_rows"])
def _softmax_rows(s_ptr, p_ptr, S, P_STRIDE, scale, BLOCK: tl.constexpr): # block size
row = tl.program_id(0).to(tl.int64)
base = s_ptr + row * P_STRIDE
m = tl.full([BLOCK], float("-inf"), tl.float32)
l = tl.zeros([BLOCK], tl.float32)
for k0 in range(0, S, BLOCK):
cols = k0 + tl.arange(0, BLOCK)
x = tl.load(base + cols, mask = cols < S, other = float("-inf")) * scale
m_new = tl.maximum(m, x)
alpha = tl.where(m_new == float("-inf"), 0.0, tl.exp(m - m_new))
l = l * alpha + tl.where(x == float("-inf"), 0.0, tl.exp(x - m_new))
m = m_new
m_row = tl.max(m, axis = 0)
l_row = tl.sum(l * tl.where(m == float("-inf"), 0.0, tl.exp(m - m_row)), axis = 0)
inv = 1.0 / l_row
for k0 in range(0, P_STRIDE, BLOCK):
cols = k0 + tl.arange(0, BLOCK)
x = tl.load(base + cols, mask = cols < S, other = float("-inf")) * scale
p = tl.where(cols < S, tl.exp(x - m_row) * inv, 0.0)
tl.store(
p_ptr + row * P_STRIDE + cols, p.to(p_ptr.dtype.element_ty), mask = cols < P_STRIDE
)
@triton.jit(do_not_specialize = _SHAPE_ARGS["_dup_up_add"])
def _dup_up_add(
o_ptr, x_ptr, C, T_o, H_o, W_o, t_off, repeats,
osb, osc, ost, osh, osw, xsb, xsc, xst, xsh, xsw,
FT: tl.constexpr, FS: tl.constexpr, BLOCK_P: tl.constexpr, BLOCK_C: tl.constexpr # FT temporal upsample factor, FS spatial upsample factor, BLOCK_P pixels per program, BLOCK_C channels per program
): # fmt: skip
pid_p = tl.program_id(0)
bt = tl.program_id(1)
pid_c = tl.program_id(2)
t = bt % T_o
b = bt // T_o
p = pid_p * BLOCK_P + tl.arange(0, BLOCK_P)
pm = p < H_o * W_o
ho = p // W_o
wo = p - ho * W_o
c = pid_c * BLOCK_C + tl.arange(0, BLOCK_C)
cm = c < C
m = pm[:, None] & cm[None, :]
tt = t + t_off
ti = tt // FT
a = tt - ti * FT
hi = ho // FS
i = ho - hi * FS
wi = wo // FS
j = wo - wi * FS
k = ((c[None, :] * FT + a) * FS + i[:, None]) * FS + j[:, None]
src = k // repeats
x_off = (b.to(tl.int64) * xsb + ti.to(tl.int64) * xst + (hi.to(tl.int64) * xsh + wi.to(tl.int64) * xsw)[:, None]
+ src.to(tl.int64) * xsc) # fmt: skip
o_off = (b.to(tl.int64) * osb + t.to(tl.int64) * ost + (ho.to(tl.int64) * osh + wo.to(tl.int64) * osw)[:, None]
+ c.to(tl.int64)[None, :] * osc) # fmt: skip
xv = tl.load(x_ptr + x_off, mask = m, other = 0.0).to(tl.float32)
ov = tl.load(o_ptr + o_off, mask = m, other = 0.0).to(tl.float32)
tl.store(o_ptr + o_off, (ov + xv).to(o_ptr.dtype.element_ty), mask = m)
@triton.jit(do_not_specialize = _SHAPE_ARGS["_up_nearest2x"])
def _up_nearest2x(
x_ptr, out_ptr, C, T_o, H, W, xsb, xsc, xst, xsh, xsw,
INTERLEAVE: tl.constexpr, BLOCK_P: tl.constexpr, BLOCK_C: tl.constexpr # INTERLEAVE wan time_conv channel interleave, BLOCK_P pixels per program, BLOCK_C channels per program
): # fmt: skip
# INTERLEAVE: x is Wan time_conv output (B, 2C, T_o/2, H, W); frame 2t+j takes channels [jC, (j+1)C).
pid_p = tl.program_id(0)
bt = tl.program_id(1)
pid_c = tl.program_id(2)
t = bt % T_o
b = bt // T_o
p = pid_p * BLOCK_P + tl.arange(0, BLOCK_P)
pm = p < 4 * H * W
ho = p // (2 * W)
wo = p - ho * (2 * W)
c = pid_c * BLOCK_C + tl.arange(0, BLOCK_C)
cm = c < C
m = pm[:, None] & cm[None, :]
if INTERLEAVE:
ti = t // 2
ch = (t - ti * 2) * C + c
else:
ti = t
ch = c
x_off = (b.to(tl.int64) * xsb + ti.to(tl.int64) * xst
+ ((ho // 2).to(tl.int64) * xsh + (wo // 2).to(tl.int64) * xsw)[:, None] + ch.to(tl.int64)[None, :] * xsc) # fmt: skip
v = tl.load(x_ptr + x_off, mask = m, other = 0.0)
o_off = (bt.to(tl.int64) * (4 * H * W) + p.to(tl.int64))[:, None] * C + c[None, :]
tl.store(out_ptr + o_off, v, mask = m)
return types.SimpleNamespace(
gn_partials = _gn_partials,
gn_combine = _gn_combine,
gn_apply = _gn_apply,
rms_act = _rms_act,
bias_residual = _bias_residual,
dcae_up_add = _dcae_up_add,
softmax_rows = _softmax_rows,
dup_up_add = _dup_up_add,
up_nearest2x = _up_nearest2x,
)
def _next_pow2(n: int) -> int:
return 1 << max(0, int(n) - 1).bit_length()
def runtime_ok() -> bool:
"""NVIDIA CUDA + a usable Triton; everything else keeps the stock path."""
if fused_vae_disabled():
return False
try:
import torch
except Exception: # noqa: BLE001
return False
if not torch.cuda.is_available() or is_rocm_torch(torch):
return False
return _triton_version_ok() and _toolchain_ok() and _kernels() is not None
def _as5d(x: Any) -> tuple:
"""(tensor viewed as B,C,T,H,W, was_4d)."""
if x.dim() == 4:
return x.unsqueeze(2), True
if x.dim() == 3: # (B, C, L): attention's GroupNorm input
return x.unsqueeze(2).unsqueeze(2), True
return x, False
def group_norm_reference(
x: Any,
norm: Any,
act: bool,
in_bias: Any = None,
) -> Any:
import torch.nn.functional as F
if in_bias is not None:
x = x + in_bias.view(1, -1, *([1] * (x.dim() - 2))).to(x.dtype)
y = F.group_norm(x, norm.num_groups, norm.weight, norm.bias, norm.eps)
return F.silu(y) if act else y
def _gn_fusable(x: Any, norm: Any) -> bool:
c = x.shape[1] if x.dim() >= 2 else 0
g = int(getattr(norm, "num_groups", 0) or 0)
return (
x.is_cuda
and x.dim() in (3, 4, 5)
and g > 0
and c % g == 0
and c & (c - 1) == 0
and g & (g - 1) == 0
and c <= 1024
and getattr(norm, "affine", True)
and norm.weight is not None
and x.dtype in (_torch().float16, _torch().bfloat16, _torch().float32)
and x.numel() > 0
)
def _torch():
import torch
return torch
_NO_SCOPE = contextlib.nullcontext()
def _on_card(t: Any) -> Any:
"""Triton launches on the current device: a tensor on another card fails the launch."""
torch = _torch()
if not t.is_cuda or torch.compiler.is_compiling():
return _NO_SCOPE
index = t.device.index
# same card (every single-card render): no device switch, ~1 us instead of ~4 us per launch
if index is None or index == torch.cuda.current_device():
return _NO_SCOPE
if os.environ.get(DEVICE_SCOPE_ENV, "").strip().lower() in _FALSE:
return _NO_SCOPE
return torch.cuda.device(index)
def _stock_in(t: Any) -> Any:
"""A kernel input as the kernel may read it."""
cls = globals().get("_FusedActCls")
if cls is None or type(t) is not cls:
return t
if t.__dict__.get("_unsloth_is_stock", False):
return _plain(t)
t._unsloth_abort()
def group_norm_stats(
x5: Any,
norm: Any,
in_bias: Any = None,
) -> tuple:
"""fp32 ``(mean, rstd)`` per (B*T, group) of ``x5 + in_bias`` (a fusable 5-D B,C,T,H,W view; stats per frame)."""
torch = _torch()
k = _kernels()
b, c, t, h, w = x5.shape
g = int(norm.num_groups)
hw = h * w
sb, sc, st, sh, sw = x5.stride()
ib = in_bias if in_bias is not None else x5
budget = 4096 if x5.element_size() == 4 else 8192
block_p = max(1, min(256, budget // c))
n_chunks = (hw + block_p - 1) // block_p
rows = b * t
pmean, pm2 = torch.empty((2, rows * n_chunks * g), dtype = torch.float32, device = x5.device)
pn = torch.empty(rows * n_chunks, dtype = torch.float32, device = x5.device)
with _on_card(x5):
k.gn_partials[(n_chunks, rows)](
x5, ib, pn, pmean, pm2, t, hw, w, n_chunks, sb, sc, st, sh, sw,
C = c, G = g, CPG = c // g, BLOCK_P = block_p, HAS_BIAS = in_bias is not None, num_warps = 4,
) # fmt: skip
mean = torch.empty(rows * g, dtype = torch.float32, device = x5.device)
rstd = torch.empty(rows * g, dtype = torch.float32, device = x5.device)
with _on_card(pn):
k.gn_combine[(rows * g,)](
pn, pmean, pm2, mean, rstd, n_chunks, g, float(norm.eps), BLOCK = 1024, num_warps = 4
)
return mean, rstd
def group_norm_act(
x: Any,
norm: Any,
act: bool = True,
in_bias: Any = None,
on_kernel: Any = None,
) -> Any:
"""``silu?(group_norm(x + in_bias))`` for 3/4/5-D ``x``; channels-last output; 5-D stats span T like nn.GroupNorm.
``on_kernel``: called once the Triton kernels completed for this call (not on a stock fallback)."""
torch = _torch()
x, in_bias = _stock_in(x), _stock_in(in_bias)
if not _gn_fusable(x, norm):
return group_norm_reference(x, norm, act, in_bias)
ndim = x.dim()
x5, _ = _as5d(x)
b, c, t, h, w = x5.shape
# nn.GroupNorm pools the whole (T, H, W) extent: flatten T into H when strides allow, else fall back
if t > 1:
if x5.stride(2) == h * x5.stride(3):
x5 = x5.as_strided(
(b, c, 1, t * h, w), (x5.stride(0), x5.stride(1), 0, x5.stride(3), x5.stride(4))
)
b, c, t, h, w = x5.shape
else:
return group_norm_reference(x, norm, act, in_bias)
k = _kernels()
g = int(norm.num_groups)
cpg = c // g
hw = h * w
sb, sc, st, sh, sw = x5.stride()
ib = in_bias if in_bias is not None else x5
rows = b * t
mean, rstd = group_norm_stats(x5, norm, in_bias)
out = torch.empty((b, t, h, w, c), dtype = x.dtype, device = x.device)
block_c = min(128, _next_pow2(c))
block_p2 = max(1, 4096 // block_c)
grid = ((hw + block_p2 - 1) // block_p2, rows, (c + block_c - 1) // block_c)
with _on_card(x5):
k.gn_apply[grid](
x5, out, mean, rstd, norm.weight, norm.bias, ib, c, t, hw, w, g, sb, sc, st, sh, sw,
CPG = cpg, ACT = bool(act), HAS_IN_BIAS = in_bias is not None, BLOCK_P = block_p2, BLOCK_C = block_c,
num_warps = 4,
) # fmt: skip
if on_kernel is not None and not torch.compiler.is_compiling():
on_kernel() # read by the structural calibration: only a completed Triton launch counts
out = out.permute(0, 4, 1, 2, 3) # B, C, T', H', W'
shape = x.shape
if ndim == 5:
return out.reshape(b, c, *shape[2:]) if out.shape != shape else out
if ndim == 4:
return out[:, :, 0]
return out[:, :, 0].reshape(shape) # (B, C, L) view over the (B, L, C) buffer
def rms_norm_reference(
x: Any,
norm: Any,
act: bool,
in_bias: Any = None,
) -> Any:
import torch.nn.functional as F
if in_bias is not None:
x = x + in_bias.view(1, -1, *([1] * (x.dim() - 2))).to(x.dtype)
y = x if norm is None else norm(x)
return F.silu(y) if act else y
def _is_pixel_norm(norm: Any) -> bool:
return type(norm).__name__ == "PerChannelRMSNorm" and getattr(norm, "channel_dim", 1) == 1
def _rms_params(norm: Any) -> Optional[tuple]:
torch = _torch()
if _is_pixel_norm(norm):
return None, None, False, 1.0
gamma = norm.gamma.reshape(-1)
bias = norm.bias
has_bias = isinstance(bias, torch.Tensor)
if not has_bias and float(bias) != 0.0:
return None
return gamma, (bias.reshape(-1) if has_bias else gamma), has_bias, float(norm.scale)
def _rms_fusable(x: Any, norm: Any, cache: Any) -> bool:
torch = _torch()
c = x.shape[1]
return (
x.is_cuda
and x.dim() == 5
and c <= _MAX_C
and x.dtype in (torch.float16, torch.bfloat16, torch.float32)
and x.numel() > 0
and (
cache is None
or (
torch.is_tensor(cache)
and cache.dim() == 5
and cache.shape[:2] == x.shape[:2]
and cache.shape[3:] == x.shape[3:]
and cache.device == x.device
)
)
and (
norm is None
or _is_pixel_norm(norm)
or (getattr(norm, "channel_first", False) and _rms_params(norm) is not None)
)
)
def rms_norm_act(
x: Any,
norm: Any,
act: bool = True,
*,
front: int = 0,
cache: Any = None,
in_bias: Any = None,
replicate_pad: Optional[tuple] = None,
back: int = 0,
) -> Any:
"""``cat([cache or zeros] -> front frames, silu?(rms_norm(x + in_bias)))`` along T, one channels-last_3d tensor.
``replicate_pad=(ph, pw)``: HunyuanVideo-1.5 replicate padding instead (``cache`` must be None); ``back`` repeats the
last frame (LTX-2)."""
x, in_bias = _stock_in(x), _stock_in(in_bias)
torch = _torch()
if replicate_pad is not None and cache is not None:
raise ValueError("replicate padding takes no cache")
if back and replicate_pad is None:
raise ValueError("back frames need replicate padding")
if not _rms_fusable(x, norm, cache):
y = rms_norm_reference(x, norm, act, in_bias)
if replicate_pad is not None:
import torch.nn.functional as F
ph, pw = replicate_pad
y = F.pad(y.float(), (pw, pw, ph, ph, front, back), mode = "replicate").to(y.dtype)
return y.contiguous(memory_format = torch.channels_last_3d)
if front:
import torch.nn.functional as F
n_cache = 0
if cache is not None:
cache = cache[:, :, -front:].to(y.dtype)
n_cache = cache.shape[2]
y = torch.cat([cache, y], dim = 2)
y = F.pad(y, (0, 0, 0, 0, front - n_cache, 0))
return y.contiguous(memory_format = torch.channels_last_3d)
k = _kernels()
mean_sq = norm is not None and _is_pixel_norm(norm)
if norm is not None and not mean_sq:
gamma, bias, has_bias, scale = _rms_params(norm)
else:
gamma, bias, has_bias, scale = x, x, False, 1.0
eps = float(getattr(norm, "eps", 0.0) or 0.0) if mean_sq else 0.0
b, c, t, h, w = x.shape
ph, pw = replicate_pad if replicate_pad is not None else (0, 0)
ho, wo = h + 2 * ph, w + 2 * pw
to = t + front + back
n_cache = 0 if cache is None else min(front, cache.shape[2])
cache_t = x if cache is None else cache[:, :, cache.shape[2] - n_cache :]
out = torch.empty((b, to, ho, wo, c), dtype = x.dtype, device = x.device)
block_c = _next_pow2(c)
block_p = max(1, min(128, 8192 // block_c))
grid = ((ho * wo + block_p - 1) // block_p, b * to)
with _on_card(x):
k.rms_act[grid](
x, out, gamma, bias, in_bias if in_bias is not None else x, cache_t,
c, t, h, w, ho, wo, ph, pw, to, front, n_cache, scale, eps,
*x.stride(), *cache_t.stride(),
NORM = norm is not None, ACT = bool(act), HAS_BIAS = has_bias, HAS_IN_BIAS = in_bias is not None,
REPLICATE = replicate_pad is not None, MEAN_SQ = mean_sq, BLOCK_P = block_p, BLOCK_C = block_c,
num_warps = 4 if block_c * block_p <= 4096 else 8,
) # fmt: skip
return out.permute(0, 4, 1, 2, 3)
def _conv_kind(conv: Any) -> Optional[str]:
torch = _torch()
pad = getattr(conv, "_padding", None)
if pad is None or getattr(conv, "padding_mode", "zeros") != "zeros" or conv.groups != 1:
return None
if (
isinstance(conv, torch.nn.Conv3d)
and len(pad) == 6
and pad[5] == 0
and pad[0] == pad[1]
and pad[2] == pad[3]
):
return "3d"
if (
isinstance(conv, torch.nn.Conv2d)
and len(pad) == 4
and pad[0] == pad[1]
and pad[2] == pad[3]
):
return "2d"
return None
def causal_conv(
conv: Any,
x: Any,
cache: Any = None,
*,
norm: Any = None,
act: bool = False,
in_bias: Any = None,
with_bias: bool = True,
) -> tuple:
"""Wan-lineage causal conv of ``silu?(rms?(x + in_bias))``; returns (out, new_cache).
``new_cache`` is a compact copy so the cache never pins the whole input."""
import torch.nn.functional as F
kind = _conv_kind(conv)
bias = conv.bias if with_bias else None
if kind == "2d": # Qwen-Image-2.1: the one-frame specialisation, never cached
if cache is not None:
raise ValueError("2D causal conv takes no cache")
y = (
x
if (norm is None and not act and in_bias is None)
else rms_norm_act(x, norm, act, in_bias = in_bias)
)
pw, _, ph, _ = conv._padding
out = F.conv2d(y[:, :, 0], conv.weight, bias, conv.stride, (ph, pw), conv.dilation)
return out.unsqueeze(2), None
if kind != "3d":
raise ValueError("unsupported causal conv " + type(conv).__name__)
pw, _, ph, _, front, _ = conv._padding
kt = conv.kernel_size[0]
spatial = (0, ph, pw)
t = x.shape[2]
if front == 0:
y = (
x
if (norm is None and not act and in_bias is None)
else rms_norm_act(x, norm, act, in_bias = in_bias)
)
return F.conv3d(y, conv.weight, bias, conv.stride, spatial, conv.dilation), None
if (
cache is None
and t == 1
and front == kt - 1
and conv.stride[0] == 1
and conv.dilation[0] == 1
):
y = rms_norm_act(x, norm, act, in_bias = in_bias)
weight = conv.weight[:, :, -1:]
return F.conv3d(y, weight, bias, conv.stride, spatial, conv.dilation), y
p = rms_norm_act(x, norm, act, front = front, cache = cache, in_bias = in_bias)
out = F.conv3d(p, conv.weight, bias, conv.stride, spatial, conv.dilation)
new_cache = p[:, :, -2:]
return out, (new_cache if p.shape[2] <= 2 else new_cache.clone())
# Single-head VAE attention: head_dim > 256 has no flash / cuDNN kernel, so two GEMMs + fp32 softmax beat SDPA.
_ATTN_SCORE_BYTES = 256 * 2**20
def single_head_attention(q: Any, k: Any, v: Any) -> Any:
"""softmax(q k^T / sqrt(d)) v for (B, L, D) half-precision q/k/v, query-chunked to bound the fp32 scores."""
torch = _torch()
b, length, d = q.shape
s_len = k.shape[1]
scale = d**-0.5
# keys padded to a multiple of 64 (cuBLAS slow path on odd extents); padded keys get probability exactly 0
s_pad = (s_len + 63) // 64 * 64
if s_pad != s_len:
kp = torch.zeros((b, s_pad, d), dtype = k.dtype, device = k.device)
vp = torch.zeros((b, s_pad, v.shape[-1]), dtype = v.dtype, device = v.device)
kp[:, :s_len] = k
vp[:, :s_len] = v
k, v = kp, vp
rows = int(max(64, min(length, _ATTN_SCORE_BYTES // max(1, b * s_pad * 4))) // 64 * 64)
kt = k.transpose(1, 2)
out = torch.empty((b, length, v.shape[-1]), dtype = v.dtype, device = v.device)
kern = _kernels()
for i in range(0, length, rows):
scores = torch.bmm(q[:, i : i + rows], kt, out_dtype = torch.float32)
r = scores.shape[1]
probs = torch.empty((b, r, s_pad), dtype = v.dtype, device = v.device)
with _on_card(scores):
kern.softmax_rows[(b * r,)](scores, probs, s_len, s_pad, scale, BLOCK = 2048, num_warps = 8)
del scores
torch.bmm(probs, v, out = out[:, i : i + rows])
return out
def _mm_attention_ok(q: Any) -> bool:
torch = _torch()
return q.is_cuda and q.dtype in (torch.float16, torch.bfloat16) and q.shape[-1] > 256
class FusedSingleHeadProcessor:
"""``AttnProcessor2_0`` using :func:`single_head_attention`; anything else goes to ``fallback``."""
def __init__(self, fallback: Any):
self.fallback = fallback
def __call__(self, attn: Any, hidden_states: Any, encoder_hidden_states: Any = None, attention_mask: Any = None,
temb: Any = None, *args: Any, **kwargs: Any,) -> Any: # fmt: skip
if (
args
or kwargs
or encoder_hidden_states is not None
or attention_mask is not None
or attn.heads != 1
or attn.spatial_norm is not None
or attn.norm_q is not None
or attn.norm_k is not None
or hidden_states.ndim != 4
or getattr(attn, "_unsloth_vae_fused_failed", False)
or not _attention_input_ok(attn, hidden_states)
):
return self.fallback(
attn, hidden_states, encoder_hidden_states, attention_mask, temb, *args, **kwargs
)
try:
return self._fused(attn, hidden_states)
except Exception as exc: # noqa: BLE001
# OOM falls back for this call only; anything else for good. Outside the handler so failed tensors are freed.
if not _is_oom(exc):
attn._unsloth_vae_fused_failed = True
return self.fallback(attn, hidden_states, encoder_hidden_states, attention_mask, temb)
@staticmethod
def _fused(attn: Any, hidden_states: Any) -> Any:
residual = hidden_states
b, c, h, w = hidden_states.shape
x = hidden_states.view(b, c, h * w).transpose(1, 2)
if attn.group_norm is not None:
x = attn.group_norm(x.transpose(1, 2)).transpose(1, 2)
q, k, v = attn.to_q(x), attn.to_k(x), attn.to_v(x)
o = single_head_attention(q, k, v)
o = attn.to_out[1](attn.to_out[0](o))
o = o.transpose(-1, -2).reshape(b, c, h, w)
if attn.residual_connection:
o = o + residual
return o / attn.rescale_output_factor
def _attention_input_ok(attn: Any, hidden_states: Any) -> bool:
"""Checked before any work, so an fp32 (upcast) or CPU VAE never pays for projections it then discards."""
torch = _torch()
width = getattr(getattr(attn, "to_q", None), "out_features", 0)
return (
hidden_states.is_cuda
and hidden_states.dtype in (torch.float16, torch.bfloat16)
and width > 256
)
def install_attention_processors(part: Any) -> int:
n = 0
for m in part.modules():
if (
type(m).__name__ == "Attention"
and getattr(m, "heads", None) == 1
and callable(getattr(m, "processor", None))
):
if not isinstance(m.processor, FusedSingleHeadProcessor):
m.processor = FusedSingleHeadProcessor(m.processor)
n += 1
return n
def _fast_wan_attention(block: Any) -> Any:
import torch.nn.functional as F
def fast(x: Any) -> Any:
identity = x
b, c, t, h, w = x.shape
y = rms_norm_act(x, block.norm, False)
y = y.permute(0, 2, 1, 3, 4).reshape(b * t, c, h, w) # a channels-last view, no copy
qkv = F.conv2d(y, block.to_qkv.weight, block.to_qkv.bias)
qkv = qkv.permute(0, 2, 3, 1).reshape(b * t, h * w, 3 * c)
q, k, v = qkv[..., :c], qkv[..., c : 2 * c], qkv[..., 2 * c :]
if not _mm_attention_ok(q):
return _stock_forward(block)(x)
o = single_head_attention(q, k, v) # (b*t, hw, c)
o = F.linear(o, block.proj.weight.view(c, c), block.proj.bias)
o = o.view(b, t, h, w, c).permute(0, 4, 1, 2, 3)
return add_bias_residual(
o.contiguous(memory_format = _torch().channels_last_3d), None, identity
)
return fast
def conv_nobias(conv: Any, x: Any) -> Any:
"""``conv(x)`` minus the bias; the caller adds ``conv.bias`` in a later fused pass."""
import torch.nn.functional as F
if getattr(conv, "padding_mode", "zeros") != "zeros":
raise ValueError("conv_nobias: padding_mode " + str(conv.padding_mode))
fn = F.conv2d if x.dim() == 4 else F.conv3d
return fn(x, conv.weight, None, conv.stride, conv.padding, conv.dilation, conv.groups)
def _cl_contig(t: Any) -> bool:
torch = _torch()
fmt = torch.channels_last if t.dim() == 4 else torch.channels_last_3d
return t.is_contiguous(memory_format = fmt)
def add_bias_residual(
out: Any,
out_bias: Any,
res: Any,
res_bias: Any = None,
scale: float = 1.0,
) -> Any:
"""``(out + out_bias + res + res_bias) / scale`` written into ``out`` (a fresh channels-last conv output)."""
torch = _torch()
k = _kernels()
if (
k is None
or not out.is_cuda
or out.dim() not in (4, 5)
or not _cl_contig(out)
or res.shape != out.shape
):
if out_bias is not None:
out.add_(out_bias.view(1, -1, *([1] * (out.dim() - 2))).to(out.dtype))
if res_bias is not None:
res = res + res_bias.view(1, -1, *([1] * (res.dim() - 2))).to(res.dtype)
out.add_(res)
return out if scale == 1.0 else out.div_(scale)
r5, _ = _as5d(res)
o5, _ = _as5d(out)
b, c, t, h, w = o5.shape
pixels = b * t * h * w
block_c = min(128, _next_pow2(c))
block_p = max(1, 4096 // block_c)
grid = ((pixels + block_p - 1) // block_p, (c + block_c - 1) // block_c)
with _on_card(out):
k.bias_residual[grid](
out, out_bias if out_bias is not None else out, res, res_bias if res_bias is not None else res,
pixels, c, t, h * w, w, *r5.stride(), 1.0 / float(scale),
HAS_OB = out_bias is not None, HAS_RB = res_bias is not None, SCALE = float(scale) != 1.0,
BLOCK_P = block_p, BLOCK_C = block_c, num_warps = 4,
) # fmt: skip
return out
def _is_oom(exc: BaseException) -> bool:
try:
if isinstance(exc, _torch().cuda.OutOfMemoryError):
return True
except Exception: # noqa: BLE001
pass
return "out of memory" in str(exc).lower()
def _guard(
module: Any,
fast: Any,
stock: Any,
label: str,
logger: Any,
oom_stock: bool = False,
) -> None:
"""Falls back to stock for good once fast raises what stock does not; causal-cache state is restored before the stock retry; OOM retries stock once."""
def forward(*args, **kwargs):
if getattr(module, "_unsloth_vae_fused_failed", False):
return stock(*args, **kwargs)
lists = [a for a in (*args, *kwargs.values()) if isinstance(a, list)]
snap = [a[:] for a in lists]
try:
return fast(*args, **kwargs)
except Exception as exc: # noqa: BLE001
if isinstance(exc, FusedActivationRead):
raise
for a, saved in zip(lists, snap):
a[:] = saved
if not _is_oom(exc):
out = stock(*args, **kwargs)
module._unsloth_vae_fused_failed = True
if logger is not None:
logger.warning(
"diffusion.vae_fused: fused %s failed, using the stock path: %s", label, exc
)
return out
if not oom_stock:
raise
return stock(
*args, **kwargs
) # outside the handler, so the failed call's tensors are freed first
forward._unsloth_vae_fused = True
module.forward = forward
def _stock_forward(module: Any) -> Any:
return type(module).forward.__get__(module)
def _resnet2d_fusable(block: Any) -> bool:
torch = _torch()
return (
isinstance(getattr(block, "norm1", None), torch.nn.GroupNorm)
and isinstance(getattr(block, "norm2", None), torch.nn.GroupNorm)
and isinstance(getattr(block, "nonlinearity", None), torch.nn.SiLU)
and getattr(block, "upsample", None) is None
and getattr(block, "downsample", None) is None
and getattr(block, "time_emb_proj", None) is None
# "group" / "spatial" name the temb norm; without temb (a VAE) both run plain norm2 like "default"
and getattr(block, "time_embedding_norm", "default") in ("default", "group")
and isinstance(getattr(block, "conv1", None), torch.nn.Conv2d)
and isinstance(getattr(block, "conv2", None), torch.nn.Conv2d)
)
def _fast_resnet2d(block: Any) -> Any:
def fast(
input_tensor: Any,
temb: Any = None,
*args: Any,
**kwargs: Any,
) -> Any:
if temb is not None or args or kwargs or block.training:
return _stock_forward(block)(input_tensor, temb, *args, **kwargs)
h = group_norm_act(input_tensor, block.norm1, act = True)
h = conv_nobias(block.conv1, h)
h = group_norm_act(h, block.norm2, act = True, in_bias = block.conv1.bias)
h = conv_nobias(block.conv2, h)
shortcut = (
block.conv_shortcut(input_tensor) if block.conv_shortcut is not None else input_tensor
)
return add_bias_residual(
h, block.conv2.bias, shortcut, None, float(block.output_scale_factor)
)
return fast
def _fast_norm(norm: Any, act: bool) -> Any:
def fast(x: Any) -> Any:
return group_norm_act(x, norm, act = act)
return fast
def install_group_norm_vae(
vae: Any,
logger: Any = None,
encoder: bool = True,
) -> int:
"""Patch every eligible ResnetBlock2D / GroupNorm of a 2-D (AutoencoderKL-family) VAE. Returns patch count."""
torch = _torch()
if vae is None or not runtime_ok():
return 0
n = 0
fused_act_norms = set()
for part_name in ("encoder", "decoder") if encoder else ("decoder",):
part = getattr(vae, part_name, None)
if part is None:
continue
n += install_attention_processors(part)
norm_out = getattr(part, "conv_norm_out", None)
act = getattr(part, "conv_act", None)
if isinstance(norm_out, torch.nn.GroupNorm) and isinstance(act, torch.nn.SiLU):
fused_act_norms.add(id(norm_out))
_guard(
norm_out,
_fast_norm(norm_out, True),
lambda x, _n = norm_out: torch.nn.functional.silu(torch.nn.GroupNorm.forward(_n, x)),
"conv_norm_out",
logger,
)
act.forward = lambda x: x
act._unsloth_vae_fused = True
n += 1
for module in part.modules():
if getattr(module.forward, "_unsloth_vae_fused", False):
continue
if type(module).__name__ == "ResnetBlock2D" and _resnet2d_fusable(module):
_guard(
module, _fast_resnet2d(module), _stock_forward(module), "ResnetBlock2D", logger
)
n += 1
for module in part.modules():
if (
isinstance(module, torch.nn.GroupNorm)
and id(module) not in fused_act_norms
and not getattr(module.forward, "_unsloth_vae_fused", False)
):
_guard(
module, _fast_norm(module, False), _stock_forward(module), "GroupNorm", logger
)
n += 1
return n
_STRUCTURAL_LAYER = "vae_fused_structural"
def _restore_structural_decode(vae: Any) -> None:
"""Remove the structural calibrator's ``_decode`` layer: exactly what it replaced comes back when it is outermost
(an instance-bound override included); under a later layer (the OOM retry) it becomes a pass-through."""
from . import decode_layers # noqa: PLC0415
decode_layers.unwrap(decode_layers.find(vae, "_decode", _STRUCTURAL_LAYER))
def uninstall(vae: Any) -> None:
_restore_structural_decode(vae)
_restore_contiguous_layout(vae)
vae.__dict__.pop("_unsloth_vae_structural", None)
vae.__dict__.pop("_unsloth_vae_fused_structural", None)
vae.__dict__.pop("_unsloth_vae_fused_installed", None)
vae.__dict__.pop("_unsloth_vae_fused_cl_weights", None)
vae.__dict__.pop("_unsloth_vae_strips", None)
for m in vae.modules():
if isinstance(getattr(m, "processor", None), FusedSingleHeadProcessor):
m.processor = m.processor.fallback
for attr in ("tiled_decode", "blend_v", "blend_h", "blend_t", "_unsloth_wide_stock_decode"):
if getattr(vae.__dict__.get(attr), "_unsloth_vae_fused", False):
delattr(vae, attr)
for module in vae.modules():
if getattr(getattr(module, "forward", None), "_unsloth_vae_fused", False) or getattr(
module, "_unsloth_vae_fused", False
):
try:
del module.forward
except AttributeError:
pass
module.__dict__.pop("_unsloth_vae_fused", None)
module.__dict__.pop("_unsloth_vae_fused_failed", None)
module.__dict__.pop("_unsloth_vae_strips_failed", None)
module.__dict__.pop("prepare_causal_attention_mask", None)
_WAN_VAES = frozenset({"AutoencoderKLWan", "AutoencoderKLQwenImage", "AutoencoderKLQwenImage21"})
def _is_rms(norm: Any) -> bool:
return (
type(norm).__name__.endswith("RMS_norm")
and hasattr(norm, "gamma")
and hasattr(norm, "scale")
)
def _wan_resblock_fusable(block: Any) -> bool:
torch = _torch()
return (
_is_rms(getattr(block, "norm1", None))
and _is_rms(getattr(block, "norm2", None))
and isinstance(getattr(block, "nonlinearity", None), torch.nn.SiLU)
and _conv_kind(getattr(block, "conv1", None)) is not None
and _conv_kind(getattr(block, "conv2", None)) is not None
)
def _fast_wan_resblock(block: Any) -> Any:
import torch.nn.functional as F
def fast(
x: Any,
feat_cache: Any = None,
feat_idx: Any = None,
*args: Any,
**kwargs: Any,
) -> Any:
if args or kwargs or block.training or (block.dropout.p and block.dropout.training):
return _stock_forward(block)(
x, feat_cache, feat_idx if feat_idx is not None else [0], *args, **kwargs
)
sc = block.conv_shortcut
if isinstance(sc, _torch().nn.Identity):
h = x
elif _conv_kind(sc) == "2d":
h = F.conv2d(x[:, :, 0], sc.weight, sc.bias, sc.stride, 0).unsqueeze(2)
else:
h = F.conv3d(x, sc.weight, sc.bias, sc.stride)
if feat_cache is not None:
idx = feat_idx[0]
out, new = causal_conv(
block.conv1, x, feat_cache[idx], norm = block.norm1, act = True, with_bias = False
)
feat_cache[idx] = new
feat_idx[0] += 1
idx = feat_idx[0]
out, new = causal_conv(
block.conv2, out, feat_cache[idx], norm = block.norm2, act = True, in_bias = block.conv1.bias,
with_bias = False,
) # fmt: skip
feat_cache[idx] = new
feat_idx[0] += 1
else:
out, _ = causal_conv(block.conv1, x, None, norm = block.norm1, act = True, with_bias = False)
out, _ = causal_conv(
block.conv2,
out,
None,
norm = block.norm2,
act = True,
in_bias = block.conv1.bias,
with_bias = False,
)
return add_bias_residual(out, block.conv2.bias, h)
return fast
def _fast_causal_conv(conv: Any) -> Any:
def fast(x: Any, cache_x: Any = None) -> Any:
return causal_conv(conv, x, cache_x)[0]
return fast
def _fast_rms_act(norm: Any) -> Any:
def fast(x: Any) -> Any:
if x.dim() != 5:
return _torch().nn.functional.silu(type(norm).forward(norm, x))
return rms_norm_act(x, norm, True)
return fast
def dup_up_add(out: Any, x: Any, dup: Any, first_chunk: bool) -> Any:
"""``out + DupUp3D(x, first_chunk)`` added into ``out`` (a fresh tensor) without materialising the shortcut."""
k = _kernels()
ft, fs, rep = int(dup.factor_t), int(dup.factor_s), int(dup.repeats)
b, c, t_o, h_o, w_o = out.shape
t_off = ft - 1 if first_chunk else 0
if (x.shape[2] * ft - t_off, x.shape[3] * fs, x.shape[4] * fs) != (
t_o,
h_o,
w_o,
) or c != dup.out_channels:
raise ValueError("dup_up_add: shape mismatch")
block_c = min(128, _next_pow2(c))
block_p = max(1, 4096 // block_c)
grid = ((h_o * w_o + block_p - 1) // block_p, b * t_o, (c + block_c - 1) // block_c)
with _on_card(out):
k.dup_up_add[grid](
out, x, c, t_o, h_o, w_o, t_off, rep, *out.stride(), *x.stride(),
FT = ft, FS = fs, BLOCK_P = block_p, BLOCK_C = block_c, num_warps = 4,
) # fmt: skip
return out
def up_nearest2x(x: Any, interleave: bool) -> Any:
"""Channels-last nearest 2x upsample; ``interleave`` also does Wan's time_conv frame interleave."""
torch = _torch()
k = _kernels()
b, c2, t, h, w = x.shape
c = c2 // 2 if interleave else c2
t_o = 2 * t if interleave else t
out = torch.empty((b * t_o, 2 * h, 2 * w, c), dtype = x.dtype, device = x.device)
block_c = min(128, _next_pow2(c))
block_p = max(1, 4096 // block_c)
grid = ((4 * h * w + block_p - 1) // block_p, b * t_o, (c + block_c - 1) // block_c)
with _on_card(x):
k.up_nearest2x[grid](
x,
out,
c,
t_o,
h,
w,
*x.stride(),
INTERLEAVE = interleave,
BLOCK_P = block_p,
BLOCK_C = block_c,
num_warps = 4,
)
return out.permute(0, 3, 1, 2), t_o
def _fast_wan_resample(mod: Any) -> Any:
import torch.nn.functional as F
conv = mod.resample[1]
def fast(
x: Any,
feat_cache: Any = None,
feat_idx: Any = None,
) -> Any:
if feat_idx is None:
feat_idx = [0]
b = x.shape[0]
interleave = False
y = x
if mod.mode == "upsample3d" and feat_cache is not None:
idx = feat_idx[0]
if feat_cache[idx] is None:
feat_cache[idx] = "Rep"
else:
prev = None if isinstance(feat_cache[idx], str) else feat_cache[idx]
y, new = causal_conv(mod.time_conv, x, prev)
feat_cache[idx] = new
interleave = True
feat_idx[0] += 1
up, t_o = up_nearest2x(y, interleave)
out = F.conv2d(up, conv.weight, conv.bias, conv.stride, conv.padding)
return out.view(b, t_o, out.shape[1], out.shape[2], out.shape[3]).permute(0, 2, 1, 3, 4)
return fast
def _wan_resample_ok(mod: Any) -> bool:
torch = _torch()
seq = getattr(mod, "resample", None)
return (
getattr(mod, "mode", None) in ("upsample2d", "upsample3d")
and isinstance(seq, torch.nn.Sequential)
and len(seq) == 2
and isinstance(seq[0], torch.nn.Upsample)
and seq[0].mode in ("nearest", "nearest-exact")
and tuple(float(f) for f in (seq[0].scale_factor or ())) == (2.0, 2.0)
and isinstance(seq[1], torch.nn.Conv2d)
and (mod.mode == "upsample2d" or _conv_kind(getattr(mod, "time_conv", None)) == "3d")
)
def _fast_wan_residual_up(block: Any) -> Any:
def fast(
x: Any,
feat_cache: Any = None,
feat_idx: Any = None,
first_chunk: bool = False,
) -> Any:
if feat_idx is None:
feat_idx = [0]
x_in = x # the stock ``x.clone()`` guards nothing: no block below writes its input
for resnet in block.resnets:
x = (
resnet(x, feat_cache = feat_cache, feat_idx = feat_idx)
if feat_cache is not None
else resnet(x)
)
if block.upsampler is not None:
if feat_cache is not None:
x = block.upsampler(x, feat_cache = feat_cache, feat_idx = feat_idx)
else:
x = block.upsampler(x)
if block.avg_shortcut is not None:
if x.data_ptr() == x_in.data_ptr():
x = x.clone()
x = dup_up_add(x, x_in, block.avg_shortcut, first_chunk)
return x
return fast
def install_wan_vae(
vae: Any,
logger: Any = None,
encoder: bool = True,
) -> int:
"""Patch the residual blocks, causal convs and output heads of a Wan-lineage VAE. Returns patch count."""
torch = _torch()
if vae is None or not runtime_ok():
return 0
n = 0
for part_name in ("encoder", "decoder") if encoder else ("decoder",):
part = getattr(vae, part_name, None)
if part is None:
continue
norm_out = getattr(part, "norm_out", None)
act = getattr(part, "nonlinearity", None)
if _is_rms(norm_out) and isinstance(act, torch.nn.SiLU):
_guard(
norm_out,
_fast_rms_act(norm_out),
lambda x, _n = norm_out: torch.nn.functional.silu(type(_n).forward(_n, x)),
"norm_out",
logger,
)
act.forward = lambda x: x
act._unsloth_vae_fused = True
n += 1
for module in part.modules():
if "forward" in module.__dict__:
continue # already patched (by us or another speed path)
name = type(module).__name__
if name.endswith("ResidualBlock") and _wan_resblock_fusable(module):
_guard(
module,
_fast_wan_resblock(module),
_stock_forward(module),
"residual block",
logger,
)
n += 1
elif (
name.endswith("ResidualUpBlock")
and hasattr(module, "avg_shortcut")
and hasattr(module, "resnets")
):
_guard(
module,
_fast_wan_residual_up(module),
_stock_forward(module),
"residual up block",
logger,
)
n += 1
elif name.endswith("Resample") and _wan_resample_ok(module):
_guard(
module, _fast_wan_resample(module), _stock_forward(module), "resample", logger
)
n += 1
elif (
name.endswith("AttentionBlock")
and _is_rms(getattr(module, "norm", None))
and isinstance(getattr(module, "to_qkv", None), torch.nn.Conv2d)
and isinstance(getattr(module, "proj", None), torch.nn.Conv2d)
):
_guard(
module, _fast_wan_attention(module), _stock_forward(module), "attention", logger,
oom_stock = True,
) # fmt: skip
n += 1
for module in part.modules():
if "forward" in module.__dict__:
continue
if type(module).__name__.endswith("CausalConv3d") and _conv_kind(module) is not None:
_guard(
module, _fast_causal_conv(module), _stock_forward(module), "causal conv", logger
)
n += 1
elif (
isinstance(module, torch.nn.Upsample)
and type(module).__name__.endswith("Upsample")
and module.mode in ("nearest", "nearest-exact")
):
_guard(module, _fast_upsample(module), _stock_forward(module), "upsample", logger)
n += 1
if install_wan_tile_batch(vae, logger):
n += 1
return n
def _fast_upsample(mod: Any) -> Any:
"""Wan's nearest upsample in the input dtype: bit-identical to its fp32 round trip, without two copies."""
def fast(x: Any) -> Any:
import torch.nn.functional as F
return F.interpolate(
x,
mod.size,
mod.scale_factor,
mod.mode,
mod.align_corners,
recompute_scale_factor = mod.recompute_scale_factor,
)
return fast
TILE_BATCH_ENV = "UNSLOTH_VAE_TILE_BATCH"
_TILE_BATCH_MAX = 4
# measured decode peak per 256 px Wan-2.1 tile (fp16, 4 frames, 96 full-res channels) ~= 1 GiB = 24x one activation
_TILE_PEAK_FACTOR = 24
def _tile_batch_cap(vae: Any, z: Any) -> int:
"""Tiles per decoder call."""
raw = os.environ.get(TILE_BATCH_ENV, "").strip()
if raw.isdigit() and int(raw) > 0:
return int(raw)
try:
from .diffusion_memory import user_budget_reserve_bytes
free, total = _torch().cuda.mem_get_info(z.device)
free = max(0, int(free) - user_budget_reserve_bytes(z.device, total))
norm_out = getattr(getattr(vae, "decoder", None), "norm_out", None)
c_last = int(norm_out.gamma.numel()) if norm_out is not None else 128
px = int(vae.tile_sample_min_height) * int(vae.tile_sample_min_width)
frames = int(getattr(vae, "temporal_compression_ratio", 4) or 4)
elem = next(vae.decoder.parameters()).element_size()
per_tile = _TILE_PEAK_FACTOR * px * frames * c_last * elem
return int(max(1, min(_TILE_BATCH_MAX, (free // 2) // max(1, per_tile))))
except Exception: # noqa: BLE001
return 1
def _blend_weights(extent: int, device: Any) -> tuple:
torch = _torch()
# stock python-scalar blend weights as fp32 opmath, built on device: a host list is a sync H2D copy per seam
y = torch.arange(extent, dtype = torch.float64, device = device) / extent
wb = y.float()
wa = (1 - y).float()
return wa, wb
def blend_seam(a: Any, b: Any, extent: int, dim: int) -> Any:
"""Vectorised, bit-identical ``blend_v`` (dim=-2) / ``blend_h`` (dim=-1): one pass instead of ``extent`` x 4."""
torch = _torch()
extent = min(a.shape[dim], b.shape[dim], extent)
if extent <= 0:
return b
wa, wb = _blend_weights(extent, b.device)
shape = [1] * b.dim()
shape[dim] = extent
wa, wb = wa.view(shape), wb.view(shape)
sa = a.narrow(dim, a.shape[dim] - extent, extent)
sb = b.narrow(dim, 0, extent)
dt = b.dtype
pa = (sa.float() * wa).to(dt)
pb = (sb.float() * wb).to(dt)
sb.copy_((pa.float() + pb.float()).to(dt))
return b
def _accepts_first_chunk(decoder: Any) -> bool:
import inspect
try:
return "first_chunk" in inspect.signature(decoder.forward).parameters
except (TypeError, ValueError):
return False
def _wan_batched_tiled_decode(
self: Any,
z: Any,
return_dict: bool = True,
clamp: bool = True,
) -> Any:
"""``tiled_decode`` with same-shaped tiles decoded as one batch and vectorised seam blending."""
torch = _torch()
from diffusers.models.autoencoders.vae import DecoderOutput
_, _, num_frames, height, width = z.shape
ratio = self.spatial_compression_ratio
sample_height, sample_width = height * ratio, width * ratio
tmh, tmw = self.tile_sample_min_height // ratio, self.tile_sample_min_width // ratio
tsh, tsw = self.tile_sample_stride_height // ratio, self.tile_sample_stride_width // ratio
out_sh, out_sw = self.tile_sample_stride_height, self.tile_sample_stride_width
patch = getattr(self.config, "patch_size", None)
if patch is not None:
sample_height, sample_width = sample_height // patch, sample_width // patch
out_sh, out_sw = out_sh // patch, out_sw // patch
blend_h = self.tile_sample_min_height // patch - out_sh
blend_w = self.tile_sample_min_width // patch - out_sw
else:
blend_h = self.tile_sample_min_height - out_sh
blend_w = self.tile_sample_min_width - out_sw
first_chunk = _accepts_first_chunk(self.decoder)
rows_i = list(range(0, height, tsh))
cols_j = list(range(0, width, tsw))
groups: dict = {}
for i in rows_i:
for j in cols_j:
groups.setdefault((min(tmh, height - i), min(tmw, width - j)), []).append((i, j))
cap = _tile_batch_cap(self, z)
tiles: dict = {}
def decode_tiles(chunk: list) -> Any:
zt = torch.cat([z[:, :, :, i : i + tmh, j : j + tmw] for i, j in chunk], 0)
self.clear_cache()
frames = []
for k in range(num_frames):
self._conv_idx = [0]
tile = self.post_quant_conv(zt[:, :, k : k + 1])
kw = {"first_chunk": k == 0} if first_chunk else {}
frames.append(
self.decoder(tile, feat_cache = self._feat_map, feat_idx = self._conv_idx, **kw)
)
return torch.cat(frames, dim = 2)
for members in groups.values():
s = 0
while s < len(members):
chunk = members[s : s + cap]
try:
out = decode_tiles(chunk)
except Exception as exc: # noqa: BLE001
if len(chunk) == 1 or not _is_oom(exc):
raise
self.clear_cache()
torch.cuda.empty_cache()
cap = 1
continue
for n, key in enumerate(chunk):
tiles[key] = out[n : n + 1]
s += len(chunk)
self.clear_cache()
rows = [[tiles[(i, j)] for j in cols_j] for i in rows_i]
result_rows = []
for ri, row in enumerate(rows):
result_row = []
for ci, tile in enumerate(row):
if ri > 0:
tile = blend_seam(rows[ri - 1][ci], tile, blend_h, -2)
if ci > 0:
tile = blend_seam(row[ci - 1], tile, blend_w, -1)
result_row.append(tile[:, :, :, :out_sh, :out_sw])
result_rows.append(torch.cat(result_row, dim = -1))
dec = torch.cat(result_rows, dim = 3)[:, :, :, :sample_height, :sample_width]
if patch is not None:
from diffusers.models.autoencoders.autoencoder_kl_wan import unpatchify
dec = unpatchify(dec, patch_size = patch)
if clamp: # as stock tiled_decode: Wan and Qwen-Image-2.1 clamp, Qwen-Image does not
dec = torch.clamp(dec, min = -1.0, max = 1.0)
if not return_dict:
return (dec,)
return DecoderOutput(sample = dec)
@lru_cache(maxsize = None)
def _stock_tiled_decode_clamps(cls: type) -> Optional[bool]:
"""Whether ``cls.tiled_decode`` clamps to [-1, 1], read from the diffusers source; None (keep stock) if unreadable."""
import inspect
import re
try:
src = inspect.getsource(cls.tiled_decode)
except (OSError, TypeError, AttributeError):
return None
return re.search(r"\bclamp\(\s*dec\b", src) is not None
def install_wan_tile_batch(vae: Any, logger: Any = None) -> bool:
# Under the wide tiles the batched loop only backs their stock-tile fallback.
wide = vae is not None and bool(getattr(vae, "_unsloth_wide_tiles", False))
if (
vae is None
or ("tiled_decode" in vae.__dict__ and not wide)
or not callable(getattr(vae, "tiled_decode", None))
):
return False
clamp = _stock_tiled_decode_clamps(type(vae))
if clamp is None:
return False
stock = _stock_forward_attr(vae, "tiled_decode")
def tiled_decode(z: Any, return_dict: bool = True) -> Any:
if z.shape[0] != 1 or getattr(vae, "_unsloth_vae_fused_failed", False):
return stock(z, return_dict = return_dict)
try:
return _wan_batched_tiled_decode(vae, z, return_dict, clamp)
except Exception as exc: # noqa: BLE001
if _is_oom(exc):
raise
vae._unsloth_vae_fused_failed = True
if logger is not None:
logger.warning(
"diffusion.vae_fused: batched tiled decode failed, using the stock loop: %s",
exc,
)
return stock(z, return_dict = return_dict)
tiled_decode._unsloth_vae_fused = True
if wide:
vae._unsloth_wide_stock_decode = tiled_decode
return False
vae.tiled_decode = tiled_decode
return True
def _stock_forward_attr(obj: Any, name: str) -> Any:
return getattr(type(obj), name).__get__(obj)
def _hv_conv_ok(conv: Any) -> bool:
torch = _torch()
inner = getattr(conv, "conv", None)
pad = getattr(conv, "time_causal_padding", None)
return (
isinstance(inner, torch.nn.Conv3d)
and getattr(conv, "pad_mode", None) == "replicate"
and pad is not None
and len(pad) == 6
and pad[0] == pad[1]
and pad[2] == pad[3]
and pad[5] == 0
and inner.padding == (0, 0, 0)
and inner.groups == 1
and inner.padding_mode == "zeros"
)
def hv_conv(
conv: Any,
x: Any,
*,
norm: Any = None,
act: bool = False,
in_bias: Any = None,
with_bias: bool = True,
) -> Any:
"""``conv(pad_replicate(silu?(rms?(x + in_bias))))`` with the pad, norm and act in one channels-last pass."""
import torch.nn.functional as F
pw, _, ph, _, front, _ = conv.time_causal_padding
inner = conv.conv
y = rms_norm_act(x, norm, act, front = front, in_bias = in_bias, replicate_pad = (ph, pw))
return F.conv3d(
y, inner.weight, inner.bias if with_bias else None, inner.stride, 0, inner.dilation
)
def _hv_resnet_ok(block: Any) -> bool:
torch = _torch()
return (
_is_rms(getattr(block, "norm1", None))
and _is_rms(getattr(block, "norm2", None))
and isinstance(getattr(block, "nonlinearity", None), torch.nn.SiLU)
and _hv_conv_ok(getattr(block, "conv1", None))
and _hv_conv_ok(getattr(block, "conv2", None))
)
def _fast_hv_resnet(block: Any) -> Any:
import torch.nn.functional as F
def fast(x: Any) -> Any:
h = hv_conv(block.conv1, x, norm = block.norm1, act = True, with_bias = False)
h = hv_conv(
block.conv2,
h,
norm = block.norm2,
act = True,
in_bias = block.conv1.conv.bias,
with_bias = False,
)
sc = block.conv_shortcut
res = x if sc is None else F.conv3d(x, sc.weight, sc.bias, sc.stride, sc.padding)
return add_bias_residual(h, block.conv2.conv.bias, res)
return fast
def _fast_hv_conv(conv: Any) -> Any:
def fast(x: Any) -> Any:
return hv_conv(conv, x)
return fast
def _fast_hv_norm_act(norm: Any) -> Any:
def fast(x: Any) -> Any:
if x.dim() != 5:
return _torch().nn.functional.silu(type(norm).forward(norm, x))
return rms_norm_act(x, norm, True)
return fast
def hv_upsample(up: Any, x: Any) -> Any:
"""``HunyuanVideo15Upsample.forward``: conv, then depth-to-space + repeat_interleave shortcut + add in one pass."""
torch = _torch()
k = _kernels()
conv = up.conv
h = hv_conv(conv, x, with_bias = False)
b, cin, f, hh, ww = x.shape
cp = h.shape[1]
temporal = bool(up.add_temporal_upsample)
factor = 8 if temporal else 4
c = cp // factor
rep = int(up.repeats)
if cp % factor or (temporal and (f < 1 or rep % 2 or cin % 4)) or rep * cin != factor * c:
raise ValueError("hv_upsample: unexpected shapes")
fo = 1 + (f - 1) * 2 if temporal else f
out = torch.empty((b, fo, 2 * hh, 2 * ww, c), dtype = x.dtype, device = x.device)
block_c = min(128, _next_pow2(c))
block_p = max(1, 4096 // block_c)
grid = ((4 * hh * ww + block_p - 1) // block_p, b * fo, (c + block_c - 1) // block_c)
bias = conv.conv.bias
with _on_card(h):
k.dcae_up_add[grid](
h, bias if bias is not None else h, x, out, c, cin, f, hh, ww, fo, rep, *h.stride(), *x.stride(),
TEMPORAL = temporal, HAS_HB = bias is not None, BLOCK_P = block_p, BLOCK_C = block_c, num_warps = 4,
) # fmt: skip
return out.permute(0, 4, 1, 2, 3)
def _hv_causal_mask(
n_frame: int,
n_hw: int,
dtype: Any,
device: Any,
batch_size: Any = None,
) -> Any:
"""Vectorised ``prepare_causal_attention_mask``: 0 where key frame <= query frame, else -inf."""
torch = _torch()
seq_len = n_frame * n_hw
mask = torch.zeros((seq_len, seq_len), dtype = dtype, device = device)
later = torch.ones((n_frame, n_frame), dtype = torch.bool, device = device).triu_(1)
mask.view(n_frame, n_hw, n_frame, n_hw).masked_fill_(later[:, None, :, None], float("-inf"))
if batch_size is not None:
mask = mask.unsqueeze(0).expand(batch_size, -1, -1)
return mask
def install_hv15_vae(
vae: Any,
logger: Any = None,
encoder: bool = True,
) -> int:
torch = _torch()
if vae is None or not runtime_ok():
return 0
n = 0
for part_name in ("encoder", "decoder") if encoder else ("decoder",):
part = getattr(vae, part_name, None)
if part is None:
continue
norm_out, act = getattr(part, "norm_out", None), getattr(part, "conv_act", None)
if (
_is_rms(norm_out)
and isinstance(act, torch.nn.SiLU)
and "forward" not in norm_out.__dict__
):
_guard(
norm_out,
_fast_hv_norm_act(norm_out),
lambda x, _n = norm_out: torch.nn.functional.silu(type(_n).forward(_n, x)),
"norm_out",
logger,
)
act.forward = lambda x: x
act._unsloth_vae_fused = True
n += 1
for module in part.modules():
if "forward" in module.__dict__:
continue
name = type(module).__name__
if name.endswith("ResnetBlock") and _hv_resnet_ok(module):
_guard(module, _fast_hv_resnet(module), _stock_forward(module), "resnet", logger)
n += 1
for module in part.modules():
if "forward" in module.__dict__:
continue
name = type(module).__name__
if (
name.endswith("Upsample")
and _hv_conv_ok(getattr(module, "conv", None))
and hasattr(module, "repeats")
):
_guard(
module,
lambda x, _m = module: hv_upsample(_m, x),
_stock_forward(module),
"upsample",
logger,
)
n += 1
elif name.endswith("AttnBlock") and callable(
getattr(module, "prepare_causal_attention_mask", None)
):
module.prepare_causal_attention_mask = _hv_causal_mask
n += 1
for module in part.modules():
if "forward" in module.__dict__:
continue
if type(module).__name__.endswith("CausalConv3d") and _hv_conv_ok(module):
_guard(module, _fast_hv_conv(module), _stock_forward(module), "causal conv", logger)
n += 1
return n
def _ltx_conv_ok(conv: Any) -> bool:
torch = _torch()
inner = getattr(conv, "conv", None)
return (
isinstance(inner, torch.nn.Conv3d)
and inner.padding_mode == "zeros"
and inner.padding[0] == 0
and inner.stride[0] == 1
and inner.dilation[0] == 1
and len(getattr(conv, "kernel_size", ())) == 3
and conv.kernel_size[0] % 2 == 1
)
def ltx_conv(
conv: Any,
x: Any,
causal: bool,
*,
norm: Any = None,
act: bool = False,
in_bias: Any = None,
with_bias: bool = True,
) -> Any:
"""``LTX2VideoCausalConv3d.forward`` over ``silu?(pixel_norm?(x + in_bias))``, pad + norm + act in one pass."""
import torch.nn.functional as F
kt = conv.kernel_size[0]
front, back = (kt - 1, 0) if causal else ((kt - 1) // 2, (kt - 1) // 2)
inner = conv.conv
if kt == 1 and norm is None and not act and in_bias is None:
y = x
else:
y = rms_norm_act(
x, norm, act, front = front, back = back, in_bias = in_bias, replicate_pad = (0, 0)
)
return F.conv3d(
y,
inner.weight,
inner.bias if with_bias else None,
inner.stride,
inner.padding,
inner.dilation,
)
def _ltx_resnet_ok(block: Any) -> bool:
torch = _torch()
return (
_is_pixel_norm(getattr(block, "norm1", None))
and _is_pixel_norm(getattr(block, "norm2", None))
and isinstance(getattr(block, "nonlinearity", None), torch.nn.SiLU)
and _ltx_conv_ok(getattr(block, "conv1", None))
and _ltx_conv_ok(getattr(block, "conv2", None))
and getattr(block, "per_channel_scale1", None) is None
and getattr(block, "per_channel_scale2", None) is None
and getattr(block, "scale_shift_table", None) is None
)
def _fast_ltx_resnet(block: Any) -> Any:
import torch.nn.functional as F
def fast(
inputs: Any,
temb: Any = None,
generator: Any = None,
causal: bool = True,
) -> Any:
if block.training and block.dropout.p:
return _stock_forward(block)(inputs, temb, generator, causal = causal)
h = ltx_conv(block.conv1, inputs, causal, norm = block.norm1, act = True, with_bias = False)
h = ltx_conv(
block.conv2,
h,
causal,
norm = block.norm2,
act = True,
in_bias = block.conv1.conv.bias,
with_bias = False,
)
res = inputs
if block.norm3 is not None:
res = block.norm3(res.movedim(1, -1)).movedim(-1, 1)
if block.conv_shortcut is not None:
sc = block.conv_shortcut
res = F.conv3d(res, sc.weight, sc.bias, sc.stride, sc.padding)
return add_bias_residual(h, block.conv2.conv.bias, res)
return fast
def _fast_ltx_conv(conv: Any) -> Any:
def fast(hidden_states: Any, causal: bool = True) -> Any:
return ltx_conv(conv, hidden_states, causal)
return fast
def _fast_pixel_norm_act(norm: Any) -> Any:
def fast(x: Any, channel_dim: Any = None) -> Any:
if x.dim() != 5 or channel_dim not in (None, 1):
return _torch().nn.functional.silu(type(norm).forward(norm, x, channel_dim))
return rms_norm_act(x, norm, True)
return fast
def _ltx2_convs_fusable(vae: Any) -> bool:
"""LTX-2's reflect-padded decoder fuses no conv: norm_out alone decoded 0.97-0.98x (L4 / A100 / RTX PRO 6000)."""
decoder = getattr(vae, "decoder", None)
return decoder is not None and any(
type(m).__name__ == "LTX2VideoCausalConv3d" and _ltx_conv_ok(m) for m in decoder.modules()
)
def install_ltx2_vae(vae: Any, logger: Any = None) -> int:
torch = _torch()
if vae is None or not runtime_ok():
return 0
n = 0
# decoder only: the causal one-frame encode (i2v conditioning) measured 33 -> 35 ms fused, so it stays stock
for part_name in ("decoder",):
part = getattr(vae, part_name, None)
if part is None:
continue
norm_out, act = getattr(part, "norm_out", None), getattr(part, "conv_act", None)
if (
_is_pixel_norm(norm_out)
and isinstance(act, torch.nn.SiLU)
and getattr(part, "time_embedder", None) is None
and "forward" not in norm_out.__dict__
):
_guard(
norm_out,
_fast_pixel_norm_act(norm_out),
lambda x, channel_dim = None, _n = norm_out: torch.nn.functional.silu(
type(_n).forward(_n, x, channel_dim)
),
"norm_out",
logger,
)
act.forward = lambda x: x
act._unsloth_vae_fused = True
n += 1
for module in part.modules():
if "forward" in module.__dict__:
continue
if type(module).__name__.endswith("ResnetBlock3d") and _ltx_resnet_ok(module):
_guard(module, _fast_ltx_resnet(module), _stock_forward(module), "resnet", logger)
n += 1
for module in part.modules():
if "forward" in module.__dict__:
continue
if type(module).__name__ == "LTX2VideoCausalConv3d" and _ltx_conv_ok(module):
_guard(
module, _fast_ltx_conv(module), _stock_forward(module), "causal conv", logger
)
n += 1
return n
def install_vectorised_blend(vae: Any) -> int:
"""Replace per-row ``blend_v`` / ``blend_h`` / ``blend_t`` loops with the bit-identical :func:`blend_seam`."""
n = 0
for name, dim in (("blend_v", -2), ("blend_h", -1), ("blend_t", -3)):
if callable(getattr(vae, name, None)) and name not in vae.__dict__:
fn = lambda a, b, blend_extent, _d = dim: blend_seam(a, b, blend_extent, _d) # noqa: E731
fn._unsloth_vae_fused = True
setattr(vae, name, fn)
n += 1
return n
_CL_WEIGHT_VAES = frozenset(
{"AutoencoderKL", "AutoencoderKLFlux2", "AutoencoderKLHunyuanVideo15", "AutoencoderKLLTX2Video"}
)
def channels_last_weights(vae: Any) -> int:
"""The fused passes hand cuDNN channels-last activations; contiguous weights would be re-laid-out on every call."""
torch = _torch()
n = 0
for part in (getattr(vae, "encoder", None), getattr(vae, "decoder", None)):
if part is None:
continue
for m in part.modules():
w = getattr(m, "weight", None)
if (
isinstance(m, (torch.nn.Conv2d, torch.nn.Conv3d))
and isinstance(w, torch.Tensor)
and w.dim() in (4, 5)
):
fmt = torch.channels_last if w.dim() == 4 else torch.channels_last_3d
if not w.is_contiguous(memory_format = fmt):
w.data = w.data.contiguous(memory_format = fmt)
n += 1
return n
def _fuses_a_conv(vae: Any) -> bool:
torch = _torch()
return any(
getattr(m.__dict__.get("forward"), "_unsloth_vae_fused", False)
and any(isinstance(c, (torch.nn.Conv2d, torch.nn.Conv3d)) for c in m.modules())
for m in vae.modules()
)
SUPPORTED_VAES = frozenset(
{
"AutoencoderKL",
"AutoencoderKLFlux2",
"AutoencoderKLWan",
"AutoencoderKLQwenImage",
"AutoencoderKLQwenImage21",
"AutoencoderKLHunyuanVideo15",
"AutoencoderKLLTX2Video",
}
)
def will_install(vae: Any) -> bool:
"""Whether :func:`install` would engage on ``vae`` here (class covered or matched by structure, NVIDIA CUDA,
usable Triton, not disabled)."""
return (
vae is not None
and (
(
type(vae).__name__ in SUPPORTED_VAES
and (type(vae).__name__ != "AutoencoderKLLTX2Video" or _ltx2_convs_fusable(vae))
)
or structural_candidate(vae)
)
and runtime_ok()
)
def install(
vae: Any,
logger: Any = None,
level: str = "fused",
encoder: bool = True,
) -> int:
"""Install every fused path that applies to ``vae``'s class. Returns the number of patched modules.
``encoder=False`` patches the decoder only, for a family whose denoising attends to the encoded conditioning clip at
every step, which amplifies the fused passes' rounding into a different sample (video_families.vae_fused_encoder)."""
if vae is None or not will_install(vae):
return 0
done = getattr(vae, "_unsloth_vae_fused_installed", 0)
if done:
return done # idempotent: a dual-DiT family runs the speed layer once per expert over the same VAE
n = _install(vae, logger, encoder)
if not n and type(vae).__name__ not in SUPPORTED_VAES:
n = install_structural(vae, logger) # decoder only, so it honours encoder=False as is
if n:
vae._unsloth_vae_fused_structural = True
# Only where measured faster, and only when a fused pass feeds a conv (relaid weights alone slow LTX-2's decode).
if (
n
and type(vae).__name__ in _CL_WEIGHT_VAES
and os.environ.get("UNSLOTH_VAE_FUSED_CL_WEIGHTS", "1") != "0"
and _fuses_a_conv(vae)
):
channels_last_weights(vae)
vae._unsloth_vae_fused_cl_weights = True
if n:
vae._unsloth_vae_fused_installed = n
if logger is not None:
logger.info("diffusion.vae_fused: fused %d VAE modules on %s", n, type(vae).__name__)
return n
def _install(
vae: Any,
logger: Any = None,
encoder: bool = True,
) -> int:
name = type(vae).__name__
if name in ("AutoencoderKL", "AutoencoderKLFlux2"):
n = install_group_norm_vae(vae, logger, encoder)
if n:
from . import diffusion_vae_strips # noqa: PLC0415
n += int(diffusion_vae_strips.install(vae, logger))
return n + (install_vectorised_blend(vae) if n else 0)
if name in _WAN_VAES:
n = install_wan_vae(vae, logger, encoder)
return n + (install_vectorised_blend(vae) if n else 0)
if name == "AutoencoderKLHunyuanVideo15":
n = install_hv15_vae(vae, logger, encoder)
return n + (install_vectorised_blend(vae) if n else 0)
if name == "AutoencoderKLLTX2Video":
if not _ltx2_convs_fusable(vae):
return 0
n = install_ltx2_vae(vae, logger)
return n + (install_vectorised_blend(vae) if n else 0)
return 0
# Structural path for unmeasured VAEs: a GroupNorm pairs with SiLU only if every output fed an nn.SiLU (hook trace of
# one tiny decode); the fused decode must match stock within tolerance or the patch is removed.
# Kill switch ``UNSLOTH_VAE_FUSED_STRUCTURAL=0``.
STRUCTURAL_ENV = "UNSLOTH_VAE_FUSED_STRUCTURAL"
# Calibration crop sized for the kernels: 16 latents keeps every spatial extent a multiple of 16,
# the same Triton specialisations a full decode uses; an 8-latent crop added Triton compiles that a
# full decode would not.
_PROBE_LATENTS = 16
_PROBE_FRAMES = 2
# Calibration verdicts are kept on disk per (class, config, decoder layout, torch, triton, device capability), so a
# later process skips the trace and the checks. Off: UNSLOTH_VAE_FUSED_STRUCTURAL_CACHE=0.
STRUCTURAL_CACHE_ENV = "UNSLOTH_VAE_FUSED_STRUCTURAL_CACHE"
_STRUCTURAL_CACHE_VERSION = 3
_PROBE_MAX_ABS = 0.125
_PROBE_MEAN_ABS = 0.01
def structural_disabled() -> bool:
return os.environ.get(STRUCTURAL_ENV, "").strip().lower() in _FALSE
def _plain_group_norms(part: Any) -> list:
torch = _torch()
return [
m
for m in part.modules()
if isinstance(m, torch.nn.GroupNorm)
and type(m).forward is torch.nn.GroupNorm.forward
and "forward" not in m.__dict__
]
def _plain_silus(part: Any) -> list:
torch = _torch()
return [
m
for m in part.modules()
if isinstance(m, torch.nn.SiLU)
and type(m).forward is torch.nn.SiLU.forward
and "forward" not in m.__dict__
]
def structural_candidate(vae: Any) -> bool:
"""A VAE outside the measured set whose decoder holds plain nn.GroupNorm modules, reached through ``_decode``."""
if vae is None or structural_disabled() or type(vae).__name__ in SUPPORTED_VAES:
return False
if getattr(vae, "_unsloth_decode_blocks_bypassed", False):
return False
decoder = getattr(vae, "decoder", None)
if decoder is None or not callable(getattr(vae, "_decode", None)):
return False
try:
return bool(_plain_group_norms(decoder))
except Exception: # noqa: BLE001
return False
def _decoded_tensor(out: Any) -> Any:
if isinstance(out, (tuple, list)):
return out[0]
return getattr(out, "sample", out)
def _probe_latent(z: Any) -> Any:
sl = [slice(None)] * z.dim()
sl[-1] = slice(0, min(_PROBE_LATENTS, z.shape[-1]))
sl[-2] = slice(0, min(_PROBE_LATENTS, z.shape[-2]))
if z.dim() == 5:
sl[2] = slice(0, min(_PROBE_FRAMES, z.shape[2]))
return z[tuple(sl)].contiguous()
def structural_cache_disabled() -> bool:
return (os.environ.get(STRUCTURAL_CACHE_ENV) or "").strip().lower() in (
"0",
"off",
"false",
"no",
)
def _structural_cache_dir() -> Any:
from pathlib import Path
try:
from .diffusion_compile_cache import cache_root
return Path(cache_root()) / "vae_structural"
except Exception: # noqa: BLE001
return None
def _vae_config_json(vae: Any) -> str:
import json
cfg = getattr(vae, "config", None)
try:
data = dict(cfg) if cfg is not None else {}
except Exception: # noqa: BLE001
data = dict(getattr(cfg, "__dict__", {}) or {})
return json.dumps(
{k: data[k] for k in sorted(data) if not str(k).startswith("_")},
sort_keys = True,
default = str,
)
_FINGERPRINT_ELEMS = 64
def _decoder_fingerprint(decoder: Any) -> Optional[str]:
"""The checkpoint's identity, cheaply: every decoder tensor's name, shape and dtype, plus its first
``_FINGERPRINT_ELEMS`` values (one device-to-host copy). Another checkpoint of the same config (a fine-tune, a
different dtype) is another fingerprint. None when the values cannot be read (meta / offloaded weights)."""
import hashlib
torch = _torch()
try:
named = [*decoder.named_parameters(), *decoder.named_buffers()]
if any(getattr(t, "is_meta", False) for _, t in named):
return None
h = hashlib.sha256(
repr([(n, tuple(t.shape), str(t.dtype), str(t.device.type)) for n, t in named]).encode()
)
with torch.no_grad():
heads = [
t.detach().reshape(-1)[:_FINGERPRINT_ELEMS].float() for _, t in named if t.numel()
]
if heads:
by_device: dict = {}
for t in heads:
by_device.setdefault(t.device, []).append(t)
for dev in sorted(by_device, key = str):
h.update(torch.cat(by_device[dev]).cpu().numpy().tobytes())
return h.hexdigest()[:32]
except Exception: # noqa: BLE001 - no identity: the caller does not use the cache
return None
def structural_cache_key(vae: Any, norms: list, silus: list, device: Any) -> Optional[str]:
"""Key of a calibration verdict: everything the pairing and the kernels' numerics depend on. Any change (another
class, config, decoder layout, checkpoint weights or dtype, hooks, torch, triton or device capability) is a new
key, so it is traced and checked again. None (no cache) when the weights' identity cannot be read."""
import hashlib
torch = _torch()
try:
import triton
triton_version = getattr(triton, "__version__", "?")
except Exception: # noqa: BLE001
triton_version = None
try:
cap = (
tuple(torch.cuda.get_device_capability(device))
if getattr(device, "type", None) == "cuda"
else None
)
except Exception: # noqa: BLE001
cap = None
fingerprint = _decoder_fingerprint(vae.decoder)
if fingerprint is None:
return None # no cheap identity for these weights: no cached verdict, trace and check every process
names = {id(m): n for n, m in vae.decoder.named_modules()}
layout = [
(
names.get(id(m), "?"),
type(m).__name__,
int(getattr(m, "num_groups", 0)),
int(getattr(m, "num_channels", 0)),
)
for m in norms
] + [(names.get(id(m), "?"), type(m).__name__) for m in silus]
blob = repr((
_STRUCTURAL_CACHE_VERSION, f"{type(vae).__module__}.{type(vae).__qualname__}", _vae_config_json(vae), layout,
str(torch.__version__), triton_version, cap, _PROBE_LATENTS, _PROBE_MAX_ABS, _PROBE_MEAN_ABS, fingerprint,
_global_module_hooks(), [_has_user_hooks(m) for m in (*norms, *silus)],
)) # fmt: skip
return hashlib.sha256(blob.encode()).hexdigest()[:32]
def _load_pairing(key: Optional[str], n_norms: int) -> Optional[dict]:
import json
root = _structural_cache_dir()
if key is None or root is None or structural_cache_disabled():
return None
try:
data = json.loads((root / f"{key}.json").read_text(encoding = "utf-8"))
except Exception: # noqa: BLE001 - absent or unreadable: calibrate
return None
paired = data.get("paired")
if data.get("key") != key or data.get("status") not in ("ok", "rejected", "no_match"):
return None
if (
not isinstance(paired, list)
or len(paired) != n_norms
or any(p not in (None, True, False) for p in paired)
):
return None
consumers = data.get("consumers")
if data["status"] == "ok" and (
not isinstance(consumers, list) or len(consumers) != n_norms
or any(not isinstance(c, list) or any(not isinstance(j, int) for j in c) for c in consumers)
): # fmt: skip
return None
return data
def _save_pairing(key: Optional[str], record: dict) -> None:
import json
root = _structural_cache_dir()
if key is None or root is None or structural_cache_disabled():
return
try:
root.mkdir(parents = True, exist_ok = True)
tmp = root / f".{key}.{os.getpid()}.tmp"
tmp.write_text(json.dumps({**record, "key": key}, default = str), encoding = "utf-8")
os.replace(tmp, root / f"{key}.json")
except Exception: # noqa: BLE001 - a cache that cannot be written only costs the next process a calibration
pass
def _has_user_hooks(module: Any) -> bool:
for name in ("_forward_hooks", "_forward_pre_hooks", "_backward_hooks", "_backward_pre_hooks"):
if getattr(module, name, None):
return True
return False
def _global_module_hooks() -> bool:
try:
from torch.nn.modules import module as mm
except Exception: # noqa: BLE001
return True
return any(
getattr(mm, name, None)
for name in ("_global_forward_hooks", "_global_forward_pre_hooks", "_global_backward_hooks")
)
# Tensor methods that only read metadata: not a reader of the values.
_META_FUNCS = frozenset((
"__get__", "dim", "size", "stride", "numel", "nelement", "ndimension", "element_size", "is_contiguous",
"storage_offset", "is_floating_point", "is_complex", "get_device", "__len__", "data_ptr",
)) # fmt: skip
# The reader callback of the structural trace is per thread, and traces run one at a time (once per VAE per process):
# two first decodes on two threads can neither clear nor overwrite each other's callback.
_TRACE_LOCAL = threading.local()
_TRACE_LOCK = threading.RLock()
def _tracked_tensor_class() -> Any:
"""A tensor subclass marking a norm's output: every torch function that touches it reports in, and its results
come back as plain tensors, so only the direct readers of a norm output are seen (cheap: no per-op hook on the
rest of the decoder)."""
torch = _torch()
cls = globals().get("_TrackedCls")
if cls is not None:
return cls
from torch.utils._pytree import tree_leaves
class _Tracked(torch.Tensor):
@classmethod
def __torch_function__(
cls,
func,
types_,
args = (),
kwargs = None,
):
kwargs = kwargs or {}
# this thread's trace only: a concurrent trace (another VAE, another thread) has its own callback
trace = getattr(_TRACE_LOCAL, "read", None)
if trace is not None and getattr(func, "__name__", "") not in _META_FUNCS:
for leaf in tree_leaves((args, kwargs)):
if isinstance(leaf, _Tracked):
trace(leaf)
with torch._C.DisableTorchFunctionSubclass():
return func(*args, **kwargs)
globals()["_TrackedCls"] = _Tracked
return _Tracked
def _trace_group_norm_silu(norms: list, silus: list, run: Any) -> tuple:
"""(calls per norm, outputs of each norm consumed ONLY by an nn.SiLU, run output, info) over one ``run()``.
A norm's output counts as consumed by the SiLU only when the trace proves the SiLU is its sole reader: the output
is marked (a tensor subclass), every torch function reading it (a view, an in-place update, a residual add) is
recorded, and any such read outside the SiLU call marks it shared. ``info["shared"]``: per norm, reads elsewhere.
Norms or SiLUs carrying hooks of their own, global module hooks, or no subclass support: nothing counts as
exclusively consumed."""
n = len(norms)
calls, consumed, shared = [0] * n, [0] * n, [0] * n
produced: dict = {}
keep: list = []
handles = []
in_silu: list = []
hooked = _global_module_hooks() or any(_has_user_hooks(m) for m in (*norms, *silus))
try:
tracked = _tracked_tensor_class()
except Exception: # noqa: BLE001 - no way to see every reader: fuse GroupNorm alone
tracked = None
def norm_hook(idx: int):
def hook(_m, _inp, out):
calls[idx] += 1
if tracked is None or type(out) is not _torch().Tensor:
return None
marked = out.as_subclass(tracked)
produced[id(marked)] = idx
keep.append(marked)
return marked
return hook
silu_index = {id(m): j for j, m in enumerate(silus)}
readers: list = [set() for _ in range(n)]
def silu_pre(m, inp):
idx = produced.get(id(inp[0])) if inp else None
if idx is not None:
consumed[idx] += 1
readers[idx].add(silu_index.get(id(m), -1))
in_silu.append(idx)
def silu_post(_m, _inp, _out):
if in_silu:
in_silu.pop()
def read(t: Any) -> None:
idx = produced.get(id(t))
if idx is not None and not (in_silu and in_silu[-1] == idx):
shared[idx] += 1
with _TRACE_LOCK:
previous = getattr(_TRACE_LOCAL, "read", None)
try:
for i, m in enumerate(norms):
handles.append(m.register_forward_hook(norm_hook(i)))
for m in silus:
handles.append(m.register_forward_pre_hook(silu_pre))
handles.append(m.register_forward_hook(silu_post))
if tracked is not None:
_TRACE_LOCAL.read = read
out = run()
finally:
_TRACE_LOCAL.read = previous
for h in handles:
h.remove()
keep.clear()
if hooked or tracked is None:
consumed = [0] * n
else:
consumed = [k if sh == 0 else 0 for k, sh in zip(consumed, shared)]
if out is not None and tracked is not None and isinstance(out, tracked):
out = out.as_subclass(_torch().Tensor)
return calls, consumed, out, {"shared": shared, "consumers": [sorted(r) for r in readers]}
def _site_hooks(norms: list, paired: list) -> tuple:
"""Forward hooks comparing each fused site with stock on the input it is given, during the crop decode itself
(one pass: no extra kernel specialisations). Returns (handles, worst) where worst is [max abs, mean abs, sites]."""
torch = _torch()
worst = [0.0, 0.0, 0]
handles = []
for norm, act in zip(norms, paired):
if act is None:
continue
def hook(
m,
inp,
out,
act = act,
):
out = _plain(out)
ref = _stock_forward(m)(inp[0])
if act:
ref = torch.nn.functional.silu(ref)
err = (out.float() - ref.float()).abs()
worst[0] = max(worst[0], float(err.max()))
worst[1] = max(worst[1], float(err.mean()))
worst[2] += 1
handles.append(norm.register_forward_hook(hook))
return handles, worst
# Tensor properties that are metadata only (a read of these is not a read of the values).
_META_PROPS = frozenset((
"shape", "dtype", "device", "is_cuda", "layout", "ndim", "requires_grad", "is_leaf", "grad_fn", "names",
"is_sparse", "is_quantized", "is_meta", "is_mkldnn", "is_nested", "itemsize", "nbytes",
)) # fmt: skip
_RAW_READS = frozenset(("data_ptr", "untyped_storage", "storage", "_typed_storage"))
def _is_meta_read(func: Any) -> bool:
name = getattr(func, "__name__", "")
if name == "__get__":
return getattr(getattr(func, "__self__", None), "__name__", "") in _META_PROPS
return name in _META_FUNCS and name not in _RAW_READS
class FusedActivationRead(RuntimeError):
"""A fused GroupNorm+SiLU output was read by something other than its paired SiLU. It holds only the activated
values: the stock GroupNorm value such a reader needs was never materialised, and recomputing it later from the
norm's input is not exact (the decoder may have written to that input in place since, and inference tensors,
which Studio renders with, keep no version counter to tell). So that decode stops here; the structural decode
layer turns activation fusion off for this VAE (GroupNorm stays fused), records that, and decodes again."""
def _fused_act_class() -> Any:
"""The output of a fused GroupNorm+SiLU: it holds the ACTIVATED values, which only the SiLU module(s) the trace
paired with it may read (once: the first consumes the mark). Any other read of the values (a residual, another
norm, a view, an in-place write, a raw pointer, a SiLU after the mark was consumed), whatever the shape or
dataflow of the decode that does it, raises FusedActivationRead: the decode is redone without activation fusion,
so a wrong value is never returned and nothing aliases the norm's (mutable) input. After an in-place paired SiLU
the storage IS the stock tensor's (silu_ overwrote it), so reads then see it as a plain tensor."""
cls = globals().get("_FusedActCls")
if cls is not None:
return cls
torch = _torch()
from torch.utils._pytree import tree_leaves
class _FusedAct(torch.Tensor):
@classmethod
def __torch_function__(
cls,
func,
types_,
args = (),
kwargs = None,
):
kwargs = kwargs or {}
if not _is_meta_read(func):
for leaf in tree_leaves((args, kwargs)):
if isinstance(leaf, _FusedAct) and not leaf.__dict__.get(
"_unsloth_is_stock", False
):
leaf._unsloth_abort(func)
with torch._C.DisableTorchFunctionSubclass():
return func(*args, **kwargs)
def _unsloth_abort(self, func: Any = None) -> None:
state = self.__dict__.get("_unsloth_state")
if state is not None:
state.foreign_reads = getattr(state, "foreign_reads", 0) + 1
raise FusedActivationRead(
f"fused GroupNorm+SiLU output read by {getattr(func, '__name__', 'a kernel')} before / instead of its "
"paired SiLU"
)
globals()["_FusedActCls"] = _FusedAct
return _FusedAct
def _plain(t: Any) -> Any:
"""``t`` as a plain tensor: a fused GroupNorm+SiLU output's activated values (no recompute)."""
cls = globals().get("_FusedActCls")
if cls is None or type(t) is not cls:
return t
torch = _torch()
with torch._C.DisableTorchFunctionSubclass():
return t.as_subclass(torch.Tensor)
def _compiling() -> bool:
try:
return bool(_torch().compiler.is_compiling())
except Exception: # noqa: BLE001
return False
def _fast_gn_maybe_act(
norm: Any,
act: bool,
state: Any,
silus: Any = None,
) -> Any:
"""The fused GroupNorm(+SiLU) forward. A fused activation's output is a _FusedAct bound to exactly the SiLU
module(s) the trace saw consume it (``silus``; None: any one SiLU): the first of those to read it gets the
activated values and consumes the mark; every other reader gets the stock GroupNorm output. Under torch.compile
(no tensor subclass in the graph) the activation is left to the SiLU, which inductor fuses itself."""
token = "any" if silus is None else frozenset(id(m) for m in silus)
fused_cls = _fused_act_class() if act else None
torch = _torch()
def fast(x: Any) -> Any:
# any patched norm (fused activation or GroupNorm only) reading a fused output reads its stock value: the
# kernel reads raw memory
x = _stock_in(x)
fuse = act and not _compiling()
ran: list = []
y = group_norm_act(x, norm, act = fuse, on_kernel = lambda: ran.append(1))
if ran:
state.kernel_calls += 1
probe = getattr(getattr(state, "local", None), "kernel_runs", None)
if probe is not None:
probe.append(1)
state.calls += 1
if fuse:
with torch._C.DisableTorchFunctionSubclass():
y = y.as_subclass(fused_cls)
y._unsloth_state, y._unsloth_silu_for = state, token
return y
return fast
def _silu_passthrough(silu: Any) -> Any:
torch = _torch()
fused_cls = _fused_act_class()
inplace = bool(getattr(silu, "inplace", False))
def forward(x: Any) -> Any:
token = getattr(x, "_unsloth_silu_for", None) if type(x) is fused_cls else None
if token is not None and (token == "any" or id(silu) in token):
x._unsloth_silu_for = None
if inplace:
x._unsloth_is_stock = (
True # stock silu_ overwrote the norm output: its storage is now exactly that
)
return _plain(x)
return torch.nn.functional.silu(x, inplace = inplace)
forward._unsloth_vae_fused = True
return forward
def _structural_patch(
norms: list,
paired: list,
silus: list,
state: Any,
logger: Any,
consumers: Optional[list] = None,
) -> int:
"""``consumers``: per norm, the indices into ``silus`` of the SiLU module(s) the trace saw read its output; only
those SiLUs consume its fused-activation mark (None: any SiLU, once)."""
n = 0
patched_silus: list = []
for i, (norm, act) in enumerate(zip(norms, paired)):
if act is None:
continue
mine = None
if act and consumers is not None:
mine = [silus[j] for j in consumers[i] if 0 <= j < len(silus)]
patched_silus += mine
_guard(
norm,
_fast_gn_maybe_act(norm, act, state, mine),
_stock_forward(norm),
"GroupNorm+SiLU" if act else "GroupNorm",
logger,
)
n += 1
if any(paired):
targets = silus if consumers is None else patched_silus
for silu in {id(m): m for m in targets}.values():
silu.forward = _silu_passthrough(silu)
return n
def _structural_unpatch(
norms: list,
silus: list,
vae: Any = None,
) -> None:
for m in (*norms, *silus):
m.__dict__.pop("forward", None)
m.__dict__.pop("_unsloth_vae_fused_failed", None)
if vae is not None:
_restore_contiguous_layout(vae)
def _restore_contiguous_layout(vae: Any) -> int:
"""Undo the channels_last(_3d) relayout the speed layer applied because fusion was planned: without the fused
passes that layout decodes slower. Only the conv weights it recorded are touched."""
modules = vae.__dict__.pop("_unsloth_channels_last_modules", None) or ()
n = 0
for module in modules:
weight = getattr(module, "weight", None)
if weight is not None and not weight.is_contiguous():
weight.data = weight.data.contiguous()
n += 1
return n
def _clear_decoder_cache(vae: Any) -> None:
"""Reset a causal decoder's frame / feature cache (Wan, Qwen-Image, LTX-2, HunyuanVideo expose clear_cache)."""
clear = getattr(vae, "clear_cache", None)
if callable(clear):
try:
clear()
except Exception: # noqa: BLE001 - a VAE without a usable cache has nothing to reset
pass
def install_structural(vae: Any, logger: Any = None) -> int:
"""Arm the dataflow-matched fused GroupNorm(+SiLU) path on ``vae``'s decoder; it is calibrated (paired, checked
against stock) on the first CUDA decode and removed if the check fails. Returns the plain GroupNorm count."""
if not structural_candidate(vae) or not runtime_ok():
return 0
if getattr(vae, "_unsloth_vae_structural", None) is not None:
return vae._unsloth_vae_structural.norms
decoder = vae.decoder
norms = _plain_group_norms(decoder)
silus = _plain_silus(decoder)
state = types.SimpleNamespace(
status = "pending", norms = len(norms), gn_silu = 0, gn = 0, calls = 0, kernel_calls = 0,
max_abs = None, mean_abs = None, site_max_abs = None, site_mean_abs = None, local = threading.local(),
paired = None, consumers = None, cached = False, key = None, act_off = False, foreign_reads = 0,
) # fmt: skip
inner = vae._decode
torch = _torch()
def _fresh(fn: Any) -> Any:
def call(*a: Any, **k: Any) -> Any:
_clear_decoder_cache(vae)
return fn(*a, **k)
return call
def calibrate(z: Any, args: tuple, kwargs: dict) -> None:
key = state.key = structural_cache_key(vae, norms, silus, getattr(z, "device", None))
cached = _load_pairing(key, len(norms))
if cached is not None:
# a verdict for this exact class / config / layout / torch / triton / device: no trace, no checks
state.cached = True
if cached["status"] != "ok":
state.status = cached["status"]
_structural_unpatch(norms, silus, vae)
return
paired = state.paired = cached["paired"]
state.consumers = cached["consumers"]
_structural_patch(norms, paired, silus, state, logger, cached["consumers"])
state.status = "ok"
state.gn_silu = sum(1 for p in paired if p)
state.gn = sum(1 for p in paired if p is False)
state.max_abs, state.mean_abs = cached.get("max_abs"), cached.get("mean_abs")
state.site_max_abs, state.site_mean_abs = (
cached.get("site_max_abs"),
cached.get("site_mean_abs"),
)
state.calls = 0
return
_calibrate(z, args, kwargs)
if state.status in ("ok", "rejected", "no_match"):
_save_pairing(key, {
"status": state.status, "paired": state.paired, "consumers": state.consumers,
"max_abs": state.max_abs, "mean_abs": state.mean_abs,
"site_max_abs": state.site_max_abs, "site_mean_abs": state.site_mean_abs,
}) # fmt: skip
def _calibrate(z: Any, args: tuple, kwargs: dict) -> None:
tiny = _probe_latent(z)
with torch.no_grad():
calls, consumed, ref, info = _trace_group_norm_silu(
norms, silus, lambda: _decoded_tensor(_fresh(inner)(tiny, *args, **kwargs))
)
# fused with its SiLU only when that SiLU is the output's sole reader; otherwise GroupNorm alone
paired = [None if c == 0 else (c == k) for c, k in zip(calls, consumed)]
state.paired, state.consumers = paired, info["consumers"]
n = _structural_patch(norms, paired, silus, state, logger, info["consumers"])
if n == 0:
state.status = "no_match"
_structural_unpatch(norms, silus, vae)
return
state.local.kernel_runs = []
handles, worst = _site_hooks(norms, paired)
try:
out = _decoded_tensor(_fresh(inner)(tiny, *args, **kwargs))
fused_ran = bool(state.local.kernel_runs)
finally:
state.local.kernel_runs = None
for h in handles:
h.remove()
site = (worst[0], worst[1]) if worst[2] else None
err = (out.float() - ref.float()).abs()
state.max_abs, state.mean_abs = float(err.max()), float(err.mean())
state.site_max_abs, state.site_mean_abs = site if site is not None else (None, None)
sites_ok = site is not None and site[0] <= _PROBE_MAX_ABS and site[1] <= _PROBE_MEAN_ABS
if (
fused_ran
and sites_ok
and state.max_abs <= _PROBE_MAX_ABS
and state.mean_abs <= _PROBE_MEAN_ABS
):
state.status = "ok"
state.gn_silu = sum(1 for p in paired if p)
state.gn = sum(1 for p in paired if p is False)
state.calls = 0
if logger is not None:
logger.info(
"diffusion.vae_fused: %s matched by structure: %d GroupNorm+SiLU, %d GroupNorm fused "
"(probe max |d| %.4f, mean |d| %.5f)",
type(vae).__name__, state.gn_silu, state.gn, state.max_abs, state.mean_abs,
) # fmt: skip
else:
state.status = "rejected"
_structural_unpatch(norms, silus, vae)
if logger is not None:
logger.warning(
"diffusion.vae_fused: %s structural fusion rejected (probe max |d| %s, mean |d| %s)",
type(vae).__name__, state.max_abs, state.mean_abs,
) # fmt: skip
def _decode(z: Any, *args: Any, **kwargs: Any) -> Any:
if state.status == "pending" and getattr(z, "is_cuda", False):
with _TRACE_LOCK:
if state.status == "pending":
try:
calibrate(z, args, kwargs)
except Exception as exc: # noqa: BLE001 - never fail a decode over the probe
state.status = "rejected"
_structural_unpatch(norms, silus, vae)
if logger is not None:
logger.warning("diffusion.vae_fused: structural probe failed: %s", exc)
_clear_decoder_cache(
vae
) # the real decode must not see the probe's causal cache
aborted = False
try:
return inner(z, *args, **kwargs)
except FusedActivationRead:
if state.status != "ok":
raise
aborted = True
if aborted:
_activation_fusion_off()
_clear_decoder_cache(vae)
return inner(z, *args, **kwargs)
def _activation_fusion_off() -> None:
"""A fused activation was read out of turn at this decode's shape / dataflow: from now on (and in later
processes: the cached verdict is downgraded) this VAE fuses GroupNorm only and every SiLU runs stock."""
with _TRACE_LOCK:
if not any(state.paired or ()):
return
paired = [False if p else p for p in state.paired]
_structural_unpatch(norms, silus)
_structural_patch(norms, paired, silus, state, logger)
state.paired, state.consumers = paired, [[] for _ in norms]
state.gn += state.gn_silu
state.gn_silu = 0
state.act_off = True
if logger is not None:
logger.warning(
"diffusion.vae_fused: %s read a fused GroupNorm+SiLU output out of turn; decoding again with "
"GroupNorm fused alone", type(vae).__name__,
) # fmt: skip
_save_pairing(getattr(state, "key", None), {
"status": "ok", "paired": paired, "consumers": state.consumers,
"max_abs": state.max_abs, "mean_abs": state.mean_abs,
"site_max_abs": state.site_max_abs, "site_mean_abs": state.site_mean_abs,
}) # fmt: skip
_decode._unsloth_vae_fused = True
vae._unsloth_vae_structural = state
from . import decode_layers # noqa: PLC0415
decode_layers.wrap(vae, "_decode", _STRUCTURAL_LAYER, lambda _inner, _layer: _decode)
if logger is not None:
logger.info(
"diffusion.vae_fused: %s is outside the measured set; %d GroupNorms will be matched by structure "
"on the first decode",
type(vae).__name__,
len(norms),
)
return len(norms)