Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: faster MiniMax-H3 video VAE encode and decode (#11801)

Daniel Han14354db43cbd
diff --git a/studio/backend/core/inference/diffusion_speed.py b/studio/backend/core/inference/diffusion_speed.py
index f2a07df3582..97becc5c7ec 100644
--- a/studio/backend/core/inference/diffusion_speed.py
+++ b/studio/backend/core/inference/diffusion_speed.py
@@ -35,8 +35,10 @@ from __future__ import annotations
import os
import sys
+import threading
+from contextlib import contextmanager
from functools import lru_cache
-from typing import Any, Optional
+from typing import Any, Iterator, Optional
from . import diffusion_gguf_compile as gguf_compile
@@ -58,6 +60,52 @@ _INDUCTOR_FLAGS = (
_INDUCTOR_TRITON_FLAGS = (("unique_kernel_names", "inductor_triton_unique_kernel_names"),)
+# allow_fp16_accumulation has one owner: writes during an open scope go to the recorded process value, so closing
+# the scope restores the latest value, not a stale copy.
+_FP16_ACCUM_LOCK = threading.RLock()
+_fp16_accum_scopes: list = []
+_fp16_accum_base: Optional[bool] = None
+
+
+def _read_fp16_accum(matmul: Any) -> bool:
+ with _FP16_ACCUM_LOCK:
+ if _fp16_accum_scopes:
+ return bool(_fp16_accum_base)
+ return bool(matmul.allow_fp16_accumulation)
+
+
+def _write_fp16_accum(matmul: Any, value: Any) -> None:
+ global _fp16_accum_base
+ with _FP16_ACCUM_LOCK:
+ if _fp16_accum_scopes:
+ _fp16_accum_base = bool(value)
+ else:
+ matmul.allow_fp16_accumulation = bool(value)
+
+
+@contextmanager
+def fp16_accumulation_scope(value: bool) -> Iterator[None]:
+ """Hold ``allow_fp16_accumulation`` at ``value`` for the body, then restore the latest process value."""
+ global _fp16_accum_base
+ import torch
+
+ matmul = torch.backends.cuda.matmul
+ token = object()
+ with _FP16_ACCUM_LOCK:
+ if not _fp16_accum_scopes:
+ _fp16_accum_base = bool(matmul.allow_fp16_accumulation)
+ _fp16_accum_scopes.append((token, bool(value)))
+ matmul.allow_fp16_accumulation = bool(value)
+ try:
+ yield
+ finally:
+ with _FP16_ACCUM_LOCK:
+ _fp16_accum_scopes[:] = [e for e in _fp16_accum_scopes if e[0] is not token]
+ matmul.allow_fp16_accumulation = (
+ _fp16_accum_scopes[-1][1] if _fp16_accum_scopes else bool(_fp16_accum_base)
+ )
+
+
def snapshot_backend_flags() -> Optional[dict]:
"""Capture the process-wide torch backend flags this layer may mutate, for restore on unload. None
without torch. Each flag is read defensively: a build missing one still captures the rest."""
@@ -70,7 +118,7 @@ def snapshot_backend_flags() -> Optional[dict]:
if matmul is not None and hasattr(matmul, "allow_tf32"):
state["matmul_tf32"] = bool(matmul.allow_tf32)
if matmul is not None and hasattr(matmul, "allow_fp16_accumulation"):
- state["matmul_fp16_accum"] = bool(matmul.allow_fp16_accumulation)
+ state["matmul_fp16_accum"] = _read_fp16_accum(matmul)
cudnn = getattr(torch.backends, "cudnn", None)
if cudnn is not None:
if hasattr(cudnn, "allow_tf32"):
@@ -122,7 +170,15 @@ def restore_backend_flags(state: Optional[dict]) -> None:
pass
matmul = getattr(getattr(torch.backends, "cuda", None), "matmul", None)
_set(matmul, "allow_tf32", "matmul_tf32")
- _set(matmul, "allow_fp16_accumulation", "matmul_fp16_accum")
+ if (
+ matmul is not None
+ and "matmul_fp16_accum" in state
+ and hasattr(matmul, "allow_fp16_accumulation")
+ ):
+ try:
+ _write_fp16_accum(matmul, state["matmul_fp16_accum"])
+ except Exception: # noqa: BLE001 - best-effort per-flag restore
+ pass
cudnn = getattr(torch.backends, "cudnn", None)
_set(cudnn, "allow_tf32", "cudnn_tf32")
_set(cudnn, "benchmark", "cudnn_benchmark")
@@ -861,7 +917,7 @@ def _enable_fp16_accumulation(
if not _is_consumer_gpu():
return False
- matmul.allow_fp16_accumulation = True
+ _write_fp16_accum(matmul, True)
return True
except Exception as exc: # noqa: BLE001 - optimisation only
_warn(logger, "fp16_accum", exc)
diff --git a/studio/backend/core/inference/video.py b/studio/backend/core/inference/video.py
index 5adeeb2dcb4..fce79d8d0da 100644
--- a/studio/backend/core/inference/video.py
+++ b/studio/backend/core/inference/video.py
@@ -5136,6 +5136,7 @@ class VideoBackend:
# Resolved BEFORE the placement below, not after it: the dense pin is a speed optimisation by its own reasoning,
# so "off" has to be known in time to decline it.
effective_speed = resolve_speed_mode(speed_mode, is_gguf = False, dense_default = SPEED_DEFAULT)
+ h3_vae_speed = effective_speed
if effective_speed == SPEED_MAX:
# SPEED_MAX compiles static until a dimension changes. H3's packed sequence length carries the caption's
# token rows, so a static graph recompiles on new prompts: measured 0.957-1.000 s/step static against
@@ -5308,6 +5309,18 @@ class VideoBackend:
speed_optims = tuple(k for k, v in applied.items() if v)
except Exception as exc: # noqa: BLE001 -- optimisation only, never fail a load
logger.warning("video.h3_speed_optims failed, continuing unoptimised: %s", exc)
+ # nothing here compiles, so it follows the REQUESTED tier, not the denoiser's eager downgrade above
+ try:
+ from .video_minimax_h3_vae import apply_h3_vae_speedups
+ vae_levers = apply_h3_vae_speedups(
+ getattr(pipe, "vae", None),
+ speed_mode = h3_vae_speed,
+ workflow = workflow,
+ logger = logger,
+ )
+ speed_optims = speed_optims + tuple(f"h3_vae_{name}" for name in vae_levers)
+ except Exception as exc: # noqa: BLE001 -- optimisation only, never fail a load
+ logger.warning("video.h3_vae_fast failed, keeping the stock VAE: %s", exc)
resolved = build_resolved_record(
{
@@ -6330,6 +6343,9 @@ class VideoBackend:
# must not keep reporting it compiled (a forced-compile quantised load runs ~30x slower eager),
# whether this render finished, was cancelled or failed.
settle_compile_fallback(state, pipe, logger)
+ if fam.modular_workflow:
+ from .video_minimax_h3_vae import settle_h3_vae_fallback
+ settle_h3_vae_fallback(state, pipe)
if cancel.is_set():
raise RuntimeError(VIDEO_CANCELLED_MSG)
diff --git a/studio/backend/core/inference/video_minimax_h3_vae.py b/studio/backend/core/inference/video_minimax_h3_vae.py
new file mode 100644
index 00000000000..826c1571340
--- /dev/null
+++ b/studio/backend/core/inference/video_minimax_h3_vae.py
@@ -0,0 +1,1309 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Faster MiniMax-H3 video VAE encode and decode: Triton-fused GroupNorm/SiLU/pad, RMSNorm/rope/SwiGLU and
+tile batching around Diffusers' own matmuls and convolutions.
+
+Levers: fused_encoder, fp16_encoder, fused_decoder, tile_batch, fp16_accum, int8_decoder (opt-in,
+``UNSLOTH_H3_VAE_INT8=1``). ``UNSLOTH_H3_VAE_FAST=0`` disables all. NVIDIA CUDA with Triton only; else stock.
+"""
+
+from __future__ import annotations
+
+import os
+import sys
+import types
+from contextlib import nullcontext
+from functools import lru_cache
+from typing import Any, Optional
+
+H3_VAE_FAST_ENV = "UNSLOTH_H3_VAE_FAST"
+H3_VAE_INT8_ENV = "UNSLOTH_H3_VAE_INT8"
+H3_VAE_TILE_BATCH_ENV = "UNSLOTH_H3_VAE_TILE_BATCH"
+FP16_ACCUM_DISABLE_ENV = "UNSLOTH_DISABLE_FP16_ACCUM"
+
+LEVER_FUSED_ENCODER = "fused_encoder"
+LEVER_FP16_ENCODER = "fp16_encoder"
+LEVER_FUSED_DECODER = "fused_decoder"
+LEVER_TILE_BATCH = "tile_batch"
+LEVER_INT8_DECODER = "int8_decoder"
+LEVER_FP16_ACCUM = "fp16_accum"
+
+_TILE_BATCH_MAX = 4
+_TILE_BATCH_BYTES_PER_TILE = 256 * 2**20
+H3_VAE_INT8_ROT_GROUP = 256
+# int8 error in early blocks propagates through the rest, so the first half stays float.
+H3_VAE_INT8_FLOAT_BLOCKS = 18
+# Triton 3.2 silently computes wrong GroupNorm statistics for a channels-last input.
+H3_VAE_MIN_TRITON = (3, 3)
+
+_SPEED_OFF = "off"
+_SPEED_EAGER = "eager"
+_SPEED_MAX = "max"
+_TRUE = ("1", "true", "yes", "on")
+_FALSE = ("0", "false", "no", "off")
+
+
+def _env(name: str) -> str:
+ return os.environ.get(name, "").strip().lower()
+
+
+def h3_vae_fast_disabled() -> bool:
+ return _env(H3_VAE_FAST_ENV) in _FALSE
+
+
+def plan_h3_vae_levers(
+ speed_mode: Optional[str],
+ *,
+ workflow: Optional[str] = None,
+ consumer_gpu: bool = False,
+) -> tuple[str, ...]:
+ """Which levers a load should engage. Pure: no torch, no device probe."""
+ mode = str(speed_mode or "").strip().lower()
+ if h3_vae_fast_disabled() or mode == _SPEED_OFF:
+ return ()
+ levers: list[str] = []
+ if workflow != "t2va":
+ levers.append(LEVER_FUSED_ENCODER)
+ if mode != _SPEED_EAGER:
+ levers.append(LEVER_FP16_ENCODER)
+ levers += [LEVER_FUSED_DECODER, LEVER_TILE_BATCH]
+ if _env(H3_VAE_INT8_ENV) in _TRUE:
+ levers.append(LEVER_INT8_DECODER)
+ if mode == _SPEED_MAX and consumer_gpu and _env(FP16_ACCUM_DISABLE_ENV) not in _TRUE:
+ levers.append(LEVER_FP16_ACCUM)
+ return tuple(levers)
+
+
+@lru_cache(maxsize = 1)
+def _kernels() -> Optional[types.SimpleNamespace]:
+ """Compile-on-first-use Triton kernels, or None when Triton is unavailable."""
+ try:
+ import triton
+ import triton.language as tl
+ from triton.language.extra import libdevice
+ except Exception: # noqa: BLE001 - no Triton means the stock path
+ return None
+ # Triton <= 3.2 resolves string annotations against module globals, not this function's locals
+ globals().update(triton = triton, tl = tl, libdevice = libdevice)
+
+ @triton.jit
+ def _gn_partials(
+ x_ptr,
+ bias_ptr,
+ pn_ptr,
+ pmean_ptr,
+ pm2_ptr,
+ T,
+ HW,
+ W,
+ n_chunks,
+ sb,
+ sc,
+ st,
+ sh,
+ sw,
+ C: tl.constexpr,
+ G: tl.constexpr,
+ CPG: tl.constexpr,
+ BLOCK_P: tl.constexpr,
+ HAS_BIAS: tl.constexpr,
+ ):
+ chunk = tl.program_id(0)
+ bt = tl.program_id(1)
+ t = bt % T
+ b = bt // T
+ p = chunk * BLOCK_P + tl.arange(0, BLOCK_P)
+ pmask = p < HW
+ hh = p // W
+ ww = p - hh * W
+ c = tl.arange(0, C)
+ off = (
+ b.to(tl.int64) * sb
+ + t.to(tl.int64) * st
+ + hh.to(tl.int64)[:, None] * sh
+ + ww.to(tl.int64)[:, None] * sw
+ + c.to(tl.int64)[None, :] * sc
+ )
+ v = tl.load(x_ptr + off, mask = pmask[:, None], other = 0.0).to(tl.float32)
+ if HAS_BIAS:
+ v = v + tl.load(bias_ptr + c).to(tl.float32)[None, :]
+ v = tl.where(pmask[:, None], v, 0.0)
+ n_pix = tl.sum(pmask.to(tl.float32), axis = 0)
+ v3 = tl.reshape(v, (BLOCK_P, G, CPG))
+ mean = tl.sum(tl.sum(v3, axis = 2), axis = 0) / (n_pix * CPG)
+ d = tl.where(pmask[:, None, None], v3 - mean[None, :, None], 0.0)
+ m2 = tl.sum(tl.sum(d * d, axis = 2), axis = 0)
+ slot = bt.to(tl.int64) * n_chunks + chunk
+ g = tl.arange(0, G)
+ tl.store(pmean_ptr + slot * G + g, mean)
+ tl.store(pm2_ptr + slot * G + g, m2)
+ tl.store(pn_ptr + slot, n_pix * CPG)
+
+ @triton.jit
+ def _gn_combine(
+ pn_ptr, pmean_ptr, pm2_ptr, mean_ptr, rstd_ptr, n_chunks, G, eps, BLOCK: tl.constexpr
+ ):
+ row = tl.program_id(0)
+ g = row % G
+ bt = (row // G).to(tl.int64)
+ zero = tl.sum(tl.zeros([BLOCK], dtype = tl.float32), axis = 0)
+ acc_n = zero
+ acc_s = zero
+ for k0 in range(0, n_chunks, BLOCK):
+ k = k0 + tl.arange(0, BLOCK)
+ km = k < n_chunks
+ n = tl.load(pn_ptr + bt * n_chunks + k, mask = km, other = 0.0)
+ mu = tl.load(pmean_ptr + (bt * n_chunks + k) * G + g, mask = km, other = 0.0)
+ acc_n += tl.sum(n, axis = 0)
+ acc_s += tl.sum(n * mu, axis = 0)
+ mean = acc_s / acc_n
+ acc_m2 = zero
+ for k0 in range(0, n_chunks, BLOCK):
+ k = k0 + tl.arange(0, BLOCK)
+ km = k < n_chunks
+ n = tl.load(pn_ptr + bt * n_chunks + k, mask = km, other = 0.0)
+ mu = tl.load(pmean_ptr + (bt * n_chunks + k) * G + g, mask = km, other = 0.0)
+ m2 = tl.load(pm2_ptr + (bt * n_chunks + k) * G + g, mask = km, other = 0.0)
+ dm = mu - mean
+ acc_m2 += tl.sum(m2 + n * dm * dm, axis = 0)
+ tl.store(mean_ptr + row, mean)
+ tl.store(rstd_ptr + row, 1.0 / tl.sqrt(acc_m2 / acc_n + eps))
+
+ @triton.jit
+ def _gn_silu_pad(
+ x_ptr,
+ out_ptr,
+ mean_ptr,
+ rstd_ptr,
+ w_ptr,
+ b_ptr,
+ ib_ptr,
+ C,
+ T,
+ H,
+ W,
+ To,
+ Ho,
+ Wo,
+ G,
+ front,
+ pad_t,
+ pad_l,
+ sb,
+ sc,
+ st,
+ sh,
+ sw,
+ CPG: tl.constexpr,
+ HAS_NORM: tl.constexpr,
+ HAS_IN_BIAS: tl.constexpr,
+ BLOCK_W: tl.constexpr,
+ BLOCK_C: tl.constexpr,
+ ):
+ pid_row = tl.program_id(0)
+ pid_w = tl.program_id(1)
+ pid_c = tl.program_id(2)
+ h_o = pid_row % Ho
+ bt = pid_row // Ho
+ t_o = bt % To
+ b = bt // To
+ t_i = t_o - front
+ t_ok = t_i >= 0
+ t_c = tl.maximum(t_i, 0)
+ r = h_o - pad_t
+ r = tl.where(r < 0, -r, r)
+ r = tl.where(r >= H, 2 * (H - 1) - r, r)
+ in_base = x_ptr + (b.to(tl.int64) * sb + t_c.to(tl.int64) * st + r.to(tl.int64) * sh)
+ out_base = out_ptr + (((b.to(tl.int64) * To + t_o) * Ho + h_o) * Wo) * C
+ wo = pid_w * BLOCK_W + tl.arange(0, BLOCK_W)
+ wmask = wo < Wo
+ q = wo - pad_l
+ q = tl.where(q < 0, -q, q)
+ q = tl.where(q >= W, 2 * (W - 1) - q, q)
+ c = pid_c * BLOCK_C + tl.arange(0, BLOCK_C)
+ cmask = c < C
+ m = wmask[:, None] & cmask[None, :]
+ # int64: an NCDHW channel stride x 127 passes 2^31 on a large untiled encode
+ in_off = q.to(tl.int64)[:, None] * sw + c.to(tl.int64)[None, :] * sc
+ v = tl.load(in_base + in_off, mask = m & t_ok, other = 0.0).to(tl.float32)
+ if HAS_IN_BIAS:
+ v = v + tl.load(ib_ptr + c, mask = cmask, other = 0.0).to(tl.float32)[None, :]
+ if HAS_NORM:
+ gidx = (b * T + t_c) * G + c // CPG
+ mu = tl.load(mean_ptr + gidx, mask = cmask, other = 0.0)
+ rs = tl.load(rstd_ptr + gidx, mask = cmask, other = 0.0)
+ gw = tl.load(w_ptr + c, mask = cmask, other = 0.0).to(tl.float32)
+ gb = tl.load(b_ptr + c, mask = cmask, other = 0.0).to(tl.float32)
+ a = rs * gw
+ v = v * a[None, :] + (gb - mu * a)[None, :]
+ v = v / (1.0 + tl.exp(-v))
+ v = tl.where(t_ok, v, 0.0)
+ tl.store(out_base + (wo[:, None] * C + c[None, :]), v.to(out_ptr.dtype.element_ty), mask = m)
+
+ @triton.jit
+ def _add_residual(
+ o_ptr,
+ ob_ptr,
+ r_ptr,
+ rb_ptr,
+ P,
+ T,
+ H,
+ W,
+ sb,
+ sc,
+ st,
+ sh,
+ sw,
+ C: tl.constexpr,
+ HAS_OB: tl.constexpr,
+ HAS_RB: tl.constexpr,
+ R_SAME: tl.constexpr,
+ BLOCK_P: tl.constexpr,
+ ):
+ p = tl.program_id(0) * BLOCK_P + tl.arange(0, BLOCK_P)
+ pmask = p < P
+ c = tl.arange(0, C)
+ base = tl.program_id(0).to(tl.int64) * BLOCK_P * C
+ local = (tl.arange(0, BLOCK_P) * C)[:, None] + c[None, :]
+ m = pmask[:, None]
+ o = tl.load(o_ptr + base + local, mask = m, other = 0.0).to(tl.float32)
+ if HAS_OB:
+ o = o + tl.load(ob_ptr + c).to(tl.float32)[None, :]
+ if R_SAME:
+ r = tl.load(r_ptr + base + local, mask = m, other = 0.0).to(tl.float32)
+ else:
+ w = p % W
+ rest = p // W
+ h = rest % H
+ rest = rest // H
+ t = rest % T
+ b = rest // T
+ poff = (
+ b.to(tl.int64) * sb
+ + t.to(tl.int64) * st
+ + h.to(tl.int64) * sh
+ + w.to(tl.int64) * sw
+ )
+ r = tl.load(
+ r_ptr + poff[:, None] + (c.to(tl.int64) * sc)[None, :], mask = m, other = 0.0
+ ).to(tl.float32)
+ if HAS_RB:
+ r = r + tl.load(rb_ptr + c).to(tl.float32)[None, :]
+ tl.store(o_ptr + base + local, (o + r).to(o_ptr.dtype.element_ty), mask = m)
+
+ @triton.jit
+ def _add_rmsnorm(
+ h_ptr, o_ptr, s_ptr, w_ptr, n_ptr, N, eps, HAS_RES: tl.constexpr, BLOCK_N: tl.constexpr
+ ):
+ row = tl.program_id(0).to(tl.int64)
+ cols = tl.arange(0, BLOCK_N)
+ mask = cols < N
+ h = tl.load(h_ptr + row * N + cols, mask = mask, other = 0.0).to(tl.float32)
+ if HAS_RES:
+ o = tl.load(o_ptr + row * N + cols, mask = mask, other = 0.0).to(tl.float32)
+ s = tl.load(s_ptr + cols, mask = mask, other = 0.0).to(tl.float32)
+ # the reference's separate multiply and add, each rounded (no fma contraction)
+ h = libdevice.add_rn(h, libdevice.mul_rn(o, s))
+ tl.store(h_ptr + row * N + cols, h.to(h_ptr.dtype.element_ty), mask = mask)
+ ms = tl.sum(h * h, axis = 0) / N
+ w = tl.load(w_ptr + cols, mask = mask, other = 0.0).to(tl.float32)
+ y = h * (1.0 / tl.sqrt(ms + eps)) * w
+ tl.store(n_ptr + row * N + cols, y.to(n_ptr.dtype.element_ty), mask = mask)
+
+ @triton.jit
+ def _qk_norm_rope(
+ x_ptr,
+ cos_ptr,
+ sin_ptr,
+ n_head_rows,
+ heads,
+ cs_stride,
+ eps,
+ HEAD_D: tl.constexpr,
+ ROT: tl.constexpr,
+ BLOCK: tl.constexpr,
+ ):
+ hr = tl.program_id(0).to(tl.int64) * BLOCK + tl.arange(0, BLOCK)
+ rmask = hr < n_head_rows
+ d = tl.arange(0, HEAD_D)
+ half = ROT // 2
+ partner = tl.where(d < half, d + half, tl.where(d < ROT, d - half, d))
+ base = hr[:, None] * HEAD_D
+ x = tl.load(x_ptr + base + d[None, :], mask = rmask[:, None], other = 0.0).to(tl.float32)
+ xp = tl.load(x_ptr + base + partner[None, :], mask = rmask[:, None], other = 0.0).to(tl.float32)
+ rstd = 1.0 / tl.sqrt(tl.sum(x * x, axis = 1) / HEAD_D + eps)
+ dt = x_ptr.dtype.element_ty
+ xn = (x * rstd[:, None]).to(dt).to(tl.float32)
+ xpn = (xp * rstd[:, None]).to(dt).to(tl.float32)
+ rot = d < ROT
+ token = hr // heads
+ cs_off = token[:, None] * cs_stride + tl.where(rot, d, 0)[None, :]
+ cs_m = rmask[:, None] & rot[None, :]
+ cos = tl.load(cos_ptr + cs_off, mask = cs_m, other = 1.0).to(tl.float32)
+ sin = tl.load(sin_ptr + cs_off, mask = cs_m, other = 0.0).to(tl.float32)
+ # mul_rn / add_rn: without them the compiler contracts into one fma and skips eager's rounding.
+ xps = tl.where((d < half)[None, :], -xpn, xpn)
+ a = libdevice.mul_rn(xn, cos).to(dt).to(tl.float32)
+ bb = libdevice.mul_rn(xps, sin).to(dt).to(tl.float32)
+ y = tl.where(rot[None, :], libdevice.add_rn(a, bb), xn)
+ tl.store(x_ptr + base + d[None, :], y.to(dt), mask = rmask[:, None])
+
+ @triton.jit
+ def _swiglu(x_ptr, out_ptr, N, BLOCK_N: tl.constexpr):
+ row = tl.program_id(0).to(tl.int64)
+ pid_n = tl.program_id(1)
+ cols = pid_n * BLOCK_N + tl.arange(0, BLOCK_N)
+ mask = cols < N
+ hid = tl.load(x_ptr + row * 2 * N + cols, mask = mask, other = 0.0).to(tl.float32)
+ gate = tl.load(x_ptr + row * 2 * N + N + cols, mask = mask, other = 0.0).to(tl.float32)
+ dt = out_ptr.dtype.element_ty
+ act = (gate / (1.0 + tl.exp(-gate))).to(dt).to(tl.float32)
+ tl.store(out_ptr + row * N + cols, (hid * act).to(dt), mask = mask)
+
+ @triton.jit
+ def _quant_rows(x_ptr, q_ptr, s_ptr, N, BLOCK_N: tl.constexpr):
+ # IEEE division + half-to-even rounding: the same codes as the torch path.
+ row = tl.program_id(0).to(tl.int64)
+ cols = tl.arange(0, BLOCK_N)
+ mask = cols < N
+ x = tl.load(x_ptr + row * N + cols, mask = mask, other = 0.0).to(tl.float32)
+ amax = tl.maximum(tl.max(tl.abs(x), axis = 0), 1e-12)
+ s = amax / 127.0
+ q = libdevice.rint(libdevice.div_rn(x, s))
+ q = tl.minimum(tl.maximum(q, -127.0), 127.0)
+ tl.store(q_ptr + row * N + cols, q.to(tl.int8), mask = mask)
+ tl.store(s_ptr + row, s)
+
+ @triton.jit
+ def _dequant_epilogue(
+ acc_ptr, xs_ptr, ws_ptr, bias_ptr, out_ptr, N, HAS_BIAS: tl.constexpr, BLOCK_N: tl.constexpr
+ ):
+ row = tl.program_id(0).to(tl.int64)
+ pid_n = tl.program_id(1)
+ cols = pid_n * BLOCK_N + tl.arange(0, BLOCK_N)
+ mask = cols < N
+ acc = tl.load(acc_ptr + row * N + cols, mask = mask, other = 0).to(tl.float32)
+ xs = tl.load(xs_ptr + row)
+ ws = tl.load(ws_ptr + cols, mask = mask, other = 0.0)
+ y = acc * xs * ws
+ if HAS_BIAS:
+ y = y + tl.load(bias_ptr + cols, mask = mask, other = 0.0).to(tl.float32)
+ tl.store(out_ptr + row * N + cols, y.to(out_ptr.dtype.element_ty), mask = mask)
+
+ return types.SimpleNamespace(
+ gn_partials = _gn_partials,
+ gn_combine = _gn_combine,
+ gn_silu_pad = _gn_silu_pad,
+ add_residual = _add_residual,
+ add_rmsnorm = _add_rmsnorm,
+ qk_norm_rope = _qk_norm_rope,
+ swiglu = _swiglu,
+ quant_rows = _quant_rows,
+ dequant_epilogue = _dequant_epilogue,
+ )
+
+
+def _next_pow2(n: int) -> int:
+ return 1 << max(0, int(n) - 1).bit_length()
+
+
+def _is_oom(exc: BaseException) -> bool:
+ try:
+ import torch
+ if isinstance(exc, torch.cuda.OutOfMemoryError):
+ return True
+ except Exception: # noqa: BLE001
+ pass
+ return "out of memory" in str(exc).lower()
+
+
+def _guarded(fast: Any, stock: Any, label: str) -> Any:
+ """Runs ``fast`` until it raises something ``stock`` does not, then logs once and runs ``stock``."""
+
+ def forward(self, *args, **kwargs):
+ if getattr(self, "_unsloth_fast_failed", False):
+ return stock(self, *args, **kwargs)
+ try:
+ return fast(self, *args, **kwargs)
+ except Exception as exc: # noqa: BLE001
+ if _is_oom(exc):
+ raise
+ # the stock path rejects this input too: caller's error, keep the fast path
+ out = stock(self, *args, **kwargs)
+ self._unsloth_fast_failed = True
+ log = getattr(self, "_unsloth_logger", None)
+ if log is not None:
+ log.warning(
+ "video.h3_vae_fast: the fused %s failed, using the stock path: %s", label, exc
+ )
+ return out
+
+ return forward
+
+
+# Activations travel as (tensor, pending_bias): the next fused pass adds the conv bias, avoiding PyTorch's
+# separate strided bias pass. A 1x1 conv folds its input's pending bias through its weight exactly.
+
+
+def _bias_view(bias: Any) -> Any:
+ return bias.view(1, -1, 1, 1, 1)
+
+
+def norm_silu_pad_reference(
+ x: Any,
+ norm: Any,
+ pad: tuple,
+ front: int,
+ in_bias: Any = None,
+) -> Any:
+ """Unfused reference of the kernel, for tests and hosts without Triton."""
+ import torch
+ import torch.nn.functional as F
+
+ if norm is not None:
+ b, c, t, h, w = x.shape
+ y = x.float() if in_bias is None else x.float() + _bias_view(in_bias).float()
+ y = y.permute(0, 2, 1, 3, 4).reshape(b * t, c, h, w)
+ y = F.group_norm(y, norm.num_groups, norm.weight.float(), norm.bias.float(), norm.eps)
+ x = F.silu(y).view(b, t, c, h, w).permute(0, 2, 1, 3, 4).to(x.dtype)
+ elif in_bias is not None:
+ x = (x.float() + _bias_view(in_bias).float()).to(x.dtype)
+ return _pad_reference(x, pad, front).contiguous(memory_format = torch.channels_last_3d)
+
+
+def _pad_reference(x: Any, pad: tuple, front: int) -> Any:
+ """Reflect ``(left, right, top, bottom)`` spatially, then ``front`` causal zero frames."""
+ import torch.nn.functional as F
+
+ if any(pad):
+ x = F.pad(x, (*pad, 0, 0), mode = "reflect")
+ return F.pad(x, (0, 0, 0, 0, front, 0)) if front else x
+
+
+def _fusable(x: Any, pad: tuple, norm: Any) -> bool:
+ c = x.shape[1]
+ return (
+ _kernels() is not None
+ and x.is_cuda
+ and max(pad[0], pad[1]) < x.shape[-1]
+ and max(pad[2], pad[3]) < x.shape[-2]
+ and (norm is None or (c & (c - 1) == 0 and c % int(norm.num_groups) == 0 and c <= 1024))
+ )
+
+
+def norm_silu_pad(
+ x: Any,
+ norm: Any,
+ pad: tuple,
+ front: int,
+ in_bias: Any = None,
+) -> Any:
+ """Fused pending-bias add + per-frame GroupNorm + SiLU + reflect/causal pad, channels-last out."""
+ import torch
+
+ if not _fusable(x, pad, norm):
+ return norm_silu_pad_reference(x, norm, pad, front, in_bias)
+ k = _kernels()
+ left, right, top, bottom = pad
+ b, c, t, h, w = x.shape
+ to, ho, wo = t + front, h + top + bottom, w + left + right
+ out = torch.empty((b, to, ho, wo, c), dtype = x.dtype, device = x.device).permute(0, 4, 1, 2, 3)
+ sb, sc, st, sh, sw = x.stride()
+ ib = in_bias if in_bias is not None else x
+ if norm is not None:
+ g = int(norm.num_groups)
+ cpg = c // g
+ hw = h * w
+ budget = 4096 if sw == 1 and x.element_size() == 4 else 8192
+ block_p = max(1, min(128, budget // c))
+ n_chunks = (hw + block_p - 1) // block_p
+ rows = b * t
+ pmean, pm2 = torch.empty((2, rows * n_chunks * g), dtype = torch.float32, device = x.device)
+ pn = torch.empty(rows * n_chunks, dtype = torch.float32, device = x.device)
+ # chunks on grid axis 0: axis 1 stops at 65535, and a large untiled frame has more chunks than that
+ k.gn_partials[(n_chunks, rows)](
+ x,
+ ib,
+ pn,
+ pmean,
+ pm2,
+ t,
+ hw,
+ w,
+ n_chunks,
+ sb,
+ sc,
+ st,
+ sh,
+ sw,
+ C = c,
+ G = g,
+ CPG = cpg,
+ BLOCK_P = block_p,
+ HAS_BIAS = in_bias is not None,
+ num_warps = 4,
+ )
+ mean = torch.empty(rows * g, dtype = torch.float32, device = x.device)
+ rstd = torch.empty(rows * g, dtype = torch.float32, device = x.device)
+ k.gn_combine[(rows * g,)](
+ pn, pmean, pm2, mean, rstd, n_chunks, g, float(norm.eps), BLOCK = 1024, num_warps = 4
+ )
+ weight, bias = norm.weight, norm.bias
+ else:
+ # unused without a norm; constant, so the pad-only kernel compiles once for every channel count
+ g, cpg = 1, 1
+ mean = rstd = weight = bias = x
+ block_c = min(128, _next_pow2(c))
+ block_w = max(16, 4096 // block_c)
+ grid = (b * to * ho, (wo + block_w - 1) // block_w, (c + block_c - 1) // block_c)
+ k.gn_silu_pad[grid](
+ x,
+ out,
+ mean,
+ rstd,
+ weight,
+ bias,
+ ib,
+ c,
+ t,
+ h,
+ w,
+ to,
+ ho,
+ wo,
+ g,
+ front,
+ top,
+ left,
+ sb,
+ sc,
+ st,
+ sh,
+ sw,
+ CPG = cpg,
+ HAS_NORM = norm is not None,
+ HAS_IN_BIAS = in_bias is not None,
+ BLOCK_W = block_w,
+ BLOCK_C = block_c,
+ num_warps = 4,
+ )
+ return out
+
+
+def add_residual(out: Any, out_bias: Any, res: Any, res_bias: Any) -> Any:
+ """``out + out_bias + res + res_bias`` written into ``out`` (the conv's own fresh output)."""
+ import torch
+
+ k = _kernels()
+ c = out.shape[1]
+ if (
+ k is None
+ or not out.is_cuda
+ or not out.is_contiguous(memory_format = torch.channels_last_3d)
+ or res.shape != out.shape
+ or c & (c - 1)
+ or c > 1024
+ ):
+ if out_bias is not None:
+ out.add_(_bias_view(out_bias).to(out.dtype))
+ if res_bias is not None:
+ res = res + _bias_view(res_bias).to(res.dtype)
+ return out.add_(res)
+ b, _, t, h, w = out.shape
+ pixels = b * t * h * w
+ block_p = max(1, 4096 // c)
+ k.add_residual[((pixels + block_p - 1) // block_p,)](
+ out,
+ out_bias if out_bias is not None else out,
+ res,
+ res_bias if res_bias is not None else res,
+ pixels,
+ t,
+ h,
+ w,
+ *res.stride(),
+ C = c,
+ HAS_OB = out_bias is not None,
+ HAS_RB = res_bias is not None,
+ R_SAME = res.is_contiguous(memory_format = torch.channels_last_3d),
+ BLOCK_P = block_p,
+ num_warps = 4,
+ )
+ return out
+
+
+def _last_tap(weight: Any) -> Any:
+ """A conv weight's last temporal tap. Not cached: must follow offload hooks' ``.to()``."""
+ import torch
+ return weight[:, :, -1:].contiguous(memory_format = torch.channels_last_3d)
+
+
+def _causal_conv(
+ conv: Any,
+ x: Any,
+ x_bias: Any = None,
+ *,
+ norm: Any = None,
+ spatial_pad: Optional[tuple] = None,
+) -> tuple:
+ """``conv(pad(silu(norm(x + x_bias))))`` WITHOUT the conv's bias, which is returned as the output's pending bias."""
+ import torch.nn.functional as F
+
+ kt = conv.kernel_size[0]
+ sp = int(getattr(conv, "spatial_padding", 0) or 0)
+ front = int(getattr(conv, "temporal_padding", 0) or 0)
+ pad = (sp, sp, sp, sp) if spatial_pad is None else tuple(spatial_pad)
+ weight = conv.weight
+ bias = conv.bias
+ if x.shape[2] == 1 and kt > 1 and front == kt - 1:
+ front = 0
+ weight = _last_tap(weight)
+ if norm is not None or front or any(pad):
+ if x.shape[1] % 8 == 0 or norm is not None:
+ x = norm_silu_pad(x, norm, pad, front, in_bias = x_bias)
+ else:
+ # conv_in's 3 channels: cuDNN would convert channels-last back
+ if x_bias is not None:
+ x = x + _bias_view(x_bias).to(x.dtype)
+ x = _pad_reference(x, pad, front)
+ elif x_bias is not None:
+ # a 1x1x1 conv without padding is linear in its input, so the pending bias folds through the weight exactly
+ folded = weight.reshape(weight.shape[0], -1).float() @ x_bias.float()
+ bias = folded if bias is None else bias.float() + folded
+ out = F.conv3d(x, weight, None, stride = conv.stride)
+ return out, bias
+
+
+def _fast_encoder_forward(self: Any, hidden_states: Any) -> Any:
+ """``MiniMaxH3VideoEncoder3d.forward`` over the fused passes. Same modules, same order."""
+ import torch
+
+ # Triton launches on the current device, which on a multi-GPU host need not be the tensor's
+ with torch.cuda.device(hidden_states.device) if hidden_states.is_cuda else nullcontext():
+ return _fast_encoder_body(self, hidden_states)
+
+
+def _fast_encoder_body(self: Any, hidden_states: Any) -> Any:
+ dtype = getattr(self, "_unsloth_compute_dtype", None) or hidden_states.dtype
+ out_dtype = getattr(self, "_unsloth_out_dtype", None) or hidden_states.dtype
+ h, hb = _causal_conv(self.conv_in, hidden_states.to(dtype))
+ for down_block in self.down_blocks:
+ for resnet in down_block.resnets:
+ x, xb = h, hb
+ h, hb = _causal_conv(resnet.conv1, x, xb, norm = resnet.norm1)
+ h, hb = _causal_conv(resnet.conv2, h, hb, norm = resnet.norm2)
+ if resnet.conv_shortcut is None:
+ r, rb = x, xb
+ else:
+ r, rb = _causal_conv(resnet.conv_shortcut, x, xb)
+ h, hb = add_residual(h, hb, r, rb), None
+ if down_block.downsamplers is not None:
+ for ds in down_block.downsamplers:
+ pad = (0, 1, 0, 1) if ds.spatial_stride == 2 else None
+ h, hb = _causal_conv(ds.conv, h, hb, spatial_pad = pad)
+ h, hb = _causal_conv(self.conv_out, h, hb, norm = self.norm_out)
+ h = h.float()
+ if hb is not None:
+ h = h + _bias_view(hb).float()
+ return h.to(out_dtype).contiguous()
+
+
+def _install_encoder(vae: Any, *, fp16: bool) -> bool:
+ import torch
+
+ encoder = getattr(vae, "encoder", None)
+ if encoder is None or not hasattr(encoder, "down_blocks"):
+ return False
+ # the kernel pads by reflection, which is what the released config uses; any other mode keeps the stock path
+ modes = {getattr(m, "spatial_padding_mode", "reflect") for m in encoder.modules()}
+ if modes - {"reflect"}:
+ return False
+ convs = [m for m in encoder.modules() if isinstance(m, torch.nn.Conv3d)]
+ stock_dtype = encoder.conv_in.weight.dtype
+ with torch.no_grad():
+ # build every tensor before assigning: a mid-way OOM must leave the stock encoder intact
+ new = []
+ for module in convs:
+ w = module.weight.data.to(torch.float16) if fp16 else module.weight.data
+ if module is not encoder.conv_in:
+ w = w.contiguous(memory_format = torch.channels_last_3d)
+ bias = module.bias
+ b = None if bias is None else (bias.data.to(torch.float16) if fp16 else bias.data)
+ new.append((module, w, b))
+ for module, w, b in new:
+ module.weight.data = w
+ if b is not None:
+ module.bias.data = b
+ encoder._unsloth_compute_dtype = torch.float16 if fp16 else None
+ # quant_conv stays in the stock dtype: hand back what the stock encoder would
+ encoder._unsloth_out_dtype = stock_dtype if fp16 else None
+ encoder._unsloth_stock_forward = encoder.forward
+ encoder.forward = types.MethodType(
+ _guarded(_fast_encoder_forward, _stock_encoder_forward, "encoder"), encoder
+ )
+ return True
+
+
+def _stock_encoder_forward(self: Any, hidden_states: Any) -> Any:
+ """Diffusers' encoder forward, under float16 autocast when the weights were cast."""
+ import torch
+
+ if getattr(self, "_unsloth_compute_dtype", None) is not torch.float16:
+ return self._unsloth_stock_forward(hidden_states)
+ out_dtype = getattr(self, "_unsloth_out_dtype", None) or hidden_states.dtype
+ if hidden_states.is_cuda:
+ with torch.autocast("cuda", dtype = torch.float16):
+ return self._unsloth_stock_forward(hidden_states).to(out_dtype)
+ return self._unsloth_stock_forward(hidden_states.to(torch.float16)).to(out_dtype)
+
+
+def _add_rmsnorm(h: Any, o: Any, scale: Any, norm: Any, out_dtype: Any) -> Any:
+ """``h += o * scale`` (skipped when ``o`` is None), then ``rmsnorm(h) * norm.weight`` in ``out_dtype``."""
+ import torch
+
+ k = _kernels()
+ n_cols = h.shape[-1]
+ eps = norm.eps if norm.eps is not None else torch.finfo(torch.float32).eps
+ if k is None or not h.is_cuda:
+ if o is not None:
+ h.add_(o * scale)
+ return torch.nn.functional.rms_norm(h.float(), (n_cols,), norm.weight.float(), eps).to(
+ out_dtype
+ )
+ h2 = h.view(-1, n_cols)
+ normed = torch.empty(h2.shape, dtype = out_dtype, device = h.device)
+ k.add_rmsnorm[(h2.shape[0],)](
+ h2,
+ o if o is not None else h2,
+ scale if scale is not None else norm.weight,
+ norm.weight,
+ normed,
+ n_cols,
+ float(eps),
+ HAS_RES = o is not None,
+ BLOCK_N = _next_pow2(n_cols),
+ num_warps = 8,
+ )
+ return normed.view(*h.shape[:-1], n_cols)
+
+
+def qk_norm_rope_reference(x: Any, cos: Any, sin: Any, heads: int, eps: float) -> Any:
+ """The reference processor's per-head float32 RMSNorm then partial split-half rope in ``x.dtype``."""
+ import torch
+
+ shape = x.shape
+ q = x.reshape(-1, heads, shape[-1] // heads)
+ q = torch.nn.functional.rms_norm(q.float(), (q.shape[-1],), None, eps).to(x.dtype)
+ rot = cos.shape[-1]
+ c = cos.reshape(-1, 1, rot).to(x.dtype)
+ s = sin.reshape(-1, 1, rot).to(x.dtype)
+ q_rot, q_pass = q[..., :rot], q[..., rot:]
+ first, second = q_rot.chunk(2, dim = -1)
+ rotated = torch.cat([-second, first], dim = -1)
+ return torch.cat([q_rot * c + rotated * s, q_pass], dim = -1).reshape(shape)
+
+
+def _qk_norm_rope_(x: Any, cos: Any, sin: Any, heads: int, eps: float) -> None:
+ k = _kernels()
+ if k is None or not x.is_cuda or not x.is_contiguous():
+ x.copy_(qk_norm_rope_reference(x, cos, sin, heads, eps))
+ return
+ head_d = x.shape[-1] // heads
+ n = x.numel() // head_d
+ block = 32
+ k.qk_norm_rope[((n + block - 1) // block,)](
+ x,
+ cos,
+ sin,
+ n,
+ heads,
+ cos.stride(-2),
+ float(eps),
+ HEAD_D = head_d,
+ ROT = cos.shape[-1],
+ BLOCK = block,
+ num_warps = 4,
+ )
+
+
+def _swiglu(x: Any) -> Any:
+ import torch
+
+ k = _kernels()
+ n = x.shape[-1] // 2
+ if k is None or not x.is_cuda:
+ hidden, gate = x.chunk(2, dim = -1)
+ return hidden * torch.nn.functional.silu(gate)
+ rows = x.numel() // x.shape[-1]
+ out = torch.empty((*x.shape[:-1], n), dtype = x.dtype, device = x.device)
+ block_n = 2048
+ k.swiglu[(rows, (n + block_n - 1) // block_n)](x, out, n, BLOCK_N = block_n, num_warps = 4)
+ return out
+
+
+def _fast_block_stack(decoder: Any, hidden_states: Any, cos: Any, sin: Any, gemm_dtype: Any) -> Any:
+ """The transformer blocks; the kernels store normed activations in ``gemm_dtype``."""
+ import torch.nn.functional as F
+
+ blocks = decoder.transformer_blocks
+ batch, seq, dim = hidden_states.shape
+ h = hidden_states.contiguous()
+ normed = _add_rmsnorm(h, None, None, blocks[0].norm1, gemm_dtype)
+ cos2 = cos.reshape(-1, cos.shape[-1])
+ sin2 = sin.reshape(-1, sin.shape[-1])
+ for i, block in enumerate(blocks):
+ attn = block.attn
+ query = attn.to_q(normed)
+ key = attn.to_k(normed)
+ value = attn.to_v(normed)
+ _qk_norm_rope_(query, cos2, sin2, attn.heads, attn.norm_q.eps)
+ _qk_norm_rope_(key, cos2, sin2, attn.heads, attn.norm_k.eps)
+ query, key, value = (
+ t.view(batch, seq, attn.heads, -1).permute(0, 2, 1, 3) for t in (query, key, value)
+ )
+ out = F.scaled_dot_product_attention(query, key, value)
+ out = out.permute(0, 2, 1, 3).flatten(2, 3)
+ out = attn.to_out[0](out)
+ normed = _add_rmsnorm(h, out, block.scale1, block.norm2, gemm_dtype)
+ ff = block.ff.net
+ proj = ff[0].proj(normed)
+ out = ff[2](_swiglu(proj))
+ if i + 1 < len(blocks):
+ normed = _add_rmsnorm(h, out, block.scale2, blocks[i + 1].norm1, gemm_dtype)
+ else:
+ h.addcmul_(out, block.scale2)
+ return h
+
+
+def _fast_decoder_forward(self: Any, hidden_states: Any) -> Any:
+ """``MiniMaxH3VideoViTDecoder3d.forward`` with the block stack over the fused kernels."""
+ import torch
+ with torch.cuda.device(hidden_states.device) if hidden_states.is_cuda else nullcontext():
+ return _fast_decoder_body(self, hidden_states)
+
+
+def _fast_decoder_body(self: Any, hidden_states: Any) -> Any:
+ import torch
+
+ batch_size, num_channels, num_frames, height, width = hidden_states.shape
+ hidden_states = hidden_states.permute(0, 2, 3, 4, 1).reshape(
+ batch_size, num_frames * height * width, num_channels
+ )
+ hidden_states = self.proj_in(hidden_states)
+ num_patches = hidden_states.shape[1]
+ register_tokens = self.register_tokens.expand(batch_size, -1, -1)
+ cls_token = torch.zeros_like(hidden_states[:, :1, :])
+ hidden_states = torch.cat([hidden_states, register_tokens, cls_token], dim = 1)
+ grids = [
+ 2.0 * (torch.arange(0.5, size, dtype = torch.float32, device = hidden_states.device) / size)
+ - 1.0
+ for size in (num_frames, height, width)
+ ]
+ position_ids = torch.stack(torch.meshgrid(*grids, indexing = "ij"), dim = -1).flatten(0, 2)
+ position_ids = position_ids.unsqueeze(0).expand(batch_size, -1, -1)
+ suffix_ids = position_ids.new_zeros((batch_size, self.num_register_tokens + 1, 3))
+ position_ids = torch.cat([position_ids, suffix_ids], dim = 1)
+ cos, sin = self.rope(position_ids)
+ # the reference casts the tables to the query dtype, which is the GEMM (autocast) dtype
+ q_dtype = (
+ torch.get_autocast_dtype("cuda")
+ if torch.is_autocast_enabled("cuda")
+ else hidden_states.dtype
+ )
+ cos = cos.to(q_dtype).contiguous()
+ sin = sin.to(q_dtype).contiguous()
+ hidden_states = _fast_block_stack(self, hidden_states, cos, sin, q_dtype)
+ hidden_states = self.norm_out(hidden_states)
+ hidden_states = self.proj_out(hidden_states)
+ hidden_states = hidden_states[:, :num_patches, :]
+ p, pt = self.patch_size, self.patch_size_t
+ hidden_states = hidden_states.view(
+ batch_size, num_frames, height, width, self.out_channels, pt, p, p
+ )
+ hidden_states = hidden_states.permute(0, 4, 1, 5, 2, 6, 3, 7).contiguous()
+ return hidden_states.reshape(
+ batch_size, self.out_channels, num_frames * pt, height * p, width * p
+ )
+
+
+def _decoder_fusable(decoder: Any) -> bool:
+ try:
+ blocks = decoder.transformer_blocks
+ attn = blocks[0].attn
+ return (
+ hasattr(attn, "to_q")
+ and hasattr(attn, "to_k")
+ and hasattr(attn, "to_v")
+ and hasattr(blocks[0].ff, "net")
+ and hasattr(blocks[0].ff.net[0], "proj")
+ and hasattr(decoder, "rope")
+ and hasattr(decoder, "register_tokens")
+ and attn.dim_head & (attn.dim_head - 1) == 0
+ )
+ except Exception: # noqa: BLE001
+ return False
+
+
+def _install_decoder(vae: Any) -> bool:
+ decoder = getattr(vae, "decoder", None)
+ if decoder is None or not _decoder_fusable(decoder):
+ return False
+ decoder._unsloth_stock_forward = decoder.forward
+ decoder.forward = types.MethodType(
+ _guarded(_fast_decoder_forward, lambda self, x: self._unsloth_stock_forward(x), "decoder"),
+ decoder,
+ )
+ return True
+
+
+def _tile_batch_size(z: Any) -> int:
+ import torch
+
+ env = _env(H3_VAE_TILE_BATCH_ENV)
+ if env.isdigit() and int(env) > 0:
+ return int(env)
+ try:
+ free, _ = torch.cuda.mem_get_info(z.device)
+ except Exception: # noqa: BLE001
+ return 1
+ return int(
+ max(1, min(_TILE_BATCH_MAX, free // (_TILE_BATCH_BYTES_PER_TILE * max(1, z.shape[0]))))
+ )
+
+
+def _batched_decode_clip(self: Any, z: Any) -> Any:
+ """``_decode_clip`` with several tiles per decoder call (``_split_tiles`` makes them equal-sized)."""
+ import torch
+
+ if not self.use_tiling:
+ return self.decoder(self.post_quant_conv(z))
+ height = z.shape[-2] * self.spatial_compression_ratio
+ width = z.shape[-1] * self.spatial_compression_ratio
+ y_indices, y_lengths, y_overlaps = self._split_tiles(
+ height, self.tile_sample_min_height, self.tile_sample_min_overlap_height
+ )
+ x_indices, x_lengths, x_overlaps = self._split_tiles(
+ width, self.tile_sample_min_width, self.tile_sample_min_overlap_width
+ )
+ ratio = self.spatial_compression_ratio
+ tiles = [
+ z[
+ ...,
+ i_pos // ratio : i_pos // ratio + i_len // ratio,
+ j_pos // ratio : j_pos // ratio + j_len // ratio,
+ ]
+ for i_pos, i_len in zip(y_indices, y_lengths)
+ for j_pos, j_len in zip(x_indices, x_lengths)
+ ]
+ per = z.shape[0]
+ batch = _tile_batch_size(z)
+ decoded: list = []
+ start = 0
+ while start < len(tiles):
+ group = tiles[start : start + batch]
+ try:
+ out = self.decoder(
+ self.post_quant_conv(torch.cat(group) if len(group) > 1 else group[0])
+ )
+ except Exception as exc: # noqa: BLE001
+ # batching is what raised the peak, so an OOM with several tiles retries them one at a time
+ if len(group) == 1 or not _is_oom(exc):
+ raise
+ torch.cuda.empty_cache()
+ batch = 1
+ continue
+ decoded.extend(out.split(per) if len(group) > 1 else [out])
+ start += len(group)
+ cols = len(x_indices)
+ rows = [decoded[r * cols : (r + 1) * cols] for r in range(len(y_indices))]
+ return self._stitch_tiles(rows, y_overlaps, x_overlaps)
+
+
+def _install_tile_batch(vae: Any) -> bool:
+ if not all(
+ hasattr(vae, n)
+ for n in ("_decode_clip", "_split_tiles", "_stitch_tiles", "post_quant_conv")
+ ):
+ return False
+ vae._decode_clip = types.MethodType(_batched_decode_clip, vae)
+ return True
+
+
+def _int8_linear(linear: Any, x: Any) -> Any:
+ """W8A8: per-token int8 activations, per-output-channel int8 weights, int32 accumulate via ``torch._int_mm``."""
+ import torch
+
+ if x.is_cuda and x.device.index is not None and x.device.index != torch.cuda.current_device():
+ with torch.cuda.device(x.device):
+ return _int8_linear(linear, x)
+ k = _kernels()
+ wq, ws = linear._unsloth_int8
+ out_dtype = torch.get_autocast_dtype("cuda") if torch.is_autocast_enabled("cuda") else x.dtype
+ rot = getattr(linear, "_unsloth_rot", None)
+ if rot is not None:
+ # ConvRot: weight stored as W @ H.T, so x @ H keeps the product while spreading outlier channels
+ g = rot.shape[0]
+ x = (x.reshape(-1, x.shape[-1] // g, g).to(out_dtype) @ rot.to(out_dtype)).reshape(x.shape)
+ x2 = x.reshape(-1, x.shape[-1])
+ if not x2.is_contiguous():
+ x2 = x2.contiguous()
+ rows, n_in = x2.shape
+ if rows <= 16:
+ w = (wq.float() * ws[:, None]).to(out_dtype)
+ return torch.nn.functional.linear(x2.to(out_dtype), w, linear.bias).view(*x.shape[:-1], -1)
+ if k is None or not x.is_cuda:
+ xf = x2.float()
+ xs = xf.abs().amax(dim = 1).clamp(min = 1e-12) / 127.0
+ xq = (xf / xs[:, None]).round().clamp(-127, 127).to(torch.int8)
+ acc = torch._int_mm(xq, wq.t()).float() * xs[:, None] * ws[None, :]
+ if linear.bias is not None:
+ acc = acc + linear.bias.float()
+ return acc.to(out_dtype).view(*x.shape[:-1], -1)
+ xq = torch.empty((rows, n_in), dtype = torch.int8, device = x.device)
+ xs = torch.empty(rows, dtype = torch.float32, device = x.device)
+ k.quant_rows[(rows,)](x2, xq, xs, n_in, BLOCK_N = _next_pow2(n_in), num_warps = 8)
+ acc = torch._int_mm(xq, wq.t())
+ n_out = acc.shape[1]
+ out = torch.empty((rows, n_out), dtype = out_dtype, device = x.device)
+ block_n = 1024
+ bias = linear.bias
+ k.dequant_epilogue[(rows, (n_out + block_n - 1) // block_n)](
+ acc,
+ xs,
+ ws,
+ bias if bias is not None else ws,
+ out,
+ n_out,
+ HAS_BIAS = bias is not None,
+ BLOCK_N = block_n,
+ num_warps = 4,
+ )
+ return out.view(*x.shape[:-1], n_out)
+
+
+def _quantize_int8_weight(w: Any, hadamard: Any) -> tuple:
+ """Per-output-channel int8 of the float32 ``w``, ConvRot-rotated first when ``hadamard`` is given."""
+ import torch
+
+ if hadamard is not None:
+ group = hadamard.shape[0]
+ w = (w.reshape(w.shape[0], -1, group) @ hadamard.T).reshape(w.shape)
+ scale = w.abs().amax(dim = 1).clamp(min = 1e-12) / 127.0
+ wq = (w / scale[:, None]).round_().clamp_(-127, 127).to(torch.int8)
+ return wq, scale.to(torch.float32).view(torch.int32)
+
+
+def _install_int8_decoder(vae: Any, *, keep_blocks: Optional[tuple] = None) -> int:
+ """Quantize decoder block Linears to ConvRot W8A8, leaving ``keep_blocks`` float. Every weight is quantized
+ before any module is touched, so a failure leaves the decoder fully float. Returns the bytes freed."""
+ import torch
+
+ decoder = getattr(vae, "decoder", None)
+ if decoder is None:
+ return 0
+ from .diffusion_convrot import build_convrot_hadamard
+
+ group = H3_VAE_INT8_ROT_GROUP
+ hadamard = rot16 = None
+ staged = []
+ with torch.no_grad():
+ if keep_blocks is None:
+ keep_blocks = tuple(range(H3_VAE_INT8_FLOAT_BLOCKS))
+ for index, block in enumerate(decoder.transformer_blocks):
+ if index in keep_blocks:
+ continue
+ for linear in block.modules():
+ if (
+ not isinstance(linear, torch.nn.Linear)
+ or getattr(linear, "_unsloth_int8", None) is not None
+ ):
+ continue
+ if linear.in_features % 8 or linear.out_features % 8:
+ continue
+ w = linear.weight.data.float()
+ rotate = linear.in_features % group == 0
+ if rotate and hadamard is None:
+ hadamard = build_convrot_hadamard(group, device = w.device, dtype = torch.float32)
+ # exact in float16: the entries are +-2^-k
+ rot16 = hadamard.to(torch.float16)
+ wq, ws = _quantize_int8_weight(w, hadamard if rotate else None)
+ staged.append((linear, wq, ws, rotate))
+ cls = _int8_linear_class()
+ freed = 0
+ for linear, wq, ws, rotate in staged:
+ if rotate:
+ linear.register_buffer("_unsloth_rot", rot16, persistent = False)
+ freed += linear.weight.numel() * linear.weight.element_size() - wq.numel()
+ linear.register_buffer("_unsloth_wq", wq)
+ linear.register_buffer("_unsloth_ws", ws)
+ del linear.weight
+ linear.weight = None
+ linear.__class__ = cls
+ return freed
+
+
+@lru_cache(maxsize = 1)
+def _int8_linear_class():
+ import torch
+ class H3Int8Linear(torch.nn.Linear):
+ """int8 Linear; fp32 scales stored as int32 so an offload hook's ``.to(dtype)`` cannot round them."""
+
+ @property
+ def _unsloth_int8(self):
+ return self._unsloth_wq, self._unsloth_ws.view(torch.float32)
+
+ def forward(self, x):
+ return _int8_linear(self, x)
+
+ return H3Int8Linear
+
+
+def _install_decode_scope(vae: Any, *, fp16_accum: bool) -> bool:
+ """Hold ``allow_fp16_accumulation`` ON only when planned, else OFF, for the decode, via the shared owner."""
+ import torch
+
+ from .diffusion_speed import fp16_accumulation_scope
+
+ if not hasattr(torch.backends.cuda.matmul, "allow_fp16_accumulation"):
+ return False
+ stock = vae.decode
+
+ def decode(self, *args, **kwargs):
+ with fp16_accumulation_scope(bool(fp16_accum)):
+ return stock(*args, **kwargs)
+
+ vae.decode = types.MethodType(decode, vae)
+ return True
+
+
+@lru_cache(maxsize = 4)
+def _triton_version_ok(version: Optional[str] = None) -> bool:
+ if version is None:
+ try:
+ import triton
+ version = str(triton.__version__)
+ except Exception: # noqa: BLE001
+ return False
+ import re
+
+ match = re.match(r"(\d+)\.(\d+)", version)
+ return bool(match) and (int(match.group(1)), int(match.group(2))) >= H3_VAE_MIN_TRITON
+
+
+@lru_cache(maxsize = 1)
+def _triton_jit_toolchain_ok() -> bool:
+ """Windows: Triton needs the MSVC CRT headers. Elsewhere, or if the probe fails, True."""
+ if sys.platform != "win32":
+ return True
+ try:
+ from .._msvc_env import crt_headers_reachable
+ return bool(crt_headers_reachable())
+ except Exception: # noqa: BLE001
+ return True
+
+
+def cuda_fast_path_available(vae: Any = None) -> bool:
+ try:
+ import torch
+ except Exception: # noqa: BLE001
+ return False
+ if not torch.cuda.is_available() or getattr(torch.version, "hip", None):
+ return False
+ if vae is not None:
+ try:
+ if type(vae).__name__ != "AutoencoderKLMiniMaxH3":
+ return False
+ except Exception: # noqa: BLE001
+ return False
+ return _triton_version_ok() and _kernels() is not None and _triton_jit_toolchain_ok()
+
+
+def apply_h3_vae_speedups(
+ vae: Any,
+ *,
+ speed_mode: Optional[str],
+ workflow: Optional[str] = None,
+ consumer_gpu: Optional[bool] = None,
+ logger: Any = None,
+) -> tuple[str, ...]:
+ """Engage the planned levers; never raises (a failing lever keeps stock). Returns engaged lever names."""
+ if vae is None:
+ return ()
+ done = getattr(vae, "_unsloth_h3_vae_levers", None)
+ if done is not None:
+ return tuple(done)
+ if consumer_gpu is None:
+ try:
+ from .diffusion_transformer_quant import _is_consumer_gpu
+ consumer_gpu = bool(_is_consumer_gpu())
+ except Exception: # noqa: BLE001
+ consumer_gpu = False
+ planned = plan_h3_vae_levers(speed_mode, workflow = workflow, consumer_gpu = consumer_gpu)
+ engaged: list[str] = []
+ if not planned or not cuda_fast_path_available(vae):
+ try:
+ vae._unsloth_h3_vae_levers = ()
+ except Exception: # noqa: BLE001
+ pass
+ return ()
+
+ def _try(name: str, fn) -> None:
+ try:
+ if fn():
+ engaged.append(name)
+ except Exception as exc: # noqa: BLE001 - optimisation only
+ if logger is not None:
+ logger.warning(
+ "video.h3_vae_fast: %s failed, keeping the stock path: %s", name, exc
+ )
+
+ for holder in (vae, getattr(vae, "encoder", None), getattr(vae, "decoder", None)):
+ if holder is not None:
+ try:
+ holder._unsloth_logger = logger
+ except Exception: # noqa: BLE001
+ pass
+ if LEVER_FUSED_ENCODER in planned:
+ _try(LEVER_FUSED_ENCODER, lambda: _install_encoder(vae, fp16 = LEVER_FP16_ENCODER in planned))
+ if LEVER_FUSED_ENCODER in engaged and LEVER_FP16_ENCODER in planned:
+ engaged.append(LEVER_FP16_ENCODER)
+ if LEVER_INT8_DECODER in planned:
+ _try(LEVER_INT8_DECODER, lambda: _install_int8_decoder(vae) > 0)
+ if LEVER_FUSED_DECODER in planned:
+ _try(LEVER_FUSED_DECODER, lambda: _install_decoder(vae))
+ if LEVER_TILE_BATCH in planned:
+ _try(LEVER_TILE_BATCH, lambda: _install_tile_batch(vae))
+ fp16_accum = LEVER_FP16_ACCUM in planned
+ # always installed: also holds the flag OFF when float16 accumulation is not planned
+ try:
+ if _install_decode_scope(vae, fp16_accum = fp16_accum) and fp16_accum:
+ engaged.append(LEVER_FP16_ACCUM)
+ except Exception as exc: # noqa: BLE001 - optimisation only
+ if logger is not None:
+ logger.warning(
+ "video.h3_vae_fast: decode_scope failed, keeping the stock decode: %s", exc
+ )
+ vae._unsloth_h3_vae_levers = tuple(engaged)
+ if logger is not None:
+ logger.info("video.h3_vae_fast: engaged %s", ", ".join(engaged) or "nothing")
+ return tuple(engaged)
+
+
+def settle_h3_vae_fallback(state: Any, pipe: Any) -> None:
+ """Drop fused levers the runtime fell back from out of ``state.speed_optims``. Never raises."""
+ try:
+ vae = getattr(pipe, "vae", None)
+ failed = [
+ f"h3_vae_{lever}"
+ for lever, part in ((LEVER_FUSED_ENCODER, "encoder"), (LEVER_FUSED_DECODER, "decoder"))
+ if getattr(getattr(vae, part, None), "_unsloth_fast_failed", False)
+ ]
+ optims = tuple(getattr(state, "speed_optims", None) or ())
+ updated = tuple(o for o in optims if o not in failed)
+ if updated != optims:
+ # the load states are frozen dataclasses
+ object.__setattr__(state, "speed_optims", updated)
+ except Exception: # noqa: BLE001 - a status fix-up must never fail a render
+ pass
diff --git a/studio/backend/tests/test_video_minimax_h3_vae.py b/studio/backend/tests/test_video_minimax_h3_vae.py
new file mode 100644
index 00000000000..d3648f1b9c2
--- /dev/null
+++ b/studio/backend/tests/test_video_minimax_h3_vae.py
@@ -0,0 +1,970 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Tests for the MiniMax-H3 video VAE speed layer; CPU tests hermetic, CUDA tests skip without a GPU."""
+
+from __future__ import annotations
+
+import copy
+import types
+
+import pytest
+import torch
+
+from core.inference import video_minimax_h3_vae as H
+
+CUDA = torch.cuda.is_available() and not getattr(torch.version, "hip", None)
+needs_cuda = pytest.mark.skipif(
+ not CUDA or H._kernels() is None or not H._triton_version_ok(),
+ reason = "needs an NVIDIA GPU with a Triton the kernels are verified on",
+)
+
+
[email protected](autouse = True)
+def _clean_env(monkeypatch):
+ for name in (
+ H.H3_VAE_FAST_ENV,
+ H.H3_VAE_INT8_ENV,
+ H.H3_VAE_TILE_BATCH_ENV,
+ H.FP16_ACCUM_DISABLE_ENV,
+ ):
+ monkeypatch.delenv(name, raising = False)
+
+
+def test_off_tier_engages_nothing():
+ assert H.plan_h3_vae_levers("off", workflow = "fl2va") == ()
+ assert H.plan_h3_vae_levers("OFF", workflow = "fl2va", consumer_gpu = True) == ()
+
+
[email protected]("tier", ["default", "max", None, ""])
+def test_default_and_max_get_the_near_lossless_set(tier):
+ plan = H.plan_h3_vae_levers(tier, workflow = "fl2va", consumer_gpu = False)
+ assert plan == (
+ H.LEVER_FUSED_ENCODER,
+ H.LEVER_FP16_ENCODER,
+ H.LEVER_FUSED_DECODER,
+ H.LEVER_TILE_BATCH,
+ )
+
+
+def test_eager_stays_rounding_level():
+ plan = H.plan_h3_vae_levers("eager", workflow = "fl2va", consumer_gpu = True)
+ assert plan == (H.LEVER_FUSED_ENCODER, H.LEVER_FUSED_DECODER, H.LEVER_TILE_BATCH)
+
+
+def test_fp16_accumulation_only_under_max_on_consumer_gpus(monkeypatch):
+ assert H.LEVER_FP16_ACCUM in H.plan_h3_vae_levers("max", workflow = "fl2va", consumer_gpu = True)
+ assert H.LEVER_FP16_ACCUM not in H.plan_h3_vae_levers(
+ "max", workflow = "fl2va", consumer_gpu = False
+ )
+ for tier in ("eager", "default"):
+ assert H.LEVER_FP16_ACCUM not in H.plan_h3_vae_levers(
+ tier, workflow = "fl2va", consumer_gpu = True
+ )
+ monkeypatch.setenv(H.FP16_ACCUM_DISABLE_ENV, "1")
+ assert H.LEVER_FP16_ACCUM not in H.plan_h3_vae_levers(
+ "max", workflow = "fl2va", consumer_gpu = True
+ )
+
+
+def test_int8_decoder_is_opt_in_only(monkeypatch):
+ for tier in ("eager", "default", "max"):
+ assert H.LEVER_INT8_DECODER not in H.plan_h3_vae_levers(
+ tier, workflow = "fl2va", consumer_gpu = True
+ )
+ monkeypatch.setenv(H.H3_VAE_INT8_ENV, "1")
+ assert H.LEVER_INT8_DECODER in H.plan_h3_vae_levers("default", workflow = "fl2va")
+
+
+def test_t2va_skips_the_encoder_levers():
+ plan = H.plan_h3_vae_levers("default", workflow = "t2va")
+ assert H.LEVER_FUSED_ENCODER not in plan and H.LEVER_FP16_ENCODER not in plan
+ assert H.LEVER_FUSED_DECODER in plan
+
+
+def test_env_switches(monkeypatch):
+ monkeypatch.setenv(H.H3_VAE_FAST_ENV, "0")
+ assert H.plan_h3_vae_levers("max", workflow = "fl2va", consumer_gpu = True) == ()
+ monkeypatch.delenv(H.H3_VAE_FAST_ENV)
+ monkeypatch.setenv(H.H3_VAE_INT8_ENV, "1")
+ assert H.plan_h3_vae_levers("off", workflow = "fl2va") == ()
+
+
+def test_apply_is_a_no_op_off_cuda(monkeypatch):
+ monkeypatch.setattr(H, "cuda_fast_path_available", lambda vae = None: False)
+ vae = types.SimpleNamespace(decode = object(), encoder = object())
+ stock_decode = vae.decode
+ assert H.apply_h3_vae_speedups(vae, speed_mode = "max", workflow = "fl2va", consumer_gpu = True) == ()
+ assert vae.decode is stock_decode
+ assert H.apply_h3_vae_speedups(None, speed_mode = "max") == ()
+
+
[email protected](CUDA, reason = "checks the real gate on a host without an NVIDIA GPU")
[email protected]("tier", ["eager", "default", "max"])
+def test_without_cuda_the_real_gate_leaves_the_stock_vae_untouched(tier):
+ vae = _tiny_vae()
+ encoder_forward, decoder_forward, decode = (
+ vae.encoder.forward,
+ vae.decoder.forward,
+ vae.decode,
+ )
+ weights = {k: v.clone() for k, v in vae.state_dict().items()}
+ assert H.plan_h3_vae_levers(tier, workflow = "fl2va")
+ assert H.apply_h3_vae_speedups(vae, speed_mode = tier, workflow = "fl2va") == ()
+ assert vae.encoder.forward == encoder_forward and vae.decoder.forward == decoder_forward
+ assert vae.decode == decode
+ for k, v in vae.state_dict().items():
+ assert v.dtype == weights[k].dtype and torch.equal(v, weights[k]), k
+
+
+def test_decode_scope_pins_and_restores_fp16_accumulation():
+ matmul = torch.backends.cuda.matmul
+ if not hasattr(matmul, "allow_fp16_accumulation"):
+ pytest.skip("torch without allow_fp16_accumulation")
+ seen = []
+
+ class _VAE:
+ def decode(self, z):
+ seen.append(matmul.allow_fp16_accumulation)
+ return z
+
+ prev = matmul.allow_fp16_accumulation
+ try:
+ for flag_before, planned in ((True, False), (False, True), (False, False)):
+ vae = _VAE()
+ matmul.allow_fp16_accumulation = flag_before
+ assert H._install_decode_scope(vae, fp16_accum = planned)
+ assert vae.decode(3) == 3
+ assert (
+ seen[-1] is planned
+ ), "the decode must run under the planned flag, not the process-wide one"
+ assert (
+ matmul.allow_fp16_accumulation is flag_before
+ ), "the process-wide flag must be restored"
+ finally:
+ matmul.allow_fp16_accumulation = prev
+
+
+def _tiny_vae():
+ diffusers = pytest.importorskip("diffusers")
+ cls = getattr(diffusers, "AutoencoderKLMiniMaxH3", None)
+ if cls is None:
+ pytest.skip("diffusers without AutoencoderKLMiniMaxH3")
+ torch.manual_seed(0)
+ vae = cls(
+ block_out_channels = (16, 32),
+ layers_per_block = 2,
+ spatial_downsample_factors = (2, 2),
+ temporal_downsample_factors = (2, 1),
+ norm_num_groups = 8,
+ latent_channels = 8,
+ decoder_num_layers = 2,
+ decoder_num_attention_heads = 2,
+ decoder_attention_head_dim = 16,
+ decoder_num_register_tokens = 2,
+ decoder_ffn_mult = 2,
+ decoder_rope_dim_ratio = 0.75,
+ clip_length = 9,
+ token_drop = 1,
+ ).eval()
+ with torch.no_grad():
+ for name, p in vae.named_parameters():
+ if name.endswith("scale1") or name.endswith("scale2"):
+ p.normal_(0, 0.5)
+ elif "norm" in name and name.endswith("weight"):
+ p.normal_(1.0, 0.2)
+ elif name.endswith("bias"):
+ p.normal_(0, 0.1)
+ vae.tile_sample_min_height = vae.tile_sample_min_width = 16
+ vae.tile_sample_min_overlap_height = vae.tile_sample_min_overlap_width = 4
+ return vae
+
+
+DEVICES = ["cpu", pytest.param("cuda", marks = needs_cuda)]
+
+
[email protected]("device", DEVICES)
[email protected]("frames", [1, 9, 18])
+def test_fused_encoder_matches_the_stock_encoder(frames, device, monkeypatch):
+ monkeypatch.setattr(torch.backends.cudnn, "allow_tf32", False)
+ vae = _tiny_vae().to(device)
+ fast = copy.deepcopy(vae)
+ assert H._install_encoder(fast, fp16 = False)
+ x = torch.randn(1, 3, frames, 24, 40, device = device)
+ with torch.no_grad():
+ ref = vae.encoder(x)
+ got = fast.encoder(x)
+ assert not getattr(
+ fast.encoder, "_unsloth_fast_failed", False
+ ), "the fused path fell back to stock"
+ assert got.shape == ref.shape and got.dtype == ref.dtype
+ torch.testing.assert_close(got, ref, rtol = 1e-4, atol = 1e-4)
+
+
[email protected]("device", DEVICES)
+def test_fused_fp16_encoder_returns_the_input_dtype_and_stays_close(device):
+ vae = _tiny_vae().to(device)
+ ref = copy.deepcopy(vae)
+ assert H._install_encoder(vae, fp16 = True)
+ x = torch.randn(1, 3, 9, 24, 40, device = device)
+ with torch.no_grad():
+ got, want = vae.encoder(x), ref.encoder(x)
+ assert not getattr(vae.encoder, "_unsloth_fast_failed", False)
+ assert got.dtype is torch.float32
+ assert ((got - want).norm() / want.norm()) < 1e-2
+
+
+def test_fused_encoder_fp16_casts_weights_and_returns_the_input_dtype():
+ vae = _tiny_vae()
+ assert H._install_encoder(vae, fp16 = True)
+ convs = [m for m in vae.encoder.modules() if isinstance(m, torch.nn.Conv3d)]
+ assert convs and all(m.weight.dtype is torch.float16 for m in convs)
+ assert all(
+ m.weight.is_contiguous(memory_format = torch.channels_last_3d)
+ for m in convs
+ if m is not vae.encoder.conv_in
+ )
+ assert vae.quant_conv.weight.dtype is torch.float32
+
+
[email protected]("device", DEVICES)
[email protected]("frames", [1, 9])
+def test_the_condition_encode_recipe_runs_through_the_fp16_encoder(frames, device):
+ # keyframe path: vae.encode feeds the encoder output to a float32 quant_conv, so it must be float32
+ from core.inference.video_minimax_h3 import trim_h3_video_vae
+
+ vae = _tiny_vae().to(device)
+ vae.register_to_config(latents_mean = [0.0] * 8, latents_std = [1.0] * 8)
+ trim_h3_video_vae(vae, workflow = "fl2va")
+ ref = copy.deepcopy(vae)
+ assert H._install_encoder(vae, fp16 = True)
+ pixels = torch.randint(0, 256, (1, 3, frames, 32, 48), dtype = torch.uint8, device = device)
+ with torch.no_grad():
+ try:
+ from diffusers.modular_pipelines.minimax_h3.encoders import encode_vae_condition
+ except ImportError:
+ x = pixels.float() / 127.5 - 1.0
+ got, want = (m.encode(x).latent_dist.mean for m in (vae, ref))
+ else:
+ args = (pixels, (0.485, 0.456, 0.406), (0.229, 0.224, 0.225))
+ got, want = encode_vae_condition(vae, *args), encode_vae_condition(ref, *args)
+ assert not getattr(vae.encoder, "_unsloth_fast_failed", False)
+ assert vae.quant_conv.weight.dtype is torch.float32
+ assert got.dtype is torch.float32 and got.shape == want.shape
+ assert ((got - want).norm() / want.norm()) < 1e-2
+
+
+@needs_cuda
+def test_the_decoder_residual_stream_stays_float32_under_the_decode_autocast(monkeypatch):
+ # the stock blocks keep the residual in float32 under float16 autocast, so must we
+ from core.inference.video_minimax_h3 import trim_h3_video_vae
+
+ vae = _tiny_vae().cuda()
+ trim_h3_video_vae(vae, workflow = "fl2va")
+ ref = copy.deepcopy(vae)
+ assert H._install_decoder(vae)
+ seen = []
+ stack = H._fast_block_stack
+
+ def spy(decoder, hidden_states, *args):
+ seen.append(hidden_states.dtype)
+ out = stack(decoder, hidden_states, *args)
+ seen.append(out.dtype)
+ return out
+
+ monkeypatch.setattr(H, "_fast_block_stack", spy)
+ z = torch.randn(1, 8, 3, 4, 5, device = "cuda")
+ with torch.no_grad(), torch.autocast("cuda", dtype = torch.float16):
+ got, want = vae.decoder(z), ref.decoder(z)
+ assert not getattr(vae.decoder, "_unsloth_fast_failed", False)
+ assert seen == [torch.float32, torch.float32]
+ assert got.dtype == want.dtype
+ torch.testing.assert_close(got.float(), want.float(), rtol = 2e-3, atol = 2e-3)
+
+
+def test_norm_silu_pad_reference_matches_diffusers_ops():
+ from torch.nn import functional as F
+
+ vae = _tiny_vae()
+ resnet = vae.encoder.down_blocks[0].resnets[0]
+ x = torch.randn(1, 16, 5, 12, 10)
+ ref = F.pad(F.silu(resnet.norm1(x)), (1, 1, 1, 1, 0, 0), mode = "reflect")
+ ref = F.pad(ref, (0, 0, 0, 0, 2, 0))
+ got = H.norm_silu_pad_reference(x, resnet.norm1, (1, 1, 1, 1), 2)
+ assert got.is_contiguous(memory_format = torch.channels_last_3d)
+ torch.testing.assert_close(got, ref, rtol = 1e-5, atol = 1e-5)
+
+
+def test_single_frame_last_tap_equals_convolving_the_zero_frames():
+ from torch.nn import functional as F
+
+ vae = _tiny_vae()
+ conv = vae.encoder.down_blocks[0].resnets[0].conv1
+ x = torch.randn(1, 16, 1, 8, 8)
+ padded = F.pad(F.pad(x, (1, 1, 1, 1, 0, 0), mode = "reflect"), (0, 0, 0, 0, 2, 0))
+ ref = F.conv3d(padded, conv.weight, conv.bias)
+ got, bias = H._causal_conv(conv, x)
+ torch.testing.assert_close(got + bias.view(1, -1, 1, 1, 1), ref, rtol = 1e-5, atol = 1e-5)
+
+
+def test_pending_bias_folds_through_a_1x1_conv():
+ from torch.nn import functional as F
+
+ conv = torch.nn.Conv3d(16, 32, 1)
+ x = torch.randn(1, 16, 3, 5, 7)
+ pending = torch.randn(16)
+ ref = F.conv3d(x + pending.view(1, -1, 1, 1, 1), conv.weight, conv.bias)
+ got, bias = H._causal_conv(conv, x, pending)
+ torch.testing.assert_close(got + bias.view(1, -1, 1, 1, 1), ref, rtol = 1e-5, atol = 1e-5)
+
+
+def test_pending_bias_is_added_before_the_norm_and_the_pad():
+ from torch.nn import functional as F
+
+ norm = torch.nn.GroupNorm(4, 16, eps = 1e-6)
+ x = torch.randn(1, 16, 4, 6, 6)
+ pending = torch.randn(16)
+ y = x + pending.view(1, -1, 1, 1, 1)
+ frames = y.permute(0, 2, 1, 3, 4).reshape(4, 16, 6, 6)
+ ref = (
+ F.silu(F.group_norm(frames, 4, norm.weight, norm.bias, 1e-6))
+ .reshape(1, 4, 16, 6, 6)
+ .permute(0, 2, 1, 3, 4)
+ )
+ ref = F.pad(F.pad(ref, (1, 1, 1, 1, 0, 0), mode = "reflect"), (0, 0, 0, 0, 2, 0))
+ torch.testing.assert_close(
+ H.norm_silu_pad(x, norm, (1, 1, 1, 1), 2, in_bias = pending), ref, rtol = 1e-5, atol = 1e-5
+ )
+ # a zero frame stays zero: the bias belongs to real frames only
+ got = H.norm_silu_pad(x, None, (0, 1, 0, 1), 2, in_bias = pending)
+ assert got[:, :, :2].abs().max() == 0
+
+
[email protected]("device", DEVICES)
+def test_fused_decoder_matches_the_stock_decoder(device):
+ vae = _tiny_vae().to(device)
+ fast = copy.deepcopy(vae)
+ assert H._install_decoder(fast)
+ z = torch.randn(1, 8, 3, 4, 5, device = device)
+ with torch.no_grad():
+ ref = vae.decoder(z)
+ got = fast.decoder(z)
+ assert not getattr(
+ fast.decoder, "_unsloth_fast_failed", False
+ ), "the fused path fell back to stock"
+ assert got.shape == ref.shape
+ torch.testing.assert_close(got, ref, rtol = 1e-4, atol = 1e-4)
+
+
+def test_tile_batching_matches_the_stock_tile_loop(monkeypatch):
+ vae = _tiny_vae()
+ fast = copy.deepcopy(vae)
+ assert H._install_tile_batch(fast)
+ z = torch.randn(1, 8, 3, 6, 9) # 24x36 px: 2x3 tiles of 16 px
+ with torch.no_grad():
+ ref = vae._decode_clip(z)
+ for batch in ("1", "2", "4", "8"):
+ monkeypatch.setenv(H.H3_VAE_TILE_BATCH_ENV, batch)
+ torch.testing.assert_close(fast._decode_clip(z), ref, rtol = 1e-5, atol = 1e-5)
+
+
+def test_int8_decoder_replaces_block_linears_and_stays_close():
+ vae = _tiny_vae()
+ fast = copy.deepcopy(vae)
+ freed = H._install_int8_decoder(fast, keep_blocks = ())
+ assert freed > 0
+ lin = fast.decoder.transformer_blocks[0].attn.to_q
+ assert lin.weight is None and lin._unsloth_wq.dtype is torch.int8
+ assert fast.decoder.proj_out.weight is not None
+ z = torch.randn(1, 8, 3, 4, 5)
+ with torch.no_grad():
+ ref = vae.decoder(z)
+ got = fast.decoder(z)
+ rel = (got - ref).norm() / ref.norm()
+ assert rel < 0.05
+
+
+def test_apply_skips_when_already_applied(monkeypatch):
+ monkeypatch.setattr(H, "cuda_fast_path_available", lambda vae = None: True)
+ vae = _tiny_vae()
+ engaged = H.apply_h3_vae_speedups(
+ vae, speed_mode = "default", workflow = "fl2va", consumer_gpu = True
+ )
+ assert set(engaged) == {
+ H.LEVER_FUSED_ENCODER,
+ H.LEVER_FP16_ENCODER,
+ H.LEVER_FUSED_DECODER,
+ H.LEVER_TILE_BATCH,
+ }
+ encoder_forward = vae.encoder.forward
+ assert H.apply_h3_vae_speedups(vae, speed_mode = "max", workflow = "fl2va") == engaged
+ assert vae.encoder.forward is encoder_forward
+
+
+@needs_cuda
[email protected]("dtype", [torch.float32, torch.float16])
[email protected]("channels_last", [False, True])
+ "frames,pad,front,with_norm",
+ [(9, (1, 1, 1, 1), 2, True), (1, (1, 1, 1, 1), 0, True), (7, (0, 1, 0, 1), 2, False)],
+)
+def test_norm_silu_pad_kernel(dtype, channels_last, frames, pad, front, with_norm):
+ torch.manual_seed(0)
+ norm = torch.nn.GroupNorm(8, 64, eps = 1e-6).cuda()
+ with torch.no_grad():
+ norm.weight.normal_()
+ norm.bias.normal_()
+ x = (torch.randn(1, 64, frames, 20, 36, device = "cuda") * 3 + 1).to(dtype)
+ if channels_last:
+ x = x.contiguous(memory_format = torch.channels_last_3d)
+ n = norm if with_norm else None
+ ref = H.norm_silu_pad_reference(x, n, pad, front)
+ got = H.norm_silu_pad(x, n, pad, front)
+ assert got.shape == ref.shape and got.stride() == ref.stride()
+ tol = 1e-5 if dtype is torch.float32 else 1e-2
+ torch.testing.assert_close(got.float(), ref.float(), rtol = tol, atol = tol)
+
+
+@needs_cuda
+def test_add_rmsnorm_kernel():
+ torch.manual_seed(0)
+ norm = torch.nn.RMSNorm(256, eps = 1e-5).cuda()
+ with torch.no_grad():
+ norm.weight.normal_(1, 0.1)
+ h = torch.randn(300, 256, device = "cuda")
+ o = torch.randn(300, 256, device = "cuda").half()
+ s = torch.randn(256, device = "cuda")
+ ref_h = h + o * s
+ ref_n = norm(ref_h).half()
+ got_n = H._add_rmsnorm(h, o, s, norm, torch.float16)
+ assert torch.equal(h, ref_h)
+ torch.testing.assert_close(got_n, ref_n, rtol = 2e-3, atol = 2e-3)
+
+
+@needs_cuda
+def test_qk_norm_rope_kernel_matches_the_reference_processor():
+ torch.manual_seed(0)
+ heads, dim = 4, 64
+ x = torch.randn(2, 100, heads * dim, device = "cuda").half()
+ angles = torch.rand(2, 100, 1, 24, device = "cuda") * 6.28
+ angles = angles.tile(2)
+ cos, sin = angles.cos().half(), angles.sin().half()
+ ref = H.qk_norm_rope_reference(x, cos, sin, heads, 1e-5)
+ got = x.clone()
+ H._qk_norm_rope_(got, cos.reshape(-1, 48), sin.reshape(-1, 48), heads, 1e-5)
+ assert (got == ref).float().mean() > 0.99
+ torch.testing.assert_close(got.float(), ref.float(), rtol = 2e-3, atol = 2e-3)
+
+
+@needs_cuda
+def test_swiglu_kernel():
+ x = torch.randn(64, 2 * 3000, device = "cuda").half()
+ hidden, gate = x.chunk(2, dim = -1)
+ got, ref = H._swiglu(x), hidden * torch.nn.functional.silu(gate)
+ assert (got == ref).float().mean() > 0.999
+ torch.testing.assert_close(got, ref, rtol = 2e-3, atol = 2e-3)
+
+
+@needs_cuda
+def test_int8_linear_kernel_path_matches_the_torch_path():
+ torch.manual_seed(0)
+ lin = torch.nn.Linear(256, 512).cuda().half()
+ ref_lin = copy.deepcopy(lin)
+ holder = torch.nn.Module()
+ holder.transformer_blocks = torch.nn.ModuleList([torch.nn.ModuleDict({"l": lin})])
+ vae = types.SimpleNamespace(decoder = holder)
+ assert H._install_int8_decoder(vae, keep_blocks = ()) > 0
+ x = torch.randn(3, 40, 256, device = "cuda").half()
+ got = lin(x)
+ ref = ref_lin(x)
+ assert got.shape == ref.shape and got.dtype == ref.dtype
+ assert ((got.float() - ref.float()).norm() / ref.float().norm()) < 0.02
+
+
+@needs_cuda
[email protected]("dtype", [torch.float16, torch.bfloat16, torch.float32])
+def test_int8_quant_rows_kernel_matches_the_torch_codes(dtype):
+ g = torch.Generator(device = "cuda").manual_seed(0)
+ rows, n = 4 * 1797, 2048
+ x = torch.randn(rows, n, device = "cuda", generator = g)
+ x = (x * (torch.rand(n, device = "cuda", generator = g) * 4 + 0.1)).to(dtype)
+ # exact halves: absmax 127 gives s = 1, where half-to-even and half-away-from-zero disagree
+ x[0, :6] = torch.tensor([127.0, 2.5, -3.5, 0.5, -0.5, 126.5], dtype = dtype)
+ x[0, 6:] = 0
+ xf = x.float()
+ ref_s = xf.abs().amax(dim = 1).clamp(min = 1e-12) / 127.0
+ ref_q = (xf / ref_s[:, None]).round().clamp(-127, 127).to(torch.int8)
+ q = torch.empty(rows, n, dtype = torch.int8, device = "cuda")
+ s = torch.empty(rows, dtype = torch.float32, device = "cuda")
+ H._kernels().quant_rows[(rows,)](x, q, s, n, BLOCK_N = n, num_warps = 8)
+ assert torch.equal(s, ref_s)
+ assert int((q != ref_q).sum()) == 0
+ assert q[0, :6].tolist() == [127, 2, -4, 0, 0, 126]
+
+
+@needs_cuda
[email protected]("res_layout", ["channels_last", "contiguous"])
+def test_add_residual_kernel(res_layout):
+ torch.manual_seed(0)
+ out = (
+ torch.randn(1, 64, 3, 10, 12, device = "cuda")
+ .half()
+ .contiguous(memory_format = torch.channels_last_3d)
+ )
+ res = torch.randn(1, 64, 3, 10, 12, device = "cuda").half()
+ if res_layout == "channels_last":
+ res = res.contiguous(memory_format = torch.channels_last_3d)
+ ob, rb = torch.randn(64, device = "cuda").half(), torch.randn(64, device = "cuda")
+ ref = (
+ out.float() + ob.float().view(1, -1, 1, 1, 1) + res.float() + rb.view(1, -1, 1, 1, 1)
+ ).half()
+ got = H.add_residual(out.clone(memory_format = torch.channels_last_3d), ob, res, rb)
+ torch.testing.assert_close(got.float(), ref.float(), rtol = 2e-3, atol = 2e-3)
+
+
+@needs_cuda
+def test_norm_silu_pad_kernel_with_pending_bias():
+ torch.manual_seed(0)
+ norm = torch.nn.GroupNorm(32, 256, eps = 1e-6).cuda()
+ x = (
+ torch.randn(1, 256, 5, 17, 23, device = "cuda")
+ .half()
+ .contiguous(memory_format = torch.channels_last_3d)
+ )
+ pending = torch.randn(256, device = "cuda")
+ ref = H.norm_silu_pad_reference(x, norm, (1, 1, 1, 1), 2, in_bias = pending)
+ got = H.norm_silu_pad(x, norm, (1, 1, 1, 1), 2, in_bias = pending)
+ torch.testing.assert_close(got.float(), ref.float(), rtol = 1e-2, atol = 1e-2)
+
+
+def test_int8_decoder_keeps_the_early_blocks_float_by_default(monkeypatch):
+ monkeypatch.setattr(H, "H3_VAE_INT8_FLOAT_BLOCKS", 1)
+ vae = _tiny_vae()
+ blocks = vae.decoder.transformer_blocks
+ H._install_int8_decoder(vae)
+ assert blocks[0].attn.to_q.weight is not None
+ assert blocks[1].attn.to_q.weight is None
+
+
+def test_a_failing_fused_path_falls_back_to_stock_once_and_logs():
+ calls = []
+
+ def fast(self, x):
+ calls.append("fast")
+ raise RuntimeError("triton: no kernel image")
+
+ def stock(self, x):
+ calls.append("stock")
+ return x + 1
+
+ warnings = []
+ holder = types.SimpleNamespace(
+ _unsloth_logger = types.SimpleNamespace(warning = lambda *a: warnings.append(a))
+ )
+ forward = H._guarded(fast, stock, "decoder")
+ assert forward(holder, 1) == 2
+ assert forward(holder, 2) == 3
+ assert calls == [
+ "fast",
+ "stock",
+ "stock",
+ ], "after one failure the fused path must not be retried"
+ assert len(warnings) == 1
+
+
+def test_an_input_the_stock_path_rejects_too_raises_without_disabling_the_fast_path():
+ def fast(self, x):
+ raise RuntimeError("Padding size should be less than the corresponding input dimension")
+
+ def stock(self, x):
+ raise RuntimeError("Padding size should be less than the corresponding input dimension")
+
+ holder = types.SimpleNamespace()
+ with pytest.raises(RuntimeError, match = "Padding size"):
+ H._guarded(fast, stock, "encoder")(holder, 1)
+ assert not getattr(holder, "_unsloth_fast_failed", False)
+
+
+def test_a_non_reflect_padding_mode_keeps_the_stock_encoder():
+ vae = _tiny_vae()
+ for m in vae.encoder.modules():
+ if hasattr(m, "spatial_padding_mode"):
+ m.spatial_padding_mode = "replicate"
+ forward = vae.encoder.forward
+ assert not H._install_encoder(vae, fp16 = True)
+ assert vae.encoder.forward == forward
+ assert all(p.dtype is torch.float32 for p in vae.encoder.parameters())
+
+
+def test_a_non_power_of_two_head_keeps_the_stock_decoder():
+ vae = _tiny_vae()
+ for block in vae.decoder.transformer_blocks:
+ block.attn.dim_head = 12
+ assert not H._install_decoder(vae)
+
+
+def test_an_oom_in_the_fused_path_is_not_masked():
+ def fast(self, x):
+ raise RuntimeError("CUDA out of memory. Tried to allocate 2.00 GiB")
+
+ forward = H._guarded(fast, lambda self, x: x, "decoder")
+ holder = types.SimpleNamespace()
+ with pytest.raises(RuntimeError, match = "out of memory"):
+ forward(holder, 1)
+ assert not getattr(holder, "_unsloth_fast_failed", False)
+
+
+def test_the_fp16_encoder_falls_back_to_the_stock_encoder(monkeypatch):
+ vae = _tiny_vae()
+ ref_vae = copy.deepcopy(vae)
+ assert H._install_encoder(vae, fp16 = True)
+ x = torch.randn(1, 3, 9, 24, 40)
+ with torch.no_grad():
+ fused = vae.encoder(x)
+
+ def boom(*a, **k):
+ raise RuntimeError("no triton")
+
+ # the installed guard resolves the fused body at call time, so breaking it exercises the real fallback
+ monkeypatch.setattr(H, "_fast_encoder_body", boom)
+ with torch.no_grad():
+ stock = vae.encoder(x)
+ want = ref_vae.encoder(x)
+ assert vae.encoder._unsloth_fast_failed
+ assert stock.dtype is torch.float32
+ # the stock forward over the float16 weights: float16 arithmetic, so close to both, equal to neither
+ assert ((stock - want).norm() / want.norm()) < 1e-2
+ assert ((stock - fused).norm() / fused.norm()) < 1e-2
+
+
+def test_tile_batching_retries_one_tile_at_a_time_after_an_oom(monkeypatch):
+ vae = _tiny_vae()
+ ref = copy.deepcopy(vae)
+ assert H._install_tile_batch(vae)
+ monkeypatch.setenv(H.H3_VAE_TILE_BATCH_ENV, "4")
+ stock_decoder = vae.decoder.forward
+ seen = []
+
+ def decoder(z):
+ seen.append(z.shape[0])
+ if z.shape[0] > 1:
+ raise RuntimeError("CUDA out of memory")
+ return stock_decoder(z)
+
+ monkeypatch.setattr(vae.decoder, "forward", decoder)
+ monkeypatch.setattr(torch.cuda, "empty_cache", lambda: None)
+ z = torch.randn(1, 8, 3, 6, 9)
+ with torch.no_grad():
+ torch.testing.assert_close(vae._decode_clip(z), ref._decode_clip(z), rtol = 1e-5, atol = 1e-5)
+ assert seen[0] == 4 and set(seen[1:]) == {1}
+
+
+def test_settle_drops_a_lever_that_fell_back_from_the_status():
+ import dataclasses
+
+ @dataclasses.dataclass(frozen = True)
+ class _State:
+ speed_optims: tuple
+
+ state = _State(
+ ("cudnn_benchmark", "h3_vae_fused_encoder", "h3_vae_fp16_encoder", "h3_vae_fused_decoder")
+ )
+ pipe = types.SimpleNamespace(
+ vae = types.SimpleNamespace(
+ encoder = types.SimpleNamespace(_unsloth_fast_failed = False),
+ decoder = types.SimpleNamespace(_unsloth_fast_failed = True),
+ )
+ )
+ H.settle_h3_vae_fallback(state, pipe)
+ assert state.speed_optims == ("cudnn_benchmark", "h3_vae_fused_encoder", "h3_vae_fp16_encoder")
+ H.settle_h3_vae_fallback(state, types.SimpleNamespace()) # no vae: a no-op, never raises
+
+
+def test_int8_small_row_path_matches_the_int_mm_path():
+ holder = torch.nn.Module()
+ lin = torch.nn.Linear(64, 32)
+ holder.transformer_blocks = torch.nn.ModuleList([torch.nn.ModuleDict({"l": lin})])
+ assert H._install_int8_decoder(types.SimpleNamespace(decoder = holder), keep_blocks = ()) > 0
+ x = torch.randn(64, 64)
+ whole = lin(x) # 64 rows: torch._int_mm
+ sliced = torch.cat(
+ [lin(x[i : i + 8]) for i in range(0, 64, 8)]
+ ) # 8 rows: the dequantised float path
+ assert ((whole - sliced).norm() / whole.norm()) < 0.02
+
+
+def test_apply_holds_fp16_accumulation_off_outside_the_plan(monkeypatch):
+ matmul = torch.backends.cuda.matmul
+ if not hasattr(matmul, "allow_fp16_accumulation"):
+ pytest.skip("torch without allow_fp16_accumulation")
+ monkeypatch.setattr(H, "cuda_fast_path_available", lambda vae = None: True)
+ seen = []
+ vae = types.SimpleNamespace(decode = lambda z: seen.append(matmul.allow_fp16_accumulation) or z)
+ prev = matmul.allow_fp16_accumulation
+ try:
+ matmul.allow_fp16_accumulation = True
+ engaged = H.apply_h3_vae_speedups(
+ vae, speed_mode = "default", workflow = "t2va", consumer_gpu = True
+ )
+ assert H.LEVER_FP16_ACCUM not in engaged
+ vae.decode(0)
+ assert seen == [False] and matmul.allow_fp16_accumulation is True
+ finally:
+ matmul.allow_fp16_accumulation = prev
+
+
+def test_the_fast_path_is_for_the_h3_vae_class_only(monkeypatch):
+ monkeypatch.setattr(torch.cuda, "is_available", lambda: True)
+ monkeypatch.setattr(H, "_kernels", lambda: object())
+ monkeypatch.setattr(H, "_triton_version_ok", lambda: True)
+ monkeypatch.setattr(H, "_triton_jit_toolchain_ok", lambda: True)
+
+ class AutoencoderKLWan:
+ pass
+
+ class AutoencoderKLMiniMaxH3:
+ pass
+
+ if getattr(torch.version, "hip", None):
+ pytest.skip("ROCm build")
+ assert not H.cuda_fast_path_available(AutoencoderKLWan())
+ assert H.cuda_fast_path_available(AutoencoderKLMiniMaxH3())
+
+
+ "platform, probe, expected",
+ [
+ ("win32", lambda: False, False), # Triton present but its JIT cannot find the CRT headers
+ ("win32", lambda: True, True),
+ (
+ "win32",
+ lambda: 1 / 0,
+ True,
+ ), # the probe itself failing is not evidence; per-call fallback covers it
+ ("linux", lambda: False, True), # never asked off Windows
+ ],
+)
+def test_the_fast_path_asks_the_windows_triton_toolchain(monkeypatch, platform, probe, expected):
+ import core._msvc_env as msvc
+
+ monkeypatch.setattr(torch.cuda, "is_available", lambda: True)
+ monkeypatch.setattr(torch.version, "hip", None, raising = False)
+ monkeypatch.setattr(H, "_kernels", lambda: object())
+ monkeypatch.setattr(H, "_triton_version_ok", lambda: True)
+ monkeypatch.setattr(H.sys, "platform", platform)
+ monkeypatch.setattr(msvc, "crt_headers_reachable", probe)
+ H._triton_jit_toolchain_ok.cache_clear()
+ try:
+
+ class AutoencoderKLMiniMaxH3:
+ pass
+
+ assert H.cuda_fast_path_available(AutoencoderKLMiniMaxH3()) is expected
+ finally:
+ H._triton_jit_toolchain_ok.cache_clear()
+
+
[email protected]("device", DEVICES)
[email protected]("stock_fallback", [False, True])
+def test_the_fp16_encoder_survives_diffusers_casting_the_pixels(device, stock_fallback):
+ # diffusers 0.40.0 encode() casts pixels to the encoder dtype and feeds a float32 quant_conv
+ get_parameter_dtype = pytest.importorskip("diffusers.models.modeling_utils").get_parameter_dtype
+
+ vae = _tiny_vae().to(device)
+ ref_vae = copy.deepcopy(vae)
+ assert H._install_encoder(vae, fp16 = True)
+ assert get_parameter_dtype(vae.encoder) is torch.float16
+ vae.encoder._unsloth_fast_failed = stock_fallback
+ x = torch.randn(1, 3, 9, 16, 16, device = device)
+ with torch.no_grad():
+ got = vae.encode(x.to(get_parameter_dtype(vae.encoder)), return_dict = False)[0].mean
+ ref = ref_vae.encode(x, return_dict = False)[0].mean
+ assert got.dtype is torch.float32 and vae.quant_conv.weight.dtype is torch.float32
+ assert (got - ref).norm() / ref.norm() < 2e-2
+
+
+def test_the_video_backend_wires_the_layer_into_the_h3_load_only():
+ import ast
+ import pathlib
+
+ source = (pathlib.Path(H.__file__).parent / "video.py").read_text(encoding = "utf-8")
+ tree = ast.parse(source)
+
+ def calls(node, name):
+ return [
+ c
+ for c in ast.walk(node)
+ if isinstance(c, ast.Call)
+ and (getattr(c.func, "id", None) == name or getattr(c.func, "attr", None) == name)
+ ]
+
+ methods = {
+ n.name: n for n in ast.walk(tree) if isinstance(n, (ast.FunctionDef, ast.AsyncFunctionDef))
+ }
+ load = methods["_load_h3_modular_pipeline"]
+ (apply_call,) = calls(load, "apply_h3_vae_speedups")
+ assert len(calls(tree, "apply_h3_vae_speedups")) == 1, "only the H3 load engages the layer"
+ assert {k.arg for k in apply_call.keywords} >= {"speed_mode", "workflow", "logger"}
+ assert calls(load, "apply_speed_optims")[0].lineno < apply_call.lineno
+ (settle,) = calls(methods["generate"], "settle_h3_vae_fallback")
+ (compile_settle,) = calls(methods["generate"], "settle_compile_fallback")
+ assert compile_settle.lineno < settle.lineno
+
+
[email protected](H._kernels() is None, reason = "needs Triton")
+def test_kernel_annotations_resolve_against_the_module_globals():
+ # Triton <= 3.2 resolves kernel annotations in module globals; a closure-local ``tl`` NameErrors there
+ k = H._kernels()
+ names = [n for n in vars(k) if not n.startswith("_")]
+ assert names
+ for name in names:
+ fn = getattr(getattr(k, name), "fn", None)
+ assert fn is not None, name
+ for arg, ann in fn.__annotations__.items():
+ if isinstance(ann, str):
+ eval(ann, fn.__globals__) # noqa: S307 - a Triton annotation such as "tl.constexpr"
+
+
+ "version, ok",
+ [
+ ("3.2.0", False), # compiles, but wrong GroupNorm statistics for a channels-last input
+ ("3.1.0", False),
+ ("3.3.1", True),
+ ("3.6.0", True),
+ ("3.8.0.post28", True), # triton-windows
+ ("3.2.0+git35c6c7c6", False),
+ ("not a version", False),
+ ],
+)
+def test_the_fast_path_needs_a_verified_triton(monkeypatch, version, ok):
+ assert H._triton_version_ok(version) is ok
+ monkeypatch.setattr(torch.cuda, "is_available", lambda: True)
+ monkeypatch.setattr(torch.version, "hip", None, raising = False)
+ monkeypatch.setattr(H, "_kernels", lambda: object())
+ monkeypatch.setattr(H, "_triton_jit_toolchain_ok", lambda: True)
+ monkeypatch.setattr(H, "_triton_version_ok", lambda: ok)
+
+ class AutoencoderKLMiniMaxH3:
+ pass
+
+ assert H.cuda_fast_path_available(AutoencoderKLMiniMaxH3()) is ok
+
+
+ "before, planned, mid",
+ [
+ (True, False, False), # the bf16 model that turned it on unloads mid-decode
+ (False, False, True), # a bf16 model loads and turns it on mid-decode
+ (True, True, False),
+ (False, True, True),
+ ],
+)
+def test_decode_scope_keeps_a_flag_write_that_lands_mid_decode(before, planned, mid):
+ from core.inference import diffusion_speed as S
+
+ matmul = torch.backends.cuda.matmul
+ if not hasattr(matmul, "allow_fp16_accumulation"):
+ pytest.skip("torch without allow_fp16_accumulation")
+ seen = {}
+
+ class _VAE:
+ def decode(self, z):
+ seen["snapshot"] = S.snapshot_backend_flags()["matmul_fp16_accum"]
+ S.restore_backend_flags({"matmul_fp16_accum": mid})
+ seen["after_write"] = matmul.allow_fp16_accumulation
+ return z
+
+ prev = matmul.allow_fp16_accumulation
+ try:
+ matmul.allow_fp16_accumulation = before
+ vae = _VAE()
+ assert H._install_decode_scope(vae, fp16_accum = planned)
+ vae.decode(0)
+ assert seen["snapshot"] is before, "a snapshot taken mid-decode must see the process value"
+ assert seen["after_write"] is planned, "the decode keeps its own value until it ends"
+ assert (
+ matmul.allow_fp16_accumulation is mid
+ ), "the write made during the decode must survive it"
+ assert not S._fp16_accum_scopes
+ finally:
+ matmul.allow_fp16_accumulation = prev
+
+
+def test_fp16_accumulation_scopes_that_close_out_of_order():
+ from core.inference import diffusion_speed as S
+
+ matmul = torch.backends.cuda.matmul
+ if not hasattr(matmul, "allow_fp16_accumulation"):
+ pytest.skip("torch without allow_fp16_accumulation")
+ prev = matmul.allow_fp16_accumulation
+ try:
+ matmul.allow_fp16_accumulation = True
+ a, b = S.fp16_accumulation_scope(False), S.fp16_accumulation_scope(True)
+ a.__enter__()
+ assert matmul.allow_fp16_accumulation is False
+ b.__enter__()
+ assert matmul.allow_fp16_accumulation is True
+ a.__exit__(None, None, None)
+ assert matmul.allow_fp16_accumulation is True, "the scope still open keeps its value"
+ b.__exit__(None, None, None)
+ assert matmul.allow_fp16_accumulation is True and not S._fp16_accum_scopes
+ finally:
+ matmul.allow_fp16_accumulation = prev
+
+
+def test_int8_install_leaves_the_decoder_float_when_quantization_fails(monkeypatch):
+ vae = _tiny_vae()
+ before = {k: v.clone() for k, v in vae.decoder.state_dict().items()}
+ real, calls = H._quantize_int8_weight, []
+
+ def flaky(w, hadamard):
+ calls.append(1)
+ if len(calls) == 3:
+ raise RuntimeError("out of host memory")
+ return real(w, hadamard)
+
+ monkeypatch.setattr(H, "_quantize_int8_weight", flaky)
+ with pytest.raises(RuntimeError):
+ H._install_int8_decoder(vae, keep_blocks = ())
+ assert len(calls) == 3
+ int8_cls = H._int8_linear_class()
+ for module in vae.decoder.modules():
+ assert type(module) is not int8_cls
+ assert not hasattr(module, "_unsloth_wq") and not hasattr(module, "_unsloth_rot")
+ after = vae.decoder.state_dict()
+ assert after.keys() == before.keys()
+ for k, v in before.items():
+ assert torch.equal(after[k], v), k
+
+
+def test_apply_reports_int8_off_and_keeps_float_weights_when_it_fails(monkeypatch):
+ monkeypatch.setattr(H, "cuda_fast_path_available", lambda vae = None: True)
+ monkeypatch.setenv(H.H3_VAE_INT8_ENV, "1")
+
+ def boom(w, hadamard):
+ raise RuntimeError("out of host memory")
+
+ monkeypatch.setattr(H, "_quantize_int8_weight", boom)
+ vae = _tiny_vae()
+ engaged = H.apply_h3_vae_speedups(
+ vae, speed_mode = "default", workflow = "t2va", consumer_gpu = False
+ )
+ assert H.LEVER_INT8_DECODER not in engaged
+ assert all(
+ m.weight is not None for m in vae.decoder.modules() if isinstance(m, torch.nn.Linear)
+ )
+
+
+def test_int8_decoder_rotated_path_stays_close(monkeypatch):
+ monkeypatch.setattr(H, "H3_VAE_INT8_ROT_GROUP", 16)
+ vae = _tiny_vae()
+ fast = copy.deepcopy(vae)
+ H._install_int8_decoder(fast, keep_blocks = ())
+ lin = fast.decoder.transformer_blocks[0].attn.to_q
+ assert lin.weight is None and lin._unsloth_rot.dtype is torch.float16
+ z = torch.randn(1, 8, 3, 4, 5)
+ with torch.no_grad():
+ ref = vae.decoder(z)
+ got = fast.decoder(z)
+ assert (got - ref).norm() / ref.norm() < 0.05