Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: faster MiniMax-H3 video VAE encode and decode (#11801)
Daniel Han14354db43cbd
diff --git a/studio/backend/core/inference/diffusion_speed.py b/studio/backend/core/inference/diffusion_speed.pyindex f2a07df3582..97becc5c7ec 100644--- a/studio/backend/core/inference/diffusion_speed.py+++ b/studio/backend/core/inference/diffusion_speed.py@@ -35,8 +35,10 @@ from __future__ import annotationsimport osimport sys+import threading+from contextlib import contextmanagerfrom functools import lru_cache-from typing import Any, Optional+from typing import Any, Iterator, Optionalfrom . import diffusion_gguf_compile as gguf_compile@@ -58,6 +60,52 @@ _INDUCTOR_FLAGS = (_INDUCTOR_TRITON_FLAGS = (("unique_kernel_names", "inductor_triton_unique_kernel_names"),)+# allow_fp16_accumulation has one owner: writes during an open scope go to the recorded process value, so closing+# the scope restores the latest value, not a stale copy.+_FP16_ACCUM_LOCK = threading.RLock()+_fp16_accum_scopes: list = []+_fp16_accum_base: Optional[bool] = None+++def _read_fp16_accum(matmul: Any) -> bool:+ with _FP16_ACCUM_LOCK:+ if _fp16_accum_scopes:+ return bool(_fp16_accum_base)+ return bool(matmul.allow_fp16_accumulation)+++def _write_fp16_accum(matmul: Any, value: Any) -> None:+ global _fp16_accum_base+ with _FP16_ACCUM_LOCK:+ if _fp16_accum_scopes:+ _fp16_accum_base = bool(value)+ else:+ matmul.allow_fp16_accumulation = bool(value)+++@contextmanager+def fp16_accumulation_scope(value: bool) -> Iterator[None]:+ """Hold ``allow_fp16_accumulation`` at ``value`` for the body, then restore the latest process value."""+ global _fp16_accum_base+ import torch++ matmul = torch.backends.cuda.matmul+ token = object()+ with _FP16_ACCUM_LOCK:+ if not _fp16_accum_scopes:+ _fp16_accum_base = bool(matmul.allow_fp16_accumulation)+ _fp16_accum_scopes.append((token, bool(value)))+ matmul.allow_fp16_accumulation = bool(value)+ try:+ yield+ finally:+ with _FP16_ACCUM_LOCK:+ _fp16_accum_scopes[:] = [e for e in _fp16_accum_scopes if e[0] is not token]+ matmul.allow_fp16_accumulation = (+ _fp16_accum_scopes[-1][1] if _fp16_accum_scopes else bool(_fp16_accum_base)+ )++def snapshot_backend_flags() -> Optional[dict]:"""Capture the process-wide torch backend flags this layer may mutate, for restore on unload. Nonewithout torch. Each flag is read defensively: a build missing one still captures the rest."""@@ -70,7 +118,7 @@ def snapshot_backend_flags() -> Optional[dict]:if matmul is not None and hasattr(matmul, "allow_tf32"):state["matmul_tf32"] = bool(matmul.allow_tf32)if matmul is not None and hasattr(matmul, "allow_fp16_accumulation"):- state["matmul_fp16_accum"] = bool(matmul.allow_fp16_accumulation)+ state["matmul_fp16_accum"] = _read_fp16_accum(matmul)cudnn = getattr(torch.backends, "cudnn", None)if cudnn is not None:if hasattr(cudnn, "allow_tf32"):@@ -122,7 +170,15 @@ def restore_backend_flags(state: Optional[dict]) -> None:passmatmul = getattr(getattr(torch.backends, "cuda", None), "matmul", None)_set(matmul, "allow_tf32", "matmul_tf32")- _set(matmul, "allow_fp16_accumulation", "matmul_fp16_accum")+ if (+ matmul is not None+ and "matmul_fp16_accum" in state+ and hasattr(matmul, "allow_fp16_accumulation")+ ):+ try:+ _write_fp16_accum(matmul, state["matmul_fp16_accum"])+ except Exception: # noqa: BLE001 - best-effort per-flag restore+ passcudnn = getattr(torch.backends, "cudnn", None)_set(cudnn, "allow_tf32", "cudnn_tf32")_set(cudnn, "benchmark", "cudnn_benchmark")@@ -861,7 +917,7 @@ def _enable_fp16_accumulation(if not _is_consumer_gpu():return False- matmul.allow_fp16_accumulation = True+ _write_fp16_accum(matmul, True)return Trueexcept Exception as exc: # noqa: BLE001 - optimisation only_warn(logger, "fp16_accum", exc)diff --git a/studio/backend/core/inference/video.py b/studio/backend/core/inference/video.pyindex 5adeeb2dcb4..fce79d8d0da 100644--- a/studio/backend/core/inference/video.py+++ b/studio/backend/core/inference/video.py@@ -5136,6 +5136,7 @@ class VideoBackend:# Resolved BEFORE the placement below, not after it: the dense pin is a speed optimisation by its own reasoning,# so "off" has to be known in time to decline it.effective_speed = resolve_speed_mode(speed_mode, is_gguf = False, dense_default = SPEED_DEFAULT)+ h3_vae_speed = effective_speedif effective_speed == SPEED_MAX:# SPEED_MAX compiles static until a dimension changes. H3's packed sequence length carries the caption's# token rows, so a static graph recompiles on new prompts: measured 0.957-1.000 s/step static against@@ -5308,6 +5309,18 @@ class VideoBackend:speed_optims = tuple(k for k, v in applied.items() if v)except Exception as exc: # noqa: BLE001 -- optimisation only, never fail a loadlogger.warning("video.h3_speed_optims failed, continuing unoptimised: %s", exc)+ # nothing here compiles, so it follows the REQUESTED tier, not the denoiser's eager downgrade above+ try:+ from .video_minimax_h3_vae import apply_h3_vae_speedups+ vae_levers = apply_h3_vae_speedups(+ getattr(pipe, "vae", None),+ speed_mode = h3_vae_speed,+ workflow = workflow,+ logger = logger,+ )+ speed_optims = speed_optims + tuple(f"h3_vae_{name}" for name in vae_levers)+ except Exception as exc: # noqa: BLE001 -- optimisation only, never fail a load+ logger.warning("video.h3_vae_fast failed, keeping the stock VAE: %s", exc)resolved = build_resolved_record({@@ -6330,6 +6343,9 @@ class VideoBackend:# must not keep reporting it compiled (a forced-compile quantised load runs ~30x slower eager),# whether this render finished, was cancelled or failed.settle_compile_fallback(state, pipe, logger)+ if fam.modular_workflow:+ from .video_minimax_h3_vae import settle_h3_vae_fallback+ settle_h3_vae_fallback(state, pipe)if cancel.is_set():raise RuntimeError(VIDEO_CANCELLED_MSG)diff --git a/studio/backend/core/inference/video_minimax_h3_vae.py b/studio/backend/core/inference/video_minimax_h3_vae.pynew file mode 100644index 00000000000..826c1571340--- /dev/null+++ b/studio/backend/core/inference/video_minimax_h3_vae.py@@ -0,0 +1,1309 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""Faster MiniMax-H3 video VAE encode and decode: Triton-fused GroupNorm/SiLU/pad, RMSNorm/rope/SwiGLU and+tile batching around Diffusers' own matmuls and convolutions.++Levers: fused_encoder, fp16_encoder, fused_decoder, tile_batch, fp16_accum, int8_decoder (opt-in,+``UNSLOTH_H3_VAE_INT8=1``). ``UNSLOTH_H3_VAE_FAST=0`` disables all. NVIDIA CUDA with Triton only; else stock.+"""++from __future__ import annotations++import os+import sys+import types+from contextlib import nullcontext+from functools import lru_cache+from typing import Any, Optional++H3_VAE_FAST_ENV = "UNSLOTH_H3_VAE_FAST"+H3_VAE_INT8_ENV = "UNSLOTH_H3_VAE_INT8"+H3_VAE_TILE_BATCH_ENV = "UNSLOTH_H3_VAE_TILE_BATCH"+FP16_ACCUM_DISABLE_ENV = "UNSLOTH_DISABLE_FP16_ACCUM"++LEVER_FUSED_ENCODER = "fused_encoder"+LEVER_FP16_ENCODER = "fp16_encoder"+LEVER_FUSED_DECODER = "fused_decoder"+LEVER_TILE_BATCH = "tile_batch"+LEVER_INT8_DECODER = "int8_decoder"+LEVER_FP16_ACCUM = "fp16_accum"++_TILE_BATCH_MAX = 4+_TILE_BATCH_BYTES_PER_TILE = 256 * 2**20+H3_VAE_INT8_ROT_GROUP = 256+# int8 error in early blocks propagates through the rest, so the first half stays float.+H3_VAE_INT8_FLOAT_BLOCKS = 18+# Triton 3.2 silently computes wrong GroupNorm statistics for a channels-last input.+H3_VAE_MIN_TRITON = (3, 3)++_SPEED_OFF = "off"+_SPEED_EAGER = "eager"+_SPEED_MAX = "max"+_TRUE = ("1", "true", "yes", "on")+_FALSE = ("0", "false", "no", "off")+++def _env(name: str) -> str:+ return os.environ.get(name, "").strip().lower()+++def h3_vae_fast_disabled() -> bool:+ return _env(H3_VAE_FAST_ENV) in _FALSE+++def plan_h3_vae_levers(+ speed_mode: Optional[str],+ *,+ workflow: Optional[str] = None,+ consumer_gpu: bool = False,+) -> tuple[str, ...]:+ """Which levers a load should engage. Pure: no torch, no device probe."""+ mode = str(speed_mode or "").strip().lower()+ if h3_vae_fast_disabled() or mode == _SPEED_OFF:+ return ()+ levers: list[str] = []+ if workflow != "t2va":+ levers.append(LEVER_FUSED_ENCODER)+ if mode != _SPEED_EAGER:+ levers.append(LEVER_FP16_ENCODER)+ levers += [LEVER_FUSED_DECODER, LEVER_TILE_BATCH]+ if _env(H3_VAE_INT8_ENV) in _TRUE:+ levers.append(LEVER_INT8_DECODER)+ if mode == _SPEED_MAX and consumer_gpu and _env(FP16_ACCUM_DISABLE_ENV) not in _TRUE:+ levers.append(LEVER_FP16_ACCUM)+ return tuple(levers)+++@lru_cache(maxsize = 1)+def _kernels() -> Optional[types.SimpleNamespace]:+ """Compile-on-first-use Triton kernels, or None when Triton is unavailable."""+ try:+ import triton+ import triton.language as tl+ from triton.language.extra import libdevice+ except Exception: # noqa: BLE001 - no Triton means the stock path+ return None+ # Triton <= 3.2 resolves string annotations against module globals, not this function's locals+ globals().update(triton = triton, tl = tl, libdevice = libdevice)++ @triton.jit+ def _gn_partials(+ x_ptr,+ bias_ptr,+ pn_ptr,+ pmean_ptr,+ pm2_ptr,+ T,+ HW,+ W,+ n_chunks,+ sb,+ sc,+ st,+ sh,+ sw,+ C: tl.constexpr,+ G: tl.constexpr,+ CPG: tl.constexpr,+ BLOCK_P: tl.constexpr,+ HAS_BIAS: tl.constexpr,+ ):+ chunk = tl.program_id(0)+ bt = tl.program_id(1)+ t = bt % T+ b = bt // T+ p = chunk * BLOCK_P + tl.arange(0, BLOCK_P)+ pmask = p < HW+ hh = p // W+ ww = p - hh * W+ c = tl.arange(0, C)+ off = (+ b.to(tl.int64) * sb+ + t.to(tl.int64) * st+ + hh.to(tl.int64)[:, None] * sh+ + ww.to(tl.int64)[:, None] * sw+ + c.to(tl.int64)[None, :] * sc+ )+ v = tl.load(x_ptr + off, mask = pmask[:, None], other = 0.0).to(tl.float32)+ if HAS_BIAS:+ v = v + tl.load(bias_ptr + c).to(tl.float32)[None, :]+ v = tl.where(pmask[:, None], v, 0.0)+ n_pix = tl.sum(pmask.to(tl.float32), axis = 0)+ v3 = tl.reshape(v, (BLOCK_P, G, CPG))+ mean = tl.sum(tl.sum(v3, axis = 2), axis = 0) / (n_pix * CPG)+ d = tl.where(pmask[:, None, None], v3 - mean[None, :, None], 0.0)+ m2 = tl.sum(tl.sum(d * d, axis = 2), axis = 0)+ slot = bt.to(tl.int64) * n_chunks + chunk+ g = tl.arange(0, G)+ tl.store(pmean_ptr + slot * G + g, mean)+ tl.store(pm2_ptr + slot * G + g, m2)+ tl.store(pn_ptr + slot, n_pix * CPG)++ @triton.jit+ def _gn_combine(+ pn_ptr, pmean_ptr, pm2_ptr, mean_ptr, rstd_ptr, n_chunks, G, eps, BLOCK: tl.constexpr+ ):+ row = tl.program_id(0)+ g = row % G+ bt = (row // G).to(tl.int64)+ zero = tl.sum(tl.zeros([BLOCK], dtype = tl.float32), axis = 0)+ acc_n = zero+ acc_s = zero+ for k0 in range(0, n_chunks, BLOCK):+ k = k0 + tl.arange(0, BLOCK)+ km = k < n_chunks+ n = tl.load(pn_ptr + bt * n_chunks + k, mask = km, other = 0.0)+ mu = tl.load(pmean_ptr + (bt * n_chunks + k) * G + g, mask = km, other = 0.0)+ acc_n += tl.sum(n, axis = 0)+ acc_s += tl.sum(n * mu, axis = 0)+ mean = acc_s / acc_n+ acc_m2 = zero+ for k0 in range(0, n_chunks, BLOCK):+ k = k0 + tl.arange(0, BLOCK)+ km = k < n_chunks+ n = tl.load(pn_ptr + bt * n_chunks + k, mask = km, other = 0.0)+ mu = tl.load(pmean_ptr + (bt * n_chunks + k) * G + g, mask = km, other = 0.0)+ m2 = tl.load(pm2_ptr + (bt * n_chunks + k) * G + g, mask = km, other = 0.0)+ dm = mu - mean+ acc_m2 += tl.sum(m2 + n * dm * dm, axis = 0)+ tl.store(mean_ptr + row, mean)+ tl.store(rstd_ptr + row, 1.0 / tl.sqrt(acc_m2 / acc_n + eps))++ @triton.jit+ def _gn_silu_pad(+ x_ptr,+ out_ptr,+ mean_ptr,+ rstd_ptr,+ w_ptr,+ b_ptr,+ ib_ptr,+ C,+ T,+ H,+ W,+ To,+ Ho,+ Wo,+ G,+ front,+ pad_t,+ pad_l,+ sb,+ sc,+ st,+ sh,+ sw,+ CPG: tl.constexpr,+ HAS_NORM: tl.constexpr,+ HAS_IN_BIAS: tl.constexpr,+ BLOCK_W: tl.constexpr,+ BLOCK_C: tl.constexpr,+ ):+ pid_row = tl.program_id(0)+ pid_w = tl.program_id(1)+ pid_c = tl.program_id(2)+ h_o = pid_row % Ho+ bt = pid_row // Ho+ t_o = bt % To+ b = bt // To+ t_i = t_o - front+ t_ok = t_i >= 0+ t_c = tl.maximum(t_i, 0)+ r = h_o - pad_t+ r = tl.where(r < 0, -r, r)+ r = tl.where(r >= H, 2 * (H - 1) - r, r)+ in_base = x_ptr + (b.to(tl.int64) * sb + t_c.to(tl.int64) * st + r.to(tl.int64) * sh)+ out_base = out_ptr + (((b.to(tl.int64) * To + t_o) * Ho + h_o) * Wo) * C+ wo = pid_w * BLOCK_W + tl.arange(0, BLOCK_W)+ wmask = wo < Wo+ q = wo - pad_l+ q = tl.where(q < 0, -q, q)+ q = tl.where(q >= W, 2 * (W - 1) - q, q)+ c = pid_c * BLOCK_C + tl.arange(0, BLOCK_C)+ cmask = c < C+ m = wmask[:, None] & cmask[None, :]+ # int64: an NCDHW channel stride x 127 passes 2^31 on a large untiled encode+ in_off = q.to(tl.int64)[:, None] * sw + c.to(tl.int64)[None, :] * sc+ v = tl.load(in_base + in_off, mask = m & t_ok, other = 0.0).to(tl.float32)+ if HAS_IN_BIAS:+ v = v + tl.load(ib_ptr + c, mask = cmask, other = 0.0).to(tl.float32)[None, :]+ if HAS_NORM:+ gidx = (b * T + t_c) * G + c // CPG+ mu = tl.load(mean_ptr + gidx, mask = cmask, other = 0.0)+ rs = tl.load(rstd_ptr + gidx, mask = cmask, other = 0.0)+ gw = tl.load(w_ptr + c, mask = cmask, other = 0.0).to(tl.float32)+ gb = tl.load(b_ptr + c, mask = cmask, other = 0.0).to(tl.float32)+ a = rs * gw+ v = v * a[None, :] + (gb - mu * a)[None, :]+ v = v / (1.0 + tl.exp(-v))+ v = tl.where(t_ok, v, 0.0)+ tl.store(out_base + (wo[:, None] * C + c[None, :]), v.to(out_ptr.dtype.element_ty), mask = m)++ @triton.jit+ def _add_residual(+ o_ptr,+ ob_ptr,+ r_ptr,+ rb_ptr,+ P,+ T,+ H,+ W,+ sb,+ sc,+ st,+ sh,+ sw,+ C: tl.constexpr,+ HAS_OB: tl.constexpr,+ HAS_RB: tl.constexpr,+ R_SAME: tl.constexpr,+ BLOCK_P: tl.constexpr,+ ):+ p = tl.program_id(0) * BLOCK_P + tl.arange(0, BLOCK_P)+ pmask = p < P+ c = tl.arange(0, C)+ base = tl.program_id(0).to(tl.int64) * BLOCK_P * C+ local = (tl.arange(0, BLOCK_P) * C)[:, None] + c[None, :]+ m = pmask[:, None]+ o = tl.load(o_ptr + base + local, mask = m, other = 0.0).to(tl.float32)+ if HAS_OB:+ o = o + tl.load(ob_ptr + c).to(tl.float32)[None, :]+ if R_SAME:+ r = tl.load(r_ptr + base + local, mask = m, other = 0.0).to(tl.float32)+ else:+ w = p % W+ rest = p // W+ h = rest % H+ rest = rest // H+ t = rest % T+ b = rest // T+ poff = (+ b.to(tl.int64) * sb+ + t.to(tl.int64) * st+ + h.to(tl.int64) * sh+ + w.to(tl.int64) * sw+ )+ r = tl.load(+ r_ptr + poff[:, None] + (c.to(tl.int64) * sc)[None, :], mask = m, other = 0.0+ ).to(tl.float32)+ if HAS_RB:+ r = r + tl.load(rb_ptr + c).to(tl.float32)[None, :]+ tl.store(o_ptr + base + local, (o + r).to(o_ptr.dtype.element_ty), mask = m)++ @triton.jit+ def _add_rmsnorm(+ h_ptr, o_ptr, s_ptr, w_ptr, n_ptr, N, eps, HAS_RES: tl.constexpr, BLOCK_N: tl.constexpr+ ):+ row = tl.program_id(0).to(tl.int64)+ cols = tl.arange(0, BLOCK_N)+ mask = cols < N+ h = tl.load(h_ptr + row * N + cols, mask = mask, other = 0.0).to(tl.float32)+ if HAS_RES:+ o = tl.load(o_ptr + row * N + cols, mask = mask, other = 0.0).to(tl.float32)+ s = tl.load(s_ptr + cols, mask = mask, other = 0.0).to(tl.float32)+ # the reference's separate multiply and add, each rounded (no fma contraction)+ h = libdevice.add_rn(h, libdevice.mul_rn(o, s))+ tl.store(h_ptr + row * N + cols, h.to(h_ptr.dtype.element_ty), mask = mask)+ ms = tl.sum(h * h, axis = 0) / N+ w = tl.load(w_ptr + cols, mask = mask, other = 0.0).to(tl.float32)+ y = h * (1.0 / tl.sqrt(ms + eps)) * w+ tl.store(n_ptr + row * N + cols, y.to(n_ptr.dtype.element_ty), mask = mask)++ @triton.jit+ def _qk_norm_rope(+ x_ptr,+ cos_ptr,+ sin_ptr,+ n_head_rows,+ heads,+ cs_stride,+ eps,+ HEAD_D: tl.constexpr,+ ROT: tl.constexpr,+ BLOCK: tl.constexpr,+ ):+ hr = tl.program_id(0).to(tl.int64) * BLOCK + tl.arange(0, BLOCK)+ rmask = hr < n_head_rows+ d = tl.arange(0, HEAD_D)+ half = ROT // 2+ partner = tl.where(d < half, d + half, tl.where(d < ROT, d - half, d))+ base = hr[:, None] * HEAD_D+ x = tl.load(x_ptr + base + d[None, :], mask = rmask[:, None], other = 0.0).to(tl.float32)+ xp = tl.load(x_ptr + base + partner[None, :], mask = rmask[:, None], other = 0.0).to(tl.float32)+ rstd = 1.0 / tl.sqrt(tl.sum(x * x, axis = 1) / HEAD_D + eps)+ dt = x_ptr.dtype.element_ty+ xn = (x * rstd[:, None]).to(dt).to(tl.float32)+ xpn = (xp * rstd[:, None]).to(dt).to(tl.float32)+ rot = d < ROT+ token = hr // heads+ cs_off = token[:, None] * cs_stride + tl.where(rot, d, 0)[None, :]+ cs_m = rmask[:, None] & rot[None, :]+ cos = tl.load(cos_ptr + cs_off, mask = cs_m, other = 1.0).to(tl.float32)+ sin = tl.load(sin_ptr + cs_off, mask = cs_m, other = 0.0).to(tl.float32)+ # mul_rn / add_rn: without them the compiler contracts into one fma and skips eager's rounding.+ xps = tl.where((d < half)[None, :], -xpn, xpn)+ a = libdevice.mul_rn(xn, cos).to(dt).to(tl.float32)+ bb = libdevice.mul_rn(xps, sin).to(dt).to(tl.float32)+ y = tl.where(rot[None, :], libdevice.add_rn(a, bb), xn)+ tl.store(x_ptr + base + d[None, :], y.to(dt), mask = rmask[:, None])++ @triton.jit+ def _swiglu(x_ptr, out_ptr, N, BLOCK_N: tl.constexpr):+ row = tl.program_id(0).to(tl.int64)+ pid_n = tl.program_id(1)+ cols = pid_n * BLOCK_N + tl.arange(0, BLOCK_N)+ mask = cols < N+ hid = tl.load(x_ptr + row * 2 * N + cols, mask = mask, other = 0.0).to(tl.float32)+ gate = tl.load(x_ptr + row * 2 * N + N + cols, mask = mask, other = 0.0).to(tl.float32)+ dt = out_ptr.dtype.element_ty+ act = (gate / (1.0 + tl.exp(-gate))).to(dt).to(tl.float32)+ tl.store(out_ptr + row * N + cols, (hid * act).to(dt), mask = mask)++ @triton.jit+ def _quant_rows(x_ptr, q_ptr, s_ptr, N, BLOCK_N: tl.constexpr):+ # IEEE division + half-to-even rounding: the same codes as the torch path.+ row = tl.program_id(0).to(tl.int64)+ cols = tl.arange(0, BLOCK_N)+ mask = cols < N+ x = tl.load(x_ptr + row * N + cols, mask = mask, other = 0.0).to(tl.float32)+ amax = tl.maximum(tl.max(tl.abs(x), axis = 0), 1e-12)+ s = amax / 127.0+ q = libdevice.rint(libdevice.div_rn(x, s))+ q = tl.minimum(tl.maximum(q, -127.0), 127.0)+ tl.store(q_ptr + row * N + cols, q.to(tl.int8), mask = mask)+ tl.store(s_ptr + row, s)++ @triton.jit+ def _dequant_epilogue(+ acc_ptr, xs_ptr, ws_ptr, bias_ptr, out_ptr, N, HAS_BIAS: tl.constexpr, BLOCK_N: tl.constexpr+ ):+ row = tl.program_id(0).to(tl.int64)+ pid_n = tl.program_id(1)+ cols = pid_n * BLOCK_N + tl.arange(0, BLOCK_N)+ mask = cols < N+ acc = tl.load(acc_ptr + row * N + cols, mask = mask, other = 0).to(tl.float32)+ xs = tl.load(xs_ptr + row)+ ws = tl.load(ws_ptr + cols, mask = mask, other = 0.0)+ y = acc * xs * ws+ if HAS_BIAS:+ y = y + tl.load(bias_ptr + cols, mask = mask, other = 0.0).to(tl.float32)+ tl.store(out_ptr + row * N + cols, y.to(out_ptr.dtype.element_ty), mask = mask)++ return types.SimpleNamespace(+ gn_partials = _gn_partials,+ gn_combine = _gn_combine,+ gn_silu_pad = _gn_silu_pad,+ add_residual = _add_residual,+ add_rmsnorm = _add_rmsnorm,+ qk_norm_rope = _qk_norm_rope,+ swiglu = _swiglu,+ quant_rows = _quant_rows,+ dequant_epilogue = _dequant_epilogue,+ )+++def _next_pow2(n: int) -> int:+ return 1 << max(0, int(n) - 1).bit_length()+++def _is_oom(exc: BaseException) -> bool:+ try:+ import torch+ if isinstance(exc, torch.cuda.OutOfMemoryError):+ return True+ except Exception: # noqa: BLE001+ pass+ return "out of memory" in str(exc).lower()+++def _guarded(fast: Any, stock: Any, label: str) -> Any:+ """Runs ``fast`` until it raises something ``stock`` does not, then logs once and runs ``stock``."""++ def forward(self, *args, **kwargs):+ if getattr(self, "_unsloth_fast_failed", False):+ return stock(self, *args, **kwargs)+ try:+ return fast(self, *args, **kwargs)+ except Exception as exc: # noqa: BLE001+ if _is_oom(exc):+ raise+ # the stock path rejects this input too: caller's error, keep the fast path+ out = stock(self, *args, **kwargs)+ self._unsloth_fast_failed = True+ log = getattr(self, "_unsloth_logger", None)+ if log is not None:+ log.warning(+ "video.h3_vae_fast: the fused %s failed, using the stock path: %s", label, exc+ )+ return out++ return forward+++# Activations travel as (tensor, pending_bias): the next fused pass adds the conv bias, avoiding PyTorch's+# separate strided bias pass. A 1x1 conv folds its input's pending bias through its weight exactly.+++def _bias_view(bias: Any) -> Any:+ return bias.view(1, -1, 1, 1, 1)+++def norm_silu_pad_reference(+ x: Any,+ norm: Any,+ pad: tuple,+ front: int,+ in_bias: Any = None,+) -> Any:+ """Unfused reference of the kernel, for tests and hosts without Triton."""+ import torch+ import torch.nn.functional as F++ if norm is not None:+ b, c, t, h, w = x.shape+ y = x.float() if in_bias is None else x.float() + _bias_view(in_bias).float()+ y = y.permute(0, 2, 1, 3, 4).reshape(b * t, c, h, w)+ y = F.group_norm(y, norm.num_groups, norm.weight.float(), norm.bias.float(), norm.eps)+ x = F.silu(y).view(b, t, c, h, w).permute(0, 2, 1, 3, 4).to(x.dtype)+ elif in_bias is not None:+ x = (x.float() + _bias_view(in_bias).float()).to(x.dtype)+ return _pad_reference(x, pad, front).contiguous(memory_format = torch.channels_last_3d)+++def _pad_reference(x: Any, pad: tuple, front: int) -> Any:+ """Reflect ``(left, right, top, bottom)`` spatially, then ``front`` causal zero frames."""+ import torch.nn.functional as F++ if any(pad):+ x = F.pad(x, (*pad, 0, 0), mode = "reflect")+ return F.pad(x, (0, 0, 0, 0, front, 0)) if front else x+++def _fusable(x: Any, pad: tuple, norm: Any) -> bool:+ c = x.shape[1]+ return (+ _kernels() is not None+ and x.is_cuda+ and max(pad[0], pad[1]) < x.shape[-1]+ and max(pad[2], pad[3]) < x.shape[-2]+ and (norm is None or (c & (c - 1) == 0 and c % int(norm.num_groups) == 0 and c <= 1024))+ )+++def norm_silu_pad(+ x: Any,+ norm: Any,+ pad: tuple,+ front: int,+ in_bias: Any = None,+) -> Any:+ """Fused pending-bias add + per-frame GroupNorm + SiLU + reflect/causal pad, channels-last out."""+ import torch++ if not _fusable(x, pad, norm):+ return norm_silu_pad_reference(x, norm, pad, front, in_bias)+ k = _kernels()+ left, right, top, bottom = pad+ b, c, t, h, w = x.shape+ to, ho, wo = t + front, h + top + bottom, w + left + right+ out = torch.empty((b, to, ho, wo, c), dtype = x.dtype, device = x.device).permute(0, 4, 1, 2, 3)+ sb, sc, st, sh, sw = x.stride()+ ib = in_bias if in_bias is not None else x+ if norm is not None:+ g = int(norm.num_groups)+ cpg = c // g+ hw = h * w+ budget = 4096 if sw == 1 and x.element_size() == 4 else 8192+ block_p = max(1, min(128, budget // c))+ n_chunks = (hw + block_p - 1) // block_p+ rows = b * t+ pmean, pm2 = torch.empty((2, rows * n_chunks * g), dtype = torch.float32, device = x.device)+ pn = torch.empty(rows * n_chunks, dtype = torch.float32, device = x.device)+ # chunks on grid axis 0: axis 1 stops at 65535, and a large untiled frame has more chunks than that+ k.gn_partials[(n_chunks, rows)](+ x,+ ib,+ pn,+ pmean,+ pm2,+ t,+ hw,+ w,+ n_chunks,+ sb,+ sc,+ st,+ sh,+ sw,+ C = c,+ G = g,+ CPG = cpg,+ BLOCK_P = block_p,+ HAS_BIAS = in_bias is not None,+ num_warps = 4,+ )+ mean = torch.empty(rows * g, dtype = torch.float32, device = x.device)+ rstd = torch.empty(rows * g, dtype = torch.float32, device = x.device)+ k.gn_combine[(rows * g,)](+ pn, pmean, pm2, mean, rstd, n_chunks, g, float(norm.eps), BLOCK = 1024, num_warps = 4+ )+ weight, bias = norm.weight, norm.bias+ else:+ # unused without a norm; constant, so the pad-only kernel compiles once for every channel count+ g, cpg = 1, 1+ mean = rstd = weight = bias = x+ block_c = min(128, _next_pow2(c))+ block_w = max(16, 4096 // block_c)+ grid = (b * to * ho, (wo + block_w - 1) // block_w, (c + block_c - 1) // block_c)+ k.gn_silu_pad[grid](+ x,+ out,+ mean,+ rstd,+ weight,+ bias,+ ib,+ c,+ t,+ h,+ w,+ to,+ ho,+ wo,+ g,+ front,+ top,+ left,+ sb,+ sc,+ st,+ sh,+ sw,+ CPG = cpg,+ HAS_NORM = norm is not None,+ HAS_IN_BIAS = in_bias is not None,+ BLOCK_W = block_w,+ BLOCK_C = block_c,+ num_warps = 4,+ )+ return out+++def add_residual(out: Any, out_bias: Any, res: Any, res_bias: Any) -> Any:+ """``out + out_bias + res + res_bias`` written into ``out`` (the conv's own fresh output)."""+ import torch++ k = _kernels()+ c = out.shape[1]+ if (+ k is None+ or not out.is_cuda+ or not out.is_contiguous(memory_format = torch.channels_last_3d)+ or res.shape != out.shape+ or c & (c - 1)+ or c > 1024+ ):+ if out_bias is not None:+ out.add_(_bias_view(out_bias).to(out.dtype))+ if res_bias is not None:+ res = res + _bias_view(res_bias).to(res.dtype)+ return out.add_(res)+ b, _, t, h, w = out.shape+ pixels = b * t * h * w+ block_p = max(1, 4096 // c)+ k.add_residual[((pixels + block_p - 1) // block_p,)](+ out,+ out_bias if out_bias is not None else out,+ res,+ res_bias if res_bias is not None else res,+ pixels,+ t,+ h,+ w,+ *res.stride(),+ C = c,+ HAS_OB = out_bias is not None,+ HAS_RB = res_bias is not None,+ R_SAME = res.is_contiguous(memory_format = torch.channels_last_3d),+ BLOCK_P = block_p,+ num_warps = 4,+ )+ return out+++def _last_tap(weight: Any) -> Any:+ """A conv weight's last temporal tap. Not cached: must follow offload hooks' ``.to()``."""+ import torch+ return weight[:, :, -1:].contiguous(memory_format = torch.channels_last_3d)+++def _causal_conv(+ conv: Any,+ x: Any,+ x_bias: Any = None,+ *,+ norm: Any = None,+ spatial_pad: Optional[tuple] = None,+) -> tuple:+ """``conv(pad(silu(norm(x + x_bias))))`` WITHOUT the conv's bias, which is returned as the output's pending bias."""+ import torch.nn.functional as F++ kt = conv.kernel_size[0]+ sp = int(getattr(conv, "spatial_padding", 0) or 0)+ front = int(getattr(conv, "temporal_padding", 0) or 0)+ pad = (sp, sp, sp, sp) if spatial_pad is None else tuple(spatial_pad)+ weight = conv.weight+ bias = conv.bias+ if x.shape[2] == 1 and kt > 1 and front == kt - 1:+ front = 0+ weight = _last_tap(weight)+ if norm is not None or front or any(pad):+ if x.shape[1] % 8 == 0 or norm is not None:+ x = norm_silu_pad(x, norm, pad, front, in_bias = x_bias)+ else:+ # conv_in's 3 channels: cuDNN would convert channels-last back+ if x_bias is not None:+ x = x + _bias_view(x_bias).to(x.dtype)+ x = _pad_reference(x, pad, front)+ elif x_bias is not None:+ # a 1x1x1 conv without padding is linear in its input, so the pending bias folds through the weight exactly+ folded = weight.reshape(weight.shape[0], -1).float() @ x_bias.float()+ bias = folded if bias is None else bias.float() + folded+ out = F.conv3d(x, weight, None, stride = conv.stride)+ return out, bias+++def _fast_encoder_forward(self: Any, hidden_states: Any) -> Any:+ """``MiniMaxH3VideoEncoder3d.forward`` over the fused passes. Same modules, same order."""+ import torch++ # Triton launches on the current device, which on a multi-GPU host need not be the tensor's+ with torch.cuda.device(hidden_states.device) if hidden_states.is_cuda else nullcontext():+ return _fast_encoder_body(self, hidden_states)+++def _fast_encoder_body(self: Any, hidden_states: Any) -> Any:+ dtype = getattr(self, "_unsloth_compute_dtype", None) or hidden_states.dtype+ out_dtype = getattr(self, "_unsloth_out_dtype", None) or hidden_states.dtype+ h, hb = _causal_conv(self.conv_in, hidden_states.to(dtype))+ for down_block in self.down_blocks:+ for resnet in down_block.resnets:+ x, xb = h, hb+ h, hb = _causal_conv(resnet.conv1, x, xb, norm = resnet.norm1)+ h, hb = _causal_conv(resnet.conv2, h, hb, norm = resnet.norm2)+ if resnet.conv_shortcut is None:+ r, rb = x, xb+ else:+ r, rb = _causal_conv(resnet.conv_shortcut, x, xb)+ h, hb = add_residual(h, hb, r, rb), None+ if down_block.downsamplers is not None:+ for ds in down_block.downsamplers:+ pad = (0, 1, 0, 1) if ds.spatial_stride == 2 else None+ h, hb = _causal_conv(ds.conv, h, hb, spatial_pad = pad)+ h, hb = _causal_conv(self.conv_out, h, hb, norm = self.norm_out)+ h = h.float()+ if hb is not None:+ h = h + _bias_view(hb).float()+ return h.to(out_dtype).contiguous()+++def _install_encoder(vae: Any, *, fp16: bool) -> bool:+ import torch++ encoder = getattr(vae, "encoder", None)+ if encoder is None or not hasattr(encoder, "down_blocks"):+ return False+ # the kernel pads by reflection, which is what the released config uses; any other mode keeps the stock path+ modes = {getattr(m, "spatial_padding_mode", "reflect") for m in encoder.modules()}+ if modes - {"reflect"}:+ return False+ convs = [m for m in encoder.modules() if isinstance(m, torch.nn.Conv3d)]+ stock_dtype = encoder.conv_in.weight.dtype+ with torch.no_grad():+ # build every tensor before assigning: a mid-way OOM must leave the stock encoder intact+ new = []+ for module in convs:+ w = module.weight.data.to(torch.float16) if fp16 else module.weight.data+ if module is not encoder.conv_in:+ w = w.contiguous(memory_format = torch.channels_last_3d)+ bias = module.bias+ b = None if bias is None else (bias.data.to(torch.float16) if fp16 else bias.data)+ new.append((module, w, b))+ for module, w, b in new:+ module.weight.data = w+ if b is not None:+ module.bias.data = b+ encoder._unsloth_compute_dtype = torch.float16 if fp16 else None+ # quant_conv stays in the stock dtype: hand back what the stock encoder would+ encoder._unsloth_out_dtype = stock_dtype if fp16 else None+ encoder._unsloth_stock_forward = encoder.forward+ encoder.forward = types.MethodType(+ _guarded(_fast_encoder_forward, _stock_encoder_forward, "encoder"), encoder+ )+ return True+++def _stock_encoder_forward(self: Any, hidden_states: Any) -> Any:+ """Diffusers' encoder forward, under float16 autocast when the weights were cast."""+ import torch++ if getattr(self, "_unsloth_compute_dtype", None) is not torch.float16:+ return self._unsloth_stock_forward(hidden_states)+ out_dtype = getattr(self, "_unsloth_out_dtype", None) or hidden_states.dtype+ if hidden_states.is_cuda:+ with torch.autocast("cuda", dtype = torch.float16):+ return self._unsloth_stock_forward(hidden_states).to(out_dtype)+ return self._unsloth_stock_forward(hidden_states.to(torch.float16)).to(out_dtype)+++def _add_rmsnorm(h: Any, o: Any, scale: Any, norm: Any, out_dtype: Any) -> Any:+ """``h += o * scale`` (skipped when ``o`` is None), then ``rmsnorm(h) * norm.weight`` in ``out_dtype``."""+ import torch++ k = _kernels()+ n_cols = h.shape[-1]+ eps = norm.eps if norm.eps is not None else torch.finfo(torch.float32).eps+ if k is None or not h.is_cuda:+ if o is not None:+ h.add_(o * scale)+ return torch.nn.functional.rms_norm(h.float(), (n_cols,), norm.weight.float(), eps).to(+ out_dtype+ )+ h2 = h.view(-1, n_cols)+ normed = torch.empty(h2.shape, dtype = out_dtype, device = h.device)+ k.add_rmsnorm[(h2.shape[0],)](+ h2,+ o if o is not None else h2,+ scale if scale is not None else norm.weight,+ norm.weight,+ normed,+ n_cols,+ float(eps),+ HAS_RES = o is not None,+ BLOCK_N = _next_pow2(n_cols),+ num_warps = 8,+ )+ return normed.view(*h.shape[:-1], n_cols)+++def qk_norm_rope_reference(x: Any, cos: Any, sin: Any, heads: int, eps: float) -> Any:+ """The reference processor's per-head float32 RMSNorm then partial split-half rope in ``x.dtype``."""+ import torch++ shape = x.shape+ q = x.reshape(-1, heads, shape[-1] // heads)+ q = torch.nn.functional.rms_norm(q.float(), (q.shape[-1],), None, eps).to(x.dtype)+ rot = cos.shape[-1]+ c = cos.reshape(-1, 1, rot).to(x.dtype)+ s = sin.reshape(-1, 1, rot).to(x.dtype)+ q_rot, q_pass = q[..., :rot], q[..., rot:]+ first, second = q_rot.chunk(2, dim = -1)+ rotated = torch.cat([-second, first], dim = -1)+ return torch.cat([q_rot * c + rotated * s, q_pass], dim = -1).reshape(shape)+++def _qk_norm_rope_(x: Any, cos: Any, sin: Any, heads: int, eps: float) -> None:+ k = _kernels()+ if k is None or not x.is_cuda or not x.is_contiguous():+ x.copy_(qk_norm_rope_reference(x, cos, sin, heads, eps))+ return+ head_d = x.shape[-1] // heads+ n = x.numel() // head_d+ block = 32+ k.qk_norm_rope[((n + block - 1) // block,)](+ x,+ cos,+ sin,+ n,+ heads,+ cos.stride(-2),+ float(eps),+ HEAD_D = head_d,+ ROT = cos.shape[-1],+ BLOCK = block,+ num_warps = 4,+ )+++def _swiglu(x: Any) -> Any:+ import torch++ k = _kernels()+ n = x.shape[-1] // 2+ if k is None or not x.is_cuda:+ hidden, gate = x.chunk(2, dim = -1)+ return hidden * torch.nn.functional.silu(gate)+ rows = x.numel() // x.shape[-1]+ out = torch.empty((*x.shape[:-1], n), dtype = x.dtype, device = x.device)+ block_n = 2048+ k.swiglu[(rows, (n + block_n - 1) // block_n)](x, out, n, BLOCK_N = block_n, num_warps = 4)+ return out+++def _fast_block_stack(decoder: Any, hidden_states: Any, cos: Any, sin: Any, gemm_dtype: Any) -> Any:+ """The transformer blocks; the kernels store normed activations in ``gemm_dtype``."""+ import torch.nn.functional as F++ blocks = decoder.transformer_blocks+ batch, seq, dim = hidden_states.shape+ h = hidden_states.contiguous()+ normed = _add_rmsnorm(h, None, None, blocks[0].norm1, gemm_dtype)+ cos2 = cos.reshape(-1, cos.shape[-1])+ sin2 = sin.reshape(-1, sin.shape[-1])+ for i, block in enumerate(blocks):+ attn = block.attn+ query = attn.to_q(normed)+ key = attn.to_k(normed)+ value = attn.to_v(normed)+ _qk_norm_rope_(query, cos2, sin2, attn.heads, attn.norm_q.eps)+ _qk_norm_rope_(key, cos2, sin2, attn.heads, attn.norm_k.eps)+ query, key, value = (+ t.view(batch, seq, attn.heads, -1).permute(0, 2, 1, 3) for t in (query, key, value)+ )+ out = F.scaled_dot_product_attention(query, key, value)+ out = out.permute(0, 2, 1, 3).flatten(2, 3)+ out = attn.to_out[0](out)+ normed = _add_rmsnorm(h, out, block.scale1, block.norm2, gemm_dtype)+ ff = block.ff.net+ proj = ff[0].proj(normed)+ out = ff[2](_swiglu(proj))+ if i + 1 < len(blocks):+ normed = _add_rmsnorm(h, out, block.scale2, blocks[i + 1].norm1, gemm_dtype)+ else:+ h.addcmul_(out, block.scale2)+ return h+++def _fast_decoder_forward(self: Any, hidden_states: Any) -> Any:+ """``MiniMaxH3VideoViTDecoder3d.forward`` with the block stack over the fused kernels."""+ import torch+ with torch.cuda.device(hidden_states.device) if hidden_states.is_cuda else nullcontext():+ return _fast_decoder_body(self, hidden_states)+++def _fast_decoder_body(self: Any, hidden_states: Any) -> Any:+ import torch++ batch_size, num_channels, num_frames, height, width = hidden_states.shape+ hidden_states = hidden_states.permute(0, 2, 3, 4, 1).reshape(+ batch_size, num_frames * height * width, num_channels+ )+ hidden_states = self.proj_in(hidden_states)+ num_patches = hidden_states.shape[1]+ register_tokens = self.register_tokens.expand(batch_size, -1, -1)+ cls_token = torch.zeros_like(hidden_states[:, :1, :])+ hidden_states = torch.cat([hidden_states, register_tokens, cls_token], dim = 1)+ grids = [+ 2.0 * (torch.arange(0.5, size, dtype = torch.float32, device = hidden_states.device) / size)+ - 1.0+ for size in (num_frames, height, width)+ ]+ position_ids = torch.stack(torch.meshgrid(*grids, indexing = "ij"), dim = -1).flatten(0, 2)+ position_ids = position_ids.unsqueeze(0).expand(batch_size, -1, -1)+ suffix_ids = position_ids.new_zeros((batch_size, self.num_register_tokens + 1, 3))+ position_ids = torch.cat([position_ids, suffix_ids], dim = 1)+ cos, sin = self.rope(position_ids)+ # the reference casts the tables to the query dtype, which is the GEMM (autocast) dtype+ q_dtype = (+ torch.get_autocast_dtype("cuda")+ if torch.is_autocast_enabled("cuda")+ else hidden_states.dtype+ )+ cos = cos.to(q_dtype).contiguous()+ sin = sin.to(q_dtype).contiguous()+ hidden_states = _fast_block_stack(self, hidden_states, cos, sin, q_dtype)+ hidden_states = self.norm_out(hidden_states)+ hidden_states = self.proj_out(hidden_states)+ hidden_states = hidden_states[:, :num_patches, :]+ p, pt = self.patch_size, self.patch_size_t+ hidden_states = hidden_states.view(+ batch_size, num_frames, height, width, self.out_channels, pt, p, p+ )+ hidden_states = hidden_states.permute(0, 4, 1, 5, 2, 6, 3, 7).contiguous()+ return hidden_states.reshape(+ batch_size, self.out_channels, num_frames * pt, height * p, width * p+ )+++def _decoder_fusable(decoder: Any) -> bool:+ try:+ blocks = decoder.transformer_blocks+ attn = blocks[0].attn+ return (+ hasattr(attn, "to_q")+ and hasattr(attn, "to_k")+ and hasattr(attn, "to_v")+ and hasattr(blocks[0].ff, "net")+ and hasattr(blocks[0].ff.net[0], "proj")+ and hasattr(decoder, "rope")+ and hasattr(decoder, "register_tokens")+ and attn.dim_head & (attn.dim_head - 1) == 0+ )+ except Exception: # noqa: BLE001+ return False+++def _install_decoder(vae: Any) -> bool:+ decoder = getattr(vae, "decoder", None)+ if decoder is None or not _decoder_fusable(decoder):+ return False+ decoder._unsloth_stock_forward = decoder.forward+ decoder.forward = types.MethodType(+ _guarded(_fast_decoder_forward, lambda self, x: self._unsloth_stock_forward(x), "decoder"),+ decoder,+ )+ return True+++def _tile_batch_size(z: Any) -> int:+ import torch++ env = _env(H3_VAE_TILE_BATCH_ENV)+ if env.isdigit() and int(env) > 0:+ return int(env)+ try:+ free, _ = torch.cuda.mem_get_info(z.device)+ except Exception: # noqa: BLE001+ return 1+ return int(+ max(1, min(_TILE_BATCH_MAX, free // (_TILE_BATCH_BYTES_PER_TILE * max(1, z.shape[0]))))+ )+++def _batched_decode_clip(self: Any, z: Any) -> Any:+ """``_decode_clip`` with several tiles per decoder call (``_split_tiles`` makes them equal-sized)."""+ import torch++ if not self.use_tiling:+ return self.decoder(self.post_quant_conv(z))+ height = z.shape[-2] * self.spatial_compression_ratio+ width = z.shape[-1] * self.spatial_compression_ratio+ y_indices, y_lengths, y_overlaps = self._split_tiles(+ height, self.tile_sample_min_height, self.tile_sample_min_overlap_height+ )+ x_indices, x_lengths, x_overlaps = self._split_tiles(+ width, self.tile_sample_min_width, self.tile_sample_min_overlap_width+ )+ ratio = self.spatial_compression_ratio+ tiles = [+ z[+ ...,+ i_pos // ratio : i_pos // ratio + i_len // ratio,+ j_pos // ratio : j_pos // ratio + j_len // ratio,+ ]+ for i_pos, i_len in zip(y_indices, y_lengths)+ for j_pos, j_len in zip(x_indices, x_lengths)+ ]+ per = z.shape[0]+ batch = _tile_batch_size(z)+ decoded: list = []+ start = 0+ while start < len(tiles):+ group = tiles[start : start + batch]+ try:+ out = self.decoder(+ self.post_quant_conv(torch.cat(group) if len(group) > 1 else group[0])+ )+ except Exception as exc: # noqa: BLE001+ # batching is what raised the peak, so an OOM with several tiles retries them one at a time+ if len(group) == 1 or not _is_oom(exc):+ raise+ torch.cuda.empty_cache()+ batch = 1+ continue+ decoded.extend(out.split(per) if len(group) > 1 else [out])+ start += len(group)+ cols = len(x_indices)+ rows = [decoded[r * cols : (r + 1) * cols] for r in range(len(y_indices))]+ return self._stitch_tiles(rows, y_overlaps, x_overlaps)+++def _install_tile_batch(vae: Any) -> bool:+ if not all(+ hasattr(vae, n)+ for n in ("_decode_clip", "_split_tiles", "_stitch_tiles", "post_quant_conv")+ ):+ return False+ vae._decode_clip = types.MethodType(_batched_decode_clip, vae)+ return True+++def _int8_linear(linear: Any, x: Any) -> Any:+ """W8A8: per-token int8 activations, per-output-channel int8 weights, int32 accumulate via ``torch._int_mm``."""+ import torch++ if x.is_cuda and x.device.index is not None and x.device.index != torch.cuda.current_device():+ with torch.cuda.device(x.device):+ return _int8_linear(linear, x)+ k = _kernels()+ wq, ws = linear._unsloth_int8+ out_dtype = torch.get_autocast_dtype("cuda") if torch.is_autocast_enabled("cuda") else x.dtype+ rot = getattr(linear, "_unsloth_rot", None)+ if rot is not None:+ # ConvRot: weight stored as W @ H.T, so x @ H keeps the product while spreading outlier channels+ g = rot.shape[0]+ x = (x.reshape(-1, x.shape[-1] // g, g).to(out_dtype) @ rot.to(out_dtype)).reshape(x.shape)+ x2 = x.reshape(-1, x.shape[-1])+ if not x2.is_contiguous():+ x2 = x2.contiguous()+ rows, n_in = x2.shape+ if rows <= 16:+ w = (wq.float() * ws[:, None]).to(out_dtype)+ return torch.nn.functional.linear(x2.to(out_dtype), w, linear.bias).view(*x.shape[:-1], -1)+ if k is None or not x.is_cuda:+ xf = x2.float()+ xs = xf.abs().amax(dim = 1).clamp(min = 1e-12) / 127.0+ xq = (xf / xs[:, None]).round().clamp(-127, 127).to(torch.int8)+ acc = torch._int_mm(xq, wq.t()).float() * xs[:, None] * ws[None, :]+ if linear.bias is not None:+ acc = acc + linear.bias.float()+ return acc.to(out_dtype).view(*x.shape[:-1], -1)+ xq = torch.empty((rows, n_in), dtype = torch.int8, device = x.device)+ xs = torch.empty(rows, dtype = torch.float32, device = x.device)+ k.quant_rows[(rows,)](x2, xq, xs, n_in, BLOCK_N = _next_pow2(n_in), num_warps = 8)+ acc = torch._int_mm(xq, wq.t())+ n_out = acc.shape[1]+ out = torch.empty((rows, n_out), dtype = out_dtype, device = x.device)+ block_n = 1024+ bias = linear.bias+ k.dequant_epilogue[(rows, (n_out + block_n - 1) // block_n)](+ acc,+ xs,+ ws,+ bias if bias is not None else ws,+ out,+ n_out,+ HAS_BIAS = bias is not None,+ BLOCK_N = block_n,+ num_warps = 4,+ )+ return out.view(*x.shape[:-1], n_out)+++def _quantize_int8_weight(w: Any, hadamard: Any) -> tuple:+ """Per-output-channel int8 of the float32 ``w``, ConvRot-rotated first when ``hadamard`` is given."""+ import torch++ if hadamard is not None:+ group = hadamard.shape[0]+ w = (w.reshape(w.shape[0], -1, group) @ hadamard.T).reshape(w.shape)+ scale = w.abs().amax(dim = 1).clamp(min = 1e-12) / 127.0+ wq = (w / scale[:, None]).round_().clamp_(-127, 127).to(torch.int8)+ return wq, scale.to(torch.float32).view(torch.int32)+++def _install_int8_decoder(vae: Any, *, keep_blocks: Optional[tuple] = None) -> int:+ """Quantize decoder block Linears to ConvRot W8A8, leaving ``keep_blocks`` float. Every weight is quantized+ before any module is touched, so a failure leaves the decoder fully float. Returns the bytes freed."""+ import torch++ decoder = getattr(vae, "decoder", None)+ if decoder is None:+ return 0+ from .diffusion_convrot import build_convrot_hadamard++ group = H3_VAE_INT8_ROT_GROUP+ hadamard = rot16 = None+ staged = []+ with torch.no_grad():+ if keep_blocks is None:+ keep_blocks = tuple(range(H3_VAE_INT8_FLOAT_BLOCKS))+ for index, block in enumerate(decoder.transformer_blocks):+ if index in keep_blocks:+ continue+ for linear in block.modules():+ if (+ not isinstance(linear, torch.nn.Linear)+ or getattr(linear, "_unsloth_int8", None) is not None+ ):+ continue+ if linear.in_features % 8 or linear.out_features % 8:+ continue+ w = linear.weight.data.float()+ rotate = linear.in_features % group == 0+ if rotate and hadamard is None:+ hadamard = build_convrot_hadamard(group, device = w.device, dtype = torch.float32)+ # exact in float16: the entries are +-2^-k+ rot16 = hadamard.to(torch.float16)+ wq, ws = _quantize_int8_weight(w, hadamard if rotate else None)+ staged.append((linear, wq, ws, rotate))+ cls = _int8_linear_class()+ freed = 0+ for linear, wq, ws, rotate in staged:+ if rotate:+ linear.register_buffer("_unsloth_rot", rot16, persistent = False)+ freed += linear.weight.numel() * linear.weight.element_size() - wq.numel()+ linear.register_buffer("_unsloth_wq", wq)+ linear.register_buffer("_unsloth_ws", ws)+ del linear.weight+ linear.weight = None+ linear.__class__ = cls+ return freed+++@lru_cache(maxsize = 1)+def _int8_linear_class():+ import torch+ class H3Int8Linear(torch.nn.Linear):+ """int8 Linear; fp32 scales stored as int32 so an offload hook's ``.to(dtype)`` cannot round them."""++ @property+ def _unsloth_int8(self):+ return self._unsloth_wq, self._unsloth_ws.view(torch.float32)++ def forward(self, x):+ return _int8_linear(self, x)++ return H3Int8Linear+++def _install_decode_scope(vae: Any, *, fp16_accum: bool) -> bool:+ """Hold ``allow_fp16_accumulation`` ON only when planned, else OFF, for the decode, via the shared owner."""+ import torch++ from .diffusion_speed import fp16_accumulation_scope++ if not hasattr(torch.backends.cuda.matmul, "allow_fp16_accumulation"):+ return False+ stock = vae.decode++ def decode(self, *args, **kwargs):+ with fp16_accumulation_scope(bool(fp16_accum)):+ return stock(*args, **kwargs)++ vae.decode = types.MethodType(decode, vae)+ return True+++@lru_cache(maxsize = 4)+def _triton_version_ok(version: Optional[str] = None) -> bool:+ if version is None:+ try:+ import triton+ version = str(triton.__version__)+ except Exception: # noqa: BLE001+ return False+ import re++ match = re.match(r"(\d+)\.(\d+)", version)+ return bool(match) and (int(match.group(1)), int(match.group(2))) >= H3_VAE_MIN_TRITON+++@lru_cache(maxsize = 1)+def _triton_jit_toolchain_ok() -> bool:+ """Windows: Triton needs the MSVC CRT headers. Elsewhere, or if the probe fails, True."""+ if sys.platform != "win32":+ return True+ try:+ from .._msvc_env import crt_headers_reachable+ return bool(crt_headers_reachable())+ except Exception: # noqa: BLE001+ return True+++def cuda_fast_path_available(vae: Any = None) -> bool:+ try:+ import torch+ except Exception: # noqa: BLE001+ return False+ if not torch.cuda.is_available() or getattr(torch.version, "hip", None):+ return False+ if vae is not None:+ try:+ if type(vae).__name__ != "AutoencoderKLMiniMaxH3":+ return False+ except Exception: # noqa: BLE001+ return False+ return _triton_version_ok() and _kernels() is not None and _triton_jit_toolchain_ok()+++def apply_h3_vae_speedups(+ vae: Any,+ *,+ speed_mode: Optional[str],+ workflow: Optional[str] = None,+ consumer_gpu: Optional[bool] = None,+ logger: Any = None,+) -> tuple[str, ...]:+ """Engage the planned levers; never raises (a failing lever keeps stock). Returns engaged lever names."""+ if vae is None:+ return ()+ done = getattr(vae, "_unsloth_h3_vae_levers", None)+ if done is not None:+ return tuple(done)+ if consumer_gpu is None:+ try:+ from .diffusion_transformer_quant import _is_consumer_gpu+ consumer_gpu = bool(_is_consumer_gpu())+ except Exception: # noqa: BLE001+ consumer_gpu = False+ planned = plan_h3_vae_levers(speed_mode, workflow = workflow, consumer_gpu = consumer_gpu)+ engaged: list[str] = []+ if not planned or not cuda_fast_path_available(vae):+ try:+ vae._unsloth_h3_vae_levers = ()+ except Exception: # noqa: BLE001+ pass+ return ()++ def _try(name: str, fn) -> None:+ try:+ if fn():+ engaged.append(name)+ except Exception as exc: # noqa: BLE001 - optimisation only+ if logger is not None:+ logger.warning(+ "video.h3_vae_fast: %s failed, keeping the stock path: %s", name, exc+ )++ for holder in (vae, getattr(vae, "encoder", None), getattr(vae, "decoder", None)):+ if holder is not None:+ try:+ holder._unsloth_logger = logger+ except Exception: # noqa: BLE001+ pass+ if LEVER_FUSED_ENCODER in planned:+ _try(LEVER_FUSED_ENCODER, lambda: _install_encoder(vae, fp16 = LEVER_FP16_ENCODER in planned))+ if LEVER_FUSED_ENCODER in engaged and LEVER_FP16_ENCODER in planned:+ engaged.append(LEVER_FP16_ENCODER)+ if LEVER_INT8_DECODER in planned:+ _try(LEVER_INT8_DECODER, lambda: _install_int8_decoder(vae) > 0)+ if LEVER_FUSED_DECODER in planned:+ _try(LEVER_FUSED_DECODER, lambda: _install_decoder(vae))+ if LEVER_TILE_BATCH in planned:+ _try(LEVER_TILE_BATCH, lambda: _install_tile_batch(vae))+ fp16_accum = LEVER_FP16_ACCUM in planned+ # always installed: also holds the flag OFF when float16 accumulation is not planned+ try:+ if _install_decode_scope(vae, fp16_accum = fp16_accum) and fp16_accum:+ engaged.append(LEVER_FP16_ACCUM)+ except Exception as exc: # noqa: BLE001 - optimisation only+ if logger is not None:+ logger.warning(+ "video.h3_vae_fast: decode_scope failed, keeping the stock decode: %s", exc+ )+ vae._unsloth_h3_vae_levers = tuple(engaged)+ if logger is not None:+ logger.info("video.h3_vae_fast: engaged %s", ", ".join(engaged) or "nothing")+ return tuple(engaged)+++def settle_h3_vae_fallback(state: Any, pipe: Any) -> None:+ """Drop fused levers the runtime fell back from out of ``state.speed_optims``. Never raises."""+ try:+ vae = getattr(pipe, "vae", None)+ failed = [+ f"h3_vae_{lever}"+ for lever, part in ((LEVER_FUSED_ENCODER, "encoder"), (LEVER_FUSED_DECODER, "decoder"))+ if getattr(getattr(vae, part, None), "_unsloth_fast_failed", False)+ ]+ optims = tuple(getattr(state, "speed_optims", None) or ())+ updated = tuple(o for o in optims if o not in failed)+ if updated != optims:+ # the load states are frozen dataclasses+ object.__setattr__(state, "speed_optims", updated)+ except Exception: # noqa: BLE001 - a status fix-up must never fail a render+ passdiff --git a/studio/backend/tests/test_video_minimax_h3_vae.py b/studio/backend/tests/test_video_minimax_h3_vae.pynew file mode 100644index 00000000000..d3648f1b9c2--- /dev/null+++ b/studio/backend/tests/test_video_minimax_h3_vae.py@@ -0,0 +1,970 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""Tests for the MiniMax-H3 video VAE speed layer; CPU tests hermetic, CUDA tests skip without a GPU."""++from __future__ import annotations++import copy+import types++import pytest+import torch++from core.inference import video_minimax_h3_vae as H++CUDA = torch.cuda.is_available() and not getattr(torch.version, "hip", None)+needs_cuda = pytest.mark.skipif(+ not CUDA or H._kernels() is None or not H._triton_version_ok(),+ reason = "needs an NVIDIA GPU with a Triton the kernels are verified on",+)++[email protected](autouse = True)+def _clean_env(monkeypatch):+ for name in (+ H.H3_VAE_FAST_ENV,+ H.H3_VAE_INT8_ENV,+ H.H3_VAE_TILE_BATCH_ENV,+ H.FP16_ACCUM_DISABLE_ENV,+ ):+ monkeypatch.delenv(name, raising = False)+++def test_off_tier_engages_nothing():+ assert H.plan_h3_vae_levers("off", workflow = "fl2va") == ()+ assert H.plan_h3_vae_levers("OFF", workflow = "fl2va", consumer_gpu = True) == ()++[email protected]("tier", ["default", "max", None, ""])+def test_default_and_max_get_the_near_lossless_set(tier):+ plan = H.plan_h3_vae_levers(tier, workflow = "fl2va", consumer_gpu = False)+ assert plan == (+ H.LEVER_FUSED_ENCODER,+ H.LEVER_FP16_ENCODER,+ H.LEVER_FUSED_DECODER,+ H.LEVER_TILE_BATCH,+ )+++def test_eager_stays_rounding_level():+ plan = H.plan_h3_vae_levers("eager", workflow = "fl2va", consumer_gpu = True)+ assert plan == (H.LEVER_FUSED_ENCODER, H.LEVER_FUSED_DECODER, H.LEVER_TILE_BATCH)+++def test_fp16_accumulation_only_under_max_on_consumer_gpus(monkeypatch):+ assert H.LEVER_FP16_ACCUM in H.plan_h3_vae_levers("max", workflow = "fl2va", consumer_gpu = True)+ assert H.LEVER_FP16_ACCUM not in H.plan_h3_vae_levers(+ "max", workflow = "fl2va", consumer_gpu = False+ )+ for tier in ("eager", "default"):+ assert H.LEVER_FP16_ACCUM not in H.plan_h3_vae_levers(+ tier, workflow = "fl2va", consumer_gpu = True+ )+ monkeypatch.setenv(H.FP16_ACCUM_DISABLE_ENV, "1")+ assert H.LEVER_FP16_ACCUM not in H.plan_h3_vae_levers(+ "max", workflow = "fl2va", consumer_gpu = True+ )+++def test_int8_decoder_is_opt_in_only(monkeypatch):+ for tier in ("eager", "default", "max"):+ assert H.LEVER_INT8_DECODER not in H.plan_h3_vae_levers(+ tier, workflow = "fl2va", consumer_gpu = True+ )+ monkeypatch.setenv(H.H3_VAE_INT8_ENV, "1")+ assert H.LEVER_INT8_DECODER in H.plan_h3_vae_levers("default", workflow = "fl2va")+++def test_t2va_skips_the_encoder_levers():+ plan = H.plan_h3_vae_levers("default", workflow = "t2va")+ assert H.LEVER_FUSED_ENCODER not in plan and H.LEVER_FP16_ENCODER not in plan+ assert H.LEVER_FUSED_DECODER in plan+++def test_env_switches(monkeypatch):+ monkeypatch.setenv(H.H3_VAE_FAST_ENV, "0")+ assert H.plan_h3_vae_levers("max", workflow = "fl2va", consumer_gpu = True) == ()+ monkeypatch.delenv(H.H3_VAE_FAST_ENV)+ monkeypatch.setenv(H.H3_VAE_INT8_ENV, "1")+ assert H.plan_h3_vae_levers("off", workflow = "fl2va") == ()+++def test_apply_is_a_no_op_off_cuda(monkeypatch):+ monkeypatch.setattr(H, "cuda_fast_path_available", lambda vae = None: False)+ vae = types.SimpleNamespace(decode = object(), encoder = object())+ stock_decode = vae.decode+ assert H.apply_h3_vae_speedups(vae, speed_mode = "max", workflow = "fl2va", consumer_gpu = True) == ()+ assert vae.decode is stock_decode+ assert H.apply_h3_vae_speedups(None, speed_mode = "max") == ()++[email protected](CUDA, reason = "checks the real gate on a host without an NVIDIA GPU")[email protected]("tier", ["eager", "default", "max"])+def test_without_cuda_the_real_gate_leaves_the_stock_vae_untouched(tier):+ vae = _tiny_vae()+ encoder_forward, decoder_forward, decode = (+ vae.encoder.forward,+ vae.decoder.forward,+ vae.decode,+ )+ weights = {k: v.clone() for k, v in vae.state_dict().items()}+ assert H.plan_h3_vae_levers(tier, workflow = "fl2va")+ assert H.apply_h3_vae_speedups(vae, speed_mode = tier, workflow = "fl2va") == ()+ assert vae.encoder.forward == encoder_forward and vae.decoder.forward == decoder_forward+ assert vae.decode == decode+ for k, v in vae.state_dict().items():+ assert v.dtype == weights[k].dtype and torch.equal(v, weights[k]), k+++def test_decode_scope_pins_and_restores_fp16_accumulation():+ matmul = torch.backends.cuda.matmul+ if not hasattr(matmul, "allow_fp16_accumulation"):+ pytest.skip("torch without allow_fp16_accumulation")+ seen = []++ class _VAE:+ def decode(self, z):+ seen.append(matmul.allow_fp16_accumulation)+ return z++ prev = matmul.allow_fp16_accumulation+ try:+ for flag_before, planned in ((True, False), (False, True), (False, False)):+ vae = _VAE()+ matmul.allow_fp16_accumulation = flag_before+ assert H._install_decode_scope(vae, fp16_accum = planned)+ assert vae.decode(3) == 3+ assert (+ seen[-1] is planned+ ), "the decode must run under the planned flag, not the process-wide one"+ assert (+ matmul.allow_fp16_accumulation is flag_before+ ), "the process-wide flag must be restored"+ finally:+ matmul.allow_fp16_accumulation = prev+++def _tiny_vae():+ diffusers = pytest.importorskip("diffusers")+ cls = getattr(diffusers, "AutoencoderKLMiniMaxH3", None)+ if cls is None:+ pytest.skip("diffusers without AutoencoderKLMiniMaxH3")+ torch.manual_seed(0)+ vae = cls(+ block_out_channels = (16, 32),+ layers_per_block = 2,+ spatial_downsample_factors = (2, 2),+ temporal_downsample_factors = (2, 1),+ norm_num_groups = 8,+ latent_channels = 8,+ decoder_num_layers = 2,+ decoder_num_attention_heads = 2,+ decoder_attention_head_dim = 16,+ decoder_num_register_tokens = 2,+ decoder_ffn_mult = 2,+ decoder_rope_dim_ratio = 0.75,+ clip_length = 9,+ token_drop = 1,+ ).eval()+ with torch.no_grad():+ for name, p in vae.named_parameters():+ if name.endswith("scale1") or name.endswith("scale2"):+ p.normal_(0, 0.5)+ elif "norm" in name and name.endswith("weight"):+ p.normal_(1.0, 0.2)+ elif name.endswith("bias"):+ p.normal_(0, 0.1)+ vae.tile_sample_min_height = vae.tile_sample_min_width = 16+ vae.tile_sample_min_overlap_height = vae.tile_sample_min_overlap_width = 4+ return vae+++DEVICES = ["cpu", pytest.param("cuda", marks = needs_cuda)]++[email protected]("device", DEVICES)[email protected]("frames", [1, 9, 18])+def test_fused_encoder_matches_the_stock_encoder(frames, device, monkeypatch):+ monkeypatch.setattr(torch.backends.cudnn, "allow_tf32", False)+ vae = _tiny_vae().to(device)+ fast = copy.deepcopy(vae)+ assert H._install_encoder(fast, fp16 = False)+ x = torch.randn(1, 3, frames, 24, 40, device = device)+ with torch.no_grad():+ ref = vae.encoder(x)+ got = fast.encoder(x)+ assert not getattr(+ fast.encoder, "_unsloth_fast_failed", False+ ), "the fused path fell back to stock"+ assert got.shape == ref.shape and got.dtype == ref.dtype+ torch.testing.assert_close(got, ref, rtol = 1e-4, atol = 1e-4)++[email protected]("device", DEVICES)+def test_fused_fp16_encoder_returns_the_input_dtype_and_stays_close(device):+ vae = _tiny_vae().to(device)+ ref = copy.deepcopy(vae)+ assert H._install_encoder(vae, fp16 = True)+ x = torch.randn(1, 3, 9, 24, 40, device = device)+ with torch.no_grad():+ got, want = vae.encoder(x), ref.encoder(x)+ assert not getattr(vae.encoder, "_unsloth_fast_failed", False)+ assert got.dtype is torch.float32+ assert ((got - want).norm() / want.norm()) < 1e-2+++def test_fused_encoder_fp16_casts_weights_and_returns_the_input_dtype():+ vae = _tiny_vae()+ assert H._install_encoder(vae, fp16 = True)+ convs = [m for m in vae.encoder.modules() if isinstance(m, torch.nn.Conv3d)]+ assert convs and all(m.weight.dtype is torch.float16 for m in convs)+ assert all(+ m.weight.is_contiguous(memory_format = torch.channels_last_3d)+ for m in convs+ if m is not vae.encoder.conv_in+ )+ assert vae.quant_conv.weight.dtype is torch.float32++[email protected]("device", DEVICES)[email protected]("frames", [1, 9])+def test_the_condition_encode_recipe_runs_through_the_fp16_encoder(frames, device):+ # keyframe path: vae.encode feeds the encoder output to a float32 quant_conv, so it must be float32+ from core.inference.video_minimax_h3 import trim_h3_video_vae++ vae = _tiny_vae().to(device)+ vae.register_to_config(latents_mean = [0.0] * 8, latents_std = [1.0] * 8)+ trim_h3_video_vae(vae, workflow = "fl2va")+ ref = copy.deepcopy(vae)+ assert H._install_encoder(vae, fp16 = True)+ pixels = torch.randint(0, 256, (1, 3, frames, 32, 48), dtype = torch.uint8, device = device)+ with torch.no_grad():+ try:+ from diffusers.modular_pipelines.minimax_h3.encoders import encode_vae_condition+ except ImportError:+ x = pixels.float() / 127.5 - 1.0+ got, want = (m.encode(x).latent_dist.mean for m in (vae, ref))+ else:+ args = (pixels, (0.485, 0.456, 0.406), (0.229, 0.224, 0.225))+ got, want = encode_vae_condition(vae, *args), encode_vae_condition(ref, *args)+ assert not getattr(vae.encoder, "_unsloth_fast_failed", False)+ assert vae.quant_conv.weight.dtype is torch.float32+ assert got.dtype is torch.float32 and got.shape == want.shape+ assert ((got - want).norm() / want.norm()) < 1e-2+++@needs_cuda+def test_the_decoder_residual_stream_stays_float32_under_the_decode_autocast(monkeypatch):+ # the stock blocks keep the residual in float32 under float16 autocast, so must we+ from core.inference.video_minimax_h3 import trim_h3_video_vae++ vae = _tiny_vae().cuda()+ trim_h3_video_vae(vae, workflow = "fl2va")+ ref = copy.deepcopy(vae)+ assert H._install_decoder(vae)+ seen = []+ stack = H._fast_block_stack++ def spy(decoder, hidden_states, *args):+ seen.append(hidden_states.dtype)+ out = stack(decoder, hidden_states, *args)+ seen.append(out.dtype)+ return out++ monkeypatch.setattr(H, "_fast_block_stack", spy)+ z = torch.randn(1, 8, 3, 4, 5, device = "cuda")+ with torch.no_grad(), torch.autocast("cuda", dtype = torch.float16):+ got, want = vae.decoder(z), ref.decoder(z)+ assert not getattr(vae.decoder, "_unsloth_fast_failed", False)+ assert seen == [torch.float32, torch.float32]+ assert got.dtype == want.dtype+ torch.testing.assert_close(got.float(), want.float(), rtol = 2e-3, atol = 2e-3)+++def test_norm_silu_pad_reference_matches_diffusers_ops():+ from torch.nn import functional as F++ vae = _tiny_vae()+ resnet = vae.encoder.down_blocks[0].resnets[0]+ x = torch.randn(1, 16, 5, 12, 10)+ ref = F.pad(F.silu(resnet.norm1(x)), (1, 1, 1, 1, 0, 0), mode = "reflect")+ ref = F.pad(ref, (0, 0, 0, 0, 2, 0))+ got = H.norm_silu_pad_reference(x, resnet.norm1, (1, 1, 1, 1), 2)+ assert got.is_contiguous(memory_format = torch.channels_last_3d)+ torch.testing.assert_close(got, ref, rtol = 1e-5, atol = 1e-5)+++def test_single_frame_last_tap_equals_convolving_the_zero_frames():+ from torch.nn import functional as F++ vae = _tiny_vae()+ conv = vae.encoder.down_blocks[0].resnets[0].conv1+ x = torch.randn(1, 16, 1, 8, 8)+ padded = F.pad(F.pad(x, (1, 1, 1, 1, 0, 0), mode = "reflect"), (0, 0, 0, 0, 2, 0))+ ref = F.conv3d(padded, conv.weight, conv.bias)+ got, bias = H._causal_conv(conv, x)+ torch.testing.assert_close(got + bias.view(1, -1, 1, 1, 1), ref, rtol = 1e-5, atol = 1e-5)+++def test_pending_bias_folds_through_a_1x1_conv():+ from torch.nn import functional as F++ conv = torch.nn.Conv3d(16, 32, 1)+ x = torch.randn(1, 16, 3, 5, 7)+ pending = torch.randn(16)+ ref = F.conv3d(x + pending.view(1, -1, 1, 1, 1), conv.weight, conv.bias)+ got, bias = H._causal_conv(conv, x, pending)+ torch.testing.assert_close(got + bias.view(1, -1, 1, 1, 1), ref, rtol = 1e-5, atol = 1e-5)+++def test_pending_bias_is_added_before_the_norm_and_the_pad():+ from torch.nn import functional as F++ norm = torch.nn.GroupNorm(4, 16, eps = 1e-6)+ x = torch.randn(1, 16, 4, 6, 6)+ pending = torch.randn(16)+ y = x + pending.view(1, -1, 1, 1, 1)+ frames = y.permute(0, 2, 1, 3, 4).reshape(4, 16, 6, 6)+ ref = (+ F.silu(F.group_norm(frames, 4, norm.weight, norm.bias, 1e-6))+ .reshape(1, 4, 16, 6, 6)+ .permute(0, 2, 1, 3, 4)+ )+ ref = F.pad(F.pad(ref, (1, 1, 1, 1, 0, 0), mode = "reflect"), (0, 0, 0, 0, 2, 0))+ torch.testing.assert_close(+ H.norm_silu_pad(x, norm, (1, 1, 1, 1), 2, in_bias = pending), ref, rtol = 1e-5, atol = 1e-5+ )+ # a zero frame stays zero: the bias belongs to real frames only+ got = H.norm_silu_pad(x, None, (0, 1, 0, 1), 2, in_bias = pending)+ assert got[:, :, :2].abs().max() == 0++[email protected]("device", DEVICES)+def test_fused_decoder_matches_the_stock_decoder(device):+ vae = _tiny_vae().to(device)+ fast = copy.deepcopy(vae)+ assert H._install_decoder(fast)+ z = torch.randn(1, 8, 3, 4, 5, device = device)+ with torch.no_grad():+ ref = vae.decoder(z)+ got = fast.decoder(z)+ assert not getattr(+ fast.decoder, "_unsloth_fast_failed", False+ ), "the fused path fell back to stock"+ assert got.shape == ref.shape+ torch.testing.assert_close(got, ref, rtol = 1e-4, atol = 1e-4)+++def test_tile_batching_matches_the_stock_tile_loop(monkeypatch):+ vae = _tiny_vae()+ fast = copy.deepcopy(vae)+ assert H._install_tile_batch(fast)+ z = torch.randn(1, 8, 3, 6, 9) # 24x36 px: 2x3 tiles of 16 px+ with torch.no_grad():+ ref = vae._decode_clip(z)+ for batch in ("1", "2", "4", "8"):+ monkeypatch.setenv(H.H3_VAE_TILE_BATCH_ENV, batch)+ torch.testing.assert_close(fast._decode_clip(z), ref, rtol = 1e-5, atol = 1e-5)+++def test_int8_decoder_replaces_block_linears_and_stays_close():+ vae = _tiny_vae()+ fast = copy.deepcopy(vae)+ freed = H._install_int8_decoder(fast, keep_blocks = ())+ assert freed > 0+ lin = fast.decoder.transformer_blocks[0].attn.to_q+ assert lin.weight is None and lin._unsloth_wq.dtype is torch.int8+ assert fast.decoder.proj_out.weight is not None+ z = torch.randn(1, 8, 3, 4, 5)+ with torch.no_grad():+ ref = vae.decoder(z)+ got = fast.decoder(z)+ rel = (got - ref).norm() / ref.norm()+ assert rel < 0.05+++def test_apply_skips_when_already_applied(monkeypatch):+ monkeypatch.setattr(H, "cuda_fast_path_available", lambda vae = None: True)+ vae = _tiny_vae()+ engaged = H.apply_h3_vae_speedups(+ vae, speed_mode = "default", workflow = "fl2va", consumer_gpu = True+ )+ assert set(engaged) == {+ H.LEVER_FUSED_ENCODER,+ H.LEVER_FP16_ENCODER,+ H.LEVER_FUSED_DECODER,+ H.LEVER_TILE_BATCH,+ }+ encoder_forward = vae.encoder.forward+ assert H.apply_h3_vae_speedups(vae, speed_mode = "max", workflow = "fl2va") == engaged+ assert vae.encoder.forward is encoder_forward+++@needs_cuda[email protected]("dtype", [torch.float32, torch.float16])[email protected]("channels_last", [False, True])+ "frames,pad,front,with_norm",+ [(9, (1, 1, 1, 1), 2, True), (1, (1, 1, 1, 1), 0, True), (7, (0, 1, 0, 1), 2, False)],+)+def test_norm_silu_pad_kernel(dtype, channels_last, frames, pad, front, with_norm):+ torch.manual_seed(0)+ norm = torch.nn.GroupNorm(8, 64, eps = 1e-6).cuda()+ with torch.no_grad():+ norm.weight.normal_()+ norm.bias.normal_()+ x = (torch.randn(1, 64, frames, 20, 36, device = "cuda") * 3 + 1).to(dtype)+ if channels_last:+ x = x.contiguous(memory_format = torch.channels_last_3d)+ n = norm if with_norm else None+ ref = H.norm_silu_pad_reference(x, n, pad, front)+ got = H.norm_silu_pad(x, n, pad, front)+ assert got.shape == ref.shape and got.stride() == ref.stride()+ tol = 1e-5 if dtype is torch.float32 else 1e-2+ torch.testing.assert_close(got.float(), ref.float(), rtol = tol, atol = tol)+++@needs_cuda+def test_add_rmsnorm_kernel():+ torch.manual_seed(0)+ norm = torch.nn.RMSNorm(256, eps = 1e-5).cuda()+ with torch.no_grad():+ norm.weight.normal_(1, 0.1)+ h = torch.randn(300, 256, device = "cuda")+ o = torch.randn(300, 256, device = "cuda").half()+ s = torch.randn(256, device = "cuda")+ ref_h = h + o * s+ ref_n = norm(ref_h).half()+ got_n = H._add_rmsnorm(h, o, s, norm, torch.float16)+ assert torch.equal(h, ref_h)+ torch.testing.assert_close(got_n, ref_n, rtol = 2e-3, atol = 2e-3)+++@needs_cuda+def test_qk_norm_rope_kernel_matches_the_reference_processor():+ torch.manual_seed(0)+ heads, dim = 4, 64+ x = torch.randn(2, 100, heads * dim, device = "cuda").half()+ angles = torch.rand(2, 100, 1, 24, device = "cuda") * 6.28+ angles = angles.tile(2)+ cos, sin = angles.cos().half(), angles.sin().half()+ ref = H.qk_norm_rope_reference(x, cos, sin, heads, 1e-5)+ got = x.clone()+ H._qk_norm_rope_(got, cos.reshape(-1, 48), sin.reshape(-1, 48), heads, 1e-5)+ assert (got == ref).float().mean() > 0.99+ torch.testing.assert_close(got.float(), ref.float(), rtol = 2e-3, atol = 2e-3)+++@needs_cuda+def test_swiglu_kernel():+ x = torch.randn(64, 2 * 3000, device = "cuda").half()+ hidden, gate = x.chunk(2, dim = -1)+ got, ref = H._swiglu(x), hidden * torch.nn.functional.silu(gate)+ assert (got == ref).float().mean() > 0.999+ torch.testing.assert_close(got, ref, rtol = 2e-3, atol = 2e-3)+++@needs_cuda+def test_int8_linear_kernel_path_matches_the_torch_path():+ torch.manual_seed(0)+ lin = torch.nn.Linear(256, 512).cuda().half()+ ref_lin = copy.deepcopy(lin)+ holder = torch.nn.Module()+ holder.transformer_blocks = torch.nn.ModuleList([torch.nn.ModuleDict({"l": lin})])+ vae = types.SimpleNamespace(decoder = holder)+ assert H._install_int8_decoder(vae, keep_blocks = ()) > 0+ x = torch.randn(3, 40, 256, device = "cuda").half()+ got = lin(x)+ ref = ref_lin(x)+ assert got.shape == ref.shape and got.dtype == ref.dtype+ assert ((got.float() - ref.float()).norm() / ref.float().norm()) < 0.02+++@needs_cuda[email protected]("dtype", [torch.float16, torch.bfloat16, torch.float32])+def test_int8_quant_rows_kernel_matches_the_torch_codes(dtype):+ g = torch.Generator(device = "cuda").manual_seed(0)+ rows, n = 4 * 1797, 2048+ x = torch.randn(rows, n, device = "cuda", generator = g)+ x = (x * (torch.rand(n, device = "cuda", generator = g) * 4 + 0.1)).to(dtype)+ # exact halves: absmax 127 gives s = 1, where half-to-even and half-away-from-zero disagree+ x[0, :6] = torch.tensor([127.0, 2.5, -3.5, 0.5, -0.5, 126.5], dtype = dtype)+ x[0, 6:] = 0+ xf = x.float()+ ref_s = xf.abs().amax(dim = 1).clamp(min = 1e-12) / 127.0+ ref_q = (xf / ref_s[:, None]).round().clamp(-127, 127).to(torch.int8)+ q = torch.empty(rows, n, dtype = torch.int8, device = "cuda")+ s = torch.empty(rows, dtype = torch.float32, device = "cuda")+ H._kernels().quant_rows[(rows,)](x, q, s, n, BLOCK_N = n, num_warps = 8)+ assert torch.equal(s, ref_s)+ assert int((q != ref_q).sum()) == 0+ assert q[0, :6].tolist() == [127, 2, -4, 0, 0, 126]+++@needs_cuda[email protected]("res_layout", ["channels_last", "contiguous"])+def test_add_residual_kernel(res_layout):+ torch.manual_seed(0)+ out = (+ torch.randn(1, 64, 3, 10, 12, device = "cuda")+ .half()+ .contiguous(memory_format = torch.channels_last_3d)+ )+ res = torch.randn(1, 64, 3, 10, 12, device = "cuda").half()+ if res_layout == "channels_last":+ res = res.contiguous(memory_format = torch.channels_last_3d)+ ob, rb = torch.randn(64, device = "cuda").half(), torch.randn(64, device = "cuda")+ ref = (+ out.float() + ob.float().view(1, -1, 1, 1, 1) + res.float() + rb.view(1, -1, 1, 1, 1)+ ).half()+ got = H.add_residual(out.clone(memory_format = torch.channels_last_3d), ob, res, rb)+ torch.testing.assert_close(got.float(), ref.float(), rtol = 2e-3, atol = 2e-3)+++@needs_cuda+def test_norm_silu_pad_kernel_with_pending_bias():+ torch.manual_seed(0)+ norm = torch.nn.GroupNorm(32, 256, eps = 1e-6).cuda()+ x = (+ torch.randn(1, 256, 5, 17, 23, device = "cuda")+ .half()+ .contiguous(memory_format = torch.channels_last_3d)+ )+ pending = torch.randn(256, device = "cuda")+ ref = H.norm_silu_pad_reference(x, norm, (1, 1, 1, 1), 2, in_bias = pending)+ got = H.norm_silu_pad(x, norm, (1, 1, 1, 1), 2, in_bias = pending)+ torch.testing.assert_close(got.float(), ref.float(), rtol = 1e-2, atol = 1e-2)+++def test_int8_decoder_keeps_the_early_blocks_float_by_default(monkeypatch):+ monkeypatch.setattr(H, "H3_VAE_INT8_FLOAT_BLOCKS", 1)+ vae = _tiny_vae()+ blocks = vae.decoder.transformer_blocks+ H._install_int8_decoder(vae)+ assert blocks[0].attn.to_q.weight is not None+ assert blocks[1].attn.to_q.weight is None+++def test_a_failing_fused_path_falls_back_to_stock_once_and_logs():+ calls = []++ def fast(self, x):+ calls.append("fast")+ raise RuntimeError("triton: no kernel image")++ def stock(self, x):+ calls.append("stock")+ return x + 1++ warnings = []+ holder = types.SimpleNamespace(+ _unsloth_logger = types.SimpleNamespace(warning = lambda *a: warnings.append(a))+ )+ forward = H._guarded(fast, stock, "decoder")+ assert forward(holder, 1) == 2+ assert forward(holder, 2) == 3+ assert calls == [+ "fast",+ "stock",+ "stock",+ ], "after one failure the fused path must not be retried"+ assert len(warnings) == 1+++def test_an_input_the_stock_path_rejects_too_raises_without_disabling_the_fast_path():+ def fast(self, x):+ raise RuntimeError("Padding size should be less than the corresponding input dimension")++ def stock(self, x):+ raise RuntimeError("Padding size should be less than the corresponding input dimension")++ holder = types.SimpleNamespace()+ with pytest.raises(RuntimeError, match = "Padding size"):+ H._guarded(fast, stock, "encoder")(holder, 1)+ assert not getattr(holder, "_unsloth_fast_failed", False)+++def test_a_non_reflect_padding_mode_keeps_the_stock_encoder():+ vae = _tiny_vae()+ for m in vae.encoder.modules():+ if hasattr(m, "spatial_padding_mode"):+ m.spatial_padding_mode = "replicate"+ forward = vae.encoder.forward+ assert not H._install_encoder(vae, fp16 = True)+ assert vae.encoder.forward == forward+ assert all(p.dtype is torch.float32 for p in vae.encoder.parameters())+++def test_a_non_power_of_two_head_keeps_the_stock_decoder():+ vae = _tiny_vae()+ for block in vae.decoder.transformer_blocks:+ block.attn.dim_head = 12+ assert not H._install_decoder(vae)+++def test_an_oom_in_the_fused_path_is_not_masked():+ def fast(self, x):+ raise RuntimeError("CUDA out of memory. Tried to allocate 2.00 GiB")++ forward = H._guarded(fast, lambda self, x: x, "decoder")+ holder = types.SimpleNamespace()+ with pytest.raises(RuntimeError, match = "out of memory"):+ forward(holder, 1)+ assert not getattr(holder, "_unsloth_fast_failed", False)+++def test_the_fp16_encoder_falls_back_to_the_stock_encoder(monkeypatch):+ vae = _tiny_vae()+ ref_vae = copy.deepcopy(vae)+ assert H._install_encoder(vae, fp16 = True)+ x = torch.randn(1, 3, 9, 24, 40)+ with torch.no_grad():+ fused = vae.encoder(x)++ def boom(*a, **k):+ raise RuntimeError("no triton")++ # the installed guard resolves the fused body at call time, so breaking it exercises the real fallback+ monkeypatch.setattr(H, "_fast_encoder_body", boom)+ with torch.no_grad():+ stock = vae.encoder(x)+ want = ref_vae.encoder(x)+ assert vae.encoder._unsloth_fast_failed+ assert stock.dtype is torch.float32+ # the stock forward over the float16 weights: float16 arithmetic, so close to both, equal to neither+ assert ((stock - want).norm() / want.norm()) < 1e-2+ assert ((stock - fused).norm() / fused.norm()) < 1e-2+++def test_tile_batching_retries_one_tile_at_a_time_after_an_oom(monkeypatch):+ vae = _tiny_vae()+ ref = copy.deepcopy(vae)+ assert H._install_tile_batch(vae)+ monkeypatch.setenv(H.H3_VAE_TILE_BATCH_ENV, "4")+ stock_decoder = vae.decoder.forward+ seen = []++ def decoder(z):+ seen.append(z.shape[0])+ if z.shape[0] > 1:+ raise RuntimeError("CUDA out of memory")+ return stock_decoder(z)++ monkeypatch.setattr(vae.decoder, "forward", decoder)+ monkeypatch.setattr(torch.cuda, "empty_cache", lambda: None)+ z = torch.randn(1, 8, 3, 6, 9)+ with torch.no_grad():+ torch.testing.assert_close(vae._decode_clip(z), ref._decode_clip(z), rtol = 1e-5, atol = 1e-5)+ assert seen[0] == 4 and set(seen[1:]) == {1}+++def test_settle_drops_a_lever_that_fell_back_from_the_status():+ import dataclasses++ @dataclasses.dataclass(frozen = True)+ class _State:+ speed_optims: tuple++ state = _State(+ ("cudnn_benchmark", "h3_vae_fused_encoder", "h3_vae_fp16_encoder", "h3_vae_fused_decoder")+ )+ pipe = types.SimpleNamespace(+ vae = types.SimpleNamespace(+ encoder = types.SimpleNamespace(_unsloth_fast_failed = False),+ decoder = types.SimpleNamespace(_unsloth_fast_failed = True),+ )+ )+ H.settle_h3_vae_fallback(state, pipe)+ assert state.speed_optims == ("cudnn_benchmark", "h3_vae_fused_encoder", "h3_vae_fp16_encoder")+ H.settle_h3_vae_fallback(state, types.SimpleNamespace()) # no vae: a no-op, never raises+++def test_int8_small_row_path_matches_the_int_mm_path():+ holder = torch.nn.Module()+ lin = torch.nn.Linear(64, 32)+ holder.transformer_blocks = torch.nn.ModuleList([torch.nn.ModuleDict({"l": lin})])+ assert H._install_int8_decoder(types.SimpleNamespace(decoder = holder), keep_blocks = ()) > 0+ x = torch.randn(64, 64)+ whole = lin(x) # 64 rows: torch._int_mm+ sliced = torch.cat(+ [lin(x[i : i + 8]) for i in range(0, 64, 8)]+ ) # 8 rows: the dequantised float path+ assert ((whole - sliced).norm() / whole.norm()) < 0.02+++def test_apply_holds_fp16_accumulation_off_outside_the_plan(monkeypatch):+ matmul = torch.backends.cuda.matmul+ if not hasattr(matmul, "allow_fp16_accumulation"):+ pytest.skip("torch without allow_fp16_accumulation")+ monkeypatch.setattr(H, "cuda_fast_path_available", lambda vae = None: True)+ seen = []+ vae = types.SimpleNamespace(decode = lambda z: seen.append(matmul.allow_fp16_accumulation) or z)+ prev = matmul.allow_fp16_accumulation+ try:+ matmul.allow_fp16_accumulation = True+ engaged = H.apply_h3_vae_speedups(+ vae, speed_mode = "default", workflow = "t2va", consumer_gpu = True+ )+ assert H.LEVER_FP16_ACCUM not in engaged+ vae.decode(0)+ assert seen == [False] and matmul.allow_fp16_accumulation is True+ finally:+ matmul.allow_fp16_accumulation = prev+++def test_the_fast_path_is_for_the_h3_vae_class_only(monkeypatch):+ monkeypatch.setattr(torch.cuda, "is_available", lambda: True)+ monkeypatch.setattr(H, "_kernels", lambda: object())+ monkeypatch.setattr(H, "_triton_version_ok", lambda: True)+ monkeypatch.setattr(H, "_triton_jit_toolchain_ok", lambda: True)++ class AutoencoderKLWan:+ pass++ class AutoencoderKLMiniMaxH3:+ pass++ if getattr(torch.version, "hip", None):+ pytest.skip("ROCm build")+ assert not H.cuda_fast_path_available(AutoencoderKLWan())+ assert H.cuda_fast_path_available(AutoencoderKLMiniMaxH3())+++ "platform, probe, expected",+ [+ ("win32", lambda: False, False), # Triton present but its JIT cannot find the CRT headers+ ("win32", lambda: True, True),+ (+ "win32",+ lambda: 1 / 0,+ True,+ ), # the probe itself failing is not evidence; per-call fallback covers it+ ("linux", lambda: False, True), # never asked off Windows+ ],+)+def test_the_fast_path_asks_the_windows_triton_toolchain(monkeypatch, platform, probe, expected):+ import core._msvc_env as msvc++ monkeypatch.setattr(torch.cuda, "is_available", lambda: True)+ monkeypatch.setattr(torch.version, "hip", None, raising = False)+ monkeypatch.setattr(H, "_kernels", lambda: object())+ monkeypatch.setattr(H, "_triton_version_ok", lambda: True)+ monkeypatch.setattr(H.sys, "platform", platform)+ monkeypatch.setattr(msvc, "crt_headers_reachable", probe)+ H._triton_jit_toolchain_ok.cache_clear()+ try:++ class AutoencoderKLMiniMaxH3:+ pass++ assert H.cuda_fast_path_available(AutoencoderKLMiniMaxH3()) is expected+ finally:+ H._triton_jit_toolchain_ok.cache_clear()++[email protected]("device", DEVICES)[email protected]("stock_fallback", [False, True])+def test_the_fp16_encoder_survives_diffusers_casting_the_pixels(device, stock_fallback):+ # diffusers 0.40.0 encode() casts pixels to the encoder dtype and feeds a float32 quant_conv+ get_parameter_dtype = pytest.importorskip("diffusers.models.modeling_utils").get_parameter_dtype++ vae = _tiny_vae().to(device)+ ref_vae = copy.deepcopy(vae)+ assert H._install_encoder(vae, fp16 = True)+ assert get_parameter_dtype(vae.encoder) is torch.float16+ vae.encoder._unsloth_fast_failed = stock_fallback+ x = torch.randn(1, 3, 9, 16, 16, device = device)+ with torch.no_grad():+ got = vae.encode(x.to(get_parameter_dtype(vae.encoder)), return_dict = False)[0].mean+ ref = ref_vae.encode(x, return_dict = False)[0].mean+ assert got.dtype is torch.float32 and vae.quant_conv.weight.dtype is torch.float32+ assert (got - ref).norm() / ref.norm() < 2e-2+++def test_the_video_backend_wires_the_layer_into_the_h3_load_only():+ import ast+ import pathlib++ source = (pathlib.Path(H.__file__).parent / "video.py").read_text(encoding = "utf-8")+ tree = ast.parse(source)++ def calls(node, name):+ return [+ c+ for c in ast.walk(node)+ if isinstance(c, ast.Call)+ and (getattr(c.func, "id", None) == name or getattr(c.func, "attr", None) == name)+ ]++ methods = {+ n.name: n for n in ast.walk(tree) if isinstance(n, (ast.FunctionDef, ast.AsyncFunctionDef))+ }+ load = methods["_load_h3_modular_pipeline"]+ (apply_call,) = calls(load, "apply_h3_vae_speedups")+ assert len(calls(tree, "apply_h3_vae_speedups")) == 1, "only the H3 load engages the layer"+ assert {k.arg for k in apply_call.keywords} >= {"speed_mode", "workflow", "logger"}+ assert calls(load, "apply_speed_optims")[0].lineno < apply_call.lineno+ (settle,) = calls(methods["generate"], "settle_h3_vae_fallback")+ (compile_settle,) = calls(methods["generate"], "settle_compile_fallback")+ assert compile_settle.lineno < settle.lineno++[email protected](H._kernels() is None, reason = "needs Triton")+def test_kernel_annotations_resolve_against_the_module_globals():+ # Triton <= 3.2 resolves kernel annotations in module globals; a closure-local ``tl`` NameErrors there+ k = H._kernels()+ names = [n for n in vars(k) if not n.startswith("_")]+ assert names+ for name in names:+ fn = getattr(getattr(k, name), "fn", None)+ assert fn is not None, name+ for arg, ann in fn.__annotations__.items():+ if isinstance(ann, str):+ eval(ann, fn.__globals__) # noqa: S307 - a Triton annotation such as "tl.constexpr"+++ "version, ok",+ [+ ("3.2.0", False), # compiles, but wrong GroupNorm statistics for a channels-last input+ ("3.1.0", False),+ ("3.3.1", True),+ ("3.6.0", True),+ ("3.8.0.post28", True), # triton-windows+ ("3.2.0+git35c6c7c6", False),+ ("not a version", False),+ ],+)+def test_the_fast_path_needs_a_verified_triton(monkeypatch, version, ok):+ assert H._triton_version_ok(version) is ok+ monkeypatch.setattr(torch.cuda, "is_available", lambda: True)+ monkeypatch.setattr(torch.version, "hip", None, raising = False)+ monkeypatch.setattr(H, "_kernels", lambda: object())+ monkeypatch.setattr(H, "_triton_jit_toolchain_ok", lambda: True)+ monkeypatch.setattr(H, "_triton_version_ok", lambda: ok)++ class AutoencoderKLMiniMaxH3:+ pass++ assert H.cuda_fast_path_available(AutoencoderKLMiniMaxH3()) is ok+++ "before, planned, mid",+ [+ (True, False, False), # the bf16 model that turned it on unloads mid-decode+ (False, False, True), # a bf16 model loads and turns it on mid-decode+ (True, True, False),+ (False, True, True),+ ],+)+def test_decode_scope_keeps_a_flag_write_that_lands_mid_decode(before, planned, mid):+ from core.inference import diffusion_speed as S++ matmul = torch.backends.cuda.matmul+ if not hasattr(matmul, "allow_fp16_accumulation"):+ pytest.skip("torch without allow_fp16_accumulation")+ seen = {}++ class _VAE:+ def decode(self, z):+ seen["snapshot"] = S.snapshot_backend_flags()["matmul_fp16_accum"]+ S.restore_backend_flags({"matmul_fp16_accum": mid})+ seen["after_write"] = matmul.allow_fp16_accumulation+ return z++ prev = matmul.allow_fp16_accumulation+ try:+ matmul.allow_fp16_accumulation = before+ vae = _VAE()+ assert H._install_decode_scope(vae, fp16_accum = planned)+ vae.decode(0)+ assert seen["snapshot"] is before, "a snapshot taken mid-decode must see the process value"+ assert seen["after_write"] is planned, "the decode keeps its own value until it ends"+ assert (+ matmul.allow_fp16_accumulation is mid+ ), "the write made during the decode must survive it"+ assert not S._fp16_accum_scopes+ finally:+ matmul.allow_fp16_accumulation = prev+++def test_fp16_accumulation_scopes_that_close_out_of_order():+ from core.inference import diffusion_speed as S++ matmul = torch.backends.cuda.matmul+ if not hasattr(matmul, "allow_fp16_accumulation"):+ pytest.skip("torch without allow_fp16_accumulation")+ prev = matmul.allow_fp16_accumulation+ try:+ matmul.allow_fp16_accumulation = True+ a, b = S.fp16_accumulation_scope(False), S.fp16_accumulation_scope(True)+ a.__enter__()+ assert matmul.allow_fp16_accumulation is False+ b.__enter__()+ assert matmul.allow_fp16_accumulation is True+ a.__exit__(None, None, None)+ assert matmul.allow_fp16_accumulation is True, "the scope still open keeps its value"+ b.__exit__(None, None, None)+ assert matmul.allow_fp16_accumulation is True and not S._fp16_accum_scopes+ finally:+ matmul.allow_fp16_accumulation = prev+++def test_int8_install_leaves_the_decoder_float_when_quantization_fails(monkeypatch):+ vae = _tiny_vae()+ before = {k: v.clone() for k, v in vae.decoder.state_dict().items()}+ real, calls = H._quantize_int8_weight, []++ def flaky(w, hadamard):+ calls.append(1)+ if len(calls) == 3:+ raise RuntimeError("out of host memory")+ return real(w, hadamard)++ monkeypatch.setattr(H, "_quantize_int8_weight", flaky)+ with pytest.raises(RuntimeError):+ H._install_int8_decoder(vae, keep_blocks = ())+ assert len(calls) == 3+ int8_cls = H._int8_linear_class()+ for module in vae.decoder.modules():+ assert type(module) is not int8_cls+ assert not hasattr(module, "_unsloth_wq") and not hasattr(module, "_unsloth_rot")+ after = vae.decoder.state_dict()+ assert after.keys() == before.keys()+ for k, v in before.items():+ assert torch.equal(after[k], v), k+++def test_apply_reports_int8_off_and_keeps_float_weights_when_it_fails(monkeypatch):+ monkeypatch.setattr(H, "cuda_fast_path_available", lambda vae = None: True)+ monkeypatch.setenv(H.H3_VAE_INT8_ENV, "1")++ def boom(w, hadamard):+ raise RuntimeError("out of host memory")++ monkeypatch.setattr(H, "_quantize_int8_weight", boom)+ vae = _tiny_vae()+ engaged = H.apply_h3_vae_speedups(+ vae, speed_mode = "default", workflow = "t2va", consumer_gpu = False+ )+ assert H.LEVER_INT8_DECODER not in engaged+ assert all(+ m.weight is not None for m in vae.decoder.modules() if isinstance(m, torch.nn.Linear)+ )+++def test_int8_decoder_rotated_path_stays_close(monkeypatch):+ monkeypatch.setattr(H, "H3_VAE_INT8_ROT_GROUP", 16)+ vae = _tiny_vae()+ fast = copy.deepcopy(vae)+ H._install_int8_decoder(fast, keep_blocks = ())+ lin = fast.decoder.transformer_blocks[0].attn.to_q+ assert lin.weight is None and lin._unsloth_rot.dtype is torch.float16+ z = torch.randn(1, 8, 3, 4, 5)+ with torch.no_grad():+ ref = vae.decoder(z)+ got = fast.decoder(z)+ assert (got - ref).norm() / ref.norm() < 0.05