Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio MiniMax-H3: opt-in FlashInfer SM120 fused block kernels for FP8 and NVFP4 (#16875)

Daniel Han790595d468ce
diff --git a/studio/backend/core/inference/video.py b/studio/backend/core/inference/video.py
index 76adffd1244..1d6be78e675 100644
--- a/studio/backend/core/inference/video.py
+++ b/studio/backend/core/inference/video.py
@@ -11635,6 +11635,19 @@ class VideoBackend:
h3_attn_levers += ("h3_strided_attn",)
except Exception as exc: # noqa: BLE001 -- stock processor stays
logger.warning("video.h3_attn: strided attention skipped: %s", exc)
+ # FlashInfer SM120 H3 block kernels (opt-in, UNSLOTH_H3_FLASHINFER=1): resident fp8 / nvfp4 blocks, before
+ # the regional compile traces them
+ if effective_speed != SPEED_OFF:
+ try:
+ from .video_minimax_h3_flashinfer import install_h3_flashinfer
+ if install_h3_flashinfer(
+ getattr(pipe, denoiser_component, None),
+ resident = bool(denoiser_pinned) and not denoiser_streamed,
+ logger = logger,
+ ).get("blocks"):
+ h3_attn_levers += ("h3_flashinfer",)
+ except Exception as exc: # noqa: BLE001 -- optimisation only, the stock blocks stay
+ logger.warning("video.h3_flashinfer: skipped: %s", exc)
h3_speed_target = types.SimpleNamespace(
device = device,
dtype = dtype,
diff --git a/studio/backend/core/inference/video_minimax_h3_flashinfer.py b/studio/backend/core/inference/video_minimax_h3_flashinfer.py
new file mode 100644
index 00000000000..a43adf0588c
--- /dev/null
+++ b/studio/backend/core/inference/video_minimax_h3_flashinfer.py
@@ -0,0 +1,891 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""MiniMax-H3 FlashInfer SM120 block kernels (``flashinfer.diffusion_ops``, FlashInfer >= 0.7.3), opt-in.
+
+UNSLOTH_H3_FLASHINFER=1 swaps each resident FP8 / NVFP4 H3 block's forward for FlashInfer's fused SM120 operators:
+pre-attention (norm1 + adaLN + QKV GEMM + q/k RMSNorm + RoPE), the out-projection with its gated residual and the MLP
+(norm2 + adaLN + FC1 + SwiGLU + FC2 + gated residual). Attention stays on SDPA. Per-lever kill switches, each on by
+default under the master switch: UNSLOTH_H3_FI_PRE, UNSLOTH_H3_FI_OUT, UNSLOTH_H3_FI_MLP.
+
+Gate: CUDA sm_120 (the operators raise elsewhere), flashinfer >= 0.7.3 importable with the ops, H3 dimensions, a
+resident denoiser, torchao Float8Tensor (per-row) or NVFP4Tensor weights. The operators JIT-compile on first use
+(nvcc >= 12.8 and ninja; FlashInfer's jit-cache wheel does not carry them), hence opt-in. Weights are repacked from
+Studio's own quantized tensors, bit-exact, into buffers on each block (they follow the module and die with it); the
+replaced projections' own weights are then released (UNSLOTH_H3_FI_KEEP_STOCK=1 keeps them). NVFP4 pre-attention
+needs one per-tensor scale for to_q / to_k / to_v (a prequant built with a shared QKV scale); other files keep the
+stock pre-attention. A probe self-check against the stock block runs at install; any failure keeps the stock forward.
+
+NVFP4 activation global scales: a1 / a2 / attention output from their amax per call (torchao's dynamic recipe);
+the SwiGLU output y, which the fused MLP quantizes in-kernel, from its amax on the block's first call at each sequence
+length, x UNSLOTH_H3_FI_NV_Y_MARGIN (default 4).
+"""
+
+from __future__ import annotations
+
+import os
+import threading
+import types
+from functools import lru_cache
+from typing import Any, Optional
+
+MASTER_ENV = "UNSLOTH_H3_FLASHINFER"
+PRE_ENV = "UNSLOTH_H3_FI_PRE"
+OUT_ENV = "UNSLOTH_H3_FI_OUT"
+MLP_ENV = "UNSLOTH_H3_FI_MLP"
+KEEP_STOCK_ENV = "UNSLOTH_H3_FI_KEEP_STOCK"
+NV_Y_MARGIN_ENV = "UNSLOTH_H3_FI_NV_Y_MARGIN"
+
+MIN_FLASHINFER = (0, 7, 3)
+HIDDEN, HEADS, HEAD_DIM, FFN, ADALN_ROWS, ROPE_DIM = 5376, 56, 128, 14336, 9, 96
+E2M1_X_E4M3 = 448.0 * 6.0
+# Probe self-check bound on the relative L2 of the block's residual delta vs the stock block: a layout or scale bug
+# lands near 1; FlashInfer vs stock on the same quantized weights measured 0.01-0.11 on real H3 inputs.
+SELF_CHECK_TOL = {"fp8": 0.10, "nvfp4": 0.25}
+_BUF = "_h3fi_"
+_NS = "unsloth_h3fi"
+_LOCK = threading.Lock()
+# Host-side facts of each installed block's operands, keyed by the operand buffer's data pointer: the NVFP4 global
+# weight scale (the kernels take alpha as a host float) and the calibrated y scale per sequence length. Ops read them
+# inside the opaque op, so nothing per block reaches the traced graph: all 50 blocks share one compiled graph (the
+# block buffers are graph inputs), with no recompile per block.
+_HOST: dict = {}
+_CAL: dict = {}
+# The installed model's switches, the same for every block (read by the traced forward; constants for dynamo).
+_CFG: dict = {
+ "levers": (),
+ "scheme": None,
+ "cudnn": True,
+ "released": False,
+ "eps": 1e-5,
+ "y_margin": 4.0,
+}
+# torch / F / SDPA handles for the traced forward, filled by register_ops(): no import statements in the region.
+_T: dict = {}
+CALLS: dict = {"pre": 0, "out": 0, "mlp": 0, "stock": 0, "y_calibrations": 0}
+
+
+# Attention backends the fused block reproduces: unrestricted SDPA ("native" / unset) and cuDNN-pinned SDPA.
+_SDPA_BACKENDS = ("", "native", "_native_cudnn")
+
+
+def _env(name: str, default: str) -> str:
+ return (os.environ.get(name) or default).strip().lower()
+
+
+def _on(name: str, default: str = "1") -> bool:
+ return _env(name, default) not in ("0", "off", "false", "no", "")
+
+
+def enabled() -> bool:
+ return _on(MASTER_ENV, "0")
+
+
+def _version_tuple(text: str) -> tuple:
+ """Leading release numbers: "0.7.3" -> (0, 7, 3), "0.7.3rc1" -> (0, 7, 3), "0.7.3+cu130" -> (0, 7, 3)."""
+ import re
+
+ out = []
+ for part in str(text).split("+")[0].split("."):
+ m = re.match(r"\d+", part)
+ if m is None:
+ break
+ out.append(int(m.group()))
+ if m.end() != len(part):
+ break
+ return tuple(out)
+
+
+def flashinfer_unavailable(device: Any = None) -> Optional[str]:
+ """Why the FlashInfer H3 operators cannot run here, or None."""
+ try:
+ import torch
+ except Exception as exc: # noqa: BLE001
+ return f"torch unavailable: {exc}"
+ from .rocm_bf16 import is_rocm_torch
+
+ if not torch.cuda.is_available() or is_rocm_torch(torch):
+ return "needs an NVIDIA CUDA device"
+ try:
+ cap = tuple(torch.cuda.get_device_capability(device))
+ except Exception as exc: # noqa: BLE001
+ return f"no device capability: {exc}"
+ if cap != (12, 0):
+ return f"FlashInfer's MiniMax-H3 operators are sm_120 only (this device is sm_{cap[0]}{cap[1]})"
+ try:
+ import flashinfer
+ from flashinfer import diffusion_ops as fd
+ except Exception as exc: # noqa: BLE001
+ return f"flashinfer not importable: {type(exc).__name__}: {exc}"
+ if _version_tuple(getattr(flashinfer, "__version__", "0")) < MIN_FLASHINFER:
+ return f"flashinfer {getattr(flashinfer, '__version__', '?')} < 0.7.3"
+ need = (
+ "minimax_h3_fp8_pre_attention",
+ "minimax_h3_nvfp4_pre_attention",
+ "minimax_h3_fp8_out_proj",
+ "minimax_h3_nvfp4_out_proj",
+ "minimax_h3_mlp_fp8_sm120",
+ "minimax_h3_mlp_nvfp4_sm120",
+ )
+ missing = [n for n in need if not hasattr(fd, n)]
+ return f"flashinfer.diffusion_ops lacks {missing}" if missing else None
+
+
+# ---------------------------------------------------------------------------------------------------------------- ops
+def _host(t: Any) -> float:
+ return _HOST[t.data_ptr()]
+
+
+def _buf(block: Any, name: str) -> Any:
+ return getattr(block, _BUF + name)
+
+
+def _calibrated(key: tuple, amax_fn, margin: float, device: Any) -> tuple:
+ hit = _CAL.get(key)
+ if hit is None:
+ import torch
+
+ g = E2M1_X_E4M3 / max(float(amax_fn()) * margin, 1e-12)
+ hit = (torch.tensor([g], dtype = torch.float32, device = device), g)
+ with _LOCK:
+ if len(_CAL) > 4096:
+ _CAL.clear()
+ _CAL[key] = hit
+ CALLS["y_calibrations"] += 1
+ return hit
+
+
+@lru_cache(maxsize = None)
+def register_ops() -> bool:
+ import torch
+ import torch.nn.functional as F
+ from flashinfer import diffusion_ops as fd
+ from torch.nn.attention import SDPBackend, sdpa_kernel
+
+ _T.update(
+ {"torch": torch, "F": F, "cudnn": SDPBackend.CUDNN_ATTENTION, "sdpa_kernel": sdpa_kernel}
+ )
+
+ @torch.library.custom_op(
+ f"{_NS}::pre",
+ mutates_args = (),
+ schema = "(Tensor x, Tensor scale9, Tensor shift9, Tensor idx, Tensor cs, Tensor ag, Tensor n1, "
+ "Tensor qn, Tensor kn, Tensor wq, Tensor ws, float eps, int nv) -> (Tensor, Tensor, Tensor)",
+ )
+ def pre(x, scale9, shift9, idx, cs, ag, n1, qn, kn, wq, ws, eps, nv):
+ CALLS["pre"] += 1
+ if not nv:
+ r = fd.minimax_h3_fp8_pre_attention(
+ x, n1, scale9, shift9, idx, wq, ws, qn, kn, cs, eps = eps, out_mode = "bf16"
+ )
+ else:
+ wg = _host(wq)
+ r = fd.minimax_h3_nvfp4_pre_attention(
+ x,
+ n1,
+ scale9,
+ shift9,
+ idx,
+ wq,
+ ws,
+ wg,
+ ag,
+ qn,
+ kn,
+ cs,
+ eps = eps,
+ out_mode = "bf16",
+ alpha = 1.0 / (float(ag.item()) * wg),
+ )
+ return r.q, r.k, r.v
+
+ @pre.register_fake
+ def _pre_fake(x, scale9, shift9, idx, cs, ag, n1, qn, kn, wq, ws, eps, nv):
+ m = x.shape[0]
+ return (
+ x.new_empty((m, HEADS, HEAD_DIM)),
+ x.new_empty((m, HEADS, HEAD_DIM)),
+ x.new_empty((m, HEADS, HEAD_DIM)),
+ )
+
+ @torch.library.custom_op(
+ f"{_NS}::out",
+ mutates_args = (),
+ schema = "(Tensor o, Tensor gate9, Tensor idx, Tensor residual, Tensor ag, Tensor wq, Tensor ws, "
+ "int nv) -> Tensor",
+ )
+ def out(o, gate9, idx, residual, ag, wq, ws, nv):
+ CALLS["out"] += 1
+ if not nv:
+ return fd.minimax_h3_fp8_out_proj(o, wq, ws, gate9, idx, residual)
+ return fd.minimax_h3_nvfp4_out_proj(o, wq, ws, _host(wq), ag, gate9, idx, residual)
+
+ @out.register_fake
+ def _out_fake(o, gate9, idx, residual, ag, wq, ws, nv):
+ return residual.new_empty(residual.shape)
+
+ @torch.library.custom_op(
+ f"{_NS}::mlp",
+ mutates_args = (),
+ schema = "(Tensor h, Tensor scale, Tensor shift, Tensor idx, Tensor gate, Tensor ag, Tensor n2, "
+ "Tensor w1q, Tensor w1s, Tensor w2q, Tensor w2s, float eps, int nv) -> Tensor",
+ )
+ def mlp(h, scale, shift, idx, gate, ag, n2, w1q, w1s, w2q, w2s, eps, nv):
+ CALLS["mlp"] += 1
+ if not nv:
+ return fd.minimax_h3_mlp_fp8_sm120(
+ h, n2, scale, shift, idx, gate, h, w1q, w1s, w2q, w2s, eps = eps
+ )
+ ag_h = float(ag.item())
+ g1, g2 = _host(w1q), _host(w2q)
+
+ def y_amax():
+ # y as the fused MLP forms it (FlashInfer's FC1 + SwiGLU on the same FC1 operands), once per length
+ y = fd.minimax_h3_fc1_swiglu_nvfp4(
+ h,
+ n2,
+ _pad_rows(scale),
+ _pad_rows(shift),
+ idx.int(),
+ ag,
+ w1q,
+ w1s,
+ torch.tensor([1.0 / (ag_h * g1)], dtype = torch.float32, device = h.device),
+ eps = eps,
+ )
+ return y.abs().amax()
+
+ yg_t, yg_h = _calibrated(
+ (w1q.data_ptr(), int(h.shape[0])), y_amax, _CFG["y_margin"], h.device
+ )
+ return fd.minimax_h3_mlp_nvfp4_sm120(
+ h,
+ n2,
+ scale,
+ shift,
+ idx,
+ gate,
+ h,
+ ag,
+ w1q,
+ w1s,
+ 1.0 / (ag_h * g1),
+ yg_t,
+ w2q,
+ w2s,
+ 1.0 / (yg_h * g2),
+ eps = eps,
+ )
+
+ @mlp.register_fake
+ def _mlp_fake(h, scale, shift, idx, gate, ag, n2, w1q, w1s, w2q, w2s, eps, nv):
+ return h.new_empty(h.shape)
+
+ return True
+
+
+# ---------------------------------------------------------------------------------------------------------- weights
+def _nvfp4_parts(weight: Any) -> tuple:
+ """(codes uint8 [N, K / 2], row-major e4m3 block scales [N, K / 16], per-tensor scale float) of a torchao NVFP4Tensor."""
+ import torch
+ from torchao.prototype.mx_formats.utils import from_blocked
+
+ n, k = weight.shape
+ scale = weight.scale
+ if getattr(weight, "is_swizzled_scales", False):
+ scale = from_blocked(scale, n, k // 16)
+ pts = weight.per_tensor_scale
+ if pts is None:
+ raise ValueError("NVFP4 weight without a per-tensor scale")
+ return (
+ weight.qdata.contiguous().view(torch.uint8).reshape(n, k // 2),
+ scale.reshape(n, k // 16).contiguous(),
+ float(pts.float().reshape(()).item()),
+ )
+
+
+def _nvfp4_dequant(codes: Any, scale: Any, pts: float) -> Any:
+ from torchao.prototype.mx_formats.kernels import f4_unpacked_to_f32, unpack_uint4
+
+ n = codes.shape[0]
+ vals = f4_unpacked_to_f32(unpack_uint4(codes.contiguous())).view(
+ n, -1, 16
+ ) * scale.float().view(n, -1, 1)
+ return vals.view(n, -1) * pts
+
+
+def _swizzle(scale: Any) -> Any:
+ """Row-major e4m3 [N, K / 16] -> FlashInfer's 128x4 swizzled flat uint8 (the cuBLAS block-scale layout)."""
+ import torch
+ from torchao.prototype.mx_formats.utils import to_blocked
+
+ return to_blocked(scale.view(torch.float8_e4m3fn)).contiguous().view(torch.uint8).reshape(-1)
+
+
+def _qkv_rows(t: Any) -> Any:
+ """[q rows; k rows; v rows] -> FlashInfer's head / kind / dim row order."""
+ import torch
+
+ q, k, v = t.chunk(3, 0)
+ tail = tuple(q.shape[1:])
+ st = torch.stack(
+ [
+ q.reshape(HEADS, HEAD_DIM, *tail),
+ k.reshape(HEADS, HEAD_DIM, *tail),
+ v.reshape(HEADS, HEAD_DIM, *tail),
+ ],
+ 1,
+ )
+ return st.reshape(3 * HEADS * HEAD_DIM, *tail).contiguous()
+
+
+def _fc1_rows(t: Any) -> Any:
+ """diffusers ``ff.net.0.proj`` rows [value; gate] -> the SM120 FC1 order: per 8 output columns, their 8 gate rows
+ then their 8 value rows (FlashInfer's ``interleave_minimax_h3_fc1_rows_sm120`` of [gate; value])."""
+ val, gate = t.chunk(2, 0)
+ ffn = val.shape[0]
+ tail = tuple(t.shape[1:])
+ st = _stack_rows(gate.reshape(ffn // 8, 8, *tail), val.reshape(ffn // 8, 8, *tail))
+ return st.reshape(2 * ffn, *tail).contiguous()
+
+
+def _stack_rows(a: Any, b: Any) -> Any:
+ import torch
+ return torch.stack([a, b], 1)
+
+
+def _scheme_of(weight: Any) -> Optional[str]:
+ name = type(weight).__name__
+ if name == "NVFP4Tensor":
+ return "nvfp4"
+ if name == "Float8Tensor":
+ return "fp8"
+ return None
+
+
+def _linears(block: Any) -> dict:
+ attn, ff = block.attn, block.ff
+ return {
+ "q": getattr(attn, "to_q", None),
+ "k": getattr(attn, "to_k", None),
+ "v": getattr(attn, "to_v", None),
+ "o": attn.to_out[0],
+ "fc1": ff.net[0].proj,
+ "fc2": ff.net[2],
+ }
+
+
+def _pack(block: Any, scheme: str, levers: tuple) -> tuple:
+ """Register FlashInfer's operands as non-persistent buffers on ``block``; returns (eps, levers it can take)."""
+ import torch
+
+ attn = block.attn
+ lin = _linears(block)
+ shapes = {
+ "q": (HEADS * HEAD_DIM, HIDDEN),
+ "k": (HEADS * HEAD_DIM, HIDDEN),
+ "v": (HEADS * HEAD_DIM, HIDDEN),
+ "o": (HIDDEN, HEADS * HEAD_DIM),
+ "fc1": (2 * FFN, HIDDEN),
+ "fc2": (HIDDEN, FFN),
+ }
+ for name, shape in shapes.items():
+ m = lin[name]
+ if (
+ m is None
+ or _scheme_of(getattr(m, "weight", None)) != scheme
+ or getattr(m, "bias", None) is not None
+ ):
+ raise ValueError(f"{name}: not a bias-free {scheme} torchao Linear")
+ if tuple(m.weight.shape) != shape:
+ raise ValueError(f"{name}: shape {tuple(m.weight.shape)} is not H3's {shape}")
+ eps = {
+ float(getattr(block.norm1, "eps", None) or 1e-5),
+ float(getattr(block.norm2, "eps", None) or 1e-5),
+ float(getattr(attn.norm_q, "eps", None) or 1e-5),
+ float(getattr(attn.norm_k, "eps", None) or 1e-5),
+ }
+ if len(eps) != 1:
+ raise ValueError(f"the fused kernels take one RMSNorm eps, the block has {sorted(eps)}")
+ dev = lin["q"].weight.qdata.device
+
+ def norm_w(m, n):
+ w = getattr(m, "weight", None)
+ return (
+ (w if w is not None else torch.ones(n, device = dev))
+ .detach()
+ .to(dev, torch.bfloat16)
+ .contiguous()
+ )
+
+ bufs = {
+ "n1": norm_w(block.norm1, HIDDEN),
+ "n2": norm_w(block.norm2, HIDDEN),
+ "qn": norm_w(attn.norm_q, HEAD_DIM),
+ "kn": norm_w(attn.norm_k, HEAD_DIM),
+ }
+ scales: dict = {}
+ levers = list(levers)
+ with torch.no_grad():
+ if scheme == "fp8":
+ parts = {n: (lin[n].weight.qdata, lin[n].weight.scale.float().reshape(-1)) for n in lin}
+ for n, (q, sc) in parts.items():
+ if q.dtype != torch.float8_e4m3fn or sc.numel() != q.shape[0]:
+ raise ValueError(f"{n}: not a per-row e4m3 Float8Tensor")
+ if "pre" in levers:
+ bufs["qkv_q"] = _qkv_rows(
+ torch.cat([parts[n][0].view(torch.uint8) for n in "qkv"], 0)
+ ).view(torch.float8_e4m3fn)
+ bufs["qkv_s"] = _qkv_rows(torch.cat([parts[n][1] for n in "qkv"], 0))
+ if "out" in levers:
+ bufs["o_q"], bufs["o_s"] = parts["o"][0].contiguous(), parts["o"][1].contiguous()
+ if "mlp" in levers:
+ bufs["fc1_q"] = _fc1_rows(parts["fc1"][0].view(torch.uint8)).view(
+ torch.float8_e4m3fn
+ )
+ bufs["fc1_s"] = _fc1_rows(parts["fc1"][1])
+ bufs["fc2_q"], bufs["fc2_s"] = (
+ parts["fc2"][0].contiguous(),
+ parts["fc2"][1].contiguous(),
+ )
+ else:
+ parts = {n: _nvfp4_parts(lin[n].weight) for n in lin}
+ if "pre" in levers:
+ pts = [parts[n][2] for n in "qkv"]
+ if pts[0] == pts[1] == pts[2]:
+ codes = torch.cat([parts[n][0] for n in "qkv"], 0)
+ sc = torch.cat([parts[n][1].view(torch.uint8) for n in "qkv"], 0)
+ bufs["qkv_q"], bufs["qkv_s"] = _qkv_rows(codes), _swizzle(_qkv_rows(sc))
+ scales["qkv"] = 1.0 / pts[0]
+ else:
+ # the fused QKV GEMM takes ONE global scale: re-scaling Studio's codes would requantize them
+ levers.remove("pre")
+
+ def same_layout(name, sc):
+ # the stored 128x4 swizzled scales ARE FlashInfer's layout when no row permutation applies: share them
+ w = lin[name].weight
+ if getattr(w, "is_swizzled_scales", False) and w.scale.is_contiguous():
+ return w.scale.view(torch.uint8).reshape(-1)
+ return _swizzle(sc)
+
+ if "out" in levers:
+ c, sc, t = parts["o"]
+ bufs["o_q"], bufs["o_s"], scales["o"] = c, same_layout("o", sc), 1.0 / t
+ if "mlp" in levers:
+ c, sc, t = parts["fc1"]
+ bufs["fc1_q"], bufs["fc1_s"], scales["fc1"] = (
+ _fc1_rows(c),
+ _swizzle(_fc1_rows(sc.view(torch.uint8))),
+ 1.0 / t,
+ )
+ c, sc, t = parts["fc2"]
+ bufs["fc2_q"], bufs["fc2_s"], scales["fc2"] = c, same_layout("fc2", sc), 1.0 / t
+ for name, t in bufs.items():
+ block.register_buffer(_BUF + name, t, persistent = False)
+ with _LOCK:
+ for name, g in scales.items():
+ _HOST[bufs[f"{name}_q"].data_ptr()] = g
+ return eps.pop(), tuple(levers)
+
+
+def _unpack(block: Any) -> None:
+ for name in [n for n, _ in block.named_buffers(recurse = False) if n.startswith(_BUF)]:
+ delattr(block, name)
+ block.__dict__.pop("_unsloth_h3fi_stock", None)
+
+
+def _release_stock(block: Any, levers: tuple) -> None:
+ """Drop the stock weights the FlashInfer levers replaced: the block's buffers hold the same codes (the
+ out-projection / FC2 buffers share the stock storage, QKV / FC1 are reordered copies)."""
+ import torch
+
+ lin = _linears(block)
+ names = (
+ (("q", "k", "v") if "pre" in levers else ())
+ + (("o",) if "out" in levers else ())
+ + (("fc1", "fc2") if "mlp" in levers else ())
+ )
+ for n in names:
+ m = lin[n]
+ m.weight = torch.nn.Parameter(
+ torch.empty((0,), device = m.weight.device, dtype = torch.bfloat16), requires_grad = False
+ )
+ m._unsloth_h3fi_released = True
+
+
+# ------------------------------------------------------------------------------------------------------ block forward
+def _pad_rows(t: Any) -> Any:
+ """[rows, C] -> the [9, C] table the fused kernels index (no branch on rows: it is unbacked under compile)."""
+ return _T["torch"].cat([t, t.new_zeros((ADALN_ROWS - t.shape[0], t.shape[1]))], 0)
+
+
+def _ag(t: Any) -> Any:
+ return (E2M1_X_E4M3 / t.abs().amax().float().clamp_min(1e-12)).reshape(1)
+
+
+def _rope(x: Any, cos: Any, sin: Any) -> Any:
+ """diffusers' ``_apply_rotary_emb`` on [M, H, D] (rotate-half over the first ``cos.shape[-1]`` channels)."""
+ torch = _T["torch"]
+ rd = cos.shape[-1]
+ xr, xp = x[..., :rd], x[..., rd:]
+ c, s = cos.to(x.dtype)[:, None, :], sin.to(x.dtype)[:, None, :]
+ x1, x2 = xr.chunk(2, dim = -1)
+ return torch.cat((xr * c + torch.cat((-x2, x1), dim = -1) * s, xp), dim = -1)
+
+
+def _fi_one(self: Any, x: Any, mods: tuple, idx64: Any, cos: Any, sin: Any) -> Any:
+ """One packed sequence [M, C] through the block. Everything per block is a module tensor (a graph input), the
+ switches are _CFG's model-wide constants: one compiled graph serves all blocks."""
+ torch, F = _T["torch"], _T["F"]
+ shift_msa, scale_msa, gate_msa, shift_mlp, scale_mlp, gate_mlp = mods
+ levers, nv, eps = _CFG["levers"], int(_CFG["scheme"] == "nvfp4"), _CFG["eps"]
+ idx32 = idx64.int()
+ empty = x.new_empty((1,), dtype = torch.float32)
+ attn = self.attn
+ if "pre" in levers:
+ cs = torch.cat([cos[:, : ROPE_DIM // 2], sin[:, : ROPE_DIM // 2]], -1).to(torch.bfloat16)
+ ag1 = empty
+ if nv:
+ n = F.rms_norm(x, (x.shape[-1],), self.norm1.weight, self.norm1.eps)
+ ag1 = _ag(
+ n * (1.0 + scale_msa.index_select(0, idx64)) + shift_msa.index_select(0, idx64)
+ )
+ q, k, v = torch.ops.unsloth_h3fi.pre(
+ x,
+ _pad_rows(scale_msa),
+ _pad_rows(shift_msa),
+ idx32,
+ cs,
+ ag1,
+ self._h3fi_n1,
+ self._h3fi_qn,
+ self._h3fi_kn,
+ self._h3fi_qkv_q,
+ self._h3fi_qkv_s,
+ eps,
+ nv,
+ )
+ else:
+ n = self.norm1(x)
+ am = n * (1.0 + scale_msa.index_select(0, idx64)) + shift_msa.index_select(0, idx64)
+ q = attn.norm_q(attn.to_q(am).unflatten(-1, (attn.heads, -1)))
+ k = attn.norm_k(attn.to_k(am).unflatten(-1, (attn.heads, -1)))
+ v = attn.to_v(am).unflatten(-1, (attn.heads, -1))
+ q, k = _rope(q, cos, sin), _rope(k, cos, sin)
+ qp, kp, vp = (t[None].permute(0, 2, 1, 3) for t in (q, k, v))
+ if _CFG["cudnn"]:
+ with _T["sdpa_kernel"](_T["cudnn"]):
+ o = F.scaled_dot_product_attention(qp, kp, vp, dropout_p = 0.0, is_causal = False)
+ else:
+ o = F.scaled_dot_product_attention(qp, kp, vp, dropout_p = 0.0, is_causal = False)
+ o = o.permute(0, 2, 1, 3)[0].reshape(x.shape[0], -1).to(torch.bfloat16)
+ if "out" in levers:
+ h = torch.ops.unsloth_h3fi.out(
+ o,
+ _pad_rows(gate_msa),
+ idx32,
+ x,
+ _ag(o) if nv else empty,
+ self._h3fi_o_q,
+ self._h3fi_o_s,
+ nv,
+ )
+ else:
+ h = x + gate_msa.index_select(0, idx64) * attn.to_out[0](o)
+ if "mlp" in levers:
+ ag2 = empty
+ if nv:
+ n = F.rms_norm(h, (h.shape[-1],), self.norm2.weight, self.norm2.eps)
+ ag2 = _ag(
+ n * (1.0 + scale_mlp.index_select(0, idx64)) + shift_mlp.index_select(0, idx64)
+ )
+ return torch.ops.unsloth_h3fi.mlp(
+ h,
+ scale_mlp.contiguous(),
+ shift_mlp.contiguous(),
+ idx64,
+ gate_mlp.contiguous(),
+ ag2,
+ self._h3fi_n2,
+ self._h3fi_fc1_q,
+ self._h3fi_fc1_s,
+ self._h3fi_fc2_q,
+ self._h3fi_fc2_s,
+ eps,
+ nv,
+ )
+ n = self.norm2(h)
+ am = n * (1.0 + scale_mlp.index_select(0, idx64)) + shift_mlp.index_select(0, idx64)
+ return h + gate_mlp.index_select(0, idx64) * self.ff(am)
+
+
+def _fi_block_forward(
+ self: Any,
+ hidden_states: Any,
+ temb: Any,
+ adaln_indices: Any,
+ rotary_emb: Any,
+ attention_mask: Any = None,
+) -> Any:
+ torch = _T["torch"]
+ mods = self.adaln_proj(temb)
+ if (
+ attention_mask is not None
+ or hidden_states.dim() != 3
+ or hidden_states.dtype != torch.bfloat16
+ or (not torch.compiler.is_compiling() and mods[0].shape[0] > ADALN_ROWS)
+ ):
+ if _CFG["released"]:
+ raise RuntimeError(
+ "MiniMax-H3 FlashInfer blocks (stock weights released) take no attention mask, bf16 "
+ "only, at most 9 adaLN rows; set UNSLOTH_H3_FLASHINFER=0"
+ )
+ CALLS["stock"] += 1
+ return self._unsloth_h3fi_stock(
+ hidden_states, temb, adaln_indices, rotary_emb, attention_mask
+ )
+ # the adaLN row count (3 x distinct timesteps) is data-dependent under compile: no guard on it there, only a check
+ torch._check(
+ mods[0].shape[0] <= ADALN_ROWS,
+ lambda: "MiniMax-H3 FlashInfer blocks take at most 9 adaLN rows",
+ )
+ idx64 = adaln_indices.reshape(-1).long()
+ cos, sin = rotary_emb
+ outs = [
+ _fi_one(self, hidden_states[b], mods, idx64, cos, sin)
+ for b in range(hidden_states.shape[0])
+ ]
+ return outs[0][None] if len(outs) == 1 else torch.stack(outs, 0)
+
+
+# ------------------------------------------------------------------------------------------------------------ install
+def _probe(block: Any, m: int = 384) -> tuple:
+ import torch
+
+ dev = next(block.parameters()).device
+ g = torch.Generator(device = "cpu").manual_seed(1234)
+ x = torch.randn(1, m, HIDDEN, generator = g).to(dev, torch.bfloat16)
+ lin = block.adaln_proj.linear
+ temb = (torch.randn(1, lin.in_features, generator = g) * 0.5).to(dev, lin.weight.dtype)
+ idx = (torch.arange(m) % 3).to(dev)
+ pos = torch.stack(
+ [torch.arange(m) // 64, (torch.arange(m) // 8) % 8, torch.arange(m) % 8], -1
+ ).float()
+ inv = 1.0 / (10000.0 ** (torch.arange(0, 32, 2, dtype = torch.float32) / 32))
+ fr = torch.cat([pos[:, i : i + 1] * inv[None] for i in range(3)], -1)
+ fr = torch.cat([fr, fr], -1).to(dev)
+ return x, temb, idx, (fr.cos(), fr.sin())
+
+
+def _self_check(block: Any) -> float:
+ import torch
+
+ x, temb, idx, rot = _probe(block)
+ with torch.no_grad():
+ ref = block._unsloth_h3fi_stock(x, temb, idx, rot, None).float() - x.float()
+ got = _fi_block_forward(block, x, temb, idx, rot, None).float() - x.float()
+ rel = ((got - ref).norm() / ref.norm().clamp_min(1e-30)).item()
+ _CAL.clear() # the probe must not leave its calibration behind
+ return rel if rel == rel else float("inf")
+
+
+def _qkv_shared(block: Any) -> bool:
+ lin = _linears(block)
+ pts = [getattr(lin[n].weight, "per_tensor_scale", None) for n in "qkv"]
+ if any(p is None for p in pts):
+ return False
+ return bool(torch_equal(pts[0], pts[1]) and torch_equal(pts[0], pts[2]))
+
+
+def torch_equal(a: Any, b: Any) -> bool:
+ import torch
+ return bool(torch.equal(a.float().reshape(-1), b.float().reshape(-1)))
+
+
+_TOOLCHAIN_HINT = (
+ "first use JIT-compiles these kernels: needs nvcc >= 12.8 (PATH or CUDA_HOME) and ninja"
+)
+
+
+def _failure_reason(exc: BaseException, limit: int = 400) -> str:
+ """One log line for an install failure. A failed JIT build's message starts with ninja echoing whole nvcc command
+ lines; keep its diagnosis (e.g. "nvcc: not found", the compiler error) and name the toolchain it needs."""
+ text = str(exc)
+ lines = [ln.strip() for ln in text.splitlines() if ln.strip()]
+ toolchain = "Ninja build failed" in text or "nvcc" in text or "ninja" in text
+ if "Ninja build failed" in text:
+ lines = [
+ ln
+ for ln in lines
+ if len(ln) <= 300 and not ln.startswith(("[", "ninja: Entering", "Ninja build failed"))
+ ] or lines[:1]
+ lines = ["JIT build failed"] + lines
+ msg = f"{type(exc).__name__}: " + " | ".join(lines)
+ if toolchain:
+ msg += f" ({_TOOLCHAIN_HINT})"
+ if len(msg) > limit:
+ msg = msg[: limit // 4] + " ... " + msg[-(limit - limit // 4 - 5) :]
+ return msg
+
+
+def install_h3_flashinfer(
+ transformer: Any,
+ *,
+ resident: bool = True,
+ logger: Any = None,
+) -> dict:
+ """Swap the resident H3 blocks' forward for the FlashInfer SM120 operators; returns what engaged."""
+ result: dict = {"blocks": 0, "scheme": None, "levers": (), "reason": None}
+ if not enabled():
+ return result
+ blocks = list(getattr(transformer, "transformer_blocks", None) or [])
+ if not blocks:
+ result["reason"] = "no transformer_blocks"
+ return result
+ first = blocks[0]
+ try:
+ device = next(first.parameters()).device
+ except Exception: # noqa: BLE001
+ device = None
+ # the pure-Python gates first: a streamed or int8 / bf16 denoiser never imports flashinfer
+ reason = None
+ if not resident:
+ reason = "the denoiser is streamed / offloaded (the operators need it resident)"
+ q_lin = getattr(
+ getattr(first, "attn", None), "to_q", None
+ ) # absent when Studio fused the int8 QKV
+ scheme = _scheme_of(getattr(q_lin, "weight", None))
+ if reason is None and scheme is None:
+ reason = (
+ f"blocks are not torchao fp8 / nvfp4 ({type(getattr(q_lin, 'weight', None)).__name__})"
+ )
+ levers = tuple(
+ n for n, env in (("pre", PRE_ENV), ("out", OUT_ENV), ("mlp", MLP_ENV)) if _on(env)
+ )
+ if reason is None and not levers:
+ reason = "every lever switched off"
+ if reason is None:
+ reason = flashinfer_unavailable(device)
+ if reason is not None:
+ result["reason"] = reason
+ if logger is not None:
+ logger.info("video.h3_flashinfer: off: %s", reason)
+ return result
+ register_ops()
+ proc = getattr(first.attn, "processor", None)
+ backend = str(
+ getattr(
+ getattr(proc, "_attention_backend", None),
+ "value",
+ getattr(proc, "_attention_backend", ""),
+ )
+ or ""
+ )
+ if backend not in _SDPA_BACKENDS:
+ # the fused block calls SDPA itself (cuDNN-pinned or unrestricted); any other selected backend would be overridden
+ result["reason"] = f"attention backend {backend!r} is not reproduced by the fused block"
+ if logger is not None:
+ logger.info("video.h3_flashinfer: off: %s", result["reason"])
+ return result
+ try:
+ margin = float(_env(NV_Y_MARGIN_ENV, "4"))
+ except ValueError:
+ margin = 4.0
+ # 0 / negative / nan / inf would turn the y scale into garbage
+ if not 0.0 < margin < float("inf"):
+ margin = 4.0
+ if scheme == "nvfp4" and "pre" in levers and not all(_qkv_shared(b) for b in blocks):
+ # the fused QKV GEMM takes ONE global scale: re-scaling Studio's per-projection codes would requantize them
+ levers = tuple(n for n in levers if n != "pre")
+ if logger is not None:
+ logger.info(
+ "video.h3_flashinfer: q / k / v do not share one NVFP4 scale: stock pre-attention"
+ )
+ if not levers:
+ result["reason"] = (
+ "only the pre-attention lever was on and the NVFP4 q / k / v scales differ"
+ )
+ return result
+ _CFG.update(
+ {
+ "levers": levers,
+ "scheme": scheme,
+ "cudnn": backend == "_native_cudnn",
+ "released": False,
+ "y_margin": margin,
+ }
+ )
+ import torch
+
+ def pack(blk: Any) -> float:
+ eps, took = _pack(blk, scheme, levers)
+ if took != levers:
+ _unpack(blk)
+ raise ValueError(f"block took {took}, not {levers}")
+ blk._unsloth_h3fi_stock = blk.forward
+ return eps
+
+ try:
+ _CFG["eps"] = pack(first)
+ rel = _self_check(first)
+ if not rel <= SELF_CHECK_TOL[scheme]:
+ raise RuntimeError(
+ f"self-check: block delta rel L2 {rel:.4f} > {SELF_CHECK_TOL[scheme]}"
+ )
+ except Exception as exc: # noqa: BLE001 -- the stock forward stays
+ _unpack(first)
+ _CFG["levers"] = ()
+ result["reason"] = _failure_reason(exc)
+ if logger is not None:
+ logger.warning("video.h3_flashinfer: kept the stock blocks: %s", result["reason"])
+ return result
+ keep = _on(KEEP_STOCK_ENV, "0")
+ _CFG["released"] = not keep
+ on_cuda = device is not None and device.type == "cuda"
+ freed = 0
+ installed: list = []
+ # one block at a time: pack, switch, release, so the load-time peak holds one block's copies, not all of them
+ for blk in blocks:
+ try:
+ if blk is not first and pack(blk) != _CFG["eps"]:
+ _unpack(blk)
+ raise ValueError("blocks disagree on the RMSNorm eps")
+ except Exception as exc: # noqa: BLE001 -- this block and the rest stay stock
+ result["reason"] = f"block {len(installed)}: {type(exc).__name__}: {exc}"[:400]
+ if logger is not None:
+ logger.warning(
+ "video.h3_flashinfer: blocks from %d on kept stock: %s",
+ len(installed),
+ result["reason"],
+ )
+ break
+ blk.forward = types.MethodType(_fi_block_forward, blk)
+ if not keep:
+ before = torch.cuda.memory_allocated(device) if on_cuda else 0
+ _release_stock(blk, levers)
+ freed += before - (torch.cuda.memory_allocated(device) if on_cuda else 0)
+ installed.append(blk)
+ if on_cuda and not keep:
+ # hand the released weights' allocator segments back to the driver (-3.5 GiB render peak at 960x544x124)
+ torch.cuda.empty_cache()
+ engaged = set(levers)
+ result.update(
+ {
+ "blocks": len(installed),
+ "scheme": scheme,
+ "levers": tuple(sorted(engaged)),
+ "self_check": rel,
+ "released_bytes": freed,
+ }
+ )
+ if logger is not None:
+ logger.info(
+ "video.h3_flashinfer: %d %s blocks on FlashInfer (%s), self-check rel %.4f, %.2f GB of stock "
+ "weights released",
+ len(installed),
+ scheme,
+ "+".join(sorted(engaged)),
+ rel,
+ freed / 1e9,
+ )
+ return result
diff --git a/studio/backend/tests/test_video_minimax_h3_flashinfer.py b/studio/backend/tests/test_video_minimax_h3_flashinfer.py
new file mode 100644
index 00000000000..1fd10c84f28
--- /dev/null
+++ b/studio/backend/tests/test_video_minimax_h3_flashinfer.py
@@ -0,0 +1,453 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""MiniMax-H3 FlashInfer SM120 block levers (CPU): off by default, refused off sm_120 / without flashinfer >= 0.7.3 /
+for streamed or non-fp8/nvfp4 denoisers, exact weight repacks (row orders, NVFP4 scales, shared storage), the stock
+forward on calls the operators cannot take, and the release of the replaced stock weights."""
+
+import types
+
+import pytest
+import torch
+
+from core.inference import video_minimax_h3_flashinfer as fi
+
+ENVS = (fi.MASTER_ENV, fi.PRE_ENV, fi.OUT_ENV, fi.MLP_ENV, fi.KEEP_STOCK_ENV, fi.NV_Y_MARGIN_ENV)
+
+
[email protected](autouse = True)
+def _clean_env(monkeypatch):
+ for name in ENVS:
+ monkeypatch.delenv(name, raising = False)
+
+
+def _fake_transformer(n = 2):
+ blocks = []
+ for _ in range(n):
+ blk = torch.nn.Module()
+ blk.attn = torch.nn.Module()
+ blk.attn.to_q = torch.nn.Linear(8, 8, bias = False)
+ blk.forward = lambda *a, **k: "stock"
+ blocks.append(blk)
+ return types.SimpleNamespace(transformer_blocks = blocks)
+
+
+def test_off_by_default_touches_nothing():
+ t = _fake_transformer()
+ before = [b.forward for b in t.transformer_blocks]
+ out = fi.install_h3_flashinfer(t)
+ assert out["blocks"] == 0 and out["reason"] is None
+ assert [b.forward for b in t.transformer_blocks] == before
+ assert not any(hasattr(b, "_unsloth_h3fi") for b in t.transformer_blocks)
+
+
+def test_refused_without_sm120_keeps_stock(monkeypatch):
+ monkeypatch.setenv(fi.MASTER_ENV, "1")
+ monkeypatch.setattr(
+ fi,
+ "flashinfer_unavailable",
+ lambda device = None: "FlashInfer's MiniMax-H3 operators are sm_120 only",
+ )
+ monkeypatch.setattr(fi, "_scheme_of", lambda w: "fp8")
+ t = _fake_transformer()
+ before = [b.forward for b in t.transformer_blocks]
+ out = fi.install_h3_flashinfer(t)
+ assert out["blocks"] == 0 and "sm_120" in out["reason"]
+ assert [b.forward for b in t.transformer_blocks] == before
+
+
+def test_unavailable_reason_on_this_host():
+ if torch.cuda.is_available() and tuple(torch.cuda.get_device_capability()) == (12, 0):
+ pytest.skip("an sm_120 host: the answer depends on the installed flashinfer")
+ reason = fi.flashinfer_unavailable()
+ assert isinstance(reason, str) and reason
+
+
[email protected]("resident,expect", [(False, "resident"), (True, "torchao fp8 / nvfp4")])
+def test_streamed_or_bf16_denoiser_is_refused(monkeypatch, resident, expect):
+ monkeypatch.setenv(fi.MASTER_ENV, "1")
+ monkeypatch.setattr(fi, "flashinfer_unavailable", lambda device = None: None)
+ out = fi.install_h3_flashinfer(_fake_transformer(), resident = resident)
+ assert out["blocks"] == 0 and expect in out["reason"]
+
+
+def test_int8_fused_qkv_blocks_are_refused_not_crashed(monkeypatch):
+ monkeypatch.setenv(fi.MASTER_ENV, "1")
+ monkeypatch.setattr(fi, "flashinfer_unavailable", lambda device = None: None)
+ t = _fake_transformer()
+ for b in t.transformer_blocks:
+ del b.attn.to_q # Studio's int8 path fuses to_q / to_k / to_v into to_qkv
+ b.attn.to_qkv = torch.nn.Linear(8, 24, bias = False)
+ out = fi.install_h3_flashinfer(t)
+ assert out["blocks"] == 0 and "fp8 / nvfp4" in out["reason"]
+
+
+def test_every_lever_off_is_a_reason(monkeypatch):
+ monkeypatch.setenv(fi.MASTER_ENV, "1")
+ for env in (fi.PRE_ENV, fi.OUT_ENV, fi.MLP_ENV):
+ monkeypatch.setenv(env, "0")
+ monkeypatch.setattr(fi, "flashinfer_unavailable", lambda device = None: None)
+ monkeypatch.setattr(fi, "_scheme_of", lambda w: "fp8")
+ out = fi.install_h3_flashinfer(_fake_transformer())
+ assert out["blocks"] == 0 and "switched off" in out["reason"]
+
+
+def _stub_install(
+ monkeypatch,
+ events,
+ fail_pack_at = None,
+ self_check = 0.01,
+):
+ monkeypatch.setenv(fi.MASTER_ENV, "1")
+ monkeypatch.setattr(fi, "flashinfer_unavailable", lambda device = None: None)
+ monkeypatch.setattr(fi, "_scheme_of", lambda w: "fp8")
+ monkeypatch.setattr(fi, "register_ops", lambda: True)
+ t = _fake_transformer(3)
+ pos = {id(b): i for i, b in enumerate(t.transformer_blocks)}
+
+ def pack(blk, scheme, levers):
+ if pos[id(blk)] == fail_pack_at:
+ raise RuntimeError("pack failed")
+ events.append(("pack", pos[id(blk)]))
+ return 1e-6, tuple(levers)
+
+ def release(blk, levers):
+ events.append(("release", pos[id(blk)]))
+
+ monkeypatch.setattr(fi, "_pack", pack)
+ monkeypatch.setattr(
+ fi, "_self_check", lambda blk: events.append(("check", pos[id(blk)])) or self_check
+ )
+ monkeypatch.setattr(fi, "_release_stock", release)
+ return t
+
+
+def test_install_packs_and_releases_one_block_at_a_time(monkeypatch):
+ events = []
+ t = _stub_install(monkeypatch, events)
+ out = fi.install_h3_flashinfer(t)
+ assert out["blocks"] == 3 and out["reason"] is None
+ # the load-time peak holds one block's copies: each block's stock weights go before the next block is packed
+ assert events == [
+ ("pack", 0),
+ ("check", 0),
+ ("release", 0),
+ ("pack", 1),
+ ("release", 1),
+ ("pack", 2),
+ ("release", 2),
+ ]
+ assert all(b.forward.__func__ is fi._fi_block_forward for b in t.transformer_blocks)
+
+
+def test_install_failed_self_check_releases_nothing(monkeypatch):
+ events = []
+ t = _stub_install(monkeypatch, events, self_check = 1.0)
+ before = [b.forward for b in t.transformer_blocks]
+ out = fi.install_h3_flashinfer(t)
+ assert out["blocks"] == 0 and "self-check" in out["reason"]
+ assert [b.forward for b in t.transformer_blocks] == before
+ assert not any(e[0] == "release" for e in events)
+
+
+def test_install_pack_failure_midway_keeps_the_rest_stock(monkeypatch):
+ events = []
+ t = _stub_install(monkeypatch, events, fail_pack_at = 2)
+ stock2 = t.transformer_blocks[2].forward
+ out = fi.install_h3_flashinfer(t)
+ assert out["blocks"] == 2 and "block 2" in out["reason"]
+ assert t.transformer_blocks[2].forward is stock2
+ assert ("release", 2) not in events
+
+
+ "text,ok",
+ [
+ ("0.7.3", True),
+ ("0.7.3rc1", True),
+ ("0.7.3+cu130", True),
+ ("0.8.0", True),
+ ("0.7.2", False),
+ ("0.6.6", False),
+ ("1.0", True),
+ ],
+)
+def test_version_floor(text, ok):
+ assert (fi._version_tuple(text) >= fi.MIN_FLASHINFER) is ok
+
+
+def _block_stub():
+ calls = []
+
+ def stock(*a):
+ calls.append(a)
+ return "stock-out"
+
+ blk = types.SimpleNamespace(
+ _unsloth_h3fi_stock = stock,
+ adaln_proj = lambda temb: tuple(torch.zeros(3, 4) for _ in range(6)),
+ )
+ return blk, calls
+
+
+def test_unsupported_calls_take_the_stock_forward(monkeypatch):
+ monkeypatch.setitem(fi._T, "torch", torch)
+ monkeypatch.setitem(fi._CFG, "released", False)
+ blk, calls = _block_stub()
+ before = fi.CALLS["stock"]
+ x = torch.zeros(1, 4, 8, dtype = torch.bfloat16)
+ assert (
+ fi._fi_block_forward(blk, x, None, None, None, attention_mask = torch.ones(1)) == "stock-out"
+ )
+ assert fi._fi_block_forward(blk, x.float(), None, None, None) == "stock-out"
+ assert len(calls) == 2 and fi.CALLS["stock"] == before + 2
+
+
+def test_released_blocks_refuse_instead_of_running_without_weights(monkeypatch):
+ monkeypatch.setitem(fi._T, "torch", torch)
+ monkeypatch.setitem(fi._CFG, "released", True)
+ blk, calls = _block_stub()
+ with pytest.raises(RuntimeError, match = "UNSLOTH_H3_FLASHINFER=0"):
+ fi._fi_block_forward(
+ blk,
+ torch.zeros(1, 4, 8, dtype = torch.bfloat16),
+ None,
+ None,
+ None,
+ attention_mask = torch.ones(1),
+ )
+ assert not calls
+
+
+def test_qkv_row_order_is_head_kind_dim():
+ rows = torch.arange(3 * fi.HEADS * fi.HEAD_DIM)
+ out = fi._qkv_rows(rows.view(-1, 1)).view(-1)
+ hd = fi.HEADS * fi.HEAD_DIM
+ for h in (0, 7, fi.HEADS - 1):
+ for kind in range(3):
+ for d in (0, 5, fi.HEAD_DIM - 1):
+ assert (
+ out[h * 3 * fi.HEAD_DIM + kind * fi.HEAD_DIM + d]
+ == kind * hd + h * fi.HEAD_DIM + d
+ )
+
+
+def test_fc1_rows_gate_then_value_interleaved_by_eight():
+ rows = torch.arange(2 * fi.FFN) # diffusers order: value rows, then gate rows
+ out = fi._fc1_rows(rows.view(-1, 1)).view(-1)
+ assert out[:8].tolist() == list(range(fi.FFN, fi.FFN + 8)) # gate of columns 0..7
+ assert out[8:16].tolist() == list(range(0, 8)) # value of the same columns
+ assert out[16:24].tolist() == list(range(fi.FFN + 8, fi.FFN + 16))
+ assert sorted(out.tolist()) == rows.tolist()
+
+
+def test_fc1_rows_match_flashinfer_interleave():
+ mod = pytest.importorskip("flashinfer.diffusion_ops.cake_minimax_h3_sm120_quant_fc1_swiglu")
+ rows = torch.arange(2 * fi.FFN).view(-1, 1)
+ val, gate = rows.chunk(2, 0)
+ assert torch.equal(
+ fi._fc1_rows(rows), mod.interleave_minimax_h3_fc1_rows_sm120(torch.cat([gate, val], 0))
+ )
+
+
+def _nvfp4(w, pts):
+ from torchao.prototype.mx_formats.nvfp4_tensor import NVFP4Tensor
+ return NVFP4Tensor.to_nvfp4(w, per_tensor_scale = pts, is_swizzled_scales = True)
+
+
+def test_nvfp4_parts_dequantize_exactly_and_reswizzle_to_the_stored_scales():
+ pytest.importorskip("torchao.prototype.mx_formats.nvfp4_tensor")
+ g = torch.Generator().manual_seed(0)
+ w = torch.randn(256, 512, generator = g) * 0.02
+ t = _nvfp4(w, (w.abs().amax() / fi.E2M1_X_E4M3).float())
+ codes, scale, pts = fi._nvfp4_parts(t)
+ assert codes.shape == (256, 256) and scale.shape == (256, 32)
+ assert torch.allclose(
+ fi._nvfp4_dequant(codes, scale, pts), t.dequantize(torch.float32), rtol = 0, atol = 1e-8
+ )
+ assert torch.equal(fi._swizzle(scale), t.scale.contiguous().view(torch.uint8).reshape(-1))
+
+
+class _Small:
+ """H3-shaped block at small sizes (module constants patched): real torchao NVFP4 weights on CPU."""
+
+ def __init__(self, monkeypatch, shared_qkv):
+ pytest.importorskip("torchao.prototype.mx_formats.nvfp4_tensor")
+ monkeypatch.setattr(fi, "HIDDEN", 64)
+ monkeypatch.setattr(fi, "HEADS", 2)
+ monkeypatch.setattr(fi, "HEAD_DIM", 32)
+ monkeypatch.setattr(fi, "FFN", 128)
+ g = torch.Generator().manual_seed(3)
+ blk = torch.nn.Module()
+ blk.norm1, blk.norm2 = torch.nn.RMSNorm(64, eps = 1e-5), torch.nn.RMSNorm(64, eps = 1e-5)
+ blk.attn = torch.nn.Module()
+ blk.attn.norm_q, blk.attn.norm_k = (
+ torch.nn.RMSNorm(32, eps = 1e-5),
+ torch.nn.RMSNorm(32, eps = 1e-5),
+ )
+ blk.ff = torch.nn.Module()
+ blk.ff.net = torch.nn.ModuleList([torch.nn.Module(), torch.nn.Identity(), None])
+ shapes = {
+ "to_q": (64, 64),
+ "to_k": (64, 64),
+ "to_v": (64, 64),
+ "o": (64, 64),
+ "fc1": (256, 64),
+ "fc2": (64, 128),
+ }
+ raw = {
+ k: torch.randn(*s, generator = g) * (0.05 if k != "to_k" else 0.02)
+ for k, s in shapes.items()
+ }
+ shared = max(raw[k].abs().amax() for k in ("to_q", "to_k", "to_v")) / fi.E2M1_X_E4M3
+ self.raw = raw
+
+ def lin(k):
+ m = torch.nn.Linear(raw[k].shape[1], raw[k].shape[0], bias = False)
+ pts = (
+ shared
+ if (shared_qkv and k in ("to_q", "to_k", "to_v"))
+ else raw[k].abs().amax() / fi.E2M1_X_E4M3
+ )
+ m.weight = torch.nn.Parameter(_nvfp4(raw[k], pts.float()), requires_grad = False)
+ return m
+
+ blk.attn.to_q, blk.attn.to_k, blk.attn.to_v = lin("to_q"), lin("to_k"), lin("to_v")
+ blk.attn.to_out = torch.nn.ModuleList([lin("o"), torch.nn.Dropout(0.0)])
+ blk.ff.net[0].proj = lin("fc1")
+ blk.ff.net[2] = lin("fc2")
+ self.blk = blk
+
+
[email protected]("shared_qkv", [True, False])
+def test_nvfp4_pack_is_exact_and_pre_needs_one_qkv_scale(monkeypatch, shared_qkv):
+ b = _Small(monkeypatch, shared_qkv)
+ blk = b.blk
+ o_codes, fc2_codes = blk.attn.to_out[0].weight.qdata, blk.ff.net[2].weight.qdata
+ eps, took = fi._pack(blk, "nvfp4", ("pre", "out", "mlp"))
+ assert eps == 1e-5
+ assert took == (("pre", "out", "mlp") if shared_qkv else ("out", "mlp"))
+ # out-projection / FC2: the very same storage as the stock weight, scales in FlashInfer's 128x4 layout
+ assert fi._buf(blk, "o_q").data_ptr() == o_codes.data_ptr()
+ assert fi._buf(blk, "fc2_q").data_ptr() == fc2_codes.data_ptr()
+ for name, lin in (("o", blk.attn.to_out[0]), ("fc2", blk.ff.net[2])):
+ codes, sc, pts = fi._nvfp4_parts(lin.weight)
+ assert torch.equal(fi._buf(blk, f"{name}_s"), fi._swizzle(sc))
+ assert fi._HOST[fi._buf(blk, f"{name}_q").data_ptr()] == 1.0 / pts
+ # FC1: the same codes, rows permuted
+ codes, sc, pts = fi._nvfp4_parts(blk.ff.net[0].proj.weight)
+ assert torch.equal(fi._buf(blk, "fc1_q"), fi._fc1_rows(codes))
+ assert fi._qkv_shared(blk) is shared_qkv
+ if shared_qkv:
+ q = torch.cat(
+ [fi._nvfp4_parts(getattr(blk.attn, n).weight)[0] for n in ("to_q", "to_k", "to_v")], 0
+ )
+ assert torch.equal(fi._buf(blk, "qkv_q"), fi._qkv_rows(q))
+ else:
+ assert not hasattr(blk, "_h3fi_qkv_q")
+ # buffers are non-persistent: the checkpoint's state dict is unchanged
+ assert not any(k.startswith("_h3fi_") for k in blk.state_dict())
+ fi._unpack(blk)
+ assert not any(n.startswith("_h3fi_") for n, _ in blk.named_buffers())
+
+
+def test_release_drops_only_the_replaced_stock_weights(monkeypatch):
+ b = _Small(monkeypatch, shared_qkv = False)
+ blk = b.blk
+ _, took = fi._pack(blk, "nvfp4", ("pre", "out", "mlp"))
+ fi._release_stock(blk, took)
+ assert blk.attn.to_q.weight.numel() > 0 # pre stayed stock (unshared QKV scales): kept
+ for m in (blk.attn.to_out[0], blk.ff.net[0].proj, blk.ff.net[2]):
+ assert m.weight.numel() == 0 and m._unsloth_h3fi_released
+ assert fi._buf(blk, "o_q").numel() > 0 # the shared storage lives on in the buffer
+
+
+ "backend, engaged",
+ [
+ (None, True),
+ ("native", True),
+ ("_native_cudnn", True),
+ ("_native_flash", False),
+ ("sage", False),
+ ],
+)
+def test_a_selected_attention_backend_the_block_cannot_reproduce_keeps_stock(
+ monkeypatch, backend, engaged
+):
+ events = []
+ t = _stub_install(monkeypatch, events)
+ if backend is not None:
+ for b in t.transformer_blocks:
+ b.attn.processor = types.SimpleNamespace(_attention_backend = backend)
+ before = [b.forward for b in t.transformer_blocks]
+ out = fi.install_h3_flashinfer(t)
+ if engaged:
+ assert out["blocks"] == 3 and out["reason"] is None
+ else:
+ assert out["blocks"] == 0 and repr(backend) in out["reason"]
+ assert [b.forward for b in t.transformer_blocks] == before and events == []
+
+
+_NINJA_NO_NVCC = (
+ "Ninja build failed. Ninja output:\n"
+ "ninja: Entering directory `/root/.cache/flashinfer/0.7.3/120a/cached_ops/minimax_h3_sm120_quant_out_proj'\n"
+ "[1/2] /usr/local/cuda/bin/nvcc --generate-dependencies-with-compile " + "-I/x " * 200 + "\n"
+ "FAILED: [code=127] /root/.cache/flashinfer/0.7.3/120a/cached_ops/out_proj.cuda.o\n"
+ " /usr/local/cuda/bin/nvcc --generate-dependencies-with-compile " + "-I/x " * 200 + "\n"
+ "/bin/sh: 1: /usr/local/cuda/bin/nvcc: not found\n"
+ "ninja: build stopped: subcommand failed.\n"
+)
+
+
+ "exc,expect",
+ [
+ (RuntimeError(_NINJA_NO_NVCC), "nvcc: not found"),
+ (FileNotFoundError(2, "No such file or directory", "ninja"), "ninja"),
+ (
+ RuntimeError(
+ "Could not find nvcc and default cuda_home='/usr/local/cuda' doesn't exist"
+ ),
+ "Could not find nvcc",
+ ),
+ ],
+)
+def test_a_missing_jit_toolchain_keeps_stock_with_a_one_line_reason(monkeypatch, exc, expect):
+ events = []
+ t = _stub_install(monkeypatch, events)
+
+ def self_check(blk):
+ raise exc
+
+ monkeypatch.setattr(fi, "_self_check", self_check)
+ before = [b.forward for b in t.transformer_blocks]
+ out = fi.install_h3_flashinfer(t)
+ assert out["blocks"] == 0 and [b.forward for b in t.transformer_blocks] == before
+ reason = out["reason"]
+ assert expect in reason and "\n" not in reason and len(reason) <= 400
+ assert "nvcc >= 12.8" in reason and "--generate-dependencies" not in reason
+ assert not any(e[0] == "release" for e in events)
+
+
[email protected]("resident,swap", [(False, False), (True, True)])
+def test_streamed_or_int8_denoiser_never_probes_flashinfer(monkeypatch, resident, swap):
+ monkeypatch.setenv(fi.MASTER_ENV, "1")
+
+ def probe(device = None):
+ raise AssertionError("flashinfer probed for a denoiser the levers refuse anyway")
+
+ monkeypatch.setattr(fi, "flashinfer_unavailable", probe)
+ if not swap:
+ monkeypatch.setattr(fi, "_scheme_of", lambda w: "fp8")
+ out = fi.install_h3_flashinfer(_fake_transformer(), resident = resident)
+ assert out["blocks"] == 0 and out["reason"]
+
+
[email protected]("margin", ["0", "-2", "nan", "inf", "x"])
+def test_a_bad_y_margin_falls_back_to_the_default(monkeypatch, margin):
+ events = []
+ t = _stub_install(monkeypatch, events)
+ monkeypatch.setenv(fi.NV_Y_MARGIN_ENV, margin)
+ fi.install_h3_flashinfer(t)
+ assert fi._CFG["y_margin"] == 4.0