Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio NVFP4: one-pass activation quantiser for torchao's to_nvfp4 (#13804)

Daniel Han02299874f656
diff --git a/studio/backend/core/inference/diffusion_nvfp4_act_quant.py b/studio/backend/core/inference/diffusion_nvfp4_act_quant.py
new file mode 100644
index 00000000000..a80e42bce1f
--- /dev/null
+++ b/studio/backend/core/inference/diffusion_nvfp4_act_quant.py
@@ -0,0 +1,351 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""One-pass NVFP4 quantiser for torchao's ``NVFP4Tensor.to_nvfp4`` (two-level scales, swizzled block scales).
+
+Under Inductor torchao's version is three kernels (quantise, pack, swizzle); this ``triton_op`` does all three in
+one pass.
+
+Arithmetic is torchao's, in torchao's order (block max / 6, divide by the per-tensor scale, clamp to
+[2^-6, 448], e4m3, then ``x * ((1 / s) / scale_fp8)``, clamp to +-6, round-to-nearest-even onto e2m1 by the
+same bit trick), so the packed bytes and scales are bit-identical to torchao's own path; ``verify()`` checks
+that on the device before the patch engages. ``UNSLOTH_NVFP4_FAST_ACT_QUANT=0`` restores torchao.
+"""
+
+from __future__ import annotations
+
+import os
+import threading
+from typing import Any
+
+FAST_ACT_QUANT_ENV = "UNSLOTH_NVFP4_FAST_ACT_QUANT"
+
+_LOCK = threading.Lock()
+_STATE: dict = {"installed": False, "verified": None, "reason": "", "fallback": 0}
+_OP = None
+# Tile shape (rows x 16*g elements per program): the fastest bit-identical one of 48 swept on B200.
+_TILE = {"rows": 8, "g": 16, "warps": 2}
+_ORIGINAL = None
+
+
+try: # Module scope, not a closure: Inductor re-imports a user kernel and its helpers by their global names.
+ import triton
+ import triton.language as tl
+except Exception: # noqa: BLE001 - no Triton means torchao's quantiser
+ triton = None
+
+if triton is not None:
+
+ @triton.jit
+ def _e2m1_code(y):
+ # Clamp to +-6, then e2m1 round-to-nearest-even by the bit trick torchao's f32 -> f4 lowering emits.
+ y = tl.minimum(tl.maximum(y, -6.0), 6.0)
+ bits = y.to(tl.int32, bitcast = True)
+ sign = bits & -2147483648
+ mag_bits = bits ^ sign
+ mag = mag_bits.to(tl.float32, bitcast = True)
+ sat = mag >= 6.0
+ denorm = (sat == 0) & (mag < 1.0)
+ normal = (sat | denorm) == 0
+ n_code = ((mag_bits + -1054867457 + ((mag_bits >> 22) & 1)) >> 22).to(tl.uint8)
+ d_code = ((mag + 4194304.0).to(tl.int32, bitcast = True) - 1249902592).to(tl.uint8)
+ code = tl.where(normal, n_code, tl.where(denorm, d_code, 7).to(tl.uint8))
+ return code | ((sign >> 28).to(tl.uint8) & 8)
+
+ @triton.jit
+ def _nvfp4_act_quant_kernel(
+ x_ptr,
+ s_ptr,
+ q_ptr,
+ sf_ptr,
+ M,
+ K: tl.constexpr,
+ NCB: tl.constexpr,
+ ROWS: tl.constexpr,
+ G: tl.constexpr,
+ ):
+ # One program: ROWS rows x G blocks of 16 elements, held as [ROWS, G, 8] even / odd halves so each packed
+ # byte is (even | odd << 4) with no reshape or split (layout conversions some tile shapes miscompiled).
+ pid_m = tl.program_id(0)
+ pid_k = tl.program_id(1)
+ rows = pid_m * ROWS + tl.arange(0, ROWS)
+ blocks = pid_k * G + tl.arange(0, G)
+ lane = tl.arange(0, 8)
+ rmask = (rows < M)[:, None, None]
+ base = (
+ rows[:, None, None].to(tl.int64) * K
+ + blocks[None, :, None] * 16
+ + lane[None, None, :] * 2
+ )
+ xe = tl.load(x_ptr + base, mask = rmask, other = 0.0).to(tl.float32)
+ xo = tl.load(x_ptr + base + 1, mask = rmask, other = 0.0).to(tl.float32)
+ s = tl.load(s_ptr).to(tl.float32)
+ bmax = tl.maximum(tl.max(tl.abs(xe), axis = 2), tl.max(tl.abs(xo), axis = 2))
+ # torchao: block_scale = max_abs / 6 (Inductor folds it to * (1/6)); scaled = block_scale / s.
+ bscale = (bmax * 0.16666666666666666) / s
+ bscale = tl.minimum(tl.maximum(bscale, 0.015625), 448.0)
+ sf8 = bscale.to(tl.float8e4nv)
+ sf32 = sf8.to(tl.float32)
+ recip = (((1.0 / s) * 1.0) / sf32)[:, :, None]
+ lo = _e2m1_code(xe * recip)
+ hi = _e2m1_code(xo * recip)
+ packed = lo | (hi << 4)
+ qoffs = (
+ rows[:, None, None].to(tl.int64) * (K // 2)
+ + blocks[None, :, None] * 8
+ + lane[None, None, :]
+ )
+ tl.store(q_ptr + qoffs, packed, mask = rmask)
+ # cuBLAS / torchao ``to_blocked`` layout: 128 x 4 tiles, (row % 32) * 16 + (row // 32 % 4) * 4 + col % 4.
+ rr = rows % 128
+ tile = (rows // 128)[:, None] * NCB + (blocks // 4)[None, :]
+ soffs = (
+ tile.to(tl.int64) * 512
+ + ((rr % 32) * 16 + (rr // 32) * 4)[:, None]
+ + (blocks % 4)[None, :]
+ )
+ tl.store(sf_ptr + soffs, sf8.to(tl.uint8, bitcast = True), mask = (rows < M)[:, None])
+
+
+def fast_act_quant_enabled() -> bool:
+ return os.environ.get(FAST_ACT_QUANT_ENV, "1").strip().lower() not in (
+ "0",
+ "false",
+ "no",
+ "off",
+ )
+
+
+def describe() -> dict:
+ return dict(_STATE)
+
+
+def _build_op():
+ """Register ``unsloth_nvfp4::act_quant(x, per_tensor_scale) -> (qdata u8 [M, K/2], scales u8 flat)``."""
+ global _OP
+ if _OP is not None:
+ return _OP
+ import torch
+ from torch.library import triton_op, wrap_triton
+
+ if triton is None:
+ raise RuntimeError("triton unavailable")
+
+ # ``from __future__ import annotations`` leaves the op's annotations as strings that infer_schema resolves here.
+ globals().setdefault("torch", torch)
+
+ @triton_op("unsloth_nvfp4::act_quant", mutates_args = ())
+ def act_quant(
+ x: torch.Tensor, per_tensor_scale: torch.Tensor
+ ) -> tuple[torch.Tensor, torch.Tensor]:
+ m, k = x.shape
+ kb = k // 16
+ ncb = (kb + 3) // 4
+ m_pad = (m + 127) // 128 * 128
+ q = torch.empty((m, k // 2), device = x.device, dtype = torch.uint8)
+ numel = m_pad * ncb * 4
+ if m_pad != m or ncb * 4 != kb:
+ sf = torch.zeros((numel,), device = x.device, dtype = torch.uint8)
+ else:
+ sf = torch.empty((numel,), device = x.device, dtype = torch.uint8)
+ g = _TILE["g"]
+ while kb % g:
+ g //= 2
+ rows = _TILE["rows"]
+ grid = (triton.cdiv(m, rows), kb // g)
+ wrap_triton(_nvfp4_act_quant_kernel)[grid](
+ x, per_tensor_scale, q, sf, m, K = k, NCB = ncb, ROWS = rows, G = g, num_warps = _TILE["warps"]
+ )
+ return q, sf
+
+ _OP = act_quant
+ return _OP
+
+
+def _eligible(
+ data_hp: Any, block_size: int, per_tensor_scale: Any, is_swizzled: bool, use_triton: bool
+) -> bool:
+ import torch
+ return (
+ is_swizzled
+ and not use_triton
+ and block_size == 16
+ and per_tensor_scale is not None
+ and isinstance(per_tensor_scale, torch.Tensor)
+ and per_tensor_scale.numel() == 1
+ and per_tensor_scale.device
+ == data_hp.device # a host scalar would hand the kernel a host pointer
+ and data_hp.dim() == 2
+ and data_hp.dtype == torch.bfloat16
+ and data_hp.is_cuda
+ and data_hp.shape[-1] % 16 == 0
+ and data_hp.is_contiguous()
+ )
+
+
+def _traced(data_hp: Any) -> bool:
+ """Whether this call is being traced into a graph rather than run eagerly.
+
+ ``torch.compiler.is_compiling()`` alone is not enough: torchao's NVFP4 linear reaches ``to_nvfp4`` from the
+ tensor subclass's dispatch, which AOTAutograd traces after Dynamo has finished, with that flag False. The
+ tensors there are fake, so a fake input (or an active tracing context) marks the call as traced."""
+ import torch
+
+ if torch.compiler.is_compiling():
+ return True
+ try:
+ from torch._subclasses.fake_tensor import is_fake
+ if is_fake(data_hp):
+ return True
+ except Exception: # noqa: BLE001 - older torch: fall through to the tracing context
+ pass
+ try:
+ from torch._guards import TracingContext
+ return TracingContext.try_get() is not None
+ except Exception: # noqa: BLE001
+ return False
+
+
+def _patched_to_nvfp4(
+ data_hp,
+ block_size: int = 16,
+ per_tensor_scale = None,
+ act_per_tensor_scale = None,
+ is_swizzled_scales: bool = False,
+ use_triton_kernel: bool = False,
+ act_quant_kwargs = None,
+):
+ # Nothing mutable is touched while tracing: a guard on a counter would recompile every call.
+ import torch
+
+ counting = not _traced(data_hp)
+ # Eager calls (the load-time weight pass, speed tier off) keep torchao's own eager arithmetic, which divides by 6
+ # where Inductor multiplies by 1/6: the kernel matches the COMPILED path bit for bit, so only traced calls use it.
+ if counting or not _eligible(
+ data_hp, block_size, per_tensor_scale, is_swizzled_scales, use_triton_kernel
+ ):
+ if counting:
+ _STATE["fallback"] += 1
+ return _ORIGINAL(
+ data_hp,
+ block_size,
+ per_tensor_scale,
+ act_per_tensor_scale,
+ is_swizzled_scales,
+ use_triton_kernel,
+ act_quant_kwargs,
+ )
+ from torchao.prototype.mx_formats.nvfp4_tensor import NVFP4Tensor
+ from torchao.prototype.mx_formats.utils import hp_data_dims_to_swizzled_scale_dims_nvfp4
+
+ m, k = data_hp.shape
+ # A Triton launch lands on the CURRENT device, not the tensor's: guard it like every other NVFP4 launch.
+ with torch.cuda.device(data_hp.device):
+ q, sf = torch.ops.unsloth_nvfp4.act_quant(
+ data_hp, per_tensor_scale.reshape(1).to(torch.float32)
+ )
+ scale_m, scale_k = hp_data_dims_to_swizzled_scale_dims_nvfp4(m, k)
+ sf = sf.view(torch.float8_e4m3fn).view(scale_m, scale_k)
+ return NVFP4Tensor(
+ q,
+ sf,
+ block_size,
+ data_hp.dtype,
+ per_tensor_scale,
+ act_per_tensor_scale,
+ is_swizzled_scales,
+ use_triton_kernel,
+ act_quant_kwargs,
+ )
+
+
+def verify(device: Any = None) -> tuple:
+ """Bit-identity against Inductor-compiled torchao ``to_nvfp4``. Eager torchao may differ in the last bit of a
+ block scale (divide vs multiply by 1/6); that is recorded, not gated."""
+ import torch
+ from torchao.prototype.mx_formats.nvfp4_tensor import NVFP4Tensor, per_tensor_amax_to_scale
+
+ original = _ORIGINAL or NVFP4Tensor.to_nvfp4
+ op = _build_op()
+ device = device or torch.device("cuda", torch.cuda.current_device())
+
+ def reference(x):
+ s = per_tensor_amax_to_scale(torch.max(torch.abs(x)))
+ t = original(x, 16, s, None, True, False, None)
+ return t.qdata.view(torch.uint8), t.scale.reshape(-1).view(torch.uint8), s
+
+ compiled = torch.compile(reference, fullgraph = True, dynamic = False)
+ op_compiled = torch.compile(lambda x, s: op(x, s), fullgraph = True, dynamic = False)
+ with torch.cuda.device(device):
+ return _verify_shapes(device, reference, compiled, op, op_compiled)
+
+
+def _verify_shapes(device, reference, compiled, op, op_compiled) -> tuple:
+ import torch
+
+ gen = torch.Generator(device = device).manual_seed(0)
+ eager_diff = 0
+ for m, k in ((256, 512), (77, 3840), (300, 2560), (1, 64)):
+ x = torch.randn(m, k, device = device, dtype = torch.float32, generator = gen)
+ x = (x * torch.rand(m, 1, device = device, generator = gen) * 8).to(torch.bfloat16)
+ x[0, :16] = 0
+ x[-1, 16:32] = -1e-6
+ want_q, want_sf, s = compiled(x)
+ with torch.cuda.device(x.device):
+ q, sf = op(x, s.reshape(1).to(torch.float32))
+ if not torch.equal(want_q, q):
+ return False, f"packed data differs from compiled torchao at {m}x{k}"
+ if not torch.equal(want_sf, sf):
+ return False, f"block scales differ from compiled torchao at {m}x{k}"
+ # Inside a compiled graph too: catches a kernel Inductor cannot re-import.
+ cq, csf = op_compiled(x, s.reshape(1).to(torch.float32))
+ if not (torch.equal(cq, q) and torch.equal(csf, sf)):
+ return False, f"the compiled op differs from the eager op at {m}x{k}"
+ eq, esf, _ = reference(x)
+ eager_diff += int((eq != q).sum().item()) + int((esf != sf).sum().item())
+ _STATE["eager_bytes_differing"] = eager_diff
+ return True, "bit-identical to compiled torchao to_nvfp4"
+
+
+def install_nvfp4_act_quant_patch(logger: Any = None) -> bool:
+ """Swap ``NVFP4Tensor.to_nvfp4`` for the one-pass kernel after an on-device bit-identity check. Idempotent."""
+ global _ORIGINAL
+ if not fast_act_quant_enabled():
+ _STATE["reason"] = f"{FAST_ACT_QUANT_ENV}=0"
+ return False
+ with _LOCK:
+ if _STATE["installed"]:
+ return bool(_STATE["verified"])
+ try:
+ import torch
+ from torchao.prototype.mx_formats.nvfp4_tensor import NVFP4Tensor
+
+ if not torch.cuda.is_available():
+ _STATE["reason"] = "cuda unavailable"
+ return False
+ _ORIGINAL = NVFP4Tensor.to_nvfp4
+ ok, reason = verify()
+ except Exception as exc: # noqa: BLE001 - any failure keeps torchao's quantiser
+ ok, reason = False, f"{type(exc).__name__}: {str(exc)[:200]}"
+ _STATE["verified"] = ok
+ _STATE["reason"] = reason
+ if ok:
+ NVFP4Tensor.to_nvfp4 = staticmethod(_patched_to_nvfp4)
+ _STATE["installed"] = True
+ if logger is not None:
+ try:
+ logger.info(
+ f"[nvfp4] one-pass activation quantiser: {'on' if ok else 'off'} ({reason})"
+ )
+ except Exception: # noqa: BLE001
+ pass
+ return ok
+
+
+def uninstall_nvfp4_act_quant_patch() -> None:
+ with _LOCK:
+ if _STATE["installed"] and _ORIGINAL is not None:
+ from torchao.prototype.mx_formats.nvfp4_tensor import NVFP4Tensor
+ NVFP4Tensor.to_nvfp4 = staticmethod(_ORIGINAL)
+ _STATE["installed"] = False
+ _STATE["verified"] = None
diff --git a/studio/backend/core/inference/diffusion_prequant.py b/studio/backend/core/inference/diffusion_prequant.py
index d91b71d9b26..c1ad36aa59b 100644
--- a/studio/backend/core/inference/diffusion_prequant.py
+++ b/studio/backend/core/inference/diffusion_prequant.py
@@ -2406,8 +2406,13 @@ def load_prequantized_transformer(
apply_activation_rotation(transformer, metadata, logger = logger)
if scheme == "nvfp4":
+ from .diffusion_nvfp4_act_quant import install_nvfp4_act_quant_patch
from .diffusion_nvfp4_linear import convert_nvfp4_backend
+
+ # Layers left on torchao quantise activations through NVFP4Tensor.to_nvfp4 at run time.
+ install_nvfp4_act_quant_patch(logger)
from .diffusion_nvfp4_ops import select_nvfp4_backend
+
convert_nvfp4_backend(
transformer, metadata, select_nvfp4_backend(device), logger = logger
)
diff --git a/studio/backend/core/inference/diffusion_transformer_quant.py b/studio/backend/core/inference/diffusion_transformer_quant.py
index 1a9b8814b6e..b9e24287f54 100644
--- a/studio/backend/core/inference/diffusion_transformer_quant.py
+++ b/studio/backend/core/inference/diffusion_transformer_quant.py
@@ -1712,6 +1712,11 @@ def _make_quant_config(scheme: str, fast_accum: Optional[bool] = None) -> Any:
if scheme == TQ_NVFP4:
from torchao.prototype.mx_formats import NVFP4DynamicActivationNVFP4WeightConfig
+ from .diffusion_nvfp4_act_quant import install_nvfp4_act_quant_patch
+
+ # Before quantize_: the load-time weight pass stays on torchao (eager); compiled activations use the kernel.
+ install_nvfp4_act_quant_patch()
+
# Select the CUTLASS FP4 path, not the default Triton kernel (which needs MSLK): on a Blackwell box with
# CUTLASS FP4 but no MSLK the default fails the smoke probe and falls back to GGUF.
try:
diff --git a/studio/backend/tests/test_diffusion_nvfp4_act_quant.py b/studio/backend/tests/test_diffusion_nvfp4_act_quant.py
new file mode 100644
index 00000000000..f850c97dbe6
--- /dev/null
+++ b/studio/backend/tests/test_diffusion_nvfp4_act_quant.py
@@ -0,0 +1,91 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""The one-pass NVFP4 activation quantiser: kill switch, eligibility, and (on Blackwell) bit identity."""
+
+import pytest
+
+torch = pytest.importorskip("torch")
+
+from core.inference import diffusion_nvfp4_act_quant as aq # noqa: E402
+
+
+def _blackwell() -> bool:
+ try:
+ return torch.cuda.is_available() and torch.cuda.get_device_capability()[0] >= 10
+ except Exception: # noqa: BLE001
+ return False
+
+
+def test_kill_switch(monkeypatch):
+ monkeypatch.setenv(aq.FAST_ACT_QUANT_ENV, "0")
+ assert not aq.fast_act_quant_enabled()
+ assert aq.install_nvfp4_act_quant_patch() is False
+ assert aq.describe()["reason"].endswith("=0")
+ monkeypatch.delenv(aq.FAST_ACT_QUANT_ENV)
+ assert aq.fast_act_quant_enabled()
+
+
+def test_eligibility_is_structural():
+ s = torch.tensor([0.01])
+ x = torch.zeros(4, 32, dtype = torch.bfloat16)
+ # CPU tensors, wrong dtype, unswizzled scales, missing per-tensor scale, odd K: all stay on torchao.
+ assert not aq._eligible(x, 16, s, True, False)
+ assert not aq._eligible(x.float(), 16, s, True, False)
+ assert not aq._eligible(x, 16, None, True, False)
+ assert not aq._eligible(x, 16, s, False, False)
+ assert not aq._eligible(x, 16, s, True, True)
+
+
+def test_host_scalar_scale_falls_back_to_torchao():
+ # torchao accepts a CPU 0-d per-tensor scale with a CUDA activation; the kernel would read a host pointer.
+ from torch._subclasses.fake_tensor import FakeTensorMode
+ with FakeTensorMode(allow_non_fake_inputs = True):
+ x = torch.empty(4, 32, dtype = torch.bfloat16, device = "cuda")
+ assert aq._eligible(x, 16, torch.empty((), device = "cuda"), True, False)
+ assert not aq._eligible(x, 16, torch.empty(()), True, False)
+
+
+def test_install_without_cuda_is_a_noop(monkeypatch):
+ if torch.cuda.is_available():
+ pytest.skip("CUDA present")
+ pytest.importorskip("torchao.prototype.mx_formats.nvfp4_tensor")
+ monkeypatch.setitem(aq._STATE, "installed", False)
+ assert aq.install_nvfp4_act_quant_patch() is False
+ assert aq.describe()["reason"] == "cuda unavailable"
+
+
[email protected](not _blackwell(), reason = "needs sm_100+ for torchao NVFP4")
+def test_bit_identical_to_compiled_torchao():
+ pytest.importorskip("triton")
+ ok, reason = aq.verify()
+ assert ok, reason
+
+
+def test_fake_tensors_count_as_traced():
+ # AOTAutograd traces torchao's subclass dispatch with fake tensors while is_compiling() is False.
+ from torch._subclasses.fake_tensor import FakeTensorMode
+ assert not aq._traced(torch.zeros(2, 16))
+ with FakeTensorMode():
+ assert aq._traced(torch.empty(2, 16))
+
+
[email protected](not _blackwell(), reason = "needs sm_100+ for torchao NVFP4")
+def test_compiled_nvfp4_linear_launches_the_kernel():
+ pytest.importorskip("triton")
+ from torch.profiler import ProfilerActivity, profile
+ from torchao.prototype.mx_formats import NVFP4DynamicActivationNVFP4WeightConfig
+ from torchao.quantization import quantize_
+
+ assert aq.install_nvfp4_act_quant_patch()
+ m = torch.nn.Sequential(torch.nn.Linear(512, 512), torch.nn.GELU(), torch.nn.Linear(512, 512))
+ m = m.cuda().to(torch.bfloat16)
+ quantize_(m, NVFP4DynamicActivationNVFP4WeightConfig(use_triton_kernel = False))
+ f = torch.compile(m, fullgraph = True)
+ x = torch.randn(256, 512, device = "cuda", dtype = torch.bfloat16)
+ with torch.inference_mode():
+ f(x)
+ with profile(activities = [ProfilerActivity.CUDA]) as p:
+ f(x)
+ torch.cuda.synchronize()
+ assert sum(1 for e in p.events() if "act_quant_kernel" in e.name) == 2