Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio NVFP4: 4over6 block scales at load and in the prequant builder (#17468)

Daniel Han952f49f7b314
diff --git a/scripts/build_prequant_checkpoint.py b/scripts/build_prequant_checkpoint.py
index 0199930b69b..00735571676 100644
--- a/scripts/build_prequant_checkpoint.py
+++ b/scripts/build_prequant_checkpoint.py
@@ -6,6 +6,7 @@
from __future__ import annotations
import argparse
+import os
import sys
import time
from pathlib import Path
@@ -207,6 +208,13 @@ def main(argv = None) -> int:
"exact fqn list is recorded in the checkpoint and the loader rotates the "
"activations of that list and nothing else. Writes the v2 format tag.",
)
+ p.add_argument(
+ "--nvfp4-weight-quant",
+ default = None,
+ choices = ["max6", "4over6"],
+ help = "nvfp4 block-scale choice: max6 (torchao's) or 4over6 (per 16-block max/6 vs max/4, "
+ "per-tensor scale for an E4M3 top of 256). Default: UNSLOTH_NVFP4_WEIGHT_QUANT, else 4over6.",
+ )
p.add_argument(
"--upload-repo", default = None, help = "optional HF repo id to upload the checkpoint to"
)
@@ -228,6 +236,9 @@ def main(argv = None) -> int:
flush = True,
)
return 2
+ if (args.scheme or "").strip().lower() != "nvfp4" and args.nvfp4_weight_quant:
+ print("error: --nvfp4-weight-quant applies to --scheme nvfp4 only", flush = True)
+ return 2
sys.path.insert(0, str(BACKEND))
import torch
@@ -365,6 +376,14 @@ def main(argv = None) -> int:
flush = True,
)
+ weight_quant = None
+ if scheme == "nvfp4":
+ from core.inference.diffusion_nvfp4_weight_quant import WEIGHT_QUANT_ENV, resolve_method
+
+ weight_quant = resolve_method(args.nvfp4_weight_quant)
+ os.environ[WEIGHT_QUANT_ENV] = weight_quant
+ print(f" nvfp4 weight scales: {weight_quant}", flush = True)
+
quantize_(transformer, _make_quant_config(scheme), filter_fn = filter_fn)
state_dict = {
@@ -388,6 +407,8 @@ def main(argv = None) -> int:
"torchao_version": getattr(torchao, "__version__", "?"),
"diffusers_version": diffusers.__version__,
}
+ if weight_quant is not None:
+ metadata["nvfp4_weight_quant"] = weight_quant
from core.inference.diffusion_prequant import canonical_weight_fingerprint
# Version-neutral, so a file built on one torchao generation verifies on every other.
diff --git a/studio/backend/core/inference/diffusion_nvfp4_weight_quant.py b/studio/backend/core/inference/diffusion_nvfp4_weight_quant.py
new file mode 100644
index 00000000000..0a0d7280b8c
--- /dev/null
+++ b/studio/backend/core/inference/diffusion_nvfp4_weight_quant.py
@@ -0,0 +1,383 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""NVFP4 weight quantisers (``max6`` and ``4over6``); output is the standard NVFP4 layout, so kernels are unchanged.
+
+``4over6`` maps each 16-block max to 6 or 4 (lower squared error); its per-tensor scale is ``amax / (6 * 256)`` so
+the largest block's /4 scale (384) still fits E4M3 (448).
+"""
+
+from __future__ import annotations
+
+import os
+import threading
+from typing import Any, NamedTuple, Optional
+
+WEIGHT_QUANT_ENV = "UNSLOTH_NVFP4_WEIGHT_QUANT"
+METHOD_MAX6 = "max6"
+METHOD_4OVER6 = "4over6"
+METHODS = (METHOD_MAX6, METHOD_4OVER6)
+DEFAULT_METHOD = METHOD_4OVER6
+_ALIASES = {
+ "max6": METHOD_MAX6,
+ "max/6": METHOD_MAX6,
+ "rtn": METHOD_MAX6,
+ "torchao": METHOD_MAX6,
+ "off": METHOD_MAX6,
+ "0": METHOD_MAX6,
+ "4over6": METHOD_4OVER6,
+ "4/6": METHOD_4OVER6,
+ "four_over_six": METHOD_4OVER6,
+ "fouroversix": METHOD_4OVER6,
+ "on": METHOD_4OVER6,
+ "1": METHOD_4OVER6,
+}
+
+BLOCK = 16
+E2M1_MAX = 6.0
+E4M3_MAX = 448.0
+E4M3_MAX_4OVER6 = 256.0
+E4M3_TINY = 2.0**-6 # torch.finfo(torch.float8_e4m3fn).tiny
+_CHUNK_ELEMS = 1 << 24
+
+_STATS_LOCK = threading.Lock()
+_STATS: dict = {"weights": 0, "blocks": 0, "blocks_at_4": 0, "passthrough": {}}
+
+
+class NVFP4Weight(NamedTuple):
+ """``scale`` is row-major [N, K/16], not swizzled."""
+
+ qdata: Any
+ scale: Any
+ per_tensor_scale: Any
+ method: str
+ blocks_at_4: int = 0
+
+
+def resolve_method(value: Optional[str] = None) -> str:
+ raw = value if value is not None else os.environ.get(WEIGHT_QUANT_ENV)
+ if raw is None:
+ return DEFAULT_METHOD
+ return _ALIASES.get(str(raw).strip().lower().replace("-", "_").replace(" ", ""), DEFAULT_METHOD)
+
+
+def e4m3_top(method: str) -> float:
+ return E4M3_MAX_4OVER6 if method == METHOD_4OVER6 else E4M3_MAX
+
+
+def per_tensor_scale_for(w: Any, method: str = METHOD_4OVER6) -> Any:
+ import torch
+ amax = torch.max(torch.abs(w)).to(torch.float32)
+ # An all-zero weight would give 0 and 0/0 block scales; any positive scale encodes it as zeros.
+ return (amax / (e4m3_top(method) * E2M1_MAX)).clamp_min(torch.finfo(torch.float32).tiny)
+
+
+def e2m1_round(x: Any) -> Any:
+ """fp32 values clamped to +-6 and rounded to the e2m1 grid, ties to even (torchao ``f32_to_f4_unpacked``)."""
+ import torch
+
+ x = x.clamp(-E2M1_MAX, E2M1_MAX)
+ a = x.abs()
+ # torch.round is round-half-even.
+ step = torch.where(a < 2.0, 0.5, torch.where(a < 4.0, 1.0, 2.0))
+ return torch.round(x / step) * step
+
+
+_E2M1_GRID = (0.0, 0.5, 1.0, 1.5, 2.0, 3.0, 4.0, 6.0)
+
+
+def e2m1_encode(q: Any) -> Any:
+ """Sign in bit 3; -0.0 keeps its sign."""
+ import torch
+
+ a = q.abs()
+ idx = torch.where(a < 2.0, (a * 2.0), torch.where(a < 4.0, a + 2.0, a / 2.0 + 4.0)).to(
+ torch.uint8
+ )
+ sign = torch.signbit(q).to(torch.uint8) << 3
+ return idx | sign
+
+
+def e2m1_decode(nibbles: Any) -> Any:
+ import torch
+
+ grid = torch.tensor(_E2M1_GRID, dtype = torch.float32, device = nibbles.device)
+ mag = grid[(nibbles & 7).long()]
+ return torch.where((nibbles & 8).bool(), -mag, mag)
+
+
+def pack_e2m1(q: Any) -> Any:
+ """Element 2i in the low nibble."""
+ n = e2m1_encode(q)
+ return n[..., 0::2] | (n[..., 1::2] << 4)
+
+
+def unpack_e2m1(qdata: Any) -> Any:
+ import torch
+
+ lo = e2m1_decode(qdata & 0xF)
+ hi = e2m1_decode(qdata >> 4)
+ return torch.stack((lo, hi), dim = -1).reshape(*qdata.shape[:-1], qdata.shape[-1] * 2)
+
+
+def _blocks(x: Any, rows: int, k: int) -> Any:
+ return x.float().reshape(rows, k // BLOCK, BLOCK)
+
+
+def _candidate(xb: Any, amax: Any, pts: Any, denom: float):
+ import torch
+
+ sf8 = torch.clamp((amax / denom) / pts, min = E4M3_TINY, max = E4M3_MAX).to(torch.float8_e4m3fn)
+ sf32 = sf8.to(torch.float32)
+ codes = e2m1_round(xb * ((1.0 / pts) / sf32).unsqueeze(-1))
+ return codes, sf8, sf32
+
+
+def quantize_blocks(
+ xb: Any,
+ pts: Any,
+ method: str,
+ err_weight: Optional[Any] = None,
+) -> tuple:
+ """Returns (codes, e4m3 scale, took4); ``err_weight`` weights the 4over6 error."""
+ import torch
+
+ amax = torch.amax(torch.abs(xb), dim = -1)
+ codes, sf8, sf32 = _candidate(xb, amax, pts, E2M1_MAX)
+ if method != METHOD_4OVER6:
+ return codes, sf8, torch.zeros_like(amax, dtype = torch.bool)
+ codes4, sf8_4, sf32_4 = _candidate(xb, amax, pts, 4.0)
+ e6 = (codes * (sf32 * pts).unsqueeze(-1) - xb) ** 2
+ e4 = (codes4 * (sf32_4 * pts).unsqueeze(-1) - xb) ** 2
+ if err_weight is not None:
+ e6 = e6 * err_weight
+ e4 = e4 * err_weight
+ take = e4.sum(-1) < e6.sum(-1) # ties keep max/6
+ return (
+ torch.where(take.unsqueeze(-1), codes4, codes),
+ torch.where(take, sf8_4, sf8),
+ take,
+ )
+
+
+def _check_shape(w: Any) -> tuple:
+ if w.dim() != 2:
+ raise ValueError(
+ f"NVFP4 weight quantisation needs a 2-D weight, got shape {tuple(w.shape)}"
+ )
+ n, k = w.shape
+ if k % BLOCK:
+ raise ValueError(f"NVFP4 needs in_features divisible by {BLOCK}, got {k}")
+ return n, k
+
+
+def quantize_nvfp4_weight(
+ w: Any,
+ method: Optional[str] = None,
+ *,
+ per_tensor_scale: Optional[Any] = None,
+ err_weight: Optional[Any] = None,
+) -> NVFP4Weight:
+ """Round-to-nearest NVFP4 of a 2-D [N, K] weight; ``max6`` is bit-identical to ``NVFP4Tensor.to_nvfp4``."""
+ import torch
+
+ method = resolve_method(method)
+ n, k = _check_shape(w)
+ pts = per_tensor_scale_for(w, method) if per_tensor_scale is None else per_tensor_scale
+ pts = pts.to(device = w.device, dtype = torch.float32).reshape(())
+ ew = None
+ if err_weight is not None:
+ ew = err_weight.to(device = w.device, dtype = torch.float32).reshape(1, k // BLOCK, BLOCK)
+ qdata = torch.empty((n, k // 2), dtype = torch.uint8, device = w.device)
+ scale = torch.empty((n, k // BLOCK), dtype = torch.float8_e4m3fn, device = w.device)
+ at4 = 0
+ step = max(1, _CHUNK_ELEMS // max(k, 1))
+ for r0 in range(0, n, step):
+ r1 = min(n, r0 + step)
+ codes, sf8, took = quantize_blocks(_blocks(w[r0:r1], r1 - r0, k), pts, method, ew)
+ qdata[r0:r1] = pack_e2m1(codes.reshape(r1 - r0, k))
+ scale[r0:r1] = sf8
+ if method == METHOD_4OVER6:
+ at4 += int(took.sum())
+ _record(n * (k // BLOCK), at4)
+ return NVFP4Weight(qdata, scale, pts, method, at4)
+
+
+def dequantize_nvfp4_weight(q: NVFP4Weight, dtype: Any = None) -> Any:
+ import torch
+
+ n, half = q.qdata.shape
+ k = half * 2
+ codes = unpack_e2m1(q.qdata).reshape(n, k // BLOCK, BLOCK)
+ s = q.scale.to(torch.float32) * q.per_tensor_scale.to(torch.float32)
+ out = (codes * s.unsqueeze(-1)).reshape(n, k)
+ return out if dtype is None else out.to(dtype)
+
+
+def fake_quantize_nvfp4_weight(
+ w: Any,
+ method: Optional[str] = None,
+ *,
+ err_weight: Optional[Any] = None,
+) -> Any:
+ return dequantize_nvfp4_weight(quantize_nvfp4_weight(w, method, err_weight = err_weight), w.dtype)
+
+
+def fake_quantize_nvfp4_activation(x: Any) -> Any:
+ """Dynamic NVFP4 activation quantisation round trip."""
+ import torch
+
+ sh = x.shape
+ x2 = x.reshape(-1, sh[-1]).float()
+ pts = per_tensor_scale_for(x2, METHOD_MAX6).clamp_min(1e-30)
+ codes, sf8, _ = quantize_blocks(_blocks(x2, x2.shape[0], sh[-1]), pts, METHOD_MAX6)
+ out = codes * (sf8.to(torch.float32) * pts).unsqueeze(-1)
+ return out.reshape(sh).to(x.dtype)
+
+
+def to_torchao_tensor(
+ q: NVFP4Weight,
+ orig_dtype: Any,
+ *,
+ act_per_tensor_scale: Optional[Any] = None,
+ use_triton_kernel: bool = False,
+ act_quant_kwargs: Optional[Any] = None,
+) -> Any:
+ """Same ``NVFP4Tensor`` (swizzled scales) the weight transform builds."""
+ from torchao.prototype.mx_formats.nvfp4_tensor import NVFP4Tensor
+ from torchao.prototype.mx_formats.utils import (
+ hp_data_dims_to_swizzled_scale_dims_nvfp4,
+ to_blocked,
+ )
+
+ if act_quant_kwargs is None:
+ from torchao.prototype.mx_formats.nvfp4_tensor import QuantizeTensorToNVFP4Kwargs
+ act_quant_kwargs = QuantizeTensorToNVFP4Kwargs(is_swizzled_scales = True)
+ n, half = q.qdata.shape
+ k = half * 2
+ scale_m, scale_k = hp_data_dims_to_swizzled_scale_dims_nvfp4(n, k)
+ return NVFP4Tensor(
+ q.qdata,
+ to_blocked(q.scale).flatten().view(scale_m, scale_k),
+ BLOCK,
+ orig_dtype,
+ q.per_tensor_scale,
+ act_per_tensor_scale,
+ True,
+ use_triton_kernel,
+ act_quant_kwargs,
+ )
+
+
+def _record(blocks: int, at4: int) -> None:
+ with _STATS_LOCK:
+ _STATS["weights"] += 1
+ _STATS["blocks"] += int(blocks)
+ _STATS["blocks_at_4"] += int(at4)
+
+
+def _passthrough(why: str) -> None:
+ with _STATS_LOCK:
+ _STATS["passthrough"][why] = _STATS["passthrough"].get(why, 0) + 1
+
+
+def describe() -> dict:
+ with _STATS_LOCK:
+ out = dict(_STATS)
+ out["passthrough"] = dict(_STATS["passthrough"])
+ out["method"] = resolve_method()
+ return out
+
+
+def _refusal(
+ data_hp: Any, block_size: int, per_tensor_scale: Any, is_swizzled: bool, use_triton: bool
+) -> str:
+ import torch
+
+ if not is_swizzled or use_triton or block_size != BLOCK:
+ return "layout"
+ if not isinstance(per_tensor_scale, torch.Tensor) or per_tensor_scale.numel() != 1:
+ return "no per-tensor scale"
+ if data_hp.dim() != 2 or data_hp.shape[-1] % BLOCK:
+ return "shape"
+ if data_hp.dtype not in (torch.bfloat16, torch.float16, torch.float32):
+ return f"dtype {data_hp.dtype}"
+ return ""
+
+
+def make_weight_to_nvfp4(inner: Any) -> Any:
+ """Weight calls (they carry ``act_quant_kwargs``) use the selected method; all else goes to ``inner``."""
+
+ def to_nvfp4(
+ data_hp,
+ block_size: int = 16,
+ per_tensor_scale = None,
+ act_per_tensor_scale = None,
+ is_swizzled_scales: bool = False,
+ use_triton_kernel: bool = False,
+ act_quant_kwargs = None,
+ ):
+ import torch
+ if act_quant_kwargs is not None and not torch.compiler.is_compiling():
+ method = resolve_method()
+ why = "" if method != METHOD_MAX6 else "max6"
+ if not why:
+ why = _refusal(
+ data_hp, block_size, per_tensor_scale, is_swizzled_scales, use_triton_kernel
+ )
+ if not why:
+ if data_hp.is_cuda:
+ with torch.cuda.device(data_hp.device):
+ q = quantize_nvfp4_weight(data_hp, method)
+ else:
+ q = quantize_nvfp4_weight(data_hp, method)
+ pts = q.per_tensor_scale.reshape(per_tensor_scale.shape)
+ return to_torchao_tensor(
+ q._replace(per_tensor_scale = pts),
+ data_hp.dtype,
+ act_per_tensor_scale = act_per_tensor_scale,
+ use_triton_kernel = use_triton_kernel,
+ act_quant_kwargs = act_quant_kwargs,
+ )
+ _passthrough(why)
+ return inner(
+ data_hp,
+ block_size,
+ per_tensor_scale,
+ act_per_tensor_scale,
+ is_swizzled_scales,
+ use_triton_kernel,
+ act_quant_kwargs,
+ )
+
+ to_nvfp4._unsloth_weight_quant = True
+ to_nvfp4._unsloth_inner = inner
+ return to_nvfp4
+
+
+_PATCH_LOCK = threading.Lock()
+
+
+def install_nvfp4_weight_quant_patch() -> bool:
+ """Idempotent; re-wraps if another patch replaced ``to_nvfp4``. False when torchao is missing."""
+ with _PATCH_LOCK:
+ try:
+ from torchao.prototype.mx_formats.nvfp4_tensor import NVFP4Tensor
+ except Exception: # noqa: BLE001 - no torchao: nothing quantises to NVFP4 through it
+ return False
+ current = NVFP4Tensor.to_nvfp4
+ if getattr(current, "_unsloth_weight_quant", False):
+ return True
+ NVFP4Tensor.to_nvfp4 = staticmethod(make_weight_to_nvfp4(current))
+ return True
+
+
+def uninstall_nvfp4_weight_quant_patch() -> None:
+ with _PATCH_LOCK:
+ try:
+ from torchao.prototype.mx_formats.nvfp4_tensor import NVFP4Tensor
+ except Exception: # noqa: BLE001
+ return
+ current = NVFP4Tensor.to_nvfp4
+ if getattr(current, "_unsloth_weight_quant", False):
+ NVFP4Tensor.to_nvfp4 = staticmethod(current._unsloth_inner)
diff --git a/studio/backend/core/inference/diffusion_transformer_quant.py b/studio/backend/core/inference/diffusion_transformer_quant.py
index 068ba48e648..fe20f0c26df 100644
--- a/studio/backend/core/inference/diffusion_transformer_quant.py
+++ b/studio/backend/core/inference/diffusion_transformer_quant.py
@@ -2131,8 +2131,12 @@ def _make_quant_config(scheme: str, fast_accum: Optional[bool] = None) -> Any:
from .diffusion_nvfp4_act_quant import install_nvfp4_act_quant_patch
+ from .diffusion_nvfp4_weight_quant import install_nvfp4_weight_quant_patch
+
# Before quantize_: the load-time weight pass stays on torchao (eager); compiled activations use the kernel.
install_nvfp4_act_quant_patch()
+ # Weight scales per UNSLOTH_NVFP4_WEIGHT_QUANT.
+ install_nvfp4_weight_quant_patch()
# Select the CUTLASS FP4 path, not the default Triton kernel (which needs MSLK): on a Blackwell box with
# CUTLASS FP4 but no MSLK the default fails the smoke probe and falls back to GGUF.
diff --git a/studio/backend/tests/test_build_prequant_checkpoint.py b/studio/backend/tests/test_build_prequant_checkpoint.py
index fef6cc05be3..7821dd7bb1f 100644
--- a/studio/backend/tests/test_build_prequant_checkpoint.py
+++ b/studio/backend/tests/test_build_prequant_checkpoint.py
@@ -362,3 +362,19 @@ def test_int8_and_fp8_keep_their_pickle_upload_names():
zimage = detect_family("Tongyi-MAI/Z-Image-Turbo")
assert build.upload_destination(zimage, "int8", rotated = False) == "transformer_int8.pt"
assert build.upload_destination(zimage, "fp8", rotated = False) == "transformer_fp8.pt"
+
+
+def test_the_weight_quant_flag_is_refused_off_nvfp4_before_any_load(capsys):
+ build = _script()
+ argv = [
+ "--base",
+ "Tongyi-MAI/Z-Image-Turbo",
+ "--family",
+ "z-image",
+ "--scheme",
+ "fp8",
+ "--out",
+ "x.pt",
+ ]
+ assert build.main([*argv, "--nvfp4-weight-quant", "4over6"]) == 2
+ assert "error:" in capsys.readouterr().out
diff --git a/studio/backend/tests/test_diffusion_nvfp4_weight_quant.py b/studio/backend/tests/test_diffusion_nvfp4_weight_quant.py
new file mode 100644
index 00000000000..804809a218e
--- /dev/null
+++ b/studio/backend/tests/test_diffusion_nvfp4_weight_quant.py
@@ -0,0 +1,229 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""NVFP4 weight quantisers (max6 / 4over6): bit-identity with torchao, layout, round trip, patch."""
+
+from __future__ import annotations
+
+import pytest
+
+torch = pytest.importorskip("torch")
+nvfp4_tensor = pytest.importorskip("torchao.prototype.mx_formats.nvfp4_tensor")
+
+from torchao.prototype.mx_formats.kernels import f32_to_f4_unpacked, pack_uint4, unpack_uint4
+from torchao.prototype.mx_formats.nvfp4_tensor import (
+ NVFP4Tensor,
+ QuantizeTensorToNVFP4Kwargs,
+ per_tensor_amax_to_scale,
+)
+from torchao.prototype.mx_formats.utils import from_blocked
+
+from core.inference import diffusion_nvfp4_weight_quant as wq
+
+
+def _weight(
+ n = 256,
+ k = 512,
+ seed = 0,
+ dtype = torch.bfloat16,
+):
+ g = torch.Generator().manual_seed(seed)
+ w = torch.randn(n, k, generator = g) * (torch.rand(n, 1, generator = g) * 0.1)
+ w[0, :16] = 0
+ w[1, 16:32] = -1e-7
+ w[2, 3] = 0.9
+ return w.to(dtype)
+
+
+def _torchao_weight(w):
+ return NVFP4Tensor.to_nvfp4(
+ w,
+ per_tensor_scale = per_tensor_amax_to_scale(torch.max(torch.abs(w))),
+ is_swizzled_scales = True,
+ act_quant_kwargs = QuantizeTensorToNVFP4Kwargs(is_swizzled_scales = True),
+ )
+
+
+def test_e2m1_round_and_pack_match_torchao():
+ x = torch.linspace(-7.0, 7.0, 28001)
+ x = torch.cat(
+ [x, torch.tensor([0.25, 0.75, 1.25, 1.75, 2.5, 3.5, 5.0, -0.25, -5.0, -0.0, 0.0])]
+ )
+ x = x[: x.numel() // 2 * 2]
+ want = pack_uint4(f32_to_f4_unpacked(x.clamp(-6, 6)))
+ assert torch.equal(wq.pack_e2m1(wq.e2m1_round(x)), want)
+ assert torch.equal(wq.unpack_e2m1(want), wq.e2m1_round(x))
+
+
[email protected]("dtype", [torch.bfloat16, torch.float32])
[email protected]("shape", [(256, 512), (77, 3840), (300, 2560), (16, 64)])
+def test_max6_is_bit_identical_to_torchao(shape, dtype):
+ w = _weight(*shape, dtype = dtype)
+ ref = _torchao_weight(w)
+ q = wq.quantize_nvfp4_weight(w, "max6")
+ assert torch.equal(q.qdata, ref.qdata)
+ got = wq.to_torchao_tensor(q, w.dtype)
+ assert got.scale.shape == ref.scale.shape
+ assert torch.equal(got.scale.view(torch.uint8), ref.scale.view(torch.uint8))
+ assert float(q.per_tensor_scale) == float(ref.per_tensor_scale)
+ assert torch.equal(got.dequantize(torch.float32), ref.dequantize(torch.float32))
+
+
+def test_4over6_layout_scale_top_and_error():
+ w = _weight(512, 1024)
+ q = wq.quantize_nvfp4_weight(w, "4over6")
+ amax = float(w.float().abs().max())
+ assert float(q.per_tensor_scale) == pytest.approx(amax / (6 * 256), rel = 1e-7)
+ assert float(q.scale.float().max()) <= 448.0
+ assert 0 < q.blocks_at_4 < q.scale.numel()
+ t = wq.to_torchao_tensor(q, w.dtype)
+ assert (
+ t.is_swizzled_scales and t.qdata.dtype == torch.uint8 and tuple(t.shape) == tuple(w.shape)
+ )
+ assert torch.equal(
+ from_blocked(t.scale, 512, 1024 // 16).view(torch.uint8), q.scale.view(torch.uint8)
+ )
+ assert torch.equal(t.dequantize(torch.float32), wq.dequantize_nvfp4_weight(q))
+ e6 = ((wq.fake_quantize_nvfp4_weight(w.float(), "max6") - w.float()) ** 2).sum()
+ e46 = ((wq.dequantize_nvfp4_weight(q) - w.float()) ** 2).sum()
+ assert e46 < 0.95 * e6
+
+
+def test_4over6_per_block_choice_is_never_worse():
+ w = _weight(128, 256).float()
+ pts = wq.per_tensor_scale_for(w, "4over6")
+ xb = w.reshape(128, 16, 16)
+ c46, s46, _ = wq.quantize_blocks(xb, pts, "4over6")
+ c6, s6, _ = wq.quantize_blocks(xb, pts, "max6")
+ e46 = ((c46 * (s46.float() * pts).unsqueeze(-1) - xb) ** 2).sum(-1)
+ e6 = ((c6 * (s6.float() * pts).unsqueeze(-1) - xb) ** 2).sum(-1)
+ assert bool((e46 <= e6).all())
+
+
+def test_activation_weighted_choice_uses_weights():
+ w = _weight(64, 256, seed = 3).float()
+ ew = torch.ones(256)
+ ew[::16] = 1e6
+ q = wq.quantize_nvfp4_weight(w, "4over6", err_weight = ew)
+ dq = wq.dequantize_nvfp4_weight(q)
+ plain = wq.dequantize_nvfp4_weight(wq.quantize_nvfp4_weight(w, "4over6"))
+ assert ((dq - w)[:, ::16] ** 2).sum() <= ((plain - w)[:, ::16] ** 2).sum()
+
+
+def test_chunked_equals_unchunked(monkeypatch):
+ w = _weight(300, 512)
+ full = wq.quantize_nvfp4_weight(w, "4over6")
+ monkeypatch.setattr(wq, "_CHUNK_ELEMS", 512 * 7)
+ part = wq.quantize_nvfp4_weight(w, "4over6")
+ assert torch.equal(full.qdata, part.qdata)
+ assert torch.equal(full.scale.view(torch.uint8), part.scale.view(torch.uint8))
+
+
+def test_resolve_method(monkeypatch):
+ monkeypatch.delenv(wq.WEIGHT_QUANT_ENV, raising = False)
+ assert wq.resolve_method() == wq.DEFAULT_METHOD
+ monkeypatch.setenv(wq.WEIGHT_QUANT_ENV, "max6")
+ assert wq.resolve_method() == "max6"
+ monkeypatch.setenv(wq.WEIGHT_QUANT_ENV, "0")
+ assert wq.resolve_method() == "max6"
+ monkeypatch.setenv(wq.WEIGHT_QUANT_ENV, "4over6")
+ assert wq.resolve_method() == "4over6"
+ monkeypatch.setenv(wq.WEIGHT_QUANT_ENV, "garbage")
+ assert wq.resolve_method() == wq.DEFAULT_METHOD
+
+
+def test_patch_routes_weights_only(monkeypatch):
+ inner = NVFP4Tensor.to_nvfp4
+ calls = []
+ w = _weight(256, 512)
+ ref = _torchao_weight(w).qdata
+
+ def spy(*args, **kwargs):
+ calls.append(args[6] if len(args) > 6 else kwargs.get("act_quant_kwargs"))
+ return inner(*args, **kwargs)
+
+ monkeypatch.setattr(NVFP4Tensor, "to_nvfp4", staticmethod(spy))
+ wq.install_nvfp4_weight_quant_patch()
+ wq.install_nvfp4_weight_quant_patch()
+ assert getattr(NVFP4Tensor.to_nvfp4, "_unsloth_weight_quant", False)
+ assert NVFP4Tensor.to_nvfp4._unsloth_inner is spy
+ kw = QuantizeTensorToNVFP4Kwargs(is_swizzled_scales = True)
+ s = per_tensor_amax_to_scale(torch.max(torch.abs(w)))
+
+ monkeypatch.setenv(wq.WEIGHT_QUANT_ENV, "4over6")
+ t = NVFP4Tensor.to_nvfp4(w, 16, s, None, True, False, kw)
+ want = wq.quantize_nvfp4_weight(w, "4over6")
+ assert isinstance(t, NVFP4Tensor) and t.act_quant_kwargs is kw and t.is_swizzled_scales
+ assert t.per_tensor_scale.shape == s.shape
+ assert float(t.per_tensor_scale) == float(want.per_tensor_scale)
+ assert torch.equal(t.qdata, want.qdata)
+ assert calls == []
+
+ a = NVFP4Tensor.to_nvfp4(w, 16, s, None, True, False, None)
+ assert calls == [None] and torch.equal(a.qdata, ref)
+ monkeypatch.setenv(wq.WEIGHT_QUANT_ENV, "max6")
+ m = NVFP4Tensor.to_nvfp4(w, 16, s, None, True, False, kw)
+ assert len(calls) == 2 and torch.equal(m.qdata, ref)
+ monkeypatch.setenv(wq.WEIGHT_QUANT_ENV, "4over6")
+ NVFP4Tensor.to_nvfp4(w, 16, s, None, False, False, kw)
+ assert len(calls) == 3
+ wq.uninstall_nvfp4_weight_quant_patch()
+ assert NVFP4Tensor.to_nvfp4 is spy
+
+
+def test_prequant_safetensors_round_trip(tmp_path):
+ pytest.importorskip("safetensors")
+ from core.inference.prequant_safetensors import (
+ safetensors_prequant_supported,
+ save_prequant_safetensors,
+ )
+
+ if not safetensors_prequant_supported():
+ pytest.skip("this torchao cannot write NVFP4 safetensors")
+ from core.inference.diffusion_prequant import _load_prequant_checkpoint
+
+ w = _weight(256, 512)
+ sd = {
+ "a.weight": wq.to_torchao_tensor(wq.quantize_nvfp4_weight(w, "4over6"), w.dtype),
+ "b.weight": wq.to_torchao_tensor(wq.quantize_nvfp4_weight(w, "max6"), w.dtype),
+ }
+ path = tmp_path / "t.safetensors"
+ save_prequant_safetensors(str(path), fmt = "nvfp4", state_dict = sd, metadata = {"scheme": "nvfp4"})
+ ck = _load_prequant_checkpoint(str(path), scheme = "nvfp4")
+ for key, want in sd.items():
+ got = ck["state_dict"][key]
+ assert type(got).__name__ == "NVFP4Tensor" and got.is_swizzled_scales
+ assert torch.equal(got.qdata.view(torch.uint8), want.qdata)
+ assert torch.equal(
+ got.scale.reshape(-1).view(torch.uint8), want.scale.reshape(-1).view(torch.uint8)
+ )
+ assert float(got.per_tensor_scale) == float(want.per_tensor_scale)
+
+
+def test_make_quant_config_installs_the_weight_patch():
+ from core.inference.diffusion_transformer_quant import _make_quant_config
+ try:
+ _make_quant_config("nvfp4")
+ assert getattr(NVFP4Tensor.to_nvfp4, "_unsloth_weight_quant", False)
+ finally:
+ wq.uninstall_nvfp4_weight_quant_patch()
+
+
+class _Tiny(torch.nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.a = torch.nn.Linear(512, 512)
+ self.b = torch.nn.Linear(512, 528)
+ self.c = torch.nn.Linear(528, 512)
+
+ def forward(self, x):
+ return self.c(torch.relu(self.b(self.a(x))))
+
+
[email protected]("method", ["max6", "4over6"])
+def test_an_all_zero_weight_round_trips_to_zeros(method):
+ w = torch.zeros(64, 128, dtype = torch.bfloat16)
+ q = wq.quantize_nvfp4_weight(w, method)
+ assert float(q.per_tensor_scale) > 0
+ out = wq.dequantize_nvfp4_weight(q)
+ assert torch.isfinite(out).all() and not out.any()