Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio NVFP4: 4over6 block scales at load and in the prequant builder (#17468)
Daniel Han952f49f7b314
diff --git a/scripts/build_prequant_checkpoint.py b/scripts/build_prequant_checkpoint.pyindex 0199930b69b..00735571676 100644--- a/scripts/build_prequant_checkpoint.py+++ b/scripts/build_prequant_checkpoint.py@@ -6,6 +6,7 @@from __future__ import annotationsimport argparse+import osimport sysimport timefrom pathlib import Path@@ -207,6 +208,13 @@ def main(argv = None) -> int:"exact fqn list is recorded in the checkpoint and the loader rotates the ""activations of that list and nothing else. Writes the v2 format tag.",)+ p.add_argument(+ "--nvfp4-weight-quant",+ default = None,+ choices = ["max6", "4over6"],+ help = "nvfp4 block-scale choice: max6 (torchao's) or 4over6 (per 16-block max/6 vs max/4, "+ "per-tensor scale for an E4M3 top of 256). Default: UNSLOTH_NVFP4_WEIGHT_QUANT, else 4over6.",+ )p.add_argument("--upload-repo", default = None, help = "optional HF repo id to upload the checkpoint to")@@ -228,6 +236,9 @@ def main(argv = None) -> int:flush = True,)return 2+ if (args.scheme or "").strip().lower() != "nvfp4" and args.nvfp4_weight_quant:+ print("error: --nvfp4-weight-quant applies to --scheme nvfp4 only", flush = True)+ return 2sys.path.insert(0, str(BACKEND))import torch@@ -365,6 +376,14 @@ def main(argv = None) -> int:flush = True,)+ weight_quant = None+ if scheme == "nvfp4":+ from core.inference.diffusion_nvfp4_weight_quant import WEIGHT_QUANT_ENV, resolve_method++ weight_quant = resolve_method(args.nvfp4_weight_quant)+ os.environ[WEIGHT_QUANT_ENV] = weight_quant+ print(f" nvfp4 weight scales: {weight_quant}", flush = True)+quantize_(transformer, _make_quant_config(scheme), filter_fn = filter_fn)state_dict = {@@ -388,6 +407,8 @@ def main(argv = None) -> int:"torchao_version": getattr(torchao, "__version__", "?"),"diffusers_version": diffusers.__version__,}+ if weight_quant is not None:+ metadata["nvfp4_weight_quant"] = weight_quantfrom core.inference.diffusion_prequant import canonical_weight_fingerprint# Version-neutral, so a file built on one torchao generation verifies on every other.diff --git a/studio/backend/core/inference/diffusion_nvfp4_weight_quant.py b/studio/backend/core/inference/diffusion_nvfp4_weight_quant.pynew file mode 100644index 00000000000..0a0d7280b8c--- /dev/null+++ b/studio/backend/core/inference/diffusion_nvfp4_weight_quant.py@@ -0,0 +1,383 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""NVFP4 weight quantisers (``max6`` and ``4over6``); output is the standard NVFP4 layout, so kernels are unchanged.++``4over6`` maps each 16-block max to 6 or 4 (lower squared error); its per-tensor scale is ``amax / (6 * 256)`` so+the largest block's /4 scale (384) still fits E4M3 (448).+"""++from __future__ import annotations++import os+import threading+from typing import Any, NamedTuple, Optional++WEIGHT_QUANT_ENV = "UNSLOTH_NVFP4_WEIGHT_QUANT"+METHOD_MAX6 = "max6"+METHOD_4OVER6 = "4over6"+METHODS = (METHOD_MAX6, METHOD_4OVER6)+DEFAULT_METHOD = METHOD_4OVER6+_ALIASES = {+ "max6": METHOD_MAX6,+ "max/6": METHOD_MAX6,+ "rtn": METHOD_MAX6,+ "torchao": METHOD_MAX6,+ "off": METHOD_MAX6,+ "0": METHOD_MAX6,+ "4over6": METHOD_4OVER6,+ "4/6": METHOD_4OVER6,+ "four_over_six": METHOD_4OVER6,+ "fouroversix": METHOD_4OVER6,+ "on": METHOD_4OVER6,+ "1": METHOD_4OVER6,+}++BLOCK = 16+E2M1_MAX = 6.0+E4M3_MAX = 448.0+E4M3_MAX_4OVER6 = 256.0+E4M3_TINY = 2.0**-6 # torch.finfo(torch.float8_e4m3fn).tiny+_CHUNK_ELEMS = 1 << 24++_STATS_LOCK = threading.Lock()+_STATS: dict = {"weights": 0, "blocks": 0, "blocks_at_4": 0, "passthrough": {}}+++class NVFP4Weight(NamedTuple):+ """``scale`` is row-major [N, K/16], not swizzled."""++ qdata: Any+ scale: Any+ per_tensor_scale: Any+ method: str+ blocks_at_4: int = 0+++def resolve_method(value: Optional[str] = None) -> str:+ raw = value if value is not None else os.environ.get(WEIGHT_QUANT_ENV)+ if raw is None:+ return DEFAULT_METHOD+ return _ALIASES.get(str(raw).strip().lower().replace("-", "_").replace(" ", ""), DEFAULT_METHOD)+++def e4m3_top(method: str) -> float:+ return E4M3_MAX_4OVER6 if method == METHOD_4OVER6 else E4M3_MAX+++def per_tensor_scale_for(w: Any, method: str = METHOD_4OVER6) -> Any:+ import torch+ amax = torch.max(torch.abs(w)).to(torch.float32)+ # An all-zero weight would give 0 and 0/0 block scales; any positive scale encodes it as zeros.+ return (amax / (e4m3_top(method) * E2M1_MAX)).clamp_min(torch.finfo(torch.float32).tiny)+++def e2m1_round(x: Any) -> Any:+ """fp32 values clamped to +-6 and rounded to the e2m1 grid, ties to even (torchao ``f32_to_f4_unpacked``)."""+ import torch++ x = x.clamp(-E2M1_MAX, E2M1_MAX)+ a = x.abs()+ # torch.round is round-half-even.+ step = torch.where(a < 2.0, 0.5, torch.where(a < 4.0, 1.0, 2.0))+ return torch.round(x / step) * step+++_E2M1_GRID = (0.0, 0.5, 1.0, 1.5, 2.0, 3.0, 4.0, 6.0)+++def e2m1_encode(q: Any) -> Any:+ """Sign in bit 3; -0.0 keeps its sign."""+ import torch++ a = q.abs()+ idx = torch.where(a < 2.0, (a * 2.0), torch.where(a < 4.0, a + 2.0, a / 2.0 + 4.0)).to(+ torch.uint8+ )+ sign = torch.signbit(q).to(torch.uint8) << 3+ return idx | sign+++def e2m1_decode(nibbles: Any) -> Any:+ import torch++ grid = torch.tensor(_E2M1_GRID, dtype = torch.float32, device = nibbles.device)+ mag = grid[(nibbles & 7).long()]+ return torch.where((nibbles & 8).bool(), -mag, mag)+++def pack_e2m1(q: Any) -> Any:+ """Element 2i in the low nibble."""+ n = e2m1_encode(q)+ return n[..., 0::2] | (n[..., 1::2] << 4)+++def unpack_e2m1(qdata: Any) -> Any:+ import torch++ lo = e2m1_decode(qdata & 0xF)+ hi = e2m1_decode(qdata >> 4)+ return torch.stack((lo, hi), dim = -1).reshape(*qdata.shape[:-1], qdata.shape[-1] * 2)+++def _blocks(x: Any, rows: int, k: int) -> Any:+ return x.float().reshape(rows, k // BLOCK, BLOCK)+++def _candidate(xb: Any, amax: Any, pts: Any, denom: float):+ import torch++ sf8 = torch.clamp((amax / denom) / pts, min = E4M3_TINY, max = E4M3_MAX).to(torch.float8_e4m3fn)+ sf32 = sf8.to(torch.float32)+ codes = e2m1_round(xb * ((1.0 / pts) / sf32).unsqueeze(-1))+ return codes, sf8, sf32+++def quantize_blocks(+ xb: Any,+ pts: Any,+ method: str,+ err_weight: Optional[Any] = None,+) -> tuple:+ """Returns (codes, e4m3 scale, took4); ``err_weight`` weights the 4over6 error."""+ import torch++ amax = torch.amax(torch.abs(xb), dim = -1)+ codes, sf8, sf32 = _candidate(xb, amax, pts, E2M1_MAX)+ if method != METHOD_4OVER6:+ return codes, sf8, torch.zeros_like(amax, dtype = torch.bool)+ codes4, sf8_4, sf32_4 = _candidate(xb, amax, pts, 4.0)+ e6 = (codes * (sf32 * pts).unsqueeze(-1) - xb) ** 2+ e4 = (codes4 * (sf32_4 * pts).unsqueeze(-1) - xb) ** 2+ if err_weight is not None:+ e6 = e6 * err_weight+ e4 = e4 * err_weight+ take = e4.sum(-1) < e6.sum(-1) # ties keep max/6+ return (+ torch.where(take.unsqueeze(-1), codes4, codes),+ torch.where(take, sf8_4, sf8),+ take,+ )+++def _check_shape(w: Any) -> tuple:+ if w.dim() != 2:+ raise ValueError(+ f"NVFP4 weight quantisation needs a 2-D weight, got shape {tuple(w.shape)}"+ )+ n, k = w.shape+ if k % BLOCK:+ raise ValueError(f"NVFP4 needs in_features divisible by {BLOCK}, got {k}")+ return n, k+++def quantize_nvfp4_weight(+ w: Any,+ method: Optional[str] = None,+ *,+ per_tensor_scale: Optional[Any] = None,+ err_weight: Optional[Any] = None,+) -> NVFP4Weight:+ """Round-to-nearest NVFP4 of a 2-D [N, K] weight; ``max6`` is bit-identical to ``NVFP4Tensor.to_nvfp4``."""+ import torch++ method = resolve_method(method)+ n, k = _check_shape(w)+ pts = per_tensor_scale_for(w, method) if per_tensor_scale is None else per_tensor_scale+ pts = pts.to(device = w.device, dtype = torch.float32).reshape(())+ ew = None+ if err_weight is not None:+ ew = err_weight.to(device = w.device, dtype = torch.float32).reshape(1, k // BLOCK, BLOCK)+ qdata = torch.empty((n, k // 2), dtype = torch.uint8, device = w.device)+ scale = torch.empty((n, k // BLOCK), dtype = torch.float8_e4m3fn, device = w.device)+ at4 = 0+ step = max(1, _CHUNK_ELEMS // max(k, 1))+ for r0 in range(0, n, step):+ r1 = min(n, r0 + step)+ codes, sf8, took = quantize_blocks(_blocks(w[r0:r1], r1 - r0, k), pts, method, ew)+ qdata[r0:r1] = pack_e2m1(codes.reshape(r1 - r0, k))+ scale[r0:r1] = sf8+ if method == METHOD_4OVER6:+ at4 += int(took.sum())+ _record(n * (k // BLOCK), at4)+ return NVFP4Weight(qdata, scale, pts, method, at4)+++def dequantize_nvfp4_weight(q: NVFP4Weight, dtype: Any = None) -> Any:+ import torch++ n, half = q.qdata.shape+ k = half * 2+ codes = unpack_e2m1(q.qdata).reshape(n, k // BLOCK, BLOCK)+ s = q.scale.to(torch.float32) * q.per_tensor_scale.to(torch.float32)+ out = (codes * s.unsqueeze(-1)).reshape(n, k)+ return out if dtype is None else out.to(dtype)+++def fake_quantize_nvfp4_weight(+ w: Any,+ method: Optional[str] = None,+ *,+ err_weight: Optional[Any] = None,+) -> Any:+ return dequantize_nvfp4_weight(quantize_nvfp4_weight(w, method, err_weight = err_weight), w.dtype)+++def fake_quantize_nvfp4_activation(x: Any) -> Any:+ """Dynamic NVFP4 activation quantisation round trip."""+ import torch++ sh = x.shape+ x2 = x.reshape(-1, sh[-1]).float()+ pts = per_tensor_scale_for(x2, METHOD_MAX6).clamp_min(1e-30)+ codes, sf8, _ = quantize_blocks(_blocks(x2, x2.shape[0], sh[-1]), pts, METHOD_MAX6)+ out = codes * (sf8.to(torch.float32) * pts).unsqueeze(-1)+ return out.reshape(sh).to(x.dtype)+++def to_torchao_tensor(+ q: NVFP4Weight,+ orig_dtype: Any,+ *,+ act_per_tensor_scale: Optional[Any] = None,+ use_triton_kernel: bool = False,+ act_quant_kwargs: Optional[Any] = None,+) -> Any:+ """Same ``NVFP4Tensor`` (swizzled scales) the weight transform builds."""+ from torchao.prototype.mx_formats.nvfp4_tensor import NVFP4Tensor+ from torchao.prototype.mx_formats.utils import (+ hp_data_dims_to_swizzled_scale_dims_nvfp4,+ to_blocked,+ )++ if act_quant_kwargs is None:+ from torchao.prototype.mx_formats.nvfp4_tensor import QuantizeTensorToNVFP4Kwargs+ act_quant_kwargs = QuantizeTensorToNVFP4Kwargs(is_swizzled_scales = True)+ n, half = q.qdata.shape+ k = half * 2+ scale_m, scale_k = hp_data_dims_to_swizzled_scale_dims_nvfp4(n, k)+ return NVFP4Tensor(+ q.qdata,+ to_blocked(q.scale).flatten().view(scale_m, scale_k),+ BLOCK,+ orig_dtype,+ q.per_tensor_scale,+ act_per_tensor_scale,+ True,+ use_triton_kernel,+ act_quant_kwargs,+ )+++def _record(blocks: int, at4: int) -> None:+ with _STATS_LOCK:+ _STATS["weights"] += 1+ _STATS["blocks"] += int(blocks)+ _STATS["blocks_at_4"] += int(at4)+++def _passthrough(why: str) -> None:+ with _STATS_LOCK:+ _STATS["passthrough"][why] = _STATS["passthrough"].get(why, 0) + 1+++def describe() -> dict:+ with _STATS_LOCK:+ out = dict(_STATS)+ out["passthrough"] = dict(_STATS["passthrough"])+ out["method"] = resolve_method()+ return out+++def _refusal(+ data_hp: Any, block_size: int, per_tensor_scale: Any, is_swizzled: bool, use_triton: bool+) -> str:+ import torch++ if not is_swizzled or use_triton or block_size != BLOCK:+ return "layout"+ if not isinstance(per_tensor_scale, torch.Tensor) or per_tensor_scale.numel() != 1:+ return "no per-tensor scale"+ if data_hp.dim() != 2 or data_hp.shape[-1] % BLOCK:+ return "shape"+ if data_hp.dtype not in (torch.bfloat16, torch.float16, torch.float32):+ return f"dtype {data_hp.dtype}"+ return ""+++def make_weight_to_nvfp4(inner: Any) -> Any:+ """Weight calls (they carry ``act_quant_kwargs``) use the selected method; all else goes to ``inner``."""++ def to_nvfp4(+ data_hp,+ block_size: int = 16,+ per_tensor_scale = None,+ act_per_tensor_scale = None,+ is_swizzled_scales: bool = False,+ use_triton_kernel: bool = False,+ act_quant_kwargs = None,+ ):+ import torch+ if act_quant_kwargs is not None and not torch.compiler.is_compiling():+ method = resolve_method()+ why = "" if method != METHOD_MAX6 else "max6"+ if not why:+ why = _refusal(+ data_hp, block_size, per_tensor_scale, is_swizzled_scales, use_triton_kernel+ )+ if not why:+ if data_hp.is_cuda:+ with torch.cuda.device(data_hp.device):+ q = quantize_nvfp4_weight(data_hp, method)+ else:+ q = quantize_nvfp4_weight(data_hp, method)+ pts = q.per_tensor_scale.reshape(per_tensor_scale.shape)+ return to_torchao_tensor(+ q._replace(per_tensor_scale = pts),+ data_hp.dtype,+ act_per_tensor_scale = act_per_tensor_scale,+ use_triton_kernel = use_triton_kernel,+ act_quant_kwargs = act_quant_kwargs,+ )+ _passthrough(why)+ return inner(+ data_hp,+ block_size,+ per_tensor_scale,+ act_per_tensor_scale,+ is_swizzled_scales,+ use_triton_kernel,+ act_quant_kwargs,+ )++ to_nvfp4._unsloth_weight_quant = True+ to_nvfp4._unsloth_inner = inner+ return to_nvfp4+++_PATCH_LOCK = threading.Lock()+++def install_nvfp4_weight_quant_patch() -> bool:+ """Idempotent; re-wraps if another patch replaced ``to_nvfp4``. False when torchao is missing."""+ with _PATCH_LOCK:+ try:+ from torchao.prototype.mx_formats.nvfp4_tensor import NVFP4Tensor+ except Exception: # noqa: BLE001 - no torchao: nothing quantises to NVFP4 through it+ return False+ current = NVFP4Tensor.to_nvfp4+ if getattr(current, "_unsloth_weight_quant", False):+ return True+ NVFP4Tensor.to_nvfp4 = staticmethod(make_weight_to_nvfp4(current))+ return True+++def uninstall_nvfp4_weight_quant_patch() -> None:+ with _PATCH_LOCK:+ try:+ from torchao.prototype.mx_formats.nvfp4_tensor import NVFP4Tensor+ except Exception: # noqa: BLE001+ return+ current = NVFP4Tensor.to_nvfp4+ if getattr(current, "_unsloth_weight_quant", False):+ NVFP4Tensor.to_nvfp4 = staticmethod(current._unsloth_inner)diff --git a/studio/backend/core/inference/diffusion_transformer_quant.py b/studio/backend/core/inference/diffusion_transformer_quant.pyindex 068ba48e648..fe20f0c26df 100644--- a/studio/backend/core/inference/diffusion_transformer_quant.py+++ b/studio/backend/core/inference/diffusion_transformer_quant.py@@ -2131,8 +2131,12 @@ def _make_quant_config(scheme: str, fast_accum: Optional[bool] = None) -> Any:from .diffusion_nvfp4_act_quant import install_nvfp4_act_quant_patch+ from .diffusion_nvfp4_weight_quant import install_nvfp4_weight_quant_patch+# Before quantize_: the load-time weight pass stays on torchao (eager); compiled activations use the kernel.install_nvfp4_act_quant_patch()+ # Weight scales per UNSLOTH_NVFP4_WEIGHT_QUANT.+ install_nvfp4_weight_quant_patch()# Select the CUTLASS FP4 path, not the default Triton kernel (which needs MSLK): on a Blackwell box with# CUTLASS FP4 but no MSLK the default fails the smoke probe and falls back to GGUF.diff --git a/studio/backend/tests/test_build_prequant_checkpoint.py b/studio/backend/tests/test_build_prequant_checkpoint.pyindex fef6cc05be3..7821dd7bb1f 100644--- a/studio/backend/tests/test_build_prequant_checkpoint.py+++ b/studio/backend/tests/test_build_prequant_checkpoint.py@@ -362,3 +362,19 @@ def test_int8_and_fp8_keep_their_pickle_upload_names():zimage = detect_family("Tongyi-MAI/Z-Image-Turbo")assert build.upload_destination(zimage, "int8", rotated = False) == "transformer_int8.pt"assert build.upload_destination(zimage, "fp8", rotated = False) == "transformer_fp8.pt"+++def test_the_weight_quant_flag_is_refused_off_nvfp4_before_any_load(capsys):+ build = _script()+ argv = [+ "--base",+ "Tongyi-MAI/Z-Image-Turbo",+ "--family",+ "z-image",+ "--scheme",+ "fp8",+ "--out",+ "x.pt",+ ]+ assert build.main([*argv, "--nvfp4-weight-quant", "4over6"]) == 2+ assert "error:" in capsys.readouterr().outdiff --git a/studio/backend/tests/test_diffusion_nvfp4_weight_quant.py b/studio/backend/tests/test_diffusion_nvfp4_weight_quant.pynew file mode 100644index 00000000000..804809a218e--- /dev/null+++ b/studio/backend/tests/test_diffusion_nvfp4_weight_quant.py@@ -0,0 +1,229 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""NVFP4 weight quantisers (max6 / 4over6): bit-identity with torchao, layout, round trip, patch."""++from __future__ import annotations++import pytest++torch = pytest.importorskip("torch")+nvfp4_tensor = pytest.importorskip("torchao.prototype.mx_formats.nvfp4_tensor")++from torchao.prototype.mx_formats.kernels import f32_to_f4_unpacked, pack_uint4, unpack_uint4+from torchao.prototype.mx_formats.nvfp4_tensor import (+ NVFP4Tensor,+ QuantizeTensorToNVFP4Kwargs,+ per_tensor_amax_to_scale,+)+from torchao.prototype.mx_formats.utils import from_blocked++from core.inference import diffusion_nvfp4_weight_quant as wq+++def _weight(+ n = 256,+ k = 512,+ seed = 0,+ dtype = torch.bfloat16,+):+ g = torch.Generator().manual_seed(seed)+ w = torch.randn(n, k, generator = g) * (torch.rand(n, 1, generator = g) * 0.1)+ w[0, :16] = 0+ w[1, 16:32] = -1e-7+ w[2, 3] = 0.9+ return w.to(dtype)+++def _torchao_weight(w):+ return NVFP4Tensor.to_nvfp4(+ w,+ per_tensor_scale = per_tensor_amax_to_scale(torch.max(torch.abs(w))),+ is_swizzled_scales = True,+ act_quant_kwargs = QuantizeTensorToNVFP4Kwargs(is_swizzled_scales = True),+ )+++def test_e2m1_round_and_pack_match_torchao():+ x = torch.linspace(-7.0, 7.0, 28001)+ x = torch.cat(+ [x, torch.tensor([0.25, 0.75, 1.25, 1.75, 2.5, 3.5, 5.0, -0.25, -5.0, -0.0, 0.0])]+ )+ x = x[: x.numel() // 2 * 2]+ want = pack_uint4(f32_to_f4_unpacked(x.clamp(-6, 6)))+ assert torch.equal(wq.pack_e2m1(wq.e2m1_round(x)), want)+ assert torch.equal(wq.unpack_e2m1(want), wq.e2m1_round(x))++[email protected]("dtype", [torch.bfloat16, torch.float32])[email protected]("shape", [(256, 512), (77, 3840), (300, 2560), (16, 64)])+def test_max6_is_bit_identical_to_torchao(shape, dtype):+ w = _weight(*shape, dtype = dtype)+ ref = _torchao_weight(w)+ q = wq.quantize_nvfp4_weight(w, "max6")+ assert torch.equal(q.qdata, ref.qdata)+ got = wq.to_torchao_tensor(q, w.dtype)+ assert got.scale.shape == ref.scale.shape+ assert torch.equal(got.scale.view(torch.uint8), ref.scale.view(torch.uint8))+ assert float(q.per_tensor_scale) == float(ref.per_tensor_scale)+ assert torch.equal(got.dequantize(torch.float32), ref.dequantize(torch.float32))+++def test_4over6_layout_scale_top_and_error():+ w = _weight(512, 1024)+ q = wq.quantize_nvfp4_weight(w, "4over6")+ amax = float(w.float().abs().max())+ assert float(q.per_tensor_scale) == pytest.approx(amax / (6 * 256), rel = 1e-7)+ assert float(q.scale.float().max()) <= 448.0+ assert 0 < q.blocks_at_4 < q.scale.numel()+ t = wq.to_torchao_tensor(q, w.dtype)+ assert (+ t.is_swizzled_scales and t.qdata.dtype == torch.uint8 and tuple(t.shape) == tuple(w.shape)+ )+ assert torch.equal(+ from_blocked(t.scale, 512, 1024 // 16).view(torch.uint8), q.scale.view(torch.uint8)+ )+ assert torch.equal(t.dequantize(torch.float32), wq.dequantize_nvfp4_weight(q))+ e6 = ((wq.fake_quantize_nvfp4_weight(w.float(), "max6") - w.float()) ** 2).sum()+ e46 = ((wq.dequantize_nvfp4_weight(q) - w.float()) ** 2).sum()+ assert e46 < 0.95 * e6+++def test_4over6_per_block_choice_is_never_worse():+ w = _weight(128, 256).float()+ pts = wq.per_tensor_scale_for(w, "4over6")+ xb = w.reshape(128, 16, 16)+ c46, s46, _ = wq.quantize_blocks(xb, pts, "4over6")+ c6, s6, _ = wq.quantize_blocks(xb, pts, "max6")+ e46 = ((c46 * (s46.float() * pts).unsqueeze(-1) - xb) ** 2).sum(-1)+ e6 = ((c6 * (s6.float() * pts).unsqueeze(-1) - xb) ** 2).sum(-1)+ assert bool((e46 <= e6).all())+++def test_activation_weighted_choice_uses_weights():+ w = _weight(64, 256, seed = 3).float()+ ew = torch.ones(256)+ ew[::16] = 1e6+ q = wq.quantize_nvfp4_weight(w, "4over6", err_weight = ew)+ dq = wq.dequantize_nvfp4_weight(q)+ plain = wq.dequantize_nvfp4_weight(wq.quantize_nvfp4_weight(w, "4over6"))+ assert ((dq - w)[:, ::16] ** 2).sum() <= ((plain - w)[:, ::16] ** 2).sum()+++def test_chunked_equals_unchunked(monkeypatch):+ w = _weight(300, 512)+ full = wq.quantize_nvfp4_weight(w, "4over6")+ monkeypatch.setattr(wq, "_CHUNK_ELEMS", 512 * 7)+ part = wq.quantize_nvfp4_weight(w, "4over6")+ assert torch.equal(full.qdata, part.qdata)+ assert torch.equal(full.scale.view(torch.uint8), part.scale.view(torch.uint8))+++def test_resolve_method(monkeypatch):+ monkeypatch.delenv(wq.WEIGHT_QUANT_ENV, raising = False)+ assert wq.resolve_method() == wq.DEFAULT_METHOD+ monkeypatch.setenv(wq.WEIGHT_QUANT_ENV, "max6")+ assert wq.resolve_method() == "max6"+ monkeypatch.setenv(wq.WEIGHT_QUANT_ENV, "0")+ assert wq.resolve_method() == "max6"+ monkeypatch.setenv(wq.WEIGHT_QUANT_ENV, "4over6")+ assert wq.resolve_method() == "4over6"+ monkeypatch.setenv(wq.WEIGHT_QUANT_ENV, "garbage")+ assert wq.resolve_method() == wq.DEFAULT_METHOD+++def test_patch_routes_weights_only(monkeypatch):+ inner = NVFP4Tensor.to_nvfp4+ calls = []+ w = _weight(256, 512)+ ref = _torchao_weight(w).qdata++ def spy(*args, **kwargs):+ calls.append(args[6] if len(args) > 6 else kwargs.get("act_quant_kwargs"))+ return inner(*args, **kwargs)++ monkeypatch.setattr(NVFP4Tensor, "to_nvfp4", staticmethod(spy))+ wq.install_nvfp4_weight_quant_patch()+ wq.install_nvfp4_weight_quant_patch()+ assert getattr(NVFP4Tensor.to_nvfp4, "_unsloth_weight_quant", False)+ assert NVFP4Tensor.to_nvfp4._unsloth_inner is spy+ kw = QuantizeTensorToNVFP4Kwargs(is_swizzled_scales = True)+ s = per_tensor_amax_to_scale(torch.max(torch.abs(w)))++ monkeypatch.setenv(wq.WEIGHT_QUANT_ENV, "4over6")+ t = NVFP4Tensor.to_nvfp4(w, 16, s, None, True, False, kw)+ want = wq.quantize_nvfp4_weight(w, "4over6")+ assert isinstance(t, NVFP4Tensor) and t.act_quant_kwargs is kw and t.is_swizzled_scales+ assert t.per_tensor_scale.shape == s.shape+ assert float(t.per_tensor_scale) == float(want.per_tensor_scale)+ assert torch.equal(t.qdata, want.qdata)+ assert calls == []++ a = NVFP4Tensor.to_nvfp4(w, 16, s, None, True, False, None)+ assert calls == [None] and torch.equal(a.qdata, ref)+ monkeypatch.setenv(wq.WEIGHT_QUANT_ENV, "max6")+ m = NVFP4Tensor.to_nvfp4(w, 16, s, None, True, False, kw)+ assert len(calls) == 2 and torch.equal(m.qdata, ref)+ monkeypatch.setenv(wq.WEIGHT_QUANT_ENV, "4over6")+ NVFP4Tensor.to_nvfp4(w, 16, s, None, False, False, kw)+ assert len(calls) == 3+ wq.uninstall_nvfp4_weight_quant_patch()+ assert NVFP4Tensor.to_nvfp4 is spy+++def test_prequant_safetensors_round_trip(tmp_path):+ pytest.importorskip("safetensors")+ from core.inference.prequant_safetensors import (+ safetensors_prequant_supported,+ save_prequant_safetensors,+ )++ if not safetensors_prequant_supported():+ pytest.skip("this torchao cannot write NVFP4 safetensors")+ from core.inference.diffusion_prequant import _load_prequant_checkpoint++ w = _weight(256, 512)+ sd = {+ "a.weight": wq.to_torchao_tensor(wq.quantize_nvfp4_weight(w, "4over6"), w.dtype),+ "b.weight": wq.to_torchao_tensor(wq.quantize_nvfp4_weight(w, "max6"), w.dtype),+ }+ path = tmp_path / "t.safetensors"+ save_prequant_safetensors(str(path), fmt = "nvfp4", state_dict = sd, metadata = {"scheme": "nvfp4"})+ ck = _load_prequant_checkpoint(str(path), scheme = "nvfp4")+ for key, want in sd.items():+ got = ck["state_dict"][key]+ assert type(got).__name__ == "NVFP4Tensor" and got.is_swizzled_scales+ assert torch.equal(got.qdata.view(torch.uint8), want.qdata)+ assert torch.equal(+ got.scale.reshape(-1).view(torch.uint8), want.scale.reshape(-1).view(torch.uint8)+ )+ assert float(got.per_tensor_scale) == float(want.per_tensor_scale)+++def test_make_quant_config_installs_the_weight_patch():+ from core.inference.diffusion_transformer_quant import _make_quant_config+ try:+ _make_quant_config("nvfp4")+ assert getattr(NVFP4Tensor.to_nvfp4, "_unsloth_weight_quant", False)+ finally:+ wq.uninstall_nvfp4_weight_quant_patch()+++class _Tiny(torch.nn.Module):+ def __init__(self):+ super().__init__()+ self.a = torch.nn.Linear(512, 512)+ self.b = torch.nn.Linear(512, 528)+ self.c = torch.nn.Linear(528, 512)++ def forward(self, x):+ return self.c(torch.relu(self.b(self.a(x))))++[email protected]("method", ["max6", "4over6"])+def test_an_all_zero_weight_round_trips_to_zeros(method):+ w = torch.zeros(64, 128, dtype = torch.bfloat16)+ q = wq.quantize_nvfp4_weight(w, method)+ assert float(q.per_tensor_scale) > 0+ out = wq.dequantize_nvfp4_weight(q)+ assert torch.isfinite(out).all() and not out.any()