Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

QAT: qat_scheme = "nvfp4", and strip NVFP4 / MX QAT linears before saving (#14512)

Daniel Han68de924b692e
diff --git a/tests/test_qat_microscaling_schemes.py b/tests/test_qat_microscaling_schemes.py
new file mode 100644
index 00000000000..13a2c76672c
--- /dev/null
+++ b/tests/test_qat_microscaling_schemes.py
@@ -0,0 +1,252 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""qat_scheme = "nvfp4" shorthand and saving torchao NVFP4 / MX QAT linears, CPU only."""
+
+from __future__ import annotations
+
+import importlib.util
+import warnings
+
+import pytest
+import torch
+
+pytestmark = pytest.mark.skipif(
+ importlib.util.find_spec("torchao") is None, reason = "require torchao"
+)
+
+import unsloth.models._utils as qat_utils
+import unsloth.save as save_mod
+from unsloth.models._utils import TorchAOConfig, _prepare_model_for_qat
+
+
+def _tiny_qwen3():
+ from transformers import Qwen3Config, Qwen3ForCausalLM
+
+ torch.manual_seed(0)
+ config = Qwen3Config(
+ vocab_size = 256,
+ hidden_size = 128,
+ intermediate_size = 256,
+ num_hidden_layers = 1,
+ num_attention_heads = 4,
+ num_key_value_heads = 1,
+ head_dim = 32,
+ tie_word_embeddings = False,
+ )
+ return Qwen3ForCausalLM(config).to(torch.bfloat16)
+
+
+def _mx_linear():
+ from torchao.prototype.qat import MXFakeQuantizedLinear
+ return MXFakeQuantizedLinear
+
+
+def _nvfp4_linear():
+ from torchao.prototype.qat import NVFP4FakeQuantizedLinear
+ return NVFP4FakeQuantizedLinear
+
+
+def test_nvfp4_scheme_refuses_without_fp4_gemm():
+ model = _tiny_qwen3()
+ with pytest.raises(RuntimeError, match = "FP4 tensor cores"):
+ _prepare_model_for_qat(model, "nvfp4")
+ assert not any(isinstance(m, _nvfp4_linear()) for m in model.modules())
+
+
+def test_nvfp4_scheme_layers_where_fp4_gemm_runs(monkeypatch):
+ from torchao.prototype.mx_formats import NVFP4DynamicActivationNVFP4WeightConfig
+
+ monkeypatch.setattr(qat_utils, "_qat_forward_runs", lambda *a: True)
+ model = _tiny_qwen3()
+ model.model.layers[0].odd = torch.nn.Linear(128, 40, bias = False)
+ with warnings.catch_warnings(record = True):
+ warnings.simplefilter("always")
+ model = _prepare_model_for_qat(model, "nvfp4")
+ ((base_config, _),) = model._torchao_config.base_config_and_filter_fns
+ assert isinstance(base_config, NVFP4DynamicActivationNVFP4WeightConfig)
+ assert (
+ base_config.use_triton_kernel is False
+ ) # the Triton quantizer needs the optional MSLK package
+ layer = model.model.layers[0]
+ for proj in (layer.self_attn.q_proj, layer.self_attn.k_proj, layer.mlp.down_proj):
+ assert isinstance(proj, _nvfp4_linear())
+ assert type(model.lm_head) is torch.nn.Linear # 4-bit head costs too much accuracy
+ assert type(layer.odd) is torch.nn.Linear
+
+
+def test_nvfp4_plain_module_without_hf_heads(monkeypatch):
+ monkeypatch.setattr(qat_utils, "_qat_forward_runs", lambda *a: True)
+ model = torch.nn.Sequential(torch.nn.Linear(64, 128, bias = False)).to(torch.bfloat16)
+ model = _prepare_model_for_qat(model, "nvfp4")
+ assert isinstance(model[0], _nvfp4_linear())
+
+
+def test_unknown_scheme_still_rejected():
+ with pytest.raises(ValueError, match = "Unexpected QAT scheme"):
+ _prepare_model_for_qat(_tiny_qwen3(), "mxfp8")
+
+
+def _mx_torchao_config():
+ from torchao.prototype.mx_formats import MXDynamicActivationMXWeightConfig
+ from torchao.quantization.quantize_.common.kernel_preference import KernelPreference
+ return TorchAOConfig(
+ qat_scheme = "custom-mx",
+ base_config_and_filter_fns = [
+ (
+ MXDynamicActivationMXWeightConfig(kernel_preference = KernelPreference.EMULATED),
+ lambda m, fqn: isinstance(m, torch.nn.Linear) and "lm_head" not in fqn,
+ )
+ ],
+ )
+
+
+def test_strip_restores_plain_linears():
+ model = _prepare_model_for_qat(_tiny_qwen3(), _mx_torchao_config())
+ q_proj = model.model.layers[0].self_attn.q_proj
+ weight = q_proj.weight
+ assert isinstance(q_proj, _mx_linear())
+ qat_utils._strip_torchao_fake_quantization(model)
+ assert not any(isinstance(m, _mx_linear()) for m in model.modules())
+ q_proj = model.model.layers[0].self_attn.q_proj
+ assert type(q_proj) is torch.nn.Linear
+ assert torch.equal(q_proj.weight, weight)
+
+
+def test_peft_save_merges_plain_linears(monkeypatch, tmp_path):
+ from peft import LoraConfig, get_peft_model
+
+ seen = {}
+ monkeypatch.setattr(
+ save_mod, "_unsloth_save_torchao_with_given_config", lambda **kw: seen.update(kw)
+ )
+ model = get_peft_model(_tiny_qwen3(), LoraConfig(r = 16, target_modules = ["q_proj", "down_proj"]))
+ model = _prepare_model_for_qat(model, _mx_torchao_config())
+ assert any(isinstance(m, _mx_linear()) for m in model.modules())
+ save_mod._unsloth_save_torchao_with_attached_config(
+ model = model, save_directory = str(tmp_path), tokenizer = object()
+ )
+ assert not any(isinstance(m, _mx_linear()) for m in model.modules())
+ assert seen["torchao_config"] is not None
+
+
+def _probe_ok(
+ monkeypatch,
+ bias_grad = True,
+ calls = None,
+ bad_widths = (),
+):
+ def forward_runs(
+ base_config,
+ device,
+ dtype = torch.bfloat16,
+ out_features = 128,
+ ):
+ if calls is not None and out_features == 128:
+ calls.append((str(device), dtype))
+ return out_features not in bad_widths
+
+ monkeypatch.setattr(qat_utils, "_qat_forward_runs", forward_runs)
+ monkeypatch.setattr(qat_utils, "_qat_bias_grad_supported", lambda *a: bias_grad, raising = False)
+
+
+def test_nvfp4_refuses_float16_layers(monkeypatch):
+ # torchao's non-Triton NVFP4 quantizer asserts bfloat16 / float32 at the first forward.
+ _probe_ok(monkeypatch)
+ model = _tiny_qwen3().to(torch.float16)
+ with pytest.raises(RuntimeError, match = "bfloat16 or float32"):
+ _prepare_model_for_qat(model, "nvfp4")
+ assert not any(isinstance(m, _nvfp4_linear()) for m in model.modules())
+
+
+def test_nvfp4_skips_peft_modules_to_save_head(monkeypatch):
+ from peft import LoraConfig, get_peft_model
+
+ _probe_ok(monkeypatch)
+ model = get_peft_model(
+ _tiny_qwen3(),
+ LoraConfig(r = 16, target_modules = ["q_proj"], modules_to_save = ["lm_head"]),
+ )
+ with warnings.catch_warnings():
+ warnings.simplefilter("ignore")
+ model = _prepare_model_for_qat(model, "nvfp4")
+ head = model.base_model.model.lm_head
+ assert not any(isinstance(m, _nvfp4_linear()) for m in head.modules())
+ assert isinstance(model.base_model.model.model.layers[0].mlp.down_proj, _nvfp4_linear())
+
+
+def test_nvfp4_probes_every_layer_placement(monkeypatch):
+ calls = []
+ _probe_ok(monkeypatch, calls = calls)
+ model = _tiny_qwen3()
+ model.model.layers[0].mlp.down_proj.to(torch.float32)
+ with warnings.catch_warnings():
+ warnings.simplefilter("ignore")
+ _prepare_model_for_qat(model, "nvfp4")
+ assert sorted(calls, key = str) == sorted(
+ [("cpu", torch.bfloat16), ("cpu", torch.float32)], key = str
+ )
+
+
+def test_nvfp4_keeps_failing_narrow_widths_in_full_precision(monkeypatch):
+ # tiny Qwen3 k_proj / v_proj have out_features 32 (one KV head of 32).
+ _probe_ok(monkeypatch, bad_widths = (32,))
+ model = _tiny_qwen3()
+ attn = model.model.layers[0].self_attn
+ with warnings.catch_warnings(record = True) as caught:
+ warnings.simplefilter("always")
+ model = _prepare_model_for_qat(model, "nvfp4")
+ assert not isinstance(attn.k_proj, _nvfp4_linear())
+ assert not isinstance(attn.v_proj, _nvfp4_linear())
+ assert isinstance(attn.q_proj, _nvfp4_linear())
+ assert any("out_features [32]" in str(w.message) for w in caught)
+
+
+def test_nvfp4_converts_narrow_widths_that_run(monkeypatch):
+ _probe_ok(monkeypatch)
+ model = _tiny_qwen3()
+ model = _prepare_model_for_qat(model, "nvfp4")
+ assert isinstance(model.model.layers[0].self_attn.k_proj, _nvfp4_linear())
+
+
+def test_nvfp4_probes_opted_in_adapter_placements(monkeypatch):
+ calls = []
+ _probe_ok(monkeypatch, calls = calls)
+ monkeypatch.setenv("UNSLOTH_QAT_FAKE_QUANTIZE_LORA", "1")
+ model = _tiny_qwen3()
+ layer = model.model.layers[0].mlp
+ layer.lora_A = torch.nn.Linear(256, 16, bias = False).to(torch.float32)
+ with warnings.catch_warnings():
+ warnings.simplefilter("ignore")
+ _prepare_model_for_qat(model, "nvfp4")
+ assert ("cpu", torch.float32) in calls
+
+
[email protected]("bias_grad", [False, True])
+def test_nvfp4_biased_layers_need_bias_grad(monkeypatch, bias_grad):
+ _probe_ok(monkeypatch, bias_grad = bias_grad)
+ model = _tiny_qwen3()
+ layer = model.model.layers[0]
+ layer.self_attn.q_proj = torch.nn.Linear(128, 128, bias = True).to(torch.bfloat16)
+ with warnings.catch_warnings(record = True) as caught:
+ warnings.simplefilter("always")
+ model = _prepare_model_for_qat(model, "nvfp4")
+ assert isinstance(layer.self_attn.q_proj, _nvfp4_linear()) is bias_grad
+ assert isinstance(layer.mlp.down_proj, _nvfp4_linear())
+ assert any("no bias" in str(w.message) for w in caught) is not bias_grad
+
+
+def test_nvfp4_accepts_rank4_inputs(monkeypatch):
+ linear_cls = _nvfp4_linear()
+
+ def fake_2d_forward(self, x):
+ assert x.dim() == 2, x.shape # torchao's addmm dispatch takes 2D operands only
+ return x @ self.weight.t()
+
+ _probe_ok(monkeypatch)
+ model = torch.nn.Sequential(torch.nn.Linear(64, 128, bias = False)).to(torch.bfloat16)
+ model = _prepare_model_for_qat(model, "nvfp4")
+ monkeypatch.setattr(linear_cls, "forward", fake_2d_forward)
+ for shape in [(6, 64), (2, 3, 64), (2, 3, 5, 64)]:
+ out = model(torch.randn(*shape, dtype = torch.bfloat16))
+ assert out.shape == (*shape[:-1], 128)
diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.py
index 3ab36f0bc29..2a061a81b70 100644
--- a/unsloth/models/_utils.py
+++ b/unsloth/models/_utils.py
@@ -7698,10 +7698,24 @@ def _strip_torchao_fake_quantization(model):
from torchao.quantization import quantize_
from torchao.quantization.qat import QATConfig
+ # torchao's NVFP4 / MX QAT linears fake-quantize inside forward() and carry no weight_fake_quantizer.
+ try:
+ from torchao.prototype import qat as prototype_qat
+ except Exception:
+ prototype_qat = None
+ forward_fake_quant = tuple(
+ cls
+ for cls in (
+ getattr(prototype_qat, "NVFP4FakeQuantizedLinear", None),
+ getattr(prototype_qat, "MXFakeQuantizedLinear", None),
+ )
+ if isinstance(cls, type)
+ )
quantize_(
model,
QATConfig(step = "convert"),
- filter_fn = lambda m, _: hasattr(m, "weight_fake_quantizer"),
+ filter_fn = lambda m, _: hasattr(m, "weight_fake_quantizer")
+ or isinstance(m, forward_fake_quant),
)
@@ -7863,6 +7877,122 @@ def _group_size_linear_filter(model, group_size, qat_scheme):
return filter_fn
+def _output_head_module_ids(model):
+ """Ids of the output head and every module under it: a PEFT modules_to_save wrapper holds nn.Linear copies."""
+ get_output_embeddings = getattr(model, "get_output_embeddings", None)
+ try:
+ out_head = get_output_embeddings() if get_output_embeddings is not None else None
+ except Exception:
+ out_head = None
+ if out_head is None:
+ return set()
+ return {id(m) for m in out_head.modules()}
+
+
+def _block_linear_filter(model, block_size, qat_scheme, skip_output_head):
+ """Linears with both dims divisible by the scale block, optionally excluding the output head."""
+ out_head_ids = _output_head_module_ids(model) if skip_output_head else set()
+
+ def eligible(m):
+ return isinstance(m, torch.nn.Linear) and id(m) not in out_head_ids
+
+ def blocks_ok(m):
+ return m.in_features % block_size == 0 and m.out_features % block_size == 0
+
+ filter_fn = lambda m, _: eligible(m) and blocks_ok(m)
+ skipped = [
+ name
+ for name, module in model.named_modules()
+ if eligible(module) and not blocks_ok(module) and not _is_lora_adapter_name(name)
+ ]
+ if skipped:
+ warnings.warn(
+ f"Unsloth: qat_scheme='{qat_scheme}' uses {block_size}-element scale blocks; these Linear "
+ f"layers have a dim not divisible by {block_size} and stay in full precision: "
+ + ", ".join(skipped[:8])
+ + (f", ... ({len(skipped) - 8} more)" if len(skipped) > 8 else ""),
+ stacklevel = 3,
+ )
+ return filter_fn
+
+
+def _qat_forward_runs(
+ base_config,
+ device,
+ dtype = torch.bfloat16,
+ out_features = 128,
+) -> bool:
+ """One tiny QAT forward with this base config, so a GPU / torch without the low precision GEMM is caught at prepare time."""
+ from torchao.quantization import quantize_
+ from torchao.quantization.qat import QATConfig
+
+ try:
+ probe = torch.nn.Sequential(torch.nn.Linear(64, out_features, bias = False)).to(
+ device = device, dtype = dtype
+ )
+ quantize_(probe, QATConfig(base_config, step = "prepare"))
+ with torch.no_grad():
+ probe(torch.ones(16, 64, device = device, dtype = dtype))
+ return True
+ except Exception:
+ return False
+
+
+def _qat_bias_grad_supported(
+ base_config,
+ device,
+ dtype = torch.bfloat16,
+) -> bool:
+ """torchao <= 0.18's NVFP4 QAT backward returns None for the bias: probe a biased layer's backward."""
+ from torchao.quantization import quantize_
+ from torchao.quantization.qat import QATConfig
+
+ try:
+ probe = torch.nn.Sequential(torch.nn.Linear(64, 128, bias = True)).to(
+ device = device, dtype = dtype
+ )
+ quantize_(probe, QATConfig(base_config, step = "prepare"))
+ x = torch.ones(16, 64, device = device, dtype = dtype, requires_grad = True)
+ probe(x).float().sum().backward()
+ return probe[0].bias.grad is not None
+ except Exception:
+ return False
+
+
+def _eligible_layer_placements(model, filter_fn):
+ """Distinct (device, dtype) of the layers a QAT filter selects, adapters excluded unless they are fake quantized too."""
+ with_adapters = os.environ.get("UNSLOTH_QAT_FAKE_QUANTIZE_LORA", "0") == "1"
+ placements = []
+ for name, module in model.named_modules():
+ if (_is_lora_adapter_name(name) and not with_adapters) or not filter_fn(module, name):
+ continue
+ key = (module.weight.device, module.weight.dtype)
+ if key not in placements:
+ placements.append(key)
+ return placements
+
+
+def _nvfp4_any_rank_forward(self, x):
+ # torchao's NVFP4FakeQuantizedLinear flattens only rank-3 inputs; vision layers see rank 4+.
+ from torchao.prototype.qat import NVFP4FakeQuantizedLinear
+
+ if x.dim() == 2:
+ return NVFP4FakeQuantizedLinear.forward(self, x)
+ out = NVFP4FakeQuantizedLinear.forward(self, x.reshape(-1, x.shape[-1]))
+ return out.view(*x.shape[:-1], out.shape[-1])
+
+
+def _accept_any_rank_nvfp4_inputs(model) -> None:
+ import types
+ try:
+ from torchao.prototype.qat import NVFP4FakeQuantizedLinear
+ except Exception:
+ return
+ for module in model.modules():
+ if isinstance(module, NVFP4FakeQuantizedLinear):
+ module.forward = types.MethodType(_nvfp4_any_rank_forward, module)
+
+
def _is_lora_adapter_name(fqn: str) -> bool:
return any(part.startswith("lora_") for part in fqn.split("."))
@@ -7925,6 +8055,93 @@ def _prepare_model_for_qat(
torchao_config = TorchAOConfig(
qat_scheme = qat_scheme, base_config_and_filter_fns = [(base_config, None)]
)
+ elif qat_scheme == "nvfp4":
+ try:
+ from torchao.prototype.mx_formats import NVFP4DynamicActivationNVFP4WeightConfig
+ except ImportError:
+ raise ImportError(TORCHAO_MSG)
+ # use_triton_kernel = False: the Triton quantizer needs optional MSLK, also at checkpoint load.
+ try:
+ base_config = NVFP4DynamicActivationNVFP4WeightConfig(use_triton_kernel = False)
+ except TypeError: # older torchao without the knob
+ base_config = NVFP4DynamicActivationNVFP4WeightConfig()
+ candidates = _block_linear_filter(model, 16, qat_scheme, skip_output_head = True)
+ placements = _eligible_layer_placements(model, candidates)
+ unsupported = sorted(
+ {
+ str(dtype)
+ for _, dtype in placements
+ if dtype not in (torch.bfloat16, torch.float32)
+ }
+ )
+ if unsupported:
+ raise RuntimeError(
+ "Unsloth: qat_scheme='nvfp4' quantizes bfloat16 or float32 activations only (torchao's "
+ f"NVFP4 quantizer); this model has {', '.join(unsupported)} layers. Load it with "
+ "dtype = torch.bfloat16."
+ )
+ for device, dtype in placements:
+ if not _qat_forward_runs(base_config, device, dtype):
+ raise RuntimeError(
+ "Unsloth: qat_scheme='nvfp4' runs NVFP4 matmuls in the forward pass and needs a GPU "
+ "with FP4 tensor cores (compute capability 10.0+: B200, GB200, RTX 50 series) and a "
+ f"torch / torchao that support it. Layers of this model are on {device}. Use "
+ "qat_scheme='fp8-fp8' instead."
+ )
+ # torchao's own NVFP4 filter skips out_features <= 64 (cuBLAS): probe each such width instead.
+ small_widths = sorted(
+ {
+ module.out_features
+ for name, module in model.named_modules()
+ if candidates(module, name) and module.out_features <= 64
+ }
+ )
+ bad_widths = {
+ n
+ for n in small_widths
+ if not all(_qat_forward_runs(base_config, d, t, n) for d, t in placements)
+ }
+ if bad_widths:
+ narrow = [
+ name
+ for name, module in model.named_modules()
+ if not _is_lora_adapter_name(name)
+ and candidates(module, name)
+ and module.out_features in bad_widths
+ ]
+ warnings.warn(
+ "Unsloth: qat_scheme='nvfp4': the NVFP4 matmul fails for out_features "
+ f"{sorted(bad_widths)} here, so these Linear layers stay in full precision: "
+ + ", ".join(narrow[:8])
+ + (f", ... ({len(narrow) - 8} more)" if len(narrow) > 8 else ""),
+ stacklevel = 2,
+ )
+ candidates = lambda m, fqn, _c = candidates, _bad = frozenset(bad_widths): (
+ _c(m, fqn) and m.out_features not in _bad
+ )
+ biased = [
+ name
+ for name, module in model.named_modules()
+ if not _is_lora_adapter_name(name)
+ and candidates(module, name)
+ and module.bias is not None
+ ]
+ filter_fn = candidates
+ if biased and not all(
+ _qat_bias_grad_supported(base_config, d, t) for d, t in placements
+ ):
+ warnings.warn(
+ "Unsloth: qat_scheme='nvfp4': this torchao's NVFP4 QAT backward returns no bias "
+ "gradient, so these biased Linear layers stay in full precision: "
+ + ", ".join(biased[:8])
+ + (f", ... ({len(biased) - 8} more)" if len(biased) > 8 else ""),
+ stacklevel = 2,
+ )
+ filter_fn = lambda m, fqn, _c = candidates: _c(m, fqn) and m.bias is None
+ torchao_config = TorchAOConfig(
+ qat_scheme = qat_scheme,
+ base_config_and_filter_fns = [(base_config, filter_fn)],
+ )
elif qat_scheme == "int8-int4":
try:
from torchao.quantization import (
@@ -8058,6 +8275,7 @@ def _prepare_model_for_qat(
for base_config, filter_fn in torchao_config.base_config_and_filter_fns:
quantize_(model, QATConfig(base_config, step = "prepare"), filter_fn = filter_fn)
_restore_fake_quantized_embedding_subclasses(model, embedding_subclasses)
+ _accept_any_rank_nvfp4_inputs(model)
return model