Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
QAT: qat_scheme = "nvfp4", and strip NVFP4 / MX QAT linears before saving (#14512)
Daniel Han68de924b692e
diff --git a/tests/test_qat_microscaling_schemes.py b/tests/test_qat_microscaling_schemes.pynew file mode 100644index 00000000000..13a2c76672c--- /dev/null+++ b/tests/test_qat_microscaling_schemes.py@@ -0,0 +1,252 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""qat_scheme = "nvfp4" shorthand and saving torchao NVFP4 / MX QAT linears, CPU only."""++from __future__ import annotations++import importlib.util+import warnings++import pytest+import torch++pytestmark = pytest.mark.skipif(+ importlib.util.find_spec("torchao") is None, reason = "require torchao"+)++import unsloth.models._utils as qat_utils+import unsloth.save as save_mod+from unsloth.models._utils import TorchAOConfig, _prepare_model_for_qat+++def _tiny_qwen3():+ from transformers import Qwen3Config, Qwen3ForCausalLM++ torch.manual_seed(0)+ config = Qwen3Config(+ vocab_size = 256,+ hidden_size = 128,+ intermediate_size = 256,+ num_hidden_layers = 1,+ num_attention_heads = 4,+ num_key_value_heads = 1,+ head_dim = 32,+ tie_word_embeddings = False,+ )+ return Qwen3ForCausalLM(config).to(torch.bfloat16)+++def _mx_linear():+ from torchao.prototype.qat import MXFakeQuantizedLinear+ return MXFakeQuantizedLinear+++def _nvfp4_linear():+ from torchao.prototype.qat import NVFP4FakeQuantizedLinear+ return NVFP4FakeQuantizedLinear+++def test_nvfp4_scheme_refuses_without_fp4_gemm():+ model = _tiny_qwen3()+ with pytest.raises(RuntimeError, match = "FP4 tensor cores"):+ _prepare_model_for_qat(model, "nvfp4")+ assert not any(isinstance(m, _nvfp4_linear()) for m in model.modules())+++def test_nvfp4_scheme_layers_where_fp4_gemm_runs(monkeypatch):+ from torchao.prototype.mx_formats import NVFP4DynamicActivationNVFP4WeightConfig++ monkeypatch.setattr(qat_utils, "_qat_forward_runs", lambda *a: True)+ model = _tiny_qwen3()+ model.model.layers[0].odd = torch.nn.Linear(128, 40, bias = False)+ with warnings.catch_warnings(record = True):+ warnings.simplefilter("always")+ model = _prepare_model_for_qat(model, "nvfp4")+ ((base_config, _),) = model._torchao_config.base_config_and_filter_fns+ assert isinstance(base_config, NVFP4DynamicActivationNVFP4WeightConfig)+ assert (+ base_config.use_triton_kernel is False+ ) # the Triton quantizer needs the optional MSLK package+ layer = model.model.layers[0]+ for proj in (layer.self_attn.q_proj, layer.self_attn.k_proj, layer.mlp.down_proj):+ assert isinstance(proj, _nvfp4_linear())+ assert type(model.lm_head) is torch.nn.Linear # 4-bit head costs too much accuracy+ assert type(layer.odd) is torch.nn.Linear+++def test_nvfp4_plain_module_without_hf_heads(monkeypatch):+ monkeypatch.setattr(qat_utils, "_qat_forward_runs", lambda *a: True)+ model = torch.nn.Sequential(torch.nn.Linear(64, 128, bias = False)).to(torch.bfloat16)+ model = _prepare_model_for_qat(model, "nvfp4")+ assert isinstance(model[0], _nvfp4_linear())+++def test_unknown_scheme_still_rejected():+ with pytest.raises(ValueError, match = "Unexpected QAT scheme"):+ _prepare_model_for_qat(_tiny_qwen3(), "mxfp8")+++def _mx_torchao_config():+ from torchao.prototype.mx_formats import MXDynamicActivationMXWeightConfig+ from torchao.quantization.quantize_.common.kernel_preference import KernelPreference+ return TorchAOConfig(+ qat_scheme = "custom-mx",+ base_config_and_filter_fns = [+ (+ MXDynamicActivationMXWeightConfig(kernel_preference = KernelPreference.EMULATED),+ lambda m, fqn: isinstance(m, torch.nn.Linear) and "lm_head" not in fqn,+ )+ ],+ )+++def test_strip_restores_plain_linears():+ model = _prepare_model_for_qat(_tiny_qwen3(), _mx_torchao_config())+ q_proj = model.model.layers[0].self_attn.q_proj+ weight = q_proj.weight+ assert isinstance(q_proj, _mx_linear())+ qat_utils._strip_torchao_fake_quantization(model)+ assert not any(isinstance(m, _mx_linear()) for m in model.modules())+ q_proj = model.model.layers[0].self_attn.q_proj+ assert type(q_proj) is torch.nn.Linear+ assert torch.equal(q_proj.weight, weight)+++def test_peft_save_merges_plain_linears(monkeypatch, tmp_path):+ from peft import LoraConfig, get_peft_model++ seen = {}+ monkeypatch.setattr(+ save_mod, "_unsloth_save_torchao_with_given_config", lambda **kw: seen.update(kw)+ )+ model = get_peft_model(_tiny_qwen3(), LoraConfig(r = 16, target_modules = ["q_proj", "down_proj"]))+ model = _prepare_model_for_qat(model, _mx_torchao_config())+ assert any(isinstance(m, _mx_linear()) for m in model.modules())+ save_mod._unsloth_save_torchao_with_attached_config(+ model = model, save_directory = str(tmp_path), tokenizer = object()+ )+ assert not any(isinstance(m, _mx_linear()) for m in model.modules())+ assert seen["torchao_config"] is not None+++def _probe_ok(+ monkeypatch,+ bias_grad = True,+ calls = None,+ bad_widths = (),+):+ def forward_runs(+ base_config,+ device,+ dtype = torch.bfloat16,+ out_features = 128,+ ):+ if calls is not None and out_features == 128:+ calls.append((str(device), dtype))+ return out_features not in bad_widths++ monkeypatch.setattr(qat_utils, "_qat_forward_runs", forward_runs)+ monkeypatch.setattr(qat_utils, "_qat_bias_grad_supported", lambda *a: bias_grad, raising = False)+++def test_nvfp4_refuses_float16_layers(monkeypatch):+ # torchao's non-Triton NVFP4 quantizer asserts bfloat16 / float32 at the first forward.+ _probe_ok(monkeypatch)+ model = _tiny_qwen3().to(torch.float16)+ with pytest.raises(RuntimeError, match = "bfloat16 or float32"):+ _prepare_model_for_qat(model, "nvfp4")+ assert not any(isinstance(m, _nvfp4_linear()) for m in model.modules())+++def test_nvfp4_skips_peft_modules_to_save_head(monkeypatch):+ from peft import LoraConfig, get_peft_model++ _probe_ok(monkeypatch)+ model = get_peft_model(+ _tiny_qwen3(),+ LoraConfig(r = 16, target_modules = ["q_proj"], modules_to_save = ["lm_head"]),+ )+ with warnings.catch_warnings():+ warnings.simplefilter("ignore")+ model = _prepare_model_for_qat(model, "nvfp4")+ head = model.base_model.model.lm_head+ assert not any(isinstance(m, _nvfp4_linear()) for m in head.modules())+ assert isinstance(model.base_model.model.model.layers[0].mlp.down_proj, _nvfp4_linear())+++def test_nvfp4_probes_every_layer_placement(monkeypatch):+ calls = []+ _probe_ok(monkeypatch, calls = calls)+ model = _tiny_qwen3()+ model.model.layers[0].mlp.down_proj.to(torch.float32)+ with warnings.catch_warnings():+ warnings.simplefilter("ignore")+ _prepare_model_for_qat(model, "nvfp4")+ assert sorted(calls, key = str) == sorted(+ [("cpu", torch.bfloat16), ("cpu", torch.float32)], key = str+ )+++def test_nvfp4_keeps_failing_narrow_widths_in_full_precision(monkeypatch):+ # tiny Qwen3 k_proj / v_proj have out_features 32 (one KV head of 32).+ _probe_ok(monkeypatch, bad_widths = (32,))+ model = _tiny_qwen3()+ attn = model.model.layers[0].self_attn+ with warnings.catch_warnings(record = True) as caught:+ warnings.simplefilter("always")+ model = _prepare_model_for_qat(model, "nvfp4")+ assert not isinstance(attn.k_proj, _nvfp4_linear())+ assert not isinstance(attn.v_proj, _nvfp4_linear())+ assert isinstance(attn.q_proj, _nvfp4_linear())+ assert any("out_features [32]" in str(w.message) for w in caught)+++def test_nvfp4_converts_narrow_widths_that_run(monkeypatch):+ _probe_ok(monkeypatch)+ model = _tiny_qwen3()+ model = _prepare_model_for_qat(model, "nvfp4")+ assert isinstance(model.model.layers[0].self_attn.k_proj, _nvfp4_linear())+++def test_nvfp4_probes_opted_in_adapter_placements(monkeypatch):+ calls = []+ _probe_ok(monkeypatch, calls = calls)+ monkeypatch.setenv("UNSLOTH_QAT_FAKE_QUANTIZE_LORA", "1")+ model = _tiny_qwen3()+ layer = model.model.layers[0].mlp+ layer.lora_A = torch.nn.Linear(256, 16, bias = False).to(torch.float32)+ with warnings.catch_warnings():+ warnings.simplefilter("ignore")+ _prepare_model_for_qat(model, "nvfp4")+ assert ("cpu", torch.float32) in calls++[email protected]("bias_grad", [False, True])+def test_nvfp4_biased_layers_need_bias_grad(monkeypatch, bias_grad):+ _probe_ok(monkeypatch, bias_grad = bias_grad)+ model = _tiny_qwen3()+ layer = model.model.layers[0]+ layer.self_attn.q_proj = torch.nn.Linear(128, 128, bias = True).to(torch.bfloat16)+ with warnings.catch_warnings(record = True) as caught:+ warnings.simplefilter("always")+ model = _prepare_model_for_qat(model, "nvfp4")+ assert isinstance(layer.self_attn.q_proj, _nvfp4_linear()) is bias_grad+ assert isinstance(layer.mlp.down_proj, _nvfp4_linear())+ assert any("no bias" in str(w.message) for w in caught) is not bias_grad+++def test_nvfp4_accepts_rank4_inputs(monkeypatch):+ linear_cls = _nvfp4_linear()++ def fake_2d_forward(self, x):+ assert x.dim() == 2, x.shape # torchao's addmm dispatch takes 2D operands only+ return x @ self.weight.t()++ _probe_ok(monkeypatch)+ model = torch.nn.Sequential(torch.nn.Linear(64, 128, bias = False)).to(torch.bfloat16)+ model = _prepare_model_for_qat(model, "nvfp4")+ monkeypatch.setattr(linear_cls, "forward", fake_2d_forward)+ for shape in [(6, 64), (2, 3, 64), (2, 3, 5, 64)]:+ out = model(torch.randn(*shape, dtype = torch.bfloat16))+ assert out.shape == (*shape[:-1], 128)diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.pyindex 3ab36f0bc29..2a061a81b70 100644--- a/unsloth/models/_utils.py+++ b/unsloth/models/_utils.py@@ -7698,10 +7698,24 @@ def _strip_torchao_fake_quantization(model):from torchao.quantization import quantize_from torchao.quantization.qat import QATConfig+ # torchao's NVFP4 / MX QAT linears fake-quantize inside forward() and carry no weight_fake_quantizer.+ try:+ from torchao.prototype import qat as prototype_qat+ except Exception:+ prototype_qat = None+ forward_fake_quant = tuple(+ cls+ for cls in (+ getattr(prototype_qat, "NVFP4FakeQuantizedLinear", None),+ getattr(prototype_qat, "MXFakeQuantizedLinear", None),+ )+ if isinstance(cls, type)+ )quantize_(model,QATConfig(step = "convert"),- filter_fn = lambda m, _: hasattr(m, "weight_fake_quantizer"),+ filter_fn = lambda m, _: hasattr(m, "weight_fake_quantizer")+ or isinstance(m, forward_fake_quant),)@@ -7863,6 +7877,122 @@ def _group_size_linear_filter(model, group_size, qat_scheme):return filter_fn+def _output_head_module_ids(model):+ """Ids of the output head and every module under it: a PEFT modules_to_save wrapper holds nn.Linear copies."""+ get_output_embeddings = getattr(model, "get_output_embeddings", None)+ try:+ out_head = get_output_embeddings() if get_output_embeddings is not None else None+ except Exception:+ out_head = None+ if out_head is None:+ return set()+ return {id(m) for m in out_head.modules()}+++def _block_linear_filter(model, block_size, qat_scheme, skip_output_head):+ """Linears with both dims divisible by the scale block, optionally excluding the output head."""+ out_head_ids = _output_head_module_ids(model) if skip_output_head else set()++ def eligible(m):+ return isinstance(m, torch.nn.Linear) and id(m) not in out_head_ids++ def blocks_ok(m):+ return m.in_features % block_size == 0 and m.out_features % block_size == 0++ filter_fn = lambda m, _: eligible(m) and blocks_ok(m)+ skipped = [+ name+ for name, module in model.named_modules()+ if eligible(module) and not blocks_ok(module) and not _is_lora_adapter_name(name)+ ]+ if skipped:+ warnings.warn(+ f"Unsloth: qat_scheme='{qat_scheme}' uses {block_size}-element scale blocks; these Linear "+ f"layers have a dim not divisible by {block_size} and stay in full precision: "+ + ", ".join(skipped[:8])+ + (f", ... ({len(skipped) - 8} more)" if len(skipped) > 8 else ""),+ stacklevel = 3,+ )+ return filter_fn+++def _qat_forward_runs(+ base_config,+ device,+ dtype = torch.bfloat16,+ out_features = 128,+) -> bool:+ """One tiny QAT forward with this base config, so a GPU / torch without the low precision GEMM is caught at prepare time."""+ from torchao.quantization import quantize_+ from torchao.quantization.qat import QATConfig++ try:+ probe = torch.nn.Sequential(torch.nn.Linear(64, out_features, bias = False)).to(+ device = device, dtype = dtype+ )+ quantize_(probe, QATConfig(base_config, step = "prepare"))+ with torch.no_grad():+ probe(torch.ones(16, 64, device = device, dtype = dtype))+ return True+ except Exception:+ return False+++def _qat_bias_grad_supported(+ base_config,+ device,+ dtype = torch.bfloat16,+) -> bool:+ """torchao <= 0.18's NVFP4 QAT backward returns None for the bias: probe a biased layer's backward."""+ from torchao.quantization import quantize_+ from torchao.quantization.qat import QATConfig++ try:+ probe = torch.nn.Sequential(torch.nn.Linear(64, 128, bias = True)).to(+ device = device, dtype = dtype+ )+ quantize_(probe, QATConfig(base_config, step = "prepare"))+ x = torch.ones(16, 64, device = device, dtype = dtype, requires_grad = True)+ probe(x).float().sum().backward()+ return probe[0].bias.grad is not None+ except Exception:+ return False+++def _eligible_layer_placements(model, filter_fn):+ """Distinct (device, dtype) of the layers a QAT filter selects, adapters excluded unless they are fake quantized too."""+ with_adapters = os.environ.get("UNSLOTH_QAT_FAKE_QUANTIZE_LORA", "0") == "1"+ placements = []+ for name, module in model.named_modules():+ if (_is_lora_adapter_name(name) and not with_adapters) or not filter_fn(module, name):+ continue+ key = (module.weight.device, module.weight.dtype)+ if key not in placements:+ placements.append(key)+ return placements+++def _nvfp4_any_rank_forward(self, x):+ # torchao's NVFP4FakeQuantizedLinear flattens only rank-3 inputs; vision layers see rank 4+.+ from torchao.prototype.qat import NVFP4FakeQuantizedLinear++ if x.dim() == 2:+ return NVFP4FakeQuantizedLinear.forward(self, x)+ out = NVFP4FakeQuantizedLinear.forward(self, x.reshape(-1, x.shape[-1]))+ return out.view(*x.shape[:-1], out.shape[-1])+++def _accept_any_rank_nvfp4_inputs(model) -> None:+ import types+ try:+ from torchao.prototype.qat import NVFP4FakeQuantizedLinear+ except Exception:+ return+ for module in model.modules():+ if isinstance(module, NVFP4FakeQuantizedLinear):+ module.forward = types.MethodType(_nvfp4_any_rank_forward, module)++def _is_lora_adapter_name(fqn: str) -> bool:return any(part.startswith("lora_") for part in fqn.split("."))@@ -7925,6 +8055,93 @@ def _prepare_model_for_qat(torchao_config = TorchAOConfig(qat_scheme = qat_scheme, base_config_and_filter_fns = [(base_config, None)])+ elif qat_scheme == "nvfp4":+ try:+ from torchao.prototype.mx_formats import NVFP4DynamicActivationNVFP4WeightConfig+ except ImportError:+ raise ImportError(TORCHAO_MSG)+ # use_triton_kernel = False: the Triton quantizer needs optional MSLK, also at checkpoint load.+ try:+ base_config = NVFP4DynamicActivationNVFP4WeightConfig(use_triton_kernel = False)+ except TypeError: # older torchao without the knob+ base_config = NVFP4DynamicActivationNVFP4WeightConfig()+ candidates = _block_linear_filter(model, 16, qat_scheme, skip_output_head = True)+ placements = _eligible_layer_placements(model, candidates)+ unsupported = sorted(+ {+ str(dtype)+ for _, dtype in placements+ if dtype not in (torch.bfloat16, torch.float32)+ }+ )+ if unsupported:+ raise RuntimeError(+ "Unsloth: qat_scheme='nvfp4' quantizes bfloat16 or float32 activations only (torchao's "+ f"NVFP4 quantizer); this model has {', '.join(unsupported)} layers. Load it with "+ "dtype = torch.bfloat16."+ )+ for device, dtype in placements:+ if not _qat_forward_runs(base_config, device, dtype):+ raise RuntimeError(+ "Unsloth: qat_scheme='nvfp4' runs NVFP4 matmuls in the forward pass and needs a GPU "+ "with FP4 tensor cores (compute capability 10.0+: B200, GB200, RTX 50 series) and a "+ f"torch / torchao that support it. Layers of this model are on {device}. Use "+ "qat_scheme='fp8-fp8' instead."+ )+ # torchao's own NVFP4 filter skips out_features <= 64 (cuBLAS): probe each such width instead.+ small_widths = sorted(+ {+ module.out_features+ for name, module in model.named_modules()+ if candidates(module, name) and module.out_features <= 64+ }+ )+ bad_widths = {+ n+ for n in small_widths+ if not all(_qat_forward_runs(base_config, d, t, n) for d, t in placements)+ }+ if bad_widths:+ narrow = [+ name+ for name, module in model.named_modules()+ if not _is_lora_adapter_name(name)+ and candidates(module, name)+ and module.out_features in bad_widths+ ]+ warnings.warn(+ "Unsloth: qat_scheme='nvfp4': the NVFP4 matmul fails for out_features "+ f"{sorted(bad_widths)} here, so these Linear layers stay in full precision: "+ + ", ".join(narrow[:8])+ + (f", ... ({len(narrow) - 8} more)" if len(narrow) > 8 else ""),+ stacklevel = 2,+ )+ candidates = lambda m, fqn, _c = candidates, _bad = frozenset(bad_widths): (+ _c(m, fqn) and m.out_features not in _bad+ )+ biased = [+ name+ for name, module in model.named_modules()+ if not _is_lora_adapter_name(name)+ and candidates(module, name)+ and module.bias is not None+ ]+ filter_fn = candidates+ if biased and not all(+ _qat_bias_grad_supported(base_config, d, t) for d, t in placements+ ):+ warnings.warn(+ "Unsloth: qat_scheme='nvfp4': this torchao's NVFP4 QAT backward returns no bias "+ "gradient, so these biased Linear layers stay in full precision: "+ + ", ".join(biased[:8])+ + (f", ... ({len(biased) - 8} more)" if len(biased) > 8 else ""),+ stacklevel = 2,+ )+ filter_fn = lambda m, fqn, _c = candidates: _c(m, fqn) and m.bias is None+ torchao_config = TorchAOConfig(+ qat_scheme = qat_scheme,+ base_config_and_filter_fns = [(base_config, filter_fn)],+ )elif qat_scheme == "int8-int4":try:from torchao.quantization import (@@ -8058,6 +8275,7 @@ def _prepare_model_for_qat(for base_config, filter_fn in torchao_config.base_config_and_filter_fns:quantize_(model, QATConfig(base_config, step = "prepare"), filter_fn = filter_fn)_restore_fake_quantized_embedding_subclasses(model, embedding_subclasses)+ _accept_any_rank_nvfp4_inputs(model)return model