Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
QAT: fall back to the module forward for torchao NVFP4 / MX linears in fused LoRA (#13280)
Daniel Han7d132bb047e0
diff --git a/tests/test_qat_forward_fake_quant_fallback.py b/tests/test_qat_forward_fake_quant_fallback.pynew file mode 100644index 00000000000..6a21d217ca9--- /dev/null+++ b/tests/test_qat_forward_fake_quant_fallback.py@@ -0,0 +1,120 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""Fused LoRA must not bypass torchao QAT linears that fake-quantize inside forward().++torchao's QATConfig(step="prepare") swaps nn.Linear for NVFP4FakeQuantizedLinear / MXFakeQuantizedLinear+when the base config is NVFP4 or MX. Those classes fake-quantize in forward() and expose no+weight_fake_quantizer / activation_fake_quantizer, so the fused kernels, which read base_layer.weight+directly, trained on the unquantized weights. They must take the module-forward fallback instead,+while ordinary LoRA and the int4 weight-only scheme keep the fused path.+"""++import pytest+import torch++pytest.importorskip("torchao.quantization.qat")+peft = pytest.importorskip("peft")++import unsloth # noqa: F401+import unsloth.kernels.fast_lora as fast_lora+from unsloth.models._utils import TorchAOConfig, _prepare_model_for_qat++DIM = 64+++class _Block(torch.nn.Module):+ def __init__(self):+ super().__init__()+ self.o_proj = torch.nn.Linear(DIM, DIM, bias = False)++ def forward(self, x):+ return self.o_proj(x)+++def _base_config(scheme):+ if scheme == "nvfp4":+ mx = pytest.importorskip("torchao.prototype.mx_formats")+ return mx.NVFP4DynamicActivationNVFP4WeightConfig()+ if scheme == "mx":+ mx = pytest.importorskip("torchao.prototype.mx_formats")+ return mx.MXDynamicActivationMXWeightConfig(+ activation_dtype = torch.float8_e4m3fn, weight_dtype = torch.float8_e4m3fn+ )+ if scheme == "int4":+ from torchao.quantization import Int4WeightOnlyConfig+ return Int4WeightOnlyConfig(group_size = DIM)+ return None+++def _lora_block(scheme):+ torch.manual_seed(0)+ model = peft.get_peft_model(+ _Block(), peft.LoraConfig(r = 16, lora_alpha = 16, target_modules = ["o_proj"], lora_dropout = 0.0)+ )+ base_config = _base_config(scheme)+ if base_config is not None:+ only_base = lambda m, fqn: isinstance(m, torch.nn.Linear) and fqn.endswith("base_layer")+ _prepare_model_for_qat(+ model,+ TorchAOConfig(qat_scheme = None, base_config_and_filter_fns = [(base_config, only_base)]),+ )+ return model.base_model.model+++def _route(block, monkeypatch):+ calls = {"fused": 0, "module": 0}++ def fused(function, *args):+ calls["fused"] += 1+ return args[0]++ def module_forward(x):+ calls["module"] += 1+ return x++ monkeypatch.setattr(fast_lora, "_apply", fused)+ monkeypatch.setattr(block.o_proj, "forward", module_forward)+ fast_lora.apply_lora_o(block, torch.randn(2, 3, DIM))+ return calls++[email protected]("scheme", ["nvfp4", "mx"])+def test_forward_fake_quant_base_layer_falls_back_to_module_forward(scheme, monkeypatch):+ block = _lora_block(scheme)+ expected = {"nvfp4": "NVFP4FakeQuantizedLinear", "mx": "MXFakeQuantizedLinear"}[scheme]+ base_layer = block.o_proj.base_layer+ assert type(base_layer).__name__ == expected+ # The premise: no fake quantizer children, so the existing child check cannot see it.+ assert getattr(base_layer, "weight_fake_quantizer", None) is None+ assert base_layer._modules.get("activation_fake_quantizer") is None++ assert fast_lora._has_activation_fake_quantizer(block.o_proj)+ assert _route(block, monkeypatch) == {"fused": 0, "module": 1}++[email protected]("scheme", ["none", "int4"])+def test_plain_and_weight_only_qat_keep_fused_path(scheme, monkeypatch):+ block = _lora_block(scheme)+ assert not fast_lora._has_activation_fake_quantizer(block.o_proj)+ assert _route(block, monkeypatch) == {"fused": 1, "module": 0}+++def test_direct_torchao_quantize_also_falls_back(monkeypatch):+ # A user calling torchao's quantize_ directly (not through qat_scheme) gets the same fallback.+ from torchao.quantization import quantize_+ from torchao.quantization.qat import QATConfig++ mx = pytest.importorskip("torchao.prototype.mx_formats")+ torch.manual_seed(0)+ model = peft.get_peft_model(+ _Block(), peft.LoraConfig(r = 16, lora_alpha = 16, target_modules = ["o_proj"], lora_dropout = 0.0)+ )+ quantize_(+ model,+ QATConfig(mx.NVFP4DynamicActivationNVFP4WeightConfig(), step = "prepare"),+ filter_fn = lambda m, fqn: isinstance(m, torch.nn.Linear) and fqn.endswith("base_layer"),+ )+ block = model.base_model.model+ assert fast_lora._has_activation_fake_quantizer(block.o_proj)+ assert _route(block, monkeypatch) == {"fused": 0, "module": 1}diff --git a/unsloth/kernels/fast_lora.py b/unsloth/kernels/fast_lora.pyindex 26b683f0544..fdb4b981c1a 100644--- a/unsloth/kernels/fast_lora.py+++ b/unsloth/kernels/fast_lora.py@@ -9,6 +9,7 @@# See the License for the specific language governing permissions and# limitations under the License.+import sysimport torchfrom unsloth_zoo.utils import Versionfrom .utils import (@@ -39,10 +40,35 @@ if not TRACE_LORA_FUNCTIONS:_apply = torch._dynamo.disable(_apply)+# torchao's NVFP4 / MX QAT linears fake-quantize inside forward() with no quantizer children, so fused LoRA+# would train unquantized. Read lazily from sys.modules so importing Unsloth never imports torchao.+_FORWARD_FAKE_QUANT_LINEARS = None+++def _forward_fake_quant_linears():+ global _FORWARD_FAKE_QUANT_LINEARS+ if _FORWARD_FAKE_QUANT_LINEARS is None:+ qat = sys.modules.get("torchao.prototype.qat")+ if qat is None:+ return ()+ _FORWARD_FAKE_QUANT_LINEARS = tuple(+ cls+ for cls in (+ getattr(qat, "NVFP4FakeQuantizedLinear", None),+ getattr(qat, "MXFakeQuantizedLinear", None),+ )+ if isinstance(cls, type)+ )+ return _FORWARD_FAKE_QUANT_LINEARS++def _has_activation_fake_quantizer(proj):# Fused LoRA skips per-layer activation fake quantizers; explicit children, as proj.modules() is slow per call.+ forward_fake_quant_linears = _forward_fake_quant_linears()for module in (getattr(proj, "base_layer", proj), *proj.lora_A.values(), *proj.lora_B.values()):- if module._modules.get("activation_fake_quantizer") is not None:+ if module._modules.get("activation_fake_quantizer") is not None or isinstance(+ module, forward_fake_quant_linears+ ):return Truereturn False