Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
fix: preserve activation QAT in LoRA MLPs (#12558)
cherry77-cloud2db18414fd6a
diff --git a/tests/test_qat_activation_fused_lora.py b/tests/test_qat_activation_fused_lora.pynew file mode 100644index 00000000000..63b886f473f--- /dev/null+++ b/tests/test_qat_activation_fused_lora.py@@ -0,0 +1,129 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""Activation QAT must reach every fake quantizer: the fused LoRA paths only saw the shared input."""++from __future__ import annotations++import pytest+import torch+import unsloth # noqa: F401++from real_accelerator import has_real_cuda++pytestmark = [+ pytest.mark.gpu,+ pytest.mark.skipif(not has_real_cuda(), reason = "needs a real CUDA device"),+]+torchao_qat = pytest.importorskip("torchao.quantization.qat")++H, I, R = 64, 128, 32+TARGETS = ["q_proj", "k_proj", "v_proj", "o_proj", "gate_proj", "up_proj", "down_proj"]+++class _Block(torch.nn.Module):+ def __init__(self):+ super().__init__()+ self.q_proj = torch.nn.Linear(H, H, bias = False)+ self.k_proj = torch.nn.Linear(H, H, bias = False)+ self.v_proj = torch.nn.Linear(H, H, bias = False)+ self.o_proj = torch.nn.Linear(H, H, bias = False)+ self.gate_proj = torch.nn.Linear(H, I, bias = False)+ self.up_proj = torch.nn.Linear(H, I, bias = False)+ self.down_proj = torch.nn.Linear(I, H, bias = False)+ self.act_fn = torch.nn.SiLU()+++def _qat_block(base_config, filter_fn = None):+ from peft import LoraConfig, get_peft_model+ from torchao.quantization import quantize_++ torch.manual_seed(3407)+ cfg = LoraConfig(r = R, lora_alpha = R, target_modules = TARGETS, init_lora_weights = False)+ model = get_peft_model(_Block(), cfg).to("cuda", torch.bfloat16)+ # Prepared after LoRA is attached, as _prepare_model_for_qat does.+ quantize_(model, torchao_qat.QATConfig(base_config, step = "prepare"), filter_fn = filter_fn)+ for name, p in model.named_parameters():+ p.requires_grad_("lora_" in name)+ return model, model.base_model.model+++def _int8_act_int4_weight():+ from torchao.quantization import Int8DynamicActivationIntxWeightConfig+ from torchao.quantization.granularity import PerGroup+ return Int8DynamicActivationIntxWeightConfig(+ weight_dtype = torch.int4, weight_granularity = PerGroup(32)+ )+++def _run(fn, model, X):+ X = X.clone().requires_grad_(True)+ out = fn(X)+ out = out if isinstance(out, tuple) else (out,)+ sum(+ (o.float() * torch.linspace(-1, 1, o.shape[-1], device = "cuda")).sum() for o in out+ ).backward()+ grads = {n: p.grad.clone() for n, p in model.named_parameters() if p.grad is not None}+ model.zero_grad(set_to_none = True)+ return out, X.grad, grads+++def _assert_matches_modules(+ model,+ block,+ blocks = ("qkv", "o", "mlp"),+):+ from unsloth.kernels import apply_lora_mlp_swiglu, apply_lora_o, apply_lora_qkv++ X = torch.randn(2, 5, H, device = "cuda", dtype = torch.bfloat16)+ pairs = {+ "qkv": (+ lambda X: apply_lora_qkv(block, X),+ lambda X: (block.q_proj(X), block.k_proj(X), block.v_proj(X)),+ ),+ "o": (lambda X: apply_lora_o(block, X), block.o_proj),+ "mlp": (+ lambda X: apply_lora_mlp_swiglu(block, X),+ lambda X: block.down_proj(block.act_fn(block.gate_proj(X)) * block.up_proj(X)),+ ),+ }+ for fast, ref in (pairs[b] for b in blocks):+ got, got_dX, got_grads = _run(fast, model, X)+ exp, exp_dX, exp_grads = _run(ref, model, X)+ for g, e in zip(got, exp):+ torch.testing.assert_close(g, e, rtol = 0, atol = 0)+ torch.testing.assert_close(got_dX, exp_dX, rtol = 0, atol = 0)+ assert got_grads.keys() == exp_grads.keys()+ for n in exp_grads:+ torch.testing.assert_close(got_grads[n], exp_grads[n], rtol = 0, atol = 0)+++def test_activation_qat_matches_module_forward():+ _assert_matches_modules(*_qat_block(_int8_act_int4_weight()))+++ "suffix, blocks", [("down_proj.base_layer", ("mlp",)), ("lora_B.default", ("qkv", "o", "mlp"))]+)+def test_selective_activation_qat_matches_module_forward(suffix, blocks):+ # A quantizer on an inner linear only (not the shared input) is still honoured.+ only = lambda m, fqn: isinstance(m, torch.nn.Linear) and fqn.endswith(suffix)+ _assert_matches_modules(*_qat_block(_int8_act_int4_weight(), only), blocks)+++def test_weight_only_qat_keeps_fused_path(monkeypatch):+ from torchao.quantization import IntxWeightOnlyConfig+ from torchao.quantization.granularity import PerAxis++ import unsloth.kernels.fast_lora as fast_lora++ _, block = _qat_block(IntxWeightOnlyConfig(weight_dtype = torch.int8, granularity = PerAxis(0)))+ fused = []+ real_apply = fast_lora._apply+ monkeypatch.setattr(fast_lora, "_apply", lambda fn, *a: fused.append(fn) or real_apply(fn, *a))+ X = torch.randn(2, 5, H, device = "cuda", dtype = torch.bfloat16)+ with torch.no_grad():+ fast_lora.apply_lora_qkv(block, X.clone())+ fast_lora.apply_lora_o(block, X.clone())+ fast_lora.apply_lora_mlp_swiglu(block, X.clone())+ assert fused == [fast_lora.LoRA_QKV, fast_lora.LoRA_W, fast_lora.LoRA_MLP]diff --git a/unsloth/kernels/fast_lora.py b/unsloth/kernels/fast_lora.pyindex 36042f4efd0..19a4f0a393a 100644--- a/unsloth/kernels/fast_lora.py+++ b/unsloth/kernels/fast_lora.py@@ -13,7 +13,6 @@ import torchfrom unsloth_zoo.utils import Versionfrom .utils import (_has_multiple_active_adapters,- _maybe_fake_quantize_activations,addmm_,fast_dequantize,QUANT_STATE,@@ -38,6 +37,14 @@ if not TRACE_LORA_FUNCTIONS:_apply = torch._dynamo.disable(_apply)+def _has_activation_fake_quantizer(proj):+ # Fused LoRA skips per-layer activation fake quantizers; explicit children, as proj.modules() is slow per call.+ for module in (getattr(proj, "base_layer", proj), *proj.lora_A.values(), *proj.lora_B.values()):+ if module._modules.get("activation_fake_quantizer") is not None:+ return True+ return False++class LoRA_MLP(torch.autograd.Function):"""### LoRA weights@@ -236,11 +243,10 @@ def apply_lora_mlp_swiglu(inplace = True,):if any(- _has_multiple_active_adapters(proj)+ _has_multiple_active_adapters(proj) or _has_activation_fake_quantizer(proj)for proj in (self.gate_proj, self.up_proj, self.down_proj)):return self.down_proj(self.act_fn(self.gate_proj(X)) * self.up_proj(X))- X = _maybe_fake_quantize_activations(X, self.gate_proj)gateW, gateW_quant, gateA, gateB, gateS = get_lora_parameters(self.gate_proj)upW, upW_quant, upA, upB, upS = get_lora_parameters(self.up_proj)downW, downW_quant, downA, downB, downS = get_lora_parameters(self.down_proj)@@ -278,11 +284,10 @@ def apply_lora_mlp_geglu_exact(inplace = True,):if any(- _has_multiple_active_adapters(proj)+ _has_multiple_active_adapters(proj) or _has_activation_fake_quantizer(proj)for proj in (self.gate_proj, self.up_proj, self.down_proj)):return self.down_proj(self.act_fn(self.gate_proj(X)) * self.up_proj(X))- X = _maybe_fake_quantize_activations(X, self.gate_proj)gateW, gateW_quant, gateA, gateB, gateS = get_lora_parameters(self.gate_proj)upW, upW_quant, upA, upB, upS = get_lora_parameters(self.up_proj)downW, downW_quant, downA, downB, downS = get_lora_parameters(self.down_proj)@@ -316,11 +321,10 @@ from .geglu import geglu_approx_forward_kernel, geglu_approx_backward_kerneldef apply_lora_mlp_geglu_approx(self, X):if any(- _has_multiple_active_adapters(proj)+ _has_multiple_active_adapters(proj) or _has_activation_fake_quantizer(proj)for proj in (self.gate_proj, self.up_proj, self.down_proj)):return self.down_proj(self.act_fn(self.gate_proj(X)) * self.up_proj(X))- X = _maybe_fake_quantize_activations(X, self.gate_proj)gateW, gateW_quant, gateA, gateB, gateS = get_lora_parameters(self.gate_proj)upW, upW_quant, upA, upB, upS = get_lora_parameters(self.up_proj)downW, downW_quant, downA, downB, downS = get_lora_parameters(self.down_proj)@@ -539,9 +543,11 @@ def apply_lora_qkv(X,inplace = True,):- if any(_has_multiple_active_adapters(proj) for proj in (self.q_proj, self.k_proj, self.v_proj)):+ if any(+ _has_multiple_active_adapters(proj) or _has_activation_fake_quantizer(proj)+ for proj in (self.q_proj, self.k_proj, self.v_proj)+ ):return self.q_proj(X), self.k_proj(X), self.v_proj(X)- X = _maybe_fake_quantize_activations(X, self.q_proj)QW, QW_quant, QA, QB, QS = get_lora_parameters(self.q_proj)KW, KW_quant, KA, KB, KS = get_lora_parameters(self.k_proj)VW, VW_quant, VA, VB, VS = get_lora_parameters(self.v_proj)@@ -643,9 +649,8 @@ class LoRA_W(torch.autograd.Function):def apply_lora_o(self, X):- if _has_multiple_active_adapters(self.o_proj):+ if _has_multiple_active_adapters(self.o_proj) or _has_activation_fake_quantizer(self.o_proj):return self.o_proj(X)- X = _maybe_fake_quantize_activations(X, self.o_proj)OW, OW_quant, OA, OB, OS = get_lora_parameters(self.o_proj)O = _apply(LoRA_W, X, OW, OW_quant, OA, OB, OS)return Odiff --git a/unsloth/kernels/utils.py b/unsloth/kernels/utils.pyindex 5027392d3c7..f98d6f1a07b 100644--- a/unsloth/kernels/utils.py+++ b/unsloth/kernels/utils.py@@ -489,17 +489,6 @@ def get_lora_parameters_bias(proj):)-def _maybe_fake_quantize_activations(X: torch.Tensor, proj: torch.nn.Module) -> torch.Tensor:- """Fake-quantize input activations if QAT is enabled, else return as-is.- Weights are fake-quantized separately in `get_lora_parameters`.- """- base_layer = getattr(proj, "base_layer", proj)- activation_fake_quantizer = getattr(base_layer, "activation_fake_quantizer", None)- if activation_fake_quantizer is not None:- X = activation_fake_quantizer(X)- return X--def _unpack_quant_state(quant_state):"""(absmax, shape, dtype, blocksize, code, code2, absmax2, offset, blocksize2) from a bnb4bit quant state, class or legacy list form. The nested fields are None (and blocksize2 0)