Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

fix: preserve activation QAT in LoRA MLPs (#12558)

cherry77-cloud2db18414fd6a
diff --git a/tests/test_qat_activation_fused_lora.py b/tests/test_qat_activation_fused_lora.py
new file mode 100644
index 00000000000..63b886f473f
--- /dev/null
+++ b/tests/test_qat_activation_fused_lora.py
@@ -0,0 +1,129 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""Activation QAT must reach every fake quantizer: the fused LoRA paths only saw the shared input."""
+
+from __future__ import annotations
+
+import pytest
+import torch
+import unsloth # noqa: F401
+
+from real_accelerator import has_real_cuda
+
+pytestmark = [
+ pytest.mark.gpu,
+ pytest.mark.skipif(not has_real_cuda(), reason = "needs a real CUDA device"),
+]
+torchao_qat = pytest.importorskip("torchao.quantization.qat")
+
+H, I, R = 64, 128, 32
+TARGETS = ["q_proj", "k_proj", "v_proj", "o_proj", "gate_proj", "up_proj", "down_proj"]
+
+
+class _Block(torch.nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.q_proj = torch.nn.Linear(H, H, bias = False)
+ self.k_proj = torch.nn.Linear(H, H, bias = False)
+ self.v_proj = torch.nn.Linear(H, H, bias = False)
+ self.o_proj = torch.nn.Linear(H, H, bias = False)
+ self.gate_proj = torch.nn.Linear(H, I, bias = False)
+ self.up_proj = torch.nn.Linear(H, I, bias = False)
+ self.down_proj = torch.nn.Linear(I, H, bias = False)
+ self.act_fn = torch.nn.SiLU()
+
+
+def _qat_block(base_config, filter_fn = None):
+ from peft import LoraConfig, get_peft_model
+ from torchao.quantization import quantize_
+
+ torch.manual_seed(3407)
+ cfg = LoraConfig(r = R, lora_alpha = R, target_modules = TARGETS, init_lora_weights = False)
+ model = get_peft_model(_Block(), cfg).to("cuda", torch.bfloat16)
+ # Prepared after LoRA is attached, as _prepare_model_for_qat does.
+ quantize_(model, torchao_qat.QATConfig(base_config, step = "prepare"), filter_fn = filter_fn)
+ for name, p in model.named_parameters():
+ p.requires_grad_("lora_" in name)
+ return model, model.base_model.model
+
+
+def _int8_act_int4_weight():
+ from torchao.quantization import Int8DynamicActivationIntxWeightConfig
+ from torchao.quantization.granularity import PerGroup
+ return Int8DynamicActivationIntxWeightConfig(
+ weight_dtype = torch.int4, weight_granularity = PerGroup(32)
+ )
+
+
+def _run(fn, model, X):
+ X = X.clone().requires_grad_(True)
+ out = fn(X)
+ out = out if isinstance(out, tuple) else (out,)
+ sum(
+ (o.float() * torch.linspace(-1, 1, o.shape[-1], device = "cuda")).sum() for o in out
+ ).backward()
+ grads = {n: p.grad.clone() for n, p in model.named_parameters() if p.grad is not None}
+ model.zero_grad(set_to_none = True)
+ return out, X.grad, grads
+
+
+def _assert_matches_modules(
+ model,
+ block,
+ blocks = ("qkv", "o", "mlp"),
+):
+ from unsloth.kernels import apply_lora_mlp_swiglu, apply_lora_o, apply_lora_qkv
+
+ X = torch.randn(2, 5, H, device = "cuda", dtype = torch.bfloat16)
+ pairs = {
+ "qkv": (
+ lambda X: apply_lora_qkv(block, X),
+ lambda X: (block.q_proj(X), block.k_proj(X), block.v_proj(X)),
+ ),
+ "o": (lambda X: apply_lora_o(block, X), block.o_proj),
+ "mlp": (
+ lambda X: apply_lora_mlp_swiglu(block, X),
+ lambda X: block.down_proj(block.act_fn(block.gate_proj(X)) * block.up_proj(X)),
+ ),
+ }
+ for fast, ref in (pairs[b] for b in blocks):
+ got, got_dX, got_grads = _run(fast, model, X)
+ exp, exp_dX, exp_grads = _run(ref, model, X)
+ for g, e in zip(got, exp):
+ torch.testing.assert_close(g, e, rtol = 0, atol = 0)
+ torch.testing.assert_close(got_dX, exp_dX, rtol = 0, atol = 0)
+ assert got_grads.keys() == exp_grads.keys()
+ for n in exp_grads:
+ torch.testing.assert_close(got_grads[n], exp_grads[n], rtol = 0, atol = 0)
+
+
+def test_activation_qat_matches_module_forward():
+ _assert_matches_modules(*_qat_block(_int8_act_int4_weight()))
+
+
+ "suffix, blocks", [("down_proj.base_layer", ("mlp",)), ("lora_B.default", ("qkv", "o", "mlp"))]
+)
+def test_selective_activation_qat_matches_module_forward(suffix, blocks):
+ # A quantizer on an inner linear only (not the shared input) is still honoured.
+ only = lambda m, fqn: isinstance(m, torch.nn.Linear) and fqn.endswith(suffix)
+ _assert_matches_modules(*_qat_block(_int8_act_int4_weight(), only), blocks)
+
+
+def test_weight_only_qat_keeps_fused_path(monkeypatch):
+ from torchao.quantization import IntxWeightOnlyConfig
+ from torchao.quantization.granularity import PerAxis
+
+ import unsloth.kernels.fast_lora as fast_lora
+
+ _, block = _qat_block(IntxWeightOnlyConfig(weight_dtype = torch.int8, granularity = PerAxis(0)))
+ fused = []
+ real_apply = fast_lora._apply
+ monkeypatch.setattr(fast_lora, "_apply", lambda fn, *a: fused.append(fn) or real_apply(fn, *a))
+ X = torch.randn(2, 5, H, device = "cuda", dtype = torch.bfloat16)
+ with torch.no_grad():
+ fast_lora.apply_lora_qkv(block, X.clone())
+ fast_lora.apply_lora_o(block, X.clone())
+ fast_lora.apply_lora_mlp_swiglu(block, X.clone())
+ assert fused == [fast_lora.LoRA_QKV, fast_lora.LoRA_W, fast_lora.LoRA_MLP]
diff --git a/unsloth/kernels/fast_lora.py b/unsloth/kernels/fast_lora.py
index 36042f4efd0..19a4f0a393a 100644
--- a/unsloth/kernels/fast_lora.py
+++ b/unsloth/kernels/fast_lora.py
@@ -13,7 +13,6 @@ import torch
from unsloth_zoo.utils import Version
from .utils import (
_has_multiple_active_adapters,
- _maybe_fake_quantize_activations,
addmm_,
fast_dequantize,
QUANT_STATE,
@@ -38,6 +37,14 @@ if not TRACE_LORA_FUNCTIONS:
_apply = torch._dynamo.disable(_apply)
+def _has_activation_fake_quantizer(proj):
+ # Fused LoRA skips per-layer activation fake quantizers; explicit children, as proj.modules() is slow per call.
+ for module in (getattr(proj, "base_layer", proj), *proj.lora_A.values(), *proj.lora_B.values()):
+ if module._modules.get("activation_fake_quantizer") is not None:
+ return True
+ return False
+
+
class LoRA_MLP(torch.autograd.Function):
"""
### LoRA weights
@@ -236,11 +243,10 @@ def apply_lora_mlp_swiglu(
inplace = True,
):
if any(
- _has_multiple_active_adapters(proj)
+ _has_multiple_active_adapters(proj) or _has_activation_fake_quantizer(proj)
for proj in (self.gate_proj, self.up_proj, self.down_proj)
):
return self.down_proj(self.act_fn(self.gate_proj(X)) * self.up_proj(X))
- X = _maybe_fake_quantize_activations(X, self.gate_proj)
gateW, gateW_quant, gateA, gateB, gateS = get_lora_parameters(self.gate_proj)
upW, upW_quant, upA, upB, upS = get_lora_parameters(self.up_proj)
downW, downW_quant, downA, downB, downS = get_lora_parameters(self.down_proj)
@@ -278,11 +284,10 @@ def apply_lora_mlp_geglu_exact(
inplace = True,
):
if any(
- _has_multiple_active_adapters(proj)
+ _has_multiple_active_adapters(proj) or _has_activation_fake_quantizer(proj)
for proj in (self.gate_proj, self.up_proj, self.down_proj)
):
return self.down_proj(self.act_fn(self.gate_proj(X)) * self.up_proj(X))
- X = _maybe_fake_quantize_activations(X, self.gate_proj)
gateW, gateW_quant, gateA, gateB, gateS = get_lora_parameters(self.gate_proj)
upW, upW_quant, upA, upB, upS = get_lora_parameters(self.up_proj)
downW, downW_quant, downA, downB, downS = get_lora_parameters(self.down_proj)
@@ -316,11 +321,10 @@ from .geglu import geglu_approx_forward_kernel, geglu_approx_backward_kernel
def apply_lora_mlp_geglu_approx(self, X):
if any(
- _has_multiple_active_adapters(proj)
+ _has_multiple_active_adapters(proj) or _has_activation_fake_quantizer(proj)
for proj in (self.gate_proj, self.up_proj, self.down_proj)
):
return self.down_proj(self.act_fn(self.gate_proj(X)) * self.up_proj(X))
- X = _maybe_fake_quantize_activations(X, self.gate_proj)
gateW, gateW_quant, gateA, gateB, gateS = get_lora_parameters(self.gate_proj)
upW, upW_quant, upA, upB, upS = get_lora_parameters(self.up_proj)
downW, downW_quant, downA, downB, downS = get_lora_parameters(self.down_proj)
@@ -539,9 +543,11 @@ def apply_lora_qkv(
X,
inplace = True,
):
- if any(_has_multiple_active_adapters(proj) for proj in (self.q_proj, self.k_proj, self.v_proj)):
+ if any(
+ _has_multiple_active_adapters(proj) or _has_activation_fake_quantizer(proj)
+ for proj in (self.q_proj, self.k_proj, self.v_proj)
+ ):
return self.q_proj(X), self.k_proj(X), self.v_proj(X)
- X = _maybe_fake_quantize_activations(X, self.q_proj)
QW, QW_quant, QA, QB, QS = get_lora_parameters(self.q_proj)
KW, KW_quant, KA, KB, KS = get_lora_parameters(self.k_proj)
VW, VW_quant, VA, VB, VS = get_lora_parameters(self.v_proj)
@@ -643,9 +649,8 @@ class LoRA_W(torch.autograd.Function):
def apply_lora_o(self, X):
- if _has_multiple_active_adapters(self.o_proj):
+ if _has_multiple_active_adapters(self.o_proj) or _has_activation_fake_quantizer(self.o_proj):
return self.o_proj(X)
- X = _maybe_fake_quantize_activations(X, self.o_proj)
OW, OW_quant, OA, OB, OS = get_lora_parameters(self.o_proj)
O = _apply(LoRA_W, X, OW, OW_quant, OA, OB, OS)
return O
diff --git a/unsloth/kernels/utils.py b/unsloth/kernels/utils.py
index 5027392d3c7..f98d6f1a07b 100644
--- a/unsloth/kernels/utils.py
+++ b/unsloth/kernels/utils.py
@@ -489,17 +489,6 @@ def get_lora_parameters_bias(proj):
)
-def _maybe_fake_quantize_activations(X: torch.Tensor, proj: torch.nn.Module) -> torch.Tensor:
- """Fake-quantize input activations if QAT is enabled, else return as-is.
- Weights are fake-quantized separately in `get_lora_parameters`.
- """
- base_layer = getattr(proj, "base_layer", proj)
- activation_fake_quantizer = getattr(base_layer, "activation_fake_quantizer", None)
- if activation_fake_quantizer is not None:
- X = activation_fake_quantizer(X)
- return X
-
-
def _unpack_quant_state(quant_state):
"""(absmax, shape, dtype, blocksize, code, code2, absmax2, offset, blocksize2) from a bnb
4bit quant state, class or legacy list form. The nested fields are None (and blocksize2 0)