Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Fused LoRA: run PEFT for a partially merged block whose input needs no grad (#14748)

Daniel Han42591cad3fd7
diff --git a/tests/test_fused_lora_peft_fallbacks.py b/tests/test_fused_lora_peft_fallbacks.py
index c9e41396bc0..55fccdd055a 100644
--- a/tests/test_fused_lora_peft_fallbacks.py
+++ b/tests/test_fused_lora_peft_fallbacks.py
@@ -385,3 +385,33 @@ def test_unloaded_plain_linear_projections_run_the_fused_paths(
fused if isinstance(fused, tuple) else (fused,), ref if isinstance(ref, tuple) else (ref,)
):
torch.testing.assert_close(a, b, **tol)
+
+
[email protected]("device,dtype,path", [c for c in CASES if c[2] != "o"])
[email protected]("dropout", [0.0, 0.3])
+def test_partially_merged_block_with_frozen_input_matches_peft(
+ monkeypatch, device, dtype, path, dropout
+):
+ # X needs no grad, but the block's other active adapters train, so the fused backward would run with
+ # no LoRA weights for the merged projection ('NoneType' object has no attribute 'to').
+ model, block = _peft_block(device, dtype, dropout)
+ block.k_proj.merge()
+ block.gate_proj.merge()
+ model.eval() # dropout off, so the fused and PEFT passes are deterministic
+ X = torch.randn(2, 5, H, device = device, dtype = dtype)
+ spy = _Spy(monkeypatch)
+ results = []
+ for fn in (PATHS[path][0](block), PATHS[path][1](block)):
+ out = fn(X)
+ outs = out if isinstance(out, tuple) else (out,)
+ sum(o.double().square().sum() for o in outs).backward()
+ grads = {n: p.grad.clone() for n, p in model.named_parameters() if p.grad is not None}
+ model.zero_grad(set_to_none = True)
+ results.append(([o.detach() for o in outs], grads))
+ assert spy.calls == [], "a fused LoRA Function ran"
+ (fused, fused_grads), (ref, ref_grads) = results
+ for a, b in zip(fused, ref):
+ torch.testing.assert_close(a, b, rtol = 0, atol = 0)
+ assert fused_grads.keys() == ref_grads.keys() and fused_grads
+ for name in ref_grads:
+ torch.testing.assert_close(fused_grads[name], ref_grads[name], rtol = 0, atol = 0)
diff --git a/unsloth/kernels/fast_lora.py b/unsloth/kernels/fast_lora.py
index 8b71fc21ccf..99645830019 100644
--- a/unsloth/kernels/fast_lora.py
+++ b/unsloth/kernels/fast_lora.py
@@ -154,6 +154,20 @@ def _base_weight_requires_grad(proj):
_is_any_autocast_enabled = getattr(torch._C, "_is_any_autocast_enabled", None)
+def _active_lora_requires_grad(projs):
+ # A trainable active adapter elsewhere in the block makes autograd run the fused backward even when X
+ # needs no grad (frozen embeddings without an input-grad hook) and only some projections are merged.
+ for proj in projs:
+ adapter = _active_lora_adapter(proj)
+ if adapter is None:
+ continue
+ for module in (proj.lora_A[adapter], proj.lora_B[adapter]):
+ for p in module.parameters():
+ if p.requires_grad:
+ return True
+ return False
+
+
def _autocast_dtype_mismatch(X):
# Under autocast the fused matmuls return the autocast dtype but the in-place LoRA accumulation runs
# in X's dtype (in-place ops are not autocast), so a different autocast dtype cannot be fused.
@@ -230,7 +244,11 @@ def _lora_dropout_and_bias(projs, X):
# forward of DPO / GRPO), but their backward needs LoRA weights, so PEFT runs it under grad.
if grad_enabled is None:
grad_enabled = torch.is_grad_enabled()
- if grad_enabled and (X.requires_grad or _base_weight_requires_grad(proj)):
+ if grad_enabled and (
+ X.requires_grad
+ or _base_weight_requires_grad(proj)
+ or _active_lora_requires_grad(projs)
+ ):
return None
dropout_p.append(0.0)
biases.append(None)