Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Fused LoRA: run PEFT for a partially merged block whose input needs no grad (#14748)
Daniel Han42591cad3fd7
diff --git a/tests/test_fused_lora_peft_fallbacks.py b/tests/test_fused_lora_peft_fallbacks.pyindex c9e41396bc0..55fccdd055a 100644--- a/tests/test_fused_lora_peft_fallbacks.py+++ b/tests/test_fused_lora_peft_fallbacks.py@@ -385,3 +385,33 @@ def test_unloaded_plain_linear_projections_run_the_fused_paths(fused if isinstance(fused, tuple) else (fused,), ref if isinstance(ref, tuple) else (ref,)):torch.testing.assert_close(a, b, **tol)++[email protected]("device,dtype,path", [c for c in CASES if c[2] != "o"])[email protected]("dropout", [0.0, 0.3])+def test_partially_merged_block_with_frozen_input_matches_peft(+ monkeypatch, device, dtype, path, dropout+):+ # X needs no grad, but the block's other active adapters train, so the fused backward would run with+ # no LoRA weights for the merged projection ('NoneType' object has no attribute 'to').+ model, block = _peft_block(device, dtype, dropout)+ block.k_proj.merge()+ block.gate_proj.merge()+ model.eval() # dropout off, so the fused and PEFT passes are deterministic+ X = torch.randn(2, 5, H, device = device, dtype = dtype)+ spy = _Spy(monkeypatch)+ results = []+ for fn in (PATHS[path][0](block), PATHS[path][1](block)):+ out = fn(X)+ outs = out if isinstance(out, tuple) else (out,)+ sum(o.double().square().sum() for o in outs).backward()+ grads = {n: p.grad.clone() for n, p in model.named_parameters() if p.grad is not None}+ model.zero_grad(set_to_none = True)+ results.append(([o.detach() for o in outs], grads))+ assert spy.calls == [], "a fused LoRA Function ran"+ (fused, fused_grads), (ref, ref_grads) = results+ for a, b in zip(fused, ref):+ torch.testing.assert_close(a, b, rtol = 0, atol = 0)+ assert fused_grads.keys() == ref_grads.keys() and fused_grads+ for name in ref_grads:+ torch.testing.assert_close(fused_grads[name], ref_grads[name], rtol = 0, atol = 0)diff --git a/unsloth/kernels/fast_lora.py b/unsloth/kernels/fast_lora.pyindex 8b71fc21ccf..99645830019 100644--- a/unsloth/kernels/fast_lora.py+++ b/unsloth/kernels/fast_lora.py@@ -154,6 +154,20 @@ def _base_weight_requires_grad(proj):_is_any_autocast_enabled = getattr(torch._C, "_is_any_autocast_enabled", None)+def _active_lora_requires_grad(projs):+ # A trainable active adapter elsewhere in the block makes autograd run the fused backward even when X+ # needs no grad (frozen embeddings without an input-grad hook) and only some projections are merged.+ for proj in projs:+ adapter = _active_lora_adapter(proj)+ if adapter is None:+ continue+ for module in (proj.lora_A[adapter], proj.lora_B[adapter]):+ for p in module.parameters():+ if p.requires_grad:+ return True+ return False++def _autocast_dtype_mismatch(X):# Under autocast the fused matmuls return the autocast dtype but the in-place LoRA accumulation runs# in X's dtype (in-place ops are not autocast), so a different autocast dtype cannot be fused.@@ -230,7 +244,11 @@ def _lora_dropout_and_bias(projs, X):# forward of DPO / GRPO), but their backward needs LoRA weights, so PEFT runs it under grad.if grad_enabled is None:grad_enabled = torch.is_grad_enabled()- if grad_enabled and (X.requires_grad or _base_weight_requires_grad(proj)):+ if grad_enabled and (+ X.requires_grad+ or _base_weight_requires_grad(proj)+ or _active_lora_requires_grad(projs)+ ):return Nonedropout_p.append(0.0)biases.append(None)