Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Decline the fused LoRA kernels under FSDP, where the weights are shard views (#11132)
Daniel Haneaa56d6bad93
diff --git a/tests/test_fsdp_declines_fused_lora.py b/tests/test_fsdp_declines_fused_lora.pynew file mode 100644index 00000000000..ad4bb08850b--- /dev/null+++ b/tests/test_fsdp_declines_fused_lora.py@@ -0,0 +1,151 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""unsloth#409: under FSDP the fused LoRA kernels read shard views (`matmul_lora`:+`size mismatch, got input (20), mat (20x896), vec (3584)` on Qwen2.5-0.5B FULL_SHARD), so+patch_peft_model must decline them. No GPU: the probe is env-driven, the wiring check textual.+"""++from __future__ import annotations++import ast+import os+from pathlib import Path++import pytest++from unsloth.models.loader_utils import fsdp_will_wrap++_LLAMA = Path(__file__).resolve().parent.parent / "unsloth" / "models" / "llama.py"++_FSDP_ENV = (+ "ACCELERATE_USE_FSDP",+ "FSDP_VERSION",+ "UNSLOTH_FORCE_FUSED_LORA",+)++[email protected](autouse = True)+def _clean_env(monkeypatch):+ for key in _FSDP_ENV:+ monkeypatch.delenv(key, raising = False)+++def test_a_plain_launch_keeps_the_fused_kernels(monkeypatch):+ """DDP and single-GPU launches export neither variable and keep the fast path."""+ assert fsdp_will_wrap() is False+ monkeypatch.setenv("ACCELERATE_MIXED_PRECISION", "bf16")+ assert fsdp_will_wrap() is False++[email protected]("value", ["true", "True", "1", "yes", "ON"])+def test_accelerate_use_fsdp_declines(monkeypatch, value):+ """`accelerate launch` with an FSDP config exports the literal "true"."""+ monkeypatch.setenv("ACCELERATE_USE_FSDP", value)+ assert fsdp_will_wrap() is True++[email protected]("value", ["false", "0", "", "no"])+def test_a_falsy_accelerate_use_fsdp_keeps_the_fused_kernels(monkeypatch, value):+ monkeypatch.setenv("ACCELERATE_USE_FSDP", value)+ assert fsdp_will_wrap() is False++[email protected]("value,expected", [("1", True), ("2", True), ("0", False), ("", False)])+def test_fsdp_version_alone_is_enough(monkeypatch, value, expected):+ """torchrun users set it by hand; "0" is how a config says "not FSDP"."""+ monkeypatch.setenv("FSDP_VERSION", value)+ assert fsdp_will_wrap() is expected+++def test_the_override_wins_over_every_signal(monkeypatch):+ monkeypatch.setenv("ACCELERATE_USE_FSDP", "true")+ monkeypatch.setenv("FSDP_VERSION", "2")+ monkeypatch.setenv("UNSLOTH_FORCE_FUSED_LORA", "1")+ assert fsdp_will_wrap() is False+++def test_a_live_accelerator_state_is_consulted_first(monkeypatch):+ """A caller-built Accelerator carries no env, so its live state must be read."""+ accelerate_state = pytest.importorskip("accelerate.state")+ shared = accelerate_state.AcceleratorState._shared_state+ monkeypatch.setitem(shared, "distributed_type", "DistributedType.FSDP")+ assert fsdp_will_wrap() is True+++def test_the_fused_lora_install_is_gated_on_the_probe():+ """The three fused installs share one `if`; splitting them must carry the gate."""+ source = _LLAMA.read_text(encoding = "utf-8")+ assert "fsdp_will_wrap" in source, "llama.py no longer probes for FSDP"++ tree = ast.parse(source)+ guarded = []+ for node in ast.walk(tree):+ if not isinstance(node, ast.If):+ continue+ test = ast.unparse(node.test)+ if "fused_lora_declined_for_fsdp" not in test:+ continue+ body = ast.unparse(node)+ if "apply_qkv" in body or "apply_o" in body or "_apply_lora_mlp" in body:+ guarded.append(test)+ assert guarded, (+ "no `if` guarding apply_qkv / apply_o / apply_lora_mlp consults "+ "fused_lora_declined_for_fsdp; under FSDP the fused kernels will read "+ "shard views again"+ )+++def test_the_declined_message_names_the_override():+ from unsloth.models.llama import _fused_lora_skip_reason+ assert _fused_lora_skip_reason(0, "none") == ""+ assert "UNSLOTH_FORCE_FUSED_LORA" in _fused_lora_skip_reason(0, "none", fsdp = True)+++def _fused_model(llama):+ import types++ import torch++ attn = torch.nn.Module()+ attn.apply_qkv = llama.apply_lora_qkv+ attn.apply_o = llama.apply_lora_o+ mlp = torch.nn.Linear(2, 2)+ mlp.forward = types.MethodType(llama.apply_lora_mlp_swiglu, mlp)+ tiled = torch.nn.Linear(2, 2)+ tiled._unsloth_forward = types.MethodType(llama.apply_lora_mlp_geglu_approx, tiled)+ root = torch.nn.Module()+ root.attn, root.mlp, root.tiled = attn, mlp, tiled+ return root+++def test_a_trainer_side_fsdp_gets_peft_forwards_back():+ """`SFTConfig(fsdp=...)` exports no env: the Trainer undoes the fused installs once it knows."""+ import torch+ from unsloth.models import llama++ root = _fused_model(llama)+ assert llama._decline_fused_lora_for_fsdp(root) == 4+ assert root.attn.apply_qkv is llama.original_apply_qkv+ assert root.attn.apply_o is llama.original_apply_o+ assert "forward" not in root.mlp.__dict__+ assert root.tiled._unsloth_forward is torch.nn.Linear.forward+ assert llama._decline_fused_lora_for_fsdp(root) == 0+++def test_the_trainer_side_decline_honours_the_override(monkeypatch):+ from unsloth.models import llama++ monkeypatch.setenv("UNSLOTH_FORCE_FUSED_LORA", "1")+ root = _fused_model(llama)+ assert llama._decline_fused_lora_for_fsdp(root) == 0+ assert root.attn.apply_qkv is llama.apply_lora_qkv+++def test_the_trainer_init_wrapper_calls_the_decline():+ import inspect++ from unsloth.models import _utils++ source = inspect.getsource(_utils.patch_gradient_accumulation_fix)+ assert "is_fsdp_enabled" in source and "_decline_fused_lora_for_fsdp" in sourcediff --git a/tests/test_lora_dropout_patch_summary.py b/tests/test_lora_dropout_patch_summary.pyindex 72f66675ab5..4ea4bebbf66 100644--- a/tests/test_lora_dropout_patch_summary.py+++ b/tests/test_lora_dropout_patch_summary.py@@ -64,13 +64,20 @@ def test_reason_appends_cleanly_to_the_summary_sentence():@pytest.mark.parametrize(- "lora_dropout,bias",- list(itertools.product([0, 0.0, 0.1, 0.5], ["none", "all", "lora_only"])),+ "lora_dropout,bias,float32_base,fsdp",+ list(+ itertools.product(+ [0, 0.0, 0.1, 0.5], ["none", "all", "lora_only"], [False, True], [False, True]+ )+ ),)-def test_reason_is_non_empty_exactly_when_the_fused_gate_is_closed(lora_dropout, bias):- """Parity with the `lora_dropout == 0 and bias == "none"` gate in patch_peft_model."""- fused_installed = lora_dropout == 0 and bias == "none"- assert bool(_fused_lora_skip_reason(lora_dropout, bias)) is not fused_installed+def test_reason_is_non_empty_exactly_when_the_fused_gate_is_closed(+ lora_dropout, bias, float32_base, fsdp+):+ """Parity with the fused-kernel gate in patch_peft_model."""+ fused_installed = lora_dropout == 0 and bias == "none" and not float32_base and not fsdp+ reason = _fused_lora_skip_reason(lora_dropout, bias, float32_base, fsdp = fsdp)+ assert bool(reason) is not fused_installeddef _gate_tests(source: str) -> list[str]:@@ -87,15 +94,20 @@ def _gate_tests(source: str) -> list[str]:def test_patch_peft_model_still_gates_the_fused_kernels_on_the_same_values():- """If the gate grows a third condition, _fused_lora_skip_reason has to grow with it or- the summary starts reporting zero counts with no reason again."""+ """Every gate condition needs a matching _fused_lora_skip_reason clause."""source = inspect.getsource(llama_module.FastLlamaModel.patch_peft_model)- assert _gate_tests(source) == ["lora_dropout == 0 and bias == 'none' and (not float32_base)"]+ assert _gate_tests(source) == [+ "lora_dropout == 0 and bias == 'none' and (not float32_base) and (not fused_lora_declined_for_fsdp)"+ ]def test_the_summary_call_carries_the_reason():source = inspect.getsource(llama_module.FastLlamaModel.patch_peft_model)- assert "unfused_reason = _fused_lora_skip_reason(lora_dropout, bias, float32_base)" in source+ flat = "".join(source.split())+ assert (+ "unfused_reason=_fused_lora_skip_reason(lora_dropout,bias,float32_base,fsdp=fused_lora_declined_for_fsdp)"+ in flat+ )assert "MLP layers.{unfused_reason}" in sourcediff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.pyindex b79e67ccccf..780fa4f6a47 100644--- a/unsloth/models/_utils.py+++ b/unsloth/models/_utils.py@@ -4882,6 +4882,9 @@ def patch_gradient_accumulation_fix(Trainer):passexcept Exception:pass+ if getattr(self, "is_fsdp_enabled", False):+ from .llama import _decline_fused_lora_for_fsdp+ _decline_fused_lora_for_fsdp(getattr(self, "model", None))_unsloth_trainer_init.__wrapped__ = _original_trainer_initTrainer.__init__ = _unsloth_trainer_initdiff --git a/unsloth/models/llama.py b/unsloth/models/llama.pyindex 8852f8458a6..70e355d5616 100644--- a/unsloth/models/llama.py+++ b/unsloth/models/llama.py@@ -37,6 +37,7 @@ from .loader_utils import (_get_fp8_mode_and_check_settings,_restore_dropped_fp8_scales,_prepare_compressed_tensors_model,+ fsdp_will_wrap,planner_class_mismatch_reason,planner_model_class,planner_config_overrides,@@ -2293,11 +2294,12 @@ def _fused_lora_skip_reason(lora_dropout,bias,float32_base = False,+ fsdp = False,) -> str:"""Why patch_peft_model skipped the fused LoRA kernels, for the patched layers summary.Returns "" when nothing disabled them, so the common summary line is unchanged. The- conditions mirror the `lora_dropout == 0 and bias == "none"` gate in patch_peft_model.+ conditions mirror the fused-kernel gate in patch_peft_model."""reasons = []if lora_dropout != 0:@@ -2306,6 +2308,8 @@ def _fused_lora_skip_reason(reasons.append(f"bias = '{bias}'")if float32_base:reasons.append("the base weights are float32")+ if fsdp:+ reasons.append("FSDP shards the weights they read (UNSLOTH_FORCE_FUSED_LORA=1 overrides)")if not reasons:return ""return (@@ -2320,6 +2324,42 @@ def _base_weight_dtype(proj):return quant_state.dtype if quant_state is not None else weight.dtype+_FUSED_LORA_MLPS = (apply_lora_mlp_swiglu, apply_lora_mlp_geglu_exact, apply_lora_mlp_geglu_approx)+++def _decline_fused_lora_for_fsdp(model) -> int:+ """Put peft's forwards back on layers patch_peft_model gave a fused LoRA kernel.++ `TrainingArguments(fsdp=...)` only reaches the Accelerator inside `Trainer.__init__`, after+ patch_peft_model ran with no launcher env to see, so the Trainer calls this once it knows.+ """+ if model is None or os.environ.get("UNSLOTH_FORCE_FUSED_LORA", "0") == "1":+ return 0+ n = 0+ for module in model.modules():+ if getattr(module, "apply_qkv", None) is apply_lora_qkv:+ module.apply_qkv = original_apply_qkv+ n += 1+ if getattr(module, "apply_o", None) is apply_lora_o:+ module.apply_o = original_apply_o+ n += 1+ for name in ("forward", "_unsloth_forward"):+ func = getattr(module.__dict__.get(name), "__func__", None)+ if getattr(func, "func", func) not in _FUSED_LORA_MLPS:+ continue+ if name == "forward":+ delattr(module, "forward")+ else:+ module._unsloth_forward = module.__class__.forward+ n += 1+ if n:+ logger.warning_once(+ f"Unsloth: the Trainer enabled FSDP, so {n} fused LoRA projections were switched back "+ "to peft's forward, which FSDP can unshard. Set UNSLOTH_FORCE_FUSED_LORA=1 to keep them."+ )+ return n++class FastLlamaModel:@staticmethoddef _prepare_for_qat(model, qat_scheme):@@ -3934,8 +3974,15 @@ class FastLlamaModel:float32_base = (_base_weight_dtype(model.model.model.layers[0].self_attn.q_proj) == torch.float32)+ # Fused kernels read `.weight` directly, bypassing FSDP's unshard hook, so under FSDP they see shard views (#409).+ fused_lora_declined_for_fsdp = fsdp_will_wrap()- if lora_dropout == 0 and bias == "none" and not float32_base:+ if (+ lora_dropout == 0+ and bias == "none"+ and not float32_base+ and not fused_lora_declined_for_fsdp+ ):for idx, layer in enumerate(model.model.model.layers):if model_type != "falcon_h1":# LoRAMLP.apply has no gate/down multiplier support yet, so falcon h1 is not patched for now.@@ -4014,7 +4061,9 @@ class FastLlamaModel:)# A zero count reads as a failure, so say why the fused kernels were skipped.- unfused_reason = _fused_lora_skip_reason(lora_dropout, bias, float32_base)+ unfused_reason = _fused_lora_skip_reason(+ lora_dropout, bias, float32_base, fsdp = fused_lora_declined_for_fsdp+ )logger.warning_once(f"Unsloth {__version__} patched {len(model.model.model.layers)} layers with "f"{n_qkv} QKV layers, {n_o} O layers and {n_mlp} MLP layers.{unfused_reason}",diff --git a/unsloth/models/loader_utils.py b/unsloth/models/loader_utils.pyindex f57af4b6f3a..58cdf31cc0e 100644--- a/unsloth/models/loader_utils.py+++ b/unsloth/models/loader_utils.py@@ -127,6 +127,33 @@ def _infer_local_rank(rank):return local_rank+_TRUTHY = ("1", "true", "yes", "on")+++def fsdp_will_wrap():+ """True when this process will hand the model to FSDP (unsloth#409).++ patch_peft_model runs before the Trainer builds its Accelerator, so the launcher env+ (`accelerate launch` exports ACCELERATE_USE_FSDP + FSDP_VERSION for FSDP configs, neither+ for DDP) is usually the only signal. `UNSLOTH_FORCE_FUSED_LORA=1` opts back in.+ """+ if os.environ.get("UNSLOTH_FORCE_FUSED_LORA", "0") == "1":+ return False+ try:+ from accelerate.state import AcceleratorState+ distributed_type = AcceleratorState._shared_state.get("distributed_type", None)+ if distributed_type is not None and "FSDP" in str(distributed_type).upper():+ return True+ except Exception:+ pass+ if str(os.environ.get("ACCELERATE_USE_FSDP", "")).strip().lower() in _TRUTHY:+ return True+ # "0" is how an accelerate config says "not FSDP".+ if str(os.environ.get("FSDP_VERSION", "")).strip() not in ("", "0"):+ return True+ return False++def prepare_device_map():rank, world_size = _infer_distributed_ranks()distributed = (world_size or 1) > 1 or (rank is not None and rank > 0)