Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Decline the fused LoRA kernels under FSDP, where the weights are shard views (#11132)

Daniel Haneaa56d6bad93
diff --git a/tests/test_fsdp_declines_fused_lora.py b/tests/test_fsdp_declines_fused_lora.py
new file mode 100644
index 00000000000..ad4bb08850b
--- /dev/null
+++ b/tests/test_fsdp_declines_fused_lora.py
@@ -0,0 +1,151 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""unsloth#409: under FSDP the fused LoRA kernels read shard views (`matmul_lora`:
+`size mismatch, got input (20), mat (20x896), vec (3584)` on Qwen2.5-0.5B FULL_SHARD), so
+patch_peft_model must decline them. No GPU: the probe is env-driven, the wiring check textual.
+"""
+
+from __future__ import annotations
+
+import ast
+import os
+from pathlib import Path
+
+import pytest
+
+from unsloth.models.loader_utils import fsdp_will_wrap
+
+_LLAMA = Path(__file__).resolve().parent.parent / "unsloth" / "models" / "llama.py"
+
+_FSDP_ENV = (
+ "ACCELERATE_USE_FSDP",
+ "FSDP_VERSION",
+ "UNSLOTH_FORCE_FUSED_LORA",
+)
+
+
[email protected](autouse = True)
+def _clean_env(monkeypatch):
+ for key in _FSDP_ENV:
+ monkeypatch.delenv(key, raising = False)
+
+
+def test_a_plain_launch_keeps_the_fused_kernels(monkeypatch):
+ """DDP and single-GPU launches export neither variable and keep the fast path."""
+ assert fsdp_will_wrap() is False
+ monkeypatch.setenv("ACCELERATE_MIXED_PRECISION", "bf16")
+ assert fsdp_will_wrap() is False
+
+
[email protected]("value", ["true", "True", "1", "yes", "ON"])
+def test_accelerate_use_fsdp_declines(monkeypatch, value):
+ """`accelerate launch` with an FSDP config exports the literal "true"."""
+ monkeypatch.setenv("ACCELERATE_USE_FSDP", value)
+ assert fsdp_will_wrap() is True
+
+
[email protected]("value", ["false", "0", "", "no"])
+def test_a_falsy_accelerate_use_fsdp_keeps_the_fused_kernels(monkeypatch, value):
+ monkeypatch.setenv("ACCELERATE_USE_FSDP", value)
+ assert fsdp_will_wrap() is False
+
+
[email protected]("value,expected", [("1", True), ("2", True), ("0", False), ("", False)])
+def test_fsdp_version_alone_is_enough(monkeypatch, value, expected):
+ """torchrun users set it by hand; "0" is how a config says "not FSDP"."""
+ monkeypatch.setenv("FSDP_VERSION", value)
+ assert fsdp_will_wrap() is expected
+
+
+def test_the_override_wins_over_every_signal(monkeypatch):
+ monkeypatch.setenv("ACCELERATE_USE_FSDP", "true")
+ monkeypatch.setenv("FSDP_VERSION", "2")
+ monkeypatch.setenv("UNSLOTH_FORCE_FUSED_LORA", "1")
+ assert fsdp_will_wrap() is False
+
+
+def test_a_live_accelerator_state_is_consulted_first(monkeypatch):
+ """A caller-built Accelerator carries no env, so its live state must be read."""
+ accelerate_state = pytest.importorskip("accelerate.state")
+ shared = accelerate_state.AcceleratorState._shared_state
+ monkeypatch.setitem(shared, "distributed_type", "DistributedType.FSDP")
+ assert fsdp_will_wrap() is True
+
+
+def test_the_fused_lora_install_is_gated_on_the_probe():
+ """The three fused installs share one `if`; splitting them must carry the gate."""
+ source = _LLAMA.read_text(encoding = "utf-8")
+ assert "fsdp_will_wrap" in source, "llama.py no longer probes for FSDP"
+
+ tree = ast.parse(source)
+ guarded = []
+ for node in ast.walk(tree):
+ if not isinstance(node, ast.If):
+ continue
+ test = ast.unparse(node.test)
+ if "fused_lora_declined_for_fsdp" not in test:
+ continue
+ body = ast.unparse(node)
+ if "apply_qkv" in body or "apply_o" in body or "_apply_lora_mlp" in body:
+ guarded.append(test)
+ assert guarded, (
+ "no `if` guarding apply_qkv / apply_o / apply_lora_mlp consults "
+ "fused_lora_declined_for_fsdp; under FSDP the fused kernels will read "
+ "shard views again"
+ )
+
+
+def test_the_declined_message_names_the_override():
+ from unsloth.models.llama import _fused_lora_skip_reason
+ assert _fused_lora_skip_reason(0, "none") == ""
+ assert "UNSLOTH_FORCE_FUSED_LORA" in _fused_lora_skip_reason(0, "none", fsdp = True)
+
+
+def _fused_model(llama):
+ import types
+
+ import torch
+
+ attn = torch.nn.Module()
+ attn.apply_qkv = llama.apply_lora_qkv
+ attn.apply_o = llama.apply_lora_o
+ mlp = torch.nn.Linear(2, 2)
+ mlp.forward = types.MethodType(llama.apply_lora_mlp_swiglu, mlp)
+ tiled = torch.nn.Linear(2, 2)
+ tiled._unsloth_forward = types.MethodType(llama.apply_lora_mlp_geglu_approx, tiled)
+ root = torch.nn.Module()
+ root.attn, root.mlp, root.tiled = attn, mlp, tiled
+ return root
+
+
+def test_a_trainer_side_fsdp_gets_peft_forwards_back():
+ """`SFTConfig(fsdp=...)` exports no env: the Trainer undoes the fused installs once it knows."""
+ import torch
+ from unsloth.models import llama
+
+ root = _fused_model(llama)
+ assert llama._decline_fused_lora_for_fsdp(root) == 4
+ assert root.attn.apply_qkv is llama.original_apply_qkv
+ assert root.attn.apply_o is llama.original_apply_o
+ assert "forward" not in root.mlp.__dict__
+ assert root.tiled._unsloth_forward is torch.nn.Linear.forward
+ assert llama._decline_fused_lora_for_fsdp(root) == 0
+
+
+def test_the_trainer_side_decline_honours_the_override(monkeypatch):
+ from unsloth.models import llama
+
+ monkeypatch.setenv("UNSLOTH_FORCE_FUSED_LORA", "1")
+ root = _fused_model(llama)
+ assert llama._decline_fused_lora_for_fsdp(root) == 0
+ assert root.attn.apply_qkv is llama.apply_lora_qkv
+
+
+def test_the_trainer_init_wrapper_calls_the_decline():
+ import inspect
+
+ from unsloth.models import _utils
+
+ source = inspect.getsource(_utils.patch_gradient_accumulation_fix)
+ assert "is_fsdp_enabled" in source and "_decline_fused_lora_for_fsdp" in source
diff --git a/tests/test_lora_dropout_patch_summary.py b/tests/test_lora_dropout_patch_summary.py
index 72f66675ab5..4ea4bebbf66 100644
--- a/tests/test_lora_dropout_patch_summary.py
+++ b/tests/test_lora_dropout_patch_summary.py
@@ -64,13 +64,20 @@ def test_reason_appends_cleanly_to_the_summary_sentence():
@pytest.mark.parametrize(
- "lora_dropout,bias",
- list(itertools.product([0, 0.0, 0.1, 0.5], ["none", "all", "lora_only"])),
+ "lora_dropout,bias,float32_base,fsdp",
+ list(
+ itertools.product(
+ [0, 0.0, 0.1, 0.5], ["none", "all", "lora_only"], [False, True], [False, True]
+ )
+ ),
)
-def test_reason_is_non_empty_exactly_when_the_fused_gate_is_closed(lora_dropout, bias):
- """Parity with the `lora_dropout == 0 and bias == "none"` gate in patch_peft_model."""
- fused_installed = lora_dropout == 0 and bias == "none"
- assert bool(_fused_lora_skip_reason(lora_dropout, bias)) is not fused_installed
+def test_reason_is_non_empty_exactly_when_the_fused_gate_is_closed(
+ lora_dropout, bias, float32_base, fsdp
+):
+ """Parity with the fused-kernel gate in patch_peft_model."""
+ fused_installed = lora_dropout == 0 and bias == "none" and not float32_base and not fsdp
+ reason = _fused_lora_skip_reason(lora_dropout, bias, float32_base, fsdp = fsdp)
+ assert bool(reason) is not fused_installed
def _gate_tests(source: str) -> list[str]:
@@ -87,15 +94,20 @@ def _gate_tests(source: str) -> list[str]:
def test_patch_peft_model_still_gates_the_fused_kernels_on_the_same_values():
- """If the gate grows a third condition, _fused_lora_skip_reason has to grow with it or
- the summary starts reporting zero counts with no reason again."""
+ """Every gate condition needs a matching _fused_lora_skip_reason clause."""
source = inspect.getsource(llama_module.FastLlamaModel.patch_peft_model)
- assert _gate_tests(source) == ["lora_dropout == 0 and bias == 'none' and (not float32_base)"]
+ assert _gate_tests(source) == [
+ "lora_dropout == 0 and bias == 'none' and (not float32_base) and (not fused_lora_declined_for_fsdp)"
+ ]
def test_the_summary_call_carries_the_reason():
source = inspect.getsource(llama_module.FastLlamaModel.patch_peft_model)
- assert "unfused_reason = _fused_lora_skip_reason(lora_dropout, bias, float32_base)" in source
+ flat = "".join(source.split())
+ assert (
+ "unfused_reason=_fused_lora_skip_reason(lora_dropout,bias,float32_base,fsdp=fused_lora_declined_for_fsdp)"
+ in flat
+ )
assert "MLP layers.{unfused_reason}" in source
diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.py
index b79e67ccccf..780fa4f6a47 100644
--- a/unsloth/models/_utils.py
+++ b/unsloth/models/_utils.py
@@ -4882,6 +4882,9 @@ def patch_gradient_accumulation_fix(Trainer):
pass
except Exception:
pass
+ if getattr(self, "is_fsdp_enabled", False):
+ from .llama import _decline_fused_lora_for_fsdp
+ _decline_fused_lora_for_fsdp(getattr(self, "model", None))
_unsloth_trainer_init.__wrapped__ = _original_trainer_init
Trainer.__init__ = _unsloth_trainer_init
diff --git a/unsloth/models/llama.py b/unsloth/models/llama.py
index 8852f8458a6..70e355d5616 100644
--- a/unsloth/models/llama.py
+++ b/unsloth/models/llama.py
@@ -37,6 +37,7 @@ from .loader_utils import (
_get_fp8_mode_and_check_settings,
_restore_dropped_fp8_scales,
_prepare_compressed_tensors_model,
+ fsdp_will_wrap,
planner_class_mismatch_reason,
planner_model_class,
planner_config_overrides,
@@ -2293,11 +2294,12 @@ def _fused_lora_skip_reason(
lora_dropout,
bias,
float32_base = False,
+ fsdp = False,
) -> str:
"""Why patch_peft_model skipped the fused LoRA kernels, for the patched layers summary.
Returns "" when nothing disabled them, so the common summary line is unchanged. The
- conditions mirror the `lora_dropout == 0 and bias == "none"` gate in patch_peft_model.
+ conditions mirror the fused-kernel gate in patch_peft_model.
"""
reasons = []
if lora_dropout != 0:
@@ -2306,6 +2308,8 @@ def _fused_lora_skip_reason(
reasons.append(f"bias = '{bias}'")
if float32_base:
reasons.append("the base weights are float32")
+ if fsdp:
+ reasons.append("FSDP shards the weights they read (UNSLOTH_FORCE_FUSED_LORA=1 overrides)")
if not reasons:
return ""
return (
@@ -2320,6 +2324,42 @@ def _base_weight_dtype(proj):
return quant_state.dtype if quant_state is not None else weight.dtype
+_FUSED_LORA_MLPS = (apply_lora_mlp_swiglu, apply_lora_mlp_geglu_exact, apply_lora_mlp_geglu_approx)
+
+
+def _decline_fused_lora_for_fsdp(model) -> int:
+ """Put peft's forwards back on layers patch_peft_model gave a fused LoRA kernel.
+
+ `TrainingArguments(fsdp=...)` only reaches the Accelerator inside `Trainer.__init__`, after
+ patch_peft_model ran with no launcher env to see, so the Trainer calls this once it knows.
+ """
+ if model is None or os.environ.get("UNSLOTH_FORCE_FUSED_LORA", "0") == "1":
+ return 0
+ n = 0
+ for module in model.modules():
+ if getattr(module, "apply_qkv", None) is apply_lora_qkv:
+ module.apply_qkv = original_apply_qkv
+ n += 1
+ if getattr(module, "apply_o", None) is apply_lora_o:
+ module.apply_o = original_apply_o
+ n += 1
+ for name in ("forward", "_unsloth_forward"):
+ func = getattr(module.__dict__.get(name), "__func__", None)
+ if getattr(func, "func", func) not in _FUSED_LORA_MLPS:
+ continue
+ if name == "forward":
+ delattr(module, "forward")
+ else:
+ module._unsloth_forward = module.__class__.forward
+ n += 1
+ if n:
+ logger.warning_once(
+ f"Unsloth: the Trainer enabled FSDP, so {n} fused LoRA projections were switched back "
+ "to peft's forward, which FSDP can unshard. Set UNSLOTH_FORCE_FUSED_LORA=1 to keep them."
+ )
+ return n
+
+
class FastLlamaModel:
@staticmethod
def _prepare_for_qat(model, qat_scheme):
@@ -3934,8 +3974,15 @@ class FastLlamaModel:
float32_base = (
_base_weight_dtype(model.model.model.layers[0].self_attn.q_proj) == torch.float32
)
+ # Fused kernels read `.weight` directly, bypassing FSDP's unshard hook, so under FSDP they see shard views (#409).
+ fused_lora_declined_for_fsdp = fsdp_will_wrap()
- if lora_dropout == 0 and bias == "none" and not float32_base:
+ if (
+ lora_dropout == 0
+ and bias == "none"
+ and not float32_base
+ and not fused_lora_declined_for_fsdp
+ ):
for idx, layer in enumerate(model.model.model.layers):
if model_type != "falcon_h1":
# LoRAMLP.apply has no gate/down multiplier support yet, so falcon h1 is not patched for now.
@@ -4014,7 +4061,9 @@ class FastLlamaModel:
)
# A zero count reads as a failure, so say why the fused kernels were skipped.
- unfused_reason = _fused_lora_skip_reason(lora_dropout, bias, float32_base)
+ unfused_reason = _fused_lora_skip_reason(
+ lora_dropout, bias, float32_base, fsdp = fused_lora_declined_for_fsdp
+ )
logger.warning_once(
f"Unsloth {__version__} patched {len(model.model.model.layers)} layers with "
f"{n_qkv} QKV layers, {n_o} O layers and {n_mlp} MLP layers.{unfused_reason}",
diff --git a/unsloth/models/loader_utils.py b/unsloth/models/loader_utils.py
index f57af4b6f3a..58cdf31cc0e 100644
--- a/unsloth/models/loader_utils.py
+++ b/unsloth/models/loader_utils.py
@@ -127,6 +127,33 @@ def _infer_local_rank(rank):
return local_rank
+_TRUTHY = ("1", "true", "yes", "on")
+
+
+def fsdp_will_wrap():
+ """True when this process will hand the model to FSDP (unsloth#409).
+
+ patch_peft_model runs before the Trainer builds its Accelerator, so the launcher env
+ (`accelerate launch` exports ACCELERATE_USE_FSDP + FSDP_VERSION for FSDP configs, neither
+ for DDP) is usually the only signal. `UNSLOTH_FORCE_FUSED_LORA=1` opts back in.
+ """
+ if os.environ.get("UNSLOTH_FORCE_FUSED_LORA", "0") == "1":
+ return False
+ try:
+ from accelerate.state import AcceleratorState
+ distributed_type = AcceleratorState._shared_state.get("distributed_type", None)
+ if distributed_type is not None and "FSDP" in str(distributed_type).upper():
+ return True
+ except Exception:
+ pass
+ if str(os.environ.get("ACCELERATE_USE_FSDP", "")).strip().lower() in _TRUTHY:
+ return True
+ # "0" is how an accelerate config says "not FSDP".
+ if str(os.environ.get("FSDP_VERSION", "")).strip() not in ("", "0"):
+ return True
+ return False
+
+
def prepare_device_map():
rank, world_size = _infer_distributed_ranks()
distributed = (world_size or 1) > 1 or (rank is not None and rank > 0)