Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Refuse an explicit offload_layers swap under FSDP / DeepSpeed (#13857)

Daniel Hanefcaebeae7e1
diff --git a/tests/test_block_swap_auto_ddp.py b/tests/test_block_swap_auto_ddp.py
index d8df0679c0e..794d66f161e 100644
--- a/tests/test_block_swap_auto_ddp.py
+++ b/tests/test_block_swap_auto_ddp.py
@@ -421,7 +421,6 @@ def test_a_sharded_trainer_refuses_an_auto_swap(monkeypatch, backend):
# Zero-swap ranks refuse too, else peers hang.
with pytest.raises(ValueError, match = "swapped 0 decoder layers"):
check(_swapped_model(swapped = (), **{backend: True}))
- assert check(_swapped_model(mark = False, **{backend: True})) == []
# get_peft_model(offload_layers = "auto") alone: every rank refuses, also a zero-swap one.
peft_auto = _swapped_model(mark = False, **{backend: True})
peft_auto.model._unsloth_offload_layers_auto = 1
@@ -432,6 +431,20 @@ def test_a_sharded_trainer_refuses_an_auto_swap(monkeypatch, backend):
check(peft_auto)
[email protected]("backend", ["fsdp", "deepspeed"])
+def test_a_sharded_trainer_refuses_an_explicit_swap(monkeypatch, backend):
+ """offload_layers = N must not reach FSDP / DeepSpeed either: FSDP1 writeback fails on the
+ rebound placeholders ("Cannot writeback when the parameter shape changes")."""
+ ns = _ignore_helpers(monkeypatch)
+ check = ns["_block_swap_for_trainer_backend"]
+ with pytest.raises(ValueError, match = r"offload_layers = 2 .*offload_layers = 0"):
+ check(_swapped_model(mark = False, swapped = (2, 3), **{backend: True}))
+ assert check(_swapped_model(mark = False, swapped = (), **{backend: True})) == []
+ no_swap = _swapped_model(mark = False, **{backend: True})
+ no_swap.model._unsloth_block_swap = None
+ assert check(no_swap) == []
+
+
def test_auto_declines_up_front_on_a_launch_that_shows_a_sharded_backend():
"""A resolver veto or an FSDP / DeepSpeed launch neither swaps nor marks."""
for name in ("llama.py", "vision.py"):
diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.py
index 43cd60adf32..304c4a6a6b5 100644
--- a/unsloth/models/_utils.py
+++ b/unsloth/models/_utils.py
@@ -8497,8 +8497,8 @@ def _ignore_block_swap_params_for_ddp(model):
def _block_swap_for_trainer_backend(trainer):
- """DDP: ignore swapped weights. FSDP / DeepSpeed (invisible at load, so "auto" planned for DDP):
- refuse, since sharding would flatten the 0-element placeholders."""
+ """DDP: ignore swapped weights. FSDP / DeepSpeed (invisible at load): refuse any block swap, since
+ sharding flattens the 0-element placeholders the swap later rebinds."""
model = getattr(trainer, "model", None)
backend = (
"FSDP"
@@ -8523,6 +8523,14 @@ def _block_swap_for_trainer_backend(trainer):
"swapped layers. Load with offload_layers = 0 (and get_peft_model(offload_layers = 0)), "
f"or launch without {backend}."
)
+ # An explicit count swaps the same layers on every rank, so every rank refuses alike.
+ if swapped:
+ raise ValueError(
+ f"Unsloth: offload_layers = {swapped} (decoder layers swapped to host RAM) cannot be "
+ f"combined with {backend}, which shards the weights itself and cannot wrap swapped "
+ "layers. Load with offload_layers = 0 (and get_peft_model(offload_layers = 0)), "
+ f"or launch without {backend}."
+ )
return []