Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Allow offload_layers = "auto" under a DDP launch (#13364)

Daniel Handbb698e1ba0e
diff --git a/tests/test_block_swap_auto_ddp.py b/tests/test_block_swap_auto_ddp.py
new file mode 100644
index 00000000000..d8df0679c0e
--- /dev/null
+++ b/tests/test_block_swap_auto_ddp.py
@@ -0,0 +1,544 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+"""from_pretrained(offload_layers = "auto") under a DDP launch. No GPU needed; extracted with ast."""
+
+import ast
+import importlib.util
+import os
+import sys
+import types
+
+import pytest
+import torch
+
+HERE = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+MODELS = os.path.join(HERE, "unsloth", "models")
+LOADER_UTILS = os.path.join(MODELS, "loader_utils.py")
+UTILS = os.path.join(MODELS, "_utils.py")
+_FAKE_PKG = "_block_swap_auto_ddp_pkg"
+
+
+def _extract(
+ path,
+ functions,
+ assigns = (),
+):
+ src = open(path, encoding = "utf-8").read()
+ out = []
+ for node in ast.parse(src).body:
+ if isinstance(node, ast.FunctionDef) and node.name in functions:
+ out.append(ast.get_source_segment(src, node))
+ elif isinstance(node, ast.Assign) and getattr(node.targets[0], "id", None) in assigns:
+ out.append(ast.get_source_segment(src, node))
+ return "\n\n".join(out)
+
+
+class _FakeCuda:
+ def __init__(
+ self,
+ count = 2,
+ free = 8 << 30,
+ ):
+ self.count, self.free = count, free
+
+ def is_available(self):
+ return True
+
+ def device_count(self):
+ return self.count
+
+ def current_device(self):
+ return 0
+
+
+class _Plan:
+ def __init__(
+ self,
+ layers,
+ offload_embedding = False,
+ ):
+ self.layers, self.offload_embedding, self.device_plan = layers, offload_embedding, None
+
+ def describe(self):
+ return "block swap: fabricated plan"
+
+
+def resolve(monkeypatch):
+ """resolve_auto_block_swap over a fabricated CUDA, planner and launcher env."""
+ for key in (
+ "ACCELERATE_USE_FSDP",
+ "FSDP_VERSION",
+ "ACCELERATE_USE_DEEPSPEED",
+ "UNSLOTH_FORCE_FUSED_LORA",
+ ):
+ monkeypatch.delenv(key, raising = False)
+ state = types.SimpleNamespace(distributed = False, calls = [], layers = 6)
+
+ def plan_block_swap(
+ model_name,
+ max_memory,
+ seq_len,
+ offload_embedding = None,
+ **options,
+ ):
+ call = dict(max_memory = max_memory, **options)
+ if offload_embedding is not None:
+ call["offload_embedding"] = offload_embedding
+ state.calls.append(call)
+ return _Plan(state.layers, bool(offload_embedding))
+
+ planner = types.ModuleType("unsloth_zoo.device_map_planner")
+ planner.plan_block_swap = plan_block_swap
+ monkeypatch.setitem(sys.modules, "unsloth_zoo.device_map_planner", planner)
+ package = types.ModuleType(_FAKE_PKG)
+ package.__path__ = []
+ utils = types.ModuleType(f"{_FAKE_PKG}._utils")
+ utils.usable_cuda_bytes = lambda device: 8 << 30
+ utils.auto_plan_depth = lambda depth: depth
+ monkeypatch.setitem(sys.modules, _FAKE_PKG, package)
+ monkeypatch.setitem(sys.modules, f"{_FAKE_PKG}._utils", utils)
+ fake_torch = types.SimpleNamespace(
+ cuda = _FakeCuda(), device = torch.device, tensor = torch.tensor, int64 = torch.int64
+ )
+ ns = {
+ "__name__": f"{_FAKE_PKG}.loader_utils",
+ "__package__": _FAKE_PKG,
+ "os": os,
+ "inspect": __import__("inspect"),
+ "torch": fake_torch,
+ "DEVICE_TYPE_TORCH": "cuda",
+ "is_distributed": lambda: state.distributed,
+ }
+ exec(
+ _extract(
+ LOADER_UTILS,
+ (
+ "resolve_auto_block_swap",
+ "_sharded_data_parallel_launch",
+ "_single_card_device_map",
+ "_single_device_index",
+ "ddp_auto_block_swap_launch",
+ "auto_offload_for_launch",
+ "_as_bytes",
+ ),
+ (
+ "_TRUTHY",
+ "_SIZE_UNITS",
+ "UNSLOTH_DEVICE_MAP",
+ "UNSLOTH_BALANCED_DEVICE_MAP",
+ "_PLANNED_DEVICE_MAPS",
+ "TRANSFORMERS_PLACEMENT_STRATEGIES",
+ "_BLOCK_SWAP_PLANNER_KEYS",
+ ),
+ ),
+ ns,
+ )
+ state.fn = lambda device_map, **kw: ns["resolve_auto_block_swap"](
+ device_map, "m", max_seq_length = 2048, **kw
+ )
+ state.ns = ns
+ return state
+
+
+def test_a_ddp_rank_plans_its_own_card(resolve):
+ """torchrun gives each rank {"": "cuda:LOCAL_RANK"}: plan that card, keep the embedding on it."""
+ resolve.distributed = True
+ layers, device_map, embedding = resolve.fn({"": "cuda:1"}, offload_embedding = True)
+ assert (layers, device_map, embedding) == (6, {"": 1}, False)
+ assert len(resolve.calls) == 1
+ assert list(resolve.calls[0]["max_memory"]) == [1]
+ assert "offload_embedding" not in resolve.calls[0]
+
+
[email protected]("device_map, card", [(1, 1), ("cuda:1", 1)])
+def test_a_ddp_rank_plans_a_bare_card_map(resolve, device_map, card):
+ """device_map = 1 or "cuda:1" (no dict) is the same single-card load as {"": "cuda:1"}."""
+ resolve.distributed = True
+ layers, out_map, embedding = resolve.fn(device_map)
+ assert (layers, out_map, embedding) == (6, {"": card}, False)
+ assert list(resolve.calls[0]["max_memory"]) == [card]
+
+
+def test_a_ddp_rank_with_room_swaps_nothing(resolve):
+ resolve.distributed, resolve.layers = True, 0
+ assert resolve.fn({"": "cuda:0"}) == (0, {"": "cuda:0"}, False)
+
+
+def test_a_ddp_rank_honours_a_max_memory_for_its_card(resolve):
+ resolve.distributed = True
+ resolve.fn({"": "cuda:1"}, planner_kwargs = {"max_memory": {1: "2GiB"}})
+ assert resolve.calls[0]["max_memory"] == {1: 2 << 30}
+
+
+ "env",
+ [
+ {"ACCELERATE_USE_FSDP": "true"},
+ {"FSDP_VERSION": "2"},
+ {"ACCELERATE_USE_DEEPSPEED": "true"},
+ ],
+)
+def test_fsdp_and_deepspeed_launches_stay_refused(resolve, monkeypatch, capsys, env):
+ resolve.distributed = True
+ for key, value in env.items():
+ monkeypatch.setenv(key, value)
+ assert resolve.fn({"": "cuda:0"}) == (0, {"": "cuda:0"}, None)
+ assert resolve.calls == []
+ assert "shards the weights" in capsys.readouterr().out
+
+
+ "device_map",
+ ["unsloth", "unsloth_balanced", "auto", "sequential", None, {"a": 0, "b": 1}, {"": "cpu"}],
+)
+def test_ddp_never_plans_a_multi_gpu_split_or_a_non_cuda_map(resolve, device_map):
+ resolve.distributed = True
+ layers, returned, embedding = resolve.fn(device_map)
+ assert (layers, embedding) == (0, None)
+ assert returned == device_map
+ assert resolve.calls == []
+
+
+def test_a_single_process_still_offers_the_embedding(resolve):
+ """Unchanged outside a distributed launch."""
+ resolve.fn({"": "cuda:0"}, offload_embedding = True)
+ assert resolve.calls[0]["offload_embedding"] is True
+
+
+def test_planning_never_runs_a_collective(resolve, monkeypatch):
+ """No collective at load: a loader only some ranks reach would hang."""
+ import torch.distributed as dist
+
+ calls = []
+
+ def collective(*args, **kwargs):
+ calls.append(args)
+ raise AssertionError("collective inside from_pretrained")
+
+ monkeypatch.setattr(dist, "is_available", lambda: True)
+ monkeypatch.setattr(dist, "is_initialized", lambda: True)
+ monkeypatch.setattr(dist, "get_backend", lambda *a, **k: "gloo")
+ for name in (
+ "all_reduce",
+ "all_gather",
+ "all_gather_object",
+ "broadcast",
+ "broadcast_object_list",
+ "barrier",
+ ):
+ monkeypatch.setattr(dist, name, collective)
+ resolve.distributed = True
+ assert resolve.fn({"": "cuda:1"}) == (6, {"": 1}, False)
+ assert calls == []
+
+
+def _ignore_helpers(monkeypatch, distributed = True):
+ zoo_root = importlib.util.find_spec("unsloth_zoo").submodule_search_locations[0]
+ spec = importlib.util.spec_from_file_location(
+ "_block_swap_for_ddp_test", os.path.join(zoo_root, "block_swap.py")
+ )
+ block_swap = importlib.util.module_from_spec(spec)
+ spec.loader.exec_module(block_swap)
+ package = types.ModuleType(_FAKE_PKG)
+ package.__path__ = []
+ loader_utils = types.ModuleType(f"{_FAKE_PKG}.loader_utils")
+ loader_utils.is_distributed = lambda: distributed
+ monkeypatch.setitem(sys.modules, _FAKE_PKG, package)
+ monkeypatch.setitem(sys.modules, f"{_FAKE_PKG}.loader_utils", loader_utils)
+ ns = {
+ "__name__": f"{_FAKE_PKG}._utils",
+ "__package__": _FAKE_PKG,
+ "torch": torch,
+ "find_decoder_layers": block_swap.find_decoder_layers,
+ }
+ exec(
+ _extract(
+ UTILS,
+ (
+ "mark_auto_offload_for_ddp",
+ "_ignore_block_swap_params_for_ddp",
+ "_block_swap_for_trainer_backend",
+ ),
+ ),
+ ns,
+ )
+ return ns
+
+
+def _tiny_peft_model(mark = True):
+ transformers = pytest.importorskip("transformers")
+ peft = pytest.importorskip("peft")
+ config = transformers.LlamaConfig(
+ vocab_size = 64,
+ hidden_size = 32,
+ intermediate_size = 64,
+ num_hidden_layers = 3,
+ num_attention_heads = 4,
+ num_key_value_heads = 2,
+ )
+ torch.manual_seed(0)
+ model = transformers.LlamaForCausalLM(config)
+ if mark:
+ model._unsloth_offload_layers_auto_ddp = True
+ # What _exclude_rope_inv_freq_from_ddp leaves on the base model before get_peft_model.
+ model._ddp_params_and_buffers_to_ignore = ["model.rotary_emb.inv_freq"]
+ lora = peft.LoraConfig(r = 4, target_modules = ["q_proj", "v_proj", "down_proj"])
+ return peft.get_peft_model(model, lora)
+
+
+def test_ignore_list_names_the_frozen_decoder_weights_of_the_wrapped_model(monkeypatch):
+ ns = _ignore_helpers(monkeypatch)
+ model = _tiny_peft_model()
+ layer = model.base_model.model.model.layers[2]
+ for p in layer.parameters():
+ if not p.requires_grad:
+ p.data = torch.empty(0, dtype = p.dtype)
+ added = ns["_ignore_block_swap_params_for_ddp"](model)
+ params = dict(model.named_parameters())
+ assert added and all(name in params for name in added)
+ assert not any(params[name].requires_grad for name in added)
+ assert not any("lora_" in name for name in added)
+ frozen_layer_params = {
+ name for name, p in params.items() if ".layers." in name and not p.requires_grad
+ }
+ assert set(added) == frozen_layer_params
+ # Embedding, final norm and lm_head stay synced; the rotary entry survives the merge.
+ assert not any(
+ "embed_tokens" in n or "lm_head" in n or n.endswith("model.norm.weight") for n in added
+ )
+ ignored = model._ddp_params_and_buffers_to_ignore
+ assert "model.rotary_emb.inv_freq" in ignored and set(added) <= set(ignored)
+ assert ns["_ignore_block_swap_params_for_ddp"](model) == []
+ assert len(model._ddp_params_and_buffers_to_ignore) == len(set(ignored))
+
+
+def test_ignore_list_is_the_same_whatever_this_rank_swapped(monkeypatch):
+ """Rank 0 swapped layer 2, rank 1 nothing: both must hand DDP the same list."""
+ ns = _ignore_helpers(monkeypatch)
+ swapped, resident = _tiny_peft_model(), _tiny_peft_model()
+ for p in swapped.base_model.model.model.layers[2].parameters():
+ if not p.requires_grad:
+ p.data = torch.empty(0, dtype = p.dtype)
+ assert ns["_ignore_block_swap_params_for_ddp"](swapped) == ns[
+ "_ignore_block_swap_params_for_ddp"
+ ](resident)
+
+
+def test_ignore_list_untouched_without_auto_or_outside_ddp(monkeypatch):
+ ns = _ignore_helpers(monkeypatch)
+ model = _tiny_peft_model(mark = False)
+ assert ns["_ignore_block_swap_params_for_ddp"](model) == []
+ ns = _ignore_helpers(monkeypatch, distributed = False)
+ assert ns["_ignore_block_swap_params_for_ddp"](_tiny_peft_model()) == []
+
+
+def test_mark_follows_the_loaders_ddp_decision(monkeypatch):
+ ns = _ignore_helpers(monkeypatch)
+ for ddp_auto in (True, False):
+ model = types.SimpleNamespace()
+ ns["mark_auto_offload_for_ddp"](model, ddp_auto)
+ assert getattr(model, "_unsloth_offload_layers_auto_ddp", False) is ddp_auto
+
+
+def test_ddp_auto_launch_needs_ddp_and_one_card(resolve, monkeypatch):
+ """A launch already showing FSDP / DeepSpeed keeps main's fallback."""
+ launch = resolve.ns["ddp_auto_block_swap_launch"]
+ assert not launch({"": "cuda:0"})
+ resolve.distributed = True
+ assert launch({"": "cuda:0"}) and launch(0) and launch("cuda:1")
+ assert not launch("auto") and not launch({"a": 0, "b": 1})
+ monkeypatch.setenv("ACCELERATE_USE_FSDP", "true")
+ assert not launch({"": "cuda:0"})
+
+
+def test_loaders_keep_the_embedding_on_the_card_for_a_ddp_plan():
+ """The loaders must drop the caller's offload_embedding themselves."""
+ for name, var, veto in (
+ ("llama.py", "offload_layers", "offload_embedding = _offload_embedding_mode = False"),
+ ("vision.py", "_offload_layers", "offload_embedding = False"),
+ ):
+ src = open(os.path.join(MODELS, name), encoding = "utf-8").read()
+ call = src.index(f"{var}, device_map, _embedding_needed = resolve_auto_block_swap(")
+ before, after = src[call - 200 : call], src[call : call + 4000]
+ assert (
+ "_ddp_auto_swap = ddp_auto_block_swap_launch(requested_device_map(device_map))"
+ in before
+ )
+ assert (
+ "if _ddp_auto_swap:" in after and veto in after.split("if _ddp_auto_swap:", 1)[1][:200]
+ )
+ assert "mark_auto_offload_for_ddp(model, _ddp_auto_swap)" in src
+
+
+def test_wiring_trainer_init_and_both_loaders():
+ utils = open(UTILS, encoding = "utf-8").read()
+ init = next(
+ n
+ for n in ast.walk(ast.parse(utils))
+ if isinstance(n, ast.FunctionDef) and n.name == "_unsloth_trainer_init"
+ )
+ assert "_block_swap_for_trainer_backend(self)" in ast.get_source_segment(utils, init)
+ for name in ("llama.py", "vision.py"):
+ src = open(os.path.join(MODELS, name), encoding = "utf-8").read()
+ assert "mark_auto_offload_for_ddp(model, _ddp_auto_swap)" in src, name
+ # The pre-wrapped PEFT branch installs the swap after its rope pass, so it applies the list itself.
+ llama = open(os.path.join(MODELS, "llama.py"), encoding = "utf-8").read()
+ install = llama.index("install_block_swap(\n model,")
+ assert "_ignore_block_swap_params_for_ddp(model)" in llama[install : install + 600]
+
+
+def _swapped_model(
+ fsdp = False,
+ deepspeed = False,
+ swapped = (2,),
+ mark = True,
+):
+ model = types.SimpleNamespace(_unsloth_block_swap = types.SimpleNamespace(indices = list(swapped)))
+ if mark:
+ model._unsloth_offload_layers_auto_ddp = True
+ return types.SimpleNamespace(model = model, is_fsdp_enabled = fsdp, is_deepspeed_enabled = deepspeed)
+
+
[email protected]("backend", ["fsdp", "deepspeed"])
+def test_a_sharded_trainer_refuses_an_auto_swap(monkeypatch, backend):
+ """FSDP / DeepSpeed invisible at load must not wrap 0-element placeholders."""
+ ns = _ignore_helpers(monkeypatch)
+ check = ns["_block_swap_for_trainer_backend"]
+ with pytest.raises(ValueError, match = "offload_layers = 0"):
+ check(_swapped_model(**{backend: True}))
+ # Zero-swap ranks refuse too, else peers hang.
+ with pytest.raises(ValueError, match = "swapped 0 decoder layers"):
+ check(_swapped_model(swapped = (), **{backend: True}))
+ assert check(_swapped_model(mark = False, **{backend: True})) == []
+ # get_peft_model(offload_layers = "auto") alone: every rank refuses, also a zero-swap one.
+ peft_auto = _swapped_model(mark = False, **{backend: True})
+ peft_auto.model._unsloth_offload_layers_auto = 1
+ with pytest.raises(ValueError, match = "swapped 1 decoder layers"):
+ check(peft_auto)
+ peft_auto.model._unsloth_block_swap.indices = []
+ with pytest.raises(ValueError, match = "swapped 0 decoder layers"):
+ check(peft_auto)
+
+
+def test_auto_declines_up_front_on_a_launch_that_shows_a_sharded_backend():
+ """A resolver veto or an FSDP / DeepSpeed launch neither swaps nor marks."""
+ for name in ("llama.py", "vision.py"):
+ src = open(os.path.join(MODELS, name), encoding = "utf-8").read()
+ veto = src.index("_ddp_auto_swap = _ddp_auto_swap and _embedding_needed is not None")
+ assert (
+ veto
+ < src.index("if _ddp_auto_swap:", veto)
+ < src.index("mark_auto_offload_for_ddp(model, _ddp_auto_swap)")
+ ), name
+ for name, calls in (("llama.py", 2), ("vision.py", 1)):
+ src = open(os.path.join(MODELS, name), encoding = "utf-8").read()
+ assert src.count("install_block_swap(") == calls
+ assert src.count("auto_offload_for_launch(offload_layers),") == calls, name
+
+
+def test_get_peft_auto_declines_on_a_launch_that_shows_a_sharded_backend(resolve, monkeypatch):
+ decline = resolve.ns["auto_offload_for_launch"]
+ assert decline("auto") == "auto" and decline(4) == 4
+ resolve.distributed = True
+ assert decline("auto") == "auto"
+ monkeypatch.setenv("ACCELERATE_USE_DEEPSPEED", "true")
+ assert decline("auto") == 0 and decline(4) == 4
+
+
+def test_ignore_list_is_applied_at_the_end_of_get_peft_model():
+ """Custom loops skip the Trainer, so both get_peft_model paths apply the list."""
+ for name, owner in (("llama.py", "FastLlamaModel"), ("vision.py", "FastBaseModel")):
+ src = open(os.path.join(MODELS, name), encoding = "utf-8").read()
+ cls = next(
+ n for n in ast.walk(ast.parse(src)) if isinstance(n, ast.ClassDef) and n.name == owner
+ )
+ fn = next(
+ n for n in cls.body if isinstance(n, ast.FunctionDef) and n.name == "get_peft_model"
+ )
+ body = ast.get_source_segment(src, fn)
+ tail = body[body.rindex("_exclude_rope_inv_freq_from_ddp(model)") :]
+ assert "_ignore_block_swap_params_for_ddp(model)" in tail.split("return model")[0], name
+ rope = _extract(LOADER_UTILS, ("_exclude_rope_inv_freq_from_ddp",))
+ assert "_ignore_block_swap_params_for_ddp" not in rope
+
+
+def test_load_time_frozen_set_is_rank_dependent(monkeypatch):
+ """Why the list is never built at load: there only swapped layers are frozen."""
+ ns = _ignore_helpers(monkeypatch)
+ transformers = pytest.importorskip("transformers")
+ config = transformers.LlamaConfig(
+ vocab_size = 64,
+ hidden_size = 32,
+ intermediate_size = 64,
+ num_hidden_layers = 3,
+ num_attention_heads = 4,
+ num_key_value_heads = 2,
+ )
+ frozen = []
+ for swapped in (1, 2):
+ model = transformers.LlamaForCausalLM(config)
+ model._unsloth_offload_layers_auto_ddp = True
+ for layer in model.model.layers[3 - swapped :]:
+ layer.requires_grad_(False)
+ frozen.append({n for n, p in model.named_parameters() if not p.requires_grad})
+ assert ns["_ignore_block_swap_params_for_ddp"](model) == []
+ assert not getattr(model, "_ddp_params_and_buffers_to_ignore", None)
+ assert frozen[0] != frozen[1]
+
+
+def test_ignore_list_is_applied_when_from_pretrained_loads_an_adapter():
+ """Loaded adapters skip get_peft_model: both loader returns apply the list."""
+ src = open(os.path.join(MODELS, "loader.py"), encoding = "utf-8").read()
+ rope = "model = _exclude_rope_inv_freq_from_ddp(model)\n"
+ assert src.count(rope) == 2
+ for at in [i for i in range(len(src)) if src.startswith(rope, i)]:
+ assert "_ignore_block_swap_params_for_ddp(model)" in src[at : at + 300]
+
+
+def test_full_finetuning_never_ignores_a_trainable_weight(monkeypatch):
+ """Trainable decoder weights are refused, never ignored."""
+ ns = _ignore_helpers(monkeypatch)
+ model = _tiny_peft_model()
+ for p in model.parameters():
+ p.requires_grad_(True)
+ with pytest.raises(ValueError, match = "decoder weights are trainable"):
+ ns["_ignore_block_swap_params_for_ddp"](model)
+ assert model._ddp_params_and_buffers_to_ignore == ["model.rotary_emb.inv_freq"]
+
+
[email protected]("bias, refused", [("none", False), ("lora_only", True), ("all", True)])
+def test_trainable_decoder_biases_are_refused_on_every_rank(monkeypatch, bias, refused):
+ """PEFT bias refusal reads the config, so zero-swap ranks refuse too."""
+ ns = _ignore_helpers(monkeypatch)
+ transformers = pytest.importorskip("transformers")
+ peft = pytest.importorskip("peft")
+ config = transformers.LlamaConfig(
+ vocab_size = 64,
+ hidden_size = 32,
+ intermediate_size = 64,
+ num_hidden_layers = 2,
+ num_attention_heads = 4,
+ num_key_value_heads = 2,
+ attention_bias = True,
+ )
+ model = transformers.LlamaForCausalLM(config)
+ model._unsloth_offload_layers_auto_ddp = True
+ lora = peft.LoraConfig(r = 4, target_modules = ["q_proj", "v_proj"], bias = bias)
+ model = peft.get_peft_model(model, lora)
+ if refused:
+ with pytest.raises(ValueError, match = "bias = 'none'"):
+ ns["_ignore_block_swap_params_for_ddp"](model)
+ else:
+ assert ns["_ignore_block_swap_params_for_ddp"](model)
diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.py
index c39d148403f..7157e3024a9 100644
--- a/unsloth/models/_utils.py
+++ b/unsloth/models/_utils.py
@@ -108,6 +108,7 @@ __all__ = [
"planned_prefetch_depth",
"trim_config_for_block_swap",
"attach_offload_layers",
+ "mark_auto_offload_for_ddp",
"skip_swapped_checkpoint_keys",
"get_moe_target_parameters",
"get_moe_target_modules",
@@ -5265,6 +5266,7 @@ def patch_gradient_accumulation_fix(Trainer):
from .llama import _decline_fused_lora_for_fsdp
_decline_fused_lora_for_fsdp(getattr(self, "model", None))
_replan_auto_offload_safely(self)
+ _block_swap_for_trainer_backend(self)
_unsloth_trainer_init.__wrapped__ = _original_trainer_init
Trainer.__init__ = _unsloth_trainer_init
@@ -7870,6 +7872,90 @@ def _replan_auto_offload_safely(trainer):
return None
+def mark_auto_offload_for_ddp(model, ddp_auto):
+ """Mark an "auto" DDP plan on every rank, even zero-swap ones, so all build the same ignore list."""
+ if ddp_auto and model is not None:
+ model._unsloth_offload_layers_auto_ddp = True
+ return model
+
+
+def _ignore_block_swap_params_for_ddp(model):
+ """Keep frozen decoder weights out of DDP's init broadcast: ranks may swap different layers to
+ 0-element placeholders. The list must match on every rank, so it follows the request and
+ architecture, never this rank's plan. Returns the names added."""
+ from .loader_utils import is_distributed
+
+ if model is None or find_decoder_layers is None or not is_distributed():
+ return []
+ if not getattr(model, "_unsloth_offload_layers_auto_ddp", False) and (
+ getattr(model, "_unsloth_offload_layers_auto", None) is None
+ ):
+ return []
+ # Before PEFT froze the base only swapped layers are frozen, so the list would differ per rank.
+ if getattr(model, "peft_config", None) is None:
+ return []
+ try:
+ layer_params = {id(p) for p in find_decoder_layers(model).parameters()}
+ except Exception:
+ return []
+ # Trainable decoder weights (bias, modules_to_save) need DDP sync; refuse on every rank alike.
+ trainable = [
+ name
+ for name, p in model.named_parameters()
+ if id(p) in layer_params and p.requires_grad and "lora_" not in name
+ ]
+ if trainable:
+ raise ValueError(
+ "Unsloth: offload_layers = 'auto' under DDP keeps the frozen decoder weights out of DDP, "
+ f"but {len(trainable)} decoder weights are trainable (e.g. {trainable[0]}). Use "
+ "bias = 'none' and no modules_to_save inside the decoder layers, or offload_layers = 0."
+ )
+ names = [
+ name
+ for name, p in model.named_parameters()
+ if id(p) in layer_params and not p.requires_grad
+ ]
+ ignored = list(getattr(model, "_ddp_params_and_buffers_to_ignore", None) or [])
+ seen = set(ignored)
+ added = [name for name in names if name not in seen]
+ if not added:
+ return []
+ from torch.nn.parallel import DistributedDataParallel
+
+ DistributedDataParallel._set_params_and_buffers_to_ignore_for_model(model, ignored + added)
+ return added
+
+
+def _block_swap_for_trainer_backend(trainer):
+ """DDP: ignore swapped weights. FSDP / DeepSpeed (invisible at load, so "auto" planned for DDP):
+ refuse, since sharding would flatten the 0-element placeholders."""
+ model = getattr(trainer, "model", None)
+ backend = (
+ "FSDP"
+ if getattr(trainer, "is_fsdp_enabled", False)
+ else "DeepSpeed"
+ if getattr(trainer, "is_deepspeed_enabled", False)
+ else None
+ )
+ if backend is None:
+ return _ignore_block_swap_params_for_ddp(model)
+ swapper = getattr(model, "_unsloth_block_swap", None) if model is not None else None
+ swapped = len(getattr(swapper, "indices", None) or ())
+ # Every auto rank refuses, even zero-swap ones: a one-sided refusal would hang the peers.
+ auto = model is not None and (
+ getattr(model, "_unsloth_offload_layers_auto_ddp", False)
+ or getattr(model, "_unsloth_offload_layers_auto", None) is not None
+ )
+ if auto:
+ raise ValueError(
+ f"Unsloth: offload_layers = 'auto' (this rank swapped {swapped} decoder layers to host "
+ f"RAM) cannot be combined with {backend}, which shards the weights itself and cannot wrap "
+ "swapped layers. Load with offload_layers = 0 (and get_peft_model(offload_layers = 0)), "
+ f"or launch without {backend}."
+ )
+ return []
+
+
def _trainer_offload_replan_skip(trainer):
"""Why the trainer must keep the attach-time plan, or None."""
args = getattr(trainer, "args", None)
diff --git a/unsloth/models/llama.py b/unsloth/models/llama.py
index f6efd38db96..8dc484b9bbb 100644
--- a/unsloth/models/llama.py
+++ b/unsloth/models/llama.py
@@ -21,6 +21,7 @@ from typing import Optional, Tuple, List, Union
from ._utils import *
from ._utils import apply_unsloth_gradient_checkpointing
+from ._utils import _ignore_block_swap_params_for_ddp
from ._utils import __version__, importlib_version
from ._utils import move_to_device
from ._utils import per_layer_device
@@ -57,6 +58,8 @@ from .loader_utils import (
requested_device_map,
resolve_unsloth_device_map,
resolve_auto_block_swap,
+ ddp_auto_block_swap_launch,
+ auto_offload_for_launch,
sync_load_when_quantizing,
warn_if_bitsandbytes_quantized_nothing,
)
@@ -3173,7 +3176,9 @@ class FastLlamaModel:
offload_layers = refuse_block_swap_load(
offload_layers, "does not support fast_inference or classification heads."
)
+ _ddp_auto_swap = False
if offload_layers == "auto":
+ _ddp_auto_swap = ddp_auto_block_swap_launch(requested_device_map(device_map))
offload_layers, device_map, _embedding_needed = resolve_auto_block_swap(
requested_device_map(device_map),
model_name,
@@ -3205,6 +3210,10 @@ class FastLlamaModel:
extra_skip_modules = ["out_proj"] if IS_FALCON_H1 else None,
),
)
+ _ddp_auto_swap = _ddp_auto_swap and _embedding_needed is not None
+ if _ddp_auto_swap:
+ # DDP refuses a CPU parameter.
+ offload_embedding = _offload_embedding_mode = False
# Here, not in loader.py: the mapper there can still substitute the repo (a -bnb-4bit name
# resolving to its 16-bit twin), so a plan sized for the caller's name is the wrong plan.
device_map = resolve_unsloth_device_map(
@@ -3549,6 +3558,7 @@ class FastLlamaModel:
model, tokenizer = patch_tokenizer(model, tokenizer)
model, tokenizer = model_patcher.post_patch(model, tokenizer, correct_dtype = dtype)
+ mark_auto_offload_for_ddp(model, _ddp_auto_swap)
attach_offload_layers(
model,
_block_swap_saved,
@@ -3994,10 +4004,11 @@ class FastLlamaModel:
model = _exclude_rope_inv_freq_from_ddp(model)
install_block_swap(
model,
- offload_layers,
+ auto_offload_for_launch(offload_layers),
prefetch_depth = prefetch_depth,
use_gradient_checkpointing = use_gradient_checkpointing,
)
+ _ignore_block_swap_params_for_ddp(model)
skip_checkpointing(model, checkpoint_skip_layers)
return model
else:
@@ -4305,7 +4316,7 @@ class FastLlamaModel:
offload_embedding_if_tight(model)
install_block_swap(
model,
- offload_layers,
+ auto_offload_for_launch(offload_layers),
prefetch_depth = prefetch_depth,
use_gradient_checkpointing = use_gradient_checkpointing,
)
@@ -4408,6 +4419,8 @@ class FastLlamaModel:
# otherwise poison (see prepare_for_training_mode).
_unsloth_install_pretrain_detector(model)
model = _exclude_rope_inv_freq_from_ddp(model)
+ # Not at load: the frozen set matches across ranks only after PEFT; custom loops skip Trainer.
+ _ignore_block_swap_params_for_ddp(model)
return model
@staticmethod
diff --git a/unsloth/models/loader.py b/unsloth/models/loader.py
index 8a190973690..4be41871452 100644
--- a/unsloth/models/loader.py
+++ b/unsloth/models/loader.py
@@ -123,6 +123,7 @@ from ._utils import (
_requested_float32,
_mark_requested_float32,
_mark_forced_float32,
+ _ignore_block_swap_params_for_ddp,
_mark_full_finetuning,
_get_text_only_config,
resolve_model_class,
@@ -1781,6 +1782,8 @@ class FastLanguageModel(FastLlamaModel):
model = _fix_rope_inv_freq(model)
model = _exclude_rope_inv_freq_from_ddp(model)
+ # Loaded adapters skip get_peft_model.
+ _ignore_block_swap_params_for_ddp(model)
# This path never sets UNSLOTH_FORCE_FLOAT32, so answer False rather than let the trainer read whatever a later load writes to the environment.
model = _mark_forced_float32(model, False)
# Full finetuning delegates to FastModel above, so this path is always LoRA.
@@ -3089,6 +3092,8 @@ class FastModel(FastBaseModel):
model = _fix_rope_inv_freq(model)
model = _exclude_rope_inv_freq_from_ddp(model)
+ # Loaded adapters skip get_peft_model.
+ _ignore_block_swap_params_for_ddp(model)
model = _mark_forced_float32(model, do_forced_float32)
model = _mark_full_finetuning(model, full_finetuning)
return _mark_requested_float32(model, user_float32), tokenizer
diff --git a/unsloth/models/loader_utils.py b/unsloth/models/loader_utils.py
index 95e57ff18fc..765ce6daa1b 100644
--- a/unsloth/models/loader_utils.py
+++ b/unsloth/models/loader_utils.py
@@ -758,6 +758,57 @@ def resolve_unsloth_device_map(
return plan.device_map
+def _sharded_data_parallel_launch():
+ """ "FSDP" / "DeepSpeed" when this process will hand the model to one, else None."""
+ try:
+ from accelerate.state import AcceleratorState
+ distributed_type = str(AcceleratorState._shared_state.get("distributed_type", "")).upper()
+ except Exception:
+ distributed_type = ""
+ if "FSDP" in distributed_type:
+ return "FSDP"
+ if "DEEPSPEED" in distributed_type:
+ return "DeepSpeed"
+ if str(os.environ.get("ACCELERATE_USE_FSDP", "")).strip().lower() in _TRUTHY or str(
+ os.environ.get("FSDP_VERSION", "")
+ ).strip() not in ("", "0"):
+ return "FSDP"
+ if str(os.environ.get("ACCELERATE_USE_DEEPSPEED", "")).strip().lower() in _TRUTHY:
+ return "DeepSpeed"
+ return None
+
+
+def _single_card_device_map(device_map):
+ if isinstance(device_map, dict):
+ if set(device_map) != {""}:
+ return False
+ device_map = device_map[""]
+ if isinstance(device_map, int) and not isinstance(device_map, bool):
+ return True
+ if isinstance(device_map, torch.device):
+ return device_map.type == "cuda"
+ return isinstance(device_map, str) and (device_map == "cuda" or device_map.startswith("cuda:"))
+
+
+def ddp_auto_block_swap_launch(device_map):
+ return (
+ is_distributed()
+ and _sharded_data_parallel_launch() is None
+ and _single_card_device_map(device_map)
+ )
+
+
+def auto_offload_for_launch(offload_layers):
+ """No "auto" swap on a launch that already shows FSDP / DeepSpeed."""
+ if (
+ offload_layers == "auto"
+ and is_distributed()
+ and _sharded_data_parallel_launch() is not None
+ ):
+ return 0
+ return offload_layers
+
+
_BLOCK_SWAP_PLANNER_KEYS = (
"batch_size",
"lora_rank",
@@ -798,8 +849,15 @@ def resolve_auto_block_swap(
return _none(skip_reason)
if DEVICE_TYPE_TORCH != "cuda" or not torch.cuda.is_available():
return _none("block swap needs a CUDA or ROCm GPU")
- if is_distributed():
- return _none("each rank of a distributed launch owns its own device")
+ distributed = is_distributed()
+ if distributed:
+ sharded = _sharded_data_parallel_launch()
+ if sharded is not None:
+ return _none(f"{sharded} shards the weights across the ranks itself")
+ if not _single_card_device_map(device_map):
+ return _none("each rank of a distributed launch owns its own device")
+ # DDP refuses a CPU parameter.
+ offload_embedding = False
try:
from unsloth_zoo.device_map_planner import plan_block_swap
except ImportError:
@@ -825,6 +883,11 @@ def resolve_auto_block_swap(
if target.type != "cuda":
return _none(f"the load places the model on {target}")
target = target.index if target.index is not None else torch.cuda.current_device()
+ elif (
+ isinstance(device_map, (int, torch.device))
+ and _single_device_index(device_map) is not None
+ ):
+ target = _single_device_index(device_map)
elif device_map is None or (
isinstance(device_map, str)
and (device_map in TRANSFORMERS_PLACEMENT_STRATEGIES or device_map.startswith("cuda"))
@@ -873,6 +936,11 @@ def resolve_auto_block_swap(
+ plan.describe().splitlines()[0].replace("block swap:", "offload_layers = 'auto':")
)
embedding = bool(getattr(plan, "offload_embedding", False))
+ if distributed:
+ # No collective: a loader only some ranks run would hang. Plans may differ per rank.
+ layers = int(plan.layers or 0)
+ print(f"Unsloth: offload_layers = 'auto' plans {layers} layers on this rank.")
+ return layers, ({"": target} if layers else device_map), False
if not plan.layers:
return 0, device_map, embedding
if multi and plan.device_plan is not None:
diff --git a/unsloth/models/vision.py b/unsloth/models/vision.py
index 610db015f7b..5ce46cf0e0f 100644
--- a/unsloth/models/vision.py
+++ b/unsloth/models/vision.py
@@ -132,6 +132,7 @@ from ._utils import (
_is_seq2seq_lm_config,
)
from ._utils import *
+from ._utils import _ignore_block_swap_params_for_ddp
from ._uma_safetensors import is_integrated_unified_memory_gpu
from ._utils import estimate_training_reserve_bytes as _zoo_reserve_estimate
from ._remote_code_buffers import restore_remote_code_non_persistent_buffers
@@ -161,6 +162,8 @@ from .loader_utils import (
raise_if_bnb_cpu_spill,
requested_device_map,
resolve_auto_block_swap,
+ ddp_auto_block_swap_launch,
+ auto_offload_for_launch,
resolve_unsloth_device_map,
sync_load_when_quantizing,
warn_if_bitsandbytes_quantized_nothing,
@@ -3476,7 +3479,9 @@ class FastBaseModel:
"needs every layer on one GPU; pass offload_layers to get_peft_model instead.",
)
_embedding_needed = None
+ _ddp_auto_swap = False
if _offload_layers == "auto":
+ _ddp_auto_swap = ddp_auto_block_swap_launch(requested_device_map(device_map))
_offload_layers, device_map, _embedding_needed = resolve_auto_block_swap(
requested_device_map(device_map),
model_name,
@@ -3513,6 +3518,10 @@ class FastBaseModel:
extra_skip_modules = _architecture_skip_modules(model_types) or None,
),
)
+ _ddp_auto_swap = _ddp_auto_swap and _embedding_needed is not None
+ if _ddp_auto_swap:
+ # DDP refuses a CPU parameter.
+ offload_embedding = False
_block_swap_device = block_swap_load_device(device_map)
if (_offload_layers or _embedding_needed) and _block_swap_device is not None:
# A placement strategy would spill the host-bound weights to the CPU, which 4-bit loads refuse.
@@ -4338,6 +4347,7 @@ class FastBaseModel:
finish_block_swap_load(
model, _block_swap_state, planned_prefetch_depth(device_map_planner_kwargs)
)
+ mark_auto_offload_for_ddp(model, _ddp_auto_swap)
# LAST, like the llama loader: patch_model_and_tokenizer below REPLACES the embedding and lm_head with fresh modules carrying the weights but not the _hf_hook, so a model split across cards would still meet the original cross-device index_select. Idempotent.
# Embeddings offloaded later sit on the host on purpose: a dispatch hook would pull them back.
@@ -4804,7 +4814,7 @@ class FastBaseModel:
offload_embedding_if_tight(model)
install_block_swap(
model,
- offload_layers,
+ auto_offload_for_launch(offload_layers),
prefetch_depth = prefetch_depth,
use_gradient_checkpointing = use_gradient_checkpointing,
)
@@ -4825,6 +4835,8 @@ class FastBaseModel:
# Detect a stray pre-train forward so train() can drop the torch.compile graph cache it would otherwise poison (see prepare_for_training_mode).
_unsloth_install_pretrain_detector(model)
model = _exclude_rope_inv_freq_from_ddp(model)
+ # Not at load: the frozen set matches across ranks only after PEFT; custom loops skip Trainer.
+ _ignore_block_swap_params_for_ddp(model)
return model
@staticmethod