Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Allow offload_layers = "auto" under a DDP launch (#13364)
Daniel Handbb698e1ba0e
diff --git a/tests/test_block_swap_auto_ddp.py b/tests/test_block_swap_auto_ddp.pynew file mode 100644index 00000000000..d8df0679c0e--- /dev/null+++ b/tests/test_block_swap_auto_ddp.py@@ -0,0 +1,544 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.+# This program is free software: you can redistribute it and/or modify+# it under the terms of the GNU Affero General Public License as published by+# the Free Software Foundation, either version 3 of the License, or+# (at your option) any later version.+# This program is distributed in the hope that it will be useful,+# but WITHOUT ANY WARRANTY; without even the implied warranty of+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+# GNU Affero General Public License for more details.+# You should have received a copy of the GNU Affero General Public License+# along with this program. If not, see <https://www.gnu.org/licenses/>.++"""from_pretrained(offload_layers = "auto") under a DDP launch. No GPU needed; extracted with ast."""++import ast+import importlib.util+import os+import sys+import types++import pytest+import torch++HERE = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))+MODELS = os.path.join(HERE, "unsloth", "models")+LOADER_UTILS = os.path.join(MODELS, "loader_utils.py")+UTILS = os.path.join(MODELS, "_utils.py")+_FAKE_PKG = "_block_swap_auto_ddp_pkg"+++def _extract(+ path,+ functions,+ assigns = (),+):+ src = open(path, encoding = "utf-8").read()+ out = []+ for node in ast.parse(src).body:+ if isinstance(node, ast.FunctionDef) and node.name in functions:+ out.append(ast.get_source_segment(src, node))+ elif isinstance(node, ast.Assign) and getattr(node.targets[0], "id", None) in assigns:+ out.append(ast.get_source_segment(src, node))+ return "\n\n".join(out)+++class _FakeCuda:+ def __init__(+ self,+ count = 2,+ free = 8 << 30,+ ):+ self.count, self.free = count, free++ def is_available(self):+ return True++ def device_count(self):+ return self.count++ def current_device(self):+ return 0+++class _Plan:+ def __init__(+ self,+ layers,+ offload_embedding = False,+ ):+ self.layers, self.offload_embedding, self.device_plan = layers, offload_embedding, None++ def describe(self):+ return "block swap: fabricated plan"+++def resolve(monkeypatch):+ """resolve_auto_block_swap over a fabricated CUDA, planner and launcher env."""+ for key in (+ "ACCELERATE_USE_FSDP",+ "FSDP_VERSION",+ "ACCELERATE_USE_DEEPSPEED",+ "UNSLOTH_FORCE_FUSED_LORA",+ ):+ monkeypatch.delenv(key, raising = False)+ state = types.SimpleNamespace(distributed = False, calls = [], layers = 6)++ def plan_block_swap(+ model_name,+ max_memory,+ seq_len,+ offload_embedding = None,+ **options,+ ):+ call = dict(max_memory = max_memory, **options)+ if offload_embedding is not None:+ call["offload_embedding"] = offload_embedding+ state.calls.append(call)+ return _Plan(state.layers, bool(offload_embedding))++ planner = types.ModuleType("unsloth_zoo.device_map_planner")+ planner.plan_block_swap = plan_block_swap+ monkeypatch.setitem(sys.modules, "unsloth_zoo.device_map_planner", planner)+ package = types.ModuleType(_FAKE_PKG)+ package.__path__ = []+ utils = types.ModuleType(f"{_FAKE_PKG}._utils")+ utils.usable_cuda_bytes = lambda device: 8 << 30+ utils.auto_plan_depth = lambda depth: depth+ monkeypatch.setitem(sys.modules, _FAKE_PKG, package)+ monkeypatch.setitem(sys.modules, f"{_FAKE_PKG}._utils", utils)+ fake_torch = types.SimpleNamespace(+ cuda = _FakeCuda(), device = torch.device, tensor = torch.tensor, int64 = torch.int64+ )+ ns = {+ "__name__": f"{_FAKE_PKG}.loader_utils",+ "__package__": _FAKE_PKG,+ "os": os,+ "inspect": __import__("inspect"),+ "torch": fake_torch,+ "DEVICE_TYPE_TORCH": "cuda",+ "is_distributed": lambda: state.distributed,+ }+ exec(+ _extract(+ LOADER_UTILS,+ (+ "resolve_auto_block_swap",+ "_sharded_data_parallel_launch",+ "_single_card_device_map",+ "_single_device_index",+ "ddp_auto_block_swap_launch",+ "auto_offload_for_launch",+ "_as_bytes",+ ),+ (+ "_TRUTHY",+ "_SIZE_UNITS",+ "UNSLOTH_DEVICE_MAP",+ "UNSLOTH_BALANCED_DEVICE_MAP",+ "_PLANNED_DEVICE_MAPS",+ "TRANSFORMERS_PLACEMENT_STRATEGIES",+ "_BLOCK_SWAP_PLANNER_KEYS",+ ),+ ),+ ns,+ )+ state.fn = lambda device_map, **kw: ns["resolve_auto_block_swap"](+ device_map, "m", max_seq_length = 2048, **kw+ )+ state.ns = ns+ return state+++def test_a_ddp_rank_plans_its_own_card(resolve):+ """torchrun gives each rank {"": "cuda:LOCAL_RANK"}: plan that card, keep the embedding on it."""+ resolve.distributed = True+ layers, device_map, embedding = resolve.fn({"": "cuda:1"}, offload_embedding = True)+ assert (layers, device_map, embedding) == (6, {"": 1}, False)+ assert len(resolve.calls) == 1+ assert list(resolve.calls[0]["max_memory"]) == [1]+ assert "offload_embedding" not in resolve.calls[0]++[email protected]("device_map, card", [(1, 1), ("cuda:1", 1)])+def test_a_ddp_rank_plans_a_bare_card_map(resolve, device_map, card):+ """device_map = 1 or "cuda:1" (no dict) is the same single-card load as {"": "cuda:1"}."""+ resolve.distributed = True+ layers, out_map, embedding = resolve.fn(device_map)+ assert (layers, out_map, embedding) == (6, {"": card}, False)+ assert list(resolve.calls[0]["max_memory"]) == [card]+++def test_a_ddp_rank_with_room_swaps_nothing(resolve):+ resolve.distributed, resolve.layers = True, 0+ assert resolve.fn({"": "cuda:0"}) == (0, {"": "cuda:0"}, False)+++def test_a_ddp_rank_honours_a_max_memory_for_its_card(resolve):+ resolve.distributed = True+ resolve.fn({"": "cuda:1"}, planner_kwargs = {"max_memory": {1: "2GiB"}})+ assert resolve.calls[0]["max_memory"] == {1: 2 << 30}+++ "env",+ [+ {"ACCELERATE_USE_FSDP": "true"},+ {"FSDP_VERSION": "2"},+ {"ACCELERATE_USE_DEEPSPEED": "true"},+ ],+)+def test_fsdp_and_deepspeed_launches_stay_refused(resolve, monkeypatch, capsys, env):+ resolve.distributed = True+ for key, value in env.items():+ monkeypatch.setenv(key, value)+ assert resolve.fn({"": "cuda:0"}) == (0, {"": "cuda:0"}, None)+ assert resolve.calls == []+ assert "shards the weights" in capsys.readouterr().out+++ "device_map",+ ["unsloth", "unsloth_balanced", "auto", "sequential", None, {"a": 0, "b": 1}, {"": "cpu"}],+)+def test_ddp_never_plans_a_multi_gpu_split_or_a_non_cuda_map(resolve, device_map):+ resolve.distributed = True+ layers, returned, embedding = resolve.fn(device_map)+ assert (layers, embedding) == (0, None)+ assert returned == device_map+ assert resolve.calls == []+++def test_a_single_process_still_offers_the_embedding(resolve):+ """Unchanged outside a distributed launch."""+ resolve.fn({"": "cuda:0"}, offload_embedding = True)+ assert resolve.calls[0]["offload_embedding"] is True+++def test_planning_never_runs_a_collective(resolve, monkeypatch):+ """No collective at load: a loader only some ranks reach would hang."""+ import torch.distributed as dist++ calls = []++ def collective(*args, **kwargs):+ calls.append(args)+ raise AssertionError("collective inside from_pretrained")++ monkeypatch.setattr(dist, "is_available", lambda: True)+ monkeypatch.setattr(dist, "is_initialized", lambda: True)+ monkeypatch.setattr(dist, "get_backend", lambda *a, **k: "gloo")+ for name in (+ "all_reduce",+ "all_gather",+ "all_gather_object",+ "broadcast",+ "broadcast_object_list",+ "barrier",+ ):+ monkeypatch.setattr(dist, name, collective)+ resolve.distributed = True+ assert resolve.fn({"": "cuda:1"}) == (6, {"": 1}, False)+ assert calls == []+++def _ignore_helpers(monkeypatch, distributed = True):+ zoo_root = importlib.util.find_spec("unsloth_zoo").submodule_search_locations[0]+ spec = importlib.util.spec_from_file_location(+ "_block_swap_for_ddp_test", os.path.join(zoo_root, "block_swap.py")+ )+ block_swap = importlib.util.module_from_spec(spec)+ spec.loader.exec_module(block_swap)+ package = types.ModuleType(_FAKE_PKG)+ package.__path__ = []+ loader_utils = types.ModuleType(f"{_FAKE_PKG}.loader_utils")+ loader_utils.is_distributed = lambda: distributed+ monkeypatch.setitem(sys.modules, _FAKE_PKG, package)+ monkeypatch.setitem(sys.modules, f"{_FAKE_PKG}.loader_utils", loader_utils)+ ns = {+ "__name__": f"{_FAKE_PKG}._utils",+ "__package__": _FAKE_PKG,+ "torch": torch,+ "find_decoder_layers": block_swap.find_decoder_layers,+ }+ exec(+ _extract(+ UTILS,+ (+ "mark_auto_offload_for_ddp",+ "_ignore_block_swap_params_for_ddp",+ "_block_swap_for_trainer_backend",+ ),+ ),+ ns,+ )+ return ns+++def _tiny_peft_model(mark = True):+ transformers = pytest.importorskip("transformers")+ peft = pytest.importorskip("peft")+ config = transformers.LlamaConfig(+ vocab_size = 64,+ hidden_size = 32,+ intermediate_size = 64,+ num_hidden_layers = 3,+ num_attention_heads = 4,+ num_key_value_heads = 2,+ )+ torch.manual_seed(0)+ model = transformers.LlamaForCausalLM(config)+ if mark:+ model._unsloth_offload_layers_auto_ddp = True+ # What _exclude_rope_inv_freq_from_ddp leaves on the base model before get_peft_model.+ model._ddp_params_and_buffers_to_ignore = ["model.rotary_emb.inv_freq"]+ lora = peft.LoraConfig(r = 4, target_modules = ["q_proj", "v_proj", "down_proj"])+ return peft.get_peft_model(model, lora)+++def test_ignore_list_names_the_frozen_decoder_weights_of_the_wrapped_model(monkeypatch):+ ns = _ignore_helpers(monkeypatch)+ model = _tiny_peft_model()+ layer = model.base_model.model.model.layers[2]+ for p in layer.parameters():+ if not p.requires_grad:+ p.data = torch.empty(0, dtype = p.dtype)+ added = ns["_ignore_block_swap_params_for_ddp"](model)+ params = dict(model.named_parameters())+ assert added and all(name in params for name in added)+ assert not any(params[name].requires_grad for name in added)+ assert not any("lora_" in name for name in added)+ frozen_layer_params = {+ name for name, p in params.items() if ".layers." in name and not p.requires_grad+ }+ assert set(added) == frozen_layer_params+ # Embedding, final norm and lm_head stay synced; the rotary entry survives the merge.+ assert not any(+ "embed_tokens" in n or "lm_head" in n or n.endswith("model.norm.weight") for n in added+ )+ ignored = model._ddp_params_and_buffers_to_ignore+ assert "model.rotary_emb.inv_freq" in ignored and set(added) <= set(ignored)+ assert ns["_ignore_block_swap_params_for_ddp"](model) == []+ assert len(model._ddp_params_and_buffers_to_ignore) == len(set(ignored))+++def test_ignore_list_is_the_same_whatever_this_rank_swapped(monkeypatch):+ """Rank 0 swapped layer 2, rank 1 nothing: both must hand DDP the same list."""+ ns = _ignore_helpers(monkeypatch)+ swapped, resident = _tiny_peft_model(), _tiny_peft_model()+ for p in swapped.base_model.model.model.layers[2].parameters():+ if not p.requires_grad:+ p.data = torch.empty(0, dtype = p.dtype)+ assert ns["_ignore_block_swap_params_for_ddp"](swapped) == ns[+ "_ignore_block_swap_params_for_ddp"+ ](resident)+++def test_ignore_list_untouched_without_auto_or_outside_ddp(monkeypatch):+ ns = _ignore_helpers(monkeypatch)+ model = _tiny_peft_model(mark = False)+ assert ns["_ignore_block_swap_params_for_ddp"](model) == []+ ns = _ignore_helpers(monkeypatch, distributed = False)+ assert ns["_ignore_block_swap_params_for_ddp"](_tiny_peft_model()) == []+++def test_mark_follows_the_loaders_ddp_decision(monkeypatch):+ ns = _ignore_helpers(monkeypatch)+ for ddp_auto in (True, False):+ model = types.SimpleNamespace()+ ns["mark_auto_offload_for_ddp"](model, ddp_auto)+ assert getattr(model, "_unsloth_offload_layers_auto_ddp", False) is ddp_auto+++def test_ddp_auto_launch_needs_ddp_and_one_card(resolve, monkeypatch):+ """A launch already showing FSDP / DeepSpeed keeps main's fallback."""+ launch = resolve.ns["ddp_auto_block_swap_launch"]+ assert not launch({"": "cuda:0"})+ resolve.distributed = True+ assert launch({"": "cuda:0"}) and launch(0) and launch("cuda:1")+ assert not launch("auto") and not launch({"a": 0, "b": 1})+ monkeypatch.setenv("ACCELERATE_USE_FSDP", "true")+ assert not launch({"": "cuda:0"})+++def test_loaders_keep_the_embedding_on_the_card_for_a_ddp_plan():+ """The loaders must drop the caller's offload_embedding themselves."""+ for name, var, veto in (+ ("llama.py", "offload_layers", "offload_embedding = _offload_embedding_mode = False"),+ ("vision.py", "_offload_layers", "offload_embedding = False"),+ ):+ src = open(os.path.join(MODELS, name), encoding = "utf-8").read()+ call = src.index(f"{var}, device_map, _embedding_needed = resolve_auto_block_swap(")+ before, after = src[call - 200 : call], src[call : call + 4000]+ assert (+ "_ddp_auto_swap = ddp_auto_block_swap_launch(requested_device_map(device_map))"+ in before+ )+ assert (+ "if _ddp_auto_swap:" in after and veto in after.split("if _ddp_auto_swap:", 1)[1][:200]+ )+ assert "mark_auto_offload_for_ddp(model, _ddp_auto_swap)" in src+++def test_wiring_trainer_init_and_both_loaders():+ utils = open(UTILS, encoding = "utf-8").read()+ init = next(+ n+ for n in ast.walk(ast.parse(utils))+ if isinstance(n, ast.FunctionDef) and n.name == "_unsloth_trainer_init"+ )+ assert "_block_swap_for_trainer_backend(self)" in ast.get_source_segment(utils, init)+ for name in ("llama.py", "vision.py"):+ src = open(os.path.join(MODELS, name), encoding = "utf-8").read()+ assert "mark_auto_offload_for_ddp(model, _ddp_auto_swap)" in src, name+ # The pre-wrapped PEFT branch installs the swap after its rope pass, so it applies the list itself.+ llama = open(os.path.join(MODELS, "llama.py"), encoding = "utf-8").read()+ install = llama.index("install_block_swap(\n model,")+ assert "_ignore_block_swap_params_for_ddp(model)" in llama[install : install + 600]+++def _swapped_model(+ fsdp = False,+ deepspeed = False,+ swapped = (2,),+ mark = True,+):+ model = types.SimpleNamespace(_unsloth_block_swap = types.SimpleNamespace(indices = list(swapped)))+ if mark:+ model._unsloth_offload_layers_auto_ddp = True+ return types.SimpleNamespace(model = model, is_fsdp_enabled = fsdp, is_deepspeed_enabled = deepspeed)++[email protected]("backend", ["fsdp", "deepspeed"])+def test_a_sharded_trainer_refuses_an_auto_swap(monkeypatch, backend):+ """FSDP / DeepSpeed invisible at load must not wrap 0-element placeholders."""+ ns = _ignore_helpers(monkeypatch)+ check = ns["_block_swap_for_trainer_backend"]+ with pytest.raises(ValueError, match = "offload_layers = 0"):+ check(_swapped_model(**{backend: True}))+ # Zero-swap ranks refuse too, else peers hang.+ with pytest.raises(ValueError, match = "swapped 0 decoder layers"):+ check(_swapped_model(swapped = (), **{backend: True}))+ assert check(_swapped_model(mark = False, **{backend: True})) == []+ # get_peft_model(offload_layers = "auto") alone: every rank refuses, also a zero-swap one.+ peft_auto = _swapped_model(mark = False, **{backend: True})+ peft_auto.model._unsloth_offload_layers_auto = 1+ with pytest.raises(ValueError, match = "swapped 1 decoder layers"):+ check(peft_auto)+ peft_auto.model._unsloth_block_swap.indices = []+ with pytest.raises(ValueError, match = "swapped 0 decoder layers"):+ check(peft_auto)+++def test_auto_declines_up_front_on_a_launch_that_shows_a_sharded_backend():+ """A resolver veto or an FSDP / DeepSpeed launch neither swaps nor marks."""+ for name in ("llama.py", "vision.py"):+ src = open(os.path.join(MODELS, name), encoding = "utf-8").read()+ veto = src.index("_ddp_auto_swap = _ddp_auto_swap and _embedding_needed is not None")+ assert (+ veto+ < src.index("if _ddp_auto_swap:", veto)+ < src.index("mark_auto_offload_for_ddp(model, _ddp_auto_swap)")+ ), name+ for name, calls in (("llama.py", 2), ("vision.py", 1)):+ src = open(os.path.join(MODELS, name), encoding = "utf-8").read()+ assert src.count("install_block_swap(") == calls+ assert src.count("auto_offload_for_launch(offload_layers),") == calls, name+++def test_get_peft_auto_declines_on_a_launch_that_shows_a_sharded_backend(resolve, monkeypatch):+ decline = resolve.ns["auto_offload_for_launch"]+ assert decline("auto") == "auto" and decline(4) == 4+ resolve.distributed = True+ assert decline("auto") == "auto"+ monkeypatch.setenv("ACCELERATE_USE_DEEPSPEED", "true")+ assert decline("auto") == 0 and decline(4) == 4+++def test_ignore_list_is_applied_at_the_end_of_get_peft_model():+ """Custom loops skip the Trainer, so both get_peft_model paths apply the list."""+ for name, owner in (("llama.py", "FastLlamaModel"), ("vision.py", "FastBaseModel")):+ src = open(os.path.join(MODELS, name), encoding = "utf-8").read()+ cls = next(+ n for n in ast.walk(ast.parse(src)) if isinstance(n, ast.ClassDef) and n.name == owner+ )+ fn = next(+ n for n in cls.body if isinstance(n, ast.FunctionDef) and n.name == "get_peft_model"+ )+ body = ast.get_source_segment(src, fn)+ tail = body[body.rindex("_exclude_rope_inv_freq_from_ddp(model)") :]+ assert "_ignore_block_swap_params_for_ddp(model)" in tail.split("return model")[0], name+ rope = _extract(LOADER_UTILS, ("_exclude_rope_inv_freq_from_ddp",))+ assert "_ignore_block_swap_params_for_ddp" not in rope+++def test_load_time_frozen_set_is_rank_dependent(monkeypatch):+ """Why the list is never built at load: there only swapped layers are frozen."""+ ns = _ignore_helpers(monkeypatch)+ transformers = pytest.importorskip("transformers")+ config = transformers.LlamaConfig(+ vocab_size = 64,+ hidden_size = 32,+ intermediate_size = 64,+ num_hidden_layers = 3,+ num_attention_heads = 4,+ num_key_value_heads = 2,+ )+ frozen = []+ for swapped in (1, 2):+ model = transformers.LlamaForCausalLM(config)+ model._unsloth_offload_layers_auto_ddp = True+ for layer in model.model.layers[3 - swapped :]:+ layer.requires_grad_(False)+ frozen.append({n for n, p in model.named_parameters() if not p.requires_grad})+ assert ns["_ignore_block_swap_params_for_ddp"](model) == []+ assert not getattr(model, "_ddp_params_and_buffers_to_ignore", None)+ assert frozen[0] != frozen[1]+++def test_ignore_list_is_applied_when_from_pretrained_loads_an_adapter():+ """Loaded adapters skip get_peft_model: both loader returns apply the list."""+ src = open(os.path.join(MODELS, "loader.py"), encoding = "utf-8").read()+ rope = "model = _exclude_rope_inv_freq_from_ddp(model)\n"+ assert src.count(rope) == 2+ for at in [i for i in range(len(src)) if src.startswith(rope, i)]:+ assert "_ignore_block_swap_params_for_ddp(model)" in src[at : at + 300]+++def test_full_finetuning_never_ignores_a_trainable_weight(monkeypatch):+ """Trainable decoder weights are refused, never ignored."""+ ns = _ignore_helpers(monkeypatch)+ model = _tiny_peft_model()+ for p in model.parameters():+ p.requires_grad_(True)+ with pytest.raises(ValueError, match = "decoder weights are trainable"):+ ns["_ignore_block_swap_params_for_ddp"](model)+ assert model._ddp_params_and_buffers_to_ignore == ["model.rotary_emb.inv_freq"]++[email protected]("bias, refused", [("none", False), ("lora_only", True), ("all", True)])+def test_trainable_decoder_biases_are_refused_on_every_rank(monkeypatch, bias, refused):+ """PEFT bias refusal reads the config, so zero-swap ranks refuse too."""+ ns = _ignore_helpers(monkeypatch)+ transformers = pytest.importorskip("transformers")+ peft = pytest.importorskip("peft")+ config = transformers.LlamaConfig(+ vocab_size = 64,+ hidden_size = 32,+ intermediate_size = 64,+ num_hidden_layers = 2,+ num_attention_heads = 4,+ num_key_value_heads = 2,+ attention_bias = True,+ )+ model = transformers.LlamaForCausalLM(config)+ model._unsloth_offload_layers_auto_ddp = True+ lora = peft.LoraConfig(r = 4, target_modules = ["q_proj", "v_proj"], bias = bias)+ model = peft.get_peft_model(model, lora)+ if refused:+ with pytest.raises(ValueError, match = "bias = 'none'"):+ ns["_ignore_block_swap_params_for_ddp"](model)+ else:+ assert ns["_ignore_block_swap_params_for_ddp"](model)diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.pyindex c39d148403f..7157e3024a9 100644--- a/unsloth/models/_utils.py+++ b/unsloth/models/_utils.py@@ -108,6 +108,7 @@ __all__ = ["planned_prefetch_depth","trim_config_for_block_swap","attach_offload_layers",+ "mark_auto_offload_for_ddp","skip_swapped_checkpoint_keys","get_moe_target_parameters","get_moe_target_modules",@@ -5265,6 +5266,7 @@ def patch_gradient_accumulation_fix(Trainer):from .llama import _decline_fused_lora_for_fsdp_decline_fused_lora_for_fsdp(getattr(self, "model", None))_replan_auto_offload_safely(self)+ _block_swap_for_trainer_backend(self)_unsloth_trainer_init.__wrapped__ = _original_trainer_initTrainer.__init__ = _unsloth_trainer_init@@ -7870,6 +7872,90 @@ def _replan_auto_offload_safely(trainer):return None+def mark_auto_offload_for_ddp(model, ddp_auto):+ """Mark an "auto" DDP plan on every rank, even zero-swap ones, so all build the same ignore list."""+ if ddp_auto and model is not None:+ model._unsloth_offload_layers_auto_ddp = True+ return model+++def _ignore_block_swap_params_for_ddp(model):+ """Keep frozen decoder weights out of DDP's init broadcast: ranks may swap different layers to+ 0-element placeholders. The list must match on every rank, so it follows the request and+ architecture, never this rank's plan. Returns the names added."""+ from .loader_utils import is_distributed++ if model is None or find_decoder_layers is None or not is_distributed():+ return []+ if not getattr(model, "_unsloth_offload_layers_auto_ddp", False) and (+ getattr(model, "_unsloth_offload_layers_auto", None) is None+ ):+ return []+ # Before PEFT froze the base only swapped layers are frozen, so the list would differ per rank.+ if getattr(model, "peft_config", None) is None:+ return []+ try:+ layer_params = {id(p) for p in find_decoder_layers(model).parameters()}+ except Exception:+ return []+ # Trainable decoder weights (bias, modules_to_save) need DDP sync; refuse on every rank alike.+ trainable = [+ name+ for name, p in model.named_parameters()+ if id(p) in layer_params and p.requires_grad and "lora_" not in name+ ]+ if trainable:+ raise ValueError(+ "Unsloth: offload_layers = 'auto' under DDP keeps the frozen decoder weights out of DDP, "+ f"but {len(trainable)} decoder weights are trainable (e.g. {trainable[0]}). Use "+ "bias = 'none' and no modules_to_save inside the decoder layers, or offload_layers = 0."+ )+ names = [+ name+ for name, p in model.named_parameters()+ if id(p) in layer_params and not p.requires_grad+ ]+ ignored = list(getattr(model, "_ddp_params_and_buffers_to_ignore", None) or [])+ seen = set(ignored)+ added = [name for name in names if name not in seen]+ if not added:+ return []+ from torch.nn.parallel import DistributedDataParallel++ DistributedDataParallel._set_params_and_buffers_to_ignore_for_model(model, ignored + added)+ return added+++def _block_swap_for_trainer_backend(trainer):+ """DDP: ignore swapped weights. FSDP / DeepSpeed (invisible at load, so "auto" planned for DDP):+ refuse, since sharding would flatten the 0-element placeholders."""+ model = getattr(trainer, "model", None)+ backend = (+ "FSDP"+ if getattr(trainer, "is_fsdp_enabled", False)+ else "DeepSpeed"+ if getattr(trainer, "is_deepspeed_enabled", False)+ else None+ )+ if backend is None:+ return _ignore_block_swap_params_for_ddp(model)+ swapper = getattr(model, "_unsloth_block_swap", None) if model is not None else None+ swapped = len(getattr(swapper, "indices", None) or ())+ # Every auto rank refuses, even zero-swap ones: a one-sided refusal would hang the peers.+ auto = model is not None and (+ getattr(model, "_unsloth_offload_layers_auto_ddp", False)+ or getattr(model, "_unsloth_offload_layers_auto", None) is not None+ )+ if auto:+ raise ValueError(+ f"Unsloth: offload_layers = 'auto' (this rank swapped {swapped} decoder layers to host "+ f"RAM) cannot be combined with {backend}, which shards the weights itself and cannot wrap "+ "swapped layers. Load with offload_layers = 0 (and get_peft_model(offload_layers = 0)), "+ f"or launch without {backend}."+ )+ return []++def _trainer_offload_replan_skip(trainer):"""Why the trainer must keep the attach-time plan, or None."""args = getattr(trainer, "args", None)diff --git a/unsloth/models/llama.py b/unsloth/models/llama.pyindex f6efd38db96..8dc484b9bbb 100644--- a/unsloth/models/llama.py+++ b/unsloth/models/llama.py@@ -21,6 +21,7 @@ from typing import Optional, Tuple, List, Unionfrom ._utils import *from ._utils import apply_unsloth_gradient_checkpointing+from ._utils import _ignore_block_swap_params_for_ddpfrom ._utils import __version__, importlib_versionfrom ._utils import move_to_devicefrom ._utils import per_layer_device@@ -57,6 +58,8 @@ from .loader_utils import (requested_device_map,resolve_unsloth_device_map,resolve_auto_block_swap,+ ddp_auto_block_swap_launch,+ auto_offload_for_launch,sync_load_when_quantizing,warn_if_bitsandbytes_quantized_nothing,)@@ -3173,7 +3176,9 @@ class FastLlamaModel:offload_layers = refuse_block_swap_load(offload_layers, "does not support fast_inference or classification heads.")+ _ddp_auto_swap = Falseif offload_layers == "auto":+ _ddp_auto_swap = ddp_auto_block_swap_launch(requested_device_map(device_map))offload_layers, device_map, _embedding_needed = resolve_auto_block_swap(requested_device_map(device_map),model_name,@@ -3205,6 +3210,10 @@ class FastLlamaModel:extra_skip_modules = ["out_proj"] if IS_FALCON_H1 else None,),)+ _ddp_auto_swap = _ddp_auto_swap and _embedding_needed is not None+ if _ddp_auto_swap:+ # DDP refuses a CPU parameter.+ offload_embedding = _offload_embedding_mode = False# Here, not in loader.py: the mapper there can still substitute the repo (a -bnb-4bit name# resolving to its 16-bit twin), so a plan sized for the caller's name is the wrong plan.device_map = resolve_unsloth_device_map(@@ -3549,6 +3558,7 @@ class FastLlamaModel:model, tokenizer = patch_tokenizer(model, tokenizer)model, tokenizer = model_patcher.post_patch(model, tokenizer, correct_dtype = dtype)+ mark_auto_offload_for_ddp(model, _ddp_auto_swap)attach_offload_layers(model,_block_swap_saved,@@ -3994,10 +4004,11 @@ class FastLlamaModel:model = _exclude_rope_inv_freq_from_ddp(model)install_block_swap(model,- offload_layers,+ auto_offload_for_launch(offload_layers),prefetch_depth = prefetch_depth,use_gradient_checkpointing = use_gradient_checkpointing,)+ _ignore_block_swap_params_for_ddp(model)skip_checkpointing(model, checkpoint_skip_layers)return modelelse:@@ -4305,7 +4316,7 @@ class FastLlamaModel:offload_embedding_if_tight(model)install_block_swap(model,- offload_layers,+ auto_offload_for_launch(offload_layers),prefetch_depth = prefetch_depth,use_gradient_checkpointing = use_gradient_checkpointing,)@@ -4408,6 +4419,8 @@ class FastLlamaModel:# otherwise poison (see prepare_for_training_mode)._unsloth_install_pretrain_detector(model)model = _exclude_rope_inv_freq_from_ddp(model)+ # Not at load: the frozen set matches across ranks only after PEFT; custom loops skip Trainer.+ _ignore_block_swap_params_for_ddp(model)return model@staticmethoddiff --git a/unsloth/models/loader.py b/unsloth/models/loader.pyindex 8a190973690..4be41871452 100644--- a/unsloth/models/loader.py+++ b/unsloth/models/loader.py@@ -123,6 +123,7 @@ from ._utils import (_requested_float32,_mark_requested_float32,_mark_forced_float32,+ _ignore_block_swap_params_for_ddp,_mark_full_finetuning,_get_text_only_config,resolve_model_class,@@ -1781,6 +1782,8 @@ class FastLanguageModel(FastLlamaModel):model = _fix_rope_inv_freq(model)model = _exclude_rope_inv_freq_from_ddp(model)+ # Loaded adapters skip get_peft_model.+ _ignore_block_swap_params_for_ddp(model)# This path never sets UNSLOTH_FORCE_FLOAT32, so answer False rather than let the trainer read whatever a later load writes to the environment.model = _mark_forced_float32(model, False)# Full finetuning delegates to FastModel above, so this path is always LoRA.@@ -3089,6 +3092,8 @@ class FastModel(FastBaseModel):model = _fix_rope_inv_freq(model)model = _exclude_rope_inv_freq_from_ddp(model)+ # Loaded adapters skip get_peft_model.+ _ignore_block_swap_params_for_ddp(model)model = _mark_forced_float32(model, do_forced_float32)model = _mark_full_finetuning(model, full_finetuning)return _mark_requested_float32(model, user_float32), tokenizerdiff --git a/unsloth/models/loader_utils.py b/unsloth/models/loader_utils.pyindex 95e57ff18fc..765ce6daa1b 100644--- a/unsloth/models/loader_utils.py+++ b/unsloth/models/loader_utils.py@@ -758,6 +758,57 @@ def resolve_unsloth_device_map(return plan.device_map+def _sharded_data_parallel_launch():+ """ "FSDP" / "DeepSpeed" when this process will hand the model to one, else None."""+ try:+ from accelerate.state import AcceleratorState+ distributed_type = str(AcceleratorState._shared_state.get("distributed_type", "")).upper()+ except Exception:+ distributed_type = ""+ if "FSDP" in distributed_type:+ return "FSDP"+ if "DEEPSPEED" in distributed_type:+ return "DeepSpeed"+ if str(os.environ.get("ACCELERATE_USE_FSDP", "")).strip().lower() in _TRUTHY or str(+ os.environ.get("FSDP_VERSION", "")+ ).strip() not in ("", "0"):+ return "FSDP"+ if str(os.environ.get("ACCELERATE_USE_DEEPSPEED", "")).strip().lower() in _TRUTHY:+ return "DeepSpeed"+ return None+++def _single_card_device_map(device_map):+ if isinstance(device_map, dict):+ if set(device_map) != {""}:+ return False+ device_map = device_map[""]+ if isinstance(device_map, int) and not isinstance(device_map, bool):+ return True+ if isinstance(device_map, torch.device):+ return device_map.type == "cuda"+ return isinstance(device_map, str) and (device_map == "cuda" or device_map.startswith("cuda:"))+++def ddp_auto_block_swap_launch(device_map):+ return (+ is_distributed()+ and _sharded_data_parallel_launch() is None+ and _single_card_device_map(device_map)+ )+++def auto_offload_for_launch(offload_layers):+ """No "auto" swap on a launch that already shows FSDP / DeepSpeed."""+ if (+ offload_layers == "auto"+ and is_distributed()+ and _sharded_data_parallel_launch() is not None+ ):+ return 0+ return offload_layers++_BLOCK_SWAP_PLANNER_KEYS = ("batch_size","lora_rank",@@ -798,8 +849,15 @@ def resolve_auto_block_swap(return _none(skip_reason)if DEVICE_TYPE_TORCH != "cuda" or not torch.cuda.is_available():return _none("block swap needs a CUDA or ROCm GPU")- if is_distributed():- return _none("each rank of a distributed launch owns its own device")+ distributed = is_distributed()+ if distributed:+ sharded = _sharded_data_parallel_launch()+ if sharded is not None:+ return _none(f"{sharded} shards the weights across the ranks itself")+ if not _single_card_device_map(device_map):+ return _none("each rank of a distributed launch owns its own device")+ # DDP refuses a CPU parameter.+ offload_embedding = Falsetry:from unsloth_zoo.device_map_planner import plan_block_swapexcept ImportError:@@ -825,6 +883,11 @@ def resolve_auto_block_swap(if target.type != "cuda":return _none(f"the load places the model on {target}")target = target.index if target.index is not None else torch.cuda.current_device()+ elif (+ isinstance(device_map, (int, torch.device))+ and _single_device_index(device_map) is not None+ ):+ target = _single_device_index(device_map)elif device_map is None or (isinstance(device_map, str)and (device_map in TRANSFORMERS_PLACEMENT_STRATEGIES or device_map.startswith("cuda"))@@ -873,6 +936,11 @@ def resolve_auto_block_swap(+ plan.describe().splitlines()[0].replace("block swap:", "offload_layers = 'auto':"))embedding = bool(getattr(plan, "offload_embedding", False))+ if distributed:+ # No collective: a loader only some ranks run would hang. Plans may differ per rank.+ layers = int(plan.layers or 0)+ print(f"Unsloth: offload_layers = 'auto' plans {layers} layers on this rank.")+ return layers, ({"": target} if layers else device_map), Falseif not plan.layers:return 0, device_map, embeddingif multi and plan.device_plan is not None:diff --git a/unsloth/models/vision.py b/unsloth/models/vision.pyindex 610db015f7b..5ce46cf0e0f 100644--- a/unsloth/models/vision.py+++ b/unsloth/models/vision.py@@ -132,6 +132,7 @@ from ._utils import (_is_seq2seq_lm_config,)from ._utils import *+from ._utils import _ignore_block_swap_params_for_ddpfrom ._uma_safetensors import is_integrated_unified_memory_gpufrom ._utils import estimate_training_reserve_bytes as _zoo_reserve_estimatefrom ._remote_code_buffers import restore_remote_code_non_persistent_buffers@@ -161,6 +162,8 @@ from .loader_utils import (raise_if_bnb_cpu_spill,requested_device_map,resolve_auto_block_swap,+ ddp_auto_block_swap_launch,+ auto_offload_for_launch,resolve_unsloth_device_map,sync_load_when_quantizing,warn_if_bitsandbytes_quantized_nothing,@@ -3476,7 +3479,9 @@ class FastBaseModel:"needs every layer on one GPU; pass offload_layers to get_peft_model instead.",)_embedding_needed = None+ _ddp_auto_swap = Falseif _offload_layers == "auto":+ _ddp_auto_swap = ddp_auto_block_swap_launch(requested_device_map(device_map))_offload_layers, device_map, _embedding_needed = resolve_auto_block_swap(requested_device_map(device_map),model_name,@@ -3513,6 +3518,10 @@ class FastBaseModel:extra_skip_modules = _architecture_skip_modules(model_types) or None,),)+ _ddp_auto_swap = _ddp_auto_swap and _embedding_needed is not None+ if _ddp_auto_swap:+ # DDP refuses a CPU parameter.+ offload_embedding = False_block_swap_device = block_swap_load_device(device_map)if (_offload_layers or _embedding_needed) and _block_swap_device is not None:# A placement strategy would spill the host-bound weights to the CPU, which 4-bit loads refuse.@@ -4338,6 +4347,7 @@ class FastBaseModel:finish_block_swap_load(model, _block_swap_state, planned_prefetch_depth(device_map_planner_kwargs))+ mark_auto_offload_for_ddp(model, _ddp_auto_swap)# LAST, like the llama loader: patch_model_and_tokenizer below REPLACES the embedding and lm_head with fresh modules carrying the weights but not the _hf_hook, so a model split across cards would still meet the original cross-device index_select. Idempotent.# Embeddings offloaded later sit on the host on purpose: a dispatch hook would pull them back.@@ -4804,7 +4814,7 @@ class FastBaseModel:offload_embedding_if_tight(model)install_block_swap(model,- offload_layers,+ auto_offload_for_launch(offload_layers),prefetch_depth = prefetch_depth,use_gradient_checkpointing = use_gradient_checkpointing,)@@ -4825,6 +4835,8 @@ class FastBaseModel:# Detect a stray pre-train forward so train() can drop the torch.compile graph cache it would otherwise poison (see prepare_for_training_mode)._unsloth_install_pretrain_detector(model)model = _exclude_rope_inv_freq_from_ddp(model)+ # Not at load: the frozen set matches across ranks only after PEFT; custom loops skip Trainer.+ _ignore_block_swap_params_for_ddp(model)return model@staticmethod