Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Native FSDP2: keep blocks that share parameters in the root unit (Zamba2) (#14384)
Daniel Handc47ce158410
diff --git a/tests/test_fsdp2_native_shared_params_gloo.py b/tests/test_fsdp2_native_shared_params_gloo.pynew file mode 100644index 00000000000..5c247088701--- /dev/null+++ b/tests/test_fsdp2_native_shared_params_gloo.py@@ -0,0 +1,237 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""Native FSDP2 must keep parameters shared across layers (Zamba2's shared transformer) tied."""++import multiprocessing+import os++if multiprocessing.parent_process() is not None or getattr(+ multiprocessing.current_process(), "_inheriting", False+):+ os.environ["CUDA_VISIBLE_DEVICES"] = "" # spawned gloo ranks: CPU only++import sys # noqa: E402++import pytest # noqa: E402+import torch # noqa: E402++sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))+from test_fsdp2_native_units import _load_native # noqa: E402++transformers = pytest.importorskip("transformers")+if not hasattr(transformers, "Zamba2Config"):+ pytest.skip("transformers without Zamba2", allow_module_level = True)++KINDS = ["mamba", "hybrid", "mamba", "hybrid", "mamba", "hybrid"]+++def _free_port():+ from tests._rendezvous_port import free_port+ return free_port()+++def _zamba2(device = None):+ config = transformers.Zamba2Config(+ vocab_size = 128,+ hidden_size = 32,+ num_hidden_layers = len(KINDS),+ layers_block_type = KINDS,+ num_mem_blocks = 1,+ num_attention_heads = 4,+ num_key_value_heads = 4,+ attention_head_dim = 8,+ mamba_d_state = 8,+ mamba_headdim = 8,+ n_mamba_heads = 8,+ intermediate_size = 64,+ use_shared_attention_adapter = False,+ adapter_rank = 4,+ chunk_size = 16,+ )+ torch.manual_seed(0)+ if device == "meta":+ with torch.device("meta"):+ return transformers.AutoModelForCausalLM.from_config(config, torch_dtype = torch.float32)+ return transformers.AutoModelForCausalLM.from_config(config, torch_dtype = torch.float32)+++def _shared_ids(model):+ owners = {}+ # Per path: transformers 4.x reuses one shared_transformer instance, which modules() yields once.+ for name, module in model.named_modules(remove_duplicate = False):+ for p in module.parameters(recurse = False):+ owners.setdefault(id(p), set()).add(name)+ return {k for k, v in owners.items() if len(v) > 1}+++def test_zamba2_has_shared_parameters_across_layers():+ model = _zamba2("meta")+ assert _shared_ids(model), "the fixture no longer ties the hybrid layers"+++class _Block(torch.nn.Module):+ def __init__(self, shared = None):+ super().__init__()+ self.lin = torch.nn.Linear(4, 4)+ if shared is not None:+ self.shared = shared+++def _reused_module_model():+ # transformers 4.x Zamba2 layout: one module instance reused by several layers.+ shared = torch.nn.Linear(4, 4)+ model = torch.nn.Module()+ model.layers = torch.nn.ModuleList([_Block(shared), _Block(), _Block(shared), _Block()])+ return model+++def test_reused_module_instance_counts_as_shared():+ native = _load_native()+ model = _reused_module_model()+ units = native.find_shard_units(model)+ assert units == [model.layers[1], model.layers[3]]+ with pytest.raises(RuntimeError, match = "tied / shared"):+ native.check_shard_units(model, list(model.layers))+++def test_embedding_inside_a_dropped_block_is_not_an_extra_unit():+ native = _load_native()+ model = _reused_module_model()+ model.layers[0].table = torch.nn.Embedding(512, 4)+ units = native.find_shard_units(model)+ assert model.layers[0] not in units+ assert native._large_root_embeddings(model, units, 0.1) == []+++def test_explicitly_ignored_shared_params_pass_the_check():+ native = _load_native()+ model = _reused_module_model()+ ignored = set(model.layers[0].shared.parameters())+ assert native.check_shard_units(model, list(model.layers), ignored)+ assert native.find_shard_units(model, ignored) == list(model.layers)+++def test_find_shard_units_keeps_sharing_blocks_in_the_root():+ native = _load_native()+ model = _zamba2("meta")+ units = native.find_shard_units(model)+ assert [type(u).__name__ for u in units] == ["Zamba2MambaDecoderLayer"] * 3+ shared = _shared_ids(model)+ for unit in units:+ assert not any(id(p) in shared for p in unit.parameters())+++def test_explicit_units_splitting_a_shared_parameter_raise():+ native = _load_native()+ model = _zamba2("meta")+ with pytest.raises(RuntimeError, match = "tied / shared"):+ native.check_shard_units(model, list(model.model.layers))+++def _worker(+ rank,+ world,+ port,+ out,+ frozen = None,+):+ os.environ.update(+ MASTER_ADDR = "127.0.0.1",+ MASTER_PORT = str(port),+ RANK = str(rank),+ WORLD_SIZE = str(world),+ LOCAL_RANK = str(rank),+ UNSLOTH_FSDP2 = "1",+ )+ import datetime++ import torch.distributed as dist++ dist.init_process_group(+ "gloo", rank = rank, world_size = world, timeout = datetime.timedelta(seconds = 300)+ )+ try:+ from torch.distributed.tensor import DTensor++ native = _load_native()+ ref, model = _zamba2(), _zamba2()+ layers = model.model.layers+ hybrid = [i for i, k in enumerate(KINDS) if k == "hybrid"]+ config = native.FSDP2Config(param_dtype = torch.float32, device = "cpu")+ if frozen:+ # Frozen shared weights replicated (shard_frozen=False): per-layer units are fine.+ for m in (ref, model):+ for p in m.model.layers[hybrid[0]].shared_transformer.parameters():+ p.requires_grad_(False)+ if frozen == "explicit":+ config.units = list(layers)+ if frozen == "fit":+ # The fit layout replicates frozen weights on its own; keep per-block trainable units.+ config.fit_layout = True+ os.environ["UNSLOTH_FSDP2_TRAINABLE_UNITS"] = "blocks"+ else:+ config.shard_frozen = False+ native.shard_model(model, config)+ first = dict(layers[hybrid[0]].shared_transformer.named_parameters())+ tied = all(+ dict(layers[i].shared_transformer.named_parameters())[n] is p+ for i in hybrid[1:]+ for n, p in first.items()+ )+ ids = torch.randint(0, 128, (1, 16), generator = torch.Generator().manual_seed(0))+ loss = model(input_ids = ids, labels = ids, use_cache = False).loss+ loss.backward()+ ref_loss = ref(input_ids = ids, labels = ids, use_cache = False).loss+ ref_loss.backward()+ ref_params = dict(ref.named_parameters())+ diff = 0.0+ missing = []+ for name, p in model.named_parameters():+ if not p.requires_grad:+ continue+ if p.grad is None:+ missing.append(name)+ continue+ g = p.grad.full_tensor() if isinstance(p.grad, DTensor) else p.grad+ diff = max(diff, float((g - ref_params[name].grad).abs().max()))+ res = {+ "units": [type(u).__name__.removeprefix("FSDP") for u in model._unsloth_fsdp2_units],+ "tied": tied,+ "loss_diff": abs(float(loss) - float(ref_loss)),+ "grad_diff": diff,+ "missing": missing,+ }+ out.put(res)+ except Exception as exc:+ import traceback+ out.put({"error": f"rank {rank} {type(exc).__name__}: {exc}\n{traceback.format_exc()}"})+ finally:+ dist.destroy_process_group()++[email protected]("frozen", [None, "explicit", "auto", "fit"])+def test_zamba2_shards_with_shared_transformer_tied(frozen):+ import torch.multiprocessing as mp++ ctx = mp.get_context("spawn")+ out = ctx.Queue()+ mp.start_processes(+ _worker,+ args = (2, _free_port(), out, frozen),+ nprocs = 2,+ join = True,+ start_method = "spawn",+ )+ results = [out.get(timeout = 5) for _ in range(2)]+ expected = (+ [type(layer).__name__ for layer in _zamba2("meta").model.layers]+ if frozen+ else (["Zamba2MambaDecoderLayer"] * 3)+ )+ for r in results:+ assert "error" not in r, r["error"]+ assert r["units"] == expected, r+ assert r["tied"] is True, r+ assert not r["missing"], r+ assert r["loss_diff"] < 1e-5 and r["grad_diff"] < 1e-5, rdiff --git a/unsloth/models/fsdp2_native.py b/unsloth/models/fsdp2_native.pyindex 76b7cbeea3b..249e8ae68ad 100644--- a/unsloth/models/fsdp2_native.py+++ b/unsloth/models/fsdp2_native.py@@ -356,7 +356,7 @@ def _is_block_list(container, no_split):return True-def find_shard_units(model) -> list:+def find_shard_units(model, ignored_params = ()) -> list:"""Repeated decoder / vision / audio blocks in module order: every child of the outermostnn.ModuleLists of blocks (model.layers, language_model.layers, vision_tower.*.layers, ...); never aunit inside a unit (MoE experts, fused-LoRA Linears stay in their block's unit). Only when no such list@@ -390,16 +390,72 @@ def find_shard_units(model) -> list:# Module (pre-)order = forward order for the usual layouts; the fallback units are re-sorted in.order = {id(m): i for i, m in enumerate(model.modules())}units.sort(key = lambda m: order.get(id(m), 0))- check_shard_units(model, units)+ units, dropped = _drop_units_sharing_params(model, units, ignored_params)+ if dropped:+ _print_rank0(+ f"Unsloth: native FSDP2 keeps {len(dropped)} blocks ({', '.join(sorted(set(dropped)))}) in the "+ "root unit: they share parameters with other blocks or modules (Zamba2's shared transformer), "+ "which one FSDP2 unit each would silently untie."+ )+ check_shard_units(model, units, ignored_params)return units+def _units_sharing_params(+ model,+ units,+ ignored = (),+):+ """Indices of units sharing a parameter with another unit or an outside module: fully_shard(unit)+ would untie it. Counted per unit, so one module instance reused by two units is caught too."""+ skip = {id(p) for p in ignored}+ holders = {}+ for i, unit in enumerate(units):+ for p in unit.parameters():+ holders.setdefault(id(p), set()).add(i)+ unit_ids = {id(u) for u in units}+ outside = set()+ stack = [model]+ while stack:+ module = stack.pop()+ if id(module) in unit_ids:+ continue+ outside.update(id(p) for p in module.parameters(recurse = False))+ stack.extend(module.children())+ bad = []+ for i, unit in enumerate(units):+ for p in unit.parameters():+ if id(p) not in skip and (id(p) in outside or len(holders[id(p)]) > 1):+ bad.append(i)+ break+ return bad+++def _drop_units_sharing_params(+ model,+ units,+ ignored_params = (),+):+ """Drop sharing units into the root, repeated: the root may then share with another unit."""+ dropped = []+ while True:+ bad = set(_units_sharing_params(model, units, ignored_params))+ if not bad:+ return units, dropped+ dropped += [type(units[i]).__name__ for i in bad]+ units = [u for i, u in enumerate(units) if i not in bad]++def _block_compiled(module):"""Unsloth's block compile installs a compiled `forward` on the decoder-layer class (_block_compile)."""return bool(getattr(getattr(type(module), "forward", None), "_unsloth_block_compiled", False))-def check_shard_units(model, units):+def check_shard_units(+ model,+ units,+ ignored_params = (),+):"""The one hard torch.compile rule: no FSDP unit inside a (compiled) decoder block or inside anotherunit. Unsloth compiles each decoder block's forward (fullgraph=False); FSDP2's hooks are dynamodisabled, so a unit nested inside the block (an expert, a router, a LoRA / norm submodule) is a silent@@ -434,9 +490,30 @@ def check_shard_units(model, units):"routers and LoRA inside their block's unit.")ancestor = parents.get(id(ancestor))+ if ignored_params is not None:+ _refuse_units_sharing_params(model, units, ignored_params, names)return True+def _refuse_units_sharing_params(+ model,+ units,+ ignored_params,+ names = None,+):+ shared = _units_sharing_params(model, units, ignored_params)+ if not shared:+ return+ unit = units[shared[0]]+ if names is None:+ names = {id(m): n for n, m in model.named_modules(remove_duplicate = False)}+ raise RuntimeError(+ f"Unsloth: native FSDP2 shard unit `{names.get(id(unit), type(unit).__name__)}` holds parameters "+ "that another unit or module also holds (tied / shared weights). One FSDP2 unit per holder would "+ "silently untie them: keep every holder of a shared parameter in one unit (or the root)."+ )++# --------------------------------------------------------------------------------------------------# Sharding# --------------------------------------------------------------------------------------------------@@ -1048,9 +1125,15 @@ def _large_root_embeddings(model, units, fraction):for module in model.modules():for p in module.parameters(recurse = False):owners.setdefault(id(p), set()).add(id(module))+ # Tables inside a decoder block (one kept in the root for sharing weights) would nest in it.+ in_block = set()+ for module in model.modules():+ if isinstance(module, nn.ModuleList) and _is_block_list(module, ()):+ for child in module.children():+ in_block.update(id(m) for m in child.modules())extra = []for _, module in model.named_modules():- if not isinstance(module, nn.Embedding):+ if not isinstance(module, nn.Embedding) or id(module) in in_block:continueparams = list(module.parameters())if not params or any(id(p) in inside or id(p) in skip for p in params):@@ -1216,13 +1299,9 @@ def shard_model(model, config: Optional[FSDP2Config] = None):"wrap it again.")config = config or FSDP2Config.from_env()- units = list(config.units) if config.units is not None else find_shard_units(model)- extra_units = (- _large_root_embeddings(model, units, config.split_large_root_embeddings)- if config.units is None- else []- )- check_shard_units(model, units + extra_units)+ if config.units is not None:+ # Sharing is checked once the replicated set is known (ignored and, if replicated, frozen params).+ check_shard_units(model, list(config.units), ignored_params = None)_refuse_tensor_parallel_model(model)device = _shard_device(model, config.device)dist = _ensure_process_group(device, config.backend, config.timeout)@@ -1275,6 +1354,20 @@ def shard_model(model, config: Optional[FSDP2Config] = None):# the agreed value, or a rank that fit alone keeps its blocks gathered while the others reshard, and# their backward collectives no longer match.resolve_reshard_after_forward(layout, config.reshard_after_forward, note = False)+ # Units after the layout: frozen weights it replicates may be shared by blocks that keep their own unit.+ ignored_params = set(config.ignored_params or ())+ if layout.replicate_frozen and _supports_ignored_params(fully_shard):+ ignored_params.update(p for p in model.parameters() if not p.requires_grad)+ units = (+ list(config.units) if config.units is not None else find_shard_units(model, ignored_params)+ )+ extra_units = (+ _large_root_embeddings(model, units, config.split_large_root_embeddings)+ if config.units is None+ else []+ )+ if extra_units:+ check_shard_units(model, units + extra_units, ignored_params = None)gather_dtype = param_dtypeif (config.param_dtype is None@@ -1321,6 +1414,7 @@ def shard_model(model, config: Optional[FSDP2Config] = None):replicated = set(ignored)if layout.replicate_frozen:replicated.update(p for p in model.parameters() if not p.requires_grad)+ _refuse_units_sharing_params(model, units + list(extra_units), replicated)if replicated:if not _supports_ignored_params(fully_shard):raise RuntimeError(