Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Native FSDP2: keep blocks that share parameters in the root unit (Zamba2) (#14384)

Daniel Handc47ce158410
diff --git a/tests/test_fsdp2_native_shared_params_gloo.py b/tests/test_fsdp2_native_shared_params_gloo.py
new file mode 100644
index 00000000000..5c247088701
--- /dev/null
+++ b/tests/test_fsdp2_native_shared_params_gloo.py
@@ -0,0 +1,237 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""Native FSDP2 must keep parameters shared across layers (Zamba2's shared transformer) tied."""
+
+import multiprocessing
+import os
+
+if multiprocessing.parent_process() is not None or getattr(
+ multiprocessing.current_process(), "_inheriting", False
+):
+ os.environ["CUDA_VISIBLE_DEVICES"] = "" # spawned gloo ranks: CPU only
+
+import sys # noqa: E402
+
+import pytest # noqa: E402
+import torch # noqa: E402
+
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+from test_fsdp2_native_units import _load_native # noqa: E402
+
+transformers = pytest.importorskip("transformers")
+if not hasattr(transformers, "Zamba2Config"):
+ pytest.skip("transformers without Zamba2", allow_module_level = True)
+
+KINDS = ["mamba", "hybrid", "mamba", "hybrid", "mamba", "hybrid"]
+
+
+def _free_port():
+ from tests._rendezvous_port import free_port
+ return free_port()
+
+
+def _zamba2(device = None):
+ config = transformers.Zamba2Config(
+ vocab_size = 128,
+ hidden_size = 32,
+ num_hidden_layers = len(KINDS),
+ layers_block_type = KINDS,
+ num_mem_blocks = 1,
+ num_attention_heads = 4,
+ num_key_value_heads = 4,
+ attention_head_dim = 8,
+ mamba_d_state = 8,
+ mamba_headdim = 8,
+ n_mamba_heads = 8,
+ intermediate_size = 64,
+ use_shared_attention_adapter = False,
+ adapter_rank = 4,
+ chunk_size = 16,
+ )
+ torch.manual_seed(0)
+ if device == "meta":
+ with torch.device("meta"):
+ return transformers.AutoModelForCausalLM.from_config(config, torch_dtype = torch.float32)
+ return transformers.AutoModelForCausalLM.from_config(config, torch_dtype = torch.float32)
+
+
+def _shared_ids(model):
+ owners = {}
+ # Per path: transformers 4.x reuses one shared_transformer instance, which modules() yields once.
+ for name, module in model.named_modules(remove_duplicate = False):
+ for p in module.parameters(recurse = False):
+ owners.setdefault(id(p), set()).add(name)
+ return {k for k, v in owners.items() if len(v) > 1}
+
+
+def test_zamba2_has_shared_parameters_across_layers():
+ model = _zamba2("meta")
+ assert _shared_ids(model), "the fixture no longer ties the hybrid layers"
+
+
+class _Block(torch.nn.Module):
+ def __init__(self, shared = None):
+ super().__init__()
+ self.lin = torch.nn.Linear(4, 4)
+ if shared is not None:
+ self.shared = shared
+
+
+def _reused_module_model():
+ # transformers 4.x Zamba2 layout: one module instance reused by several layers.
+ shared = torch.nn.Linear(4, 4)
+ model = torch.nn.Module()
+ model.layers = torch.nn.ModuleList([_Block(shared), _Block(), _Block(shared), _Block()])
+ return model
+
+
+def test_reused_module_instance_counts_as_shared():
+ native = _load_native()
+ model = _reused_module_model()
+ units = native.find_shard_units(model)
+ assert units == [model.layers[1], model.layers[3]]
+ with pytest.raises(RuntimeError, match = "tied / shared"):
+ native.check_shard_units(model, list(model.layers))
+
+
+def test_embedding_inside_a_dropped_block_is_not_an_extra_unit():
+ native = _load_native()
+ model = _reused_module_model()
+ model.layers[0].table = torch.nn.Embedding(512, 4)
+ units = native.find_shard_units(model)
+ assert model.layers[0] not in units
+ assert native._large_root_embeddings(model, units, 0.1) == []
+
+
+def test_explicitly_ignored_shared_params_pass_the_check():
+ native = _load_native()
+ model = _reused_module_model()
+ ignored = set(model.layers[0].shared.parameters())
+ assert native.check_shard_units(model, list(model.layers), ignored)
+ assert native.find_shard_units(model, ignored) == list(model.layers)
+
+
+def test_find_shard_units_keeps_sharing_blocks_in_the_root():
+ native = _load_native()
+ model = _zamba2("meta")
+ units = native.find_shard_units(model)
+ assert [type(u).__name__ for u in units] == ["Zamba2MambaDecoderLayer"] * 3
+ shared = _shared_ids(model)
+ for unit in units:
+ assert not any(id(p) in shared for p in unit.parameters())
+
+
+def test_explicit_units_splitting_a_shared_parameter_raise():
+ native = _load_native()
+ model = _zamba2("meta")
+ with pytest.raises(RuntimeError, match = "tied / shared"):
+ native.check_shard_units(model, list(model.model.layers))
+
+
+def _worker(
+ rank,
+ world,
+ port,
+ out,
+ frozen = None,
+):
+ os.environ.update(
+ MASTER_ADDR = "127.0.0.1",
+ MASTER_PORT = str(port),
+ RANK = str(rank),
+ WORLD_SIZE = str(world),
+ LOCAL_RANK = str(rank),
+ UNSLOTH_FSDP2 = "1",
+ )
+ import datetime
+
+ import torch.distributed as dist
+
+ dist.init_process_group(
+ "gloo", rank = rank, world_size = world, timeout = datetime.timedelta(seconds = 300)
+ )
+ try:
+ from torch.distributed.tensor import DTensor
+
+ native = _load_native()
+ ref, model = _zamba2(), _zamba2()
+ layers = model.model.layers
+ hybrid = [i for i, k in enumerate(KINDS) if k == "hybrid"]
+ config = native.FSDP2Config(param_dtype = torch.float32, device = "cpu")
+ if frozen:
+ # Frozen shared weights replicated (shard_frozen=False): per-layer units are fine.
+ for m in (ref, model):
+ for p in m.model.layers[hybrid[0]].shared_transformer.parameters():
+ p.requires_grad_(False)
+ if frozen == "explicit":
+ config.units = list(layers)
+ if frozen == "fit":
+ # The fit layout replicates frozen weights on its own; keep per-block trainable units.
+ config.fit_layout = True
+ os.environ["UNSLOTH_FSDP2_TRAINABLE_UNITS"] = "blocks"
+ else:
+ config.shard_frozen = False
+ native.shard_model(model, config)
+ first = dict(layers[hybrid[0]].shared_transformer.named_parameters())
+ tied = all(
+ dict(layers[i].shared_transformer.named_parameters())[n] is p
+ for i in hybrid[1:]
+ for n, p in first.items()
+ )
+ ids = torch.randint(0, 128, (1, 16), generator = torch.Generator().manual_seed(0))
+ loss = model(input_ids = ids, labels = ids, use_cache = False).loss
+ loss.backward()
+ ref_loss = ref(input_ids = ids, labels = ids, use_cache = False).loss
+ ref_loss.backward()
+ ref_params = dict(ref.named_parameters())
+ diff = 0.0
+ missing = []
+ for name, p in model.named_parameters():
+ if not p.requires_grad:
+ continue
+ if p.grad is None:
+ missing.append(name)
+ continue
+ g = p.grad.full_tensor() if isinstance(p.grad, DTensor) else p.grad
+ diff = max(diff, float((g - ref_params[name].grad).abs().max()))
+ res = {
+ "units": [type(u).__name__.removeprefix("FSDP") for u in model._unsloth_fsdp2_units],
+ "tied": tied,
+ "loss_diff": abs(float(loss) - float(ref_loss)),
+ "grad_diff": diff,
+ "missing": missing,
+ }
+ out.put(res)
+ except Exception as exc:
+ import traceback
+ out.put({"error": f"rank {rank} {type(exc).__name__}: {exc}\n{traceback.format_exc()}"})
+ finally:
+ dist.destroy_process_group()
+
+
[email protected]("frozen", [None, "explicit", "auto", "fit"])
+def test_zamba2_shards_with_shared_transformer_tied(frozen):
+ import torch.multiprocessing as mp
+
+ ctx = mp.get_context("spawn")
+ out = ctx.Queue()
+ mp.start_processes(
+ _worker,
+ args = (2, _free_port(), out, frozen),
+ nprocs = 2,
+ join = True,
+ start_method = "spawn",
+ )
+ results = [out.get(timeout = 5) for _ in range(2)]
+ expected = (
+ [type(layer).__name__ for layer in _zamba2("meta").model.layers]
+ if frozen
+ else (["Zamba2MambaDecoderLayer"] * 3)
+ )
+ for r in results:
+ assert "error" not in r, r["error"]
+ assert r["units"] == expected, r
+ assert r["tied"] is True, r
+ assert not r["missing"], r
+ assert r["loss_diff"] < 1e-5 and r["grad_diff"] < 1e-5, r
diff --git a/unsloth/models/fsdp2_native.py b/unsloth/models/fsdp2_native.py
index 76b7cbeea3b..249e8ae68ad 100644
--- a/unsloth/models/fsdp2_native.py
+++ b/unsloth/models/fsdp2_native.py
@@ -356,7 +356,7 @@ def _is_block_list(container, no_split):
return True
-def find_shard_units(model) -> list:
+def find_shard_units(model, ignored_params = ()) -> list:
"""Repeated decoder / vision / audio blocks in module order: every child of the outermost
nn.ModuleLists of blocks (model.layers, language_model.layers, vision_tower.*.layers, ...); never a
unit inside a unit (MoE experts, fused-LoRA Linears stay in their block's unit). Only when no such list
@@ -390,16 +390,72 @@ def find_shard_units(model) -> list:
# Module (pre-)order = forward order for the usual layouts; the fallback units are re-sorted in.
order = {id(m): i for i, m in enumerate(model.modules())}
units.sort(key = lambda m: order.get(id(m), 0))
- check_shard_units(model, units)
+ units, dropped = _drop_units_sharing_params(model, units, ignored_params)
+ if dropped:
+ _print_rank0(
+ f"Unsloth: native FSDP2 keeps {len(dropped)} blocks ({', '.join(sorted(set(dropped)))}) in the "
+ "root unit: they share parameters with other blocks or modules (Zamba2's shared transformer), "
+ "which one FSDP2 unit each would silently untie."
+ )
+ check_shard_units(model, units, ignored_params)
return units
+def _units_sharing_params(
+ model,
+ units,
+ ignored = (),
+):
+ """Indices of units sharing a parameter with another unit or an outside module: fully_shard(unit)
+ would untie it. Counted per unit, so one module instance reused by two units is caught too."""
+ skip = {id(p) for p in ignored}
+ holders = {}
+ for i, unit in enumerate(units):
+ for p in unit.parameters():
+ holders.setdefault(id(p), set()).add(i)
+ unit_ids = {id(u) for u in units}
+ outside = set()
+ stack = [model]
+ while stack:
+ module = stack.pop()
+ if id(module) in unit_ids:
+ continue
+ outside.update(id(p) for p in module.parameters(recurse = False))
+ stack.extend(module.children())
+ bad = []
+ for i, unit in enumerate(units):
+ for p in unit.parameters():
+ if id(p) not in skip and (id(p) in outside or len(holders[id(p)]) > 1):
+ bad.append(i)
+ break
+ return bad
+
+
+def _drop_units_sharing_params(
+ model,
+ units,
+ ignored_params = (),
+):
+ """Drop sharing units into the root, repeated: the root may then share with another unit."""
+ dropped = []
+ while True:
+ bad = set(_units_sharing_params(model, units, ignored_params))
+ if not bad:
+ return units, dropped
+ dropped += [type(units[i]).__name__ for i in bad]
+ units = [u for i, u in enumerate(units) if i not in bad]
+
+
def _block_compiled(module):
"""Unsloth's block compile installs a compiled `forward` on the decoder-layer class (_block_compile)."""
return bool(getattr(getattr(type(module), "forward", None), "_unsloth_block_compiled", False))
-def check_shard_units(model, units):
+def check_shard_units(
+ model,
+ units,
+ ignored_params = (),
+):
"""The one hard torch.compile rule: no FSDP unit inside a (compiled) decoder block or inside another
unit. Unsloth compiles each decoder block's forward (fullgraph=False); FSDP2's hooks are dynamo
disabled, so a unit nested inside the block (an expert, a router, a LoRA / norm submodule) is a silent
@@ -434,9 +490,30 @@ def check_shard_units(model, units):
"routers and LoRA inside their block's unit."
)
ancestor = parents.get(id(ancestor))
+ if ignored_params is not None:
+ _refuse_units_sharing_params(model, units, ignored_params, names)
return True
+def _refuse_units_sharing_params(
+ model,
+ units,
+ ignored_params,
+ names = None,
+):
+ shared = _units_sharing_params(model, units, ignored_params)
+ if not shared:
+ return
+ unit = units[shared[0]]
+ if names is None:
+ names = {id(m): n for n, m in model.named_modules(remove_duplicate = False)}
+ raise RuntimeError(
+ f"Unsloth: native FSDP2 shard unit `{names.get(id(unit), type(unit).__name__)}` holds parameters "
+ "that another unit or module also holds (tied / shared weights). One FSDP2 unit per holder would "
+ "silently untie them: keep every holder of a shared parameter in one unit (or the root)."
+ )
+
+
# --------------------------------------------------------------------------------------------------
# Sharding
# --------------------------------------------------------------------------------------------------
@@ -1048,9 +1125,15 @@ def _large_root_embeddings(model, units, fraction):
for module in model.modules():
for p in module.parameters(recurse = False):
owners.setdefault(id(p), set()).add(id(module))
+ # Tables inside a decoder block (one kept in the root for sharing weights) would nest in it.
+ in_block = set()
+ for module in model.modules():
+ if isinstance(module, nn.ModuleList) and _is_block_list(module, ()):
+ for child in module.children():
+ in_block.update(id(m) for m in child.modules())
extra = []
for _, module in model.named_modules():
- if not isinstance(module, nn.Embedding):
+ if not isinstance(module, nn.Embedding) or id(module) in in_block:
continue
params = list(module.parameters())
if not params or any(id(p) in inside or id(p) in skip for p in params):
@@ -1216,13 +1299,9 @@ def shard_model(model, config: Optional[FSDP2Config] = None):
"wrap it again."
)
config = config or FSDP2Config.from_env()
- units = list(config.units) if config.units is not None else find_shard_units(model)
- extra_units = (
- _large_root_embeddings(model, units, config.split_large_root_embeddings)
- if config.units is None
- else []
- )
- check_shard_units(model, units + extra_units)
+ if config.units is not None:
+ # Sharing is checked once the replicated set is known (ignored and, if replicated, frozen params).
+ check_shard_units(model, list(config.units), ignored_params = None)
_refuse_tensor_parallel_model(model)
device = _shard_device(model, config.device)
dist = _ensure_process_group(device, config.backend, config.timeout)
@@ -1275,6 +1354,20 @@ def shard_model(model, config: Optional[FSDP2Config] = None):
# the agreed value, or a rank that fit alone keeps its blocks gathered while the others reshard, and
# their backward collectives no longer match.
resolve_reshard_after_forward(layout, config.reshard_after_forward, note = False)
+ # Units after the layout: frozen weights it replicates may be shared by blocks that keep their own unit.
+ ignored_params = set(config.ignored_params or ())
+ if layout.replicate_frozen and _supports_ignored_params(fully_shard):
+ ignored_params.update(p for p in model.parameters() if not p.requires_grad)
+ units = (
+ list(config.units) if config.units is not None else find_shard_units(model, ignored_params)
+ )
+ extra_units = (
+ _large_root_embeddings(model, units, config.split_large_root_embeddings)
+ if config.units is None
+ else []
+ )
+ if extra_units:
+ check_shard_units(model, units + extra_units, ignored_params = None)
gather_dtype = param_dtype
if (
config.param_dtype is None
@@ -1321,6 +1414,7 @@ def shard_model(model, config: Optional[FSDP2Config] = None):
replicated = set(ignored)
if layout.replicate_frozen:
replicated.update(p for p in model.parameters() if not p.requires_grad)
+ _refuse_units_sharing_params(model, units + list(extra_units), replicated)
if replicated:
if not _supports_ignored_params(fully_shard):
raise RuntimeError(