Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Native FSDP2: sync initial trainable weights across ranks; shard FastSentenceTransformer and FastDiffusionModel (#15012)

Daniel Hanc52ef020c1d9
diff --git a/tests/diffusion/test_diffusion_distributed_device_map.py b/tests/diffusion/test_diffusion_distributed_device_map.py
new file mode 100644
index 00000000000..8bd9e6f377b
--- /dev/null
+++ b/tests/diffusion/test_diffusion_distributed_device_map.py
@@ -0,0 +1,56 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""FastDiffusionModel under torchrun: each rank loads one whole model on its own device, as
+FastLanguageModel / FastModel do. With the default device_map = "auto" every rank spread the model
+over all visible cards, and a 2-rank native FSDP2 launch died in transformers' infer_auto_device_map
+(IndexError) before training."""
+
+import pytest
+
+torch = pytest.importorskip("torch")
+transformers = pytest.importorskip("transformers")
+if not hasattr(transformers, "DiffusionGemmaForBlockDiffusion"):
+ pytest.skip("transformers without DiffusionGemma", allow_module_level = True)
+
+TINY = "trl-internal-testing/tiny-DiffusionGemmaForBlockDiffusion"
+
+
+class _Stop(Exception):
+ pass
+
+
+def _placement(monkeypatch, env, **kwargs):
+ import unsloth.models.diffusion as diffusion
+
+ for key in ("WORLD_SIZE", "LOCAL_RANK", "RANK", "LOCAL_WORLD_SIZE"):
+ monkeypatch.delenv(key, raising = False)
+ for key, value in env.items():
+ monkeypatch.setenv(key, value)
+ seen = {}
+
+ def resolve(device_map, *args, **kw):
+ seen["device_map"] = device_map
+ raise _Stop
+
+ monkeypatch.setattr(diffusion, "resolve_unsloth_device_map", resolve)
+ with pytest.raises(_Stop):
+ diffusion.FastDiffusionModel.from_pretrained(TINY, dtype = torch.float32, **kwargs)
+ return seen["device_map"]
+
+
+RANK1 = {"WORLD_SIZE": "2", "RANK": "1", "LOCAL_RANK": "1", "LOCAL_WORLD_SIZE": "2"}
+
+
+def test_each_rank_loads_on_its_own_device(monkeypatch):
+ from unsloth.models.loader_utils import DEVICE_TYPE_TORCH
+ assert _placement(monkeypatch, RANK1) == {"": f"{DEVICE_TYPE_TORCH}:1"}
+
+
+def test_a_device_the_caller_named_is_kept(monkeypatch):
+ assert _placement(monkeypatch, RANK1, device_map = "cpu") == "cpu"
+ assert _placement(monkeypatch, RANK1, device_map = {"": "cpu"}) == {"": "cpu"}
+
+
+def test_a_single_process_keeps_auto(monkeypatch):
+ assert _placement(monkeypatch, {}) == "auto"
diff --git a/tests/python/test_fast_sentence_transformer_lora_seed.py b/tests/python/test_fast_sentence_transformer_lora_seed.py
new file mode 100644
index 00000000000..e635abe7a2e
--- /dev/null
+++ b/tests/python/test_fast_sentence_transformer_lora_seed.py
@@ -0,0 +1,48 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""FastSentenceTransformer.get_peft_model seeds the adapter init with random_state on the fast
+encoder path (BERT, ModernBERT, ...), as FastModel.get_peft_model does. It was ignored there, so the
+LoRA matrices depended on whatever had used the RNG before, and under torchrun every rank drew its own
+adapters (native FSDP2 then sharded a mix of them)."""
+
+import pytest
+from real_accelerator import has_real_accelerator
+
+torch = pytest.importorskip("torch")
+pytest.importorskip("sentence_transformers")
+
+# The loader reads device properties even with device_map = "cpu".
+pytestmark = pytest.mark.skipif(
+ not has_real_accelerator(), reason = "the loader needs an accelerator"
+)
+
+ENCODER = "sentence-transformers-testing/stsb-bert-tiny-safetensors"
+
+
+def _adapters(random_state, burn):
+ from unsloth import FastSentenceTransformer
+
+ model = FastSentenceTransformer.from_pretrained(
+ ENCODER, max_seq_length = 32, dtype = torch.float32, device_map = "cpu"
+ )
+ assert getattr(model, "_unsloth_fast_encoder", False) # the path under test
+ torch.rand(burn) # a different RNG history before the adapters are made
+ model = FastSentenceTransformer.get_peft_model(
+ model, r = 4, lora_alpha = 8, target_modules = ["query", "value"], random_state = random_state
+ )
+ return {n: p.detach().clone() for n, p in model.named_parameters() if "lora_A" in n}
+
+
+def test_random_state_fixes_the_adapters():
+ a = _adapters(7, burn = 1)
+ b = _adapters(7, burn = 1000)
+ assert a and a.keys() == b.keys()
+ for name in a:
+ assert torch.equal(a[name], b[name]), name
+
+
+def test_a_different_random_state_gives_different_adapters():
+ a = _adapters(7, burn = 1)
+ c = _adapters(8, burn = 1)
+ assert any(not torch.equal(a[n], c[n]) for n in a)
diff --git a/tests/test_fsdp2_native_sync_initial_weights_gloo.py b/tests/test_fsdp2_native_sync_initial_weights_gloo.py
new file mode 100644
index 00000000000..14e4cd8ce0f
--- /dev/null
+++ b/tests/test_fsdp2_native_sync_initial_weights_gloo.py
@@ -0,0 +1,144 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""Native FSDP2 starts every rank from the same trainable weights, on a gloo CPU world of 2.
+
+DDP broadcasts rank 0's parameters when it wraps the model. fully_shard keeps each rank's own slice,
+so an adapter initialised without a common seed on each rank (FastSentenceTransformer.get_peft_model
+ignored random_state) was sharded as a mix of both ranks' values: rank 0 held its half of the LoRA
+matrices, rank 1 its own, and the step-0 gradient norm under FSDP2 was 12% off one process (DDP: 2%).
+shard_model now checks a checksum per trainable tensor and broadcasts the first rank's values of the
+ones that differ."""
+
+import multiprocessing
+import os
+
+if multiprocessing.parent_process() is not None or getattr(
+ multiprocessing.current_process(), "_inheriting", False
+):
+ os.environ["CUDA_VISIBLE_DEVICES"] = "" # spawned gloo ranks: CPU only, also on GPU hosts
+
+import sys # noqa: E402
+
+import pytest # noqa: E402
+import torch # noqa: E402
+
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+from test_fsdp2_native_units import _load_native # noqa: E402
+from test_fsdp2_native_trainer_ckpt_gloo import _free_port, _full # noqa: E402
+from test_fsdp2_native_trainer_args_gloo import _lora # noqa: E402
+
+
+def _worker(rank, world, port, out, case):
+ os.environ.update(
+ MASTER_ADDR = "127.0.0.1",
+ MASTER_PORT = str(port),
+ RANK = str(rank),
+ WORLD_SIZE = str(world),
+ LOCAL_RANK = str(rank),
+ LOCAL_WORLD_SIZE = str(world),
+ )
+ import datetime
+ import io
+ from contextlib import redirect_stdout
+
+ import torch.distributed as dist
+
+ dist.init_process_group(
+ "gloo", rank = rank, world_size = world, timeout = datetime.timedelta(seconds = 120)
+ )
+ try:
+ native = _load_native()
+ # Same base weights everywhere; the adapters differ per rank unless the case says otherwise.
+ seed = 1 + rank if case == "differ" else 1
+ model = _lora(seed)
+ first = _lora(1) # rank 0's starting adapters, built locally for the comparison
+ if case == "nan":
+ # Identical non-finite weights on every rank (tiny Llama4 ships some) are not a mismatch.
+ for m in (model, first):
+ next(p for n, p in m.named_parameters() if p.requires_grad).data.view(-1)[0] = (
+ float("nan")
+ )
+ if case in ("permuted", "nan_elsewhere") and rank == 1:
+ # Same sum and L1 norm as rank 0, different contents: the elements reversed.
+ w = next(p for n, p in model.named_parameters() if p.requires_grad).data.view(-1)
+ w.copy_(w.flip(0))
+ if case == "nan_elsewhere":
+ # A shared NaN must not hide a difference in the other elements.
+ for m in (model, first):
+ next(p for n, p in m.named_parameters() if p.requires_grad).data.view(-1)[0] = (
+ float("nan")
+ )
+ want = {n: p.detach().clone() for n, p in first.named_parameters() if p.requires_grad}
+ buf = io.StringIO()
+ with redirect_stdout(buf):
+ native.shard_model(model, native.FSDP2Config(param_dtype = torch.float32, device = "cpu"))
+ got = _full(model, trainable_only = True)
+ res = {
+ "n": len(want),
+ "max_diff": max((got[n] - want[n]).nan_to_num(0.0).abs().max().item() for n in want),
+ "message": "differed across ranks" in buf.getvalue(),
+ }
+ out.put((rank, res))
+ except Exception as exc:
+ import traceback
+ out.put((rank, {"crash": f"{type(exc).__name__}: {exc}\n{traceback.format_exc()}"}))
+ finally:
+ dist.destroy_process_group()
+
+
+def _run(case):
+ import torch.multiprocessing as mp
+
+ ctx = mp.get_context("spawn")
+ out = ctx.Queue()
+ mp.start_processes(
+ _worker, args = (2, _free_port(), out, case), nprocs = 2, join = True, start_method = "spawn"
+ )
+ results = {}
+ while not out.empty():
+ rank, r = out.get(timeout = 5)
+ results[rank] = r
+ assert sorted(results) == [0, 1], results
+ for rank, r in results.items():
+ print("SYNC_RESULT", case, rank, r)
+ assert "crash" not in r, r["crash"]
+ return results
+
+
+def test_ranks_with_different_adapters_start_from_the_first_ranks():
+ pytest.importorskip("transformers")
+ pytest.importorskip("peft")
+ results = _run("differ")
+ for rank, r in results.items():
+ assert r["n"] > 0, r
+ assert r["max_diff"] == 0.0, (rank, r) # rank 0's adapters, bit for bit, on both ranks
+ assert results[0]["message"] and not results[1]["message"], results # said once, on rank 0
+
+
+def test_ranks_that_already_agree_are_left_alone():
+ pytest.importorskip("transformers")
+ pytest.importorskip("peft")
+ for rank, r in _run("same").items():
+ assert r["max_diff"] == 0.0, (rank, r)
+ assert not r["message"], (rank, r)
+
+
+def test_identical_non_finite_weights_are_not_a_mismatch():
+ pytest.importorskip("transformers")
+ pytest.importorskip("peft")
+ for rank, r in _run("nan").items():
+ assert r["max_diff"] == 0.0, (rank, r)
+ assert not r["message"], (rank, r)
+
+
+def test_a_permuted_adapter_is_not_mistaken_for_the_same():
+ for rank, r in _run("permuted").items():
+ assert r["max_diff"] == 0.0, (rank, r)
+ assert r["message"] or rank != 0, (rank, r)
+
+
+def test_a_shared_nan_does_not_hide_other_differences():
+ for rank, r in _run("nan_elsewhere").items():
+ assert r["max_diff"] == 0.0, (rank, r)
+ assert r["message"] or rank != 0, (rank, r)
diff --git a/tests/test_fsdp2_native_trainer_init_any_loader_gloo.py b/tests/test_fsdp2_native_trainer_init_any_loader_gloo.py
new file mode 100644
index 00000000000..2156498d594
--- /dev/null
+++ b/tests/test_fsdp2_native_trainer_init_any_loader_gloo.py
@@ -0,0 +1,282 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""UNSLOTH_FSDP2=1 for a model no FastLanguageModel / FastModel load built.
+
+Native FSDP2 shards inside Unsloth's Trainer.__init__ wrapper, which only those loads (and
+DecisionTrainer) installed. A FastSentenceTransformer encoder (BERT, ModernBERT) never loads through
+them, so under `UNSLOTH_FSDP2=1 torchrun` its SentenceTransformerTrainer trained with plain DDP:
+nothing sharded and no message. `import unsloth` now installs the wrapper under the flag
+(fsdp2_launch.install_native_fsdp2_trainer_init).
+
+* the installer wraps Trainer under the flag, and only there;
+* the GPU import path calls it outside any try / except;
+* on a gloo CPU world of 2, a SentenceTransformer-shaped model (a plain encoder plus mean pooling and
+ a Trainer subclass with its own compute_loss, as sentence-transformers builds it) is sharded by the
+ installed wrapper and trains like one process over the same 4 rows."""
+
+import multiprocessing
+import os
+
+if multiprocessing.parent_process() is not None or getattr(
+ multiprocessing.current_process(), "_inheriting", False
+):
+ os.environ["CUDA_VISIBLE_DEVICES"] = "" # spawned gloo ranks: CPU only, also on GPU hosts
+
+import ast # noqa: E402
+import copy # noqa: E402
+import importlib.util # noqa: E402
+import sys # noqa: E402
+import types # noqa: E402
+
+import pytest # noqa: E402
+import torch # noqa: E402
+
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+from test_fsdp2_native_units import REPO, _load_native # noqa: E402
+from test_fsdp2_native_trainer_ckpt_gloo import _free_port, _stub_unsloth_save # noqa: E402
+from test_fsdp2_native_trainer_args_gloo import _install_wrapped_trainer_init # noqa: E402
+
+ENCODER = "hf-internal-testing/tiny-random-BertModel"
+STEPS = 2
+
+
+def _load_launch(calls):
+ """fsdp2_launch.py with `.models._utils` stubbed (the real one needs the GPU package)."""
+ pkg = sys.modules.get("unsloth") or types.ModuleType("unsloth")
+ if not hasattr(pkg, "__path__"):
+ pkg.__path__ = [os.path.join(REPO, "unsloth")]
+ models = sys.modules.get("unsloth.models") or types.ModuleType("unsloth.models")
+ if not hasattr(models, "__path__"):
+ models.__path__ = [os.path.join(REPO, "unsloth", "models")]
+ utils = types.ModuleType("unsloth.models._utils")
+
+ def patch_gradient_accumulation_fix(trainer_cls):
+ calls.append(trainer_cls)
+ _install_wrapped_trainer_init()
+
+ utils.patch_gradient_accumulation_fix = patch_gradient_accumulation_fix
+ sys.modules["unsloth"] = pkg
+ sys.modules["unsloth.models"] = models
+ sys.modules["unsloth.models._utils"] = utils
+ spec = importlib.util.spec_from_file_location(
+ "unsloth.fsdp2_launch", os.path.join(REPO, "unsloth", "fsdp2_launch.py")
+ )
+ module = importlib.util.module_from_spec(spec)
+ sys.modules["unsloth.fsdp2_launch"] = module
+ spec.loader.exec_module(module)
+ return module
+
+
+ "env,applies",
+ [
+ ({"UNSLOTH_FSDP2": "1", "WORLD_SIZE": "2"}, True),
+ ({"UNSLOTH_FSDP2": "1", "WORLD_SIZE": "1"}, False),
+ ({"WORLD_SIZE": "2"}, False),
+ ({"UNSLOTH_FSDP2": "1", "WORLD_SIZE": "2", "UNSLOTH_FSDP2_BACKEND": "accelerate"}, False),
+ ],
+)
+def test_installer_wraps_trainer_only_under_native_fsdp2(monkeypatch, env, applies):
+ from transformers import Trainer
+
+ for key in ("UNSLOTH_FSDP2", "WORLD_SIZE", "UNSLOTH_FSDP2_BACKEND", "ACCELERATE_USE_FSDP"):
+ monkeypatch.delenv(key, raising = False)
+ for key, value in env.items():
+ monkeypatch.setenv(key, value)
+ for name in ("unsloth", "unsloth.models", "unsloth.models._utils", "unsloth.fsdp2_launch"):
+ monkeypatch.setitem(sys.modules, name, sys.modules.get(name))
+ monkeypatch.setattr(Trainer, "__init__", Trainer.__init__)
+ calls = []
+ launch = _load_launch(calls)
+ assert launch.install_native_fsdp2_trainer_init() is applies
+ assert calls == ([Trainer] if applies else [])
+
+
+def test_gpu_import_path_installs_it():
+ tree = ast.parse(open(os.path.join(REPO, "unsloth", "__init__.py"), encoding = "utf-8").read())
+ found = [
+ node
+ for node in ast.walk(tree)
+ if isinstance(node, ast.Call)
+ and isinstance(node.func, ast.Name)
+ and node.func.id == "_install_native_trainer_init"
+ ]
+ assert found, "unsloth/__init__.py no longer installs the native FSDP2 Trainer wrapper"
+ # Not swallowed: a wrapper that failed to install would silently fall back to DDP.
+ for node in ast.walk(tree):
+ if isinstance(node, ast.Try):
+ inside = {id(n) for stmt in node.body for n in ast.walk(stmt)}
+ assert not any(id(call) in inside for call in found)
+
+
+class _MeanPooled(torch.nn.Module):
+ """A SentenceTransformer's shape: the HF encoder plus mean pooling, no PreTrainedModel root."""
+
+ def __init__(self, encoder):
+ super().__init__()
+ self.encoder = encoder
+
+ def forward(self, input_ids, attention_mask):
+ hidden = self.encoder(input_ids = input_ids, attention_mask = attention_mask).last_hidden_state
+ mask = attention_mask.unsqueeze(-1).to(hidden.dtype)
+ return (hidden * mask).sum(1) / mask.sum(1)
+
+
+def _pair_loss(model, batch):
+ a = model(batch["a_ids"], batch["a_mask"])
+ b = model(batch["b_ids"], batch["b_mask"])
+ cos = torch.nn.functional.cosine_similarity(a, b)
+ return torch.nn.functional.mse_loss(cos, batch["score"])
+
+
+def _rows():
+ g = torch.Generator().manual_seed(0)
+ return [
+ {
+ "a_ids": torch.randint(5, 90, (12,), generator = g),
+ "a_mask": torch.ones(12, dtype = torch.long),
+ "b_ids": torch.randint(5, 90, (12,), generator = g),
+ "b_mask": torch.ones(12, dtype = torch.long),
+ "score": torch.rand((), generator = g),
+ }
+ for _ in range(4)
+ ]
+
+
+def _model():
+ import transformers
+
+ config = transformers.AutoConfig.from_pretrained(ENCODER)
+ config.hidden_dropout_prob = config.attention_probs_dropout_prob = 0.0 # deterministic steps
+ torch.manual_seed(0)
+ return _MeanPooled(transformers.AutoModel.from_config(config, torch_dtype = torch.float32))
+
+
+def _reference(model, rows):
+ """One process: the same AdamW steps on all 4 rows (constant LR, clip 1.0)."""
+ model = copy.deepcopy(model)
+ batch = {k: torch.stack([r[k] for r in rows]) for k in rows[0]}
+ opt = torch.optim.AdamW(model.parameters(), lr = 1e-3, weight_decay = 0.0)
+ for _ in range(STEPS):
+ opt.zero_grad()
+ _pair_loss(model, batch).backward()
+ torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)
+ opt.step()
+ return {n: p.detach().clone() for n, p in model.named_parameters()}
+
+
+def _worker(rank, world, port, out, tmp):
+ os.environ.update(
+ MASTER_ADDR = "127.0.0.1",
+ MASTER_PORT = str(port),
+ RANK = str(rank),
+ WORLD_SIZE = str(world),
+ LOCAL_RANK = str(rank),
+ LOCAL_WORLD_SIZE = str(world),
+ UNSLOTH_FSDP2 = "1",
+ )
+ import datetime
+
+ import torch.distributed as dist
+
+ dist.init_process_group(
+ "gloo", rank = rank, world_size = world, timeout = datetime.timedelta(seconds = 120)
+ )
+ try:
+ native = _load_native()
+ _stub_unsloth_save()
+ from transformers import Trainer, TrainingArguments
+
+ res = {"wrapped_before": Trainer.__init__.__name__ == "_unsloth_trainer_init"}
+ launch = _load_launch([])
+ res["installed"] = launch.install_native_fsdp2_trainer_init()
+
+ class PairTrainer(Trainer):
+ # sentence-transformers' own compute_loss: the model returns embeddings, not a loss.
+ def compute_loss(
+ self,
+ model,
+ inputs,
+ return_outputs = False,
+ num_items_in_batch = None,
+ ):
+ return _pair_loss(model, inputs)
+
+ def collate(rows):
+ return {k: torch.stack([r[k] for r in rows]) for k in rows[0]}
+
+ rows = _rows()
+ model = _model()
+ want = _reference(model, rows)
+ args = TrainingArguments(
+ output_dir = os.path.join(tmp, "out"),
+ report_to = [],
+ use_cpu = True,
+ per_device_train_batch_size = 2,
+ max_steps = STEPS,
+ learning_rate = 1e-3,
+ lr_scheduler_type = "constant",
+ optim = "adamw_torch",
+ weight_decay = 0.0,
+ max_grad_norm = 1.0,
+ save_strategy = "no",
+ logging_steps = 1,
+ disable_tqdm = True,
+ remove_unused_columns = False,
+ dataloader_drop_last = True,
+ seed = 0,
+ )
+ trainer = PairTrainer(model = model, args = args, train_dataset = rows, data_collator = collate)
+ res["sharded"] = native.is_native_fsdp2(trainer.model)
+ res["units"] = sum(
+ isinstance(m, torch.distributed.fsdp.FSDPModule) for m in trainer.model.modules()
+ )
+ trainer.train()
+ from torch.distributed.tensor import DTensor
+
+ got = {
+ n: (p.full_tensor() if isinstance(p, DTensor) else p).detach()
+ for n, p in trainer.model.named_parameters()
+ }
+ res["n_params"] = len(got)
+ res["missing"] = sorted(set(want) - set(got))[:5]
+ res["max_diff"] = max((got[n] - want[n]).abs().max().item() for n in want if n in got)
+ res["moved"] = max((want[n] - p).abs().max().item() for n, p in _model().named_parameters())
+ out.put((rank, res))
+ except Exception as exc:
+ import traceback
+ out.put((rank, {"crash": f"{type(exc).__name__}: {exc}\n{traceback.format_exc()}"}))
+ finally:
+ dist.destroy_process_group()
+
+
+def test_sentence_transformer_shaped_model_shards_and_matches_one_process(tmp_path):
+ pytest.importorskip("transformers")
+ import torch.multiprocessing as mp
+
+ ctx = mp.get_context("spawn")
+ out = ctx.Queue()
+ mp.start_processes(
+ _worker,
+ args = (2, _free_port(), out, str(tmp_path)),
+ nprocs = 2,
+ join = True,
+ start_method = "spawn",
+ )
+ results = {}
+ while not out.empty():
+ rank, r = out.get(timeout = 5)
+ results[rank] = r
+ assert sorted(results) == [0, 1], results
+ for rank, r in results.items():
+ print("ANY_LOADER_RESULT", rank, r)
+ assert "crash" not in r, r["crash"]
+ assert r["wrapped_before"] is False, r # nothing else installed the wrapper here
+ assert r["installed"] is True and r["sharded"] is True, r
+ assert r["units"] >= 2, r # encoder blocks + root
+ assert not r["missing"], r
+ assert (
+ r["moved"] > 1e-4
+ ), r # the steps changed the weights, so the comparison means something
+ assert r["max_diff"] < 2e-5, r # fp32, reduction order only
diff --git a/unsloth/__init__.py b/unsloth/__init__.py
index 4e2d54a8052..8c8e15f8573 100644
--- a/unsloth/__init__.py
+++ b/unsloth/__init__.py
@@ -1717,6 +1717,14 @@ else:
except Exception:
pass
+ # Native FSDP2 shards inside the Trainer.__init__ wrapper that FastLanguageModel / FastModel loads
+ # install. A FastSentenceTransformer encoder never loads through them, so under UNSLOTH_FSDP2=1 its
+ # Trainer trained with DDP instead, silently. Install the wrapper up front under the flag.
+ from .fsdp2_launch import install_native_fsdp2_trainer_init as _install_native_trainer_init
+
+ _install_native_trainer_init()
+ del _install_native_trainer_init
+
def get_gpu_memory_stats():
"""Return CUDA/ROCm/XPU/NPU device stats, peak memory, and total memory in GiB."""
try:
diff --git a/unsloth/fsdp2_launch.py b/unsloth/fsdp2_launch.py
index b558f3d3d8d..8e1a9b489bc 100644
--- a/unsloth/fsdp2_launch.py
+++ b/unsloth/fsdp2_launch.py
@@ -16,6 +16,7 @@ __all__ = [
"native_backend_conflicts",
"refuse_native_backend_conflicts",
"install_raw_accelerator_fsdp2_fallback",
+ "install_native_fsdp2_trainer_init",
]
_TRUTHY = ("1", "true", "yes", "on")
@@ -457,3 +458,19 @@ def install_raw_accelerator_fsdp2_fallback():
Accelerator.__init__ = __init__
setattr(Accelerator, _RAW_ACCELERATOR_PATCHED, True)
return True
+
+
+def install_native_fsdp2_trainer_init():
+ """UNSLOTH_FSDP2=1 under WORLD_SIZE > 1 shards natively inside Unsloth's Trainer.__init__ wrapper
+ (installed by `_utils.patch_gradient_accumulation_fix`). FastLanguageModel / FastModel loads and
+ DecisionTrainer install it; a model built without them (a FastSentenceTransformer encoder) left the
+ Trainer unwrapped, so it trained under DDP with nothing sharded and no message. Installs the wrapper
+ once the flag selects native FSDP2. Returns whether it applies."""
+ if not unsloth_fsdp2_native_requested():
+ return False
+ from transformers import Trainer
+
+ from .models._utils import patch_gradient_accumulation_fix
+
+ patch_gradient_accumulation_fix(Trainer)
+ return True
diff --git a/unsloth/models/diffusion.py b/unsloth/models/diffusion.py
index d19a92c3879..9134cfa8270 100644
--- a/unsloth/models/diffusion.py
+++ b/unsloth/models/diffusion.py
@@ -38,10 +38,12 @@ from .diffusion_profiles import (
)
from .llama import logger
from .loader_utils import (
+ is_automatic_device_map,
planner_config_overrides,
planner_hub_kwargs,
planner_kwargs_with_max_memory,
planner_quantization_kwargs,
+ prepare_device_map,
requested_device_map,
resolve_unsloth_device_map,
)
@@ -356,8 +358,15 @@ class FastDiffusionModel:
# Same leaf-level resolution as llama.py and vision.py: an unresolved "unsloth" becomes
# torch.device("unsloth") in transformers and raises instead of loading.
+ device_map = requested_device_map(device_map)
+ if is_automatic_device_map(device_map):
+ # Under torchrun / accelerate launch each rank loads one whole model on its own device, as
+ # FastLanguageModel / FastModel do: "auto" would spread every rank over all the visible cards.
+ distributed_device_map, is_dist = prepare_device_map()
+ if is_dist:
+ device_map = distributed_device_map
device_map = resolve_unsloth_device_map(
- requested_device_map(device_map),
+ device_map,
model_name,
full_finetuning = full_finetuning,
skip_reason = (
diff --git a/unsloth/models/fsdp2_native.py b/unsloth/models/fsdp2_native.py
index 85c661c7233..ae0c85fe953 100644
--- a/unsloth/models/fsdp2_native.py
+++ b/unsloth/models/fsdp2_native.py
@@ -1240,6 +1240,85 @@ def _agree_over_ranks(
return [int(v) == ranks for v in t.tolist()]
+def _mesh_broadcast_from_first(tensor, mesh, dist):
+ """Broadcast `tensor` in place from the mesh's first rank to every rank of `mesh` (the world when
+ None): one broadcast per mesh dimension, each from its group's first rank."""
+ if mesh is None:
+ dist.broadcast(tensor, src = 0)
+ return
+ for dim in range(mesh.ndim):
+ group = mesh.get_group(dim)
+ dist.broadcast(tensor, src = dist.get_global_rank(group, 0), group = group)
+
+
+_FINGERPRINT_CHUNK = (
+ 1 << 20
+) # 8 MiB per float64 buffer: a few of them before fully_shard, never a full copy
+
+
+def _content_fingerprint(p):
+ """Four float64 numbers that tell two ranks' copies of a tensor apart: sum, L1 norm, a sum weighted
+ by a position-dependent cos(i * golden ratio) (so a permutation such as [1, -1] vs [-1, 1] differs),
+ and the count of non-finite elements. Non-finite values are mapped to fixed finite stand-ins first,
+ so a NaN cannot hide the other elements while identical NaN patterns still agree. Chunked, so no
+ float64 copy of a large weight is made: each chunk holds a few 8 MiB float64 buffers."""
+ flat = p.detach().reshape(-1)
+ out = torch.zeros(4, dtype = torch.float64, device = flat.device)
+ for start in range(0, flat.numel(), _FINGERPRINT_CHUNK):
+ chunk = flat[start : start + _FINGERPRINT_CHUNK].to(torch.float64)
+ bad = ~torch.isfinite(chunk)
+ chunk = torch.nan_to_num(chunk, nan = 3.0, posinf = 5.0, neginf = -7.0)
+ index = torch.arange(start, start + chunk.numel(), dtype = torch.float64, device = chunk.device)
+ weight = torch.cos(index * 0.6180339887498949)
+ out += torch.stack(
+ (chunk.sum(), chunk.abs().sum(), (chunk * weight).sum(), bad.sum().double())
+ )
+ return out
+
+
+def _sync_trainable_from_first_rank(
+ model,
+ dist,
+ device,
+ mesh = None,
+):
+ """Every rank must start from the same trainable weights. DDP broadcasts rank 0's parameters when
+ it wraps the model; fully_shard keeps each rank's own slice instead, so a weight initialised
+ without a seed on each rank (a LoRA adapter whose loader ignored random_state, a fresh head) became
+ a mix of the ranks' values, and the ranks trained different models. Compares a content fingerprint per
+ trainable tensor over the ranks of `mesh` and broadcasts the first rank's values of the tensors
+ that differ (only those, so a full finetune is not re-sent). Returns how many were synced."""
+ params = [
+ p
+ for p in model.parameters()
+ if p.requires_grad and p.device.type != "meta" and _floating(p)
+ ]
+ if not params or dist.get_world_size() == 1:
+ return 0
+ from .fsdp2_save import mesh_all_reduce
+
+ with torch.no_grad():
+ local = torch.stack([_content_fingerprint(p).to(device) for p in params])
+ first = local.clone()
+ _mesh_broadcast_from_first(first, mesh, dist)
+ differs = (local != first).any(dim = 1).to(torch.int32)
+ mesh_all_reduce(differs, mesh, dist)
+ index = [i for i, n in enumerate(differs.tolist()) if n]
+ for i in index:
+ p = params[i]
+ data = p.data if p.device == device else p.data.to(device)
+ _mesh_broadcast_from_first(data, mesh, dist)
+ if data is not p.data:
+ p.data.copy_(data)
+ if index:
+ _print_rank0(
+ f"Unsloth: native FSDP2: {len(index)} trainable tensors differed across ranks before "
+ "sharding (initialised without a common seed); every rank now starts from the first "
+ "rank's values, as DDP does."
+ )
+ return len(index)
+
+
def _place_replicated(params, device):
"""Parameters left out of FSDP (ignored_params) are never moved by it: they must already sit on this
rank's device. Moves plain ones that do not; refuses meta (a meta / broadcast load must materialise
@@ -1304,6 +1383,7 @@ def shard_model(model, config: Optional[FSDP2Config] = None):
if device.type == "cuda":
torch.cuda.set_device(device)
mesh = config.mesh if config.mesh is not None else _default_mesh(device, dist)
+ _sync_trainable_from_first_rank(model, dist, device, config.mesh)
param_dtype = _resolve_param_dtype(model, config)
mp_policy = MixedPrecisionPolicy(param_dtype = param_dtype, reduce_dtype = config.reduce_dtype)
_patch_recompute_input_cast() # before any fully_shard registers FSDPState._pre_forward
diff --git a/unsloth/models/sentence_transformer.py b/unsloth/models/sentence_transformer.py
index 036c47432f0..1c1ccbcf760 100644
--- a/unsloth/models/sentence_transformer.py
+++ b/unsloth/models/sentence_transformer.py
@@ -3227,6 +3227,12 @@ class FastSentenceTransformer(FastModel):
f"Unsloth Warning: {inner_model.__class__.__name__} does not support gradient checkpointing. Skipping."
)
+ # Seed the adapter init like FastModel.get_peft_model: random_state was ignored on this
+ # path, so the LoRA matrices depended on whatever ran before, and every torchrun rank
+ # drew its own.
+ from transformers import set_seed as transformers_set_seed
+
+ transformers_set_seed(random_state)
lora_config = LoraConfig(
r = r,
lora_alpha = lora_alpha,