Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Native FSDP2: sync initial trainable weights across ranks; shard FastSentenceTransformer and FastDiffusionModel (#15012)
Daniel Hanc52ef020c1d9
diff --git a/tests/diffusion/test_diffusion_distributed_device_map.py b/tests/diffusion/test_diffusion_distributed_device_map.pynew file mode 100644index 00000000000..8bd9e6f377b--- /dev/null+++ b/tests/diffusion/test_diffusion_distributed_device_map.py@@ -0,0 +1,56 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""FastDiffusionModel under torchrun: each rank loads one whole model on its own device, as+FastLanguageModel / FastModel do. With the default device_map = "auto" every rank spread the model+over all visible cards, and a 2-rank native FSDP2 launch died in transformers' infer_auto_device_map+(IndexError) before training."""++import pytest++torch = pytest.importorskip("torch")+transformers = pytest.importorskip("transformers")+if not hasattr(transformers, "DiffusionGemmaForBlockDiffusion"):+ pytest.skip("transformers without DiffusionGemma", allow_module_level = True)++TINY = "trl-internal-testing/tiny-DiffusionGemmaForBlockDiffusion"+++class _Stop(Exception):+ pass+++def _placement(monkeypatch, env, **kwargs):+ import unsloth.models.diffusion as diffusion++ for key in ("WORLD_SIZE", "LOCAL_RANK", "RANK", "LOCAL_WORLD_SIZE"):+ monkeypatch.delenv(key, raising = False)+ for key, value in env.items():+ monkeypatch.setenv(key, value)+ seen = {}++ def resolve(device_map, *args, **kw):+ seen["device_map"] = device_map+ raise _Stop++ monkeypatch.setattr(diffusion, "resolve_unsloth_device_map", resolve)+ with pytest.raises(_Stop):+ diffusion.FastDiffusionModel.from_pretrained(TINY, dtype = torch.float32, **kwargs)+ return seen["device_map"]+++RANK1 = {"WORLD_SIZE": "2", "RANK": "1", "LOCAL_RANK": "1", "LOCAL_WORLD_SIZE": "2"}+++def test_each_rank_loads_on_its_own_device(monkeypatch):+ from unsloth.models.loader_utils import DEVICE_TYPE_TORCH+ assert _placement(monkeypatch, RANK1) == {"": f"{DEVICE_TYPE_TORCH}:1"}+++def test_a_device_the_caller_named_is_kept(monkeypatch):+ assert _placement(monkeypatch, RANK1, device_map = "cpu") == "cpu"+ assert _placement(monkeypatch, RANK1, device_map = {"": "cpu"}) == {"": "cpu"}+++def test_a_single_process_keeps_auto(monkeypatch):+ assert _placement(monkeypatch, {}) == "auto"diff --git a/tests/python/test_fast_sentence_transformer_lora_seed.py b/tests/python/test_fast_sentence_transformer_lora_seed.pynew file mode 100644index 00000000000..e635abe7a2e--- /dev/null+++ b/tests/python/test_fast_sentence_transformer_lora_seed.py@@ -0,0 +1,48 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""FastSentenceTransformer.get_peft_model seeds the adapter init with random_state on the fast+encoder path (BERT, ModernBERT, ...), as FastModel.get_peft_model does. It was ignored there, so the+LoRA matrices depended on whatever had used the RNG before, and under torchrun every rank drew its own+adapters (native FSDP2 then sharded a mix of them)."""++import pytest+from real_accelerator import has_real_accelerator++torch = pytest.importorskip("torch")+pytest.importorskip("sentence_transformers")++# The loader reads device properties even with device_map = "cpu".+pytestmark = pytest.mark.skipif(+ not has_real_accelerator(), reason = "the loader needs an accelerator"+)++ENCODER = "sentence-transformers-testing/stsb-bert-tiny-safetensors"+++def _adapters(random_state, burn):+ from unsloth import FastSentenceTransformer++ model = FastSentenceTransformer.from_pretrained(+ ENCODER, max_seq_length = 32, dtype = torch.float32, device_map = "cpu"+ )+ assert getattr(model, "_unsloth_fast_encoder", False) # the path under test+ torch.rand(burn) # a different RNG history before the adapters are made+ model = FastSentenceTransformer.get_peft_model(+ model, r = 4, lora_alpha = 8, target_modules = ["query", "value"], random_state = random_state+ )+ return {n: p.detach().clone() for n, p in model.named_parameters() if "lora_A" in n}+++def test_random_state_fixes_the_adapters():+ a = _adapters(7, burn = 1)+ b = _adapters(7, burn = 1000)+ assert a and a.keys() == b.keys()+ for name in a:+ assert torch.equal(a[name], b[name]), name+++def test_a_different_random_state_gives_different_adapters():+ a = _adapters(7, burn = 1)+ c = _adapters(8, burn = 1)+ assert any(not torch.equal(a[n], c[n]) for n in a)diff --git a/tests/test_fsdp2_native_sync_initial_weights_gloo.py b/tests/test_fsdp2_native_sync_initial_weights_gloo.pynew file mode 100644index 00000000000..14e4cd8ce0f--- /dev/null+++ b/tests/test_fsdp2_native_sync_initial_weights_gloo.py@@ -0,0 +1,144 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""Native FSDP2 starts every rank from the same trainable weights, on a gloo CPU world of 2.++DDP broadcasts rank 0's parameters when it wraps the model. fully_shard keeps each rank's own slice,+so an adapter initialised without a common seed on each rank (FastSentenceTransformer.get_peft_model+ignored random_state) was sharded as a mix of both ranks' values: rank 0 held its half of the LoRA+matrices, rank 1 its own, and the step-0 gradient norm under FSDP2 was 12% off one process (DDP: 2%).+shard_model now checks a checksum per trainable tensor and broadcasts the first rank's values of the+ones that differ."""++import multiprocessing+import os++if multiprocessing.parent_process() is not None or getattr(+ multiprocessing.current_process(), "_inheriting", False+):+ os.environ["CUDA_VISIBLE_DEVICES"] = "" # spawned gloo ranks: CPU only, also on GPU hosts++import sys # noqa: E402++import pytest # noqa: E402+import torch # noqa: E402++sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))+from test_fsdp2_native_units import _load_native # noqa: E402+from test_fsdp2_native_trainer_ckpt_gloo import _free_port, _full # noqa: E402+from test_fsdp2_native_trainer_args_gloo import _lora # noqa: E402+++def _worker(rank, world, port, out, case):+ os.environ.update(+ MASTER_ADDR = "127.0.0.1",+ MASTER_PORT = str(port),+ RANK = str(rank),+ WORLD_SIZE = str(world),+ LOCAL_RANK = str(rank),+ LOCAL_WORLD_SIZE = str(world),+ )+ import datetime+ import io+ from contextlib import redirect_stdout++ import torch.distributed as dist++ dist.init_process_group(+ "gloo", rank = rank, world_size = world, timeout = datetime.timedelta(seconds = 120)+ )+ try:+ native = _load_native()+ # Same base weights everywhere; the adapters differ per rank unless the case says otherwise.+ seed = 1 + rank if case == "differ" else 1+ model = _lora(seed)+ first = _lora(1) # rank 0's starting adapters, built locally for the comparison+ if case == "nan":+ # Identical non-finite weights on every rank (tiny Llama4 ships some) are not a mismatch.+ for m in (model, first):+ next(p for n, p in m.named_parameters() if p.requires_grad).data.view(-1)[0] = (+ float("nan")+ )+ if case in ("permuted", "nan_elsewhere") and rank == 1:+ # Same sum and L1 norm as rank 0, different contents: the elements reversed.+ w = next(p for n, p in model.named_parameters() if p.requires_grad).data.view(-1)+ w.copy_(w.flip(0))+ if case == "nan_elsewhere":+ # A shared NaN must not hide a difference in the other elements.+ for m in (model, first):+ next(p for n, p in m.named_parameters() if p.requires_grad).data.view(-1)[0] = (+ float("nan")+ )+ want = {n: p.detach().clone() for n, p in first.named_parameters() if p.requires_grad}+ buf = io.StringIO()+ with redirect_stdout(buf):+ native.shard_model(model, native.FSDP2Config(param_dtype = torch.float32, device = "cpu"))+ got = _full(model, trainable_only = True)+ res = {+ "n": len(want),+ "max_diff": max((got[n] - want[n]).nan_to_num(0.0).abs().max().item() for n in want),+ "message": "differed across ranks" in buf.getvalue(),+ }+ out.put((rank, res))+ except Exception as exc:+ import traceback+ out.put((rank, {"crash": f"{type(exc).__name__}: {exc}\n{traceback.format_exc()}"}))+ finally:+ dist.destroy_process_group()+++def _run(case):+ import torch.multiprocessing as mp++ ctx = mp.get_context("spawn")+ out = ctx.Queue()+ mp.start_processes(+ _worker, args = (2, _free_port(), out, case), nprocs = 2, join = True, start_method = "spawn"+ )+ results = {}+ while not out.empty():+ rank, r = out.get(timeout = 5)+ results[rank] = r+ assert sorted(results) == [0, 1], results+ for rank, r in results.items():+ print("SYNC_RESULT", case, rank, r)+ assert "crash" not in r, r["crash"]+ return results+++def test_ranks_with_different_adapters_start_from_the_first_ranks():+ pytest.importorskip("transformers")+ pytest.importorskip("peft")+ results = _run("differ")+ for rank, r in results.items():+ assert r["n"] > 0, r+ assert r["max_diff"] == 0.0, (rank, r) # rank 0's adapters, bit for bit, on both ranks+ assert results[0]["message"] and not results[1]["message"], results # said once, on rank 0+++def test_ranks_that_already_agree_are_left_alone():+ pytest.importorskip("transformers")+ pytest.importorskip("peft")+ for rank, r in _run("same").items():+ assert r["max_diff"] == 0.0, (rank, r)+ assert not r["message"], (rank, r)+++def test_identical_non_finite_weights_are_not_a_mismatch():+ pytest.importorskip("transformers")+ pytest.importorskip("peft")+ for rank, r in _run("nan").items():+ assert r["max_diff"] == 0.0, (rank, r)+ assert not r["message"], (rank, r)+++def test_a_permuted_adapter_is_not_mistaken_for_the_same():+ for rank, r in _run("permuted").items():+ assert r["max_diff"] == 0.0, (rank, r)+ assert r["message"] or rank != 0, (rank, r)+++def test_a_shared_nan_does_not_hide_other_differences():+ for rank, r in _run("nan_elsewhere").items():+ assert r["max_diff"] == 0.0, (rank, r)+ assert r["message"] or rank != 0, (rank, r)diff --git a/tests/test_fsdp2_native_trainer_init_any_loader_gloo.py b/tests/test_fsdp2_native_trainer_init_any_loader_gloo.pynew file mode 100644index 00000000000..2156498d594--- /dev/null+++ b/tests/test_fsdp2_native_trainer_init_any_loader_gloo.py@@ -0,0 +1,282 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""UNSLOTH_FSDP2=1 for a model no FastLanguageModel / FastModel load built.++Native FSDP2 shards inside Unsloth's Trainer.__init__ wrapper, which only those loads (and+DecisionTrainer) installed. A FastSentenceTransformer encoder (BERT, ModernBERT) never loads through+them, so under `UNSLOTH_FSDP2=1 torchrun` its SentenceTransformerTrainer trained with plain DDP:+nothing sharded and no message. `import unsloth` now installs the wrapper under the flag+(fsdp2_launch.install_native_fsdp2_trainer_init).++* the installer wraps Trainer under the flag, and only there;+* the GPU import path calls it outside any try / except;+* on a gloo CPU world of 2, a SentenceTransformer-shaped model (a plain encoder plus mean pooling and+ a Trainer subclass with its own compute_loss, as sentence-transformers builds it) is sharded by the+ installed wrapper and trains like one process over the same 4 rows."""++import multiprocessing+import os++if multiprocessing.parent_process() is not None or getattr(+ multiprocessing.current_process(), "_inheriting", False+):+ os.environ["CUDA_VISIBLE_DEVICES"] = "" # spawned gloo ranks: CPU only, also on GPU hosts++import ast # noqa: E402+import copy # noqa: E402+import importlib.util # noqa: E402+import sys # noqa: E402+import types # noqa: E402++import pytest # noqa: E402+import torch # noqa: E402++sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))+from test_fsdp2_native_units import REPO, _load_native # noqa: E402+from test_fsdp2_native_trainer_ckpt_gloo import _free_port, _stub_unsloth_save # noqa: E402+from test_fsdp2_native_trainer_args_gloo import _install_wrapped_trainer_init # noqa: E402++ENCODER = "hf-internal-testing/tiny-random-BertModel"+STEPS = 2+++def _load_launch(calls):+ """fsdp2_launch.py with `.models._utils` stubbed (the real one needs the GPU package)."""+ pkg = sys.modules.get("unsloth") or types.ModuleType("unsloth")+ if not hasattr(pkg, "__path__"):+ pkg.__path__ = [os.path.join(REPO, "unsloth")]+ models = sys.modules.get("unsloth.models") or types.ModuleType("unsloth.models")+ if not hasattr(models, "__path__"):+ models.__path__ = [os.path.join(REPO, "unsloth", "models")]+ utils = types.ModuleType("unsloth.models._utils")++ def patch_gradient_accumulation_fix(trainer_cls):+ calls.append(trainer_cls)+ _install_wrapped_trainer_init()++ utils.patch_gradient_accumulation_fix = patch_gradient_accumulation_fix+ sys.modules["unsloth"] = pkg+ sys.modules["unsloth.models"] = models+ sys.modules["unsloth.models._utils"] = utils+ spec = importlib.util.spec_from_file_location(+ "unsloth.fsdp2_launch", os.path.join(REPO, "unsloth", "fsdp2_launch.py")+ )+ module = importlib.util.module_from_spec(spec)+ sys.modules["unsloth.fsdp2_launch"] = module+ spec.loader.exec_module(module)+ return module+++ "env,applies",+ [+ ({"UNSLOTH_FSDP2": "1", "WORLD_SIZE": "2"}, True),+ ({"UNSLOTH_FSDP2": "1", "WORLD_SIZE": "1"}, False),+ ({"WORLD_SIZE": "2"}, False),+ ({"UNSLOTH_FSDP2": "1", "WORLD_SIZE": "2", "UNSLOTH_FSDP2_BACKEND": "accelerate"}, False),+ ],+)+def test_installer_wraps_trainer_only_under_native_fsdp2(monkeypatch, env, applies):+ from transformers import Trainer++ for key in ("UNSLOTH_FSDP2", "WORLD_SIZE", "UNSLOTH_FSDP2_BACKEND", "ACCELERATE_USE_FSDP"):+ monkeypatch.delenv(key, raising = False)+ for key, value in env.items():+ monkeypatch.setenv(key, value)+ for name in ("unsloth", "unsloth.models", "unsloth.models._utils", "unsloth.fsdp2_launch"):+ monkeypatch.setitem(sys.modules, name, sys.modules.get(name))+ monkeypatch.setattr(Trainer, "__init__", Trainer.__init__)+ calls = []+ launch = _load_launch(calls)+ assert launch.install_native_fsdp2_trainer_init() is applies+ assert calls == ([Trainer] if applies else [])+++def test_gpu_import_path_installs_it():+ tree = ast.parse(open(os.path.join(REPO, "unsloth", "__init__.py"), encoding = "utf-8").read())+ found = [+ node+ for node in ast.walk(tree)+ if isinstance(node, ast.Call)+ and isinstance(node.func, ast.Name)+ and node.func.id == "_install_native_trainer_init"+ ]+ assert found, "unsloth/__init__.py no longer installs the native FSDP2 Trainer wrapper"+ # Not swallowed: a wrapper that failed to install would silently fall back to DDP.+ for node in ast.walk(tree):+ if isinstance(node, ast.Try):+ inside = {id(n) for stmt in node.body for n in ast.walk(stmt)}+ assert not any(id(call) in inside for call in found)+++class _MeanPooled(torch.nn.Module):+ """A SentenceTransformer's shape: the HF encoder plus mean pooling, no PreTrainedModel root."""++ def __init__(self, encoder):+ super().__init__()+ self.encoder = encoder++ def forward(self, input_ids, attention_mask):+ hidden = self.encoder(input_ids = input_ids, attention_mask = attention_mask).last_hidden_state+ mask = attention_mask.unsqueeze(-1).to(hidden.dtype)+ return (hidden * mask).sum(1) / mask.sum(1)+++def _pair_loss(model, batch):+ a = model(batch["a_ids"], batch["a_mask"])+ b = model(batch["b_ids"], batch["b_mask"])+ cos = torch.nn.functional.cosine_similarity(a, b)+ return torch.nn.functional.mse_loss(cos, batch["score"])+++def _rows():+ g = torch.Generator().manual_seed(0)+ return [+ {+ "a_ids": torch.randint(5, 90, (12,), generator = g),+ "a_mask": torch.ones(12, dtype = torch.long),+ "b_ids": torch.randint(5, 90, (12,), generator = g),+ "b_mask": torch.ones(12, dtype = torch.long),+ "score": torch.rand((), generator = g),+ }+ for _ in range(4)+ ]+++def _model():+ import transformers++ config = transformers.AutoConfig.from_pretrained(ENCODER)+ config.hidden_dropout_prob = config.attention_probs_dropout_prob = 0.0 # deterministic steps+ torch.manual_seed(0)+ return _MeanPooled(transformers.AutoModel.from_config(config, torch_dtype = torch.float32))+++def _reference(model, rows):+ """One process: the same AdamW steps on all 4 rows (constant LR, clip 1.0)."""+ model = copy.deepcopy(model)+ batch = {k: torch.stack([r[k] for r in rows]) for k in rows[0]}+ opt = torch.optim.AdamW(model.parameters(), lr = 1e-3, weight_decay = 0.0)+ for _ in range(STEPS):+ opt.zero_grad()+ _pair_loss(model, batch).backward()+ torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)+ opt.step()+ return {n: p.detach().clone() for n, p in model.named_parameters()}+++def _worker(rank, world, port, out, tmp):+ os.environ.update(+ MASTER_ADDR = "127.0.0.1",+ MASTER_PORT = str(port),+ RANK = str(rank),+ WORLD_SIZE = str(world),+ LOCAL_RANK = str(rank),+ LOCAL_WORLD_SIZE = str(world),+ UNSLOTH_FSDP2 = "1",+ )+ import datetime++ import torch.distributed as dist++ dist.init_process_group(+ "gloo", rank = rank, world_size = world, timeout = datetime.timedelta(seconds = 120)+ )+ try:+ native = _load_native()+ _stub_unsloth_save()+ from transformers import Trainer, TrainingArguments++ res = {"wrapped_before": Trainer.__init__.__name__ == "_unsloth_trainer_init"}+ launch = _load_launch([])+ res["installed"] = launch.install_native_fsdp2_trainer_init()++ class PairTrainer(Trainer):+ # sentence-transformers' own compute_loss: the model returns embeddings, not a loss.+ def compute_loss(+ self,+ model,+ inputs,+ return_outputs = False,+ num_items_in_batch = None,+ ):+ return _pair_loss(model, inputs)++ def collate(rows):+ return {k: torch.stack([r[k] for r in rows]) for k in rows[0]}++ rows = _rows()+ model = _model()+ want = _reference(model, rows)+ args = TrainingArguments(+ output_dir = os.path.join(tmp, "out"),+ report_to = [],+ use_cpu = True,+ per_device_train_batch_size = 2,+ max_steps = STEPS,+ learning_rate = 1e-3,+ lr_scheduler_type = "constant",+ optim = "adamw_torch",+ weight_decay = 0.0,+ max_grad_norm = 1.0,+ save_strategy = "no",+ logging_steps = 1,+ disable_tqdm = True,+ remove_unused_columns = False,+ dataloader_drop_last = True,+ seed = 0,+ )+ trainer = PairTrainer(model = model, args = args, train_dataset = rows, data_collator = collate)+ res["sharded"] = native.is_native_fsdp2(trainer.model)+ res["units"] = sum(+ isinstance(m, torch.distributed.fsdp.FSDPModule) for m in trainer.model.modules()+ )+ trainer.train()+ from torch.distributed.tensor import DTensor++ got = {+ n: (p.full_tensor() if isinstance(p, DTensor) else p).detach()+ for n, p in trainer.model.named_parameters()+ }+ res["n_params"] = len(got)+ res["missing"] = sorted(set(want) - set(got))[:5]+ res["max_diff"] = max((got[n] - want[n]).abs().max().item() for n in want if n in got)+ res["moved"] = max((want[n] - p).abs().max().item() for n, p in _model().named_parameters())+ out.put((rank, res))+ except Exception as exc:+ import traceback+ out.put((rank, {"crash": f"{type(exc).__name__}: {exc}\n{traceback.format_exc()}"}))+ finally:+ dist.destroy_process_group()+++def test_sentence_transformer_shaped_model_shards_and_matches_one_process(tmp_path):+ pytest.importorskip("transformers")+ import torch.multiprocessing as mp++ ctx = mp.get_context("spawn")+ out = ctx.Queue()+ mp.start_processes(+ _worker,+ args = (2, _free_port(), out, str(tmp_path)),+ nprocs = 2,+ join = True,+ start_method = "spawn",+ )+ results = {}+ while not out.empty():+ rank, r = out.get(timeout = 5)+ results[rank] = r+ assert sorted(results) == [0, 1], results+ for rank, r in results.items():+ print("ANY_LOADER_RESULT", rank, r)+ assert "crash" not in r, r["crash"]+ assert r["wrapped_before"] is False, r # nothing else installed the wrapper here+ assert r["installed"] is True and r["sharded"] is True, r+ assert r["units"] >= 2, r # encoder blocks + root+ assert not r["missing"], r+ assert (+ r["moved"] > 1e-4+ ), r # the steps changed the weights, so the comparison means something+ assert r["max_diff"] < 2e-5, r # fp32, reduction order onlydiff --git a/unsloth/__init__.py b/unsloth/__init__.pyindex 4e2d54a8052..8c8e15f8573 100644--- a/unsloth/__init__.py+++ b/unsloth/__init__.py@@ -1717,6 +1717,14 @@ else:except Exception:pass+ # Native FSDP2 shards inside the Trainer.__init__ wrapper that FastLanguageModel / FastModel loads+ # install. A FastSentenceTransformer encoder never loads through them, so under UNSLOTH_FSDP2=1 its+ # Trainer trained with DDP instead, silently. Install the wrapper up front under the flag.+ from .fsdp2_launch import install_native_fsdp2_trainer_init as _install_native_trainer_init++ _install_native_trainer_init()+ del _install_native_trainer_init+def get_gpu_memory_stats():"""Return CUDA/ROCm/XPU/NPU device stats, peak memory, and total memory in GiB."""try:diff --git a/unsloth/fsdp2_launch.py b/unsloth/fsdp2_launch.pyindex b558f3d3d8d..8e1a9b489bc 100644--- a/unsloth/fsdp2_launch.py+++ b/unsloth/fsdp2_launch.py@@ -16,6 +16,7 @@ __all__ = ["native_backend_conflicts","refuse_native_backend_conflicts","install_raw_accelerator_fsdp2_fallback",+ "install_native_fsdp2_trainer_init",]_TRUTHY = ("1", "true", "yes", "on")@@ -457,3 +458,19 @@ def install_raw_accelerator_fsdp2_fallback():Accelerator.__init__ = __init__setattr(Accelerator, _RAW_ACCELERATOR_PATCHED, True)return True+++def install_native_fsdp2_trainer_init():+ """UNSLOTH_FSDP2=1 under WORLD_SIZE > 1 shards natively inside Unsloth's Trainer.__init__ wrapper+ (installed by `_utils.patch_gradient_accumulation_fix`). FastLanguageModel / FastModel loads and+ DecisionTrainer install it; a model built without them (a FastSentenceTransformer encoder) left the+ Trainer unwrapped, so it trained under DDP with nothing sharded and no message. Installs the wrapper+ once the flag selects native FSDP2. Returns whether it applies."""+ if not unsloth_fsdp2_native_requested():+ return False+ from transformers import Trainer++ from .models._utils import patch_gradient_accumulation_fix++ patch_gradient_accumulation_fix(Trainer)+ return Truediff --git a/unsloth/models/diffusion.py b/unsloth/models/diffusion.pyindex d19a92c3879..9134cfa8270 100644--- a/unsloth/models/diffusion.py+++ b/unsloth/models/diffusion.py@@ -38,10 +38,12 @@ from .diffusion_profiles import ()from .llama import loggerfrom .loader_utils import (+ is_automatic_device_map,planner_config_overrides,planner_hub_kwargs,planner_kwargs_with_max_memory,planner_quantization_kwargs,+ prepare_device_map,requested_device_map,resolve_unsloth_device_map,)@@ -356,8 +358,15 @@ class FastDiffusionModel:# Same leaf-level resolution as llama.py and vision.py: an unresolved "unsloth" becomes# torch.device("unsloth") in transformers and raises instead of loading.+ device_map = requested_device_map(device_map)+ if is_automatic_device_map(device_map):+ # Under torchrun / accelerate launch each rank loads one whole model on its own device, as+ # FastLanguageModel / FastModel do: "auto" would spread every rank over all the visible cards.+ distributed_device_map, is_dist = prepare_device_map()+ if is_dist:+ device_map = distributed_device_mapdevice_map = resolve_unsloth_device_map(- requested_device_map(device_map),+ device_map,model_name,full_finetuning = full_finetuning,skip_reason = (diff --git a/unsloth/models/fsdp2_native.py b/unsloth/models/fsdp2_native.pyindex 85c661c7233..ae0c85fe953 100644--- a/unsloth/models/fsdp2_native.py+++ b/unsloth/models/fsdp2_native.py@@ -1240,6 +1240,85 @@ def _agree_over_ranks(return [int(v) == ranks for v in t.tolist()]+def _mesh_broadcast_from_first(tensor, mesh, dist):+ """Broadcast `tensor` in place from the mesh's first rank to every rank of `mesh` (the world when+ None): one broadcast per mesh dimension, each from its group's first rank."""+ if mesh is None:+ dist.broadcast(tensor, src = 0)+ return+ for dim in range(mesh.ndim):+ group = mesh.get_group(dim)+ dist.broadcast(tensor, src = dist.get_global_rank(group, 0), group = group)+++_FINGERPRINT_CHUNK = (+ 1 << 20+) # 8 MiB per float64 buffer: a few of them before fully_shard, never a full copy+++def _content_fingerprint(p):+ """Four float64 numbers that tell two ranks' copies of a tensor apart: sum, L1 norm, a sum weighted+ by a position-dependent cos(i * golden ratio) (so a permutation such as [1, -1] vs [-1, 1] differs),+ and the count of non-finite elements. Non-finite values are mapped to fixed finite stand-ins first,+ so a NaN cannot hide the other elements while identical NaN patterns still agree. Chunked, so no+ float64 copy of a large weight is made: each chunk holds a few 8 MiB float64 buffers."""+ flat = p.detach().reshape(-1)+ out = torch.zeros(4, dtype = torch.float64, device = flat.device)+ for start in range(0, flat.numel(), _FINGERPRINT_CHUNK):+ chunk = flat[start : start + _FINGERPRINT_CHUNK].to(torch.float64)+ bad = ~torch.isfinite(chunk)+ chunk = torch.nan_to_num(chunk, nan = 3.0, posinf = 5.0, neginf = -7.0)+ index = torch.arange(start, start + chunk.numel(), dtype = torch.float64, device = chunk.device)+ weight = torch.cos(index * 0.6180339887498949)+ out += torch.stack(+ (chunk.sum(), chunk.abs().sum(), (chunk * weight).sum(), bad.sum().double())+ )+ return out+++def _sync_trainable_from_first_rank(+ model,+ dist,+ device,+ mesh = None,+):+ """Every rank must start from the same trainable weights. DDP broadcasts rank 0's parameters when+ it wraps the model; fully_shard keeps each rank's own slice instead, so a weight initialised+ without a seed on each rank (a LoRA adapter whose loader ignored random_state, a fresh head) became+ a mix of the ranks' values, and the ranks trained different models. Compares a content fingerprint per+ trainable tensor over the ranks of `mesh` and broadcasts the first rank's values of the tensors+ that differ (only those, so a full finetune is not re-sent). Returns how many were synced."""+ params = [+ p+ for p in model.parameters()+ if p.requires_grad and p.device.type != "meta" and _floating(p)+ ]+ if not params or dist.get_world_size() == 1:+ return 0+ from .fsdp2_save import mesh_all_reduce++ with torch.no_grad():+ local = torch.stack([_content_fingerprint(p).to(device) for p in params])+ first = local.clone()+ _mesh_broadcast_from_first(first, mesh, dist)+ differs = (local != first).any(dim = 1).to(torch.int32)+ mesh_all_reduce(differs, mesh, dist)+ index = [i for i, n in enumerate(differs.tolist()) if n]+ for i in index:+ p = params[i]+ data = p.data if p.device == device else p.data.to(device)+ _mesh_broadcast_from_first(data, mesh, dist)+ if data is not p.data:+ p.data.copy_(data)+ if index:+ _print_rank0(+ f"Unsloth: native FSDP2: {len(index)} trainable tensors differed across ranks before "+ "sharding (initialised without a common seed); every rank now starts from the first "+ "rank's values, as DDP does."+ )+ return len(index)++def _place_replicated(params, device):"""Parameters left out of FSDP (ignored_params) are never moved by it: they must already sit on thisrank's device. Moves plain ones that do not; refuses meta (a meta / broadcast load must materialise@@ -1304,6 +1383,7 @@ def shard_model(model, config: Optional[FSDP2Config] = None):if device.type == "cuda":torch.cuda.set_device(device)mesh = config.mesh if config.mesh is not None else _default_mesh(device, dist)+ _sync_trainable_from_first_rank(model, dist, device, config.mesh)param_dtype = _resolve_param_dtype(model, config)mp_policy = MixedPrecisionPolicy(param_dtype = param_dtype, reduce_dtype = config.reduce_dtype)_patch_recompute_input_cast() # before any fully_shard registers FSDPState._pre_forwarddiff --git a/unsloth/models/sentence_transformer.py b/unsloth/models/sentence_transformer.pyindex 036c47432f0..1c1ccbcf760 100644--- a/unsloth/models/sentence_transformer.py+++ b/unsloth/models/sentence_transformer.py@@ -3227,6 +3227,12 @@ class FastSentenceTransformer(FastModel):f"Unsloth Warning: {inner_model.__class__.__name__} does not support gradient checkpointing. Skipping.")+ # Seed the adapter init like FastModel.get_peft_model: random_state was ignored on this+ # path, so the LoRA matrices depended on whatever ran before, and every torchrun rank+ # drew its own.+ from transformers import set_seed as transformers_set_seed++ transformers_set_seed(random_state)lora_config = LoraConfig(r = r,lora_alpha = lora_alpha,