Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Native FSDP2: clip gradients through Unsloth's clip, not the mixed mesh path (#15708)

Daniel Hand546b0c7ae9c
diff --git a/tests/test_fsdp2_native_trainer_fsdp_clip_gloo.py b/tests/test_fsdp2_native_trainer_fsdp_clip_gloo.py
index 2b3ec88212f..ba1d6b1f458 100644
--- a/tests/test_fsdp2_native_trainer_fsdp_clip_gloo.py
+++ b/tests/test_fsdp2_native_trainer_fsdp_clip_gloo.py
@@ -116,6 +116,57 @@ def _case_clip(native):
return res
+def _case_trainer_clip(native):
+ """Trainer._clip_grad_norm with sharded bfloat16 and replicated float32 grads."""
+ from accelerate import Accelerator
+ from torch.distributed.device_mesh import init_device_mesh
+ from torch.distributed.fsdp import fully_shard
+ from torch.distributed.tensor import DTensor
+ from transformers import Trainer
+
+ native.install_native_fsdp2_hooks()
+ model = _Net()
+ for name in ("a", "b", "dead"):
+ getattr(model, name).to(torch.bfloat16)
+ mesh = init_device_mesh("cpu", (2,))
+ for name in ("a", "b", "dead"):
+ fully_shard(getattr(model, name), mesh = mesh)
+ fully_shard(model, mesh = mesh, ignored_params = {model.rep})
+ setattr(model, native._NATIVE_FLAG, True)
+ x = torch.randn(64, 7, generator = torch.Generator().manual_seed(1)).to(torch.bfloat16)
+
+ class _T:
+ _has_mixed_mesh_grads = Trainer._has_mixed_mesh_grads
+ _clip_grad_norm = Trainer._clip_grad_norm
+ _mixed_mesh_grad_norm = getattr(Trainer, "_mixed_mesh_grad_norm", None)
+
+ t = _T()
+ t.args = types.SimpleNamespace(max_grad_norm = 1.0)
+ t._mixed_mesh_grads = None
+ t.model = model
+ t.accelerator = Accelerator(cpu = True)
+ t.accelerator._models.append(model)
+ model.zero_grad(set_to_none = True)
+ (model(x).float() * 50.0).backward()
+ grads = [
+ p.grad.full_tensor() if isinstance(p.grad, DTensor) else p.grad for p in model.parameters()
+ ]
+ dtypes = sorted({str(g.dtype) for g in grads})
+ fp64 = math.sqrt(sum(float(g.double().square().sum()) for g in grads))
+ one_gpu = float(torch.nn.utils.get_total_norm([g.clone() for g in grads], 2.0))
+ want = float(native.native_clip_grad_norm_(list(model.parameters()), math.inf))
+ mixed = float(t._mixed_mesh_grad_norm(model, math.inf)) if t._mixed_mesh_grad_norm else None
+ got = float(t._clip_grad_norm(model))
+ return {
+ "got": got,
+ "want": want,
+ "mixed": mixed,
+ "one_gpu": one_gpu,
+ "fp64": fp64,
+ "dtypes": dtypes,
+ }
+
+
def _mllama():
from transformers import MllamaConfig, MllamaForConditionalGeneration
@@ -198,6 +249,8 @@ def _worker(rank, world, port, out, case, tmp):
res = _case_trainer_fsdp(native, tmp)
elif case == "clip":
res = _case_clip(native)
+ elif case == "trainer_clip":
+ res = _case_trainer_clip(native)
elif case == "mllama":
res = _case_mllama(native, rank)
out.put((rank, res))
@@ -275,6 +328,21 @@ def test_native_clip_matches_torch_for_every_norm_type(tmp_path):
assert v["grad_diff"] < 1e-6, (rank, norm_type, v)
+def test_trainer_clip_with_replicated_and_sharded_grads_uses_native_clip(tmp_path):
+ # Trainer._mixed_mesh_grad_norm combined bf16 norms in bf16, off by up to 2e-3.
+ transformers = pytest.importorskip("transformers")
+ pytest.importorskip("accelerate")
+ if not hasattr(transformers.Trainer, "_has_mixed_mesh_grads"):
+ pytest.skip("this transformers has no mixed DTensor / plain gradient norm path")
+ for rank, r in _run("trainer_clip", tmp_path).items():
+ assert r["dtypes"] == ["torch.bfloat16", "torch.float32"], (rank, r)
+ assert r["got"] == r["want"], (rank, r)
+ if r["mixed"] is not None: # this batch separates the two combines
+ assert not math.isclose(r["mixed"], r["want"], rel_tol = 1e-4), (rank, r)
+ # One extra 16-bit rounding over single-GPU clip_grad_norm_.
+ assert math.isclose(r["got"], r["one_gpu"], rel_tol = 2**-6), (rank, r)
+
+
def test_mllama_mixed_image_batch_raises_instead_of_hanging(tmp_path):
pytest.importorskip("transformers")
pytest.importorskip("transformers.models.mllama")
diff --git a/unsloth/models/fsdp2_native.py b/unsloth/models/fsdp2_native.py
index 350c1333aea..51854ef4318 100644
--- a/unsloth/models/fsdp2_native.py
+++ b/unsloth/models/fsdp2_native.py
@@ -3327,6 +3327,23 @@ def _install_trainer_hooks():
Trainer._load_best_model = _load_best_model
+ original_mixed_mesh = getattr(Trainer, "_has_mixed_mesh_grads", None)
+ if original_mixed_mesh is not None:
+
+ @functools.wraps(original_mixed_mesh)
+ def _has_mixed_mesh_grads(self, model):
+ # _mixed_mesh_grad_norm skips our clip + fp16 unscale and sums 16-bit norms in 16 bits;
+ # native FSDP2 mixes replicated and sharded grads, so stay on the accelerator path.
+ if (
+ is_native_fsdp2(model)
+ or is_native_fsdp2(getattr(self, "model", None))
+ or _native_models(getattr(self, "accelerator", None))
+ ):
+ return False
+ return original_mixed_mesh(self, model)
+
+ Trainer._has_mixed_mesh_grads = _has_mixed_mesh_grads
+
original_create_accelerator = getattr(Trainer, "create_accelerator_and_postprocess", None)
if original_create_accelerator is not None: