Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Tensor parallel: average fp16 dp gradients in fp32 so finite gradients cannot sum to inf (#15415)

Daniel Han91bbf469066a
diff --git a/tests/test_tp_env_gloo.py b/tests/test_tp_env_gloo.py
index 8ecb8414731..0e2033f689d 100644
--- a/tests/test_tp_env_gloo.py
+++ b/tests/test_tp_env_gloo.py
@@ -357,6 +357,39 @@ def _case_dp_average(rank, world, tmp):
}
+def _case_dp_average_dtypes(rank, world, tmp):
+ """fp16 grads whose sum overflows but mean does not; bf16 / fp32 match own-dtype all-reduce."""
+ error = _import_checkout_unsloth()
+ if error is not None:
+ return {"skip": error}
+ import torch.distributed as dist
+ from torch.distributed.device_mesh import init_device_mesh
+
+ from unsloth.models import tp_native
+
+ dist.init_process_group("gloo")
+ mesh = init_device_mesh("cpu", (world,), mesh_dim_names = ("dp",))
+ torch.manual_seed(rank)
+ out, params = {}, {}
+ for dtype in (torch.float16, torch.bfloat16, torch.float32):
+ big = torch.nn.Parameter(torch.zeros(4, dtype = dtype))
+ big.grad = torch.tensor([40000.0, 40000.0 if rank else 30000.0, 1e-3, -6e4], dtype = dtype)
+ noise = torch.nn.Parameter(torch.zeros(64, dtype = dtype))
+ noise.grad = (torch.randn(64) * 10.0 ** torch.randint(-6, 4, (64,))).to(dtype)
+ params[dtype] = (big, noise)
+ reference = []
+ for p in (big, noise):
+ g = p.grad.clone()
+ dist.all_reduce(g)
+ reference.append(g.div_(world))
+ out[str(dtype)] = {"reference": reference}
+ tp_native._dp_average([p for pair in params.values() for p in pair], mesh["dp"])
+ for dtype, pair in params.items():
+ out[str(dtype)]["got"] = [p.grad.clone() for p in pair]
+ out[str(dtype)]["dtype"] = [p.grad.dtype for p in pair]
+ return out
+
+
def _case_accelerate_fsdp(rank, world, tmp, ckpt):
"""UNSLOTH_FSDP2=1 that resolves to accelerate's FSDP: the dp dimension would be left unsynced."""
error = _import_checkout_unsloth()
@@ -390,6 +423,7 @@ CASES = {
"units": _case_units,
"env": _case_env,
"dp_average": _case_dp_average,
+ "dp_average_dtypes": _case_dp_average_dtypes,
"accelerate_fsdp": _case_accelerate_fsdp,
}
@@ -625,6 +659,24 @@ def test_dp_average_matches_ddp_for_gradients_on_some_replicas():
assert res["none"] is None and res["frozen"] is None, res
+def test_dp_average_keeps_fp16_finite_and_bf16_fp32_unchanged():
+ results = _spawn("dp_average_dtypes", 2)
+ for res in results:
+ fp16 = res[str(torch.float16)]
+ big, noise = fp16["got"]
+ assert fp16["dtype"] == [torch.float16] * 2
+ assert torch.isfinite(big).all() and torch.isfinite(noise).all(), big
+ assert big.tolist() == [40000.0, 35008.0, 0.0010004043579101562, -6e4], big.tolist()
+ assert not torch.isfinite(fp16["reference"][0]).all()
+ for dtype in (torch.bfloat16, torch.float32):
+ entry = res[str(dtype)]
+ assert entry["dtype"] == [dtype] * 2
+ for got, want in zip(entry["got"], entry["reference"]):
+ assert torch.equal(got, want), (dtype, got, want)
+ for a, b in zip(results[0][str(torch.float16)]["got"], results[1][str(torch.float16)]["got"]):
+ assert torch.equal(a, b)
+
+
def test_env_tp_x_accelerate_fsdp_is_refused():
env = {"UNSLOTH_TP_SIZE": "2", "UNSLOTH_FSDP2": "1", "ACCELERATE_USE_FSDP": "true"}
for res in _spawn("accelerate_fsdp", 4, env):
diff --git a/unsloth/models/tp_native.py b/unsloth/models/tp_native.py
index c16088ce52d..59441a54e13 100644
--- a/unsloth/models/tp_native.py
+++ b/unsloth/models/tp_native.py
@@ -2575,6 +2575,9 @@ def _dp_average(params, mesh):
with torch.no_grad():
for tensors in buckets.values():
flat = _flatten_dense_tensors(tensors)
+ if flat.dtype == torch.float16:
+ # fp16 sum can overflow past 65504 though the mean is finite.
+ flat = flat.float()
dist.all_reduce(flat, group = group)
flat.div_(n)
for t, value in zip(tensors, _unflatten_dense_tensors(flat, tensors)):