Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Native FSDP2: faster native_clip_grad_norm_ that follows torch clip_grad_norm_ dtypes (#14973)

Daniel Han9df97d910c42
diff --git a/tests/test_fsdp2_native_clip_dtype_gloo.py b/tests/test_fsdp2_native_clip_dtype_gloo.py
new file mode 100644
index 00000000000..8417c906471
--- /dev/null
+++ b/tests/test_fsdp2_native_clip_dtype_gloo.py
@@ -0,0 +1,174 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""native_clip_grad_norm_ matches torch.nn.utils.clip_grad_norm_ in dtype and value on a 2-rank gloo
+world; a float16 sharded norm above 256 stays finite."""
+
+import multiprocessing
+import os
+
+if multiprocessing.parent_process() is not None or getattr(
+ multiprocessing.current_process(), "_inheriting", False
+):
+ os.environ["CUDA_VISIBLE_DEVICES"] = "" # spawned gloo ranks: CPU only, also on GPU hosts
+
+import math # noqa: E402
+import sys # noqa: E402
+import time # noqa: E402
+
+import pytest # noqa: E402
+import torch # noqa: E402
+
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+from test_fsdp2_native_units import _load_native # noqa: E402
+from test_fsdp2_native_trainer_ckpt_gloo import _free_port # noqa: E402
+
+NORM_TYPES = (0.0, 1.0, 2.0, 3.0, math.inf, -math.inf)
+# 16-bit: shard norm + combine round once more than torch's single norm
+CASES = {
+ "float32": (torch.float32, 1e-5),
+ "bfloat16": (torch.bfloat16, 2**-6),
+ "float16": (torch.float16, 2**-8),
+ "mixed": (None, 1e-5),
+ # complex64 norms are float32: the rank with the empty shard must gather float32 too
+ "complex64": (torch.complex64, 1e-5),
+}
+SHAPES = ((3, 4), (1, 5), (6, 2), (4,)) # (3, 4) uneven over 2 ranks, (1, 5) empty on rank 1
+
+
+def _grads(dtype, scale):
+ g = torch.Generator().manual_seed(7)
+ out = []
+ for i, shape in enumerate(SHAPES):
+ d = dtype or (torch.bfloat16 if i % 2 else torch.float32)
+ t = torch.randn(*shape, generator = g) * scale
+ if d.is_complex:
+ t = torch.complex(t, torch.randn(*shape, generator = g) * scale)
+ if i == 2:
+ t[0] = 0.0 # a zero row: p = 0 counts tensors, -inf sees a zero
+ out.append(t.to(d))
+ return out
+
+
+def _case(native, rank):
+ from torch.distributed.device_mesh import init_device_mesh
+ from torch.distributed.tensor import DTensor, Shard, distribute_tensor
+
+ mesh = init_device_mesh("cpu", (2,))
+ res = {}
+ for name, (dtype, _) in CASES.items():
+ for scale in (1.0, 200.0) if name == "float16" else (1.0,):
+ for norm_type in NORM_TYPES:
+ full = _grads(dtype, scale)
+ ref = [torch.nn.Parameter(torch.zeros_like(t)) for t in full]
+ for q, t in zip(ref, full):
+ q.grad = t.clone()
+ params = []
+ for i, t in enumerate(full):
+ if i == len(full) - 1: # a replicated (ignored_params) gradient
+ q = torch.nn.Parameter(torch.zeros_like(t))
+ q.grad = t.clone()
+ else:
+ q = torch.nn.Parameter(
+ distribute_tensor(torch.zeros_like(t), mesh, [Shard(0)])
+ )
+ q.grad = distribute_tensor(t.clone(), mesh, [Shard(0)])
+ params.append(q)
+ want = torch.nn.utils.clip_grad_norm_(ref, 0.5, norm_type = norm_type)
+ got = native.native_clip_grad_norm_(params, 0.5, norm_type = norm_type)
+ diff = 0.0
+ for q, r in zip(params, ref):
+ gq = q.grad.full_tensor() if isinstance(q.grad, DTensor) else q.grad
+ scale_r = max(r.grad.abs().max().item(), 1e-30)
+ diff = max(diff, ((gq - r.grad).abs().float().max() / scale_r).item())
+ res[f"{name}|{scale}|{norm_type}"] = {
+ "got": float(got),
+ "want": float(want),
+ "got_dtype": str(got.dtype),
+ "want_dtype": str(want.dtype),
+ "grad_rel_diff": diff,
+ }
+ return res
+
+
+def _worker(rank, world, port, out):
+ os.environ.update(
+ MASTER_ADDR = "127.0.0.1",
+ MASTER_PORT = str(port),
+ RANK = str(rank),
+ WORLD_SIZE = str(world),
+ LOCAL_RANK = str(rank),
+ LOCAL_WORLD_SIZE = str(world),
+ )
+ import datetime
+
+ import torch.distributed as dist
+
+ dist.init_process_group(
+ "gloo", rank = rank, world_size = world, timeout = datetime.timedelta(seconds = 30)
+ )
+ try:
+ out.put((rank, _case(_load_native(), rank)))
+ except Exception as exc:
+ import traceback
+ out.put((rank, {"crash": f"{type(exc).__name__}: {exc}\n{traceback.format_exc()}"}))
+ finally:
+ try:
+ dist.destroy_process_group()
+ except Exception:
+ pass
+
+
+def _run(timeout = 180):
+ import torch.multiprocessing as mp
+
+ ctx = mp.get_context("spawn")
+ out = ctx.Queue()
+ procs = mp.start_processes(
+ _worker, args = (2, _free_port(), out), nprocs = 2, join = False, start_method = "spawn"
+ )
+ deadline = time.time() + timeout
+ try:
+ while not procs.join(timeout = 5):
+ if time.time() > deadline:
+ for p in procs.processes:
+ p.kill()
+ pytest.fail(f"the ranks did not finish within {timeout} s (hung collective)")
+ except mp.ProcessExitedException as exc:
+ pytest.fail(f"a rank died ({exc})")
+ results = {}
+ while not out.empty():
+ rank, r = out.get(timeout = 5)
+ results[rank] = r
+ assert sorted(results) == [0, 1], results
+ for r in results.values():
+ assert "crash" not in r, r["crash"]
+ return results
+
+
[email protected](scope = "module")
+def results():
+ return _run()
+
+
[email protected]("case", list(CASES))
+def test_native_clip_follows_torch_dtype_and_value(results, case):
+ tol = CASES[case][1]
+ for rank, r in results.items():
+ keys = [k for k in r if k.startswith(case + "|")]
+ assert keys, (rank, case)
+ for k in keys:
+ v = r[k]
+ assert v["got_dtype"] == v["want_dtype"], (rank, k, v)
+ assert math.isfinite(v["got"]) == math.isfinite(v["want"]), (rank, k, v)
+ if math.isfinite(v["want"]):
+ assert math.isclose(v["got"], v["want"], rel_tol = tol, abs_tol = 1e-6), (rank, k, v)
+ assert v["grad_rel_diff"] <= 2 * tol, (rank, k, v)
+ assert all(results[0][k] == results[1][k] for k in results[0] if k.startswith(case + "|"))
+
+
+def test_float16_sharded_norm_above_256_stays_finite(results):
+ for rank, r in results.items():
+ v = r["float16|200.0|2.0"]
+ assert v["want"] > 256 and math.isfinite(v["want"]), v
+ assert math.isfinite(v["got"]), (rank, v)
diff --git a/tests/test_fsdp2_native_clip_fast_path_gloo.py b/tests/test_fsdp2_native_clip_fast_path_gloo.py
new file mode 100644
index 00000000000..2efbf03533f
--- /dev/null
+++ b/tests/test_fsdp2_native_clip_fast_path_gloo.py
@@ -0,0 +1,187 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""Batched native_clip_grad_norm_ equals a one-tensor-at-a-time reference on a 2-rank gloo world
+(bitwise, or within one float32 rounding), incl. empty shards and mixed dtypes."""
+
+import multiprocessing
+import os
+
+if multiprocessing.parent_process() is not None or getattr(
+ multiprocessing.current_process(), "_inheriting", False
+):
+ os.environ["CUDA_VISIBLE_DEVICES"] = "" # spawned gloo ranks: CPU only, also on GPU hosts
+
+import math # noqa: E402
+import sys # noqa: E402
+import time # noqa: E402
+
+import pytest # noqa: E402
+import torch # noqa: E402
+
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+from test_fsdp2_native_units import _load_native # noqa: E402
+from test_fsdp2_native_trainer_ckpt_gloo import _free_port # noqa: E402
+
+NORM_TYPES = (0.0, 1.0, 2.0, math.inf)
+
+
+class _Many(torch.nn.Module):
+ """Many small sharded Linears, optionally a 1-row (empty shard) and a float16 one."""
+
+ def __init__(
+ self,
+ empty_shard = False,
+ mixed = False,
+ ):
+ super().__init__()
+ torch.manual_seed(0)
+ self.layers = torch.nn.ModuleList(torch.nn.Linear(6, 6) for _ in range(12))
+ self.one = torch.nn.Linear(6, 1) if empty_shard else None
+ self.lin16 = torch.nn.Linear(6, 4).half() if mixed else None
+
+ def forward(self, x):
+ h = x
+ for layer in self.layers:
+ h = torch.tanh(layer(h))
+ out = h.square().sum()
+ if self.one is not None:
+ out = out + self.one(h).square().sum()
+ if self.lin16 is not None:
+ out = out + self.lin16(h.half()).float().square().sum()
+ return out
+
+
+def _reference(native_dtensor, params, max_norm, p):
+ """The same computation one tensor at a time."""
+ import torch.distributed as dist
+
+ norms = {} # (sharded, dtype) -> norms; order: plain then sharded
+ for q in params:
+ g = q.grad
+ sharded = isinstance(g, native_dtensor)
+ local = g.to_local() if sharded else g
+ if local.numel():
+ n = torch._foreach_norm([local.detach()], p)[0]
+ else:
+ n = torch.tensor(math.inf if p < 0 else 0.0, dtype = local.dtype)
+ if sharded:
+ parts = [torch.empty_like(n) for _ in range(dist.get_world_size())]
+ dist.all_gather(parts, n)
+ n = torch.linalg.vector_norm(torch.stack(parts), p)
+ norms.setdefault((sharded, local.dtype), []).append(n)
+ ordered = [n for key in sorted(norms, key = lambda k: k[0]) for n in norms[key]]
+ total = torch.linalg.vector_norm(torch.stack(ordered), p)
+ coef = torch.clamp(float(max_norm) / (total + 1e-6), max = 1.0)
+ return total, coef
+
+
+def _case(native, empty_shard, mixed):
+ from torch.distributed.device_mesh import init_device_mesh
+ from torch.distributed.fsdp import fully_shard
+ from torch.distributed.tensor import DTensor
+
+ model = _Many(empty_shard = empty_shard, mixed = mixed)
+ mesh = init_device_mesh("cpu", (2,))
+ if model.lin16 is not None:
+ fully_shard(model.lin16, mesh = mesh)
+ fully_shard(model, mesh = mesh)
+ x = torch.randn(3, 6, generator = torch.Generator().manual_seed(1))
+ res = {}
+ for norm_type in NORM_TYPES:
+ model.zero_grad(set_to_none = True)
+ model(x).backward()
+ params = list(model.parameters())
+ want_total, coef = _reference(DTensor, params, 0.05, norm_type)
+ want_grads = [
+ (q.grad.to_local() if isinstance(q.grad, DTensor) else q.grad).clone() for q in params
+ ]
+ got = native.native_clip_grad_norm_(params, 0.05, norm_type = norm_type)
+ grads_equal = all(
+ torch.allclose(
+ q.grad.to_local() if isinstance(q.grad, DTensor) else q.grad,
+ w.clone().mul_(coef) if w.numel() else w,
+ )
+ for q, w in zip(params, want_grads)
+ )
+ res[str(norm_type)] = {
+ "got": float(got),
+ "want": float(want_total),
+ "bitwise": bool(torch.equal(got, want_total)),
+ "close": bool(torch.allclose(got, want_total, rtol = 2**-22, atol = 0)),
+ "grads_equal": grads_equal,
+ "empty_local": any(
+ isinstance(q.grad, DTensor) and q.grad.to_local().numel() == 0 for q in params
+ ),
+ "dtypes": sorted({str(q.grad.dtype) for q in params}),
+ }
+ return res
+
+
+def _worker(rank, world, port, out, case):
+ os.environ.update(
+ MASTER_ADDR = "127.0.0.1",
+ MASTER_PORT = str(port),
+ RANK = str(rank),
+ WORLD_SIZE = str(world),
+ LOCAL_RANK = str(rank),
+ UNSLOTH_FSDP2 = "1",
+ )
+ import datetime
+
+ import torch.distributed as dist
+
+ dist.init_process_group(
+ "gloo", rank = rank, world_size = world, timeout = datetime.timedelta(seconds = 30)
+ )
+ try:
+ native = _load_native()
+ out.put((rank, _case(native, empty_shard = case == "empty", mixed = case == "mixed")))
+ except Exception as exc:
+ import traceback
+ out.put((rank, {"crash": f"{type(exc).__name__}: {exc}\n{traceback.format_exc()}"}))
+ finally:
+ try:
+ dist.destroy_process_group()
+ except Exception:
+ pass
+
+
+def _run(case, timeout = 180):
+ import torch.multiprocessing as mp
+
+ out = mp.get_context("spawn").Queue()
+ procs = mp.start_processes(
+ _worker, args = (2, _free_port(), out, case), nprocs = 2, join = False, start_method = "spawn"
+ )
+ deadline = time.time() + timeout
+ try:
+ while not procs.join(timeout = 5):
+ if time.time() > deadline:
+ for p in procs.processes:
+ p.kill()
+ pytest.fail(f"{case}: the ranks did not finish within {timeout} s")
+ except mp.ProcessExitedException as exc:
+ pytest.fail(f"{case}: a rank died ({exc})")
+ results = {}
+ while not out.empty():
+ rank, r = out.get(timeout = 5)
+ results[rank] = r
+ assert sorted(results) == [0, 1], results
+ for r in results.values():
+ assert "crash" not in r, r["crash"]
+ return results
+
+
[email protected]("case", ["fast", "empty", "mixed"])
+def test_clip_batched_equal_to_per_tensor_norms(case):
+ results = _run(case)
+ for rank, r in results.items():
+ for norm_type in NORM_TYPES:
+ v = r[str(norm_type)]
+ assert v["bitwise"] if case != "empty" else v["close"], (case, rank, norm_type, v)
+ assert v["grads_equal"], (case, rank, norm_type, v)
+ if case == "empty":
+ assert v["empty_local"] == (rank == 1), (rank, v)
+ if case == "mixed":
+ assert v["dtypes"] == ["torch.float16", "torch.float32"], v
diff --git a/unsloth/models/fsdp2_native.py b/unsloth/models/fsdp2_native.py
index 36c0099af9c..8fcb4ba60e5 100644
--- a/unsloth/models/fsdp2_native.py
+++ b/unsloth/models/fsdp2_native.py
@@ -1507,7 +1507,9 @@ def native_clip_grad_norm_(
norm_type = 2.0,
):
"""Global gradient norm over DTensor (sharded) and plain gradients; clips in place and returns a
- plain float32 tensor (the same value on every rank)."""
+ plain tensor (the same value on every rank), matching torch.nn.utils.clip_grad_norm_ dtypes.
+ Shard norms are gathered and combined by vector_norm, not summed as p-th powers: a float16
+ power sum overflows from a norm of 256 where one GPU's vector_norm does not."""
import torch.distributed as dist
try:
@@ -1527,7 +1529,10 @@ def native_clip_grad_norm_(
for p in params:
g = p.grad
if isinstance(g, DTensor):
- local = g.to_local()
+ # to_local() adds an autograd op per call.
+ local = getattr(g, "_local_tensor", None)
+ if local is None:
+ local = g.to_local()
dims = tuple(i for i, pl in enumerate(g.placements) if pl.is_shard())
groups.setdefault((g.device_mesh, dims), []).append(local)
sharded.append(local)
@@ -1544,61 +1549,56 @@ def native_clip_grad_norm_(
p = norm_type
def _per_tensor(grads):
- """One float64 norm per tensor (torch.linalg.vector_norm semantics); an empty local shard
- (uneven sharding: dim 0 smaller than the world) is the identity of the combine. 0 for p >= 0;
- +inf for every negative p (its p-th power is 0, and -inf takes the min), where vector_norm
- itself refuses an empty tensor."""
- if not grads:
- return torch.zeros(0, device = device, dtype = torch.float64)
- empty = math.inf if p < 0 else 0.0
- live = [g.detach() for g in grads if g.numel()]
- try:
- norms = list(torch._foreach_norm(live, p)) if live else []
- except Exception:
- norms = [torch.linalg.vector_norm(g, p) for g in live]
- it = iter(norms)
- return torch.stack(
- [
- next(it).double().to(device)
- if g.numel()
- else torch.tensor(empty, device = device, dtype = torch.float64)
- for g in grads
- ]
- )
-
- # torch.nn.utils.clip_grad_norm_ is vector_norm(stack(per-tensor norms), p): for p = 0 the number
- # of tensors with a nonzero element, for +-inf the max / min, else the p-th root of the sum of
- # p-th powers. A sharded tensor's norm combines over ranks first (counts and powers add up).
- replicated_norms = _per_tensor(replicated)
-
- def _over_shards(value, op, mesh, dims):
- if initialized:
- for dim in dims:
- dist.all_reduce(value, op = op, group = mesh.get_group(dim))
- return value
-
- if p == 0:
- total = (replicated_norms > 0).sum().double()
- for (mesh, dims), grads in groups.items():
- counts = _over_shards(_per_tensor(grads), dist.ReduceOp.SUM, mesh, dims)
- total = total + (counts > 0).sum().double()
- elif p in (math.inf, -math.inf):
- pick, op = (torch.max, dist.ReduceOp.MAX) if p > 0 else (torch.min, dist.ReduceOp.MIN)
- total = torch.tensor(0.0 if p > 0 else math.inf, device = device, dtype = torch.float64)
- parts = [pick(replicated_norms)] if replicated_norms.numel() else []
- for (mesh, dims), grads in groups.items():
- parts.append(_over_shards(pick(_per_tensor(grads)), op, mesh, dims))
- if parts:
- total = pick(torch.stack(parts))
- else:
- total = replicated_norms.pow(p).sum()
- for (mesh, dims), grads in groups.items():
- powers = _per_tensor(grads).pow(p).sum()
- total = total + _over_shards(powers, dist.ReduceOp.SUM, mesh, dims)
- total = total.pow(1.0 / p)
+ """Per-tensor norms in each gradient's dtype, one 1-D tensor per dtype. An empty local shard
+ gets the combine identity (0, or +inf for p < 0); vector_norm refuses empty tensors."""
+ by_dtype = {}
+ for i, g in enumerate(grads):
+ by_dtype.setdefault(g.dtype, []).append(i)
+ chunks = []
+ for dtype, idx in by_dtype.items():
+ live = [k for k, i in enumerate(idx) if grads[i].numel()]
+ norms = []
+ if live:
+ tensors = [grads[idx[k]].detach() for k in live]
+ try:
+ norms = list(torch._foreach_norm(tensors, p))
+ except Exception:
+ norms = [torch.linalg.vector_norm(g, p) for g in tensors]
+ if len({n.device for n in norms}) > 1:
+ norms = [n.to(device) for n in norms]
+ if len(live) == len(idx):
+ values = torch.stack(norms).to(device)
+ else:
+ # complex grads give real norms; every rank must gather the same dtype.
+ norm_dtype = dtype.to_real() if dtype.is_complex else dtype
+ values = torch.full(
+ (len(idx),), math.inf if p < 0 else 0.0, device = device, dtype = norm_dtype
+ )
+ if live:
+ values[torch.tensor(live, device = device)] = torch.stack(norms).to(device)
+ chunks.append(values)
+ return chunks
+
+ def _over_shards(chunks, mesh, dims):
+ """Full-tensor norms from local shard norms (all-gather per shard dim, then vector_norm)."""
+ out = []
+ for values in chunks:
+ rows = values.unsqueeze(0)
+ if initialized:
+ for dim in dims:
+ group = mesh.get_group(dim)
+ parts = [torch.empty_like(rows) for _ in range(dist.get_world_size(group))]
+ dist.all_gather(parts, rows, group = group)
+ rows = torch.cat(parts, dim = 0)
+ out.append(torch.linalg.vector_norm(rows, p, dim = 0))
+ return out
+
+ norms = _per_tensor(replicated)
+ for (mesh, dims), grads in groups.items():
+ norms.extend(_over_shards(_per_tensor(grads), mesh, dims))
+ total = torch.linalg.vector_norm(torch.cat(norms), p)
clip_coef = float(max_norm) / (total + 1e-6)
- clip_coef = torch.clamp(clip_coef, max = 1.0).float()
- total = total.float()
+ clip_coef = torch.clamp(clip_coef, max = 1.0)
grads = sharded + replicated
if grads:
by_device = {}