Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Native FSDP2: faster native_clip_grad_norm_ that follows torch clip_grad_norm_ dtypes (#14973)
Daniel Han9df97d910c42
diff --git a/tests/test_fsdp2_native_clip_dtype_gloo.py b/tests/test_fsdp2_native_clip_dtype_gloo.pynew file mode 100644index 00000000000..8417c906471--- /dev/null+++ b/tests/test_fsdp2_native_clip_dtype_gloo.py@@ -0,0 +1,174 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""native_clip_grad_norm_ matches torch.nn.utils.clip_grad_norm_ in dtype and value on a 2-rank gloo+world; a float16 sharded norm above 256 stays finite."""++import multiprocessing+import os++if multiprocessing.parent_process() is not None or getattr(+ multiprocessing.current_process(), "_inheriting", False+):+ os.environ["CUDA_VISIBLE_DEVICES"] = "" # spawned gloo ranks: CPU only, also on GPU hosts++import math # noqa: E402+import sys # noqa: E402+import time # noqa: E402++import pytest # noqa: E402+import torch # noqa: E402++sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))+from test_fsdp2_native_units import _load_native # noqa: E402+from test_fsdp2_native_trainer_ckpt_gloo import _free_port # noqa: E402++NORM_TYPES = (0.0, 1.0, 2.0, 3.0, math.inf, -math.inf)+# 16-bit: shard norm + combine round once more than torch's single norm+CASES = {+ "float32": (torch.float32, 1e-5),+ "bfloat16": (torch.bfloat16, 2**-6),+ "float16": (torch.float16, 2**-8),+ "mixed": (None, 1e-5),+ # complex64 norms are float32: the rank with the empty shard must gather float32 too+ "complex64": (torch.complex64, 1e-5),+}+SHAPES = ((3, 4), (1, 5), (6, 2), (4,)) # (3, 4) uneven over 2 ranks, (1, 5) empty on rank 1+++def _grads(dtype, scale):+ g = torch.Generator().manual_seed(7)+ out = []+ for i, shape in enumerate(SHAPES):+ d = dtype or (torch.bfloat16 if i % 2 else torch.float32)+ t = torch.randn(*shape, generator = g) * scale+ if d.is_complex:+ t = torch.complex(t, torch.randn(*shape, generator = g) * scale)+ if i == 2:+ t[0] = 0.0 # a zero row: p = 0 counts tensors, -inf sees a zero+ out.append(t.to(d))+ return out+++def _case(native, rank):+ from torch.distributed.device_mesh import init_device_mesh+ from torch.distributed.tensor import DTensor, Shard, distribute_tensor++ mesh = init_device_mesh("cpu", (2,))+ res = {}+ for name, (dtype, _) in CASES.items():+ for scale in (1.0, 200.0) if name == "float16" else (1.0,):+ for norm_type in NORM_TYPES:+ full = _grads(dtype, scale)+ ref = [torch.nn.Parameter(torch.zeros_like(t)) for t in full]+ for q, t in zip(ref, full):+ q.grad = t.clone()+ params = []+ for i, t in enumerate(full):+ if i == len(full) - 1: # a replicated (ignored_params) gradient+ q = torch.nn.Parameter(torch.zeros_like(t))+ q.grad = t.clone()+ else:+ q = torch.nn.Parameter(+ distribute_tensor(torch.zeros_like(t), mesh, [Shard(0)])+ )+ q.grad = distribute_tensor(t.clone(), mesh, [Shard(0)])+ params.append(q)+ want = torch.nn.utils.clip_grad_norm_(ref, 0.5, norm_type = norm_type)+ got = native.native_clip_grad_norm_(params, 0.5, norm_type = norm_type)+ diff = 0.0+ for q, r in zip(params, ref):+ gq = q.grad.full_tensor() if isinstance(q.grad, DTensor) else q.grad+ scale_r = max(r.grad.abs().max().item(), 1e-30)+ diff = max(diff, ((gq - r.grad).abs().float().max() / scale_r).item())+ res[f"{name}|{scale}|{norm_type}"] = {+ "got": float(got),+ "want": float(want),+ "got_dtype": str(got.dtype),+ "want_dtype": str(want.dtype),+ "grad_rel_diff": diff,+ }+ return res+++def _worker(rank, world, port, out):+ os.environ.update(+ MASTER_ADDR = "127.0.0.1",+ MASTER_PORT = str(port),+ RANK = str(rank),+ WORLD_SIZE = str(world),+ LOCAL_RANK = str(rank),+ LOCAL_WORLD_SIZE = str(world),+ )+ import datetime++ import torch.distributed as dist++ dist.init_process_group(+ "gloo", rank = rank, world_size = world, timeout = datetime.timedelta(seconds = 30)+ )+ try:+ out.put((rank, _case(_load_native(), rank)))+ except Exception as exc:+ import traceback+ out.put((rank, {"crash": f"{type(exc).__name__}: {exc}\n{traceback.format_exc()}"}))+ finally:+ try:+ dist.destroy_process_group()+ except Exception:+ pass+++def _run(timeout = 180):+ import torch.multiprocessing as mp++ ctx = mp.get_context("spawn")+ out = ctx.Queue()+ procs = mp.start_processes(+ _worker, args = (2, _free_port(), out), nprocs = 2, join = False, start_method = "spawn"+ )+ deadline = time.time() + timeout+ try:+ while not procs.join(timeout = 5):+ if time.time() > deadline:+ for p in procs.processes:+ p.kill()+ pytest.fail(f"the ranks did not finish within {timeout} s (hung collective)")+ except mp.ProcessExitedException as exc:+ pytest.fail(f"a rank died ({exc})")+ results = {}+ while not out.empty():+ rank, r = out.get(timeout = 5)+ results[rank] = r+ assert sorted(results) == [0, 1], results+ for r in results.values():+ assert "crash" not in r, r["crash"]+ return results++[email protected](scope = "module")+def results():+ return _run()++[email protected]("case", list(CASES))+def test_native_clip_follows_torch_dtype_and_value(results, case):+ tol = CASES[case][1]+ for rank, r in results.items():+ keys = [k for k in r if k.startswith(case + "|")]+ assert keys, (rank, case)+ for k in keys:+ v = r[k]+ assert v["got_dtype"] == v["want_dtype"], (rank, k, v)+ assert math.isfinite(v["got"]) == math.isfinite(v["want"]), (rank, k, v)+ if math.isfinite(v["want"]):+ assert math.isclose(v["got"], v["want"], rel_tol = tol, abs_tol = 1e-6), (rank, k, v)+ assert v["grad_rel_diff"] <= 2 * tol, (rank, k, v)+ assert all(results[0][k] == results[1][k] for k in results[0] if k.startswith(case + "|"))+++def test_float16_sharded_norm_above_256_stays_finite(results):+ for rank, r in results.items():+ v = r["float16|200.0|2.0"]+ assert v["want"] > 256 and math.isfinite(v["want"]), v+ assert math.isfinite(v["got"]), (rank, v)diff --git a/tests/test_fsdp2_native_clip_fast_path_gloo.py b/tests/test_fsdp2_native_clip_fast_path_gloo.pynew file mode 100644index 00000000000..2efbf03533f--- /dev/null+++ b/tests/test_fsdp2_native_clip_fast_path_gloo.py@@ -0,0 +1,187 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""Batched native_clip_grad_norm_ equals a one-tensor-at-a-time reference on a 2-rank gloo world+(bitwise, or within one float32 rounding), incl. empty shards and mixed dtypes."""++import multiprocessing+import os++if multiprocessing.parent_process() is not None or getattr(+ multiprocessing.current_process(), "_inheriting", False+):+ os.environ["CUDA_VISIBLE_DEVICES"] = "" # spawned gloo ranks: CPU only, also on GPU hosts++import math # noqa: E402+import sys # noqa: E402+import time # noqa: E402++import pytest # noqa: E402+import torch # noqa: E402++sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))+from test_fsdp2_native_units import _load_native # noqa: E402+from test_fsdp2_native_trainer_ckpt_gloo import _free_port # noqa: E402++NORM_TYPES = (0.0, 1.0, 2.0, math.inf)+++class _Many(torch.nn.Module):+ """Many small sharded Linears, optionally a 1-row (empty shard) and a float16 one."""++ def __init__(+ self,+ empty_shard = False,+ mixed = False,+ ):+ super().__init__()+ torch.manual_seed(0)+ self.layers = torch.nn.ModuleList(torch.nn.Linear(6, 6) for _ in range(12))+ self.one = torch.nn.Linear(6, 1) if empty_shard else None+ self.lin16 = torch.nn.Linear(6, 4).half() if mixed else None++ def forward(self, x):+ h = x+ for layer in self.layers:+ h = torch.tanh(layer(h))+ out = h.square().sum()+ if self.one is not None:+ out = out + self.one(h).square().sum()+ if self.lin16 is not None:+ out = out + self.lin16(h.half()).float().square().sum()+ return out+++def _reference(native_dtensor, params, max_norm, p):+ """The same computation one tensor at a time."""+ import torch.distributed as dist++ norms = {} # (sharded, dtype) -> norms; order: plain then sharded+ for q in params:+ g = q.grad+ sharded = isinstance(g, native_dtensor)+ local = g.to_local() if sharded else g+ if local.numel():+ n = torch._foreach_norm([local.detach()], p)[0]+ else:+ n = torch.tensor(math.inf if p < 0 else 0.0, dtype = local.dtype)+ if sharded:+ parts = [torch.empty_like(n) for _ in range(dist.get_world_size())]+ dist.all_gather(parts, n)+ n = torch.linalg.vector_norm(torch.stack(parts), p)+ norms.setdefault((sharded, local.dtype), []).append(n)+ ordered = [n for key in sorted(norms, key = lambda k: k[0]) for n in norms[key]]+ total = torch.linalg.vector_norm(torch.stack(ordered), p)+ coef = torch.clamp(float(max_norm) / (total + 1e-6), max = 1.0)+ return total, coef+++def _case(native, empty_shard, mixed):+ from torch.distributed.device_mesh import init_device_mesh+ from torch.distributed.fsdp import fully_shard+ from torch.distributed.tensor import DTensor++ model = _Many(empty_shard = empty_shard, mixed = mixed)+ mesh = init_device_mesh("cpu", (2,))+ if model.lin16 is not None:+ fully_shard(model.lin16, mesh = mesh)+ fully_shard(model, mesh = mesh)+ x = torch.randn(3, 6, generator = torch.Generator().manual_seed(1))+ res = {}+ for norm_type in NORM_TYPES:+ model.zero_grad(set_to_none = True)+ model(x).backward()+ params = list(model.parameters())+ want_total, coef = _reference(DTensor, params, 0.05, norm_type)+ want_grads = [+ (q.grad.to_local() if isinstance(q.grad, DTensor) else q.grad).clone() for q in params+ ]+ got = native.native_clip_grad_norm_(params, 0.05, norm_type = norm_type)+ grads_equal = all(+ torch.allclose(+ q.grad.to_local() if isinstance(q.grad, DTensor) else q.grad,+ w.clone().mul_(coef) if w.numel() else w,+ )+ for q, w in zip(params, want_grads)+ )+ res[str(norm_type)] = {+ "got": float(got),+ "want": float(want_total),+ "bitwise": bool(torch.equal(got, want_total)),+ "close": bool(torch.allclose(got, want_total, rtol = 2**-22, atol = 0)),+ "grads_equal": grads_equal,+ "empty_local": any(+ isinstance(q.grad, DTensor) and q.grad.to_local().numel() == 0 for q in params+ ),+ "dtypes": sorted({str(q.grad.dtype) for q in params}),+ }+ return res+++def _worker(rank, world, port, out, case):+ os.environ.update(+ MASTER_ADDR = "127.0.0.1",+ MASTER_PORT = str(port),+ RANK = str(rank),+ WORLD_SIZE = str(world),+ LOCAL_RANK = str(rank),+ UNSLOTH_FSDP2 = "1",+ )+ import datetime++ import torch.distributed as dist++ dist.init_process_group(+ "gloo", rank = rank, world_size = world, timeout = datetime.timedelta(seconds = 30)+ )+ try:+ native = _load_native()+ out.put((rank, _case(native, empty_shard = case == "empty", mixed = case == "mixed")))+ except Exception as exc:+ import traceback+ out.put((rank, {"crash": f"{type(exc).__name__}: {exc}\n{traceback.format_exc()}"}))+ finally:+ try:+ dist.destroy_process_group()+ except Exception:+ pass+++def _run(case, timeout = 180):+ import torch.multiprocessing as mp++ out = mp.get_context("spawn").Queue()+ procs = mp.start_processes(+ _worker, args = (2, _free_port(), out, case), nprocs = 2, join = False, start_method = "spawn"+ )+ deadline = time.time() + timeout+ try:+ while not procs.join(timeout = 5):+ if time.time() > deadline:+ for p in procs.processes:+ p.kill()+ pytest.fail(f"{case}: the ranks did not finish within {timeout} s")+ except mp.ProcessExitedException as exc:+ pytest.fail(f"{case}: a rank died ({exc})")+ results = {}+ while not out.empty():+ rank, r = out.get(timeout = 5)+ results[rank] = r+ assert sorted(results) == [0, 1], results+ for r in results.values():+ assert "crash" not in r, r["crash"]+ return results++[email protected]("case", ["fast", "empty", "mixed"])+def test_clip_batched_equal_to_per_tensor_norms(case):+ results = _run(case)+ for rank, r in results.items():+ for norm_type in NORM_TYPES:+ v = r[str(norm_type)]+ assert v["bitwise"] if case != "empty" else v["close"], (case, rank, norm_type, v)+ assert v["grads_equal"], (case, rank, norm_type, v)+ if case == "empty":+ assert v["empty_local"] == (rank == 1), (rank, v)+ if case == "mixed":+ assert v["dtypes"] == ["torch.float16", "torch.float32"], vdiff --git a/unsloth/models/fsdp2_native.py b/unsloth/models/fsdp2_native.pyindex 36c0099af9c..8fcb4ba60e5 100644--- a/unsloth/models/fsdp2_native.py+++ b/unsloth/models/fsdp2_native.py@@ -1507,7 +1507,9 @@ def native_clip_grad_norm_(norm_type = 2.0,):"""Global gradient norm over DTensor (sharded) and plain gradients; clips in place and returns a- plain float32 tensor (the same value on every rank)."""+ plain tensor (the same value on every rank), matching torch.nn.utils.clip_grad_norm_ dtypes.+ Shard norms are gathered and combined by vector_norm, not summed as p-th powers: a float16+ power sum overflows from a norm of 256 where one GPU's vector_norm does not."""import torch.distributed as disttry:@@ -1527,7 +1529,10 @@ def native_clip_grad_norm_(for p in params:g = p.gradif isinstance(g, DTensor):- local = g.to_local()+ # to_local() adds an autograd op per call.+ local = getattr(g, "_local_tensor", None)+ if local is None:+ local = g.to_local()dims = tuple(i for i, pl in enumerate(g.placements) if pl.is_shard())groups.setdefault((g.device_mesh, dims), []).append(local)sharded.append(local)@@ -1544,61 +1549,56 @@ def native_clip_grad_norm_(p = norm_typedef _per_tensor(grads):- """One float64 norm per tensor (torch.linalg.vector_norm semantics); an empty local shard- (uneven sharding: dim 0 smaller than the world) is the identity of the combine. 0 for p >= 0;- +inf for every negative p (its p-th power is 0, and -inf takes the min), where vector_norm- itself refuses an empty tensor."""- if not grads:- return torch.zeros(0, device = device, dtype = torch.float64)- empty = math.inf if p < 0 else 0.0- live = [g.detach() for g in grads if g.numel()]- try:- norms = list(torch._foreach_norm(live, p)) if live else []- except Exception:- norms = [torch.linalg.vector_norm(g, p) for g in live]- it = iter(norms)- return torch.stack(- [- next(it).double().to(device)- if g.numel()- else torch.tensor(empty, device = device, dtype = torch.float64)- for g in grads- ]- )-- # torch.nn.utils.clip_grad_norm_ is vector_norm(stack(per-tensor norms), p): for p = 0 the number- # of tensors with a nonzero element, for +-inf the max / min, else the p-th root of the sum of- # p-th powers. A sharded tensor's norm combines over ranks first (counts and powers add up).- replicated_norms = _per_tensor(replicated)-- def _over_shards(value, op, mesh, dims):- if initialized:- for dim in dims:- dist.all_reduce(value, op = op, group = mesh.get_group(dim))- return value-- if p == 0:- total = (replicated_norms > 0).sum().double()- for (mesh, dims), grads in groups.items():- counts = _over_shards(_per_tensor(grads), dist.ReduceOp.SUM, mesh, dims)- total = total + (counts > 0).sum().double()- elif p in (math.inf, -math.inf):- pick, op = (torch.max, dist.ReduceOp.MAX) if p > 0 else (torch.min, dist.ReduceOp.MIN)- total = torch.tensor(0.0 if p > 0 else math.inf, device = device, dtype = torch.float64)- parts = [pick(replicated_norms)] if replicated_norms.numel() else []- for (mesh, dims), grads in groups.items():- parts.append(_over_shards(pick(_per_tensor(grads)), op, mesh, dims))- if parts:- total = pick(torch.stack(parts))- else:- total = replicated_norms.pow(p).sum()- for (mesh, dims), grads in groups.items():- powers = _per_tensor(grads).pow(p).sum()- total = total + _over_shards(powers, dist.ReduceOp.SUM, mesh, dims)- total = total.pow(1.0 / p)+ """Per-tensor norms in each gradient's dtype, one 1-D tensor per dtype. An empty local shard+ gets the combine identity (0, or +inf for p < 0); vector_norm refuses empty tensors."""+ by_dtype = {}+ for i, g in enumerate(grads):+ by_dtype.setdefault(g.dtype, []).append(i)+ chunks = []+ for dtype, idx in by_dtype.items():+ live = [k for k, i in enumerate(idx) if grads[i].numel()]+ norms = []+ if live:+ tensors = [grads[idx[k]].detach() for k in live]+ try:+ norms = list(torch._foreach_norm(tensors, p))+ except Exception:+ norms = [torch.linalg.vector_norm(g, p) for g in tensors]+ if len({n.device for n in norms}) > 1:+ norms = [n.to(device) for n in norms]+ if len(live) == len(idx):+ values = torch.stack(norms).to(device)+ else:+ # complex grads give real norms; every rank must gather the same dtype.+ norm_dtype = dtype.to_real() if dtype.is_complex else dtype+ values = torch.full(+ (len(idx),), math.inf if p < 0 else 0.0, device = device, dtype = norm_dtype+ )+ if live:+ values[torch.tensor(live, device = device)] = torch.stack(norms).to(device)+ chunks.append(values)+ return chunks++ def _over_shards(chunks, mesh, dims):+ """Full-tensor norms from local shard norms (all-gather per shard dim, then vector_norm)."""+ out = []+ for values in chunks:+ rows = values.unsqueeze(0)+ if initialized:+ for dim in dims:+ group = mesh.get_group(dim)+ parts = [torch.empty_like(rows) for _ in range(dist.get_world_size(group))]+ dist.all_gather(parts, rows, group = group)+ rows = torch.cat(parts, dim = 0)+ out.append(torch.linalg.vector_norm(rows, p, dim = 0))+ return out++ norms = _per_tensor(replicated)+ for (mesh, dims), grads in groups.items():+ norms.extend(_over_shards(_per_tensor(grads), mesh, dims))+ total = torch.linalg.vector_norm(torch.cat(norms), p)clip_coef = float(max_norm) / (total + 1e-6)- clip_coef = torch.clamp(clip_coef, max = 1.0).float()- total = total.float()+ clip_coef = torch.clamp(clip_coef, max = 1.0)grads = sharded + replicatedif grads:by_device = {}