Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Tensor parallel float16: unscale DTensor and plain grads apart in the native ShardedGradScaler (#15551)
Daniel Handef336453f71
diff --git a/tests/test_tp_scaler_mixed_dtensor_grads_gloo.py b/tests/test_tp_scaler_mixed_dtensor_grads_gloo.pynew file mode 100644index 00000000000..e8696fe6ceb--- /dev/null+++ b/tests/test_tp_scaler_mixed_dtensor_grads_gloo.py@@ -0,0 +1,161 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""float16 TP with DTensor and plain grads in one optimizer (Qwen3-VL, UNSLOTH_TP_SIZE=2) on a gloo CPU world of 2."""++import multiprocessing+import os++if multiprocessing.parent_process() is not None or getattr(+ multiprocessing.current_process(), "_inheriting", False+):+ os.environ["CUDA_VISIBLE_DEVICES"] = "" # spawned gloo ranks: CPU only, also on GPU hosts++import sys # noqa: E402+import types # noqa: E402++import pytest # noqa: E402+import torch # noqa: E402++sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))+from test_fsdp2_native_units import _load_native # noqa: E402+from test_fsdp2_native_trainer_ckpt_gloo import _free_port # noqa: E402++_SCALE = 2.0**8+++class _Handler:+ def to_kwargs(self):+ return {"device": "cpu", "init_scale": _SCALE}+++def _run(optim, inf_rank = None):+ from torch.distributed.device_mesh import init_device_mesh+ from torch.distributed.tensor import Shard, distribute_tensor++ native = _load_native()+ rank = torch.distributed.get_rank()+ trainer = types.SimpleNamespace(+ accelerator = types.SimpleNamespace(+ scaler = torch.amp.GradScaler("cpu", init_scale = _SCALE), scaler_handler = _Handler()+ )+ )+ assert native._scaler_for_native(trainer) is True+ scaler = trainer.accelerator.scaler+ mesh = init_device_mesh("cpu", (2,), mesh_dim_names = ("tp",))+ torch.manual_seed(0)+ sharded = torch.nn.Parameter(distribute_tensor(torch.randn(8, 4), mesh, [Shard(0)]))+ plain = torch.nn.Parameter(torch.randn(4, 4))+ if optim == "adamw":+ opt = torch.optim.AdamW([sharded, plain], lr = 1e-2, foreach = False)+ else:+ opt = torch.optim.SGD([sharded, plain], lr = 1e-2, momentum = 0.9, foreach = False)+ out = {"unscaled": None}+ for step in range(3):+ loss = sharded.to_local().pow(2).sum() + plain.pow(2).sum()+ scaler.scale(loss).backward()+ if step == 1 and inf_rank in (rank, "all"):+ plain.grad[0, 0] = float("inf")+ scaler.unscale_(opt)+ if step == 0:+ out["unscaled"] = (+ torch.allclose(sharded.grad.to_local(), 2 * sharded.detach().to_local()),+ torch.allclose(plain.grad, 2 * plain.detach()),+ )+ scaler.step(opt)+ scaler.update()+ opt.zero_grad(set_to_none = True)+ out["steps"] = (+ (float(opt.state[sharded]["step"]), float(opt.state[plain]["step"]))+ if optim == "adamw"+ else None+ )+ out["params"] = [sharded.detach().to_local().clone(), plain.detach().clone()]+ return out+++def _sparse_fp16():+ """A float16 sparse plain grad (nn.Embedding(sparse = True)) beside a DTensor grad: upstream coalesces+ it and assigns the result back to param.grad, so that assignment has to reach the real parameter."""+ from torch.distributed.device_mesh import init_device_mesh+ from torch.distributed.tensor import Shard, distribute_tensor++ native = _load_native()+ trainer = types.SimpleNamespace(+ accelerator = types.SimpleNamespace(+ scaler = torch.amp.GradScaler("cpu", init_scale = _SCALE), scaler_handler = _Handler()+ )+ )+ assert native._scaler_for_native(trainer) is True+ scaler = trainer.accelerator.scaler+ mesh = init_device_mesh("cpu", (2,), mesh_dim_names = ("tp",))+ sharded = torch.nn.Parameter(distribute_tensor(torch.zeros(8, 4), mesh, [Shard(0)]))+ embed = torch.nn.Parameter(torch.zeros(4, 2, dtype = torch.float16))+ opt = torch.optim.SGD([sharded, embed], lr = 1e-2)+ scaler.scale(torch.zeros(()))+ sharded.grad = distribute_tensor(torch.full((8, 4), _SCALE), mesh, [Shard(0)])+ # rows 0, 0, 2: uncoalesced, row 0 sums to 2 * _SCALE+ embed.grad = torch.sparse_coo_tensor(+ torch.tensor([[0, 0, 2]]), torch.full((3, 2), _SCALE, dtype = torch.float16), (4, 2)+ )+ scaler.unscale_(opt)+ return sharded.grad.to_local().clone(), embed.grad.to_dense().float().clone()+++def _worker(rank, world, port, out):+ os.environ.update(MASTER_ADDR = "127.0.0.1", MASTER_PORT = str(port))+ torch.distributed.init_process_group("gloo", rank = rank, world_size = world)+ try:+ res = {}+ for optim in ("adamw", "sgd"):+ for inf_rank in (None, 1, "all"):+ try:+ res[optim, inf_rank] = _run(optim, inf_rank)+ except Exception as e:+ res[optim, inf_rank] = f"{type(e).__name__}: {e}"+ try:+ res["sparse_fp16"] = _sparse_fp16()+ except Exception as e:+ res["sparse_fp16"] = f"{type(e).__name__}: {e}"+ torch.save(res, os.path.join(out, f"rank{rank}.pt"))+ finally:+ torch.distributed.destroy_process_group()++[email protected](scope = "module")+def ranks(tmp_path_factory):+ out = str(tmp_path_factory.mktemp("tp_scaler"))+ torch.multiprocessing.spawn(_worker, args = (2, _free_port(), out), nprocs = 2, join = True)+ return [torch.load(os.path.join(out, f"rank{r}.pt")) for r in range(2)]++[email protected]("optim", ["adamw", "sgd"])+ "inf_rank", [None, 1, "all"], ids = ["no_inf", "one_rank_inf", "every_rank_inf"]+)+def test_mixed_dtensor_and_plain_grads_unscale_and_skip(ranks, optim, inf_rank):+ for res in ranks:+ got = res[optim, inf_rank]+ assert not isinstance(got, str), got+ assert got["unscaled"] == (True, True), got["unscaled"]+ if optim == "adamw":+ expected = (+ 3.0 if inf_rank is None else 2.0+ ) # an overflow on any rank skips it on every rank+ assert got["steps"] == (expected, expected), got["steps"]+ assert all(torch.isfinite(p).all() for p in got["params"])+ torch.testing.assert_close(+ ranks[0][optim, inf_rank]["params"][1], ranks[1][optim, inf_rank]["params"][1]+ )+++def test_mixed_grads_unscale_float16_sparse_plain_grad(ranks):+ for res in ranks:+ got = res["sparse_fp16"]+ assert not isinstance(got, str), got+ dense, sparse = got+ assert torch.equal(dense, torch.ones(4, 4)), dense+ # unscaled on the parameter itself, not on a stand-in's copy+ assert torch.equal(+ sparse, torch.tensor([[2.0, 2.0], [0.0, 0.0], [1.0, 1.0], [0.0, 0.0]])+ ), sparsediff --git a/unsloth/models/fsdp2_native.py b/unsloth/models/fsdp2_native.pyindex a5ad76d4a90..329ef399ecf 100644--- a/unsloth/models/fsdp2_native.py+++ b/unsloth/models/fsdp2_native.py@@ -23,6 +23,7 @@ import functoolsimport mathimport osimport sys+import typesfrom typing import Callable, Optional, Unionimport torch@@ -3346,6 +3347,31 @@ def _has_cpu_tensors(optimizer, grads = False):return False+def _local_grads_if_mixed(optimizer):+ """Swap DTensor grads for their local shards when DTensor and plain grads mix: one foreach unscale+ takes one tensor kind ("got mixed torch.Tensor and DTensor"). Elementwise, so the shard result is the same."""+ try:+ from torch.distributed.tensor import DTensor+ except ImportError:+ return optimizer+ groups, dtensor, plain = [], False, False+ for group in optimizer.param_groups:+ params = []+ for p in group["params"]:+ grad = p.grad+ if isinstance(grad, DTensor):+ dtensor = True+ p = types.SimpleNamespace(grad = grad._local_tensor)+ elif grad is not None:+ plain = True+ # Plain grads keep their parameter: upstream reassigns a coalesced float16 sparse grad to it.+ params.append(p)+ groups.append({"params": params})+ if not (dtensor and plain):+ return optimizer+ return types.SimpleNamespace(param_groups = groups)++def _scaler_for_native(trainer):"""float16: the GradScaler's inf check must agree over ranks, or one rank skips a step theothers take. ShardedGradScaler all-reduces it."""@@ -3399,7 +3425,9 @@ def _scaler_for_native(trainer):# CPU grads get non_blocking copies of CUDA inv_scale / found_inf, read before they land.if inv_scale.device.type != "cpu" and _has_cpu_tensors(optimizer, grads = True):inv_scale, found_inf = inv_scale.cpu(), found_inf.cpu()- return super()._unscale_grads_(optimizer, inv_scale, found_inf, allow_fp16)+ return super()._unscale_grads_(+ _local_grads_if_mixed(optimizer), inv_scale, found_inf, allow_fp16+ )def step(self, optimizer, *args, **kwargs):# Same race for a fused optimizer's grad_scale / found_inf on CPU parameters: use host copies.