Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

FSDP2 float16 with CPU offload: unscale CPU DTensor grads as local shards (#16398)

Daniel Hana098d99a54da
diff --git a/tests/test_fsdp2_native_scaler_cpu_offload_dtensor.py b/tests/test_fsdp2_native_scaler_cpu_offload_dtensor.py
new file mode 100644
index 00000000000..9285becf0d0
--- /dev/null
+++ b/tests/test_fsdp2_native_scaler_cpu_offload_dtensor.py
@@ -0,0 +1,75 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""CPUOffloadPolicy fp16: DTensor grads live on the CPU, and DTensor's found_inf reduce is a CPU collective a
+NCCL-only group cannot run (2 x T4 failed with "No backend type associated with device type cpu"). The scaler must
+unscale local shards and leave found_inf agreement to ShardedGradScaler."""
+
+import os
+import sys
+import types
+
+import pytest
+import torch
+
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+from test_fsdp2_native_units import _load_native # noqa: E402
+from test_fsdp2_native_trainer_ckpt_gloo import _free_port # noqa: E402
+
+
+class _Handler:
+ def to_kwargs(self):
+ return {"device": "cpu", "init_scale": 2.0**8}
+
+
[email protected](scope = "module")
+def cpu_mesh():
+ import torch.distributed as dist
+ from torch.distributed.device_mesh import init_device_mesh
+
+ os.environ.update(MASTER_ADDR = "127.0.0.1", MASTER_PORT = str(_free_port()))
+ dist.init_process_group("gloo", rank = 0, world_size = 1)
+ try:
+ yield init_device_mesh("cpu", (1,))
+ finally:
+ dist.destroy_process_group()
+
+
[email protected]("overflow", [False, True], ids = ["finite", "inf"])
+def test_cpu_dtensor_grads_unscale_as_local_shards(cpu_mesh, monkeypatch, overflow):
+ from torch.distributed.fsdp.sharded_grad_scaler import ShardedGradScaler
+ from torch.distributed.tensor import DTensor, Shard, distribute_tensor
+
+ native = _load_native()
+ old = torch.amp.GradScaler("cpu", init_scale = 2.0**8)
+ trainer = types.SimpleNamespace(
+ accelerator = types.SimpleNamespace(scaler = old, scaler_handler = _Handler())
+ )
+ assert native._scaler_for_native(trainer) is True
+ scaler = trainer.accelerator.scaler
+ torch.manual_seed(0)
+ p = torch.nn.Parameter(distribute_tensor(torch.randn(8, 8), cpu_mesh, [Shard(0)]))
+ grad = torch.randn(8, 8) * 256.0
+ if overflow:
+ grad[0, 0] = float("inf")
+ p.grad = distribute_tensor(grad, cpu_mesh, [Shard(0)])
+ opt = torch.optim.AdamW([p], lr = 1e-2, foreach = True)
+ seen = []
+ parent = ShardedGradScaler._unscale_grads_
+
+ def recording(self, optimizer, *a, **k):
+ seen.extend(
+ type(q.grad) for g in optimizer.param_groups for q in g["params"] if q.grad is not None
+ )
+ return parent(self, optimizer, *a, **k)
+
+ monkeypatch.setattr(ShardedGradScaler, "_unscale_grads_", recording)
+ scaler.scale(torch.ones(())) # initializes the scale; the grads above stand for scaled ones
+ scaler.unscale_(opt)
+ assert seen and not any(issubclass(t, DTensor) for t in seen), seen
+ assert isinstance(p.grad, DTensor)
+ state = scaler._per_optimizer_states[id(opt)]
+ found = sum(float(v.sum()) for v in state["found_inf_per_device"].values())
+ assert found == (1.0 if overflow else 0.0)
+ if not overflow:
+ torch.testing.assert_close(p.grad.full_tensor(), grad / 256.0)
diff --git a/unsloth/models/fsdp2_native.py b/unsloth/models/fsdp2_native.py
index 144299baf62..378360f4559 100644
--- a/unsloth/models/fsdp2_native.py
+++ b/unsloth/models/fsdp2_native.py
@@ -3492,25 +3492,28 @@ def _has_cpu_tensors(optimizer, grads = False):
def _local_grads_if_mixed(optimizer):
"""Swap DTensor grads for their local shards when DTensor and plain grads mix: one foreach unscale
- takes one tensor kind ("got mixed torch.Tensor and DTensor"). Elementwise, so the shard result is the same."""
+ takes one tensor kind ("got mixed torch.Tensor and DTensor"), or when a DTensor grad is on the CPU
+ (CPUOffloadPolicy): DTensor's found_inf reduce would need a CPU collective a NCCL-only group lacks.
+ Elementwise, so the shard result is the same; ShardedGradScaler.unscale_ agrees found_inf over ranks."""
try:
from torch.distributed.tensor import DTensor
except ImportError:
return optimizer
- groups, dtensor, plain = [], False, False
+ groups, dtensor, plain, cpu_dtensor = [], False, False, False
for group in optimizer.param_groups:
params = []
for p in group["params"]:
grad = p.grad
if isinstance(grad, DTensor):
dtensor = True
+ cpu_dtensor = cpu_dtensor or grad.device.type == "cpu"
p = types.SimpleNamespace(grad = grad._local_tensor)
elif grad is not None:
plain = True
# Plain grads keep their parameter: upstream reassigns a coalesced float16 sparse grad to it.
params.append(p)
groups.append({"params": params})
- if not (dtensor and plain):
+ if not ((dtensor and plain) or cpu_dtensor):
return optimizer
return types.SimpleNamespace(param_groups = groups)