Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
FSDP2 float16 with CPU offload: unscale CPU DTensor grads as local shards (#16398)
Daniel Hana098d99a54da
diff --git a/tests/test_fsdp2_native_scaler_cpu_offload_dtensor.py b/tests/test_fsdp2_native_scaler_cpu_offload_dtensor.pynew file mode 100644index 00000000000..9285becf0d0--- /dev/null+++ b/tests/test_fsdp2_native_scaler_cpu_offload_dtensor.py@@ -0,0 +1,75 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""CPUOffloadPolicy fp16: DTensor grads live on the CPU, and DTensor's found_inf reduce is a CPU collective a+NCCL-only group cannot run (2 x T4 failed with "No backend type associated with device type cpu"). The scaler must+unscale local shards and leave found_inf agreement to ShardedGradScaler."""++import os+import sys+import types++import pytest+import torch++sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))+from test_fsdp2_native_units import _load_native # noqa: E402+from test_fsdp2_native_trainer_ckpt_gloo import _free_port # noqa: E402+++class _Handler:+ def to_kwargs(self):+ return {"device": "cpu", "init_scale": 2.0**8}++[email protected](scope = "module")+def cpu_mesh():+ import torch.distributed as dist+ from torch.distributed.device_mesh import init_device_mesh++ os.environ.update(MASTER_ADDR = "127.0.0.1", MASTER_PORT = str(_free_port()))+ dist.init_process_group("gloo", rank = 0, world_size = 1)+ try:+ yield init_device_mesh("cpu", (1,))+ finally:+ dist.destroy_process_group()++[email protected]("overflow", [False, True], ids = ["finite", "inf"])+def test_cpu_dtensor_grads_unscale_as_local_shards(cpu_mesh, monkeypatch, overflow):+ from torch.distributed.fsdp.sharded_grad_scaler import ShardedGradScaler+ from torch.distributed.tensor import DTensor, Shard, distribute_tensor++ native = _load_native()+ old = torch.amp.GradScaler("cpu", init_scale = 2.0**8)+ trainer = types.SimpleNamespace(+ accelerator = types.SimpleNamespace(scaler = old, scaler_handler = _Handler())+ )+ assert native._scaler_for_native(trainer) is True+ scaler = trainer.accelerator.scaler+ torch.manual_seed(0)+ p = torch.nn.Parameter(distribute_tensor(torch.randn(8, 8), cpu_mesh, [Shard(0)]))+ grad = torch.randn(8, 8) * 256.0+ if overflow:+ grad[0, 0] = float("inf")+ p.grad = distribute_tensor(grad, cpu_mesh, [Shard(0)])+ opt = torch.optim.AdamW([p], lr = 1e-2, foreach = True)+ seen = []+ parent = ShardedGradScaler._unscale_grads_++ def recording(self, optimizer, *a, **k):+ seen.extend(+ type(q.grad) for g in optimizer.param_groups for q in g["params"] if q.grad is not None+ )+ return parent(self, optimizer, *a, **k)++ monkeypatch.setattr(ShardedGradScaler, "_unscale_grads_", recording)+ scaler.scale(torch.ones(())) # initializes the scale; the grads above stand for scaled ones+ scaler.unscale_(opt)+ assert seen and not any(issubclass(t, DTensor) for t in seen), seen+ assert isinstance(p.grad, DTensor)+ state = scaler._per_optimizer_states[id(opt)]+ found = sum(float(v.sum()) for v in state["found_inf_per_device"].values())+ assert found == (1.0 if overflow else 0.0)+ if not overflow:+ torch.testing.assert_close(p.grad.full_tensor(), grad / 256.0)diff --git a/unsloth/models/fsdp2_native.py b/unsloth/models/fsdp2_native.pyindex 144299baf62..378360f4559 100644--- a/unsloth/models/fsdp2_native.py+++ b/unsloth/models/fsdp2_native.py@@ -3492,25 +3492,28 @@ def _has_cpu_tensors(optimizer, grads = False):def _local_grads_if_mixed(optimizer):"""Swap DTensor grads for their local shards when DTensor and plain grads mix: one foreach unscale- takes one tensor kind ("got mixed torch.Tensor and DTensor"). Elementwise, so the shard result is the same."""+ takes one tensor kind ("got mixed torch.Tensor and DTensor"), or when a DTensor grad is on the CPU+ (CPUOffloadPolicy): DTensor's found_inf reduce would need a CPU collective a NCCL-only group lacks.+ Elementwise, so the shard result is the same; ShardedGradScaler.unscale_ agrees found_inf over ranks."""try:from torch.distributed.tensor import DTensorexcept ImportError:return optimizer- groups, dtensor, plain = [], False, False+ groups, dtensor, plain, cpu_dtensor = [], False, False, Falsefor group in optimizer.param_groups:params = []for p in group["params"]:grad = p.gradif isinstance(grad, DTensor):dtensor = True+ cpu_dtensor = cpu_dtensor or grad.device.type == "cpu"p = types.SimpleNamespace(grad = grad._local_tensor)elif grad is not None:plain = True# Plain grads keep their parameter: upstream reassigns a coalesced float16 sparse grad to it.params.append(p)groups.append({"params": params})- if not (dtensor and plain):+ if not ((dtensor and plain) or cpu_dtensor):return optimizerreturn types.SimpleNamespace(param_groups = groups)