Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Native FSDP2 float16: start the loss scale where one GPU's scaled backward fits (#15033)

Daniel Hane472a560b24e
diff --git a/tests/test_fsdp2_native_fp16_scale_parity_gloo.py b/tests/test_fsdp2_native_fp16_scale_parity_gloo.py
new file mode 100644
index 00000000000..29eb1e48d15
--- /dev/null
+++ b/tests/test_fsdp2_native_fp16_scale_parity_gloo.py
@@ -0,0 +1,131 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""Native FSDP2 float16 starts the loss scale divided by the data parallel size, so it takes the steps
+one GPU takes (gloo CPU world of 2)."""
+
+import multiprocessing
+import os
+
+if multiprocessing.parent_process() is not None or getattr(
+ multiprocessing.current_process(), "_inheriting", False
+):
+ os.environ["CUDA_VISIBLE_DEVICES"] = ""
+
+import sys # noqa: E402
+import types # noqa: E402
+
+import pytest # noqa: E402
+import torch # noqa: E402
+
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+from test_fsdp2_native_units import _load_native # noqa: E402
+from test_fsdp2_native_trainer_ckpt_gloo import _free_port # noqa: E402
+
+SCALE = 2.0**16
+TOKENS = 8
+FP16_GRAD = 40000.0 # one GPU's scaled per-token output gradient: below the float16 max (65504)
+
+
+def _data(world):
+ torch.manual_seed(0)
+ x = torch.rand(TOKENS, 8) * 1e-3
+ weight = torch.randn(8, 8) * 0.1
+ coef = FP16_GRAD * TOKENS / SCALE
+ return x, weight, coef
+
+
+def _loss(out, coef, n_global, world):
+ # transformers Trainer with num_items_in_batch: sum / global token count, times the world size.
+ return (out.float() * coef).sum() / n_global * world
+
+
+def _one_gpu(world):
+ x, weight, coef = _data(world)
+ w = torch.nn.Parameter(weight.clone())
+ scaler = torch.amp.GradScaler("cpu", init_scale = SCALE)
+ opt = torch.optim.SGD([w], lr = 1e-3)
+ out = torch.nn.functional.linear(x.half(), w.half())
+ scaler.scale(_loss(out, coef, TOKENS, 1)).backward()
+ scaler.step(opt)
+ scaler.update()
+ return float(scaler.get_scale()), w.detach().clone()
+
+
+def _native(world, rank):
+ native = _load_native()
+ from torch.distributed.fsdp import MixedPrecisionPolicy, fully_shard
+
+ x, weight, coef = _data(world)
+ model = torch.nn.Linear(8, 8, bias = False)
+ with torch.no_grad():
+ model.weight.copy_(weight)
+ fully_shard(
+ model,
+ mp_policy = MixedPrecisionPolicy(param_dtype = torch.float16, reduce_dtype = torch.float32),
+ )
+ old = torch.amp.GradScaler("cpu", init_scale = SCALE)
+ trainer = types.SimpleNamespace(
+ model = model, accelerator = types.SimpleNamespace(scaler = old, scaler_handler = None)
+ )
+ assert native._scaler_for_native(trainer) is True
+ scaler = trainer.accelerator.scaler
+ opt = torch.optim.SGD(model.parameters(), lr = 1e-3)
+ share = TOKENS // world
+ out = model(x[rank * share : (rank + 1) * share].half())
+ scaler.scale(_loss(out, coef, TOKENS, world)).backward()
+ scaler.unscale_(opt)
+ found_inf = float(
+ sum(
+ v.item() for v in scaler._per_optimizer_states[id(opt)]["found_inf_per_device"].values()
+ )
+ )
+ scaler.step(opt)
+ scaler.update()
+ full = model.weight.full_tensor().detach().clone()
+ return found_inf, float(scaler.get_scale()), full
+
+
+def _worker(rank, world, port, out):
+ os.environ.update(MASTER_ADDR = "127.0.0.1", MASTER_PORT = str(port))
+ torch.distributed.init_process_group("gloo", rank = rank, world_size = world)
+ try:
+ res = {"one_gpu": _one_gpu(world)}
+ try:
+ res["native"] = _native(world, rank)
+ except Exception as e:
+ res["native"] = repr(e)
+ torch.save(res, os.path.join(out, f"rank{rank}.pt"))
+ finally:
+ torch.distributed.destroy_process_group()
+
+
[email protected](scope = "module")
+def ranks(tmp_path_factory):
+ out = str(tmp_path_factory.mktemp("fp16_scale"))
+ torch.multiprocessing.spawn(_worker, args = (2, _free_port(), out), nprocs = 2, join = True)
+ return [torch.load(os.path.join(out, f"rank{r}.pt")) for r in range(2)]
+
+
+def test_one_gpu_takes_the_step(ranks):
+ for res in ranks:
+ scale, _ = res["one_gpu"]
+ assert scale == SCALE
+
+
+def test_native_fp16_takes_the_step_one_gpu_takes(ranks):
+ for res in ranks:
+ assert not isinstance(res["native"], str), res["native"]
+ found_inf, scale, weight = res["native"]
+ assert found_inf == 0.0, "native FSDP2 overflowed where one GPU did not"
+ assert scale == SCALE / 2, scale
+ _, one_gpu_weight = res["one_gpu"]
+ torch.testing.assert_close(weight, one_gpu_weight, rtol = 1e-3, atol = 1e-6)
+
+
+def test_data_parallel_size_reads_the_mesh():
+ native = _load_native()
+ model = torch.nn.Linear(2, 2)
+ model.__dict__["_unsloth_fsdp2_mesh"] = types.SimpleNamespace(size = lambda: 4)
+ assert native._data_parallel_size(model) == 4
+ assert native._data_parallel_size(None) in (1, int(os.environ.get("WORLD_SIZE", "1")))
diff --git a/tests/test_fsdp2_native_scaler_fused_gloo.py b/tests/test_fsdp2_native_scaler_fused_gloo.py
index e512e57cb61..b8343c8ad1b 100644
--- a/tests/test_fsdp2_native_scaler_fused_gloo.py
+++ b/tests/test_fsdp2_native_scaler_fused_gloo.py
@@ -140,7 +140,8 @@ def test_fused_adamw_steps_through_the_native_fp16_scaler(ranks, both):
assert not isinstance(out, str), f"fused={fused}: {out}"
steps, scale, params = out
assert steps == 2.0, (fused, steps) # 3 steps, the overflow one skipped on every rank
- assert scale == 2.0**7, (fused, scale) # backed off once
+ # user init_scale is kept, not divided by the ranks
+ assert scale == 2.0**7, (fused, scale)
assert all(torch.isfinite(p).all() for p in params), fused
for a, b in zip(res[False, both][2], res[True, both][2]):
torch.testing.assert_close(a, b, rtol = 1e-5, atol = 1e-6)
diff --git a/unsloth/models/fsdp2_native.py b/unsloth/models/fsdp2_native.py
index 7470baf1a41..3392e23f72a 100644
--- a/unsloth/models/fsdp2_native.py
+++ b/unsloth/models/fsdp2_native.py
@@ -458,11 +458,8 @@ def check_shard_units(
units,
ignored_params = (),
):
- """The one hard torch.compile rule: no FSDP unit inside a (compiled) decoder block or inside another
- unit. Unsloth compiles each decoder block's forward (fullgraph=False); FSDP2's hooks are dynamo
- disabled, so a unit nested inside the block (an expert, a router, a LoRA / norm submodule) is a silent
- graph break per block (an error under fullgraph). Experts, routers and LoRA stay inside their block's
- unit. Raises RuntimeError naming the offending unit."""
+ """No FSDP unit inside a compiled decoder block or another unit: FSDP2 hooks are dynamo disabled,
+ so that is a graph break per block. Raises RuntimeError naming the offending unit."""
blocks = set()
for module in model.modules():
if isinstance(module, nn.ModuleList) and _is_block_list(module, ()):
@@ -557,12 +554,8 @@ def _refuse_gpu_quantized_on_cpu_mesh(model):
def _shard_device(model, requested = None):
- """The device FSDP2 shards onto: the model's own, except that a model still entirely on the CPU
- (loaded with device_map = None; meta tensors stay meta) goes to this rank's GPU when CUDA is used,
- like the Trainer would have placed it: fully_shard moves each unit's states to the mesh device as
- it shards it, _place_replicated the replicated ones. A deliberate CPU / disk offload (any
- device_map entry on cpu / disk) and CPU 4-bit weights are refused instead. requested = "cpu" (the
- Trainer's use_cpu) keeps the CPU mesh."""
+ """The device FSDP2 shards onto: the model's own, or this rank's GPU for an all-CPU model. CPU /
+ disk offload and CPU 4-bit weights are refused; requested = "cpu" keeps the CPU mesh."""
if requested is not None and torch.device(requested).type == "cpu":
_refuse_gpu_quantized_on_cpu_mesh(model)
return torch.device("cpu")
@@ -838,11 +831,9 @@ def _unify_dtypes(
param_dtype,
can_ignore = True,
):
- """FSDP2 wants one original dtype over a unit's trainable parameters (all of them on torch < 2.8).
- Unsloth's full finetuning keeps 16-bit linears next to float32 norms; one GPU updates each in its own
- dtype. To keep that, a small minority dtype group (norms) is left out of the unit: replicated, its
- gradient averaged over ranks like DDP. A large one, or a torch without `ignored_params`, gets float32
- masters instead (gathered in `param_dtype`). Returns (ignored parameter set, number upcast)."""
+ """FSDP2 wants one original dtype per unit's trainable parameters. A small minority dtype group
+ (norms) is left out (replicated, DDP-averaged); a large one gets float32 masters.
+ Returns (ignored parameter set, number upcast)."""
frozen_too = _fsdp2_checks_frozen_dtypes()
groups = [list(u.parameters()) for u in units]
inside = set()
@@ -1057,13 +1048,9 @@ def _recompute_skips_input_cast(original):
def _patch_recompute_input_cast():
- """torch < 2.13 skips FSDP2's forward-input cast when a checkpointed unit is recomputed in backward, so
- the recompute sees the uncast (e.g. float32) inputs while the forward saw param_dtype ones: anything
- cast to the input dtype (LoRA A / B, a dequantized 4-bit weight) is saved in another dtype and
- non-reentrant checkpointing raises CheckpointError (Kaggle T4, torch 2.10, fp16 MoE / VLM 4-bit).
- torch 2.13 casts there (`return self._cast_forward_inputs(args, kwargs)`); do the same on older
- builds. Must run before fully_shard registers the hook. Native FSDP2 only;
- UNSLOTH_FSDP2_RECOMPUTE_CAST=0 turns it off (diagnostics)."""
+ """torch < 2.13 skips FSDP2's input cast on checkpoint recompute, so recompute sees uncast inputs and
+ non-reentrant checkpointing raises CheckpointError. Cast there like torch 2.13 does. Must run before
+ fully_shard registers the hook; UNSLOTH_FSDP2_RECOMPUTE_CAST=0 turns it off."""
if os.environ.get("UNSLOTH_FSDP2_RECOMPUTE_CAST", "1").strip().lower() in (
"0",
"false",
@@ -1296,16 +1283,8 @@ def _refuse_block_swap(model):
def shard_model(model, config: Optional[FSDP2Config] = None):
- """fully_shard(unit) for each unit of `find_shard_units`, then fully_shard(model): the root holds
- embeddings, final norm, lm_head and any leftover parameters, so Unsloth's direct lm_head.weight
- read in the fused CE runs while the root is unsharded. Pass the outermost module the training
- loop calls (the PeftModel for LoRA): peft calls the inner model's `.forward` directly, which
- would skip an inner root's hooks. Idempotent.
-
- Layout (fsdp2_perf.plan_fsdp2_layout): when the model fits on one GPU and reshard_after_forward is
- auto, frozen weights stay replicated (never gathered, like DDP), only trainable ones are sharded and
- kept gathered from forward to backward with next-block prefetch, and a small trainable set (LoRA) is
- one root unit. Otherwise each block reshards after forward."""
+ """fully_shard each unit of `find_shard_units`, then the root (embeddings, final norm, lm_head,
+ leftovers). Pass the outermost module the loop calls (the PeftModel for LoRA). Idempotent."""
if getattr(model, _NATIVE_FLAG, False):
return model
_refuse_block_swap(model)
@@ -2135,12 +2114,8 @@ def _peft_saved_key(key, adapter):
def _peft_save_keys(root, adapter):
- """model key -> checkpoint key for `adapter`, from PEFT's own save path
- (peft.utils.get_peft_model_state_dict, peft/utils/save_and_load.py): every key rule the installed
- PEFT applies on save (adapter name removal, DoRA's lora_magnitude_vector, modules_to_save, later
- ones) is inverted here. PEFT is handed empty placeholder tensors (no collective, no copy) and the
- returned keys are matched back by tensor identity; {} when PEFT cannot be asked, and the rule above
- stands in."""
+ """model key -> checkpoint key for `adapter`, inverted from PEFT's own save path using placeholder
+ tensors matched by identity; {} when PEFT cannot be asked."""
try:
from peft.utils import get_peft_model_state_dict
except Exception:
@@ -2562,10 +2537,31 @@ def _scaler_for_native(trainer):
if handler is not None and hasattr(handler, "to_kwargs"):
kwargs = handler.to_kwargs()
kwargs.setdefault("device", getattr(scaler, "_device", "cuda"))
+ # The Trainer scales each rank's loss by world size, so float16 activation grads are world times
+ # one GPU's and overflow at its scale. Divide init_scale by the DP size unless the user set it.
+ world = _data_parallel_size(getattr(trainer, "model", None))
+ if world > 1 and "init_scale" not in kwargs:
+ kwargs["init_scale"] = 2.0**16 / world
accelerator.scaler = _ShardedGradScaler(**kwargs)
return True
+def _data_parallel_size(model):
+ mesh = getattr(model, "__dict__", {}).get("_unsloth_fsdp2_mesh") if model is not None else None
+ try:
+ if mesh is not None:
+ return int(mesh.size())
+ except Exception:
+ pass
+ try:
+ import torch.distributed as dist
+ if dist.is_available() and dist.is_initialized():
+ return int(dist.get_world_size())
+ except Exception:
+ pass
+ return 1
+
+
def _native_optimizer(optimizer):
"""Whether an optimizer steps parameters shard_model produced (marked on each of them)."""
groups = getattr(_inner_optimizer(optimizer), "param_groups", None) or []
@@ -2977,12 +2973,8 @@ def _refuse_projection_optim(training_args):
def _refuse_accelerate_fp8(training_args):
- """accelerate's FP8 training (mixed_precision = "fp8": Transformer Engine, torchao or MS-AMP)
- converts the model's Linear layers in Accelerator.prepare, after native sharding captured the
- DTensors, so the optimizer would target replacement parameters FSDP2 does not manage. The Trainer
- gets it from ACCELERATE_MIXED_PRECISION (accelerate launch / config) unless fp16 / bf16 is set:
- transformers 5 stores the result as args.mixed_precision, 4.x leaves it in the env. Unsloth's own
- float8 weights are not affected (kept replicated, never converted by accelerate)."""
+ """accelerate FP8 training converts Linear layers in prepare, after sharding captured the DTensors,
+ so the optimizer would target parameters FSDP2 does not manage."""
mode = getattr(training_args, "mixed_precision", None) if training_args is not None else None
if mode is None:
mode = os.environ.get("ACCELERATE_MIXED_PRECISION", "no")
@@ -3000,11 +2992,8 @@ def _refuse_accelerate_fp8(training_args):
def _refuse_accelerate_backends(training_args):
- """Every accelerate backend native FSDP2 cannot run under (fsdp2_launch.NATIVE_BACKEND_TABLE:
- DeepSpeed, Megatron-LM, XLA, NPU / MLU / SDAA / MUSA / HPU / XPU / Neuron devices, SageMaker DP, a
- ParallelismConfig env), from the env and the TrainingArguments (`deepspeed`, `deepspeed_plugin`,
- `hf_deepspeed_config`, ...): the import-time check only sees the launch env, and accelerate would
- otherwise build that backend over the DTensor model. Before any model change."""
+ """Refuse accelerate backends native FSDP2 cannot run under (fsdp2_launch.NATIVE_BACKEND_TABLE),
+ from the env and the TrainingArguments. Before any model change."""
from ..fsdp2_launch import refuse_native_backend_conflicts
refuse_native_backend_conflicts(training_args, _trainer_uses_cpu(training_args))