Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Fix FSDP2 checkpoint planning across parameter swaps and accumulation

Daniel Han99a8bdfbc9f1
diff --git a/tests/test_gc_skip_auto.py b/tests/test_gc_skip_auto.py
index be1a889d9e8..142f7af6f17 100644
--- a/tests/test_gc_skip_auto.py
+++ b/tests/test_gc_skip_auto.py
@@ -1,12 +1,11 @@
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
-"""checkpoint_skip_layers = "auto" / UNSLOTH_GC_KEEP_LAYERS, extracted from _utils.py with ast."""
import ast, logging, os
import pytest
import torch
-from real_accelerator import has_real_cuda # tests/_shared, on sys.path via tests/conftest.py
+from real_accelerator import has_real_cuda
transformers = pytest.importorskip("transformers")
from transformers import Qwen3Config, Qwen3ForCausalLM
@@ -20,6 +19,7 @@ NAMES = (
"_AutoCheckpointSkip",
"usable_cuda_bytes",
"_size_auto_checkpoint_skip_for_trainer",
+ "_local_numel",
)
@@ -181,7 +181,7 @@ def test_auto_ignores_allocation_spike_before_first_step():
model = _model(device = "cuda")
ns["skip_checkpointing"](model, "auto")
planner = model._unsloth_auto_skip
- # 3 GiB usable; a 2 GiB loading spike before training must not be read as activations.
+
ns["usable_cuda_bytes"] = lambda device: 3 << 30
spike = torch.empty(2 << 30, dtype = torch.uint8, device = "cuda")
del spike
@@ -215,11 +215,11 @@ def test_auto_reserves_optimizer_state_only_when_not_allocated(optimizer_stepped
ids = torch.randint(0, 32000, (2, 64), generator = torch.Generator().manual_seed(1)).cuda()
model = _big_model()
numel = sum(p.numel() for p in model.parameters())
- assert numel * 8 > (1 << 30) # AdamW state well above the planner's slack
+ assert numel * 8 > (1 << 30)
ns["skip_checkpointing"](model, "auto")
planner = model._unsloth_auto_skip
optimizer = torch.optim.AdamW(model.parameters(), lr = 1e-5)
- # Usable after step 1: grads + foreach AdamW workspace + 1.6 GiB.
+
ns["usable_cuda_bytes"] = lambda device: numel * 8 + int(1.6 * (1 << 30))
model(input_ids = ids, labels = ids).loss.backward()
if optimizer_stepped:
@@ -227,10 +227,8 @@ def test_auto_reserves_optimizer_state_only_when_not_allocated(optimizer_stepped
optimizer.zero_grad(set_to_none = True)
model(input_ids = ids, labels = ids).loss.backward()
if optimizer_stepped:
- # Full finetuning: the state is in memory_allocated already and must not be reserved again.
assert planner.kept == [0, 1, 2, 3, 4, 5]
else:
- # Gradient accumulation: the state does not exist yet, so it is reserved and nothing fits.
assert planner.kept == []
del optimizer, model
torch.cuda.empty_cache()
@@ -260,7 +258,7 @@ def test_auto_reserves_state_for_params_the_optimizer_has_not_seen():
ids = torch.randint(0, 32000, (2, 64), generator = torch.Generator().manual_seed(1)).cuda()
model = _big_model()
numel = sum(p.numel() for p in model.parameters())
- # A trainable param the first batch never reaches (an unrouted MoE expert, an unused tower).
+
model.unused = torch.nn.Parameter(torch.zeros(1 << 27, device = "cuda"))
assert model.unused.numel() * 8 >= (1 << 30)
ns["skip_checkpointing"](model, "auto")
@@ -273,7 +271,7 @@ def test_auto_reserves_state_for_params_the_optimizer_has_not_seen():
optimizer.zero_grad(set_to_none = True)
assert model.unused not in optimizer.state
model(input_ids = ids, labels = ids).loss.backward()
- # main: one step happened, so nothing is reserved for `unused` and every layer is kept.
+
assert planner.kept == []
del optimizer, model
torch.cuda.empty_cache()
@@ -289,7 +287,7 @@ def test_auto_does_not_scale_persistent_grads_by_sequence_length():
ns["skip_checkpointing"](model, "auto")
planner = model._unsloth_auto_skip
optimizer = torch.optim.AdamW(model.parameters(), lr = 1e-5)
- # Room for 16x the short batch's activations, but not for 16x the fp32 grads it allocated.
+
usable = numel * 4 * 6
ns["usable_cuda_bytes"] = lambda device: usable
model(input_ids = short, labels = short).loss.backward()
@@ -297,15 +295,13 @@ def test_auto_does_not_scale_persistent_grads_by_sequence_length():
optimizer.zero_grad(set_to_none = True)
model(input_ids = short, labels = short).loss.backward()
assert planner.layer_rows == 64
- # main: (activations + grads) x 16 leaves no headroom, so nothing is kept.
+
assert planner.kept == [0, 1, 2, 3, 4, 5]
del optimizer, model
torch.cuda.empty_cache()
class _FakeLayer(torch.nn.Module):
- """A decoder layer whose recompute leaves `cost` fake CUDA bytes behind."""
-
def __init__(self, cost, memory):
super().__init__()
self.weight = torch.nn.Parameter(torch.zeros(1), requires_grad = False)
@@ -336,7 +332,6 @@ class _FakeModel(torch.nn.Module):
def _fake_cuda(monkeypatch, ns, usable):
- """CPU stand-ins for the CUDA calls the planner makes; memory_allocated is a counter."""
memory = {"allocated": 0}
monkeypatch.setattr(torch.cuda, "is_available", lambda: True)
monkeypatch.setattr(torch.Tensor, "is_cuda", property(lambda self: True))
@@ -348,7 +343,7 @@ def _fake_cuda(monkeypatch, ns, usable):
def _fake_step(model, rows):
- """One reentrant-checkpointed step: no_grad forward of every layer, recompute in reverse."""
+ """Simulate checkpoint forwards and recomputations without running autograd."""
hidden = torch.zeros(1, rows, 4)
model(hidden)
with torch.no_grad():
@@ -369,7 +364,7 @@ def test_auto_sizes_by_the_most_expensive_layer_type(monkeypatch):
_fake_step(model, rows = 16)
model(torch.zeros(1, 16, 4))
assert planner.layer_bytes == gib
- # (10 GiB - 4.5 GiB step) * 0.75 - 1 GiB = 3.125 GiB: 3 one-GiB layers, not 25 eighth-GiB ones.
+
assert len(planner.kept) == 3
monkeypatch.undo()
@@ -380,7 +375,7 @@ def test_auto_scales_eager_attention_quadratically(monkeypatch, attn, exponent):
gib = 1 << 30
memory = _fake_cuda(monkeypatch, ns, usable = 12 * gib)
model = _FakeModel([gib // 16] * 8, memory, attn = attn)
- model.max_seq_length = 64 # 4x the probed 16 rows
+ model.max_seq_length = 64
ns["skip_checkpointing"](model, "auto")
planner = model._unsloth_auto_skip
_fake_step(model, rows = 16)
@@ -389,7 +384,7 @@ def test_auto_scales_eager_attention_quadratically(monkeypatch, attn, exponent):
step = 8 * (gib // 16)
budget = (12 * gib - step * scale) * 0.75 - gib
assert len(planner.kept) == min(8, int(budget // (gib // 16 * scale)))
- # Linear: 8 of 8 fit. Quadratic: a 4x longer batch costs 16x, so only 2 do.
+
assert len(planner.kept) == (8 if exponent == 1 else 2)
monkeypatch.undo()
@@ -454,9 +449,9 @@ def test_auto_sizes_for_the_trainer_batch_not_the_first_one(monkeypatch):
trainer.model = model
trainer.args = type("Args", (), {"per_device_train_batch_size": 4, "max_length": None})()
ns["_size_auto_checkpoint_skip_for_trainer"](trainer)
- _fake_step(model, rows = 16) # first batch: 1 row at max_seq_length
+ _fake_step(model, rows = 16)
model(torch.zeros(1, 16, 4))
- # Sized for 4 rows: (4 GiB - 2 GiB) * 0.75 - 1 GiB = 0.5 GiB, two 0.25 GiB layers (not all 8).
+
assert planner.target_batch == 4
assert len(planner.kept) == 2
monkeypatch.undo()
@@ -467,16 +462,15 @@ def test_auto_ignores_gradients_on_another_gpu(monkeypatch):
gib = 1 << 30
memory = _fake_cuda(monkeypatch, ns, usable = 5 * gib)
model = _FakeModel([gib // 16] * 8, memory)
- model.max_seq_length = 64 # 4x the probed rows
+ model.max_seq_length = 64
# A trainable vision tower on another card ("meta" stands in for cuda:1).
model.tower = torch.nn.Parameter(torch.empty(gib // 8, device = "meta"))
ns["skip_checkpointing"](model, "auto")
planner = model._unsloth_auto_skip
_fake_step(model, rows = 16)
- model.tower.grad = torch.empty(gib // 8, device = "meta") # 0.5 GiB, not on the planned card
+ model.tower.grad = torch.empty(gib // 8, device = "meta")
model(torch.zeros(1, 16, 4))
- # 0.5 GiB step x 4 = 2 GiB activations, tower AdamW state on its own card: (5 - 2) * 0.75 - 1 =
- # 1.25 GiB, five 0.25 GiB layers.
+
assert len(planner.kept) == 5
monkeypatch.undo()
@@ -489,7 +483,7 @@ def test_auto_still_plans_when_the_probe_layer_moves_to_host(monkeypatch):
model = _FakeModel([gib // 16] * 8, memory)
ns["skip_checkpointing"](model, "auto")
planner = model._unsloth_auto_skip
- # offload_layers = "auto" re-plans for the trainer and swaps the last layer (the probe) to host.
+
model.model.layers[-1].to("meta")
_fake_step(model, rows = 16)
model(torch.zeros(1, 16, 4))
@@ -501,14 +495,13 @@ def test_auto_never_scales_below_the_measured_length(monkeypatch):
ns, found = _load()
gib = 1 << 30
memory = _fake_cuda(monkeypatch, ns, usable = 4 * gib)
- model = _FakeModel([gib // 16] * 8, memory) # max_seq_length 16
+ model = _FakeModel([gib // 16] * 8, memory)
ns["skip_checkpointing"](model, "auto")
planner = model._unsloth_auto_skip
planner.target_batch = 4
- _fake_step(model, rows = 64) # a collator expanded the batch past max_seq_length
+ _fake_step(model, rows = 64)
model(torch.zeros(1, 64, 4))
- # 4 rows x 64 measured tokens: scale 4, not 4 x 16 / 64 = 1.
- # (4 GiB - 2 GiB) * 0.75 - 1 GiB = 0.5 GiB, two 0.25 GiB layers (before: all 8).
+
assert len(planner.kept) == 2
monkeypatch.undo()
@@ -523,8 +516,7 @@ def _trainable_head(model, numel):
def test_auto_counts_resident_accumulated_grads_once(monkeypatch, accumulating):
ns, found = _load()
gib = 1 << 30
- numel = gib // 8 # fp32 grad 0.5 GiB, AdamW state + foreach workspace 1.5 GiB
- # Resident grads are allocated, so the card has that much less usable memory.
+ numel = gib // 8
memory = _fake_cuda(monkeypatch, ns, usable = 4 * gib - (gib // 2 if accumulating else 0))
model = _FakeModel([gib // 16] * 8, memory)
head = _trainable_head(model, numel)
@@ -538,7 +530,7 @@ def test_auto_counts_resident_accumulated_grads_once(monkeypatch, accumulating):
head.grad = None
memory["allocated"] -= gib // 2
model(torch.zeros(1, 16, 4))
- # Both: (4 GiB - 0.5 GiB activations - 0.5 GiB grads - 1.5 GiB) * 0.75 - 1 GiB = 0.125 GiB.
+
assert len(planner.kept) == 2
monkeypatch.undo()
@@ -557,11 +549,11 @@ def test_auto_reserves_the_foreach_optimizer_workspace(monkeypatch):
memory["allocated"] += gib // 2
optimizer = torch.optim.AdamW([head])
planner._optimizer_pre_hook(optimizer, (), {})
- optimizer.state[head] = {"step": torch.tensor(1.0)} # state allocated by the first step
+ optimizer.state[head] = {"step": torch.tensor(1.0)}
head.grad = None
memory["allocated"] -= gib // 2
model(torch.zeros(1, 16, 4))
- # 0.5 GiB foreach workspace still reserved: (3 - 1 - 0.5) * 0.75 - 1 = 0.125 GiB, two layers.
+
assert len(planner.kept) == 2
monkeypatch.undo()
@@ -593,7 +585,7 @@ def test_auto_reserves_pending_state_only_on_the_planned_card(monkeypatch):
planner = model._unsloth_auto_skip
_fake_step(model, rows = 16)
model(torch.zeros(1, 16, 4))
- # (3 - 0.5) * 0.75 - 1 = 0.875 GiB: all 8 layers (before: 8 GiB tower state, nothing kept).
+
assert len(planner.kept) == 8
monkeypatch.undo()
@@ -609,7 +601,6 @@ def test_counts_apply_only_to_checkpointable_layers():
@pytest.mark.skipif(not has_real_cuda(), reason = "the auto planner sizes CUDA memory")
@pytest.mark.parametrize("mode", ["fullgraph", "dynamic"])
def test_auto_planner_stays_out_of_compiled_regions(mode):
- # TrainingArguments(torch_compile = True) compiles the whole model; the planner must stay out of it.
from torch._dynamo.utils import counters
class Wrapped(torch.nn.Module):
diff --git a/tests/test_gc_skip_auto_dist_cells.py b/tests/test_gc_skip_auto_dist_cells.py
index b4bd7941122..2007040975c 100644
--- a/tests/test_gc_skip_auto_dist_cells.py
+++ b/tests/test_gc_skip_auto_dist_cells.py
@@ -1,10 +1,9 @@
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
-"""checkpoint_skip_layers = "auto" under torchrun, on CPU gloo: which runs take the collective, that
-every rank takes it at the same forward, and that DDP training is unchanged by the plan."""
import datetime
+import math
import multiprocessing
import os
import sys
@@ -27,8 +26,6 @@ def _load(
tensor_parallel = False,
pipeline_parallel = False,
):
- """The planner from _utils.py, with its package-relative imports (is_distributed,
- is_tensor_parallel, is_pipeline_parallel) answered by stubs instead of falling back."""
sys.path.insert(0, HERE)
import test_gc_skip_auto as T
@@ -53,7 +50,6 @@ def _load(
def _spoof_cuda(ns, usable, memory):
- # Process-local CPU stand-ins for the CUDA calls the planner makes (as T._fake_cuda does).
torch.cuda.is_available = lambda: True
torch.Tensor.is_cuda = property(lambda self: True)
torch.cuda.reset_peak_memory_stats = lambda device = None: None
@@ -77,7 +73,6 @@ def _count_collectives():
def _fake_scenario(rank, scenario, usable):
- """The fake 8-layer model of test_gc_skip_auto, with rank-local extra forwards."""
T, ns = _load(
tensor_parallel = scenario == "tp",
pipeline_parallel = scenario == "pp",
@@ -89,11 +84,9 @@ def _fake_scenario(rank, scenario, usable):
ns["skip_checkpointing"](model, "auto")
planner = model._unsloth_auto_skip
if scenario == "rank0_pretrain_forward" and rank == 0:
- # A grad-enabled training forward only rank 0 runs before step 1 (no backward).
model(torch.zeros(1, 16, 4))
T._fake_step(model, rows = 16)
if scenario == "rank0_eval_and_generate" and rank == 0:
- # Rank-0-only evaluation / sample generation between steps: eval mode, no grad.
model.eval()
model(torch.zeros(1, 16, 4))
model.train()
@@ -108,7 +101,6 @@ def _fake_scenario(rank, scenario, usable):
def _ddp_scenario(rank, scenario, usable):
- """A tiny Qwen3 under DDP (gloo), HF reentrant gradient checkpointing, 4 optimizer steps."""
from torch.nn.parallel import DistributedDataParallel as DDP
static_graph, find_unused, accumulation, auto = scenario
@@ -122,7 +114,7 @@ def _ddp_scenario(rank, scenario, usable):
model = T._model(layers = 4)
if auto:
_spoof_cuda(ns, usable, memory)
- # Each grad-mode layer call "allocates" 1/16 GiB, so the planner measures a cost.
+
for layer in model.model.layers:
layer.register_forward_hook(
lambda m, a, o: memory.__setitem__(
@@ -172,8 +164,67 @@ def _ddp_scenario(rank, scenario, usable):
}
+def _fsdp2_grad_bytes_scenario(rank, scenario, usable):
+ from torch.distributed.device_mesh import init_device_mesh
+ from torch.distributed.fsdp import fully_shard
+
+ root_only, mode = scenario
+ mesh = init_device_mesh("cpu", (torch.distributed.get_world_size(),))
+ T, ns = _load()
+ model = T._model(layers = 2)
+ global_bytes = sum(p.numel() * p.element_size() for p in model.parameters())
+ memory = {"allocated": 0}
+ _spoof_cuda(ns, usable, memory)
+ for layer in model.model.layers:
+ layer.register_forward_hook(
+ lambda m, a, o: memory.__setitem__(
+ "allocated", memory["allocated"] + (GIB // 16 if torch.is_grad_enabled() else 0)
+ )
+ )
+ ns["skip_checkpointing"](model, "auto")
+ planner = model._unsloth_auto_skip
+ if not root_only:
+ for layer in model.model.layers:
+ fully_shard(layer, mesh = mesh)
+ fully_shard(model, mesh = mesh)
+ optimizer = torch.optim.AdamW(model.parameters(), lr = 0.001)
+ # Only the planner's CUDA predicates are spoofed; AdamW still runs on CPU.
+ optimizer._accelerator_graph_capture_health_check = lambda: None
+ parameters = list(model.parameters())
+ local_numel = sum(p.to_local().numel() for p in parameters)
+ if mode != "sync":
+ model.set_requires_gradient_sync(False)
+ model.set_reshard_after_backward(mode != "defer")
+ ids = torch.randint(0, 128, (2, 16), generator = torch.Generator().manual_seed(rank))
+ model(input_ids = ids, labels = ids).loss.backward()
+ expected_grad = (
+ sum(p.grad.to_local().numel() * p.grad.element_size() for p in parameters)
+ if mode == "sync"
+ else global_bytes
+ )
+ measured_grad = planner._fresh_grad_bytes(torch.device("cpu"))
+ memory["allocated"] += expected_grad
+ if mode == "sync":
+ optimizer.step()
+ optimizer.zero_grad(set_to_none = True)
+ memory["allocated"] -= expected_grad
+ model.set_requires_gradient_sync(True)
+ model.set_reshard_after_backward(True)
+ activation = 2 * (GIB // 16) * 4 + (expected_grad if mode == "sync" else 0)
+ optimizer_bytes = local_numel * (4 if mode == "sync" else 12)
+ available = math.ceil(activation + optimizer_bytes + (GIB + GIB // 4) / 0.75) + 128
+ ns["usable_cuda_bytes"] = lambda device: available
+ model(input_ids = ids, labels = ids).loss.backward()
+ optimizer.step()
+ return {
+ "utils": T.UTILS,
+ "planner": measured_grad,
+ "expected": expected_grad,
+ "kept": planner.kept,
+ }
+
+
def _join_scenario(rank, scenario, usable):
- """DDP.join(): rank 0 runs out of inputs before rank 1's planning forward."""
from torch.nn.parallel import DistributedDataParallel as DDP
T, ns = _load()
@@ -230,7 +281,11 @@ def _worker(rank, world, port, kind, scenario, usable, queue):
world_size = world,
timeout = datetime.timedelta(seconds = 30),
)
- run = {"fake": _fake_scenario, "join": _join_scenario}.get(kind, _ddp_scenario)
+ run = {
+ "fake": _fake_scenario,
+ "join": _join_scenario,
+ "fsdp2_grad_bytes": _fsdp2_grad_bytes_scenario,
+ }.get(kind, _ddp_scenario)
queue.put((rank, run(rank, scenario, usable[rank]), None))
except Exception:
import traceback
@@ -266,7 +321,6 @@ def _launch(kind, scenario, usable):
@pytest.mark.parametrize("scenario", ["rank0_pretrain_forward", "rank0_eval_and_generate"])
def test_rank_local_forwards_do_not_desync_the_collective(scenario):
- # Rank 0 has room for 8 layers, rank 1 for 2; rank-local forwards must not move rank 0's plan.
results = _launch("fake", scenario, [4 * GIB, 2 * GIB])
assert results[0]["kept"] == results[1]["kept"] == [3, 7]
assert results[0]["all_reduce"] == results[1]["all_reduce"] == 1
@@ -318,7 +372,7 @@ def test_single_process_plans_alone_without_a_collective(monkeypatch, world_size
planner = model._unsloth_auto_skip
T._fake_step(model, rows = 16)
model(torch.zeros(1, 16, 4))
- # (4 GiB - 0.5 GiB step) * 0.75 - 1 GiB = 1.6 GiB: all 8 sixteenth-GiB layers, as before.
+
assert planner.kept == list(range(8))
monkeypatch.undo()
@@ -341,7 +395,6 @@ def _assert_same_training(auto, reference):
[(False, False, 1), (False, False, 2), (True, False, 1)],
)
def test_ddp_training_unchanged_and_ranks_agree(static_graph, find_unused, accumulation):
- # Rank 0 fits 4, rank 1 fits 2: both keep 2; losses / weights match full checkpointing.
usable = [8 * GIB, 2 * GIB + GIB // 4]
auto = _launch("ddp", (static_graph, find_unused, accumulation, True), usable)
reference = _launch("ddp", (static_graph, find_unused, accumulation, False), usable)
@@ -361,7 +414,6 @@ def _fsdp2_runs_on_cpu():
@pytest.mark.skipif(not _fsdp2_runs_on_cpu(), reason = "this torch has no CPU FSDP2 path")
@pytest.mark.parametrize("accumulation", [1, 2])
def test_fsdp2_training_unchanged_and_ranks_agree(accumulation):
- # The collective runs in the root forward pre-hook, next to FSDP2's own all-gathers.
usable = [8 * GIB, 2 * GIB + GIB // 4]
auto = _launch("ddp", ("fsdp2", False, accumulation, True), usable)
reference = _launch("ddp", ("fsdp2", False, accumulation, False), usable)
@@ -372,6 +424,17 @@ def test_fsdp2_training_unchanged_and_ranks_agree(accumulation):
_assert_same_training(auto[rank], reference[rank])
[email protected](not _fsdp2_runs_on_cpu(), reason = "this torch has no CPU FSDP2 path")
[email protected]("root_only", [False, True])
[email protected]("mode", ["sync", "skip", "defer"])
+def test_fsdp2_gradient_bytes_are_rank_local(root_only, mode):
+ results = _launch("fsdp2_grad_bytes", (root_only, mode), [8 * GIB, 8 * GIB])
+ for rank, r in results.items():
+ print("FSDP2_GRAD_BYTES", rank, r)
+ assert r["planner"] == r["expected"] > 0, r
+ assert r["kept"] == [1], r
+
+
def test_ddp_join_with_uneven_inputs_keeps_every_layer_checkpointed():
# A joined rank only shadows DDP's collectives, so a planning all_reduce would mispair.
usable = [8 * GIB, 8 * GIB]
diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.py
index 445fae78acf..590ca93823a 100644
--- a/unsloth/models/_utils.py
+++ b/unsloth/models/_utils.py
@@ -10641,15 +10641,16 @@ def _gc_keep_layers_env():
return 0
+def _local_numel(t):
+ """Elements this rank holds: an FSDP2 DTensor's numel() is the global size, its shard is local."""
+ local = getattr(t, "_local_tensor", None)
+ return (local if isinstance(local, torch.Tensor) else t).numel()
+
+
class _AutoCheckpointSkip:
- """checkpoint_skip_layers = "auto": once, after step 1, un-checkpoint as many layers as fit.
- Per-layer cost = the largest allocation any eligible layer's reentrant recompute leaves in the
- first backward (hybrid stacks mix attention and Mamba blocks); AdamW state is reserved for every
- trainable param the optimizer has no state for yet. Sized for the trainer batch at the longest of
- max_length, max_seq_length and the measured length: inputs a collator expands past that (images,
- audio) need max_seq_length set to their bound. Under torchrun (DDP / native FSDP2) every rank
- plans and the smallest K wins, so every rank keeps the same number of layers; tensor / pipeline parallel
- models stay fully checkpointed."""
+ """Plan checkpoint skips once from measured memory, using the smallest count across ranks.
+ Multimodal inputs must stay within max_seq_length; tensor/pipeline parallel models are excluded.
+ """
MARGIN = 1 << 30
FRACTION = 0.75
@@ -10675,6 +10676,7 @@ class _AutoCheckpointSkip:
self.optimizers = []
self.fresh_grads = None
self.grad_bytes = None
+ self.fsdp_params = {}
@classmethod
def _no_cuda(cls):
@@ -10756,6 +10758,53 @@ class _AutoCheckpointSkip:
return p.device
return None
+ def _parameters(self):
+ """Resolve optimizer-owned shards while FSDP exposes temporary gathered parameters."""
+ aliases = {}
+ seen = set()
+ for module in self.model.modules():
+ get_state = getattr(module, "_get_fsdp_state", None)
+ if not callable(get_state):
+ continue
+ state = get_state()
+ if id(state) in seen:
+ continue
+ seen.add(id(state))
+ groups = getattr(state, "_fsdp_param_groups", None)
+ if groups is None:
+ group = getattr(state, "_fsdp_param_group", None)
+ groups = () if group is None else (group,)
+ for group in groups:
+ for fp in group.fsdp_params:
+ for name in (
+ "sharded_param",
+ "_unsharded_param",
+ "_sharded_post_forward_param",
+ ):
+ param = getattr(fp, name, None)
+ if param is not None:
+ aliases[id(param)] = fp
+ self.fsdp_params = {}
+ parameters = {}
+ for param in self.model.parameters():
+ fp = aliases.get(id(param))
+ if fp is not None:
+ param = fp.sharded_param
+ self.fsdp_params[id(param)] = fp
+ parameters[id(param)] = param
+ return parameters.values()
+
+ def _gradient(self, param):
+ if param.grad is not None:
+ return param.grad
+ fp = self.fsdp_params.get(id(param))
+ if fp is None:
+ return None
+ accumulated = getattr(fp, "unsharded_accumulated_grad", None)
+ if accumulated is not None:
+ return accumulated
+ return getattr(getattr(fp, "_unsharded_param", None), "grad", None)
+
def _model_pre_hook(self, module, args):
# Planning under torch.compile breaks fullgraph/dynamic tracing; compiled models stay fully checkpointed.
if torch.compiler.is_compiling():
@@ -10769,9 +10818,8 @@ class _AutoCheckpointSkip:
# Step-local peak: a model-loading spike must not count as activations.
torch.cuda.reset_peak_memory_stats(device)
self.started = torch.cuda.memory_allocated(device)
- # Grads the first backward allocates persist: they are not sequence-dependent.
self.fresh_grads = [
- p for p in self.model.parameters() if p.requires_grad and p.grad is None
+ p for p in self._parameters() if p.requires_grad and self._gradient(p) is None
]
return
if self.layer_bytes is not None:
@@ -10831,7 +10879,7 @@ class _AutoCheckpointSkip:
def _optimizer_pre_hook(self, optimizer, args, kwargs):
if self.started is None or self.stepped:
return
- trainable = {id(p) for p in self.model.parameters() if p.requires_grad}
+ trainable = {id(p) for p in self._parameters() if p.requires_grad}
if not any(id(p) in trainable for group in optimizer.param_groups for p in group["params"]):
return
if not any(o() is optimizer for o in self.optimizers):
@@ -10846,11 +10894,10 @@ class _AutoCheckpointSkip:
self.stepped = True
def _fresh_grad_bytes(self, device):
- # Only grads on the planned card are in its step peak (a vision tower may sit on another GPU).
return sum(
- p.grad.numel() * p.grad.element_size()
+ _local_numel(grad) * grad.element_size()
for p in (self.fresh_grads or ())
- if p.grad is not None and p.grad.is_cuda and p.grad.device == device
+ if (grad := self._gradient(p)) is not None and grad.is_cuda and grad.device == device
)
def _has_optimizer_state(self, param):
@@ -10914,10 +10961,8 @@ class _AutoCheckpointSkip:
if self.step_peak is not None
else torch.cuda.max_memory_allocated(device)
)
- # A short or small first batch (dynamic padding, packing, token-budget batches) under-measures:
- # size for the trainer's batch x max length tokens, else the measured batch at max_seq_length.
+ # A short first batch must still reserve activations for the trainer's maximum batch/length.
rows = max(1, self.layer_rows)
- # Never below the measured length (multimodal collators can expand past max_seq_length).
max_seq = max(rows, self.target_seq or getattr(self.model, "max_seq_length", None) or rows)
batch = max(self.target_batch or 0, self.layer_batch)
tokens = max(1.0, batch * max_seq / (self.layer_batch * rows))
@@ -10933,18 +10978,17 @@ class _AutoCheckpointSkip:
resident = self._fresh_grad_bytes(device) if self.fresh_grads else 0
activation_peak = (step_bytes - grad_bytes) * scale + max(0, grad_bytes - resident)
self.fresh_grads = None
- # State an optimizer step already allocated is excluded by usable_cuda_bytes; reserve the rest
- # (sparse MoE experts, unused towers and gradient accumulation have none yet).
- trainable = [p for p in self.model.parameters() if p.requires_grad]
+ # Existing optimizer state is already excluded by usable_cuda_bytes.
+ trainable = [p for p in self._parameters() if p.requires_grad]
optimizer = sum(
- p.numel() * 8
+ _local_numel(p) * 8
for p in trainable
if p.device == device and not self._has_optimizer_state(p)
)
- # foreach AdamW allocates about one fp32 copy of the params (sqrt of exp_avg_sq) every step,
- # on each param's own card.
- optimizer += sum(p.numel() * 4 for p in trainable if p.device == device)
+ # foreach AdamW needs one temporary fp32 parameter-sized buffer on the parameter's device.
+ optimizer += sum(_local_numel(p) * 4 for p in trainable if p.device == device)
self.optimizers = []
+ self.fsdp_params.clear()
headroom = usable_cuda_bytes(device) - activation_peak - optimizer
budget = headroom * self.FRACTION - self.MARGIN
per_layer = self.layer_bytes * scale