Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Fix FSDP2 checkpoint planning across parameter swaps and accumulation
Daniel Han99a8bdfbc9f1
diff --git a/tests/test_gc_skip_auto.py b/tests/test_gc_skip_auto.pyindex be1a889d9e8..142f7af6f17 100644--- a/tests/test_gc_skip_auto.py+++ b/tests/test_gc_skip_auto.py@@ -1,12 +1,11 @@# SPDX-License-Identifier: AGPL-3.0-only# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.-"""checkpoint_skip_layers = "auto" / UNSLOTH_GC_KEEP_LAYERS, extracted from _utils.py with ast."""import ast, logging, osimport pytestimport torch-from real_accelerator import has_real_cuda # tests/_shared, on sys.path via tests/conftest.py+from real_accelerator import has_real_cudatransformers = pytest.importorskip("transformers")from transformers import Qwen3Config, Qwen3ForCausalLM@@ -20,6 +19,7 @@ NAMES = ("_AutoCheckpointSkip","usable_cuda_bytes","_size_auto_checkpoint_skip_for_trainer",+ "_local_numel",)@@ -181,7 +181,7 @@ def test_auto_ignores_allocation_spike_before_first_step():model = _model(device = "cuda")ns["skip_checkpointing"](model, "auto")planner = model._unsloth_auto_skip- # 3 GiB usable; a 2 GiB loading spike before training must not be read as activations.+ns["usable_cuda_bytes"] = lambda device: 3 << 30spike = torch.empty(2 << 30, dtype = torch.uint8, device = "cuda")del spike@@ -215,11 +215,11 @@ def test_auto_reserves_optimizer_state_only_when_not_allocated(optimizer_steppedids = torch.randint(0, 32000, (2, 64), generator = torch.Generator().manual_seed(1)).cuda()model = _big_model()numel = sum(p.numel() for p in model.parameters())- assert numel * 8 > (1 << 30) # AdamW state well above the planner's slack+ assert numel * 8 > (1 << 30)ns["skip_checkpointing"](model, "auto")planner = model._unsloth_auto_skipoptimizer = torch.optim.AdamW(model.parameters(), lr = 1e-5)- # Usable after step 1: grads + foreach AdamW workspace + 1.6 GiB.+ns["usable_cuda_bytes"] = lambda device: numel * 8 + int(1.6 * (1 << 30))model(input_ids = ids, labels = ids).loss.backward()if optimizer_stepped:@@ -227,10 +227,8 @@ def test_auto_reserves_optimizer_state_only_when_not_allocated(optimizer_steppedoptimizer.zero_grad(set_to_none = True)model(input_ids = ids, labels = ids).loss.backward()if optimizer_stepped:- # Full finetuning: the state is in memory_allocated already and must not be reserved again.assert planner.kept == [0, 1, 2, 3, 4, 5]else:- # Gradient accumulation: the state does not exist yet, so it is reserved and nothing fits.assert planner.kept == []del optimizer, modeltorch.cuda.empty_cache()@@ -260,7 +258,7 @@ def test_auto_reserves_state_for_params_the_optimizer_has_not_seen():ids = torch.randint(0, 32000, (2, 64), generator = torch.Generator().manual_seed(1)).cuda()model = _big_model()numel = sum(p.numel() for p in model.parameters())- # A trainable param the first batch never reaches (an unrouted MoE expert, an unused tower).+model.unused = torch.nn.Parameter(torch.zeros(1 << 27, device = "cuda"))assert model.unused.numel() * 8 >= (1 << 30)ns["skip_checkpointing"](model, "auto")@@ -273,7 +271,7 @@ def test_auto_reserves_state_for_params_the_optimizer_has_not_seen():optimizer.zero_grad(set_to_none = True)assert model.unused not in optimizer.statemodel(input_ids = ids, labels = ids).loss.backward()- # main: one step happened, so nothing is reserved for `unused` and every layer is kept.+assert planner.kept == []del optimizer, modeltorch.cuda.empty_cache()@@ -289,7 +287,7 @@ def test_auto_does_not_scale_persistent_grads_by_sequence_length():ns["skip_checkpointing"](model, "auto")planner = model._unsloth_auto_skipoptimizer = torch.optim.AdamW(model.parameters(), lr = 1e-5)- # Room for 16x the short batch's activations, but not for 16x the fp32 grads it allocated.+usable = numel * 4 * 6ns["usable_cuda_bytes"] = lambda device: usablemodel(input_ids = short, labels = short).loss.backward()@@ -297,15 +295,13 @@ def test_auto_does_not_scale_persistent_grads_by_sequence_length():optimizer.zero_grad(set_to_none = True)model(input_ids = short, labels = short).loss.backward()assert planner.layer_rows == 64- # main: (activations + grads) x 16 leaves no headroom, so nothing is kept.+assert planner.kept == [0, 1, 2, 3, 4, 5]del optimizer, modeltorch.cuda.empty_cache()class _FakeLayer(torch.nn.Module):- """A decoder layer whose recompute leaves `cost` fake CUDA bytes behind."""-def __init__(self, cost, memory):super().__init__()self.weight = torch.nn.Parameter(torch.zeros(1), requires_grad = False)@@ -336,7 +332,6 @@ class _FakeModel(torch.nn.Module):def _fake_cuda(monkeypatch, ns, usable):- """CPU stand-ins for the CUDA calls the planner makes; memory_allocated is a counter."""memory = {"allocated": 0}monkeypatch.setattr(torch.cuda, "is_available", lambda: True)monkeypatch.setattr(torch.Tensor, "is_cuda", property(lambda self: True))@@ -348,7 +343,7 @@ def _fake_cuda(monkeypatch, ns, usable):def _fake_step(model, rows):- """One reentrant-checkpointed step: no_grad forward of every layer, recompute in reverse."""+ """Simulate checkpoint forwards and recomputations without running autograd."""hidden = torch.zeros(1, rows, 4)model(hidden)with torch.no_grad():@@ -369,7 +364,7 @@ def test_auto_sizes_by_the_most_expensive_layer_type(monkeypatch):_fake_step(model, rows = 16)model(torch.zeros(1, 16, 4))assert planner.layer_bytes == gib- # (10 GiB - 4.5 GiB step) * 0.75 - 1 GiB = 3.125 GiB: 3 one-GiB layers, not 25 eighth-GiB ones.+assert len(planner.kept) == 3monkeypatch.undo()@@ -380,7 +375,7 @@ def test_auto_scales_eager_attention_quadratically(monkeypatch, attn, exponent):gib = 1 << 30memory = _fake_cuda(monkeypatch, ns, usable = 12 * gib)model = _FakeModel([gib // 16] * 8, memory, attn = attn)- model.max_seq_length = 64 # 4x the probed 16 rows+ model.max_seq_length = 64ns["skip_checkpointing"](model, "auto")planner = model._unsloth_auto_skip_fake_step(model, rows = 16)@@ -389,7 +384,7 @@ def test_auto_scales_eager_attention_quadratically(monkeypatch, attn, exponent):step = 8 * (gib // 16)budget = (12 * gib - step * scale) * 0.75 - gibassert len(planner.kept) == min(8, int(budget // (gib // 16 * scale)))- # Linear: 8 of 8 fit. Quadratic: a 4x longer batch costs 16x, so only 2 do.+assert len(planner.kept) == (8 if exponent == 1 else 2)monkeypatch.undo()@@ -454,9 +449,9 @@ def test_auto_sizes_for_the_trainer_batch_not_the_first_one(monkeypatch):trainer.model = modeltrainer.args = type("Args", (), {"per_device_train_batch_size": 4, "max_length": None})()ns["_size_auto_checkpoint_skip_for_trainer"](trainer)- _fake_step(model, rows = 16) # first batch: 1 row at max_seq_length+ _fake_step(model, rows = 16)model(torch.zeros(1, 16, 4))- # Sized for 4 rows: (4 GiB - 2 GiB) * 0.75 - 1 GiB = 0.5 GiB, two 0.25 GiB layers (not all 8).+assert planner.target_batch == 4assert len(planner.kept) == 2monkeypatch.undo()@@ -467,16 +462,15 @@ def test_auto_ignores_gradients_on_another_gpu(monkeypatch):gib = 1 << 30memory = _fake_cuda(monkeypatch, ns, usable = 5 * gib)model = _FakeModel([gib // 16] * 8, memory)- model.max_seq_length = 64 # 4x the probed rows+ model.max_seq_length = 64# A trainable vision tower on another card ("meta" stands in for cuda:1).model.tower = torch.nn.Parameter(torch.empty(gib // 8, device = "meta"))ns["skip_checkpointing"](model, "auto")planner = model._unsloth_auto_skip_fake_step(model, rows = 16)- model.tower.grad = torch.empty(gib // 8, device = "meta") # 0.5 GiB, not on the planned card+ model.tower.grad = torch.empty(gib // 8, device = "meta")model(torch.zeros(1, 16, 4))- # 0.5 GiB step x 4 = 2 GiB activations, tower AdamW state on its own card: (5 - 2) * 0.75 - 1 =- # 1.25 GiB, five 0.25 GiB layers.+assert len(planner.kept) == 5monkeypatch.undo()@@ -489,7 +483,7 @@ def test_auto_still_plans_when_the_probe_layer_moves_to_host(monkeypatch):model = _FakeModel([gib // 16] * 8, memory)ns["skip_checkpointing"](model, "auto")planner = model._unsloth_auto_skip- # offload_layers = "auto" re-plans for the trainer and swaps the last layer (the probe) to host.+model.model.layers[-1].to("meta")_fake_step(model, rows = 16)model(torch.zeros(1, 16, 4))@@ -501,14 +495,13 @@ def test_auto_never_scales_below_the_measured_length(monkeypatch):ns, found = _load()gib = 1 << 30memory = _fake_cuda(monkeypatch, ns, usable = 4 * gib)- model = _FakeModel([gib // 16] * 8, memory) # max_seq_length 16+ model = _FakeModel([gib // 16] * 8, memory)ns["skip_checkpointing"](model, "auto")planner = model._unsloth_auto_skipplanner.target_batch = 4- _fake_step(model, rows = 64) # a collator expanded the batch past max_seq_length+ _fake_step(model, rows = 64)model(torch.zeros(1, 64, 4))- # 4 rows x 64 measured tokens: scale 4, not 4 x 16 / 64 = 1.- # (4 GiB - 2 GiB) * 0.75 - 1 GiB = 0.5 GiB, two 0.25 GiB layers (before: all 8).+assert len(planner.kept) == 2monkeypatch.undo()@@ -523,8 +516,7 @@ def _trainable_head(model, numel):def test_auto_counts_resident_accumulated_grads_once(monkeypatch, accumulating):ns, found = _load()gib = 1 << 30- numel = gib // 8 # fp32 grad 0.5 GiB, AdamW state + foreach workspace 1.5 GiB- # Resident grads are allocated, so the card has that much less usable memory.+ numel = gib // 8memory = _fake_cuda(monkeypatch, ns, usable = 4 * gib - (gib // 2 if accumulating else 0))model = _FakeModel([gib // 16] * 8, memory)head = _trainable_head(model, numel)@@ -538,7 +530,7 @@ def test_auto_counts_resident_accumulated_grads_once(monkeypatch, accumulating):head.grad = Nonememory["allocated"] -= gib // 2model(torch.zeros(1, 16, 4))- # Both: (4 GiB - 0.5 GiB activations - 0.5 GiB grads - 1.5 GiB) * 0.75 - 1 GiB = 0.125 GiB.+assert len(planner.kept) == 2monkeypatch.undo()@@ -557,11 +549,11 @@ def test_auto_reserves_the_foreach_optimizer_workspace(monkeypatch):memory["allocated"] += gib // 2optimizer = torch.optim.AdamW([head])planner._optimizer_pre_hook(optimizer, (), {})- optimizer.state[head] = {"step": torch.tensor(1.0)} # state allocated by the first step+ optimizer.state[head] = {"step": torch.tensor(1.0)}head.grad = Nonememory["allocated"] -= gib // 2model(torch.zeros(1, 16, 4))- # 0.5 GiB foreach workspace still reserved: (3 - 1 - 0.5) * 0.75 - 1 = 0.125 GiB, two layers.+assert len(planner.kept) == 2monkeypatch.undo()@@ -593,7 +585,7 @@ def test_auto_reserves_pending_state_only_on_the_planned_card(monkeypatch):planner = model._unsloth_auto_skip_fake_step(model, rows = 16)model(torch.zeros(1, 16, 4))- # (3 - 0.5) * 0.75 - 1 = 0.875 GiB: all 8 layers (before: 8 GiB tower state, nothing kept).+assert len(planner.kept) == 8monkeypatch.undo()@@ -609,7 +601,6 @@ def test_counts_apply_only_to_checkpointable_layers():@pytest.mark.skipif(not has_real_cuda(), reason = "the auto planner sizes CUDA memory")@pytest.mark.parametrize("mode", ["fullgraph", "dynamic"])def test_auto_planner_stays_out_of_compiled_regions(mode):- # TrainingArguments(torch_compile = True) compiles the whole model; the planner must stay out of it.from torch._dynamo.utils import countersclass Wrapped(torch.nn.Module):diff --git a/tests/test_gc_skip_auto_dist_cells.py b/tests/test_gc_skip_auto_dist_cells.pyindex b4bd7941122..2007040975c 100644--- a/tests/test_gc_skip_auto_dist_cells.py+++ b/tests/test_gc_skip_auto_dist_cells.py@@ -1,10 +1,9 @@# SPDX-License-Identifier: AGPL-3.0-only# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.-"""checkpoint_skip_layers = "auto" under torchrun, on CPU gloo: which runs take the collective, that-every rank takes it at the same forward, and that DDP training is unchanged by the plan."""import datetime+import mathimport multiprocessingimport osimport sys@@ -27,8 +26,6 @@ def _load(tensor_parallel = False,pipeline_parallel = False,):- """The planner from _utils.py, with its package-relative imports (is_distributed,- is_tensor_parallel, is_pipeline_parallel) answered by stubs instead of falling back."""sys.path.insert(0, HERE)import test_gc_skip_auto as T@@ -53,7 +50,6 @@ def _load(def _spoof_cuda(ns, usable, memory):- # Process-local CPU stand-ins for the CUDA calls the planner makes (as T._fake_cuda does).torch.cuda.is_available = lambda: Truetorch.Tensor.is_cuda = property(lambda self: True)torch.cuda.reset_peak_memory_stats = lambda device = None: None@@ -77,7 +73,6 @@ def _count_collectives():def _fake_scenario(rank, scenario, usable):- """The fake 8-layer model of test_gc_skip_auto, with rank-local extra forwards."""T, ns = _load(tensor_parallel = scenario == "tp",pipeline_parallel = scenario == "pp",@@ -89,11 +84,9 @@ def _fake_scenario(rank, scenario, usable):ns["skip_checkpointing"](model, "auto")planner = model._unsloth_auto_skipif scenario == "rank0_pretrain_forward" and rank == 0:- # A grad-enabled training forward only rank 0 runs before step 1 (no backward).model(torch.zeros(1, 16, 4))T._fake_step(model, rows = 16)if scenario == "rank0_eval_and_generate" and rank == 0:- # Rank-0-only evaluation / sample generation between steps: eval mode, no grad.model.eval()model(torch.zeros(1, 16, 4))model.train()@@ -108,7 +101,6 @@ def _fake_scenario(rank, scenario, usable):def _ddp_scenario(rank, scenario, usable):- """A tiny Qwen3 under DDP (gloo), HF reentrant gradient checkpointing, 4 optimizer steps."""from torch.nn.parallel import DistributedDataParallel as DDPstatic_graph, find_unused, accumulation, auto = scenario@@ -122,7 +114,7 @@ def _ddp_scenario(rank, scenario, usable):model = T._model(layers = 4)if auto:_spoof_cuda(ns, usable, memory)- # Each grad-mode layer call "allocates" 1/16 GiB, so the planner measures a cost.+for layer in model.model.layers:layer.register_forward_hook(lambda m, a, o: memory.__setitem__(@@ -172,8 +164,67 @@ def _ddp_scenario(rank, scenario, usable):}+def _fsdp2_grad_bytes_scenario(rank, scenario, usable):+ from torch.distributed.device_mesh import init_device_mesh+ from torch.distributed.fsdp import fully_shard++ root_only, mode = scenario+ mesh = init_device_mesh("cpu", (torch.distributed.get_world_size(),))+ T, ns = _load()+ model = T._model(layers = 2)+ global_bytes = sum(p.numel() * p.element_size() for p in model.parameters())+ memory = {"allocated": 0}+ _spoof_cuda(ns, usable, memory)+ for layer in model.model.layers:+ layer.register_forward_hook(+ lambda m, a, o: memory.__setitem__(+ "allocated", memory["allocated"] + (GIB // 16 if torch.is_grad_enabled() else 0)+ )+ )+ ns["skip_checkpointing"](model, "auto")+ planner = model._unsloth_auto_skip+ if not root_only:+ for layer in model.model.layers:+ fully_shard(layer, mesh = mesh)+ fully_shard(model, mesh = mesh)+ optimizer = torch.optim.AdamW(model.parameters(), lr = 0.001)+ # Only the planner's CUDA predicates are spoofed; AdamW still runs on CPU.+ optimizer._accelerator_graph_capture_health_check = lambda: None+ parameters = list(model.parameters())+ local_numel = sum(p.to_local().numel() for p in parameters)+ if mode != "sync":+ model.set_requires_gradient_sync(False)+ model.set_reshard_after_backward(mode != "defer")+ ids = torch.randint(0, 128, (2, 16), generator = torch.Generator().manual_seed(rank))+ model(input_ids = ids, labels = ids).loss.backward()+ expected_grad = (+ sum(p.grad.to_local().numel() * p.grad.element_size() for p in parameters)+ if mode == "sync"+ else global_bytes+ )+ measured_grad = planner._fresh_grad_bytes(torch.device("cpu"))+ memory["allocated"] += expected_grad+ if mode == "sync":+ optimizer.step()+ optimizer.zero_grad(set_to_none = True)+ memory["allocated"] -= expected_grad+ model.set_requires_gradient_sync(True)+ model.set_reshard_after_backward(True)+ activation = 2 * (GIB // 16) * 4 + (expected_grad if mode == "sync" else 0)+ optimizer_bytes = local_numel * (4 if mode == "sync" else 12)+ available = math.ceil(activation + optimizer_bytes + (GIB + GIB // 4) / 0.75) + 128+ ns["usable_cuda_bytes"] = lambda device: available+ model(input_ids = ids, labels = ids).loss.backward()+ optimizer.step()+ return {+ "utils": T.UTILS,+ "planner": measured_grad,+ "expected": expected_grad,+ "kept": planner.kept,+ }++def _join_scenario(rank, scenario, usable):- """DDP.join(): rank 0 runs out of inputs before rank 1's planning forward."""from torch.nn.parallel import DistributedDataParallel as DDPT, ns = _load()@@ -230,7 +281,11 @@ def _worker(rank, world, port, kind, scenario, usable, queue):world_size = world,timeout = datetime.timedelta(seconds = 30),)- run = {"fake": _fake_scenario, "join": _join_scenario}.get(kind, _ddp_scenario)+ run = {+ "fake": _fake_scenario,+ "join": _join_scenario,+ "fsdp2_grad_bytes": _fsdp2_grad_bytes_scenario,+ }.get(kind, _ddp_scenario)queue.put((rank, run(rank, scenario, usable[rank]), None))except Exception:import traceback@@ -266,7 +321,6 @@ def _launch(kind, scenario, usable):@pytest.mark.parametrize("scenario", ["rank0_pretrain_forward", "rank0_eval_and_generate"])def test_rank_local_forwards_do_not_desync_the_collective(scenario):- # Rank 0 has room for 8 layers, rank 1 for 2; rank-local forwards must not move rank 0's plan.results = _launch("fake", scenario, [4 * GIB, 2 * GIB])assert results[0]["kept"] == results[1]["kept"] == [3, 7]assert results[0]["all_reduce"] == results[1]["all_reduce"] == 1@@ -318,7 +372,7 @@ def test_single_process_plans_alone_without_a_collective(monkeypatch, world_sizeplanner = model._unsloth_auto_skipT._fake_step(model, rows = 16)model(torch.zeros(1, 16, 4))- # (4 GiB - 0.5 GiB step) * 0.75 - 1 GiB = 1.6 GiB: all 8 sixteenth-GiB layers, as before.+assert planner.kept == list(range(8))monkeypatch.undo()@@ -341,7 +395,6 @@ def _assert_same_training(auto, reference):[(False, False, 1), (False, False, 2), (True, False, 1)],)def test_ddp_training_unchanged_and_ranks_agree(static_graph, find_unused, accumulation):- # Rank 0 fits 4, rank 1 fits 2: both keep 2; losses / weights match full checkpointing.usable = [8 * GIB, 2 * GIB + GIB // 4]auto = _launch("ddp", (static_graph, find_unused, accumulation, True), usable)reference = _launch("ddp", (static_graph, find_unused, accumulation, False), usable)@@ -361,7 +414,6 @@ def _fsdp2_runs_on_cpu():@pytest.mark.skipif(not _fsdp2_runs_on_cpu(), reason = "this torch has no CPU FSDP2 path")@pytest.mark.parametrize("accumulation", [1, 2])def test_fsdp2_training_unchanged_and_ranks_agree(accumulation):- # The collective runs in the root forward pre-hook, next to FSDP2's own all-gathers.usable = [8 * GIB, 2 * GIB + GIB // 4]auto = _launch("ddp", ("fsdp2", False, accumulation, True), usable)reference = _launch("ddp", ("fsdp2", False, accumulation, False), usable)@@ -372,6 +424,17 @@ def test_fsdp2_training_unchanged_and_ranks_agree(accumulation):_assert_same_training(auto[rank], reference[rank])[email protected](not _fsdp2_runs_on_cpu(), reason = "this torch has no CPU FSDP2 path")[email protected]("root_only", [False, True])[email protected]("mode", ["sync", "skip", "defer"])+def test_fsdp2_gradient_bytes_are_rank_local(root_only, mode):+ results = _launch("fsdp2_grad_bytes", (root_only, mode), [8 * GIB, 8 * GIB])+ for rank, r in results.items():+ print("FSDP2_GRAD_BYTES", rank, r)+ assert r["planner"] == r["expected"] > 0, r+ assert r["kept"] == [1], r++def test_ddp_join_with_uneven_inputs_keeps_every_layer_checkpointed():# A joined rank only shadows DDP's collectives, so a planning all_reduce would mispair.usable = [8 * GIB, 8 * GIB]diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.pyindex 445fae78acf..590ca93823a 100644--- a/unsloth/models/_utils.py+++ b/unsloth/models/_utils.py@@ -10641,15 +10641,16 @@ def _gc_keep_layers_env():return 0+def _local_numel(t):+ """Elements this rank holds: an FSDP2 DTensor's numel() is the global size, its shard is local."""+ local = getattr(t, "_local_tensor", None)+ return (local if isinstance(local, torch.Tensor) else t).numel()++class _AutoCheckpointSkip:- """checkpoint_skip_layers = "auto": once, after step 1, un-checkpoint as many layers as fit.- Per-layer cost = the largest allocation any eligible layer's reentrant recompute leaves in the- first backward (hybrid stacks mix attention and Mamba blocks); AdamW state is reserved for every- trainable param the optimizer has no state for yet. Sized for the trainer batch at the longest of- max_length, max_seq_length and the measured length: inputs a collator expands past that (images,- audio) need max_seq_length set to their bound. Under torchrun (DDP / native FSDP2) every rank- plans and the smallest K wins, so every rank keeps the same number of layers; tensor / pipeline parallel- models stay fully checkpointed."""+ """Plan checkpoint skips once from measured memory, using the smallest count across ranks.+ Multimodal inputs must stay within max_seq_length; tensor/pipeline parallel models are excluded.+ """MARGIN = 1 << 30FRACTION = 0.75@@ -10675,6 +10676,7 @@ class _AutoCheckpointSkip:self.optimizers = []self.fresh_grads = Noneself.grad_bytes = None+ self.fsdp_params = {}@classmethoddef _no_cuda(cls):@@ -10756,6 +10758,53 @@ class _AutoCheckpointSkip:return p.devicereturn None+ def _parameters(self):+ """Resolve optimizer-owned shards while FSDP exposes temporary gathered parameters."""+ aliases = {}+ seen = set()+ for module in self.model.modules():+ get_state = getattr(module, "_get_fsdp_state", None)+ if not callable(get_state):+ continue+ state = get_state()+ if id(state) in seen:+ continue+ seen.add(id(state))+ groups = getattr(state, "_fsdp_param_groups", None)+ if groups is None:+ group = getattr(state, "_fsdp_param_group", None)+ groups = () if group is None else (group,)+ for group in groups:+ for fp in group.fsdp_params:+ for name in (+ "sharded_param",+ "_unsharded_param",+ "_sharded_post_forward_param",+ ):+ param = getattr(fp, name, None)+ if param is not None:+ aliases[id(param)] = fp+ self.fsdp_params = {}+ parameters = {}+ for param in self.model.parameters():+ fp = aliases.get(id(param))+ if fp is not None:+ param = fp.sharded_param+ self.fsdp_params[id(param)] = fp+ parameters[id(param)] = param+ return parameters.values()++ def _gradient(self, param):+ if param.grad is not None:+ return param.grad+ fp = self.fsdp_params.get(id(param))+ if fp is None:+ return None+ accumulated = getattr(fp, "unsharded_accumulated_grad", None)+ if accumulated is not None:+ return accumulated+ return getattr(getattr(fp, "_unsharded_param", None), "grad", None)+def _model_pre_hook(self, module, args):# Planning under torch.compile breaks fullgraph/dynamic tracing; compiled models stay fully checkpointed.if torch.compiler.is_compiling():@@ -10769,9 +10818,8 @@ class _AutoCheckpointSkip:# Step-local peak: a model-loading spike must not count as activations.torch.cuda.reset_peak_memory_stats(device)self.started = torch.cuda.memory_allocated(device)- # Grads the first backward allocates persist: they are not sequence-dependent.self.fresh_grads = [- p for p in self.model.parameters() if p.requires_grad and p.grad is None+ p for p in self._parameters() if p.requires_grad and self._gradient(p) is None]returnif self.layer_bytes is not None:@@ -10831,7 +10879,7 @@ class _AutoCheckpointSkip:def _optimizer_pre_hook(self, optimizer, args, kwargs):if self.started is None or self.stepped:return- trainable = {id(p) for p in self.model.parameters() if p.requires_grad}+ trainable = {id(p) for p in self._parameters() if p.requires_grad}if not any(id(p) in trainable for group in optimizer.param_groups for p in group["params"]):returnif not any(o() is optimizer for o in self.optimizers):@@ -10846,11 +10894,10 @@ class _AutoCheckpointSkip:self.stepped = Truedef _fresh_grad_bytes(self, device):- # Only grads on the planned card are in its step peak (a vision tower may sit on another GPU).return sum(- p.grad.numel() * p.grad.element_size()+ _local_numel(grad) * grad.element_size()for p in (self.fresh_grads or ())- if p.grad is not None and p.grad.is_cuda and p.grad.device == device+ if (grad := self._gradient(p)) is not None and grad.is_cuda and grad.device == device)def _has_optimizer_state(self, param):@@ -10914,10 +10961,8 @@ class _AutoCheckpointSkip:if self.step_peak is not Noneelse torch.cuda.max_memory_allocated(device))- # A short or small first batch (dynamic padding, packing, token-budget batches) under-measures:- # size for the trainer's batch x max length tokens, else the measured batch at max_seq_length.+ # A short first batch must still reserve activations for the trainer's maximum batch/length.rows = max(1, self.layer_rows)- # Never below the measured length (multimodal collators can expand past max_seq_length).max_seq = max(rows, self.target_seq or getattr(self.model, "max_seq_length", None) or rows)batch = max(self.target_batch or 0, self.layer_batch)tokens = max(1.0, batch * max_seq / (self.layer_batch * rows))@@ -10933,18 +10978,17 @@ class _AutoCheckpointSkip:resident = self._fresh_grad_bytes(device) if self.fresh_grads else 0activation_peak = (step_bytes - grad_bytes) * scale + max(0, grad_bytes - resident)self.fresh_grads = None- # State an optimizer step already allocated is excluded by usable_cuda_bytes; reserve the rest- # (sparse MoE experts, unused towers and gradient accumulation have none yet).- trainable = [p for p in self.model.parameters() if p.requires_grad]+ # Existing optimizer state is already excluded by usable_cuda_bytes.+ trainable = [p for p in self._parameters() if p.requires_grad]optimizer = sum(- p.numel() * 8+ _local_numel(p) * 8for p in trainableif p.device == device and not self._has_optimizer_state(p))- # foreach AdamW allocates about one fp32 copy of the params (sqrt of exp_avg_sq) every step,- # on each param's own card.- optimizer += sum(p.numel() * 4 for p in trainable if p.device == device)+ # foreach AdamW needs one temporary fp32 parameter-sized buffer on the parameter's device.+ optimizer += sum(_local_numel(p) * 4 for p in trainable if p.device == device)self.optimizers = []+ self.fsdp_params.clear()headroom = usable_cuda_bytes(device) - activation_peak - optimizerbudget = headroom * self.FRACTION - self.MARGINper_layer = self.layer_bytes * scale