Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

checkpoint_skip_layers="auto" under torchrun: every rank keeps the smallest planned count (#15545)

Daniel Han570b9b827baf
diff --git a/tests/test_gc_skip_auto_dist_cells.py b/tests/test_gc_skip_auto_dist_cells.py
new file mode 100644
index 00000000000..c4ddbd811e3
--- /dev/null
+++ b/tests/test_gc_skip_auto_dist_cells.py
@@ -0,0 +1,337 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""checkpoint_skip_layers = "auto" under torchrun, on CPU gloo: which runs take the collective, that
+every rank takes it at the same forward, and that DDP training is unchanged by the plan."""
+
+import datetime
+import multiprocessing
+import os
+import socket
+import sys
+import types
+
+import pytest
+import torch
+
+HERE = os.path.dirname(os.path.abspath(__file__))
+GIB = 1 << 30
+
+
+def _free_port():
+ with socket.socket() as s:
+ s.bind(("127.0.0.1", 0))
+ return s.getsockname()[1]
+
+
+def _load(
+ distributed = None,
+ tensor_parallel = False,
+ pipeline_parallel = False,
+):
+ """The planner from _utils.py, with its package-relative imports (is_distributed,
+ is_tensor_parallel, is_pipeline_parallel) answered by stubs instead of falling back."""
+ sys.path.insert(0, HERE)
+ import test_gc_skip_auto as T
+
+ ns, _ = T._load()
+ ns["__package__"] = "unsloth_s9stub.models"
+ package = types.ModuleType("unsloth_s9stub")
+ package.__path__ = []
+ models = types.ModuleType("unsloth_s9stub.models")
+ models.__path__ = []
+ loader = types.ModuleType("unsloth_s9stub.models.loader_utils")
+ if distributed is None:
+ loader.is_distributed = lambda: int(os.environ.get("WORLD_SIZE", "1")) > 1
+ else:
+ loader.is_distributed = lambda: distributed
+ tp = types.ModuleType("unsloth_s9stub.models.tp_native")
+ tp.is_tensor_parallel = lambda model: tensor_parallel
+ pp = types.ModuleType("unsloth_s9stub.models.pp_native")
+ pp.is_pipeline_parallel = lambda model: pipeline_parallel
+ for module in (package, models, loader, tp, pp):
+ sys.modules[module.__name__] = module
+ return T, ns
+
+
+def _spoof_cuda(ns, usable, memory):
+ # Process-local CPU stand-ins for the CUDA calls the planner makes (as T._fake_cuda does).
+ torch.cuda.is_available = lambda: True
+ torch.Tensor.is_cuda = property(lambda self: True)
+ torch.cuda.reset_peak_memory_stats = lambda device = None: None
+ torch.cuda.memory_allocated = lambda device = None: memory["allocated"]
+ torch.cuda.max_memory_allocated = lambda device = None: memory["allocated"]
+ ns["usable_cuda_bytes"] = lambda device: usable
+
+
+def _count_collectives():
+ import torch.distributed as dist
+
+ calls = {"all_reduce": 0}
+ real = dist.all_reduce
+
+ def counted(*args, **kwargs):
+ calls["all_reduce"] += 1
+ return real(*args, **kwargs)
+
+ dist.all_reduce = counted
+ return calls
+
+
+def _fake_scenario(rank, scenario, usable):
+ """The fake 8-layer model of test_gc_skip_auto, with rank-local extra forwards."""
+ T, ns = _load(
+ tensor_parallel = scenario == "tp",
+ pipeline_parallel = scenario == "pp",
+ )
+ memory = {"allocated": 0}
+ _spoof_cuda(ns, usable, memory)
+ calls = _count_collectives()
+ model = T._FakeModel([GIB // 16] * 8, memory)
+ ns["skip_checkpointing"](model, "auto")
+ planner = model._unsloth_auto_skip
+ if scenario == "rank0_pretrain_forward" and rank == 0:
+ # A grad-enabled training forward only rank 0 runs before step 1 (no backward).
+ model(torch.zeros(1, 16, 4))
+ T._fake_step(model, rows = 16)
+ if scenario == "rank0_eval_and_generate" and rank == 0:
+ # Rank-0-only evaluation / sample generation between steps: eval mode, no grad.
+ model.eval()
+ model(torch.zeros(1, 16, 4))
+ model.train()
+ with torch.no_grad():
+ model(torch.zeros(1, 16, 4))
+ with torch.inference_mode():
+ model(torch.zeros(1, 16, 4))
+ model(torch.zeros(1, 16, 4))
+ T._fake_step(model, rows = 16)
+ model(torch.zeros(1, 16, 4))
+ return {"kept": list(planner.kept), "all_reduce": calls["all_reduce"]}
+
+
+def _ddp_scenario(rank, scenario, usable):
+ """A tiny Qwen3 under DDP (gloo), HF reentrant gradient checkpointing, 4 optimizer steps."""
+ from torch.nn.parallel import DistributedDataParallel as DDP
+
+ static_graph, find_unused, accumulation, auto = scenario
+ if static_graph == "fsdp2":
+ from torch.distributed.device_mesh import init_device_mesh
+
+ # A CPU mesh built before the CUDA stand-ins, which would make it pick cuda / NCCL.
+ mesh = init_device_mesh("cpu", (torch.distributed.get_world_size(),))
+ T, ns = _load()
+ memory = {"allocated": 0}
+ model = T._model(layers = 4)
+ if auto:
+ _spoof_cuda(ns, usable, memory)
+ # Each grad-mode layer call "allocates" 1/16 GiB, so the planner measures a cost.
+ for layer in model.model.layers:
+ layer.register_forward_hook(
+ lambda m, a, o: memory.__setitem__(
+ "allocated", memory["allocated"] + (GIB // 16 if torch.is_grad_enabled() else 0)
+ )
+ )
+ ns["skip_checkpointing"](model, "auto")
+ planner = getattr(model, "_unsloth_auto_skip", None)
+ calls = _count_collectives()
+ if static_graph == "fsdp2":
+ try:
+ from torch.distributed.fsdp import fully_shard
+ except ImportError:
+ from torch.distributed._composable.fsdp import fully_shard
+ for layer in model.model.layers:
+ fully_shard(layer, mesh = mesh)
+ fully_shard(model, mesh = mesh)
+ ddp = model
+ ddp.no_sync = _null
+ else:
+ ddp = DDP(model, static_graph = static_graph, find_unused_parameters = find_unused)
+ optimizer = torch.optim.SGD(model.parameters(), lr = 0.1)
+ generator = torch.Generator().manual_seed(100 + rank)
+ losses = []
+ for step in range(4):
+ for micro in range(accumulation):
+ ids = torch.randint(0, 128, (2, 32), generator = generator)
+ context = ddp.no_sync() if micro < accumulation - 1 else _null()
+ with context:
+ loss = ddp(input_ids = ids, labels = ids).loss / accumulation
+ loss.backward()
+ losses.append(float(loss.detach()))
+ optimizer.step()
+ optimizer.zero_grad(set_to_none = True)
+ digest = float(
+ sum(
+ (p.full_tensor() if hasattr(p, "full_tensor") else p).detach().double().sum()
+ for p in model.parameters()
+ )
+ )
+ return {
+ "kept": None if planner is None else (None if planner.kept is None else list(planner.kept)),
+ "checkpointed": [bool(layer.gradient_checkpointing) for layer in model.model.layers],
+ "losses": losses,
+ "digest": digest,
+ "all_reduce": calls["all_reduce"],
+ }
+
+
+class _null:
+ def __enter__(self):
+ return self
+
+ def __exit__(self, *exc):
+ return False
+
+
+def _worker(rank, world, port, kind, scenario, usable, queue):
+ import torch.distributed as dist
+
+ os.environ["WORLD_SIZE"] = str(world)
+ os.environ["RANK"] = str(rank)
+ try:
+ dist.init_process_group(
+ "gloo",
+ init_method = f"tcp://127.0.0.1:{port}",
+ rank = rank,
+ world_size = world,
+ timeout = datetime.timedelta(seconds = 30),
+ )
+ run = _fake_scenario if kind == "fake" else _ddp_scenario
+ queue.put((rank, run(rank, scenario, usable[rank]), None))
+ except Exception:
+ import traceback
+ queue.put((rank, None, traceback.format_exc()))
+ finally:
+ if dist.is_initialized():
+ dist.destroy_process_group()
+
+
+def _launch(kind, scenario, usable):
+ ctx = multiprocessing.get_context("spawn")
+ queue = ctx.Queue()
+ port = _free_port()
+ procs = [
+ ctx.Process(target = _worker, args = (r, len(usable), port, kind, scenario, usable, queue))
+ for r in range(len(usable))
+ ]
+ for p in procs:
+ p.start()
+ results = dict()
+ try:
+ for _ in procs:
+ rank, result, error = queue.get(timeout = 240)
+ assert error is None, error
+ results[rank] = result
+ finally:
+ for p in procs:
+ p.join(timeout = 60)
+ if p.is_alive():
+ p.kill()
+ return results
+
+
[email protected]("scenario", ["rank0_pretrain_forward", "rank0_eval_and_generate"])
+def test_rank_local_forwards_do_not_desync_the_collective(scenario):
+ # Rank 0 has room for 8 layers, rank 1 for 2; rank-local forwards must not move rank 0's plan.
+ results = _launch("fake", scenario, [4 * GIB, 2 * GIB])
+ assert results[0]["kept"] == results[1]["kept"] == [3, 7]
+ assert results[0]["all_reduce"] == results[1]["all_reduce"] == 1
+
+
[email protected]("scenario", ["tp", "pp"])
+def test_tensor_and_pipeline_parallel_stay_checkpointed_without_a_collective(scenario, capfd):
+ results = _launch("fake", scenario, [4 * GIB, 4 * GIB])
+ for rank in (0, 1):
+ assert results[rank]["kept"] == []
+ assert results[rank]["all_reduce"] == 0
+
+
+def test_process_group_missing_under_torchrun_keeps_every_layer(monkeypatch, capsys):
+ monkeypatch.setenv("WORLD_SIZE", "2")
+ T, ns = _load(distributed = True)
+ memory = T._fake_cuda(monkeypatch, ns, usable = 4 * GIB)
+ import torch.distributed as dist
+
+ assert not dist.is_initialized()
+ monkeypatch.setattr(
+ dist, "all_reduce", lambda *a, **k: pytest.fail("collective without a group")
+ )
+ model = T._FakeModel([GIB // 16] * 8, memory)
+ ns["skip_checkpointing"](model, "auto")
+ planner = model._unsloth_auto_skip
+ T._fake_step(model, rows = 16)
+ model(torch.zeros(1, 16, 4))
+ assert planner.kept == []
+ assert "needs the process group before the second step" in capsys.readouterr().out
+ monkeypatch.undo()
+
+
[email protected]("world_size_env", [None, "1"])
+def test_single_process_plans_alone_without_a_collective(monkeypatch, world_size_env):
+ if world_size_env is None:
+ monkeypatch.delenv("WORLD_SIZE", raising = False)
+ else:
+ monkeypatch.setenv("WORLD_SIZE", world_size_env)
+ T, ns = _load()
+ memory = T._fake_cuda(monkeypatch, ns, usable = 4 * GIB)
+ import torch.distributed as dist
+
+ monkeypatch.setattr(
+ dist, "all_reduce", lambda *a, **k: pytest.fail("collective on one process")
+ )
+ model = T._FakeModel([GIB // 16] * 8, memory)
+ ns["skip_checkpointing"](model, "auto")
+ planner = model._unsloth_auto_skip
+ T._fake_step(model, rows = 16)
+ model(torch.zeros(1, 16, 4))
+ # (4 GiB - 0.5 GiB step) * 0.75 - 1 GiB = 1.6 GiB: all 8 sixteenth-GiB layers, as before.
+ assert planner.kept == list(range(8))
+ monkeypatch.undo()
+
+
+def test_world_size_one_process_group_plans_alone():
+ results = _launch("fake", "agree", [4 * GIB])
+ assert results[0]["kept"] == list(range(8))
+ assert results[0]["all_reduce"] == 0
+
+
+def _assert_same_training(auto, reference):
+ # Skipping the recompute may change fp32 rounding by an ulp on some torch versions (2.13 CPU).
+ torch.testing.assert_close(auto["losses"], reference["losses"], rtol = 1e-6, atol = 1e-6)
+ torch.testing.assert_close(auto["digest"], reference["digest"], rtol = 1e-6, atol = 1e-6)
+
+
+ "static_graph, find_unused, accumulation",
+ # static_graph + no_sync and find_unused_parameters fail with reentrant checkpointing regardless (DDP limits).
+ [(False, False, 1), (False, False, 2), (True, False, 1)],
+)
+def test_ddp_training_unchanged_and_ranks_agree(static_graph, find_unused, accumulation):
+ # Rank 0 fits 4, rank 1 fits 2: both keep 2; losses / weights match full checkpointing.
+ usable = [8 * GIB, 2 * GIB + GIB // 4]
+ auto = _launch("ddp", (static_graph, find_unused, accumulation, True), usable)
+ reference = _launch("ddp", (static_graph, find_unused, accumulation, False), usable)
+ assert auto[0]["kept"] == auto[1]["kept"]
+ assert 0 < len(auto[1]["kept"]) < 4
+ assert auto[0]["checkpointed"] == auto[1]["checkpointed"]
+ for rank in (0, 1):
+ assert reference[rank]["checkpointed"] == [True] * 4
+ _assert_same_training(auto[rank], reference[rank])
+
+
+def _fsdp2_runs_on_cpu():
+ # torch 2.6's FSDP2 all-gather records events on a CPU stream that has no record_event.
+ return hasattr(torch.cpu.current_stream(), "record_event")
+
+
[email protected](not _fsdp2_runs_on_cpu(), reason = "this torch has no CPU FSDP2 path")
[email protected]("accumulation", [1, 2])
+def test_fsdp2_training_unchanged_and_ranks_agree(accumulation):
+ # The collective runs in the root forward pre-hook, next to FSDP2's own all-gathers.
+ usable = [8 * GIB, 2 * GIB + GIB // 4]
+ auto = _launch("ddp", ("fsdp2", False, accumulation, True), usable)
+ reference = _launch("ddp", ("fsdp2", False, accumulation, False), usable)
+ assert auto[0]["kept"] == auto[1]["kept"]
+ assert 0 < len(auto[1]["kept"]) < 4
+ for rank in (0, 1):
+ assert reference[rank]["checkpointed"] == [True] * 4
+ _assert_same_training(auto[rank], reference[rank])
diff --git a/tests/test_gc_skip_auto_dist_gloo.py b/tests/test_gc_skip_auto_dist_gloo.py
new file mode 100644
index 00000000000..17bf1e5c624
--- /dev/null
+++ b/tests/test_gc_skip_auto_dist_gloo.py
@@ -0,0 +1,76 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""checkpoint_skip_layers = "auto" under torchrun: every rank keeps the same (smallest) set of layers."""
+
+import multiprocessing
+import os
+import socket
+import sys
+
+import pytest
+import torch
+
+HERE = os.path.dirname(os.path.abspath(__file__))
+GIB = 1 << 30
+
+
+def _free_port():
+ with socket.socket() as s:
+ s.bind(("127.0.0.1", 0))
+ return s.getsockname()[1]
+
+
+def _worker(rank, world, port, usable, queue):
+ import torch.distributed as dist
+ sys.path.insert(0, HERE)
+ try:
+ dist.init_process_group(
+ "gloo", init_method = f"tcp://127.0.0.1:{port}", rank = rank, world_size = world
+ )
+ import test_gc_skip_auto as T
+
+ ns, _ = T._load()
+ memory = {"allocated": 0}
+ # Process-local CPU stand-ins for the CUDA calls the planner makes (as T._fake_cuda does).
+ torch.cuda.is_available = lambda: True
+ torch.Tensor.is_cuda = property(lambda self: True)
+ torch.cuda.reset_peak_memory_stats = lambda device = None: None
+ torch.cuda.memory_allocated = lambda device = None: memory["allocated"]
+ torch.cuda.max_memory_allocated = lambda device = None: memory["allocated"]
+ ns["usable_cuda_bytes"] = lambda device: usable
+ model = T._FakeModel([GIB // 16] * 8, memory)
+ ns["skip_checkpointing"](model, "auto")
+ planner = model._unsloth_auto_skip
+ T._fake_step(model, rows = 16)
+ model(torch.zeros(1, 16, 4))
+ queue.put((rank, list(planner.kept), None))
+ except Exception as error: # reported to the parent instead of hanging it
+ import traceback
+ queue.put((rank, None, traceback.format_exc()))
+ finally:
+ if dist.is_initialized():
+ dist.destroy_process_group()
+
+
+def test_auto_agrees_on_the_smallest_count_across_ranks():
+ # Rank 0 has room for all 8 layers, rank 1 for 2: both must keep the same 2.
+ # (4 GiB - 0.5 GiB step) * 0.75 - 1 GiB = 1.6 GiB -> 8; (2 GiB - 0.5) * 0.75 - 1 = 0.125 GiB -> 2.
+ ctx = multiprocessing.get_context("spawn")
+ queue = ctx.Queue()
+ port = _free_port()
+ usable = [4 * GIB, 2 * GIB]
+ procs = [ctx.Process(target = _worker, args = (r, 2, port, usable[r], queue)) for r in range(2)]
+ for p in procs:
+ p.start()
+ results = dict()
+ for _ in procs:
+ rank, kept, error = queue.get(timeout = 200)
+ assert error is None, error
+ results[rank] = kept
+ for p in procs:
+ p.join(timeout = 60)
+ assert p.exitcode == 0
+ assert len(results[1]) == 2
+ assert results[0] == results[1]
diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.py
index 67c386c59f6..5fb1d11d2d0 100644
--- a/unsloth/models/_utils.py
+++ b/unsloth/models/_utils.py
@@ -10055,8 +10055,9 @@ class _AutoCheckpointSkip:
first backward (hybrid stacks mix attention and Mamba blocks); AdamW state is reserved for every
trainable param the optimizer has no state for yet. Sized for the trainer batch at the longest of
max_length, max_seq_length and the measured length: inputs a collator expands past that (images,
- audio) need max_seq_length set to their bound. Single process only: under torchrun each rank
- would pick its own K."""
+ audio) need max_seq_length set to their bound. Under torchrun (DDP / native FSDP2) every rank
+ plans and the smallest K wins, so every rank keeps the same number of layers; tensor / pipeline parallel
+ models stay fully checkpointed."""
MARGIN = 1 << 30
FRACTION = 0.75
@@ -10096,16 +10097,6 @@ class _AutoCheckpointSkip:
def install(cls, model):
if not torch.cuda.is_available():
return cls._no_cuda()
- try:
- from .loader_utils import is_distributed
- distributed = is_distributed()
- except Exception:
- distributed = False
- if distributed:
- print(
- 'Unsloth: checkpoint_skip_layers = "auto" is single process only; every layer stays checkpointed.'
- )
- return None
previous = getattr(model, "_unsloth_auto_skip", None)
if previous is not None:
previous.remove()
@@ -10277,6 +10268,40 @@ class _AutoCheckpointSkip:
return True
return False
+ def _agree_across_ranks(self, count, device):
+ """torchrun: MIN of count over ranks (one collective, at the planning forward every rank reaches)."""
+ try:
+ from .loader_utils import is_distributed
+ distributed = is_distributed()
+ except Exception:
+ distributed = False
+ dist = getattr(torch, "distributed", None)
+ initialized = dist is not None and dist.is_available() and dist.is_initialized()
+ if not distributed and not (initialized and dist.get_world_size() > 1):
+ return count
+ try:
+ from .tp_native import is_tensor_parallel
+ from .pp_native import is_pipeline_parallel
+ sharded = is_tensor_parallel(self.model) or is_pipeline_parallel(self.model)
+ except Exception:
+ sharded = False
+ if sharded:
+ # Stages / shards hold different layers and memory: one K does not fit all ranks.
+ print(
+ 'Unsloth: checkpoint_skip_layers = "auto" does not plan tensor / pipeline parallel models; every layer stays checkpointed.'
+ )
+ return 0
+ if not initialized:
+ print(
+ 'Unsloth: checkpoint_skip_layers = "auto" under torchrun needs the process group before the second step; every layer stays checkpointed.'
+ )
+ return 0
+ backend = str(dist.get_backend())
+ on = device if "nccl" in backend and device is not None else torch.device("cpu")
+ agreed = torch.tensor([int(count)], dtype = torch.int64, device = on)
+ dist.all_reduce(agreed, op = dist.ReduceOp.MIN)
+ return int(agreed.item())
+
def plan(self, device):
self.remove()
allocated_start = self.started
@@ -10320,6 +10345,7 @@ class _AutoCheckpointSkip:
budget = headroom * self.FRACTION - self.MARGIN
per_layer = self.layer_bytes * scale
count = int(budget // per_layer) if per_layer > 0 and budget > 0 else 0
+ count = self._agree_across_ranks(count, device)
self.kept = skip_checkpointing(self.model, count) if count > 0 else []
if self.kept:
# Smart offloading learnt the checkpointed-call count in the first step; recount it.