Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Parallel capabilities: follow the measured composition evidence (packed CP, Unsloth checkpoint, tiling, block swap) (#16773)

Daniel Han30949ea01dac
diff --git a/tests/test_parallel_capabilities_evidence.py b/tests/test_parallel_capabilities_evidence.py
new file mode 100644
index 00000000000..26d2fb4c207
--- /dev/null
+++ b/tests/test_parallel_capabilities_evidence.py
@@ -0,0 +1,336 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""The registry follows the measured composition evidence. CPU only."""
+
+import dataclasses
+import types
+
+import pytest
+import torch
+
+import unsloth # noqa: F401
+from unsloth import parallel_capabilities as caps
+from unsloth.parallel_env import ParallelLayout
+
+TINY = "trl-internal-testing/tiny-Qwen3ForCausalLM"
+
+_ENV = (
+ "UNSLOTH_PARALLEL",
+ "UNSLOTH_TP_SIZE",
+ "UNSLOTH_PP_SIZE",
+ "UNSLOTH_CP_SIZE",
+ "UNSLOTH_FSDP2",
+ "UNSLOTH_PARALLEL_STRICT",
+ "WORLD_SIZE",
+)
+
+
[email protected](autouse = True)
+def clean_env(monkeypatch):
+ for key in _ENV:
+ monkeypatch.delenv(key, raising = False)
+ yield monkeypatch
+
+
+def _tiny():
+ from transformers import AutoConfig, AutoModelForCausalLM
+
+ try:
+ config = AutoConfig.from_pretrained(TINY)
+ except Exception as e: # offline host without the tiny repo cached
+ pytest.skip(f"{TINY}: {e}")
+ config.num_hidden_layers = 2
+ torch.manual_seed(0)
+ return AutoModelForCausalLM.from_config(config)
+
+
+def _packed(**kw):
+ base = dict(
+ trainer = "sft",
+ family = "qwen3",
+ packing = True,
+ cp = 2,
+ dp = 1,
+ cp_backend = "ulysses",
+ attention = "sdpa",
+ compile = "eager",
+ mode = "lora",
+ torch = (2, 13),
+ )
+ base.update(kw)
+ return caps.Query(**base)
+
+
+# Packed / padding-free rows x cp Ulysses.
+
+
+ "kw",
+ [
+ {},
+ {"family": "gemma3", "windowed": True, "attention": "eager"},
+ {"family": "mistral", "windowed": True},
+ {"family": "gemma2", "windowed": True, "attention": "eager"},
+ {"mode": "full"},
+ ],
+)
+def test_packed_rows_under_generic_ulysses_eager_are_supported(kw):
+ status, cell = caps.cell_status(_packed(**kw))
+ assert status == "supported", cell
+ assert "packed rows x cp Ulysses" in cell.name and not cell.missing()
+ assert "test_generic_ulysses_matches_one_process" in cell.parity
+ caps.check(_packed(**kw))
+
+
+def test_packed_rows_through_query_from_reach_the_eager_cell():
+ model = _tiny()
+ layout = types.SimpleNamespace(dp = 1, tp = 1, pp = 1, cp = 2, fsdp2 = False, cp_backend = "ulysses")
+ args = types.SimpleNamespace(packing = False, padding_free = True, gradient_checkpointing = False)
+ trl = pytest.importorskip("trl")
+ query = caps.query_from(
+ layout, trl.SFTTrainer, model, args, cp_backend = "ulysses", peft_config = object()
+ )
+ assert query.packing and query.family == "qwen3" and query.memory == "none"
+ status, cell = caps.cell_status(query)
+ assert status == "supported" and "packed rows" in cell.name, (status, cell)
+
+
+def test_packed_rows_under_block_compile_stay_unverified_with_the_gap(clean_env):
+ query = _packed(compile = "block")
+ status, detail = caps.cell_status(query)
+ assert status == "unverified"
+ assert "lacks engagement, compile evidence" in detail and "breaks the graph" in detail
+ caps.check(query) # not gated: runs, as before, outside strict mode
+ clean_env.setenv("UNSLOTH_PARALLEL_STRICT", "1")
+ with pytest.raises(NotImplementedError, match = "breaks the graph"):
+ caps.check(query)
+
+
+ "kw",
+ [
+ {"dp": 2}, # dp 2 x cp 2 was not measured
+ {"fsdp2": True},
+ {"family": "llama"},
+ {"kernels": True, "loss": "fused_ce"}, # Unsloth's fast attention is not the generic path
+ {"memory": "unsloth_offload"},
+ ],
+)
+def test_packed_rows_outside_the_measured_cell_are_not_supported(kw):
+ assert caps.cell_status(_packed(**kw))[0] != "supported"
+
+
+def test_kept_refusals_flex_and_ring_with_packing_or_window():
+ with pytest.raises(NotImplementedError, match = "flex attention block masks"):
+ caps.check(_packed(attention = "flex"))
+ with pytest.raises(NotImplementedError, match = "ring context parallel with packed rows"):
+ caps.check(_packed(cp_backend = "ring", sm = 9))
+ with pytest.raises(NotImplementedError, match = "sliding window"):
+ caps.check(_packed(cp_backend = "ring", packing = False, windowed = True, sm = 9))
+
+
+# Memory: Unsloth's checkpoint and tiling.
+
+
+ "fn_name, offloads, want",
+ [
+ ("unsloth_checkpoint", None, "unsloth_offload"), # path A (UnslothCheckpointFunction)
+ ("_nonre_checkpoint", True, "unsloth_offload"), # path B with offload hooks
+ # path B without offload: a plain non-reentrant checkpoint
+ ("_nonre_checkpoint", False, "gc"),
+ ("unsloth_offloaded_gradient_checkpoint", None, "unsloth_offload_legacy"), # not measured
+ ("unsloth_gradient_checkpoint", None, "unsloth_gc"),
+ ("checkpoint", None, "gc"),
+ ],
+)
+def test_memory_reports_unsloth_checkpoints(monkeypatch, fn_name, offloads, want):
+ import torch.utils.checkpoint as checkpoint
+
+ def fn(function, *args, **kwargs):
+ return function(*args, **kwargs)
+
+ fn.__name__ = fn_name
+ if offloads is not None:
+ fn._unsloth_offloads = offloads
+ monkeypatch.setattr(checkpoint, "checkpoint", fn)
+ args = types.SimpleNamespace(gradient_checkpointing = True)
+ assert caps._memory(None, args) == want
+ assert caps._memory(None, types.SimpleNamespace(gradient_checkpointing = False)) == "none"
+
+
+def test_memory_reports_the_zoo_unsloth_checkpoint(monkeypatch):
+ zoo = pytest.importorskip("unsloth_zoo.gradient_checkpointing")
+ import torch.utils.checkpoint as checkpoint
+
+ monkeypatch.setattr(checkpoint, "checkpoint", zoo.unsloth_checkpoint)
+ assert (
+ caps._memory(None, types.SimpleNamespace(gradient_checkpointing = True)) == "unsloth_offload"
+ )
+
+
+def test_unsloth_checkpoint_is_seen_and_covered_by_the_cells_that_measured_it(monkeypatch):
+ zoo = pytest.importorskip("unsloth_zoo.gradient_checkpointing")
+ import torch.utils.checkpoint as checkpoint
+
+ monkeypatch.setattr(checkpoint, "checkpoint", zoo.unsloth_checkpoint)
+ memory = caps._memory(None, types.SimpleNamespace(gradient_checkpointing = True))
+ assert memory == "unsloth_offload"
+ for kw, name in (
+ (dict(family = "llama", tp = 2), "cp x tp (DDP dp), eager"),
+ (dict(family = "llama", tp = 2, dp = 2, fsdp2 = True), "cp x tp (FSDP2 dp), eager"),
+ (dict(family = "llama", tp = 2, compile = "block"), "cp x tp, Unsloth block compile"),
+ (dict(family = "llama", fsdp2 = True), "cp x native FSDP2 (dp_shard x cp), eager"),
+ (
+ dict(family = "qwen3_moe", moe = True, expert_lora = True),
+ "MoE qwen3_moe x lora+experts x aux off x cp",
+ ),
+ (
+ dict(family = "qwen3_moe", moe = True, mode = "full", compile = "block"),
+ "MoE qwen3_moe x full x aux off x cp, block compile",
+ ),
+ ):
+ query = caps.Query(
+ trainer = "sft", cp = 2, cp_backend = "ulysses", memory = memory, torch = (2, 13), **kw
+ )
+ status, cell = caps.cell_status(query)
+ assert status == "supported" and cell.name == name, (kw, status, cell)
+ assert "unsloth_checkpointing" in cell.note, cell.note
+ caps.check(query)
+
+
+def test_unsloth_checkpoint_keeps_an_unverified_moe_row_refused_under_strict(
+ clean_env, monkeypatch
+):
+ zoo = pytest.importorskip("unsloth_zoo.gradient_checkpointing")
+ import torch.utils.checkpoint as checkpoint
+
+ monkeypatch.setattr(checkpoint, "checkpoint", zoo.unsloth_checkpoint)
+ memory = caps._memory(None, types.SimpleNamespace(gradient_checkpointing = True))
+ query = caps.Query(trainer = "sft", family = "qwen3_moe", moe = True, dp = 2, memory = memory)
+ assert caps.cell_status(query)[0] == "unverified"
+ clean_env.setenv("UNSLOTH_PARALLEL_STRICT", "1")
+ with pytest.raises(NotImplementedError, match = "MoE qwen3_moe x lora x aux off x ddp"):
+ caps.check(query)
+
+
+def test_tiling_is_its_own_flag():
+ pytest.importorskip("unsloth_zoo.tiling_registry")
+ from unsloth.tiling import tile_mlp
+
+ model = _tiny()
+ args = types.SimpleNamespace(gradient_checkpointing = True)
+ assert not caps.query_from(None, None, model, args).tiled
+ tile_mlp(model)
+ query = caps.query_from(None, None, model, args)
+ assert query.tiled and query.memory != "tiled_mlp"
+ assert {"tiled_mlp", "gradient_checkpointing"} <= set(caps.check_features(query))
+
+
+def test_every_feature_module_path_exists():
+ import os
+ import re
+
+ root = os.path.dirname(caps.__file__)
+ for feature in caps.FEATURES:
+ local = feature.module.split("unsloth_zoo")[0]
+ for path in re.findall(r"[\w/]+\.py", local):
+ assert os.path.exists(os.path.join(root, path)), (feature.name, path)
+
+
+# Block swap.
+
+
+def _swap(model, variant):
+ layers = model.model.layers
+ if variant == "count":
+ swapper = types.SimpleNamespace(indices = [1])
+ layers._unsloth_block_swap = swapper
+ model._unsloth_block_swap = swapper
+ elif variant == "empty": # a swapper that swapped nothing (not auto): nothing to refuse
+ layers._unsloth_block_swap = types.SimpleNamespace(indices = [])
+ elif variant == "auto":
+ model._unsloth_offload_layers_auto = "auto"
+ elif variant == "auto_ddp":
+ model._unsloth_offload_layers_auto_ddp = True
+ return model
+
+
[email protected]("variant", ["none", "count", "empty", "auto", "auto_ddp"])
+def test_block_swap_flag_agrees_with_the_fsdp2_shard_refusal(variant):
+ from unsloth.models import fsdp2_native
+
+ model = _swap(_tiny(), variant)
+ try:
+ fsdp2_native._refuse_block_swap(model)
+ refused = False
+ except ValueError:
+ refused = True
+ assert caps._block_swap(model) == refused == (variant in ("count", "auto", "auto_ddp"))
+
+
[email protected]("variant", ["count", "auto"])
+def test_block_swap_under_fsdp2_is_refused_by_check_layout_before_weights_move(variant):
+ model = _swap(_tiny(), variant)
+ before = {n: (p.data_ptr(), p.detach().clone()) for n, p in model.named_parameters()}
+ layout = ParallelLayout(tp = 2, dp = 2, world = 4, fsdp2 = True)
+ with pytest.raises(NotImplementedError, match = "block swap"):
+ caps.check_layout(layout, None, model, None)
+ for name, param in model.named_parameters():
+ assert param.data_ptr() == before[name][0] and torch.equal(param.detach(), before[name][1])
+ assert "block_swap" in caps.markdown_table("features")
+ assert "block swap (offload_layers) under native FSDP2" in caps.markdown_table("rules")
+
+
+def test_block_swap_without_fsdp2_passes_the_rule():
+ query = caps.Query(dp = 2, block_swap = True)
+ assert "block_swap" in caps.check_features(query)
+ caps.check(query)
+
+
[email protected]("size", [2, 3, 16])
[email protected]("axis", ["tp", "pp"])
+def test_block_swap_under_tp_or_pp_is_unverified(clean_env, axis, size):
+ query = caps.Query(trainer = "sft", block_swap = True, **{axis: size})
+ status, detail = caps.cell_status(query)
+ assert status == "unverified" and f"block swap x {axis}" in detail
+ caps.check(query) # a single axis: runs outside strict mode
+ clean_env.setenv("UNSLOTH_PARALLEL_STRICT", "1")
+ with pytest.raises(NotImplementedError, match = "block swap check"):
+ caps.check(query)
+
+
+ "layout",
+ [
+ dict(family = "llama", cp = 2, tp = 2, cp_backend = "ulysses"),
+ dict(family = "qwen3_moe", moe = True, expert_lora = True, pp = 2, pp_schedule = "gpipe"),
+ ],
+)
+def test_block_swap_refusal_on_a_gated_layout_names_offload_layers(clean_env, layout):
+ query = caps.Query(trainer = "sft", torch = (2, 13), **layout)
+ assert caps.cell_status(query)[0] == "supported"
+ query = dataclasses.replace(query, block_swap = True)
+ with pytest.raises(NotImplementedError) as e:
+ caps.check(query)
+ assert "block swap x" in str(e.value) and "offload_layers = 0" in str(e.value)
+
+
+# MoE qwen3_moe x qlora x pp: measured with one micro-batch only, which the registry cannot see.
+
+
+def test_qwen3_moe_qlora_pp_keeps_its_refusal_with_the_micro_batch_evidence():
+ query = caps.Query(
+ trainer = "sft",
+ family = "qwen3_moe",
+ moe = True,
+ router_aux_loss = True,
+ mode = "qlora",
+ pp = 2,
+ pp_schedule = "gpipe",
+ torch = (2, 13),
+ )
+ with pytest.raises(NotImplementedError, match = "UNSLOTH_PP_MICROBATCHES=1") as e:
+ caps.check(query)
+ assert "qwen3_moe x qlora x aux on x pp" in str(e.value)
diff --git a/tests/test_parallel_cp_compose_gloo.py b/tests/test_parallel_cp_compose_gloo.py
index da00662a39b..c9dab621313 100644
--- a/tests/test_parallel_cp_compose_gloo.py
+++ b/tests/test_parallel_cp_compose_gloo.py
@@ -103,6 +103,110 @@ def _block_compile_on_cpu(model):
bc.torch.compile = real_compile
+def _install_unsloth_checkpoint(model, gc, stats):
+ """Unsloth's checkpoint on CPU: "unsloth" = path A (UnslothCheckpointFunction, never offloads on CPU),
+ "unsloth_nonre" = path B (vision.py _nonre_checkpoint) with CPU device seams: bookkeeping real, copies not."""
+ import contextlib
+
+ import torch.utils.checkpoint as torch_checkpoint
+ import transformers.modeling_utils as hf_modeling_utils
+ import unsloth_zoo.gradient_checkpointing as G
+
+ original_forward = G.UnslothCheckpointFunction.forward
+
+ def counted_forward(ctx, run_function, preserve_rng_state, *args):
+ stats["checkpoint_calls"] += 1
+ return original_forward(ctx, run_function, preserve_rng_state, *args)
+
+ G.UnslothCheckpointFunction.forward = staticmethod(counted_forward)
+ if gc == "unsloth":
+ # No CUDA: no pinned host buffers and no double buffer (this worker process only).
+ os.environ["UNSLOTH_DISABLE_PINNED_MEMORY"] = "1"
+ os.environ["UNSLOTH_DISABLE_DOUBLE_BUFFER"] = "1"
+ G.patch_unsloth_smart_gradient_checkpointing(dtype = torch.float32)
+ reentrant = True
+ else:
+ from unsloth.models import vision
+
+ G.unpatch_unsloth_smart_gradient_checkpointing()
+
+ class _Stream:
+ def wait_event(self, event):
+ pass
+
+ def wait_stream(self, other):
+ pass
+
+ class _Event:
+ def record(self, stream = None):
+ pass
+
+ def query(self):
+ return True
+
+ compute = _Stream()
+
+ class CpuSeamOffloader(G.NonReentrantActivationOffloader):
+ def _on_host(self, x):
+ return x.numel() == 0 # torch's checkpoint sentinel
+
+ def _streamable(self, x):
+ return not G._is_compiling()
+
+ def _new_stream(self, device_index):
+ return _Stream()
+
+ def _current_stream(self, device_index):
+ return compute
+
+ def _stream_ctx(self, stream):
+ return contextlib.nullcontext()
+
+ def _new_event(self):
+ return _Event()
+
+ def _record_stream(self, x, stream):
+ pass
+
+ def _host_copy(
+ self,
+ x,
+ non_blocking = True,
+ ):
+ stats["offloaded"] += 1
+ return x.detach().clone()
+
+ def _device_copy(self, host, device):
+ stats["restored"] += 1
+ return host.clone()
+
+ def _empty_on_device(self, host, device):
+ return torch.empty_like(host)
+
+ G._reserve_pinned = lambda nbytes: True
+ G._release_pinned = lambda nbytes: None
+ G.PINNED_MEMORY_AVAILABLE = True
+ G._new_activation_offloader = lambda: CpuSeamOffloader()
+ original = getattr(
+ torch_checkpoint.checkpoint, "_unsloth_original", torch_checkpoint.checkpoint
+ )
+ vision._register_gc_offload_root(model, True)
+
+ def _nonre_checkpoint(function, *args, **kwargs):
+ stats["checkpoint_calls"] += 1
+ kwargs["use_reentrant"] = False
+ with vision._offload_activation_hooks(vision._gc_checkpoint_owner(function)):
+ return original(function, *args, **kwargs)
+
+ _nonre_checkpoint._unsloth_original = original
+ _nonre_checkpoint._unsloth_offloads = True
+ torch_checkpoint.checkpoint = _nonre_checkpoint
+ hf_modeling_utils.checkpoint = _nonre_checkpoint
+ reentrant = False
+ model.gradient_checkpointing_enable(gradient_checkpointing_kwargs = {"use_reentrant": reentrant})
+ model.enable_input_require_grads()
+
+
def _selected_moe_backend():
try:
from unsloth_zoo.temporary_patches import moe_utils
@@ -185,6 +289,7 @@ def _case_train(
device = "cpu",
bypass_pre = False,
bypass_post = False,
+ gc = "none",
):
error = _import_checkout_unsloth()
if error is not None:
@@ -278,6 +383,9 @@ def _case_train(
load = tp_env.tensor_parallel_env_load(load)
peft = tp_env.tensor_parallel_env_peft(peft)
model = load(ckpt, load_in_4bit = False)
+ gc_stats = {"checkpoint_calls": 0, "offloaded": 0, "restored": 0}
+ if gc != "none":
+ _install_unsloth_checkpoint(model, gc, gc_stats)
if mode in ("lora", "qlora"):
model = peft(model)
else:
@@ -461,6 +569,10 @@ def _case_train(
"moe_backend": _selected_moe_backend(),
"device_name": torch.cuda.get_device_name() if device != "cpu" else "cpu",
"dist_backend": dist.get_backend() if dist.is_initialized() else None,
+ "gc": dict(gc_stats),
+ "memory": __import__("unsloth.parallel_capabilities", fromlist = ["x"])._memory(
+ trainer.model, args
+ ),
}
if dist.is_initialized():
dist.barrier()
@@ -917,3 +1029,135 @@ def test_moe_with_router_aux_loss_under_cp_is_refused(config_flag):
assert "MoE x pp / cp" in error and "router_aux_loss_coef = 0.0" in error, (
error or res["losses"]
)
+
+
+# Unsloth's checkpoint (paths A and B) in the gated cells above, against one process without checkpointing.
+_UNSLOTH_GC = pytest.mark.parametrize("gc", ["unsloth", "unsloth_nonre"])
+
+
+def _checkpoint_engaged(res, gc):
+ assert res["memory"] == "unsloth_offload", res["memory"]
+ assert res["gc"]["checkpoint_calls"] > 0, res["gc"]
+ if gc == "unsloth_nonre":
+ assert res["gc"]["offloaded"] > 0 and res["gc"]["restored"] == res["gc"]["offloaded"], res[
+ "gc"
+ ]
+
+
+@_UNSLOTH_GC
+ "world, flag, ref_bs, layout",
+ [
+ (4, "tp=2,cp=2", 2, "cp 2 x tp 2"),
+ (8, "tp=2,cp=2", 4, "dp 2 (DDP) x cp 2 x tp 2"),
+ (8, "dp=2,tp=2,cp=2,fsdp=1", 4, "dp 2 (native FSDP2) x cp 2 x tp 2"),
+ ],
+ ids = ["cp2_tp2", "dp2_cp2_tp2", "fsdp2_cp2_tp2"],
+)
+def test_cp_x_tp_with_unsloth_checkpointing_matches_one_process(world, flag, ref_bs, layout, gc):
+ ref = _ref(ref_bs)
+ results = _spawn("train", world, {"UNSLOTH_PARALLEL": flag}, bs = 2, cp = 2, sft_cp = False, gc = gc)
+ for rank, res in enumerate(results):
+ assert "error" not in res, (rank, res.get("error"), res.get("trace"))
+ assert res["layout"] == layout and res["is_tp"], res["layout"]
+ assert res["cp_backend"] == "ulysses" and res["seen_lengths"] == [SEQ // 2]
+ assert res["ulysses_calls"] >= N_LAYERS, res["ulysses_calls"]
+ _checkpoint_engaged(res, gc)
+ _close(res, ref, what = (flag, gc, rank))
+
+
+@needs_block_compile_torch
+@_UNSLOTH_GC
+ "world, flag, ref_bs",
+ [(4, "tp=2,cp=2", 2), (8, "dp=2,tp=2,cp=2,fsdp=1", 4)],
+ ids = ["cp2_tp2", "fsdp2_cp2_tp2"],
+)
+def test_block_compile_with_unsloth_checkpointing_composes_like_one_process(
+ world, flag, ref_bs, gc
+):
+ from unsloth.utils.compile_counters import not_above
+
+ eager = _ref(ref_bs)
+ # The baseline runs under 1-rank DDP, which refuses reentrant checkpointing ("marked ready twice"), so
+ # path A is compared to the compiled run without checkpointing.
+ single = _ref(ref_bs, compile = True, gc = "none" if gc == "unsloth" else gc)
+ assert single["compile"]["unique_graphs"] > 0
+ if gc != "unsloth":
+ _checkpoint_engaged(single, gc)
+ _close(single, eager, what = ("one process compiled", gc))
+ results = _spawn(
+ "train", world, {"UNSLOTH_PARALLEL": flag}, bs = 2, cp = 2, sft_cp = False, compile = True, gc = gc
+ )
+ for rank, res in enumerate(results):
+ assert "error" not in res, (rank, res.get("error"), res.get("trace"))
+ assert res["compiled_classes"] == ["LlamaDecoderLayer"] and res["seen_lengths"] == [
+ SEQ // 2
+ ]
+ _checkpoint_engaged(res, gc)
+ assert not not_above(res["compile"], single["compile"]), (
+ rank,
+ res["compile"],
+ single["compile"],
+ )
+ _close(res, eager, what = (flag, "compiled", gc, rank))
+
+
+@_UNSLOTH_GC
+ "env, sft_cp",
+ [({"UNSLOTH_FSDP2": "1"}, True), ({"UNSLOTH_PARALLEL": "cp=2,fsdp=1"}, False)],
+ ids = ["fsdp2_env_cp_config", "unsloth_parallel"],
+)
+def test_cp_with_native_fsdp2_and_unsloth_checkpointing_matches_one_process(env, sft_cp, gc):
+ ref = _ref(4)
+ for rank, res in enumerate(_spawn("train", 4, env, bs = 2, cp = 2, sft_cp = sft_cp, gc = gc)):
+ assert "error" not in res, (rank, res.get("error"), res.get("trace"))
+ assert (
+ res["native_fsdp2"]
+ and res["seen_lengths"] == [SEQ // 2]
+ and res["ulysses_calls"] >= N_LAYERS
+ )
+ _checkpoint_engaged(res, gc)
+ _close(res, ref, what = (env, gc, rank))
+
+
+@_UNSLOTH_GC
+ "mode, world, env",
+ [("lora", 2, {}), ("lora", 4, {"UNSLOTH_FSDP2": "1"}), ("full", 2, {})],
+ ids = ["lora-cp2", "lora-fsdp2_cp2", "full-cp2"],
+)
+def test_moe_under_cp_with_unsloth_checkpointing_matches_one_process(mode, world, env, gc):
+ ref = _ref(2 * world // 2, arch = "qwen3_moe", mode = mode)
+ results = _spawn("train", world, env, bs = 2, cp = 2, arch = "qwen3_moe", mode = mode, gc = gc)
+ _same_rows(results, ref, what = (mode, world, env, gc))
+ for rank, res in enumerate(results):
+ assert "error" not in res, (rank, res.get("error"), res.get("trace"))
+ assert not res["output_router_logits"]
+ experts = [k for k in res["grads"] if ".experts." in k]
+ assert experts and all(res["grads"][k].abs().sum() > 0 for k in experts), experts
+ assert res["seen_lengths"] == [SEQ // 2] and res["ulysses_calls"] >= 2
+ _checkpoint_engaged(res, gc)
+ _close(
+ res, ref, what = (mode, world, env, gc, rank), state_atol = 1e-5 if mode == "full" else 1e-6
+ )
+
+
+@needs_block_compile_torch
+@_UNSLOTH_GC
+def test_moe_block_compile_under_cp_with_unsloth_checkpointing_is_compile_clean(gc):
+ from unsloth.utils.compile_counters import not_above
+
+ ref = _ref(2, arch = "qwen3_moe", mode = "full")
+ single = _ref(2, compile = True, arch = "qwen3_moe", mode = "full", gc = gc)
+ _checkpoint_engaged(single, gc)
+ _close(single, ref, what = ("qwen3_moe one process compiled", gc), state_atol = 1e-5)
+ results = _spawn("train", 2, bs = 2, cp = 2, arch = "qwen3_moe", mode = "full", compile = True, gc = gc)
+ _same_rows(results, ref, what = ("compiled", gc))
+ for rank, res in enumerate(results):
+ assert "error" not in res, (rank, res.get("error"), res.get("trace"))
+ assert res["compiled_classes"] and res["seen_lengths"] == [SEQ // 2]
+ _checkpoint_engaged(res, gc)
+ assert not not_above(res["compile"], single["compile"]), (res["compile"], single["compile"])
+ _close(res, ref, what = ("qwen3_moe cp2 compiled", gc, rank), state_atol = 1e-5)
diff --git a/tests/test_parallel_env.py b/tests/test_parallel_env.py
index cf323bd7994..fb5123d8243 100644
--- a/tests/test_parallel_env.py
+++ b/tests/test_parallel_env.py
@@ -647,7 +647,8 @@ def test_every_cell_carries_three_kinds_of_evidence_that_exist():
@pytest.mark.parametrize(
"extra, missing",
[
- (dict(memory = "unsloth_offload"), "memory='unsloth_offload'"),
+ # unsloth_offload (Unsloth's checkpoint) is measured in the cp x tp cells; the legacy shim is not.
+ (dict(memory = "unsloth_gc"), "memory='unsloth_gc'"),
(dict(mode = "qlora"), "mode='qlora'"),
(dict(kernels = True), "kernels=True"),
(dict(family = "qwen3"), "family='qwen3'"),
@@ -719,17 +720,23 @@ def test_query_reads_compile_memory_and_attention_from_the_model():
Layer.forward._unsloth_block_compiled = True
original = checkpoint.checkpoint
- def unsloth_offloaded_gradient_checkpoint(*a, **k):
+ def unsloth_checkpoint(*a, **k):
return original(*a, **k)
- checkpoint.checkpoint = unsloth_offloaded_gradient_checkpoint
- try:
- q = caps.query_from(
- None, None, model, types.SimpleNamespace(gradient_checkpointing = True, packing = True)
- )
- finally:
- checkpoint.checkpoint = original
- assert (q.compile, q.memory, q.packing) == ("block", "unsloth_offload", True)
+ def unsloth_offloaded_gradient_checkpoint(*a, **k): # the legacy zoo shim
+ return original(*a, **k)
+
+ seen = []
+ for fn in (unsloth_checkpoint, unsloth_offloaded_gradient_checkpoint):
+ checkpoint.checkpoint = fn
+ try:
+ q = caps.query_from(
+ None, None, model, types.SimpleNamespace(gradient_checkpointing = True, packing = True)
+ )
+ finally:
+ checkpoint.checkpoint = original
+ seen.append((q.compile, q.memory, q.packing))
+ assert seen == [("block", "unsloth_offload", True), ("block", "unsloth_offload_legacy", True)]
def test_compile_counters_count_breaks_and_recompiles():
diff --git a/unsloth/parallel_capabilities.py b/unsloth/parallel_capabilities.py
index 6456eda0d3f..f1a39f17936 100644
--- a/unsloth/parallel_capabilities.py
+++ b/unsloth/parallel_capabilities.py
@@ -81,7 +81,8 @@ class Query:
pp_schedule: Optional[str] = None # gpipe, 1f1b, interleaved
attention: str = "sdpa" # sdpa, eager, flash, flash_varlen, fa3, fa4, flex, flex_sparse
compile: str = "eager" # eager, block (Unsloth's per decoder layer torch.compile)
- memory: str = "none" # none, gc (torch checkpoint), unsloth_gc, unsloth_offload, tiled_mlp
+ # none, gc, unsloth_gc, unsloth_offload (Unsloth's checkpoint), unsloth_offload_legacy (old zoo shim), tiled_mlp
+ memory: str = "none"
kernels: bool = False # Unsloth's fused RoPE / RMSNorm / SwiGLU / LoRA kernels
loss: str = "standard" # standard, fused_ce, chunked_ce, fused_rl
moe: bool = False # routed experts
@@ -93,6 +94,9 @@ class Query:
"" # UNSLOTH_MOE_BACKEND: grouped_mm, unsloth_triton, native_torch ("" = default)
)
optim: str = "" # the optimizer (args.optim or the class of a prebuilt one), lower case
+ tiled: bool = False
+ # offload_layers = N or "auto" (block swap).
+ block_swap: bool = False
@dataclasses.dataclass(frozen = True)
@@ -153,6 +157,7 @@ RULES = (
"full finetuning under cp x tp has no parity evidence yet (tensor parallel alone trains it)",
"use LoRA, or UNSLOTH_FSDP2=1 with cp for full finetuning",
),
+ # Ring carries no packed_seq_lengths and drops window / softcap (context_parallel.py).
Rule(
"cp backend",
"ring context parallel with packed rows",
@@ -223,6 +228,7 @@ RULES = (
"one stage per rank in v1",
"pp_schedule=gpipe or 1f1b",
),
+ # Kept: flex backward cannot run on CPU (no parity evidence) and generic would lose block sparsity.
Rule(
"cp x attention",
"context parallel with flex attention block masks",
@@ -262,6 +268,14 @@ RULES = (
"torch.optim.Muon (it steps DTensor shards as whole matrices), adamw_torch / adamw_8bit, or DDP "
"(dp without fsdp) for another Muon",
),
+ Rule(
+ "fsdp2 x memory",
+ "block swap (offload_layers) under native FSDP2",
+ lambda q: q.fsdp2 and q.block_swap,
+ "block swap keeps decoder layers in host RAM behind placeholders it rebinds each step, and fully_shard "
+ "would turn those into DTensors (models/fsdp2_native.py refuses it in shard_model on every rank)",
+ "offload_layers = 0 at load (and get_peft_model(offload_layers = 0)), or launch without UNSLOTH_FSDP2",
+ ),
Rule(
"tp x model",
"tensor parallel on attention / MLP blocks that own a parameter directly (gpt-oss attention sinks)",
@@ -345,7 +359,7 @@ class Feature:
def _gc(q):
- return q.memory in ("gc", "unsloth_gc", "unsloth_offload")
+ return q.memory in ("gc", "unsloth_gc", "unsloth_offload", "unsloth_offload_legacy")
FEATURES = (
@@ -518,10 +532,17 @@ FEATURES = (
Feature(
"tiled_mlp",
"memory",
- "models/_tiled_mlp_ddp.py + unsloth_zoo tiled MLP",
- lambda q: q.memory == "tiled_mlp",
+ "tiling.py (tile_mlp, unsloth_tiled_mlp) + unsloth_zoo tiling_registry",
+ lambda q: q.tiled or q.memory == "tiled_mlp",
hooks = ("decoder_layer.forward",),
),
+ Feature(
+ "block_swap",
+ "memory",
+ "models/_utils.py (offload_layers) + unsloth_zoo block_swap",
+ lambda q: q.block_swap,
+ hooks = ("from_pretrained", "get_peft_model", "decoder_layer.forward"),
+ ),
# Kernels and losses.
Feature(
"unsloth_kernels",
@@ -608,6 +629,8 @@ _GLOO = "tests/test_parallel_cp_compose_gloo.py::"
# Block compile cells: their compile tests pass on torch 2.13 and 2.15; torch 2.10 recompiles more than one
# process (6 > 1 under cp x tp, 1 > 0 for MoE under cp), so older versions run them eager.
BLOCK_COMPILE_TORCH = (2, 13)
+_UNSLOTH_GC = "Unsloth's checkpoint (path A and path B): " + _GLOO
+_WITH_UNSLOTH_GC = ("none", "gc", "unsloth_offload")
_CP_TP = {
"trainer": ("sft",),
"family": ("llama",),
@@ -619,16 +642,45 @@ _CP_TP = {
"cp_backend": ("ulysses",),
"pp": (1,),
"attention": ("sdpa", "eager"),
- "memory": ("none", "gc"),
+ "memory": _WITH_UNSLOTH_GC,
"kernels": (False,),
"loss": ("standard",),
"vllm": (False,),
"moe": (False,),
+ "block_swap": (False,),
}
_PARITY = _GLOO + "test_cp_x_tp_from_unsloth_parallel_matches_one_process"
-_CP_DP = {**_CP_TP, "tp": (1,), "moe": (False,)}
+_CP_DP = {k: v for k, v in _CP_TP.items() if k != "block_swap"}
+_CP_DP.update(tp = (1,), moe = (False,))
_FSDP2_CP = _GLOO + "test_cp_with_native_fsdp2_matches_one_process"
_MOE_CP = _GLOO + "test_moe_without_aux_loss_under_cp_matches_one_process"
+# Packed rows under generic Ulysses (HF attention, DDP over cp).
+_PACKED_CP = {
+ "trainer": ("sft",),
+ "family": ("qwen3", "mistral", "gemma2", "gemma3"),
+ "hybrid": (False,),
+ "multimodal": (False,),
+ "windowed": (False, True),
+ "mode": ("lora", "full"),
+ "packing": (True,),
+ "cp_backend": ("ulysses",),
+ "dp": (1,),
+ "cp": (2,),
+ "tp": (1,),
+ "pp": (1,),
+ "fsdp2": (False,),
+ "attention": ("sdpa", "eager"),
+ "memory": ("none",),
+ "kernels": (False,),
+ "loss": ("standard",),
+ "vllm": (False,),
+ "moe": (False,),
+ "block_swap": (False,),
+}
+_PACKED_ULYSSES = (
+ "tests/test_cp_ulysses_generic_gloo.py::test_generic_ulysses_matches_one_process"
+ "[gemma3-eager-True|qwen3-sdpa-True]"
+)
CELLS = (
Cell(
"cp x native FSDP2 (dp_shard x cp), eager",
@@ -637,11 +689,15 @@ CELLS = (
engagement = _FSDP2_CP
+ " (FSDP2 mesh covers the cp group, seq / cp tokens, Ulysses exchanges)",
compile = "eager: no compiled region",
+ note = _UNSLOTH_GC
+ + "test_cp_with_native_fsdp2_and_unsloth_checkpointing_matches_one_process",
),
Cell(
"cp x tp (DDP dp), eager",
{**_CP_TP, "fsdp2": (False,), "compile": ("eager",), "dp": (1, 2), "tp": (2,), "cp": (2,)},
parity = _PARITY + "[cp2_tp2|dp2_cp2_tp2]",
+ note = _UNSLOTH_GC
+ + "test_cp_x_tp_with_unsloth_checkpointing_matches_one_process[cp2_tp2|dp2_cp2_tp2]",
engagement = _PARITY
+ " (seq / cp tokens per rank, one Ulysses exchange per layer, cp group != tp group)",
compile = "eager: no compiled region",
@@ -650,6 +706,8 @@ CELLS = (
"cp x tp (FSDP2 dp), eager",
{**_CP_TP, "fsdp2": (True,), "compile": ("eager",), "dp": (2,), "tp": (2,), "cp": (2,)},
parity = _PARITY + "[fsdp2_cp2_tp2]",
+ note = _UNSLOTH_GC
+ + "test_cp_x_tp_with_unsloth_checkpointing_matches_one_process[fsdp2_cp2_tp2]",
engagement = _PARITY
+ "[fsdp2_cp2_tp2] (as above, FSDP2 over the flattened (dp_shard, cp) mesh)",
compile = "eager: no compiled region",
@@ -669,8 +727,30 @@ CELLS = (
+ "test_block_compile_composes_like_one_process (compiled classes, seq / cp tokens)",
compile = _GLOO
+ "test_block_compile_composes_like_one_process (graph breaks / recompiles <= one process)",
+ note = _UNSLOTH_GC
+ + "test_block_compile_with_unsloth_checkpointing_composes_like_one_process",
min_torch = BLOCK_COMPILE_TORCH,
),
+ Cell(
+ "packed rows x cp Ulysses (DDP), eager",
+ {**_PACKED_CP, "compile": ("eager",)},
+ parity = _PACKED_ULYSSES,
+ engagement = _PACKED_ULYSSES
+ + " (one exchange per layer over the whole row, document lengths reach the attention)",
+ compile = "eager: no compiled region",
+ note = "a 2-rank CPU matrix also matched one process for TRL packing and padding-free rows on Qwen3, "
+ "Mistral (window), Gemma-2 (window, softcap) and Gemma-3 (window, replicated kv head), LoRA and full",
+ ),
+ Cell(
+ "packed rows x cp Ulysses (DDP), Unsloth block compile",
+ {**_PACKED_CP, "compile": ("block",)},
+ parity = "tests/test_cp_ulysses_generic_gloo.py::test_compiled_layers_follow_each_packed_row",
+ engagement = None,
+ compile = None,
+ note = "compiled layers now rebuild each packed row's metadata, but the data-dependent cu_seqlens tail "
+ "check in utils/ulysses_hybrid.py still breaks the graph on every packed attention call, so there is no "
+ "compile evidence (graph breaks / graphs not above one process) yet; eager is proven",
+ ),
)
@@ -732,9 +812,13 @@ _MOE_COMMON = {
"packing": (False,),
"vllm": (False,),
}
+# Covers Unsloth's checkpoint too, so such a run stays unverified instead of falling to a single-axis layout.
+_MOE_UNVERIFIED_MEMORY = _WITH_UNSLOTH_GC
_MOE_TEST = _GLOO + "test_moe_under_data_and_context_parallel_matches_one_process"
_MOE_AUX_TEST = _GLOO + "test_moe_with_router_aux_loss_under_ddp_matches_per_micro_batch"
_MOE_COMPILE = _GLOO + "test_moe_block_compile_under_cp_is_compile_clean"
+_MOE_CP_GC = "test_moe_under_cp_with_unsloth_checkpointing_matches_one_process"
+_MOE_CP_GC_COMPILE = "test_moe_block_compile_under_cp_with_unsloth_checkpointing_is_compile_clean"
_MOE_PP = "tests/test_pp_moe_aux_gloo.py::test_pp2_moe_aux_matches_gradient_accumulation"
# (family, precision, aux, mode) -> (parity, engagement, compile) for the rows a test proves.
_MOE_EVIDENCE = {
@@ -791,6 +875,13 @@ def _moe_note(family, precision, aux, mode):
return "pipeline parallel carries the router auxiliary loss for Qwen3-MoE only (refused at the split)"
if precision == "full":
return "refused by the pp x mode rule (pipeline parallel trains LoRA adapters)"
+ if precision == "qlora" and mode == "pp" and family == "qwen3_moe" and aux:
+ # Not opened: micro-batches resolve at the first step (pp_env.resolve_microbatches), not in the layout.
+ return (
+ "with UNSLOTH_PP_MICROBATCHES=1 and gpipe it matched one process on 2 T4 (tiny and 128-wide Qwen3-MoE, "
+ "4-bit experts); more micro-batches are refused at the first step (the router loss would be summed "
+ "per micro-batch), and the registry cannot see the micro-batch count before then"
+ )
if precision == "qlora":
# Measured on one G4 (2 NCCL ranks sharing the card): qwen3_moe QLoRA under DDP equals one process over the
# same micro-batches (bitsandbytes 4-bit expert path engaged); cp / FSDP2 x QLoRA have not run.
@@ -817,17 +908,31 @@ def _moe_cells():
}
name = f"MoE {family} x {precision} x aux {'on' if aux else 'off'} x {mode}"
if evidence:
- cells.append(Cell(name, match, *evidence))
+ note = compiled_note = ""
+ if mode in ("cp", "fsdp2_cp"):
+ match = {**match, "memory": _WITH_UNSLOTH_GC}
+ case = f"{precision.split('+')[0]}-{mode}2" # lora-cp2, lora-fsdp2_cp2, full-cp2
+ note = (
+ _UNSLOTH_GC + _MOE_CP_GC + f"[{case}-unsloth|{case}-unsloth_nonre]"
+ )
+ compiled_note = _UNSLOTH_GC + _MOE_CP_GC_COMPILE
+ if mode in ("pp", "tp"):
+ # No evidence run swapped layers; block swap x tp / pp is its own cell.
+ match = {**match, "block_swap": (False,)}
+ cells.append(Cell(name, match, *evidence, note = note))
if not evidence[2].startswith("eager:"):
cells.append(
Cell(
name + ", block compile",
{**match, "compile": ("block",)},
*evidence,
+ note = compiled_note,
min_torch = BLOCK_COMPILE_TORCH,
)
)
else:
+ if match["memory"] == _MOE_COMMON["memory"]:
+ match = {**match, "memory": _MOE_UNVERIFIED_MEMORY}
cells.append(
Cell(
name,
@@ -841,7 +946,44 @@ def _moe_cells():
return tuple(cells)
-CELLS = CELLS + _moe_cells()
+class _AboveOne:
+ """Any size above 1: parallel_env accepts any positive size, so (2, 4, 8) would miss 3 or 16."""
+
+ def __contains__(self, size):
+ return isinstance(size, int) and size > 1
+
+ def __iter__(self):
+ return iter(("2+",))
+
+
+# The refusal names the knob to turn off: dropping an axis is not the fix for a swapped model.
+_NO_BLOCK_SWAP = (
+ "; load with offload_layers = 0 (and get_peft_model(offload_layers = 0)) to run this layout"
+)
+CELLS = (
+ CELLS
+ + _moe_cells()
+ + (
+ Cell(
+ "block swap x tp",
+ {"block_swap": (True,), "tp": _AboveOne()},
+ None,
+ None,
+ None,
+ note = "models/tp_native.py has no block swap check and no run shards swapped layers yet"
+ + _NO_BLOCK_SWAP,
+ ),
+ Cell(
+ "block swap x pp",
+ {"block_swap": (True,), "pp": _AboveOne()},
+ None,
+ None,
+ None,
+ note = "models/pp_native.py has no block swap check and no run splits swapped layers yet"
+ + _NO_BLOCK_SWAP,
+ ),
+ )
+)
def _strict() -> bool:
@@ -1131,21 +1273,66 @@ def _attention(model, packing):
def _memory(model, args):
try:
import torch.utils.checkpoint as checkpoint
- name = getattr(checkpoint.checkpoint, "__name__", "")
+ fn = checkpoint.checkpoint
except Exception:
- name = ""
+ fn = None
+ name = getattr(fn, "__name__", "")
gc = bool(getattr(args, "gradient_checkpointing", False)) or bool(
getattr(model, "is_gradient_checkpointing", False)
)
if not gc:
return "none"
if name == "unsloth_offloaded_gradient_checkpoint":
+ # Legacy zoo shim: unmeasured, so it keeps its own name.
+ return "unsloth_offload_legacy"
+ try:
+ from ._gc_kwargs import _checkpoint_offloads
+ offloads = _checkpoint_offloads(fn)
+ except Exception:
+ offloads = name in ("unsloth_checkpoint", "unsloth_offloaded_gradient_checkpoint")
+ if offloads:
return "unsloth_offload"
if name == "unsloth_gradient_checkpoint":
return "unsloth_gc"
return "gc"
+def _tiled(model) -> bool:
+ if (
+ model is None
+ or isinstance(model, str)
+ or not callable(getattr(model, "named_modules", None))
+ ):
+ return False
+ try:
+ from unsloth_zoo.tiling_registry import tiled_modules
+ except Exception:
+ return False
+ try:
+ return bool(tiled_modules(model))
+ except Exception:
+ return False
+
+
+def _block_swap(model) -> bool:
+ """As fsdp2_native._refuse_block_swap: "auto" counts even on a rank that swapped none."""
+ if model is None or isinstance(model, str) or not callable(getattr(model, "modules", None)):
+ return False
+ try:
+ for module in model.modules():
+ d = getattr(module, "__dict__", {})
+ swapper = d.get("_unsloth_block_swap", None)
+ if swapper is not None and len(getattr(swapper, "indices", None) or ()) > 0:
+ return True
+ if d.get("_unsloth_offload_layers_auto_ddp", False) or (
+ d.get("_unsloth_offload_layers_auto", None) is not None
+ ):
+ return True
+ except Exception:
+ pass
+ return False
+
+
def _kernels(model):
if model is None or isinstance(model, str):
return False
@@ -1304,6 +1491,8 @@ def query_from(
expert_lora = _expert_lora(model) or _pending_expert_lora(model, peft_config),
moe_backend = __import__("os").environ.get("UNSLOTH_MOE_BACKEND", "").strip().lower(),
optim = _optimizer_name(args, optimizer),
+ tiled = _tiled(model),
+ block_swap = _block_swap(model),
)