Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Parallel capabilities: follow the measured composition evidence (packed CP, Unsloth checkpoint, tiling, block swap) (#16773)
Daniel Han30949ea01dac
diff --git a/tests/test_parallel_capabilities_evidence.py b/tests/test_parallel_capabilities_evidence.pynew file mode 100644index 00000000000..26d2fb4c207--- /dev/null+++ b/tests/test_parallel_capabilities_evidence.py@@ -0,0 +1,336 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""The registry follows the measured composition evidence. CPU only."""++import dataclasses+import types++import pytest+import torch++import unsloth # noqa: F401+from unsloth import parallel_capabilities as caps+from unsloth.parallel_env import ParallelLayout++TINY = "trl-internal-testing/tiny-Qwen3ForCausalLM"++_ENV = (+ "UNSLOTH_PARALLEL",+ "UNSLOTH_TP_SIZE",+ "UNSLOTH_PP_SIZE",+ "UNSLOTH_CP_SIZE",+ "UNSLOTH_FSDP2",+ "UNSLOTH_PARALLEL_STRICT",+ "WORLD_SIZE",+)++[email protected](autouse = True)+def clean_env(monkeypatch):+ for key in _ENV:+ monkeypatch.delenv(key, raising = False)+ yield monkeypatch+++def _tiny():+ from transformers import AutoConfig, AutoModelForCausalLM++ try:+ config = AutoConfig.from_pretrained(TINY)+ except Exception as e: # offline host without the tiny repo cached+ pytest.skip(f"{TINY}: {e}")+ config.num_hidden_layers = 2+ torch.manual_seed(0)+ return AutoModelForCausalLM.from_config(config)+++def _packed(**kw):+ base = dict(+ trainer = "sft",+ family = "qwen3",+ packing = True,+ cp = 2,+ dp = 1,+ cp_backend = "ulysses",+ attention = "sdpa",+ compile = "eager",+ mode = "lora",+ torch = (2, 13),+ )+ base.update(kw)+ return caps.Query(**base)+++# Packed / padding-free rows x cp Ulysses.+++ "kw",+ [+ {},+ {"family": "gemma3", "windowed": True, "attention": "eager"},+ {"family": "mistral", "windowed": True},+ {"family": "gemma2", "windowed": True, "attention": "eager"},+ {"mode": "full"},+ ],+)+def test_packed_rows_under_generic_ulysses_eager_are_supported(kw):+ status, cell = caps.cell_status(_packed(**kw))+ assert status == "supported", cell+ assert "packed rows x cp Ulysses" in cell.name and not cell.missing()+ assert "test_generic_ulysses_matches_one_process" in cell.parity+ caps.check(_packed(**kw))+++def test_packed_rows_through_query_from_reach_the_eager_cell():+ model = _tiny()+ layout = types.SimpleNamespace(dp = 1, tp = 1, pp = 1, cp = 2, fsdp2 = False, cp_backend = "ulysses")+ args = types.SimpleNamespace(packing = False, padding_free = True, gradient_checkpointing = False)+ trl = pytest.importorskip("trl")+ query = caps.query_from(+ layout, trl.SFTTrainer, model, args, cp_backend = "ulysses", peft_config = object()+ )+ assert query.packing and query.family == "qwen3" and query.memory == "none"+ status, cell = caps.cell_status(query)+ assert status == "supported" and "packed rows" in cell.name, (status, cell)+++def test_packed_rows_under_block_compile_stay_unverified_with_the_gap(clean_env):+ query = _packed(compile = "block")+ status, detail = caps.cell_status(query)+ assert status == "unverified"+ assert "lacks engagement, compile evidence" in detail and "breaks the graph" in detail+ caps.check(query) # not gated: runs, as before, outside strict mode+ clean_env.setenv("UNSLOTH_PARALLEL_STRICT", "1")+ with pytest.raises(NotImplementedError, match = "breaks the graph"):+ caps.check(query)+++ "kw",+ [+ {"dp": 2}, # dp 2 x cp 2 was not measured+ {"fsdp2": True},+ {"family": "llama"},+ {"kernels": True, "loss": "fused_ce"}, # Unsloth's fast attention is not the generic path+ {"memory": "unsloth_offload"},+ ],+)+def test_packed_rows_outside_the_measured_cell_are_not_supported(kw):+ assert caps.cell_status(_packed(**kw))[0] != "supported"+++def test_kept_refusals_flex_and_ring_with_packing_or_window():+ with pytest.raises(NotImplementedError, match = "flex attention block masks"):+ caps.check(_packed(attention = "flex"))+ with pytest.raises(NotImplementedError, match = "ring context parallel with packed rows"):+ caps.check(_packed(cp_backend = "ring", sm = 9))+ with pytest.raises(NotImplementedError, match = "sliding window"):+ caps.check(_packed(cp_backend = "ring", packing = False, windowed = True, sm = 9))+++# Memory: Unsloth's checkpoint and tiling.+++ "fn_name, offloads, want",+ [+ ("unsloth_checkpoint", None, "unsloth_offload"), # path A (UnslothCheckpointFunction)+ ("_nonre_checkpoint", True, "unsloth_offload"), # path B with offload hooks+ # path B without offload: a plain non-reentrant checkpoint+ ("_nonre_checkpoint", False, "gc"),+ ("unsloth_offloaded_gradient_checkpoint", None, "unsloth_offload_legacy"), # not measured+ ("unsloth_gradient_checkpoint", None, "unsloth_gc"),+ ("checkpoint", None, "gc"),+ ],+)+def test_memory_reports_unsloth_checkpoints(monkeypatch, fn_name, offloads, want):+ import torch.utils.checkpoint as checkpoint++ def fn(function, *args, **kwargs):+ return function(*args, **kwargs)++ fn.__name__ = fn_name+ if offloads is not None:+ fn._unsloth_offloads = offloads+ monkeypatch.setattr(checkpoint, "checkpoint", fn)+ args = types.SimpleNamespace(gradient_checkpointing = True)+ assert caps._memory(None, args) == want+ assert caps._memory(None, types.SimpleNamespace(gradient_checkpointing = False)) == "none"+++def test_memory_reports_the_zoo_unsloth_checkpoint(monkeypatch):+ zoo = pytest.importorskip("unsloth_zoo.gradient_checkpointing")+ import torch.utils.checkpoint as checkpoint++ monkeypatch.setattr(checkpoint, "checkpoint", zoo.unsloth_checkpoint)+ assert (+ caps._memory(None, types.SimpleNamespace(gradient_checkpointing = True)) == "unsloth_offload"+ )+++def test_unsloth_checkpoint_is_seen_and_covered_by_the_cells_that_measured_it(monkeypatch):+ zoo = pytest.importorskip("unsloth_zoo.gradient_checkpointing")+ import torch.utils.checkpoint as checkpoint++ monkeypatch.setattr(checkpoint, "checkpoint", zoo.unsloth_checkpoint)+ memory = caps._memory(None, types.SimpleNamespace(gradient_checkpointing = True))+ assert memory == "unsloth_offload"+ for kw, name in (+ (dict(family = "llama", tp = 2), "cp x tp (DDP dp), eager"),+ (dict(family = "llama", tp = 2, dp = 2, fsdp2 = True), "cp x tp (FSDP2 dp), eager"),+ (dict(family = "llama", tp = 2, compile = "block"), "cp x tp, Unsloth block compile"),+ (dict(family = "llama", fsdp2 = True), "cp x native FSDP2 (dp_shard x cp), eager"),+ (+ dict(family = "qwen3_moe", moe = True, expert_lora = True),+ "MoE qwen3_moe x lora+experts x aux off x cp",+ ),+ (+ dict(family = "qwen3_moe", moe = True, mode = "full", compile = "block"),+ "MoE qwen3_moe x full x aux off x cp, block compile",+ ),+ ):+ query = caps.Query(+ trainer = "sft", cp = 2, cp_backend = "ulysses", memory = memory, torch = (2, 13), **kw+ )+ status, cell = caps.cell_status(query)+ assert status == "supported" and cell.name == name, (kw, status, cell)+ assert "unsloth_checkpointing" in cell.note, cell.note+ caps.check(query)+++def test_unsloth_checkpoint_keeps_an_unverified_moe_row_refused_under_strict(+ clean_env, monkeypatch+):+ zoo = pytest.importorskip("unsloth_zoo.gradient_checkpointing")+ import torch.utils.checkpoint as checkpoint++ monkeypatch.setattr(checkpoint, "checkpoint", zoo.unsloth_checkpoint)+ memory = caps._memory(None, types.SimpleNamespace(gradient_checkpointing = True))+ query = caps.Query(trainer = "sft", family = "qwen3_moe", moe = True, dp = 2, memory = memory)+ assert caps.cell_status(query)[0] == "unverified"+ clean_env.setenv("UNSLOTH_PARALLEL_STRICT", "1")+ with pytest.raises(NotImplementedError, match = "MoE qwen3_moe x lora x aux off x ddp"):+ caps.check(query)+++def test_tiling_is_its_own_flag():+ pytest.importorskip("unsloth_zoo.tiling_registry")+ from unsloth.tiling import tile_mlp++ model = _tiny()+ args = types.SimpleNamespace(gradient_checkpointing = True)+ assert not caps.query_from(None, None, model, args).tiled+ tile_mlp(model)+ query = caps.query_from(None, None, model, args)+ assert query.tiled and query.memory != "tiled_mlp"+ assert {"tiled_mlp", "gradient_checkpointing"} <= set(caps.check_features(query))+++def test_every_feature_module_path_exists():+ import os+ import re++ root = os.path.dirname(caps.__file__)+ for feature in caps.FEATURES:+ local = feature.module.split("unsloth_zoo")[0]+ for path in re.findall(r"[\w/]+\.py", local):+ assert os.path.exists(os.path.join(root, path)), (feature.name, path)+++# Block swap.+++def _swap(model, variant):+ layers = model.model.layers+ if variant == "count":+ swapper = types.SimpleNamespace(indices = [1])+ layers._unsloth_block_swap = swapper+ model._unsloth_block_swap = swapper+ elif variant == "empty": # a swapper that swapped nothing (not auto): nothing to refuse+ layers._unsloth_block_swap = types.SimpleNamespace(indices = [])+ elif variant == "auto":+ model._unsloth_offload_layers_auto = "auto"+ elif variant == "auto_ddp":+ model._unsloth_offload_layers_auto_ddp = True+ return model++[email protected]("variant", ["none", "count", "empty", "auto", "auto_ddp"])+def test_block_swap_flag_agrees_with_the_fsdp2_shard_refusal(variant):+ from unsloth.models import fsdp2_native++ model = _swap(_tiny(), variant)+ try:+ fsdp2_native._refuse_block_swap(model)+ refused = False+ except ValueError:+ refused = True+ assert caps._block_swap(model) == refused == (variant in ("count", "auto", "auto_ddp"))++[email protected]("variant", ["count", "auto"])+def test_block_swap_under_fsdp2_is_refused_by_check_layout_before_weights_move(variant):+ model = _swap(_tiny(), variant)+ before = {n: (p.data_ptr(), p.detach().clone()) for n, p in model.named_parameters()}+ layout = ParallelLayout(tp = 2, dp = 2, world = 4, fsdp2 = True)+ with pytest.raises(NotImplementedError, match = "block swap"):+ caps.check_layout(layout, None, model, None)+ for name, param in model.named_parameters():+ assert param.data_ptr() == before[name][0] and torch.equal(param.detach(), before[name][1])+ assert "block_swap" in caps.markdown_table("features")+ assert "block swap (offload_layers) under native FSDP2" in caps.markdown_table("rules")+++def test_block_swap_without_fsdp2_passes_the_rule():+ query = caps.Query(dp = 2, block_swap = True)+ assert "block_swap" in caps.check_features(query)+ caps.check(query)++[email protected]("size", [2, 3, 16])[email protected]("axis", ["tp", "pp"])+def test_block_swap_under_tp_or_pp_is_unverified(clean_env, axis, size):+ query = caps.Query(trainer = "sft", block_swap = True, **{axis: size})+ status, detail = caps.cell_status(query)+ assert status == "unverified" and f"block swap x {axis}" in detail+ caps.check(query) # a single axis: runs outside strict mode+ clean_env.setenv("UNSLOTH_PARALLEL_STRICT", "1")+ with pytest.raises(NotImplementedError, match = "block swap check"):+ caps.check(query)+++ "layout",+ [+ dict(family = "llama", cp = 2, tp = 2, cp_backend = "ulysses"),+ dict(family = "qwen3_moe", moe = True, expert_lora = True, pp = 2, pp_schedule = "gpipe"),+ ],+)+def test_block_swap_refusal_on_a_gated_layout_names_offload_layers(clean_env, layout):+ query = caps.Query(trainer = "sft", torch = (2, 13), **layout)+ assert caps.cell_status(query)[0] == "supported"+ query = dataclasses.replace(query, block_swap = True)+ with pytest.raises(NotImplementedError) as e:+ caps.check(query)+ assert "block swap x" in str(e.value) and "offload_layers = 0" in str(e.value)+++# MoE qwen3_moe x qlora x pp: measured with one micro-batch only, which the registry cannot see.+++def test_qwen3_moe_qlora_pp_keeps_its_refusal_with_the_micro_batch_evidence():+ query = caps.Query(+ trainer = "sft",+ family = "qwen3_moe",+ moe = True,+ router_aux_loss = True,+ mode = "qlora",+ pp = 2,+ pp_schedule = "gpipe",+ torch = (2, 13),+ )+ with pytest.raises(NotImplementedError, match = "UNSLOTH_PP_MICROBATCHES=1") as e:+ caps.check(query)+ assert "qwen3_moe x qlora x aux on x pp" in str(e.value)diff --git a/tests/test_parallel_cp_compose_gloo.py b/tests/test_parallel_cp_compose_gloo.pyindex da00662a39b..c9dab621313 100644--- a/tests/test_parallel_cp_compose_gloo.py+++ b/tests/test_parallel_cp_compose_gloo.py@@ -103,6 +103,110 @@ def _block_compile_on_cpu(model):bc.torch.compile = real_compile+def _install_unsloth_checkpoint(model, gc, stats):+ """Unsloth's checkpoint on CPU: "unsloth" = path A (UnslothCheckpointFunction, never offloads on CPU),+ "unsloth_nonre" = path B (vision.py _nonre_checkpoint) with CPU device seams: bookkeeping real, copies not."""+ import contextlib++ import torch.utils.checkpoint as torch_checkpoint+ import transformers.modeling_utils as hf_modeling_utils+ import unsloth_zoo.gradient_checkpointing as G++ original_forward = G.UnslothCheckpointFunction.forward++ def counted_forward(ctx, run_function, preserve_rng_state, *args):+ stats["checkpoint_calls"] += 1+ return original_forward(ctx, run_function, preserve_rng_state, *args)++ G.UnslothCheckpointFunction.forward = staticmethod(counted_forward)+ if gc == "unsloth":+ # No CUDA: no pinned host buffers and no double buffer (this worker process only).+ os.environ["UNSLOTH_DISABLE_PINNED_MEMORY"] = "1"+ os.environ["UNSLOTH_DISABLE_DOUBLE_BUFFER"] = "1"+ G.patch_unsloth_smart_gradient_checkpointing(dtype = torch.float32)+ reentrant = True+ else:+ from unsloth.models import vision++ G.unpatch_unsloth_smart_gradient_checkpointing()++ class _Stream:+ def wait_event(self, event):+ pass++ def wait_stream(self, other):+ pass++ class _Event:+ def record(self, stream = None):+ pass++ def query(self):+ return True++ compute = _Stream()++ class CpuSeamOffloader(G.NonReentrantActivationOffloader):+ def _on_host(self, x):+ return x.numel() == 0 # torch's checkpoint sentinel++ def _streamable(self, x):+ return not G._is_compiling()++ def _new_stream(self, device_index):+ return _Stream()++ def _current_stream(self, device_index):+ return compute++ def _stream_ctx(self, stream):+ return contextlib.nullcontext()++ def _new_event(self):+ return _Event()++ def _record_stream(self, x, stream):+ pass++ def _host_copy(+ self,+ x,+ non_blocking = True,+ ):+ stats["offloaded"] += 1+ return x.detach().clone()++ def _device_copy(self, host, device):+ stats["restored"] += 1+ return host.clone()++ def _empty_on_device(self, host, device):+ return torch.empty_like(host)++ G._reserve_pinned = lambda nbytes: True+ G._release_pinned = lambda nbytes: None+ G.PINNED_MEMORY_AVAILABLE = True+ G._new_activation_offloader = lambda: CpuSeamOffloader()+ original = getattr(+ torch_checkpoint.checkpoint, "_unsloth_original", torch_checkpoint.checkpoint+ )+ vision._register_gc_offload_root(model, True)++ def _nonre_checkpoint(function, *args, **kwargs):+ stats["checkpoint_calls"] += 1+ kwargs["use_reentrant"] = False+ with vision._offload_activation_hooks(vision._gc_checkpoint_owner(function)):+ return original(function, *args, **kwargs)++ _nonre_checkpoint._unsloth_original = original+ _nonre_checkpoint._unsloth_offloads = True+ torch_checkpoint.checkpoint = _nonre_checkpoint+ hf_modeling_utils.checkpoint = _nonre_checkpoint+ reentrant = False+ model.gradient_checkpointing_enable(gradient_checkpointing_kwargs = {"use_reentrant": reentrant})+ model.enable_input_require_grads()++def _selected_moe_backend():try:from unsloth_zoo.temporary_patches import moe_utils@@ -185,6 +289,7 @@ def _case_train(device = "cpu",bypass_pre = False,bypass_post = False,+ gc = "none",):error = _import_checkout_unsloth()if error is not None:@@ -278,6 +383,9 @@ def _case_train(load = tp_env.tensor_parallel_env_load(load)peft = tp_env.tensor_parallel_env_peft(peft)model = load(ckpt, load_in_4bit = False)+ gc_stats = {"checkpoint_calls": 0, "offloaded": 0, "restored": 0}+ if gc != "none":+ _install_unsloth_checkpoint(model, gc, gc_stats)if mode in ("lora", "qlora"):model = peft(model)else:@@ -461,6 +569,10 @@ def _case_train("moe_backend": _selected_moe_backend(),"device_name": torch.cuda.get_device_name() if device != "cpu" else "cpu","dist_backend": dist.get_backend() if dist.is_initialized() else None,+ "gc": dict(gc_stats),+ "memory": __import__("unsloth.parallel_capabilities", fromlist = ["x"])._memory(+ trainer.model, args+ ),}if dist.is_initialized():dist.barrier()@@ -917,3 +1029,135 @@ def test_moe_with_router_aux_loss_under_cp_is_refused(config_flag):assert "MoE x pp / cp" in error and "router_aux_loss_coef = 0.0" in error, (error or res["losses"])+++# Unsloth's checkpoint (paths A and B) in the gated cells above, against one process without checkpointing.+_UNSLOTH_GC = pytest.mark.parametrize("gc", ["unsloth", "unsloth_nonre"])+++def _checkpoint_engaged(res, gc):+ assert res["memory"] == "unsloth_offload", res["memory"]+ assert res["gc"]["checkpoint_calls"] > 0, res["gc"]+ if gc == "unsloth_nonre":+ assert res["gc"]["offloaded"] > 0 and res["gc"]["restored"] == res["gc"]["offloaded"], res[+ "gc"+ ]+++@_UNSLOTH_GC+ "world, flag, ref_bs, layout",+ [+ (4, "tp=2,cp=2", 2, "cp 2 x tp 2"),+ (8, "tp=2,cp=2", 4, "dp 2 (DDP) x cp 2 x tp 2"),+ (8, "dp=2,tp=2,cp=2,fsdp=1", 4, "dp 2 (native FSDP2) x cp 2 x tp 2"),+ ],+ ids = ["cp2_tp2", "dp2_cp2_tp2", "fsdp2_cp2_tp2"],+)+def test_cp_x_tp_with_unsloth_checkpointing_matches_one_process(world, flag, ref_bs, layout, gc):+ ref = _ref(ref_bs)+ results = _spawn("train", world, {"UNSLOTH_PARALLEL": flag}, bs = 2, cp = 2, sft_cp = False, gc = gc)+ for rank, res in enumerate(results):+ assert "error" not in res, (rank, res.get("error"), res.get("trace"))+ assert res["layout"] == layout and res["is_tp"], res["layout"]+ assert res["cp_backend"] == "ulysses" and res["seen_lengths"] == [SEQ // 2]+ assert res["ulysses_calls"] >= N_LAYERS, res["ulysses_calls"]+ _checkpoint_engaged(res, gc)+ _close(res, ref, what = (flag, gc, rank))+++@needs_block_compile_torch+@_UNSLOTH_GC+ "world, flag, ref_bs",+ [(4, "tp=2,cp=2", 2), (8, "dp=2,tp=2,cp=2,fsdp=1", 4)],+ ids = ["cp2_tp2", "fsdp2_cp2_tp2"],+)+def test_block_compile_with_unsloth_checkpointing_composes_like_one_process(+ world, flag, ref_bs, gc+):+ from unsloth.utils.compile_counters import not_above++ eager = _ref(ref_bs)+ # The baseline runs under 1-rank DDP, which refuses reentrant checkpointing ("marked ready twice"), so+ # path A is compared to the compiled run without checkpointing.+ single = _ref(ref_bs, compile = True, gc = "none" if gc == "unsloth" else gc)+ assert single["compile"]["unique_graphs"] > 0+ if gc != "unsloth":+ _checkpoint_engaged(single, gc)+ _close(single, eager, what = ("one process compiled", gc))+ results = _spawn(+ "train", world, {"UNSLOTH_PARALLEL": flag}, bs = 2, cp = 2, sft_cp = False, compile = True, gc = gc+ )+ for rank, res in enumerate(results):+ assert "error" not in res, (rank, res.get("error"), res.get("trace"))+ assert res["compiled_classes"] == ["LlamaDecoderLayer"] and res["seen_lengths"] == [+ SEQ // 2+ ]+ _checkpoint_engaged(res, gc)+ assert not not_above(res["compile"], single["compile"]), (+ rank,+ res["compile"],+ single["compile"],+ )+ _close(res, eager, what = (flag, "compiled", gc, rank))+++@_UNSLOTH_GC+ "env, sft_cp",+ [({"UNSLOTH_FSDP2": "1"}, True), ({"UNSLOTH_PARALLEL": "cp=2,fsdp=1"}, False)],+ ids = ["fsdp2_env_cp_config", "unsloth_parallel"],+)+def test_cp_with_native_fsdp2_and_unsloth_checkpointing_matches_one_process(env, sft_cp, gc):+ ref = _ref(4)+ for rank, res in enumerate(_spawn("train", 4, env, bs = 2, cp = 2, sft_cp = sft_cp, gc = gc)):+ assert "error" not in res, (rank, res.get("error"), res.get("trace"))+ assert (+ res["native_fsdp2"]+ and res["seen_lengths"] == [SEQ // 2]+ and res["ulysses_calls"] >= N_LAYERS+ )+ _checkpoint_engaged(res, gc)+ _close(res, ref, what = (env, gc, rank))+++@_UNSLOTH_GC+ "mode, world, env",+ [("lora", 2, {}), ("lora", 4, {"UNSLOTH_FSDP2": "1"}), ("full", 2, {})],+ ids = ["lora-cp2", "lora-fsdp2_cp2", "full-cp2"],+)+def test_moe_under_cp_with_unsloth_checkpointing_matches_one_process(mode, world, env, gc):+ ref = _ref(2 * world // 2, arch = "qwen3_moe", mode = mode)+ results = _spawn("train", world, env, bs = 2, cp = 2, arch = "qwen3_moe", mode = mode, gc = gc)+ _same_rows(results, ref, what = (mode, world, env, gc))+ for rank, res in enumerate(results):+ assert "error" not in res, (rank, res.get("error"), res.get("trace"))+ assert not res["output_router_logits"]+ experts = [k for k in res["grads"] if ".experts." in k]+ assert experts and all(res["grads"][k].abs().sum() > 0 for k in experts), experts+ assert res["seen_lengths"] == [SEQ // 2] and res["ulysses_calls"] >= 2+ _checkpoint_engaged(res, gc)+ _close(+ res, ref, what = (mode, world, env, gc, rank), state_atol = 1e-5 if mode == "full" else 1e-6+ )+++@needs_block_compile_torch+@_UNSLOTH_GC+def test_moe_block_compile_under_cp_with_unsloth_checkpointing_is_compile_clean(gc):+ from unsloth.utils.compile_counters import not_above++ ref = _ref(2, arch = "qwen3_moe", mode = "full")+ single = _ref(2, compile = True, arch = "qwen3_moe", mode = "full", gc = gc)+ _checkpoint_engaged(single, gc)+ _close(single, ref, what = ("qwen3_moe one process compiled", gc), state_atol = 1e-5)+ results = _spawn("train", 2, bs = 2, cp = 2, arch = "qwen3_moe", mode = "full", compile = True, gc = gc)+ _same_rows(results, ref, what = ("compiled", gc))+ for rank, res in enumerate(results):+ assert "error" not in res, (rank, res.get("error"), res.get("trace"))+ assert res["compiled_classes"] and res["seen_lengths"] == [SEQ // 2]+ _checkpoint_engaged(res, gc)+ assert not not_above(res["compile"], single["compile"]), (res["compile"], single["compile"])+ _close(res, ref, what = ("qwen3_moe cp2 compiled", gc, rank), state_atol = 1e-5)diff --git a/tests/test_parallel_env.py b/tests/test_parallel_env.pyindex cf323bd7994..fb5123d8243 100644--- a/tests/test_parallel_env.py+++ b/tests/test_parallel_env.py@@ -647,7 +647,8 @@ def test_every_cell_carries_three_kinds_of_evidence_that_exist():@pytest.mark.parametrize("extra, missing",[- (dict(memory = "unsloth_offload"), "memory='unsloth_offload'"),+ # unsloth_offload (Unsloth's checkpoint) is measured in the cp x tp cells; the legacy shim is not.+ (dict(memory = "unsloth_gc"), "memory='unsloth_gc'"),(dict(mode = "qlora"), "mode='qlora'"),(dict(kernels = True), "kernels=True"),(dict(family = "qwen3"), "family='qwen3'"),@@ -719,17 +720,23 @@ def test_query_reads_compile_memory_and_attention_from_the_model():Layer.forward._unsloth_block_compiled = Trueoriginal = checkpoint.checkpoint- def unsloth_offloaded_gradient_checkpoint(*a, **k):+ def unsloth_checkpoint(*a, **k):return original(*a, **k)- checkpoint.checkpoint = unsloth_offloaded_gradient_checkpoint- try:- q = caps.query_from(- None, None, model, types.SimpleNamespace(gradient_checkpointing = True, packing = True)- )- finally:- checkpoint.checkpoint = original- assert (q.compile, q.memory, q.packing) == ("block", "unsloth_offload", True)+ def unsloth_offloaded_gradient_checkpoint(*a, **k): # the legacy zoo shim+ return original(*a, **k)++ seen = []+ for fn in (unsloth_checkpoint, unsloth_offloaded_gradient_checkpoint):+ checkpoint.checkpoint = fn+ try:+ q = caps.query_from(+ None, None, model, types.SimpleNamespace(gradient_checkpointing = True, packing = True)+ )+ finally:+ checkpoint.checkpoint = original+ seen.append((q.compile, q.memory, q.packing))+ assert seen == [("block", "unsloth_offload", True), ("block", "unsloth_offload_legacy", True)]def test_compile_counters_count_breaks_and_recompiles():diff --git a/unsloth/parallel_capabilities.py b/unsloth/parallel_capabilities.pyindex 6456eda0d3f..f1a39f17936 100644--- a/unsloth/parallel_capabilities.py+++ b/unsloth/parallel_capabilities.py@@ -81,7 +81,8 @@ class Query:pp_schedule: Optional[str] = None # gpipe, 1f1b, interleavedattention: str = "sdpa" # sdpa, eager, flash, flash_varlen, fa3, fa4, flex, flex_sparsecompile: str = "eager" # eager, block (Unsloth's per decoder layer torch.compile)- memory: str = "none" # none, gc (torch checkpoint), unsloth_gc, unsloth_offload, tiled_mlp+ # none, gc, unsloth_gc, unsloth_offload (Unsloth's checkpoint), unsloth_offload_legacy (old zoo shim), tiled_mlp+ memory: str = "none"kernels: bool = False # Unsloth's fused RoPE / RMSNorm / SwiGLU / LoRA kernelsloss: str = "standard" # standard, fused_ce, chunked_ce, fused_rlmoe: bool = False # routed experts@@ -93,6 +94,9 @@ class Query:"" # UNSLOTH_MOE_BACKEND: grouped_mm, unsloth_triton, native_torch ("" = default))optim: str = "" # the optimizer (args.optim or the class of a prebuilt one), lower case+ tiled: bool = False+ # offload_layers = N or "auto" (block swap).+ block_swap: bool = False@dataclasses.dataclass(frozen = True)@@ -153,6 +157,7 @@ RULES = ("full finetuning under cp x tp has no parity evidence yet (tensor parallel alone trains it)","use LoRA, or UNSLOTH_FSDP2=1 with cp for full finetuning",),+ # Ring carries no packed_seq_lengths and drops window / softcap (context_parallel.py).Rule("cp backend","ring context parallel with packed rows",@@ -223,6 +228,7 @@ RULES = ("one stage per rank in v1","pp_schedule=gpipe or 1f1b",),+ # Kept: flex backward cannot run on CPU (no parity evidence) and generic would lose block sparsity.Rule("cp x attention","context parallel with flex attention block masks",@@ -262,6 +268,14 @@ RULES = ("torch.optim.Muon (it steps DTensor shards as whole matrices), adamw_torch / adamw_8bit, or DDP ""(dp without fsdp) for another Muon",),+ Rule(+ "fsdp2 x memory",+ "block swap (offload_layers) under native FSDP2",+ lambda q: q.fsdp2 and q.block_swap,+ "block swap keeps decoder layers in host RAM behind placeholders it rebinds each step, and fully_shard "+ "would turn those into DTensors (models/fsdp2_native.py refuses it in shard_model on every rank)",+ "offload_layers = 0 at load (and get_peft_model(offload_layers = 0)), or launch without UNSLOTH_FSDP2",+ ),Rule("tp x model","tensor parallel on attention / MLP blocks that own a parameter directly (gpt-oss attention sinks)",@@ -345,7 +359,7 @@ class Feature:def _gc(q):- return q.memory in ("gc", "unsloth_gc", "unsloth_offload")+ return q.memory in ("gc", "unsloth_gc", "unsloth_offload", "unsloth_offload_legacy")FEATURES = (@@ -518,10 +532,17 @@ FEATURES = (Feature("tiled_mlp","memory",- "models/_tiled_mlp_ddp.py + unsloth_zoo tiled MLP",- lambda q: q.memory == "tiled_mlp",+ "tiling.py (tile_mlp, unsloth_tiled_mlp) + unsloth_zoo tiling_registry",+ lambda q: q.tiled or q.memory == "tiled_mlp",hooks = ("decoder_layer.forward",),),+ Feature(+ "block_swap",+ "memory",+ "models/_utils.py (offload_layers) + unsloth_zoo block_swap",+ lambda q: q.block_swap,+ hooks = ("from_pretrained", "get_peft_model", "decoder_layer.forward"),+ ),# Kernels and losses.Feature("unsloth_kernels",@@ -608,6 +629,8 @@ _GLOO = "tests/test_parallel_cp_compose_gloo.py::"# Block compile cells: their compile tests pass on torch 2.13 and 2.15; torch 2.10 recompiles more than one# process (6 > 1 under cp x tp, 1 > 0 for MoE under cp), so older versions run them eager.BLOCK_COMPILE_TORCH = (2, 13)+_UNSLOTH_GC = "Unsloth's checkpoint (path A and path B): " + _GLOO+_WITH_UNSLOTH_GC = ("none", "gc", "unsloth_offload")_CP_TP = {"trainer": ("sft",),"family": ("llama",),@@ -619,16 +642,45 @@ _CP_TP = {"cp_backend": ("ulysses",),"pp": (1,),"attention": ("sdpa", "eager"),- "memory": ("none", "gc"),+ "memory": _WITH_UNSLOTH_GC,"kernels": (False,),"loss": ("standard",),"vllm": (False,),"moe": (False,),+ "block_swap": (False,),}_PARITY = _GLOO + "test_cp_x_tp_from_unsloth_parallel_matches_one_process"-_CP_DP = {**_CP_TP, "tp": (1,), "moe": (False,)}+_CP_DP = {k: v for k, v in _CP_TP.items() if k != "block_swap"}+_CP_DP.update(tp = (1,), moe = (False,))_FSDP2_CP = _GLOO + "test_cp_with_native_fsdp2_matches_one_process"_MOE_CP = _GLOO + "test_moe_without_aux_loss_under_cp_matches_one_process"+# Packed rows under generic Ulysses (HF attention, DDP over cp).+_PACKED_CP = {+ "trainer": ("sft",),+ "family": ("qwen3", "mistral", "gemma2", "gemma3"),+ "hybrid": (False,),+ "multimodal": (False,),+ "windowed": (False, True),+ "mode": ("lora", "full"),+ "packing": (True,),+ "cp_backend": ("ulysses",),+ "dp": (1,),+ "cp": (2,),+ "tp": (1,),+ "pp": (1,),+ "fsdp2": (False,),+ "attention": ("sdpa", "eager"),+ "memory": ("none",),+ "kernels": (False,),+ "loss": ("standard",),+ "vllm": (False,),+ "moe": (False,),+ "block_swap": (False,),+}+_PACKED_ULYSSES = (+ "tests/test_cp_ulysses_generic_gloo.py::test_generic_ulysses_matches_one_process"+ "[gemma3-eager-True|qwen3-sdpa-True]"+)CELLS = (Cell("cp x native FSDP2 (dp_shard x cp), eager",@@ -637,11 +689,15 @@ CELLS = (engagement = _FSDP2_CP+ " (FSDP2 mesh covers the cp group, seq / cp tokens, Ulysses exchanges)",compile = "eager: no compiled region",+ note = _UNSLOTH_GC+ + "test_cp_with_native_fsdp2_and_unsloth_checkpointing_matches_one_process",),Cell("cp x tp (DDP dp), eager",{**_CP_TP, "fsdp2": (False,), "compile": ("eager",), "dp": (1, 2), "tp": (2,), "cp": (2,)},parity = _PARITY + "[cp2_tp2|dp2_cp2_tp2]",+ note = _UNSLOTH_GC+ + "test_cp_x_tp_with_unsloth_checkpointing_matches_one_process[cp2_tp2|dp2_cp2_tp2]",engagement = _PARITY+ " (seq / cp tokens per rank, one Ulysses exchange per layer, cp group != tp group)",compile = "eager: no compiled region",@@ -650,6 +706,8 @@ CELLS = ("cp x tp (FSDP2 dp), eager",{**_CP_TP, "fsdp2": (True,), "compile": ("eager",), "dp": (2,), "tp": (2,), "cp": (2,)},parity = _PARITY + "[fsdp2_cp2_tp2]",+ note = _UNSLOTH_GC+ + "test_cp_x_tp_with_unsloth_checkpointing_matches_one_process[fsdp2_cp2_tp2]",engagement = _PARITY+ "[fsdp2_cp2_tp2] (as above, FSDP2 over the flattened (dp_shard, cp) mesh)",compile = "eager: no compiled region",@@ -669,8 +727,30 @@ CELLS = (+ "test_block_compile_composes_like_one_process (compiled classes, seq / cp tokens)",compile = _GLOO+ "test_block_compile_composes_like_one_process (graph breaks / recompiles <= one process)",+ note = _UNSLOTH_GC+ + "test_block_compile_with_unsloth_checkpointing_composes_like_one_process",min_torch = BLOCK_COMPILE_TORCH,),+ Cell(+ "packed rows x cp Ulysses (DDP), eager",+ {**_PACKED_CP, "compile": ("eager",)},+ parity = _PACKED_ULYSSES,+ engagement = _PACKED_ULYSSES+ + " (one exchange per layer over the whole row, document lengths reach the attention)",+ compile = "eager: no compiled region",+ note = "a 2-rank CPU matrix also matched one process for TRL packing and padding-free rows on Qwen3, "+ "Mistral (window), Gemma-2 (window, softcap) and Gemma-3 (window, replicated kv head), LoRA and full",+ ),+ Cell(+ "packed rows x cp Ulysses (DDP), Unsloth block compile",+ {**_PACKED_CP, "compile": ("block",)},+ parity = "tests/test_cp_ulysses_generic_gloo.py::test_compiled_layers_follow_each_packed_row",+ engagement = None,+ compile = None,+ note = "compiled layers now rebuild each packed row's metadata, but the data-dependent cu_seqlens tail "+ "check in utils/ulysses_hybrid.py still breaks the graph on every packed attention call, so there is no "+ "compile evidence (graph breaks / graphs not above one process) yet; eager is proven",+ ),)@@ -732,9 +812,13 @@ _MOE_COMMON = {"packing": (False,),"vllm": (False,),}+# Covers Unsloth's checkpoint too, so such a run stays unverified instead of falling to a single-axis layout.+_MOE_UNVERIFIED_MEMORY = _WITH_UNSLOTH_GC_MOE_TEST = _GLOO + "test_moe_under_data_and_context_parallel_matches_one_process"_MOE_AUX_TEST = _GLOO + "test_moe_with_router_aux_loss_under_ddp_matches_per_micro_batch"_MOE_COMPILE = _GLOO + "test_moe_block_compile_under_cp_is_compile_clean"+_MOE_CP_GC = "test_moe_under_cp_with_unsloth_checkpointing_matches_one_process"+_MOE_CP_GC_COMPILE = "test_moe_block_compile_under_cp_with_unsloth_checkpointing_is_compile_clean"_MOE_PP = "tests/test_pp_moe_aux_gloo.py::test_pp2_moe_aux_matches_gradient_accumulation"# (family, precision, aux, mode) -> (parity, engagement, compile) for the rows a test proves._MOE_EVIDENCE = {@@ -791,6 +875,13 @@ def _moe_note(family, precision, aux, mode):return "pipeline parallel carries the router auxiliary loss for Qwen3-MoE only (refused at the split)"if precision == "full":return "refused by the pp x mode rule (pipeline parallel trains LoRA adapters)"+ if precision == "qlora" and mode == "pp" and family == "qwen3_moe" and aux:+ # Not opened: micro-batches resolve at the first step (pp_env.resolve_microbatches), not in the layout.+ return (+ "with UNSLOTH_PP_MICROBATCHES=1 and gpipe it matched one process on 2 T4 (tiny and 128-wide Qwen3-MoE, "+ "4-bit experts); more micro-batches are refused at the first step (the router loss would be summed "+ "per micro-batch), and the registry cannot see the micro-batch count before then"+ )if precision == "qlora":# Measured on one G4 (2 NCCL ranks sharing the card): qwen3_moe QLoRA under DDP equals one process over the# same micro-batches (bitsandbytes 4-bit expert path engaged); cp / FSDP2 x QLoRA have not run.@@ -817,17 +908,31 @@ def _moe_cells():}name = f"MoE {family} x {precision} x aux {'on' if aux else 'off'} x {mode}"if evidence:- cells.append(Cell(name, match, *evidence))+ note = compiled_note = ""+ if mode in ("cp", "fsdp2_cp"):+ match = {**match, "memory": _WITH_UNSLOTH_GC}+ case = f"{precision.split('+')[0]}-{mode}2" # lora-cp2, lora-fsdp2_cp2, full-cp2+ note = (+ _UNSLOTH_GC + _MOE_CP_GC + f"[{case}-unsloth|{case}-unsloth_nonre]"+ )+ compiled_note = _UNSLOTH_GC + _MOE_CP_GC_COMPILE+ if mode in ("pp", "tp"):+ # No evidence run swapped layers; block swap x tp / pp is its own cell.+ match = {**match, "block_swap": (False,)}+ cells.append(Cell(name, match, *evidence, note = note))if not evidence[2].startswith("eager:"):cells.append(Cell(name + ", block compile",{**match, "compile": ("block",)},*evidence,+ note = compiled_note,min_torch = BLOCK_COMPILE_TORCH,))else:+ if match["memory"] == _MOE_COMMON["memory"]:+ match = {**match, "memory": _MOE_UNVERIFIED_MEMORY}cells.append(Cell(name,@@ -841,7 +946,44 @@ def _moe_cells():return tuple(cells)-CELLS = CELLS + _moe_cells()+class _AboveOne:+ """Any size above 1: parallel_env accepts any positive size, so (2, 4, 8) would miss 3 or 16."""++ def __contains__(self, size):+ return isinstance(size, int) and size > 1++ def __iter__(self):+ return iter(("2+",))+++# The refusal names the knob to turn off: dropping an axis is not the fix for a swapped model.+_NO_BLOCK_SWAP = (+ "; load with offload_layers = 0 (and get_peft_model(offload_layers = 0)) to run this layout"+)+CELLS = (+ CELLS+ + _moe_cells()+ + (+ Cell(+ "block swap x tp",+ {"block_swap": (True,), "tp": _AboveOne()},+ None,+ None,+ None,+ note = "models/tp_native.py has no block swap check and no run shards swapped layers yet"+ + _NO_BLOCK_SWAP,+ ),+ Cell(+ "block swap x pp",+ {"block_swap": (True,), "pp": _AboveOne()},+ None,+ None,+ None,+ note = "models/pp_native.py has no block swap check and no run splits swapped layers yet"+ + _NO_BLOCK_SWAP,+ ),+ )+)def _strict() -> bool:@@ -1131,21 +1273,66 @@ def _attention(model, packing):def _memory(model, args):try:import torch.utils.checkpoint as checkpoint- name = getattr(checkpoint.checkpoint, "__name__", "")+ fn = checkpoint.checkpointexcept Exception:- name = ""+ fn = None+ name = getattr(fn, "__name__", "")gc = bool(getattr(args, "gradient_checkpointing", False)) or bool(getattr(model, "is_gradient_checkpointing", False))if not gc:return "none"if name == "unsloth_offloaded_gradient_checkpoint":+ # Legacy zoo shim: unmeasured, so it keeps its own name.+ return "unsloth_offload_legacy"+ try:+ from ._gc_kwargs import _checkpoint_offloads+ offloads = _checkpoint_offloads(fn)+ except Exception:+ offloads = name in ("unsloth_checkpoint", "unsloth_offloaded_gradient_checkpoint")+ if offloads:return "unsloth_offload"if name == "unsloth_gradient_checkpoint":return "unsloth_gc"return "gc"+def _tiled(model) -> bool:+ if (+ model is None+ or isinstance(model, str)+ or not callable(getattr(model, "named_modules", None))+ ):+ return False+ try:+ from unsloth_zoo.tiling_registry import tiled_modules+ except Exception:+ return False+ try:+ return bool(tiled_modules(model))+ except Exception:+ return False+++def _block_swap(model) -> bool:+ """As fsdp2_native._refuse_block_swap: "auto" counts even on a rank that swapped none."""+ if model is None or isinstance(model, str) or not callable(getattr(model, "modules", None)):+ return False+ try:+ for module in model.modules():+ d = getattr(module, "__dict__", {})+ swapper = d.get("_unsloth_block_swap", None)+ if swapper is not None and len(getattr(swapper, "indices", None) or ()) > 0:+ return True+ if d.get("_unsloth_offload_layers_auto_ddp", False) or (+ d.get("_unsloth_offload_layers_auto", None) is not None+ ):+ return True+ except Exception:+ pass+ return False++def _kernels(model):if model is None or isinstance(model, str):return False@@ -1304,6 +1491,8 @@ def query_from(expert_lora = _expert_lora(model) or _pending_expert_lora(model, peft_config),moe_backend = __import__("os").environ.get("UNSLOTH_MOE_BACKEND", "").strip().lower(),optim = _optimizer_name(args, optimizer),+ tiled = _tiled(model),+ block_swap = _block_swap(model),)