Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Pipeline parallel: every torch.distributed.pipelining schedule (Interleaved1F1B, LoopedBFS, zero bubble, ZBV, DualPipeV) (#15655)

Daniel Han323696d36d09
diff --git a/tests/test_pp_schedules_gloo.py b/tests/test_pp_schedules_gloo.py
new file mode 100644
index 00000000000..68d219e7b7b
--- /dev/null
+++ b/tests/test_pp_schedules_gloo.py
@@ -0,0 +1,708 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""Multi-stage pipeline schedules (Interleaved1F1B, LoopedBFS, InterleavedZeroBubble, ZBVZeroBubble, DualPipeV)
+of apply_pipeline_parallel on a gloo CPU world (2 / 4 ranks, 2 stages per rank) vs the same model in one
+process, fp32; refusals before the model is cut."""
+
+import multiprocessing
+import os
+
+if multiprocessing.parent_process() is not None or getattr(
+ multiprocessing.current_process(), "_inheriting", False
+):
+ os.environ["CUDA_VISIBLE_DEVICES"] = "" # spawned gloo ranks: CPU only, also on GPU hosts
+
+import sys # noqa: E402
+import tempfile # noqa: E402
+import types # noqa: E402
+import weakref # noqa: E402
+import gc as gc_module # noqa: E402
+
+import pytest # noqa: E402
+import torch # noqa: E402
+
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+from test_pp_native_gloo import ( # noqa: E402
+ GEMMA3,
+ LLAMA,
+ MISTRAL,
+ QWEN3,
+ _enable_reentrant_gc,
+ _free_port,
+ _read_adapter,
+ _reference,
+ _torch_pipelining_error,
+ batch,
+ build,
+ ppn,
+)
+
+ARCHS = {"llama": LLAMA, "qwen3": QWEN3, "gemma3": GEMMA3, "mistral": MISTRAL}
+MULTI = ["interleaved1f1b", "loopedbfs", "interleavedzerobubble", "zbvzerobubble", "dualpipev"]
+V_SCHEDULES = ("zbvzerobubble", "dualpipev")
+N_LAYERS = 8
+
+
+def _v_supported():
+ import importlib
+ return importlib.import_module("unsloth.models.pp_schedules").v_schedules_supported()
+
+
+def _skip_v(schedule):
+ if schedule in V_SCHEDULES and not _v_supported():
+ pytest.skip(
+ f"V schedules need torch >= 2.12 (refused on {torch.__version__}, see the refusal test)"
+ )
+
+
+def _stage_map(
+ pp,
+ schedule,
+ v = 2,
+):
+ """{global stage: pp rank} as the schedule places them."""
+ out = {}
+ for r in range(pp):
+ if schedule in ("zbvzerobubble", "dualpipev"):
+ ks = (r, 2 * pp - 1 - r)
+ else:
+ ks = tuple(r + k * pp for k in range(v))
+ for k in ks:
+ out[k] = r
+ return out
+
+
+def _case_multi(
+ rank,
+ world,
+ tmp,
+ arch,
+ schedule,
+ n_mb = 4,
+ layers = N_LAYERS,
+ gc = None,
+ steps = 3,
+ tie = None,
+ bsz = 4,
+ compile_layers = False,
+):
+ from torch.distributed.device_mesh import init_device_mesh
+
+ repo = ARCHS[arch]
+ ref = build(repo, layers = layers, tie = tie)
+ ids, labels = batch(ref, bsz = bsz)
+ if gc == "reentrant":
+ _enable_reentrant_gc(ref)
+ elif gc == "non_reentrant":
+ ref.gradient_checkpointing_enable(gradient_checkpointing_kwargs = {"use_reentrant": False})
+ ref_res = _reference(ref, ids, labels, steps)
+ if rank == 0:
+ ref.save_pretrained(os.path.join(tmp, "ref"))
+ model = build(repo, layers = layers, tie = tie)
+ if gc == "reentrant":
+ _enable_reentrant_gc(model)
+ elif gc == "non_reentrant":
+ model.gradient_checkpointing_enable(gradient_checkpointing_kwargs = {"use_reentrant": False})
+ mesh = init_device_mesh("cpu", (world,), mesh_dim_names = ("pp",))
+ originals = {
+ "embed": weakref.ref(model.get_input_embeddings()),
+ "head": weakref.ref(model.get_output_embeddings()),
+ }
+ if compile_layers:
+ # Unsloth compiles each decoder block (torch.compile regions): AOTAutograd's compiled backward donates
+ # buffers, which a split (zero bubble) backward with retain_graph cannot run.
+ from torch import _dynamo
+ _dynamo.config.cache_size_limit = 64
+ for layer in ppn._find_parts(model).layers:
+ layer.compile(dynamic = False)
+ from torch._functorch import config as functorch_config
+
+ donated_before = getattr(functorch_config, "donated_buffer", None)
+ pp = ppn.apply_pipeline_parallel(model, mesh["pp"], n_microbatches = n_mb, schedule = schedule)
+ donated_after = getattr(functorch_config, "donated_buffer", None)
+ gc_module.collect()
+ alive = {k: r() is not None for k, r in originals.items()}
+ outs = {k: [] for k in pp.local_stages}
+ handles = [
+ module.register_forward_hook(
+ lambda m, a, kw, o, k = k: outs[k].append(o.detach().clone()), with_kwargs = True
+ )
+ for k, module in zip(pp.local_stages, pp.stage_modules)
+ ]
+ calls = {}
+ for i, layer in enumerate(pp.parts.layers):
+ if type(layer).__name__ != "_PPPassThrough":
+ handles.append(
+ layer.register_forward_pre_hook(
+ lambda m, a, i = i: calls.__setitem__(i, calls.get(i, 0) + 1)
+ )
+ )
+ opt = torch.optim.AdamW(pp.parameters(), lr = 1e-2, foreach = False)
+ losses, norms, grads = [], [], None
+ for step in range(steps):
+ loss = pp.step(ids, labels)
+ if step == 0:
+ grads = {
+ n: p.grad.detach().clone()
+ for n, p in model.named_parameters()
+ if p.grad is not None
+ }
+ for h in handles:
+ h.remove()
+ norms.append(float(pp.clip_grad_norm_(1.0)))
+ opt.step()
+ opt.zero_grad(set_to_none = True)
+ losses.append(float(loss))
+ built = next(iter(pp._shapes.values()))
+ torch_map = dict(getattr(built, "stage_index_to_group_rank", {}) or {})
+ pp.save_adapter(os.path.join(tmp, "pp"))
+ # Every layer module of the model is back in place after the stage forwards (no swap leaked).
+ layer_types = [type(layer).__name__ for layer in pp.parts.layers]
+ direct_error = None
+ try:
+ model(input_ids = ids, labels = labels)
+ except RuntimeError as exc:
+ direct_error = str(exc)
+ return {
+ "ref": {k: v for k, v in ref_res.items() if k != "hidden"},
+ "hidden": ref_res["hidden"],
+ "losses": losses,
+ "norms": norms,
+ "grads": grads,
+ "outs": outs,
+ "bounds": pp.bounds,
+ "local_stages": pp.local_stages,
+ "owned": {n for n, p in model.named_parameters() if p.requires_grad},
+ "trained": {
+ n.replace(".default.", "."): p.detach().clone()
+ for n, p in model.named_parameters()
+ if p.requires_grad
+ },
+ "alive": alive,
+ "layer_types": layer_types,
+ "direct_error": direct_error,
+ "info": dict(model.__dict__.get("_unsloth_pp") or {}),
+ "repr": repr(pp),
+ "torch_map": torch_map,
+ "layer_calls": calls,
+ "donated": (donated_before, donated_after),
+ "gc": gc,
+ "n_mb": n_mb,
+ }
+
+
+def _check_multi(
+ results,
+ saved,
+ pp,
+ schedule,
+ layers = N_LAYERS,
+ v = 2,
+):
+ total = pp * v
+ placement = _stage_map(pp, schedule, v)
+ ref = results[0]["ref"]
+ union = set()
+ for rank, res in enumerate(results):
+ bounds = res["bounds"]
+ assert len(bounds) == total and bounds[0][0] == 0 and bounds[-1][1] == layers, bounds
+ assert all(bounds[k][1] == bounds[k + 1][0] for k in range(total - 1)), bounds
+ assert set(res["local_stages"]) == {k for k, r in placement.items() if r == rank}
+ # torch's schedule places the stages where this rank built them.
+ assert res["torch_map"] == placement, (res["torch_map"], placement)
+ assert res["info"]["stages_per_rank"] == v and tuple(res["info"]["local_stages"]) == tuple(
+ res["local_stages"]
+ )
+ assert res["losses"] == pytest.approx(ref["losses"], rel = 1e-5, abs = 1e-6), (
+ rank,
+ res["losses"],
+ ref["losses"],
+ )
+ assert res["norms"][0] == pytest.approx(ref["norms"][0], rel = 1e-5), (rank, res["norms"])
+ assert res["norms"] == pytest.approx(ref["norms"], rel = 1e-4), (
+ rank,
+ res["norms"],
+ ref["norms"],
+ )
+ assert set(res["grads"]) == res["owned"], sorted(set(res["grads"]) ^ res["owned"])[:6]
+ assert not union & set(res["grads"]), "a trainable parameter lives on two ranks"
+ union |= set(res["grads"])
+ for name, grad in res["grads"].items():
+ torch.testing.assert_close(grad, ref["grads"][name], rtol = 1e-4, atol = 1e-6, msg = name)
+ # Each local stage's output (micro-batch by micro-batch) = the one-process hidden state entering the
+ # next stage's first layer; the last stage returns the loss.
+ for k, chunks in res["outs"].items():
+ assert len(chunks) == res["n_mb"], (k, len(chunks))
+ if k < total - 1:
+ torch.testing.assert_close(
+ torch.cat(chunks), res["hidden"][bounds[k][1]], rtol = 1e-5, atol = 1e-5, msg = str(k)
+ )
+ else:
+ assert all(o.dim() == 0 for o in chunks)
+ owns_first, owns_last = 0 in res["local_stages"], total - 1 in res["local_stages"]
+ assert res["alive"] == {"embed": owns_first, "head": owns_last}, (rank, res["alive"])
+ owned_layers = {i for k in res["local_stages"] for i in range(*bounds[k])}
+ if res["gc"] is None:
+ # Each owned layer runs once per micro-batch: a stage never runs this rank's other stages' layers.
+ assert res["layer_calls"] == {i: res["n_mb"] for i in owned_layers}, (
+ rank,
+ res["layer_calls"],
+ )
+ for i, name in enumerate(res["layer_types"]):
+ assert (name == "_PPPassThrough") == (i not in owned_layers), (rank, i, name)
+ assert "split for pipeline parallelism" in (res["direct_error"] or "")
+ # Zero bubble (split backward) turns AOTAutograd's donated buffers off; other schedules leave it alone.
+ before, after = res["donated"]
+ if before is not None:
+ split = schedule in ("interleavedzerobubble", "zbvzerobubble", "dualpipev")
+ assert after == (False if split else before), (schedule, res["donated"])
+ assert union == set(ref["grads"]), sorted(union ^ set(ref["grads"]))[:6]
+ assert set(saved["pp"]) == set(saved["ref"]), sorted(set(saved["pp"]) ^ set(saved["ref"]))[:6]
+ trained = {}
+ for res in results:
+ trained.update(res["trained"])
+ for key, value in saved["pp"].items():
+ assert torch.equal(value, trained[key]), key
+ for key, value in saved["ref"].items():
+ torch.testing.assert_close(saved["pp"][key], value, rtol = 1e-3, atol = 2e-3, msg = key)
+
+
+def _case_refusals(rank, world, tmp):
+ """Every refusal leaves the model uncut: no pass-through layer, the embedding and lm_head in place."""
+ from torch.distributed.device_mesh import init_device_mesh
+
+ mesh = init_device_mesh("cpu", (world,), mesh_dim_names = ("pp",))["pp"]
+ out = {}
+
+ def attempt(key, model, **kwargs):
+ embed, head = model.get_input_embeddings(), model.get_output_embeddings()
+ try:
+ ppn.apply_pipeline_parallel(model, mesh, **kwargs)
+ out[key] = None
+ except (ValueError, RuntimeError) as exc:
+ out[key] = str(exc)
+ layers = ppn._find_parts(model).layers
+ out[key + ":uncut"] = (
+ not any(type(layer).__name__ == "_PPPassThrough" for layer in layers)
+ and model.get_input_embeddings() is embed
+ and model.get_output_embeddings() is head
+ and not ppn.is_pipeline_parallel(model)
+ )
+
+ attempt("unknown", build(LLAMA), schedule = "pipedream")
+ attempt(
+ "gpipe_two_stages",
+ build(LLAMA),
+ schedule = "gpipe",
+ config = ppn.PPConfig(stages_per_rank = 2),
+ )
+ attempt(
+ "zbv_three_stages",
+ build(LLAMA, layers = 8),
+ schedule = "zbvzerobubble",
+ n_microbatches = 4,
+ config = ppn.PPConfig(stages_per_rank = 3),
+ )
+ # DualPipeV needs at least as many micro-batches as stages (4 here): torch's own check, run up front.
+ attempt("dualpipev_few_microbatches", build(LLAMA), schedule = "dualpipev", n_microbatches = 2)
+ # Interleaved 1F1B: micro-batches a multiple of the rounds (5 // 2 = 2 rounds).
+ attempt("interleaved_rounds", build(LLAMA), schedule = "interleaved1f1b", n_microbatches = 5)
+ attempt("too_many_stages", build(LLAMA, layers = 3), schedule = "interleaved1f1b", n_microbatches = 4)
+
+ reentrant = build(LLAMA)
+ _enable_reentrant_gc(reentrant)
+ attempt("zb_reentrant_gc", reentrant, schedule = "interleavedzerobubble", n_microbatches = 4)
+ attempt("v_schedule", build(LLAMA), schedule = "zbvzerobubble", n_microbatches = 4)
+
+ def apply_lora_qkv(self, x): # stands in for unsloth.kernels.fast_lora's fused kernel
+ raise AssertionError("not called")
+
+ apply_lora_qkv.__module__ = "unsloth.kernels.fast_lora"
+ fused = build(LLAMA)
+ for layer in ppn._find_parts(fused).layers:
+ layer.self_attn.apply_qkv = apply_lora_qkv
+ attempt("zb_fused_lora", fused, schedule = "interleavedzerobubble", n_microbatches = 4)
+ fused_ok = build(LLAMA)
+ for layer in ppn._find_parts(fused_ok).layers:
+ layer.self_attn.apply_qkv = apply_lora_qkv
+ attempt("interleaved_fused_lora", fused_ok, schedule = "interleaved1f1b", n_microbatches = 4)
+
+ fake = types.ModuleType("unsloth_zoo.gradient_checkpointing") # pre-#2071 zoo: no slot leases
+ saved = sys.modules.get("unsloth_zoo.gradient_checkpointing")
+ sys.modules["unsloth_zoo.gradient_checkpointing"] = fake
+ try:
+ old_zoo = build(LLAMA)
+ _enable_reentrant_gc(old_zoo)
+ attempt("interleaved_old_zoo", old_zoo, schedule = "interleaved1f1b", n_microbatches = 4)
+ bfs = build(LLAMA)
+ _enable_reentrant_gc(bfs)
+ # LoopedBFS runs every forward of a step before its backwards: smart checkpointing is safe there.
+ attempt("loopedbfs_old_zoo", bfs, schedule = "loopedbfs", n_microbatches = 4)
+ finally:
+ if saved is None:
+ sys.modules.pop("unsloth_zoo.gradient_checkpointing", None)
+ else:
+ sys.modules["unsloth_zoo.gradient_checkpointing"] = saved
+ return out
+
+
+def _case_dp_refusal(rank, world, tmp):
+ """pp 2 x dp 2 mesh: several stages per rank do not compose with native FSDP2 over dp (refused before the cut);
+ the same mesh with one stage per rank is still accepted."""
+ from torch.distributed.device_mesh import init_device_mesh
+
+ mesh = init_device_mesh("cpu", (2, 2), mesh_dim_names = ("pp", "dp"))
+ model = build(LLAMA)
+ out = {}
+ try:
+ ppn.apply_pipeline_parallel(model, mesh["pp"], n_microbatches = 4, schedule = "interleaved1f1b")
+ out["multi"] = None
+ except ValueError as exc:
+ out["multi"] = str(exc)
+ out["uncut"] = not ppn.is_pipeline_parallel(model)
+ pp = ppn.apply_pipeline_parallel(build(LLAMA), mesh["pp"], n_microbatches = 2, schedule = "gpipe")
+ out["single"] = pp.stages_per_rank
+ return out
+
+
+def _case_compiled_refusal(rank, world, tmp, schedule):
+ """A split-backward schedule over compiled blocks on a torch whose split backward cannot run them."""
+ from torch.distributed.device_mesh import init_device_mesh
+
+ mesh = init_device_mesh("cpu", (world,), mesh_dim_names = ("pp",))["pp"]
+ model = build(LLAMA)
+ for layer in ppn._find_parts(model).layers:
+ layer.compile(dynamic = False)
+ embed, head = model.get_input_embeddings(), model.get_output_embeddings()
+ out = {}
+ try:
+ ppn.apply_pipeline_parallel(model, mesh, n_microbatches = 4, schedule = schedule)
+ out["error"] = None
+ except (ValueError, RuntimeError) as exc:
+ out["error"] = str(exc)
+ out["uncut"] = (
+ not any(type(layer).__name__ == "_PPPassThrough" for layer in ppn._find_parts(model).layers)
+ and model.get_input_embeddings() is embed
+ and model.get_output_embeddings() is head
+ and not ppn.is_pipeline_parallel(model)
+ )
+ return out
+
+
+CASES = {
+ "multi": _case_multi,
+ "refusals": _case_refusals,
+ "dp_refusal": _case_dp_refusal,
+ "compiled_refusal": _case_compiled_refusal,
+}
+
+
+def _worker(rank, world, port, case, tmp, kwargs):
+ os.environ.update(
+ MASTER_ADDR = "127.0.0.1",
+ MASTER_PORT = str(port),
+ RANK = str(rank),
+ WORLD_SIZE = str(world),
+ LOCAL_RANK = str(rank),
+ LOCAL_WORLD_SIZE = str(world),
+ )
+ import datetime
+
+ import torch.distributed as dist
+
+ torch.set_num_threads(2)
+ dist.init_process_group(
+ "gloo", rank = rank, world_size = world, timeout = datetime.timedelta(seconds = 180)
+ )
+ try:
+ res = CASES[case](rank, world, tmp, **kwargs)
+ torch.save(res, os.path.join(tmp, f"result_{rank}.pt"))
+ finally:
+ dist.destroy_process_group()
+
+
+def _spawn(
+ case,
+ world = 2,
+ keep = None,
+ **kwargs,
+):
+ if _torch_pipelining_error():
+ pytest.skip(_torch_pipelining_error())
+ ctx = multiprocessing.get_context("spawn")
+ port = _free_port()
+ with tempfile.TemporaryDirectory() as tmp:
+ procs = [
+ ctx.Process(target = _worker, args = (r, world, port, case, tmp, kwargs))
+ for r in range(world)
+ ]
+ for p in procs:
+ p.start()
+ for p in procs:
+ p.join(600)
+ codes = [p.exitcode for p in procs]
+ assert codes == [0] * world, f"rank exit codes {codes}"
+ results = [
+ torch.load(os.path.join(tmp, f"result_{r}.pt"), weights_only = False)
+ for r in range(world)
+ ]
+ if keep is not None:
+ keep["ref"] = _read_adapter(os.path.join(tmp, "ref"))
+ keep["pp"] = _read_adapter(os.path.join(tmp, "pp"))
+ return results
+
+
[email protected]("schedule", MULTI)
+def test_pp2_multi_stage_schedules_match_one_process(schedule):
+ _skip_v(schedule)
+ saved = {}
+ results = _spawn("multi", 2, keep = saved, arch = "llama", schedule = schedule)
+ _check_multi(results, saved, 2, schedule)
+
+
[email protected]("schedule", ["interleaved1f1b", "zbvzerobubble"])
+def test_pp4_multi_stage_schedules_match_one_process(schedule):
+ _skip_v(schedule)
+ saved = {}
+ results = _spawn("multi", 4, keep = saved, arch = "mistral", schedule = schedule, n_mb = 4)
+ _check_multi(results, saved, 4, schedule)
+
+
[email protected]("arch", ["qwen3", "gemma3"])
+def test_pp2_interleaved1f1b_other_archs_match_one_process(arch):
+ saved = {}
+ results = _spawn("multi", 2, keep = saved, arch = arch, schedule = "interleaved1f1b", n_mb = 2, bsz = 4)
+ _check_multi(results, saved, 2, "interleaved1f1b")
+
+
+def test_pp2_dualpipev_tied_embeddings_both_ends_on_rank0():
+ # V placement puts the first and the last stage on rank 0: the tied embedding / lm_head stay there only.
+ _skip_v("dualpipev")
+ saved = {}
+ results = _spawn("multi", 2, keep = saved, arch = "gemma3", schedule = "dualpipev", tie = True)
+ _check_multi(results, saved, 2, "dualpipev")
+ assert results[0]["alive"] == {"embed": True, "head": True}
+ assert results[1]["alive"] == {"embed": False, "head": False}
+
+
[email protected]("schedule", ["interleaved1f1b", "loopedbfs"])
+def test_pp2_multi_stage_reentrant_gradient_checkpointing_matches_one_process(schedule):
+ saved = {}
+ results = _spawn("multi", 2, keep = saved, arch = "qwen3", schedule = schedule, gc = "reentrant")
+ _check_multi(results, saved, 2, schedule)
+
+
[email protected]("schedule", ["interleavedzerobubble", "zbvzerobubble"])
+def test_pp2_zero_bubble_with_non_reentrant_checkpointing_matches_one_process(schedule):
+ _skip_v(schedule)
+ saved = {}
+ results = _spawn("multi", 2, keep = saved, arch = "llama", schedule = schedule, gc = "non_reentrant")
+ _check_multi(results, saved, 2, schedule)
+
+
[email protected]("schedule", ["interleavedzerobubble", "zbvzerobubble"])
+def test_pp2_zero_bubble_with_compiled_blocks_matches_one_process(schedule):
+ sched = sys.modules["unsloth.models.pp_schedules"]
+ if not sched.split_backward_keeps_function_nodes():
+ # torch <= 2.12: split backward reads _input_metadata of compiled (AOTAutograd) nodes whose graph root
+ # is already released; refused before the model is cut.
+ # V schedules on torch < 2.12 hit the earlier V-placement refusal first.
+ v_refused = schedule in V_SCHEDULES and not _v_supported()
+ for res in _spawn("compiled_refusal", 2, schedule = schedule):
+ if v_refused:
+ assert "(torch >= 2.12)" in (res["error"] or ""), res
+ else:
+ assert "cannot run over torch.compile'd blocks" in (res["error"] or ""), res
+ assert schedule in res["error"] and "interleaved1f1b" in res["error"], res
+ assert res["uncut"], res
+ return
+ _skip_v(schedule)
+ saved = {}
+ results = _spawn("multi", 2, keep = saved, arch = "llama", schedule = schedule, compile_layers = True)
+ _check_multi(results, saved, 2, schedule)
+
+
+def test_pp4_interleaved1f1b_x_tp2_matches_one_process():
+ from test_pp_fsdp2_gloo import _check, _spawn as compose_spawn
+ dims = {"pp": 2, "tp": 2}
+ _check(
+ compose_spawn(
+ "compose", world = 4, arch = "llama", dims = dims, schedule = "interleaved1f1b", n_mb = 4
+ ),
+ dims,
+ )
+
+
+def test_multi_stage_refusals_leave_the_model_uncut():
+ res = _spawn("refusals", 2)[0]
+ for key in [k for k in res if not k.endswith(":uncut")]:
+ if key in ("loopedbfs_old_zoo", "interleaved_fused_lora") or (
+ key == "v_schedule" and _v_supported()
+ ):
+ continue
+ assert res[key + ":uncut"], (key, res[key])
+ assert "pipeline schedule must be one of" in res["unknown"]
+ assert "runs one stage per rank" in res["gpipe_two_stages"]
+ assert "exactly 2 stages per rank" in res["zbv_three_stages"]
+ assert (
+ "DualPipeV requires at least as many microbatches as stages"
+ if _v_supported()
+ else "(torch >= 2.12)"
+ ) in res["dualpipev_few_microbatches"]
+ assert "multiple of the number of rounds" in res["interleaved_rounds"]
+ assert "cannot split 3 decoder layers into 4 pipeline stages" in res["too_many_stages"]
+ assert "splits each backward into an input and a weight pass" in res["zb_reentrant_gc"]
+ assert "interleaves forwards and backwards" in res["interleaved_old_zoo"]
+ assert res["loopedbfs_old_zoo"] is None
+ assert (
+ "runs the backward of Unsloth's fused LoRA kernels (apply_lora_qkv) twice"
+ in res["zb_fused_lora"]
+ )
+ assert res["interleaved_fused_lora"] is None # full-backward schedules keep the fused kernels
+ if _v_supported():
+ assert res["v_schedule"] is None
+ else:
+ assert "(torch >= 2.12)" in res["v_schedule"]
+
+
+def test_multi_stage_on_a_pp_x_dp_mesh_is_refused_before_the_cut():
+ res = _spawn("dp_refusal", 4)
+ for r in res:
+ assert "several stages per rank is not supported" in (r["multi"] or ""), r
+ assert r["uncut"] and r["single"] == 1
+
+
+def test_schedule_names_normalize():
+ sched = sys.modules["unsloth.models.pp_schedules"]
+ assert sched.schedule_spec("Interleaved1F1B").name == "interleaved1f1b"
+ assert sched.schedule_spec("ScheduleDualPipeV").name == "dualpipev"
+ assert sched.schedule_spec("zbv").name == "zbvzerobubble"
+ assert sched.schedule_spec("looped_bfs").name == "loopedbfs"
+ assert sched.schedule_spec("GPipe").name == "gpipe"
+
+
+def test_split_backward_probe_matches_torch_node_ownership():
+ # THPFunction owns its node (c10::intrusive_ptr) from torch 2.13; up to 2.12 it is a std::weak_ptr.
+ sched = sys.modules["unsloth.models.pp_schedules"]
+ major, minor = (int(v) for v in torch.__version__.split(".")[:2])
+ assert sched.split_backward_keeps_function_nodes() == ((major, minor) >= (2, 13))
+
+
+def test_compiled_blocks_refused_for_split_backward_on_affected_torch(monkeypatch):
+ sched = sys.modules["unsloth.models.pp_schedules"]
+ split = {"interleaved": True, "split_backward": True}
+ full = {"interleaved": True, "split_backward": False}
+
+ def model_with(install):
+ block = torch.nn.Linear(2, 2)
+ install(block)
+ return torch.nn.Sequential(torch.nn.Linear(2, 2), block)
+
+ def marked(block):
+ cls = type("Block", (torch.nn.Linear,), {})
+ forward = lambda self, x: torch.nn.Linear.forward(self, x) # noqa: E731
+ forward._unsloth_block_compiled = True
+ cls.forward = forward
+ block.__class__ = cls
+
+ installs = {
+ "module_compile": lambda b: b.compile(dynamic = False),
+ "block_compile": marked,
+ "compiled_forward": lambda b: setattr(b, "forward", torch.compile(b.forward)),
+ }
+ assert sched.compiled_blocks(torch.nn.Sequential(torch.nn.Linear(2, 2))) == []
+ assert sched.compiled_blocks(torch.compile(torch.nn.Linear(2, 2)))
+ for schedule in ("interleavedzerobubble", "zbvzerobubble", "dualpipev"):
+ spec = sched.schedule_spec(schedule)
+ for kind, install in installs.items():
+ assert sched.compiled_blocks(model_with(install)) == ["1"], kind
+ monkeypatch.setattr(ppn, "split_backward_keeps_function_nodes", lambda: False)
+ with pytest.raises(ValueError, match = "cannot run over torch.compile'd blocks") as exc:
+ ppn._check_schedule(model_with(install), spec, split)
+ assert schedule in str(exc.value) and "UNSLOTH_BLOCK_COMPILE=0" in str(exc.value)
+ ppn._check_schedule(model_with(install), spec, full) # full backward: not affected
+ monkeypatch.setattr(ppn, "split_backward_keeps_function_nodes", lambda: True)
+ ppn._check_schedule(model_with(install), spec, split) # torch >= 2.13: allowed
+ monkeypatch.setattr(ppn, "split_backward_keeps_function_nodes", lambda: False)
+ ppn._check_schedule(torch.nn.Sequential(torch.nn.Linear(2, 2)), spec, split) # nothing compiled
+
+
+def test_forward_calling_compiled_function_refused_for_split_backward(monkeypatch):
+ # unsloth_zoo's generated unsloth_compiled_module_* files keep the class forward eager and call a module-level
+ # compiled function from it (Gemma3MLP.forward -> Gemma3MLP_forward), through torch_compile_with_fallback's
+ # functools.wraps wrapper. Its backward is a CompiledFunctionBackward node: torch <= 2.12's split backward fails
+ # on it after the cut ("_input_metadata is invalid"), as on a compiled block.
+ import functools
+
+ sched = sys.modules["unsloth.models.pp_schedules"]
+ split = {"interleaved": True, "split_backward": True}
+
+ def eager(self, x):
+ return torch.nn.functional.linear(x, self.weight, self.bias)
+
+ compiled = torch.compile(eager, dynamic = False)
+
+ @functools.wraps(compiled)
+ def dispatch(*args, **kwargs): # unsloth_zoo _eager_during_decode's shape
+ return compiled(*args, **kwargs)
+
+ dispatch.__wrapped__ = eager
+ dispatch._unsloth_compiled_func = compiled
+
+ def block_calling(target):
+ scope = {"Block_forward": target}
+ exec("def forward(self, x):\n return Block_forward(self, x)\n", scope)
+ cls = type("Block", (torch.nn.Linear,), {"forward": scope["forward"]})
+ block = torch.nn.Linear(2, 2)
+ block.__class__ = cls
+ return torch.nn.Sequential(torch.nn.Linear(2, 2), block)
+
+ for kind, target in {"torch_compile": compiled, "zoo_wrapper": dispatch}.items():
+ assert sched.compiled_blocks(block_calling(target)) == ["1"], kind
+ spec = sched.schedule_spec("interleavedzerobubble")
+ monkeypatch.setattr(ppn, "split_backward_keeps_function_nodes", lambda: False)
+ with pytest.raises(ValueError, match = "cannot run over torch.compile'd blocks"):
+ ppn._check_schedule(block_calling(target), spec, split)
+ monkeypatch.setattr(ppn, "split_backward_keeps_function_nodes", lambda: True)
+ ppn._check_schedule(block_calling(target), spec, split) # torch >= 2.13: allowed
+
+ # A compiled helper captured from an enclosing scope (a closure, not a global) is found too.
+ def closure_forward(self, x):
+ return compiled(self, x)
+
+ closure_block = torch.nn.Linear(2, 2)
+ closure_block.__class__ = type("Block", (torch.nn.Linear,), {"forward": closure_forward})
+ assert sched.compiled_blocks(torch.nn.Sequential(torch.nn.Linear(2, 2), closure_block)) == ["1"]
+ # An eager helper called the same way is not a compiled block.
+ assert sched.compiled_blocks(block_calling(eager)) == []
+
+
+def test_single_backward_kernels_unwraps_partials():
+ # patch_peft_model binds Cohere's MLP kernel as MethodType(partial(apply_lora_mlp, inplace=False), mlp).
+ import functools
+
+ sched = sys.modules["unsloth.models.pp_schedules"]
+
+ def apply_lora_mlp_swiglu(
+ self,
+ X,
+ inplace = True,
+ ): # stands in for fast_lora's fused MLP kernel
+ raise AssertionError("not called")
+
+ apply_lora_mlp_swiglu.__module__ = "unsloth.kernels.fast_lora"
+ cohere = functools.partial(apply_lora_mlp_swiglu, inplace = False)
+ installs = {
+ "plain": lambda m: types.MethodType(apply_lora_mlp_swiglu, m),
+ "partial": lambda m: types.MethodType(cohere, m),
+ "nested_partial": lambda m: types.MethodType(functools.partial(cohere), m),
+ "unbound_partial": lambda m: cohere,
+ }
+ for attr in ("forward", "_unsloth_forward"):
+ for kind, install in installs.items():
+ mlp = torch.nn.Linear(2, 2)
+ setattr(mlp, attr, install(mlp))
+ model = torch.nn.Sequential(mlp)
+ assert sched.single_backward_kernels(model) == ["apply_lora_mlp_swiglu"], (attr, kind)
+ assert sched.single_backward_kernels(torch.nn.Sequential(torch.nn.Linear(2, 2))) == []
diff --git a/unsloth/models/pp_native.py b/unsloth/models/pp_native.py
index f7a91da3ef6..233015fac8d 100644
--- a/unsloth/models/pp_native.py
+++ b/unsloth/models/pp_native.py
@@ -1,8 +1,9 @@
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
-"""Pipeline parallel, one stage per rank; LoRA first; order pp outermost then dp then tp, split
-before parallelizing; save_adapter is collective, rank 0 writes; param names unchanged."""
+"""Pipeline parallel, one or more stages per rank (every torch schedule, pp_schedules.py); LoRA first; order
+pp outermost then dp then tp, split before parallelizing; save_adapter is collective, rank 0 writes; param
+names unchanged."""
from __future__ import annotations
@@ -21,6 +22,20 @@ from typing import Optional
import torch
import torch.nn as nn
+from .pp_schedules import (
+ SCHEDULE_NAMES,
+ allow_split_backward,
+ compiled_blocks,
+ non_reentrant_checkpointing_only,
+ split_backward_keeps_function_nodes,
+ single_backward_kernels,
+ resolve_stages_per_rank,
+ schedule_spec,
+ stage_indices,
+ torch_schedule_class,
+ validate_schedule,
+)
+
__all__ = [
"PPConfig",
"PipelineParallel",
@@ -34,7 +49,7 @@ __all__ = [
]
_PP_FLAG = "_unsloth_pp"
-_SCHEDULES = ("gpipe", "1f1b")
+_SCHEDULES = SCHEDULE_NAMES
_MOE_CONFIG_KEYS = ("num_experts", "num_local_experts", "n_routed_experts", "moe_num_experts")
# Layers reading another layer's state, or extra per-layer inputs: a stage boundary would cut them.
_CROSS_LAYER_CONFIG_KEYS = (
@@ -45,6 +60,11 @@ _CROSS_LAYER_CONFIG_KEYS = (
)
_ATTENTION_LAYER_TYPES = {"full_attention", "sliding_attention", "chunked_attention", "attention"}
_STEP_KWARGS = ("attention_mask", "position_ids")
+_MULTI_STAGE_DP = (
+ "Unsloth: native FSDP2 data parallel over a pipeline with several stages per rank is not supported "
+ "(each FSDP2 root must be one stage's module); use one stage per rank (schedule = 'gpipe' / '1f1b') "
+ "with a dp mesh, or no dp mesh."
+)
_SIMPLE = (bool, int, float, str, type(None))
@@ -60,6 +80,9 @@ class PPConfig:
sync_trainable: bool = True
# (micro-batch size, sequence length) shapes whose stage / schedule stay built.
max_cached_shapes: int = 4
+ # Stages per rank for multi-stage schedules (interleaved1f1b, loopedbfs, ...); None = the schedule's
+ # default (1 for gpipe / 1f1b, 2 otherwise). The decoder is cut into pp x stages_per_rank stages.
+ stages_per_rank: Optional[int] = None
class _StageOutput(BaseException):
@@ -218,6 +241,8 @@ def check_pp_dp_composition(model, dp_mesh):
raise ValueError(
"Unsloth: this pipeline stage is already sharded over a data parallel mesh."
)
+ if stage.stages_per_rank > 1:
+ raise ValueError(_MULTI_STAGE_DP)
if dp_mesh is None:
raise ValueError(
"Unsloth: native FSDP2 over a pipeline parallel model needs FSDP2Config(mesh = mesh['dp']), the "
@@ -731,6 +756,14 @@ class _StageContext:
self.tuple_format = tuple_format
self.active = False
self.stage_input = None
+ # The step's token count, given to every stage forward from here rather than as a schedule kwarg:
+ # a non-tensor kwarg reaches torch's split backward (zero bubble) as a stage input on torch < 2.13.
+ self.num_items_in_batch = None
+
+ def configure(self, start, end, is_first, is_last):
+ """The stage about to run (several stages of one rank share this context, one runs at a time)."""
+ self.start, self.end = start, end
+ self.is_first, self.is_last = is_first, is_last
def pass_through(self, index, args, kwargs):
if not self.active:
@@ -748,17 +781,44 @@ class _StageContext:
class _PPStageModule(nn.Module):
- """What PipelineStage runs: the whole model's forward on this stage's slice."""
+ """What PipelineStage runs: the whole model's forward on this stage's slice.
- def __init__(self, model, ctx, dtype, hidden_size):
+ With several stages per rank, `stage` = (start, end, is_first, is_last, swaps): the context is set to this
+ stage and, for the duration of the forward, `swaps` [(container, key, stand-in)] replace this rank's
+ modules that belong to its other stages (their layers by pass-throughs, the embedding by its stub)."""
+
+ def __init__(
+ self,
+ model,
+ ctx,
+ dtype,
+ hidden_size,
+ stage = None,
+ ):
super().__init__()
self.model = model
self.__dict__["_pp_ctx"] = ctx
self.__dict__["_pp_dtype"] = dtype
self.__dict__["_pp_hidden"] = hidden_size
self.__dict__["_pp_warned"] = False
+ self.__dict__["_pp_stage"] = stage
def forward(self, x, **kwargs):
+ stage = self._pp_stage
+ if stage is None:
+ return self._stage_forward(x, **kwargs)
+ start, end, is_first, is_last, swaps = stage
+ self._pp_ctx.configure(start, end, is_first, is_last)
+ originals = [(container, key, container._modules[key]) for container, key, _ in swaps]
+ for container, key, stand_in in swaps:
+ container._modules[key] = stand_in
+ try:
+ return self._stage_forward(x, **kwargs)
+ finally:
+ for container, key, original in originals:
+ container._modules[key] = original
+
+ def _stage_forward(self, x, **kwargs):
ctx = self._pp_ctx
if ctx.is_first:
input_ids = x
@@ -769,6 +829,8 @@ class _PPStageModule(nn.Module):
try:
# Pass-through layers return hidden states only: no per-layer attentions / hidden states.
kwargs.update(output_attentions = False, output_hidden_states = False)
+ if ctx.num_items_in_batch is not None:
+ kwargs["num_items_in_batch"] = ctx.num_items_in_batch
out = self.model(input_ids = input_ids, use_cache = False, **kwargs)
except _StageOutput as stop:
hidden = stop.hidden
@@ -841,21 +903,48 @@ def _check_torch_pipelining():
)
-def _check_schedule(model, schedule):
- if schedule not in _SCHEDULES:
- raise ValueError(
- f"Unsloth: pipeline schedule must be one of {_SCHEDULES}, got {schedule!r}."
- )
+def _check_schedule(model, spec, props):
+ """Refusals for the schedule `spec` whose compute order has `props` (pp_schedules.validate_schedule)."""
if (
- schedule == "1f1b"
+ props["interleaved"]
and _gradient_checkpointing_on(model)
and _zoo_slot_lease_present() is False
):
raise ValueError(
- "Unsloth: the 1F1B pipeline schedule interleaves forwards and backwards, which this "
+ f"Unsloth: the {spec.name} pipeline schedule interleaves forwards and backwards, which this "
"unsloth_zoo's smart gradient checkpointing does not support (offloaded activations of a pending "
- "micro-batch would be overwritten). Use schedule = 'gpipe', or update unsloth_zoo (host slot "
- "leases, unsloth-zoo PR #2071)."
+ "micro-batch would be overwritten). Use schedule = 'gpipe' (or 'loopedbfs'), or update unsloth_zoo "
+ "(host slot leases, unsloth-zoo PR #2071)."
+ )
+ fused = single_backward_kernels(model) if props["split_backward"] else []
+ if fused:
+ raise ValueError(
+ f"Unsloth: the {spec.name} pipeline schedule splits each backward into an input and a weight pass, "
+ f"which runs the backward of Unsloth's fused LoRA kernels ({', '.join(fused)}) twice; they can run it "
+ "once. Use schedule = 'interleaved1f1b' (or 'loopedbfs'), or load the model without the fused LoRA "
+ "path."
+ )
+ if props["split_backward"] and not split_backward_keeps_function_nodes():
+ compiled = compiled_blocks(model)
+ if compiled:
+ shown = ", ".join(compiled[:3]) + (
+ f" and {len(compiled) - 3} more" if len(compiled) > 3 else ""
+ )
+ raise ValueError(
+ f"Unsloth: the {spec.name} pipeline schedule splits each backward into an input and a weight "
+ f"pass, which torch {torch.__version__} cannot run over torch.compile'd blocks ({shown}): its "
+ "weight pass reads _input_metadata of the compiled (AOTAutograd) backward nodes after their graph "
+ "root is released, which needs torch >= 2.13. Use schedule = 'interleaved1f1b' (or 'loopedbfs', "
+ "a non zero-bubble schedule), disable the compiled blocks (UNSLOTH_BLOCK_COMPILE=0, UNSLOTH_COMPILE_DISABLE=1, no "
+ "module.compile()), or upgrade torch. The model was not modified."
+ )
+ if props["split_backward"] and non_reentrant_checkpointing_only(model) is False:
+ raise ValueError(
+ f"Unsloth: the {spec.name} pipeline schedule splits each backward into an input and a weight pass "
+ "(torch.autograd.grad), which reentrant gradient checkpointing (transformers' default, Unsloth "
+ "smart checkpointing) cannot run. Turn gradient checkpointing off, use torch's non-reentrant "
+ "checkpointing (gradient_checkpointing_kwargs = {'use_reentrant': False}), or use "
+ "schedule = 'interleaved1f1b'."
)
@@ -873,19 +962,41 @@ class PipelineParallel:
"""This rank's pipeline stage of a model split by apply_pipeline_parallel."""
def __init__(
- self, model, mesh, parts, bounds, n_microbatches, schedule, config, tuple_format, dtype
+ self,
+ model,
+ mesh,
+ parts,
+ bounds,
+ n_microbatches,
+ schedule,
+ config,
+ tuple_format,
+ dtype,
+ stages_per_rank = 1,
):
import torch.distributed as dist
self.model = model
self.mesh = mesh
self.group = mesh.get_group()
- self.num_stages = mesh.size()
- self.stage_index = mesh.get_local_rank()
- self.bounds = bounds
- self.start, self.end = bounds[self.stage_index]
+ self.num_stages = mesh.size() # pp ranks
+ self.stage_index = (
+ mesh.get_local_rank()
+ ) # this rank's pp index (its stage with one stage per rank)
+ self.bounds = bounds # [start, end) decoder layers of every stage (pp x stages_per_rank)
+ self.spec = schedule_spec(schedule)
+ self.stages_per_rank = int(stages_per_rank)
+ self.total_stages = self.num_stages * self.stages_per_rank
+ if len(bounds) != self.total_stages:
+ raise ValueError(
+ f"Unsloth: {len(bounds)} layer ranges for {self.total_stages} pipeline stages."
+ )
+ self.local_stages = stage_indices(
+ self.spec, self.stage_index, self.num_stages, self.stages_per_rank
+ )
+ self.start, self.end = bounds[self.local_stages[0]]
self.n_microbatches = n_microbatches
- self.schedule = schedule
+ self.schedule = self.spec.name
self.config = config
self.parts = parts
self._foreign_prefixes = () # set by apply_pipeline_parallel
@@ -894,7 +1005,15 @@ class PipelineParallel:
hidden = getattr(_text_config(model), "hidden_size", None) or parts.embed.weight.shape[-1]
self.hidden_size = int(hidden)
self.module = _PPStageModule(model, self.ctx, dtype, self.hidden_size)
+ self.stage_modules = [self.module] # one per local stage (set by apply_pipeline_parallel)
self._global_ranks = [dist.get_global_rank(self.group, i) for i in range(self.num_stages)]
+ last_owner = next(
+ r
+ for r in range(self.num_stages)
+ if self.total_stages - 1
+ in stage_indices(self.spec, r, self.num_stages, self.stages_per_rank)
+ )
+ self._last_global_rank = self._global_ranks[last_owner]
self._shapes = collections.OrderedDict()
self._scaler = None
self._step_scaler = (
@@ -962,6 +1081,8 @@ class PipelineParallel:
one post_backward in perform_reduce_grad), as torchtitan's stage submodule does."""
from torch.distributed.fsdp import MixedPrecisionPolicy, fully_shard
+ if self.stages_per_rank > 1:
+ raise ValueError(_MULTI_STAGE_DP)
policy = MixedPrecisionPolicy(
param_dtype = None,
reduce_dtype = getattr(mp_policy, "reduce_dtype", None),
@@ -973,11 +1094,13 @@ class PipelineParallel:
@property
def is_first(self):
- return self.stage_index == 0
+ """This rank holds the first stage (the input embedding)."""
+ return 0 in self.local_stages
@property
def is_last(self):
- return self.stage_index == self.num_stages - 1
+ """This rank holds the last stage (final norm, lm_head and the loss)."""
+ return self.total_stages - 1 in self.local_stages
@property
def device(self):
@@ -991,6 +1114,14 @@ class PipelineParallel:
return [p for p in self.module.parameters() if p.requires_grad]
def __repr__(self):
+ if self.stages_per_rank > 1:
+ ranges = ", ".join(
+ f"{k}: [{self.bounds[k][0]}, {self.bounds[k][1]})" for k in self.local_stages
+ )
+ return (
+ f"PipelineParallel(rank {self.stage_index} / {self.num_stages}, stages {{{ranges}}} of "
+ f"{self.total_stages}, {self.schedule}, n_microbatches = {self.n_microbatches})"
+ )
return (
f"PipelineParallel(stage {self.stage_index} / {self.num_stages}, layers [{self.start}, "
f"{self.end}), {self.schedule}, n_microbatches = {self.n_microbatches})"
@@ -1006,43 +1137,49 @@ class PipelineParallel:
return output if self._scaler is None else self._scaler.scale(output)
def _schedule_for(self, mb, seq):
- from torch.distributed.pipelining import PipelineStage, Schedule1F1B, ScheduleGPipe
+ from torch.distributed.pipelining import PipelineStage
key = (mb, seq)
if key in self._shapes:
self._shapes.move_to_end(key)
return self._shapes[key]
device = self.device
- if self.is_first:
- example_in = torch.empty((mb, seq), dtype = torch.long, device = "meta")
- else:
- # requires_grad is part of the static metadata: received hidden states need their grad.
- example_in = torch.empty(
- (mb, seq, self.hidden_size),
- dtype = self.activation_dtype,
- device = "meta",
- requires_grad = True,
- )
- if self.is_last:
- example_out = torch.empty((), dtype = torch.float32, device = "meta", requires_grad = True)
- else:
- example_out = torch.empty(
- (mb, seq, self.hidden_size),
- dtype = self.activation_dtype,
- device = "meta",
- requires_grad = True,
+ hidden = (mb, seq, self.hidden_size)
+ stages = []
+ for index, module in zip(self.local_stages, self.stage_modules):
+ if index == 0:
+ example_in = torch.empty((mb, seq), dtype = torch.long, device = "meta")
+ else:
+ # requires_grad is part of the static metadata: received hidden states need their grad.
+ example_in = torch.empty(
+ hidden, dtype = self.activation_dtype, device = "meta", requires_grad = True
+ )
+ if index == self.total_stages - 1:
+ example_out = torch.empty(
+ (), dtype = torch.float32, device = "meta", requires_grad = True
+ )
+ else:
+ example_out = torch.empty(
+ hidden, dtype = self.activation_dtype, device = "meta", requires_grad = True
+ )
+ stages.append(
+ PipelineStage(
+ module,
+ index,
+ self.total_stages,
+ device,
+ input_args = example_in,
+ output_args = example_out,
+ group = self.group,
+ )
)
- stage = PipelineStage(
- self.module,
- self.stage_index,
- self.num_stages,
- device,
- input_args = example_in,
- output_args = example_out,
- group = self.group,
+ cls = torch_schedule_class(self.spec)
+ schedule = cls(
+ stages if self.spec.multi else stages[0],
+ self.n_microbatches,
+ loss_fn = self._loss_fn,
+ scale_grads = False,
)
- cls = ScheduleGPipe if self.schedule == "gpipe" else Schedule1F1B
- schedule = cls(stage, self.n_microbatches, loss_fn = self._loss_fn, scale_grads = False)
self._shapes[key] = schedule
while len(self._shapes) > max(1, int(self.config.max_cached_shapes)):
self._shapes.popitem(last = False)
@@ -1118,7 +1255,7 @@ class PipelineParallel:
value = value.to(device)
step_kwargs[key] = value
step_kwargs["labels"] = labels.to(device)
- step_kwargs["num_items_in_batch"] = num_items_in_batch
+ self.ctx.num_items_in_batch = num_items_in_batch
if scaler is not None and scaler.is_enabled():
scaler.scale(
torch.zeros((), device = device)
@@ -1133,6 +1270,7 @@ class PipelineParallel:
schedule.step(*args, target = target, return_outputs = False, **step_kwargs)
finally:
self._scaler = None
+ self.ctx.num_items_in_batch = None
if self.dp_mesh is not None:
self._average_replicated_grads()
if self.is_last:
@@ -1145,7 +1283,7 @@ class PipelineParallel:
else:
total = torch.zeros(1, dtype = torch.float32, device = device)
if self.num_stages > 1:
- dist.broadcast(total, src = self._global_ranks[-1], group = self.group)
+ dist.broadcast(total, src = self._last_global_rank, group = self.group)
if dp_group is not None:
dist.all_reduce(total, group = dp_group) # each replica's sum / global tokens
return total[0]
@@ -1354,9 +1492,10 @@ class PipelineParallel:
}
if self.num_stages == 1:
return local if not self.composed or dist.get_rank() == 0 else None
- gathered = [None] * self.num_stages if self.is_first else None
+ first = self.stage_index == 0
+ gathered = [None] * self.num_stages if first else None
dist.gather_object(local, gathered, dst = self._global_ranks[0], group = self.group)
- if not self.is_first or (self.composed and dist.get_rank() != 0):
+ if not first or (self.composed and dist.get_rank() != 0):
return None # one pp group (the one holding global rank 0) writes
merged = {adapter: {} for adapter in local}
for part in gathered:
@@ -1396,6 +1535,33 @@ class PipelineParallel:
dist.barrier(group = self.group)
+def _local_stage_modules(pp_stage, parts, owned):
+ """One _PPStageModule per local stage of a multi-stage rank. Every one runs the same model; while stage k
+ runs, this rank's layers of its other stages are swapped for pass-throughs and, unless k is the first
+ stage, the embedding for its stub (so it is not looked up for nothing)."""
+ layers = parts.layers
+ stand_ins = {i: _PPPassThrough(i, pp_stage.ctx, layers[i]) for i in owned}
+ embed_owner = _owner(parts.decoder, parts.embed) if pp_stage.is_first else None
+ embed_stub = _PPEmbeddingStub(parts.embed) if pp_stage.is_first else None
+ modules = []
+ for k in pp_stage.local_stages:
+ start, end = pp_stage.bounds[k]
+ swaps = [(layers, str(i), stand_ins[i]) for i in owned if not (start <= i < end)]
+ if k != 0 and embed_owner is not None:
+ swaps.append((embed_owner[0], embed_owner[1], embed_stub))
+ stage = (start, end, k == 0, k == pp_stage.total_stages - 1, swaps)
+ modules.append(
+ _PPStageModule(
+ pp_stage.model,
+ pp_stage.ctx,
+ pp_stage.activation_dtype,
+ pp_stage.hidden_size,
+ stage = stage,
+ )
+ )
+ return modules
+
+
def _sync_trainable(model, group, src):
import torch.distributed as dist
with torch.no_grad():
@@ -1434,7 +1600,9 @@ def apply_pipeline_parallel(
return existing
_check_torch_pipelining()
config = config or PPConfig()
- schedule = str(schedule).lower().replace("-", "").replace("_", "")
+ spec = schedule_spec(schedule)
+ schedule = spec.name
+ stages_per_rank = resolve_stages_per_rank(spec, config.stages_per_rank)
if pp_mesh is None:
pp_mesh = _default_mesh(model)
if pp_mesh.ndim != 1:
@@ -1466,23 +1634,43 @@ def apply_pipeline_parallel(
)
parts = _find_parts(model)
_refuse_unsupported(model, parts)
- _check_schedule(model, schedule)
+ total_stages = pp * stages_per_rank
+ rank = pp_mesh.get_local_rank()
+ if stages_per_rank > 1:
+ root = _root_mesh(pp_mesh)
+ names = tuple(getattr(root, "mesh_dim_names", None) or ()) if root is not None else ()
+ if "dp" in names:
+ raise ValueError(_MULTI_STAGE_DP)
+ # The torch schedule's own checks, on placeholder stages: before anything is cut.
+ props = validate_schedule(spec, pp_mesh.get_group(), pp, stages_per_rank, n_microbatches, rank)
+ _check_schedule(model, spec, props)
tuple_format = _layer_call_format(parts.decoder, parts.layers_name) == "tuple"
- bounds = derive_pp_split(model, pp, config)
+ bounds = derive_pp_split(model, total_stages, config)
_refuse_shared_across_stages(parts, bounds)
dtype = config.activation_dtype or parts.embed.weight.dtype
- rank = pp_mesh.get_local_rank()
if config.sync_trainable and pp > 1:
_sync_trainable(model, pp_mesh.get_group(), dist.get_global_rank(pp_mesh.get_group(), 0))
+ donated_off = props["split_backward"] and allow_split_backward()
pp_stage = PipelineParallel(
- model, pp_mesh, parts, bounds, n_microbatches, schedule, config, tuple_format, dtype
+ model,
+ pp_mesh,
+ parts,
+ bounds,
+ n_microbatches,
+ schedule,
+ config,
+ tuple_format,
+ dtype,
+ stages_per_rank = stages_per_rank,
)
- start, end = bounds[rank]
- is_first, is_last = rank == 0, rank == pp - 1
+ local = pp_stage.local_stages
+ owned = sorted({i for k in local for i in range(*bounds[k])})
+ start, end = bounds[local[0]]
+ is_first, is_last = pp_stage.is_first, pp_stage.is_last
layers = parts.layers
for i in range(len(layers)):
- if not (start <= i < end):
+ if i not in owned:
layers[i] = _PPPassThrough(i, pp_stage.ctx, layers[i])
embed, lm_head = parts.embed, parts.lm_head
if not is_first:
@@ -1509,6 +1697,9 @@ def apply_pipeline_parallel(
)
# The stage object must not pin a dropped embedding / lm_head (vocabulary-sized) alive.
parts = pp_stage.parts = dataclasses.replace(parts, embed = embed, lm_head = lm_head)
+ if stages_per_rank > 1:
+ pp_stage.stage_modules = _local_stage_modules(pp_stage, parts, owned)
+ pp_stage.module = pp_stage.stage_modules[0]
info = {
"size": pp,
"rank": rank,
@@ -1516,6 +1707,8 @@ def apply_pipeline_parallel(
"bounds": bounds,
"source": parts.source,
"schedule": schedule,
+ "stages_per_rank": stages_per_rank,
+ "local_stages": tuple(local),
}
inner = parts.inner
for level in _model_chain(model):
@@ -1531,9 +1724,16 @@ def apply_pipeline_parallel(
torch.cuda.empty_cache()
if (dist.get_rank() if dist.is_initialized() else rank) == 0:
print(
- f"Unsloth: pipeline parallel over {pp} stages ({schedule}, {n_microbatches} micro-batches, "
- f"{parts.source} parts): decoder layers per stage {[e - s for s, e in bounds]}"
+ f"Unsloth: pipeline parallel over {pp} ranks x {stages_per_rank} stages ({schedule}, "
+ f"{n_microbatches} micro-batches, {parts.source} parts): decoder layers per stage "
+ f"{[e - s for s, e in bounds]}"
+ (", tied embeddings kept on the first and last stage" if parts.tied else "")
+ + (
+ "; compiled backward graphs built without donated buffers (zero bubble runs autograd.grad "
+ "with retain_graph)"
+ if donated_off
+ else ""
+ )
+ "."
)
return pp_stage
diff --git a/unsloth/models/pp_schedules.py b/unsloth/models/pp_schedules.py
new file mode 100644
index 00000000000..ca8e601324a
--- /dev/null
+++ b/unsloth/models/pp_schedules.py
@@ -0,0 +1,328 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""Pipeline schedules for apply_pipeline_parallel: one stage per rank (GPipe, 1F1B) or several (looped / V).
+
+ pp = apply_pipeline_parallel(model, mesh["pp"], n_microbatches = 8, schedule = "interleaved1f1b")
+
+Rank r keeps stages r, r + pp, ... (looped) or r and 2 pp - 1 - r (V). Every local stage runs the same model
+with only its own layers active, so parameter names and adapter keys are unchanged. The schedule is first
+built on placeholder stages to learn whether a rank runs a forward after a backward (smart checkpointing
+offload slots must then be leased per checkpoint, unsloth-zoo PR #2071) and whether it splits the backward
+(zero bubble: torch.autograd.grad, which reentrant checkpointing cannot run).
+"""
+
+from __future__ import annotations
+
+import dataclasses
+import functools
+from typing import Optional
+
+import torch
+import torch.nn as nn
+
+__all__ = [
+ "ScheduleSpec",
+ "schedule_spec",
+ "stage_indices",
+ "validate_schedule",
+ "SCHEDULE_NAMES",
+]
+
+
[email protected](frozen = True)
+class ScheduleSpec:
+ name: str # normalized, e.g. "interleaved1f1b"
+ torch_name: str # torch.distributed.pipelining.schedules class
+ multi: bool # PipelineScheduleMulti: a list of local stages
+ style: str # "single", "loop" or "v": which stages a rank keeps
+ default_stages_per_rank: int
+
+
+_SPECS = {
+ "gpipe": ScheduleSpec("gpipe", "ScheduleGPipe", False, "single", 1),
+ "1f1b": ScheduleSpec("1f1b", "Schedule1F1B", False, "single", 1),
+ "interleaved1f1b": ScheduleSpec("interleaved1f1b", "ScheduleInterleaved1F1B", True, "loop", 2),
+ "loopedbfs": ScheduleSpec("loopedbfs", "ScheduleLoopedBFS", True, "loop", 2),
+ "interleavedzerobubble": ScheduleSpec(
+ "interleavedzerobubble", "ScheduleInterleavedZeroBubble", True, "loop", 2
+ ),
+ "zbvzerobubble": ScheduleSpec("zbvzerobubble", "ScheduleZBVZeroBubble", True, "v", 2),
+ "dualpipev": ScheduleSpec("dualpipev", "ScheduleDualPipeV", True, "v", 2),
+}
+_ALIASES = {"zbv": "zbvzerobubble", "interleavedzb": "interleavedzerobubble"}
+SCHEDULE_NAMES = tuple(_SPECS)
+
+
+def _normalize(name):
+ key = str(name).lower().replace("-", "").replace("_", "").replace(" ", "")
+ if key.startswith("schedule"):
+ key = key[len("schedule") :]
+ return _ALIASES.get(key, key)
+
+
+def schedule_spec(name) -> ScheduleSpec:
+ spec = _SPECS.get(_normalize(name))
+ if spec is None:
+ raise ValueError(
+ f"Unsloth: pipeline schedule must be one of {SCHEDULE_NAMES}, got {name!r}."
+ )
+ return spec
+
+
+def torch_schedule_class(spec):
+ from torch.distributed.pipelining import schedules
+
+ cls = getattr(schedules, spec.torch_name, None)
+ if cls is None:
+ raise RuntimeError(
+ f"Unsloth: torch {torch.__version__} has no {spec.torch_name} "
+ f"(pipeline schedule {spec.name!r})."
+ )
+ return cls
+
+
+def resolve_stages_per_rank(spec, stages_per_rank):
+ """stages_per_rank for `spec` (None = its default); ValueError when the schedule cannot run it."""
+ v = spec.default_stages_per_rank if stages_per_rank is None else int(stages_per_rank)
+ if v < 1:
+ raise ValueError("Unsloth: PPConfig.stages_per_rank must be >= 1.")
+ if spec.style == "single" and v != 1:
+ raise ValueError(
+ f"Unsloth: the {spec.name} schedule runs one stage per rank (PPConfig.stages_per_rank = {v}); "
+ "use 'interleaved1f1b' or 'loopedbfs' for several."
+ )
+ if spec.style == "v" and v != 2:
+ raise ValueError(
+ f"Unsloth: the {spec.name} schedule runs exactly 2 stages per rank (V placement), got "
+ f"PPConfig.stages_per_rank = {v}."
+ )
+ return v
+
+
+def stage_indices(spec, rank, pp, stages_per_rank):
+ """Global stage indices pp rank `rank` keeps, in execution order of its forward."""
+ if spec.style == "single":
+ return (rank,)
+ if spec.style == "loop":
+ return tuple(rank + k * pp for k in range(stages_per_rank))
+ return (rank, 2 * pp - 1 - rank) # "v"
+
+
+class _NoParams(nn.Module):
+ def forward(self, x):
+ return x
+
+
+def v_schedules_supported():
+ """V placement needs PipelineStage._is_same_rank (torch >= 2.12); older stage setup fails."""
+ try:
+ from torch.distributed.pipelining import PipelineStage
+ except ImportError:
+ return False
+ return hasattr(PipelineStage, "_is_same_rank")
+
+
+def validate_schedule(spec, group, pp, stages_per_rank, n_microbatches, rank):
+ """Build the schedule on placeholder stages so torch's checks run before the model is cut.
+ Returns {"interleaved": bool, "split_backward": bool} over every rank's compute order."""
+ from torch.distributed.pipelining import PipelineStage
+
+ cls = torch_schedule_class(spec)
+ if spec.style == "v" and not v_schedules_supported():
+ raise ValueError(
+ f"Unsloth: the {spec.name} pipeline schedule (V placement: two adjacent stages on one rank) needs "
+ f"a torch whose PipelineStage handles adjacent stages on one rank (torch >= 2.12); on torch "
+ f"{torch.__version__} torch's stage setup between them fails. Use 'interleaved1f1b' or "
+ "'interleavedzerobubble'."
+ )
+ num_stages = pp * stages_per_rank
+ local = stage_indices(spec, rank, pp, stages_per_rank)
+ example = torch.empty((1, 1), device = "meta")
+ stages = [
+ PipelineStage(
+ _NoParams(),
+ k,
+ num_stages,
+ torch.device("cpu"),
+ input_args = example,
+ output_args = example,
+ group = group,
+ )
+ for k in local
+ ]
+ loss_fn = lambda output, target: output.sum() # noqa: E731
+ try:
+ if spec.multi:
+ schedule = cls(stages, n_microbatches, loss_fn = loss_fn, scale_grads = False)
+ else:
+ schedule = cls(stages[0], n_microbatches, loss_fn = loss_fn, scale_grads = False)
+ except (ValueError, AssertionError, RuntimeError) as exc:
+ raise ValueError(
+ f"Unsloth: the {spec.name} pipeline schedule cannot run {n_microbatches} micro-batches over "
+ f"{num_stages} stages ({pp} ranks x {stages_per_rank}): {exc}"
+ ) from None
+ order = getattr(schedule, "pipeline_order", None)
+ if not order:
+ # GPipe runs every forward first; 1F1B interleaves.
+ return {"interleaved": spec.name == "1f1b" and pp > 1, "split_backward": False}
+ interleaved = split = False
+ for actions in order.values():
+ seen_backward = False
+ for action in actions:
+ if action is None:
+ continue
+ # By name: the enum's values are ints on torch 2.10, letters later.
+ kind = getattr(action.computation_type, "name", str(action.computation_type))
+ if kind == "FORWARD":
+ interleaved = interleaved or seen_backward
+ elif kind in ("FULL_BACKWARD", "BACKWARD_INPUT", "BACKWARD_WEIGHT"):
+ seen_backward = True
+ split = split or kind != "FULL_BACKWARD"
+ return {"interleaved": interleaved, "split_backward": split}
+
+
+def non_reentrant_checkpointing_only(model) -> Optional[bool]:
+ """None: no checkpointing; True: all non-reentrant torch checkpointing; False otherwise."""
+ import torch.utils.checkpoint as checkpointing
+
+ # unsloth_zoo smart checkpointing patches checkpoint (original kept as _old_checkpoint).
+ original = getattr(checkpointing, "_old_checkpoint", None) or checkpointing.checkpoint
+ patched = checkpointing.checkpoint is not original
+ found = False
+ for module in model.modules():
+ if getattr(module, "gradient_checkpointing", False) is not True:
+ continue
+ found = True
+ func = getattr(module, "_gradient_checkpointing_func", None)
+ if patched or not isinstance(func, functools.partial) or func.func is not original:
+ return False
+ if func.keywords.get("use_reentrant", True) is not False:
+ return False
+ if not found and bool(getattr(model, "is_gradient_checkpointing", False)):
+ return False
+ return True if found else None
+
+
+def allow_split_backward():
+ """Disable AOTAutograd donated buffers: zero bubble backward uses retain_graph, which a compiled backward
+ with donated buffers refuses. Returns True if changed; call before the first forward (already compiled
+ blocks keep their buffers)."""
+ try:
+ import torch._functorch.config as functorch_config
+ except ImportError:
+ return False
+ if getattr(functorch_config, "donated_buffer", False):
+ functorch_config.donated_buffer = False
+ return True
+ return False
+
+
[email protected]_cache(maxsize = None)
+def split_backward_keeps_function_nodes() -> bool:
+ """True if an autograd.Function node (e.g. CompiledFunctionBackward) keeps `_input_metadata` after its
+ output dies; split backward's weight pass needs it. torch <= 2.12 holds the node by weak_ptr, so it fails.
+ Probed rather than version-checked."""
+
+ class _Probe(torch.autograd.Function):
+ @staticmethod
+ def forward(ctx, x):
+ return x * 2
+
+ @staticmethod
+ def backward(ctx, grad):
+ return grad * 2
+
+ try:
+ with torch.enable_grad():
+ out = _Probe.apply(torch.ones(1, requires_grad = True))
+ node = out.grad_fn
+ del out
+ node._input_metadata
+ except RuntimeError:
+ return False
+ except Exception:
+ # Do not refuse on a probe we cannot interpret.
+ return True
+ return True
+
+
+def compiled_blocks(model):
+ """Names of submodules that run through torch.compile (backward is a CompiledFunctionBackward node)."""
+ found = []
+ calls = {}
+ for name, module in model.named_modules():
+ cls_forward = getattr(type(module), "forward", None)
+ inst_forward = module.__dict__.get("forward")
+ if id(cls_forward) not in calls:
+ calls[id(cls_forward)] = _calls_compiled(cls_forward)
+ if (
+ getattr(cls_forward, "_unsloth_block_compiled", False)
+ or module.__dict__.get("_compiled_call_impl") is not None
+ or type(module).__name__ == "OptimizedModule"
+ or hasattr(_unwrap_callable(inst_forward), "_torchdynamo_orig_callable")
+ or hasattr(inst_forward, "_torchdynamo_orig_callable")
+ or calls[id(cls_forward)]
+ or (inst_forward is not None and _calls_compiled(inst_forward))
+ ):
+ found.append(name or type(module).__name__)
+ return found
+
+
+def _is_compiled(value):
+ try:
+ return (
+ hasattr(value, "_torchdynamo_orig_callable")
+ or hasattr(value, "_unsloth_compiled_func")
+ or type(value).__name__ == "OptimizedModule"
+ )
+ except Exception:
+ return False
+
+
+def _calls_compiled(forward):
+ """True if `forward` calls a compiled global or closure: unsloth_zoo compiled modules keep the class
+ forward eager and call a compiled `Gemma3MLP_forward`-style function."""
+ fn = _unwrap_callable(forward)
+ code = getattr(fn, "__code__", None)
+ scope = getattr(fn, "__globals__", None)
+ if code is None or not isinstance(scope, dict):
+ return False
+ if any(_is_compiled(scope.get(name)) for name in code.co_names if name in scope):
+ return True
+ for cell in getattr(fn, "__closure__", None) or ():
+ try:
+ value = cell.cell_contents
+ except ValueError:
+ continue
+ if _is_compiled(value):
+ return True
+ return False
+
+
+_SINGLE_BACKWARD_MODULES = ("unsloth.kernels.fast_lora",)
+
+
+def _unwrap_callable(value):
+ """Strip bound methods and functools.partial (Cohere MLP is MethodType(partial(apply_lora_mlp, ...)))."""
+ for _ in range(16):
+ if isinstance(value, functools.partial):
+ value = value.func
+ elif hasattr(value, "__func__"):
+ value = value.__func__
+ else:
+ break
+ return value
+
+
+def single_backward_kernels(model):
+ """Fused LoRA kernels on `model`: their backward mutates saved tensors, so it cannot run twice as split
+ backward (zero bubble) requires."""
+ found = set()
+ for module in model.modules():
+ candidates = list(module.__dict__.values()) + [getattr(type(module), "forward", None)]
+ for value in candidates:
+ fn = _unwrap_callable(value)
+ if callable(fn) and getattr(fn, "__module__", None) in _SINGLE_BACKWARD_MODULES:
+ found.add(getattr(fn, "__name__", repr(fn)))
+ return sorted(found)