Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Pipeline parallel: every torch.distributed.pipelining schedule (Interleaved1F1B, LoopedBFS, zero bubble, ZBV, DualPipeV) (#15655)
Daniel Han323696d36d09
diff --git a/tests/test_pp_schedules_gloo.py b/tests/test_pp_schedules_gloo.pynew file mode 100644index 00000000000..68d219e7b7b--- /dev/null+++ b/tests/test_pp_schedules_gloo.py@@ -0,0 +1,708 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""Multi-stage pipeline schedules (Interleaved1F1B, LoopedBFS, InterleavedZeroBubble, ZBVZeroBubble, DualPipeV)+of apply_pipeline_parallel on a gloo CPU world (2 / 4 ranks, 2 stages per rank) vs the same model in one+process, fp32; refusals before the model is cut."""++import multiprocessing+import os++if multiprocessing.parent_process() is not None or getattr(+ multiprocessing.current_process(), "_inheriting", False+):+ os.environ["CUDA_VISIBLE_DEVICES"] = "" # spawned gloo ranks: CPU only, also on GPU hosts++import sys # noqa: E402+import tempfile # noqa: E402+import types # noqa: E402+import weakref # noqa: E402+import gc as gc_module # noqa: E402++import pytest # noqa: E402+import torch # noqa: E402++sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))+from test_pp_native_gloo import ( # noqa: E402+ GEMMA3,+ LLAMA,+ MISTRAL,+ QWEN3,+ _enable_reentrant_gc,+ _free_port,+ _read_adapter,+ _reference,+ _torch_pipelining_error,+ batch,+ build,+ ppn,+)++ARCHS = {"llama": LLAMA, "qwen3": QWEN3, "gemma3": GEMMA3, "mistral": MISTRAL}+MULTI = ["interleaved1f1b", "loopedbfs", "interleavedzerobubble", "zbvzerobubble", "dualpipev"]+V_SCHEDULES = ("zbvzerobubble", "dualpipev")+N_LAYERS = 8+++def _v_supported():+ import importlib+ return importlib.import_module("unsloth.models.pp_schedules").v_schedules_supported()+++def _skip_v(schedule):+ if schedule in V_SCHEDULES and not _v_supported():+ pytest.skip(+ f"V schedules need torch >= 2.12 (refused on {torch.__version__}, see the refusal test)"+ )+++def _stage_map(+ pp,+ schedule,+ v = 2,+):+ """{global stage: pp rank} as the schedule places them."""+ out = {}+ for r in range(pp):+ if schedule in ("zbvzerobubble", "dualpipev"):+ ks = (r, 2 * pp - 1 - r)+ else:+ ks = tuple(r + k * pp for k in range(v))+ for k in ks:+ out[k] = r+ return out+++def _case_multi(+ rank,+ world,+ tmp,+ arch,+ schedule,+ n_mb = 4,+ layers = N_LAYERS,+ gc = None,+ steps = 3,+ tie = None,+ bsz = 4,+ compile_layers = False,+):+ from torch.distributed.device_mesh import init_device_mesh++ repo = ARCHS[arch]+ ref = build(repo, layers = layers, tie = tie)+ ids, labels = batch(ref, bsz = bsz)+ if gc == "reentrant":+ _enable_reentrant_gc(ref)+ elif gc == "non_reentrant":+ ref.gradient_checkpointing_enable(gradient_checkpointing_kwargs = {"use_reentrant": False})+ ref_res = _reference(ref, ids, labels, steps)+ if rank == 0:+ ref.save_pretrained(os.path.join(tmp, "ref"))+ model = build(repo, layers = layers, tie = tie)+ if gc == "reentrant":+ _enable_reentrant_gc(model)+ elif gc == "non_reentrant":+ model.gradient_checkpointing_enable(gradient_checkpointing_kwargs = {"use_reentrant": False})+ mesh = init_device_mesh("cpu", (world,), mesh_dim_names = ("pp",))+ originals = {+ "embed": weakref.ref(model.get_input_embeddings()),+ "head": weakref.ref(model.get_output_embeddings()),+ }+ if compile_layers:+ # Unsloth compiles each decoder block (torch.compile regions): AOTAutograd's compiled backward donates+ # buffers, which a split (zero bubble) backward with retain_graph cannot run.+ from torch import _dynamo+ _dynamo.config.cache_size_limit = 64+ for layer in ppn._find_parts(model).layers:+ layer.compile(dynamic = False)+ from torch._functorch import config as functorch_config++ donated_before = getattr(functorch_config, "donated_buffer", None)+ pp = ppn.apply_pipeline_parallel(model, mesh["pp"], n_microbatches = n_mb, schedule = schedule)+ donated_after = getattr(functorch_config, "donated_buffer", None)+ gc_module.collect()+ alive = {k: r() is not None for k, r in originals.items()}+ outs = {k: [] for k in pp.local_stages}+ handles = [+ module.register_forward_hook(+ lambda m, a, kw, o, k = k: outs[k].append(o.detach().clone()), with_kwargs = True+ )+ for k, module in zip(pp.local_stages, pp.stage_modules)+ ]+ calls = {}+ for i, layer in enumerate(pp.parts.layers):+ if type(layer).__name__ != "_PPPassThrough":+ handles.append(+ layer.register_forward_pre_hook(+ lambda m, a, i = i: calls.__setitem__(i, calls.get(i, 0) + 1)+ )+ )+ opt = torch.optim.AdamW(pp.parameters(), lr = 1e-2, foreach = False)+ losses, norms, grads = [], [], None+ for step in range(steps):+ loss = pp.step(ids, labels)+ if step == 0:+ grads = {+ n: p.grad.detach().clone()+ for n, p in model.named_parameters()+ if p.grad is not None+ }+ for h in handles:+ h.remove()+ norms.append(float(pp.clip_grad_norm_(1.0)))+ opt.step()+ opt.zero_grad(set_to_none = True)+ losses.append(float(loss))+ built = next(iter(pp._shapes.values()))+ torch_map = dict(getattr(built, "stage_index_to_group_rank", {}) or {})+ pp.save_adapter(os.path.join(tmp, "pp"))+ # Every layer module of the model is back in place after the stage forwards (no swap leaked).+ layer_types = [type(layer).__name__ for layer in pp.parts.layers]+ direct_error = None+ try:+ model(input_ids = ids, labels = labels)+ except RuntimeError as exc:+ direct_error = str(exc)+ return {+ "ref": {k: v for k, v in ref_res.items() if k != "hidden"},+ "hidden": ref_res["hidden"],+ "losses": losses,+ "norms": norms,+ "grads": grads,+ "outs": outs,+ "bounds": pp.bounds,+ "local_stages": pp.local_stages,+ "owned": {n for n, p in model.named_parameters() if p.requires_grad},+ "trained": {+ n.replace(".default.", "."): p.detach().clone()+ for n, p in model.named_parameters()+ if p.requires_grad+ },+ "alive": alive,+ "layer_types": layer_types,+ "direct_error": direct_error,+ "info": dict(model.__dict__.get("_unsloth_pp") or {}),+ "repr": repr(pp),+ "torch_map": torch_map,+ "layer_calls": calls,+ "donated": (donated_before, donated_after),+ "gc": gc,+ "n_mb": n_mb,+ }+++def _check_multi(+ results,+ saved,+ pp,+ schedule,+ layers = N_LAYERS,+ v = 2,+):+ total = pp * v+ placement = _stage_map(pp, schedule, v)+ ref = results[0]["ref"]+ union = set()+ for rank, res in enumerate(results):+ bounds = res["bounds"]+ assert len(bounds) == total and bounds[0][0] == 0 and bounds[-1][1] == layers, bounds+ assert all(bounds[k][1] == bounds[k + 1][0] for k in range(total - 1)), bounds+ assert set(res["local_stages"]) == {k for k, r in placement.items() if r == rank}+ # torch's schedule places the stages where this rank built them.+ assert res["torch_map"] == placement, (res["torch_map"], placement)+ assert res["info"]["stages_per_rank"] == v and tuple(res["info"]["local_stages"]) == tuple(+ res["local_stages"]+ )+ assert res["losses"] == pytest.approx(ref["losses"], rel = 1e-5, abs = 1e-6), (+ rank,+ res["losses"],+ ref["losses"],+ )+ assert res["norms"][0] == pytest.approx(ref["norms"][0], rel = 1e-5), (rank, res["norms"])+ assert res["norms"] == pytest.approx(ref["norms"], rel = 1e-4), (+ rank,+ res["norms"],+ ref["norms"],+ )+ assert set(res["grads"]) == res["owned"], sorted(set(res["grads"]) ^ res["owned"])[:6]+ assert not union & set(res["grads"]), "a trainable parameter lives on two ranks"+ union |= set(res["grads"])+ for name, grad in res["grads"].items():+ torch.testing.assert_close(grad, ref["grads"][name], rtol = 1e-4, atol = 1e-6, msg = name)+ # Each local stage's output (micro-batch by micro-batch) = the one-process hidden state entering the+ # next stage's first layer; the last stage returns the loss.+ for k, chunks in res["outs"].items():+ assert len(chunks) == res["n_mb"], (k, len(chunks))+ if k < total - 1:+ torch.testing.assert_close(+ torch.cat(chunks), res["hidden"][bounds[k][1]], rtol = 1e-5, atol = 1e-5, msg = str(k)+ )+ else:+ assert all(o.dim() == 0 for o in chunks)+ owns_first, owns_last = 0 in res["local_stages"], total - 1 in res["local_stages"]+ assert res["alive"] == {"embed": owns_first, "head": owns_last}, (rank, res["alive"])+ owned_layers = {i for k in res["local_stages"] for i in range(*bounds[k])}+ if res["gc"] is None:+ # Each owned layer runs once per micro-batch: a stage never runs this rank's other stages' layers.+ assert res["layer_calls"] == {i: res["n_mb"] for i in owned_layers}, (+ rank,+ res["layer_calls"],+ )+ for i, name in enumerate(res["layer_types"]):+ assert (name == "_PPPassThrough") == (i not in owned_layers), (rank, i, name)+ assert "split for pipeline parallelism" in (res["direct_error"] or "")+ # Zero bubble (split backward) turns AOTAutograd's donated buffers off; other schedules leave it alone.+ before, after = res["donated"]+ if before is not None:+ split = schedule in ("interleavedzerobubble", "zbvzerobubble", "dualpipev")+ assert after == (False if split else before), (schedule, res["donated"])+ assert union == set(ref["grads"]), sorted(union ^ set(ref["grads"]))[:6]+ assert set(saved["pp"]) == set(saved["ref"]), sorted(set(saved["pp"]) ^ set(saved["ref"]))[:6]+ trained = {}+ for res in results:+ trained.update(res["trained"])+ for key, value in saved["pp"].items():+ assert torch.equal(value, trained[key]), key+ for key, value in saved["ref"].items():+ torch.testing.assert_close(saved["pp"][key], value, rtol = 1e-3, atol = 2e-3, msg = key)+++def _case_refusals(rank, world, tmp):+ """Every refusal leaves the model uncut: no pass-through layer, the embedding and lm_head in place."""+ from torch.distributed.device_mesh import init_device_mesh++ mesh = init_device_mesh("cpu", (world,), mesh_dim_names = ("pp",))["pp"]+ out = {}++ def attempt(key, model, **kwargs):+ embed, head = model.get_input_embeddings(), model.get_output_embeddings()+ try:+ ppn.apply_pipeline_parallel(model, mesh, **kwargs)+ out[key] = None+ except (ValueError, RuntimeError) as exc:+ out[key] = str(exc)+ layers = ppn._find_parts(model).layers+ out[key + ":uncut"] = (+ not any(type(layer).__name__ == "_PPPassThrough" for layer in layers)+ and model.get_input_embeddings() is embed+ and model.get_output_embeddings() is head+ and not ppn.is_pipeline_parallel(model)+ )++ attempt("unknown", build(LLAMA), schedule = "pipedream")+ attempt(+ "gpipe_two_stages",+ build(LLAMA),+ schedule = "gpipe",+ config = ppn.PPConfig(stages_per_rank = 2),+ )+ attempt(+ "zbv_three_stages",+ build(LLAMA, layers = 8),+ schedule = "zbvzerobubble",+ n_microbatches = 4,+ config = ppn.PPConfig(stages_per_rank = 3),+ )+ # DualPipeV needs at least as many micro-batches as stages (4 here): torch's own check, run up front.+ attempt("dualpipev_few_microbatches", build(LLAMA), schedule = "dualpipev", n_microbatches = 2)+ # Interleaved 1F1B: micro-batches a multiple of the rounds (5 // 2 = 2 rounds).+ attempt("interleaved_rounds", build(LLAMA), schedule = "interleaved1f1b", n_microbatches = 5)+ attempt("too_many_stages", build(LLAMA, layers = 3), schedule = "interleaved1f1b", n_microbatches = 4)++ reentrant = build(LLAMA)+ _enable_reentrant_gc(reentrant)+ attempt("zb_reentrant_gc", reentrant, schedule = "interleavedzerobubble", n_microbatches = 4)+ attempt("v_schedule", build(LLAMA), schedule = "zbvzerobubble", n_microbatches = 4)++ def apply_lora_qkv(self, x): # stands in for unsloth.kernels.fast_lora's fused kernel+ raise AssertionError("not called")++ apply_lora_qkv.__module__ = "unsloth.kernels.fast_lora"+ fused = build(LLAMA)+ for layer in ppn._find_parts(fused).layers:+ layer.self_attn.apply_qkv = apply_lora_qkv+ attempt("zb_fused_lora", fused, schedule = "interleavedzerobubble", n_microbatches = 4)+ fused_ok = build(LLAMA)+ for layer in ppn._find_parts(fused_ok).layers:+ layer.self_attn.apply_qkv = apply_lora_qkv+ attempt("interleaved_fused_lora", fused_ok, schedule = "interleaved1f1b", n_microbatches = 4)++ fake = types.ModuleType("unsloth_zoo.gradient_checkpointing") # pre-#2071 zoo: no slot leases+ saved = sys.modules.get("unsloth_zoo.gradient_checkpointing")+ sys.modules["unsloth_zoo.gradient_checkpointing"] = fake+ try:+ old_zoo = build(LLAMA)+ _enable_reentrant_gc(old_zoo)+ attempt("interleaved_old_zoo", old_zoo, schedule = "interleaved1f1b", n_microbatches = 4)+ bfs = build(LLAMA)+ _enable_reentrant_gc(bfs)+ # LoopedBFS runs every forward of a step before its backwards: smart checkpointing is safe there.+ attempt("loopedbfs_old_zoo", bfs, schedule = "loopedbfs", n_microbatches = 4)+ finally:+ if saved is None:+ sys.modules.pop("unsloth_zoo.gradient_checkpointing", None)+ else:+ sys.modules["unsloth_zoo.gradient_checkpointing"] = saved+ return out+++def _case_dp_refusal(rank, world, tmp):+ """pp 2 x dp 2 mesh: several stages per rank do not compose with native FSDP2 over dp (refused before the cut);+ the same mesh with one stage per rank is still accepted."""+ from torch.distributed.device_mesh import init_device_mesh++ mesh = init_device_mesh("cpu", (2, 2), mesh_dim_names = ("pp", "dp"))+ model = build(LLAMA)+ out = {}+ try:+ ppn.apply_pipeline_parallel(model, mesh["pp"], n_microbatches = 4, schedule = "interleaved1f1b")+ out["multi"] = None+ except ValueError as exc:+ out["multi"] = str(exc)+ out["uncut"] = not ppn.is_pipeline_parallel(model)+ pp = ppn.apply_pipeline_parallel(build(LLAMA), mesh["pp"], n_microbatches = 2, schedule = "gpipe")+ out["single"] = pp.stages_per_rank+ return out+++def _case_compiled_refusal(rank, world, tmp, schedule):+ """A split-backward schedule over compiled blocks on a torch whose split backward cannot run them."""+ from torch.distributed.device_mesh import init_device_mesh++ mesh = init_device_mesh("cpu", (world,), mesh_dim_names = ("pp",))["pp"]+ model = build(LLAMA)+ for layer in ppn._find_parts(model).layers:+ layer.compile(dynamic = False)+ embed, head = model.get_input_embeddings(), model.get_output_embeddings()+ out = {}+ try:+ ppn.apply_pipeline_parallel(model, mesh, n_microbatches = 4, schedule = schedule)+ out["error"] = None+ except (ValueError, RuntimeError) as exc:+ out["error"] = str(exc)+ out["uncut"] = (+ not any(type(layer).__name__ == "_PPPassThrough" for layer in ppn._find_parts(model).layers)+ and model.get_input_embeddings() is embed+ and model.get_output_embeddings() is head+ and not ppn.is_pipeline_parallel(model)+ )+ return out+++CASES = {+ "multi": _case_multi,+ "refusals": _case_refusals,+ "dp_refusal": _case_dp_refusal,+ "compiled_refusal": _case_compiled_refusal,+}+++def _worker(rank, world, port, case, tmp, kwargs):+ os.environ.update(+ MASTER_ADDR = "127.0.0.1",+ MASTER_PORT = str(port),+ RANK = str(rank),+ WORLD_SIZE = str(world),+ LOCAL_RANK = str(rank),+ LOCAL_WORLD_SIZE = str(world),+ )+ import datetime++ import torch.distributed as dist++ torch.set_num_threads(2)+ dist.init_process_group(+ "gloo", rank = rank, world_size = world, timeout = datetime.timedelta(seconds = 180)+ )+ try:+ res = CASES[case](rank, world, tmp, **kwargs)+ torch.save(res, os.path.join(tmp, f"result_{rank}.pt"))+ finally:+ dist.destroy_process_group()+++def _spawn(+ case,+ world = 2,+ keep = None,+ **kwargs,+):+ if _torch_pipelining_error():+ pytest.skip(_torch_pipelining_error())+ ctx = multiprocessing.get_context("spawn")+ port = _free_port()+ with tempfile.TemporaryDirectory() as tmp:+ procs = [+ ctx.Process(target = _worker, args = (r, world, port, case, tmp, kwargs))+ for r in range(world)+ ]+ for p in procs:+ p.start()+ for p in procs:+ p.join(600)+ codes = [p.exitcode for p in procs]+ assert codes == [0] * world, f"rank exit codes {codes}"+ results = [+ torch.load(os.path.join(tmp, f"result_{r}.pt"), weights_only = False)+ for r in range(world)+ ]+ if keep is not None:+ keep["ref"] = _read_adapter(os.path.join(tmp, "ref"))+ keep["pp"] = _read_adapter(os.path.join(tmp, "pp"))+ return results++[email protected]("schedule", MULTI)+def test_pp2_multi_stage_schedules_match_one_process(schedule):+ _skip_v(schedule)+ saved = {}+ results = _spawn("multi", 2, keep = saved, arch = "llama", schedule = schedule)+ _check_multi(results, saved, 2, schedule)++[email protected]("schedule", ["interleaved1f1b", "zbvzerobubble"])+def test_pp4_multi_stage_schedules_match_one_process(schedule):+ _skip_v(schedule)+ saved = {}+ results = _spawn("multi", 4, keep = saved, arch = "mistral", schedule = schedule, n_mb = 4)+ _check_multi(results, saved, 4, schedule)++[email protected]("arch", ["qwen3", "gemma3"])+def test_pp2_interleaved1f1b_other_archs_match_one_process(arch):+ saved = {}+ results = _spawn("multi", 2, keep = saved, arch = arch, schedule = "interleaved1f1b", n_mb = 2, bsz = 4)+ _check_multi(results, saved, 2, "interleaved1f1b")+++def test_pp2_dualpipev_tied_embeddings_both_ends_on_rank0():+ # V placement puts the first and the last stage on rank 0: the tied embedding / lm_head stay there only.+ _skip_v("dualpipev")+ saved = {}+ results = _spawn("multi", 2, keep = saved, arch = "gemma3", schedule = "dualpipev", tie = True)+ _check_multi(results, saved, 2, "dualpipev")+ assert results[0]["alive"] == {"embed": True, "head": True}+ assert results[1]["alive"] == {"embed": False, "head": False}++[email protected]("schedule", ["interleaved1f1b", "loopedbfs"])+def test_pp2_multi_stage_reentrant_gradient_checkpointing_matches_one_process(schedule):+ saved = {}+ results = _spawn("multi", 2, keep = saved, arch = "qwen3", schedule = schedule, gc = "reentrant")+ _check_multi(results, saved, 2, schedule)++[email protected]("schedule", ["interleavedzerobubble", "zbvzerobubble"])+def test_pp2_zero_bubble_with_non_reentrant_checkpointing_matches_one_process(schedule):+ _skip_v(schedule)+ saved = {}+ results = _spawn("multi", 2, keep = saved, arch = "llama", schedule = schedule, gc = "non_reentrant")+ _check_multi(results, saved, 2, schedule)++[email protected]("schedule", ["interleavedzerobubble", "zbvzerobubble"])+def test_pp2_zero_bubble_with_compiled_blocks_matches_one_process(schedule):+ sched = sys.modules["unsloth.models.pp_schedules"]+ if not sched.split_backward_keeps_function_nodes():+ # torch <= 2.12: split backward reads _input_metadata of compiled (AOTAutograd) nodes whose graph root+ # is already released; refused before the model is cut.+ # V schedules on torch < 2.12 hit the earlier V-placement refusal first.+ v_refused = schedule in V_SCHEDULES and not _v_supported()+ for res in _spawn("compiled_refusal", 2, schedule = schedule):+ if v_refused:+ assert "(torch >= 2.12)" in (res["error"] or ""), res+ else:+ assert "cannot run over torch.compile'd blocks" in (res["error"] or ""), res+ assert schedule in res["error"] and "interleaved1f1b" in res["error"], res+ assert res["uncut"], res+ return+ _skip_v(schedule)+ saved = {}+ results = _spawn("multi", 2, keep = saved, arch = "llama", schedule = schedule, compile_layers = True)+ _check_multi(results, saved, 2, schedule)+++def test_pp4_interleaved1f1b_x_tp2_matches_one_process():+ from test_pp_fsdp2_gloo import _check, _spawn as compose_spawn+ dims = {"pp": 2, "tp": 2}+ _check(+ compose_spawn(+ "compose", world = 4, arch = "llama", dims = dims, schedule = "interleaved1f1b", n_mb = 4+ ),+ dims,+ )+++def test_multi_stage_refusals_leave_the_model_uncut():+ res = _spawn("refusals", 2)[0]+ for key in [k for k in res if not k.endswith(":uncut")]:+ if key in ("loopedbfs_old_zoo", "interleaved_fused_lora") or (+ key == "v_schedule" and _v_supported()+ ):+ continue+ assert res[key + ":uncut"], (key, res[key])+ assert "pipeline schedule must be one of" in res["unknown"]+ assert "runs one stage per rank" in res["gpipe_two_stages"]+ assert "exactly 2 stages per rank" in res["zbv_three_stages"]+ assert (+ "DualPipeV requires at least as many microbatches as stages"+ if _v_supported()+ else "(torch >= 2.12)"+ ) in res["dualpipev_few_microbatches"]+ assert "multiple of the number of rounds" in res["interleaved_rounds"]+ assert "cannot split 3 decoder layers into 4 pipeline stages" in res["too_many_stages"]+ assert "splits each backward into an input and a weight pass" in res["zb_reentrant_gc"]+ assert "interleaves forwards and backwards" in res["interleaved_old_zoo"]+ assert res["loopedbfs_old_zoo"] is None+ assert (+ "runs the backward of Unsloth's fused LoRA kernels (apply_lora_qkv) twice"+ in res["zb_fused_lora"]+ )+ assert res["interleaved_fused_lora"] is None # full-backward schedules keep the fused kernels+ if _v_supported():+ assert res["v_schedule"] is None+ else:+ assert "(torch >= 2.12)" in res["v_schedule"]+++def test_multi_stage_on_a_pp_x_dp_mesh_is_refused_before_the_cut():+ res = _spawn("dp_refusal", 4)+ for r in res:+ assert "several stages per rank is not supported" in (r["multi"] or ""), r+ assert r["uncut"] and r["single"] == 1+++def test_schedule_names_normalize():+ sched = sys.modules["unsloth.models.pp_schedules"]+ assert sched.schedule_spec("Interleaved1F1B").name == "interleaved1f1b"+ assert sched.schedule_spec("ScheduleDualPipeV").name == "dualpipev"+ assert sched.schedule_spec("zbv").name == "zbvzerobubble"+ assert sched.schedule_spec("looped_bfs").name == "loopedbfs"+ assert sched.schedule_spec("GPipe").name == "gpipe"+++def test_split_backward_probe_matches_torch_node_ownership():+ # THPFunction owns its node (c10::intrusive_ptr) from torch 2.13; up to 2.12 it is a std::weak_ptr.+ sched = sys.modules["unsloth.models.pp_schedules"]+ major, minor = (int(v) for v in torch.__version__.split(".")[:2])+ assert sched.split_backward_keeps_function_nodes() == ((major, minor) >= (2, 13))+++def test_compiled_blocks_refused_for_split_backward_on_affected_torch(monkeypatch):+ sched = sys.modules["unsloth.models.pp_schedules"]+ split = {"interleaved": True, "split_backward": True}+ full = {"interleaved": True, "split_backward": False}++ def model_with(install):+ block = torch.nn.Linear(2, 2)+ install(block)+ return torch.nn.Sequential(torch.nn.Linear(2, 2), block)++ def marked(block):+ cls = type("Block", (torch.nn.Linear,), {})+ forward = lambda self, x: torch.nn.Linear.forward(self, x) # noqa: E731+ forward._unsloth_block_compiled = True+ cls.forward = forward+ block.__class__ = cls++ installs = {+ "module_compile": lambda b: b.compile(dynamic = False),+ "block_compile": marked,+ "compiled_forward": lambda b: setattr(b, "forward", torch.compile(b.forward)),+ }+ assert sched.compiled_blocks(torch.nn.Sequential(torch.nn.Linear(2, 2))) == []+ assert sched.compiled_blocks(torch.compile(torch.nn.Linear(2, 2)))+ for schedule in ("interleavedzerobubble", "zbvzerobubble", "dualpipev"):+ spec = sched.schedule_spec(schedule)+ for kind, install in installs.items():+ assert sched.compiled_blocks(model_with(install)) == ["1"], kind+ monkeypatch.setattr(ppn, "split_backward_keeps_function_nodes", lambda: False)+ with pytest.raises(ValueError, match = "cannot run over torch.compile'd blocks") as exc:+ ppn._check_schedule(model_with(install), spec, split)+ assert schedule in str(exc.value) and "UNSLOTH_BLOCK_COMPILE=0" in str(exc.value)+ ppn._check_schedule(model_with(install), spec, full) # full backward: not affected+ monkeypatch.setattr(ppn, "split_backward_keeps_function_nodes", lambda: True)+ ppn._check_schedule(model_with(install), spec, split) # torch >= 2.13: allowed+ monkeypatch.setattr(ppn, "split_backward_keeps_function_nodes", lambda: False)+ ppn._check_schedule(torch.nn.Sequential(torch.nn.Linear(2, 2)), spec, split) # nothing compiled+++def test_forward_calling_compiled_function_refused_for_split_backward(monkeypatch):+ # unsloth_zoo's generated unsloth_compiled_module_* files keep the class forward eager and call a module-level+ # compiled function from it (Gemma3MLP.forward -> Gemma3MLP_forward), through torch_compile_with_fallback's+ # functools.wraps wrapper. Its backward is a CompiledFunctionBackward node: torch <= 2.12's split backward fails+ # on it after the cut ("_input_metadata is invalid"), as on a compiled block.+ import functools++ sched = sys.modules["unsloth.models.pp_schedules"]+ split = {"interleaved": True, "split_backward": True}++ def eager(self, x):+ return torch.nn.functional.linear(x, self.weight, self.bias)++ compiled = torch.compile(eager, dynamic = False)++ @functools.wraps(compiled)+ def dispatch(*args, **kwargs): # unsloth_zoo _eager_during_decode's shape+ return compiled(*args, **kwargs)++ dispatch.__wrapped__ = eager+ dispatch._unsloth_compiled_func = compiled++ def block_calling(target):+ scope = {"Block_forward": target}+ exec("def forward(self, x):\n return Block_forward(self, x)\n", scope)+ cls = type("Block", (torch.nn.Linear,), {"forward": scope["forward"]})+ block = torch.nn.Linear(2, 2)+ block.__class__ = cls+ return torch.nn.Sequential(torch.nn.Linear(2, 2), block)++ for kind, target in {"torch_compile": compiled, "zoo_wrapper": dispatch}.items():+ assert sched.compiled_blocks(block_calling(target)) == ["1"], kind+ spec = sched.schedule_spec("interleavedzerobubble")+ monkeypatch.setattr(ppn, "split_backward_keeps_function_nodes", lambda: False)+ with pytest.raises(ValueError, match = "cannot run over torch.compile'd blocks"):+ ppn._check_schedule(block_calling(target), spec, split)+ monkeypatch.setattr(ppn, "split_backward_keeps_function_nodes", lambda: True)+ ppn._check_schedule(block_calling(target), spec, split) # torch >= 2.13: allowed++ # A compiled helper captured from an enclosing scope (a closure, not a global) is found too.+ def closure_forward(self, x):+ return compiled(self, x)++ closure_block = torch.nn.Linear(2, 2)+ closure_block.__class__ = type("Block", (torch.nn.Linear,), {"forward": closure_forward})+ assert sched.compiled_blocks(torch.nn.Sequential(torch.nn.Linear(2, 2), closure_block)) == ["1"]+ # An eager helper called the same way is not a compiled block.+ assert sched.compiled_blocks(block_calling(eager)) == []+++def test_single_backward_kernels_unwraps_partials():+ # patch_peft_model binds Cohere's MLP kernel as MethodType(partial(apply_lora_mlp, inplace=False), mlp).+ import functools++ sched = sys.modules["unsloth.models.pp_schedules"]++ def apply_lora_mlp_swiglu(+ self,+ X,+ inplace = True,+ ): # stands in for fast_lora's fused MLP kernel+ raise AssertionError("not called")++ apply_lora_mlp_swiglu.__module__ = "unsloth.kernels.fast_lora"+ cohere = functools.partial(apply_lora_mlp_swiglu, inplace = False)+ installs = {+ "plain": lambda m: types.MethodType(apply_lora_mlp_swiglu, m),+ "partial": lambda m: types.MethodType(cohere, m),+ "nested_partial": lambda m: types.MethodType(functools.partial(cohere), m),+ "unbound_partial": lambda m: cohere,+ }+ for attr in ("forward", "_unsloth_forward"):+ for kind, install in installs.items():+ mlp = torch.nn.Linear(2, 2)+ setattr(mlp, attr, install(mlp))+ model = torch.nn.Sequential(mlp)+ assert sched.single_backward_kernels(model) == ["apply_lora_mlp_swiglu"], (attr, kind)+ assert sched.single_backward_kernels(torch.nn.Sequential(torch.nn.Linear(2, 2))) == []diff --git a/unsloth/models/pp_native.py b/unsloth/models/pp_native.pyindex f7a91da3ef6..233015fac8d 100644--- a/unsloth/models/pp_native.py+++ b/unsloth/models/pp_native.py@@ -1,8 +1,9 @@# SPDX-License-Identifier: AGPL-3.0-only# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.-"""Pipeline parallel, one stage per rank; LoRA first; order pp outermost then dp then tp, split-before parallelizing; save_adapter is collective, rank 0 writes; param names unchanged."""+"""Pipeline parallel, one or more stages per rank (every torch schedule, pp_schedules.py); LoRA first; order+pp outermost then dp then tp, split before parallelizing; save_adapter is collective, rank 0 writes; param+names unchanged."""from __future__ import annotations@@ -21,6 +22,20 @@ from typing import Optionalimport torchimport torch.nn as nn+from .pp_schedules import (+ SCHEDULE_NAMES,+ allow_split_backward,+ compiled_blocks,+ non_reentrant_checkpointing_only,+ split_backward_keeps_function_nodes,+ single_backward_kernels,+ resolve_stages_per_rank,+ schedule_spec,+ stage_indices,+ torch_schedule_class,+ validate_schedule,+)+__all__ = ["PPConfig","PipelineParallel",@@ -34,7 +49,7 @@ __all__ = []_PP_FLAG = "_unsloth_pp"-_SCHEDULES = ("gpipe", "1f1b")+_SCHEDULES = SCHEDULE_NAMES_MOE_CONFIG_KEYS = ("num_experts", "num_local_experts", "n_routed_experts", "moe_num_experts")# Layers reading another layer's state, or extra per-layer inputs: a stage boundary would cut them._CROSS_LAYER_CONFIG_KEYS = (@@ -45,6 +60,11 @@ _CROSS_LAYER_CONFIG_KEYS = ()_ATTENTION_LAYER_TYPES = {"full_attention", "sliding_attention", "chunked_attention", "attention"}_STEP_KWARGS = ("attention_mask", "position_ids")+_MULTI_STAGE_DP = (+ "Unsloth: native FSDP2 data parallel over a pipeline with several stages per rank is not supported "+ "(each FSDP2 root must be one stage's module); use one stage per rank (schedule = 'gpipe' / '1f1b') "+ "with a dp mesh, or no dp mesh."+)_SIMPLE = (bool, int, float, str, type(None))@@ -60,6 +80,9 @@ class PPConfig:sync_trainable: bool = True# (micro-batch size, sequence length) shapes whose stage / schedule stay built.max_cached_shapes: int = 4+ # Stages per rank for multi-stage schedules (interleaved1f1b, loopedbfs, ...); None = the schedule's+ # default (1 for gpipe / 1f1b, 2 otherwise). The decoder is cut into pp x stages_per_rank stages.+ stages_per_rank: Optional[int] = Noneclass _StageOutput(BaseException):@@ -218,6 +241,8 @@ def check_pp_dp_composition(model, dp_mesh):raise ValueError("Unsloth: this pipeline stage is already sharded over a data parallel mesh.")+ if stage.stages_per_rank > 1:+ raise ValueError(_MULTI_STAGE_DP)if dp_mesh is None:raise ValueError("Unsloth: native FSDP2 over a pipeline parallel model needs FSDP2Config(mesh = mesh['dp']), the "@@ -731,6 +756,14 @@ class _StageContext:self.tuple_format = tuple_formatself.active = Falseself.stage_input = None+ # The step's token count, given to every stage forward from here rather than as a schedule kwarg:+ # a non-tensor kwarg reaches torch's split backward (zero bubble) as a stage input on torch < 2.13.+ self.num_items_in_batch = None++ def configure(self, start, end, is_first, is_last):+ """The stage about to run (several stages of one rank share this context, one runs at a time)."""+ self.start, self.end = start, end+ self.is_first, self.is_last = is_first, is_lastdef pass_through(self, index, args, kwargs):if not self.active:@@ -748,17 +781,44 @@ class _StageContext:class _PPStageModule(nn.Module):- """What PipelineStage runs: the whole model's forward on this stage's slice."""+ """What PipelineStage runs: the whole model's forward on this stage's slice.- def __init__(self, model, ctx, dtype, hidden_size):+ With several stages per rank, `stage` = (start, end, is_first, is_last, swaps): the context is set to this+ stage and, for the duration of the forward, `swaps` [(container, key, stand-in)] replace this rank's+ modules that belong to its other stages (their layers by pass-throughs, the embedding by its stub)."""++ def __init__(+ self,+ model,+ ctx,+ dtype,+ hidden_size,+ stage = None,+ ):super().__init__()self.model = modelself.__dict__["_pp_ctx"] = ctxself.__dict__["_pp_dtype"] = dtypeself.__dict__["_pp_hidden"] = hidden_sizeself.__dict__["_pp_warned"] = False+ self.__dict__["_pp_stage"] = stagedef forward(self, x, **kwargs):+ stage = self._pp_stage+ if stage is None:+ return self._stage_forward(x, **kwargs)+ start, end, is_first, is_last, swaps = stage+ self._pp_ctx.configure(start, end, is_first, is_last)+ originals = [(container, key, container._modules[key]) for container, key, _ in swaps]+ for container, key, stand_in in swaps:+ container._modules[key] = stand_in+ try:+ return self._stage_forward(x, **kwargs)+ finally:+ for container, key, original in originals:+ container._modules[key] = original++ def _stage_forward(self, x, **kwargs):ctx = self._pp_ctxif ctx.is_first:input_ids = x@@ -769,6 +829,8 @@ class _PPStageModule(nn.Module):try:# Pass-through layers return hidden states only: no per-layer attentions / hidden states.kwargs.update(output_attentions = False, output_hidden_states = False)+ if ctx.num_items_in_batch is not None:+ kwargs["num_items_in_batch"] = ctx.num_items_in_batchout = self.model(input_ids = input_ids, use_cache = False, **kwargs)except _StageOutput as stop:hidden = stop.hidden@@ -841,21 +903,48 @@ def _check_torch_pipelining():)-def _check_schedule(model, schedule):- if schedule not in _SCHEDULES:- raise ValueError(- f"Unsloth: pipeline schedule must be one of {_SCHEDULES}, got {schedule!r}."- )+def _check_schedule(model, spec, props):+ """Refusals for the schedule `spec` whose compute order has `props` (pp_schedules.validate_schedule)."""if (- schedule == "1f1b"+ props["interleaved"]and _gradient_checkpointing_on(model)and _zoo_slot_lease_present() is False):raise ValueError(- "Unsloth: the 1F1B pipeline schedule interleaves forwards and backwards, which this "+ f"Unsloth: the {spec.name} pipeline schedule interleaves forwards and backwards, which this ""unsloth_zoo's smart gradient checkpointing does not support (offloaded activations of a pending "- "micro-batch would be overwritten). Use schedule = 'gpipe', or update unsloth_zoo (host slot "- "leases, unsloth-zoo PR #2071)."+ "micro-batch would be overwritten). Use schedule = 'gpipe' (or 'loopedbfs'), or update unsloth_zoo "+ "(host slot leases, unsloth-zoo PR #2071)."+ )+ fused = single_backward_kernels(model) if props["split_backward"] else []+ if fused:+ raise ValueError(+ f"Unsloth: the {spec.name} pipeline schedule splits each backward into an input and a weight pass, "+ f"which runs the backward of Unsloth's fused LoRA kernels ({', '.join(fused)}) twice; they can run it "+ "once. Use schedule = 'interleaved1f1b' (or 'loopedbfs'), or load the model without the fused LoRA "+ "path."+ )+ if props["split_backward"] and not split_backward_keeps_function_nodes():+ compiled = compiled_blocks(model)+ if compiled:+ shown = ", ".join(compiled[:3]) + (+ f" and {len(compiled) - 3} more" if len(compiled) > 3 else ""+ )+ raise ValueError(+ f"Unsloth: the {spec.name} pipeline schedule splits each backward into an input and a weight "+ f"pass, which torch {torch.__version__} cannot run over torch.compile'd blocks ({shown}): its "+ "weight pass reads _input_metadata of the compiled (AOTAutograd) backward nodes after their graph "+ "root is released, which needs torch >= 2.13. Use schedule = 'interleaved1f1b' (or 'loopedbfs', "+ "a non zero-bubble schedule), disable the compiled blocks (UNSLOTH_BLOCK_COMPILE=0, UNSLOTH_COMPILE_DISABLE=1, no "+ "module.compile()), or upgrade torch. The model was not modified."+ )+ if props["split_backward"] and non_reentrant_checkpointing_only(model) is False:+ raise ValueError(+ f"Unsloth: the {spec.name} pipeline schedule splits each backward into an input and a weight pass "+ "(torch.autograd.grad), which reentrant gradient checkpointing (transformers' default, Unsloth "+ "smart checkpointing) cannot run. Turn gradient checkpointing off, use torch's non-reentrant "+ "checkpointing (gradient_checkpointing_kwargs = {'use_reentrant': False}), or use "+ "schedule = 'interleaved1f1b'.")@@ -873,19 +962,41 @@ class PipelineParallel:"""This rank's pipeline stage of a model split by apply_pipeline_parallel."""def __init__(- self, model, mesh, parts, bounds, n_microbatches, schedule, config, tuple_format, dtype+ self,+ model,+ mesh,+ parts,+ bounds,+ n_microbatches,+ schedule,+ config,+ tuple_format,+ dtype,+ stages_per_rank = 1,):import torch.distributed as distself.model = modelself.mesh = meshself.group = mesh.get_group()- self.num_stages = mesh.size()- self.stage_index = mesh.get_local_rank()- self.bounds = bounds- self.start, self.end = bounds[self.stage_index]+ self.num_stages = mesh.size() # pp ranks+ self.stage_index = (+ mesh.get_local_rank()+ ) # this rank's pp index (its stage with one stage per rank)+ self.bounds = bounds # [start, end) decoder layers of every stage (pp x stages_per_rank)+ self.spec = schedule_spec(schedule)+ self.stages_per_rank = int(stages_per_rank)+ self.total_stages = self.num_stages * self.stages_per_rank+ if len(bounds) != self.total_stages:+ raise ValueError(+ f"Unsloth: {len(bounds)} layer ranges for {self.total_stages} pipeline stages."+ )+ self.local_stages = stage_indices(+ self.spec, self.stage_index, self.num_stages, self.stages_per_rank+ )+ self.start, self.end = bounds[self.local_stages[0]]self.n_microbatches = n_microbatches- self.schedule = schedule+ self.schedule = self.spec.nameself.config = configself.parts = partsself._foreign_prefixes = () # set by apply_pipeline_parallel@@ -894,7 +1005,15 @@ class PipelineParallel:hidden = getattr(_text_config(model), "hidden_size", None) or parts.embed.weight.shape[-1]self.hidden_size = int(hidden)self.module = _PPStageModule(model, self.ctx, dtype, self.hidden_size)+ self.stage_modules = [self.module] # one per local stage (set by apply_pipeline_parallel)self._global_ranks = [dist.get_global_rank(self.group, i) for i in range(self.num_stages)]+ last_owner = next(+ r+ for r in range(self.num_stages)+ if self.total_stages - 1+ in stage_indices(self.spec, r, self.num_stages, self.stages_per_rank)+ )+ self._last_global_rank = self._global_ranks[last_owner]self._shapes = collections.OrderedDict()self._scaler = Noneself._step_scaler = (@@ -962,6 +1081,8 @@ class PipelineParallel:one post_backward in perform_reduce_grad), as torchtitan's stage submodule does."""from torch.distributed.fsdp import MixedPrecisionPolicy, fully_shard+ if self.stages_per_rank > 1:+ raise ValueError(_MULTI_STAGE_DP)policy = MixedPrecisionPolicy(param_dtype = None,reduce_dtype = getattr(mp_policy, "reduce_dtype", None),@@ -973,11 +1094,13 @@ class PipelineParallel:@propertydef is_first(self):- return self.stage_index == 0+ """This rank holds the first stage (the input embedding)."""+ return 0 in self.local_stages@propertydef is_last(self):- return self.stage_index == self.num_stages - 1+ """This rank holds the last stage (final norm, lm_head and the loss)."""+ return self.total_stages - 1 in self.local_stages@propertydef device(self):@@ -991,6 +1114,14 @@ class PipelineParallel:return [p for p in self.module.parameters() if p.requires_grad]def __repr__(self):+ if self.stages_per_rank > 1:+ ranges = ", ".join(+ f"{k}: [{self.bounds[k][0]}, {self.bounds[k][1]})" for k in self.local_stages+ )+ return (+ f"PipelineParallel(rank {self.stage_index} / {self.num_stages}, stages {{{ranges}}} of "+ f"{self.total_stages}, {self.schedule}, n_microbatches = {self.n_microbatches})"+ )return (f"PipelineParallel(stage {self.stage_index} / {self.num_stages}, layers [{self.start}, "f"{self.end}), {self.schedule}, n_microbatches = {self.n_microbatches})"@@ -1006,43 +1137,49 @@ class PipelineParallel:return output if self._scaler is None else self._scaler.scale(output)def _schedule_for(self, mb, seq):- from torch.distributed.pipelining import PipelineStage, Schedule1F1B, ScheduleGPipe+ from torch.distributed.pipelining import PipelineStagekey = (mb, seq)if key in self._shapes:self._shapes.move_to_end(key)return self._shapes[key]device = self.device- if self.is_first:- example_in = torch.empty((mb, seq), dtype = torch.long, device = "meta")- else:- # requires_grad is part of the static metadata: received hidden states need their grad.- example_in = torch.empty(- (mb, seq, self.hidden_size),- dtype = self.activation_dtype,- device = "meta",- requires_grad = True,- )- if self.is_last:- example_out = torch.empty((), dtype = torch.float32, device = "meta", requires_grad = True)- else:- example_out = torch.empty(- (mb, seq, self.hidden_size),- dtype = self.activation_dtype,- device = "meta",- requires_grad = True,+ hidden = (mb, seq, self.hidden_size)+ stages = []+ for index, module in zip(self.local_stages, self.stage_modules):+ if index == 0:+ example_in = torch.empty((mb, seq), dtype = torch.long, device = "meta")+ else:+ # requires_grad is part of the static metadata: received hidden states need their grad.+ example_in = torch.empty(+ hidden, dtype = self.activation_dtype, device = "meta", requires_grad = True+ )+ if index == self.total_stages - 1:+ example_out = torch.empty(+ (), dtype = torch.float32, device = "meta", requires_grad = True+ )+ else:+ example_out = torch.empty(+ hidden, dtype = self.activation_dtype, device = "meta", requires_grad = True+ )+ stages.append(+ PipelineStage(+ module,+ index,+ self.total_stages,+ device,+ input_args = example_in,+ output_args = example_out,+ group = self.group,+ ))- stage = PipelineStage(- self.module,- self.stage_index,- self.num_stages,- device,- input_args = example_in,- output_args = example_out,- group = self.group,+ cls = torch_schedule_class(self.spec)+ schedule = cls(+ stages if self.spec.multi else stages[0],+ self.n_microbatches,+ loss_fn = self._loss_fn,+ scale_grads = False,)- cls = ScheduleGPipe if self.schedule == "gpipe" else Schedule1F1B- schedule = cls(stage, self.n_microbatches, loss_fn = self._loss_fn, scale_grads = False)self._shapes[key] = schedulewhile len(self._shapes) > max(1, int(self.config.max_cached_shapes)):self._shapes.popitem(last = False)@@ -1118,7 +1255,7 @@ class PipelineParallel:value = value.to(device)step_kwargs[key] = valuestep_kwargs["labels"] = labels.to(device)- step_kwargs["num_items_in_batch"] = num_items_in_batch+ self.ctx.num_items_in_batch = num_items_in_batchif scaler is not None and scaler.is_enabled():scaler.scale(torch.zeros((), device = device)@@ -1133,6 +1270,7 @@ class PipelineParallel:schedule.step(*args, target = target, return_outputs = False, **step_kwargs)finally:self._scaler = None+ self.ctx.num_items_in_batch = Noneif self.dp_mesh is not None:self._average_replicated_grads()if self.is_last:@@ -1145,7 +1283,7 @@ class PipelineParallel:else:total = torch.zeros(1, dtype = torch.float32, device = device)if self.num_stages > 1:- dist.broadcast(total, src = self._global_ranks[-1], group = self.group)+ dist.broadcast(total, src = self._last_global_rank, group = self.group)if dp_group is not None:dist.all_reduce(total, group = dp_group) # each replica's sum / global tokensreturn total[0]@@ -1354,9 +1492,10 @@ class PipelineParallel:}if self.num_stages == 1:return local if not self.composed or dist.get_rank() == 0 else None- gathered = [None] * self.num_stages if self.is_first else None+ first = self.stage_index == 0+ gathered = [None] * self.num_stages if first else Nonedist.gather_object(local, gathered, dst = self._global_ranks[0], group = self.group)- if not self.is_first or (self.composed and dist.get_rank() != 0):+ if not first or (self.composed and dist.get_rank() != 0):return None # one pp group (the one holding global rank 0) writesmerged = {adapter: {} for adapter in local}for part in gathered:@@ -1396,6 +1535,33 @@ class PipelineParallel:dist.barrier(group = self.group)+def _local_stage_modules(pp_stage, parts, owned):+ """One _PPStageModule per local stage of a multi-stage rank. Every one runs the same model; while stage k+ runs, this rank's layers of its other stages are swapped for pass-throughs and, unless k is the first+ stage, the embedding for its stub (so it is not looked up for nothing)."""+ layers = parts.layers+ stand_ins = {i: _PPPassThrough(i, pp_stage.ctx, layers[i]) for i in owned}+ embed_owner = _owner(parts.decoder, parts.embed) if pp_stage.is_first else None+ embed_stub = _PPEmbeddingStub(parts.embed) if pp_stage.is_first else None+ modules = []+ for k in pp_stage.local_stages:+ start, end = pp_stage.bounds[k]+ swaps = [(layers, str(i), stand_ins[i]) for i in owned if not (start <= i < end)]+ if k != 0 and embed_owner is not None:+ swaps.append((embed_owner[0], embed_owner[1], embed_stub))+ stage = (start, end, k == 0, k == pp_stage.total_stages - 1, swaps)+ modules.append(+ _PPStageModule(+ pp_stage.model,+ pp_stage.ctx,+ pp_stage.activation_dtype,+ pp_stage.hidden_size,+ stage = stage,+ )+ )+ return modules++def _sync_trainable(model, group, src):import torch.distributed as distwith torch.no_grad():@@ -1434,7 +1600,9 @@ def apply_pipeline_parallel(return existing_check_torch_pipelining()config = config or PPConfig()- schedule = str(schedule).lower().replace("-", "").replace("_", "")+ spec = schedule_spec(schedule)+ schedule = spec.name+ stages_per_rank = resolve_stages_per_rank(spec, config.stages_per_rank)if pp_mesh is None:pp_mesh = _default_mesh(model)if pp_mesh.ndim != 1:@@ -1466,23 +1634,43 @@ def apply_pipeline_parallel()parts = _find_parts(model)_refuse_unsupported(model, parts)- _check_schedule(model, schedule)+ total_stages = pp * stages_per_rank+ rank = pp_mesh.get_local_rank()+ if stages_per_rank > 1:+ root = _root_mesh(pp_mesh)+ names = tuple(getattr(root, "mesh_dim_names", None) or ()) if root is not None else ()+ if "dp" in names:+ raise ValueError(_MULTI_STAGE_DP)+ # The torch schedule's own checks, on placeholder stages: before anything is cut.+ props = validate_schedule(spec, pp_mesh.get_group(), pp, stages_per_rank, n_microbatches, rank)+ _check_schedule(model, spec, props)tuple_format = _layer_call_format(parts.decoder, parts.layers_name) == "tuple"- bounds = derive_pp_split(model, pp, config)+ bounds = derive_pp_split(model, total_stages, config)_refuse_shared_across_stages(parts, bounds)dtype = config.activation_dtype or parts.embed.weight.dtype- rank = pp_mesh.get_local_rank()if config.sync_trainable and pp > 1:_sync_trainable(model, pp_mesh.get_group(), dist.get_global_rank(pp_mesh.get_group(), 0))+ donated_off = props["split_backward"] and allow_split_backward()pp_stage = PipelineParallel(- model, pp_mesh, parts, bounds, n_microbatches, schedule, config, tuple_format, dtype+ model,+ pp_mesh,+ parts,+ bounds,+ n_microbatches,+ schedule,+ config,+ tuple_format,+ dtype,+ stages_per_rank = stages_per_rank,)- start, end = bounds[rank]- is_first, is_last = rank == 0, rank == pp - 1+ local = pp_stage.local_stages+ owned = sorted({i for k in local for i in range(*bounds[k])})+ start, end = bounds[local[0]]+ is_first, is_last = pp_stage.is_first, pp_stage.is_lastlayers = parts.layersfor i in range(len(layers)):- if not (start <= i < end):+ if i not in owned:layers[i] = _PPPassThrough(i, pp_stage.ctx, layers[i])embed, lm_head = parts.embed, parts.lm_headif not is_first:@@ -1509,6 +1697,9 @@ def apply_pipeline_parallel()# The stage object must not pin a dropped embedding / lm_head (vocabulary-sized) alive.parts = pp_stage.parts = dataclasses.replace(parts, embed = embed, lm_head = lm_head)+ if stages_per_rank > 1:+ pp_stage.stage_modules = _local_stage_modules(pp_stage, parts, owned)+ pp_stage.module = pp_stage.stage_modules[0]info = {"size": pp,"rank": rank,@@ -1516,6 +1707,8 @@ def apply_pipeline_parallel("bounds": bounds,"source": parts.source,"schedule": schedule,+ "stages_per_rank": stages_per_rank,+ "local_stages": tuple(local),}inner = parts.innerfor level in _model_chain(model):@@ -1531,9 +1724,16 @@ def apply_pipeline_parallel(torch.cuda.empty_cache()if (dist.get_rank() if dist.is_initialized() else rank) == 0:print(- f"Unsloth: pipeline parallel over {pp} stages ({schedule}, {n_microbatches} micro-batches, "- f"{parts.source} parts): decoder layers per stage {[e - s for s, e in bounds]}"+ f"Unsloth: pipeline parallel over {pp} ranks x {stages_per_rank} stages ({schedule}, "+ f"{n_microbatches} micro-batches, {parts.source} parts): decoder layers per stage "+ f"{[e - s for s, e in bounds]}"+ (", tied embeddings kept on the first and last stage" if parts.tied else "")+ + (+ "; compiled backward graphs built without donated buffers (zero bubble runs autograd.grad "+ "with retain_graph)"+ if donated_off+ else ""+ )+ ".")return pp_stagediff --git a/unsloth/models/pp_schedules.py b/unsloth/models/pp_schedules.pynew file mode 100644index 00000000000..ca8e601324a--- /dev/null+++ b/unsloth/models/pp_schedules.py@@ -0,0 +1,328 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""Pipeline schedules for apply_pipeline_parallel: one stage per rank (GPipe, 1F1B) or several (looped / V).++ pp = apply_pipeline_parallel(model, mesh["pp"], n_microbatches = 8, schedule = "interleaved1f1b")++Rank r keeps stages r, r + pp, ... (looped) or r and 2 pp - 1 - r (V). Every local stage runs the same model+with only its own layers active, so parameter names and adapter keys are unchanged. The schedule is first+built on placeholder stages to learn whether a rank runs a forward after a backward (smart checkpointing+offload slots must then be leased per checkpoint, unsloth-zoo PR #2071) and whether it splits the backward+(zero bubble: torch.autograd.grad, which reentrant checkpointing cannot run).+"""++from __future__ import annotations++import dataclasses+import functools+from typing import Optional++import torch+import torch.nn as nn++__all__ = [+ "ScheduleSpec",+ "schedule_spec",+ "stage_indices",+ "validate_schedule",+ "SCHEDULE_NAMES",+]++[email protected](frozen = True)+class ScheduleSpec:+ name: str # normalized, e.g. "interleaved1f1b"+ torch_name: str # torch.distributed.pipelining.schedules class+ multi: bool # PipelineScheduleMulti: a list of local stages+ style: str # "single", "loop" or "v": which stages a rank keeps+ default_stages_per_rank: int+++_SPECS = {+ "gpipe": ScheduleSpec("gpipe", "ScheduleGPipe", False, "single", 1),+ "1f1b": ScheduleSpec("1f1b", "Schedule1F1B", False, "single", 1),+ "interleaved1f1b": ScheduleSpec("interleaved1f1b", "ScheduleInterleaved1F1B", True, "loop", 2),+ "loopedbfs": ScheduleSpec("loopedbfs", "ScheduleLoopedBFS", True, "loop", 2),+ "interleavedzerobubble": ScheduleSpec(+ "interleavedzerobubble", "ScheduleInterleavedZeroBubble", True, "loop", 2+ ),+ "zbvzerobubble": ScheduleSpec("zbvzerobubble", "ScheduleZBVZeroBubble", True, "v", 2),+ "dualpipev": ScheduleSpec("dualpipev", "ScheduleDualPipeV", True, "v", 2),+}+_ALIASES = {"zbv": "zbvzerobubble", "interleavedzb": "interleavedzerobubble"}+SCHEDULE_NAMES = tuple(_SPECS)+++def _normalize(name):+ key = str(name).lower().replace("-", "").replace("_", "").replace(" ", "")+ if key.startswith("schedule"):+ key = key[len("schedule") :]+ return _ALIASES.get(key, key)+++def schedule_spec(name) -> ScheduleSpec:+ spec = _SPECS.get(_normalize(name))+ if spec is None:+ raise ValueError(+ f"Unsloth: pipeline schedule must be one of {SCHEDULE_NAMES}, got {name!r}."+ )+ return spec+++def torch_schedule_class(spec):+ from torch.distributed.pipelining import schedules++ cls = getattr(schedules, spec.torch_name, None)+ if cls is None:+ raise RuntimeError(+ f"Unsloth: torch {torch.__version__} has no {spec.torch_name} "+ f"(pipeline schedule {spec.name!r})."+ )+ return cls+++def resolve_stages_per_rank(spec, stages_per_rank):+ """stages_per_rank for `spec` (None = its default); ValueError when the schedule cannot run it."""+ v = spec.default_stages_per_rank if stages_per_rank is None else int(stages_per_rank)+ if v < 1:+ raise ValueError("Unsloth: PPConfig.stages_per_rank must be >= 1.")+ if spec.style == "single" and v != 1:+ raise ValueError(+ f"Unsloth: the {spec.name} schedule runs one stage per rank (PPConfig.stages_per_rank = {v}); "+ "use 'interleaved1f1b' or 'loopedbfs' for several."+ )+ if spec.style == "v" and v != 2:+ raise ValueError(+ f"Unsloth: the {spec.name} schedule runs exactly 2 stages per rank (V placement), got "+ f"PPConfig.stages_per_rank = {v}."+ )+ return v+++def stage_indices(spec, rank, pp, stages_per_rank):+ """Global stage indices pp rank `rank` keeps, in execution order of its forward."""+ if spec.style == "single":+ return (rank,)+ if spec.style == "loop":+ return tuple(rank + k * pp for k in range(stages_per_rank))+ return (rank, 2 * pp - 1 - rank) # "v"+++class _NoParams(nn.Module):+ def forward(self, x):+ return x+++def v_schedules_supported():+ """V placement needs PipelineStage._is_same_rank (torch >= 2.12); older stage setup fails."""+ try:+ from torch.distributed.pipelining import PipelineStage+ except ImportError:+ return False+ return hasattr(PipelineStage, "_is_same_rank")+++def validate_schedule(spec, group, pp, stages_per_rank, n_microbatches, rank):+ """Build the schedule on placeholder stages so torch's checks run before the model is cut.+ Returns {"interleaved": bool, "split_backward": bool} over every rank's compute order."""+ from torch.distributed.pipelining import PipelineStage++ cls = torch_schedule_class(spec)+ if spec.style == "v" and not v_schedules_supported():+ raise ValueError(+ f"Unsloth: the {spec.name} pipeline schedule (V placement: two adjacent stages on one rank) needs "+ f"a torch whose PipelineStage handles adjacent stages on one rank (torch >= 2.12); on torch "+ f"{torch.__version__} torch's stage setup between them fails. Use 'interleaved1f1b' or "+ "'interleavedzerobubble'."+ )+ num_stages = pp * stages_per_rank+ local = stage_indices(spec, rank, pp, stages_per_rank)+ example = torch.empty((1, 1), device = "meta")+ stages = [+ PipelineStage(+ _NoParams(),+ k,+ num_stages,+ torch.device("cpu"),+ input_args = example,+ output_args = example,+ group = group,+ )+ for k in local+ ]+ loss_fn = lambda output, target: output.sum() # noqa: E731+ try:+ if spec.multi:+ schedule = cls(stages, n_microbatches, loss_fn = loss_fn, scale_grads = False)+ else:+ schedule = cls(stages[0], n_microbatches, loss_fn = loss_fn, scale_grads = False)+ except (ValueError, AssertionError, RuntimeError) as exc:+ raise ValueError(+ f"Unsloth: the {spec.name} pipeline schedule cannot run {n_microbatches} micro-batches over "+ f"{num_stages} stages ({pp} ranks x {stages_per_rank}): {exc}"+ ) from None+ order = getattr(schedule, "pipeline_order", None)+ if not order:+ # GPipe runs every forward first; 1F1B interleaves.+ return {"interleaved": spec.name == "1f1b" and pp > 1, "split_backward": False}+ interleaved = split = False+ for actions in order.values():+ seen_backward = False+ for action in actions:+ if action is None:+ continue+ # By name: the enum's values are ints on torch 2.10, letters later.+ kind = getattr(action.computation_type, "name", str(action.computation_type))+ if kind == "FORWARD":+ interleaved = interleaved or seen_backward+ elif kind in ("FULL_BACKWARD", "BACKWARD_INPUT", "BACKWARD_WEIGHT"):+ seen_backward = True+ split = split or kind != "FULL_BACKWARD"+ return {"interleaved": interleaved, "split_backward": split}+++def non_reentrant_checkpointing_only(model) -> Optional[bool]:+ """None: no checkpointing; True: all non-reentrant torch checkpointing; False otherwise."""+ import torch.utils.checkpoint as checkpointing++ # unsloth_zoo smart checkpointing patches checkpoint (original kept as _old_checkpoint).+ original = getattr(checkpointing, "_old_checkpoint", None) or checkpointing.checkpoint+ patched = checkpointing.checkpoint is not original+ found = False+ for module in model.modules():+ if getattr(module, "gradient_checkpointing", False) is not True:+ continue+ found = True+ func = getattr(module, "_gradient_checkpointing_func", None)+ if patched or not isinstance(func, functools.partial) or func.func is not original:+ return False+ if func.keywords.get("use_reentrant", True) is not False:+ return False+ if not found and bool(getattr(model, "is_gradient_checkpointing", False)):+ return False+ return True if found else None+++def allow_split_backward():+ """Disable AOTAutograd donated buffers: zero bubble backward uses retain_graph, which a compiled backward+ with donated buffers refuses. Returns True if changed; call before the first forward (already compiled+ blocks keep their buffers)."""+ try:+ import torch._functorch.config as functorch_config+ except ImportError:+ return False+ if getattr(functorch_config, "donated_buffer", False):+ functorch_config.donated_buffer = False+ return True+ return False++[email protected]_cache(maxsize = None)+def split_backward_keeps_function_nodes() -> bool:+ """True if an autograd.Function node (e.g. CompiledFunctionBackward) keeps `_input_metadata` after its+ output dies; split backward's weight pass needs it. torch <= 2.12 holds the node by weak_ptr, so it fails.+ Probed rather than version-checked."""++ class _Probe(torch.autograd.Function):+ @staticmethod+ def forward(ctx, x):+ return x * 2++ @staticmethod+ def backward(ctx, grad):+ return grad * 2++ try:+ with torch.enable_grad():+ out = _Probe.apply(torch.ones(1, requires_grad = True))+ node = out.grad_fn+ del out+ node._input_metadata+ except RuntimeError:+ return False+ except Exception:+ # Do not refuse on a probe we cannot interpret.+ return True+ return True+++def compiled_blocks(model):+ """Names of submodules that run through torch.compile (backward is a CompiledFunctionBackward node)."""+ found = []+ calls = {}+ for name, module in model.named_modules():+ cls_forward = getattr(type(module), "forward", None)+ inst_forward = module.__dict__.get("forward")+ if id(cls_forward) not in calls:+ calls[id(cls_forward)] = _calls_compiled(cls_forward)+ if (+ getattr(cls_forward, "_unsloth_block_compiled", False)+ or module.__dict__.get("_compiled_call_impl") is not None+ or type(module).__name__ == "OptimizedModule"+ or hasattr(_unwrap_callable(inst_forward), "_torchdynamo_orig_callable")+ or hasattr(inst_forward, "_torchdynamo_orig_callable")+ or calls[id(cls_forward)]+ or (inst_forward is not None and _calls_compiled(inst_forward))+ ):+ found.append(name or type(module).__name__)+ return found+++def _is_compiled(value):+ try:+ return (+ hasattr(value, "_torchdynamo_orig_callable")+ or hasattr(value, "_unsloth_compiled_func")+ or type(value).__name__ == "OptimizedModule"+ )+ except Exception:+ return False+++def _calls_compiled(forward):+ """True if `forward` calls a compiled global or closure: unsloth_zoo compiled modules keep the class+ forward eager and call a compiled `Gemma3MLP_forward`-style function."""+ fn = _unwrap_callable(forward)+ code = getattr(fn, "__code__", None)+ scope = getattr(fn, "__globals__", None)+ if code is None or not isinstance(scope, dict):+ return False+ if any(_is_compiled(scope.get(name)) for name in code.co_names if name in scope):+ return True+ for cell in getattr(fn, "__closure__", None) or ():+ try:+ value = cell.cell_contents+ except ValueError:+ continue+ if _is_compiled(value):+ return True+ return False+++_SINGLE_BACKWARD_MODULES = ("unsloth.kernels.fast_lora",)+++def _unwrap_callable(value):+ """Strip bound methods and functools.partial (Cohere MLP is MethodType(partial(apply_lora_mlp, ...)))."""+ for _ in range(16):+ if isinstance(value, functools.partial):+ value = value.func+ elif hasattr(value, "__func__"):+ value = value.__func__+ else:+ break+ return value+++def single_backward_kernels(model):+ """Fused LoRA kernels on `model`: their backward mutates saved tensors, so it cannot run twice as split+ backward (zero bubble) requires."""+ found = set()+ for module in model.modules():+ candidates = list(module.__dict__.values()) + [getattr(type(module), "forward", None)]+ for value in candidates:+ fn = _unwrap_callable(value)+ if callable(fn) and getattr(fn, "__module__", None) in _SINGLE_BACKWARD_MODULES:+ found.add(getattr(fn, "__name__", repr(fn)))+ return sorted(found)