Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Context parallel: give decoder layers no mask inside a CP step so block compile keeps one graph on packed rows (#16862)

Daniel Han6953b1cba130
diff --git a/tests/test_context_parallel.py b/tests/test_context_parallel.py
index c87ed36a87e..191cc5ee461 100644
--- a/tests/test_context_parallel.py
+++ b/tests/test_context_parallel.py
@@ -733,6 +733,50 @@ def test_attention_hooks_are_not_stacked_by_a_second_manager():
assert len(model.self_attn._forward_pre_hooks) == 1
+def test_decoder_layers_see_no_mask_inside_a_cp_step_only():
+ class Layer(torch.nn.Module):
+ def __init__(self, linear = False):
+ super().__init__()
+ self.self_attn = torch.nn.Identity()
+ if linear:
+ self.linear_attn = torch.nn.Identity()
+ self.seen = []
+
+ def forward(
+ self,
+ hidden_states,
+ attention_mask = None,
+ ):
+ self.seen.append(attention_mask)
+ return hidden_states
+
+ model = torch.nn.Module()
+ model.layers = torch.nn.ModuleList([Layer(), Layer(linear = True)])
+ for _ in range(2):
+ _manager().attach_attention_hooks(model)
+ assert len(model.layers[0]._forward_pre_hooks) == 1
+ assert (
+ len(model.layers[1]._forward_pre_hooks) == 0
+ ) # a linear attention scan reads the padding mask
+ x, mask = torch.ones(1, 4, 2), torch.ones(1, 1, 4, 4, dtype = torch.bool)
+
+ def run(masks):
+ for layer in model.layers:
+ layer.seen.clear()
+ for m in masks:
+ layer(x, attention_mask = m)
+ return [layer.seen for layer in model.layers]
+
+ # No CP manager: the exact same object passes through.
+ plain = run([mask, None])
+ assert plain[0][0] is mask and plain[0][1] is None and plain[1][0] is mask
+ with cp._manager_scope(_manager()):
+ inside = run([mask, None])
+ assert inside[0] == [None, None]
+ assert inside[1][0] is mask
+ assert run([mask])[0][0] is mask
+
+
def test_sdpa_is_restored_when_the_cp_step_raises(monkeypatch):
import torch.nn.functional as F
diff --git a/tests/test_parallel_capabilities_evidence.py b/tests/test_parallel_capabilities_evidence.py
index 26d2fb4c207..d81d0cfcb0a 100644
--- a/tests/test_parallel_capabilities_evidence.py
+++ b/tests/test_parallel_capabilities_evidence.py
@@ -96,14 +96,36 @@ def test_packed_rows_through_query_from_reach_the_eager_cell():
assert status == "supported" and "packed rows" in cell.name, (status, cell)
-def test_packed_rows_under_block_compile_stay_unverified_with_the_gap(clean_env):
- query = _packed(compile = "block")
[email protected]("family", ["qwen3", "llama"])
+def test_packed_rows_under_block_compile_are_supported_on_torch_2_13(family, clean_env):
+ query = _packed(compile = "block", family = family)
+ status, cell = caps.cell_status(query)
+ assert (
+ status == "supported" and cell.name.endswith("Unsloth block compile") and not cell.missing()
+ )
+ for kind in (cell.parity, cell.engagement, cell.compile):
+ assert "test_block_compile_packed_rows_under_cp_match_one_process" in kind
+ clean_env.setenv("UNSLOTH_PARALLEL_STRICT", "1")
+ caps.check(query)
+
+
+ "kw",
+ [
+ {"torch": (2, 12)}, # the compile test passes on torch >= 2.13 only
+ {"family": "gemma3", "windowed": True, "attention": "eager"},
+ {"family": "llama", "torch": (2, 12)},
+ {"mode": "full"},
+ {"attention": "eager"},
+ ],
+)
+def test_packed_rows_under_block_compile_outside_the_measured_cell_stay_unverified(kw, clean_env):
+ query = _packed(compile = "block", **kw)
status, detail = caps.cell_status(query)
- assert status == "unverified"
- assert "lacks engagement, compile evidence" in detail and "breaks the graph" in detail
- caps.check(query) # not gated: runs, as before, outside strict mode
+ assert status == "unverified", detail
+ assert "lacks engagement, compile evidence" in detail and "torch >= 2.13 only" in detail, detail
clean_env.setenv("UNSLOTH_PARALLEL_STRICT", "1")
- with pytest.raises(NotImplementedError, match = "breaks the graph"):
+ with pytest.raises(NotImplementedError):
caps.check(query)
diff --git a/tests/test_parallel_cp_compose_gloo.py b/tests/test_parallel_cp_compose_gloo.py
index c9dab621313..26253858779 100644
--- a/tests/test_parallel_cp_compose_gloo.py
+++ b/tests/test_parallel_cp_compose_gloo.py
@@ -19,8 +19,10 @@ if os.environ.get("UNSLOTH_TEST_RANKS_ON_GPU") != "1" and (
):
os.environ["CUDA_VISIBLE_DEVICES"] = "" # spawned gloo ranks: CPU only, also on GPU hosts
+import dataclasses # noqa: E402
import sys # noqa: E402
import tempfile # noqa: E402
+import types # noqa: E402
import pytest # noqa: E402
import torch # noqa: E402
@@ -75,6 +77,55 @@ def _checkpoint(tmp, arch = "llama"):
return path
+# Packed rows, one per micro-batch, each with other documents (compiled layers must follow every row): "padfree"
+# fills the row, "pack" leaves a one-token tail that context parallel pads to the cp multiple.
+_PACKED_DOCS = {
+ "padfree": ([5, 7, 4], [3, 9, 4], [8, 2, 6], [4, 4, 8]),
+ "pack": ([5, 7, 3], [3, 9, 3], [8, 2, 5], [6, 6, 3]),
+}
+
+
+def _packed_row(batch, kind, row):
+ """One packed row as Unsloth's packing / padding-free collators hand it on: the global document lengths,
+ positions restarting per document, and no target across a document start."""
+ lengths = _PACKED_DOCS[kind][row % len(_PACKED_DOCS[kind])]
+ n = sum(lengths)
+ input_ids, labels = batch["input_ids"][:, :n], batch["labels"][:, :n].clone()
+ starts = torch.tensor([0, *lengths[:-1]]).cumsum(0)
+ labels[:, starts] = -100
+ position_ids = torch.cat([torch.arange(k) for k in lengths])[None]
+ return {
+ "input_ids": input_ids,
+ "labels": labels,
+ "position_ids": position_ids,
+ "packed_seq_lengths": torch.tensor(lengths, dtype = torch.int32),
+ }
+
+
+def _packed_oracle(model, batch):
+ """The one-process reference itself keeps documents apart: each document's logits in the packed row equal
+ the same document alone (eager, so no compiled graph is added to the reference's counters)."""
+ ids, lengths = batch["input_ids"], batch["packed_seq_lengths"].tolist()
+ was = model.training
+ model.eval()
+ with torch.no_grad(), torch.compiler.set_stance("force_eager"):
+ full = model(
+ input_ids = ids,
+ position_ids = batch["position_ids"],
+ packed_seq_lengths = batch["packed_seq_lengths"],
+ use_cache = False,
+ ).logits[0]
+ worst, start = 0.0, 0
+ for n in lengths:
+ alone = model(input_ids = ids[:, start : start + n], use_cache = False).logits[0]
+ worst = max(
+ worst, float((full[start : start + n] - alone).abs().max() / alone.abs().max())
+ )
+ start += n
+ model.train(was)
+ return worst
+
+
def _full(t):
from torch.distributed.tensor import DTensor
return (t.full_tensor() if isinstance(t, DTensor) else t).detach().float().cpu().clone()
@@ -290,6 +341,7 @@ def _case_train(
bypass_pre = False,
bypass_post = False,
gc = "none",
+ packed = None,
):
error = _import_checkout_unsloth()
if error is not None:
@@ -421,13 +473,15 @@ def _case_train(
batch["labels"][
batch["labels"] % 5 == 0
] = -100 # uneven token counts per row and per shard
+ if packed:
+ batch.update(_packed_row(batch, packed, int(rows[0]["row"])))
return batch
class SequentialSFT(trl.SFTTrainer):
def _get_train_sampler(self, *args, **kwargs):
return torch.utils.data.SequentialSampler(self.train_dataset)
- grads = {}
+ grads, query = {}, None
class Grabber(transformers.TrainerCallback):
def on_pre_optimizer_step(
@@ -468,6 +522,10 @@ def _case_train(
dataset_kwargs = {"skip_prepare_dataset": True},
gradient_checkpointing = False,
)
+ if packed:
+ # What the registry reads for a packed run (query_from): SFTConfig.padding_free; the rows come from the
+ # collator above, so TRL's own padding-free collator is not involved.
+ args.padding_free = True
if device != "cpu" and torch.cuda.device_count() < world:
args.ddp_backend = os.environ.get("UNSLOTH_TEST_SHARED_GPU_BACKEND", "gloo")
if hasattr(args, "router_aux_loss_coef"):
@@ -483,6 +541,17 @@ def _case_train(
# enough: the stamp below is the per-model "train this one in float32" switch the trainer honours.
args.bf16 = args.fp16 = False
model._unsloth_forced_float32 = True
+ if packed:
+ from unsloth import parallel_capabilities as caps
+
+ # The registry's view of the run as configured, before the CPU route's normalize_cpu_training_arguments
+ # (unsloth_zoo/cpu/trainer.py) turns padding_free off on CPU ranks inside the trainer's __init__.
+ layout_now = types.SimpleNamespace(
+ dp = 1, tp = 1, pp = 1, cp = cp, fsdp2 = False, cp_backend = "ulysses"
+ )
+ query = dataclasses.asdict(
+ caps.query_from(layout_now, SequentialSFT, model, args, cp = cp, cp_backend = "ulysses")
+ )
tokenizer = AutoTokenizer.from_pretrained(ckpt)
trainer = SequentialSFT(
model = model,
@@ -518,10 +587,37 @@ def _case_train(
ulysses_hybrid._ulysses_hf_attention = counted
from unsloth.utils import compile_counters
+ packed_seen, oracle, exchanges = [], None, None
+ if packed:
+ # Eager, outside the compiled layers: the global document lengths of every micro-batch at its shard.
+ original_shard = cpm.ContextParallelManager._ulysses_shard
+
+ def recorded_shard(self, inputs):
+ packed_seen.append(
+ [int(x) for x in torch.as_tensor(inputs["packed_seq_lengths"]).tolist()]
+ )
+ return original_shard(self, inputs)
+
+ cpm.ContextParallelManager._ulysses_shard = recorded_shard
+ if world == 1:
+ oracle = _packed_oracle(trainer.model, collate([data[0]]))
+ rows_read.clear() # the oracle's row is not a training micro-batch
torch._dynamo.reset()
before = compile_counters.snapshot()
- trainer.train()
+ if packed:
+ # Inside the compiled layers no Python counter runs: the profiler sees each collective they issue.
+ from torch.profiler import ProfilerActivity, profile
+ with profile(activities = [ProfilerActivity.CPU]) as prof:
+ trainer.train()
+ exchanges = sum(1 for e in prof.events() if e.name == "gloo:all_to_all")
+ else:
+ trainer.train()
compile_used = compile_counters.since(before)
+ if packed:
+ # torch >= 2.13 counts data-dependent breaks (the old cu_seqlens tail check) under "unimplemented".
+ compile_used["unimplemented"] = int(
+ sum(torch._dynamo.utils.counters.get("unimplemented", {}).values())
+ )
if device != "cpu":
assert autocast_seen == {False}, ("a GPU cell ran under autocast", autocast_seen)
logs = [h for h in trainer.state.log_history if "loss" in h]
@@ -573,6 +669,11 @@ def _case_train(
"memory": __import__("unsloth.parallel_capabilities", fromlist = ["x"])._memory(
trainer.model, args
),
+ "packed_seen": packed_seen,
+ "oracle": oracle,
+ "query": query,
+ "exchanges": exchanges,
+ "n_layers": int(trainer.model.config.num_hidden_layers),
}
if dist.is_initialized():
dist.barrier()
@@ -884,6 +985,78 @@ def test_block_compile_composes_like_one_process(world, flag, ref_bs):
_close(res, eager, what = (flag, "compiled", rank))
+@needs_block_compile_torch
[email protected]("ga", [1, 2])
[email protected]("packed", ["pack", "padfree"])
[email protected]("arch", ["qwen3", "llama"])
+def test_block_compile_packed_rows_under_cp_match_one_process(arch, packed, ga):
+ """Unsloth's block compile x packed / padding-free rows x cp 2 Ulysses (the registry's packed compiled
+ cell): the same numbers as one eager process; the registry sees block compile, packing and cp Ulysses;
+ every micro-batch's documents reach the exchange; the compiled layers issue the all-to-alls; no more graph
+ breaks (also data-dependent ones) / recompiles / graphs than the same compiled script in one process."""
+ from unsloth.utils.compile_counters import not_above
+
+ kw = dict(arch = arch, packed = packed, ga = ga)
+ eager = _ref(1, **kw)
+ single = _ref(1, compile = True, **kw)
+ assert single["compiled_classes"] and single["compile"]["unique_graphs"] > 0, single["compile"]
+ assert single["oracle"] < 1e-4, (
+ "the one-process reference attends across documents",
+ single["oracle"],
+ )
+ _close(single, eager, what = "one process compiled")
+ micro = len(single["rows_read"])
+ want_docs = [list(_PACKED_DOCS[packed][r[0] % 4]) for r in single["rows_read"]]
+ results = _spawn("train", 2, bs = 1, cp = 2, compile = True, **kw)
+ _same_rows(results, single, what = kw)
+ for rank, res in enumerate(results):
+ what = (kw, rank)
+ assert res["compiled_classes"] == single["compiled_classes"], what
+ query = res["query"]
+ assert (
+ query["compile"],
+ query["packing"],
+ query["cp"],
+ query["cp_backend"],
+ query["family"],
+ ) == (
+ "block",
+ True,
+ 2,
+ "ulysses",
+ arch,
+ ), (what, query)
+ # The run as configured reaches the registry's packed compiled cell, whose evidence is this test.
+ from unsloth import parallel_capabilities as caps
+
+ status, cell = caps.cell_status(caps.Query(**query))
+ assert (
+ status == "supported" and "test_block_compile_packed_rows_under_cp" in cell.compile
+ ), (what, cell)
+ assert res["cp_backend"] == "ulysses" and res["seen_lengths"] == [SEQ // 2], (
+ what,
+ res["seen_lengths"],
+ )
+ assert res["packed_seen"] == want_docs, (what, res["packed_seen"], want_docs)
+ # At least one exchange per layer per micro-batch forward and backward (Ulysses swaps q / k / v and back).
+ assert res["exchanges"] >= 2 * res["n_layers"] * micro, (what, res["exchanges"])
+ assert not not_above(res["compile"], single["compile"]), (
+ what,
+ res["compile"],
+ single["compile"],
+ )
+ for key in ("unimplemented", "unique_graphs"):
+ assert res["compile"][key] <= single["compile"][key], (
+ what,
+ key,
+ res["compile"],
+ single["compile"],
+ )
+ # Adam (eps 1e-4) amplifies cp's fp32 reduction order on near-zero adapter grads: the eager cp arm moves
+ # the adapters by the same 1.8e-5 (qwen3 padding-free), so the band is the eager composition's.
+ _close(res, eager, what = what, state_atol = 5e-5)
+
+
@pytest.mark.parametrize(
"env, sft_cp",
[({"UNSLOTH_FSDP2": "1"}, True), ({"UNSLOTH_PARALLEL": "cp=2,fsdp=1"}, False)],
diff --git a/unsloth/context_parallel.py b/unsloth/context_parallel.py
index 5839b66455c..56cfe6606ce 100644
--- a/unsloth/context_parallel.py
+++ b/unsloth/context_parallel.py
@@ -334,6 +334,14 @@ def _self_attn_pre_forward_hook(_module, module_args, module_kwargs):
return module_args, module_kwargs
+def _decoder_layer_pre_forward_hook(_module, module_args, module_kwargs):
+ # HF's shard-local mask is None on shards with no document start, a tensor elsewhere; CP attention never
+ # reads it, so always pass None: a compiled layer guards on the argument type and keeps one graph.
+ if get_cp_manager() is not None and module_kwargs.get("attention_mask") is not None:
+ module_kwargs["attention_mask"] = None
+ return module_args, module_kwargs
+
+
class ContextParallelManager:
backend = "ring"
rotate_method = "allgather"
@@ -396,12 +404,25 @@ class ContextParallelManager:
install_hf_attention_wrapper(
module, module.config._attn_implementation, get_cp_manager
)
- for name, module in model.named_modules():
- if name.endswith("self_attn") and not getattr(module, "_unsloth_cp_hooked", False):
+ for name, module in list(model.named_modules()):
+ if not name.endswith("self_attn"):
+ continue
+ if not getattr(module, "_unsloth_cp_hooked", False):
module.register_forward_pre_hook(
_self_attn_pre_forward_hook, with_kwargs = True, prepend = True
)
module._unsloth_cp_hooked = True
+ # Skip layers with linear_attn: its scan reads the padding mask.
+ layer = model.get_submodule(name.rpartition(".")[0]) if "." in name else None
+ if (
+ layer is not None
+ and not hasattr(layer, "linear_attn")
+ and not getattr(layer, "_unsloth_cp_mask_hooked", False)
+ ):
+ layer.register_forward_pre_hook(
+ _decoder_layer_pre_forward_hook, with_kwargs = True, prepend = True
+ )
+ layer._unsloth_cp_mask_hooked = True
def _prepare_inputs(self, inputs: dict) -> None:
input_ids = inputs.get("input_ids")
diff --git a/unsloth/parallel_capabilities.py b/unsloth/parallel_capabilities.py
index f1a39f17936..021f400d566 100644
--- a/unsloth/parallel_capabilities.py
+++ b/unsloth/parallel_capabilities.py
@@ -428,7 +428,8 @@ FEATURES = (
consumes = ("input_ids", "labels", "position_ids", "num_items_in_batch"),
uses = ("packed_seq_lengths",),
produces = ("seq_shard", "shift_labels", "position_ids"),
- hooks = ("Trainer.__init__", "attention_function"),
+ # decoder_layer.forward: a pre-hook drops the per-layer mask (context_parallel.py).
+ hooks = ("Trainer.__init__", "decoder_layer.forward", "attention_function"),
exclusive = ("trainer.training_step", "trainer.prediction_step"),
),
Feature(
@@ -439,7 +440,8 @@ FEATURES = (
mesh = ("cp",),
consumes = ("input_ids", "labels", "position_ids", "num_items_in_batch"),
produces = ("seq_shard", "shift_labels", "cp_permutation", "position_ids"),
- hooks = ("Trainer.__init__", "attention_function"),
+ # decoder_layer.forward: a pre-hook drops the per-layer mask (context_parallel.py).
+ hooks = ("Trainer.__init__", "decoder_layer.forward", "attention_function"),
exclusive = ("trainer.training_step", "trainer.prediction_step"),
),
# Precision.
@@ -681,6 +683,7 @@ _PACKED_ULYSSES = (
"tests/test_cp_ulysses_generic_gloo.py::test_generic_ulysses_matches_one_process"
"[gemma3-eager-True|qwen3-sdpa-True]"
)
+_PACKED_COMPILED = _GLOO + "test_block_compile_packed_rows_under_cp_match_one_process"
CELLS = (
Cell(
"cp x native FSDP2 (dp_shard x cp), eager",
@@ -743,13 +746,40 @@ CELLS = (
),
Cell(
"packed rows x cp Ulysses (DDP), Unsloth block compile",
- {**_PACKED_CP, "compile": ("block",)},
+ {
+ **_PACKED_CP,
+ "family": ("qwen3", "llama"),
+ "windowed": (False,),
+ "mode": ("lora",),
+ "attention": ("sdpa",),
+ "compile": ("block",),
+ },
+ parity = _PACKED_COMPILED
+ + " (packing and padding-free rows, ga 1 and 2: the same numbers as one process)",
+ engagement = _PACKED_COMPILED
+ + " (compiled classes, seq / cp tokens, every micro-batch's document lengths at the shard, the compiled "
+ "layers issue the all-to-alls)",
+ compile = _PACKED_COMPILED
+ + " (graph breaks incl. data-dependent ones / recompiles / graphs <= one process)",
+ note = "per-document compiled attention also: "
+ "tests/test_cp_ulysses_generic_gloo.py::test_compiled_layers_follow_each_packed_row",
+ min_torch = BLOCK_COMPILE_TORCH,
+ ),
+ # First match wins: the rest of the packed compiled rows, and the measured one on older torch.
+ Cell(
+ "packed rows x cp Ulysses (DDP), Unsloth block compile, unmeasured",
+ {
+ **_PACKED_CP,
+ "family": _PACKED_CP["family"] + ("llama",),
+ "compile": ("block",),
+ },
parity = "tests/test_cp_ulysses_generic_gloo.py::test_compiled_layers_follow_each_packed_row",
engagement = None,
compile = None,
- note = "compiled layers now rebuild each packed row's metadata, but the data-dependent cu_seqlens tail "
- "check in utils/ulysses_hybrid.py still breaks the graph on every packed attention call, so there is no "
- "compile evidence (graph breaks / graphs not above one process) yet; eager is proven",
+ note = "compile evidence ("
+ + _PACKED_COMPILED
+ + ") covers Qwen3 / Llama, LoRA, SDPA, no window, on "
+ "torch >= 2.13 only; eager is proven",
),
)