Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Context parallel: give decoder layers no mask inside a CP step so block compile keeps one graph on packed rows (#16862)
Daniel Han6953b1cba130
diff --git a/tests/test_context_parallel.py b/tests/test_context_parallel.pyindex c87ed36a87e..191cc5ee461 100644--- a/tests/test_context_parallel.py+++ b/tests/test_context_parallel.py@@ -733,6 +733,50 @@ def test_attention_hooks_are_not_stacked_by_a_second_manager():assert len(model.self_attn._forward_pre_hooks) == 1+def test_decoder_layers_see_no_mask_inside_a_cp_step_only():+ class Layer(torch.nn.Module):+ def __init__(self, linear = False):+ super().__init__()+ self.self_attn = torch.nn.Identity()+ if linear:+ self.linear_attn = torch.nn.Identity()+ self.seen = []++ def forward(+ self,+ hidden_states,+ attention_mask = None,+ ):+ self.seen.append(attention_mask)+ return hidden_states++ model = torch.nn.Module()+ model.layers = torch.nn.ModuleList([Layer(), Layer(linear = True)])+ for _ in range(2):+ _manager().attach_attention_hooks(model)+ assert len(model.layers[0]._forward_pre_hooks) == 1+ assert (+ len(model.layers[1]._forward_pre_hooks) == 0+ ) # a linear attention scan reads the padding mask+ x, mask = torch.ones(1, 4, 2), torch.ones(1, 1, 4, 4, dtype = torch.bool)++ def run(masks):+ for layer in model.layers:+ layer.seen.clear()+ for m in masks:+ layer(x, attention_mask = m)+ return [layer.seen for layer in model.layers]++ # No CP manager: the exact same object passes through.+ plain = run([mask, None])+ assert plain[0][0] is mask and plain[0][1] is None and plain[1][0] is mask+ with cp._manager_scope(_manager()):+ inside = run([mask, None])+ assert inside[0] == [None, None]+ assert inside[1][0] is mask+ assert run([mask])[0][0] is mask++def test_sdpa_is_restored_when_the_cp_step_raises(monkeypatch):import torch.nn.functional as Fdiff --git a/tests/test_parallel_capabilities_evidence.py b/tests/test_parallel_capabilities_evidence.pyindex 26d2fb4c207..d81d0cfcb0a 100644--- a/tests/test_parallel_capabilities_evidence.py+++ b/tests/test_parallel_capabilities_evidence.py@@ -96,14 +96,36 @@ def test_packed_rows_through_query_from_reach_the_eager_cell():assert status == "supported" and "packed rows" in cell.name, (status, cell)-def test_packed_rows_under_block_compile_stay_unverified_with_the_gap(clean_env):- query = _packed(compile = "block")[email protected]("family", ["qwen3", "llama"])+def test_packed_rows_under_block_compile_are_supported_on_torch_2_13(family, clean_env):+ query = _packed(compile = "block", family = family)+ status, cell = caps.cell_status(query)+ assert (+ status == "supported" and cell.name.endswith("Unsloth block compile") and not cell.missing()+ )+ for kind in (cell.parity, cell.engagement, cell.compile):+ assert "test_block_compile_packed_rows_under_cp_match_one_process" in kind+ clean_env.setenv("UNSLOTH_PARALLEL_STRICT", "1")+ caps.check(query)+++ "kw",+ [+ {"torch": (2, 12)}, # the compile test passes on torch >= 2.13 only+ {"family": "gemma3", "windowed": True, "attention": "eager"},+ {"family": "llama", "torch": (2, 12)},+ {"mode": "full"},+ {"attention": "eager"},+ ],+)+def test_packed_rows_under_block_compile_outside_the_measured_cell_stay_unverified(kw, clean_env):+ query = _packed(compile = "block", **kw)status, detail = caps.cell_status(query)- assert status == "unverified"- assert "lacks engagement, compile evidence" in detail and "breaks the graph" in detail- caps.check(query) # not gated: runs, as before, outside strict mode+ assert status == "unverified", detail+ assert "lacks engagement, compile evidence" in detail and "torch >= 2.13 only" in detail, detailclean_env.setenv("UNSLOTH_PARALLEL_STRICT", "1")- with pytest.raises(NotImplementedError, match = "breaks the graph"):+ with pytest.raises(NotImplementedError):caps.check(query)diff --git a/tests/test_parallel_cp_compose_gloo.py b/tests/test_parallel_cp_compose_gloo.pyindex c9dab621313..26253858779 100644--- a/tests/test_parallel_cp_compose_gloo.py+++ b/tests/test_parallel_cp_compose_gloo.py@@ -19,8 +19,10 @@ if os.environ.get("UNSLOTH_TEST_RANKS_ON_GPU") != "1" and ():os.environ["CUDA_VISIBLE_DEVICES"] = "" # spawned gloo ranks: CPU only, also on GPU hosts+import dataclasses # noqa: E402import sys # noqa: E402import tempfile # noqa: E402+import types # noqa: E402import pytest # noqa: E402import torch # noqa: E402@@ -75,6 +77,55 @@ def _checkpoint(tmp, arch = "llama"):return path+# Packed rows, one per micro-batch, each with other documents (compiled layers must follow every row): "padfree"+# fills the row, "pack" leaves a one-token tail that context parallel pads to the cp multiple.+_PACKED_DOCS = {+ "padfree": ([5, 7, 4], [3, 9, 4], [8, 2, 6], [4, 4, 8]),+ "pack": ([5, 7, 3], [3, 9, 3], [8, 2, 5], [6, 6, 3]),+}+++def _packed_row(batch, kind, row):+ """One packed row as Unsloth's packing / padding-free collators hand it on: the global document lengths,+ positions restarting per document, and no target across a document start."""+ lengths = _PACKED_DOCS[kind][row % len(_PACKED_DOCS[kind])]+ n = sum(lengths)+ input_ids, labels = batch["input_ids"][:, :n], batch["labels"][:, :n].clone()+ starts = torch.tensor([0, *lengths[:-1]]).cumsum(0)+ labels[:, starts] = -100+ position_ids = torch.cat([torch.arange(k) for k in lengths])[None]+ return {+ "input_ids": input_ids,+ "labels": labels,+ "position_ids": position_ids,+ "packed_seq_lengths": torch.tensor(lengths, dtype = torch.int32),+ }+++def _packed_oracle(model, batch):+ """The one-process reference itself keeps documents apart: each document's logits in the packed row equal+ the same document alone (eager, so no compiled graph is added to the reference's counters)."""+ ids, lengths = batch["input_ids"], batch["packed_seq_lengths"].tolist()+ was = model.training+ model.eval()+ with torch.no_grad(), torch.compiler.set_stance("force_eager"):+ full = model(+ input_ids = ids,+ position_ids = batch["position_ids"],+ packed_seq_lengths = batch["packed_seq_lengths"],+ use_cache = False,+ ).logits[0]+ worst, start = 0.0, 0+ for n in lengths:+ alone = model(input_ids = ids[:, start : start + n], use_cache = False).logits[0]+ worst = max(+ worst, float((full[start : start + n] - alone).abs().max() / alone.abs().max())+ )+ start += n+ model.train(was)+ return worst++def _full(t):from torch.distributed.tensor import DTensorreturn (t.full_tensor() if isinstance(t, DTensor) else t).detach().float().cpu().clone()@@ -290,6 +341,7 @@ def _case_train(bypass_pre = False,bypass_post = False,gc = "none",+ packed = None,):error = _import_checkout_unsloth()if error is not None:@@ -421,13 +473,15 @@ def _case_train(batch["labels"][batch["labels"] % 5 == 0] = -100 # uneven token counts per row and per shard+ if packed:+ batch.update(_packed_row(batch, packed, int(rows[0]["row"])))return batchclass SequentialSFT(trl.SFTTrainer):def _get_train_sampler(self, *args, **kwargs):return torch.utils.data.SequentialSampler(self.train_dataset)- grads = {}+ grads, query = {}, Noneclass Grabber(transformers.TrainerCallback):def on_pre_optimizer_step(@@ -468,6 +522,10 @@ def _case_train(dataset_kwargs = {"skip_prepare_dataset": True},gradient_checkpointing = False,)+ if packed:+ # What the registry reads for a packed run (query_from): SFTConfig.padding_free; the rows come from the+ # collator above, so TRL's own padding-free collator is not involved.+ args.padding_free = Trueif device != "cpu" and torch.cuda.device_count() < world:args.ddp_backend = os.environ.get("UNSLOTH_TEST_SHARED_GPU_BACKEND", "gloo")if hasattr(args, "router_aux_loss_coef"):@@ -483,6 +541,17 @@ def _case_train(# enough: the stamp below is the per-model "train this one in float32" switch the trainer honours.args.bf16 = args.fp16 = Falsemodel._unsloth_forced_float32 = True+ if packed:+ from unsloth import parallel_capabilities as caps++ # The registry's view of the run as configured, before the CPU route's normalize_cpu_training_arguments+ # (unsloth_zoo/cpu/trainer.py) turns padding_free off on CPU ranks inside the trainer's __init__.+ layout_now = types.SimpleNamespace(+ dp = 1, tp = 1, pp = 1, cp = cp, fsdp2 = False, cp_backend = "ulysses"+ )+ query = dataclasses.asdict(+ caps.query_from(layout_now, SequentialSFT, model, args, cp = cp, cp_backend = "ulysses")+ )tokenizer = AutoTokenizer.from_pretrained(ckpt)trainer = SequentialSFT(model = model,@@ -518,10 +587,37 @@ def _case_train(ulysses_hybrid._ulysses_hf_attention = countedfrom unsloth.utils import compile_counters+ packed_seen, oracle, exchanges = [], None, None+ if packed:+ # Eager, outside the compiled layers: the global document lengths of every micro-batch at its shard.+ original_shard = cpm.ContextParallelManager._ulysses_shard++ def recorded_shard(self, inputs):+ packed_seen.append(+ [int(x) for x in torch.as_tensor(inputs["packed_seq_lengths"]).tolist()]+ )+ return original_shard(self, inputs)++ cpm.ContextParallelManager._ulysses_shard = recorded_shard+ if world == 1:+ oracle = _packed_oracle(trainer.model, collate([data[0]]))+ rows_read.clear() # the oracle's row is not a training micro-batchtorch._dynamo.reset()before = compile_counters.snapshot()- trainer.train()+ if packed:+ # Inside the compiled layers no Python counter runs: the profiler sees each collective they issue.+ from torch.profiler import ProfilerActivity, profile+ with profile(activities = [ProfilerActivity.CPU]) as prof:+ trainer.train()+ exchanges = sum(1 for e in prof.events() if e.name == "gloo:all_to_all")+ else:+ trainer.train()compile_used = compile_counters.since(before)+ if packed:+ # torch >= 2.13 counts data-dependent breaks (the old cu_seqlens tail check) under "unimplemented".+ compile_used["unimplemented"] = int(+ sum(torch._dynamo.utils.counters.get("unimplemented", {}).values())+ )if device != "cpu":assert autocast_seen == {False}, ("a GPU cell ran under autocast", autocast_seen)logs = [h for h in trainer.state.log_history if "loss" in h]@@ -573,6 +669,11 @@ def _case_train("memory": __import__("unsloth.parallel_capabilities", fromlist = ["x"])._memory(trainer.model, args),+ "packed_seen": packed_seen,+ "oracle": oracle,+ "query": query,+ "exchanges": exchanges,+ "n_layers": int(trainer.model.config.num_hidden_layers),}if dist.is_initialized():dist.barrier()@@ -884,6 +985,78 @@ def test_block_compile_composes_like_one_process(world, flag, ref_bs):_close(res, eager, what = (flag, "compiled", rank))+@needs_block_compile_torch[email protected]("ga", [1, 2])[email protected]("packed", ["pack", "padfree"])[email protected]("arch", ["qwen3", "llama"])+def test_block_compile_packed_rows_under_cp_match_one_process(arch, packed, ga):+ """Unsloth's block compile x packed / padding-free rows x cp 2 Ulysses (the registry's packed compiled+ cell): the same numbers as one eager process; the registry sees block compile, packing and cp Ulysses;+ every micro-batch's documents reach the exchange; the compiled layers issue the all-to-alls; no more graph+ breaks (also data-dependent ones) / recompiles / graphs than the same compiled script in one process."""+ from unsloth.utils.compile_counters import not_above++ kw = dict(arch = arch, packed = packed, ga = ga)+ eager = _ref(1, **kw)+ single = _ref(1, compile = True, **kw)+ assert single["compiled_classes"] and single["compile"]["unique_graphs"] > 0, single["compile"]+ assert single["oracle"] < 1e-4, (+ "the one-process reference attends across documents",+ single["oracle"],+ )+ _close(single, eager, what = "one process compiled")+ micro = len(single["rows_read"])+ want_docs = [list(_PACKED_DOCS[packed][r[0] % 4]) for r in single["rows_read"]]+ results = _spawn("train", 2, bs = 1, cp = 2, compile = True, **kw)+ _same_rows(results, single, what = kw)+ for rank, res in enumerate(results):+ what = (kw, rank)+ assert res["compiled_classes"] == single["compiled_classes"], what+ query = res["query"]+ assert (+ query["compile"],+ query["packing"],+ query["cp"],+ query["cp_backend"],+ query["family"],+ ) == (+ "block",+ True,+ 2,+ "ulysses",+ arch,+ ), (what, query)+ # The run as configured reaches the registry's packed compiled cell, whose evidence is this test.+ from unsloth import parallel_capabilities as caps++ status, cell = caps.cell_status(caps.Query(**query))+ assert (+ status == "supported" and "test_block_compile_packed_rows_under_cp" in cell.compile+ ), (what, cell)+ assert res["cp_backend"] == "ulysses" and res["seen_lengths"] == [SEQ // 2], (+ what,+ res["seen_lengths"],+ )+ assert res["packed_seen"] == want_docs, (what, res["packed_seen"], want_docs)+ # At least one exchange per layer per micro-batch forward and backward (Ulysses swaps q / k / v and back).+ assert res["exchanges"] >= 2 * res["n_layers"] * micro, (what, res["exchanges"])+ assert not not_above(res["compile"], single["compile"]), (+ what,+ res["compile"],+ single["compile"],+ )+ for key in ("unimplemented", "unique_graphs"):+ assert res["compile"][key] <= single["compile"][key], (+ what,+ key,+ res["compile"],+ single["compile"],+ )+ # Adam (eps 1e-4) amplifies cp's fp32 reduction order on near-zero adapter grads: the eager cp arm moves+ # the adapters by the same 1.8e-5 (qwen3 padding-free), so the band is the eager composition's.+ _close(res, eager, what = what, state_atol = 5e-5)++@pytest.mark.parametrize("env, sft_cp",[({"UNSLOTH_FSDP2": "1"}, True), ({"UNSLOTH_PARALLEL": "cp=2,fsdp=1"}, False)],diff --git a/unsloth/context_parallel.py b/unsloth/context_parallel.pyindex 5839b66455c..56cfe6606ce 100644--- a/unsloth/context_parallel.py+++ b/unsloth/context_parallel.py@@ -334,6 +334,14 @@ def _self_attn_pre_forward_hook(_module, module_args, module_kwargs):return module_args, module_kwargs+def _decoder_layer_pre_forward_hook(_module, module_args, module_kwargs):+ # HF's shard-local mask is None on shards with no document start, a tensor elsewhere; CP attention never+ # reads it, so always pass None: a compiled layer guards on the argument type and keeps one graph.+ if get_cp_manager() is not None and module_kwargs.get("attention_mask") is not None:+ module_kwargs["attention_mask"] = None+ return module_args, module_kwargs++class ContextParallelManager:backend = "ring"rotate_method = "allgather"@@ -396,12 +404,25 @@ class ContextParallelManager:install_hf_attention_wrapper(module, module.config._attn_implementation, get_cp_manager)- for name, module in model.named_modules():- if name.endswith("self_attn") and not getattr(module, "_unsloth_cp_hooked", False):+ for name, module in list(model.named_modules()):+ if not name.endswith("self_attn"):+ continue+ if not getattr(module, "_unsloth_cp_hooked", False):module.register_forward_pre_hook(_self_attn_pre_forward_hook, with_kwargs = True, prepend = True)module._unsloth_cp_hooked = True+ # Skip layers with linear_attn: its scan reads the padding mask.+ layer = model.get_submodule(name.rpartition(".")[0]) if "." in name else None+ if (+ layer is not None+ and not hasattr(layer, "linear_attn")+ and not getattr(layer, "_unsloth_cp_mask_hooked", False)+ ):+ layer.register_forward_pre_hook(+ _decoder_layer_pre_forward_hook, with_kwargs = True, prepend = True+ )+ layer._unsloth_cp_mask_hooked = Truedef _prepare_inputs(self, inputs: dict) -> None:input_ids = inputs.get("input_ids")diff --git a/unsloth/parallel_capabilities.py b/unsloth/parallel_capabilities.pyindex f1a39f17936..021f400d566 100644--- a/unsloth/parallel_capabilities.py+++ b/unsloth/parallel_capabilities.py@@ -428,7 +428,8 @@ FEATURES = (consumes = ("input_ids", "labels", "position_ids", "num_items_in_batch"),uses = ("packed_seq_lengths",),produces = ("seq_shard", "shift_labels", "position_ids"),- hooks = ("Trainer.__init__", "attention_function"),+ # decoder_layer.forward: a pre-hook drops the per-layer mask (context_parallel.py).+ hooks = ("Trainer.__init__", "decoder_layer.forward", "attention_function"),exclusive = ("trainer.training_step", "trainer.prediction_step"),),Feature(@@ -439,7 +440,8 @@ FEATURES = (mesh = ("cp",),consumes = ("input_ids", "labels", "position_ids", "num_items_in_batch"),produces = ("seq_shard", "shift_labels", "cp_permutation", "position_ids"),- hooks = ("Trainer.__init__", "attention_function"),+ # decoder_layer.forward: a pre-hook drops the per-layer mask (context_parallel.py).+ hooks = ("Trainer.__init__", "decoder_layer.forward", "attention_function"),exclusive = ("trainer.training_step", "trainer.prediction_step"),),# Precision.@@ -681,6 +683,7 @@ _PACKED_ULYSSES = ("tests/test_cp_ulysses_generic_gloo.py::test_generic_ulysses_matches_one_process""[gemma3-eager-True|qwen3-sdpa-True]")+_PACKED_COMPILED = _GLOO + "test_block_compile_packed_rows_under_cp_match_one_process"CELLS = (Cell("cp x native FSDP2 (dp_shard x cp), eager",@@ -743,13 +746,40 @@ CELLS = (),Cell("packed rows x cp Ulysses (DDP), Unsloth block compile",- {**_PACKED_CP, "compile": ("block",)},+ {+ **_PACKED_CP,+ "family": ("qwen3", "llama"),+ "windowed": (False,),+ "mode": ("lora",),+ "attention": ("sdpa",),+ "compile": ("block",),+ },+ parity = _PACKED_COMPILED+ + " (packing and padding-free rows, ga 1 and 2: the same numbers as one process)",+ engagement = _PACKED_COMPILED+ + " (compiled classes, seq / cp tokens, every micro-batch's document lengths at the shard, the compiled "+ "layers issue the all-to-alls)",+ compile = _PACKED_COMPILED+ + " (graph breaks incl. data-dependent ones / recompiles / graphs <= one process)",+ note = "per-document compiled attention also: "+ "tests/test_cp_ulysses_generic_gloo.py::test_compiled_layers_follow_each_packed_row",+ min_torch = BLOCK_COMPILE_TORCH,+ ),+ # First match wins: the rest of the packed compiled rows, and the measured one on older torch.+ Cell(+ "packed rows x cp Ulysses (DDP), Unsloth block compile, unmeasured",+ {+ **_PACKED_CP,+ "family": _PACKED_CP["family"] + ("llama",),+ "compile": ("block",),+ },parity = "tests/test_cp_ulysses_generic_gloo.py::test_compiled_layers_follow_each_packed_row",engagement = None,compile = None,- note = "compiled layers now rebuild each packed row's metadata, but the data-dependent cu_seqlens tail "- "check in utils/ulysses_hybrid.py still breaks the graph on every packed attention call, so there is no "- "compile evidence (graph breaks / graphs not above one process) yet; eager is proven",+ note = "compile evidence ("+ + _PACKED_COMPILED+ + ") covers Qwen3 / Llama, LoRA, SDPA, no window, on "+ "torch >= 2.13 only; eager is proven",),)