Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Pipeline parallel: end a non-last stage without a graph break under torch.compile (#16167)

Daniel Hanfa2a5e714955
diff --git a/tests/test_pp_native_gloo.py b/tests/test_pp_native_gloo.py
index fc2954beee3..d8d88793457 100644
--- a/tests/test_pp_native_gloo.py
+++ b/tests/test_pp_native_gloo.py
@@ -772,6 +772,40 @@ def test_layer_call_format_reads_tuple_loops():
assert ppn._layer_call_format(Decoder(), "layers") == "tuple"
+def test_non_last_stage_compiles_as_one_graph():
+ from torch._dynamo.utils import counters
+
+ class Toy(torch.nn.Module):
+ def __init__(self, ctx):
+ super().__init__()
+ self.embed = torch.nn.Embedding(10, 8)
+ self.owned = torch.nn.Linear(8, 8)
+ self.other = ppn._PPPassThrough(1, ctx, torch.nn.Linear(8, 8))
+ self.head = torch.nn.Linear(8, 10)
+
+ def forward(self, input_ids, **kwargs):
+ return self.head(self.other(self.owned(self.embed(input_ids))))
+
+ torch.manual_seed(0)
+ ctx = ppn._StageContext(0, 1, True, False, False)
+ stage = ppn._PPStageModule(Toy(ctx), ctx, torch.float32, 8)
+ x = torch.randint(0, 10, (2, 5))
+ expected = stage(x)
+ expected.sum().backward()
+ grads = [p.grad.clone() for p in stage.parameters() if p.grad is not None]
+ stage.zero_grad(set_to_none = True)
+
+ torch._dynamo.reset()
+ counters.clear()
+ out = torch.compile(stage, backend = "eager", fullgraph = True)(x)
+ out.sum().backward()
+ assert sum(counters["graph_break"].values()) == 0
+ assert torch.equal(out, expected)
+ got = [p.grad for p in stage.parameters() if p.grad is not None]
+ assert len(got) == len(grads) and all(torch.equal(a, b) for a, b in zip(got, grads))
+ torch._dynamo.reset()
+
+
def test_pp2_entry_points_are_refused():
results = _spawn("guards", 2)
if "skip" in results[0]:
diff --git a/unsloth/models/pp_native.py b/unsloth/models/pp_native.py
index 7e4d313b7a4..507db389395 100644
--- a/unsloth/models/pp_native.py
+++ b/unsloth/models/pp_native.py
@@ -104,11 +104,8 @@ class PPConfig:
class _StageOutput(BaseException):
"""Raised by the pass-through after a non-last stage's last layer: the stage output. A BaseException,
- so no `except Exception` inside a model forward swallows it."""
-
- def __init__(self, hidden):
- super().__init__()
- self.hidden = hidden
+ so no `except Exception` inside a model forward swallows it. The hidden states ride in `args`:
+ Dynamo cannot trace a custom exception `__init__` (a graph break, an error under fullgraph)."""
def _unwrap(model):
@@ -1119,7 +1116,7 @@ class _PPStageModule(nn.Module):
ctx.router_logits = []
out = self.model(input_ids = input_ids, use_cache = False, **kwargs)
except _StageOutput as stop:
- hidden = stop.hidden
+ hidden = stop.args[0]
if hidden.shape[-1] != self._pp_hidden or hidden.dim() != 3:
raise RuntimeError(
f"Unsloth: pipeline stage output has shape {tuple(hidden.shape)}, expected "