Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Pipeline parallel on CPU: run the chunked cross entropy on the last stage (#16846)

Daniel Hana2d21d9d032b
diff --git a/tests/test_pp_env_cpu_chunked_ce.py b/tests/test_pp_env_cpu_chunked_ce.py
new file mode 100644
index 00000000000..b7d9dd43f0b
--- /dev/null
+++ b/tests/test_pp_env_cpu_chunked_ce.py
@@ -0,0 +1,124 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""CPU pipeline step must arm zoo's chunked CE itself: the schedule never calls compute_loss."""
+
+import pytest
+import torch
+
+from unsloth.models import pp_env
+
+
+class _Args:
+ device = torch.device("cpu")
+ gradient_accumulation_steps = 1
+ loss_type = "nll"
+
+
+class _Accelerator:
+ scaler = None
+
+ def unwrap_model(self, model):
+ return model
+
+
+def _trainer(
+ model,
+ stage,
+ zoo_wrapped = True,
+ loss_type = "nll",
+):
+ def compute_loss(self, *a, **k):
+ raise AssertionError("the pipeline step never calls compute_loss")
+
+ if zoo_wrapped:
+ compute_loss._unsloth_cpu = (
+ True # what unsloth_zoo.cpu.trainer._patch_sft_compute_loss marks
+ )
+ cls = type(
+ "CpuSFTTrainer", (), {"compute_loss": compute_loss, "_prepare_inputs": lambda self, x: x}
+ )
+ trainer = cls()
+ trainer.args = _Args()
+ trainer.args.loss_type = loss_type
+ trainer.accelerator = _Accelerator()
+ trainer.optimizer = None
+ trainer.__dict__["_unsloth_pipeline_parallel"] = stage
+ return trainer
+
+
+class _Stage:
+ def __init__(
+ self,
+ inner,
+ fail = False,
+ ):
+ self.inner, self.fail, self.armed = inner, fail, []
+
+ def step(self, input_ids, labels, **kwargs):
+ self.armed.append(bool(getattr(self.inner, "_unsloth_cpu_chunked_ce_active", False)))
+ if self.fail:
+ raise RuntimeError("stage failed")
+ return torch.tensor(1.0)
+
+
+def _models():
+ inner = torch.nn.Linear(
+ 2, 2
+ ) # the causal LM whose forward zoo's patch_chunked_causal_lm_loss replaced
+ inner._unsloth_cpu_chunked_ce = True
+ peft = torch.nn.Module()
+ peft.base_model = torch.nn.Module()
+ peft.base_model.model = inner
+ return peft, inner
+
+
+def plain_helpers(monkeypatch):
+ monkeypatch.setattr(pp_env, "_note_trained", lambda stage: None)
+ monkeypatch.setattr(
+ pp_env, "_batch", lambda stage, inputs: (inputs["input_ids"], inputs["labels"], {})
+ )
+ monkeypatch.setattr(pp_env, "_global_items", lambda trainer, stage, n: n)
+ monkeypatch.setattr(pp_env, "_first_stage_batch", lambda stage, i, l, e, n: (i, l, e, n))
+ monkeypatch.setattr(pp_env, "resolve_microbatches", lambda stage, rows: (2, 0))
+ monkeypatch.setattr(pp_env, "_refuse_split_router_aux", lambda stage, m: None)
+ monkeypatch.setattr(pp_env, "_pad_rows", lambda i, l, e, pad: (i, l, e))
+
+
+def _step(trainer, model):
+ batch = {
+ "input_ids": torch.ones(2, 4, dtype = torch.long),
+ "labels": torch.ones(2, 4, dtype = torch.long),
+ }
+ return pp_env._pp_training_step(trainer, model, batch, num_items_in_batch = 6)
+
+
+def test_pp_step_arms_cpu_chunked_ce_like_compute_loss(plain_helpers):
+ peft, inner = _models()
+ stage = _Stage(inner)
+ _step(_trainer(peft, stage), peft)
+ assert stage.armed == [True]
+ assert inner._unsloth_cpu_chunked_ce_active is False
+
+
+def test_pp_step_disarms_when_the_stage_raises(plain_helpers):
+ peft, inner = _models()
+ stage = _Stage(inner, fail = True)
+ with pytest.raises(RuntimeError, match = "stage failed"):
+ _step(_trainer(peft, stage), peft)
+ assert stage.armed == [True] and inner._unsloth_cpu_chunked_ce_active is False
+
+
+ "zoo_wrapped,loss_type", [(False, "nll"), (True, "dft")], ids = ["no_zoo_wrapper", "dft"]
+)
+def test_pp_step_leaves_chunked_ce_off_where_compute_loss_would(
+ plain_helpers, zoo_wrapped, loss_type
+):
+ # Unwrapped compute_loss or non-nll loss keeps the model's own forward, as zoo does.
+ peft, inner = _models()
+ stage = _Stage(inner)
+ _step(_trainer(peft, stage, zoo_wrapped = zoo_wrapped, loss_type = loss_type), peft)
+ assert stage.armed == [False]
+ assert not vars(inner).get("_unsloth_cpu_chunked_ce_active", False)
diff --git a/unsloth/models/pp_env.py b/unsloth/models/pp_env.py
index bcb85b4e715..b74616e231a 100644
--- a/unsloth/models/pp_env.py
+++ b/unsloth/models/pp_env.py
@@ -752,6 +752,23 @@ def _global_items(trainer, stage, num_items_in_batch):
return int(count[0])
+def _cpu_chunked_ce_model(trainer, model):
+ """Model whose zoo CPU chunked CE compute_loss would arm, or None; the pipeline schedule never calls compute_loss."""
+ if not getattr(getattr(type(trainer), "compute_loss", None), "_unsloth_cpu", False):
+ return None
+ loss_type = getattr(trainer.args, "loss_type", None)
+ if str(getattr(loss_type, "value", loss_type) or "nll") != "nll":
+ return None
+ accelerator = getattr(trainer, "accelerator", None)
+ unwrapped = accelerator.unwrap_model(model) if accelerator is not None else model
+ base = getattr(unwrapped, "base_model", None)
+ for candidate in (unwrapped, base, getattr(base, "model", None)):
+ # vars(): PEFT forwards attribute reads to the inner model.
+ if candidate is not None and vars(candidate).get("_unsloth_cpu_chunked_ce", False):
+ return candidate
+ return None
+
+
def _pp_training_step(
self,
model,
@@ -784,15 +801,22 @@ def _pp_training_step(
m, pad = resolve_microbatches(stage, input_ids.shape[0])
_refuse_split_router_aux(stage, m)
input_ids, labels, extra = _pad_rows(input_ids, labels, extra, pad)
- loss = stage.step(
- input_ids,
- labels,
- num_items_in_batch = n,
- scaler = scaler,
- n_microbatches = m,
- loss_scale = scale,
- **extra,
- )
+ chunked = _cpu_chunked_ce_model(self, model)
+ if chunked is not None:
+ chunked._unsloth_cpu_chunked_ce_active = True
+ try:
+ loss = stage.step(
+ input_ids,
+ labels,
+ num_items_in_batch = n,
+ scaler = scaler,
+ n_microbatches = m,
+ loss_scale = scale,
+ **extra,
+ )
+ finally:
+ if chunked is not None:
+ chunked._unsloth_cpu_chunked_ce_active = False
del inputs
return loss.detach().to(self.args.device)