Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Context parallel: eval loss is the sequence's token mean, not cp_size times it (#17049)

Shubhankar Tripathy76f3834eb68a
diff --git a/tests/test_context_parallel.py b/tests/test_context_parallel.py
index 2c3c6117693..fbe1c20cb41 100644
--- a/tests/test_context_parallel.py
+++ b/tests/test_context_parallel.py
@@ -327,10 +327,19 @@ def _patched_trainer(
def compute_loss_context_manager(self):
return contextlib.nullcontext()
- def compute_loss(self, model, inputs):
+ def compute_loss(
+ self,
+ model,
+ inputs,
+ num_items_in_batch = None,
+ ):
+ # Like TRL's SFTTrainer: a per-token loss of 1 over this shard's targets, divided by the
+ # num_items_in_batch argument (an inputs["num_items_in_batch"] key is not read).
assert inputs["use_cache"] is False
targets = inputs["shift_labels"].ne(-100).sum()
- return targets.float() / inputs["num_items_in_batch"]
+ if num_items_in_batch is None:
+ num_items_in_batch = targets.clamp_min(1)
+ return targets.float() / num_items_in_batch
def train(self, *a, **k):
return "trained"
@@ -383,6 +392,71 @@ def test_eval_loss_is_the_token_mean_even_with_an_empty_shard(monkeypatch, world
trainer.prediction_step(None, batch(), False)
+def _model_normalised_loss(
+ self,
+ model,
+ inputs,
+ num_items_in_batch = None,
+):
+ # TRL < 1 hands inputs to the model: sum / inputs["num_items_in_batch"] when the key is there, else
+ # transformers' token mean over this shard, 0 / 0 on a shard without targets.
+ targets = inputs["shift_labels"].ne(-100).sum().float()
+ count = inputs.get("num_items_in_batch")
+ return targets / (targets if count is None else count)
+
+
[email protected]("compute_loss", ["trl", "model"])
+def test_eval_loss_is_not_multiplied_by_the_cp_size(monkeypatch, compute_loss):
+ import types
+
+ @contextlib.contextmanager
+ def first_half(mesh, buffers, buffer_seq_dims, no_restore_buffers):
+ for buffer in buffers:
+ buffer.data = buffer[:, : buffer.shape[1] // 2].clone()
+ yield
+
+ monkeypatch.setattr(cp, "context_parallel", first_half)
+ monkeypatch.setattr(cp.dist, "get_world_size", lambda: 2)
+ Trainer = _patched_trainer(monkeypatch)
+ if compute_loss == "model":
+ monkeypatch.setattr(Trainer, "compute_loss", _model_normalised_loss)
+ trainer = cp._bind_cp_methods(object.__new__(Trainer))
+ manager = _manager(size = 2)
+ manager.mesh = types.SimpleNamespace(get_group = lambda: "cp")
+ trainer._context_parallel_manager = manager
+ # Two ranks holding 2 of the 4 targets each, with the same per-token loss: the sum is 1, not 2.
+ monkeypatch.setattr(cp.dist, "all_reduce", lambda tensor, group = None: tensor.mul_(2))
+ labels = torch.tensor([[-100, 1, 1, -100, -100, -100, -100, 1, 1]]) # padded to 12, halves of 6
+ batch = {"input_ids": torch.ones(1, 9, dtype = torch.long), "labels": labels}
+ loss, _, _ = trainer.prediction_step(None, batch, True)
+ assert loss.item() == pytest.approx(1.0)
+
+
+def test_eval_loss_of_a_shard_without_targets_is_zero_even_when_its_mean_is_nan(monkeypatch):
+ import types
+
+ @contextlib.contextmanager
+ def first_half(mesh, buffers, buffer_seq_dims, no_restore_buffers):
+ for buffer in buffers:
+ buffer.data = buffer[:, : buffer.shape[1] // 2].clone()
+ yield
+
+ monkeypatch.setattr(cp, "context_parallel", first_half)
+ monkeypatch.setattr(cp.dist, "get_world_size", lambda: 2)
+ Trainer = _patched_trainer(monkeypatch)
+ monkeypatch.setattr(Trainer, "compute_loss", _model_normalised_loss)
+ trainer = cp._bind_cp_methods(object.__new__(Trainer))
+ manager = _manager(size = 2)
+ manager.mesh = types.SimpleNamespace(get_group = lambda: "cp")
+ trainer._context_parallel_manager = manager
+ # This rank holds the prompt (no targets), the other rank's share is the whole mean, 1.
+ monkeypatch.setattr(cp.dist, "all_reduce", lambda tensor, group = None: tensor.add_(1.0))
+ labels = torch.tensor([[-100] * 7 + [1, 1]])
+ batch = {"input_ids": torch.ones(1, 9, dtype = torch.long), "labels": labels}
+ loss, _, _ = trainer.prediction_step(None, batch, True)
+ assert loss.item() == pytest.approx(1.0)
+
+
@pytest.mark.parametrize(
"attrs",
[
diff --git a/unsloth/context_parallel.py b/unsloth/context_parallel.py
index 569fefbf5d9..7e2c292084f 100644
--- a/unsloth/context_parallel.py
+++ b/unsloth/context_parallel.py
@@ -1943,10 +1943,15 @@ def patch_sft_trainer(install_cp_methods: bool = False) -> None:
if not isinstance(shift_labels, torch.Tensor):
with manager.apply(inputs):
return original_prediction_step(self, model, inputs, *args, **kwargs)
- inputs["num_items_in_batch"] = shift_labels.ne(-100).sum().clamp_min(1)
+ # compute_loss returns this shard's token mean (TRL >= 1 ignores an inputs["num_items_in_batch"]
+ # key, older TRL leaves the model to average): weight it by the shard's share of the targets so
+ # the cp sum is the sequence's token mean. A shard without targets adds 0, even when its mean is 0 / 0.
+ count = shift_labels.ne(-100).sum().clamp_min(1)
inputs.setdefault("use_cache", False)
with manager.apply(inputs), torch.no_grad(), self.compute_loss_context_manager():
+ local = inputs["shift_labels"].ne(-100).sum()
loss = self.compute_loss(model, inputs).detach()
+ loss = torch.where(local > 0, loss * local / count, torch.zeros_like(loss))
dist.all_reduce(loss, group = manager.mesh.get_group())
return loss, None, None