Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

fix: exclude padded logits from softcapped cross entropy (#10616)

Chenghao Liu57b1d758b82a
diff --git a/tests/test_cross_entropy_softcap_padding.py b/tests/test_cross_entropy_softcap_padding.py
new file mode 100644
index 00000000000..bdbd5ce8a3f
--- /dev/null
+++ b/tests/test_cross_entropy_softcap_padding.py
@@ -0,0 +1,123 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+import pytest
+import torch
+import torch.nn.functional as F
+
+pytestmark = pytest.mark.gpu
+
+
[email protected](not torch.cuda.is_available(), reason = "CUDA Triton kernels required")
[email protected]("dtype", [torch.float32, torch.float16, torch.bfloat16])
+ "vocab_size,softcap,offset",
+ [
+ (32000, 1.0, 0.0),
+ (65537, 1.0, 0.0),
+ (256000, 30.0, -100.0),
+ (262208, 30.0, -100.0),
+ (32768, 1.0, 0.0),
+ (32000, 0.0, 0.0),
+ (65537, 0.0, 0.0),
+ ],
+)
+def test_cross_entropy_softcap_padding(dtype, vocab_size, softcap, offset):
+ from unsloth.kernels.cross_entropy_loss import fast_cross_entropy_loss
+
+ torch.manual_seed(42)
+ inputs = (torch.randn(1, 3, vocab_size, device = "cuda") + offset).to(dtype)
+ labels = torch.tensor([[0, vocab_size - 1, -100]], device = "cuda")
+ logits = inputs.clone().requires_grad_()
+ reference_logits = inputs.clone().requires_grad_()
+ transformed = reference_logits.float()
+ if softcap:
+ transformed = softcap * torch.tanh(transformed / softcap)
+ expected = F.cross_entropy(transformed.flatten(0, 1), labels.flatten())
+ expected.backward()
+
+ actual = fast_cross_entropy_loss(logits, labels, logit_softcapping = softcap)
+ actual.backward()
+
+ torch.testing.assert_close(actual, expected, rtol = 1e-5, atol = 1e-5)
+ torch.testing.assert_close(logits.grad, reference_logits.grad, rtol = 1e-2, atol = 1e-7)
+
+
[email protected](not torch.cuda.is_available(), reason = "CUDA Triton kernels required")
[email protected]("vocab_size", [32000, 65537, 256000])
[email protected]("logit_scaling", [0.0625, 0.5, 2.0])
+def test_cross_entropy_softcap_padding_with_logit_scaling(vocab_size, logit_scaling):
+ """Cohere-style logit scaling runs before the softcap, so the mask has to survive both."""
+ from unsloth.kernels.cross_entropy_loss import fast_cross_entropy_loss
+
+ softcap = 30.0
+ torch.manual_seed(42)
+ inputs = (torch.randn(1, 3, vocab_size, device = "cuda") - 100.0).float()
+ labels = torch.tensor([[0, vocab_size - 1, -100]], device = "cuda")
+ logits = inputs.clone().requires_grad_()
+ reference_logits = inputs.clone().requires_grad_()
+ transformed = logit_scaling * reference_logits.float()
+ transformed = softcap * torch.tanh(transformed / softcap)
+ expected = F.cross_entropy(transformed.flatten(0, 1), labels.flatten())
+ expected.backward()
+
+ actual = fast_cross_entropy_loss(
+ logits,
+ labels,
+ logit_softcapping = softcap,
+ logit_scaling = logit_scaling,
+ )
+ actual.backward()
+
+ torch.testing.assert_close(actual, expected, rtol = 1e-5, atol = 1e-5)
+ torch.testing.assert_close(logits.grad, reference_logits.grad, rtol = 1e-2, atol = 1e-7)
+
+
[email protected](not torch.cuda.is_available(), reason = "CUDA Triton kernels required")
[email protected]("vocab_size", [32000, 65537])
[email protected]("softcap", [0.0, 30.0])
+def test_negative_logit_scaling_does_not_nan(vocab_size, softcap):
+ """A negative scale maps the -inf padding to +inf, which used to poison the row maximum."""
+ from unsloth.kernels.cross_entropy_loss import fast_cross_entropy_loss
+
+ torch.manual_seed(42)
+ inputs = torch.randn(1, 2, vocab_size, device = "cuda").float()
+ labels = torch.tensor([[0, vocab_size - 1]], device = "cuda")
+ logits = inputs.clone().requires_grad_()
+ reference_logits = inputs.clone().requires_grad_()
+ transformed = -1.0 * reference_logits.float()
+ if softcap:
+ transformed = softcap * torch.tanh(transformed / softcap)
+ expected = F.cross_entropy(transformed.flatten(0, 1), labels.flatten())
+
+ actual = fast_cross_entropy_loss(
+ logits,
+ labels,
+ logit_softcapping = softcap,
+ logit_scaling = -1.0,
+ )
+
+ assert torch.isfinite(actual), f"loss is {actual}"
+ torch.testing.assert_close(actual, expected, rtol = 1e-5, atol = 1e-5)
+
+
[email protected](not torch.cuda.is_available(), reason = "CUDA Triton kernels required")
[email protected]("vocab_size", [32000, 65537, 256000, 262208])
+def test_softcapped_probabilities_sum_to_one(vocab_size):
+ """Guards the denominator without leaning on a loss tolerance."""
+ from unsloth.kernels.cross_entropy_loss import fast_cross_entropy_loss
+
+ softcap = 30.0
+ torch.manual_seed(42)
+ inputs = (torch.randn(1, 1, vocab_size, device = "cuda") - 100.0).float()
+ labels = torch.tensor([[0]], device = "cuda")
+
+ loss = fast_cross_entropy_loss(
+ inputs.clone().requires_grad_(), labels, logit_softcapping = softcap
+ )
+ transformed = softcap * torch.tanh(inputs.double() / softcap)
+ # A single supervised token means loss == logsumexp - transformed[label].
+ logsumexp = loss.double() + transformed[0, 0, 0]
+ mass = torch.exp(transformed - logsumexp).sum()
+
+ torch.testing.assert_close(mass, torch.ones_like(mass), rtol = 0, atol = 1e-5)
diff --git a/unsloth/kernels/cross_entropy_loss.py b/unsloth/kernels/cross_entropy_loss.py
index 6616e070604..17c744b4f55 100644
--- a/unsloth/kernels/cross_entropy_loss.py
+++ b/unsloth/kernels/cross_entropy_loss.py
@@ -83,6 +83,9 @@ def _cross_entropy_forward(
# Do logit softcapping for Gemma 2: t * tanh(1/t * x)
if DO_SOFTCAPPING:
logits = SOFTCAP * triton_tanh(logits / SOFTCAP)
+ if DO_LOGIT_SCALING or DO_SOFTCAPPING:
+ # Either transform makes the -inf padding finite: -SOFTCAP via tanh, +inf via a negative scale.
+ logits = tl.where(mask, logits, -float("inf"))
c = tl.max(logits, 0)
logsumexp = c + tl.log(tl.sum(tl.exp(logits - c), 0))
@@ -168,6 +171,9 @@ def _chunked_cross_entropy_forward(
logits = LOGIT_SCALE * logits
if DO_SOFTCAPPING:
logits = SOFTCAP * triton_tanh(logits / SOFTCAP)
+ if DO_LOGIT_SCALING or DO_SOFTCAPPING:
+ # Either transform makes the -inf padding finite: -SOFTCAP via tanh, +inf via a negative scale.
+ logits = tl.where(mask, logits, -float("inf"))
c = tl.max(logits, 0)
logsumexp = c + tl.log(tl.sum(tl.exp(logits - c), 0))