Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
fix: exclude padded logits from softcapped cross entropy (#10616)
Chenghao Liu57b1d758b82a
diff --git a/tests/test_cross_entropy_softcap_padding.py b/tests/test_cross_entropy_softcap_padding.pynew file mode 100644index 00000000000..bdbd5ce8a3f--- /dev/null+++ b/tests/test_cross_entropy_softcap_padding.py@@ -0,0 +1,123 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++import pytest+import torch+import torch.nn.functional as F++pytestmark = pytest.mark.gpu++[email protected](not torch.cuda.is_available(), reason = "CUDA Triton kernels required")[email protected]("dtype", [torch.float32, torch.float16, torch.bfloat16])+ "vocab_size,softcap,offset",+ [+ (32000, 1.0, 0.0),+ (65537, 1.0, 0.0),+ (256000, 30.0, -100.0),+ (262208, 30.0, -100.0),+ (32768, 1.0, 0.0),+ (32000, 0.0, 0.0),+ (65537, 0.0, 0.0),+ ],+)+def test_cross_entropy_softcap_padding(dtype, vocab_size, softcap, offset):+ from unsloth.kernels.cross_entropy_loss import fast_cross_entropy_loss++ torch.manual_seed(42)+ inputs = (torch.randn(1, 3, vocab_size, device = "cuda") + offset).to(dtype)+ labels = torch.tensor([[0, vocab_size - 1, -100]], device = "cuda")+ logits = inputs.clone().requires_grad_()+ reference_logits = inputs.clone().requires_grad_()+ transformed = reference_logits.float()+ if softcap:+ transformed = softcap * torch.tanh(transformed / softcap)+ expected = F.cross_entropy(transformed.flatten(0, 1), labels.flatten())+ expected.backward()++ actual = fast_cross_entropy_loss(logits, labels, logit_softcapping = softcap)+ actual.backward()++ torch.testing.assert_close(actual, expected, rtol = 1e-5, atol = 1e-5)+ torch.testing.assert_close(logits.grad, reference_logits.grad, rtol = 1e-2, atol = 1e-7)++[email protected](not torch.cuda.is_available(), reason = "CUDA Triton kernels required")[email protected]("vocab_size", [32000, 65537, 256000])[email protected]("logit_scaling", [0.0625, 0.5, 2.0])+def test_cross_entropy_softcap_padding_with_logit_scaling(vocab_size, logit_scaling):+ """Cohere-style logit scaling runs before the softcap, so the mask has to survive both."""+ from unsloth.kernels.cross_entropy_loss import fast_cross_entropy_loss++ softcap = 30.0+ torch.manual_seed(42)+ inputs = (torch.randn(1, 3, vocab_size, device = "cuda") - 100.0).float()+ labels = torch.tensor([[0, vocab_size - 1, -100]], device = "cuda")+ logits = inputs.clone().requires_grad_()+ reference_logits = inputs.clone().requires_grad_()+ transformed = logit_scaling * reference_logits.float()+ transformed = softcap * torch.tanh(transformed / softcap)+ expected = F.cross_entropy(transformed.flatten(0, 1), labels.flatten())+ expected.backward()++ actual = fast_cross_entropy_loss(+ logits,+ labels,+ logit_softcapping = softcap,+ logit_scaling = logit_scaling,+ )+ actual.backward()++ torch.testing.assert_close(actual, expected, rtol = 1e-5, atol = 1e-5)+ torch.testing.assert_close(logits.grad, reference_logits.grad, rtol = 1e-2, atol = 1e-7)++[email protected](not torch.cuda.is_available(), reason = "CUDA Triton kernels required")[email protected]("vocab_size", [32000, 65537])[email protected]("softcap", [0.0, 30.0])+def test_negative_logit_scaling_does_not_nan(vocab_size, softcap):+ """A negative scale maps the -inf padding to +inf, which used to poison the row maximum."""+ from unsloth.kernels.cross_entropy_loss import fast_cross_entropy_loss++ torch.manual_seed(42)+ inputs = torch.randn(1, 2, vocab_size, device = "cuda").float()+ labels = torch.tensor([[0, vocab_size - 1]], device = "cuda")+ logits = inputs.clone().requires_grad_()+ reference_logits = inputs.clone().requires_grad_()+ transformed = -1.0 * reference_logits.float()+ if softcap:+ transformed = softcap * torch.tanh(transformed / softcap)+ expected = F.cross_entropy(transformed.flatten(0, 1), labels.flatten())++ actual = fast_cross_entropy_loss(+ logits,+ labels,+ logit_softcapping = softcap,+ logit_scaling = -1.0,+ )++ assert torch.isfinite(actual), f"loss is {actual}"+ torch.testing.assert_close(actual, expected, rtol = 1e-5, atol = 1e-5)++[email protected](not torch.cuda.is_available(), reason = "CUDA Triton kernels required")[email protected]("vocab_size", [32000, 65537, 256000, 262208])+def test_softcapped_probabilities_sum_to_one(vocab_size):+ """Guards the denominator without leaning on a loss tolerance."""+ from unsloth.kernels.cross_entropy_loss import fast_cross_entropy_loss++ softcap = 30.0+ torch.manual_seed(42)+ inputs = (torch.randn(1, 1, vocab_size, device = "cuda") - 100.0).float()+ labels = torch.tensor([[0]], device = "cuda")++ loss = fast_cross_entropy_loss(+ inputs.clone().requires_grad_(), labels, logit_softcapping = softcap+ )+ transformed = softcap * torch.tanh(inputs.double() / softcap)+ # A single supervised token means loss == logsumexp - transformed[label].+ logsumexp = loss.double() + transformed[0, 0, 0]+ mass = torch.exp(transformed - logsumexp).sum()++ torch.testing.assert_close(mass, torch.ones_like(mass), rtol = 0, atol = 1e-5)diff --git a/unsloth/kernels/cross_entropy_loss.py b/unsloth/kernels/cross_entropy_loss.pyindex 6616e070604..17c744b4f55 100644--- a/unsloth/kernels/cross_entropy_loss.py+++ b/unsloth/kernels/cross_entropy_loss.py@@ -83,6 +83,9 @@ def _cross_entropy_forward(# Do logit softcapping for Gemma 2: t * tanh(1/t * x)if DO_SOFTCAPPING:logits = SOFTCAP * triton_tanh(logits / SOFTCAP)+ if DO_LOGIT_SCALING or DO_SOFTCAPPING:+ # Either transform makes the -inf padding finite: -SOFTCAP via tanh, +inf via a negative scale.+ logits = tl.where(mask, logits, -float("inf"))c = tl.max(logits, 0)logsumexp = c + tl.log(tl.sum(tl.exp(logits - c), 0))@@ -168,6 +171,9 @@ def _chunked_cross_entropy_forward(logits = LOGIT_SCALE * logitsif DO_SOFTCAPPING:logits = SOFTCAP * triton_tanh(logits / SOFTCAP)+ if DO_LOGIT_SCALING or DO_SOFTCAPPING:+ # Either transform makes the -inf padding finite: -SOFTCAP via tanh, +inf via a negative scale.+ logits = tl.where(mask, logits, -float("inf"))c = tl.max(logits, 0)logsumexp = c + tl.log(tl.sum(tl.exp(logits - c), 0))