Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Cross entropy: keep the loss and label gradient of confident tokens (#15140)

Ming Liu80fabbebf78f
diff --git a/tests/test_cross_entropy_confident_labels.py b/tests/test_cross_entropy_confident_labels.py
new file mode 100644
index 00000000000..a5c4107cc3c
--- /dev/null
+++ b/tests/test_cross_entropy_confident_labels.py
@@ -0,0 +1,114 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""Once the label logit leads the logsumexp of the others by ~16.6 nats, softmax(label) rounds to
+1 in fp32: logsumexp - x and softmax - 1 both become exactly 0 while the other classes keep their
+gradients, so the loss reads 0 and the gradient no longer sums to zero (arXiv 2610.31642)."""
+
+import math
+
+import pytest
+from real_accelerator import (
+ has_real_cuda,
+) # tests/_shared, on sys.path via tests/conftest.py
+import torch
+import torch.nn.functional as F
+
+pytestmark = pytest.mark.gpu
+
+
+def _reference(
+ logits,
+ labels,
+ softcap = 0.0,
+):
+ """fp64 loss and gradient written as softplus(logsumexp(others) - x)."""
+ z = logits.double().clone().requires_grad_()
+ t = softcap * torch.tanh(z / softcap) if softcap else z
+ target = t.gather(-1, labels.clamp(min = 0)[..., None])
+ others = (t - target).masked_fill(
+ F.one_hot(labels.clamp(min = 0), t.shape[-1]).bool(), float("-inf")
+ )
+ loss = F.softplus(torch.logsumexp(others, -1)) * (labels != -100)
+ loss.sum().backward()
+ return loss.detach(), z.grad
+
+
[email protected](not has_real_cuda(), reason = "CUDA Triton kernels required")
[email protected]("vocab_size", [32000, 65537, 262144])
[email protected]("margin", [5.0, 10.0, 17.0, 20.0, 40.0, 80.0])
+def test_confident_label_keeps_loss_and_gradient(vocab_size, margin):
+ from unsloth.kernels.cross_entropy_loss import Fast_CrossEntropyLoss
+
+ # margin = x - logsumexp(others) with the vocab_size - 1 others at 0
+ inputs = torch.zeros(3, vocab_size, device = "cuda")
+ labels = torch.tensor([3, vocab_size - 1, -100], device = "cuda")
+ inputs[0, 3] = inputs[1, vocab_size - 1] = margin + math.log(vocab_size - 1)
+ logits = inputs.clone().requires_grad_()
+
+ loss = Fast_CrossEntropyLoss.apply(logits, labels, 0, 0)
+ loss.sum().backward()
+ expected_loss, expected_grad = _reference(inputs, labels)
+
+ rows = torch.arange(2, device = "cuda")
+ torch.testing.assert_close(loss.double(), expected_loss, rtol = 1e-5, atol = 0)
+ torch.testing.assert_close(
+ logits.grad[rows, labels[:2]].double(), expected_grad[rows, labels[:2]], rtol = 1e-5, atol = 0
+ )
+ # Used to be +exp(-margin): the label's share was rounded away.
+ assert logits.grad[:2].double().sum(-1).abs().max() <= 1e-4 * math.exp(-margin)
+
+
[email protected](not has_real_cuda(), reason = "CUDA Triton kernels required")
[email protected]("vocab_size", [32000, 65537, 262144])
[email protected]("softcap", [0.0, 30.0])
+def test_random_logits_match_fp64(vocab_size, softcap):
+ from unsloth.kernels.cross_entropy_loss import Fast_CrossEntropyLoss
+
+ torch.manual_seed(0)
+ inputs = torch.randn(4, vocab_size, device = "cuda") * 4
+ labels = torch.tensor([0, 17, -100, vocab_size - 1], device = "cuda")
+ logits = inputs.clone().requires_grad_()
+
+ loss = Fast_CrossEntropyLoss.apply(logits, labels, softcap, 0)
+ loss.sum().backward()
+ expected_loss, expected_grad = _reference(inputs, labels, softcap)
+
+ torch.testing.assert_close(loss.double(), expected_loss, rtol = 1e-5, atol = 1e-6)
+ torch.testing.assert_close(logits.grad.double(), expected_grad, rtol = 1e-4, atol = 1e-9)
+
+
[email protected](not has_real_cuda(), reason = "CUDA Triton kernels required")
+def test_confident_bf16_label():
+ from unsloth.kernels.cross_entropy_loss import fast_cross_entropy_loss
+
+ inputs = torch.zeros(1, 1, 32000, device = "cuda", dtype = torch.bfloat16)
+ inputs[0, 0, 0] = 32.0 # ~21.6 nats over the others
+ labels = torch.tensor([[0]], device = "cuda")
+ logits = inputs.clone().requires_grad_()
+
+ loss = fast_cross_entropy_loss(logits, labels)
+ loss.backward()
+ expected_loss, expected_grad = _reference(inputs[0].float(), labels[0])
+
+ torch.testing.assert_close(loss.double(), expected_loss[0], rtol = 1e-5, atol = 0)
+ torch.testing.assert_close(
+ logits.grad[0, 0, 0].double(), expected_grad[0, 0], rtol = 1e-2, atol = 0
+ )
+
+
[email protected](not has_real_cuda(), reason = "CUDA Triton kernels required")
[email protected]("vocab_size, label_logit, other", [(2, 0.0, 103.0), (262144, -91.0, 0.0)])
+def test_hopeless_label_gradient_stays_at_minus_one(vocab_size, label_logit, other):
+ from unsloth.kernels.cross_entropy_loss import Fast_CrossEntropyLoss
+
+ # Loss ~100: exp(-loss) is subnormal, the label gradient must be -1 and never past it.
+ inputs = torch.full((1, vocab_size), other, device = "cuda")
+ inputs[0, 0] = label_logit
+ labels = torch.tensor([0], device = "cuda")
+ logits = inputs.clone().requires_grad_()
+
+ Fast_CrossEntropyLoss.apply(logits, labels, 0, 0).sum().backward()
+ _, expected_grad = _reference(inputs, labels)
+ torch.testing.assert_close(logits.grad[0, 0].double(), expected_grad[0, 0], rtol = 1e-6, atol = 0)
+ assert logits.grad[0, 0].item() >= -1.0
diff --git a/tests/test_cross_entropy_eviction_hints.py b/tests/test_cross_entropy_eviction_hints.py
index cbf63bc0c60..aa8b0006fc8 100644
--- a/tests/test_cross_entropy_eviction_hints.py
+++ b/tests/test_cross_entropy_eviction_hints.py
@@ -27,6 +27,7 @@ _SIGNATURE = {
"logits_ptr": "*bf16",
"loss_ptr": "*fp32",
"logsumexp_ptr": "*fp32",
+ "logsumexp_others_ptr": "*fp32",
"labels_ptr": "*i64",
}
diff --git a/unsloth/kernels/cross_entropy_loss.py b/unsloth/kernels/cross_entropy_loss.py
index 550e16e94bd..055cf75f5bf 100644
--- a/unsloth/kernels/cross_entropy_loss.py
+++ b/unsloth/kernels/cross_entropy_loss.py
@@ -31,6 +31,28 @@ from unsloth_zoo.loss_utils import (
)
+# log1p / expm1 from tl.log / tl.exp only (Kahan's trick), so no libdevice is needed.
+def _log1p(x):
+ # log(1 + x) for x >= 0, keeping x when 1 + x rounds to 1.
+ u = 1.0 + x
+ return tl.where(u == 1.0, x, tl.log(u) * (x / (u - 1.0)))
+
+
+def _softplus(x):
+ return tl.maximum(x, 0.0) + _log1p(tl.exp(-tl.abs(x)))
+
+
+def _expm1_neg(x):
+ # exp(-x) - 1 for x >= 0: Kahan's trick only below 1, where u - 1 cancels; past ~87 u is
+ # subnormal and log(u) drifts from -x, so the trick would push the gradient past -1.
+ u = tl.exp(-x)
+ near_zero = tl.where(u == 1.0, -x, (u - 1.0) * (-x / tl.log(u)))
+ return tl.where(x < 1.0, near_zero, u - 1.0)
+
+
def _cross_entropy_forward(
logits_ptr,
logits_row_stride: tl.constexpr, # VOCAB_SIZE
@@ -87,7 +109,9 @@ def _cross_entropy_forward(
logits = tl.where(mask, logits, -float("inf"))
c = tl.max(logits, 0)
- logsumexp = c + tl.log(tl.sum(tl.exp(logits - c), 0))
+ # One reduction that skips the label (col_offsets never equals -100); the label is added back below.
+ sum_others = tl.sum(tl.where(col_offsets == label_idx, 0.0, tl.exp(logits - c)), 0)
+ logsumexp = c + tl.log(sum_others)
if label_idx != -100:
x = tl.load(logits_ptr + label_idx).to(tl.float32)
@@ -97,7 +121,9 @@ def _cross_entropy_forward(
# Do logit softcapping for Gemma 2: t * tanh(1/t * x)
if DO_SOFTCAPPING:
x = SOFTCAP * triton_tanh(x / SOFTCAP)
- loss = logsumexp - x
+ # lse - x rounds to 0 past a ~16.6 nat margin in fp32 (softmax collapse, arXiv 2610.31642).
+ loss = _softplus(logsumexp - x)
+ logsumexp = c + tl.log(sum_others + tl.exp(x - c))
else:
loss = 0.0
tl.store(logsumexp_ptr, logsumexp)
@@ -161,6 +187,7 @@ def _chunked_cross_entropy_forward(
logits_row_stride: tl.constexpr,
loss_ptr,
logsumexp_ptr,
+ logsumexp_others_ptr,
labels_ptr,
VOCAB_SIZE: tl.constexpr,
N_CHUNKS: tl.constexpr,
@@ -194,12 +221,16 @@ def _chunked_cross_entropy_forward(
final logsumexp reduction!
Ie do: logsumexp(chunked_logsumexp) - x
+
+ The loss itself is softplus(logsumexp(chunked_logsumexp_others) - x), where
+ the others skip the label, so it does not round to 0 for confident labels.
"""
row_idx = tl.program_id(0)
chunk_idx = tl.program_id(1)
logits_ptr += row_idx * triton_cast(logits_row_stride, tl.int64)
loss_ptr += row_idx
logsumexp_ptr += row_idx * N_CHUNKS + chunk_idx
+ logsumexp_others_ptr += row_idx * N_CHUNKS + chunk_idx
labels_ptr += row_idx
col_offsets = chunk_idx * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE)
@@ -223,7 +254,22 @@ def _chunked_cross_entropy_forward(
c = tl.max(logits, 0)
# All -inf chunk (masked vocab range): shift by 0 so logsumexp is -inf, not NaN.
c = tl.where(c == -float("inf"), 0.0, c)
- logsumexp = c + tl.log(tl.sum(tl.exp(logits - c), 0))
+ exp_logits = tl.exp(logits - c)
+ # One reduction per chunk: the label chunk sums the other terms and adds the label back as a scalar.
+ has_label = (label_idx >= chunk_idx * BLOCK_SIZE) & (label_idx < (chunk_idx + 1) * BLOCK_SIZE)
+ if has_label:
+ exp_logits = tl.where(col_offsets == label_idx, 0.0, exp_logits)
+ sum_others = tl.sum(exp_logits, 0)
+ logsumexp_others = c + tl.log(sum_others)
+ logsumexp = logsumexp_others
+ if has_label:
+ x_label = tl.load(logits_ptr + label_idx).to(tl.float32)
+ if DO_LOGIT_SCALING:
+ x_label = LOGIT_SCALE * x_label
+ if DO_SOFTCAPPING:
+ x_label = SOFTCAP * triton_tanh(x_label / SOFTCAP)
+ logsumexp = c + tl.log(sum_others + tl.exp(x_label - c))
+ tl.store(logsumexp_others_ptr, logsumexp_others)
if chunk_idx == 0:
# logsumexp(chunked_logsumexp) - x, with the -x done separately.
@@ -256,6 +302,7 @@ def _cross_entropy_backward(
dloss_ptr,
dloss_row_stride: tl.constexpr,
logsumexp_ptr,
+ loss_ptr,
labels_ptr,
VOCAB_SIZE: tl.constexpr,
BLOCK_SIZE: tl.constexpr,
@@ -278,6 +325,9 @@ def _cross_entropy_backward(
If y == 0: dC/dx = 0
If y == 1 and x == label: dC/dlabel = exp[x - logsumexp] - 1
If y == 1 and x != label: dC/dx = exp[x - logsumexp]
+
+ exp[x - logsumexp] - 1 rounds to 0 for confident labels, so the label
+ uses exp[-loss] - 1 = expm1(-loss) with the accurate forward loss.
"""
row_idx = tl.program_id(0)
block_idx = tl.program_id(1)
@@ -309,10 +359,11 @@ def _cross_entropy_backward(
x = SOFTCAP * partial
logsumexp = tl.load(logsumexp_ptr + row_idx)
+ loss = tl.load(loss_ptr + row_idx)
y = tl.exp(x - logsumexp)
y = tl.where(
col_offsets == label_idx,
- y - 1.0, # exp(x - logsumexp) - 1
+ _expm1_neg(loss), # exp(x - logsumexp) - 1 = exp(-loss) - 1
y, # exp(x - logsumexp)
)
@@ -398,6 +449,7 @@ class Fast_CrossEntropyLoss(torch.autograd.Function):
dtype = torch.float32,
device = device,
)
+ logsumexp_others = torch.empty_like(logsumexp)
with torch_gpu_device(device):
_chunked_cross_entropy_forward[
@@ -410,6 +462,7 @@ class Fast_CrossEntropyLoss(torch.autograd.Function):
logits.stride(0),
losses,
logsumexp,
+ logsumexp_others,
labels,
VOCAB_SIZE = vocab_size,
N_CHUNKS = n_chunks,
@@ -422,11 +475,13 @@ class Fast_CrossEntropyLoss(torch.autograd.Function):
num_warps = num_warps,
enable_fp_fusion = True,
)
- # logsumexp(chunked_logsumexp) - x, with the -x done separately.
+ # softplus(logsumexp(chunked_logsumexp_others) - x), with the -x done separately.
logsumexp = torch.logsumexp(logsumexp, dim = 1)
- losses = (losses + logsumexp).masked_fill(labels == -100, 0) # mask padding out
+ # Out of place: in-place ops on the returned tensor zero its compiled grads (torch 2.11).
+ losses = torch.nn.functional.softplus(torch.logsumexp(logsumexp_others, dim = 1) + losses)
+ losses = losses.masked_fill(labels == -100, 0) # mask padding out
- ctx.save_for_backward(logits, logsumexp, labels)
+ ctx.save_for_backward(logits, logsumexp, losses, labels)
ctx.DO_SOFTCAPPING = DO_SOFTCAPPING
ctx.logit_softcapping = logit_softcapping
ctx.DO_LOGIT_SCALING = DO_LOGIT_SCALING
@@ -435,7 +490,7 @@ class Fast_CrossEntropyLoss(torch.autograd.Function):
@staticmethod
def backward(ctx, dlosses):
- logits, logsumexp, labels = ctx.saved_tensors
+ logits, logsumexp, losses, labels = ctx.saved_tensors
n_rows: int
vocab_size: int
n_rows, vocab_size = logits.shape
@@ -461,6 +516,7 @@ class Fast_CrossEntropyLoss(torch.autograd.Function):
dlosses,
dlosses.stride(0),
logsumexp,
+ losses,
labels,
VOCAB_SIZE = vocab_size,
BLOCK_SIZE = BLOCK_SIZE,