Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Cross entropy: keep the loss and label gradient of confident tokens (#15140)
Ming Liu80fabbebf78f
diff --git a/tests/test_cross_entropy_confident_labels.py b/tests/test_cross_entropy_confident_labels.pynew file mode 100644index 00000000000..a5c4107cc3c--- /dev/null+++ b/tests/test_cross_entropy_confident_labels.py@@ -0,0 +1,114 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""Once the label logit leads the logsumexp of the others by ~16.6 nats, softmax(label) rounds to+1 in fp32: logsumexp - x and softmax - 1 both become exactly 0 while the other classes keep their+gradients, so the loss reads 0 and the gradient no longer sums to zero (arXiv 2610.31642)."""++import math++import pytest+from real_accelerator import (+ has_real_cuda,+) # tests/_shared, on sys.path via tests/conftest.py+import torch+import torch.nn.functional as F++pytestmark = pytest.mark.gpu+++def _reference(+ logits,+ labels,+ softcap = 0.0,+):+ """fp64 loss and gradient written as softplus(logsumexp(others) - x)."""+ z = logits.double().clone().requires_grad_()+ t = softcap * torch.tanh(z / softcap) if softcap else z+ target = t.gather(-1, labels.clamp(min = 0)[..., None])+ others = (t - target).masked_fill(+ F.one_hot(labels.clamp(min = 0), t.shape[-1]).bool(), float("-inf")+ )+ loss = F.softplus(torch.logsumexp(others, -1)) * (labels != -100)+ loss.sum().backward()+ return loss.detach(), z.grad++[email protected](not has_real_cuda(), reason = "CUDA Triton kernels required")[email protected]("vocab_size", [32000, 65537, 262144])[email protected]("margin", [5.0, 10.0, 17.0, 20.0, 40.0, 80.0])+def test_confident_label_keeps_loss_and_gradient(vocab_size, margin):+ from unsloth.kernels.cross_entropy_loss import Fast_CrossEntropyLoss++ # margin = x - logsumexp(others) with the vocab_size - 1 others at 0+ inputs = torch.zeros(3, vocab_size, device = "cuda")+ labels = torch.tensor([3, vocab_size - 1, -100], device = "cuda")+ inputs[0, 3] = inputs[1, vocab_size - 1] = margin + math.log(vocab_size - 1)+ logits = inputs.clone().requires_grad_()++ loss = Fast_CrossEntropyLoss.apply(logits, labels, 0, 0)+ loss.sum().backward()+ expected_loss, expected_grad = _reference(inputs, labels)++ rows = torch.arange(2, device = "cuda")+ torch.testing.assert_close(loss.double(), expected_loss, rtol = 1e-5, atol = 0)+ torch.testing.assert_close(+ logits.grad[rows, labels[:2]].double(), expected_grad[rows, labels[:2]], rtol = 1e-5, atol = 0+ )+ # Used to be +exp(-margin): the label's share was rounded away.+ assert logits.grad[:2].double().sum(-1).abs().max() <= 1e-4 * math.exp(-margin)++[email protected](not has_real_cuda(), reason = "CUDA Triton kernels required")[email protected]("vocab_size", [32000, 65537, 262144])[email protected]("softcap", [0.0, 30.0])+def test_random_logits_match_fp64(vocab_size, softcap):+ from unsloth.kernels.cross_entropy_loss import Fast_CrossEntropyLoss++ torch.manual_seed(0)+ inputs = torch.randn(4, vocab_size, device = "cuda") * 4+ labels = torch.tensor([0, 17, -100, vocab_size - 1], device = "cuda")+ logits = inputs.clone().requires_grad_()++ loss = Fast_CrossEntropyLoss.apply(logits, labels, softcap, 0)+ loss.sum().backward()+ expected_loss, expected_grad = _reference(inputs, labels, softcap)++ torch.testing.assert_close(loss.double(), expected_loss, rtol = 1e-5, atol = 1e-6)+ torch.testing.assert_close(logits.grad.double(), expected_grad, rtol = 1e-4, atol = 1e-9)++[email protected](not has_real_cuda(), reason = "CUDA Triton kernels required")+def test_confident_bf16_label():+ from unsloth.kernels.cross_entropy_loss import fast_cross_entropy_loss++ inputs = torch.zeros(1, 1, 32000, device = "cuda", dtype = torch.bfloat16)+ inputs[0, 0, 0] = 32.0 # ~21.6 nats over the others+ labels = torch.tensor([[0]], device = "cuda")+ logits = inputs.clone().requires_grad_()++ loss = fast_cross_entropy_loss(logits, labels)+ loss.backward()+ expected_loss, expected_grad = _reference(inputs[0].float(), labels[0])++ torch.testing.assert_close(loss.double(), expected_loss[0], rtol = 1e-5, atol = 0)+ torch.testing.assert_close(+ logits.grad[0, 0, 0].double(), expected_grad[0, 0], rtol = 1e-2, atol = 0+ )++[email protected](not has_real_cuda(), reason = "CUDA Triton kernels required")[email protected]("vocab_size, label_logit, other", [(2, 0.0, 103.0), (262144, -91.0, 0.0)])+def test_hopeless_label_gradient_stays_at_minus_one(vocab_size, label_logit, other):+ from unsloth.kernels.cross_entropy_loss import Fast_CrossEntropyLoss++ # Loss ~100: exp(-loss) is subnormal, the label gradient must be -1 and never past it.+ inputs = torch.full((1, vocab_size), other, device = "cuda")+ inputs[0, 0] = label_logit+ labels = torch.tensor([0], device = "cuda")+ logits = inputs.clone().requires_grad_()++ Fast_CrossEntropyLoss.apply(logits, labels, 0, 0).sum().backward()+ _, expected_grad = _reference(inputs, labels)+ torch.testing.assert_close(logits.grad[0, 0].double(), expected_grad[0, 0], rtol = 1e-6, atol = 0)+ assert logits.grad[0, 0].item() >= -1.0diff --git a/tests/test_cross_entropy_eviction_hints.py b/tests/test_cross_entropy_eviction_hints.pyindex cbf63bc0c60..aa8b0006fc8 100644--- a/tests/test_cross_entropy_eviction_hints.py+++ b/tests/test_cross_entropy_eviction_hints.py@@ -27,6 +27,7 @@ _SIGNATURE = {"logits_ptr": "*bf16","loss_ptr": "*fp32","logsumexp_ptr": "*fp32",+ "logsumexp_others_ptr": "*fp32","labels_ptr": "*i64",}diff --git a/unsloth/kernels/cross_entropy_loss.py b/unsloth/kernels/cross_entropy_loss.pyindex 550e16e94bd..055cf75f5bf 100644--- a/unsloth/kernels/cross_entropy_loss.py+++ b/unsloth/kernels/cross_entropy_loss.py@@ -31,6 +31,28 @@ from unsloth_zoo.loss_utils import ()+# log1p / expm1 from tl.log / tl.exp only (Kahan's trick), so no libdevice is needed.+def _log1p(x):+ # log(1 + x) for x >= 0, keeping x when 1 + x rounds to 1.+ u = 1.0 + x+ return tl.where(u == 1.0, x, tl.log(u) * (x / (u - 1.0)))+++def _softplus(x):+ return tl.maximum(x, 0.0) + _log1p(tl.exp(-tl.abs(x)))+++def _expm1_neg(x):+ # exp(-x) - 1 for x >= 0: Kahan's trick only below 1, where u - 1 cancels; past ~87 u is+ # subnormal and log(u) drifts from -x, so the trick would push the gradient past -1.+ u = tl.exp(-x)+ near_zero = tl.where(u == 1.0, -x, (u - 1.0) * (-x / tl.log(u)))+ return tl.where(x < 1.0, near_zero, u - 1.0)++def _cross_entropy_forward(logits_ptr,logits_row_stride: tl.constexpr, # VOCAB_SIZE@@ -87,7 +109,9 @@ def _cross_entropy_forward(logits = tl.where(mask, logits, -float("inf"))c = tl.max(logits, 0)- logsumexp = c + tl.log(tl.sum(tl.exp(logits - c), 0))+ # One reduction that skips the label (col_offsets never equals -100); the label is added back below.+ sum_others = tl.sum(tl.where(col_offsets == label_idx, 0.0, tl.exp(logits - c)), 0)+ logsumexp = c + tl.log(sum_others)if label_idx != -100:x = tl.load(logits_ptr + label_idx).to(tl.float32)@@ -97,7 +121,9 @@ def _cross_entropy_forward(# Do logit softcapping for Gemma 2: t * tanh(1/t * x)if DO_SOFTCAPPING:x = SOFTCAP * triton_tanh(x / SOFTCAP)- loss = logsumexp - x+ # lse - x rounds to 0 past a ~16.6 nat margin in fp32 (softmax collapse, arXiv 2610.31642).+ loss = _softplus(logsumexp - x)+ logsumexp = c + tl.log(sum_others + tl.exp(x - c))else:loss = 0.0tl.store(logsumexp_ptr, logsumexp)@@ -161,6 +187,7 @@ def _chunked_cross_entropy_forward(logits_row_stride: tl.constexpr,loss_ptr,logsumexp_ptr,+ logsumexp_others_ptr,labels_ptr,VOCAB_SIZE: tl.constexpr,N_CHUNKS: tl.constexpr,@@ -194,12 +221,16 @@ def _chunked_cross_entropy_forward(final logsumexp reduction!Ie do: logsumexp(chunked_logsumexp) - x++ The loss itself is softplus(logsumexp(chunked_logsumexp_others) - x), where+ the others skip the label, so it does not round to 0 for confident labels."""row_idx = tl.program_id(0)chunk_idx = tl.program_id(1)logits_ptr += row_idx * triton_cast(logits_row_stride, tl.int64)loss_ptr += row_idxlogsumexp_ptr += row_idx * N_CHUNKS + chunk_idx+ logsumexp_others_ptr += row_idx * N_CHUNKS + chunk_idxlabels_ptr += row_idxcol_offsets = chunk_idx * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE)@@ -223,7 +254,22 @@ def _chunked_cross_entropy_forward(c = tl.max(logits, 0)# All -inf chunk (masked vocab range): shift by 0 so logsumexp is -inf, not NaN.c = tl.where(c == -float("inf"), 0.0, c)- logsumexp = c + tl.log(tl.sum(tl.exp(logits - c), 0))+ exp_logits = tl.exp(logits - c)+ # One reduction per chunk: the label chunk sums the other terms and adds the label back as a scalar.+ has_label = (label_idx >= chunk_idx * BLOCK_SIZE) & (label_idx < (chunk_idx + 1) * BLOCK_SIZE)+ if has_label:+ exp_logits = tl.where(col_offsets == label_idx, 0.0, exp_logits)+ sum_others = tl.sum(exp_logits, 0)+ logsumexp_others = c + tl.log(sum_others)+ logsumexp = logsumexp_others+ if has_label:+ x_label = tl.load(logits_ptr + label_idx).to(tl.float32)+ if DO_LOGIT_SCALING:+ x_label = LOGIT_SCALE * x_label+ if DO_SOFTCAPPING:+ x_label = SOFTCAP * triton_tanh(x_label / SOFTCAP)+ logsumexp = c + tl.log(sum_others + tl.exp(x_label - c))+ tl.store(logsumexp_others_ptr, logsumexp_others)if chunk_idx == 0:# logsumexp(chunked_logsumexp) - x, with the -x done separately.@@ -256,6 +302,7 @@ def _cross_entropy_backward(dloss_ptr,dloss_row_stride: tl.constexpr,logsumexp_ptr,+ loss_ptr,labels_ptr,VOCAB_SIZE: tl.constexpr,BLOCK_SIZE: tl.constexpr,@@ -278,6 +325,9 @@ def _cross_entropy_backward(If y == 0: dC/dx = 0If y == 1 and x == label: dC/dlabel = exp[x - logsumexp] - 1If y == 1 and x != label: dC/dx = exp[x - logsumexp]++ exp[x - logsumexp] - 1 rounds to 0 for confident labels, so the label+ uses exp[-loss] - 1 = expm1(-loss) with the accurate forward loss."""row_idx = tl.program_id(0)block_idx = tl.program_id(1)@@ -309,10 +359,11 @@ def _cross_entropy_backward(x = SOFTCAP * partiallogsumexp = tl.load(logsumexp_ptr + row_idx)+ loss = tl.load(loss_ptr + row_idx)y = tl.exp(x - logsumexp)y = tl.where(col_offsets == label_idx,- y - 1.0, # exp(x - logsumexp) - 1+ _expm1_neg(loss), # exp(x - logsumexp) - 1 = exp(-loss) - 1y, # exp(x - logsumexp))@@ -398,6 +449,7 @@ class Fast_CrossEntropyLoss(torch.autograd.Function):dtype = torch.float32,device = device,)+ logsumexp_others = torch.empty_like(logsumexp)with torch_gpu_device(device):_chunked_cross_entropy_forward[@@ -410,6 +462,7 @@ class Fast_CrossEntropyLoss(torch.autograd.Function):logits.stride(0),losses,logsumexp,+ logsumexp_others,labels,VOCAB_SIZE = vocab_size,N_CHUNKS = n_chunks,@@ -422,11 +475,13 @@ class Fast_CrossEntropyLoss(torch.autograd.Function):num_warps = num_warps,enable_fp_fusion = True,)- # logsumexp(chunked_logsumexp) - x, with the -x done separately.+ # softplus(logsumexp(chunked_logsumexp_others) - x), with the -x done separately.logsumexp = torch.logsumexp(logsumexp, dim = 1)- losses = (losses + logsumexp).masked_fill(labels == -100, 0) # mask padding out+ # Out of place: in-place ops on the returned tensor zero its compiled grads (torch 2.11).+ losses = torch.nn.functional.softplus(torch.logsumexp(logsumexp_others, dim = 1) + losses)+ losses = losses.masked_fill(labels == -100, 0) # mask padding out- ctx.save_for_backward(logits, logsumexp, labels)+ ctx.save_for_backward(logits, logsumexp, losses, labels)ctx.DO_SOFTCAPPING = DO_SOFTCAPPINGctx.logit_softcapping = logit_softcappingctx.DO_LOGIT_SCALING = DO_LOGIT_SCALING@@ -435,7 +490,7 @@ class Fast_CrossEntropyLoss(torch.autograd.Function):@staticmethoddef backward(ctx, dlosses):- logits, logsumexp, labels = ctx.saved_tensors+ logits, logsumexp, losses, labels = ctx.saved_tensorsn_rows: intvocab_size: intn_rows, vocab_size = logits.shape@@ -461,6 +516,7 @@ class Fast_CrossEntropyLoss(torch.autograd.Function):dlosses,dlosses.stride(0),logsumexp,+ losses,labels,VOCAB_SIZE = vocab_size,BLOCK_SIZE = BLOCK_SIZE,