Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

fix: preserve logits in fast cross entropy backward (#12351)

cherry77-cloud0cf426bb2f13
diff --git a/tests/test_cross_entropy_reused_logits.py b/tests/test_cross_entropy_reused_logits.py
new file mode 100644
index 00000000000..8e0e3ede7e3
--- /dev/null
+++ b/tests/test_cross_entropy_reused_logits.py
@@ -0,0 +1,105 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+import pytest
+from real_accelerator import has_real_cuda
+import torch
+import torch.nn.functional as F
+
+pytestmark = [
+ pytest.mark.gpu,
+ pytest.mark.skipif(not has_real_cuda(), reason = "CUDA Triton kernels required"),
+]
+
+
+def _reference_loss(logits, labels, softcap, scaling):
+ transformed = logits.float()
+ if scaling:
+ transformed = scaling * transformed
+ if softcap:
+ transformed = softcap * torch.tanh(transformed / softcap)
+ return F.cross_entropy(transformed.flatten(0, 1), labels.flatten())
+
+
[email protected]("dtype", [torch.float32, torch.float16, torch.bfloat16])
[email protected]("vocab_size", [37, 65537])
[email protected]("softcap,scaling", [(0.0, 0.0), (2.0, 0.0), (0.0, 0.5), (2.0, 0.5)])
+def test_two_cross_entropy_losses_share_logits(dtype, vocab_size, softcap, scaling):
+ from unsloth.kernels.cross_entropy_loss import fast_cross_entropy_loss
+
+ torch.manual_seed(42)
+ hidden = torch.randn(2, 3, 8, device = "cuda")
+ weight = torch.randn(vocab_size, 8, device = "cuda", requires_grad = True)
+ reference_weight = weight.detach().clone().requires_grad_()
+ logits = F.linear(hidden, weight).to(dtype)
+ reference_logits = F.linear(hidden, reference_weight).to(dtype)
+ original_logits = logits.detach().clone()
+ labels_a = torch.tensor([[0, 1, -100], [2, vocab_size - 1, 3]], device = "cuda")
+ labels_b = torch.tensor([[vocab_size - 1, -100, 2], [3, 1, 0]], device = "cuda")
+ assert logits.is_contiguous()
+
+ actual = fast_cross_entropy_loss(
+ logits, labels_a, logit_softcapping = softcap, logit_scaling = scaling
+ ) + 0.7 * fast_cross_entropy_loss(
+ logits, labels_b, logit_softcapping = softcap, logit_scaling = scaling
+ )
+ expected = _reference_loss(
+ reference_logits, labels_a, softcap, scaling
+ ) + 0.7 * _reference_loss(reference_logits, labels_b, softcap, scaling)
+ actual.backward()
+ expected.backward()
+
+ torch.testing.assert_close(actual, expected, rtol = 1e-5, atol = 1e-5)
+ torch.testing.assert_close(
+ weight.grad,
+ reference_weight.grad,
+ rtol = 1e-4 if dtype == torch.float32 else 2e-2,
+ atol = 2e-6 if dtype == torch.float32 else 2e-3,
+ )
+ torch.testing.assert_close(logits, original_logits, rtol = 0, atol = 0)
+
+
[email protected]("dtype", [torch.float32, torch.float16, torch.bfloat16])
[email protected]("vocab_size", [37, 65537])
[email protected]("softcap,scaling", [(0.0, 0.0), (2.0, 0.5)])
[email protected]("use_autograd_grad", [False, True], ids = ["backward", "autograd_grad"])
+def test_cross_entropy_retained_graph(dtype, vocab_size, softcap, scaling, use_autograd_grad):
+ from unsloth.kernels.cross_entropy_loss import fast_cross_entropy_loss
+
+ torch.manual_seed(42)
+ logits = torch.randn(2, 3, vocab_size, device = "cuda", dtype = dtype, requires_grad = True)
+ original_logits = logits.detach().clone()
+ reference_logits = logits.detach().clone().requires_grad_()
+ labels = torch.tensor([[0, 1, -100], [2, vocab_size - 1, 3]], device = "cuda")
+ actual = fast_cross_entropy_loss(
+ logits, labels, logit_softcapping = softcap, logit_scaling = scaling
+ )
+ expected = _reference_loss(reference_logits, labels, softcap, scaling)
+ (reference_grad,) = torch.autograd.grad(expected, reference_logits)
+
+ gradients = []
+ for multiplier in (1.0, 0.3):
+ upstream_grad = torch.full_like(actual, multiplier)
+ if use_autograd_grad:
+ (gradient,) = torch.autograd.grad(actual, logits, upstream_grad, retain_graph = True)
+ else:
+ logits.grad = None
+ actual.backward(upstream_grad, retain_graph = True)
+ gradient = logits.grad
+ gradients.append(gradient)
+ torch.testing.assert_close(
+ gradient,
+ multiplier * reference_grad,
+ rtol = 1e-4 if dtype == torch.float32 else 2e-2,
+ atol = 1e-7 if dtype == torch.float32 else 2e-3,
+ )
+
+ # A later backward must also leave previously returned gradients intact.
+ torch.testing.assert_close(
+ gradients[0],
+ reference_grad,
+ rtol = 1e-4 if dtype == torch.float32 else 2e-2,
+ atol = 1e-7 if dtype == torch.float32 else 2e-3,
+ )
+ torch.testing.assert_close(actual, expected, rtol = 1e-5, atol = 1e-5)
+ torch.testing.assert_close(logits, original_logits, rtol = 0, atol = 0)
diff --git a/unsloth/kernels/cross_entropy_loss.py b/unsloth/kernels/cross_entropy_loss.py
index 8b7f17bb0b4..b9fc9ea54f6 100644
--- a/unsloth/kernels/cross_entropy_loss.py
+++ b/unsloth/kernels/cross_entropy_loss.py
@@ -202,6 +202,7 @@ _chunked_cross_entropy_forward = triton.heuristics(
def _cross_entropy_backward(
logits_ptr,
logits_row_stride: tl.constexpr,
+ dlogits_ptr,
dloss_ptr,
dloss_row_stride: tl.constexpr,
logsumexp_ptr,
@@ -232,6 +233,7 @@ def _cross_entropy_backward(
block_idx = tl.program_id(1)
logits_ptr += row_idx * triton_cast(logits_row_stride, tl.int64)
+ dlogits_ptr += row_idx * triton_cast(logits_row_stride, tl.int64)
dloss_ptr += row_idx * dloss_row_stride
col_offsets = block_idx * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE)
mask = col_offsets < VOCAB_SIZE
@@ -273,7 +275,7 @@ def _cross_entropy_backward(
y = y * (1.0 - partial * partial)
# If y == 0 then dC/dx = 0, and it is already masked to 0, so dloss = 0.
- tl.store(logits_ptr + col_offsets, dloss * y, mask = mask)
+ tl.store(dlogits_ptr + col_offsets, dloss * y, mask = mask)
_cross_entropy_backward = triton.jit(_cross_entropy_backward)
@@ -382,6 +384,8 @@ class Fast_CrossEntropyLoss(torch.autograd.Function):
n_rows: int
vocab_size: int
n_rows, vocab_size = logits.shape
+ # Preserve saved logits for other losses and repeated backward calls.
+ dlogits = torch.empty_like(logits)
BLOCK_SIZE: int = 4096
div: int
@@ -398,6 +402,7 @@ class Fast_CrossEntropyLoss(torch.autograd.Function):
](
logits,
logits.stride(0),
+ dlogits,
dlosses,
dlosses.stride(0),
logsumexp,
@@ -411,7 +416,7 @@ class Fast_CrossEntropyLoss(torch.autograd.Function):
num_warps = 8,
)
return (
- logits,
+ dlogits,
None,
None,
None,