Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
fix: preserve logits in fast cross entropy backward (#12351)
cherry77-cloud0cf426bb2f13
diff --git a/tests/test_cross_entropy_reused_logits.py b/tests/test_cross_entropy_reused_logits.pynew file mode 100644index 00000000000..8e0e3ede7e3--- /dev/null+++ b/tests/test_cross_entropy_reused_logits.py@@ -0,0 +1,105 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++import pytest+from real_accelerator import has_real_cuda+import torch+import torch.nn.functional as F++pytestmark = [+ pytest.mark.gpu,+ pytest.mark.skipif(not has_real_cuda(), reason = "CUDA Triton kernels required"),+]+++def _reference_loss(logits, labels, softcap, scaling):+ transformed = logits.float()+ if scaling:+ transformed = scaling * transformed+ if softcap:+ transformed = softcap * torch.tanh(transformed / softcap)+ return F.cross_entropy(transformed.flatten(0, 1), labels.flatten())++[email protected]("dtype", [torch.float32, torch.float16, torch.bfloat16])[email protected]("vocab_size", [37, 65537])[email protected]("softcap,scaling", [(0.0, 0.0), (2.0, 0.0), (0.0, 0.5), (2.0, 0.5)])+def test_two_cross_entropy_losses_share_logits(dtype, vocab_size, softcap, scaling):+ from unsloth.kernels.cross_entropy_loss import fast_cross_entropy_loss++ torch.manual_seed(42)+ hidden = torch.randn(2, 3, 8, device = "cuda")+ weight = torch.randn(vocab_size, 8, device = "cuda", requires_grad = True)+ reference_weight = weight.detach().clone().requires_grad_()+ logits = F.linear(hidden, weight).to(dtype)+ reference_logits = F.linear(hidden, reference_weight).to(dtype)+ original_logits = logits.detach().clone()+ labels_a = torch.tensor([[0, 1, -100], [2, vocab_size - 1, 3]], device = "cuda")+ labels_b = torch.tensor([[vocab_size - 1, -100, 2], [3, 1, 0]], device = "cuda")+ assert logits.is_contiguous()++ actual = fast_cross_entropy_loss(+ logits, labels_a, logit_softcapping = softcap, logit_scaling = scaling+ ) + 0.7 * fast_cross_entropy_loss(+ logits, labels_b, logit_softcapping = softcap, logit_scaling = scaling+ )+ expected = _reference_loss(+ reference_logits, labels_a, softcap, scaling+ ) + 0.7 * _reference_loss(reference_logits, labels_b, softcap, scaling)+ actual.backward()+ expected.backward()++ torch.testing.assert_close(actual, expected, rtol = 1e-5, atol = 1e-5)+ torch.testing.assert_close(+ weight.grad,+ reference_weight.grad,+ rtol = 1e-4 if dtype == torch.float32 else 2e-2,+ atol = 2e-6 if dtype == torch.float32 else 2e-3,+ )+ torch.testing.assert_close(logits, original_logits, rtol = 0, atol = 0)++[email protected]("dtype", [torch.float32, torch.float16, torch.bfloat16])[email protected]("vocab_size", [37, 65537])[email protected]("softcap,scaling", [(0.0, 0.0), (2.0, 0.5)])[email protected]("use_autograd_grad", [False, True], ids = ["backward", "autograd_grad"])+def test_cross_entropy_retained_graph(dtype, vocab_size, softcap, scaling, use_autograd_grad):+ from unsloth.kernels.cross_entropy_loss import fast_cross_entropy_loss++ torch.manual_seed(42)+ logits = torch.randn(2, 3, vocab_size, device = "cuda", dtype = dtype, requires_grad = True)+ original_logits = logits.detach().clone()+ reference_logits = logits.detach().clone().requires_grad_()+ labels = torch.tensor([[0, 1, -100], [2, vocab_size - 1, 3]], device = "cuda")+ actual = fast_cross_entropy_loss(+ logits, labels, logit_softcapping = softcap, logit_scaling = scaling+ )+ expected = _reference_loss(reference_logits, labels, softcap, scaling)+ (reference_grad,) = torch.autograd.grad(expected, reference_logits)++ gradients = []+ for multiplier in (1.0, 0.3):+ upstream_grad = torch.full_like(actual, multiplier)+ if use_autograd_grad:+ (gradient,) = torch.autograd.grad(actual, logits, upstream_grad, retain_graph = True)+ else:+ logits.grad = None+ actual.backward(upstream_grad, retain_graph = True)+ gradient = logits.grad+ gradients.append(gradient)+ torch.testing.assert_close(+ gradient,+ multiplier * reference_grad,+ rtol = 1e-4 if dtype == torch.float32 else 2e-2,+ atol = 1e-7 if dtype == torch.float32 else 2e-3,+ )++ # A later backward must also leave previously returned gradients intact.+ torch.testing.assert_close(+ gradients[0],+ reference_grad,+ rtol = 1e-4 if dtype == torch.float32 else 2e-2,+ atol = 1e-7 if dtype == torch.float32 else 2e-3,+ )+ torch.testing.assert_close(actual, expected, rtol = 1e-5, atol = 1e-5)+ torch.testing.assert_close(logits, original_logits, rtol = 0, atol = 0)diff --git a/unsloth/kernels/cross_entropy_loss.py b/unsloth/kernels/cross_entropy_loss.pyindex 8b7f17bb0b4..b9fc9ea54f6 100644--- a/unsloth/kernels/cross_entropy_loss.py+++ b/unsloth/kernels/cross_entropy_loss.py@@ -202,6 +202,7 @@ _chunked_cross_entropy_forward = triton.heuristics(def _cross_entropy_backward(logits_ptr,logits_row_stride: tl.constexpr,+ dlogits_ptr,dloss_ptr,dloss_row_stride: tl.constexpr,logsumexp_ptr,@@ -232,6 +233,7 @@ def _cross_entropy_backward(block_idx = tl.program_id(1)logits_ptr += row_idx * triton_cast(logits_row_stride, tl.int64)+ dlogits_ptr += row_idx * triton_cast(logits_row_stride, tl.int64)dloss_ptr += row_idx * dloss_row_stridecol_offsets = block_idx * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE)mask = col_offsets < VOCAB_SIZE@@ -273,7 +275,7 @@ def _cross_entropy_backward(y = y * (1.0 - partial * partial)# If y == 0 then dC/dx = 0, and it is already masked to 0, so dloss = 0.- tl.store(logits_ptr + col_offsets, dloss * y, mask = mask)+ tl.store(dlogits_ptr + col_offsets, dloss * y, mask = mask)_cross_entropy_backward = triton.jit(_cross_entropy_backward)@@ -382,6 +384,8 @@ class Fast_CrossEntropyLoss(torch.autograd.Function):n_rows: intvocab_size: intn_rows, vocab_size = logits.shape+ # Preserve saved logits for other losses and repeated backward calls.+ dlogits = torch.empty_like(logits)BLOCK_SIZE: int = 4096div: int@@ -398,6 +402,7 @@ class Fast_CrossEntropyLoss(torch.autograd.Function):](logits,logits.stride(0),+ dlogits,dlosses,dlosses.stride(0),logsumexp,@@ -411,7 +416,7 @@ class Fast_CrossEntropyLoss(torch.autograd.Function):num_warps = 8,)return (- logits,+ dlogits,None,None,None,