Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Cross entropy: correct gradients under torch.compile for vocabs above 65536 (#14712)

Daniel Hanaf89fcbb70de
diff --git a/tests/test_cross_entropy_compiled_grads.py b/tests/test_cross_entropy_compiled_grads.py
new file mode 100644
index 00000000000..6e94be0868a
--- /dev/null
+++ b/tests/test_cross_entropy_compiled_grads.py
@@ -0,0 +1,42 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.
+
+"""Chunked vocabs (> 65536): in-place ops on the forward output zeroed compiled grads (torch 2.11)."""
+
+import pytest
+import torch
+
+if not torch.cuda.is_available() or torch.version.hip is not None:
+ pytest.skip(
+ "Triton cross entropy kernels under torch.compile need an NVIDIA CUDA device",
+ allow_module_level = True,
+ )
+
+from unsloth.kernels import cross_entropy_loss as ce_mod
+
+
[email protected]("vocab", [32768, 65537, 151936])
[email protected]("fullgraph", [False, True])
+def test_compiled_grads_match_the_fp64_reference(vocab, fullgraph):
+ torch._dynamo.reset()
+ generator = torch.Generator(device = "cuda").manual_seed(vocab)
+ # sm75 (T4) has no bf16 Triton under torch.compile.
+ dtype = torch.bfloat16 if torch.cuda.get_device_capability()[0] >= 8 else torch.float16
+ logits = torch.randn(2, 5, vocab, device = "cuda", generator = generator).to(dtype)
+ labels = torch.randint(0, vocab, (2, 5), device = "cuda", generator = generator)
+ labels[0, 1] = -100
+ reference = logits.double().requires_grad_(True)
+ n_items = (labels != -100).sum()
+ expected = (
+ torch.nn.functional.cross_entropy(
+ reference.flatten(0, 1), labels.flatten(), reduction = "sum"
+ )
+ / n_items
+ )
+ expected.backward()
+ x = logits.clone().requires_grad_(True)
+ loss = torch.compile(ce_mod.fast_cross_entropy_loss, fullgraph = fullgraph)(x, labels)
+ loss.backward()
+ torch.testing.assert_close(loss.double(), expected.detach(), rtol = 1e-4, atol = 1e-4)
+ rel = (x.grad.double() - reference.grad).norm() / reference.grad.norm()
+ assert rel < 1e-2, rel
diff --git a/unsloth/kernels/cross_entropy_loss.py b/unsloth/kernels/cross_entropy_loss.py
index 29ec4fc89c9..b31a5c5c99a 100644
--- a/unsloth/kernels/cross_entropy_loss.py
+++ b/unsloth/kernels/cross_entropy_loss.py
@@ -422,8 +422,7 @@ class Fast_CrossEntropyLoss(torch.autograd.Function):
)
# logsumexp(chunked_logsumexp) - x, with the -x done separately.
logsumexp = torch.logsumexp(logsumexp, dim = 1)
- losses += logsumexp
- losses.masked_fill_(labels == -100, 0) # Don't forget to mask padding out!
+ losses = (losses + logsumexp).masked_fill(labels == -100, 0) # mask padding out
ctx.save_for_backward(logits, logsumexp, labels)
ctx.DO_SOFTCAPPING = DO_SOFTCAPPING