Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Cross entropy: correct gradients under torch.compile for vocabs above 65536 (#14712)
Daniel Hanaf89fcbb70de
diff --git a/tests/test_cross_entropy_compiled_grads.py b/tests/test_cross_entropy_compiled_grads.pynew file mode 100644index 00000000000..6e94be0868a--- /dev/null+++ b/tests/test_cross_entropy_compiled_grads.py@@ -0,0 +1,42 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.++"""Chunked vocabs (> 65536): in-place ops on the forward output zeroed compiled grads (torch 2.11)."""++import pytest+import torch++if not torch.cuda.is_available() or torch.version.hip is not None:+ pytest.skip(+ "Triton cross entropy kernels under torch.compile need an NVIDIA CUDA device",+ allow_module_level = True,+ )++from unsloth.kernels import cross_entropy_loss as ce_mod++[email protected]("vocab", [32768, 65537, 151936])[email protected]("fullgraph", [False, True])+def test_compiled_grads_match_the_fp64_reference(vocab, fullgraph):+ torch._dynamo.reset()+ generator = torch.Generator(device = "cuda").manual_seed(vocab)+ # sm75 (T4) has no bf16 Triton under torch.compile.+ dtype = torch.bfloat16 if torch.cuda.get_device_capability()[0] >= 8 else torch.float16+ logits = torch.randn(2, 5, vocab, device = "cuda", generator = generator).to(dtype)+ labels = torch.randint(0, vocab, (2, 5), device = "cuda", generator = generator)+ labels[0, 1] = -100+ reference = logits.double().requires_grad_(True)+ n_items = (labels != -100).sum()+ expected = (+ torch.nn.functional.cross_entropy(+ reference.flatten(0, 1), labels.flatten(), reduction = "sum"+ )+ / n_items+ )+ expected.backward()+ x = logits.clone().requires_grad_(True)+ loss = torch.compile(ce_mod.fast_cross_entropy_loss, fullgraph = fullgraph)(x, labels)+ loss.backward()+ torch.testing.assert_close(loss.double(), expected.detach(), rtol = 1e-4, atol = 1e-4)+ rel = (x.grad.double() - reference.grad).norm() / reference.grad.norm()+ assert rel < 1e-2, reldiff --git a/unsloth/kernels/cross_entropy_loss.py b/unsloth/kernels/cross_entropy_loss.pyindex 29ec4fc89c9..b31a5c5c99a 100644--- a/unsloth/kernels/cross_entropy_loss.py+++ b/unsloth/kernels/cross_entropy_loss.py@@ -422,8 +422,7 @@ class Fast_CrossEntropyLoss(torch.autograd.Function):)# logsumexp(chunked_logsumexp) - x, with the -x done separately.logsumexp = torch.logsumexp(logsumexp, dim = 1)- losses += logsumexp- losses.masked_fill_(labels == -100, 0) # Don't forget to mask padding out!+ losses = (losses + logsumexp).masked_fill(labels == -100, 0) # mask padding outctx.save_for_backward(logits, logsumexp, labels)ctx.DO_SOFTCAPPING = DO_SOFTCAPPING