Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
fix: handle strided cross entropy inputs (#10713)
Chenghao Liu9159f4e25fb7
diff --git a/tests/test_cross_entropy_input_layout.py b/tests/test_cross_entropy_input_layout.pynew file mode 100644index 00000000000..b5a7d1f2cbd--- /dev/null+++ b/tests/test_cross_entropy_input_layout.py@@ -0,0 +1,55 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++import pytest+from real_accelerator import (+ has_real_cuda,+) # tests/_shared, on sys.path via tests/conftest.py+import torch+import torch.nn.functional as F++pytestmark = pytest.mark.gpu++[email protected](not has_real_cuda(), reason = "CUDA Triton kernels required")[email protected]("dtype", [torch.float32, torch.float16, torch.bfloat16])[email protected]("vocab_size", [80, 65540])+ "layout",+ [+ "contiguous",+ "logits_columns",+ "logits_slice",+ "logits_transpose",+ "logits_expand",+ "labels_columns",+ "labels_transpose",+ ],+)+def test_cross_entropy_input_layout(layout, vocab_size, dtype):+ from unsloth.kernels.cross_entropy_loss import fast_cross_entropy_loss++ torch.manual_seed(123)+ logits = torch.randn(2, 4, vocab_size, device = "cuda", dtype = dtype)+ labels = torch.randint(vocab_size, (2, 4), device = "cuda")+ if layout == "logits_columns":+ logits = torch.randn(2, 4, 2 * vocab_size, device = "cuda", dtype = dtype)[..., ::2]+ elif layout == "logits_slice":+ logits = torch.randn(2, 8, vocab_size, device = "cuda", dtype = dtype)[:, :4]+ elif layout == "logits_transpose":+ logits = torch.randn(4, 2, vocab_size, device = "cuda", dtype = dtype).transpose(0, 1)+ elif layout == "logits_expand":+ logits = torch.randn(1, 1, vocab_size, device = "cuda", dtype = dtype).expand(2, 4, vocab_size)+ elif layout == "labels_columns":+ labels = torch.randint(vocab_size, (2, 8), device = "cuda")[:, ::2]+ elif layout == "labels_transpose":+ labels = torch.randint(vocab_size, (4, 2), device = "cuda").t()+ labels[0, -1] = -100+ logits.requires_grad_(True)+ reference = logits.detach().clone().requires_grad_(True)+ expected = F.cross_entropy(reference.float().reshape(-1, vocab_size), labels.reshape(-1))+ expected.backward()+ actual = fast_cross_entropy_loss(logits, labels)+ actual.backward()+ torch.testing.assert_close(actual, expected, rtol = 1e-5, atol = 1e-5)+ torch.testing.assert_close(logits.grad, reference.grad, rtol = 2e-3, atol = 1e-5)diff --git a/unsloth/kernels/cross_entropy_loss.py b/unsloth/kernels/cross_entropy_loss.pyindex bdb50fe204b..8b7f17bb0b4 100644--- a/unsloth/kernels/cross_entropy_loss.py+++ b/unsloth/kernels/cross_entropy_loss.py@@ -298,7 +298,8 @@ class Fast_CrossEntropyLoss(torch.autograd.Function):vocab_size: intn_rows, vocab_size = logits.shapedevice = logits.device- labels = labels.to(device)+ logits = logits.contiguous()+ labels = labels.to(device).contiguous()div, mod = divmod(vocab_size, MAX_FUSED_SIZE)n_chunks: int = div + (mod != 0)@@ -436,8 +437,8 @@ def fast_cross_entropy_loss(device = logits.deviceloss = Fast_CrossEntropyLoss.apply(- logits.view(batch * seq_len, d),- labels.view(-1),+ logits.reshape(batch * seq_len, d),+ labels.reshape(-1),logit_softcapping,logit_scaling,)