Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

fix: handle strided cross entropy inputs (#10713)

Chenghao Liu9159f4e25fb7
diff --git a/tests/test_cross_entropy_input_layout.py b/tests/test_cross_entropy_input_layout.py
new file mode 100644
index 00000000000..b5a7d1f2cbd
--- /dev/null
+++ b/tests/test_cross_entropy_input_layout.py
@@ -0,0 +1,55 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+import pytest
+from real_accelerator import (
+ has_real_cuda,
+) # tests/_shared, on sys.path via tests/conftest.py
+import torch
+import torch.nn.functional as F
+
+pytestmark = pytest.mark.gpu
+
+
[email protected](not has_real_cuda(), reason = "CUDA Triton kernels required")
[email protected]("dtype", [torch.float32, torch.float16, torch.bfloat16])
[email protected]("vocab_size", [80, 65540])
+ "layout",
+ [
+ "contiguous",
+ "logits_columns",
+ "logits_slice",
+ "logits_transpose",
+ "logits_expand",
+ "labels_columns",
+ "labels_transpose",
+ ],
+)
+def test_cross_entropy_input_layout(layout, vocab_size, dtype):
+ from unsloth.kernels.cross_entropy_loss import fast_cross_entropy_loss
+
+ torch.manual_seed(123)
+ logits = torch.randn(2, 4, vocab_size, device = "cuda", dtype = dtype)
+ labels = torch.randint(vocab_size, (2, 4), device = "cuda")
+ if layout == "logits_columns":
+ logits = torch.randn(2, 4, 2 * vocab_size, device = "cuda", dtype = dtype)[..., ::2]
+ elif layout == "logits_slice":
+ logits = torch.randn(2, 8, vocab_size, device = "cuda", dtype = dtype)[:, :4]
+ elif layout == "logits_transpose":
+ logits = torch.randn(4, 2, vocab_size, device = "cuda", dtype = dtype).transpose(0, 1)
+ elif layout == "logits_expand":
+ logits = torch.randn(1, 1, vocab_size, device = "cuda", dtype = dtype).expand(2, 4, vocab_size)
+ elif layout == "labels_columns":
+ labels = torch.randint(vocab_size, (2, 8), device = "cuda")[:, ::2]
+ elif layout == "labels_transpose":
+ labels = torch.randint(vocab_size, (4, 2), device = "cuda").t()
+ labels[0, -1] = -100
+ logits.requires_grad_(True)
+ reference = logits.detach().clone().requires_grad_(True)
+ expected = F.cross_entropy(reference.float().reshape(-1, vocab_size), labels.reshape(-1))
+ expected.backward()
+ actual = fast_cross_entropy_loss(logits, labels)
+ actual.backward()
+ torch.testing.assert_close(actual, expected, rtol = 1e-5, atol = 1e-5)
+ torch.testing.assert_close(logits.grad, reference.grad, rtol = 2e-3, atol = 1e-5)
diff --git a/unsloth/kernels/cross_entropy_loss.py b/unsloth/kernels/cross_entropy_loss.py
index bdb50fe204b..8b7f17bb0b4 100644
--- a/unsloth/kernels/cross_entropy_loss.py
+++ b/unsloth/kernels/cross_entropy_loss.py
@@ -298,7 +298,8 @@ class Fast_CrossEntropyLoss(torch.autograd.Function):
vocab_size: int
n_rows, vocab_size = logits.shape
device = logits.device
- labels = labels.to(device)
+ logits = logits.contiguous()
+ labels = labels.to(device).contiguous()
div, mod = divmod(vocab_size, MAX_FUSED_SIZE)
n_chunks: int = div + (mod != 0)
@@ -436,8 +437,8 @@ def fast_cross_entropy_loss(
device = logits.device
loss = Fast_CrossEntropyLoss.apply(
- logits.view(batch * seq_len, d),
- labels.view(-1),
+ logits.reshape(batch * seq_len, d),
+ labels.reshape(-1),
logit_softcapping,
logit_scaling,
)