Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Cross entropy: 8192-wide chunks at 4 warps for the chunked forward on sm120 (#15744)

Daniel Hanfa96c94f3f40
diff --git a/tests/test_cross_entropy_chunk_config.py b/tests/test_cross_entropy_chunk_config.py
index a018d3e646a..094720dda0b 100644
--- a/tests/test_cross_entropy_chunk_config.py
+++ b/tests/test_cross_entropy_chunk_config.py
@@ -46,9 +46,14 @@ def test_b200_uses_8k_blocks_with_4_warps(vocab, monkeypatch):
assert _route((10, 0), monkeypatch, vocab) == (8192, 4)
[email protected]("vocab", [65537, 128256, 151936, 256000, 262144])
+def test_sm120_uses_8k_blocks_with_4_warps(vocab, monkeypatch):
+ assert _route((12, 0), monkeypatch, vocab) == (8192, 4)
+
+
@pytest.mark.parametrize(
"capability",
- [(6, 1), (7, 0), (7, 2), (8, 6), (8, 7), (8, 9), (9, 0), (10, 3), (12, 0)],
+ [(6, 1), (7, 0), (7, 2), (8, 6), (8, 7), (8, 9), (9, 0), (10, 3), (12, 1)],
)
@pytest.mark.parametrize("vocab", [128256, 262144])
def test_other_archs_keep_the_previous_config(capability, vocab, monkeypatch):
diff --git a/tests/test_cross_entropy_sm120_chunk_config.py b/tests/test_cross_entropy_sm120_chunk_config.py
new file mode 100644
index 00000000000..d0e00cd44db
--- /dev/null
+++ b/tests/test_cross_entropy_sm120_chunk_config.py
@@ -0,0 +1,34 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.
+
+"""sm120 (RTX PRO 6000, RTX 5090) routes the chunked CE forward to 8192-wide chunks at 4 warps; runs on CPU."""
+
+import pytest
+import torch
+
+from unsloth.kernels import cross_entropy_loss as ce_mod
+from unsloth.kernels.utils import MAX_FUSED_SIZE
+
+
+def _route(capability, vocab, monkeypatch):
+ monkeypatch.setattr(torch.cuda, "get_device_capability", lambda index = None: capability)
+ monkeypatch.setattr(torch.version, "hip", None)
+ ce_mod._chunked_forward_config_at.cache_clear()
+ try:
+ return ce_mod._chunked_forward_config_at("cuda", 0, vocab)
+ finally:
+ ce_mod._chunked_forward_config_at.cache_clear()
+
+
[email protected]("vocab", [65537, 128256, 151936, 201088, 256000, 262144])
+def test_sm120_uses_8k_blocks_with_4_warps(vocab, monkeypatch):
+ assert _route((12, 0), vocab, monkeypatch) == (8192, 4)
+
+
[email protected]("capability", [(12, 1), (11, 0), (9, 0), (8, 9)])
+def test_neighbouring_archs_keep_the_previous_config(capability, monkeypatch):
+ assert _route(capability, 151936, monkeypatch) == (MAX_FUSED_SIZE, 32)
+
+
+def test_sm120_entry_has_no_vocab_cap():
+ assert ce_mod._CHUNKED_FORWARD_CONFIGS[(12, 0)] == (8192, 4, None)
diff --git a/unsloth/kernels/cross_entropy_loss.py b/unsloth/kernels/cross_entropy_loss.py
index 66fbd590756..8c8ade294ad 100644
--- a/unsloth/kernels/cross_entropy_loss.py
+++ b/unsloth/kernels/cross_entropy_loss.py
@@ -154,14 +154,16 @@ def _chunked_evict_policy(device):
return _chunked_evict_at(device.type, index)
-# (BLOCK_SIZE, num_warps, max vocab) for the chunked forward, measured faster on T4 / A100 / B200.
+# (BLOCK_SIZE, num_warps, max vocab) for the chunked forward, measured faster on T4 / A100 / B200 / sm120.
# A100 at Gemma 262144 was mixed across torch versions, so larger vocabs keep the default.
# B200: 8192-wide chunks at 4 warps run 1.5-1.9x faster than one 65536-wide chunk at 32 warps
# (128256 / 151936 / 262144 vocabs, bf16), since each program then needs a fraction of the registers.
+# sm120 (RTX PRO 6000): the same 8192 / 4 config is 1.09-1.41x faster; 16K and 32K chunks tie with it.
_CHUNKED_FORWARD_CONFIGS = {
(7, 5): (16384, 8, None),
(8, 0): (16384, 8, 163840),
(10, 0): (8192, 4, None),
+ (12, 0): (8192, 4, None),
}