Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Cross entropy: 8192-wide chunks at 4 warps for the chunked forward on sm120 (#15744)
Daniel Hanfa96c94f3f40
diff --git a/tests/test_cross_entropy_chunk_config.py b/tests/test_cross_entropy_chunk_config.pyindex a018d3e646a..094720dda0b 100644--- a/tests/test_cross_entropy_chunk_config.py+++ b/tests/test_cross_entropy_chunk_config.py@@ -46,9 +46,14 @@ def test_b200_uses_8k_blocks_with_4_warps(vocab, monkeypatch):assert _route((10, 0), monkeypatch, vocab) == (8192, 4)[email protected]("vocab", [65537, 128256, 151936, 256000, 262144])+def test_sm120_uses_8k_blocks_with_4_warps(vocab, monkeypatch):+ assert _route((12, 0), monkeypatch, vocab) == (8192, 4)++@pytest.mark.parametrize("capability",- [(6, 1), (7, 0), (7, 2), (8, 6), (8, 7), (8, 9), (9, 0), (10, 3), (12, 0)],+ [(6, 1), (7, 0), (7, 2), (8, 6), (8, 7), (8, 9), (9, 0), (10, 3), (12, 1)],)@pytest.mark.parametrize("vocab", [128256, 262144])def test_other_archs_keep_the_previous_config(capability, vocab, monkeypatch):diff --git a/tests/test_cross_entropy_sm120_chunk_config.py b/tests/test_cross_entropy_sm120_chunk_config.pynew file mode 100644index 00000000000..d0e00cd44db--- /dev/null+++ b/tests/test_cross_entropy_sm120_chunk_config.py@@ -0,0 +1,34 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.++"""sm120 (RTX PRO 6000, RTX 5090) routes the chunked CE forward to 8192-wide chunks at 4 warps; runs on CPU."""++import pytest+import torch++from unsloth.kernels import cross_entropy_loss as ce_mod+from unsloth.kernels.utils import MAX_FUSED_SIZE+++def _route(capability, vocab, monkeypatch):+ monkeypatch.setattr(torch.cuda, "get_device_capability", lambda index = None: capability)+ monkeypatch.setattr(torch.version, "hip", None)+ ce_mod._chunked_forward_config_at.cache_clear()+ try:+ return ce_mod._chunked_forward_config_at("cuda", 0, vocab)+ finally:+ ce_mod._chunked_forward_config_at.cache_clear()++[email protected]("vocab", [65537, 128256, 151936, 201088, 256000, 262144])+def test_sm120_uses_8k_blocks_with_4_warps(vocab, monkeypatch):+ assert _route((12, 0), vocab, monkeypatch) == (8192, 4)++[email protected]("capability", [(12, 1), (11, 0), (9, 0), (8, 9)])+def test_neighbouring_archs_keep_the_previous_config(capability, monkeypatch):+ assert _route(capability, 151936, monkeypatch) == (MAX_FUSED_SIZE, 32)+++def test_sm120_entry_has_no_vocab_cap():+ assert ce_mod._CHUNKED_FORWARD_CONFIGS[(12, 0)] == (8192, 4, None)diff --git a/unsloth/kernels/cross_entropy_loss.py b/unsloth/kernels/cross_entropy_loss.pyindex 66fbd590756..8c8ade294ad 100644--- a/unsloth/kernels/cross_entropy_loss.py+++ b/unsloth/kernels/cross_entropy_loss.py@@ -154,14 +154,16 @@ def _chunked_evict_policy(device):return _chunked_evict_at(device.type, index)-# (BLOCK_SIZE, num_warps, max vocab) for the chunked forward, measured faster on T4 / A100 / B200.+# (BLOCK_SIZE, num_warps, max vocab) for the chunked forward, measured faster on T4 / A100 / B200 / sm120.# A100 at Gemma 262144 was mixed across torch versions, so larger vocabs keep the default.# B200: 8192-wide chunks at 4 warps run 1.5-1.9x faster than one 65536-wide chunk at 32 warps# (128256 / 151936 / 262144 vocabs, bf16), since each program then needs a fraction of the registers.+# sm120 (RTX PRO 6000): the same 8192 / 4 config is 1.09-1.41x faster; 16K and 32K chunks tie with it._CHUNKED_FORWARD_CONFIGS = {(7, 5): (16384, 8, None),(8, 0): (16384, 8, 163840),(10, 0): (8192, 4, None),+ (12, 0): (8192, 4, None),}