Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Cross entropy: 8192-wide chunks at 4 warps for the chunked forward on B200 (#15141)

Daniel Han6eb93c8955f2
diff --git a/tests/test_cross_entropy_chunk_config.py b/tests/test_cross_entropy_chunk_config.py
index ff805ee3824..a018d3e646a 100644
--- a/tests/test_cross_entropy_chunk_config.py
+++ b/tests/test_cross_entropy_chunk_config.py
@@ -41,9 +41,14 @@ def test_a100_keeps_the_previous_config_for_larger_vocabs(vocab, monkeypatch):
assert _route((8, 0), monkeypatch, vocab) == (MAX_FUSED_SIZE, 32)
[email protected]("vocab", [65537, 128256, 151936, 256000, 262144])
+def test_b200_uses_8k_blocks_with_4_warps(vocab, monkeypatch):
+ assert _route((10, 0), monkeypatch, vocab) == (8192, 4)
+
+
@pytest.mark.parametrize(
"capability",
- [(6, 1), (7, 0), (7, 2), (8, 6), (8, 7), (8, 9), (9, 0), (10, 0), (10, 3), (12, 0)],
+ [(6, 1), (7, 0), (7, 2), (8, 6), (8, 7), (8, 9), (9, 0), (10, 3), (12, 0)],
)
@pytest.mark.parametrize("vocab", [128256, 262144])
def test_other_archs_keep_the_previous_config(capability, vocab, monkeypatch):
@@ -126,11 +131,12 @@ def test_launch_uses_the_routed_config(vocab, monkeypatch):
(256000, 0.0, 0.0625),
],
)
-def test_loss_and_grads_match_across_chunk_widths(vocab, softcap, scale, monkeypatch):
[email protected]("config", [(16384, 8), (8192, 4)])
+def test_loss_and_grads_match_across_chunk_widths(vocab, softcap, scale, config, monkeypatch):
_, loss_old, grad_old = _launch(
vocab, (MAX_FUSED_SIZE, 32), monkeypatch, softcap = softcap, scale = scale
)
- _, loss_new, grad_new = _launch(vocab, (16384, 8), monkeypatch, softcap = softcap, scale = scale)
+ _, loss_new, grad_new = _launch(vocab, config, monkeypatch, softcap = softcap, scale = scale)
torch.testing.assert_close(loss_new, loss_old, rtol = 1e-5, atol = 1e-5)
torch.testing.assert_close(grad_new, grad_old, rtol = 2e-2, atol = 2e-3)
diff --git a/unsloth/kernels/cross_entropy_loss.py b/unsloth/kernels/cross_entropy_loss.py
index e2a381320ea..29ec4fc89c9 100644
--- a/unsloth/kernels/cross_entropy_loss.py
+++ b/unsloth/kernels/cross_entropy_loss.py
@@ -128,11 +128,14 @@ def _chunked_evict_policy(device):
return _chunked_evict_at(device.type, index)
-# (BLOCK_SIZE, num_warps, max vocab) for the chunked forward, measured faster on T4 / A100.
+# (BLOCK_SIZE, num_warps, max vocab) for the chunked forward, measured faster on T4 / A100 / B200.
# A100 at Gemma 262144 was mixed across torch versions, so larger vocabs keep the default.
+# B200: 8192-wide chunks at 4 warps run 1.5-1.9x faster than one 65536-wide chunk at 32 warps
+# (128256 / 151936 / 262144 vocabs, bf16), since each program then needs a fraction of the registers.
_CHUNKED_FORWARD_CONFIGS = {
(7, 5): (16384, 8, None),
(8, 0): (16384, 8, 163840),
+ (10, 0): (8192, 4, None),
}