Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Cross entropy: 8192-wide chunks at 4 warps for the chunked forward on B200 (#15141)
Daniel Han6eb93c8955f2
diff --git a/tests/test_cross_entropy_chunk_config.py b/tests/test_cross_entropy_chunk_config.pyindex ff805ee3824..a018d3e646a 100644--- a/tests/test_cross_entropy_chunk_config.py+++ b/tests/test_cross_entropy_chunk_config.py@@ -41,9 +41,14 @@ def test_a100_keeps_the_previous_config_for_larger_vocabs(vocab, monkeypatch):assert _route((8, 0), monkeypatch, vocab) == (MAX_FUSED_SIZE, 32)[email protected]("vocab", [65537, 128256, 151936, 256000, 262144])+def test_b200_uses_8k_blocks_with_4_warps(vocab, monkeypatch):+ assert _route((10, 0), monkeypatch, vocab) == (8192, 4)++@pytest.mark.parametrize("capability",- [(6, 1), (7, 0), (7, 2), (8, 6), (8, 7), (8, 9), (9, 0), (10, 0), (10, 3), (12, 0)],+ [(6, 1), (7, 0), (7, 2), (8, 6), (8, 7), (8, 9), (9, 0), (10, 3), (12, 0)],)@pytest.mark.parametrize("vocab", [128256, 262144])def test_other_archs_keep_the_previous_config(capability, vocab, monkeypatch):@@ -126,11 +131,12 @@ def test_launch_uses_the_routed_config(vocab, monkeypatch):(256000, 0.0, 0.0625),],)-def test_loss_and_grads_match_across_chunk_widths(vocab, softcap, scale, monkeypatch):[email protected]("config", [(16384, 8), (8192, 4)])+def test_loss_and_grads_match_across_chunk_widths(vocab, softcap, scale, config, monkeypatch):_, loss_old, grad_old = _launch(vocab, (MAX_FUSED_SIZE, 32), monkeypatch, softcap = softcap, scale = scale)- _, loss_new, grad_new = _launch(vocab, (16384, 8), monkeypatch, softcap = softcap, scale = scale)+ _, loss_new, grad_new = _launch(vocab, config, monkeypatch, softcap = softcap, scale = scale)torch.testing.assert_close(loss_new, loss_old, rtol = 1e-5, atol = 1e-5)torch.testing.assert_close(grad_new, grad_old, rtol = 2e-2, atol = 2e-3)diff --git a/unsloth/kernels/cross_entropy_loss.py b/unsloth/kernels/cross_entropy_loss.pyindex e2a381320ea..29ec4fc89c9 100644--- a/unsloth/kernels/cross_entropy_loss.py+++ b/unsloth/kernels/cross_entropy_loss.py@@ -128,11 +128,14 @@ def _chunked_evict_policy(device):return _chunked_evict_at(device.type, index)-# (BLOCK_SIZE, num_warps, max vocab) for the chunked forward, measured faster on T4 / A100.+# (BLOCK_SIZE, num_warps, max vocab) for the chunked forward, measured faster on T4 / A100 / B200.# A100 at Gemma 262144 was mixed across torch versions, so larger vocabs keep the default.+# B200: 8192-wide chunks at 4 warps run 1.5-1.9x faster than one 65536-wide chunk at 32 warps+# (128256 / 151936 / 262144 vocabs, bf16), since each program then needs a fraction of the registers._CHUNKED_FORWARD_CONFIGS = {(7, 5): (16384, 8, None),(8, 0): (16384, 8, 163840),+ (10, 0): (8192, 4, None),}