Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
NVFP4 fused dX: skip split K on sm89 when the output already fills the GPU (#14536)
Daniel Han0b124ea117c6
diff --git a/tests/test_nvfp4_fused_gate_shape.py b/tests/test_nvfp4_fused_gate_shape.pyindex 42811268562..627ea710d49 100644--- a/tests/test_nvfp4_fused_gate_shape.py+++ b/tests/test_nvfp4_fused_gate_shape.py@@ -127,3 +127,22 @@ def test_degenerate_weights_stay_dense(N, monkeypatch):monkeypatch.setattr(N, "_device_capability", lambda index: (12, 0))flat = torch.zeros(4096 * 2048, dtype = torch.uint8)assert not N._fused_ok(_rows(1, 4096), flat, torch.ones(1, 1), torch.tensor([1.0]), 4096)+++# (capability, SMs, dx, M, n_out, k_red, split): sm89 dX up to 32 rows skips split K once the output fills every SM.+SPLIT_CASES = [+ ((8, 9), 58, True, 1, 14336, 4096, 1), # Llama-3-8B down_proj dX: was 4+ ((8, 9), 58, True, 32, 12288, 4096, 1), # Qwen3-8B down_proj dX: was 4+ ((8, 9), 58, True, 64, 14336, 4096, 2), # past 32 rows: unchanged+ ((8, 9), 58, True, 16, 4096, 14336, 8), # 32 output tiles: still split+ ((8, 9), 58, False, 16, 4096, 14336, 4), # forward: unchanged+ ((8, 0), 108, True, 16, 14336, 4096, 4), # A100: split 4 stays fastest+ ((12, 0), 188, True, 16, 14336, 4096, 8),+]++[email protected]("cap, sms, dx, M, n_out, k_red, split", SPLIT_CASES)+def test_split_k_per_capability(N, monkeypatch, cap, sms, dx, M, n_out, k_red, split):+ monkeypatch.setattr(N, "_device_capability", lambda index: cap)+ monkeypatch.setattr(N, "_sm_count", lambda index: sms)+ assert N._fused_config(M, n_out, k_red, torch.device("cuda", 0), dx)[-1] == splitdiff --git a/tests/test_nvfp4_fused_gemm.py b/tests/test_nvfp4_fused_gemm.pyindex 4f8b5bd5627..1dd262c3fc2 100644--- a/tests/test_nvfp4_fused_gemm.py+++ b/tests/test_nvfp4_fused_gemm.py@@ -211,6 +211,29 @@ def test_split_reduction_matches_unsplit(N, fused, monkeypatch, dtype, M, N_out,)[email protected]("dtype", _dtypes())[email protected]("M", [1, 32])+def test_sm89_unsplit_dx_matches_split(N, fused, monkeypatch, dtype, M):+ # The sm89 dX rule (split 1 once the output fills every SM) against the split it replaced.+ if torch.cuda.get_device_capability() < (8, 0):+ pytest.skip("fp16 / bf16 Ampere tiles")+ packed, scale, gs = _random_nvfp4(512, 14336, seed = 11)+ G = torch.randn(M, 512, device = "cuda", dtype = dtype)+ monkeypatch.setattr(N, "_device_capability", lambda index: (8, 9))+ monkeypatch.setattr(N, "_sm_count", lambda index: 58)+ assert N._fused_config(M, 14336, 512, G.device, True)[-1] == 1+ dX = _dx(N, G, packed, scale, gs)+ old = N._fused_config(M, 14336, 512, G.device, True)[:-1] + (4,)+ monkeypatch.setattr(N, "_CONFIG_HOOK", lambda *a: old)+ dX_split = _dx(N, G, packed, scale, gs)+ W = N.nvfp4_dequantize(packed, scale, gs, dtype)+ exact = G.double() @ W.double()+ err_new, err_old = _oracle_errors(dX, dX_split, exact)+ assert err_new <= 1.25 * err_old + 1e-6, (err_new, err_old)+ _close_to_dense(dX, dX_split)+ _close_to_dense(dX, G @ W)++@pytest.mark.parametrize("dtype", _dtypes())def test_strided_operands(N, fused, monkeypatch, dtype):big_packed, big_scale, gs = _random_nvfp4(72, 512, seed = 3)diff --git a/unsloth/kernels/nvfp4.py b/unsloth/kernels/nvfp4.pyindex 7fdfc78b249..0e1e39f273e 100644--- a/unsloth/kernels/nvfp4.py+++ b/unsloth/kernels/nvfp4.py@@ -527,7 +527,8 @@ def _fused_config():"""(BLOCK_M, BLOCK_OUT, BLOCK_RED, num_warps, num_stages, split) for an [M, k_red] @ [k_red, n_out] product."""index = device.index or 0- stages = 2 if _device_capability(index) < (8, 0) else 3+ capability = _device_capability(index)+ stages = 2 if capability < (8, 0) else 3if M <= 16:bm, bo, br, warps = (16, 128, 64, 4) if dx else (16, 64, 128, 4)elif M <= 64:@@ -537,6 +538,9 @@ def _fused_config(else:bm, bo, br, warps = 128, 64, 64, 4tiles = triton.cdiv(M, bm) * triton.cdiv(n_out, bo)+ # sm89 dX up to 32 rows: an output that fills every SM by itself is 11-17% faster unsplit (L4; A100 is not).+ if dx and M <= 32 and capability == (8, 9) and tiles >= _sm_count(index):+ return bm, bo, br, warps, stages, 1# Split under four waves; fp32 partials capped at 1/8 of the 16 bit weight.split = 1while (