Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

NVFP4 fused dX: skip split K on sm89 when the output already fills the GPU (#14536)

Daniel Han0b124ea117c6
diff --git a/tests/test_nvfp4_fused_gate_shape.py b/tests/test_nvfp4_fused_gate_shape.py
index 42811268562..627ea710d49 100644
--- a/tests/test_nvfp4_fused_gate_shape.py
+++ b/tests/test_nvfp4_fused_gate_shape.py
@@ -127,3 +127,22 @@ def test_degenerate_weights_stay_dense(N, monkeypatch):
monkeypatch.setattr(N, "_device_capability", lambda index: (12, 0))
flat = torch.zeros(4096 * 2048, dtype = torch.uint8)
assert not N._fused_ok(_rows(1, 4096), flat, torch.ones(1, 1), torch.tensor([1.0]), 4096)
+
+
+# (capability, SMs, dx, M, n_out, k_red, split): sm89 dX up to 32 rows skips split K once the output fills every SM.
+SPLIT_CASES = [
+ ((8, 9), 58, True, 1, 14336, 4096, 1), # Llama-3-8B down_proj dX: was 4
+ ((8, 9), 58, True, 32, 12288, 4096, 1), # Qwen3-8B down_proj dX: was 4
+ ((8, 9), 58, True, 64, 14336, 4096, 2), # past 32 rows: unchanged
+ ((8, 9), 58, True, 16, 4096, 14336, 8), # 32 output tiles: still split
+ ((8, 9), 58, False, 16, 4096, 14336, 4), # forward: unchanged
+ ((8, 0), 108, True, 16, 14336, 4096, 4), # A100: split 4 stays fastest
+ ((12, 0), 188, True, 16, 14336, 4096, 8),
+]
+
+
[email protected]("cap, sms, dx, M, n_out, k_red, split", SPLIT_CASES)
+def test_split_k_per_capability(N, monkeypatch, cap, sms, dx, M, n_out, k_red, split):
+ monkeypatch.setattr(N, "_device_capability", lambda index: cap)
+ monkeypatch.setattr(N, "_sm_count", lambda index: sms)
+ assert N._fused_config(M, n_out, k_red, torch.device("cuda", 0), dx)[-1] == split
diff --git a/tests/test_nvfp4_fused_gemm.py b/tests/test_nvfp4_fused_gemm.py
index 4f8b5bd5627..1dd262c3fc2 100644
--- a/tests/test_nvfp4_fused_gemm.py
+++ b/tests/test_nvfp4_fused_gemm.py
@@ -211,6 +211,29 @@ def test_split_reduction_matches_unsplit(N, fused, monkeypatch, dtype, M, N_out,
)
[email protected]("dtype", _dtypes())
[email protected]("M", [1, 32])
+def test_sm89_unsplit_dx_matches_split(N, fused, monkeypatch, dtype, M):
+ # The sm89 dX rule (split 1 once the output fills every SM) against the split it replaced.
+ if torch.cuda.get_device_capability() < (8, 0):
+ pytest.skip("fp16 / bf16 Ampere tiles")
+ packed, scale, gs = _random_nvfp4(512, 14336, seed = 11)
+ G = torch.randn(M, 512, device = "cuda", dtype = dtype)
+ monkeypatch.setattr(N, "_device_capability", lambda index: (8, 9))
+ monkeypatch.setattr(N, "_sm_count", lambda index: 58)
+ assert N._fused_config(M, 14336, 512, G.device, True)[-1] == 1
+ dX = _dx(N, G, packed, scale, gs)
+ old = N._fused_config(M, 14336, 512, G.device, True)[:-1] + (4,)
+ monkeypatch.setattr(N, "_CONFIG_HOOK", lambda *a: old)
+ dX_split = _dx(N, G, packed, scale, gs)
+ W = N.nvfp4_dequantize(packed, scale, gs, dtype)
+ exact = G.double() @ W.double()
+ err_new, err_old = _oracle_errors(dX, dX_split, exact)
+ assert err_new <= 1.25 * err_old + 1e-6, (err_new, err_old)
+ _close_to_dense(dX, dX_split)
+ _close_to_dense(dX, G @ W)
+
+
@pytest.mark.parametrize("dtype", _dtypes())
def test_strided_operands(N, fused, monkeypatch, dtype):
big_packed, big_scale, gs = _random_nvfp4(72, 512, seed = 3)
diff --git a/unsloth/kernels/nvfp4.py b/unsloth/kernels/nvfp4.py
index 7fdfc78b249..0e1e39f273e 100644
--- a/unsloth/kernels/nvfp4.py
+++ b/unsloth/kernels/nvfp4.py
@@ -527,7 +527,8 @@ def _fused_config(
):
"""(BLOCK_M, BLOCK_OUT, BLOCK_RED, num_warps, num_stages, split) for an [M, k_red] @ [k_red, n_out] product."""
index = device.index or 0
- stages = 2 if _device_capability(index) < (8, 0) else 3
+ capability = _device_capability(index)
+ stages = 2 if capability < (8, 0) else 3
if M <= 16:
bm, bo, br, warps = (16, 128, 64, 4) if dx else (16, 64, 128, 4)
elif M <= 64:
@@ -537,6 +538,9 @@ def _fused_config(
else:
bm, bo, br, warps = 128, 64, 64, 4
tiles = triton.cdiv(M, bm) * triton.cdiv(n_out, bo)
+ # sm89 dX up to 32 rows: an output that fills every SM by itself is 11-17% faster unsplit (L4; A100 is not).
+ if dx and M <= 32 and capability == (8, 9) and tiles >= _sm_count(index):
+ return bm, bo, br, warps, stages, 1
# Split under four waves; fp32 partials capped at 1/8 of the 16 bit weight.
split = 1
while (