Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

NF4 GEMV: keep the table load on 256 wide tiles (#14027)

Daniel Han20e341ab7d7c
diff --git a/tests/test_nf4_triton_gemv.py b/tests/test_nf4_triton_gemv.py
index 084c88098d8..9d12519075c 100644
--- a/tests/test_nf4_triton_gemv.py
+++ b/tests/test_nf4_triton_gemv.py
@@ -377,3 +377,33 @@ def test_rocm_never_takes_the_rows_gemv_by_default(monkeypatch):
W, W_quant = utils.get_lora_parameters_bias(proj)[:2]
X = torch.randn(2, 1, 4096, dtype = torch.bfloat16, device = "cuda")
assert utils._gemv_rows(X, W, W_quant, None) is None
+
+
+def test_small_tiles_keep_the_table_load():
+ gather = 2 if nf4_gemv._HAS_TL_GATHER else 0
+ assert nf4_gemv._lut_mode(64) == 0
+ assert nf4_gemv._lut_mode(256) == 0
+ assert nf4_gemv._lut_mode(512) == gather
+ assert nf4_gemv._lut_mode(2048) == gather
+
+
[email protected]("rows", [1, 2, 4])
[email protected]("dtype", DTYPES, ids = str)
[email protected]("shape", [(1000, 192), (2048, 256), (2048, 512), (1536, 8960)], ids = str)
+def test_small_and_odd_k_tiles_match_fp32(shape, dtype, rows, kernel):
+ """K <= 256 (table load), K = 512 (gather) and an odd K = 8960 (masked tail)."""
+ from unsloth.kernels.nf4_gemv import gemv_nf4_rows
+
+ if dtype == torch.bfloat16 and not torch.cuda.is_bf16_supported():
+ pytest.skip("no bf16")
+ n, k = shape
+ q, s = _quant(n, k, dtype)
+ a = _args(q, s)
+ X = torch.randn(rows, 1, k, dtype = dtype, device = "cuda")
+ ref = X.float().view(rows, k) @ F.dequantize_4bit(q, s).float().t()
+ if rows == 1:
+ out = gemv_nf4(X, *a).view(1, n)
+ else:
+ out = torch.empty(rows, 1, n, dtype = dtype, device = "cuda")
+ gemv_nf4_rows(X, *a[:-1], out)
+ assert _rel_err(out.view(rows, n), ref) < TOL[dtype]
diff --git a/unsloth/kernels/nf4_gemv.py b/unsloth/kernels/nf4_gemv.py
index 8ead2fbe968..97c73e6c94c 100644
--- a/unsloth/kernels/nf4_gemv.py
+++ b/unsloth/kernels/nf4_gemv.py
@@ -207,6 +207,11 @@ def _gemv_config(N: int, K: int, blocksize: int, capability, words_ok: bool, for
return False, (-(-N // block_n),), block_n, block_k, num_warps
+def _lut_mode(block_k):
+ # 2 = register tl.gather, 0 = L1 table load; a 256 wide tile is too small to pay for the gather.
+ return 2 if _HAS_TL_GATHER and block_k >= 512 else 0
+
+
@functools.lru_cache(maxsize = None)
def _capability(index):
return torch.cuda.get_device_capability(index)
@@ -264,7 +269,7 @@ def _launch(
if not use_words:
# Even K skips K masks; register gather replaces LUT loads: 16-23% faster (A100/L4/T4, Triton 3.7).
constexprs["EVEN_K"] = K % block_k == 0
- constexprs["LUT_MODE"] = 2 if _HAS_TL_GATHER else 0
+ constexprs["LUT_MODE"] = _lut_mode(block_k)
if kernel is _gemv_nf4_kernel or kernel is _gemv_nf4_words_kernel:
entry = launch(
kernel,