Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
NF4 GEMV: keep the table load on 256 wide tiles (#14027)
Daniel Han20e341ab7d7c
diff --git a/tests/test_nf4_triton_gemv.py b/tests/test_nf4_triton_gemv.pyindex 084c88098d8..9d12519075c 100644--- a/tests/test_nf4_triton_gemv.py+++ b/tests/test_nf4_triton_gemv.py@@ -377,3 +377,33 @@ def test_rocm_never_takes_the_rows_gemv_by_default(monkeypatch):W, W_quant = utils.get_lora_parameters_bias(proj)[:2]X = torch.randn(2, 1, 4096, dtype = torch.bfloat16, device = "cuda")assert utils._gemv_rows(X, W, W_quant, None) is None+++def test_small_tiles_keep_the_table_load():+ gather = 2 if nf4_gemv._HAS_TL_GATHER else 0+ assert nf4_gemv._lut_mode(64) == 0+ assert nf4_gemv._lut_mode(256) == 0+ assert nf4_gemv._lut_mode(512) == gather+ assert nf4_gemv._lut_mode(2048) == gather++[email protected]("rows", [1, 2, 4])[email protected]("dtype", DTYPES, ids = str)[email protected]("shape", [(1000, 192), (2048, 256), (2048, 512), (1536, 8960)], ids = str)+def test_small_and_odd_k_tiles_match_fp32(shape, dtype, rows, kernel):+ """K <= 256 (table load), K = 512 (gather) and an odd K = 8960 (masked tail)."""+ from unsloth.kernels.nf4_gemv import gemv_nf4_rows++ if dtype == torch.bfloat16 and not torch.cuda.is_bf16_supported():+ pytest.skip("no bf16")+ n, k = shape+ q, s = _quant(n, k, dtype)+ a = _args(q, s)+ X = torch.randn(rows, 1, k, dtype = dtype, device = "cuda")+ ref = X.float().view(rows, k) @ F.dequantize_4bit(q, s).float().t()+ if rows == 1:+ out = gemv_nf4(X, *a).view(1, n)+ else:+ out = torch.empty(rows, 1, n, dtype = dtype, device = "cuda")+ gemv_nf4_rows(X, *a[:-1], out)+ assert _rel_err(out.view(rows, n), ref) < TOL[dtype]diff --git a/unsloth/kernels/nf4_gemv.py b/unsloth/kernels/nf4_gemv.pyindex 8ead2fbe968..97c73e6c94c 100644--- a/unsloth/kernels/nf4_gemv.py+++ b/unsloth/kernels/nf4_gemv.py@@ -207,6 +207,11 @@ def _gemv_config(N: int, K: int, blocksize: int, capability, words_ok: bool, forreturn False, (-(-N // block_n),), block_n, block_k, num_warps+def _lut_mode(block_k):+ # 2 = register tl.gather, 0 = L1 table load; a 256 wide tile is too small to pay for the gather.+ return 2 if _HAS_TL_GATHER and block_k >= 512 else 0++@functools.lru_cache(maxsize = None)def _capability(index):return torch.cuda.get_device_capability(index)@@ -264,7 +269,7 @@ def _launch(if not use_words:# Even K skips K masks; register gather replaces LUT loads: 16-23% faster (A100/L4/T4, Triton 3.7).constexprs["EVEN_K"] = K % block_k == 0- constexprs["LUT_MODE"] = 2 if _HAS_TL_GATHER else 0+ constexprs["LUT_MODE"] = _lut_mode(block_k)if kernel is _gemv_nf4_kernel or kernel is _gemv_nf4_words_kernel:entry = launch(kernel,