Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

NF4 GEMV: drop K masks on even K and gather the NF4 table from registers (#13829)

Daniel Han2ce16b551e38
diff --git a/unsloth/kernels/nf4_gemv.py b/unsloth/kernels/nf4_gemv.py
index 82ba41b5662..8ead2fbe968 100644
--- a/unsloth/kernels/nf4_gemv.py
+++ b/unsloth/kernels/nf4_gemv.py
@@ -15,7 +15,7 @@ import torch
import triton
import triton.language as tl
-from .nf4 import _HAS_MUL_RN, _mul
+from .nf4 import _HAS_MUL_RN, _HAS_TL_GATHER, _mul, _nf4_lut
from .triton_launch import launch, relaunch, tag_compile_cache
__all__ = [
@@ -47,6 +47,8 @@ def _gemv_nf4_kernel(
USE_MUL_RN: tl.constexpr,
BLOCK_N: tl.constexpr,
BLOCK_K: tl.constexpr,
+ EVEN_K: tl.constexpr,
+ LUT_MODE: tl.constexpr,
):
# FSDP-QLoRA packs the 4bit weight into float storage; it is bytes either way.
W = W.to(tl.pointer_type(tl.uint8))
@@ -68,18 +70,18 @@ def _gemv_nf4_kernel(
if NESTED:
offset = tl.load(OFFSET)
for k0 in range(0, K, BLOCK_K):
- kmask = (k0 + 2 * half_k) < K
+ kmask = ((k0 + 2 * half_k) < K) | EVEN_K
m = rmask[:, None] & kmask[None, :]
b = tl.load(W + rows64[:, None] * (K // 2) + (k0 // 2 + half_k)[None, :], mask = m, other = 0)
b = b.to(tl.int32)
# High nibble is the earlier element, as bitsandbytes packs it.
- hi = tl.load(LUT + (b >> 4))
- lo = tl.load(LUT + (b & 15))
+ hi = _nf4_lut(LUT, b >> 4, LUT_MODE)
+ lo = _nf4_lut(LUT, b & 15, LUT_MODE)
w = tl.reshape(tl.join(hi, lo), (BLOCK_N, BLOCK_K))
xk = k0 + ks
- x = tl.load(X + xk, mask = xk < K, other = 0.0).to(tl.float32)
+ x = tl.load(X + xk, mask = (xk < K) | EVEN_K, other = 0.0).to(tl.float32)
part = tl.sum(tl.reshape(w * x[None, :], (BLOCK_N, NB, BLOCKSIZE)), axis = 2)
- bmask = rmask[:, None] & ((k0 + nbs * BLOCKSIZE) < K)[None, :]
+ bmask = rmask[:, None] & (((k0 + nbs * BLOCKSIZE) < K) | EVEN_K)[None, :]
blk = rows64[:, None] * blocks_per_row + (k0 // BLOCKSIZE + nbs)[None, :]
if NESTED:
q = tl.load(ABSMAX + blk, mask = bmask, other = 0).to(tl.int32)
@@ -259,6 +261,10 @@ def _launch(
BLOCK_N = block_n,
BLOCK_K = block_k,
)
+ if not use_words:
+ # Even K skips K masks; register gather replaces LUT loads: 16-23% faster (A100/L4/T4, Triton 3.7).
+ constexprs["EVEN_K"] = K % block_k == 0
+ constexprs["LUT_MODE"] = 2 if _HAS_TL_GATHER else 0
if kernel is _gemv_nf4_kernel or kernel is _gemv_nf4_words_kernel:
entry = launch(
kernel,