Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
NF4 GEMV: drop K masks on even K and gather the NF4 table from registers (#13829)
Daniel Han2ce16b551e38
diff --git a/unsloth/kernels/nf4_gemv.py b/unsloth/kernels/nf4_gemv.pyindex 82ba41b5662..8ead2fbe968 100644--- a/unsloth/kernels/nf4_gemv.py+++ b/unsloth/kernels/nf4_gemv.py@@ -15,7 +15,7 @@ import torchimport tritonimport triton.language as tl-from .nf4 import _HAS_MUL_RN, _mul+from .nf4 import _HAS_MUL_RN, _HAS_TL_GATHER, _mul, _nf4_lutfrom .triton_launch import launch, relaunch, tag_compile_cache__all__ = [@@ -47,6 +47,8 @@ def _gemv_nf4_kernel(USE_MUL_RN: tl.constexpr,BLOCK_N: tl.constexpr,BLOCK_K: tl.constexpr,+ EVEN_K: tl.constexpr,+ LUT_MODE: tl.constexpr,):# FSDP-QLoRA packs the 4bit weight into float storage; it is bytes either way.W = W.to(tl.pointer_type(tl.uint8))@@ -68,18 +70,18 @@ def _gemv_nf4_kernel(if NESTED:offset = tl.load(OFFSET)for k0 in range(0, K, BLOCK_K):- kmask = (k0 + 2 * half_k) < K+ kmask = ((k0 + 2 * half_k) < K) | EVEN_Km = rmask[:, None] & kmask[None, :]b = tl.load(W + rows64[:, None] * (K // 2) + (k0 // 2 + half_k)[None, :], mask = m, other = 0)b = b.to(tl.int32)# High nibble is the earlier element, as bitsandbytes packs it.- hi = tl.load(LUT + (b >> 4))- lo = tl.load(LUT + (b & 15))+ hi = _nf4_lut(LUT, b >> 4, LUT_MODE)+ lo = _nf4_lut(LUT, b & 15, LUT_MODE)w = tl.reshape(tl.join(hi, lo), (BLOCK_N, BLOCK_K))xk = k0 + ks- x = tl.load(X + xk, mask = xk < K, other = 0.0).to(tl.float32)+ x = tl.load(X + xk, mask = (xk < K) | EVEN_K, other = 0.0).to(tl.float32)part = tl.sum(tl.reshape(w * x[None, :], (BLOCK_N, NB, BLOCKSIZE)), axis = 2)- bmask = rmask[:, None] & ((k0 + nbs * BLOCKSIZE) < K)[None, :]+ bmask = rmask[:, None] & (((k0 + nbs * BLOCKSIZE) < K) | EVEN_K)[None, :]blk = rows64[:, None] * blocks_per_row + (k0 // BLOCKSIZE + nbs)[None, :]if NESTED:q = tl.load(ABSMAX + blk, mask = bmask, other = 0).to(tl.int32)@@ -259,6 +261,10 @@ def _launch(BLOCK_N = block_n,BLOCK_K = block_k,)+ if not use_words:+ # Even K skips K masks; register gather replaces LUT loads: 16-23% faster (A100/L4/T4, Triton 3.7).+ constexprs["EVEN_K"] = K % block_k == 0+ constexprs["LUT_MODE"] = 2 if _HAS_TL_GATHER else 0if kernel is _gemv_nf4_kernel or kernel is _gemv_nf4_words_kernel:entry = launch(kernel,