Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Decode small 4bit batches with the NF4 GEMV instead of dequantizing every token (#13642)

Daniel Han26df9508df53
diff --git a/tests/test_bnb_integration_compile.py b/tests/test_bnb_integration_compile.py
index cfd14991a08..90b73da332d 100644
--- a/tests/test_bnb_integration_compile.py
+++ b/tests/test_bnb_integration_compile.py
@@ -470,7 +470,8 @@ def test_primitives_compile_fullgraph(nf4_kernels):
torch._dynamo.reset()
[email protected]("bsz", [1, 2])
+# 2 and 8 take the multi-row NF4 GEMV, 9 the dequantize path.
[email protected]("bsz", [1, 2, 8, 9])
def test_fast_linear_forward_compiles_fullgraph(nf4_kernels, bsz):
_, block = _lora_block(dtype = CDTYPE)
fn = lambda x: U.fast_linear_forward(block.o_proj, x)
diff --git a/tests/test_nf4_triton_gemv.py b/tests/test_nf4_triton_gemv.py
index 7778639f2e3..05191611a35 100644
--- a/tests/test_nf4_triton_gemv.py
+++ b/tests/test_nf4_triton_gemv.py
@@ -167,3 +167,217 @@ def test_decode_plan_is_rebuilt_or_bypassed_when_inputs_change():
assert torch.equal(utils.fast_gemv(Xm, q, s), gemv_nf4(Xm.contiguous(), *_args(q, s)))
assert getattr(copy.deepcopy(s), "_unsloth_gemv_plan", None) is None
assert getattr(pickle.loads(pickle.dumps(s)), "_unsloth_gemv_plan", None) is None
+
+
+def _linear4bit(
+ n,
+ k,
+ seed = 0,
+):
+ bnb = pytest.importorskip("bitsandbytes")
+ torch.manual_seed(seed)
+ proj = bnb.nn.Linear4bit(k, n, bias = False, compute_dtype = torch.bfloat16, quant_type = "nf4")
+ W = torch.randn(n, k, dtype = torch.bfloat16)
+ proj.weight = bnb.nn.Params4bit(W, requires_grad = False, quant_type = "nf4")
+ return proj.to("cuda")
+
+
[email protected]("nested", [True, False], ids = ["nested", "flat"])
[email protected]("bsz", [2, 3, 8])
[email protected]("shape", [(1024, 4096), (4096, 14336), (1000, 192)], ids = str)
+def test_rows_kernel_is_the_batch_1_kernel_per_row(shape, bsz, nested, kernel):
+ """Strided out, as fast_swiglu_inference passes up[:, :, :hd]."""
+ from unsloth.kernels.nf4_gemv import gemv_nf4_rows
+
+ q, s = _quant(*shape, torch.bfloat16, nested = nested)
+ a = _args(q, s)
+ X = torch.randn(bsz, 1, shape[1], dtype = torch.bfloat16, device = "cuda")
+ wide = torch.zeros(bsz, 1, shape[0] + 64, dtype = torch.bfloat16, device = "cuda")
+ gemv_nf4_rows(X, *a[:-1], wide[:, :, : shape[0]])
+ for i in range(bsz):
+ assert torch.equal(wide[i, :, : shape[0]], gemv_nf4(X[i : i + 1].contiguous(), *a)[0]), i
+ assert not wide[:, :, shape[0] :].any()
+
+
[email protected]("bsz", [2, 3, 4])
+def test_small_decode_batches_skip_the_dequantize(bsz, monkeypatch):
+ utils = _utils_with_kernels()
+ proj = _linear4bit(1024, 4096)
+ X = torch.randn(bsz, 1, 4096, dtype = torch.bfloat16, device = "cuda")
+ ref = (
+ X.float().view(bsz, -1)
+ @ F.dequantize_4bit(proj.weight.data, proj.weight.quant_state).float().t()
+ )
+ monkeypatch.setattr(utils, "_gemv_rows_cap", lambda device: 4)
+ monkeypatch.setattr(utils, "fast_dequantize", lambda *a, **k: pytest.fail("dequantized"))
+ first = utils.fast_linear_forward(proj, X)
+ assert _rel_err(first.view(bsz, -1), ref) < TOL[torch.bfloat16]
+ out = torch.empty(bsz, 1, 1024, dtype = torch.bfloat16, device = "cuda")
+ for _ in range(3):
+ assert torch.equal(utils.fast_linear_forward(proj, X, out = out), first)
+ import copy
+
+ assert getattr(copy.deepcopy(proj.weight.quant_state), "_unsloth_gemv_rows", None) is None
+
+
+def test_planned_rows_skip_a_strided_out(monkeypatch):
+ utils = _utils_with_kernels()
+ monkeypatch.setattr(utils, "_gemv_rows_cap", lambda device: 4)
+ proj = _linear4bit(1024, 4096)
+ X = torch.randn(2, 1, 4096, dtype = torch.bfloat16, device = "cuda")
+ ref = utils.fast_linear_forward(proj, X)
+ wide = torch.zeros(2, 1, 2048, dtype = torch.bfloat16, device = "cuda")
+ utils.fast_linear_forward(proj, X, out = wide[:, :, :1024])
+ strided = wide[:, :, ::2]
+ assert strided.stride(0) == 2048 and strided.stride(-1) == 2
+ assert (
+ _rel_err(utils.fast_linear_forward(proj, X, out = strided).float(), ref.float())
+ < TOL[torch.bfloat16]
+ )
+
+
+def test_one_row_into_a_taller_out_writes_only_that_row():
+ q, s = _quant(1024, 4096, torch.bfloat16, nested = True)
+ X = torch.randn(1, 1, 4096, dtype = torch.bfloat16, device = "cuda")
+ tall = torch.zeros(4, 1, 1024, dtype = torch.bfloat16, device = "cuda")
+ gemv_nf4(X, *_args(q, s), out = tall)
+ torch.cuda.synchronize()
+ assert torch.equal(tall[:1], gemv_nf4(X, *_args(q, s)))
+ assert not tall[1:].any()
+
+
+def test_planned_rows_are_reused_without_an_out(monkeypatch):
+ # fast_geglu_inference calls the gate and up projections with out = None.
+ utils = _utils_with_kernels()
+ monkeypatch.setattr(utils, "_gemv_rows_cap", lambda device: 4)
+ proj = _linear4bit(1024, 4096)
+ W, W_quant = utils.get_lora_parameters_bias(proj)[:2]
+ X = torch.randn(3, 1, 4096, dtype = torch.bfloat16, device = "cuda")
+ first = utils._gemv_rows(X, W, W_quant, None)
+ monkeypatch.setattr(utils, "_unpack_quant_state", lambda *a: pytest.fail("replanned"))
+ again = utils._gemv_rows(X, W, W_quant, None)
+ assert again.data_ptr() != first.data_ptr() and torch.equal(again, first)
+
+
+def test_rows_gemv_rejects_an_out_it_cannot_fill(monkeypatch):
+ utils = _utils_with_kernels()
+ monkeypatch.setattr(utils, "_gemv_rows_cap", lambda device: 4)
+ proj = _linear4bit(1024, 4096)
+ W, W_quant = utils.get_lora_parameters_bias(proj)[:2]
+ X = torch.randn(2, 1, 4096, dtype = torch.bfloat16, device = "cuda")
+ good = torch.empty(2, 1, 1024, dtype = torch.bfloat16, device = "cuda")
+ assert utils._gemv_rows(X, W, W_quant, good) is good
+ for bad in (
+ torch.empty(2, 1, 1023, dtype = torch.bfloat16, device = "cuda"),
+ torch.empty(1, 1, 1024, dtype = torch.bfloat16, device = "cuda"),
+ torch.empty(2, 0, 1024, dtype = torch.bfloat16, device = "cuda"),
+ torch.empty(2, 2, 1024, dtype = torch.bfloat16, device = "cuda"),
+ torch.empty(2048, dtype = torch.bfloat16, device = "cuda").as_strided(
+ (2, 1, 1024), (512, 512, 1)
+ ),
+ ):
+ assert utils._gemv_rows(X, W, W_quant, bad) is None
+ assert utils._gemv_rows(X, W, W_quant, torch.empty(2, 1, 1024, dtype = torch.bfloat16)) is None
+ for wrong_k in (4095, 4097):
+ assert (
+ utils._gemv_rows(
+ torch.randn(2, 1, wrong_k, dtype = torch.bfloat16, device = "cuda"), W, W_quant, None
+ )
+ is None
+ )
+
+
+def test_rows_gemv_needs_the_weight_on_the_activation_device(monkeypatch):
+ utils = _utils_with_kernels()
+ monkeypatch.setattr(utils, "_gemv_rows_cap", lambda device: 4)
+ proj = _linear4bit(1024, 4096)
+ W, W_quant = utils.get_lora_parameters_bias(proj)[:2]
+ X = torch.randn(2, 1, 4096, dtype = torch.bfloat16, device = "cuda")
+ # Triton rejects a CPU pointer but not another GPU's: the guard must stop the launch.
+ for name in ("gemv_nf4_rows", "gemv_nf4_rows_planned"):
+ monkeypatch.setattr(utils, name, lambda *a: pytest.fail("launched"))
+ assert utils._gemv_rows(X, W.data.cpu(), W_quant, None) is None
+ absmax = W_quant.absmax
+ monkeypatch.setattr(W_quant, "absmax", absmax.cpu())
+ assert utils._gemv_rows(X, W, W_quant, None) is None
+
+
+def test_empty_decode_batch_keeps_the_matmul_path(monkeypatch):
+ utils = _utils_with_kernels()
+ monkeypatch.setattr(utils, "_gemv_rows_cap", lambda device: 4)
+ proj = _linear4bit(1024, 4096)
+ monkeypatch.setattr(utils, "_gemv_rows", lambda *a: pytest.fail("rows GEMV for 0 rows"))
+ X = torch.randn(0, 1, 4096, dtype = torch.bfloat16, device = "cuda")
+ assert utils.fast_linear_forward(proj, X).shape == (0, 1, 1024)
+
+
+def test_larger_decode_batches_still_dequantize(monkeypatch):
+ utils = _utils_with_kernels()
+ proj = _linear4bit(1024, 4096)
+ X = torch.randn(utils._GEMV_MAX_ROWS + 1, 1, 4096, dtype = torch.bfloat16, device = "cuda")
+ calls = []
+ real = utils.fast_dequantize
+ monkeypatch.setattr(utils, "fast_dequantize", lambda *a, **k: calls.append(1) or real(*a, **k))
+ ref = (
+ X.float().view(X.shape[0], -1)
+ @ F.dequantize_4bit(proj.weight.data, proj.weight.quant_state).float().t()
+ )
+ assert (
+ _rel_err(utils.fast_linear_forward(proj, X).view(X.shape[0], -1), ref) < TOL[torch.bfloat16]
+ )
+ assert calls
+
+
+ "arch, env, wins",
+ [
+ ((7, 5), None, {2: True, 3: True, 4: True}),
+ ((8, 9), None, {2: True, 4: True}),
+ ((12, 0), None, {2: True, 4: True}),
+ ((8, 0), None, {2: True, 3: False, 4: False}),
+ ((9, 0), None, {2: False, 4: False}),
+ ((10, 0), None, {2: False, 4: False}),
+ ((10, 0), "3", {2: True, 3: True}),
+ ((7, 5), "1", {2: False}),
+ ],
+)
+def test_rows_or_dequantize_is_fixed_per_architecture(monkeypatch, arch, env, wins):
+ utils = _utils_with_kernels()
+ monkeypatch.setattr(utils, "_GEMV_ROWS_CAP", {})
+ if env is None:
+ monkeypatch.delenv("UNSLOTH_GEMV_MAX_ROWS", raising = False)
+ else:
+ monkeypatch.setenv("UNSLOTH_GEMV_MAX_ROWS", env)
+ monkeypatch.setattr(utils, "_GEMV_MAX_ROWS", int(env))
+ monkeypatch.setattr(torch.version, "hip", None)
+ monkeypatch.setattr(torch.cuda, "get_device_capability", lambda *a: arch)
+ proj = _linear4bit(1024, 4096)
+ W, W_quant = utils.get_lora_parameters_bias(proj)[:2]
+ for _ in range(2):
+ for bsz, win in wins.items():
+ X = torch.randn(bsz, 1, 4096, dtype = torch.bfloat16, device = "cuda")
+ assert (utils._gemv_rows(X, W, W_quant, None) is not None) is win
+
+
+def test_rows_above_the_cap_skip_straight_to_dequantize(monkeypatch):
+ # A100 batch 4 decoded 10% slower when a strided X was unpacked and copied before the cap said no.
+ utils = _utils_with_kernels()
+ monkeypatch.setattr(utils, "_gemv_rows_cap", lambda device: 2)
+ monkeypatch.setattr(utils, "_unpack_quant_state", lambda *a: pytest.fail("unpacked"))
+ proj = _linear4bit(1024, 4096)
+ W, W_quant = utils.get_lora_parameters_bias(proj)[:2]
+ X = torch.randn(4, 1, 8192, dtype = torch.bfloat16, device = "cuda")[:, :, :4096]
+ assert not X.is_contiguous()
+ assert utils._gemv_rows(X, W, W_quant, None) is None
+
+
+def test_rocm_never_takes_the_rows_gemv_by_default(monkeypatch):
+ utils = _utils_with_kernels()
+ monkeypatch.setattr(utils, "_GEMV_ROWS_CAP", {})
+ monkeypatch.delenv("UNSLOTH_GEMV_MAX_ROWS", raising = False)
+ monkeypatch.setattr(torch.version, "hip", "6.4")
+ monkeypatch.setattr(torch.cuda, "get_device_capability", lambda *a: (12, 0))
+ proj = _linear4bit(1024, 4096)
+ W, W_quant = utils.get_lora_parameters_bias(proj)[:2]
+ X = torch.randn(2, 1, 4096, dtype = torch.bfloat16, device = "cuda")
+ assert utils._gemv_rows(X, W, W_quant, None) is None
diff --git a/unsloth/kernels/nf4_gemv.py b/unsloth/kernels/nf4_gemv.py
index 813b04696d9..52de268b6b8 100644
--- a/unsloth/kernels/nf4_gemv.py
+++ b/unsloth/kernels/nf4_gemv.py
@@ -8,6 +8,7 @@
# Every weight row must start a new quantization block (K % blocksize == 0); callers check.
import functools
+import os
from typing import Optional
import torch
@@ -20,6 +21,8 @@ from .triton_launch import launch, relaunch, tag_compile_cache
__all__ = [
"gemv_nf4",
"gemv_nf4_planned",
+ "gemv_nf4_rows",
+ "gemv_nf4_rows_planned",
"triton_gemv_eager",
]
@@ -36,6 +39,8 @@ def _gemv_nf4_kernel(
OUT,
N,
K,
+ M,
+ OUT_STRIDE,
BLOCKSIZE: tl.constexpr,
BLOCKSIZE2: tl.constexpr,
NESTED: tl.constexpr,
@@ -45,7 +50,11 @@ def _gemv_nf4_kernel(
):
# FSDP-QLoRA packs the 4bit weight into float storage; it is bytes either way.
W = W.to(tl.pointer_type(tl.uint8))
+ # A tile's M rows run adjacently so the weight is read once.
pid = tl.program_id(0)
+ X += (pid % M) * K
+ OUT += (pid % M) * OUT_STRIDE
+ pid = pid // M
rows = pid * BLOCK_N + tl.arange(0, BLOCK_N)
rmask = rows < N
rows64 = rows.to(tl.int64)
@@ -97,6 +106,8 @@ def _gemv_nf4_words_kernel(
OUT,
N,
K,
+ M,
+ OUT_STRIDE,
BLOCKSIZE: tl.constexpr,
BLOCKSIZE2: tl.constexpr,
NESTED: tl.constexpr,
@@ -106,7 +117,11 @@ def _gemv_nf4_words_kernel(
):
# _gemv_nf4_kernel reading int32 words of 8 nibbles; faster before Triton 3.7, slower on 3.7.
W = W.to(tl.pointer_type(tl.int32))
+ # A tile's M rows run adjacently so the weight is read once.
pid = tl.program_id(0)
+ X += (pid % M) * K
+ OUT += (pid % M) * OUT_STRIDE
+ pid = pid // M
rows = pid * BLOCK_N + tl.arange(0, BLOCK_N)
rmask = rows < N
rows64 = rows.to(tl.int64)
@@ -215,6 +230,8 @@ def _launch(
# Under dynamic=True the weight sizes arrive as SymInts; specialize them (an unhashable SymInt
# in the cached config made Dynamo skip the frame).
N, K, blocksize, blocksize2 = int(N), int(K), int(blocksize), int(blocksize2)
+ # From X, not out: an out with spare rows must not read past X.
+ M = X.numel() // K
use_words, grid, block_n, block_k, num_warps = _gemv_config(
N, K, blocksize, _capability(X.device.index), _word_aligned(W), _FORCE_KERNEL
)
@@ -230,7 +247,10 @@ def _launch(
out,
N,
K,
+ M,
+ out.stride(0) if M > 1 else 0,
)
+ grid = (grid[0] * M,)
constexprs = dict(
BLOCKSIZE = blocksize,
BLOCKSIZE2 = blocksize2 if nested else 1,
@@ -391,3 +411,95 @@ def gemv_nf4_planned(
int(blocksize2) if code2 is not None else 0,
plan = True,
)
+
+
+# At 8 rows dequantize + matmul won end to end on A100 and RTX PRO 6000.
+GEMV_MAX_ROWS = int(os.environ.get("UNSLOTH_GEMV_MAX_ROWS", "4")) # 1 = always dequantize
+
+
[email protected]_op("unsloth::gemv_nf4_rows", mutates_args = ())
+def _gemv_nf4_rows_op(
+ X: torch.Tensor,
+ W: torch.Tensor,
+ absmax: torch.Tensor,
+ code2: Optional[torch.Tensor],
+ absmax2: Optional[torch.Tensor],
+ offset: Optional[torch.Tensor],
+ code: torch.Tensor,
+ blocksize: int,
+ blocksize2: int,
+ out_features: int,
+ in_features: int,
+) -> torch.Tensor:
+ out = torch.empty((X.shape[0], 1, out_features), dtype = X.dtype, device = X.device)
+ return _launch(
+ (
+ torch.library.wrap_triton(_gemv_nf4_kernel),
+ torch.library.wrap_triton(_gemv_nf4_words_kernel),
+ ),
+ X,
+ W,
+ absmax,
+ code2,
+ absmax2,
+ offset,
+ code,
+ out,
+ out_features,
+ in_features,
+ blocksize,
+ blocksize2,
+ )
+
+
+def gemv_nf4_rows(X, W_u8, absmax, code2, absmax2, offset, code, blocksize, blocksize2, shape, out):
+ """X (M, 1, K) contiguous -> out (M, 1, N), maybe strided; each row bit-identical to gemv_nf4."""
+ N, K = int(shape[0]), int(shape[1])
+ blocksize, blocksize2 = int(blocksize), int(blocksize2) if code2 is not None else 0
+ if _is_compiling():
+ result = _gemv_nf4_rows_op(
+ X, W_u8, absmax, code2, absmax2, offset, code, blocksize, blocksize2, N, K
+ )
+ out.copy_(result)
+ return out
+ args = (
+ _KERNELS,
+ X,
+ W_u8,
+ absmax,
+ code2,
+ absmax2,
+ offset,
+ code,
+ out,
+ N,
+ K,
+ blocksize,
+ blocksize2,
+ )
+ if X.device.index == _current_device():
+ return _launch(*args)
+ with torch.cuda.device(X.device):
+ return _launch(*args)
+
+
+def gemv_nf4_rows_planned(
+ X, W_u8, absmax, code2, absmax2, offset, code, blocksize, blocksize2, shape, out
+):
+ """gemv_nf4_planned for gemv_nf4_rows."""
+ return _launch(
+ _KERNELS,
+ X,
+ W_u8,
+ absmax,
+ code2,
+ absmax2,
+ offset,
+ code,
+ out,
+ int(shape[0]),
+ int(shape[1]),
+ int(blocksize),
+ int(blocksize2) if code2 is not None else 0,
+ plan = True,
+ )
diff --git a/unsloth/kernels/utils.py b/unsloth/kernels/utils.py
index 49ea13c182b..f9a59a69d0e 100644
--- a/unsloth/kernels/utils.py
+++ b/unsloth/kernels/utils.py
@@ -599,6 +599,7 @@ def _nf4_kernels_supported():
# UNSLOTH_BNB_TRITON=0 keeps the bitsandbytes ctypes path.
_USE_NF4_KERNELS = False
_TRITON_GEMV_EAGER = False
+_GEMV_MAX_ROWS = 1
_is_compiling = torch.compiler.is_compiling
_current_device = torch.cuda.current_device
if (
@@ -610,7 +611,14 @@ if (
try:
import triton
from .nf4 import dequantize_nf4, dequantize_nf4_planned
- from .nf4_gemv import gemv_nf4, gemv_nf4_planned, triton_gemv_eager
+ from .nf4_gemv import GEMV_MAX_ROWS as _GEMV_MAX_ROWS
+ from .nf4_gemv import (
+ gemv_nf4,
+ gemv_nf4_planned,
+ gemv_nf4_rows,
+ gemv_nf4_rows_planned,
+ triton_gemv_eager,
+ )
_USE_NF4_KERNELS = True
# Eager decode uses the Triton GEMV where it is faster; compiled code always does.
@@ -1420,6 +1428,132 @@ def _quant_state_dtype(quant_state):
return getattr(quant_state, "dtype", None)
+# Max GEMV decode rows per compute capability, from end-to-end decode A/Bs (#2798). Fixed, not timed, so
+# decoding is reproducible; others dequantize. UNSLOTH_GEMV_MAX_ROWS overrides where the Triton GEMV runs.
+_GEMV_ROWS_BY_ARCH = {(7, 5): 4, (8, 0): 2, (8, 9): 4, (12, 0): 4}
+_GEMV_ROWS_CAP = {}
+
+
+def _gemv_rows_cap(device):
+ index = device.index
+ cap = _GEMV_ROWS_CAP.get(index)
+ if cap is None:
+ if "UNSLOTH_GEMV_MAX_ROWS" in os.environ:
+ cap = _GEMV_MAX_ROWS
+ else:
+ try:
+ # ROCm reports gfx versions here (gfx1200 is (12, 0)), so the table is CUDA only.
+ arch = None if torch.version.hip else torch.cuda.get_device_capability(device)
+ except Exception:
+ arch = None
+ cap = min(_GEMV_MAX_ROWS, _GEMV_ROWS_BY_ARCH.get(arch, 1))
+ _GEMV_ROWS_CAP[index] = cap
+ return cap
+
+
+def _gemv_rows(X, W, quant_state, out):
+ """One gemv_nf4_rows launch for 2.._GEMV_MAX_ROWS decode rows (#2798); None = dequantize."""
+ M = X.shape[0]
+ # First, so rows above the cap cost the dequantize path nothing.
+ if not 2 <= M <= _gemv_rows_cap(X.device) or W.device != X.device:
+ return None
+ eager = (
+ not _is_compiling()
+ and type(quant_state) is not list
+ and X.is_contiguous()
+ and X.device.index == _current_device()
+ )
+ if eager:
+ plans = getattr(quant_state, "_unsloth_gemv_rows", None)
+ plan = plans.get(M) if plans is not None else None
+ if (
+ plan
+ and plan[0] == W.data_ptr()
+ and plan[1] is quant_state.absmax
+ and plan[2] is X.dtype
+ and X.data_ptr() & 15 == plan[3]
+ and X.numel() == M * plan[7]
+ and (
+ plan[4] == plan[6]
+ if out is None
+ else (
+ out.dtype is X.dtype
+ and out.data_ptr() & 15 == 0
+ and out.stride(0) == plan[4]
+ and out.stride(-1) == 1
+ and out.shape == (M, 1, plan[6])
+ and out.device == X.device
+ )
+ )
+ ):
+ if out is None:
+ out = torch_empty((M, 1, plan[6]), dtype = X.dtype, device = X.device)
+ try:
+ if plan[5](X, out):
+ return out
+ except Exception:
+ pass
+ plans.pop(M, None)
+ absmax, shape, dtype, blocksize, code, code2, absmax2, offset, blocksize2 = _unpack_quant_state(
+ quant_state
+ )
+ if (
+ shape[1] % blocksize != 0
+ or X.dtype not in (torch.float16, torch.bfloat16)
+ or X.numel() != M * shape[1]
+ or any(t is not None and t.device != X.device for t in (absmax, code, code2, absmax2))
+ ):
+ return None
+ if out is not None and (
+ out.device != X.device
+ or out.dtype != X.dtype
+ or out.stride(-1) != 1
+ or out.shape != (M, 1, shape[0])
+ or out.stride(0) < shape[0]
+ ):
+ return None
+ if not X.is_contiguous():
+ X = X.contiguous()
+ args = (absmax, code2, absmax2, offset, code, blocksize, blocksize2, shape)
+ try:
+ if out is None:
+ out = torch_empty((M, 1, shape[0]), dtype = X.dtype, device = X.device)
+ if not eager or out.data_ptr() & 15:
+ return gemv_nf4_rows(X, W, *args, out)
+ rerun = gemv_nf4_rows_planned(X, W, *args, out)
+ if rerun is not None:
+ _rows_plans(quant_state)[M] = _NF4Plan(
+ [
+ W.data_ptr(),
+ quant_state.absmax,
+ X.dtype,
+ X.data_ptr() & 15,
+ out.stride(0),
+ rerun,
+ int(shape[0]),
+ int(shape[1]),
+ ]
+ )
+ return out
+ except Exception as error:
+ if not _nf4_kernels_failed(error):
+ raise
+ return None
+
+
+class _NF4Plans(dict):
+ # Rows -> _NF4Plan; dropped by copies and pickles.
+ def __reduce__(self):
+ return (_no_plan, ())
+
+
+def _rows_plans(quant_state):
+ plans = getattr(quant_state, "_unsloth_gemv_rows", None)
+ if plans is None:
+ plans = quant_state._unsloth_gemv_rows = _NF4Plans()
+ return plans
+
+
def fast_linear_forward(
proj,
X,
@@ -1458,6 +1592,14 @@ def fast_linear_forward(
elif bsz == 1 and q_len == 1 and _quant_state_dtype(W_quant) != torch_float32:
# The 4bit gemv kernels are fp16/bf16 only.
out = fast_gemv(X, W, W_quant, out = out)
+ elif (
+ 2 <= bsz <= _GEMV_MAX_ROWS
+ and _USE_NF4_KERNELS
+ and _TRITON_GEMV_EAGER
+ and _quant_state_dtype(W_quant) != torch_float32
+ and (rows := _gemv_rows(X, W, W_quant, out)) is not None
+ ):
+ out = rows
else:
W = fast_dequantize(W.t(), W_quant, use_global_buffer = True)
out = torch_matmul(X, W, out = out)