Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Decode small 4bit batches with the NF4 GEMV instead of dequantizing every token (#13642)
Daniel Han26df9508df53
diff --git a/tests/test_bnb_integration_compile.py b/tests/test_bnb_integration_compile.pyindex cfd14991a08..90b73da332d 100644--- a/tests/test_bnb_integration_compile.py+++ b/tests/test_bnb_integration_compile.py@@ -470,7 +470,8 @@ def test_primitives_compile_fullgraph(nf4_kernels):torch._dynamo.reset()[email protected]("bsz", [1, 2])+# 2 and 8 take the multi-row NF4 GEMV, 9 the dequantize path.[email protected]("bsz", [1, 2, 8, 9])def test_fast_linear_forward_compiles_fullgraph(nf4_kernels, bsz):_, block = _lora_block(dtype = CDTYPE)fn = lambda x: U.fast_linear_forward(block.o_proj, x)diff --git a/tests/test_nf4_triton_gemv.py b/tests/test_nf4_triton_gemv.pyindex 7778639f2e3..05191611a35 100644--- a/tests/test_nf4_triton_gemv.py+++ b/tests/test_nf4_triton_gemv.py@@ -167,3 +167,217 @@ def test_decode_plan_is_rebuilt_or_bypassed_when_inputs_change():assert torch.equal(utils.fast_gemv(Xm, q, s), gemv_nf4(Xm.contiguous(), *_args(q, s)))assert getattr(copy.deepcopy(s), "_unsloth_gemv_plan", None) is Noneassert getattr(pickle.loads(pickle.dumps(s)), "_unsloth_gemv_plan", None) is None+++def _linear4bit(+ n,+ k,+ seed = 0,+):+ bnb = pytest.importorskip("bitsandbytes")+ torch.manual_seed(seed)+ proj = bnb.nn.Linear4bit(k, n, bias = False, compute_dtype = torch.bfloat16, quant_type = "nf4")+ W = torch.randn(n, k, dtype = torch.bfloat16)+ proj.weight = bnb.nn.Params4bit(W, requires_grad = False, quant_type = "nf4")+ return proj.to("cuda")++[email protected]("nested", [True, False], ids = ["nested", "flat"])[email protected]("bsz", [2, 3, 8])[email protected]("shape", [(1024, 4096), (4096, 14336), (1000, 192)], ids = str)+def test_rows_kernel_is_the_batch_1_kernel_per_row(shape, bsz, nested, kernel):+ """Strided out, as fast_swiglu_inference passes up[:, :, :hd]."""+ from unsloth.kernels.nf4_gemv import gemv_nf4_rows++ q, s = _quant(*shape, torch.bfloat16, nested = nested)+ a = _args(q, s)+ X = torch.randn(bsz, 1, shape[1], dtype = torch.bfloat16, device = "cuda")+ wide = torch.zeros(bsz, 1, shape[0] + 64, dtype = torch.bfloat16, device = "cuda")+ gemv_nf4_rows(X, *a[:-1], wide[:, :, : shape[0]])+ for i in range(bsz):+ assert torch.equal(wide[i, :, : shape[0]], gemv_nf4(X[i : i + 1].contiguous(), *a)[0]), i+ assert not wide[:, :, shape[0] :].any()++[email protected]("bsz", [2, 3, 4])+def test_small_decode_batches_skip_the_dequantize(bsz, monkeypatch):+ utils = _utils_with_kernels()+ proj = _linear4bit(1024, 4096)+ X = torch.randn(bsz, 1, 4096, dtype = torch.bfloat16, device = "cuda")+ ref = (+ X.float().view(bsz, -1)+ @ F.dequantize_4bit(proj.weight.data, proj.weight.quant_state).float().t()+ )+ monkeypatch.setattr(utils, "_gemv_rows_cap", lambda device: 4)+ monkeypatch.setattr(utils, "fast_dequantize", lambda *a, **k: pytest.fail("dequantized"))+ first = utils.fast_linear_forward(proj, X)+ assert _rel_err(first.view(bsz, -1), ref) < TOL[torch.bfloat16]+ out = torch.empty(bsz, 1, 1024, dtype = torch.bfloat16, device = "cuda")+ for _ in range(3):+ assert torch.equal(utils.fast_linear_forward(proj, X, out = out), first)+ import copy++ assert getattr(copy.deepcopy(proj.weight.quant_state), "_unsloth_gemv_rows", None) is None+++def test_planned_rows_skip_a_strided_out(monkeypatch):+ utils = _utils_with_kernels()+ monkeypatch.setattr(utils, "_gemv_rows_cap", lambda device: 4)+ proj = _linear4bit(1024, 4096)+ X = torch.randn(2, 1, 4096, dtype = torch.bfloat16, device = "cuda")+ ref = utils.fast_linear_forward(proj, X)+ wide = torch.zeros(2, 1, 2048, dtype = torch.bfloat16, device = "cuda")+ utils.fast_linear_forward(proj, X, out = wide[:, :, :1024])+ strided = wide[:, :, ::2]+ assert strided.stride(0) == 2048 and strided.stride(-1) == 2+ assert (+ _rel_err(utils.fast_linear_forward(proj, X, out = strided).float(), ref.float())+ < TOL[torch.bfloat16]+ )+++def test_one_row_into_a_taller_out_writes_only_that_row():+ q, s = _quant(1024, 4096, torch.bfloat16, nested = True)+ X = torch.randn(1, 1, 4096, dtype = torch.bfloat16, device = "cuda")+ tall = torch.zeros(4, 1, 1024, dtype = torch.bfloat16, device = "cuda")+ gemv_nf4(X, *_args(q, s), out = tall)+ torch.cuda.synchronize()+ assert torch.equal(tall[:1], gemv_nf4(X, *_args(q, s)))+ assert not tall[1:].any()+++def test_planned_rows_are_reused_without_an_out(monkeypatch):+ # fast_geglu_inference calls the gate and up projections with out = None.+ utils = _utils_with_kernels()+ monkeypatch.setattr(utils, "_gemv_rows_cap", lambda device: 4)+ proj = _linear4bit(1024, 4096)+ W, W_quant = utils.get_lora_parameters_bias(proj)[:2]+ X = torch.randn(3, 1, 4096, dtype = torch.bfloat16, device = "cuda")+ first = utils._gemv_rows(X, W, W_quant, None)+ monkeypatch.setattr(utils, "_unpack_quant_state", lambda *a: pytest.fail("replanned"))+ again = utils._gemv_rows(X, W, W_quant, None)+ assert again.data_ptr() != first.data_ptr() and torch.equal(again, first)+++def test_rows_gemv_rejects_an_out_it_cannot_fill(monkeypatch):+ utils = _utils_with_kernels()+ monkeypatch.setattr(utils, "_gemv_rows_cap", lambda device: 4)+ proj = _linear4bit(1024, 4096)+ W, W_quant = utils.get_lora_parameters_bias(proj)[:2]+ X = torch.randn(2, 1, 4096, dtype = torch.bfloat16, device = "cuda")+ good = torch.empty(2, 1, 1024, dtype = torch.bfloat16, device = "cuda")+ assert utils._gemv_rows(X, W, W_quant, good) is good+ for bad in (+ torch.empty(2, 1, 1023, dtype = torch.bfloat16, device = "cuda"),+ torch.empty(1, 1, 1024, dtype = torch.bfloat16, device = "cuda"),+ torch.empty(2, 0, 1024, dtype = torch.bfloat16, device = "cuda"),+ torch.empty(2, 2, 1024, dtype = torch.bfloat16, device = "cuda"),+ torch.empty(2048, dtype = torch.bfloat16, device = "cuda").as_strided(+ (2, 1, 1024), (512, 512, 1)+ ),+ ):+ assert utils._gemv_rows(X, W, W_quant, bad) is None+ assert utils._gemv_rows(X, W, W_quant, torch.empty(2, 1, 1024, dtype = torch.bfloat16)) is None+ for wrong_k in (4095, 4097):+ assert (+ utils._gemv_rows(+ torch.randn(2, 1, wrong_k, dtype = torch.bfloat16, device = "cuda"), W, W_quant, None+ )+ is None+ )+++def test_rows_gemv_needs_the_weight_on_the_activation_device(monkeypatch):+ utils = _utils_with_kernels()+ monkeypatch.setattr(utils, "_gemv_rows_cap", lambda device: 4)+ proj = _linear4bit(1024, 4096)+ W, W_quant = utils.get_lora_parameters_bias(proj)[:2]+ X = torch.randn(2, 1, 4096, dtype = torch.bfloat16, device = "cuda")+ # Triton rejects a CPU pointer but not another GPU's: the guard must stop the launch.+ for name in ("gemv_nf4_rows", "gemv_nf4_rows_planned"):+ monkeypatch.setattr(utils, name, lambda *a: pytest.fail("launched"))+ assert utils._gemv_rows(X, W.data.cpu(), W_quant, None) is None+ absmax = W_quant.absmax+ monkeypatch.setattr(W_quant, "absmax", absmax.cpu())+ assert utils._gemv_rows(X, W, W_quant, None) is None+++def test_empty_decode_batch_keeps_the_matmul_path(monkeypatch):+ utils = _utils_with_kernels()+ monkeypatch.setattr(utils, "_gemv_rows_cap", lambda device: 4)+ proj = _linear4bit(1024, 4096)+ monkeypatch.setattr(utils, "_gemv_rows", lambda *a: pytest.fail("rows GEMV for 0 rows"))+ X = torch.randn(0, 1, 4096, dtype = torch.bfloat16, device = "cuda")+ assert utils.fast_linear_forward(proj, X).shape == (0, 1, 1024)+++def test_larger_decode_batches_still_dequantize(monkeypatch):+ utils = _utils_with_kernels()+ proj = _linear4bit(1024, 4096)+ X = torch.randn(utils._GEMV_MAX_ROWS + 1, 1, 4096, dtype = torch.bfloat16, device = "cuda")+ calls = []+ real = utils.fast_dequantize+ monkeypatch.setattr(utils, "fast_dequantize", lambda *a, **k: calls.append(1) or real(*a, **k))+ ref = (+ X.float().view(X.shape[0], -1)+ @ F.dequantize_4bit(proj.weight.data, proj.weight.quant_state).float().t()+ )+ assert (+ _rel_err(utils.fast_linear_forward(proj, X).view(X.shape[0], -1), ref) < TOL[torch.bfloat16]+ )+ assert calls+++ "arch, env, wins",+ [+ ((7, 5), None, {2: True, 3: True, 4: True}),+ ((8, 9), None, {2: True, 4: True}),+ ((12, 0), None, {2: True, 4: True}),+ ((8, 0), None, {2: True, 3: False, 4: False}),+ ((9, 0), None, {2: False, 4: False}),+ ((10, 0), None, {2: False, 4: False}),+ ((10, 0), "3", {2: True, 3: True}),+ ((7, 5), "1", {2: False}),+ ],+)+def test_rows_or_dequantize_is_fixed_per_architecture(monkeypatch, arch, env, wins):+ utils = _utils_with_kernels()+ monkeypatch.setattr(utils, "_GEMV_ROWS_CAP", {})+ if env is None:+ monkeypatch.delenv("UNSLOTH_GEMV_MAX_ROWS", raising = False)+ else:+ monkeypatch.setenv("UNSLOTH_GEMV_MAX_ROWS", env)+ monkeypatch.setattr(utils, "_GEMV_MAX_ROWS", int(env))+ monkeypatch.setattr(torch.version, "hip", None)+ monkeypatch.setattr(torch.cuda, "get_device_capability", lambda *a: arch)+ proj = _linear4bit(1024, 4096)+ W, W_quant = utils.get_lora_parameters_bias(proj)[:2]+ for _ in range(2):+ for bsz, win in wins.items():+ X = torch.randn(bsz, 1, 4096, dtype = torch.bfloat16, device = "cuda")+ assert (utils._gemv_rows(X, W, W_quant, None) is not None) is win+++def test_rows_above_the_cap_skip_straight_to_dequantize(monkeypatch):+ # A100 batch 4 decoded 10% slower when a strided X was unpacked and copied before the cap said no.+ utils = _utils_with_kernels()+ monkeypatch.setattr(utils, "_gemv_rows_cap", lambda device: 2)+ monkeypatch.setattr(utils, "_unpack_quant_state", lambda *a: pytest.fail("unpacked"))+ proj = _linear4bit(1024, 4096)+ W, W_quant = utils.get_lora_parameters_bias(proj)[:2]+ X = torch.randn(4, 1, 8192, dtype = torch.bfloat16, device = "cuda")[:, :, :4096]+ assert not X.is_contiguous()+ assert utils._gemv_rows(X, W, W_quant, None) is None+++def test_rocm_never_takes_the_rows_gemv_by_default(monkeypatch):+ utils = _utils_with_kernels()+ monkeypatch.setattr(utils, "_GEMV_ROWS_CAP", {})+ monkeypatch.delenv("UNSLOTH_GEMV_MAX_ROWS", raising = False)+ monkeypatch.setattr(torch.version, "hip", "6.4")+ monkeypatch.setattr(torch.cuda, "get_device_capability", lambda *a: (12, 0))+ proj = _linear4bit(1024, 4096)+ W, W_quant = utils.get_lora_parameters_bias(proj)[:2]+ X = torch.randn(2, 1, 4096, dtype = torch.bfloat16, device = "cuda")+ assert utils._gemv_rows(X, W, W_quant, None) is Nonediff --git a/unsloth/kernels/nf4_gemv.py b/unsloth/kernels/nf4_gemv.pyindex 813b04696d9..52de268b6b8 100644--- a/unsloth/kernels/nf4_gemv.py+++ b/unsloth/kernels/nf4_gemv.py@@ -8,6 +8,7 @@# Every weight row must start a new quantization block (K % blocksize == 0); callers check.import functools+import osfrom typing import Optionalimport torch@@ -20,6 +21,8 @@ from .triton_launch import launch, relaunch, tag_compile_cache__all__ = ["gemv_nf4","gemv_nf4_planned",+ "gemv_nf4_rows",+ "gemv_nf4_rows_planned","triton_gemv_eager",]@@ -36,6 +39,8 @@ def _gemv_nf4_kernel(OUT,N,K,+ M,+ OUT_STRIDE,BLOCKSIZE: tl.constexpr,BLOCKSIZE2: tl.constexpr,NESTED: tl.constexpr,@@ -45,7 +50,11 @@ def _gemv_nf4_kernel():# FSDP-QLoRA packs the 4bit weight into float storage; it is bytes either way.W = W.to(tl.pointer_type(tl.uint8))+ # A tile's M rows run adjacently so the weight is read once.pid = tl.program_id(0)+ X += (pid % M) * K+ OUT += (pid % M) * OUT_STRIDE+ pid = pid // Mrows = pid * BLOCK_N + tl.arange(0, BLOCK_N)rmask = rows < Nrows64 = rows.to(tl.int64)@@ -97,6 +106,8 @@ def _gemv_nf4_words_kernel(OUT,N,K,+ M,+ OUT_STRIDE,BLOCKSIZE: tl.constexpr,BLOCKSIZE2: tl.constexpr,NESTED: tl.constexpr,@@ -106,7 +117,11 @@ def _gemv_nf4_words_kernel():# _gemv_nf4_kernel reading int32 words of 8 nibbles; faster before Triton 3.7, slower on 3.7.W = W.to(tl.pointer_type(tl.int32))+ # A tile's M rows run adjacently so the weight is read once.pid = tl.program_id(0)+ X += (pid % M) * K+ OUT += (pid % M) * OUT_STRIDE+ pid = pid // Mrows = pid * BLOCK_N + tl.arange(0, BLOCK_N)rmask = rows < Nrows64 = rows.to(tl.int64)@@ -215,6 +230,8 @@ def _launch(# Under dynamic=True the weight sizes arrive as SymInts; specialize them (an unhashable SymInt# in the cached config made Dynamo skip the frame).N, K, blocksize, blocksize2 = int(N), int(K), int(blocksize), int(blocksize2)+ # From X, not out: an out with spare rows must not read past X.+ M = X.numel() // Kuse_words, grid, block_n, block_k, num_warps = _gemv_config(N, K, blocksize, _capability(X.device.index), _word_aligned(W), _FORCE_KERNEL)@@ -230,7 +247,10 @@ def _launch(out,N,K,+ M,+ out.stride(0) if M > 1 else 0,)+ grid = (grid[0] * M,)constexprs = dict(BLOCKSIZE = blocksize,BLOCKSIZE2 = blocksize2 if nested else 1,@@ -391,3 +411,95 @@ def gemv_nf4_planned(int(blocksize2) if code2 is not None else 0,plan = True,)+++# At 8 rows dequantize + matmul won end to end on A100 and RTX PRO 6000.+GEMV_MAX_ROWS = int(os.environ.get("UNSLOTH_GEMV_MAX_ROWS", "4")) # 1 = always dequantize++[email protected]_op("unsloth::gemv_nf4_rows", mutates_args = ())+def _gemv_nf4_rows_op(+ X: torch.Tensor,+ W: torch.Tensor,+ absmax: torch.Tensor,+ code2: Optional[torch.Tensor],+ absmax2: Optional[torch.Tensor],+ offset: Optional[torch.Tensor],+ code: torch.Tensor,+ blocksize: int,+ blocksize2: int,+ out_features: int,+ in_features: int,+) -> torch.Tensor:+ out = torch.empty((X.shape[0], 1, out_features), dtype = X.dtype, device = X.device)+ return _launch(+ (+ torch.library.wrap_triton(_gemv_nf4_kernel),+ torch.library.wrap_triton(_gemv_nf4_words_kernel),+ ),+ X,+ W,+ absmax,+ code2,+ absmax2,+ offset,+ code,+ out,+ out_features,+ in_features,+ blocksize,+ blocksize2,+ )+++def gemv_nf4_rows(X, W_u8, absmax, code2, absmax2, offset, code, blocksize, blocksize2, shape, out):+ """X (M, 1, K) contiguous -> out (M, 1, N), maybe strided; each row bit-identical to gemv_nf4."""+ N, K = int(shape[0]), int(shape[1])+ blocksize, blocksize2 = int(blocksize), int(blocksize2) if code2 is not None else 0+ if _is_compiling():+ result = _gemv_nf4_rows_op(+ X, W_u8, absmax, code2, absmax2, offset, code, blocksize, blocksize2, N, K+ )+ out.copy_(result)+ return out+ args = (+ _KERNELS,+ X,+ W_u8,+ absmax,+ code2,+ absmax2,+ offset,+ code,+ out,+ N,+ K,+ blocksize,+ blocksize2,+ )+ if X.device.index == _current_device():+ return _launch(*args)+ with torch.cuda.device(X.device):+ return _launch(*args)+++def gemv_nf4_rows_planned(+ X, W_u8, absmax, code2, absmax2, offset, code, blocksize, blocksize2, shape, out+):+ """gemv_nf4_planned for gemv_nf4_rows."""+ return _launch(+ _KERNELS,+ X,+ W_u8,+ absmax,+ code2,+ absmax2,+ offset,+ code,+ out,+ int(shape[0]),+ int(shape[1]),+ int(blocksize),+ int(blocksize2) if code2 is not None else 0,+ plan = True,+ )diff --git a/unsloth/kernels/utils.py b/unsloth/kernels/utils.pyindex 49ea13c182b..f9a59a69d0e 100644--- a/unsloth/kernels/utils.py+++ b/unsloth/kernels/utils.py@@ -599,6 +599,7 @@ def _nf4_kernels_supported():# UNSLOTH_BNB_TRITON=0 keeps the bitsandbytes ctypes path._USE_NF4_KERNELS = False_TRITON_GEMV_EAGER = False+_GEMV_MAX_ROWS = 1_is_compiling = torch.compiler.is_compiling_current_device = torch.cuda.current_deviceif (@@ -610,7 +611,14 @@ if (try:import tritonfrom .nf4 import dequantize_nf4, dequantize_nf4_planned- from .nf4_gemv import gemv_nf4, gemv_nf4_planned, triton_gemv_eager+ from .nf4_gemv import GEMV_MAX_ROWS as _GEMV_MAX_ROWS+ from .nf4_gemv import (+ gemv_nf4,+ gemv_nf4_planned,+ gemv_nf4_rows,+ gemv_nf4_rows_planned,+ triton_gemv_eager,+ )_USE_NF4_KERNELS = True# Eager decode uses the Triton GEMV where it is faster; compiled code always does.@@ -1420,6 +1428,132 @@ def _quant_state_dtype(quant_state):return getattr(quant_state, "dtype", None)+# Max GEMV decode rows per compute capability, from end-to-end decode A/Bs (#2798). Fixed, not timed, so+# decoding is reproducible; others dequantize. UNSLOTH_GEMV_MAX_ROWS overrides where the Triton GEMV runs.+_GEMV_ROWS_BY_ARCH = {(7, 5): 4, (8, 0): 2, (8, 9): 4, (12, 0): 4}+_GEMV_ROWS_CAP = {}+++def _gemv_rows_cap(device):+ index = device.index+ cap = _GEMV_ROWS_CAP.get(index)+ if cap is None:+ if "UNSLOTH_GEMV_MAX_ROWS" in os.environ:+ cap = _GEMV_MAX_ROWS+ else:+ try:+ # ROCm reports gfx versions here (gfx1200 is (12, 0)), so the table is CUDA only.+ arch = None if torch.version.hip else torch.cuda.get_device_capability(device)+ except Exception:+ arch = None+ cap = min(_GEMV_MAX_ROWS, _GEMV_ROWS_BY_ARCH.get(arch, 1))+ _GEMV_ROWS_CAP[index] = cap+ return cap+++def _gemv_rows(X, W, quant_state, out):+ """One gemv_nf4_rows launch for 2.._GEMV_MAX_ROWS decode rows (#2798); None = dequantize."""+ M = X.shape[0]+ # First, so rows above the cap cost the dequantize path nothing.+ if not 2 <= M <= _gemv_rows_cap(X.device) or W.device != X.device:+ return None+ eager = (+ not _is_compiling()+ and type(quant_state) is not list+ and X.is_contiguous()+ and X.device.index == _current_device()+ )+ if eager:+ plans = getattr(quant_state, "_unsloth_gemv_rows", None)+ plan = plans.get(M) if plans is not None else None+ if (+ plan+ and plan[0] == W.data_ptr()+ and plan[1] is quant_state.absmax+ and plan[2] is X.dtype+ and X.data_ptr() & 15 == plan[3]+ and X.numel() == M * plan[7]+ and (+ plan[4] == plan[6]+ if out is None+ else (+ out.dtype is X.dtype+ and out.data_ptr() & 15 == 0+ and out.stride(0) == plan[4]+ and out.stride(-1) == 1+ and out.shape == (M, 1, plan[6])+ and out.device == X.device+ )+ )+ ):+ if out is None:+ out = torch_empty((M, 1, plan[6]), dtype = X.dtype, device = X.device)+ try:+ if plan[5](X, out):+ return out+ except Exception:+ pass+ plans.pop(M, None)+ absmax, shape, dtype, blocksize, code, code2, absmax2, offset, blocksize2 = _unpack_quant_state(+ quant_state+ )+ if (+ shape[1] % blocksize != 0+ or X.dtype not in (torch.float16, torch.bfloat16)+ or X.numel() != M * shape[1]+ or any(t is not None and t.device != X.device for t in (absmax, code, code2, absmax2))+ ):+ return None+ if out is not None and (+ out.device != X.device+ or out.dtype != X.dtype+ or out.stride(-1) != 1+ or out.shape != (M, 1, shape[0])+ or out.stride(0) < shape[0]+ ):+ return None+ if not X.is_contiguous():+ X = X.contiguous()+ args = (absmax, code2, absmax2, offset, code, blocksize, blocksize2, shape)+ try:+ if out is None:+ out = torch_empty((M, 1, shape[0]), dtype = X.dtype, device = X.device)+ if not eager or out.data_ptr() & 15:+ return gemv_nf4_rows(X, W, *args, out)+ rerun = gemv_nf4_rows_planned(X, W, *args, out)+ if rerun is not None:+ _rows_plans(quant_state)[M] = _NF4Plan(+ [+ W.data_ptr(),+ quant_state.absmax,+ X.dtype,+ X.data_ptr() & 15,+ out.stride(0),+ rerun,+ int(shape[0]),+ int(shape[1]),+ ]+ )+ return out+ except Exception as error:+ if not _nf4_kernels_failed(error):+ raise+ return None+++class _NF4Plans(dict):+ # Rows -> _NF4Plan; dropped by copies and pickles.+ def __reduce__(self):+ return (_no_plan, ())+++def _rows_plans(quant_state):+ plans = getattr(quant_state, "_unsloth_gemv_rows", None)+ if plans is None:+ plans = quant_state._unsloth_gemv_rows = _NF4Plans()+ return plans++def fast_linear_forward(proj,X,@@ -1458,6 +1592,14 @@ def fast_linear_forward(elif bsz == 1 and q_len == 1 and _quant_state_dtype(W_quant) != torch_float32:# The 4bit gemv kernels are fp16/bf16 only.out = fast_gemv(X, W, W_quant, out = out)+ elif (+ 2 <= bsz <= _GEMV_MAX_ROWS+ and _USE_NF4_KERNELS+ and _TRITON_GEMV_EAGER+ and _quant_state_dtype(W_quant) != torch_float32+ and (rows := _gemv_rows(X, W, W_quant, out)) is not None+ ):+ out = rowselse:W = fast_dequantize(W.t(), W_quant, use_global_buffer = True)out = torch_matmul(X, W, out = out)