Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

FP8 rowwise decode: one multi-row GEMV launch for 2 to 16 tokens (#14791)

Daniel Han444dcbf0ebff
diff --git a/tests/test_compressed_tensors_fp8_kernel_route.py b/tests/test_compressed_tensors_fp8_kernel_route.py
index 37473ba7f70..940b5005a40 100644
--- a/tests/test_compressed_tensors_fp8_kernel_route.py
+++ b/tests/test_compressed_tensors_fp8_kernel_route.py
@@ -286,7 +286,7 @@ def test_decode_gemv_refuses_what_it_cannot_compute():
with torch.no_grad():
assert not can_use_fp8_rowwise_gemv(X, W, torch.ones(16, 128, device = dev))
assert not can_use_fp8_rowwise_gemv(
- torch.randn(2, 16384, device = dev, dtype = torch.bfloat16), W, s
+ torch.randn(17, 16384, device = dev, dtype = torch.bfloat16), W, s
)
assert not can_use_fp8_rowwise_gemv(X.float(), W, s)
assert not can_use_fp8_rowwise_gemv(X[:, :2048], W.t(), s)
diff --git a/tests/test_fp8_rowwise_gemv_rows.py b/tests/test_fp8_rowwise_gemv_rows.py
new file mode 100644
index 00000000000..58f3147fc87
--- /dev/null
+++ b/tests/test_fp8_rowwise_gemv_rows.py
@@ -0,0 +1,154 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""Per-channel FP8 GEMV for 2..16 decode rows: routing, accuracy against fp64, split-K, kill switch."""
+
+import os
+import sys
+from contextlib import nullcontext
+from pathlib import Path
+
+import pytest
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
+
+torch = pytest.importorskip("torch")
+pytest.importorskip("triton")
+
+CUDA = torch.cuda.is_available()
+FP8_GPU = CUDA and torch.version.hip is None and torch.cuda.get_device_capability() >= (8, 9)
+needs_fp8_gpu = pytest.mark.skipif(not FP8_GPU, reason = "needs an sm89+ NVIDIA GPU")
+
+
[email protected](scope = "module")
+def F():
+ os.environ.setdefault("UNSLOTH_IS_PRESENT", "1")
+ from unsloth.kernels import fp8
+ return fp8
+
+
+def _weights(N, K, seed = 0):
+ g = torch.Generator(device = "cuda").manual_seed(seed)
+ W = torch.randn(N, K, device = "cuda", generator = g) * 0.02
+ scale = (W.abs().amax(1, keepdim = True) / 448.0).float()
+ return (W / scale).to(torch.float8_e4m3fn), scale
+
+
+class _Recorder:
+ def __init__(self):
+ self.calls = []
+
+ def __getitem__(self, grid):
+ def launch(*args, **kwargs):
+ self.calls.append((grid, args, kwargs))
+
+ return launch
+
+
+ "N, K, expected",
+ [(4096, 4096, False), (2**31 // 4096, 4096, False), (2**31 // 4096 + 1, 4096, True)],
+)
+def test_rows_launch_widens_past_int32(F, monkeypatch, N, K, expected):
+ rec = _Recorder()
+ monkeypatch.setattr(F, "_fp8_rowwise_gemv_rows_kernel", rec)
+ monkeypatch.setattr(F, "_fp8_triton_device_context", lambda *a, **k: nullcontext())
+ monkeypatch.setattr(F, "_FP8_ROWS_CONFIG_OVERRIDE", (32, 128, 1, 4, 4))
+ X = torch.empty(4, 1, K, device = "meta", dtype = torch.bfloat16)
+ W = torch.empty(N, K, device = "meta", dtype = torch.float8_e4m3fn)
+ S = torch.empty(N, 1, device = "meta", dtype = torch.float32)
+ F.fp8_rowwise_gemv(X, W, S)
+ assert len(rec.calls) == 1
+ assert rec.calls[0][2]["LONG_INDEXING"] is expected
+
+
[email protected]("dtype, N, K, split", [
+ (torch.bfloat16, 4096, 12288, 8), # long K (down_proj)
+ (torch.float16, 1024, 4096, 8), # narrow N (kv_proj)
+ (torch.bfloat16, 4096, 4096, 1),
+ (torch.float16, 12288, 4096, 1),
+ (torch.bfloat16, 512, 2048, 1),
+])
+def test_split_k_rule(F, dtype, N, K, split):
+ assert F._fp8_rows_config(N, K, dtype)[2] == split
+
+
+@needs_fp8_gpu
+def test_routing(F, monkeypatch):
+ Wq, scale = _weights(1024, 512)
+ for M, ok in ((1, True), (2, True), (16, True), (17, False)):
+ X = torch.randn(M, 1, 512, device = "cuda", dtype = torch.bfloat16)
+ assert F.can_use_fp8_rowwise_gemv(X, Wq, scale) is ok, M
+ X = torch.randn(4, 1, 512, device = "cuda", dtype = torch.bfloat16, requires_grad = True)
+ assert not F.can_use_fp8_rowwise_gemv(X, Wq, scale)
+ blocks = torch.ones(8, 4, device = "cuda")
+ assert not F.can_use_fp8_rowwise_gemv(X.detach(), Wq, blocks)
+ monkeypatch.setattr(F, "_FP8_GEMV_ROWS", False)
+ assert not F.can_use_fp8_rowwise_gemv(X.detach(), Wq, scale)
+ assert F.can_use_fp8_rowwise_gemv(X.detach()[:1], Wq, scale)
+
+
+@needs_fp8_gpu
+def test_one_row_keeps_the_single_row_kernel(F, monkeypatch):
+ def boom(*a, **k):
+ raise AssertionError("rows kernel used for one row")
+
+ monkeypatch.setattr(F, "_fp8_rowwise_gemv_rows", boom)
+ Wq, scale = _weights(512, 256)
+ X = torch.randn(1, 1, 256, device = "cuda", dtype = torch.bfloat16)
+ F.fp8_rowwise_gemv(X, Wq, scale)
+
+
+@needs_fp8_gpu
[email protected]("dtype", [torch.bfloat16, torch.float16])
[email protected]("M", [2, 3, 8, 16])
[email protected]("N, K", [(1024, 4096), (4096, 4096), (200, 320)])
[email protected]("cfg", [None, (16, 128, 8, 4, 4), (32, 256, 1, 4, 3), (64, 128, 2, 4, 4)])
+def test_rows_accuracy(F, monkeypatch, dtype, M, N, K, cfg):
+ monkeypatch.setattr(F, "_FP8_ROWS_CONFIG_OVERRIDE", cfg)
+ Wq, scale = _weights(N, K)
+ X = torch.randn(M, 1, K, device = "cuda", dtype = dtype)
+ Wref = Wq.double() * scale.double()
+ ref = (X.double().reshape(M, K) @ Wref.t()).reshape(M, 1, N)
+ y = F.fp8_rowwise_gemv(X, Wq, scale)
+ assert y.shape == (M, 1, N) and y.dtype == dtype
+ # The dequantize-then-matmul path rounds W to 16 bits first; ours reads the exact fp8 values.
+ dq = X @ (Wq.float() * scale).to(dtype).t()
+ err, err_dq = (y.double() - ref).abs(), (dq.double() - ref).abs()
+ ulp = torch.finfo(dtype).eps * ref.abs().max().item()
+ assert err.max().item() <= err_dq.max().item() + ulp
+ assert err.mean().item() <= err_dq.mean().item() * 1.05
+ # Fixed reduction order: repeat calls are bitwise equal (split-K sums partials, no atomics).
+ assert torch.equal(y, F.fp8_rowwise_gemv(X, Wq, scale))
+
+
+@needs_fp8_gpu
+def test_rows_through_fp8_linear_callers(F):
+ from unsloth.kernels.utils import fast_linear_forward
+
+ class P(torch.nn.Module):
+ def __init__(self, Wq, scale):
+ super().__init__()
+ self.weight = torch.nn.Parameter(Wq, requires_grad = False)
+ self.weight_scale = scale
+ self.bias = None
+
+ def forward(self, x):
+ raise AssertionError("not routed")
+
+ Wq, scale = _weights(1024, 512)
+ X = torch.randn(4, 1, 512, device = "cuda", dtype = torch.bfloat16)
+ with torch.no_grad():
+ out = fast_linear_forward(P(Wq, scale), X)
+ ref = X.double() @ (Wq.double() * scale.double()).t()
+ assert (out.double() - ref).abs().max().item() < 0.05 * ref.abs().max().item()
+
+
+@needs_fp8_gpu
+def test_rows_off_where_triton_miscompiles_the_gemv(F, monkeypatch):
+ # Blackwell + Triton < 3.4: only the exact one-row BLOCK_N 1 launch is trusted there.
+ monkeypatch.setattr(F, "_fp8_gemv_miscompiles", lambda device: True)
+ monkeypatch.setattr(F, "_triton_builds_blackwell", lambda: True)
+ Wq, scale = _weights(1024, 512)
+ assert not F.can_use_fp8_rowwise_gemv(torch.randn(4, 1, 512, device = "cuda", dtype = torch.bfloat16), Wq, scale)
+ assert F.can_use_fp8_rowwise_gemv(torch.randn(1, 1, 512, device = "cuda", dtype = torch.bfloat16), Wq, scale)
diff --git a/unsloth/kernels/fp8.py b/unsloth/kernels/fp8.py
index 6254c6fbbc0..88e82acabdf 100644
--- a/unsloth/kernels/fp8.py
+++ b/unsloth/kernels/fp8.py
@@ -579,12 +579,130 @@ def _fp8_rowwise_gemv_kernel(
tl.store(y_ptr + offs_n, y.to(y_ptr.dtype.element_ty), mask = offs_n < N)
+# Batched decode (2..16 rows): each weight tile is read once for every row, no 16-bit weight copy.
+# UNSLOTH_FP8_GEMV_ROWS=0 restores the dequantize path.
+_FP8_GEMV_MAX_ROWS = 16
+_FP8_GEMV_ROWS = os.environ.get("UNSLOTH_FP8_GEMV_ROWS", "1").strip().lower() not in ("0", "false", "no", "off")
+
+
[email protected](do_not_specialize = ["M"])
+def _fp8_rowwise_gemv_rows_kernel(
+ x_ptr,
+ w_ptr,
+ s_ptr,
+ y_ptr,
+ p_ptr,
+ M,
+ N,
+ K,
+ stride_wn,
+ BLOCK_M: tl.constexpr,
+ BLOCK_N: tl.constexpr,
+ BLOCK_K: tl.constexpr,
+ SPLIT_K: tl.constexpr,
+ LONG_INDEXING: tl.constexpr = False,
+):
+ pid = tl.program_id(0)
+ pid_k = tl.program_id(1)
+ if LONG_INDEXING:
+ pid = pid.to(tl.int64)
+ offs_m = tl.arange(0, BLOCK_M)
+ offs_n = pid * BLOCK_N + tl.arange(0, BLOCK_N)
+ acc = tl.zeros((BLOCK_M, BLOCK_N), dtype = tl.float32)
+ k_per = tl.cdiv(tl.cdiv(K, SPLIT_K), BLOCK_K) * BLOCK_K
+ k_lo = pid_k * k_per
+ for k0 in range(k_lo, tl.minimum(k_lo + k_per, K), BLOCK_K):
+ offs_k = k0 + tl.arange(0, BLOCK_K)
+ x = tl.load(
+ x_ptr + offs_m[:, None] * K + offs_k[None, :],
+ mask = (offs_m[:, None] < M) & (offs_k[None, :] < K),
+ other = 0.0,
+ )
+ # e4m3 values are exact in bf16 / fp16, so the dot sees the stored weights unrounded.
+ w = tl.load(
+ w_ptr + offs_n[:, None] * stride_wn + offs_k[None, :],
+ mask = (offs_n[:, None] < N) & (offs_k[None, :] < K),
+ other = 0.0,
+ ).to(x.dtype)
+ acc = tl.dot(x, tl.trans(w), acc, input_precision = "ieee", out_dtype = tl.float32)
+ mask = (offs_m[:, None] < M) & (offs_n[None, :] < N)
+ if SPLIT_K == 1:
+ s = tl.load(s_ptr + offs_n, mask = offs_n < N, other = 0.0).to(tl.float32)
+ y = acc * s[None, :]
+ tl.store(y_ptr + offs_m[:, None] * N + offs_n[None, :], y.to(y_ptr.dtype.element_ty), mask = mask)
+ else:
+ # fp32 partials, summed in a fixed order by the reduce kernel: deterministic, no atomics.
+ tl.store(p_ptr + (pid_k * M + offs_m[:, None]) * N + offs_n[None, :], acc, mask = mask)
+
+
[email protected](do_not_specialize = ["M"])
+def _fp8_rowwise_gemv_rows_reduce(p_ptr, s_ptr, y_ptr, M, N, SPLIT_K: tl.constexpr, BLOCK: tl.constexpr):
+ offs = tl.program_id(0) * BLOCK + tl.arange(0, BLOCK)
+ mask = offs < M * N
+ acc = tl.zeros((BLOCK,), dtype = tl.float32)
+ for k in tl.static_range(SPLIT_K):
+ acc += tl.load(p_ptr + k * M * N + offs, mask = mask, other = 0.0)
+ s = tl.load(s_ptr + offs % N, mask = mask, other = 0.0).to(tl.float32)
+ tl.store(y_ptr + offs, (acc * s).to(y_ptr.dtype.element_ty), mask = mask)
+
+
+def _fp8_rows_config(N, K, dtype):
+ """(BLOCK_N, BLOCK_K, SPLIT_K, num_warps, num_stages) from a sweep; split K only for long K or narrow N."""
+ if _FP8_ROWS_CONFIG_OVERRIDE is not None:
+ return _FP8_ROWS_CONFIG_OVERRIDE
+ if K >= 8192:
+ return (64 if dtype == torch.float16 else 32), 128, 8, 4, 4
+ if N <= 1024 and K >= 4096:
+ return 32, 128, 8, 4, 4
+ block_n = 16 if N <= 2048 else (64 if N >= 12288 and dtype == torch.float16 else 32)
+ return block_n, 256, 1, 4, 4
+
+
+_FP8_ROWS_CONFIG_OVERRIDE = None
+
+
+def _fp8_rowwise_gemv_rows(X, weight, scale, out, rows):
+ N, K = weight.shape
+ block_n, block_k, split_k, warps, stages = _fp8_rows_config(N, K, X.dtype)
+ partials = (
+ torch.empty((split_k, rows, N), device = X.device, dtype = torch.float32) if split_k > 1 else out
+ )
+ with _fp8_triton_device_context(X, static_device_count = True):
+ _fp8_rowwise_gemv_rows_kernel[(triton.cdiv(N, block_n), split_k)](
+ X,
+ weight,
+ scale,
+ out,
+ partials,
+ rows,
+ N,
+ K,
+ weight.stride(0),
+ BLOCK_M = _FP8_GEMV_MAX_ROWS,
+ BLOCK_N = block_n,
+ BLOCK_K = block_k,
+ SPLIT_K = split_k,
+ LONG_INDEXING = long_indexing(weight),
+ num_warps = warps,
+ num_stages = stages,
+ enable_fp_fusion = True,
+ )
+ if split_k > 1:
+ _fp8_rowwise_gemv_rows_reduce[(triton.cdiv(rows * N, 1024),)](
+ partials, scale, out, rows, N, SPLIT_K = split_k, BLOCK = 1024, enable_fp_fusion = True
+ )
+ return out
+
+
def can_use_fp8_rowwise_gemv(X, weight, weight_scale):
if not (X.is_cuda and weight.is_cuda and X.dtype in (torch.bfloat16, torch.float16)):
return False
if weight.dtype != torch.float8_e4m3fn or weight.dim() != 2 or not weight.is_contiguous():
return False
- if X.numel() != X.shape[-1] or X.shape[-1] != weight.shape[1]:
+ if X.shape[-1] != weight.shape[1]:
+ return False
+ rows = X.numel() // X.shape[-1]
+ if rows != 1 and not (2 <= rows <= _FP8_GEMV_MAX_ROWS and _FP8_GEMV_ROWS):
return False
# Exactly one scale per output row: a block grid can have N elements by coincidence.
if (
@@ -598,6 +716,8 @@ def can_use_fp8_rowwise_gemv(X, weight, weight_scale):
return False
if not _fp8_gemv_miscompiles(weight.device):
return True
+ if rows != 1:
+ return False
# Triton 3.3 is exact at BLOCK_N 1 (N <= 4096); beats dequant + matmul when N >= 256 or K <= 4096 (RTX PRO 6000).
N, K = weight.shape
return N <= 4096 and (N >= 256 or K <= 4096) and _triton_builds_blackwell()
@@ -613,12 +733,15 @@ def _triton_builds_blackwell():
def fp8_rowwise_gemv(X, weight, weight_scale):
- """X @ (weight * weight_scale).T for one token without a 16-bit weight copy."""
+ """X @ (weight * weight_scale).T for up to _FP8_GEMV_MAX_ROWS tokens without a 16-bit weight copy."""
N, K = weight.shape
out = torch.empty((*X.shape[:-1], N), device = X.device, dtype = X.dtype)
if not X.is_contiguous():
X = X.contiguous()
scale = weight_scale if weight_scale.is_contiguous() else weight_scale.contiguous()
+ rows = X.numel() // K
+ if rows != 1:
+ return _fp8_rowwise_gemv_rows(X, weight, scale, out, rows)
block_n = 1 if N <= 4096 else (4 if N <= 32768 else 8)
with _fp8_triton_device_context(X, static_device_count = True):
_fp8_rowwise_gemv_kernel[(triton.cdiv(N, block_n),)](