Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
FP8 rowwise decode: one multi-row GEMV launch for 2 to 16 tokens (#14791)
Daniel Han444dcbf0ebff
diff --git a/tests/test_compressed_tensors_fp8_kernel_route.py b/tests/test_compressed_tensors_fp8_kernel_route.pyindex 37473ba7f70..940b5005a40 100644--- a/tests/test_compressed_tensors_fp8_kernel_route.py+++ b/tests/test_compressed_tensors_fp8_kernel_route.py@@ -286,7 +286,7 @@ def test_decode_gemv_refuses_what_it_cannot_compute():with torch.no_grad():assert not can_use_fp8_rowwise_gemv(X, W, torch.ones(16, 128, device = dev))assert not can_use_fp8_rowwise_gemv(- torch.randn(2, 16384, device = dev, dtype = torch.bfloat16), W, s+ torch.randn(17, 16384, device = dev, dtype = torch.bfloat16), W, s)assert not can_use_fp8_rowwise_gemv(X.float(), W, s)assert not can_use_fp8_rowwise_gemv(X[:, :2048], W.t(), s)diff --git a/tests/test_fp8_rowwise_gemv_rows.py b/tests/test_fp8_rowwise_gemv_rows.pynew file mode 100644index 00000000000..58f3147fc87--- /dev/null+++ b/tests/test_fp8_rowwise_gemv_rows.py@@ -0,0 +1,154 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""Per-channel FP8 GEMV for 2..16 decode rows: routing, accuracy against fp64, split-K, kill switch."""++import os+import sys+from contextlib import nullcontext+from pathlib import Path++import pytest++sys.path.insert(0, str(Path(__file__).resolve().parents[1]))++torch = pytest.importorskip("torch")+pytest.importorskip("triton")++CUDA = torch.cuda.is_available()+FP8_GPU = CUDA and torch.version.hip is None and torch.cuda.get_device_capability() >= (8, 9)+needs_fp8_gpu = pytest.mark.skipif(not FP8_GPU, reason = "needs an sm89+ NVIDIA GPU")++[email protected](scope = "module")+def F():+ os.environ.setdefault("UNSLOTH_IS_PRESENT", "1")+ from unsloth.kernels import fp8+ return fp8+++def _weights(N, K, seed = 0):+ g = torch.Generator(device = "cuda").manual_seed(seed)+ W = torch.randn(N, K, device = "cuda", generator = g) * 0.02+ scale = (W.abs().amax(1, keepdim = True) / 448.0).float()+ return (W / scale).to(torch.float8_e4m3fn), scale+++class _Recorder:+ def __init__(self):+ self.calls = []++ def __getitem__(self, grid):+ def launch(*args, **kwargs):+ self.calls.append((grid, args, kwargs))++ return launch+++ "N, K, expected",+ [(4096, 4096, False), (2**31 // 4096, 4096, False), (2**31 // 4096 + 1, 4096, True)],+)+def test_rows_launch_widens_past_int32(F, monkeypatch, N, K, expected):+ rec = _Recorder()+ monkeypatch.setattr(F, "_fp8_rowwise_gemv_rows_kernel", rec)+ monkeypatch.setattr(F, "_fp8_triton_device_context", lambda *a, **k: nullcontext())+ monkeypatch.setattr(F, "_FP8_ROWS_CONFIG_OVERRIDE", (32, 128, 1, 4, 4))+ X = torch.empty(4, 1, K, device = "meta", dtype = torch.bfloat16)+ W = torch.empty(N, K, device = "meta", dtype = torch.float8_e4m3fn)+ S = torch.empty(N, 1, device = "meta", dtype = torch.float32)+ F.fp8_rowwise_gemv(X, W, S)+ assert len(rec.calls) == 1+ assert rec.calls[0][2]["LONG_INDEXING"] is expected++[email protected]("dtype, N, K, split", [+ (torch.bfloat16, 4096, 12288, 8), # long K (down_proj)+ (torch.float16, 1024, 4096, 8), # narrow N (kv_proj)+ (torch.bfloat16, 4096, 4096, 1),+ (torch.float16, 12288, 4096, 1),+ (torch.bfloat16, 512, 2048, 1),+])+def test_split_k_rule(F, dtype, N, K, split):+ assert F._fp8_rows_config(N, K, dtype)[2] == split+++@needs_fp8_gpu+def test_routing(F, monkeypatch):+ Wq, scale = _weights(1024, 512)+ for M, ok in ((1, True), (2, True), (16, True), (17, False)):+ X = torch.randn(M, 1, 512, device = "cuda", dtype = torch.bfloat16)+ assert F.can_use_fp8_rowwise_gemv(X, Wq, scale) is ok, M+ X = torch.randn(4, 1, 512, device = "cuda", dtype = torch.bfloat16, requires_grad = True)+ assert not F.can_use_fp8_rowwise_gemv(X, Wq, scale)+ blocks = torch.ones(8, 4, device = "cuda")+ assert not F.can_use_fp8_rowwise_gemv(X.detach(), Wq, blocks)+ monkeypatch.setattr(F, "_FP8_GEMV_ROWS", False)+ assert not F.can_use_fp8_rowwise_gemv(X.detach(), Wq, scale)+ assert F.can_use_fp8_rowwise_gemv(X.detach()[:1], Wq, scale)+++@needs_fp8_gpu+def test_one_row_keeps_the_single_row_kernel(F, monkeypatch):+ def boom(*a, **k):+ raise AssertionError("rows kernel used for one row")++ monkeypatch.setattr(F, "_fp8_rowwise_gemv_rows", boom)+ Wq, scale = _weights(512, 256)+ X = torch.randn(1, 1, 256, device = "cuda", dtype = torch.bfloat16)+ F.fp8_rowwise_gemv(X, Wq, scale)+++@needs_fp8_gpu[email protected]("dtype", [torch.bfloat16, torch.float16])[email protected]("M", [2, 3, 8, 16])[email protected]("N, K", [(1024, 4096), (4096, 4096), (200, 320)])[email protected]("cfg", [None, (16, 128, 8, 4, 4), (32, 256, 1, 4, 3), (64, 128, 2, 4, 4)])+def test_rows_accuracy(F, monkeypatch, dtype, M, N, K, cfg):+ monkeypatch.setattr(F, "_FP8_ROWS_CONFIG_OVERRIDE", cfg)+ Wq, scale = _weights(N, K)+ X = torch.randn(M, 1, K, device = "cuda", dtype = dtype)+ Wref = Wq.double() * scale.double()+ ref = (X.double().reshape(M, K) @ Wref.t()).reshape(M, 1, N)+ y = F.fp8_rowwise_gemv(X, Wq, scale)+ assert y.shape == (M, 1, N) and y.dtype == dtype+ # The dequantize-then-matmul path rounds W to 16 bits first; ours reads the exact fp8 values.+ dq = X @ (Wq.float() * scale).to(dtype).t()+ err, err_dq = (y.double() - ref).abs(), (dq.double() - ref).abs()+ ulp = torch.finfo(dtype).eps * ref.abs().max().item()+ assert err.max().item() <= err_dq.max().item() + ulp+ assert err.mean().item() <= err_dq.mean().item() * 1.05+ # Fixed reduction order: repeat calls are bitwise equal (split-K sums partials, no atomics).+ assert torch.equal(y, F.fp8_rowwise_gemv(X, Wq, scale))+++@needs_fp8_gpu+def test_rows_through_fp8_linear_callers(F):+ from unsloth.kernels.utils import fast_linear_forward++ class P(torch.nn.Module):+ def __init__(self, Wq, scale):+ super().__init__()+ self.weight = torch.nn.Parameter(Wq, requires_grad = False)+ self.weight_scale = scale+ self.bias = None++ def forward(self, x):+ raise AssertionError("not routed")++ Wq, scale = _weights(1024, 512)+ X = torch.randn(4, 1, 512, device = "cuda", dtype = torch.bfloat16)+ with torch.no_grad():+ out = fast_linear_forward(P(Wq, scale), X)+ ref = X.double() @ (Wq.double() * scale.double()).t()+ assert (out.double() - ref).abs().max().item() < 0.05 * ref.abs().max().item()+++@needs_fp8_gpu+def test_rows_off_where_triton_miscompiles_the_gemv(F, monkeypatch):+ # Blackwell + Triton < 3.4: only the exact one-row BLOCK_N 1 launch is trusted there.+ monkeypatch.setattr(F, "_fp8_gemv_miscompiles", lambda device: True)+ monkeypatch.setattr(F, "_triton_builds_blackwell", lambda: True)+ Wq, scale = _weights(1024, 512)+ assert not F.can_use_fp8_rowwise_gemv(torch.randn(4, 1, 512, device = "cuda", dtype = torch.bfloat16), Wq, scale)+ assert F.can_use_fp8_rowwise_gemv(torch.randn(1, 1, 512, device = "cuda", dtype = torch.bfloat16), Wq, scale)diff --git a/unsloth/kernels/fp8.py b/unsloth/kernels/fp8.pyindex 6254c6fbbc0..88e82acabdf 100644--- a/unsloth/kernels/fp8.py+++ b/unsloth/kernels/fp8.py@@ -579,12 +579,130 @@ def _fp8_rowwise_gemv_kernel(tl.store(y_ptr + offs_n, y.to(y_ptr.dtype.element_ty), mask = offs_n < N)+# Batched decode (2..16 rows): each weight tile is read once for every row, no 16-bit weight copy.+# UNSLOTH_FP8_GEMV_ROWS=0 restores the dequantize path.+_FP8_GEMV_MAX_ROWS = 16+_FP8_GEMV_ROWS = os.environ.get("UNSLOTH_FP8_GEMV_ROWS", "1").strip().lower() not in ("0", "false", "no", "off")++[email protected](do_not_specialize = ["M"])+def _fp8_rowwise_gemv_rows_kernel(+ x_ptr,+ w_ptr,+ s_ptr,+ y_ptr,+ p_ptr,+ M,+ N,+ K,+ stride_wn,+ BLOCK_M: tl.constexpr,+ BLOCK_N: tl.constexpr,+ BLOCK_K: tl.constexpr,+ SPLIT_K: tl.constexpr,+ LONG_INDEXING: tl.constexpr = False,+):+ pid = tl.program_id(0)+ pid_k = tl.program_id(1)+ if LONG_INDEXING:+ pid = pid.to(tl.int64)+ offs_m = tl.arange(0, BLOCK_M)+ offs_n = pid * BLOCK_N + tl.arange(0, BLOCK_N)+ acc = tl.zeros((BLOCK_M, BLOCK_N), dtype = tl.float32)+ k_per = tl.cdiv(tl.cdiv(K, SPLIT_K), BLOCK_K) * BLOCK_K+ k_lo = pid_k * k_per+ for k0 in range(k_lo, tl.minimum(k_lo + k_per, K), BLOCK_K):+ offs_k = k0 + tl.arange(0, BLOCK_K)+ x = tl.load(+ x_ptr + offs_m[:, None] * K + offs_k[None, :],+ mask = (offs_m[:, None] < M) & (offs_k[None, :] < K),+ other = 0.0,+ )+ # e4m3 values are exact in bf16 / fp16, so the dot sees the stored weights unrounded.+ w = tl.load(+ w_ptr + offs_n[:, None] * stride_wn + offs_k[None, :],+ mask = (offs_n[:, None] < N) & (offs_k[None, :] < K),+ other = 0.0,+ ).to(x.dtype)+ acc = tl.dot(x, tl.trans(w), acc, input_precision = "ieee", out_dtype = tl.float32)+ mask = (offs_m[:, None] < M) & (offs_n[None, :] < N)+ if SPLIT_K == 1:+ s = tl.load(s_ptr + offs_n, mask = offs_n < N, other = 0.0).to(tl.float32)+ y = acc * s[None, :]+ tl.store(y_ptr + offs_m[:, None] * N + offs_n[None, :], y.to(y_ptr.dtype.element_ty), mask = mask)+ else:+ # fp32 partials, summed in a fixed order by the reduce kernel: deterministic, no atomics.+ tl.store(p_ptr + (pid_k * M + offs_m[:, None]) * N + offs_n[None, :], acc, mask = mask)++[email protected](do_not_specialize = ["M"])+def _fp8_rowwise_gemv_rows_reduce(p_ptr, s_ptr, y_ptr, M, N, SPLIT_K: tl.constexpr, BLOCK: tl.constexpr):+ offs = tl.program_id(0) * BLOCK + tl.arange(0, BLOCK)+ mask = offs < M * N+ acc = tl.zeros((BLOCK,), dtype = tl.float32)+ for k in tl.static_range(SPLIT_K):+ acc += tl.load(p_ptr + k * M * N + offs, mask = mask, other = 0.0)+ s = tl.load(s_ptr + offs % N, mask = mask, other = 0.0).to(tl.float32)+ tl.store(y_ptr + offs, (acc * s).to(y_ptr.dtype.element_ty), mask = mask)+++def _fp8_rows_config(N, K, dtype):+ """(BLOCK_N, BLOCK_K, SPLIT_K, num_warps, num_stages) from a sweep; split K only for long K or narrow N."""+ if _FP8_ROWS_CONFIG_OVERRIDE is not None:+ return _FP8_ROWS_CONFIG_OVERRIDE+ if K >= 8192:+ return (64 if dtype == torch.float16 else 32), 128, 8, 4, 4+ if N <= 1024 and K >= 4096:+ return 32, 128, 8, 4, 4+ block_n = 16 if N <= 2048 else (64 if N >= 12288 and dtype == torch.float16 else 32)+ return block_n, 256, 1, 4, 4+++_FP8_ROWS_CONFIG_OVERRIDE = None+++def _fp8_rowwise_gemv_rows(X, weight, scale, out, rows):+ N, K = weight.shape+ block_n, block_k, split_k, warps, stages = _fp8_rows_config(N, K, X.dtype)+ partials = (+ torch.empty((split_k, rows, N), device = X.device, dtype = torch.float32) if split_k > 1 else out+ )+ with _fp8_triton_device_context(X, static_device_count = True):+ _fp8_rowwise_gemv_rows_kernel[(triton.cdiv(N, block_n), split_k)](+ X,+ weight,+ scale,+ out,+ partials,+ rows,+ N,+ K,+ weight.stride(0),+ BLOCK_M = _FP8_GEMV_MAX_ROWS,+ BLOCK_N = block_n,+ BLOCK_K = block_k,+ SPLIT_K = split_k,+ LONG_INDEXING = long_indexing(weight),+ num_warps = warps,+ num_stages = stages,+ enable_fp_fusion = True,+ )+ if split_k > 1:+ _fp8_rowwise_gemv_rows_reduce[(triton.cdiv(rows * N, 1024),)](+ partials, scale, out, rows, N, SPLIT_K = split_k, BLOCK = 1024, enable_fp_fusion = True+ )+ return out++def can_use_fp8_rowwise_gemv(X, weight, weight_scale):if not (X.is_cuda and weight.is_cuda and X.dtype in (torch.bfloat16, torch.float16)):return Falseif weight.dtype != torch.float8_e4m3fn or weight.dim() != 2 or not weight.is_contiguous():return False- if X.numel() != X.shape[-1] or X.shape[-1] != weight.shape[1]:+ if X.shape[-1] != weight.shape[1]:+ return False+ rows = X.numel() // X.shape[-1]+ if rows != 1 and not (2 <= rows <= _FP8_GEMV_MAX_ROWS and _FP8_GEMV_ROWS):return False# Exactly one scale per output row: a block grid can have N elements by coincidence.if (@@ -598,6 +716,8 @@ def can_use_fp8_rowwise_gemv(X, weight, weight_scale):return Falseif not _fp8_gemv_miscompiles(weight.device):return True+ if rows != 1:+ return False# Triton 3.3 is exact at BLOCK_N 1 (N <= 4096); beats dequant + matmul when N >= 256 or K <= 4096 (RTX PRO 6000).N, K = weight.shapereturn N <= 4096 and (N >= 256 or K <= 4096) and _triton_builds_blackwell()@@ -613,12 +733,15 @@ def _triton_builds_blackwell():def fp8_rowwise_gemv(X, weight, weight_scale):- """X @ (weight * weight_scale).T for one token without a 16-bit weight copy."""+ """X @ (weight * weight_scale).T for up to _FP8_GEMV_MAX_ROWS tokens without a 16-bit weight copy."""N, K = weight.shapeout = torch.empty((*X.shape[:-1], N), device = X.device, dtype = X.dtype)if not X.is_contiguous():X = X.contiguous()scale = weight_scale if weight_scale.is_contiguous() else weight_scale.contiguous()+ rows = X.numel() // K+ if rows != 1:+ return _fp8_rowwise_gemv_rows(X, weight, scale, out, rows)block_n = 1 if N <= 4096 else (4 if N <= 32768 else 8)with _fp8_triton_device_context(X, static_device_count = True):_fp8_rowwise_gemv_kernel[(triton.cdiv(N, block_n),)](