Skip to content

FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.

perf(linear): run nvfp4 linears on nvfp4 activations on sm_120 (#737)

Xiaoze Fan4d2e2e2104ab
diff --git a/python/freetoken/kernel/gluon/nvfp4/linear/sm120_gemv.py b/python/freetoken/kernel/gluon/nvfp4/linear/sm120_gemv.py
new file mode 100644
index 00000000..cb63149a
--- /dev/null
+++ b/python/freetoken/kernel/gluon/nvfp4/linear/sm120_gemv.py
@@ -0,0 +1,230 @@
+"""NVFP4 W4A16 GEMV and skinny GEMM for sm_120 on the row-major weight: [N, K/2] e2m1 codes, flat 128x4 e4m3 block scales, one fp32 global.
+
+:func:`gemv_fma` decodes each code word once on the CUDA cores and reuses it for every token; :func:`gemv_mma` runs bf16 mma.sync with the
+weight rows on M and the tokens on N. Both keep K a constexpr with the K loop unrolled and load the activation with evict_last, so it
+stays in L2 while the weight stream passes; the weight keeps the default policy, as evict_first on it slows the kernels after it in a
+model step (lines it would have evicted stay in L2 for them to evict). :func:`gemv_config` picks the kernel and tile for a shape.
+"""
+
+import functools
+
+import torch
+import triton
+from triton.experimental import gluon
+from triton.experimental.gluon import language as gl
+from triton.experimental.gluon.language.nvidia.ampere import mma_v2
+
+from ..utils import e2m1x8_bf16x2, scale_bf16x2
+
+# one code word (k = 8w + 0..7) times its f16x2 scale pair -> 8 fp32; e2m1 times e4m3 is exact in f16
+_DECODE = gl.constexpr("""
+{
+.reg .b8 b0, b1, b2, b3;
+.reg .b32 h0, h1, h2, h3;
+.reg .b16 l0, l1, l2, l3, l4, l5, l6, l7;
+mov.b32 {b0, b1, b2, b3}, $8;
+cvt.rn.f16x2.e2m1x2 h0, b0;
+cvt.rn.f16x2.e2m1x2 h1, b1;
+cvt.rn.f16x2.e2m1x2 h2, b2;
+cvt.rn.f16x2.e2m1x2 h3, b3;
+mul.rn.f16x2 h0, h0, $9;
+mul.rn.f16x2 h1, h1, $9;
+mul.rn.f16x2 h2, h2, $9;
+mul.rn.f16x2 h3, h3, $9;
+mov.b32 {l0, l1}, h0;
+mov.b32 {l2, l3}, h1;
+mov.b32 {l4, l5}, h2;
+mov.b32 {l6, l7}, h3;
+cvt.f32.f16 $0, l0;
+cvt.f32.f16 $1, l1;
+cvt.f32.f16 $2, l2;
+cvt.f32.f16 $3, l3;
+cvt.f32.f16 $4, l4;
+cvt.f32.f16 $5, l5;
+cvt.f32.f16 $6, l6;
+cvt.f32.f16 $7, l7;
+}
+""")
+
+
+def _scale_row(row, K_TILES: gl.constexpr):
+ # 128x4 atoms: [row tile, k tile, row % 32, (row % 128) // 32, group % 4]
+ return (row // 128).to(gl.int64) * (K_TILES * 512) + (row % 32) * 16 + (row % 128) // 32 * 4
+
+
+def _gemv_fma_kernel(x_ptr, w_ptr, s_ptr, g_ptr, out_ptr, T, stride_x, stride_o,
+ K: gl.constexpr, T_C: gl.constexpr, R: gl.constexpr, V: gl.constexpr):
+ W: gl.constexpr = gl.num_warps()
+ BN: gl.constexpr = R * W
+ BKW: gl.constexpr = 32 * V
+ KW: gl.constexpr = K // 8
+ K_TILES: gl.constexpr = (K // 16 + 3) // 4
+ # a warp owns R whole rows; each lane holds V code words and their 8 k per row and per token
+ L4: gl.constexpr = gl.BlockedLayout([T_C, R, V, 8], [1, 1, 32, 1], [1, W, 1, 1], [3, 2, 1, 0])
+ La: gl.constexpr = gl.SliceLayout(3, L4)
+ Lw: gl.constexpr = gl.SliceLayout(0, La)
+ Lx: gl.constexpr = gl.SliceLayout(1, L4)
+ Lxj: gl.constexpr = gl.SliceLayout(1, La)
+
+ # 4 consecutive CTA rows are rows r, r + 32, r + 64, r + 96: one 16-byte chunk of every 128x4 scale atom
+ flat = gl.program_id(0) * BN + gl.arange(0, BN, layout=gl.SliceLayout(1, Lw))
+ row = (flat // 128) * 128 + (flat % 128) // 4 + (flat % 4) * 32
+ wcol = gl.arange(0, BKW, layout=gl.SliceLayout(0, Lw))
+ w_base = w_ptr + row.to(gl.int64)[:, None] * KW + wcol[None, :]
+ s_row = _scale_row(row, K_TILES)
+ xt = gl.arange(0, T_C, layout=gl.SliceLayout(1, gl.SliceLayout(2, Lx)))
+ xw = gl.arange(0, BKW, layout=gl.SliceLayout(0, gl.SliceLayout(2, Lx)))
+ xj = gl.arange(0, 8, layout=gl.SliceLayout(0, gl.SliceLayout(1, Lx)))
+ x_off = xt[:, None, None] * stride_x + (xw * 8)[None, :, None] + xj[None, None, :]
+ x_ok = (xt < T)[:, None, None]
+
+ acc = gl.zeros([T_C, BN, BKW], gl.float32, La)
+ for ph in gl.static_range(KW // BKW):
+ words = gl.load(w_base + ph * BKW)
+ grp = (ph * BKW + wcol) // 2
+ sc = gl.load(s_ptr + s_row[:, None] + (grp // 4 * 512 + grp % 4)[None, :])
+ s16 = sc.to(gl.float8e4nv, bitcast=True).to(gl.float16).to(gl.uint16, bitcast=True).to(gl.uint32)
+ s2 = (s16 | (s16 << 16)).to(gl.int32, bitcast=True)
+ v0, v1, v2, v3, v4, v5, v6, v7 = gl.inline_asm_elementwise(_DECODE, "=r,=r,=r,=r,=r,=r,=r,=r,r,r", [words, s2],
+ dtype=(gl.float32,) * 8, is_pure=True, pack=1)
+ xa = gl.load(x_ptr + ph * BKW * 8 + x_off, mask=x_ok, other=0.0, eviction_policy="evict_last").to(gl.float32)
+ # in-thread split of the 8 k of each word: x0..x7 line up with v0..v7
+ ev, od = gl.split(gl.reshape(xa, [T_C, BKW, 4, 2]))
+ e0, e1 = gl.split(gl.reshape(ev, [T_C, BKW, 2, 2]))
+ o0, o1 = gl.split(gl.reshape(od, [T_C, BKW, 2, 2]))
+ x0, x4 = gl.split(e0)
+ x2, x6 = gl.split(e1)
+ x1, x5 = gl.split(o0)
+ x3, x7 = gl.split(o1)
+ acc += v0[None] * gl.convert_layout(x0, Lxj)[:, None, :]
+ acc += v1[None] * gl.convert_layout(x1, Lxj)[:, None, :]
+ acc += v2[None] * gl.convert_layout(x2, Lxj)[:, None, :]
+ acc += v3[None] * gl.convert_layout(x3, Lxj)[:, None, :]
+ acc += v4[None] * gl.convert_layout(x4, Lxj)[:, None, :]
+ acc += v5[None] * gl.convert_layout(x5, Lxj)[:, None, :]
+ acc += v6[None] * gl.convert_layout(x6, Lxj)[:, None, :]
+ acc += v7[None] * gl.convert_layout(x7, Lxj)[:, None, :]
+ res = gl.sum(acc, axis=2) * gl.load(g_ptr)
+ Lo: gl.constexpr = gl.SliceLayout(2, La)
+ ot = gl.arange(0, T_C, layout=gl.SliceLayout(1, Lo))
+ orow = gl.convert_layout(row, gl.SliceLayout(0, Lo))
+ gl.store(out_ptr + ot[:, None] * stride_o + orow[None, :], res.to(out_ptr.dtype.element_ty), mask=(ot < T)[:, None])
+
+
+def _to_bf16(q):
+ lo = (q & 0xFFFF).to(gl.uint16).to(gl.bfloat16, bitcast=True)
+ hi = (q >> 16).to(gl.uint16).to(gl.bfloat16, bitcast=True)
+ return gl.join(lo, hi)
+
+
+def _kperm(t, A: gl.constexpr, B: gl.constexpr):
+ # k_phys = 32 lid + 4 s + 2 h + e -> k_mma = 16 s + 8 h + 2 lid + e: a lane's contiguous 32 k become its fragment slots
+ t = gl.reshape(t, [A, B, 4, 8, 2, 2])
+ t = gl.permute(t, (0, 1, 3, 4, 2, 5))
+ return gl.reshape(t, [A, B, 128])
+
+
+def _gemv_mma_kernel(x_ptr, w_ptr, s_ptr, g_ptr, out_ptr, T, stride_x, stride_o,
+ K: gl.constexpr, T_C: gl.constexpr, BM: gl.constexpr, Lc: gl.constexpr, Lx: gl.constexpr, mma: gl.constexpr):
+ KW: gl.constexpr = gl.num_warps()
+ NB: gl.constexpr = K // 128
+ K_TILES: gl.constexpr = (K // 16 + 3) // 4
+ al: gl.constexpr = gl.DotOperandLayout(0, mma, 2)
+ bl: gl.constexpr = gl.DotOperandLayout(1, mma, 2)
+
+ # batched [KW, BM, T_C] mma: warp w takes the 128-k blocks w, w + KW, ...; the warps' partials are summed at the end
+ row0 = gl.program_id(0) * BM
+ cw = gl.arange(0, KW, layout=gl.SliceLayout(1, gl.SliceLayout(2, Lc)))
+ cm = gl.arange(0, BM, layout=gl.SliceLayout(0, gl.SliceLayout(2, Lc)))
+ cj = gl.arange(0, 16, layout=gl.SliceLayout(0, gl.SliceLayout(1, Lc)))
+ row = row0 + cm
+ c_off = (row.to(gl.int64) * (K // 8))[None, :, None] + (cw * 16)[:, None, None] + cj[None, None, :]
+ grp = cw[:, None, None] * 8 + (cj // 2)[None, None, :]
+ s_off = _scale_row(row, K_TILES)[None, :, None] + (grp // 4 * 512 + grp % 4)
+ xw = gl.arange(0, KW, layout=gl.SliceLayout(1, gl.SliceLayout(2, Lx)))
+ xt = gl.arange(0, T_C, layout=gl.SliceLayout(0, gl.SliceLayout(2, Lx)))
+ xk = gl.arange(0, 128, layout=gl.SliceLayout(0, gl.SliceLayout(1, Lx)))
+ x_off = (xt * stride_x)[None, :, None] + (xw * 128)[:, None, None] + xk[None, None, :]
+ x_ok = (xt < T)[None, :, None]
+
+ acc = gl.zeros([KW, BM, T_C], gl.float32, mma)
+ for it in gl.static_range((NB + KW - 1) // KW):
+ b0 = it * KW
+ if (it + 1) * KW <= NB:
+ q = gl.load(w_ptr + c_off + b0 * 16)
+ sc = gl.load(s_ptr + s_off + b0 * 1024)
+ xa = gl.load(x_ptr + x_off + b0 * 128, mask=x_ok, other=0.0, eviction_policy="evict_last")
+ else:
+ ok = (cw + b0 < NB)[:, None, None]
+ q = gl.load(w_ptr + c_off + b0 * 16, mask=ok, other=0)
+ sc = gl.load(s_ptr + s_off + b0 * 1024, mask=ok, other=0)
+ xa = gl.load(x_ptr + x_off + b0 * 128, mask=x_ok & (xw + b0 < NB)[:, None, None], other=0.0, eviction_policy="evict_last")
+ w0, w1, w2, w3 = e2m1x8_bf16x2(q.to(gl.uint32, bitcast=True), scale_bf16x2(sc))
+ # join appends its axis last: this order flattens to the k pairs w0, w1, w2, w3
+ p = gl.reshape(gl.join(gl.join(w0, w2), gl.join(w1, w3)), [KW, BM, 64])
+ a = gl.convert_layout(_kperm(gl.reshape(_to_bf16(p), [KW, BM, 128]), KW, BM), al)
+ b = gl.convert_layout(gl.permute(_kperm(xa, KW, T_C), (0, 2, 1)), bl)
+ acc = mma_v2(a, b, acc)
+ # e2m1x8_bf16x2 leaves a factor 2^-8 in the decoded weight
+ res = gl.sum(acc, axis=0) * (gl.load(g_ptr) * 256.0)
+ Lo: gl.constexpr = gl.SliceLayout(0, mma)
+ om = gl.arange(0, BM, layout=gl.SliceLayout(1, Lo))
+ ot = gl.arange(0, T_C, layout=gl.SliceLayout(0, Lo))
+ gl.store(out_ptr + ot[None, :] * stride_o + (row0 + om)[:, None], res.to(out_ptr.dtype.element_ty), mask=(ot < T)[None, :])
+
+
+def _mma_layouts(kw: int, t_c: int, bm: int):
+ warps = [[1 << i, 0, 0] for i in range(kw.bit_length() - 1)]
+ lanes = [[0, 0, 32], [0, 0, 64], [0, 1, 0], [0, 2, 0], [0, 4, 0]]
+ # codes [KW, BM, 16 words]: lane (gid, lid) loads words 4 lid .. 4 lid + 3 of rows gid, gid + 8, gid + 16, ...
+ codes = gl.DistributedLinearLayout([[0, 0, 1], [0, 0, 2], [0, 8, 0]] + [[0, 16 << i, 0] for i in range(bm.bit_length() - 5)],
+ [[0, 0, 4], [0, 0, 8], [0, 1, 0], [0, 2, 0], [0, 4, 0]], warps, [], [kw, bm, 16])
+ # activations [KW, T_C, 128]: lane (gid, lid) loads k 32 lid .. 32 lid + 31 of tokens gid, gid + 8
+ acts = gl.DistributedLinearLayout([[0, 0, 1 << i] for i in range(5)] + [[0, 8 << i, 0] for i in range(t_c.bit_length() - 4)],
+ lanes, warps, [], [kw, t_c, 128])
+ return codes, acts, gl.NVMMADistributedLayout([2, 0], [kw, 1, 1], [1, 16, 8])
+
+
+def gemv_fma(x: torch.Tensor, weight: torch.Tensor, scale: torch.Tensor, g: torch.Tensor, R: int, V: int, num_warps: int) -> torch.Tensor:
+ """x [T, K] bf16 row-contiguous, weight [N, K/2] uint8, scale flat 128x4 e4m3, g fp32 -> [T, N]; see :func:`gemv_config` for the shape rules."""
+ T, K = x.shape
+ N = weight.shape[0]
+ out = torch.empty(T, N, dtype=x.dtype, device=x.device)
+ _gemv_fma_kernel[(N // (R * num_warps),)](x, weight.view(torch.int32), scale.view(torch.uint8), g, out, T, x.stride(0), out.stride(0),
+ K=K, T_C=triton.next_power_of_2(T), R=R, V=V, num_warps=num_warps)
+ return out
+
+
+def gemv_mma(x: torch.Tensor, weight: torch.Tensor, scale: torch.Tensor, g: torch.Tensor, bm: int, num_warps: int) -> torch.Tensor:
+ """Same contract as :func:`gemv_fma`; T <= 16."""
+ T, K = x.shape
+ N = weight.shape[0]
+ t_c = max(8, triton.next_power_of_2(T))
+ codes, acts, mma = _mma_layouts(num_warps, t_c, bm)
+ out = torch.empty(T, N, dtype=x.dtype, device=x.device)
+ _gemv_mma_kernel[(N // bm,)](x, weight.view(torch.int32), scale.view(torch.uint8), g, out, T, x.stride(0), out.stride(0),
+ K=K, T_C=t_c, BM=bm, Lc=codes, Lx=acts, mma=mma, num_warps=num_warps)
+ return out
+
+
+def gemv_config(N: int, K: int, T: int, max_tokens: int = 16):
+ """(kernel, tile) for a [T, K] x [N, K] product, or None when neither kernel covers it (T > max_tokens <= 16, or an unsupported shape)."""
+ if T == 1 and K < 16384 or T == 2 and N >= 16384:
+ words = K // 8
+ if N % 128 == 0:
+ # wide layers take more rows per CTA; narrow ones longer words per lane
+ if N >= 16384 and words % 64 == 0:
+ return gemv_fma, {"R": 4, "V": 2, "num_warps": 4}
+ for V, warps in ((4, 4), (2, 8), (1, 8)):
+ if words % (32 * V) == 0 and not (N >= 16384 and V == 4):
+ return gemv_fma, {"R": 2, "V": V, "num_warps": warps}
+ if T > max_tokens or K % 128:
+ return None
+ tile = {"bm": 16, "num_warps": 8} if T <= 8 or N % 32 else {"bm": 32, "num_warps": 8}
+ return (gemv_mma, tile) if N % tile["bm"] == 0 else None
diff --git a/python/freetoken/kernel/triton/nvfp4_act_quant.py b/python/freetoken/kernel/triton/nvfp4_act_quant.py
index b52cfbe8..6f56a460 100644
--- a/python/freetoken/kernel/triton/nvfp4_act_quant.py
+++ b/python/freetoken/kernel/triton/nvfp4_act_quant.py
@@ -16,27 +16,53 @@ def _e2m1x2(hi, lo):
@triton.jit
-def _quant_kernel(x_ptr, q_ptr, s_ptr, g_ptr, K, stride_x, BLOCK_K: tl.constexpr):
- row = tl.program_id(0).to(tl.int64)
- offs = tl.arange(0, BLOCK_K)
- x = tl.load(x_ptr + row * stride_x + offs, mask=offs < K, other=0.0).to(tl.float32)
+def _quant_kernel(x_ptr, q_ptr, s_ptr, g_ptr, R, K, stride_x, k_tiles,
+ SWIZZLED: tl.constexpr, BLOCK_M: tl.constexpr, BLOCK_K: tl.constexpr):
+ # with a static global every 16-wide block quantizes on its own, so a row splits over programs
+ rows = tl.program_id(0) * BLOCK_M + tl.arange(0, BLOCK_M)
+ cols = tl.program_id(1) * BLOCK_K + tl.arange(0, BLOCK_K)
+ row_ok = rows < R
+ r64 = rows.to(tl.int64)
+ x = tl.load(x_ptr + r64[:, None] * stride_x + cols[None, :], mask=row_ok[:, None] & (cols < K)[None, :], other=0.0)
g = tl.load(g_ptr)
- xb = tl.reshape(x, [BLOCK_K // 16, 16])
- s = (tl.max(tl.abs(xb), 1) / (6.0 * g)).to(tl.float8e4nv)
+ xb = tl.reshape(x.to(tl.float32), [BLOCK_M, BLOCK_K // 16, 16])
+ s = (tl.max(tl.abs(xb), 2) / (6.0 * g)).to(tl.float8e4nv)
sf = s.to(tl.float32) * g
inv = tl.where(sf > 0, 1.0 / sf, 0.0)
- lo, hi = tl.split(tl.reshape(xb * inv[:, None], [BLOCK_K // 2, 2]))
- oq = tl.arange(0, BLOCK_K // 2)
- tl.store(q_ptr + row * (K // 2) + oq, _e2m1x2(hi, lo).to(tl.uint8), mask=oq < K // 2)
- os_ = tl.arange(0, BLOCK_K // 16)
- tl.store(s_ptr + row * (K // 16) + os_, s.to(tl.uint8, bitcast=True), mask=os_ < K // 16)
+ lo, hi = tl.split(tl.reshape(xb * inv[:, :, None], [BLOCK_M, BLOCK_K // 2, 2]))
+ oq = tl.program_id(1) * (BLOCK_K // 2) + tl.arange(0, BLOCK_K // 2)
+ tl.store(q_ptr + r64[:, None] * (K // 2) + oq[None, :], _e2m1x2(hi, lo).to(tl.uint8),
+ mask=row_ok[:, None] & (oq < K // 2)[None, :])
+ sc = tl.program_id(1) * (BLOCK_K // 16) + tl.arange(0, BLOCK_K // 16)
+ if SWIZZLED:
+ # cuBLAS / CUTLASS 128x4 block-scale atoms: [row tile, k tile, row % 32, (row % 128) // 32, col % 4]
+ s_off = ((r64 // 128)[:, None] * k_tiles + (sc // 4)[None, :]) * 512 \
+ + ((rows % 32) * 16 + (rows % 128) // 32 * 4)[:, None] + (sc % 4)[None, :]
+ else:
+ s_off = r64[:, None] * (K // 16) + sc[None, :]
+ tl.store(s_ptr + s_off, s.to(tl.uint8, bitcast=True), mask=row_ok[:, None] & (sc < K // 16)[None, :])
-def quantize_nvfp4(x: torch.Tensor, g: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
- """x [R, K] -> (codes [R, K / 2] uint8, low nibble first; block scales [R, K / 16] e4m3) under the fp32 global g [1]; K % 16 == 0."""
+def quantize_nvfp4(x: torch.Tensor, g: torch.Tensor, swizzled: bool = False) -> tuple[torch.Tensor, torch.Tensor]:
+ """x [R, K] -> (codes [R, K / 2] uint8, low nibble first; e4m3 block scales) under the fp32 global g [1]; K % 16 == 0.
+
+ The scales are [R, K / 16], or with ``swizzled`` the flat cuBLAS / CUTLASS 128x4 layout that ``torch.nn.functional.scaled_mm`` takes,
+ rows padded to 128 and K / 16 to 4 with zeros.
+ """
R, K = x.shape
+ assert K % 16 == 0 and x.stride(1) == 1
q = torch.empty(R, K // 2, dtype=torch.uint8, device=x.device)
- s = torch.empty(R, K // 16, dtype=torch.uint8, device=x.device)
+ k_tiles = triton.cdiv(K // 16, 4)
+ if swizzled:
+ n = triton.cdiv(R, 128) * 128 * k_tiles * 4
+ exact = R % 128 == 0 and (K // 16) % 4 == 0
+ s = (torch.empty if exact else torch.zeros)(n, dtype=torch.uint8, device=x.device)
+ else:
+ s = torch.empty(R, K // 16, dtype=torch.uint8, device=x.device)
if R:
- _quant_kernel[(R,)](x, q, s, g, K, x.stride(0), BLOCK_K=triton.next_power_of_2(K), num_warps=4)
+ block_k = min(triton.next_power_of_2(K), 1024)
+ block_m = max(1, 1024 // block_k)
+ grid = (triton.cdiv(R, block_m), triton.cdiv(K, block_k))
+ _quant_kernel[grid](x, q, s, g, R, K, x.stride(0), k_tiles, SWIZZLED=swizzled, BLOCK_M=block_m, BLOCK_K=block_k,
+ num_warps=4)
return q, s.view(torch.float8_e4m3fn)
diff --git a/python/freetoken/layers/quantization/linear/nvfp4.py b/python/freetoken/layers/quantization/linear/nvfp4.py
index 80f5b78d..6a131653 100644
--- a/python/freetoken/layers/quantization/linear/nvfp4.py
+++ b/python/freetoken/layers/quantization/linear/nvfp4.py
@@ -2,18 +2,24 @@
from __future__ import annotations
+import functools
from types import SimpleNamespace
from typing import Any
import torch
from freetoken.kernel import backend
+from freetoken.utils import init_logger
from ..registry import LayerKind, register_method
from ..scheme import NVFP4_GROUP as GROUP, QuantKind
from .base import LinearConfig, LinearKernel, LinearMethod
+logger = init_logger(__name__)
+
FP8 = torch.float8_e4m3fn
+# fused parts whose weight globals are further apart than this ratio get a load-time warning
+_PART_GLOBAL_WARN_RATIO = 1.05
class TritonNvfp4LinearKernel(LinearKernel):
@@ -32,6 +38,180 @@ class TritonNvfp4LinearKernel(LinearKernel):
return nvfp4_dense_linear_t(x, layer.weight, layer.weight_scale, layer.weight_global, layer.bias)
+def _warn_part_globals() -> None:
+ logger.warning_rank0(
+ "an NVFP4 linear fuses parts whose weight global scales differ by more than %d%%; the W4A4 kernels use the largest "
+ "for every part, which scales the other parts' outputs by their ratio; --quant-backend linear.nvfp4=triton keeps "
+ "each part's global", round((_PART_GLOBAL_WARN_RATIO - 1) * 100),
+ )
+
+
+def _tensor_global(layer: Any) -> torch.Tensor:
+ """The one weight global the W4A4 GEMMs and the small-batch kernels take: the largest per-part global, the block scales unchanged."""
+ g = layer.weight_global.float()
+ g_max = g.max()
+ if g_max > _PART_GLOBAL_WARN_RATIO * g.min():
+ _warn_part_globals()
+ return g_max
+
+
+def _swizzle_128x4(scale: torch.Tensor) -> torch.Tensor:
+ """[N, K/16] block scales -> the flat cuBLAS / CUTLASS 128x4 layout, rows padded to 128 and columns to 4 with zeros."""
+ rows, cols = scale.shape
+ rb, cb = -(-rows // 128), -(-cols // 4)
+ padded = torch.zeros(rb * 128, cb * 4, dtype=scale.dtype, device=scale.device)
+ padded[:rows, :cols] = scale
+ blocks = padded.view(rb, 128, cb, 4).permute(0, 2, 1, 3)
+ return blocks.reshape(-1, 4, 32, 4).transpose(1, 2).reshape(-1).contiguous()
+
+
+def _sm120_gemv(device_index: int):
+ """The row-major W4A16 GEMV / skinny GEMM module for this GPU, or None."""
+ from freetoken.kernel import gluon
+
+ if torch.cuda.get_device_capability(device_index) != (12, 0) or not gluon.available():
+ return None
+ from freetoken.kernel.gluon.nvfp4.linear import sm120_gemv
+
+ return sm120_gemv
+
+
+def _small_batch(layer: Any, x: torch.Tensor, max_tokens: int) -> torch.Tensor | None:
+ """W4A16 on the W4A4 kernels' own weight for decode-sized batches, where the W4A4 GEMMs leave bandwidth unused; None to run the GEMM."""
+ gemv = _sm120_gemv(x.device.index)
+ if gemv is None or x.dtype != torch.bfloat16 or x.shape[0] == 0:
+ return None
+ route = gemv.gemv_config(layer.weight.shape[0], x.shape[1], x.shape[0], max_tokens)
+ if route is None:
+ return None
+ kernel, tile = route
+ return kernel(x if x.stride(1) == 1 else x.contiguous(), layer.weight, layer.weight_scale, layer.weight_global, **tile)
+
+
+def _flashinfer_cuda_major() -> int:
+ """The CUDA version flashinfer's b12x GEMM checks at call time: nvcc's, else torch's."""
+ from flashinfer.jit.cpp_ext import get_cuda_version
+
+ return get_cuda_version().major
+
+
+def _small_batch_usable() -> bool:
+ return torch.cuda.is_available() and _sm120_gemv(torch.cuda.current_device()) is not None
+
+
+class CublasNvfp4LinearKernel(LinearKernel):
+ """W4A4: activations quantized under the checkpoint's input_scale, then torch's NVFP4 scaled_mm (cuBLASLt) on the row-major weight."""
+
+ name = "cublas"
+
+ def unusable_reason(self, cfg: LinearConfig) -> str | None:
+ if backend.is_rocm() or not torch.cuda.is_available():
+ return "needs CUDA"
+ if cfg.scheme is None or not cfg.scheme.has("input_scale"):
+ return "the weight declares no static activation scale (W4A16 checkpoint)"
+ if backend.device_capability() < (10, 0):
+ return "NVFP4 tensor cores need sm_100+"
+ if not hasattr(torch.nn.functional, "scaled_mm") or not hasattr(torch, "float4_e2m1fn_x2"):
+ return f"torch {torch.__version__} has no NVFP4 scaled_mm"
+ if cfg.out_features % 16 or cfg.in_features % 32:
+ return f"scaled_mm needs N % 16 == 0 and K % 32 == 0, got N={cfg.out_features} K={cfg.in_features}"
+ return None
+
+ def worth_it(self, cfg: LinearConfig) -> bool:
+ # cuBLAS alone runs decode-sized batches up to 1.4x slower than the triton GEMV; the small-batch kernels cover those on sm_120
+ return _small_batch_usable()
+
+ def finalize(self, layer: Any) -> None:
+ g_w = _tensor_global(layer)
+ layer.weight_scale = _swizzle_128x4(layer.weight_scale)
+ layer.weight_global = g_w.reshape(())
+ layer.input_scale = layer.input_scale.float().reshape(())
+
+ def apply(self, layer: Any, x: torch.Tensor) -> torch.Tensor:
+ from torch.nn import functional as F
+
+ from freetoken.kernel.triton.nvfp4_act_quant import quantize_nvfp4
+
+ *lead, K = x.shape
+ out = _small_batch(layer, x.reshape(-1, K), 16)
+ if out is not None:
+ out = out.reshape(*lead, layer.weight.shape[0])
+ return out + layer.bias.to(out.dtype) if layer.bias is not None else out
+ xq, xs = quantize_nvfp4(x.reshape(-1, K), layer.input_scale.reshape(1), swizzled=True)
+ block, tensor = F.ScalingType.BlockWise1x16, F.ScalingType.TensorWise
+ swizzle = [F.SwizzleType.SWIZZLE_32_4_4, F.SwizzleType.NO_SWIZZLE]
+ out = F.scaled_mm(
+ xq.view(torch.float4_e2m1fn_x2), layer.weight.view(torch.float4_e2m1fn_x2).t(),
+ [xs, layer.input_scale], [block, tensor], [layer.weight_scale, layer.weight_global], [block, tensor],
+ swizzle_a=swizzle, swizzle_b=swizzle, output_dtype=x.dtype,
+ ).reshape(*lead, layer.weight.shape[0])
+ return out + layer.bias.to(out.dtype) if layer.bias is not None else out
+
+
+class B12xNvfp4LinearKernel(LinearKernel):
+ """W4A4 through flashinfer alone: its activation quant and its SM12x CuTe-DSL GEMM (mm_fp4 backend b12x) on the row-major weight."""
+
+ name = "b12x"
+
+ def unusable_reason(self, cfg: LinearConfig) -> str | None:
+ if backend.is_rocm() or not torch.cuda.is_available():
+ return "needs CUDA"
+ if cfg.scheme is None or not cfg.scheme.has("input_scale"):
+ return "the weight declares no static activation scale (W4A16 checkpoint)"
+ cc = backend.device_capability()
+ if cc[0] != 12:
+ return f"b12x requires sm_120 / sm_121, got sm_{cc[0]}{cc[1]}"
+ if not backend.is_flashinfer_installed():
+ return "flashinfer is not installed"
+ drv = backend.driver_cuda_version()
+ if drv is not None and drv < 13000:
+ return f"b12x JIT-compiles through the driver and needs CUDA 13+, the driver supports {drv // 1000}.{drv % 1000 // 10}"
+ if cfg.in_features % 32 or cfg.out_features % 8:
+ # flashinfer checks only K; N % 8 != 0 returns wrong output
+ return f"b12x needs K % 32 == 0 and N % 8 == 0, got K={cfg.in_features} N={cfg.out_features}"
+ try:
+ from flashinfer import fp4_quantize, mm_fp4, nvfp4_block_scale_interleave # noqa: F401
+
+ cuda_major = _flashinfer_cuda_major()
+ except Exception as exc:
+ return f"flashinfer FP4 GEMM symbols are unusable ({exc!r})"
+ if cuda_major < 13:
+ return f"flashinfer's b12x GEMM needs a CUDA 13 toolkit (nvcc, else torch), got CUDA {cuda_major}"
+ return None
+
+ def worth_it(self, cfg: LinearConfig) -> bool:
+ return _small_batch_usable()
+
+ def finalize(self, layer: Any) -> None:
+ from flashinfer import nvfp4_block_scale_interleave
+
+ g_w = _tensor_global(layer)
+ layer.weight_scale = nvfp4_block_scale_interleave(layer.weight_scale.view(torch.uint8)).view(FP8)
+ # flashinfer takes the quant-side activation global and alpha = 1 / (that * the weight's quant-side global)
+ layer.input_scale_inv = (1.0 / layer.input_scale.float()).reshape(1)
+ layer.alpha = (layer.input_scale.float() * g_w).reshape(1)
+ layer.weight_global = g_w.reshape(())
+
+ def apply(self, layer: Any, x: torch.Tensor) -> torch.Tensor:
+ from flashinfer import fp4_quantize, mm_fp4
+
+ *lead, K = x.shape
+ x2 = x.reshape(-1, K)
+ if x2.shape[0] == 0:
+ return x.new_empty(*lead, layer.weight.shape[0])
+ # b12x's W4A4 GEMM beats the W4A16 kernels from 9 tokens on GeForce, where bf16 mma accumulates in fp32 at half rate
+ out = _small_batch(layer, x2, 8)
+ if out is not None:
+ out = out.reshape(*lead, layer.weight.shape[0])
+ return out + layer.bias.to(out.dtype) if layer.bias is not None else out
+ xq, xs = fp4_quantize(x2.contiguous(), layer.input_scale_inv, is_sf_swizzled_layout=True)
+ out = mm_fp4(xq, layer.weight.T, xs, layer.weight_scale, layer.alpha, x.dtype, backend="b12x").reshape(*lead, layer.weight.shape[0])
+ return out + layer.bias.to(out.dtype) if layer.bias is not None else out
+
+
class MarlinNvfp4LinearKernel(LinearKernel):
"""vLLM's fused FP4 Marlin GEMM (W4A16); needs one global scale for the whole weight."""
@@ -113,7 +293,7 @@ class EmulationNvfp4LinearKernel(LinearKernel):
@register_method(QuantKind.NVFP4, LayerKind.LINEAR)
class Nvfp4LinearMethod(LinearMethod):
- candidates = (TritonNvfp4LinearKernel, MarlinNvfp4LinearKernel, EmulationNvfp4LinearKernel)
+ candidates = (B12xNvfp4LinearKernel, CublasNvfp4LinearKernel, TritonNvfp4LinearKernel, MarlinNvfp4LinearKernel, EmulationNvfp4LinearKernel)
def create_weights(self, layer: Any) -> None:
g = self.cfg
diff --git a/tests/kernels/test_nvfp4_w4a4_linear.py b/tests/kernels/test_nvfp4_w4a4_linear.py
new file mode 100644
index 00000000..1840d30f
--- /dev/null
+++ b/tests/kernels/test_nvfp4_w4a4_linear.py
@@ -0,0 +1,158 @@
+"""The W4A4 NVFP4 linears: cublas (our activation quant with 128x4-swizzled scales feeding torch's NVFP4 scaled_mm) and b12x (flashinfer only),
+and the sm_120 W4A16 GEMV / skinny GEMM both run for small batches on the same row-major weight."""
+
+from __future__ import annotations
+
+import math
+from types import SimpleNamespace
+
+import pytest
+import torch
+
+pytestmark = pytest.mark.skipif(
+ not torch.cuda.is_available() or torch.cuda.get_device_capability() < (10, 0), reason="needs NVFP4 tensor cores (sm_100+)"
+)
+
+_E2M1 = torch.tensor([0.0, 0.5, 1.0, 1.5, 2.0, 3.0, 4.0, 6.0, -0.0, -0.5, -1.0, -1.5, -2.0, -3.0, -4.0, -6.0])
+
+
+def _round_e2m1(v: torch.Tensor) -> torch.Tensor:
+ """Round to the nearest e2m1 value, ties to the even code, saturating at 6 (cvt.rn.satfinite)."""
+ grid = _E2M1[:8].to(v.device)
+ a = v.abs().clamp(max=6.0)
+ hi = torch.searchsorted(grid, a.contiguous()).clamp(max=7)
+ lo = (hi - 1).clamp(min=0)
+ d_lo, d_hi = a - grid[lo], grid[hi] - a
+ code = torch.where(d_lo < d_hi, lo, torch.where(d_hi < d_lo, hi, torch.where(lo % 2 == 0, lo, hi)))
+ return torch.copysign(grid[code], v)
+
+
+def _qdq(x: torch.Tensor, g: float) -> torch.Tensor:
+ xb = x.float().view(x.shape[0], -1, 16)
+ g = torch.tensor(g, dtype=torch.float32, device=x.device)
+ sf = ((xb.abs().amax(-1) / (6.0 * g)).to(torch.float8_e4m3fn).float() * g).unsqueeze(-1)
+ inv = torch.where(sf > 0, 1.0 / sf, torch.zeros_like(sf))
+ return (_round_e2m1(xb * inv) * sf).view_as(x)
+
+
+def _dequant(packed: torch.Tensor, scale: torch.Tensor, row_global: torch.Tensor) -> torch.Tensor:
+ n, k2 = packed.shape
+ codes = torch.stack([packed & 0xF, packed >> 4], dim=-1).view(n, 2 * k2).long()
+ return _E2M1.to(packed.device)[codes] * scale.float().repeat_interleave(16, dim=1) * row_global.float().unsqueeze(1)
+
+
+def _small_batch_route(N, K, M, max_tokens=16):
+ if torch.cuda.get_device_capability() != (12, 0):
+ return None
+ from freetoken.kernel import gluon
+
+ if not gluon.available():
+ return None
+ from freetoken.kernel.gluon.nvfp4.linear.sm120_gemv import gemv_config
+
+ return gemv_config(N, K, M, max_tokens)
+
+
[email protected]("rows,K", [(1, 2048), (7, 2080), (300, 5376)])
+def test_swizzled_scales_are_the_cublas_128x4_layout(rows, K):
+ """Same codes either way; the swizzled scales are the linear ones in cuBLAS's [row tile, k tile, 32, 4, 4] order, zero padded."""
+ from freetoken.kernel.triton.nvfp4_act_quant import quantize_nvfp4
+ from freetoken.layers.quantization.linear.nvfp4 import _swizzle_128x4
+
+ x = torch.randn(rows, K, device="cuda", dtype=torch.bfloat16)
+ g = (x.float().abs().max() / 2688).reshape(1)
+ q, s = quantize_nvfp4(x, g)
+ q_swz, s_swz = quantize_nvfp4(x, g, swizzled=True)
+ assert torch.equal(q, q_swz)
+ assert torch.equal(_swizzle_128x4(s).view(torch.uint8), s_swz.view(torch.uint8))
+
+
[email protected]("kernel_name", ["cublas", "b12x"])
[email protected]("M", [1, 2, 7, 16, 300])
[email protected]("N,K", [(512, 1024), (528, 2080), (12, 2048)])
[email protected]("parts_differ", [False, True])
+def test_w4a4_linear_matches_nvfp4_activation_reference(kernel_name, M, N, K, parts_differ):
+ """finalize + apply against a torch emulation: W4A4, or W4A16 where the sm_120 small-batch kernels take the batch (M <= 16 for cublas,
+ M <= 8 for b12x, at 512 x 1024; 528 x 2080 fits neither and keeps the GEMM; N = 12 must be refused, b12x returns garbage there);
+ parts_differ gives the halves of a fused weight different globals, and the larger one applies to both."""
+ from freetoken.layers.quantization.linear.base import LinearConfig
+ from freetoken.layers.quantization.linear.nvfp4 import Nvfp4LinearMethod
+ from freetoken.layers.quantization.scheme import nvfp4_scheme
+
+ kernel = next(c for c in Nvfp4LinearMethod.candidates if c.name == kernel_name)()
+ reason = kernel.unusable_reason(LinearConfig(K, N, (N // 2, N // 2), nvfp4_scheme(input_scale=True)))
+ if reason:
+ pytest.skip(reason)
+ g = torch.Generator(device="cuda").manual_seed(0)
+ weight = torch.randint(0, 256, (N, K // 2), dtype=torch.uint8, device="cuda", generator=g)
+ scale = (torch.rand(N, K // 16, device="cuda", generator=g) + 0.25).to(torch.float8_e4m3fn)
+ row_global = torch.full((N,), 1e-3, dtype=torch.float16, device="cuda")
+ if parts_differ:
+ row_global[N // 2:] = 4e-3
+ bias = torch.randn(N, device="cuda", dtype=torch.bfloat16)
+ x = torch.randn(M, K, device="cuda", dtype=torch.bfloat16, generator=g)
+ # off amax / 2688 by an irrational factor: with amax / 2688 many values sit exactly on an e2m1 midpoint, where the kernel's
+ # approximate fp32 reciprocal and torch's exact one round apart
+ g_in = float(x.float().abs().max()) / 2688 * math.pi / 3
+ # one tensor-wise global: the largest part's, applied to every row
+ w_ref = _dequant(weight, scale, torch.full_like(row_global, float(row_global.max())))
+ small = _small_batch_route(N, K, M, 16 if kernel_name == "cublas" else 8) is not None
+ ref = (x.float() if small else _qdq(x, g_in)) @ w_ref.T + bias.float()
+
+ layer = SimpleNamespace(weight=weight.clone(), weight_scale=scale.clone(), weight_global=row_global.clone(),
+ input_scale=torch.tensor(g_in, device="cuda"), bias=bias)
+ kernel.finalize(layer)
+ out = kernel.apply(layer, x)
+ torch.testing.assert_close(out.float(), ref, rtol=1e-2, atol=1e-2 * float(ref.abs().max()))
+
+
+ (16384, 1024, 1), # fma, 4 rows per warp
+ (16384, 768, 2), # fma, one word per lane
+ (640, 2048, 1), # fma, four words per lane
+ (640, 17408, 1), # long K: mma
+ (640, 4224, 12), # mma bm32, masked last K block
+ (256, 16512, 4), # mma, masked last K block
+ (640, 2048, 5), # mma bm16, padded tokens
+])
+def test_sm120_gemv_tiles_match_dequant_reference(N, K, T):
+ """Each tile gemv_config hands out, against x @ dequant(W)^T on the flat 128x4 scales."""
+ route = _small_batch_route(N, K, T)
+ if route is None:
+ pytest.skip("needs sm_120 with Gluon")
+ from freetoken.layers.quantization.linear.nvfp4 import _swizzle_128x4
+
+ g = torch.Generator(device="cuda").manual_seed(0)
+ weight = torch.randint(0, 256, (N, K // 2), dtype=torch.uint8, device="cuda", generator=g)
+ scale = (torch.rand(N, K // 16, device="cuda", generator=g) + 0.25).to(torch.float8_e4m3fn)
+ x = torch.randn(T, K, device="cuda", dtype=torch.bfloat16, generator=g)
+ g_w = torch.tensor(1e-3, device="cuda")
+ ref = x.float() @ _dequant(weight, scale, torch.full((N,), 1e-3, device="cuda")).T
+ kernel, tile = route
+ out = kernel(x, weight, _swizzle_128x4(scale), g_w, **tile)
+ torch.testing.assert_close(out.float(), ref, rtol=1e-2, atol=1e-2 * float(ref.abs().max()))
+
+
[email protected]("kernel_name", ["cublas", "b12x"])
[email protected]("M", [0, 4, 64])
+def test_w4a4_linear_takes_row_strided_and_empty_inputs(kernel_name, M):
+ """A row-strided slice and an empty batch give the same output as a contiguous input, on both the small-batch and the GEMM route."""
+ from freetoken.layers.quantization.linear.base import LinearConfig
+ from freetoken.layers.quantization.linear.nvfp4 import Nvfp4LinearMethod
+ from freetoken.layers.quantization.scheme import nvfp4_scheme
+
+ N, K = 512, 1024
+ kernel = next(c for c in Nvfp4LinearMethod.candidates if c.name == kernel_name)()
+ reason = kernel.unusable_reason(LinearConfig(K, N, (N,), nvfp4_scheme(input_scale=True)))
+ if reason:
+ pytest.skip(reason)
+ g = torch.Generator(device="cuda").manual_seed(0)
+ layer = SimpleNamespace(weight=torch.randint(0, 256, (N, K // 2), dtype=torch.uint8, device="cuda", generator=g),
+ weight_scale=(torch.rand(N, K // 16, device="cuda", generator=g) + 0.25).to(torch.float8_e4m3fn),
+ weight_global=torch.full((N,), 1e-3, dtype=torch.float16, device="cuda"),
+ input_scale=torch.tensor(0.01, device="cuda"), bias=None)
+ kernel.finalize(layer)
+ buf = torch.randn(M, K + 64, device="cuda", dtype=torch.bfloat16, generator=g)
+ out = kernel.apply(layer, buf[:, :K])
+ assert out.shape == (M, N)
+ torch.testing.assert_close(out, kernel.apply(layer, buf[:, :K].contiguous()), rtol=0, atol=0)