FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.
perf(linear): run nvfp4 linears on nvfp4 activations on sm_120 (#737)
Xiaoze Fan4d2e2e2104ab
diff --git a/python/freetoken/kernel/gluon/nvfp4/linear/sm120_gemv.py b/python/freetoken/kernel/gluon/nvfp4/linear/sm120_gemv.pynew file mode 100644index 00000000..cb63149a--- /dev/null+++ b/python/freetoken/kernel/gluon/nvfp4/linear/sm120_gemv.py@@ -0,0 +1,230 @@+"""NVFP4 W4A16 GEMV and skinny GEMM for sm_120 on the row-major weight: [N, K/2] e2m1 codes, flat 128x4 e4m3 block scales, one fp32 global.++:func:`gemv_fma` decodes each code word once on the CUDA cores and reuses it for every token; :func:`gemv_mma` runs bf16 mma.sync with the+weight rows on M and the tokens on N. Both keep K a constexpr with the K loop unrolled and load the activation with evict_last, so it+stays in L2 while the weight stream passes; the weight keeps the default policy, as evict_first on it slows the kernels after it in a+model step (lines it would have evicted stay in L2 for them to evict). :func:`gemv_config` picks the kernel and tile for a shape.+"""++import functools++import torch+import triton+from triton.experimental import gluon+from triton.experimental.gluon import language as gl+from triton.experimental.gluon.language.nvidia.ampere import mma_v2++from ..utils import e2m1x8_bf16x2, scale_bf16x2++# one code word (k = 8w + 0..7) times its f16x2 scale pair -> 8 fp32; e2m1 times e4m3 is exact in f16+_DECODE = gl.constexpr("""+{+.reg .b8 b0, b1, b2, b3;+.reg .b32 h0, h1, h2, h3;+.reg .b16 l0, l1, l2, l3, l4, l5, l6, l7;+mov.b32 {b0, b1, b2, b3}, $8;+cvt.rn.f16x2.e2m1x2 h0, b0;+cvt.rn.f16x2.e2m1x2 h1, b1;+cvt.rn.f16x2.e2m1x2 h2, b2;+cvt.rn.f16x2.e2m1x2 h3, b3;+mul.rn.f16x2 h0, h0, $9;+mul.rn.f16x2 h1, h1, $9;+mul.rn.f16x2 h2, h2, $9;+mul.rn.f16x2 h3, h3, $9;+mov.b32 {l0, l1}, h0;+mov.b32 {l2, l3}, h1;+mov.b32 {l4, l5}, h2;+mov.b32 {l6, l7}, h3;+cvt.f32.f16 $0, l0;+cvt.f32.f16 $1, l1;+cvt.f32.f16 $2, l2;+cvt.f32.f16 $3, l3;+cvt.f32.f16 $4, l4;+cvt.f32.f16 $5, l5;+cvt.f32.f16 $6, l6;+cvt.f32.f16 $7, l7;+}+""")+++def _scale_row(row, K_TILES: gl.constexpr):+ # 128x4 atoms: [row tile, k tile, row % 32, (row % 128) // 32, group % 4]+ return (row // 128).to(gl.int64) * (K_TILES * 512) + (row % 32) * 16 + (row % 128) // 32 * 4+++def _gemv_fma_kernel(x_ptr, w_ptr, s_ptr, g_ptr, out_ptr, T, stride_x, stride_o,+ K: gl.constexpr, T_C: gl.constexpr, R: gl.constexpr, V: gl.constexpr):+ W: gl.constexpr = gl.num_warps()+ BN: gl.constexpr = R * W+ BKW: gl.constexpr = 32 * V+ KW: gl.constexpr = K // 8+ K_TILES: gl.constexpr = (K // 16 + 3) // 4+ # a warp owns R whole rows; each lane holds V code words and their 8 k per row and per token+ L4: gl.constexpr = gl.BlockedLayout([T_C, R, V, 8], [1, 1, 32, 1], [1, W, 1, 1], [3, 2, 1, 0])+ La: gl.constexpr = gl.SliceLayout(3, L4)+ Lw: gl.constexpr = gl.SliceLayout(0, La)+ Lx: gl.constexpr = gl.SliceLayout(1, L4)+ Lxj: gl.constexpr = gl.SliceLayout(1, La)++ # 4 consecutive CTA rows are rows r, r + 32, r + 64, r + 96: one 16-byte chunk of every 128x4 scale atom+ flat = gl.program_id(0) * BN + gl.arange(0, BN, layout=gl.SliceLayout(1, Lw))+ row = (flat // 128) * 128 + (flat % 128) // 4 + (flat % 4) * 32+ wcol = gl.arange(0, BKW, layout=gl.SliceLayout(0, Lw))+ w_base = w_ptr + row.to(gl.int64)[:, None] * KW + wcol[None, :]+ s_row = _scale_row(row, K_TILES)+ xt = gl.arange(0, T_C, layout=gl.SliceLayout(1, gl.SliceLayout(2, Lx)))+ xw = gl.arange(0, BKW, layout=gl.SliceLayout(0, gl.SliceLayout(2, Lx)))+ xj = gl.arange(0, 8, layout=gl.SliceLayout(0, gl.SliceLayout(1, Lx)))+ x_off = xt[:, None, None] * stride_x + (xw * 8)[None, :, None] + xj[None, None, :]+ x_ok = (xt < T)[:, None, None]++ acc = gl.zeros([T_C, BN, BKW], gl.float32, La)+ for ph in gl.static_range(KW // BKW):+ words = gl.load(w_base + ph * BKW)+ grp = (ph * BKW + wcol) // 2+ sc = gl.load(s_ptr + s_row[:, None] + (grp // 4 * 512 + grp % 4)[None, :])+ s16 = sc.to(gl.float8e4nv, bitcast=True).to(gl.float16).to(gl.uint16, bitcast=True).to(gl.uint32)+ s2 = (s16 | (s16 << 16)).to(gl.int32, bitcast=True)+ v0, v1, v2, v3, v4, v5, v6, v7 = gl.inline_asm_elementwise(_DECODE, "=r,=r,=r,=r,=r,=r,=r,=r,r,r", [words, s2],+ dtype=(gl.float32,) * 8, is_pure=True, pack=1)+ xa = gl.load(x_ptr + ph * BKW * 8 + x_off, mask=x_ok, other=0.0, eviction_policy="evict_last").to(gl.float32)+ # in-thread split of the 8 k of each word: x0..x7 line up with v0..v7+ ev, od = gl.split(gl.reshape(xa, [T_C, BKW, 4, 2]))+ e0, e1 = gl.split(gl.reshape(ev, [T_C, BKW, 2, 2]))+ o0, o1 = gl.split(gl.reshape(od, [T_C, BKW, 2, 2]))+ x0, x4 = gl.split(e0)+ x2, x6 = gl.split(e1)+ x1, x5 = gl.split(o0)+ x3, x7 = gl.split(o1)+ acc += v0[None] * gl.convert_layout(x0, Lxj)[:, None, :]+ acc += v1[None] * gl.convert_layout(x1, Lxj)[:, None, :]+ acc += v2[None] * gl.convert_layout(x2, Lxj)[:, None, :]+ acc += v3[None] * gl.convert_layout(x3, Lxj)[:, None, :]+ acc += v4[None] * gl.convert_layout(x4, Lxj)[:, None, :]+ acc += v5[None] * gl.convert_layout(x5, Lxj)[:, None, :]+ acc += v6[None] * gl.convert_layout(x6, Lxj)[:, None, :]+ acc += v7[None] * gl.convert_layout(x7, Lxj)[:, None, :]+ res = gl.sum(acc, axis=2) * gl.load(g_ptr)+ Lo: gl.constexpr = gl.SliceLayout(2, La)+ ot = gl.arange(0, T_C, layout=gl.SliceLayout(1, Lo))+ orow = gl.convert_layout(row, gl.SliceLayout(0, Lo))+ gl.store(out_ptr + ot[:, None] * stride_o + orow[None, :], res.to(out_ptr.dtype.element_ty), mask=(ot < T)[:, None])+++def _to_bf16(q):+ lo = (q & 0xFFFF).to(gl.uint16).to(gl.bfloat16, bitcast=True)+ hi = (q >> 16).to(gl.uint16).to(gl.bfloat16, bitcast=True)+ return gl.join(lo, hi)+++def _kperm(t, A: gl.constexpr, B: gl.constexpr):+ # k_phys = 32 lid + 4 s + 2 h + e -> k_mma = 16 s + 8 h + 2 lid + e: a lane's contiguous 32 k become its fragment slots+ t = gl.reshape(t, [A, B, 4, 8, 2, 2])+ t = gl.permute(t, (0, 1, 3, 4, 2, 5))+ return gl.reshape(t, [A, B, 128])+++def _gemv_mma_kernel(x_ptr, w_ptr, s_ptr, g_ptr, out_ptr, T, stride_x, stride_o,+ K: gl.constexpr, T_C: gl.constexpr, BM: gl.constexpr, Lc: gl.constexpr, Lx: gl.constexpr, mma: gl.constexpr):+ KW: gl.constexpr = gl.num_warps()+ NB: gl.constexpr = K // 128+ K_TILES: gl.constexpr = (K // 16 + 3) // 4+ al: gl.constexpr = gl.DotOperandLayout(0, mma, 2)+ bl: gl.constexpr = gl.DotOperandLayout(1, mma, 2)++ # batched [KW, BM, T_C] mma: warp w takes the 128-k blocks w, w + KW, ...; the warps' partials are summed at the end+ row0 = gl.program_id(0) * BM+ cw = gl.arange(0, KW, layout=gl.SliceLayout(1, gl.SliceLayout(2, Lc)))+ cm = gl.arange(0, BM, layout=gl.SliceLayout(0, gl.SliceLayout(2, Lc)))+ cj = gl.arange(0, 16, layout=gl.SliceLayout(0, gl.SliceLayout(1, Lc)))+ row = row0 + cm+ c_off = (row.to(gl.int64) * (K // 8))[None, :, None] + (cw * 16)[:, None, None] + cj[None, None, :]+ grp = cw[:, None, None] * 8 + (cj // 2)[None, None, :]+ s_off = _scale_row(row, K_TILES)[None, :, None] + (grp // 4 * 512 + grp % 4)+ xw = gl.arange(0, KW, layout=gl.SliceLayout(1, gl.SliceLayout(2, Lx)))+ xt = gl.arange(0, T_C, layout=gl.SliceLayout(0, gl.SliceLayout(2, Lx)))+ xk = gl.arange(0, 128, layout=gl.SliceLayout(0, gl.SliceLayout(1, Lx)))+ x_off = (xt * stride_x)[None, :, None] + (xw * 128)[:, None, None] + xk[None, None, :]+ x_ok = (xt < T)[None, :, None]++ acc = gl.zeros([KW, BM, T_C], gl.float32, mma)+ for it in gl.static_range((NB + KW - 1) // KW):+ b0 = it * KW+ if (it + 1) * KW <= NB:+ q = gl.load(w_ptr + c_off + b0 * 16)+ sc = gl.load(s_ptr + s_off + b0 * 1024)+ xa = gl.load(x_ptr + x_off + b0 * 128, mask=x_ok, other=0.0, eviction_policy="evict_last")+ else:+ ok = (cw + b0 < NB)[:, None, None]+ q = gl.load(w_ptr + c_off + b0 * 16, mask=ok, other=0)+ sc = gl.load(s_ptr + s_off + b0 * 1024, mask=ok, other=0)+ xa = gl.load(x_ptr + x_off + b0 * 128, mask=x_ok & (xw + b0 < NB)[:, None, None], other=0.0, eviction_policy="evict_last")+ w0, w1, w2, w3 = e2m1x8_bf16x2(q.to(gl.uint32, bitcast=True), scale_bf16x2(sc))+ # join appends its axis last: this order flattens to the k pairs w0, w1, w2, w3+ p = gl.reshape(gl.join(gl.join(w0, w2), gl.join(w1, w3)), [KW, BM, 64])+ a = gl.convert_layout(_kperm(gl.reshape(_to_bf16(p), [KW, BM, 128]), KW, BM), al)+ b = gl.convert_layout(gl.permute(_kperm(xa, KW, T_C), (0, 2, 1)), bl)+ acc = mma_v2(a, b, acc)+ # e2m1x8_bf16x2 leaves a factor 2^-8 in the decoded weight+ res = gl.sum(acc, axis=0) * (gl.load(g_ptr) * 256.0)+ Lo: gl.constexpr = gl.SliceLayout(0, mma)+ om = gl.arange(0, BM, layout=gl.SliceLayout(1, Lo))+ ot = gl.arange(0, T_C, layout=gl.SliceLayout(0, Lo))+ gl.store(out_ptr + ot[None, :] * stride_o + (row0 + om)[:, None], res.to(out_ptr.dtype.element_ty), mask=(ot < T)[None, :])+++def _mma_layouts(kw: int, t_c: int, bm: int):+ warps = [[1 << i, 0, 0] for i in range(kw.bit_length() - 1)]+ lanes = [[0, 0, 32], [0, 0, 64], [0, 1, 0], [0, 2, 0], [0, 4, 0]]+ # codes [KW, BM, 16 words]: lane (gid, lid) loads words 4 lid .. 4 lid + 3 of rows gid, gid + 8, gid + 16, ...+ codes = gl.DistributedLinearLayout([[0, 0, 1], [0, 0, 2], [0, 8, 0]] + [[0, 16 << i, 0] for i in range(bm.bit_length() - 5)],+ [[0, 0, 4], [0, 0, 8], [0, 1, 0], [0, 2, 0], [0, 4, 0]], warps, [], [kw, bm, 16])+ # activations [KW, T_C, 128]: lane (gid, lid) loads k 32 lid .. 32 lid + 31 of tokens gid, gid + 8+ acts = gl.DistributedLinearLayout([[0, 0, 1 << i] for i in range(5)] + [[0, 8 << i, 0] for i in range(t_c.bit_length() - 4)],+ lanes, warps, [], [kw, t_c, 128])+ return codes, acts, gl.NVMMADistributedLayout([2, 0], [kw, 1, 1], [1, 16, 8])+++def gemv_fma(x: torch.Tensor, weight: torch.Tensor, scale: torch.Tensor, g: torch.Tensor, R: int, V: int, num_warps: int) -> torch.Tensor:+ """x [T, K] bf16 row-contiguous, weight [N, K/2] uint8, scale flat 128x4 e4m3, g fp32 -> [T, N]; see :func:`gemv_config` for the shape rules."""+ T, K = x.shape+ N = weight.shape[0]+ out = torch.empty(T, N, dtype=x.dtype, device=x.device)+ _gemv_fma_kernel[(N // (R * num_warps),)](x, weight.view(torch.int32), scale.view(torch.uint8), g, out, T, x.stride(0), out.stride(0),+ K=K, T_C=triton.next_power_of_2(T), R=R, V=V, num_warps=num_warps)+ return out+++def gemv_mma(x: torch.Tensor, weight: torch.Tensor, scale: torch.Tensor, g: torch.Tensor, bm: int, num_warps: int) -> torch.Tensor:+ """Same contract as :func:`gemv_fma`; T <= 16."""+ T, K = x.shape+ N = weight.shape[0]+ t_c = max(8, triton.next_power_of_2(T))+ codes, acts, mma = _mma_layouts(num_warps, t_c, bm)+ out = torch.empty(T, N, dtype=x.dtype, device=x.device)+ _gemv_mma_kernel[(N // bm,)](x, weight.view(torch.int32), scale.view(torch.uint8), g, out, T, x.stride(0), out.stride(0),+ K=K, T_C=t_c, BM=bm, Lc=codes, Lx=acts, mma=mma, num_warps=num_warps)+ return out+++def gemv_config(N: int, K: int, T: int, max_tokens: int = 16):+ """(kernel, tile) for a [T, K] x [N, K] product, or None when neither kernel covers it (T > max_tokens <= 16, or an unsupported shape)."""+ if T == 1 and K < 16384 or T == 2 and N >= 16384:+ words = K // 8+ if N % 128 == 0:+ # wide layers take more rows per CTA; narrow ones longer words per lane+ if N >= 16384 and words % 64 == 0:+ return gemv_fma, {"R": 4, "V": 2, "num_warps": 4}+ for V, warps in ((4, 4), (2, 8), (1, 8)):+ if words % (32 * V) == 0 and not (N >= 16384 and V == 4):+ return gemv_fma, {"R": 2, "V": V, "num_warps": warps}+ if T > max_tokens or K % 128:+ return None+ tile = {"bm": 16, "num_warps": 8} if T <= 8 or N % 32 else {"bm": 32, "num_warps": 8}+ return (gemv_mma, tile) if N % tile["bm"] == 0 else Nonediff --git a/python/freetoken/kernel/triton/nvfp4_act_quant.py b/python/freetoken/kernel/triton/nvfp4_act_quant.pyindex b52cfbe8..6f56a460 100644--- a/python/freetoken/kernel/triton/nvfp4_act_quant.py+++ b/python/freetoken/kernel/triton/nvfp4_act_quant.py@@ -16,27 +16,53 @@ def _e2m1x2(hi, lo):@triton.jit-def _quant_kernel(x_ptr, q_ptr, s_ptr, g_ptr, K, stride_x, BLOCK_K: tl.constexpr):- row = tl.program_id(0).to(tl.int64)- offs = tl.arange(0, BLOCK_K)- x = tl.load(x_ptr + row * stride_x + offs, mask=offs < K, other=0.0).to(tl.float32)+def _quant_kernel(x_ptr, q_ptr, s_ptr, g_ptr, R, K, stride_x, k_tiles,+ SWIZZLED: tl.constexpr, BLOCK_M: tl.constexpr, BLOCK_K: tl.constexpr):+ # with a static global every 16-wide block quantizes on its own, so a row splits over programs+ rows = tl.program_id(0) * BLOCK_M + tl.arange(0, BLOCK_M)+ cols = tl.program_id(1) * BLOCK_K + tl.arange(0, BLOCK_K)+ row_ok = rows < R+ r64 = rows.to(tl.int64)+ x = tl.load(x_ptr + r64[:, None] * stride_x + cols[None, :], mask=row_ok[:, None] & (cols < K)[None, :], other=0.0)g = tl.load(g_ptr)- xb = tl.reshape(x, [BLOCK_K // 16, 16])- s = (tl.max(tl.abs(xb), 1) / (6.0 * g)).to(tl.float8e4nv)+ xb = tl.reshape(x.to(tl.float32), [BLOCK_M, BLOCK_K // 16, 16])+ s = (tl.max(tl.abs(xb), 2) / (6.0 * g)).to(tl.float8e4nv)sf = s.to(tl.float32) * ginv = tl.where(sf > 0, 1.0 / sf, 0.0)- lo, hi = tl.split(tl.reshape(xb * inv[:, None], [BLOCK_K // 2, 2]))- oq = tl.arange(0, BLOCK_K // 2)- tl.store(q_ptr + row * (K // 2) + oq, _e2m1x2(hi, lo).to(tl.uint8), mask=oq < K // 2)- os_ = tl.arange(0, BLOCK_K // 16)- tl.store(s_ptr + row * (K // 16) + os_, s.to(tl.uint8, bitcast=True), mask=os_ < K // 16)+ lo, hi = tl.split(tl.reshape(xb * inv[:, :, None], [BLOCK_M, BLOCK_K // 2, 2]))+ oq = tl.program_id(1) * (BLOCK_K // 2) + tl.arange(0, BLOCK_K // 2)+ tl.store(q_ptr + r64[:, None] * (K // 2) + oq[None, :], _e2m1x2(hi, lo).to(tl.uint8),+ mask=row_ok[:, None] & (oq < K // 2)[None, :])+ sc = tl.program_id(1) * (BLOCK_K // 16) + tl.arange(0, BLOCK_K // 16)+ if SWIZZLED:+ # cuBLAS / CUTLASS 128x4 block-scale atoms: [row tile, k tile, row % 32, (row % 128) // 32, col % 4]+ s_off = ((r64 // 128)[:, None] * k_tiles + (sc // 4)[None, :]) * 512 \+ + ((rows % 32) * 16 + (rows % 128) // 32 * 4)[:, None] + (sc % 4)[None, :]+ else:+ s_off = r64[:, None] * (K // 16) + sc[None, :]+ tl.store(s_ptr + s_off, s.to(tl.uint8, bitcast=True), mask=row_ok[:, None] & (sc < K // 16)[None, :])-def quantize_nvfp4(x: torch.Tensor, g: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:- """x [R, K] -> (codes [R, K / 2] uint8, low nibble first; block scales [R, K / 16] e4m3) under the fp32 global g [1]; K % 16 == 0."""+def quantize_nvfp4(x: torch.Tensor, g: torch.Tensor, swizzled: bool = False) -> tuple[torch.Tensor, torch.Tensor]:+ """x [R, K] -> (codes [R, K / 2] uint8, low nibble first; e4m3 block scales) under the fp32 global g [1]; K % 16 == 0.++ The scales are [R, K / 16], or with ``swizzled`` the flat cuBLAS / CUTLASS 128x4 layout that ``torch.nn.functional.scaled_mm`` takes,+ rows padded to 128 and K / 16 to 4 with zeros.+ """R, K = x.shape+ assert K % 16 == 0 and x.stride(1) == 1q = torch.empty(R, K // 2, dtype=torch.uint8, device=x.device)- s = torch.empty(R, K // 16, dtype=torch.uint8, device=x.device)+ k_tiles = triton.cdiv(K // 16, 4)+ if swizzled:+ n = triton.cdiv(R, 128) * 128 * k_tiles * 4+ exact = R % 128 == 0 and (K // 16) % 4 == 0+ s = (torch.empty if exact else torch.zeros)(n, dtype=torch.uint8, device=x.device)+ else:+ s = torch.empty(R, K // 16, dtype=torch.uint8, device=x.device)if R:- _quant_kernel[(R,)](x, q, s, g, K, x.stride(0), BLOCK_K=triton.next_power_of_2(K), num_warps=4)+ block_k = min(triton.next_power_of_2(K), 1024)+ block_m = max(1, 1024 // block_k)+ grid = (triton.cdiv(R, block_m), triton.cdiv(K, block_k))+ _quant_kernel[grid](x, q, s, g, R, K, x.stride(0), k_tiles, SWIZZLED=swizzled, BLOCK_M=block_m, BLOCK_K=block_k,+ num_warps=4)return q, s.view(torch.float8_e4m3fn)diff --git a/python/freetoken/layers/quantization/linear/nvfp4.py b/python/freetoken/layers/quantization/linear/nvfp4.pyindex 80f5b78d..6a131653 100644--- a/python/freetoken/layers/quantization/linear/nvfp4.py+++ b/python/freetoken/layers/quantization/linear/nvfp4.py@@ -2,18 +2,24 @@from __future__ import annotations+import functoolsfrom types import SimpleNamespacefrom typing import Anyimport torchfrom freetoken.kernel import backend+from freetoken.utils import init_loggerfrom ..registry import LayerKind, register_methodfrom ..scheme import NVFP4_GROUP as GROUP, QuantKindfrom .base import LinearConfig, LinearKernel, LinearMethod+logger = init_logger(__name__)+FP8 = torch.float8_e4m3fn+# fused parts whose weight globals are further apart than this ratio get a load-time warning+_PART_GLOBAL_WARN_RATIO = 1.05class TritonNvfp4LinearKernel(LinearKernel):@@ -32,6 +38,180 @@ class TritonNvfp4LinearKernel(LinearKernel):return nvfp4_dense_linear_t(x, layer.weight, layer.weight_scale, layer.weight_global, layer.bias)+def _warn_part_globals() -> None:+ logger.warning_rank0(+ "an NVFP4 linear fuses parts whose weight global scales differ by more than %d%%; the W4A4 kernels use the largest "+ "for every part, which scales the other parts' outputs by their ratio; --quant-backend linear.nvfp4=triton keeps "+ "each part's global", round((_PART_GLOBAL_WARN_RATIO - 1) * 100),+ )+++def _tensor_global(layer: Any) -> torch.Tensor:+ """The one weight global the W4A4 GEMMs and the small-batch kernels take: the largest per-part global, the block scales unchanged."""+ g = layer.weight_global.float()+ g_max = g.max()+ if g_max > _PART_GLOBAL_WARN_RATIO * g.min():+ _warn_part_globals()+ return g_max+++def _swizzle_128x4(scale: torch.Tensor) -> torch.Tensor:+ """[N, K/16] block scales -> the flat cuBLAS / CUTLASS 128x4 layout, rows padded to 128 and columns to 4 with zeros."""+ rows, cols = scale.shape+ rb, cb = -(-rows // 128), -(-cols // 4)+ padded = torch.zeros(rb * 128, cb * 4, dtype=scale.dtype, device=scale.device)+ padded[:rows, :cols] = scale+ blocks = padded.view(rb, 128, cb, 4).permute(0, 2, 1, 3)+ return blocks.reshape(-1, 4, 32, 4).transpose(1, 2).reshape(-1).contiguous()+++def _sm120_gemv(device_index: int):+ """The row-major W4A16 GEMV / skinny GEMM module for this GPU, or None."""+ from freetoken.kernel import gluon++ if torch.cuda.get_device_capability(device_index) != (12, 0) or not gluon.available():+ return None+ from freetoken.kernel.gluon.nvfp4.linear import sm120_gemv++ return sm120_gemv+++def _small_batch(layer: Any, x: torch.Tensor, max_tokens: int) -> torch.Tensor | None:+ """W4A16 on the W4A4 kernels' own weight for decode-sized batches, where the W4A4 GEMMs leave bandwidth unused; None to run the GEMM."""+ gemv = _sm120_gemv(x.device.index)+ if gemv is None or x.dtype != torch.bfloat16 or x.shape[0] == 0:+ return None+ route = gemv.gemv_config(layer.weight.shape[0], x.shape[1], x.shape[0], max_tokens)+ if route is None:+ return None+ kernel, tile = route+ return kernel(x if x.stride(1) == 1 else x.contiguous(), layer.weight, layer.weight_scale, layer.weight_global, **tile)+++def _flashinfer_cuda_major() -> int:+ """The CUDA version flashinfer's b12x GEMM checks at call time: nvcc's, else torch's."""+ from flashinfer.jit.cpp_ext import get_cuda_version++ return get_cuda_version().major+++def _small_batch_usable() -> bool:+ return torch.cuda.is_available() and _sm120_gemv(torch.cuda.current_device()) is not None+++class CublasNvfp4LinearKernel(LinearKernel):+ """W4A4: activations quantized under the checkpoint's input_scale, then torch's NVFP4 scaled_mm (cuBLASLt) on the row-major weight."""++ name = "cublas"++ def unusable_reason(self, cfg: LinearConfig) -> str | None:+ if backend.is_rocm() or not torch.cuda.is_available():+ return "needs CUDA"+ if cfg.scheme is None or not cfg.scheme.has("input_scale"):+ return "the weight declares no static activation scale (W4A16 checkpoint)"+ if backend.device_capability() < (10, 0):+ return "NVFP4 tensor cores need sm_100+"+ if not hasattr(torch.nn.functional, "scaled_mm") or not hasattr(torch, "float4_e2m1fn_x2"):+ return f"torch {torch.__version__} has no NVFP4 scaled_mm"+ if cfg.out_features % 16 or cfg.in_features % 32:+ return f"scaled_mm needs N % 16 == 0 and K % 32 == 0, got N={cfg.out_features} K={cfg.in_features}"+ return None++ def worth_it(self, cfg: LinearConfig) -> bool:+ # cuBLAS alone runs decode-sized batches up to 1.4x slower than the triton GEMV; the small-batch kernels cover those on sm_120+ return _small_batch_usable()++ def finalize(self, layer: Any) -> None:+ g_w = _tensor_global(layer)+ layer.weight_scale = _swizzle_128x4(layer.weight_scale)+ layer.weight_global = g_w.reshape(())+ layer.input_scale = layer.input_scale.float().reshape(())++ def apply(self, layer: Any, x: torch.Tensor) -> torch.Tensor:+ from torch.nn import functional as F++ from freetoken.kernel.triton.nvfp4_act_quant import quantize_nvfp4++ *lead, K = x.shape+ out = _small_batch(layer, x.reshape(-1, K), 16)+ if out is not None:+ out = out.reshape(*lead, layer.weight.shape[0])+ return out + layer.bias.to(out.dtype) if layer.bias is not None else out+ xq, xs = quantize_nvfp4(x.reshape(-1, K), layer.input_scale.reshape(1), swizzled=True)+ block, tensor = F.ScalingType.BlockWise1x16, F.ScalingType.TensorWise+ swizzle = [F.SwizzleType.SWIZZLE_32_4_4, F.SwizzleType.NO_SWIZZLE]+ out = F.scaled_mm(+ xq.view(torch.float4_e2m1fn_x2), layer.weight.view(torch.float4_e2m1fn_x2).t(),+ [xs, layer.input_scale], [block, tensor], [layer.weight_scale, layer.weight_global], [block, tensor],+ swizzle_a=swizzle, swizzle_b=swizzle, output_dtype=x.dtype,+ ).reshape(*lead, layer.weight.shape[0])+ return out + layer.bias.to(out.dtype) if layer.bias is not None else out+++class B12xNvfp4LinearKernel(LinearKernel):+ """W4A4 through flashinfer alone: its activation quant and its SM12x CuTe-DSL GEMM (mm_fp4 backend b12x) on the row-major weight."""++ name = "b12x"++ def unusable_reason(self, cfg: LinearConfig) -> str | None:+ if backend.is_rocm() or not torch.cuda.is_available():+ return "needs CUDA"+ if cfg.scheme is None or not cfg.scheme.has("input_scale"):+ return "the weight declares no static activation scale (W4A16 checkpoint)"+ cc = backend.device_capability()+ if cc[0] != 12:+ return f"b12x requires sm_120 / sm_121, got sm_{cc[0]}{cc[1]}"+ if not backend.is_flashinfer_installed():+ return "flashinfer is not installed"+ drv = backend.driver_cuda_version()+ if drv is not None and drv < 13000:+ return f"b12x JIT-compiles through the driver and needs CUDA 13+, the driver supports {drv // 1000}.{drv % 1000 // 10}"+ if cfg.in_features % 32 or cfg.out_features % 8:+ # flashinfer checks only K; N % 8 != 0 returns wrong output+ return f"b12x needs K % 32 == 0 and N % 8 == 0, got K={cfg.in_features} N={cfg.out_features}"+ try:+ from flashinfer import fp4_quantize, mm_fp4, nvfp4_block_scale_interleave # noqa: F401++ cuda_major = _flashinfer_cuda_major()+ except Exception as exc:+ return f"flashinfer FP4 GEMM symbols are unusable ({exc!r})"+ if cuda_major < 13:+ return f"flashinfer's b12x GEMM needs a CUDA 13 toolkit (nvcc, else torch), got CUDA {cuda_major}"+ return None++ def worth_it(self, cfg: LinearConfig) -> bool:+ return _small_batch_usable()++ def finalize(self, layer: Any) -> None:+ from flashinfer import nvfp4_block_scale_interleave++ g_w = _tensor_global(layer)+ layer.weight_scale = nvfp4_block_scale_interleave(layer.weight_scale.view(torch.uint8)).view(FP8)+ # flashinfer takes the quant-side activation global and alpha = 1 / (that * the weight's quant-side global)+ layer.input_scale_inv = (1.0 / layer.input_scale.float()).reshape(1)+ layer.alpha = (layer.input_scale.float() * g_w).reshape(1)+ layer.weight_global = g_w.reshape(())++ def apply(self, layer: Any, x: torch.Tensor) -> torch.Tensor:+ from flashinfer import fp4_quantize, mm_fp4++ *lead, K = x.shape+ x2 = x.reshape(-1, K)+ if x2.shape[0] == 0:+ return x.new_empty(*lead, layer.weight.shape[0])+ # b12x's W4A4 GEMM beats the W4A16 kernels from 9 tokens on GeForce, where bf16 mma accumulates in fp32 at half rate+ out = _small_batch(layer, x2, 8)+ if out is not None:+ out = out.reshape(*lead, layer.weight.shape[0])+ return out + layer.bias.to(out.dtype) if layer.bias is not None else out+ xq, xs = fp4_quantize(x2.contiguous(), layer.input_scale_inv, is_sf_swizzled_layout=True)+ out = mm_fp4(xq, layer.weight.T, xs, layer.weight_scale, layer.alpha, x.dtype, backend="b12x").reshape(*lead, layer.weight.shape[0])+ return out + layer.bias.to(out.dtype) if layer.bias is not None else out++class MarlinNvfp4LinearKernel(LinearKernel):"""vLLM's fused FP4 Marlin GEMM (W4A16); needs one global scale for the whole weight."""@@ -113,7 +293,7 @@ class EmulationNvfp4LinearKernel(LinearKernel):@register_method(QuantKind.NVFP4, LayerKind.LINEAR)class Nvfp4LinearMethod(LinearMethod):- candidates = (TritonNvfp4LinearKernel, MarlinNvfp4LinearKernel, EmulationNvfp4LinearKernel)+ candidates = (B12xNvfp4LinearKernel, CublasNvfp4LinearKernel, TritonNvfp4LinearKernel, MarlinNvfp4LinearKernel, EmulationNvfp4LinearKernel)def create_weights(self, layer: Any) -> None:g = self.cfgdiff --git a/tests/kernels/test_nvfp4_w4a4_linear.py b/tests/kernels/test_nvfp4_w4a4_linear.pynew file mode 100644index 00000000..1840d30f--- /dev/null+++ b/tests/kernels/test_nvfp4_w4a4_linear.py@@ -0,0 +1,158 @@+"""The W4A4 NVFP4 linears: cublas (our activation quant with 128x4-swizzled scales feeding torch's NVFP4 scaled_mm) and b12x (flashinfer only),+and the sm_120 W4A16 GEMV / skinny GEMM both run for small batches on the same row-major weight."""++from __future__ import annotations++import math+from types import SimpleNamespace++import pytest+import torch++pytestmark = pytest.mark.skipif(+ not torch.cuda.is_available() or torch.cuda.get_device_capability() < (10, 0), reason="needs NVFP4 tensor cores (sm_100+)"+)++_E2M1 = torch.tensor([0.0, 0.5, 1.0, 1.5, 2.0, 3.0, 4.0, 6.0, -0.0, -0.5, -1.0, -1.5, -2.0, -3.0, -4.0, -6.0])+++def _round_e2m1(v: torch.Tensor) -> torch.Tensor:+ """Round to the nearest e2m1 value, ties to the even code, saturating at 6 (cvt.rn.satfinite)."""+ grid = _E2M1[:8].to(v.device)+ a = v.abs().clamp(max=6.0)+ hi = torch.searchsorted(grid, a.contiguous()).clamp(max=7)+ lo = (hi - 1).clamp(min=0)+ d_lo, d_hi = a - grid[lo], grid[hi] - a+ code = torch.where(d_lo < d_hi, lo, torch.where(d_hi < d_lo, hi, torch.where(lo % 2 == 0, lo, hi)))+ return torch.copysign(grid[code], v)+++def _qdq(x: torch.Tensor, g: float) -> torch.Tensor:+ xb = x.float().view(x.shape[0], -1, 16)+ g = torch.tensor(g, dtype=torch.float32, device=x.device)+ sf = ((xb.abs().amax(-1) / (6.0 * g)).to(torch.float8_e4m3fn).float() * g).unsqueeze(-1)+ inv = torch.where(sf > 0, 1.0 / sf, torch.zeros_like(sf))+ return (_round_e2m1(xb * inv) * sf).view_as(x)+++def _dequant(packed: torch.Tensor, scale: torch.Tensor, row_global: torch.Tensor) -> torch.Tensor:+ n, k2 = packed.shape+ codes = torch.stack([packed & 0xF, packed >> 4], dim=-1).view(n, 2 * k2).long()+ return _E2M1.to(packed.device)[codes] * scale.float().repeat_interleave(16, dim=1) * row_global.float().unsqueeze(1)+++def _small_batch_route(N, K, M, max_tokens=16):+ if torch.cuda.get_device_capability() != (12, 0):+ return None+ from freetoken.kernel import gluon++ if not gluon.available():+ return None+ from freetoken.kernel.gluon.nvfp4.linear.sm120_gemv import gemv_config++ return gemv_config(N, K, M, max_tokens)++[email protected]("rows,K", [(1, 2048), (7, 2080), (300, 5376)])+def test_swizzled_scales_are_the_cublas_128x4_layout(rows, K):+ """Same codes either way; the swizzled scales are the linear ones in cuBLAS's [row tile, k tile, 32, 4, 4] order, zero padded."""+ from freetoken.kernel.triton.nvfp4_act_quant import quantize_nvfp4+ from freetoken.layers.quantization.linear.nvfp4 import _swizzle_128x4++ x = torch.randn(rows, K, device="cuda", dtype=torch.bfloat16)+ g = (x.float().abs().max() / 2688).reshape(1)+ q, s = quantize_nvfp4(x, g)+ q_swz, s_swz = quantize_nvfp4(x, g, swizzled=True)+ assert torch.equal(q, q_swz)+ assert torch.equal(_swizzle_128x4(s).view(torch.uint8), s_swz.view(torch.uint8))++[email protected]("kernel_name", ["cublas", "b12x"])[email protected]("M", [1, 2, 7, 16, 300])[email protected]("N,K", [(512, 1024), (528, 2080), (12, 2048)])[email protected]("parts_differ", [False, True])+def test_w4a4_linear_matches_nvfp4_activation_reference(kernel_name, M, N, K, parts_differ):+ """finalize + apply against a torch emulation: W4A4, or W4A16 where the sm_120 small-batch kernels take the batch (M <= 16 for cublas,+ M <= 8 for b12x, at 512 x 1024; 528 x 2080 fits neither and keeps the GEMM; N = 12 must be refused, b12x returns garbage there);+ parts_differ gives the halves of a fused weight different globals, and the larger one applies to both."""+ from freetoken.layers.quantization.linear.base import LinearConfig+ from freetoken.layers.quantization.linear.nvfp4 import Nvfp4LinearMethod+ from freetoken.layers.quantization.scheme import nvfp4_scheme++ kernel = next(c for c in Nvfp4LinearMethod.candidates if c.name == kernel_name)()+ reason = kernel.unusable_reason(LinearConfig(K, N, (N // 2, N // 2), nvfp4_scheme(input_scale=True)))+ if reason:+ pytest.skip(reason)+ g = torch.Generator(device="cuda").manual_seed(0)+ weight = torch.randint(0, 256, (N, K // 2), dtype=torch.uint8, device="cuda", generator=g)+ scale = (torch.rand(N, K // 16, device="cuda", generator=g) + 0.25).to(torch.float8_e4m3fn)+ row_global = torch.full((N,), 1e-3, dtype=torch.float16, device="cuda")+ if parts_differ:+ row_global[N // 2:] = 4e-3+ bias = torch.randn(N, device="cuda", dtype=torch.bfloat16)+ x = torch.randn(M, K, device="cuda", dtype=torch.bfloat16, generator=g)+ # off amax / 2688 by an irrational factor: with amax / 2688 many values sit exactly on an e2m1 midpoint, where the kernel's+ # approximate fp32 reciprocal and torch's exact one round apart+ g_in = float(x.float().abs().max()) / 2688 * math.pi / 3+ # one tensor-wise global: the largest part's, applied to every row+ w_ref = _dequant(weight, scale, torch.full_like(row_global, float(row_global.max())))+ small = _small_batch_route(N, K, M, 16 if kernel_name == "cublas" else 8) is not None+ ref = (x.float() if small else _qdq(x, g_in)) @ w_ref.T + bias.float()++ layer = SimpleNamespace(weight=weight.clone(), weight_scale=scale.clone(), weight_global=row_global.clone(),+ input_scale=torch.tensor(g_in, device="cuda"), bias=bias)+ kernel.finalize(layer)+ out = kernel.apply(layer, x)+ torch.testing.assert_close(out.float(), ref, rtol=1e-2, atol=1e-2 * float(ref.abs().max()))++[email protected]("N,K,T", [+ (16384, 1024, 1), # fma, 4 rows per warp+ (16384, 768, 2), # fma, one word per lane+ (640, 2048, 1), # fma, four words per lane+ (640, 17408, 1), # long K: mma+ (640, 4224, 12), # mma bm32, masked last K block+ (256, 16512, 4), # mma, masked last K block+ (640, 2048, 5), # mma bm16, padded tokens+])+def test_sm120_gemv_tiles_match_dequant_reference(N, K, T):+ """Each tile gemv_config hands out, against x @ dequant(W)^T on the flat 128x4 scales."""+ route = _small_batch_route(N, K, T)+ if route is None:+ pytest.skip("needs sm_120 with Gluon")+ from freetoken.layers.quantization.linear.nvfp4 import _swizzle_128x4++ g = torch.Generator(device="cuda").manual_seed(0)+ weight = torch.randint(0, 256, (N, K // 2), dtype=torch.uint8, device="cuda", generator=g)+ scale = (torch.rand(N, K // 16, device="cuda", generator=g) + 0.25).to(torch.float8_e4m3fn)+ x = torch.randn(T, K, device="cuda", dtype=torch.bfloat16, generator=g)+ g_w = torch.tensor(1e-3, device="cuda")+ ref = x.float() @ _dequant(weight, scale, torch.full((N,), 1e-3, device="cuda")).T+ kernel, tile = route+ out = kernel(x, weight, _swizzle_128x4(scale), g_w, **tile)+ torch.testing.assert_close(out.float(), ref, rtol=1e-2, atol=1e-2 * float(ref.abs().max()))++[email protected]("kernel_name", ["cublas", "b12x"])[email protected]("M", [0, 4, 64])+def test_w4a4_linear_takes_row_strided_and_empty_inputs(kernel_name, M):+ """A row-strided slice and an empty batch give the same output as a contiguous input, on both the small-batch and the GEMM route."""+ from freetoken.layers.quantization.linear.base import LinearConfig+ from freetoken.layers.quantization.linear.nvfp4 import Nvfp4LinearMethod+ from freetoken.layers.quantization.scheme import nvfp4_scheme++ N, K = 512, 1024+ kernel = next(c for c in Nvfp4LinearMethod.candidates if c.name == kernel_name)()+ reason = kernel.unusable_reason(LinearConfig(K, N, (N,), nvfp4_scheme(input_scale=True)))+ if reason:+ pytest.skip(reason)+ g = torch.Generator(device="cuda").manual_seed(0)+ layer = SimpleNamespace(weight=torch.randint(0, 256, (N, K // 2), dtype=torch.uint8, device="cuda", generator=g),+ weight_scale=(torch.rand(N, K // 16, device="cuda", generator=g) + 0.25).to(torch.float8_e4m3fn),+ weight_global=torch.full((N,), 1e-3, dtype=torch.float16, device="cuda"),+ input_scale=torch.tensor(0.01, device="cuda"), bias=None)+ kernel.finalize(layer)+ buf = torch.randn(M, K + 64, device="cuda", dtype=torch.bfloat16, generator=g)+ out = kernel.apply(layer, buf[:, :K])+ assert out.shape == (M, N)+ torch.testing.assert_close(out, kernel.apply(layer, buf[:, :K].contiguous()), rtol=0, atol=0)