Skip to content

FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.

perf(fp8): run per-tensor fp8 as W8A8 via scaled_mm (#7)

Xiaoze Fan586a6c16ce47
diff --git a/python/freetoken/kernel/triton/fp8_pertensor_linear.py b/python/freetoken/kernel/triton/fp8_pertensor_linear.py
index 37009e0a..28a54c94 100644
--- a/python/freetoken/kernel/triton/fp8_pertensor_linear.py
+++ b/python/freetoken/kernel/triton/fp8_pertensor_linear.py
@@ -25,8 +25,14 @@ import torch
import triton
import triton.language as tl
from freetoken.layers import BaseOP
+from freetoken.layers.base import _concat_prefix
-from freetoken.kernel.triton.e4m3_compat import e4m3_kernel_view, e4m3_native_cx, e4m3_u8_to_f32
+from freetoken.kernel.triton.e4m3_compat import (
+ e4m3_kernel_view,
+ e4m3_native,
+ e4m3_native_cx,
+ e4m3_u8_to_f32,
+)
FP8 = torch.float8_e4m3fn
_TL_DTYPE = {torch.bfloat16: tl.bfloat16, torch.float16: tl.float16, torch.float32: tl.float32}
@@ -171,23 +177,108 @@ def _gemm(a: torch.Tensor, weight: torch.Tensor, weight_scale: torch.Tensor,
return out
+# ======================================================================================
+# Batched decode (M > 1) W8A8: quantize the activation with the checkpoint's static
+# per-tensor ``input_scale`` and hand both fp8 operands to cuBLASLt via torch._scaled_mm.
+#
+# ``_gemm`` above is a *prefill* kernel: grid is (cdiv(M, BLOCK_M), cdiv(N, 128)) with no
+# split-K, so at decode batch sizes the M axis contributes a single block row and the whole
+# GEMM runs on cdiv(N, 128) CTAs -- 8 of them for N=1024, on a 188-SM part. Its runtime then
+# tracks K and ignores both N and M: latency-bound, not bandwidth-bound. Measured on
+# Qwen3.5-27B's projections (RTX PRO 6000, 1461 GB/s copy roof): 594 GB/s flat across
+# M=2..16, against 1334 GB/s for the M=1 GEMV over the same weights. Routing M>=2 to
+# cuBLASLt restores ~1300 GB/s -- 10.15 ms -> 5.56 ms of per-decode-step GEMM.
+#
+# Requires sm_89+ (torch._scaled_mm's floor; Ampere has no FP8 tensor cores) *and* a
+# checkpoint carrying ``input_scale``. Without either, ``_gemm`` still runs: glm_moe_dsa
+# quantizes to fp8 at load with genuine per-row scales and has no activation scale at all.
+# ======================================================================================
+_E4M3_MAX = 448.0 # largest finite e4m3 magnitude
+
+
+def _static_quant_kernel(x_ptr, out_ptr, scale_ptr, n_elements, BLOCK: tl.constexpr):
+ """bf16 activation -> fp8-e4m3 under one broadcast per-tensor scale."""
+ offs = tl.program_id(0) * BLOCK + tl.arange(0, BLOCK)
+ mask = offs < n_elements
+ inv = 1.0 / tl.load(scale_ptr).to(tl.float32)
+ v = tl.load(x_ptr + offs, mask=mask, other=0.0).to(tl.float32) * inv
+ v = tl.minimum(tl.maximum(v, -448.0), 448.0)
+ tl.store(out_ptr + offs, v.to(tl.float8e4nv), mask=mask)
+
+
+def _static_quant(a: torch.Tensor, input_scale: torch.Tensor) -> torch.Tensor:
+ """One launch, no host sync: the scale is read from device memory inside the kernel, so
+ this stays CUDA-graph safe (fixed shapes, no dependence on tensor *values* on the host)."""
+ n = a.numel()
+ out = torch.empty_like(a, dtype=FP8)
+ BLOCK = 1024
+ _static_quant_kernel[(triton.cdiv(n, BLOCK),)](
+ a, out, input_scale, n, BLOCK=BLOCK, num_warps=4,
+ )
+ return out
+
+
+def _scaled_mm(
+ a: torch.Tensor, weight: torch.Tensor, weight_scale: torch.Tensor,
+ input_scale: torch.Tensor, uniform_scale: bool, out_dtype: torch.dtype,
+) -> torch.Tensor:
+ """``a @ (weight_fp8 * weight_scale)^T`` as a W8A8 cuBLASLt GEMM.
+
+ ``weight`` stays in the checkpoint-native row-major ``[N, K]``: ``.t()`` is exactly the
+ column-major ``[K, N]`` operand cuBLASLt wants, so the transpose is a stride change and
+ never a copy (unlike Marlin, or our own NVFP4 path's K-major repack).
+
+ A genuine per-tensor weight (one scalar repeated down ``weight_scale``) takes the
+ tensor-wise path. A fused projection, whose ``weight_scale`` is piecewise-constant
+ because each part carries its own scalar, takes the row-wise path -- that keeps every
+ part's scale exact, where vLLM/SGLang instead requantize the parts onto a shared maximum
+ and eat the precision loss. Row-wise costs ~4% here (5.56 ms vs 5.39 ms per step)."""
+ qa = _static_quant(a, input_scale)
+ wt = weight.t() # [N, K] row-major -> [K, N] column-major, stride-only
+ if uniform_scale:
+ return torch._scaled_mm(
+ qa, wt, scale_a=input_scale.reshape(()), scale_b=weight_scale[0].reshape(()),
+ out_dtype=out_dtype,
+ )
+ return torch._scaled_mm(
+ qa, wt,
+ scale_a=input_scale.reshape(1, 1).expand(a.shape[0], 1).contiguous(),
+ scale_b=weight_scale.reshape(1, -1),
+ out_dtype=out_dtype,
+ )
+
+
def fp8_pertensor_linear(
x: torch.Tensor, weight: torch.Tensor, weight_scale: torch.Tensor,
bias: torch.Tensor | None = None,
+ input_scale: torch.Tensor | None = None,
+ uniform_scale: bool = False,
) -> torch.Tensor:
- """``y = x @ (weight_fp8 * weight_scale)^T``. Decode (M=1) -> split-K GEMV; prefill -> GEMM.
- ``weight`` [N, K] fp8-e4m3, ``weight_scale`` [N] fp32 (per output row)."""
+ """``y = x @ (weight_fp8 * weight_scale)^T``. ``weight`` [N, K] fp8-e4m3, ``weight_scale``
+ [N] fp32 (per output row).
+
+ Whether the activation is quantized is a property of the *deployment*, never of the batch:
+ with ``input_scale`` on sm_89+ every M runs W8A8, otherwise every M runs W8A16 (split-K
+ GEMV at M=1, GEMM above it). Picking per-M would make a request's numerics depend on how
+ many unrelated requests happened to be in flight, so a reply would not reproduce at bs=1 --
+ and W8A16 at M=1 is worth only ~0.5% (5.53 ms vs 5.56 ms of per-step GEMM) anyway. vLLM and
+ SGLang likewise run one scheme across all M on any GPU with FP8 tensor cores."""
*lead, K = x.shape
N = weight.shape[0]
if _USE_REF: # numeric-reference fallback (debug / A-B)
w = weight.to(x.dtype) * weight_scale.to(x.dtype)[:, None]
out = (x.reshape(-1, K) @ w.t()).reshape(*lead, N)
+ elif input_scale is not None and e4m3_native():
+ out = _scaled_mm(
+ x.reshape(-1, K), weight, weight_scale, input_scale, uniform_scale, x.dtype,
+ ).reshape(*lead, N)
+ elif x.numel() // K == 1:
+ out = _gemv(x.reshape(K), e4m3_kernel_view(weight), weight_scale, x.dtype).reshape(*lead, N)
else:
- w8 = e4m3_kernel_view(weight)
- if x.numel() // K == 1:
- out = _gemv(x.reshape(K), w8, weight_scale, x.dtype).reshape(*lead, N)
- else:
- out = _gemm(x.reshape(-1, K), w8, weight_scale, x.dtype).reshape(*lead, N)
+ out = _gemm(
+ x.reshape(-1, K), e4m3_kernel_view(weight), weight_scale, x.dtype,
+ ).reshape(*lead, N)
if bias is not None:
out = out + bias.to(out.dtype)
return out
@@ -199,7 +290,11 @@ def fp8_pertensor_linear(
class Fp8PerTensorLinear(BaseOP):
"""Replicated per-tensor-FP8 linear: fp8-e4m3 ``weight`` ``[out, in]`` + per-row fp32
``weight_scale`` ``[out]`` (a genuine per-tensor weight stores the same scalar in every
- row; a fused projection stores each part's scalar across its own rows)."""
+ row; a fused projection stores each part's scalar across its own rows).
+
+ ``input_scale`` (modelopt's calibrated per-tensor activation scale) is optional: when the
+ checkpoint ships it, batched decode runs W8A8 through cuBLASLt; when it does not (models
+ that quantize to fp8 at load, e.g. glm_moe_dsa), every batch size stays W8A16."""
def __init__(self, in_features: int, out_features: int, has_bias: bool = False):
self.in_features = in_features
@@ -207,9 +302,28 @@ class Fp8PerTensorLinear(BaseOP):
self.weight = torch.empty(out_features, in_features, dtype=FP8)
self.weight_scale = torch.empty(out_features, dtype=torch.float32)
self.bias = torch.empty(out_features) if has_bias else None
+ # Set from the state dict when present. Left as None (not a tensor) so BaseOP's
+ # reflective state_dict/load_state_dict skip it entirely on checkpoints without one.
+ self.input_scale: torch.Tensor | None = None
+ self._uniform_scale = False
+
+ def load_state_dict(self, state_dict, *, prefix: str = "", _internal: bool = False) -> None:
+ # Taken out before BaseOP's reflective pass (so it is not an "unexpected key") and
+ # put back after (so that pass does not try to pop it a second time on a reload).
+ input_scale = state_dict.pop(_concat_prefix(prefix, "input_scale"), None)
+ self.input_scale = None
+ super().load_state_dict(state_dict, prefix=prefix, _internal=_internal)
+ self.input_scale = input_scale
+ # Tensor-wise scaling needs one scalar for the whole weight; a fused projection is
+ # only piecewise-constant, so decide once here rather than syncing on every forward.
+ scale = self.weight_scale
+ self._uniform_scale = bool((scale == scale[0]).all().item())
def forward(self, x: torch.Tensor) -> torch.Tensor:
- return fp8_pertensor_linear(x, self.weight, self.weight_scale, self.bias)
+ return fp8_pertensor_linear(
+ x, self.weight, self.weight_scale, self.bias,
+ self.input_scale, self._uniform_scale,
+ )
class Fp8PerTensorColMerged(Fp8PerTensorLinear):
diff --git a/python/freetoken/models/qwen3_5_moe/weight.py b/python/freetoken/models/qwen3_5_moe/weight.py
index 926df7c8..b3414089 100644
--- a/python/freetoken/models/qwen3_5_moe/weight.py
+++ b/python/freetoken/models/qwen3_5_moe/weight.py
@@ -316,25 +316,36 @@ def _per_row_scale(scalar: torch.Tensor, rows: int) -> torch.Tensor:
return scalar.reshape(1).to(torch.float32).expand(rows)
-def _pt_fp8_fuse(base: str, weight: torch.Tensor, scalar: torch.Tensor, buf: dict):
- """Buffer an fp8 fusion part ``(weight, scalar)``; once all parts arrive emit the
- concatenated ``(.weight fp8, .weight_scale per-row fp32)``. ``[]`` while incomplete,
- ``None`` if ``base`` is not an fp8 fusion part."""
+def _pt_fp8_fuse(base: str, weight: torch.Tensor, scalar: torch.Tensor,
+ act_scale: torch.Tensor | None, buf: dict):
+ """Buffer an fp8 fusion part ``(weight, scalar, act_scale)``; once all parts arrive emit
+ the concatenated ``(.weight fp8, .weight_scale per-row fp32)`` plus the shared
+ ``.input_scale``. ``[]`` while incomplete, ``None`` if ``base`` is not an fp8 fusion part.
+
+ The fused parts all read the *same* activation, so modelopt calibrates one activation
+ range for all of them and their ``input_scale`` values come out bit-identical (verified on
+ Qwen3.8-27B-NVFP4: q/k/v all 0.2053571492, GDN qkv/z both 0.1121651828). Taking the max is
+ therefore exact here, and stays correct if a future checkpoint lets them drift."""
for fused_suffix, parts in _PT_FP8_FUSE.items():
for idx, part in enumerate(parts):
if base.endswith(part):
key = base[: -len(part)] + fused_suffix
slots = buf.setdefault(key, {})
- slots[idx] = (weight, scalar)
+ slots[idx] = (weight, scalar, act_scale)
if len(slots) < len(parts):
return []
del buf[key]
ws = [slots[i][0] for i in range(len(parts))]
ss = [_per_row_scale(slots[i][1], slots[i][0].shape[0]) for i in range(len(parts))]
- return [
+ emit = [
(key + ".weight", torch.cat(ws, dim=0)),
(key + ".weight_scale", torch.cat(ss, dim=0).contiguous()),
]
+ acts = [slots[i][2] for i in range(len(parts))]
+ if all(a is not None for a in acts):
+ emit.append((key + ".input_scale", torch.stack(
+ [a.reshape(()).to(torch.float32) for a in acts]).max()))
+ return emit
return None
@@ -466,13 +477,20 @@ def _iter_weights_attn_fp8(
if has_s and not has_s2: # per-tensor FP8 dense projection
w = f.get_tensor(raw_name) # fp8-e4m3, kept verbatim
sc = f.get_tensor(raw_base + ".weight_scale")
- emit = _pt_fp8_fuse(base, w, sc, fp8_buf)
+ # modelopt's calibrated activation scale: kept (not dropped with the
+ # other scale suffixes) so batched decode can run W8A8 instead of
+ # W8A16. Absent -> the layer stays on the W8A16 kernel.
+ act = (f.get_tensor(raw_base + ".input_scale")
+ if raw_base + ".input_scale" in keyset else None)
+ emit = _pt_fp8_fuse(base, w, sc, act, fp8_buf)
if emit is not None:
yield from emit
continue
# standalone fp8 (self_attn.o_proj, linear_attn.out_proj)
yield base + ".weight", w
yield base + ".weight_scale", _per_row_scale(sc, w.shape[0]).contiguous()
+ if act is not None:
+ yield base + ".input_scale", act.reshape(()).to(torch.float32)
continue
if has_s2: # NVFP4 dense: keep native (W4A16) where the model expects it
emit = _dense_nvfp4_emit(
diff --git a/tests/kernels/test_fp8_pertensor_linear.py b/tests/kernels/test_fp8_pertensor_linear.py
new file mode 100644
index 00000000..6b158f52
--- /dev/null
+++ b/tests/kernels/test_fp8_pertensor_linear.py
@@ -0,0 +1,127 @@
+"""Per-tensor FP8 linear: the W8A16 kernels against the dequant reference, and the W8A8
+(torch._scaled_mm) path against a W8A8 reference.
+
+Which of the two runs is fixed by the deployment -- an ``input_scale`` in the checkpoint plus
+sm_89+ -- and never by M, so each is checked against the reference matching its own contract:
+W8A16 keeps the activation in bf16 and matches the exact dequant reference, W8A8 quantizes it
+to fp8 and is held to a reference that applies the same quantization.
+"""
+
+from __future__ import annotations
+
+import pytest
+import torch
+
+if not torch.cuda.is_available(): # pragma: no cover
+ pytest.skip("CUDA required", allow_module_level=True)
+
+from freetoken.kernel.triton.e4m3_compat import e4m3_native
+
+DEV = "cuda"
+FP8 = torch.float8_e4m3fn
+
+
+def _quant_parts(part_rows: list[int], K: int, seed: int = 0):
+ """A fused per-tensor-FP8 weight: each part quantized under its own scalar, exactly how
+ modelopt stores q/k/v (and how the loader concatenates them into a per-row vector)."""
+ torch.manual_seed(seed)
+ N = sum(part_rows)
+ wf = torch.randn(N, K, device=DEV) * 0.05
+ rows, qs, scales = 0, [], []
+ for p in part_rows:
+ block = wf[rows : rows + p]
+ s = block.abs().max() / 448.0
+ qs.append((block / s).clamp(-448, 448).to(FP8))
+ scales.append(s.expand(p))
+ rows += p
+ return torch.cat(qs, 0), torch.cat(scales).contiguous().float()
+
+
+def _dequant(w8: torch.Tensor, scale: torch.Tensor) -> torch.Tensor:
+ return w8.to(torch.float32) * scale[:, None]
+
+
+# 1 = split-K GEMV; 2..16 = decode batch (the CUDA-graph ladder); 64/300 = prefill.
[email protected]("M", [1, 2, 4, 8, 16, 64, 300])
+# (5120, 14336) and (5120, 16384) are Qwen3.5-27B's fused qkv_proj / in_proj_qkvz;
+# (1024, 6144) is a standalone o_proj shape; 6112 leaves a k-mask tail.
[email protected]("K,part_rows", [
+ (5120, [12288, 1024, 1024]),
+ (1024, [6144]),
+ (6112, [512, 128]),
+])
+def test_w8a16_matches_dequant_reference(M: int, K: int, part_rows: list[int]):
+ """Without an ``input_scale`` every M stays on the W8A16 kernels, activation exact."""
+ from freetoken.kernel.triton.fp8_pertensor_linear import fp8_pertensor_linear
+
+ w8, scale = _quant_parts(part_rows, K, seed=M)
+ x = torch.randn(M, K, device=DEV, dtype=torch.bfloat16)
+ y = fp8_pertensor_linear(x, w8, scale)
+ y_ref = (x.float() @ _dequant(w8, scale).t()).to(torch.bfloat16)
+ rel = (y.float() - y_ref.float()).abs().max() / y_ref.float().abs().max().clamp(min=1e-6)
+ assert rel.item() < 2e-2, rel.item()
+
+
[email protected](not e4m3_native(), reason="torch._scaled_mm needs sm_89+")
[email protected]("M", [1, 2, 4, 16, 64])
[email protected]("part_rows,uniform", [
+ ([12288, 1024, 1024], False), # fused -> piecewise-constant scale -> row-wise
+ ([6144], True), # standalone -> one scalar -> tensor-wise
+])
+def test_w8a8_matches_w8a8_reference(M: int, part_rows: list[int], uniform: bool):
+ from freetoken.kernel.triton.fp8_pertensor_linear import fp8_pertensor_linear
+
+ K = 2048
+ w8, scale = _quant_parts(part_rows, K, seed=M)
+ x = torch.randn(M, K, device=DEV, dtype=torch.bfloat16)
+ input_scale = (x.abs().max().float() / 448.0).reshape(())
+
+ y = fp8_pertensor_linear(x, w8, scale, None, input_scale, uniform)
+
+ xq = (x.float() / input_scale).clamp(-448, 448).to(FP8)
+ y_ref = (xq.to(torch.float32) * input_scale) @ _dequant(w8, scale).t()
+ rel = ((y.float() - y_ref).norm() / y_ref.norm()).item()
+ assert rel < 1e-2, rel
+
+
[email protected](not e4m3_native(), reason="torch._scaled_mm needs sm_89+")
+def test_batch_size_does_not_change_the_numeric_scheme():
+ """A deployment that can run W8A8 must run it at every M, so that a reply reproduces at
+ bs=1 regardless of how many other requests shared its forward. Feeding the same row alone
+ and as part of a batch must therefore agree bit-for-bit."""
+ from freetoken.kernel.triton.fp8_pertensor_linear import fp8_pertensor_linear
+
+ K, part_rows = 2048, [1024, 256]
+ w8, scale = _quant_parts(part_rows, K)
+ x = torch.randn(8, K, device=DEV, dtype=torch.bfloat16)
+ input_scale = (x.abs().max().float() / 448.0).reshape(())
+
+ batched = fp8_pertensor_linear(x, w8, scale, None, input_scale, False)
+ alone = fp8_pertensor_linear(x[:1], w8, scale, None, input_scale, False)
+ assert torch.equal(alone, batched[:1])
+
+
+def test_layer_load_marks_uniform_scale_and_optional_input_scale():
+ from freetoken.kernel.triton.fp8_pertensor_linear import (
+ Fp8PerTensorColMerged,
+ Fp8PerTensorLinear,
+ )
+
+ K = 512
+ w8, scale = _quant_parts([256, 64, 64], K)
+ merged = Fp8PerTensorColMerged(K, [256, 64, 64])
+ merged.load_state_dict({
+ "weight": w8, "weight_scale": scale, "input_scale": torch.tensor(0.01, device=DEV),
+ })
+ assert merged._uniform_scale is False
+ assert merged.input_scale is not None
+
+ single = Fp8PerTensorLinear(K, 384)
+ flat = scale[:1].expand(384).contiguous()
+ single.load_state_dict({"weight": w8, "weight_scale": flat}) # no input_scale
+ assert single._uniform_scale is True
+ assert single.input_scale is None
+
+ # a reload must not trip over the input_scale it kept from the first load
+ single.load_state_dict({"weight": w8, "weight_scale": flat})
+ assert single.input_scale is None