FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.
perf(fp8): run per-tensor fp8 as W8A8 via scaled_mm (#7)
Xiaoze Fan586a6c16ce47
diff --git a/python/freetoken/kernel/triton/fp8_pertensor_linear.py b/python/freetoken/kernel/triton/fp8_pertensor_linear.pyindex 37009e0a..28a54c94 100644--- a/python/freetoken/kernel/triton/fp8_pertensor_linear.py+++ b/python/freetoken/kernel/triton/fp8_pertensor_linear.py@@ -25,8 +25,14 @@ import torchimport tritonimport triton.language as tlfrom freetoken.layers import BaseOP+from freetoken.layers.base import _concat_prefix-from freetoken.kernel.triton.e4m3_compat import e4m3_kernel_view, e4m3_native_cx, e4m3_u8_to_f32+from freetoken.kernel.triton.e4m3_compat import (+ e4m3_kernel_view,+ e4m3_native,+ e4m3_native_cx,+ e4m3_u8_to_f32,+)FP8 = torch.float8_e4m3fn_TL_DTYPE = {torch.bfloat16: tl.bfloat16, torch.float16: tl.float16, torch.float32: tl.float32}@@ -171,23 +177,108 @@ def _gemm(a: torch.Tensor, weight: torch.Tensor, weight_scale: torch.Tensor,return out+# ======================================================================================+# Batched decode (M > 1) W8A8: quantize the activation with the checkpoint's static+# per-tensor ``input_scale`` and hand both fp8 operands to cuBLASLt via torch._scaled_mm.+#+# ``_gemm`` above is a *prefill* kernel: grid is (cdiv(M, BLOCK_M), cdiv(N, 128)) with no+# split-K, so at decode batch sizes the M axis contributes a single block row and the whole+# GEMM runs on cdiv(N, 128) CTAs -- 8 of them for N=1024, on a 188-SM part. Its runtime then+# tracks K and ignores both N and M: latency-bound, not bandwidth-bound. Measured on+# Qwen3.5-27B's projections (RTX PRO 6000, 1461 GB/s copy roof): 594 GB/s flat across+# M=2..16, against 1334 GB/s for the M=1 GEMV over the same weights. Routing M>=2 to+# cuBLASLt restores ~1300 GB/s -- 10.15 ms -> 5.56 ms of per-decode-step GEMM.+#+# Requires sm_89+ (torch._scaled_mm's floor; Ampere has no FP8 tensor cores) *and* a+# checkpoint carrying ``input_scale``. Without either, ``_gemm`` still runs: glm_moe_dsa+# quantizes to fp8 at load with genuine per-row scales and has no activation scale at all.+# ======================================================================================+_E4M3_MAX = 448.0 # largest finite e4m3 magnitude+++def _static_quant_kernel(x_ptr, out_ptr, scale_ptr, n_elements, BLOCK: tl.constexpr):+ """bf16 activation -> fp8-e4m3 under one broadcast per-tensor scale."""+ offs = tl.program_id(0) * BLOCK + tl.arange(0, BLOCK)+ mask = offs < n_elements+ inv = 1.0 / tl.load(scale_ptr).to(tl.float32)+ v = tl.load(x_ptr + offs, mask=mask, other=0.0).to(tl.float32) * inv+ v = tl.minimum(tl.maximum(v, -448.0), 448.0)+ tl.store(out_ptr + offs, v.to(tl.float8e4nv), mask=mask)+++def _static_quant(a: torch.Tensor, input_scale: torch.Tensor) -> torch.Tensor:+ """One launch, no host sync: the scale is read from device memory inside the kernel, so+ this stays CUDA-graph safe (fixed shapes, no dependence on tensor *values* on the host)."""+ n = a.numel()+ out = torch.empty_like(a, dtype=FP8)+ BLOCK = 1024+ _static_quant_kernel[(triton.cdiv(n, BLOCK),)](+ a, out, input_scale, n, BLOCK=BLOCK, num_warps=4,+ )+ return out+++def _scaled_mm(+ a: torch.Tensor, weight: torch.Tensor, weight_scale: torch.Tensor,+ input_scale: torch.Tensor, uniform_scale: bool, out_dtype: torch.dtype,+) -> torch.Tensor:+ """``a @ (weight_fp8 * weight_scale)^T`` as a W8A8 cuBLASLt GEMM.++ ``weight`` stays in the checkpoint-native row-major ``[N, K]``: ``.t()`` is exactly the+ column-major ``[K, N]`` operand cuBLASLt wants, so the transpose is a stride change and+ never a copy (unlike Marlin, or our own NVFP4 path's K-major repack).++ A genuine per-tensor weight (one scalar repeated down ``weight_scale``) takes the+ tensor-wise path. A fused projection, whose ``weight_scale`` is piecewise-constant+ because each part carries its own scalar, takes the row-wise path -- that keeps every+ part's scale exact, where vLLM/SGLang instead requantize the parts onto a shared maximum+ and eat the precision loss. Row-wise costs ~4% here (5.56 ms vs 5.39 ms per step)."""+ qa = _static_quant(a, input_scale)+ wt = weight.t() # [N, K] row-major -> [K, N] column-major, stride-only+ if uniform_scale:+ return torch._scaled_mm(+ qa, wt, scale_a=input_scale.reshape(()), scale_b=weight_scale[0].reshape(()),+ out_dtype=out_dtype,+ )+ return torch._scaled_mm(+ qa, wt,+ scale_a=input_scale.reshape(1, 1).expand(a.shape[0], 1).contiguous(),+ scale_b=weight_scale.reshape(1, -1),+ out_dtype=out_dtype,+ )++def fp8_pertensor_linear(x: torch.Tensor, weight: torch.Tensor, weight_scale: torch.Tensor,bias: torch.Tensor | None = None,+ input_scale: torch.Tensor | None = None,+ uniform_scale: bool = False,) -> torch.Tensor:- """``y = x @ (weight_fp8 * weight_scale)^T``. Decode (M=1) -> split-K GEMV; prefill -> GEMM.- ``weight`` [N, K] fp8-e4m3, ``weight_scale`` [N] fp32 (per output row)."""+ """``y = x @ (weight_fp8 * weight_scale)^T``. ``weight`` [N, K] fp8-e4m3, ``weight_scale``+ [N] fp32 (per output row).++ Whether the activation is quantized is a property of the *deployment*, never of the batch:+ with ``input_scale`` on sm_89+ every M runs W8A8, otherwise every M runs W8A16 (split-K+ GEMV at M=1, GEMM above it). Picking per-M would make a request's numerics depend on how+ many unrelated requests happened to be in flight, so a reply would not reproduce at bs=1 --+ and W8A16 at M=1 is worth only ~0.5% (5.53 ms vs 5.56 ms of per-step GEMM) anyway. vLLM and+ SGLang likewise run one scheme across all M on any GPU with FP8 tensor cores."""*lead, K = x.shapeN = weight.shape[0]if _USE_REF: # numeric-reference fallback (debug / A-B)w = weight.to(x.dtype) * weight_scale.to(x.dtype)[:, None]out = (x.reshape(-1, K) @ w.t()).reshape(*lead, N)+ elif input_scale is not None and e4m3_native():+ out = _scaled_mm(+ x.reshape(-1, K), weight, weight_scale, input_scale, uniform_scale, x.dtype,+ ).reshape(*lead, N)+ elif x.numel() // K == 1:+ out = _gemv(x.reshape(K), e4m3_kernel_view(weight), weight_scale, x.dtype).reshape(*lead, N)else:- w8 = e4m3_kernel_view(weight)- if x.numel() // K == 1:- out = _gemv(x.reshape(K), w8, weight_scale, x.dtype).reshape(*lead, N)- else:- out = _gemm(x.reshape(-1, K), w8, weight_scale, x.dtype).reshape(*lead, N)+ out = _gemm(+ x.reshape(-1, K), e4m3_kernel_view(weight), weight_scale, x.dtype,+ ).reshape(*lead, N)if bias is not None:out = out + bias.to(out.dtype)return out@@ -199,7 +290,11 @@ def fp8_pertensor_linear(class Fp8PerTensorLinear(BaseOP):"""Replicated per-tensor-FP8 linear: fp8-e4m3 ``weight`` ``[out, in]`` + per-row fp32``weight_scale`` ``[out]`` (a genuine per-tensor weight stores the same scalar in every- row; a fused projection stores each part's scalar across its own rows)."""+ row; a fused projection stores each part's scalar across its own rows).++ ``input_scale`` (modelopt's calibrated per-tensor activation scale) is optional: when the+ checkpoint ships it, batched decode runs W8A8 through cuBLASLt; when it does not (models+ that quantize to fp8 at load, e.g. glm_moe_dsa), every batch size stays W8A16."""def __init__(self, in_features: int, out_features: int, has_bias: bool = False):self.in_features = in_features@@ -207,9 +302,28 @@ class Fp8PerTensorLinear(BaseOP):self.weight = torch.empty(out_features, in_features, dtype=FP8)self.weight_scale = torch.empty(out_features, dtype=torch.float32)self.bias = torch.empty(out_features) if has_bias else None+ # Set from the state dict when present. Left as None (not a tensor) so BaseOP's+ # reflective state_dict/load_state_dict skip it entirely on checkpoints without one.+ self.input_scale: torch.Tensor | None = None+ self._uniform_scale = False++ def load_state_dict(self, state_dict, *, prefix: str = "", _internal: bool = False) -> None:+ # Taken out before BaseOP's reflective pass (so it is not an "unexpected key") and+ # put back after (so that pass does not try to pop it a second time on a reload).+ input_scale = state_dict.pop(_concat_prefix(prefix, "input_scale"), None)+ self.input_scale = None+ super().load_state_dict(state_dict, prefix=prefix, _internal=_internal)+ self.input_scale = input_scale+ # Tensor-wise scaling needs one scalar for the whole weight; a fused projection is+ # only piecewise-constant, so decide once here rather than syncing on every forward.+ scale = self.weight_scale+ self._uniform_scale = bool((scale == scale[0]).all().item())def forward(self, x: torch.Tensor) -> torch.Tensor:- return fp8_pertensor_linear(x, self.weight, self.weight_scale, self.bias)+ return fp8_pertensor_linear(+ x, self.weight, self.weight_scale, self.bias,+ self.input_scale, self._uniform_scale,+ )class Fp8PerTensorColMerged(Fp8PerTensorLinear):diff --git a/python/freetoken/models/qwen3_5_moe/weight.py b/python/freetoken/models/qwen3_5_moe/weight.pyindex 926df7c8..b3414089 100644--- a/python/freetoken/models/qwen3_5_moe/weight.py+++ b/python/freetoken/models/qwen3_5_moe/weight.py@@ -316,25 +316,36 @@ def _per_row_scale(scalar: torch.Tensor, rows: int) -> torch.Tensor:return scalar.reshape(1).to(torch.float32).expand(rows)-def _pt_fp8_fuse(base: str, weight: torch.Tensor, scalar: torch.Tensor, buf: dict):- """Buffer an fp8 fusion part ``(weight, scalar)``; once all parts arrive emit the- concatenated ``(.weight fp8, .weight_scale per-row fp32)``. ``[]`` while incomplete,- ``None`` if ``base`` is not an fp8 fusion part."""+def _pt_fp8_fuse(base: str, weight: torch.Tensor, scalar: torch.Tensor,+ act_scale: torch.Tensor | None, buf: dict):+ """Buffer an fp8 fusion part ``(weight, scalar, act_scale)``; once all parts arrive emit+ the concatenated ``(.weight fp8, .weight_scale per-row fp32)`` plus the shared+ ``.input_scale``. ``[]`` while incomplete, ``None`` if ``base`` is not an fp8 fusion part.++ The fused parts all read the *same* activation, so modelopt calibrates one activation+ range for all of them and their ``input_scale`` values come out bit-identical (verified on+ Qwen3.8-27B-NVFP4: q/k/v all 0.2053571492, GDN qkv/z both 0.1121651828). Taking the max is+ therefore exact here, and stays correct if a future checkpoint lets them drift."""for fused_suffix, parts in _PT_FP8_FUSE.items():for idx, part in enumerate(parts):if base.endswith(part):key = base[: -len(part)] + fused_suffixslots = buf.setdefault(key, {})- slots[idx] = (weight, scalar)+ slots[idx] = (weight, scalar, act_scale)if len(slots) < len(parts):return []del buf[key]ws = [slots[i][0] for i in range(len(parts))]ss = [_per_row_scale(slots[i][1], slots[i][0].shape[0]) for i in range(len(parts))]- return [+ emit = [(key + ".weight", torch.cat(ws, dim=0)),(key + ".weight_scale", torch.cat(ss, dim=0).contiguous()),]+ acts = [slots[i][2] for i in range(len(parts))]+ if all(a is not None for a in acts):+ emit.append((key + ".input_scale", torch.stack(+ [a.reshape(()).to(torch.float32) for a in acts]).max()))+ return emitreturn None@@ -466,13 +477,20 @@ def _iter_weights_attn_fp8(if has_s and not has_s2: # per-tensor FP8 dense projectionw = f.get_tensor(raw_name) # fp8-e4m3, kept verbatimsc = f.get_tensor(raw_base + ".weight_scale")- emit = _pt_fp8_fuse(base, w, sc, fp8_buf)+ # modelopt's calibrated activation scale: kept (not dropped with the+ # other scale suffixes) so batched decode can run W8A8 instead of+ # W8A16. Absent -> the layer stays on the W8A16 kernel.+ act = (f.get_tensor(raw_base + ".input_scale")+ if raw_base + ".input_scale" in keyset else None)+ emit = _pt_fp8_fuse(base, w, sc, act, fp8_buf)if emit is not None:yield from emitcontinue# standalone fp8 (self_attn.o_proj, linear_attn.out_proj)yield base + ".weight", wyield base + ".weight_scale", _per_row_scale(sc, w.shape[0]).contiguous()+ if act is not None:+ yield base + ".input_scale", act.reshape(()).to(torch.float32)continueif has_s2: # NVFP4 dense: keep native (W4A16) where the model expects itemit = _dense_nvfp4_emit(diff --git a/tests/kernels/test_fp8_pertensor_linear.py b/tests/kernels/test_fp8_pertensor_linear.pynew file mode 100644index 00000000..6b158f52--- /dev/null+++ b/tests/kernels/test_fp8_pertensor_linear.py@@ -0,0 +1,127 @@+"""Per-tensor FP8 linear: the W8A16 kernels against the dequant reference, and the W8A8+(torch._scaled_mm) path against a W8A8 reference.++Which of the two runs is fixed by the deployment -- an ``input_scale`` in the checkpoint plus+sm_89+ -- and never by M, so each is checked against the reference matching its own contract:+W8A16 keeps the activation in bf16 and matches the exact dequant reference, W8A8 quantizes it+to fp8 and is held to a reference that applies the same quantization.+"""++from __future__ import annotations++import pytest+import torch++if not torch.cuda.is_available(): # pragma: no cover+ pytest.skip("CUDA required", allow_module_level=True)++from freetoken.kernel.triton.e4m3_compat import e4m3_native++DEV = "cuda"+FP8 = torch.float8_e4m3fn+++def _quant_parts(part_rows: list[int], K: int, seed: int = 0):+ """A fused per-tensor-FP8 weight: each part quantized under its own scalar, exactly how+ modelopt stores q/k/v (and how the loader concatenates them into a per-row vector)."""+ torch.manual_seed(seed)+ N = sum(part_rows)+ wf = torch.randn(N, K, device=DEV) * 0.05+ rows, qs, scales = 0, [], []+ for p in part_rows:+ block = wf[rows : rows + p]+ s = block.abs().max() / 448.0+ qs.append((block / s).clamp(-448, 448).to(FP8))+ scales.append(s.expand(p))+ rows += p+ return torch.cat(qs, 0), torch.cat(scales).contiguous().float()+++def _dequant(w8: torch.Tensor, scale: torch.Tensor) -> torch.Tensor:+ return w8.to(torch.float32) * scale[:, None]+++# 1 = split-K GEMV; 2..16 = decode batch (the CUDA-graph ladder); 64/300 = prefill.[email protected]("M", [1, 2, 4, 8, 16, 64, 300])+# (5120, 14336) and (5120, 16384) are Qwen3.5-27B's fused qkv_proj / in_proj_qkvz;+# (1024, 6144) is a standalone o_proj shape; 6112 leaves a k-mask tail.[email protected]("K,part_rows", [+ (5120, [12288, 1024, 1024]),+ (1024, [6144]),+ (6112, [512, 128]),+])+def test_w8a16_matches_dequant_reference(M: int, K: int, part_rows: list[int]):+ """Without an ``input_scale`` every M stays on the W8A16 kernels, activation exact."""+ from freetoken.kernel.triton.fp8_pertensor_linear import fp8_pertensor_linear++ w8, scale = _quant_parts(part_rows, K, seed=M)+ x = torch.randn(M, K, device=DEV, dtype=torch.bfloat16)+ y = fp8_pertensor_linear(x, w8, scale)+ y_ref = (x.float() @ _dequant(w8, scale).t()).to(torch.bfloat16)+ rel = (y.float() - y_ref.float()).abs().max() / y_ref.float().abs().max().clamp(min=1e-6)+ assert rel.item() < 2e-2, rel.item()++[email protected](not e4m3_native(), reason="torch._scaled_mm needs sm_89+")[email protected]("M", [1, 2, 4, 16, 64])[email protected]("part_rows,uniform", [+ ([12288, 1024, 1024], False), # fused -> piecewise-constant scale -> row-wise+ ([6144], True), # standalone -> one scalar -> tensor-wise+])+def test_w8a8_matches_w8a8_reference(M: int, part_rows: list[int], uniform: bool):+ from freetoken.kernel.triton.fp8_pertensor_linear import fp8_pertensor_linear++ K = 2048+ w8, scale = _quant_parts(part_rows, K, seed=M)+ x = torch.randn(M, K, device=DEV, dtype=torch.bfloat16)+ input_scale = (x.abs().max().float() / 448.0).reshape(())++ y = fp8_pertensor_linear(x, w8, scale, None, input_scale, uniform)++ xq = (x.float() / input_scale).clamp(-448, 448).to(FP8)+ y_ref = (xq.to(torch.float32) * input_scale) @ _dequant(w8, scale).t()+ rel = ((y.float() - y_ref).norm() / y_ref.norm()).item()+ assert rel < 1e-2, rel++[email protected](not e4m3_native(), reason="torch._scaled_mm needs sm_89+")+def test_batch_size_does_not_change_the_numeric_scheme():+ """A deployment that can run W8A8 must run it at every M, so that a reply reproduces at+ bs=1 regardless of how many other requests shared its forward. Feeding the same row alone+ and as part of a batch must therefore agree bit-for-bit."""+ from freetoken.kernel.triton.fp8_pertensor_linear import fp8_pertensor_linear++ K, part_rows = 2048, [1024, 256]+ w8, scale = _quant_parts(part_rows, K)+ x = torch.randn(8, K, device=DEV, dtype=torch.bfloat16)+ input_scale = (x.abs().max().float() / 448.0).reshape(())++ batched = fp8_pertensor_linear(x, w8, scale, None, input_scale, False)+ alone = fp8_pertensor_linear(x[:1], w8, scale, None, input_scale, False)+ assert torch.equal(alone, batched[:1])+++def test_layer_load_marks_uniform_scale_and_optional_input_scale():+ from freetoken.kernel.triton.fp8_pertensor_linear import (+ Fp8PerTensorColMerged,+ Fp8PerTensorLinear,+ )++ K = 512+ w8, scale = _quant_parts([256, 64, 64], K)+ merged = Fp8PerTensorColMerged(K, [256, 64, 64])+ merged.load_state_dict({+ "weight": w8, "weight_scale": scale, "input_scale": torch.tensor(0.01, device=DEV),+ })+ assert merged._uniform_scale is False+ assert merged.input_scale is not None++ single = Fp8PerTensorLinear(K, 384)+ flat = scale[:1].expand(384).contiguous()+ single.load_state_dict({"weight": w8, "weight_scale": flat}) # no input_scale+ assert single._uniform_scale is True+ assert single.input_scale is None++ # a reload must not trip over the input_scale it kept from the first load+ single.load_state_dict({"weight": w8, "weight_scale": flat})+ assert single.input_scale is None