FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.
"""Per-tensor (per-output-row) FP8 W8A16 dense linear, shared across models.
The nvidia/modelopt ``MIXED_PRECISION`` checkpoints (Qwen3.5/3.6, ...) quantize the attention
and GatedDeltaNet projections to **per-tensor FP8** (``weight`` fp8-e4m3 + a single scalar
``weight_scale``; ``weight_bf16 = weight_fp8 * weight_scale``) while the routed experts are
NVFP4. FreeToken used to *dequantize these dense weights to bf16 at load* and run them through
cuBLAS bf16 GEMV -- which is the dominant decode cost (~54% of the step) because it reads
2x the bytes of the resident FP8 weight. This module keeps the weight FP8 and reads it
directly in a W8A16 kernel (bf16 activation, fp8 weight), halving that traffic.
Fused projections (q/k/v -> ``qkv_proj``; GDN qkv|z -> ``in_proj_qkvz``) concatenate parts
that each carry their *own* per-tensor scalar, so the scalar is broadcast to a per-output-row
vector ``weight_scale`` ``[N]`` (piecewise-constant after fusion) and applied per output row.
This is exactly equal to applying each part's scalar to its own rows.
Numerics: the kernel converts fp8->f32 and accumulates in fp32 (standard W8A16; strictly more
accurate than the previous ``weight.to(bf16) * scale`` materialization, which it replaces).
"""
from __future__ import annotations
import functools
import re
import torch
import triton
import triton.language as tl
from freetoken.kernel.triton.e4m3_compat import (
e4m3_kernel_view,
e4m3_native,
e4m3_native_cx,
e4m3_u8_to_f32,
f32_to_e4m3,
)
FP8 = torch.float8_e4m3fn
_TL_DTYPE = {torch.bfloat16: tl.bfloat16, torch.float16: tl.float16, torch.float32: tl.float32}
# Row-wise _scaled_mm on sm_89 with torch < 2.12 launches its CUTLASS stream-K kernel off the
# current stream (pytorch/pytorch#177651, fixed by pytorch/pytorch@252bb4a; #182/#72/#220), and
# some builds (Windows) have no row-wise kernel (#227). Fallback: one tensor-wise GEMM per part.
def _torch_version() -> tuple[int, int]:
m = re.match(r"(\d+)\.(\d+)", torch.__version__)
return (int(m.group(1)), int(m.group(2))) if m else (0, 0)
@functools.cache
def rowwise_scaled_mm_ok() -> bool:
"""Whether row-wise ``torch._scaled_mm`` may be issued from a side stream on this GPU.
Decided once per process, at load (never under graph capture)."""
if not torch.cuda.is_available():
return True
from freetoken.gpu_select import assigned_visible_gpu
idx = assigned_visible_gpu()
dev = torch.device("cuda", torch.cuda.current_device() if idx is None else idx)
if torch.cuda.get_device_capability(dev) == (8, 9) and _torch_version() < (2, 12):
return False
# Probe on the default stream (safe even where the launch ignores the current stream); a
# build without the row-wise kernel raises here instead of at the first forward.
try:
with torch.cuda.device(dev), torch.cuda.stream(torch.cuda.default_stream(dev)):
a = torch.zeros(16, 32, dtype=FP8, device=dev)
b = torch.zeros(32, 32, dtype=FP8, device=dev)
torch._scaled_mm(
a, b.t(), scale_a=torch.ones(16, 1, device=dev),
scale_b=torch.ones(1, 32, device=dev), out_dtype=torch.bfloat16,
)
torch.cuda.synchronize(dev)
except RuntimeError:
return False
return True
def weight_scale_segments(weight_scale: torch.Tensor) -> list[tuple[int, int]]:
"""``[start, end)`` row ranges over which ``weight_scale`` is constant (the fused parts).
Syncs; call at load."""
s = weight_scale.detach().reshape(-1).float().cpu()
change = (torch.nonzero(s[1:] != s[:-1]).flatten() + 1).tolist()
bounds = [0, *change, s.numel()]
return list(zip(bounds[:-1], bounds[1:]))
_MAX_SEGMENTS = 8 # q/k/v = 3, GDN qkv|z = 2; a genuine per-row scale stays W8A16 instead
def _segments_w8a8_ok(segments: list[tuple[int, int]]) -> bool:
"""cuBLASLt needs 16-row aligned fp8 operands; more parts than a fused projection has
means a genuine per-row scale."""
return 0 < len(segments) <= _MAX_SEGMENTS and all((e - s) % 16 == 0 for s, e in segments)
# ======================================================================================
# Decode (M==1) split-K GEMV: raw fp8 x bf16 reduction in fp32, per-row scale at reduce.
# ======================================================================================
@triton.jit
def _gemv_splitk_kernel(
a_ptr, w_ptr, part_ptr, N, K, n_kb, kb_per,
stride_ak, stride_wn, stride_wk, stride_pk, stride_pn,
BLOCK_N: tl.constexpr, BLOCK_K: tl.constexpr,
):
"""Each (pid_n, pid_k) computes the partial sum over ``kb_per`` BLOCK_K chunks for a
BLOCK_N slice of outputs. ``kb_per`` ceil-tiles K so K only needs to be a multiple of
BLOCK_K is *not* required (loads are k-masked). Per-row scale is applied in the reduce."""
pid_n = tl.program_id(0)
pid_k = tl.program_id(1)
offs_n = pid_n * BLOCK_N + tl.arange(0, BLOCK_N)
n_mask = offs_n < N
kb_start = pid_k * kb_per
acc = tl.zeros((BLOCK_N,), dtype=tl.float32)
for i in range(kb_per):
kb = kb_start + i
if kb < n_kb:
offs_k = kb * BLOCK_K + tl.arange(0, BLOCK_K)
k_mask = offs_k < K
a = tl.load(a_ptr + offs_k * stride_ak, mask=k_mask, other=0.0).to(tl.float32)
if e4m3_native_cx():
w = tl.load(
w_ptr + offs_n[:, None] * stride_wn + offs_k[None, :] * stride_wk,
mask=n_mask[:, None] & k_mask[None, :], other=0.0,
).to(tl.float32)
else:
w = e4m3_u8_to_f32(tl.load(
w_ptr + offs_n[:, None] * stride_wn + offs_k[None, :] * stride_wk,
mask=n_mask[:, None] & k_mask[None, :], other=0,
))
acc += tl.sum(w * a[None, :], axis=1)
tl.store(part_ptr + pid_k * stride_pk + offs_n * stride_pn, acc, mask=n_mask)
@triton.jit
def _splitk_reduce_kernel(
part_ptr, scale_ptr, out_ptr, N, SPLIT_K: tl.constexpr,
stride_pk, stride_pn, BLOCK: tl.constexpr, OUT: tl.constexpr,
):
offs = tl.program_id(0) * BLOCK + tl.arange(0, BLOCK)
mask = offs < N
acc = tl.zeros((BLOCK,), dtype=tl.float32)
for k in tl.static_range(SPLIT_K):
acc += tl.load(part_ptr + k * stride_pk + offs * stride_pn, mask=mask, other=0.0)
scale = tl.load(scale_ptr + offs, mask=mask, other=0.0).to(tl.float32)
tl.store(out_ptr + offs, (acc * scale).to(OUT), mask=mask)
def _gemv(a: torch.Tensor, weight: torch.Tensor, weight_scale: torch.Tensor,
out_dtype: torch.dtype) -> torch.Tensor:
"""M==1 split-K GEMV. ``a`` [K] bf16; ``weight`` [N, K] fp8; ``weight_scale`` [N] fp32."""
N, K = weight.shape
BLOCK_K = 128
n_kb = triton.cdiv(K, BLOCK_K)
BLOCK_N = 16
n_tiles = triton.cdiv(N, BLOCK_N)
split_k = max(1, min(1536 // n_tiles, n_kb))
split_k = 1 << (split_k.bit_length() - 1) # pow2 -> stable reduction order
kb_per = triton.cdiv(n_kb, split_k)
part = torch.empty((split_k, N), dtype=torch.float32, device=a.device)
_gemv_splitk_kernel[(n_tiles, split_k)](
a, weight, part, N, K, n_kb, kb_per,
a.stride(0), weight.stride(0), weight.stride(1), part.stride(0), part.stride(1),
BLOCK_N=BLOCK_N, BLOCK_K=BLOCK_K, num_warps=1,
)
out = torch.empty(N, dtype=out_dtype, device=a.device)
_splitk_reduce_kernel[(triton.cdiv(N, 256),)](
part, weight_scale, out, N, split_k, part.stride(0), part.stride(1),
BLOCK=256, OUT=_TL_DTYPE[out_dtype if out_dtype in _TL_DTYPE else torch.bfloat16],
num_warps=2,
)
return out
# ======================================================================================
# Prefill (M>1) W8A16 GEMM: fp8 weight read from HBM, upcast to bf16 in-register for the
# tensor-core dot (fp8 e4m3 -> bf16 is lossless), per-row scale applied after accumulation.
# ======================================================================================
@triton.jit
def _gemm_kernel(
a_ptr, w_ptr, scale_ptr, c_ptr, M, N, K,
stride_am, stride_ak, stride_wn, stride_wk, stride_cm, stride_cn,
BLOCK_M: tl.constexpr, BLOCK_N: tl.constexpr, BLOCK_K: tl.constexpr,
compute_type: tl.constexpr,
):
pid_m = tl.program_id(0)
pid_n = tl.program_id(1)
offs_m = pid_m * BLOCK_M + tl.arange(0, BLOCK_M)
offs_n = pid_n * BLOCK_N + tl.arange(0, BLOCK_N)
offs_k = tl.arange(0, BLOCK_K)
m_mask = offs_m < M
n_mask = offs_n < N
a_ptrs = a_ptr + offs_m[:, None] * stride_am + offs_k[None, :] * stride_ak
w_ptrs = w_ptr + offs_n[:, None] * stride_wn + offs_k[None, :] * stride_wk
acc = tl.zeros((BLOCK_M, BLOCK_N), dtype=tl.float32)
for k in range(tl.cdiv(K, BLOCK_K)):
k_rem = K - k * BLOCK_K
a = tl.load(a_ptrs, mask=m_mask[:, None] & (offs_k[None, :] < k_rem), other=0.0)
w_mask = n_mask[:, None] & (offs_k[None, :] < k_rem)
if e4m3_native_cx():
w = tl.load(w_ptrs, mask=w_mask, other=0.0).to(a.dtype)
else:
w = e4m3_u8_to_f32(tl.load(w_ptrs, mask=w_mask, other=0)).to(a.dtype)
acc += tl.dot(a, tl.trans(w), out_dtype=tl.float32)
a_ptrs += BLOCK_K * stride_ak
w_ptrs += BLOCK_K * stride_wk
scale = tl.load(scale_ptr + offs_n, mask=n_mask, other=0.0).to(tl.float32)
acc = acc * scale[None, :]
c_ptrs = c_ptr + offs_m[:, None] * stride_cm + offs_n[None, :] * stride_cn
tl.store(c_ptrs, acc.to(compute_type), mask=m_mask[:, None] & n_mask[None, :])
def _gemm(a: torch.Tensor, weight: torch.Tensor, weight_scale: torch.Tensor,
out_dtype: torch.dtype) -> torch.Tensor:
"""M>1 W8A16 GEMM. ``a`` [M, K] bf16; ``weight`` [N, K] fp8; ``weight_scale`` [N] fp32."""
M, K = a.shape
N = weight.shape[0]
compute = out_dtype if out_dtype in _TL_DTYPE else torch.bfloat16
out = torch.empty((M, N), dtype=compute, device=a.device)
BLOCK_M = 64 if M >= 64 else 32
BLOCK_N, BLOCK_K = 128, 64
grid = (triton.cdiv(M, BLOCK_M), triton.cdiv(N, BLOCK_N))
_gemm_kernel[grid](
a, weight, weight_scale, out, M, N, K,
a.stride(0), a.stride(1), weight.stride(0), weight.stride(1), out.stride(0), out.stride(1),
BLOCK_M=BLOCK_M, BLOCK_N=BLOCK_N, BLOCK_K=BLOCK_K, compute_type=_TL_DTYPE[compute],
num_warps=8 if M >= 64 else 4, num_stages=3,
)
return out
# ======================================================================================
# Batched decode (M > 1) W8A8: quantize the activation with the checkpoint's static
# per-tensor ``input_scale`` and hand both fp8 operands to cuBLASLt via torch._scaled_mm.
#
# ``_gemm`` above is a *prefill* kernel: grid is (cdiv(M, BLOCK_M), cdiv(N, 128)) with no
# split-K, so at decode batch sizes the M axis contributes a single block row and the whole
# GEMM runs on cdiv(N, 128) CTAs -- 8 of them for N=1024, on a 188-SM part. Its runtime then
# tracks K and ignores both N and M: latency-bound, not bandwidth-bound. Measured on
# Qwen3.5-27B's projections (RTX PRO 6000, 1461 GB/s copy roof): 594 GB/s flat across
# M=2..16, against 1334 GB/s for the M=1 GEMV over the same weights. Routing M>=2 to
# cuBLASLt restores ~1300 GB/s -- 10.15 ms -> 5.56 ms of per-decode-step GEMM.
#
# Requires sm_89+ (torch._scaled_mm's floor; Ampere has no FP8 tensor cores) *and* a
# checkpoint carrying ``input_scale``. Without either, ``_gemm`` still runs: glm_moe_dsa
# quantizes to fp8 at load with genuine per-row scales and has no activation scale at all.
# ======================================================================================
_E4M3_MAX = 448.0 # largest finite e4m3 magnitude
@triton.jit
def _static_quant_kernel(x_ptr, out_ptr, scale_ptr, n_elements, BLOCK: tl.constexpr):
"""bf16 activation -> fp8-e4m3 under one broadcast per-tensor scale."""
offs = tl.program_id(0) * BLOCK + tl.arange(0, BLOCK)
mask = offs < n_elements
inv = 1.0 / tl.load(scale_ptr).to(tl.float32)
v = tl.load(x_ptr + offs, mask=mask, other=0.0).to(tl.float32) * inv
v = tl.minimum(tl.maximum(v, -448.0), 448.0)
tl.store(out_ptr + offs, f32_to_e4m3(v), mask=mask)
def _static_quant(a: torch.Tensor, input_scale: torch.Tensor) -> torch.Tensor:
"""One launch, no host sync: the scale is read from device memory inside the kernel, so
this stays CUDA-graph safe (fixed shapes, no dependence on tensor *values* on the host)."""
n = a.numel()
out = torch.empty_like(a, dtype=FP8)
BLOCK = 1024
_static_quant_kernel[(triton.cdiv(n, BLOCK),)](
a, out, input_scale, n, BLOCK=BLOCK, num_warps=4,
)
return out
def _scaled_mm(
a: torch.Tensor, weight: torch.Tensor, weight_scale: torch.Tensor,
input_scale: torch.Tensor, uniform_scale: bool, out_dtype: torch.dtype,
scale_segments: list[tuple[int, int]] | None = None,
) -> torch.Tensor:
"""``a @ (weight_fp8 * weight_scale)^T`` as a W8A8 cuBLASLt GEMM.
``weight`` stays in the checkpoint-native row-major ``[N, K]``: ``.t()`` is exactly the
column-major ``[K, N]`` operand cuBLASLt wants, so the transpose is a stride change and
never a copy (unlike Marlin, or our own NVFP4 path's K-major repack).
A genuine per-tensor weight (one scalar repeated down ``weight_scale``) takes the
tensor-wise path. A fused projection, whose ``weight_scale`` is piecewise-constant
because each part carries its own scalar, takes the row-wise path -- that keeps every
part's scale exact, where vLLM/SGLang instead requantize the parts onto a shared maximum
and eat the precision loss. Row-wise costs ~4% here (5.56 ms vs 5.39 ms per step).
Where row-wise is unsafe (:func:`rowwise_scaled_mm_ok`) ``scale_segments`` is passed and
each part runs its own tensor-wise GEMM over ``weight[s:e]`` (still stride-only), outputs
concatenated: the same W8A8 scheme, not bit-identical (accumulation order differs)."""
qa = _static_quant(a, input_scale)
wt = weight.t() # [N, K] row-major -> [K, N] column-major, stride-only
sa = input_scale.reshape(())
if uniform_scale:
return torch._scaled_mm(
qa, wt, scale_a=sa, scale_b=weight_scale[0].reshape(()), out_dtype=out_dtype,
)
if scale_segments is not None:
return torch.cat([
torch._scaled_mm(
qa, weight[s:e].t(), scale_a=sa, scale_b=weight_scale[s].reshape(()),
out_dtype=out_dtype,
)
for s, e in scale_segments
], dim=1)
return torch._scaled_mm(
qa, wt,
scale_a=input_scale.reshape(1, 1).expand(a.shape[0], 1).contiguous(),
scale_b=weight_scale.reshape(1, -1),
out_dtype=out_dtype,
)
def fp8_pertensor_linear(
x: torch.Tensor, weight: torch.Tensor, weight_scale: torch.Tensor,
bias: torch.Tensor | None = None,
input_scale: torch.Tensor | None = None,
uniform_scale: bool = False,
scale_segments: list[tuple[int, int]] | None = None,
) -> torch.Tensor:
"""``y = x @ (weight_fp8 * weight_scale)^T``. ``weight`` [N, K] fp8-e4m3, ``weight_scale``
[N] fp32 (per output row). ``scale_segments``: the fused parts' row ranges, precomputed at
load by the layer; derived here (with a sync) when omitted and needed.
Whether the activation is quantized is a property of the *deployment*, never of the batch:
with ``input_scale`` on sm_89+ every M runs W8A8, otherwise every M runs W8A16 (split-K
GEMV at M=1, GEMM above it). Picking per-M would make a request's numerics depend on how
many unrelated requests happened to be in flight, so a reply would not reproduce at bs=1 --
and W8A16 at M=1 is worth only ~0.5% (5.53 ms vs 5.56 ms of per-step GEMM) anyway. vLLM and
SGLang likewise run one scheme across all M on any GPU with FP8 tensor cores."""
*lead, K = x.shape
N = weight.shape[0]
w8a8 = input_scale is not None and e4m3_native()
segments = None
if w8a8 and not uniform_scale and not rowwise_scaled_mm_ok():
segments = scale_segments if scale_segments is not None else weight_scale_segments(weight_scale)
if not _segments_w8a8_ok(segments):
w8a8 = False # W8A16 below is exact for any per-row scale and never calls _scaled_mm
if w8a8:
out = _scaled_mm(
x.reshape(-1, K), weight, weight_scale, input_scale, uniform_scale, x.dtype,
scale_segments=segments,
).reshape(*lead, N)
elif x.numel() // K == 1:
out = _gemv(x.reshape(K), e4m3_kernel_view(weight), weight_scale, x.dtype).reshape(*lead, N)
else:
out = _gemm(
x.reshape(-1, K), e4m3_kernel_view(weight), weight_scale, x.dtype,
).reshape(*lead, N)
if bias is not None:
out = out + bias.to(out.dtype)
return out
# ======================================================================================
# ======================================================================================