Skip to content

FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.

24.6 KB
Raw Blame
"""Block-scaled FP8 (e4m3) linear for DeepSeek-V4 / V4.1, matching the reference numerics.

The reference (``inference/model.py`` ``linear`` + ``inference/kernel.py``
``act_quant``/``fp8_gemm``) quantizes the *activation* to FP8 with a per-``block``
power-of-two (ue8m0) scale, then runs an FP8xFP8 block-scaled GEMM against the FP8
weight (which carries its own ``block x block`` ue8m0 scale). Both operands' scales are
applied per ``block``-K slab to a separate FP32 accumulator. This module reproduces that:

  ``y = fp8_gemm(act_quant(x, block, ue8m0), weight_fp8, weight_scale_e8m0)``

``block`` is the checkpoint's ``weight_block_size``: 128 for DeepSeek-V4, 32 for V4.1.

``act_quant`` (reference): per block ``s = 2**ceil(log2(max(|x|,1e-4)/448))`` (exact
via IEEE bit ops -> matches ``fast_round_scale``), ``x_fp8 = round_e4m3(clamp(x/s,
+-448))``, scale stored e8m0. The GEMM accumulates ``sum_k (A_fp8 @ B_fp8) * s_a * s_b``
per ``block``-K slab in FP32.

Also provides ``act_quant_fp8_inplace`` -- the fused FP8 quant+dequant round-trip the
reference applies in-place to the window / compressor KV (``act_quant(..., 64, ...,
inplace=True)``), returning BF16.

Assumes ``K % block == 0`` and ``N % block == 0`` (true for every DeepSeek-V4 / V4.1 projection).
"""

from __future__ import annotations

import torch
import triton
import triton.language as tl

from freetoken.kernel.triton.e4m3_compat import (
    e4m3_act_dtype,
    e4m3_kernel_view,
    e4m3_native_cx,
    e4m3_u8_to_f32,
    f32_to_e4m3,
    round_e4m3,
)

FP8 = torch.float8_e4m3fn
_TL_DTYPE = {torch.bfloat16: tl.bfloat16, torch.float16: tl.float16, torch.float32: tl.float32}


# ======================================================================================
# Activation FP8 quantization (ue8m0 power-of-two scale), matching reference act_quant.
# ======================================================================================
@triton.jit
def _log2_ceil(v):
    """Exact ceil(log2(v)) for v > 0 via IEEE-754 bit ops (matches fast_log2_ceil)."""
    bits = v.to(tl.uint32, bitcast=True)
    exp = ((bits >> 23) & 0xFF).to(tl.int32)
    man = (bits & 0x7FFFFF).to(tl.int32)
    return exp - 127 + tl.where(man != 0, 1, 0)


@triton.jit
def _act_quant_fp8_kernel(
    x_ptr, y_ptr, s_ptr, M, N,
    stride_xm, stride_xn, stride_ym, stride_yn, stride_sm, stride_sn,
    BLOCK_M: tl.constexpr, BLOCK: tl.constexpr,
):
    """Per-row, per-``BLOCK`` FP8 quant with ue8m0 (pow2) scale. ``s`` holds e8m0 codes."""
    pid_m = tl.program_id(0)
    pid_n = tl.program_id(1)
    offs_m = pid_m * BLOCK_M + tl.arange(0, BLOCK_M)
    offs_k = pid_n * BLOCK + tl.arange(0, BLOCK)
    m_mask = offs_m < M
    x = tl.load(
        x_ptr + offs_m[:, None] * stride_xm + offs_k[None, :] * stride_xn,
        mask=m_mask[:, None], other=0.0,
    ).to(tl.float32)
    amax = tl.max(tl.abs(x), axis=1)
    amax = tl.maximum(amax, 1e-4)
    e = _log2_ceil(amax * (1.0 / 448.0))                # [BLOCK_M]
    s = tl.exp2(e.to(tl.float32))
    y = tl.clamp(x / s[:, None], -448.0, 448.0)
    if e4m3_native_cx():
        y = f32_to_e4m3(y)
    else:
        y = round_e4m3(y)  # e4m3-grid values into the wrapper's bf16 buffer
    tl.store(
        y_ptr + offs_m[:, None] * stride_ym + offs_k[None, :] * stride_yn,
        y, mask=m_mask[:, None],
    )
    code = (e + 127).to(tl.uint8)
    tl.store(s_ptr + offs_m * stride_sm + pid_n * stride_sn, code, mask=m_mask)


def act_quant_fp8(x: torch.Tensor, block: int = 128) -> tuple[torch.Tensor, torch.Tensor]:
    """Reference ``act_quant`` (ue8m0): returns ``(x_fp8 [M,K], scale_codes [M,K//block])``
    where ``scale = 2**(code-127)``. Without native fp8 the quantized values are the
    same e4m3-grid points held in bf16 (exactly representable)."""
    *lead, K = x.shape
    assert K % block == 0, (K, block)
    x2d = x.reshape(-1, K).contiguous()
    M = x2d.shape[0]
    y = torch.empty((M, K), dtype=e4m3_act_dtype(), device=x.device)
    s = torch.empty((M, K // block), dtype=torch.uint8, device=x.device)
    BLOCK_M = 32
    grid = (triton.cdiv(M, BLOCK_M), K // block)
    _act_quant_fp8_kernel[grid](
        x2d, y, s, M, K,
        x2d.stride(0), x2d.stride(1), y.stride(0), y.stride(1), s.stride(0), s.stride(1),
        BLOCK_M=BLOCK_M, BLOCK=block,
    )
    return y, s


@triton.jit
def _act_quant_inplace_kernel(
    x_ptr, o_ptr, M, N, stride_m, stride_n, stride_om, stride_on,
    FP8_MIN, FP8_MAX, INV_MAX, FP4: tl.constexpr,
    BLOCK_M: tl.constexpr, BLOCK: tl.constexpr,
):
    """Fused quant+dequant round-trip (reference ``inplace=True``), written to ``o_ptr`` as
    the input dtype (``o_ptr==x_ptr`` for true in-place; a distinct out buffer fuses the
    copy for callers that must not clobber the input). ``FP4=False`` -> FP8 e4m3 (block 64);
    ``FP4=True`` -> FP4 e2m1 (block 32)."""
    pid_m = tl.program_id(0)
    pid_n = tl.program_id(1)
    offs_m = pid_m * BLOCK_M + tl.arange(0, BLOCK_M)
    offs_k = pid_n * BLOCK + tl.arange(0, BLOCK)
    m_mask = offs_m < M
    ptrs = x_ptr + offs_m[:, None] * stride_m + offs_k[None, :] * stride_n
    x = tl.load(ptrs, mask=m_mask[:, None], other=0.0).to(tl.float32)
    amax = tl.max(tl.abs(x), axis=1)
    if FP4:
        amax = tl.maximum(amax, 6.0 * (2.0 ** -126))
    else:
        amax = tl.maximum(amax, 1e-4)
    e = _log2_ceil(amax * INV_MAX)
    s = tl.exp2(e.to(tl.float32))
    q = tl.clamp(x / s[:, None], FP8_MIN, FP8_MAX)
    if FP4:
        q = _round_fp4(q)
    elif e4m3_native_cx():
        q = f32_to_e4m3(q).to(tl.float32)
    else:
        q = round_e4m3(q)
    optrs = o_ptr + offs_m[:, None] * stride_om + offs_k[None, :] * stride_on
    y = (q * s[:, None]).to(optrs.dtype.element_ty)
    tl.store(optrs, y, mask=m_mask[:, None])


@triton.jit
def _round_fp4(x):
    """Round to nearest float4_e2m1fn value in {0,.5,1,1.5,2,3,4,6} (signed), in FP32.

    Matches the hardware ``float4_e2m1fn`` cast: round-to-nearest, ties-to-even on the
    grid magnitudes. Even-magnitude grid points are {0, 1.0, 2.0, 4.0} (even mantissa
    bit), so the odd-magnitude midpoints (0.75, 1.75, 3.5) round UP to the even neighbor
    while the even-magnitude midpoints (0.25, 1.25, 2.5, 5.0) round toward the even one.
    Verified against the tilelang reference fp4 cast (probe: 0.75->1, 1.75->2, 3.5->4)."""
    sign = tl.where(x < 0, -1.0, 1.0)
    a = tl.abs(x)
    r = tl.where(
        a <= 0.25, 0.0,          # 0.25 tie -> 0.0 (even)
        tl.where(a < 0.75, 0.5,  # 0.75 tie -> 1.0 (even)
        tl.where(a <= 1.25, 1.0, # 1.25 tie -> 1.0 (even)
        tl.where(a < 1.75, 1.5,  # 1.75 tie -> 2.0 (even)
        tl.where(a <= 2.5, 2.0,  # 2.5 tie -> 2.0 (even)
        tl.where(a < 3.5, 3.0,   # 3.5 tie -> 4.0 (even)
        tl.where(a <= 5.0, 4.0, 6.0)))))))
    return sign * r


def act_quant_fp8_inplace(x: torch.Tensor, block: int = 64) -> torch.Tensor:
    """Reference ``act_quant(x, block, ue8m0, e8m0, inplace=True)``: FP8 quant+dequant
    round-trip written back into ``x`` (BF16). Operates on the (possibly strided) view."""
    *lead, N = x.shape
    assert N % block == 0, (N, block)
    x2d = x.reshape(-1, N)
    M = x2d.shape[0]
    BLOCK_M = 32
    grid = (triton.cdiv(M, BLOCK_M), N // block)
    _act_quant_inplace_kernel[grid](
        x2d, x2d, M, N, x2d.stride(0), x2d.stride(1), x2d.stride(0), x2d.stride(1),
        -448.0, 448.0, 1.0 / 448.0, False, BLOCK_M=BLOCK_M, BLOCK=block,
    )
    return x


def act_quant_fp8_roundtrip(x: torch.Tensor, block: int = 128) -> torch.Tensor:
    """FP8 quant+dequant round-trip into a fresh contiguous BF16 tensor (fuses the copy --
    for callers that must keep ``x`` intact, e.g. the MoE expert input shared with the
    gate / shared expert). Numerically identical to ``act_quant_fp8_inplace(x.clone())``."""
    *lead, N = x.shape
    assert N % block == 0, (N, block)
    x2d = x.reshape(-1, N)
    out = torch.empty_like(x2d)
    M = x2d.shape[0]
    BLOCK_M = 32
    grid = (triton.cdiv(M, BLOCK_M), N // block)
    _act_quant_inplace_kernel[grid](
        x2d, out, M, N, x2d.stride(0), x2d.stride(1), out.stride(0), out.stride(1),
        -448.0, 448.0, 1.0 / 448.0, False, BLOCK_M=BLOCK_M, BLOCK=block,
    )
    return out.reshape(x.shape)


def fp4_act_quant_inplace(x: torch.Tensor, block: int = 32) -> torch.Tensor:
    """Reference ``fp4_act_quant(x, block, inplace=True)``: FP4 quant+dequant round-trip
    written back into ``x`` (BF16)."""
    *lead, N = x.shape
    assert N % block == 0, (N, block)
    x2d = x.reshape(-1, N)
    M = x2d.shape[0]
    BLOCK_M = 32
    grid = (triton.cdiv(M, BLOCK_M), N // block)
    _act_quant_inplace_kernel[grid](
        x2d, x2d, M, N, x2d.stride(0), x2d.stride(1), x2d.stride(0), x2d.stride(1),
        -6.0, 6.0, 1.0 / 6.0, True, BLOCK_M=BLOCK_M, BLOCK=block,
    )
    return x


# ======================================================================================
# FP8 (act) x FP8 (weight) block-scaled GEMM / GEMV.
# ======================================================================================
@triton.jit
def _fp8_act_gemm_kernel(
    a_ptr,            # [M, K] float8_e4m3fn (quantized activation)
    w_ptr,            # [N, K] float8_e4m3fn
    sa_ptr,           # [M, K//BLOCK_K] uint8 (e8m0 act codes)
    sb_ptr,           # [N//BLOCK_K, K//BLOCK_K] uint8 (e8m0 weight codes; square BLOCK_K blocks)
    c_ptr,            # [M, N] compute dtype
    M, N, K,
    stride_am, stride_ak, stride_wn, stride_wk,
    stride_sam, stride_sak, stride_sbn, stride_sbk,
    stride_cm, stride_cn,
    BLOCK_M: tl.constexpr, BLOCK_N: tl.constexpr, BLOCK_K: tl.constexpr,
    compute_type: tl.constexpr,
):
    """``BLOCK_K`` is the weight's square block edge, so each K step maps to exactly one scale
    column. When the N tile IS one scale row (``BLOCK_N == BLOCK_K``, the DeepSeek-V4 block-128
    case) the weight scale is a scalar per step and the N masks vanish -- the original kernel;
    otherwise (block 32) the tile spans ``BLOCK_N // BLOCK_K`` scale rows and N is masked."""
    SCALAR_SCALE: tl.constexpr = BLOCK_N == BLOCK_K
    pid_m = tl.program_id(0)
    pid_n = tl.program_id(1)
    offs_m = pid_m * BLOCK_M + tl.arange(0, BLOCK_M)
    offs_n = pid_n * BLOCK_N + tl.arange(0, BLOCK_N)
    offs_k = tl.arange(0, BLOCK_K)
    m_mask = offs_m < M
    a_ptrs = a_ptr + offs_m[:, None] * stride_am + offs_k[None, :] * stride_ak
    w_ptrs = w_ptr + offs_n[:, None] * stride_wn + offs_k[None, :] * stride_wk
    acc = tl.zeros((BLOCK_M, BLOCK_N), dtype=tl.float32)
    num_k = tl.cdiv(K, BLOCK_K)
    if SCALAR_SCALE:
        for k in range(num_k):
            a = tl.load(a_ptrs, mask=m_mask[:, None], other=0.0)
            w = tl.load(w_ptrs)
            if e4m3_native_cx():
                p = tl.dot(a, tl.trans(w), out_dtype=tl.float32)
            else:
                # bf16 dot on the same e4m3 grid: operands exact in bf16, fp32 acc
                p = tl.dot(a, tl.trans(e4m3_u8_to_f32(w).to(tl.bfloat16)), out_dtype=tl.float32)
            sa_code = tl.load(sa_ptr + offs_m * stride_sam + k * stride_sak, mask=m_mask, other=0)
            sca = tl.exp2(sa_code.to(tl.float32) - 127.0)            # [BLOCK_M]
            sb_code = tl.load(sb_ptr + pid_n * stride_sbn + k * stride_sbk)
            scb = tl.exp2(sb_code.to(tl.float32) - 127.0)            # scalar (one N block)
            acc += p * sca[:, None] * scb
            a_ptrs += BLOCK_K * stride_ak
            w_ptrs += BLOCK_K * stride_wk
        c_ptrs = c_ptr + offs_m[:, None] * stride_cm + offs_n[None, :] * stride_cn
        tl.store(c_ptrs, acc.to(compute_type), mask=m_mask[:, None])
    else:
        n_mask = offs_n < N  # N is a multiple of BLOCK_K, not necessarily of BLOCK_N
        sn = offs_n // BLOCK_K
        for k in range(num_k):
            a = tl.load(a_ptrs, mask=m_mask[:, None], other=0.0)
            w = tl.load(w_ptrs, mask=n_mask[:, None], other=0.0)
            if e4m3_native_cx():
                p = tl.dot(a, tl.trans(w), out_dtype=tl.float32)
            else:
                p = tl.dot(a, tl.trans(e4m3_u8_to_f32(w).to(tl.bfloat16)), out_dtype=tl.float32)
            sa_code = tl.load(sa_ptr + offs_m * stride_sam + k * stride_sak, mask=m_mask, other=0)
            sca = tl.exp2(sa_code.to(tl.float32) - 127.0)            # [BLOCK_M]
            sb_code = tl.load(sb_ptr + sn * stride_sbn + k * stride_sbk, mask=n_mask, other=0)
            scb = tl.exp2(sb_code.to(tl.float32) - 127.0)            # [BLOCK_N]
            acc += p * sca[:, None] * scb[None, :]
            a_ptrs += BLOCK_K * stride_ak
            w_ptrs += BLOCK_K * stride_wk
        c_ptrs = c_ptr + offs_m[:, None] * stride_cm + offs_n[None, :] * stride_cn
        tl.store(c_ptrs, acc.to(compute_type), mask=m_mask[:, None] & n_mask[None, :])


@triton.jit
def _fp8_gemv_kernel(
    x_ptr,            # [M, G, K] activation in the compute dtype (bf16); QUANT_ACT: quantized in-kernel per SLAB
    w_ptr,            # [G * N, K] float8_e4m3fn (or its uint8 view on pre-Ada parts)
    sb_ptr,           # [G * N // SLAB, K // SLAB] uint8 (e8m0 weight codes)
    out_ptr,          # [M, G * N] OUT dtype (SPLIT_K == 1)
    part_ptr,         # [SPLIT_K, M, G * N] fp32 partials (SPLIT_K > 1)
    M, N, K,
    stride_xm, stride_xg, stride_wn, stride_wk, stride_sbn, stride_sbk, stride_om, stride_pk, stride_pm,
    BLOCK_M: tl.constexpr, BLOCK_N: tl.constexpr, BLOCK_K: tl.constexpr, SLAB: tl.constexpr, SPLIT_K: tl.constexpr,
    QUANT_ACT: tl.constexpr, OUT: tl.constexpr,
):
    """Small-M (decode) block-scaled fp8 GEMV/GEMM: one pass over the weight at near HBM bandwidth.

    Each program owns ``BLOCK_N`` rows and ``K // SPLIT_K`` contiguous K, walked ``BLOCK_K`` (several
    ``SLAB``-wide scale slabs) at a time: a wide tile keeps enough bytes in flight per SM to saturate
    the memory system, which a slab-at-a-time walk (512 B per load at the V4.1 slab of 32) cannot.
    Every weight tile is applied to all ``M`` (<= ``BLOCK_M``) activation rows, so a decode batch
    costs one weight read like a single token does (the tensor-core GEMM path is 3-6x slower at
    M = 2..8: too few tiles, tiny K steps). ``QUANT_ACT`` quantizes the activation here rather than
    by a separate ``act_quant`` launch: every program recomputes the ue8m0 scale and the e4m3 rounding
    of the slabs it touches (the reference formula, bit-identical to ``act_quant_fp8``) from the
    L2-resident bf16 rows, so the path is one kernel (plus a small reduce when ``SPLIT_K > 1``).
    Without it the activation stays bf16 (W8A16: the reference's bf16 einsum over a dequantized
    weight, e.g. ``wo_a``). The third grid axis is an independent group (block-diagonal weights:
    group ``g`` reads ``x[:, g]`` and rows ``[g * N, (g + 1) * N)``)."""
    tl.static_assert(BLOCK_K % SLAB == 0, "the K tile holds whole scale slabs")
    NSLAB: tl.constexpr = BLOCK_K // SLAB
    pid_n = tl.program_id(0)
    pid_k = tl.program_id(1)
    g = tl.program_id(2).to(tl.int64)
    x_ptr += g * stride_xg
    offs_n = pid_n * BLOCK_N + tl.arange(0, BLOCK_N)
    n_mask = offs_n < N
    sn = (g * N + offs_n) // SLAB
    offs_n = g * N + offs_n
    rows = tl.arange(0, BLOCK_M)
    k_per = K // SPLIT_K
    k_start = pid_k * k_per
    acc = tl.zeros((BLOCK_M, BLOCK_N), dtype=tl.float32)
    for k0 in range(0, k_per, BLOCK_K):
        offs_k = k_start + k0 + tl.arange(0, BLOCK_K)
        # --- the weight tile, dequantized per slab (read once for every activation row) ---
        w_ptrs = w_ptr + offs_n[:, None] * stride_wn + offs_k[None, :] * stride_wk
        if e4m3_native_cx():
            w = tl.load(w_ptrs, mask=n_mask[:, None], other=0.0).to(tl.float32)
        else:
            w = e4m3_u8_to_f32(tl.load(w_ptrs, mask=n_mask[:, None], other=0))
        kb = (k_start + k0) // SLAB + tl.arange(0, NSLAB)
        sb_code = tl.load(sb_ptr + sn[:, None] * stride_sbn + kb[None, :] * stride_sbk, mask=n_mask[:, None], other=0)
        scb = tl.exp2(sb_code.to(tl.float32) - 127.0)  # [BLOCK_N, NSLAB]
        for m in tl.static_range(BLOCK_M):
            if m < M:
                xs = tl.reshape(tl.load(x_ptr + m * stride_xm + offs_k).to(tl.float32), (NSLAB, SLAB))
                if QUANT_ACT:
                    # --- reference act_quant on this row's slabs: s = 2**ceil(log2(max(|x|, 1e-4) / 448)) ---
                    amax = tl.maximum(tl.max(tl.abs(xs), axis=1), 1e-4)
                    sa = tl.exp2(_log2_ceil(amax * (1.0 / 448.0)).to(tl.float32))  # [NSLAB]
                    q = tl.clamp(xs / sa[:, None], -448.0, 448.0)
                    if e4m3_native_cx():
                        q = f32_to_e4m3(q).to(tl.float32)
                    else:
                        q = round_e4m3(q).to(tl.float32)
                else:
                    sa = tl.full((NSLAB,), 1.0, tl.float32)
                    q = xs
                a = tl.reshape(q, (BLOCK_K,))
                slab = tl.sum(tl.reshape(w * a[None, :], (BLOCK_N, NSLAB, SLAB)), axis=2)  # [BLOCK_N, NSLAB]
                contrib = tl.sum(slab * scb * sa[None, :], axis=1)  # [BLOCK_N]
                acc = tl.where(rows[:, None] == m, acc + contrib[None, :], acc)
    row_mask = rows < M
    if SPLIT_K == 1:
        tl.store(out_ptr + rows[:, None] * stride_om + offs_n[None, :], acc.to(OUT), mask=row_mask[:, None] & n_mask[None, :])
    else:
        tl.store(part_ptr + pid_k * stride_pk + rows[:, None] * stride_pm + offs_n[None, :], acc, mask=row_mask[:, None] & n_mask[None, :])


@triton.jit
def _splitk_reduce_kernel(part_ptr, out_ptr, M, N, SPLIT_K: tl.constexpr,
                          stride_pk, stride_pm, stride_om, BLOCK: tl.constexpr, OUT: tl.constexpr):
    m = tl.program_id(1)
    offs = tl.program_id(0) * BLOCK + tl.arange(0, BLOCK)
    mask = offs < N
    acc = tl.zeros((BLOCK,), dtype=tl.float32)
    for k in tl.static_range(SPLIT_K):
        acc += tl.load(part_ptr + k * stride_pk + m * stride_pm + offs, mask=mask, other=0.0)
    tl.store(out_ptr + m * stride_om + offs, acc.to(OUT), mask=mask)


_GEMV_BLOCK_N = 32
_GEMV_NARROW_BLOCK_N = 8
def _gemv_cfg(N: int, K: int, slab: int, sm_count: int) -> tuple[int, int, int, int]:
    """Choose whole scale tiles and enough independent programs to cover the device."""
    block_k = next((bk for bk in (512, 256, 128) if bk >= slab and K % bk == 0), slab)
    if triton.cdiv(N, _GEMV_BLOCK_N) >= sm_count:
        return _GEMV_BLOCK_N, block_k, 1, 4
    n_tiles = triton.cdiv(N, _GEMV_NARROW_BLOCK_N)
    split = 1
    while n_tiles * split < sm_count and (K // block_k) % (split * 2) == 0:
        split *= 2
    return _GEMV_NARROW_BLOCK_N, block_k, split, 4


GEMV_MAX_M = 4  # bound the unrolled row loop's register footprint


def _fp8_gemv(
    x: torch.Tensor, weight: torch.Tensor, sb: torch.Tensor, out_dtype: torch.dtype, block: int, *, groups: int = 1, quant_act: bool = True,
) -> torch.Tensor:
    """``x [M, groups, K]`` bf16 x block-diagonal ``weight [groups * N, K]`` fp8 -> ``[M, groups * N]``
    in ``out_dtype`` (``M <= GEMV_MAX_M``). ``quant_act`` applies the reference fp8 activation quant
    (W8A8); off, the activation stays bf16 (W8A16). ``groups == 1`` is the plain GEMV."""
    GN, K = weight.shape
    N = GN // groups
    x = x.reshape(-1, groups, K)
    M = x.shape[0]
    assert 1 <= M <= GEMV_MAX_M, M
    block_m = triton.next_power_of_2(M)
    block_n, block_k, split_k, num_warps = _gemv_cfg(GN, K, block, torch.cuda.get_device_properties(x.device).multi_processor_count)
    out = torch.empty((M, GN), dtype=out_dtype, device=x.device)
    part = out if split_k == 1 else torch.empty((split_k, M, GN), dtype=torch.float32, device=x.device)
    _fp8_gemv_kernel[(triton.cdiv(N, block_n), split_k, groups)](
        x, weight, sb, out, part, M, N, K,
        x.stride(0), x.stride(1), weight.stride(0), weight.stride(1), sb.stride(0), sb.stride(1), out.stride(0),
        part.stride(0) if split_k > 1 else 0, part.stride(1) if split_k > 1 else 0,
        BLOCK_M=block_m, BLOCK_N=block_n, BLOCK_K=block_k, SLAB=block, SPLIT_K=split_k, QUANT_ACT=quant_act, OUT=_TL_DTYPE[out_dtype],
        num_warps=num_warps,
    )
    if split_k > 1:
        _splitk_reduce_kernel[(triton.cdiv(GN, 256), M)](
            part, out, M, GN, split_k, part.stride(0), part.stride(1), out.stride(0),
            BLOCK=256, OUT=_TL_DTYPE[out_dtype], num_warps=2,
        )
    return out


def grouped_w8a16_gemv(x: torch.Tensor, weight: torch.Tensor, scale: torch.Tensor, *, block: int) -> torch.Tensor:
    """Decode form of the reference's grouped bf16 einsum ``tgd,grd->tgr`` over a block-diagonal fp8
    weight: ``x [t, G, d]`` bf16 (``t <= GEMV_MAX_M``), ``weight [G * r, d]`` fp8 with ``[G * r //
    block, d // block]`` e8m0 ``scale`` -> ``[t, G * r]`` bf16. The weight is dequantized in-kernel
    (exact in bf16: e4m3 x 2^k), the activation is not quantized, so this is the einsum's math with a
    different fp32 summation order."""
    assert x.ndim == 3 and weight.dtype == FP8
    G = x.shape[1]
    sb = (scale.view(torch.uint8) if scale.dtype == torch.float8_e8m0fnu else scale).contiguous()
    return _fp8_gemv(x.contiguous(), e4m3_kernel_view(weight), sb, x.dtype, block, groups=G, quant_act=False)


@triton.jit
def _dequant_block_fp8_kernel(w_ptr, sb_ptr, out_ptr, N, K, stride_wn, stride_sbn, stride_on, SLAB: tl.constexpr, BLOCK_N: tl.constexpr, BLOCK_K: tl.constexpr):
    pid_n = tl.program_id(0)
    pid_k = tl.program_id(1)
    offs_n = pid_n * BLOCK_N + tl.arange(0, BLOCK_N)
    offs_k = pid_k * BLOCK_K + tl.arange(0, BLOCK_K)
    mask = (offs_n < N)[:, None] & (offs_k < K)[None, :]
    if e4m3_native_cx():
        w = tl.load(w_ptr + offs_n[:, None] * stride_wn + offs_k[None, :], mask=mask, other=0.0).to(tl.float32)
    else:
        w = e4m3_u8_to_f32(tl.load(w_ptr + offs_n[:, None] * stride_wn + offs_k[None, :], mask=mask, other=0))
    code = tl.load(sb_ptr + (offs_n // SLAB)[:, None] * stride_sbn + (offs_k // SLAB)[None, :], mask=mask, other=0)
    tl.store(out_ptr + offs_n[:, None] * stride_on + offs_k[None, :], (w * tl.exp2(code.to(tl.float32) - 127.0)).to(tl.bfloat16), mask=mask)


def dequant_block_fp8(weight: torch.Tensor, scale: torch.Tensor, *, block: int) -> torch.Tensor:
    """``[N, K]`` block-scaled e4m3 -> bf16 (exact), one pass; the prefill-side companion of
    :func:`grouped_w8a16_gemv` for weights the reference consumes through a bf16 einsum."""
    N, K = weight.shape
    sb = (scale.view(torch.uint8) if scale.dtype == torch.float8_e8m0fnu else scale).contiguous()
    out = torch.empty((N, K), dtype=torch.bfloat16, device=weight.device)
    _dequant_block_fp8_kernel[(triton.cdiv(N, 64), triton.cdiv(K, 256))](
        e4m3_kernel_view(weight), sb, out, N, K, weight.stride(0), sb.stride(0), out.stride(0),
        SLAB=block, BLOCK_N=64, BLOCK_K=256, num_warps=4,
    )
    return out


def block_fp8_linear(
    x: torch.Tensor,
    weight: torch.Tensor,
    scale: torch.Tensor,
    bias: torch.Tensor | None = None,
    *,
    block: int = 128,
) -> torch.Tensor:
    """``y = act_quant(x) @ weight^T`` (reference FP8 path).

    ``x``: ``[..., K]`` bf16; ``weight``: ``[N, K]`` float8_e4m3fn; ``scale``:
    ``[N//block, K//block]`` float8_e8m0fnu (weight block scale). Activation is quantized
    to FP8 with a per-``block`` ue8m0 scale; the GEMM applies both scales per ``block``-K slab.
    """
    assert weight.dtype == FP8
    *lead, K = x.shape
    N = weight.shape[0]
    assert weight.shape[1] == K
    assert K % block == 0 and N % block == 0, (N, K, block)
    assert tuple(scale.shape) == (N // block, K // block), (tuple(scale.shape), N, K, block)
    compute_dtype = x.dtype if x.dtype in _TL_DTYPE else torch.bfloat16
    sb = scale.view(torch.uint8) if scale.dtype == torch.float8_e8m0fnu else scale
    sb = sb.contiguous()
    w = e4m3_kernel_view(weight)

    x2d = x.reshape(-1, K)
    if x2d.shape[0] == 1:
        # decode: one kernel quantizes the activation rows and streams the weight once
        out = _fp8_gemv(x2d.contiguous(), w, sb, compute_dtype, block).reshape(*lead, N)
        if bias is not None:
            out = out + bias.to(out.dtype)
        return out

    a_fp8, sa = act_quant_fp8(x, block)  # [M,K] fp8, [M,K//block] e8m0 codes
    M = a_fp8.shape[0]

    out = torch.empty((M, N), dtype=compute_dtype, device=x.device)
    BLOCK_M = 32
    BLOCK_N = 128
    BLOCK_K = block
    grid = (triton.cdiv(M, BLOCK_M), triton.cdiv(N, BLOCK_N))
    _fp8_act_gemm_kernel[grid](
        a_fp8, w, sa, sb, out,
        M, N, K,
        a_fp8.stride(0), a_fp8.stride(1), w.stride(0), w.stride(1),
        sa.stride(0), sa.stride(1), sb.stride(0), sb.stride(1),
        out.stride(0), out.stride(1),
        BLOCK_M=BLOCK_M, BLOCK_N=BLOCK_N, BLOCK_K=BLOCK_K,
        compute_type=_TL_DTYPE[compute_dtype], num_warps=4, num_stages=3,
    )
    out = out.reshape(*lead, N)
    if bias is not None:
        out = out + bias.to(out.dtype)
    return out


__all__ = ["block_fp8_linear", "act_quant_fp8", "act_quant_fp8_inplace", "fp4_act_quant_inplace", "grouped_w8a16_gemv", "dequant_block_fp8", "GEMV_MAX_M"]