Skip to content

FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.

23.7 KB
Raw Blame
"""Inline-dequant NVFP4 fused-MoE Triton kernels.

These kernels read the NVFP4 expert cache (packed FP4 codes + fp8 block scales +
fp32 per-row global scale) *directly* and dequantize inside the GEMM K-loop, so the
grouped MoE GEMM never materializes a BF16 copy of the experts (no separate dequant
pass, no HBM round-trip of the 4x larger BF16 weights).

Layout, for an expert weight ``W[N, K]``:
  - ``packed[slot, n, k//2]`` uint8: two FP4 codes per byte, low nibble = even k.
  - ``scale[slot, n, k//16]`` fp8-e4m3: per 16-wide block scale.
  - ``global[slot, n]`` fp32: per-output-row scale (``weight_scale_2``).
  - ``W[n, k] = E2M1[code] * scale[n, k//16] * global[n]``.

The global scale is constant along K, so it is applied once after the K-loop.

Decode (M=1) is HBM-bandwidth bound. :func:`_decode_nvfp4_marlin_kernel` is the production
decode GEMV (int32 wide loads + deferred K reduction); :func:`_decode_nvfp4_moe_kernel` is
the original LUT-gather variant kept for A/B comparison. The marlin-style wide load lifts
MiniMax-M2 decode toward the RTX 5090 read-bandwidth ceiling (~87%); the residual gap is FP4
dequant ALU, which a swizzled-layout tensor-core path (marlin / flashinfer b12x) closes but
those need sm_80-99 / CUDA>=13 respectively.
"""

from __future__ import annotations

import functools

import torch
import triton
import triton.language as tl
from triton.language import target_info
from triton.runtime.jit import constexpr_function

from freetoken.kernel.triton.e4m3_compat import e4m3_native_cx, e4m3_u8_to_f32
from freetoken.kernel.triton.nvfp4_linear import _nvfp4_pair_f32

_E2M1_VALUES = [
    0.0, 0.5, 1.0, 1.5, 2.0, 3.0, 4.0, 6.0,
    -0.0, -0.5, -1.0, -1.5, -2.0, -3.0, -4.0, -6.0,
]


@functools.lru_cache(maxsize=None)
def _e2m1_lut(device_index: int) -> torch.Tensor:
    return torch.tensor(
        _E2M1_VALUES, dtype=torch.float32, device=torch.device("cuda", device_index)
    )


@triton.jit
def _decode_nvfp4_moe_kernel(
    a_ptr,             # [M, K] activations (compute dtype)
    packed_ptr,        # [S, N, K // 2] uint8
    scale_ptr,         # [S, N, K // 16] fp8-e4m3
    global_ptr,        # [S, N] fp32
    c_ptr,             # [M, TOP_K, N] output (compute dtype)
    topk_weights_ptr,  # [M, TOP_K] fp32
    topk_ids_ptr,      # [M, TOP_K] int32 -> cache slot
    lut_ptr,           # [16] fp32
    total_routes,
    N,
    K,
    stride_am, stride_ak,
    stride_pe, stride_pn, stride_pkb,
    stride_se, stride_sn, stride_sblk,
    stride_ge, stride_gn,
    stride_cm, stride_ck, stride_cn,
    stride_tw_m, stride_tw_k,
    stride_tid_m, stride_tid_k,
    BLOCK_SIZE_N: tl.constexpr,
    BLOCK_SIZE_KB: tl.constexpr,  # bytes processed per K-iter (covers 2*KB k-values)
    TOP_K: tl.constexpr,
    A_ROW_IS_ROUTE: tl.constexpr,
    MUL_ROUTED_WEIGHT: tl.constexpr,
    compute_type: tl.constexpr,
):
    """Original LUT-gather serial-K decode GEMV. Kept as the A/B baseline; the production
    decode path is :func:`_decode_nvfp4_marlin_kernel`."""
    route_id = tl.program_id(0)
    n_block_id = tl.program_id(1)
    token_id = route_id // TOP_K
    route_k = route_id - token_id * TOP_K

    offs_n = n_block_id * BLOCK_SIZE_N + tl.arange(0, BLOCK_SIZE_N)
    n_mask = offs_n < N

    slot = tl.load(topk_ids_ptr + token_id * stride_tid_m + route_k * stride_tid_k).to(tl.int64)
    a_row = route_id if A_ROW_IS_ROUTE else token_id
    a_base = a_ptr + a_row * stride_am

    offs_kb = tl.arange(0, BLOCK_SIZE_KB)
    K_BYTES = K // 2
    accumulator = tl.zeros((BLOCK_SIZE_N,), dtype=tl.float32)

    packed_slot = packed_ptr + slot * stride_pe
    scale_slot = scale_ptr + slot * stride_se
    for kb_start in range(0, tl.cdiv(K_BYTES, BLOCK_SIZE_KB)):
        byte_idx = kb_start * BLOCK_SIZE_KB + offs_kb
        byte_mask = byte_idx < K_BYTES

        p_ptrs = packed_slot + offs_n[None, :] * stride_pn + byte_idx[:, None] * stride_pkb
        bytes_ = tl.load(
            p_ptrs, mask=byte_mask[:, None] & n_mask[None, :], other=0
        ).to(tl.int32)
        lo = bytes_ & 0xF
        hi = (bytes_ >> 4) & 0xF
        b_lo = tl.load(lut_ptr + lo)
        b_hi = tl.load(lut_ptr + hi)

        sblk = byte_idx // 8
        s_ptrs = scale_slot + offs_n[None, :] * stride_sn + sblk[:, None] * stride_sblk
        s_mask = byte_mask[:, None] & n_mask[None, :]
        if e4m3_native_cx():
            scale = tl.load(s_ptrs, mask=s_mask, other=0.0).to(tl.float32)
        else:
            scale = e4m3_u8_to_f32(tl.load(s_ptrs, mask=s_mask, other=0))
        b_lo = b_lo * scale
        b_hi = b_hi * scale

        a_lo = tl.load(a_base + (2 * byte_idx) * stride_ak, mask=byte_mask, other=0.0).to(tl.float32)
        a_hi = tl.load(a_base + (2 * byte_idx + 1) * stride_ak, mask=byte_mask, other=0.0).to(tl.float32)
        accumulator += tl.sum(a_lo[:, None] * b_lo, axis=0)
        accumulator += tl.sum(a_hi[:, None] * b_hi, axis=0)

    g = tl.load(global_ptr + slot * stride_ge + offs_n * stride_gn, mask=n_mask, other=0.0).to(tl.float32)
    accumulator = accumulator * g

    if MUL_ROUTED_WEIGHT:
        weight = tl.load(topk_weights_ptr + token_id * stride_tw_m + route_k * stride_tw_k)
        accumulator = accumulator * weight

    c_ptrs = c_ptr + token_id * stride_cm + route_k * stride_ck + offs_n * stride_cn
    tl.store(c_ptrs, accumulator.to(compute_type), mask=(route_id < total_routes) & n_mask)


@triton.jit
def _bf16x4_to_f32(v):
    """Four bf16 packed in one int64 -> four fp32 (a bf16 is the high half of its fp32)."""
    lo = v.to(tl.int32)
    hi = (v >> 32).to(tl.int32)
    return (
        (lo << 16).to(tl.float32, bitcast=True),
        (lo & -65536).to(tl.float32, bitcast=True),
        (hi << 16).to(tl.float32, bitcast=True),
        (hi & -65536).to(tl.float32, bitcast=True),
    )


@triton.jit
def _f16x4_to_f32(v):
    lo = v.to(tl.int32)
    hi = (v >> 32).to(tl.int32)
    return (
        lo.to(tl.int16).to(tl.float16, bitcast=True).to(tl.float32),
        (lo >> 16).to(tl.int16).to(tl.float16, bitcast=True).to(tl.float32),
        hi.to(tl.int16).to(tl.float16, bitcast=True).to(tl.float32),
        (hi >> 16).to(tl.int16).to(tl.float16, bitcast=True).to(tl.float32),
    )


@triton.jit
def _decode_nvfp4_marlin_kernel(
    a_ptr,             # [M, K] activations (compute dtype)
    packed_ptr,        # [S, N, K // 8] int32 (8 fp4 codes per word, nibble j -> k=8*w+j)
    scale_ptr,         # [S, N, K // 16] fp8-e4m3
    global_ptr,        # [S, N] fp32
    c_ptr,             # [M, TOP_K, N] output (compute dtype)
    topk_weights_ptr,  # [M, TOP_K] fp32
    topk_ids_ptr,      # [M, TOP_K] int32 -> cache slot
    total_routes,
    N,
    K,
    stride_am, stride_ak,
    stride_pe, stride_pn, stride_pkw,
    stride_se, stride_sn, stride_sblk,
    stride_ge, stride_gn,
    stride_cm, stride_ck, stride_cn,
    stride_tw_m, stride_tw_k,
    stride_tid_m, stride_tid_k,
    BLOCK_SIZE_N: tl.constexpr,
    BLOCK_SIZE_KW: tl.constexpr,  # int32 words per tile (covers 8*KW k-values)
    UNROLL: tl.constexpr,         # tiles per loop trip
    TOP_K: tl.constexpr,
    A_ROW_IS_ROUTE: tl.constexpr,
    A_WIDE: tl.constexpr,         # activation rows are contiguous bf16/fp16, 16-byte aligned
    MUL_ROUTED_WEIGHT: tl.constexpr,
    compute_type: tl.constexpr,
):
    """Marlin-style NVFP4 decode GEMV: wide int32 weight loads + deferred reduction.

    Versus :func:`_decode_nvfp4_moe_kernel` (which loads the packed codes one byte at a
    time and runs a cross-lane reduction *every* K-iter) this:

      * Loads the FP4 codes as **int32 words** (8 codes / 4 bytes per element) so the HBM
        read issues wide coalesced transactions -- the single biggest lever for the
        gate/up GEMM, whose mem-only ceiling jumps from ~65% to ~85% of peak read BW.
      * **Defers** the K reduction: the ``[BLOCK_N, BLOCK_KW]`` partial accumulates across
        *all* K-iters and is reduced once at the end, removing the per-iter ``tl.sum``
        barrier so the weight loads of successive iters pipeline.

    Every load in the K-loop is shaped so the compiler keeps the whole loop in the word
    tile's register layout (no shared-memory layout conversions): the block scales are
    loaded as a half tile and interleaved in registers (one e4m3 scale per 16 k-values
    == 2 words), the e2m1 codes are decoded with the fp16 bit trick of
    :func:`_nvfp4_pair_f32` (2^14 folded into the scale), and with ``A_WIDE`` the 8
    activations of a word are read as two int64 and unpacked by shifts. Mirrors the fast
    kernel's route/tile mapping and epilogue so it is a drop-in decode GEMV (CUDA-graph
    safe: fixed shapes, no host sync)."""
    route_id = tl.program_id(0)
    n_block_id = tl.program_id(1)
    token_id = route_id // TOP_K
    route_k = route_id - token_id * TOP_K

    offs_n = n_block_id * BLOCK_SIZE_N + tl.arange(0, BLOCK_SIZE_N)
    n_mask = offs_n < N

    slot = tl.load(topk_ids_ptr + token_id * stride_tid_m + route_k * stride_tid_k).to(tl.int64)
    a_row = route_id if A_ROW_IS_ROUTE else token_id
    a_base = a_ptr + a_row * stride_am

    offs_kw = tl.arange(0, BLOCK_SIZE_KW)
    offs_kh = tl.arange(0, BLOCK_SIZE_KW // 2)
    K_WORDS = K // 8
    partial = tl.zeros((BLOCK_SIZE_N, BLOCK_SIZE_KW), dtype=tl.float32)

    p_base = packed_ptr + slot * stride_pe + offs_n[:, None] * stride_pn
    s_base = scale_ptr + slot * stride_se + offs_n[:, None] * stride_sn
    for kw_start in range(0, tl.cdiv(K_WORDS, BLOCK_SIZE_KW), UNROLL):
        for u in tl.static_range(UNROLL):
            widx = (kw_start + u) * BLOCK_SIZE_KW + offs_kw
            w_mask = widx < K_WORDS
            word = tl.load(
                p_base + widx[None, :] * stride_pkw,
                mask=n_mask[:, None] & w_mask[None, :], other=0,
            )
            hidx = (kw_start + u) * (BLOCK_SIZE_KW // 2) + offs_kh
            s_ptrs = s_base + hidx[None, :] * stride_sblk
            s_mask = n_mask[:, None] & (hidx[None, :] < K_WORDS // 2)
            if e4m3_native_cx():
                s_half = tl.load(s_ptrs, mask=s_mask, other=0.0).to(tl.float32)
            else:
                s_half = e4m3_u8_to_f32(tl.load(s_ptrs, mask=s_mask, other=0))
            s_half = s_half * 16384.0
            scale = tl.interleave(s_half, s_half)

            if A_WIDE:
                a64 = a_base.to(tl.pointer_type(tl.int64), bitcast=True)
                v = tl.load(
                    a64 + widx[:, None] * 2 + tl.arange(0, 2)[None, :],
                    mask=w_mask[:, None], other=0,
                )
                v0, v1 = tl.split(v)
                if a_ptr.dtype.element_ty == tl.bfloat16:
                    a0, a1, a2, a3 = _bf16x4_to_f32(v0)
                    a4, a5, a6, a7 = _bf16x4_to_f32(v1)
                else:
                    a0, a1, a2, a3 = _f16x4_to_f32(v0)
                    a4, a5, a6, a7 = _f16x4_to_f32(v1)
            else:
                kbase = 8 * widx
                a0 = tl.load(a_base + kbase * stride_ak, mask=w_mask, other=0.0).to(tl.float32)
                a1 = tl.load(a_base + (kbase + 1) * stride_ak, mask=w_mask, other=0.0).to(tl.float32)
                a2 = tl.load(a_base + (kbase + 2) * stride_ak, mask=w_mask, other=0.0).to(tl.float32)
                a3 = tl.load(a_base + (kbase + 3) * stride_ak, mask=w_mask, other=0.0).to(tl.float32)
                a4 = tl.load(a_base + (kbase + 4) * stride_ak, mask=w_mask, other=0.0).to(tl.float32)
                a5 = tl.load(a_base + (kbase + 5) * stride_ak, mask=w_mask, other=0.0).to(tl.float32)
                a6 = tl.load(a_base + (kbase + 6) * stride_ak, mask=w_mask, other=0.0).to(tl.float32)
                a7 = tl.load(a_base + (kbase + 7) * stride_ak, mask=w_mask, other=0.0).to(tl.float32)

            acc_w = tl.zeros((BLOCK_SIZE_N, BLOCK_SIZE_KW), dtype=tl.float32)
            b0, b4 = _nvfp4_pair_f32(word)
            b1, b5 = _nvfp4_pair_f32(word >> 4)
            b2, b6 = _nvfp4_pair_f32(word >> 8)
            b3, b7 = _nvfp4_pair_f32(word >> 12)
            acc_w += a0[None, :] * b0
            acc_w += a1[None, :] * b1
            acc_w += a2[None, :] * b2
            acc_w += a3[None, :] * b3
            acc_w += a4[None, :] * b4
            acc_w += a5[None, :] * b5
            acc_w += a6[None, :] * b6
            acc_w += a7[None, :] * b7
            partial += acc_w * scale

    accumulator = tl.sum(partial, axis=1)
    g = tl.load(global_ptr + slot * stride_ge + offs_n * stride_gn, mask=n_mask, other=0.0).to(tl.float32)
    accumulator = accumulator * g

    if MUL_ROUTED_WEIGHT:
        weight = tl.load(topk_weights_ptr + token_id * stride_tw_m + route_k * stride_tw_k)
        accumulator = accumulator * weight

    c_ptrs = c_ptr + token_id * stride_cm + route_k * stride_ck + offs_n * stride_cn
    tl.store(c_ptrs, accumulator.to(compute_type), mask=(route_id < total_routes) & n_mask)


@triton.jit
def _e2m1_f16_x2p14(code):
    """e2m1 code (low 4 bits) -> fp16 value * 2^-14: magnitude bits go to fp16 [11:9], the sign to [15]; fp16 subnormals line up the e == 0 codes."""
    bits = ((code & 7) << 9) | ((code & 8) << 12)
    return bits.to(tl.uint16).to(tl.float16, bitcast=True)


@constexpr_function
def _bf16x2_mul_native():
    return target_info.cuda_capability_geq(9, 0)


@triton.jit
def _bf16x2_scale(w, s2):
    if _bf16x2_mul_native():
        return tl.inline_asm_elementwise("mul.rn.bf16x2 $0, $1, $2;", "=r,r,r", [w, s2], tl.uint32, True, 1)
    else:
        # mul.bf16x2 is sm_90+; fma with a -0.0 addend gives the same product
        return tl.inline_asm_elementwise("{ .reg .b32 z; mov.b32 z, 0x80008000; fma.rn.bf16x2 $0, $1, $2, z; }",
                                         "=r,r,r", [w, s2], tl.uint32, True, 1)


@triton.jit
def _bf16x2_halves(w):
    lo = (w & 0xFFFF).to(tl.uint16).to(tl.bfloat16, bitcast=True)
    hi = (w >> 16).to(tl.uint16).to(tl.bfloat16, bitcast=True)
    return lo, hi


@triton.jit
def _e2m1x8_bf16(q, s2, BLOCK_N: tl.constexpr, BLOCK_W: tl.constexpr):
    """[BLOCK_N, BLOCK_W] uint32 words of 8 e2m1 codes -> [BLOCK_N, 8 * BLOCK_W] bf16 in k order, times s2 * 2^-126."""
    # each 16-bit half's top nibble becomes a bf16 sign + exponent/mantissa: e2m1 * 2^-126, exact incl. e == 0
    o0 = _bf16x2_scale((q & 0x80008000) | ((q >> 6) & 0x01C001C0), s2)
    q1 = q << 4
    o1 = _bf16x2_scale((q1 & 0x80008000) | ((q1 >> 6) & 0x01C001C0), s2)
    q2 = q << 8
    o2 = _bf16x2_scale((q2 & 0x80008000) | ((q2 >> 6) & 0x01C001C0), s2)
    q3 = q << 12
    o3 = _bf16x2_scale((q3 & 0x80008000) | ((q3 >> 6) & 0x01C001C0), s2)
    # o0..o3 hold the k pairs (3, 7), (2, 6), (1, 5), (0, 4); join appends the new axis last
    k0, k4 = _bf16x2_halves(o3)
    k1, k5 = _bf16x2_halves(o2)
    k2, k6 = _bf16x2_halves(o1)
    k3, k7 = _bf16x2_halves(o0)
    w = tl.join(tl.join(tl.join(k0, k4), tl.join(k2, k6)), tl.join(tl.join(k1, k5), tl.join(k3, k7)))
    return tl.reshape(w, (BLOCK_N, 8 * BLOCK_W))


@triton.jit
def _e2m1x2_bf16(v, s2, BLOCK_N: tl.constexpr, BLOCK_KB: tl.constexpr):
    """[BLOCK_N, BLOCK_KB] bytes of 2 e2m1 codes -> [BLOCK_N, 2 * BLOCK_KB] bf16 in k order, times s2 * 2^-126."""
    # low nibble -> low bf16, high nibble -> high bf16: sign to bit 15 / 31, exponent and mantissa to bits 8..6 / 24..22
    pair = ((v & 0x8) << 12) | ((v & 0x7) << 6) | ((v & 0x80) << 24) | ((v & 0x70) << 18)
    lo, hi = _bf16x2_halves(_bf16x2_scale(pair.to(tl.uint32, bitcast=True), s2))
    return tl.reshape(tl.join(lo, hi), (BLOCK_N, 2 * BLOCK_KB))


@triton.jit
def _prefill_nvfp4_moe_kernel(
    a_ptr,             # [M, K] activations
    packed_ptr,        # [S, N, K // 2] uint8
    scale_ptr,         # [S, N, K // 16] fp8-e4m3 (uint8 view before sm_89)
    global_ptr,        # [S, N] fp32
    c_ptr,             # [num_valid_tokens, N] output (flat over M*top_k)
    topk_weights_ptr,
    sorted_token_ids_ptr,
    expert_ids_ptr,    # cache slot per M-block
    num_tokens_post_padded_ptr,
    w_desc,            # TMA descriptor over packed as [S * N, K // 2], or None
    s_desc,            # TMA descriptor over scale as [S * N, K // 16], or None
    N,
    K,
    EM,
    num_valid_tokens,
    stride_am, stride_ak,
    stride_pe, stride_pn, stride_pkb,
    stride_se, stride_sn, stride_sblk,
    stride_ge, stride_gn,
    stride_cm, stride_cn,
    stride_tw,
    BLOCK_SIZE_M: tl.constexpr,   # tokens per block, the moe_align padding
    BLOCK_SIZE_N: tl.constexpr,
    BLOCK_SIZE_KB: tl.constexpr,  # packed bytes per K step (2 values each)
    GROUP_SIZE_M: tl.constexpr,
    MUL_ROUTED_WEIGHT: tl.constexpr,
    top_k: tl.constexpr,
    compute_type: tl.constexpr,
    USE_TMA: tl.constexpr = False,     # needs K % 256 == 0
    TMA_SCALES: tl.constexpr = False,  # with USE_TMA: scales by TMA too, else pointer loads
    FAST_DECODE: tl.constexpr = 0,     # 0: fp16 pair decode, 1: packed_ptr / w_desc are int32 views of 8 codes, 2: bytes
    SCALE_WORDS: tl.constexpr = False,  # with FAST_DECODE == 1 and pointer loads: scale_ptr is an int32 view of 4 scales
):
    # out^T = W @ X^T: wgmma reads its A operand (the decoded weights) from registers, while B must be in shared memory.
    # The token block is the MMA N dimension, so a 16-token block still fills the MMA.
    pid = tl.program_id(axis=0)
    num_pid_m = tl.cdiv(EM, BLOCK_SIZE_M)
    num_pid_n = tl.cdiv(N, BLOCK_SIZE_N)
    num_pid_in_group = GROUP_SIZE_M * num_pid_n
    group_id = pid // num_pid_in_group
    first_pid_m = group_id * GROUP_SIZE_M
    group_size_m = min(num_pid_m - first_pid_m, GROUP_SIZE_M)
    pid_m = first_pid_m + ((pid % num_pid_in_group) % group_size_m)
    pid_n = (pid % num_pid_in_group) // group_size_m

    num_tokens_post_padded = tl.load(num_tokens_post_padded_ptr)
    if pid_m * BLOCK_SIZE_M >= num_tokens_post_padded:
        return

    offs_token_id = pid_m * BLOCK_SIZE_M + tl.arange(0, BLOCK_SIZE_M)
    offs_token = tl.load(sorted_token_ids_ptr + offs_token_id).to(tl.int64)
    token_mask = offs_token < num_valid_tokens

    offs_bn = (pid_n * BLOCK_SIZE_N + tl.arange(0, BLOCK_SIZE_N)) % N
    offs_kb = tl.arange(0, BLOCK_SIZE_KB)
    offs_k = tl.arange(0, 2 * BLOCK_SIZE_KB)
    offs_sb = tl.arange(0, BLOCK_SIZE_KB // 8)
    KW: tl.constexpr = BLOCK_SIZE_KB // 4
    offs_kw = tl.arange(0, KW)
    xt_ptrs = a_ptr + (offs_token[None, :] // top_k * stride_am + offs_k[:, None] * stride_ak)

    slot = tl.load(expert_ids_ptr + pid_m).to(tl.int64)
    WORDS: tl.constexpr = FAST_DECODE == 1
    if WORDS:
        p_ptrs = packed_ptr + slot * stride_pe + offs_bn[:, None] * stride_pn + offs_kw[None, :] * stride_pkb
    else:
        p_ptrs = packed_ptr + slot * stride_pe + offs_bn[:, None] * stride_pn + offs_kb[None, :] * stride_pkb
    tl.static_assert(not SCALE_WORDS or (WORDS and not USE_TMA and BLOCK_SIZE_KB >= 32))
    SW: tl.constexpr = BLOCK_SIZE_KB // 32
    if SCALE_WORDS:
        offs_sw = tl.arange(0, SW)
        s_ptrs = scale_ptr + slot * stride_se + offs_bn[:, None] * stride_sn + offs_sw[None, :] * stride_sblk
    else:
        s_ptrs = scale_ptr + slot * stride_se + offs_bn[:, None] * stride_sn + offs_sb[None, :] * stride_sblk

    accumulator = tl.zeros((BLOCK_SIZE_N, BLOCK_SIZE_M), dtype=tl.float32)
    K_BYTES = K // 2
    # rows past N belong to the next slot; their outputs are masked on store
    w_row = (slot * N + pid_n * BLOCK_SIZE_N).to(tl.int32)
    SB: tl.constexpr = BLOCK_SIZE_KB // 8
    for kb in range(0, tl.cdiv(K_BYTES, BLOCK_SIZE_KB)):
        k_mask = kb * 2 * BLOCK_SIZE_KB + offs_k < K
        if USE_TMA and WORDS:
            v = w_desc.load([w_row, kb * KW])
        elif USE_TMA:
            v = w_desc.load([w_row, kb * BLOCK_SIZE_KB]).to(tl.int32)
        if USE_TMA and TMA_SCALES:
            # a TMA box row needs >= 16 bytes, so load the 16 scales that hold this step and pick its SB of them
            s16 = s_desc.load([w_row, (kb * SB) // 16 * 16]).to(tl.uint8, bitcast=True)
            s3 = tl.reshape(s16, (BLOCK_SIZE_N, 16 // SB, SB))
            pick = tl.arange(0, 16 // SB) == (kb * SB) % 16 // SB
            s = tl.sum(tl.where(pick[None, :, None], s3, 0), axis=1).to(tl.uint8)
            s = s.to(tl.float8e4nv, bitcast=True).to(tl.float32)
        elif USE_TMA:
            sblk_mask = kb * SB + offs_sb < K_BYTES // 8
            s = tl.load(s_ptrs, mask=sblk_mask[None, :], other=0.0).to(tl.float32)
        else:
            sblk_mask = kb * SB + offs_sb < K_BYTES // 8
            if WORDS:
                v = tl.load(p_ptrs, mask=(kb * KW + offs_kw < K_BYTES // 4)[None, :], other=0)
            else:
                byte_mask = kb * BLOCK_SIZE_KB + offs_kb < K_BYTES
                v = tl.load(p_ptrs, mask=byte_mask[None, :], other=0).to(tl.int32)
            if SCALE_WORDS:
                # 4-byte loads can go through the cp.async pipeline, byte loads cannot; spread each word over the 8 weight words it scales
                sw = tl.load(s_ptrs, mask=(kb * SW + offs_sw < K_BYTES // 32)[None, :], other=0)
                sw = tl.reshape(tl.broadcast_to(sw[:, :, None], (BLOCK_SIZE_N, SW, KW // SW)), (BLOCK_SIZE_N, KW))
                s = ((sw >> ((offs_kw // 2 % 4) * 8)[None, :]) & 0xFF).to(tl.uint8)
                if e4m3_native_cx():
                    s = s.to(tl.float8e4nv, bitcast=True).to(tl.float32)
                else:
                    s = e4m3_u8_to_f32(s)
            elif e4m3_native_cx():
                s = tl.load(s_ptrs, mask=sblk_mask[None, :], other=0.0).to(tl.float32)
            else:
                s = e4m3_u8_to_f32(tl.load(s_ptrs, mask=sblk_mask[None, :], other=0))
        if FAST_DECODE:
            # 2^118 here and the decode's 2^-126 leave 2^-8, undone in the epilogue; every step is exact in bf16
            s = (s * 3.3230699894622794e35).to(tl.bfloat16).to(tl.uint16, bitcast=True).to(tl.uint32)
            s = s | (s << 16)
            if WORDS:
                if not SCALE_WORDS:
                    # one scale per 16 values, two words
                    s = tl.reshape(tl.broadcast_to(s[:, :, None], (BLOCK_SIZE_N, SB, 2)), (BLOCK_SIZE_N, KW))
                w = _e2m1x8_bf16(v.to(tl.uint32, bitcast=True), s, BLOCK_SIZE_N, KW)
            else:
                # one scale per 16 values, eight bytes
                s = tl.reshape(tl.broadcast_to(s[:, :, None], (BLOCK_SIZE_N, SB, 8)), (BLOCK_SIZE_N, BLOCK_SIZE_KB))
                w = _e2m1x2_bf16(v, s, BLOCK_SIZE_N, BLOCK_SIZE_KB)
        else:
            # one scale per 16 values; the 2^14 undoes the decode's 2^-14
            s = tl.reshape(tl.broadcast_to((s * 16384.0)[:, :, None], (BLOCK_SIZE_N, BLOCK_SIZE_KB // 8, 8)), (BLOCK_SIZE_N, BLOCK_SIZE_KB))
            s = s.to(tl.bfloat16)
            # exact in bf16: e2m1 has <= 2 significant bits and e4m3 <= 4
            w_lo = _e2m1_f16_x2p14(v & 0xF).to(tl.bfloat16) * s
            w_hi = _e2m1_f16_x2p14(v >> 4).to(tl.bfloat16) * s
            # interleave back to k = 2 * byte + {0, 1} so one dot covers the K step
            w = tl.reshape(tl.join(w_lo, w_hi), (BLOCK_SIZE_N, 2 * BLOCK_SIZE_KB))
        xt = tl.load(xt_ptrs, mask=k_mask[:, None] & token_mask[None, :], other=0.0)
        accumulator = tl.dot(w.to(xt.dtype), xt, accumulator)
        xt_ptrs += 2 * BLOCK_SIZE_KB * stride_ak
        if not USE_TMA:
            p_ptrs += (KW if WORDS else BLOCK_SIZE_KB) * stride_pkb
        if not (USE_TMA and TMA_SCALES):
            s_ptrs += (SW if SCALE_WORDS else BLOCK_SIZE_KB // 8) * stride_sblk

    g = tl.load(global_ptr + slot * stride_ge + offs_bn * stride_gn).to(tl.float32)
    if FAST_DECODE:
        g = g * 256.0
    accumulator = accumulator * g[:, None]

    if MUL_ROUTED_WEIGHT:
        moe_weight = tl.load(topk_weights_ptr + offs_token * stride_tw, mask=token_mask, other=0)
        accumulator = accumulator * moe_weight[None, :]

    accumulator = accumulator.to(compute_type)
    offs_cn = pid_n * BLOCK_SIZE_N + tl.arange(0, BLOCK_SIZE_N)
    c_ptrs = c_ptr + stride_cm * offs_token[None, :] + stride_cn * offs_cn[:, None]
    c_mask = token_mask[None, :] & (offs_cn[:, None] < N)
    tl.store(c_ptrs, accumulator, mask=c_mask)


__all__ = [
    "_decode_nvfp4_moe_kernel",
    "_decode_nvfp4_marlin_kernel",
    "_prefill_nvfp4_moe_kernel",
    "_e2m1_lut",
]