Skip to content

FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.

perf(kernel): avoid the activation layout conversion in fp8 moe decode and read mxfp4 gemv weights as column pairs (#653)

Xiaoze Fan2d8b9cc1eac2
diff --git a/python/freetoken/kernel/moe_impl.py b/python/freetoken/kernel/moe_impl.py
index ef569fa2..17fd77c4 100644
--- a/python/freetoken/kernel/moe_impl.py
+++ b/python/freetoken/kernel/moe_impl.py
@@ -484,12 +484,15 @@ def mxfp4_splitk_gemv_triton(
k_groups = K // 32
kgps = triton.cdiv(k_groups, num_splits)
lut = get_fp4_lut(x.device)
+ assert N % 2 == 0 and block_n % 2 == 0, "the kernel reads byte pairs of adjacent columns"
+ w16 = w_blocks_t.view(torch.int16)
+ s16 = w_scales_t.view(torch.int16)
grid = (triton.cdiv(N, block_n), routes * num_splits)
mxfp4_splitk_gemv_kernel[grid](
- x, w_blocks_t, w_scales_t, bias_arg, expert_ids, partial, lut, N, K,
- stride_xe, w_blocks_t.stride(0), w_blocks_t.stride(1),
- w_scales_t.stride(0), w_scales_t.stride(1), bias_stride, N,
+ x, w16, s16, bias_arg, expert_ids, partial, lut, N, K,
+ stride_xe, w16.stride(0), w16.stride(1),
+ s16.stride(0), s16.stride(1), bias_stride, N,
HAS_BIAS=has_bias, BLOCK_N=block_n, # type: ignore
NUM_K_SPLITS=num_splits, K_GROUPS_PER_SPLIT=kgps, # type: ignore
num_warps=num_warps,
diff --git a/python/freetoken/kernel/triton/fp8_blockscale_moe.py b/python/freetoken/kernel/triton/fp8_blockscale_moe.py
index 74760516..1db54a80 100644
--- a/python/freetoken/kernel/triton/fp8_blockscale_moe.py
+++ b/python/freetoken/kernel/triton/fp8_blockscale_moe.py
@@ -35,7 +35,7 @@ def _decode_fp8_moe_kernel(
stride_twm, stride_twk, stride_tidm, stride_tidk,
BLOCK_N: tl.constexpr, BLOCK_K: tl.constexpr, # BLOCK_K == 128 (one weight scale block)
TOP_K: tl.constexpr, A_ROW_IS_ROUTE: tl.constexpr, MUL_ROUTED_WEIGHT: tl.constexpr,
- compute_type: tl.constexpr,
+ compute_type: tl.constexpr, EVEN_K: tl.constexpr,
):
route_id = tl.program_id(0)
n_block = tl.program_id(1)
@@ -54,7 +54,10 @@ def _decode_fp8_moe_kernel(
acc = tl.zeros((BLOCK_N,), dtype=tl.float32)
for kb in range(tl.cdiv(K, BLOCK_K)):
offs_k = kb * BLOCK_K + tl.arange(0, BLOCK_K)
- k_mask = offs_k < K
+ if EVEN_K:
+ k_mask = tl.full((BLOCK_K,), True, tl.int1)
+ else:
+ k_mask = offs_k < K
if e4m3_native_cx():
w = tl.load(
w_slot + offs_n[:, None] * stride_wn + offs_k[None, :] * stride_wk,
@@ -66,8 +69,10 @@ def _decode_fp8_moe_kernel(
mask=n_mask[:, None] & k_mask[None, :], other=0,
))
sc = tl.load(s_slot + sn * stride_sn + kb * stride_sk, mask=n_mask, other=0.0).to(tl.float32)
- a = tl.load(a_base + offs_k * stride_ak, mask=k_mask, other=0.0).to(tl.float32)
- acc += tl.sum(w * a[None, :], axis=1) * sc
+ # load the activation row at the weight tile shape so it gets the weight tile layout (no shared-memory layout conversion)
+ a_ptrs = tl.broadcast_to(a_base + offs_k[None, :] * stride_ak, (BLOCK_N, BLOCK_K))
+ a = tl.load(a_ptrs, mask=k_mask[None, :], other=0.0).to(tl.float32)
+ acc += tl.sum(w * a, axis=1) * sc
if MUL_ROUTED_WEIGHT:
acc *= tl.load(topk_weights_ptr + token_id * stride_twm + route_k * stride_twk)
@@ -92,7 +97,7 @@ def _decode_gemm(a, w, s, c, topk_weights, topk_ids, mul_routed_weight, a_row_is
topk_weights.stride(0), topk_weights.stride(1), topk_ids.stride(0), topk_ids.stride(1),
BLOCK_N=BLOCK_N, BLOCK_K=128, TOP_K=top_k,
A_ROW_IS_ROUTE=a_row_is_route, MUL_ROUTED_WEIGHT=mul_routed_weight,
- compute_type=_TL.get(c.dtype, tl.bfloat16), num_warps=4,
+ compute_type=_TL.get(c.dtype, tl.bfloat16), EVEN_K=w.shape[2] % 128 == 0, num_warps=4,
)
diff --git a/python/freetoken/kernel/triton/mxfp4_moe.py b/python/freetoken/kernel/triton/mxfp4_moe.py
index 3981d2a3..c0237f25 100644
--- a/python/freetoken/kernel/triton/mxfp4_moe.py
+++ b/python/freetoken/kernel/triton/mxfp4_moe.py
@@ -53,30 +53,36 @@ def mxfp4_splitk_gemv_kernel(
expert_id = tl.load(expert_ids_ptr + pid_e)
offs_n = pid_n * BLOCK_N + tl.arange(0, BLOCK_N)
mask_n = offs_n < N
+ # w_ptr / s_ptr are int16 views: one element holds the bytes of columns 2p (low) and 2p+1 (high)
+ offs_p = pid_n * (BLOCK_N // 2) + tl.arange(0, BLOCK_N // 2)
+ mask_p = offs_p < N // 2
k_groups = K // 32
kg0 = pid_k * K_GROUPS_PER_SPLIT
kg1 = tl.minimum(kg0 + K_GROUPS_PER_SPLIT, k_groups)
- acc = tl.zeros([BLOCK_N], dtype=tl.float32)
+ acc_even = tl.zeros([BLOCK_N // 2], dtype=tl.float32)
+ acc_odd = tl.zeros([BLOCK_N // 2], dtype=tl.float32)
w_base = expert_id * stride_we
s_base = expert_id * stride_se
x_base = pid_e * stride_xe
for kg in range(kg0, kg1):
- s_val = tl.load(s_ptr + s_base + kg * stride_sk + offs_n, mask=mask_n, other=0)
- scale_f = _e8m0_scale(s_val)
+ s_pair = tl.load(s_ptr + s_base + kg * stride_sk + offs_p, mask=mask_p, other=0).to(tl.int32)
+ scale_even = _e8m0_scale(s_pair & 0xFF)
+ scale_odd = _e8m0_scale((s_pair >> 8) & 0xFF)
for kk in tl.static_range(16):
k_packed = kg * 16 + kk
- w_byte = tl.load(
- w_ptr + w_base + k_packed * stride_wk + offs_n, mask=mask_n, other=0
+ w_pair = tl.load(
+ w_ptr + w_base + k_packed * stride_wk + offs_p, mask=mask_p, other=0
).to(tl.int32)
- lo = w_byte & 0x0F
- hi = (w_byte >> 4) & 0x0F
x_lo = tl.load(x_ptr + x_base + kg * 32 + kk * 2).to(tl.float32)
x_hi = tl.load(x_ptr + x_base + kg * 32 + kk * 2 + 1).to(tl.float32)
- acc += _fp4_table_lut(lo, lut_ptr) * scale_f * x_lo
- acc += _fp4_table_lut(hi, lut_ptr) * scale_f * x_hi
+ acc_even += _fp4_table_lut(w_pair & 0x0F, lut_ptr) * scale_even * x_lo
+ acc_even += _fp4_table_lut((w_pair >> 4) & 0x0F, lut_ptr) * scale_even * x_hi
+ acc_odd += _fp4_table_lut((w_pair >> 8) & 0x0F, lut_ptr) * scale_odd * x_lo
+ acc_odd += _fp4_table_lut((w_pair >> 12) & 0x0F, lut_ptr) * scale_odd * x_hi
+ acc = tl.reshape(tl.join(acc_even, acc_odd), (BLOCK_N,))
if HAS_BIAS and pid_k == 0:
acc += tl.load(