FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.
perf(kernel): avoid the activation layout conversion in fp8 moe decode and read mxfp4 gemv weights as column pairs (#653)
Xiaoze Fan2d8b9cc1eac2
diff --git a/python/freetoken/kernel/moe_impl.py b/python/freetoken/kernel/moe_impl.pyindex ef569fa2..17fd77c4 100644--- a/python/freetoken/kernel/moe_impl.py+++ b/python/freetoken/kernel/moe_impl.py@@ -484,12 +484,15 @@ def mxfp4_splitk_gemv_triton(k_groups = K // 32kgps = triton.cdiv(k_groups, num_splits)lut = get_fp4_lut(x.device)+ assert N % 2 == 0 and block_n % 2 == 0, "the kernel reads byte pairs of adjacent columns"+ w16 = w_blocks_t.view(torch.int16)+ s16 = w_scales_t.view(torch.int16)grid = (triton.cdiv(N, block_n), routes * num_splits)mxfp4_splitk_gemv_kernel[grid](- x, w_blocks_t, w_scales_t, bias_arg, expert_ids, partial, lut, N, K,- stride_xe, w_blocks_t.stride(0), w_blocks_t.stride(1),- w_scales_t.stride(0), w_scales_t.stride(1), bias_stride, N,+ x, w16, s16, bias_arg, expert_ids, partial, lut, N, K,+ stride_xe, w16.stride(0), w16.stride(1),+ s16.stride(0), s16.stride(1), bias_stride, N,HAS_BIAS=has_bias, BLOCK_N=block_n, # type: ignoreNUM_K_SPLITS=num_splits, K_GROUPS_PER_SPLIT=kgps, # type: ignorenum_warps=num_warps,diff --git a/python/freetoken/kernel/triton/fp8_blockscale_moe.py b/python/freetoken/kernel/triton/fp8_blockscale_moe.pyindex 74760516..1db54a80 100644--- a/python/freetoken/kernel/triton/fp8_blockscale_moe.py+++ b/python/freetoken/kernel/triton/fp8_blockscale_moe.py@@ -35,7 +35,7 @@ def _decode_fp8_moe_kernel(stride_twm, stride_twk, stride_tidm, stride_tidk,BLOCK_N: tl.constexpr, BLOCK_K: tl.constexpr, # BLOCK_K == 128 (one weight scale block)TOP_K: tl.constexpr, A_ROW_IS_ROUTE: tl.constexpr, MUL_ROUTED_WEIGHT: tl.constexpr,- compute_type: tl.constexpr,+ compute_type: tl.constexpr, EVEN_K: tl.constexpr,):route_id = tl.program_id(0)n_block = tl.program_id(1)@@ -54,7 +54,10 @@ def _decode_fp8_moe_kernel(acc = tl.zeros((BLOCK_N,), dtype=tl.float32)for kb in range(tl.cdiv(K, BLOCK_K)):offs_k = kb * BLOCK_K + tl.arange(0, BLOCK_K)- k_mask = offs_k < K+ if EVEN_K:+ k_mask = tl.full((BLOCK_K,), True, tl.int1)+ else:+ k_mask = offs_k < Kif e4m3_native_cx():w = tl.load(w_slot + offs_n[:, None] * stride_wn + offs_k[None, :] * stride_wk,@@ -66,8 +69,10 @@ def _decode_fp8_moe_kernel(mask=n_mask[:, None] & k_mask[None, :], other=0,))sc = tl.load(s_slot + sn * stride_sn + kb * stride_sk, mask=n_mask, other=0.0).to(tl.float32)- a = tl.load(a_base + offs_k * stride_ak, mask=k_mask, other=0.0).to(tl.float32)- acc += tl.sum(w * a[None, :], axis=1) * sc+ # load the activation row at the weight tile shape so it gets the weight tile layout (no shared-memory layout conversion)+ a_ptrs = tl.broadcast_to(a_base + offs_k[None, :] * stride_ak, (BLOCK_N, BLOCK_K))+ a = tl.load(a_ptrs, mask=k_mask[None, :], other=0.0).to(tl.float32)+ acc += tl.sum(w * a, axis=1) * scif MUL_ROUTED_WEIGHT:acc *= tl.load(topk_weights_ptr + token_id * stride_twm + route_k * stride_twk)@@ -92,7 +97,7 @@ def _decode_gemm(a, w, s, c, topk_weights, topk_ids, mul_routed_weight, a_row_istopk_weights.stride(0), topk_weights.stride(1), topk_ids.stride(0), topk_ids.stride(1),BLOCK_N=BLOCK_N, BLOCK_K=128, TOP_K=top_k,A_ROW_IS_ROUTE=a_row_is_route, MUL_ROUTED_WEIGHT=mul_routed_weight,- compute_type=_TL.get(c.dtype, tl.bfloat16), num_warps=4,+ compute_type=_TL.get(c.dtype, tl.bfloat16), EVEN_K=w.shape[2] % 128 == 0, num_warps=4,)diff --git a/python/freetoken/kernel/triton/mxfp4_moe.py b/python/freetoken/kernel/triton/mxfp4_moe.pyindex 3981d2a3..c0237f25 100644--- a/python/freetoken/kernel/triton/mxfp4_moe.py+++ b/python/freetoken/kernel/triton/mxfp4_moe.py@@ -53,30 +53,36 @@ def mxfp4_splitk_gemv_kernel(expert_id = tl.load(expert_ids_ptr + pid_e)offs_n = pid_n * BLOCK_N + tl.arange(0, BLOCK_N)mask_n = offs_n < N+ # w_ptr / s_ptr are int16 views: one element holds the bytes of columns 2p (low) and 2p+1 (high)+ offs_p = pid_n * (BLOCK_N // 2) + tl.arange(0, BLOCK_N // 2)+ mask_p = offs_p < N // 2k_groups = K // 32kg0 = pid_k * K_GROUPS_PER_SPLITkg1 = tl.minimum(kg0 + K_GROUPS_PER_SPLIT, k_groups)- acc = tl.zeros([BLOCK_N], dtype=tl.float32)+ acc_even = tl.zeros([BLOCK_N // 2], dtype=tl.float32)+ acc_odd = tl.zeros([BLOCK_N // 2], dtype=tl.float32)w_base = expert_id * stride_wes_base = expert_id * stride_sex_base = pid_e * stride_xefor kg in range(kg0, kg1):- s_val = tl.load(s_ptr + s_base + kg * stride_sk + offs_n, mask=mask_n, other=0)- scale_f = _e8m0_scale(s_val)+ s_pair = tl.load(s_ptr + s_base + kg * stride_sk + offs_p, mask=mask_p, other=0).to(tl.int32)+ scale_even = _e8m0_scale(s_pair & 0xFF)+ scale_odd = _e8m0_scale((s_pair >> 8) & 0xFF)for kk in tl.static_range(16):k_packed = kg * 16 + kk- w_byte = tl.load(- w_ptr + w_base + k_packed * stride_wk + offs_n, mask=mask_n, other=0+ w_pair = tl.load(+ w_ptr + w_base + k_packed * stride_wk + offs_p, mask=mask_p, other=0).to(tl.int32)- lo = w_byte & 0x0F- hi = (w_byte >> 4) & 0x0Fx_lo = tl.load(x_ptr + x_base + kg * 32 + kk * 2).to(tl.float32)x_hi = tl.load(x_ptr + x_base + kg * 32 + kk * 2 + 1).to(tl.float32)- acc += _fp4_table_lut(lo, lut_ptr) * scale_f * x_lo- acc += _fp4_table_lut(hi, lut_ptr) * scale_f * x_hi+ acc_even += _fp4_table_lut(w_pair & 0x0F, lut_ptr) * scale_even * x_lo+ acc_even += _fp4_table_lut((w_pair >> 4) & 0x0F, lut_ptr) * scale_even * x_hi+ acc_odd += _fp4_table_lut((w_pair >> 8) & 0x0F, lut_ptr) * scale_odd * x_lo+ acc_odd += _fp4_table_lut((w_pair >> 12) & 0x0F, lut_ptr) * scale_odd * x_hi+ acc = tl.reshape(tl.join(acc_even, acc_odd), (BLOCK_N,))if HAS_BIAS and pid_k == 0:acc += tl.load(