FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.
perf(moe): skip inactive block-fp8 decode routes (#665)
cherry77-cloud7974c9cc8c54
diff --git a/python/freetoken/kernel/triton/fp8_blockscale_moe.py b/python/freetoken/kernel/triton/fp8_blockscale_moe.pyindex 429134d2..ba7284ce 100644--- a/python/freetoken/kernel/triton/fp8_blockscale_moe.py+++ b/python/freetoken/kernel/triton/fp8_blockscale_moe.py@@ -46,6 +46,11 @@ def _decode_fp8_moe_kernel(sn = offs_n // 128 # weight-scale N-block per output rowslot = tl.load(topk_ids_ptr + token_id * stride_tidm + route_k * stride_tidk).to(tl.int64)+ c_ptrs = c_ptr + token_id * stride_cm + route_k * stride_ck + offs_n * stride_cn+ if slot < 0:+ # Both GEMMs must clear inactive routes before activation/reduction, including graph replay.+ tl.store(c_ptrs, 0, mask=(route_id < total_routes) & n_mask)+ returna_row = route_id if A_ROW_IS_ROUTE else token_ida_base = a_ptr + a_row * stride_amw_slot = w_ptr + slot * stride_we@@ -76,7 +81,6 @@ def _decode_fp8_moe_kernel(if MUL_ROUTED_WEIGHT:acc *= tl.load(topk_weights_ptr + token_id * stride_twm + route_k * stride_twk)- c_ptrs = c_ptr + token_id * stride_cm + route_k * stride_ck + offs_n * stride_cntl.store(c_ptrs, acc.to(compute_type), mask=(route_id < total_routes) & n_mask)@@ -106,7 +110,7 @@ def fused_experts_decode_fp8_blockscale(topk_weights, topk_ids, activation="silu", act_alpha=1.0, act_limit=float("inf"),) -> torch.Tensor:"""Decode (bs-1) inline-dequant block-fp8 MoE. ``topk_ids`` index rows of the expert- banks (resident: expert id; offload: cache slot)."""+ banks (resident: expert id; offload: cache slot); negative slots produce zero."""from freetoken.kernel import moe_sum_reduce_tritonfrom freetoken.layers import gated_act_and_muldiff --git a/python/freetoken/layers/quantization/moe/fp8_block.py b/python/freetoken/layers/quantization/moe/fp8_block.pyindex 6cda6b1e..ab75ff96 100644--- a/python/freetoken/layers/quantization/moe/fp8_block.py+++ b/python/freetoken/layers/quantization/moe/fp8_block.py@@ -20,6 +20,7 @@ def _pad_scale(rows: int, cols: int) -> int:class TritonFp8BlockMoEKernel(MoEKernel):name = "triton"cpu_format = "fp8_block"+ supports_inactive_slots = Truedef unusable_reason(self, cfg: MoEConfig) -> str | None:reason = self._common_reject(cfg, tp_ok=False, cpu_ok=True, plain_silu_only=False)