FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.
perf(moe): route fused_topk through the in-repo triton router (#319)
Xiaoze Fane05cff83a04b
diff --git a/python/freetoken/kernel/backend.py b/python/freetoken/kernel/backend.pyindex 3037ad8d..fc42c49f 100644--- a/python/freetoken/kernel/backend.py+++ b/python/freetoken/kernel/backend.py@@ -31,17 +31,6 @@ def is_sgl_kernel_installed() -> bool:return _importable("sgl_kernel")-def is_triton_kernels_installed() -> bool:- """OpenAI's ``triton_kernels`` (the fused MoE router used by ``moe.fused.fused_topk``).-- Distinct from the ``triton`` runtime we always depend on: it ships with the Triton- source tree and has no Windows wheel. It is also not one of the six ops- ``freetoken.kernel.triton`` reimplements, so its call-site carries its own fallback.- """- return _importable("triton_kernels")--@functools.cachedef driver_cuda_version() -> int | None:"""Max CUDA version the installed NVIDIA driver supports (``13000`` == CUDA 13.0),diff --git a/python/freetoken/moe/fused.py b/python/freetoken/moe/fused.pyindex 4b9a4875..ecc45eae 100644--- a/python/freetoken/moe/fused.py+++ b/python/freetoken/moe/fused.py@@ -6,11 +6,7 @@ from typing import Dict, Tupleimport torchfrom freetoken.moe import BaseMoeBackend-from freetoken.utils import div_ceil, init_logger--logger = init_logger(__name__)--_warned_torch_topk = False+from freetoken.utils import div_ceildef _torch_fused_topk(@@ -19,7 +15,7 @@ def _torch_fused_topk(renormalize: bool,num_token_non_padded: torch.Tensor | None,) -> Tuple[torch.Tensor, torch.Tensor]:- """Pure-torch softmax router matching triton_kernels.topk (Windows fallback).+ """Pure-torch reference for the fused softmax router; tests compare the kernel against it.Softmax over all experts, select the top-k, and (when ``renormalize``) rescale theselected weights to sum to 1 -- the standard fused-MoE routing convention.@@ -44,52 +40,9 @@ def fused_topk() -> Tuple[torch.Tensor, torch.Tensor]:assert hidden_states.shape[0] == gating_output.shape[0], "Number of tokens mismatch"- from freetoken.kernel.backend import is_triton_kernels_installed-- # triton_kernels ships no Windows wheel, and unlike flashinfer/sgl_kernel it is not one- # of the six ops the in-repo triton kernels cover -- so this router needs its own fallback.- if not is_triton_kernels_installed():- global _warned_torch_topk- if not _warned_torch_topk:- _warned_torch_topk = True- # Once, not per call: this runs every MoE forward. On Linux a missing- # triton_kernels used to fail fast with ImportError; keep the misconfiguration- # visible without giving up the fallback that Windows needs.- logger.warning_rank0(- "fused_topk: triton_kernels is not installed -> pure-torch router fallback "- "(numerically equivalent, slower). Expected on Windows (no wheel); on Linux "- "install triton_kernels to restore the fused router."- )- return _torch_fused_topk(gating_output, topk, renormalize, num_token_non_padded)-- if topk & (topk - 1):- # triton_kernels.topk builds tl.arange(0, k), which must be a power of 2; a- # top-10 router (qwen4_exp) takes the equivalent vendored triton router instead.- from freetoken.kernel.triton.moe_router import fused_topk_softmax-- return fused_topk_softmax(gating_output, topk, renormalize, num_token_non_padded)-- from triton_kernels.topk import topk as triton_kernels_topk-- logits = gating_output.float()- softmax_first = not renormalize- if softmax_first:- logits = torch.softmax(logits, dim=-1)- sparse_topk = triton_kernels_topk(- logits,- topk,- apply_softmax=not softmax_first,- )- if hasattr(sparse_topk, "vals"):- topk_weights = sparse_topk.vals- topk_ids = sparse_topk.indx- else:- topk_weights, topk_ids = sparse_topk[:2]- topk_ids = topk_ids.to(torch.int32)- if num_token_non_padded is not None:- indices = torch.arange(0, topk_ids.shape[0], device=topk_ids.device)- topk_ids[indices >= num_token_non_padded, :] = -1- return topk_weights, topk_ids+ from freetoken.kernel.triton.moe_router import fused_topk_softmax++ return fused_topk_softmax(gating_output, topk, renormalize, num_token_non_padded)def moe_align_block_size(diff --git a/tests/moe/test_fused_moe.py b/tests/moe/test_fused_moe.pyindex 41a75c20..cafe7784 100644--- a/tests/moe/test_fused_moe.py+++ b/tests/moe/test_fused_moe.py@@ -277,8 +277,8 @@ def test_fused_experts_decode_activation_and_router_weight_modes(@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA is required")-def test_fused_topk_non_power_of_2_k_routes_vendored_router():- """triton_kernels.topk builds tl.arange(0, k) (power-of-2 only); k=10 must not reach it."""+def test_fused_topk_handles_non_power_of_2_k():+ """A top-10 router (qwen4_exp) must route like any other k."""from freetoken.moe.fused import _torch_fused_topk, fused_topkgating = torch.randn(5, 64, device="cuda")@@ -289,7 +289,7 @@ def test_fused_topk_non_power_of_2_k_routes_vendored_router():torch.testing.assert_close(weights, ref_w, rtol=1e-5, atol=1e-6)-# The vendored triton router behind that k=10 branch; fp32 logits keep the reference top-k tie-free.+# The in-repo triton router behind fused_topk; fp32 logits keep the reference top-k tie-free.# Ties get their own case below, because torch.topk does not break them by expert id.@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA is required")@pytest.mark.parametrize("renormalize", [True, False])