Skip to content

FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.

perf(moe): route fused_topk through the in-repo triton router (#319)

Xiaoze Fane05cff83a04b
diff --git a/python/freetoken/kernel/backend.py b/python/freetoken/kernel/backend.py
index 3037ad8d..fc42c49f 100644
--- a/python/freetoken/kernel/backend.py
+++ b/python/freetoken/kernel/backend.py
@@ -31,17 +31,6 @@ def is_sgl_kernel_installed() -> bool:
return _importable("sgl_kernel")
-def is_triton_kernels_installed() -> bool:
- """OpenAI's ``triton_kernels`` (the fused MoE router used by ``moe.fused.fused_topk``).
-
- Distinct from the ``triton`` runtime we always depend on: it ships with the Triton
- source tree and has no Windows wheel. It is also not one of the six ops
- ``freetoken.kernel.triton`` reimplements, so its call-site carries its own fallback.
- """
- return _importable("triton_kernels")
-
-
@functools.cache
def driver_cuda_version() -> int | None:
"""Max CUDA version the installed NVIDIA driver supports (``13000`` == CUDA 13.0),
diff --git a/python/freetoken/moe/fused.py b/python/freetoken/moe/fused.py
index 4b9a4875..ecc45eae 100644
--- a/python/freetoken/moe/fused.py
+++ b/python/freetoken/moe/fused.py
@@ -6,11 +6,7 @@ from typing import Dict, Tuple
import torch
from freetoken.moe import BaseMoeBackend
-from freetoken.utils import div_ceil, init_logger
-
-logger = init_logger(__name__)
-
-_warned_torch_topk = False
+from freetoken.utils import div_ceil
def _torch_fused_topk(
@@ -19,7 +15,7 @@ def _torch_fused_topk(
renormalize: bool,
num_token_non_padded: torch.Tensor | None,
) -> Tuple[torch.Tensor, torch.Tensor]:
- """Pure-torch softmax router matching triton_kernels.topk (Windows fallback).
+ """Pure-torch reference for the fused softmax router; tests compare the kernel against it.
Softmax over all experts, select the top-k, and (when ``renormalize``) rescale the
selected weights to sum to 1 -- the standard fused-MoE routing convention.
@@ -44,52 +40,9 @@ def fused_topk(
) -> Tuple[torch.Tensor, torch.Tensor]:
assert hidden_states.shape[0] == gating_output.shape[0], "Number of tokens mismatch"
- from freetoken.kernel.backend import is_triton_kernels_installed
-
- # triton_kernels ships no Windows wheel, and unlike flashinfer/sgl_kernel it is not one
- # of the six ops the in-repo triton kernels cover -- so this router needs its own fallback.
- if not is_triton_kernels_installed():
- global _warned_torch_topk
- if not _warned_torch_topk:
- _warned_torch_topk = True
- # Once, not per call: this runs every MoE forward. On Linux a missing
- # triton_kernels used to fail fast with ImportError; keep the misconfiguration
- # visible without giving up the fallback that Windows needs.
- logger.warning_rank0(
- "fused_topk: triton_kernels is not installed -> pure-torch router fallback "
- "(numerically equivalent, slower). Expected on Windows (no wheel); on Linux "
- "install triton_kernels to restore the fused router."
- )
- return _torch_fused_topk(gating_output, topk, renormalize, num_token_non_padded)
-
- if topk & (topk - 1):
- # triton_kernels.topk builds tl.arange(0, k), which must be a power of 2; a
- # top-10 router (qwen4_exp) takes the equivalent vendored triton router instead.
- from freetoken.kernel.triton.moe_router import fused_topk_softmax
-
- return fused_topk_softmax(gating_output, topk, renormalize, num_token_non_padded)
-
- from triton_kernels.topk import topk as triton_kernels_topk
-
- logits = gating_output.float()
- softmax_first = not renormalize
- if softmax_first:
- logits = torch.softmax(logits, dim=-1)
- sparse_topk = triton_kernels_topk(
- logits,
- topk,
- apply_softmax=not softmax_first,
- )
- if hasattr(sparse_topk, "vals"):
- topk_weights = sparse_topk.vals
- topk_ids = sparse_topk.indx
- else:
- topk_weights, topk_ids = sparse_topk[:2]
- topk_ids = topk_ids.to(torch.int32)
- if num_token_non_padded is not None:
- indices = torch.arange(0, topk_ids.shape[0], device=topk_ids.device)
- topk_ids[indices >= num_token_non_padded, :] = -1
- return topk_weights, topk_ids
+ from freetoken.kernel.triton.moe_router import fused_topk_softmax
+
+ return fused_topk_softmax(gating_output, topk, renormalize, num_token_non_padded)
def moe_align_block_size(
diff --git a/tests/moe/test_fused_moe.py b/tests/moe/test_fused_moe.py
index 41a75c20..cafe7784 100644
--- a/tests/moe/test_fused_moe.py
+++ b/tests/moe/test_fused_moe.py
@@ -277,8 +277,8 @@ def test_fused_experts_decode_activation_and_router_weight_modes(
@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA is required")
-def test_fused_topk_non_power_of_2_k_routes_vendored_router():
- """triton_kernels.topk builds tl.arange(0, k) (power-of-2 only); k=10 must not reach it."""
+def test_fused_topk_handles_non_power_of_2_k():
+ """A top-10 router (qwen4_exp) must route like any other k."""
from freetoken.moe.fused import _torch_fused_topk, fused_topk
gating = torch.randn(5, 64, device="cuda")
@@ -289,7 +289,7 @@ def test_fused_topk_non_power_of_2_k_routes_vendored_router():
torch.testing.assert_close(weights, ref_w, rtol=1e-5, atol=1e-6)
-# The vendored triton router behind that k=10 branch; fp32 logits keep the reference top-k tie-free.
+# The in-repo triton router behind fused_topk; fp32 logits keep the reference top-k tie-free.
# Ties get their own case below, because torch.topk does not break them by expert id.
@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA is required")
@pytest.mark.parametrize("renormalize", [True, False])