Skip to content

FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.

perf(moe): fuse sigmoid + selection-bias routing into a triton kernel (#804)

Xiaoze Fan19a60a5b6571
diff --git a/docs/ftw-hotfix.md b/docs/ftw-hotfix.md
index 0c6b6b9f..b8a18149 100644
--- a/docs/ftw-hotfix.md
+++ b/docs/ftw-hotfix.md
@@ -17,6 +17,9 @@ python scripts/ftw_hotfix.py --ftw ~/models/Qwen3.8-Flash-Next-NVFP4-FTW --sourc
# Qwen3-VL FTW converted before image input: add the vision encoder (about 1 GiB) from the Hub, in place
python scripts/ftw_hotfix.py --ftw ~/models/Qwen3-VL-8B-Instruct-FTW --repo Qwen/Qwen3-VL-8B-Instruct
+# old GLM-5.2 FTW: restore the fp32 router selection biases from the Hub, in place
+python scripts/ftw_hotfix.py --ftw ~/models/GLM-5.2-NVFP4-FTW --repo nvidia/GLM-5.2-NVFP4
+
# just show what would change
python scripts/ftw_hotfix.py --ftw ~/models/GLM-5.2-NVFP4-FTW --dry-run
```
@@ -33,13 +36,14 @@ byte range, never the whole checkpoint; `--revision` pins the Hub revision to re
| `RuntimeError: Unexpected keys ... .weight_scale` | nvidia/GLM-5.2-NVFP4 | dequantize the old runtime-fp8 weights back to bf16 | none |
| `PLE shard indices are not contiguous 0..N-1: []` | Qwen3.8-Flash-Next (FTWs converted before #420) | write the PLE table as `ple-table-*.safetensors` | 47.7 GiB |
| `ValueError: ... stores NVFP4 global scales as fp16` | NVFP4 FTWs converted by older builds | none: the fp16 rounding cannot be undone, so the script refuses; reconvert with `ft checkpoint` | - |
+| none: loads, but expert selection uses a bf16-rounded selection bias | nvidia/GLM-5.2-NVFP4 FTWs converted by older builds | restore the fp32 `e_score_correction_bias` from the source and compact every dense shard that held the old values; without `--repo` or `--source` the script only prints a note | about 75 KiB, plus the source's index and shard headers |
| none: loads, but the output is wrong | Qwen3.5 / Qwen3.6 / Qwen3.8-27B FTWs converted by older builds (norm weights stored as bf16(1 + w)) | none: the script does not detect it; reconvert with `ft checkpoint` | - |
| `ValueError: ... holds no vision encoder tensors` | image-input families (Qwen3-VL, Qwen3.6, Qwen3.8-Flash-Next, Gemma-4, GLM-5.3-Flash, Muse-Glimmer, MiniMax-M3) converted before the family served images | add the vision encoder: the tower's checkpoint tensors are fetched and the family's own weight reader emits them as the converter would | 1 to 4 GiB |
The script decides by itself: it builds the current model from the FTW's `config.json`, compares
the FTW index with the tensors the model declares, and applies only the repairs that are needed.
Before writing anything it checks that the FTW matches the model (shapes, dtypes, byte counts,
-shard files) and refuses one that does not. An FTW that loads as is is left untouched.
+shard files) and refuses one that does not. An FTW that needs no repair is left untouched.
## What it writes
diff --git a/python/freetoken/kernel/triton/moe_router.py b/python/freetoken/kernel/triton/moe_router.py
index 54ecfe32..ee698c63 100644
--- a/python/freetoken/kernel/triton/moe_router.py
+++ b/python/freetoken/kernel/triton/moe_router.py
@@ -1,7 +1,7 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the SGLang project
# Adapted from SGLang (kernels/ops/moe/moe_fused_gate.py)
-"""Fused softmax top-k MoE router (bias-free, ungrouped experts).
+"""Fused top-k MoE routers: softmax (bias-free, ungrouped) and sigmoid with a selection-only bias and optional group-limited top-k.
The ``num_token_non_padded`` row mask reads the device tensor, so it survives CUDA-graph capture.
"""
@@ -14,6 +14,7 @@ from typing import Tuple
import torch
import triton
import triton.language as tl
+from triton.language.extra import libdevice
from freetoken.utils.arch import is_sm90_supported
@@ -23,13 +24,37 @@ def _pdl_supported() -> bool:
return is_sm90_supported()
+def _keep_top_groups(ranked, offs_n, N_GROUP: tl.constexpr, TOPK_GROUP: tl.constexpr, GROUP_SIZE: tl.constexpr, BLOCK_G: tl.constexpr):
+ # DeepSeek-V3 noaux_tc: a group scores the sum of its top-2 entries; experts outside the TOPK_GROUP best groups drop to -inf.
+ offs_g = tl.arange(0, BLOCK_G)
+ group_of_n = offs_n // GROUP_SIZE
+ group_score = tl.full([ranked.shape[0], BLOCK_G], -float("inf"), dtype=tl.float32)
+ for g in tl.static_range(N_GROUP):
+ vals = tl.where(group_of_n[None, :] == g, ranked, -float("inf"))
+ top1 = tl.max(vals, axis=1)[:, None]
+ top1_lane = tl.min(tl.where(vals == top1, offs_n[None, :], offs_n.shape[0]), axis=1)[:, None]
+ top2 = tl.max(tl.where(offs_n[None, :] == top1_lane, -float("inf"), vals), axis=1)[:, None]
+ group_score = tl.where(offs_g[None, :] == g, top1 + top2, group_score)
+
+ keep = tl.zeros(ranked.shape, dtype=tl.int1)
+ for _ in tl.static_range(TOPK_GROUP):
+ best = tl.max(group_score, axis=1)[:, None]
+ win_g = tl.min(tl.where(group_score == best, offs_g[None, :], BLOCK_G), axis=1)[:, None]
+ keep = keep | (group_of_n[None, :] == win_g)
+ group_score = tl.where(offs_g[None, :] == win_g, -float("inf"), group_score)
+ return tl.where(keep, ranked, -float("inf"))
+
+
@triton.jit
def _router_triton_kernel(
scores_ptr,
+ bias_ptr,
out_weights_ptr,
out_indices_ptr,
num_token_non_padded_ptr,
M,
+ routed_scale,
stride_sm,
stride_sn,
stride_wm,
@@ -41,6 +66,10 @@ def _router_triton_kernel(
BLOCK_M: tl.constexpr,
BLOCK_N: tl.constexpr,
BLOCK_K: tl.constexpr,
+ SIGMOID: tl.constexpr,
+ N_GROUP: tl.constexpr,
+ TOPK_GROUP: tl.constexpr,
+ BLOCK_G: tl.constexpr,
RENORMALIZE: tl.constexpr,
HAS_TOKEN_LIMIT: tl.constexpr,
launch_pdl: tl.constexpr,
@@ -62,14 +91,23 @@ def _router_triton_kernel(
mask2d = mask_m[:, None] & mask_n[None, :]
logits = tl.load(row_ptr, mask=mask2d, other=0.0).to(tl.float32)
- ranked = tl.where(mask_n[None, :], logits, -float("inf"))
- row_max = tl.max(ranked, axis=1)[:, None]
- exp_row = tl.where(mask_n[None, :], tl.exp(ranked - row_max), 0.0)
- activated = exp_row / tl.sum(exp_row, axis=1)[:, None]
+ if SIGMOID:
+ # libdevice exp and round-to-nearest division match torch.sigmoid bit for bit.
+ activated = tl.div_rn(1.0, 1.0 + libdevice.exp(-logits))
+ bias = tl.load(bias_ptr + offs_n, mask=mask_n, other=0.0).to(tl.float32)
+ ranked = tl.where(mask_n[None, :], activated + bias[None, :], -float("inf"))
+ else:
+ ranked = tl.where(mask_n[None, :], logits, -float("inf"))
+ row_max = tl.max(ranked, axis=1)[:, None]
+ exp_row = tl.where(mask_n[None, :], tl.exp(ranked - row_max), 0.0)
+ activated = exp_row / tl.sum(exp_row, axis=1)[:, None]
# Map NaN -> a finite floor
ranked = tl.where(ranked == ranked, ranked, -1e30)
+ if N_GROUP > 1:
+ ranked = _keep_top_groups(ranked, offs_n, N_GROUP, TOPK_GROUP, N // N_GROUP, BLOCK_G)
+
offs_k = tl.arange(0, BLOCK_K)
mask_k = offs_k < K
selected_vals = tl.zeros([BLOCK_M, BLOCK_K], dtype=tl.float32)
@@ -93,7 +131,12 @@ def _router_triton_kernel(
if RENORMALIZE:
routed_sum = tl.sum(tl.where(mask_k[None, :], selected_vals, 0.0), axis=1)[:, None]
- selected_vals = selected_vals / tl.where(routed_sum > 0.0, routed_sum, 1.0)
+ if SIGMOID:
+ selected_vals = tl.div_rn(selected_vals, routed_sum + 1e-20)
+ else:
+ selected_vals = selected_vals / tl.where(routed_sum > 0.0, routed_sum, 1.0)
+ if SIGMOID:
+ selected_vals = selected_vals * routed_scale
if HAS_TOKEN_LIMIT:
limit = tl.load(num_token_non_padded_ptr)
@@ -106,16 +149,16 @@ def _router_triton_kernel(
tl.store(out_i_ptr, selected_idx, mask=store_mask)
-def fused_topk_softmax(
+def _launch_router(
gating_output: torch.Tensor,
+ bias: torch.Tensor | None,
topk: int,
renormalize: bool,
- num_token_non_padded: torch.Tensor | None = None,
+ routed_scale: float,
+ n_group: int,
+ topk_group: int,
+ num_token_non_padded: torch.Tensor | None,
) -> Tuple[torch.Tensor, torch.Tensor]:
- """Softmax over all experts, top-k, then renormalize; ties keep the lowest expert id.
-
- ``num_token_non_padded`` is a device scalar; rows at or past it get expert id -1.
- """
assert gating_output.ndim == 2, "gating_output must be 2D"
M, N = gating_output.shape
weights = torch.empty((M, topk), dtype=torch.float32, device=gating_output.device)
@@ -134,10 +177,12 @@ def fused_topk_softmax(
_router_triton_kernel[(triton.cdiv(M, BLOCK_M),)](
gating_output,
+ bias,
weights,
indices,
num_token_non_padded,
M,
+ routed_scale,
gating_output.stride(0),
gating_output.stride(1),
weights.stride(0),
@@ -149,6 +194,10 @@ def fused_topk_softmax(
BLOCK_M=BLOCK_M,
BLOCK_N=BLOCK_N,
BLOCK_K=BLOCK_K,
+ SIGMOID=bias is not None,
+ N_GROUP=n_group,
+ TOPK_GROUP=topk_group,
+ BLOCK_G=triton.next_power_of_2(n_group),
RENORMALIZE=renormalize,
HAS_TOKEN_LIMIT=num_token_non_padded is not None,
launch_pdl=_pdl_supported(),
@@ -157,4 +206,38 @@ def fused_topk_softmax(
return weights, indices
-__all__ = ["fused_topk_softmax"]
+def fused_topk_softmax(
+ gating_output: torch.Tensor,
+ topk: int,
+ renormalize: bool,
+ num_token_non_padded: torch.Tensor | None = None,
+) -> Tuple[torch.Tensor, torch.Tensor]:
+ """Softmax over all experts, top-k, then renormalize; ties keep the lowest expert id.
+
+ ``num_token_non_padded`` is a device scalar; rows at or past it get expert id -1.
+ """
+ return _launch_router(gating_output, None, topk, renormalize, 1.0, 1, 1, num_token_non_padded)
+
+
+def fused_topk_sigmoid(
+ gating_output: torch.Tensor,
+ bias: torch.Tensor,
+ topk: int,
+ renormalize: bool,
+ routed_scale: float = 1.0,
+ n_group: int = 1,
+ topk_group: int = 1,
+ num_token_non_padded: torch.Tensor | None = None,
+) -> Tuple[torch.Tensor, torch.Tensor]:
+ """Sigmoid scores; ``scores + bias`` picks the top-k (within the ``topk_group`` best of ``n_group`` groups when ``n_group > 1``).
+
+ The weights are the unbiased scores, renormalized to sum to 1 when ``renormalize``, then times ``routed_scale``.
+ Ties keep the lowest expert id; rows at or past ``num_token_non_padded`` get expert id -1.
+ """
+ num_experts = gating_output.shape[1]
+ assert bias.shape == (num_experts,) and bias.stride(0) == 1
+ assert n_group == 1 or (num_experts % n_group == 0 and num_experts // n_group >= 2 and 1 <= topk_group <= n_group)
+ return _launch_router(gating_output, bias, topk, renormalize, float(routed_scale), n_group, topk_group, num_token_non_padded)
+
+
+__all__ = ["fused_topk_sigmoid", "fused_topk_softmax"]
diff --git a/python/freetoken/models/glm4_moe/moe.py b/python/freetoken/models/glm4_moe/moe.py
index 648eb9c3..96b65043 100644
--- a/python/freetoken/models/glm4_moe/moe.py
+++ b/python/freetoken/models/glm4_moe/moe.py
@@ -3,7 +3,7 @@ from __future__ import annotations
from typing import TYPE_CHECKING, Tuple
import torch
-import torch.nn.functional as F
+from freetoken.kernel.triton.moe_router import fused_topk_sigmoid
from freetoken.layers import BaseOP, LinearReplicated, make_moe_layer
from .mlp import GlmGatedMLP
@@ -52,29 +52,17 @@ class Glm4MoeSparseBlock(BaseOP):
prefix=f"{prefix}.shared_experts",
)
- def _group_limited(self, scores_for_choice: torch.Tensor) -> torch.Tensor:
- m = scores_for_choice.shape[0]
- e, g = self.num_experts, self.n_group
- group_scores = scores_for_choice.view(m, g, e // g).topk(2, dim=-1)[0].sum(dim=-1)
- group_idx = torch.topk(group_scores, self.topk_group, dim=-1, sorted=False)[1]
- group_mask = torch.zeros_like(group_scores)
- group_mask.scatter_(1, group_idx, 1.0)
- score_mask = group_mask.unsqueeze(-1).expand(m, g, e // g).reshape(m, e)
- return scores_for_choice.masked_fill(~score_mask.bool(), float("-inf"))
-
def _route(self, hidden_states: torch.Tensor) -> TopK:
- # HF computes the router logits in fp32; the gate is tiny so we match it exactly.
- logits = F.linear(hidden_states.float(), self.gate.weight.float())
- scores = logits.sigmoid()
- scores_for_choice = scores + self.e_score_correction_bias.float()
- if self.n_group > 1:
- scores_for_choice = self._group_limited(scores_for_choice)
- _, topk_ids = torch.topk(scores_for_choice, self.top_k, dim=-1)
- topk_weights = scores.gather(-1, topk_ids)
- if self.norm_topk_prob:
- topk_weights = topk_weights / (topk_weights.sum(dim=-1, keepdim=True) + 1e-20)
- topk_weights = topk_weights * self.routed_scaling_factor
- return topk_weights.to(torch.float32).contiguous(), topk_ids.to(torch.int32).contiguous()
+ logits = torch.mm(hidden_states, self.gate.weight.t(), out_dtype=torch.float32)
+ return fused_topk_sigmoid(
+ logits,
+ self.e_score_correction_bias,
+ self.top_k,
+ self.norm_topk_prob,
+ routed_scale=self.routed_scaling_factor,
+ n_group=self.n_group,
+ topk_group=self.topk_group,
+ )
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
num_tokens, hidden_dim = hidden_states.shape
diff --git a/python/freetoken/models/glm5_next/moe.py b/python/freetoken/models/glm5_next/moe.py
index a0cbe711..cca39f3d 100644
--- a/python/freetoken/models/glm5_next/moe.py
+++ b/python/freetoken/models/glm5_next/moe.py
@@ -15,7 +15,7 @@ from __future__ import annotations
from typing import TYPE_CHECKING, Tuple
import torch
-import torch.nn.functional as F
+from freetoken.kernel.triton.moe_router import fused_topk_sigmoid
from freetoken.layers import BaseOP, LinearReplicated, make_moe_layer
from .mlp import Glm5NextGatedMLP
@@ -56,29 +56,17 @@ class Glm5NextSparseBlock(BaseOP):
prefix=f"{prefix}.shared_experts",
)
- def _group_limited(self, scores_for_choice: torch.Tensor) -> torch.Tensor:
- m = scores_for_choice.shape[0]
- e, g = self.num_experts, self.n_group
- group_scores = scores_for_choice.view(m, g, e // g).topk(2, dim=-1)[0].sum(dim=-1)
- group_idx = torch.topk(group_scores, self.topk_group, dim=-1, sorted=False)[1]
- group_mask = torch.zeros_like(group_scores)
- group_mask.scatter_(1, group_idx, 1.0)
- score_mask = group_mask.unsqueeze(-1).expand(m, g, e // g).reshape(m, e)
- return scores_for_choice.masked_fill(~score_mask.bool(), float("-inf"))
-
def _route(self, hidden_states: torch.Tensor) -> TopK:
- # HF computes router logits in fp32 (moe_router_dtype: float32); match exactly.
- logits = F.linear(hidden_states.float(), self.gate.weight.float())
- scores = logits.sigmoid()
- scores_for_choice = scores + self.e_score_correction_bias.float()
- if self.n_group > 1:
- scores_for_choice = self._group_limited(scores_for_choice)
- _, topk_ids = torch.topk(scores_for_choice, self.top_k, dim=-1)
- topk_weights = scores.gather(-1, topk_ids)
- if self.norm_topk_prob:
- topk_weights = topk_weights / (topk_weights.sum(dim=-1, keepdim=True) + 1e-20)
- topk_weights = topk_weights * self.routed_scaling_factor
- return topk_weights.to(torch.float32).contiguous(), topk_ids.to(torch.int32).contiguous()
+ logits = torch.mm(hidden_states, self.gate.weight.t(), out_dtype=torch.float32)
+ return fused_topk_sigmoid(
+ logits,
+ self.e_score_correction_bias,
+ self.top_k,
+ self.norm_topk_prob,
+ routed_scale=self.routed_scaling_factor,
+ n_group=self.n_group,
+ topk_group=self.topk_group,
+ )
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
num_tokens, hidden_dim = hidden_states.shape
diff --git a/python/freetoken/models/glm_moe_dsa/moe.py b/python/freetoken/models/glm_moe_dsa/moe.py
index 15ebf7cb..46139c9f 100644
--- a/python/freetoken/models/glm_moe_dsa/moe.py
+++ b/python/freetoken/models/glm_moe_dsa/moe.py
@@ -13,7 +13,7 @@ from __future__ import annotations
from typing import TYPE_CHECKING, Tuple
import torch
-import torch.nn.functional as F
+from freetoken.kernel.triton.moe_router import fused_topk_sigmoid
from freetoken.layers import BaseOP, LinearReplicated, make_moe_layer
from .mlp import GlmDsaGatedMLP
@@ -34,9 +34,7 @@ class GlmMoeDsaSparseBlock(BaseOP):
self.topk_group = config.topk_group
self.gate = LinearReplicated(config.hidden_size, config.num_experts, has_bias=False)
- # DeepSeek-style selection bias; kept fp32 in HF, stored in the model dtype and
- # upcast at use (exact enough for the argmax-style top-k selection).
- self.e_score_correction_bias = torch.empty(config.num_experts)
+ self.e_score_correction_bias = torch.empty(config.num_experts, dtype=torch.float32)
# The offload cache indexes experts by *MoE* layer (global layer minus
# first_k_dense_replace), matching how the loader packs the expert banks.
@@ -54,29 +52,17 @@ class GlmMoeDsaSparseBlock(BaseOP):
prefix=f"{prefix}.shared_experts",
)
- def _group_limited(self, scores_for_choice: torch.Tensor) -> torch.Tensor:
- m = scores_for_choice.shape[0]
- e, g = self.num_experts, self.n_group
- group_scores = scores_for_choice.view(m, g, e // g).topk(2, dim=-1)[0].sum(dim=-1)
- group_idx = torch.topk(group_scores, self.topk_group, dim=-1, sorted=False)[1]
- group_mask = torch.zeros_like(group_scores)
- group_mask.scatter_(1, group_idx, 1.0)
- score_mask = group_mask.unsqueeze(-1).expand(m, g, e // g).reshape(m, e)
- return scores_for_choice.masked_fill(~score_mask.bool(), float("-inf"))
-
def _route(self, hidden_states: torch.Tensor) -> TopK:
- # HF computes the router logits in fp32; the gate is tiny so we match it exactly.
- logits = F.linear(hidden_states.float(), self.gate.weight.float())
- scores = logits.sigmoid()
- scores_for_choice = scores + self.e_score_correction_bias.float()
- if self.n_group > 1:
- scores_for_choice = self._group_limited(scores_for_choice)
- _, topk_ids = torch.topk(scores_for_choice, self.top_k, dim=-1)
- topk_weights = scores.gather(-1, topk_ids)
- if self.norm_topk_prob:
- topk_weights = topk_weights / (topk_weights.sum(dim=-1, keepdim=True) + 1e-20)
- topk_weights = topk_weights * self.routed_scaling_factor
- return topk_weights.to(torch.float32).contiguous(), topk_ids.to(torch.int32).contiguous()
+ logits = torch.mm(hidden_states, self.gate.weight.t(), out_dtype=torch.float32)
+ return fused_topk_sigmoid(
+ logits,
+ self.e_score_correction_bias,
+ self.top_k,
+ self.norm_topk_prob,
+ routed_scale=self.routed_scaling_factor,
+ n_group=self.n_group,
+ topk_group=self.topk_group,
+ )
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
num_tokens, hidden_dim = hidden_states.shape
diff --git a/python/freetoken/models/glm_moe_dsa/weight.py b/python/freetoken/models/glm_moe_dsa/weight.py
index 900d71af..1b7a3dae 100644
--- a/python/freetoken/models/glm_moe_dsa/weight.py
+++ b/python/freetoken/models/glm_moe_dsa/weight.py
@@ -110,7 +110,7 @@ def iter_weights(
yield f"{m}.gate.weight", reader.get(f"{m}.gate.weight")
yield (
f"{m}.e_score_correction_bias",
- reader.get(f"{m}.gate.e_score_correction_bias").to(torch.bfloat16),
+ reader.get(f"{m}.gate.e_score_correction_bias").to(torch.float32),
)
for proj in ("gate_proj", "up_proj", "down_proj"):
yield f"{m}.shared_experts.{proj}.weight", reader.get(f"{m}.shared_experts.{proj}.weight")
diff --git a/python/freetoken/models/minimax_m2/moe.py b/python/freetoken/models/minimax_m2/moe.py
index 1410f388..fdec094c 100644
--- a/python/freetoken/models/minimax_m2/moe.py
+++ b/python/freetoken/models/minimax_m2/moe.py
@@ -3,6 +3,8 @@ from __future__ import annotations
from typing import TYPE_CHECKING, Tuple
import torch
+import torch.nn.functional as F
+from freetoken.kernel.triton.moe_router import fused_topk_sigmoid
from freetoken.layers import BaseOP, LinearReplicated, make_moe_layer
if TYPE_CHECKING:
@@ -21,8 +23,9 @@ class MiniMaxM2SparseMoeBlock(BaseOP):
self.num_experts = config.num_experts
self.norm_topk_prob = config.norm_topk_prob
self.gate = LinearReplicated(config.hidden_size, config.num_experts, has_bias=False)
+ self.gate.weight = torch.empty(config.num_experts, config.hidden_size, dtype=torch.float32)
# DeepSeek-style selection bias; not an nn.Parameter in HF (a registered buffer).
- self.e_score_correction_bias = torch.empty(config.num_experts)
+ self.e_score_correction_bias = torch.empty(config.num_experts, dtype=torch.float32)
self.experts = make_moe_layer(
config,
layer_id=layer_id,
@@ -31,20 +34,14 @@ class MiniMaxM2SparseMoeBlock(BaseOP):
prefix=f"{prefix}.experts",
)
- def _route(self, router_logits: torch.Tensor) -> TopK:
- scores = router_logits.float().sigmoid()
- scores_for_choice = scores + self.e_score_correction_bias.float()
- _, topk_ids = torch.topk(scores_for_choice, self.top_k, dim=-1)
- topk_weights = scores.gather(-1, topk_ids)
- if self.norm_topk_prob:
- topk_weights = topk_weights / topk_weights.sum(dim=-1, keepdim=True)
- return topk_weights.to(torch.float32).contiguous(), topk_ids.to(torch.int32).contiguous()
+ def _route(self, hidden_states: torch.Tensor) -> TopK:
+ logits = F.linear(hidden_states.float(), self.gate.weight)
+ return fused_topk_sigmoid(logits, self.e_score_correction_bias, self.top_k, self.norm_topk_prob)
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
num_tokens, hidden_dim = hidden_states.shape
hidden_states = hidden_states.view(-1, hidden_dim)
- router_logits = self.gate.forward(hidden_states)
- topk_weights, topk_ids = self._route(router_logits)
+ topk_weights, topk_ids = self._route(hidden_states)
out = self.experts.routed_forward(hidden_states, topk_weights, topk_ids)
return out.view(num_tokens, hidden_dim)
diff --git a/python/freetoken/models/minimax_m2/weight.py b/python/freetoken/models/minimax_m2/weight.py
index 1b91ead8..ac8f27e3 100644
--- a/python/freetoken/models/minimax_m2/weight.py
+++ b/python/freetoken/models/minimax_m2/weight.py
@@ -82,6 +82,8 @@ def iter_weights(
world_size=tp_info.size,
num_kv_heads=config.num_kv_heads,
)
+ if name.endswith(".e_score_correction_bias"):
+ tensor = tensor.float()
yield name, tensor
yield from iter_merged_tensors(raw(), _MERGE_RULES, model_name="minimax_m2")
diff --git a/python/freetoken/models/minimax_m3/moe.py b/python/freetoken/models/minimax_m3/moe.py
index 1bb1276e..f2e089ef 100644
--- a/python/freetoken/models/minimax_m3/moe.py
+++ b/python/freetoken/models/minimax_m3/moe.py
@@ -8,8 +8,7 @@ renormalize, scale by ``routed_scaling_factor`` (2.0). No group-limited top-k
``limit``) and the always-on shared expert (MXFP8, swigluoai) is added.
The checkpoint keeps ``e_score_correction_bias`` fp32 and the reference applies it
-fp32; the bias sits on the top-k boundary, so it is stored fp32 here too (unlike
-GLM's bf16 storage, whose checkpoint ships it fp32 but consumes it off a bf16 cast).
+fp32; the bias sits on the top-k boundary, so it is stored fp32 here too.
"""
from __future__ import annotations
@@ -18,6 +17,7 @@ from typing import TYPE_CHECKING, Tuple
import torch
import torch.nn.functional as F
+from freetoken.kernel.triton.moe_router import fused_topk_sigmoid
from freetoken.layers import BaseOP, LinearReplicated, make_moe_layer
from .mlp import MiniMaxM3MLP
@@ -72,14 +72,13 @@ class MiniMaxM3SparseMoeBlock(BaseOP):
def _route(self, hidden_states: torch.Tensor) -> TopK:
# HF/vLLM compute the router logits in fp32; the gate is stored fp32.
logits = F.linear(hidden_states.float(), self.gate.weight)
- scores = logits.sigmoid()
- scores_for_choice = scores + self.e_score_correction_bias
- _, topk_ids = torch.topk(scores_for_choice, self.top_k, dim=-1)
- topk_weights = scores.gather(-1, topk_ids)
- if self.norm_topk_prob:
- topk_weights = topk_weights / (topk_weights.sum(dim=-1, keepdim=True) + 1e-20)
- topk_weights = topk_weights * self.routed_scaling_factor
- return topk_weights.to(torch.float32).contiguous(), topk_ids.to(torch.int32).contiguous()
+ return fused_topk_sigmoid(
+ logits,
+ self.e_score_correction_bias,
+ self.top_k,
+ self.norm_topk_prob,
+ routed_scale=self.routed_scaling_factor,
+ )
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
num_tokens, hidden_dim = hidden_states.shape
diff --git a/scripts/ftw_hotfix.py b/scripts/ftw_hotfix.py
index 7b4d2257..b040a104 100644
--- a/scripts/ftw_hotfix.py
+++ b/scripts/ftw_hotfix.py
@@ -9,6 +9,7 @@ FTW index against the tensors that model declares, and repairs the differences:
- renames (DeepSeek-V4: the dense tree moved under ``model.`` and ``.scale`` became ``.weight_scale_inv``)
- fp8 weights the current model declares as bf16 (old runtime fp8 residue) are dequantized, their ``.weight_scale`` dropped
- tensors the model declares but the FTW lacks (``input_scale``) are fetched from the HF repo by byte range
+- router selection biases an older build rounded to bf16 (GLM-5.2) are restored in fp32 from the source checkpoint
- an FTW converted before its family served images gets its vision encoder: the tower's checkpoint tensors are fetched into a scratch checkpoint dir and the family's own weight reader emits them under the names the FTW needs
- a Qwen3.8-Flash-Next FTW without the PLE n-gram table gets it written as ``ple-table-*.safetensors``
@@ -63,6 +64,7 @@ _PLE_SHARD_RE = re.compile(r"\.ple\.ple_embedding\.ngram_embedding\.shard_(\d+)\
_PLE_SCALE_SUFFIX = ".ple.ple_embedding.ngram_embedding.weight_scale"
_PLE_FILE_RE = re.compile(r"^ple-table-\d{5}\.safetensors$")
_PLE_FILE_BYTES = 4 << 30
+_BIAS_SUFFIX = ".e_score_correction_bias"
# tensors an old FTW may carry that the text model does not declare; they are kept and never count as errors
_IGNORED_PREFIXES = VISION_KEY_PREFIXES
_VERBOSE = False
@@ -235,6 +237,40 @@ def plan(arch: str, entries: list[dict], expected: dict, name_map: NameMap, quan
return renames, dequants, fetches, drops, leftovers
+def rounded_biases(entries: list[dict], expected: dict, renames: dict[str, str]) -> dict[str, dict]:
+ """Router selection biases the FTW stores narrower than the fp32 the model reads; the rounding can change the selected experts."""
+ out: dict[str, dict] = {}
+ for e in entries:
+ name = renames.get(e["name"], e["name"])
+ exp = expected.get(name)
+ if (e["kind"] == "weight" and name.endswith(_BIAS_SUFFIX) and exp is not None and exp[1] == torch.float32
+ and _dtype_of(e["dtype"]) in (torch.bfloat16, torch.float16)):
+ out[name] = e
+ return out
+
+
+def resolve_biases(biases: dict[str, dict], source: TensorSource, ftw_dir: str, index: dict) -> tuple[dict[str, torch.Tensor], int, list[str]]:
+ """Fetch the fp32 source tensor of each rounded bias; a source that stores it narrow too has nothing to restore."""
+ restored: dict[str, torch.Tensor] = {}
+ same, errors = 0, []
+ for n, e in biases.items():
+ # GLM checkpoints keep the bias under the router gate; the readers drop the ``gate.``
+ names = [n, n[: -len(_BIAS_SUFFIX)] + ".gate" + _BIAS_SUFFIX]
+ src = next((c for c in names if source.has(c)), None)
+ if src is None:
+ errors.append(f"no source tensor for {n}: tried {names}")
+ elif source.meta(src)["dtype"] != "F32":
+ same += 1
+ else:
+ t = source.get(src)
+ # the FTW must hold this very tensor, rounded; anything else means a different source checkpoint
+ if torch.equal(t.to(_dtype_of(e["dtype"])), read_entry(ftw_dir, index, e)):
+ restored[n] = t
+ else:
+ errors.append(f"{src} in the source does not round to the FTW's {n}; pass the checkpoint this FTW was converted from")
+ return restored, same, errors
+
+
def resolve_fetches(fetches, source: TensorSource) -> tuple[dict[str, tuple[list[str], bool]], list[str]]:
"""Map each missing tensor to (the source tensors it is built from, reciprocal); a fused scale needs every part."""
resolved: dict[str, tuple[list[str], bool]] = {}
@@ -813,6 +849,8 @@ def main(argv: list[str] | None = None) -> int:
log(f"{arch}: model declares {len(expected)} dense tensors")
source = TensorSource(ns.repo, ns.source, ns.revision) if (ns.repo or ns.source) else None
renames, dequants, fetches, drops, leftovers = plan(arch, index["tensors"], expected, name_map, quant)
+ # only the old GLM-5.2 reader rounded the bias; other families store it as the checkpoint does
+ biases = rounded_biases(index["tensors"], expected, renames) if arch == "GlmMoeDsaForCausalLM" else {}
# the encoder comes from the family reader, not from by-name fetches
vision_missing = sorted(n for n, *_ in fetches if is_vision(n))
fetches = [f for f in fetches if not is_vision(f[0])]
@@ -823,6 +861,10 @@ def main(argv: list[str] | None = None) -> int:
ple_status, ple_detail, ple_files = chk["ple"]
need_ple = ple_status in ("missing", "partial")
next_num, orphans = shard_numbers(ns.ftw, index)
+ bias_vals: dict[str, torch.Tensor] = {}
+ bias_same, bias_errors = 0, []
+ if biases and source is not None and not problems:
+ bias_vals, bias_same, bias_errors = resolve_biases(biases, source, ns.ftw, index)
print(f"{arch}: {len(expected)} expected dense tensors, FTW has {sum(e['kind'] == 'weight' for e in index['tensors'])}")
print(f" renames {len(renames)} dequantize {len(dequants)} fetch {len(fetches)} drop {len(drops)} leftover {len(leftovers)}"
@@ -842,7 +884,7 @@ def main(argv: list[str] | None = None) -> int:
for n in leftovers[:8]:
print(f" leftover (not declared by the model): {n}")
if orphans:
- removing = not ns.out and bool(renames or dequants or fetches or drops or need_ple or dirty)
+ removing = not ns.out and bool(renames or dequants or fetches or vision_missing or bias_vals or drops or need_ple or dirty)
print(f" note: {len(orphans)} shard file(s) not referenced by the index, left by an interrupted run"
+ (", will be removed: " if removing else ", left as is: ") + str(orphans[:3]))
@@ -868,6 +910,16 @@ def main(argv: list[str] | None = None) -> int:
for msg in errors:
print(f"ERROR: {msg}", file=sys.stderr)
bad = True
+ if biases and source is None:
+ print(f" note: {len(biases)} router selection biases (e.g. {next(iter(biases))}) are stored narrower than the fp32 the model reads; "
+ "rerun with --repo or --source to restore them")
+ for msg in bias_errors:
+ print(f"ERROR: {msg}", file=sys.stderr)
+ bad = True
+ if bias_vals:
+ print(f" restore {len(bias_vals)} router selection biases in fp32 from the source")
+ if bias_same:
+ print(f" {bias_same} router selection biases are narrow in the source as well; nothing to restore")
# the converter transforms most tensors on the way in (norm offsets, fusion, packing); only the
# activation scale scalars are stored as the checkpoint has them, so only they can be fetched raw
unfetchable = [n for n, *_ in fetches if not n.endswith(".input_scale")]
@@ -905,7 +957,7 @@ def main(argv: list[str] | None = None) -> int:
print(f"ERROR: {n}: reader shape {list(tower[n].shape)} != model shape {list(expected[n][0])}", file=sys.stderr)
return 2
- replaced = dequant_names | set(fetch_srcs) | set(vision_missing)
+ replaced = dequant_names | set(fetch_srcs) | set(vision_missing) | set(bias_vals)
keep: list[tuple[dict, str]] = []
for e in index["tensors"]:
name = renames.get(e["name"], e["name"]) if e["kind"] == "weight" else e["name"]
@@ -932,7 +984,7 @@ def main(argv: list[str] | None = None) -> int:
print(f" disk: about {need / 2**30:.1f} GiB needed under {work} ({free / 2**30:.1f} GiB free)")
if ns.dry_run:
return 0
- if not (renames or dequants or fetch_srcs or vision_missing or drops or need_ple or dirty or ns.out):
+ if not (renames or dequants or fetch_srcs or vision_missing or bias_vals or drops or need_ple or dirty or ns.out):
print("nothing to do; the FTW loads as is")
return 0
if free < need + (1 << 30):
@@ -958,9 +1010,12 @@ def main(argv: list[str] | None = None) -> int:
vals = [1.0 / v for v in vals]
# a fused projection shares one activation scale; the reader takes the max over its parts
w.add_tensor(n, torch.stack(vals).max().reshape(()))
+ for n, t in bias_vals.items():
+ log(f" restore {n} in fp32")
+ w.add_tensor(n, t)
hotfix = {"from": os.path.abspath(ns.ftw), "renamed": len(renames), "dequantized": len(dequants),
- "fetched": len(fetch_srcs) + len(vision_missing), "dropped": len(drops), "compacted": 0, "source": ns.repo or ns.source}
+ "fetched": len(fetch_srcs) + len(vision_missing), "restored": len(bias_vals), "dropped": len(drops), "compacted": 0, "source": ns.repo or ns.source}
compacted = 0
if ns.out:
log(f"--out: writing a fresh FTW -> {work}")
@@ -994,7 +1049,7 @@ def main(argv: list[str] | None = None) -> int:
os.remove(os.path.join(work, f))
new_entries: list[dict] = []
new_shards: list[dict] = []
- if dequants or fetch_srcs or vision_missing:
+ if dequants or fetch_srcs or vision_missing or bias_vals:
log(f"in place: appending shard {_SHARD_FMT.format(next_num)} -> {work}")
w = ShardWriter(work, index["shard_limit"], first_shard=next_num, global_off=index["total_bytes"])
try:
@@ -1013,7 +1068,7 @@ def main(argv: list[str] | None = None) -> int:
if renames or new_entries or drops:
# rollback stays possible only while no shard gets replaced: keep the old index then
bak = os.path.join(work, INDEX_NAME + ".bak")
- if not drops and not dirty and not os.path.exists(bak):
+ if not drops and not dirty and not bias_vals and not os.path.exists(bak):
shutil.copy2(os.path.join(work, INDEX_NAME), bak)
write_index(work, new_index)
log(f"index written: {os.path.join(work, INDEX_NAME)}")
diff --git a/tests/moe/test_fused_moe.py b/tests/moe/test_fused_moe.py
index cafe7784..da34dba2 100644
--- a/tests/moe/test_fused_moe.py
+++ b/tests/moe/test_fused_moe.py
@@ -369,3 +369,52 @@ def test_fused_topk_softmax_is_cuda_graph_capturable():
torch.cuda.synchronize()
assert (ids[:3] != -1).all()
assert (ids[3:] == -1).all()
+
+
+def _reference_sigmoid_topk(logits, bias, topk, renormalize, scale, n_group, topk_group, num_token_non_padded):
+ """The torch routing of the GLM / MiniMax MoE blocks: sigmoid, bias for selection only, optional group-limited top-k."""
+ scores = logits.float().sigmoid()
+ choice = scores + bias.float()
+ if n_group > 1:
+ m, e = choice.shape
+ group_scores = choice.view(m, n_group, e // n_group).topk(2, dim=-1)[0].sum(dim=-1)
+ group_idx = torch.topk(group_scores, topk_group, dim=-1, sorted=False)[1]
+ group_mask = torch.zeros_like(group_scores).scatter_(1, group_idx, 1.0).bool()
+ choice = choice.masked_fill(~group_mask.repeat_interleave(e // n_group, dim=1), float("-inf"))
+ _, ids = torch.topk(choice, topk, dim=-1)
+ weights = scores.gather(-1, ids)
+ if renormalize:
+ weights = weights / (weights.sum(dim=-1, keepdim=True) + 1e-20)
+ ids = ids.to(torch.int32)
+ ids[torch.arange(ids.shape[0], device=ids.device) >= num_token_non_padded] = -1
+ return weights * scale, ids
+
+
[email protected](not torch.cuda.is_available(), reason="CUDA is required")
[email protected]("num_tokens", [1, 7, 2048])
+ "num_experts,topk,n_group,topk_group,renormalize,scale",
+ [
+ (128, 8, 1, 1, True, 1.0), # GLM-4.5-Air
+ (160, 8, 1, 1, True, 2.5), # GLM-4.7
+ (288, 8, 1, 1, True, 2.5), # GLM-5.3-Flash
+ (128, 4, 1, 1, True, 2.0), # MiniMax-M3
+ (256, 8, 8, 4, True, 2.5), # DeepSeek-V3 group-limited top-k
+ (96, 6, 3, 2, False, 1.0), # odd group count, no renormalization
+ ],
+)
+def test_fused_topk_sigmoid_matches_torch_reference(num_tokens, num_experts, topk, n_group, topk_group, renormalize, scale):
+ from freetoken.kernel.triton.moe_router import fused_topk_sigmoid
+
+ gen = torch.Generator(device="cuda").manual_seed(num_tokens * 31 + num_experts * 7 + topk)
+ logits = torch.randn(num_tokens, num_experts, generator=gen, device="cuda")
+ bias = torch.randn(num_experts, generator=gen, device="cuda") * 0.1
+ limit = torch.tensor(max(1, num_tokens - 3), dtype=torch.int32, device="cuda")
+
+ weights, ids = fused_topk_sigmoid(logits, bias, topk, renormalize, scale, n_group, topk_group, limit)
+ ref_w, ref_i = _reference_sigmoid_topk(logits, bias, topk, renormalize, scale, n_group, topk_group, limit)
+
+ assert weights.dtype == torch.float32 and ids.dtype == torch.int32
+ assert torch.equal(ids, ref_i)
+ torch.testing.assert_close(weights, ref_w, rtol=1e-5, atol=1e-6)