Skip to content

FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.

fix(models): compute shared experts before in-place routed experts (#463)

cherry77-cloude0886ccfb698
diff --git a/python/freetoken/layers/moe.py b/python/freetoken/layers/moe.py
index 42ad822b..c2094549 100644
--- a/python/freetoken/layers/moe.py
+++ b/python/freetoken/layers/moe.py
@@ -124,6 +124,9 @@ class MoELayer(BaseOP):
rewrites expert ids into cache slot ids); pass a fresh tensor or a clone.
The resident path does not mutate it today, but callers must not rely on
that.
+
+ ``hidden_states`` may also be overwritten by the expert kernel. Compute
+ shared branches that need the original input before calling this method.
"""
out = self._resident_gemm(hidden_states, topk_weights, topk_ids)
return self._maybe_all_reduce(out)
@@ -210,6 +213,9 @@ class OffloadMoELayer(MoELayer):
scores, selection bias, group-limited top-k, ...); identical to ``forward``
past the router. ``topk_ids`` must be safe to mutate in place (decode
rewrites expert ids into cache slot ids); pass a fresh tensor or a clone.
+
+ ``hidden_states`` may also be overwritten by the expert kernel. Compute
+ shared branches that need the original input before calling this method.
"""
ctx = get_global_ctx()
if ctx.batch.is_prefill:
diff --git a/python/freetoken/models/glm4_moe/moe.py b/python/freetoken/models/glm4_moe/moe.py
index e1b0f18b..8e595ee7 100644
--- a/python/freetoken/models/glm4_moe/moe.py
+++ b/python/freetoken/models/glm4_moe/moe.py
@@ -82,8 +82,10 @@ class Glm4MoeSparseBlock(BaseOP):
num_tokens, hidden_dim = hidden_states.shape
hidden_states = hidden_states.view(-1, hidden_dim)
topk_weights, topk_ids = self._route(hidden_states)
+ # Routed kernels may overwrite hidden_states; shared experts need the original input.
+ shared = self.shared_experts.forward(hidden_states)
out = self.experts.routed_forward(hidden_states, topk_weights, topk_ids)
- out = out + self.shared_experts.forward(hidden_states)
+ out = out + shared
return out.view(num_tokens, hidden_dim)
diff --git a/python/freetoken/models/glm5_next/moe.py b/python/freetoken/models/glm5_next/moe.py
index 1cf27ca6..a0cbe711 100644
--- a/python/freetoken/models/glm5_next/moe.py
+++ b/python/freetoken/models/glm5_next/moe.py
@@ -84,8 +84,10 @@ class Glm5NextSparseBlock(BaseOP):
num_tokens, hidden_dim = hidden_states.shape
hidden_states = hidden_states.view(-1, hidden_dim)
topk_weights, topk_ids = self._route(hidden_states)
+ # Routed kernels may overwrite hidden_states; shared experts need the original input.
+ shared = self.shared_experts.forward(hidden_states)
out = self.experts.routed_forward(hidden_states, topk_weights, topk_ids)
- out = out + self.shared_experts.forward(hidden_states)
+ out = out + shared
return out.view(num_tokens, hidden_dim)
diff --git a/python/freetoken/models/glm_moe_dsa/moe.py b/python/freetoken/models/glm_moe_dsa/moe.py
index 0e17c9be..15ebf7cb 100644
--- a/python/freetoken/models/glm_moe_dsa/moe.py
+++ b/python/freetoken/models/glm_moe_dsa/moe.py
@@ -82,8 +82,10 @@ class GlmMoeDsaSparseBlock(BaseOP):
num_tokens, hidden_dim = hidden_states.shape
hidden_states = hidden_states.view(-1, hidden_dim)
topk_weights, topk_ids = self._route(hidden_states)
+ # Routed kernels may overwrite hidden_states; shared experts need the original input.
+ shared = self.shared_experts.forward(hidden_states)
out = self.experts.routed_forward(hidden_states, topk_weights, topk_ids)
- out = out + self.shared_experts.forward(hidden_states)
+ out = out + shared
return out.view(num_tokens, hidden_dim)
diff --git a/python/freetoken/models/minimax_m3/moe.py b/python/freetoken/models/minimax_m3/moe.py
index fe1139b8..1bb1276e 100644
--- a/python/freetoken/models/minimax_m3/moe.py
+++ b/python/freetoken/models/minimax_m3/moe.py
@@ -85,8 +85,10 @@ class MiniMaxM3SparseMoeBlock(BaseOP):
num_tokens, hidden_dim = hidden_states.shape
hidden_states = hidden_states.view(-1, hidden_dim)
topk_weights, topk_ids = self._route(hidden_states)
+ # Routed kernels may overwrite hidden_states; shared experts need the original input.
+ shared = self.shared_experts.forward(hidden_states)
out = self.experts.routed_forward(hidden_states, topk_weights, topk_ids)
- out = out + self.shared_experts.forward(hidden_states)
+ out = out + shared
return out.view(num_tokens, hidden_dim)