FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.
fix(models): compute shared experts before in-place routed experts (#463)
cherry77-cloude0886ccfb698
diff --git a/python/freetoken/layers/moe.py b/python/freetoken/layers/moe.pyindex 42ad822b..c2094549 100644--- a/python/freetoken/layers/moe.py+++ b/python/freetoken/layers/moe.py@@ -124,6 +124,9 @@ class MoELayer(BaseOP):rewrites expert ids into cache slot ids); pass a fresh tensor or a clone.The resident path does not mutate it today, but callers must not rely onthat.++ ``hidden_states`` may also be overwritten by the expert kernel. Compute+ shared branches that need the original input before calling this method."""out = self._resident_gemm(hidden_states, topk_weights, topk_ids)return self._maybe_all_reduce(out)@@ -210,6 +213,9 @@ class OffloadMoELayer(MoELayer):scores, selection bias, group-limited top-k, ...); identical to ``forward``past the router. ``topk_ids`` must be safe to mutate in place (decoderewrites expert ids into cache slot ids); pass a fresh tensor or a clone.++ ``hidden_states`` may also be overwritten by the expert kernel. Compute+ shared branches that need the original input before calling this method."""ctx = get_global_ctx()if ctx.batch.is_prefill:diff --git a/python/freetoken/models/glm4_moe/moe.py b/python/freetoken/models/glm4_moe/moe.pyindex e1b0f18b..8e595ee7 100644--- a/python/freetoken/models/glm4_moe/moe.py+++ b/python/freetoken/models/glm4_moe/moe.py@@ -82,8 +82,10 @@ class Glm4MoeSparseBlock(BaseOP):num_tokens, hidden_dim = hidden_states.shapehidden_states = hidden_states.view(-1, hidden_dim)topk_weights, topk_ids = self._route(hidden_states)+ # Routed kernels may overwrite hidden_states; shared experts need the original input.+ shared = self.shared_experts.forward(hidden_states)out = self.experts.routed_forward(hidden_states, topk_weights, topk_ids)- out = out + self.shared_experts.forward(hidden_states)+ out = out + sharedreturn out.view(num_tokens, hidden_dim)diff --git a/python/freetoken/models/glm5_next/moe.py b/python/freetoken/models/glm5_next/moe.pyindex 1cf27ca6..a0cbe711 100644--- a/python/freetoken/models/glm5_next/moe.py+++ b/python/freetoken/models/glm5_next/moe.py@@ -84,8 +84,10 @@ class Glm5NextSparseBlock(BaseOP):num_tokens, hidden_dim = hidden_states.shapehidden_states = hidden_states.view(-1, hidden_dim)topk_weights, topk_ids = self._route(hidden_states)+ # Routed kernels may overwrite hidden_states; shared experts need the original input.+ shared = self.shared_experts.forward(hidden_states)out = self.experts.routed_forward(hidden_states, topk_weights, topk_ids)- out = out + self.shared_experts.forward(hidden_states)+ out = out + sharedreturn out.view(num_tokens, hidden_dim)diff --git a/python/freetoken/models/glm_moe_dsa/moe.py b/python/freetoken/models/glm_moe_dsa/moe.pyindex 0e17c9be..15ebf7cb 100644--- a/python/freetoken/models/glm_moe_dsa/moe.py+++ b/python/freetoken/models/glm_moe_dsa/moe.py@@ -82,8 +82,10 @@ class GlmMoeDsaSparseBlock(BaseOP):num_tokens, hidden_dim = hidden_states.shapehidden_states = hidden_states.view(-1, hidden_dim)topk_weights, topk_ids = self._route(hidden_states)+ # Routed kernels may overwrite hidden_states; shared experts need the original input.+ shared = self.shared_experts.forward(hidden_states)out = self.experts.routed_forward(hidden_states, topk_weights, topk_ids)- out = out + self.shared_experts.forward(hidden_states)+ out = out + sharedreturn out.view(num_tokens, hidden_dim)diff --git a/python/freetoken/models/minimax_m3/moe.py b/python/freetoken/models/minimax_m3/moe.pyindex fe1139b8..1bb1276e 100644--- a/python/freetoken/models/minimax_m3/moe.py+++ b/python/freetoken/models/minimax_m3/moe.py@@ -85,8 +85,10 @@ class MiniMaxM3SparseMoeBlock(BaseOP):num_tokens, hidden_dim = hidden_states.shapehidden_states = hidden_states.view(-1, hidden_dim)topk_weights, topk_ids = self._route(hidden_states)+ # Routed kernels may overwrite hidden_states; shared experts need the original input.+ shared = self.shared_experts.forward(hidden_states)out = self.experts.routed_forward(hidden_states, topk_weights, topk_ids)- out = out + self.shared_experts.forward(hidden_states)+ out = out + sharedreturn out.view(num_tokens, hidden_dim)