Skip to content

🧠 两小时从 0 训出你的 64M 大模型。Train your own 64M LLM from scratch in 2 hours.

[fix] keep the MoE router trainable at num_experts_per_tok=1

basil-k-aji-dev1c876435ef9f
diff --git a/model/model_minimind.py b/model/model_minimind.py
index dc080e5..96dd8fb 100755
--- a/model/model_minimind.py
+++ b/model/model_minimind.py
@@ -158,7 +158,14 @@ class MOEFeedForward(nn.Module):
x_flat = x.view(-1, hidden_dim)
scores = F.softmax(self.gate(x_flat), dim=-1)
topk_weight, topk_idx = torch.topk(scores, k=self.config.num_experts_per_tok, dim=-1, sorted=False)
- if self.config.norm_topk_prob: topk_weight = topk_weight / (topk_weight.sum(dim=-1, keepdim=True) + 1e-20)
+ # Normalising a single top-k weight by its own sum yields 1.0 for every
+ # token, so the gate drops out of the graph: topk_idx is discrete and the
+ # weight is now a constant, leaving the LM loss no path to the router. It
+ # then trains on aux_loss alone, whose optimum is a uniform distribution,
+ # and the experts never specialise. Only normalise when there is more than
+ # one weight to normalise.
+ if self.config.norm_topk_prob and self.config.num_experts_per_tok > 1:
+ topk_weight = topk_weight / (topk_weight.sum(dim=-1, keepdim=True) + 1e-20)
y = torch.zeros_like(x_flat)
for i, expert in enumerate(self.experts):
mask = (topk_idx == i)