🧠 两小时从 0 训出你的 64M 大模型。Train your own 64M LLM from scratch in 2 hours.
[fix] restore top-1 router gradient #858
jingyaogongb63a9565e305
diff --git a/model/model_minimind.py b/model/model_minimind.pyindex 96dd8fb..b2e314e 100755--- a/model/model_minimind.py+++ b/model/model_minimind.py@@ -158,14 +158,12 @@ class MOEFeedForward(nn.Module):x_flat = x.view(-1, hidden_dim)scores = F.softmax(self.gate(x_flat), dim=-1)topk_weight, topk_idx = torch.topk(scores, k=self.config.num_experts_per_tok, dim=-1, sorted=False)- # Normalising a single top-k weight by its own sum yields 1.0 for every- # token, so the gate drops out of the graph: topk_idx is discrete and the- # weight is now a constant, leaving the LM loss no path to the router. It- # then trains on aux_loss alone, whose optimum is a uniform distribution,- # and the experts never specialise. Only normalise when there is more than- # one weight to normalise.- if self.config.norm_topk_prob and self.config.num_experts_per_tok > 1:- topk_weight = topk_weight / (topk_weight.sum(dim=-1, keepdim=True) + 1e-20)+ if self.config.norm_topk_prob:+ if self.config.num_experts_per_tok > 1:+ topk_weight = topk_weight / (topk_weight.sum(dim=-1, keepdim=True) + 1e-20)+ else:+ top1 = torch.topk(F.softmax(self.gate(x_flat.detach()), dim=-1), k=1, dim=-1, sorted=False)[0]+ topk_weight = top1 - top1.detach() + 1.0y = torch.zeros_like(x_flat)for i, expert in enumerate(self.experts):mask = (topk_idx == i)