🧠 两小时从 0 训出你的 64M 大模型。Train your own 64M LLM from scratch in 2 hours.
rmsnorm float convert
jingyaogongbf81fd5f5ef4
diff --git a/model/model.py b/model/model.pyindex 1dc6478..6316a04 100644--- a/model/model.py+++ b/model/model.py@@ -14,13 +14,16 @@ from transformers.modeling_outputs import CausalLMOutputWithPastclass RMSNorm(torch.nn.Module):- def __init__(self, dim: int, eps: float):+ def __init__(self, dim: int, eps: float = 1e-6):super().__init__()self.eps = epsself.weight = nn.Parameter(torch.ones(dim))+ def _norm(self, x):+ return x * torch.rsqrt(x.pow(2).mean(-1, keepdim=True) + self.eps)+def forward(self, x):- return self.weight * (x.float() * torch.rsqrt(x.pow(2).mean(-1, keepdim=True) + self.eps)).type_as(x)+ return self.weight * self._norm(x.float()).type_as(x)def precompute_pos_cis(dim: int, end: int = int(32 * 1024), theta: float = 1e6):