🧠 两小时从 0 训出你的 64M 大模型。Train your own 64M LLM from scratch in 2 hours.
update cis init
gongjy5b65bc767e76
diff --git a/model/model.py b/model/model.pyindex 070a8f2..3f49236 100644--- a/model/model.py+++ b/model/model.py@@ -23,7 +23,7 @@ class RMSNorm(torch.nn.Module):return self.weight * (x.float() * torch.rsqrt(x.pow(2).mean(-1, keepdim=True) + self.eps)).type_as(x)-def precompute_pos_cis(dim: int, end: int, theta: float = 1e4):+def precompute_pos_cis(dim: int, end: int = int(32 * 1024), theta: float = 1e6):freqs = 1.0 / (theta ** (torch.arange(0, dim, 2)[: (dim // 2)].float() / dim))t = torch.arange(end, device=freqs.device) # type: ignorefreqs = torch.outer(t, freqs).float() # type: ignore@@ -295,8 +295,9 @@ class MiniMindLM(PreTrainedModel):self.norm = RMSNorm(params.dim, eps=params.norm_eps)self.output = nn.Linear(params.dim, params.vocab_size, bias=False)self.tok_embeddings.weight = self.output.weight- self.register_buffer("pos_cis", precompute_pos_cis(params.dim // params.n_heads, params.max_seq_len,- theta=params.rope_theta), persistent=False)+ self.register_buffer("pos_cis",+ precompute_pos_cis(dim=params.dim // params.n_heads, theta=params.rope_theta),+ persistent=False)self.OUT = CausalLMOutputWithPast()def forward(self,