🧠 两小时从 0 训出你的 64M 大模型。Train your own 64M LLM from scratch in 2 hours.
update model (fix loss bug)
gongjya87f6284007a
diff --git a/model/model.py b/model/model.pyindex e858ec7..20535fb 100644--- a/model/model.py+++ b/model/model.py@@ -369,7 +369,8 @@ class Transformer(PreTrainedModel):if targets is not None:logits = self.output(h)- self.last_loss = F.cross_entropy(logits.view(-1, logits.size(-1)), targets.view(-1), ignore_index=-1)+ self.last_loss = F.cross_entropy(logits.view(-1, logits.size(-1)), targets.view(-1),+ ignore_index=0, reduction='none')else:logits = self.output(h[:, [-1], :])self.last_loss = None