Skip to content

🧠 两小时从 0 训出你的 64M 大模型。Train your own 64M LLM from scratch in 2 hours.

update model (fix loss bug)

gongjya87f6284007a
diff --git a/model/model.py b/model/model.py
index e858ec7..20535fb 100644
--- a/model/model.py
+++ b/model/model.py
@@ -369,7 +369,8 @@ class Transformer(PreTrainedModel):
if targets is not None:
logits = self.output(h)
- self.last_loss = F.cross_entropy(logits.view(-1, logits.size(-1)), targets.view(-1), ignore_index=-1)
+ self.last_loss = F.cross_entropy(logits.view(-1, logits.size(-1)), targets.view(-1),
+ ignore_index=0, reduction='none')
else:
logits = self.output(h[:, [-1], :])
self.last_loss = None