🧠 两小时从 0 训出你的 64M 大模型。Train your own 64M LLM from scratch in 2 hours.
update dpo_loss
jingyaogong278ec760a191
diff --git a/train_dpo.py b/train_dpo.pyindex dc9054a..e0b67af 100644--- a/train_dpo.py+++ b/train_dpo.py@@ -40,11 +40,12 @@ def logits_to_probs(logits, labels):return probs-def dpo_loss(ref_probs, probs, beta):+def dpo_loss(ref_probs, probs, mask, beta):# ref_probs 和 probs 都是 shape: (batch_size, seq_len)- # 计算每个样本的平均概率- ref_probs = ref_probs.mean(dim=1)- probs = probs.mean(dim=1)+ # https://github.com/jingyaogong/minimind/issues/298+ seq_lengths = mask.sum(dim=1, keepdim=True) # (batch_size, 1)+ ref_probs = (ref_probs * mask).sum(dim=1) / seq_lengths.squeeze()+ probs = (probs * mask).sum(dim=1) / seq_lengths.squeeze()# 将 chosen 和 rejected 数据分开batch_size = ref_probs.shape[0]@@ -87,7 +88,7 @@ def train_epoch(epoch, wandb):logits = outputs.logitsprobs = logits_to_probs(logits, y)probs = probs * mask- loss = dpo_loss(ref_probs, probs, beta=0.1)+ loss = dpo_loss(ref_probs, probs, mask, beta=0.1)loss = loss / args.accumulation_stepsscaler.scale(loss).backward()@@ -183,7 +184,7 @@ if __name__ == "__main__":parser.add_argument('--local_rank', type=int, default=-1)parser.add_argument('--dim', default=512, type=int)parser.add_argument('--n_layers', default=8, type=int)- parser.add_argument('--max_seq_len', default=3000, type=int)+ parser.add_argument('--max_seq_len', default=1024, type=int)parser.add_argument('--use_moe', default=False, type=bool)parser.add_argument("--data_path", type=str, default="./dataset/dpo.jsonl")