Skip to content

🧠 两小时从 0 训出你的 64M 大模型。Train your own 64M LLM from scratch in 2 hours.

Merge pull request #821 from lycDragonSlayer/refactor/remove-redundant-update-policy

jingyaogong7f5ba40a4eed
diff --cc trainer/train_agent.py
index 63a0782,e18aada..4a72a11
--- a/trainer/train_agent.py
+++ b/trainer/train_agent.py
@@@ -471,10 -470,8 +471,9 @@@ if __name__ == "__main__"
if args.use_compile == 1:
model = torch.compile(model)
Logger('torch.compile enabled')
- rollout_engine.update_policy(model)
if dist.is_initialized():
- model = DistributedDataParallel(model, device_ids=[local_rank])
+ # 同 train_ppo:RoPE buffer 各 rank 一致,每步广播纯属浪费
+ model = DistributedDataParallel(model, device_ids=[local_rank], broadcast_buffers=False)
rollout_engine.update_policy(model)
for epoch in range(start_epoch, args.epochs):
diff --cc trainer/train_grpo.py
index ad9aa3b,664e7e5..a7b01cf
--- a/trainer/train_grpo.py
+++ b/trainer/train_grpo.py
@@@ -312,10 -310,8 +312,9 @@@ if __name__ == "__main__"
if args.use_compile == 1:
model = torch.compile(model)
Logger('torch.compile enabled')
- rollout_engine.update_policy(model)
if dist.is_initialized():
- model = DistributedDataParallel(model, device_ids=[local_rank])
+ # 同 train_ppo:RoPE buffer 各 rank 一致,每步广播纯属浪费
+ model = DistributedDataParallel(model, device_ids=[local_rank], broadcast_buffers=False)
rollout_engine.update_policy(model)
# ========== 8. 开始训练 ==========
diff --cc trainer/train_ppo.py
index 4d762b6,f0427b6..b66d4b6
--- a/trainer/train_ppo.py
+++ b/trainer/train_ppo.py
@@@ -427,11 -426,9 +427,10 @@@ if __name__ == "__main__"
if args.use_compile == 1:
actor_model = torch.compile(actor_model)
Logger('torch.compile enabled')
- rollout_engine.update_policy(actor_model)
if dist.is_initialized():
- actor_model = DistributedDataParallel(actor_model, device_ids=[local_rank])
- critic_model = DistributedDataParallel(critic_model, device_ids=[local_rank])
+ # freqs_cos/freqs_sin 各 rank 由 config 确定性算出,默认每步广播一次纯属浪费
+ actor_model = DistributedDataParallel(actor_model, device_ids=[local_rank], broadcast_buffers=False)
+ critic_model = DistributedDataParallel(critic_model, device_ids=[local_rank], broadcast_buffers=False)
rollout_engine.update_policy(actor_model)
# ========== 8. 开始训练 ==========