Skip to content

🧠 两小时从 0 训出你的 64M 大模型。Train your own 64M LLM from scratch in 2 hours.

[perf] skip the per-forward RoPE buffer broadcast in the RL trainers

basil-k-aji-dev37ac206e9ac8
diff --git a/trainer/train_agent.py b/trainer/train_agent.py
index 50520aa..c8b7f88 100644
--- a/trainer/train_agent.py
+++ b/trainer/train_agent.py
@@ -473,7 +473,9 @@ if __name__ == "__main__":
Logger('torch.compile enabled')
rollout_engine.update_policy(model)
if dist.is_initialized():
- model = DistributedDataParallel(model, device_ids=[local_rank])
+ # See train_ppo.py: the RoPE buffers are identical on every rank, so
+ # the per-forward buffer broadcast is pure overhead.
+ model = DistributedDataParallel(model, device_ids=[local_rank], broadcast_buffers=False)
rollout_engine.update_policy(model)
for epoch in range(start_epoch, args.epochs):
diff --git a/trainer/train_grpo.py b/trainer/train_grpo.py
index 0ba7cf5..9d3a232 100755
--- a/trainer/train_grpo.py
+++ b/trainer/train_grpo.py
@@ -314,7 +314,9 @@ if __name__ == "__main__":
Logger('torch.compile enabled')
rollout_engine.update_policy(model)
if dist.is_initialized():
- model = DistributedDataParallel(model, device_ids=[local_rank])
+ # See train_ppo.py: the RoPE buffers are identical on every rank, so
+ # the per-forward buffer broadcast is pure overhead.
+ model = DistributedDataParallel(model, device_ids=[local_rank], broadcast_buffers=False)
rollout_engine.update_policy(model)
# ========== 8. 开始训练 ==========
diff --git a/trainer/train_ppo.py b/trainer/train_ppo.py
index 9023c6b..0c8625e 100644
--- a/trainer/train_ppo.py
+++ b/trainer/train_ppo.py
@@ -34,6 +34,19 @@ def rep_penalty(text, n=3, cap=0.5):
# 自定义的Critic模型,继承自MiniMindLM
class CriticModel(MiniMindForCausalLM):
+ """Value head over the base model. `forward` never calls `lm_head`.
+
+ This relies on `tie_word_embeddings` (default True), which makes
+ `lm_head.weight` the *same* Parameter as `model.embed_tokens.weight`. The
+ embedding is used on every forward, so the tied parameter still receives a
+ gradient and DDP sees no unused parameters.
+
+ Untie it and `lm_head.weight` becomes a real parameter that this forward
+ never touches, so DDP raises "Expected to have finished reduction in the
+ prior iteration" unless the critic is wrapped with
+ `find_unused_parameters=True` (or `lm_head` is deleted from the critic).
+ """
+
def __init__(self, params):
super().__init__(params)
# 替换lm_head为输出单一价值的线性层
@@ -429,8 +442,12 @@ if __name__ == "__main__":
Logger('torch.compile enabled')
rollout_engine.update_policy(actor_model)
if dist.is_initialized():
- actor_model = DistributedDataParallel(actor_model, device_ids=[local_rank])
- critic_model = DistributedDataParallel(critic_model, device_ids=[local_rank])
+ # broadcast_buffers=False: the only registered buffers are the RoPE
+ # tables (freqs_cos / freqs_sin), which every rank recomputes
+ # deterministically from config and never mutates from data. The
+ # default re-broadcasts them from rank 0 on every forward for nothing.
+ actor_model = DistributedDataParallel(actor_model, device_ids=[local_rank], broadcast_buffers=False)
+ critic_model = DistributedDataParallel(critic_model, device_ids=[local_rank], broadcast_buffers=False)
rollout_engine.update_policy(actor_model)
# ========== 8. 开始训练 ==========