Skip to content

A straightforward method for training your LLM, from downloading data to generating text.

fix(ddp): make the reward model train on multiple GPUs

FareedKhan-dev9476be56043f
diff --git a/scripts/train_reward.py b/scripts/train_reward.py
index ff025bf..caec1ad 100644
--- a/scripts/train_reward.py
+++ b/scripts/train_reward.py
@@ -63,7 +63,10 @@ def main():
backbone = load_backbone_from_ckpt(cfg, cfg.sft_ckpt, ctx.device)
rm = RewardModel(backbone).to(ctx.device)
- rm = ddp_wrap(rm, ctx)
+ # find_unused_parameters=True: the reward model uses the backbone's forward_hidden + a
+ # reward head and never its lm_head, so lm_head params get no gradient. Without this flag
+ # DDP errors on the first backward.
+ rm = ddp_wrap(rm, ctx, find_unused_parameters=True)
optimizer = configure_optimizer(unwrap(rm), cfg.lr, cfg.weight_decay)
import json
@@ -114,7 +117,9 @@ def main():
metrics={"train_loss": loss.item()})
if ctx.is_main:
- acc, marg = eval_accuracy(rm, cfg, ctx)
+ # Unwrap for the final eval: other ranks are already at cleanup(), so a collective on
+ # the DDP-wrapped model here would hang (NCCL timeout). The periodic eval runs on all ranks.
+ acc, marg = eval_accuracy(unwrap(rm), cfg, ctx)
save_stage_ckpt(cfg.out_ckpt, rm, optimizer, stage="reward", cfg=cfg, step=total_steps,
metrics={"test_acc": acc, "test_margin": marg})
print(f"Done RM. test_acc {acc:.3f} margin {marg:.3f} -> {cfg.out_ckpt}")
diff --git a/src/post_training/distributed.py b/src/post_training/distributed.py
index 4d46975..71d33f4 100644
--- a/src/post_training/distributed.py
+++ b/src/post_training/distributed.py
@@ -58,16 +58,21 @@ def ddp_setup(device: str = "cuda") -> DDPContext:
return DDPContext(rank=rank, local_rank=local_rank, world_size=world_size, device=dev)
-def ddp_wrap(model: torch.nn.Module, ctx: DDPContext) -> torch.nn.Module:
+def ddp_wrap(model: torch.nn.Module, ctx: DDPContext, find_unused_parameters: bool = False) -> torch.nn.Module:
"""Wrap a model in DDP when running multi-GPU; return it unchanged otherwise.
Only the *trainable* model should be wrapped. Reference / old-policy / reward models
carry no gradients and must NOT be wrapped (they are replicated identically per rank).
+
+ Pass ``find_unused_parameters=True`` for a model where some parameters do not receive a
+ gradient on every step (for example the reward model, which uses the backbone's
+ ``forward_hidden`` and a reward head but never its ``lm_head``). Without it, DDP raises
+ a "did not get a gradient" error on the first backward.
"""
if not ctx.enabled:
return model
device_ids = [ctx.local_rank] if ctx.device.startswith("cuda") else None
- return DDP(model, device_ids=device_ids, find_unused_parameters=False)
+ return DDP(model, device_ids=device_ids, find_unused_parameters=find_unused_parameters)
def is_main_process(ctx: DDPContext) -> bool: