A straightforward method for training your LLM, from downloading data to generating text.
fix(ddp): make the reward model train on multiple GPUs
FareedKhan-dev9476be56043f
diff --git a/scripts/train_reward.py b/scripts/train_reward.pyindex ff025bf..caec1ad 100644--- a/scripts/train_reward.py+++ b/scripts/train_reward.py@@ -63,7 +63,10 @@ def main():backbone = load_backbone_from_ckpt(cfg, cfg.sft_ckpt, ctx.device)rm = RewardModel(backbone).to(ctx.device)- rm = ddp_wrap(rm, ctx)+ # find_unused_parameters=True: the reward model uses the backbone's forward_hidden + a+ # reward head and never its lm_head, so lm_head params get no gradient. Without this flag+ # DDP errors on the first backward.+ rm = ddp_wrap(rm, ctx, find_unused_parameters=True)optimizer = configure_optimizer(unwrap(rm), cfg.lr, cfg.weight_decay)import json@@ -114,7 +117,9 @@ def main():metrics={"train_loss": loss.item()})if ctx.is_main:- acc, marg = eval_accuracy(rm, cfg, ctx)+ # Unwrap for the final eval: other ranks are already at cleanup(), so a collective on+ # the DDP-wrapped model here would hang (NCCL timeout). The periodic eval runs on all ranks.+ acc, marg = eval_accuracy(unwrap(rm), cfg, ctx)save_stage_ckpt(cfg.out_ckpt, rm, optimizer, stage="reward", cfg=cfg, step=total_steps,metrics={"test_acc": acc, "test_margin": marg})print(f"Done RM. test_acc {acc:.3f} margin {marg:.3f} -> {cfg.out_ckpt}")diff --git a/src/post_training/distributed.py b/src/post_training/distributed.pyindex 4d46975..71d33f4 100644--- a/src/post_training/distributed.py+++ b/src/post_training/distributed.py@@ -58,16 +58,21 @@ def ddp_setup(device: str = "cuda") -> DDPContext:return DDPContext(rank=rank, local_rank=local_rank, world_size=world_size, device=dev)-def ddp_wrap(model: torch.nn.Module, ctx: DDPContext) -> torch.nn.Module:+def ddp_wrap(model: torch.nn.Module, ctx: DDPContext, find_unused_parameters: bool = False) -> torch.nn.Module:"""Wrap a model in DDP when running multi-GPU; return it unchanged otherwise.Only the *trainable* model should be wrapped. Reference / old-policy / reward modelscarry no gradients and must NOT be wrapped (they are replicated identically per rank).++ Pass ``find_unused_parameters=True`` for a model where some parameters do not receive a+ gradient on every step (for example the reward model, which uses the backbone's+ ``forward_hidden`` and a reward head but never its ``lm_head``). Without it, DDP raises+ a "did not get a gradient" error on the first backward."""if not ctx.enabled:return modeldevice_ids = [ctx.local_rank] if ctx.device.startswith("cuda") else None- return DDP(model, device_ids=device_ids, find_unused_parameters=False)+ return DDP(model, device_ids=device_ids, find_unused_parameters=find_unused_parameters)def is_main_process(ctx: DDPContext) -> bool: