🧠 两小时从 0 训出你的 64M 大模型。Train your own 64M LLM from scratch in 2 hours.
[fix] route RL trainer forward through DDP so gradients all-reduce
basil-k-aji-dev9c088dd3a57f
diff --git a/trainer/train_agent.py b/trainer/train_agent.pyindex 261b00c..6b4360f 100644--- a/trainer/train_agent.py+++ b/trainer/train_agent.py@@ -278,9 +278,10 @@ def rl_train_epoch(epoch, loader, iters, rollout_engine, ref_model, reward_modelrewards = calculate_rewards(prompts, completions, gt_batch, tools_batch, args.num_generations, reward_model, device=args.device, turn_outputs_batch=turn_outputs_batch, unfinished_batch=unfinished_batch)- model_unwrapped = model.module if isinstance(model, DistributedDataParallel) else modelwith autocast_ctx:- res = model_unwrapped(input_ids, attention_mask=full_mask)+ # 反向传播必须经过 DDP 包装后的模块:直接调用 .module 会跳过+ # DDP 的 prepare_for_backward,梯度不会 all-reduce,各卡静默发散。+ res = model(input_ids, attention_mask=full_mask)aux_loss = res.aux_loss if lm_config.use_moe else torch.tensor(0.0, device=args.device)logits = res.logits[:, :-1, :]per_token_logps = F.log_softmax(logits, dim=-1).gather(2, input_ids[:, 1:].unsqueeze(-1)).squeeze(-1)diff --git a/trainer/train_grpo.py b/trainer/train_grpo.pyindex 76ea045..0ba7cf5 100755--- a/trainer/train_grpo.py+++ b/trainer/train_grpo.py@@ -95,9 +95,10 @@ def grpo_train_epoch(epoch, loader, iters, rollout_engine, ref_model, reward_modrewards = calculate_rewards(prompts, completions, reward_model).to(args.device) # [B*num_gen]- model_unwrapped = model.module if isinstance(model, DistributedDataParallel) else modelwith autocast_ctx:- res = model_unwrapped(outputs, attention_mask=full_mask)+ # 反向传播必须经过 DDP 包装后的模块:直接调用 .module 会跳过+ # DDP 的 prepare_for_backward,梯度不会 all-reduce,各卡静默发散。+ res = model(outputs, attention_mask=full_mask)aux_loss = res.aux_loss if lm_config.use_moe else torch.tensor(0.0, device=args.device)per_token_logps = F.log_softmax(res.logits[:, :-1, :], dim=-1).gather(2, outputs[:, 1:].unsqueeze(-1)).squeeze(-1).gather(1, logp_pos)diff --git a/trainer/train_ppo.py b/trainer/train_ppo.pyindex 07c7738..9023c6b 100644--- a/trainer/train_ppo.py+++ b/trainer/train_ppo.py@@ -159,8 +159,6 @@ def ppo_train_epoch(epoch, loader, iters, rollout_engine, ref_model, actor_schedclipfrac_sum = 0.0aux_loss_sum = 0.0log_count = 0- actor_unwrapped = actor_model.module if isinstance(actor_model, DistributedDataParallel) else actor_model- critic_unwrapped = critic_model.module if isinstance(critic_model, DistributedDataParallel) else critic_modelfor ppo_epoch in range(args.ppo_update_iters):if stop_ppo:break@@ -168,11 +166,13 @@ def ppo_train_epoch(epoch, loader, iters, rollout_engine, ref_model, actor_schedfor i in range(0, B, mb_size):inds = b_inds[i:i + mb_size]- mb_values_seq = critic_unwrapped(input_ids=gen_out[inds], attention_mask=full_mask[inds])+ # 反向传播必须经过 DDP 包装后的模块:直接调用 .module 会跳过+ # DDP 的 prepare_for_backward,梯度不会 all-reduce,各卡静默发散。+ mb_values_seq = critic_model(input_ids=gen_out[inds], attention_mask=full_mask[inds])mb_resp_values = mb_values_seq.gather(1, logp_pos[inds])with autocast_ctx:- res = actor_unwrapped(input_ids=gen_out[inds], attention_mask=full_mask[inds])+ res = actor_model(input_ids=gen_out[inds], attention_mask=full_mask[inds])aux_loss = res.aux_loss if lm_config.use_moe else torch.tensor(0.0, device=args.device)# 在 autocast 内计算 log_softmax,避免直接对 fp16/bf16 logits# 计算造成额外数值偏差。@@ -191,7 +191,7 @@ def ppo_train_epoch(epoch, loader, iters, rollout_engine, ref_model, actor_schedf"ratio_max={torch.exp(_lrv).max().item():.6f} "f"ratio_min={torch.exp(_lrv).min().item():.6f} "f"dropout={getattr(lm_config, 'dropout', None)} "- f"training={actor_unwrapped.training}")+ f"training={actor_model.training}")approx_kl = (0.5 * (log_ratio ** 2) * resp_policy_mask[inds]).sum() / resp_policy_mask[inds].sum().clamp(min=1)# 同步各卡的 approx_kl,防止某卡 break 而其它卡继续导致 DDP 死锁