🧠 两小时从 0 训出你的 64M 大模型。Train your own 64M LLM from scratch in 2 hours.
[fix] sampler-ddp
jingyaogong463044e92a33
diff --git a/trainer/train_distill_reason.py b/trainer/train_distill_reason.pyindex ab7866c..9c1c9eb 100644--- a/trainer/train_distill_reason.py+++ b/trainer/train_distill_reason.py@@ -213,4 +213,5 @@ if __name__ == "__main__":iter_per_epoch = len(train_loader)for epoch in range(args.epochs):+ train_sampler and train_sampler.set_epoch(epoch)train_epoch(epoch, wandb)diff --git a/trainer/train_distillation.py b/trainer/train_distillation.pyindex f4e2d41..9b6c0a7 100644--- a/trainer/train_distillation.py+++ b/trainer/train_distillation.py@@ -262,4 +262,5 @@ if __name__ == "__main__":iter_per_epoch = len(train_loader)for epoch in range(args.epochs):+ train_sampler and train_sampler.set_epoch(epoch)train_epoch(epoch, wandb)diff --git a/trainer/train_dpo.py b/trainer/train_dpo.pyindex c4acf36..cdb9d92 100644--- a/trainer/train_dpo.py+++ b/trainer/train_dpo.py@@ -245,4 +245,5 @@ if __name__ == "__main__":iter_per_epoch = len(train_loader)for epoch in range(args.epochs):+ train_sampler and train_sampler.set_epoch(epoch)train_epoch(epoch, wandb)diff --git a/trainer/train_full_sft.py b/trainer/train_full_sft.pyindex 58f87aa..bae0dff 100644--- a/trainer/train_full_sft.py+++ b/trainer/train_full_sft.py@@ -199,4 +199,5 @@ if __name__ == "__main__":iter_per_epoch = len(train_loader)for epoch in range(args.epochs):+ train_sampler and train_sampler.set_epoch(epoch)train_epoch(epoch, wandb)diff --git a/trainer/train_grpo.py b/trainer/train_grpo.pyindex 0402f41..8aeb349 100755--- a/trainer/train_grpo.py+++ b/trainer/train_grpo.py@@ -313,4 +313,5 @@ if __name__ == "__main__":model = DistributedDataParallel(model, device_ids=[ddp_local_rank])for epoch in range(args.epochs):+ train_sampler and train_sampler.set_epoch(epoch)grpo_train_epoch(epoch, wandb)diff --git a/trainer/train_lora.py b/trainer/train_lora.pyindex 5da1607..4c73126 100644--- a/trainer/train_lora.py+++ b/trainer/train_lora.py@@ -205,4 +205,5 @@ if __name__ == "__main__":iter_per_epoch = len(train_loader)for epoch in range(args.epochs):+ train_sampler and train_sampler.set_epoch(epoch)train_epoch(epoch, wandb)diff --git a/trainer/train_ppo.py b/trainer/train_ppo.pyindex 1132da9..f8a8d38 100644--- a/trainer/train_ppo.py+++ b/trainer/train_ppo.py@@ -367,6 +367,7 @@ if __name__ == "__main__":old_actor_model.to(args.device)for epoch in range(args.epochs):+ train_sampler and train_sampler.set_epoch(epoch)ppo_train_epoch(epoch, wandb, old_actor_model, ref_model, actor_scheduler, critic_scheduler)if ddp:diff --git a/trainer/train_pretrain.py b/trainer/train_pretrain.pyindex 4166db5..ab6f68f 100644--- a/trainer/train_pretrain.py+++ b/trainer/train_pretrain.py@@ -197,4 +197,5 @@ if __name__ == "__main__":iter_per_epoch = len(train_loader)for epoch in range(args.epochs):+ train_sampler and train_sampler.set_epoch(epoch)train_epoch(epoch, wandb)diff --git a/trainer/train_spo.py b/trainer/train_spo.pyindex 8db67b9..7ea15bf 100755--- a/trainer/train_spo.py+++ b/trainer/train_spo.py@@ -364,4 +364,5 @@ if __name__ == "__main__":value_tracker = AutoAdaptiveValueTracker(rho_mode='kl', rho_const=0.9, D_half=0.06, clip_lower=0.5, clip_upper=0.96)for epoch in range(args.epochs):+ train_sampler and train_sampler.set_epoch(epoch)spo_train_epoch(epoch, wandb, value_tracker)