🧠 两小时从 0 训出你的 64M 大模型。Train your own 64M LLM from scratch in 2 hours.
[fix] dist cleanup
jingyaogong42a4e8c86a0e
diff --git a/trainer/train_distill_reason.py b/trainer/train_distill_reason.pyindex 3a10847..9162cca 100644--- a/trainer/train_distill_reason.py+++ b/trainer/train_distill_reason.py@@ -176,3 +176,6 @@ if __name__ == "__main__":else: # 默认从头开始loader = DataLoader(train_ds, batch_size=args.batch_size, shuffle=(train_sampler is None), sampler=train_sampler, num_workers=args.num_workers, pin_memory=True)train_epoch(epoch, loader, len(loader), tokenizer, lm_config, 0, wandb)++ # ========== 9. 清理分布进程 ==========+ if dist.is_initialized(): dist.destroy_process_group()\ No newline at end of filediff --git a/trainer/train_distillation.py b/trainer/train_distillation.pyindex 24450a3..1deec83 100644--- a/trainer/train_distillation.py+++ b/trainer/train_distillation.py@@ -226,3 +226,6 @@ if __name__ == "__main__":else: # 默认从头开始loader = DataLoader(train_ds, batch_size=args.batch_size, shuffle=(train_sampler is None), sampler=train_sampler, num_workers=args.num_workers, pin_memory=True)train_epoch(epoch, loader, len(loader), teacher_model, lm_config_student, 0, wandb, args.alpha, args.temperature)++ # ========== 9. 清理分布进程 ==========+ if dist.is_initialized(): dist.destroy_process_group()\ No newline at end of filediff --git a/trainer/train_dpo.py b/trainer/train_dpo.pyindex eccfaff..c4987df 100644--- a/trainer/train_dpo.py+++ b/trainer/train_dpo.py@@ -211,3 +211,6 @@ if __name__ == "__main__":else: # 默认从头开始loader = DataLoader(train_ds, batch_size=args.batch_size, shuffle=(train_sampler is None), sampler=train_sampler, num_workers=args.num_workers, pin_memory=True)train_epoch(epoch, loader, len(loader), ref_model, lm_config, 0, wandb, args.beta)++ # ========== 9. 清理分布进程 ==========+ if dist.is_initialized(): dist.destroy_process_group()\ No newline at end of filediff --git a/trainer/train_full_sft.py b/trainer/train_full_sft.pyindex faec3f5..2ab8f6b 100644--- a/trainer/train_full_sft.py+++ b/trainer/train_full_sft.py@@ -164,3 +164,6 @@ if __name__ == "__main__":else: # 默认从头开始loader = DataLoader(train_ds, batch_size=args.batch_size, shuffle=(train_sampler is None), sampler=train_sampler, num_workers=args.num_workers, pin_memory=True)train_epoch(epoch, loader, len(loader), 0, wandb)++ # ========== 9. 清理分布进程 ==========+ if dist.is_initialized(): dist.destroy_process_group()\ No newline at end of filediff --git a/trainer/train_grpo.py b/trainer/train_grpo.pyindex d5b30ed..1438bdd 100755--- a/trainer/train_grpo.py+++ b/trainer/train_grpo.py@@ -294,3 +294,6 @@ if __name__ == "__main__":drop_last=False, shuffle=(train_sampler is None),num_workers=args.num_workers, sampler=train_sampler)grpo_train_epoch(epoch, loader, len(loader), ref_model, reward_model, reward_tokenizer, 0, wandb)++ # ========== 9. 清理分布进程 ==========+ if dist.is_initialized(): dist.destroy_process_group()\ No newline at end of filediff --git a/trainer/train_lora.py b/trainer/train_lora.pyindex dff0f19..2af9739 100644--- a/trainer/train_lora.py+++ b/trainer/train_lora.py@@ -177,3 +177,6 @@ if __name__ == "__main__":else: # 默认从头开始loader = DataLoader(train_ds, batch_size=args.batch_size, shuffle=(train_sampler is None), sampler=train_sampler, num_workers=args.num_workers, pin_memory=True)train_epoch(epoch, loader, len(loader), lora_params, 0, wandb)++ # ========== 10. 清理分布进程 ==========+ if dist.is_initialized(): dist.destroy_process_group()\ No newline at end of filediff --git a/trainer/train_ppo.py b/trainer/train_ppo.pyindex fd094b3..d38836d 100644--- a/trainer/train_ppo.py+++ b/trainer/train_ppo.py@@ -368,3 +368,6 @@ if __name__ == "__main__":sampler=train_sampler, num_workers=args.num_workers, pin_memory=True)ppo_train_epoch(epoch, loader, len(loader), old_actor_model, ref_model,actor_scheduler, critic_scheduler, reward_model, reward_tokenizer, 0, wandb)++ # ========== 9. 清理分布进程 ==========+ if dist.is_initialized(): dist.destroy_process_group()\ No newline at end of filediff --git a/trainer/train_pretrain.py b/trainer/train_pretrain.pyindex 4596041..29c4d88 100644--- a/trainer/train_pretrain.py+++ b/trainer/train_pretrain.py@@ -163,3 +163,6 @@ if __name__ == "__main__":else: # 默认从头开始loader = DataLoader(train_ds, batch_size=args.batch_size, shuffle=(train_sampler is None), sampler=train_sampler, num_workers=args.num_workers, pin_memory=True)train_epoch(epoch, loader, len(loader), 0, wandb)++ # ========== 9. 清理分布进程 ==========+ if dist.is_initialized(): dist.destroy_process_group()\ No newline at end of filediff --git a/trainer/train_spo.py b/trainer/train_spo.pyindex a15a2ac..b81e6b5 100755--- a/trainer/train_spo.py+++ b/trainer/train_spo.py@@ -344,3 +344,6 @@ if __name__ == "__main__":drop_last=False, shuffle=(train_sampler is None),num_workers=args.num_workers, sampler=train_sampler)spo_train_epoch(epoch, loader, len(loader), ref_model, reward_model, reward_tokenizer, value_tracker, 0, wandb)++ # ========== 9. 清理分布进程 ==========+ if dist.is_initialized(): dist.destroy_process_group()\ No newline at end of file