🧠 两小时从 0 训出你的 64M 大模型。Train your own 64M LLM from scratch in 2 hours.
[feat] shuffle data
jingyaogonga82526da11ea
diff --git a/trainer/train_distill_reason.py b/trainer/train_distill_reason.pyindex 525490e..3bb15a8 100644--- a/trainer/train_distill_reason.py+++ b/trainer/train_distill_reason.py@@ -199,7 +199,7 @@ if __name__ == "__main__":batch_size=args.batch_size,pin_memory=True,drop_last=False,- shuffle=False,+ shuffle=(train_sampler is None),num_workers=args.num_workers,sampler=train_sampler)diff --git a/trainer/train_distillation.py b/trainer/train_distillation.pyindex 377e5f3..c97860c 100644--- a/trainer/train_distillation.py+++ b/trainer/train_distillation.py@@ -248,7 +248,7 @@ if __name__ == "__main__":batch_size=args.batch_size,pin_memory=True,drop_last=False,- shuffle=False,+ shuffle=(train_sampler is None),num_workers=args.num_workers,sampler=train_sampler)diff --git a/trainer/train_dpo.py b/trainer/train_dpo.pyindex 4b72afa..8c7bb45 100644--- a/trainer/train_dpo.py+++ b/trainer/train_dpo.py@@ -231,7 +231,7 @@ if __name__ == "__main__":batch_size=args.batch_size,pin_memory=True,drop_last=False,- shuffle=False,+ shuffle=(train_sampler is None),num_workers=args.num_workers,sampler=train_sampler)diff --git a/trainer/train_full_sft.py b/trainer/train_full_sft.pyindex 7c601a4..041801f 100644--- a/trainer/train_full_sft.py+++ b/trainer/train_full_sft.py@@ -185,7 +185,7 @@ if __name__ == "__main__":batch_size=args.batch_size,pin_memory=True,drop_last=False,- shuffle=False,+ shuffle=(train_sampler is None),num_workers=args.num_workers,sampler=train_sampler)diff --git a/trainer/train_grpo.py b/trainer/train_grpo.pyindex 7c33a94..704800d 100755--- a/trainer/train_grpo.py+++ b/trainer/train_grpo.py@@ -299,7 +299,7 @@ if __name__ == "__main__":train_ds = RLAIFDataset(args.data_path, tokenizer, max_length=lm_config.max_seq_len)train_sampler = DistributedSampler(train_ds) if ddp else Nonetrain_loader = DataLoader(train_ds, batch_size=args.batch_size, pin_memory=True,- drop_last=False, shuffle=False,+ drop_last=False, shuffle=(train_sampler is None),num_workers=args.num_workers, sampler=train_sampler)optimizer = optim.AdamW(model.parameters(), lr=args.learning_rate)diff --git a/trainer/train_lora.py b/trainer/train_lora.pyindex e2659f4..df9f9ae 100644--- a/trainer/train_lora.py+++ b/trainer/train_lora.py@@ -196,7 +196,7 @@ if __name__ == "__main__":batch_size=args.batch_size,pin_memory=True,drop_last=False,- shuffle=False,+ shuffle=(train_sampler is None),num_workers=args.num_workers,sampler=train_sampler)diff --git a/trainer/train_ppo.py b/trainer/train_ppo.pyindex 6ddbb64..7652c22 100644--- a/trainer/train_ppo.py+++ b/trainer/train_ppo.py@@ -344,7 +344,7 @@ if __name__ == "__main__":train_ds = RLAIFDataset(args.data_path, tokenizer, max_length=(args.max_seq_len + args.max_gen_len))train_sampler = DistributedSampler(train_ds) if ddp else Nonetrain_loader = DataLoader(train_ds, batch_size=args.batch_size, pin_memory=True,- drop_last=False, shuffle=False,+ drop_last=False, shuffle=(train_sampler is None),num_workers=args.num_workers, sampler=train_sampler)# 初始化优化器diff --git a/trainer/train_pretrain.py b/trainer/train_pretrain.pyindex 13b83fc..36a3cd8 100644--- a/trainer/train_pretrain.py+++ b/trainer/train_pretrain.py@@ -183,7 +183,7 @@ if __name__ == "__main__":batch_size=args.batch_size,pin_memory=True,drop_last=False,- shuffle=False,+ shuffle=(train_sampler is None),num_workers=args.num_workers,sampler=train_sampler)diff --git a/trainer/train_spo.py b/trainer/train_spo.pyindex 1b81791..e13e741 100755--- a/trainer/train_spo.py+++ b/trainer/train_spo.py@@ -348,7 +348,7 @@ if __name__ == "__main__":train_ds = RLAIFDataset(args.data_path, tokenizer, max_length=lm_config.max_seq_len)train_sampler = DistributedSampler(train_ds) if ddp else Nonetrain_loader = DataLoader(train_ds, batch_size=args.batch_size, pin_memory=True,- drop_last=False, shuffle=False,+ drop_last=False, shuffle=(train_sampler is None),num_workers=args.num_workers, sampler=train_sampler)optimizer = optim.AdamW(model.parameters(), lr=args.learning_rate)