🧠 两小时从 0 训出你的 64M 大模型。Train your own 64M LLM from scratch in 2 hours.
[fix] data skip
jingyaogongfea69cf338b8
diff --git a/trainer/train_distillation.py b/trainer/train_distillation.pyindex 2ec6fbe..5cc6269 100644--- a/trainer/train_distillation.py+++ b/trainer/train_distillation.py@@ -222,7 +222,7 @@ if __name__ == "__main__":for epoch in range(start_epoch, args.epochs):train_sampler and train_sampler.set_epoch(epoch)setup_seed(42 + epoch); indices = torch.randperm(len(train_ds)).tolist()- skip = start_step + 1 if (epoch == start_epoch and start_step > 0) else 0+ skip = start_step if (epoch == start_epoch and start_step > 0) else 0batch_sampler = SkipBatchSampler(train_sampler or indices, args.batch_size, skip)loader = DataLoader(train_ds, batch_sampler=batch_sampler, num_workers=args.num_workers, pin_memory=True)if skip > 0:diff --git a/trainer/train_dpo.py b/trainer/train_dpo.pyindex 31102cf..f1e567d 100644--- a/trainer/train_dpo.py+++ b/trainer/train_dpo.py@@ -206,7 +206,7 @@ if __name__ == "__main__":for epoch in range(start_epoch, args.epochs):train_sampler and train_sampler.set_epoch(epoch)setup_seed(42 + epoch); indices = torch.randperm(len(train_ds)).tolist()- skip = start_step + 1 if (epoch == start_epoch and start_step > 0) else 0+ skip = start_step if (epoch == start_epoch and start_step > 0) else 0batch_sampler = SkipBatchSampler(train_sampler or indices, args.batch_size, skip)loader = DataLoader(train_ds, batch_sampler=batch_sampler, num_workers=args.num_workers, pin_memory=True)if skip > 0:diff --git a/trainer/train_full_sft.py b/trainer/train_full_sft.pyindex 7b64a23..cc59cc7 100644--- a/trainer/train_full_sft.py+++ b/trainer/train_full_sft.py@@ -149,7 +149,7 @@ if __name__ == "__main__":for epoch in range(start_epoch, args.epochs):train_sampler and train_sampler.set_epoch(epoch)setup_seed(42 + epoch); indices = torch.randperm(len(train_ds)).tolist()- skip = start_step + 1 if (epoch == start_epoch and start_step > 0) else 0+ skip = start_step if (epoch == start_epoch and start_step > 0) else 0batch_sampler = SkipBatchSampler(train_sampler or indices, args.batch_size, skip)loader = DataLoader(train_ds, batch_sampler=batch_sampler, num_workers=args.num_workers, pin_memory=True)if skip > 0:diff --git a/trainer/train_grpo.py b/trainer/train_grpo.pyindex bab4223..5e63779 100755--- a/trainer/train_grpo.py+++ b/trainer/train_grpo.py@@ -291,7 +291,7 @@ if __name__ == "__main__":for epoch in range(start_epoch, args.epochs):train_sampler and train_sampler.set_epoch(epoch)setup_seed(42 + epoch); indices = torch.randperm(len(train_ds)).tolist()- skip = start_step + 1 if (epoch == start_epoch and start_step > 0) else 0+ skip = start_step if (epoch == start_epoch and start_step > 0) else 0batch_sampler = SkipBatchSampler(train_sampler or indices, args.batch_size, skip)loader = DataLoader(train_ds, batch_sampler=batch_sampler, num_workers=args.num_workers, pin_memory=True)if skip > 0:diff --git a/trainer/train_lora.py b/trainer/train_lora.pyindex 83d54c6..f5b0235 100644--- a/trainer/train_lora.py+++ b/trainer/train_lora.py@@ -162,7 +162,7 @@ if __name__ == "__main__":for epoch in range(start_epoch, args.epochs):train_sampler and train_sampler.set_epoch(epoch)setup_seed(42 + epoch); indices = torch.randperm(len(train_ds)).tolist()- skip = start_step + 1 if (epoch == start_epoch and start_step > 0) else 0+ skip = start_step if (epoch == start_epoch and start_step > 0) else 0batch_sampler = SkipBatchSampler(train_sampler or indices, args.batch_size, skip)loader = DataLoader(train_ds, batch_sampler=batch_sampler, num_workers=args.num_workers, pin_memory=True)if skip > 0:diff --git a/trainer/train_ppo.py b/trainer/train_ppo.pyindex 47af75a..88c11bc 100644--- a/trainer/train_ppo.py+++ b/trainer/train_ppo.py@@ -366,7 +366,7 @@ if __name__ == "__main__":for epoch in range(start_epoch, args.epochs):train_sampler and train_sampler.set_epoch(epoch)setup_seed(42 + epoch); indices = torch.randperm(len(train_ds)).tolist()- skip = start_step + 1 if (epoch == start_epoch and start_step > 0) else 0+ skip = start_step if (epoch == start_epoch and start_step > 0) else 0batch_sampler = SkipBatchSampler(train_sampler or indices, args.batch_size, skip)loader = DataLoader(train_ds, batch_sampler=batch_sampler, num_workers=args.num_workers, pin_memory=True)if skip > 0:diff --git a/trainer/train_pretrain.py b/trainer/train_pretrain.pyindex ea0ca21..a8ad97f 100644--- a/trainer/train_pretrain.py+++ b/trainer/train_pretrain.py@@ -148,7 +148,7 @@ if __name__ == "__main__":for epoch in range(start_epoch, args.epochs):train_sampler and train_sampler.set_epoch(epoch)setup_seed(42 + epoch); indices = torch.randperm(len(train_ds)).tolist()- skip = start_step + 1 if (epoch == start_epoch and start_step > 0) else 0+ skip = start_step if (epoch == start_epoch and start_step > 0) else 0batch_sampler = SkipBatchSampler(train_sampler or indices, args.batch_size, skip)loader = DataLoader(train_ds, batch_sampler=batch_sampler, num_workers=args.num_workers, pin_memory=True)if skip > 0:diff --git a/trainer/train_reason.py b/trainer/train_reason.pyindex eaae014..a9b2d55 100644--- a/trainer/train_reason.py+++ b/trainer/train_reason.py@@ -166,7 +166,7 @@ if __name__ == "__main__":for epoch in range(start_epoch, args.epochs):train_sampler and train_sampler.set_epoch(epoch)setup_seed(42 + epoch); indices = torch.randperm(len(train_ds)).tolist()- skip = start_step + 1 if (epoch == start_epoch and start_step > 0) else 0+ skip = start_step if (epoch == start_epoch and start_step > 0) else 0batch_sampler = SkipBatchSampler(train_sampler or indices, args.batch_size, skip)loader = DataLoader(train_ds, batch_sampler=batch_sampler, num_workers=args.num_workers, pin_memory=True)if skip > 0:diff --git a/trainer/train_spo.py b/trainer/train_spo.pyindex b6308d5..219ddba 100755--- a/trainer/train_spo.py+++ b/trainer/train_spo.py@@ -341,7 +341,7 @@ if __name__ == "__main__":for epoch in range(start_epoch, args.epochs):train_sampler and train_sampler.set_epoch(epoch)setup_seed(42 + epoch); indices = torch.randperm(len(train_ds)).tolist()- skip = start_step + 1 if (epoch == start_epoch and start_step > 0) else 0+ skip = start_step if (epoch == start_epoch and start_step > 0) else 0batch_sampler = SkipBatchSampler(train_sampler or indices, args.batch_size, skip)loader = DataLoader(train_ds, batch_sampler=batch_sampler, num_workers=args.num_workers, pin_memory=True)if skip > 0: