🧠 两小时从 0 训出你的 64M 大模型。Train your own 64M LLM from scratch in 2 hours.
fix deepspeed local_rank bug
gongjy4ef9c4156301
diff --git a/1-pretrain.py b/1-pretrain.pyindex d51b0e3..219ca3e 100644--- a/1-pretrain.py+++ b/1-pretrain.py@@ -147,6 +147,7 @@ if __name__ == "__main__":parser.add_argument("--warmup_iters", type=int, default=0, help="Number of warmup iterations")parser.add_argument("--log_interval", type=int, default=100, help="Logging interval")parser.add_argument("--save_interval", type=int, default=1000, help="Model saving interval")+ parser.add_argument('--local_rank', type=int, default=-1, help='local rank for distributed training')args = parser.parse_args()diff --git a/3-full_sft.py b/3-full_sft.pyindex 551043b..6bee3c1 100644--- a/3-full_sft.py+++ b/3-full_sft.py@@ -155,6 +155,7 @@ if __name__ == "__main__":parser.add_argument("--warmup_iters", type=int, default=0, help="Number of warmup iterations")parser.add_argument("--log_interval", type=int, default=100, help="Logging interval")parser.add_argument("--save_interval", type=int, default=1000, help="Model saving interval")+ parser.add_argument('--local_rank', type=int, default=-1, help='local rank for distributed training')args = parser.parse_args()