🧠 两小时从 0 训出你的 64M 大模型。Train your own 64M LLM from scratch in 2 hours.
fix dtype bug
gongjy89d260145f64
diff --git a/1-pretrain.py b/1-pretrain.pyindex ad1a45c..aa8da83 100644--- a/1-pretrain.py+++ b/1-pretrain.py@@ -179,7 +179,7 @@ if __name__ == "__main__":model = init_model()- scaler = torch.cuda.amp.GradScaler(enabled=(args.dtype == args.dtype))+ scaler = torch.cuda.amp.GradScaler(enabled=(args.dtype in ['float16', 'bfloat16']))optimizer = optim.Adam(model.parameters(), lr=args.learning_rate)if False and platform.system() != 'Windows' and float(torch.__version__.split('.')[0]) >= 2:diff --git a/3-full_sft.py b/3-full_sft.pyindex d82e662..e50f54e 100644--- a/3-full_sft.py+++ b/3-full_sft.py@@ -198,7 +198,7 @@ if __name__ == "__main__":sampler=train_sampler)- scaler = torch.cuda.amp.GradScaler(enabled=(args.dtype == args.dtype))+ scaler = torch.cuda.amp.GradScaler(enabled=(args.dtype in ['float16', 'bfloat16']))optimizer = optim.Adam(model.parameters(), lr=args.learning_rate)if False and not lm_config.use_moe and platform.system() != 'Windows' and float(torch.__version__.split('.')[0]) >= 2:diff --git a/4-lora_sft.py b/4-lora_sft.pyindex e72f8ca..f6328f1 100644--- a/4-lora_sft.py+++ b/4-lora_sft.py@@ -174,7 +174,7 @@ if __name__ == "__main__":num_workers=args.num_workers,)- scaler = torch.cuda.amp.GradScaler(enabled=(args.dtype == 'float16'))+ scaler = torch.cuda.amp.GradScaler(enabled=(args.dtype in ['float16', 'bfloat16']))optimizer = optim.Adam(model.parameters(), lr=args.learning_rate)if False and platform.system() != 'Windows' and float(torch.__version__.split('.')[0]) >= 2: