🧠 两小时从 0 训出你的 64M 大模型。Train your own 64M LLM from scratch in 2 hours.
[fix] keep resume checkpoint weights in fp32
Vincent043b22a07326
diff --git a/trainer/trainer_utils.py b/trainer/trainer_utils.pyindex b171289..610c1f4 100644--- a/trainer/trainer_utils.py+++ b/trainer/trainer_utils.py@@ -71,10 +71,10 @@ def lm_checkpoint(lm_config, weight='full_sft', model=None, optimizer=None, epocif model is not None:raw_model = model.module if isinstance(model, DistributedDataParallel) else modelraw_model = getattr(raw_model, '_orig_mod', raw_model)- state_dict = raw_model.state_dict()- state_dict = {k: v.half().cpu() for k, v in state_dict.items()}+ # 续训权重保持fp32(fp16会抹掉小于半个ulp的累计更新),推理权重另存fp16;先.cpu()避免显存增长(#559)+ state_dict = {k: v.cpu() for k, v in raw_model.state_dict().items()}ckp_tmp = ckp_path + '.tmp'- torch.save(state_dict, ckp_tmp)+ torch.save({k: v.half() for k, v in state_dict.items()}, ckp_tmp)os.replace(ckp_tmp, ckp_path)wandb_id = Noneif wandb: