🧠 两小时从 0 训出你的 64M 大模型。Train your own 64M LLM from scratch in 2 hours.
[fix] LoRA: add alpha/rank scaling with backward-compatible persistence
uphomer9012e5e0ef1c
diff --git a/model/model_lora.py b/model/model_lora.pyindex 9d96955..4b54a96 100644--- a/model/model_lora.py+++ b/model/model_lora.py@@ -4,9 +4,12 @@ from torch import optim, nn# 定义Lora网络结构class LoRA(nn.Module):- def __init__(self, in_features, out_features, rank):+ def __init__(self, in_features, out_features, rank, lora_alpha=None):super().__init__()self.rank = rank # LoRA的秩(rank),控制低秩矩阵的大小+ # 标准 LoRA 的 alpha/r 缩放;lora_alpha=None 表示 alpha=rank(scaling=1),保持历史行为+ scaling = (rank if lora_alpha is None else lora_alpha) / rank+ self.register_buffer("scaling", torch.tensor(scaling, dtype=torch.float32))self.A = nn.Linear(in_features, rank, bias=False) # 低秩矩阵Aself.B = nn.Linear(rank, out_features, bias=False) # 低秩矩阵B# 矩阵A高斯初始化@@ -15,13 +18,13 @@ class LoRA(nn.Module):self.B.weight.data.zero_()def forward(self, x):- return self.B(self.A(x))+ return self.B(self.A(x)) * self.scaling-def apply_lora(model, rank=16):+def apply_lora(model, rank=16, lora_alpha=None):for name, module in model.named_modules():if isinstance(module, nn.Linear) and module.in_features == module.out_features:- lora = LoRA(module.in_features, module.out_features, rank=rank).to(model.device)+ lora = LoRA(module.in_features, module.out_features, rank=rank, lora_alpha=lora_alpha).to(model.device)setattr(module, "lora", lora)original_forward = module.forward@@ -39,7 +42,8 @@ def load_lora(model, path):for name, module in model.named_modules():if hasattr(module, 'lora'):lora_state = {k.replace(f'{name}.lora.', ''): v for k, v in state_dict.items() if f'{name}.lora.' in k}- module.lora.load_state_dict(lora_state)+ # 兼容旧 checkpoint:旧版本没有 scaling buffer(等价于 scaling=1),缺失时保留默认值+ module.lora.load_state_dict(lora_state, strict=False)def save_lora(model, path):@@ -48,7 +52,11 @@ def save_lora(model, path):for name, module in raw_model.named_modules():if hasattr(module, 'lora'):clean_name = name[7:] if name.startswith("module.") else name- lora_state = {f'{clean_name}.lora.{k}': v.cpu().half() for k, v in module.lora.state_dict().items()}+ lora_state = {}+ for k, v in module.lora.state_dict().items():+ # 权重保持 half 以维持文件体积;scaling 等 buffer 用 float32,避免精度损失+ v = v.cpu().half() if k.endswith('.weight') else v.cpu().float()+ lora_state[f'{clean_name}.lora.{k}'] = vstate_dict.update(lora_state)torch.save(state_dict, path)@@ -61,5 +69,7 @@ def merge_lora(model, lora_path, save_path):if isinstance(module, nn.Linear) and '.lora.' not in name:state_dict[f'{name}.weight'] = module.weight.data.clone().cpu().half()if hasattr(module, 'lora'):- state_dict[f'{name}.weight'] += (module.lora.B.weight.data @ module.lora.A.weight.data).cpu().half()+ delta = module.lora.B.weight.data @ module.lora.A.weight.data+ delta = (delta * module.lora.scaling.to(delta.device)).cpu().half()+ state_dict[f'{name}.weight'] += deltatorch.save(state_dict, save_path)diff --git a/trainer/train_lora.py b/trainer/train_lora.pyindex 35a87fc..bfdcf50 100644--- a/trainer/train_lora.py+++ b/trainer/train_lora.py@@ -70,6 +70,7 @@ if __name__ == "__main__":parser = argparse.ArgumentParser(description="MiniMind LoRA Fine-tuning")parser.add_argument("--save_dir", type=str, default="../out", help="模型保存目录")parser.add_argument("--lora_rank", default=16, type=int, help="LoRA rank")+ parser.add_argument("--lora_alpha", default=None, type=float, help="LoRA alpha(默认等于 rank,即 scaling=1)")parser.add_argument("--lora_name", type=str, default="lora_medical", help="LoRA权重名称(如lora_identity/lora_medical等)")parser.add_argument("--epochs", type=int, default=10, help="训练轮数")parser.add_argument("--batch_size", type=int, default=32, help="batch size")@@ -119,7 +120,7 @@ if __name__ == "__main__":# ========== 5. 定义模型、应用LoRA、冻结非LoRA参数 ==========model, tokenizer = init_model(lm_config, args.from_weight, device=args.device)- apply_lora(model, rank=args.lora_rank)+ apply_lora(model, rank=args.lora_rank, lora_alpha=args.lora_alpha)# 统计参数total_params = sum(p.numel() for p in model.parameters())