Skip to content

🧠 两小时从 0 训出你的 64M 大模型。Train your own 64M LLM from scratch in 2 hours.

[fix] LoRA: add alpha/rank scaling with backward-compatible persistence

uphomer9012e5e0ef1c
diff --git a/model/model_lora.py b/model/model_lora.py
index 9d96955..4b54a96 100644
--- a/model/model_lora.py
+++ b/model/model_lora.py
@@ -4,9 +4,12 @@ from torch import optim, nn
# 定义Lora网络结构
class LoRA(nn.Module):
- def __init__(self, in_features, out_features, rank):
+ def __init__(self, in_features, out_features, rank, lora_alpha=None):
super().__init__()
self.rank = rank # LoRA的秩(rank),控制低秩矩阵的大小
+ # 标准 LoRA 的 alpha/r 缩放;lora_alpha=None 表示 alpha=rank(scaling=1),保持历史行为
+ scaling = (rank if lora_alpha is None else lora_alpha) / rank
+ self.register_buffer("scaling", torch.tensor(scaling, dtype=torch.float32))
self.A = nn.Linear(in_features, rank, bias=False) # 低秩矩阵A
self.B = nn.Linear(rank, out_features, bias=False) # 低秩矩阵B
# 矩阵A高斯初始化
@@ -15,13 +18,13 @@ class LoRA(nn.Module):
self.B.weight.data.zero_()
def forward(self, x):
- return self.B(self.A(x))
+ return self.B(self.A(x)) * self.scaling
-def apply_lora(model, rank=16):
+def apply_lora(model, rank=16, lora_alpha=None):
for name, module in model.named_modules():
if isinstance(module, nn.Linear) and module.in_features == module.out_features:
- lora = LoRA(module.in_features, module.out_features, rank=rank).to(model.device)
+ lora = LoRA(module.in_features, module.out_features, rank=rank, lora_alpha=lora_alpha).to(model.device)
setattr(module, "lora", lora)
original_forward = module.forward
@@ -39,7 +42,8 @@ def load_lora(model, path):
for name, module in model.named_modules():
if hasattr(module, 'lora'):
lora_state = {k.replace(f'{name}.lora.', ''): v for k, v in state_dict.items() if f'{name}.lora.' in k}
- module.lora.load_state_dict(lora_state)
+ # 兼容旧 checkpoint:旧版本没有 scaling buffer(等价于 scaling=1),缺失时保留默认值
+ module.lora.load_state_dict(lora_state, strict=False)
def save_lora(model, path):
@@ -48,7 +52,11 @@ def save_lora(model, path):
for name, module in raw_model.named_modules():
if hasattr(module, 'lora'):
clean_name = name[7:] if name.startswith("module.") else name
- lora_state = {f'{clean_name}.lora.{k}': v.cpu().half() for k, v in module.lora.state_dict().items()}
+ lora_state = {}
+ for k, v in module.lora.state_dict().items():
+ # 权重保持 half 以维持文件体积;scaling 等 buffer 用 float32,避免精度损失
+ v = v.cpu().half() if k.endswith('.weight') else v.cpu().float()
+ lora_state[f'{clean_name}.lora.{k}'] = v
state_dict.update(lora_state)
torch.save(state_dict, path)
@@ -61,5 +69,7 @@ def merge_lora(model, lora_path, save_path):
if isinstance(module, nn.Linear) and '.lora.' not in name:
state_dict[f'{name}.weight'] = module.weight.data.clone().cpu().half()
if hasattr(module, 'lora'):
- state_dict[f'{name}.weight'] += (module.lora.B.weight.data @ module.lora.A.weight.data).cpu().half()
+ delta = module.lora.B.weight.data @ module.lora.A.weight.data
+ delta = (delta * module.lora.scaling.to(delta.device)).cpu().half()
+ state_dict[f'{name}.weight'] += delta
torch.save(state_dict, save_path)
diff --git a/trainer/train_lora.py b/trainer/train_lora.py
index 35a87fc..bfdcf50 100644
--- a/trainer/train_lora.py
+++ b/trainer/train_lora.py
@@ -70,6 +70,7 @@ if __name__ == "__main__":
parser = argparse.ArgumentParser(description="MiniMind LoRA Fine-tuning")
parser.add_argument("--save_dir", type=str, default="../out", help="模型保存目录")
parser.add_argument("--lora_rank", default=16, type=int, help="LoRA rank")
+ parser.add_argument("--lora_alpha", default=None, type=float, help="LoRA alpha(默认等于 rank,即 scaling=1)")
parser.add_argument("--lora_name", type=str, default="lora_medical", help="LoRA权重名称(如lora_identity/lora_medical等)")
parser.add_argument("--epochs", type=int, default=10, help="训练轮数")
parser.add_argument("--batch_size", type=int, default=32, help="batch size")
@@ -119,7 +120,7 @@ if __name__ == "__main__":
# ========== 5. 定义模型、应用LoRA、冻结非LoRA参数 ==========
model, tokenizer = init_model(lm_config, args.from_weight, device=args.device)
- apply_lora(model, rank=args.lora_rank)
+ apply_lora(model, rank=args.lora_rank, lora_alpha=args.lora_alpha)
# 统计参数
total_params = sum(p.numel() for p in model.parameters())