Skip to content

🧠 两小时从 0 训出你的 64M 大模型。Train your own 64M LLM from scratch in 2 hours.

[fix] accumulate LoRA deltas before rounding merged weights

Afloat5240cc813308
diff --git a/model/model_lora.py b/model/model_lora.py
index d323266..37c1600 100644
--- a/model/model_lora.py
+++ b/model/model_lora.py
@@ -62,8 +62,9 @@ def merge_lora(model, lora_path, save_path):
state_dict = {k: v.cpu().half() for k, v in raw_model.state_dict().items() if '.lora.' not in k}
for name, module in raw_model.named_modules():
if isinstance(module, nn.Linear) and '.lora.' not in name:
- state_dict[f'{name}.weight'] = module.weight.data.clone().cpu().half()
+ state_dict[f'{name}.weight'] = module.weight.data.clone().cpu().float()
if hasattr(module, 'lora'):
- delta = (module.lora.B.weight.data @ module.lora.A.weight.data) * module.lora.scaling
- state_dict[f'{name}.weight'] += delta.cpu().half()
+ delta = (module.lora.B.weight.data.float() @ module.lora.A.weight.data.float()) * module.lora.scaling.float()
+ state_dict[f'{name}.weight'] += delta.cpu()
+ state_dict[f'{name}.weight'] = state_dict[f'{name}.weight'].half()
torch.save(state_dict, save_path)
diff --git a/tests/test_lora_merge_fp32.py b/tests/test_lora_merge_fp32.py
new file mode 100644
index 0000000..99b0382
--- /dev/null
+++ b/tests/test_lora_merge_fp32.py
@@ -0,0 +1,31 @@
+import unittest
+import tempfile
+from pathlib import Path
+import torch
+from model.model_lora import apply_lora, save_lora, merge_lora
+
+class LinearModel(torch.nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.projection = torch.nn.Linear(4, 4, bias=False)
+ @property
+ def device(self):
+ return self.projection.weight.device
+
+class LoRAMergePrecisionTests(unittest.TestCase):
+ def test_merge_rounds_once_after_adding_delta(self):
+ model = LinearModel()
+ model.projection.weight.data.fill_(0.9997)
+ apply_lora(model, rank=1)
+ model.projection.lora.A.weight.data.fill_(1.0)
+ model.projection.lora.B.weight.data.fill_(0.0002)
+ with tempfile.TemporaryDirectory() as directory:
+ adapter, merged = Path(directory) / 'adapter.pth', Path(directory) / 'merged.pth'
+ save_lora(model, adapter)
+ state = torch.load(adapter, weights_only=True)
+ expected = (model.projection.weight.float() + state['projection.lora.B.weight'].float() @ state['projection.lora.A.weight'].float()).half()
+ merge_lora(model, adapter, merged)
+ actual = torch.load(merged, weights_only=True)
+ torch.testing.assert_close(actual['projection.weight'], expected, rtol=0, atol=0)
+ self.assertEqual(actual['projection.weight'].dtype, torch.float16)
+ self.assertFalse(any('.lora.' in key for key in actual))