🧠 两小时从 0 训出你的 64M 大模型。Train your own 64M LLM from scratch in 2 hours.
[fix] accumulate LoRA deltas before rounding merged weights
Afloat5240cc813308
diff --git a/model/model_lora.py b/model/model_lora.pyindex d323266..37c1600 100644--- a/model/model_lora.py+++ b/model/model_lora.py@@ -62,8 +62,9 @@ def merge_lora(model, lora_path, save_path):state_dict = {k: v.cpu().half() for k, v in raw_model.state_dict().items() if '.lora.' not in k}for name, module in raw_model.named_modules():if isinstance(module, nn.Linear) and '.lora.' not in name:- state_dict[f'{name}.weight'] = module.weight.data.clone().cpu().half()+ state_dict[f'{name}.weight'] = module.weight.data.clone().cpu().float()if hasattr(module, 'lora'):- delta = (module.lora.B.weight.data @ module.lora.A.weight.data) * module.lora.scaling- state_dict[f'{name}.weight'] += delta.cpu().half()+ delta = (module.lora.B.weight.data.float() @ module.lora.A.weight.data.float()) * module.lora.scaling.float()+ state_dict[f'{name}.weight'] += delta.cpu()+ state_dict[f'{name}.weight'] = state_dict[f'{name}.weight'].half()torch.save(state_dict, save_path)diff --git a/tests/test_lora_merge_fp32.py b/tests/test_lora_merge_fp32.pynew file mode 100644index 0000000..99b0382--- /dev/null+++ b/tests/test_lora_merge_fp32.py@@ -0,0 +1,31 @@+import unittest+import tempfile+from pathlib import Path+import torch+from model.model_lora import apply_lora, save_lora, merge_lora++class LinearModel(torch.nn.Module):+ def __init__(self):+ super().__init__()+ self.projection = torch.nn.Linear(4, 4, bias=False)+ @property+ def device(self):+ return self.projection.weight.device++class LoRAMergePrecisionTests(unittest.TestCase):+ def test_merge_rounds_once_after_adding_delta(self):+ model = LinearModel()+ model.projection.weight.data.fill_(0.9997)+ apply_lora(model, rank=1)+ model.projection.lora.A.weight.data.fill_(1.0)+ model.projection.lora.B.weight.data.fill_(0.0002)+ with tempfile.TemporaryDirectory() as directory:+ adapter, merged = Path(directory) / 'adapter.pth', Path(directory) / 'merged.pth'+ save_lora(model, adapter)+ state = torch.load(adapter, weights_only=True)+ expected = (model.projection.weight.float() + state['projection.lora.B.weight'].float() @ state['projection.lora.A.weight'].float()).half()+ merge_lora(model, adapter, merged)+ actual = torch.load(merged, weights_only=True)+ torch.testing.assert_close(actual['projection.weight'], expected, rtol=0, atol=0)+ self.assertEqual(actual['projection.weight'].dtype, torch.float16)+ self.assertFalse(any('.lora.' in key for key in actual))