Skip to content

🧠 两小时从 0 训出你的 64M 大模型。Train your own 64M LLM from scratch in 2 hours.

[fix] accept boolean padding masks in manual attention

Afloat767dd65859f0
diff --git a/model/model_minimind.py b/model/model_minimind.py
old mode 100755
new mode 100644
index 3f23fdf..0704477
--- a/model/model_minimind.py
+++ b/model/model_minimind.py
@@ -127,7 +127,7 @@ class Attention(nn.Module):
else:
scores = (xq @ xk.transpose(-2, -1)).float() / math.sqrt(self.head_dim)
if self.is_causal: scores[:, :, :, -seq_len:] += torch.full((seq_len, seq_len), float("-inf"), device=scores.device).triu(1)
- if attention_mask is not None: scores += (1.0 - attention_mask.unsqueeze(1).unsqueeze(2)) * -1e9
+ if attention_mask is not None: scores += (1.0 - attention_mask.to(scores.dtype).unsqueeze(1).unsqueeze(2)) * -1e9
output = self.attn_dropout(F.softmax(scores, dim=-1).type_as(xq)) @ xv
output = output.transpose(1, 2).reshape(bsz, seq_len, -1)
output = self.resid_dropout(self.o_proj(output))
diff --git a/tests/test_attention_bool_mask.py b/tests/test_attention_bool_mask.py
new file mode 100644
index 0000000..54546c3
--- /dev/null
+++ b/tests/test_attention_bool_mask.py
@@ -0,0 +1,29 @@
+import unittest
+from unittest.mock import patch
+import torch
+from model.model_minimind import MiniMindConfig, MiniMindForCausalLM
+
+def tiny_model(vocab_size=64):
+ torch.manual_seed(17)
+ return MiniMindForCausalLM(MiniMindConfig(
+ hidden_size=16, num_hidden_layers=1, num_attention_heads=2,
+ num_key_value_heads=1, head_dim=8, intermediate_size=32,
+ vocab_size=vocab_size, max_position_embeddings=128)).eval()
+
+class BooleanAttentionMaskTests(unittest.TestCase):
+ def test_boolean_mask_matches_integer_mask(self):
+ model = tiny_model()
+ ids = torch.tensor([[0, 0, 3, 4], [0, 5, 6, 7]])
+ mask = ids.ne(0)
+ for cache in (False, True):
+ with self.subTest(cache=cache), torch.no_grad():
+ expected = model(ids, attention_mask=mask.long(), use_cache=cache)
+ actual = model(ids, attention_mask=mask, use_cache=cache)
+ torch.testing.assert_close(actual.logits, expected.logits)
+ self.assertTrue(bool(actual.logits.isfinite().all()))
+ if cache:
+ token = torch.tensor([[8], [9]])
+ extended = torch.cat((mask, torch.ones(2, 1, dtype=torch.bool)), 1)
+ actual_next = model(token, attention_mask=extended, past_key_values=actual.past_key_values)
+ expected_next = model(token, attention_mask=extended.long(), past_key_values=expected.past_key_values)
+ torch.testing.assert_close(actual_next.logits, expected_next.logits)