🧠 两小时从 0 训出你的 64M 大模型。Train your own 64M LLM from scratch in 2 hours.
[fix] accept boolean padding masks in manual attention
Afloat767dd65859f0
diff --git a/model/model_minimind.py b/model/model_minimind.pyold mode 100755new mode 100644index 3f23fdf..0704477--- a/model/model_minimind.py+++ b/model/model_minimind.py@@ -127,7 +127,7 @@ class Attention(nn.Module):else:scores = (xq @ xk.transpose(-2, -1)).float() / math.sqrt(self.head_dim)if self.is_causal: scores[:, :, :, -seq_len:] += torch.full((seq_len, seq_len), float("-inf"), device=scores.device).triu(1)- if attention_mask is not None: scores += (1.0 - attention_mask.unsqueeze(1).unsqueeze(2)) * -1e9+ if attention_mask is not None: scores += (1.0 - attention_mask.to(scores.dtype).unsqueeze(1).unsqueeze(2)) * -1e9output = self.attn_dropout(F.softmax(scores, dim=-1).type_as(xq)) @ xvoutput = output.transpose(1, 2).reshape(bsz, seq_len, -1)output = self.resid_dropout(self.o_proj(output))diff --git a/tests/test_attention_bool_mask.py b/tests/test_attention_bool_mask.pynew file mode 100644index 0000000..54546c3--- /dev/null+++ b/tests/test_attention_bool_mask.py@@ -0,0 +1,29 @@+import unittest+from unittest.mock import patch+import torch+from model.model_minimind import MiniMindConfig, MiniMindForCausalLM++def tiny_model(vocab_size=64):+ torch.manual_seed(17)+ return MiniMindForCausalLM(MiniMindConfig(+ hidden_size=16, num_hidden_layers=1, num_attention_heads=2,+ num_key_value_heads=1, head_dim=8, intermediate_size=32,+ vocab_size=vocab_size, max_position_embeddings=128)).eval()++class BooleanAttentionMaskTests(unittest.TestCase):+ def test_boolean_mask_matches_integer_mask(self):+ model = tiny_model()+ ids = torch.tensor([[0, 0, 3, 4], [0, 5, 6, 7]])+ mask = ids.ne(0)+ for cache in (False, True):+ with self.subTest(cache=cache), torch.no_grad():+ expected = model(ids, attention_mask=mask.long(), use_cache=cache)+ actual = model(ids, attention_mask=mask, use_cache=cache)+ torch.testing.assert_close(actual.logits, expected.logits)+ self.assertTrue(bool(actual.logits.isfinite().all()))+ if cache:+ token = torch.tensor([[8], [9]])+ extended = torch.cat((mask, torch.ones(2, 1, dtype=torch.bool)), 1)+ actual_next = model(token, attention_mask=extended, past_key_values=actual.past_key_values)+ expected_next = model(token, attention_mask=extended.long(), past_key_values=expected.past_key_values)+ torch.testing.assert_close(actual_next.logits, expected_next.logits)