Skip to content

🧠 两小时从 0 训出你的 64M 大模型。Train your own 64M LLM from scratch in 2 hours.

[fix] ignore sampling temperature for greedy generation

Afloat5f69faf966c9
diff --git a/model/model_minimind.py b/model/model_minimind.py
old mode 100755
new mode 100644
index 3f23fdf..9933cff
--- a/model/model_minimind.py
+++ b/model/model_minimind.py
@@ -269,13 +269,14 @@ class MiniMindForCausalLM(PreTrainedModel, GenerationMixin):
past_len = past_key_values[0][0].shape[1] if past_key_values else 0
outputs = self.forward(input_ids[:, past_len:], attention_mask, past_key_values, use_cache=use_cache, **kwargs)
attention_mask = torch.cat([attention_mask, attention_mask.new_ones(attention_mask.shape[0], 1)], -1) if attention_mask is not None else None
- logits = outputs.logits[:, -1, :] / temperature
+ logits = outputs.logits[:, -1, :]
+ if do_sample: logits = logits / temperature
if repetition_penalty != 1.0:
for i in range(input_ids.shape[0]):
seen = torch.unique(input_ids[i]); score = logits[i, seen]; logits[i, seen] = torch.where(score > 0, score / repetition_penalty, score * repetition_penalty)
- if top_k > 0:
+ if do_sample and top_k > 0:
logits[logits < torch.topk(logits, top_k)[0][..., -1, None]] = -float('inf')
- if top_p < 1.0:
+ if do_sample and top_p < 1.0:
sorted_logits, sorted_indices = torch.sort(logits, descending=True)
mask = torch.cumsum(torch.softmax(sorted_logits, dim=-1), dim=-1) > top_p
mask[..., 1:], mask[..., 0] = mask[..., :-1].clone(), 0
diff --git a/tests/test_generation_greedy_temperature.py b/tests/test_generation_greedy_temperature.py
new file mode 100644
index 0000000..efad6da
--- /dev/null
+++ b/tests/test_generation_greedy_temperature.py
@@ -0,0 +1,34 @@
+import unittest
+from unittest.mock import patch
+import torch
+from model.model_minimind import MiniMindConfig, MiniMindForCausalLM
+
+def tiny_model(vocab_size=64):
+ torch.manual_seed(17)
+ return MiniMindForCausalLM(MiniMindConfig(
+ hidden_size=16, num_hidden_layers=1, num_attention_heads=2,
+ num_key_value_heads=1, head_dim=8, intermediate_size=32,
+ vocab_size=vocab_size, max_position_embeddings=128)).eval()
+
+from types import SimpleNamespace
+
+class GreedyTemperatureTests(unittest.TestCase):
+ def test_greedy_ignores_temperature_and_sampling_filters(self):
+ model = tiny_model()
+ logits = torch.zeros(1, 1, 64)
+ logits[0, 0, 3] = 1.0
+ logits[0, 0, 7] = 2.0
+ for temperature in (0.0, -1.0, 0.7):
+ with self.subTest(temperature=temperature), patch.object(model, 'forward', return_value=SimpleNamespace(logits=logits.clone(), past_key_values=None)):
+ result = model.generate(torch.tensor([[4]]), max_new_tokens=1, do_sample=False,
+ temperature=temperature, top_k=1000, top_p=0.0)
+ self.assertEqual(result[0, -1].item(), 7)
+
+ def test_repetition_penalty_still_applies_to_greedy(self):
+ model = tiny_model()
+ logits = torch.zeros(1, 1, 64)
+ logits[0, 0, 3], logits[0, 0, 7] = 1.5, 2.0
+ with patch.object(model, 'forward', return_value=SimpleNamespace(logits=logits, past_key_values=None)):
+ result = model.generate(torch.tensor([[7]]), max_new_tokens=1, do_sample=False,
+ temperature=0.0, repetition_penalty=2.0)
+ self.assertEqual(result[0, -1].item(), 3)