🧠 两小时从 0 训出你的 64M 大模型。Train your own 64M LLM from scratch in 2 hours.
[update] fix decoding, drop tests
jingyaogongcc9629e5f464
diff --git a/model/model_minimind.py b/model/model_minimind.pyindex 8c85e3c..ba6cd26 100644--- a/model/model_minimind.py+++ b/model/model_minimind.py@@ -260,6 +260,7 @@ class MiniMindForCausalLM(PreTrainedModel, GenerationMixin):# https://github.com/jingyaogong/minimind/discussions/611@torch.inference_mode()def generate(self, inputs=None, attention_mask=None, max_new_tokens=8192, temperature=0.85, top_p=0.85, top_k=50, eos_token_id=2, streamer=None, use_cache=True, num_return_sequences=1, do_sample=True, repetition_penalty=1.0, **kwargs):+ if do_sample and not 0 < temperature < math.inf: raise ValueError("Sampling temperature must be finite and positive")input_ids = kwargs.pop("input_ids", inputs).repeat(num_return_sequences, 1)attention_mask = attention_mask.repeat(num_return_sequences, 1) if attention_mask is not None else Nonepast_key_values = kwargs.pop("past_key_values", None)@@ -269,8 +270,7 @@ class MiniMindForCausalLM(PreTrainedModel, GenerationMixin):past_len = past_key_values[0][0].shape[1] if past_key_values else 0outputs = self.forward(input_ids[:, past_len:], attention_mask, past_key_values, use_cache=use_cache, **kwargs)attention_mask = torch.cat([attention_mask, attention_mask.new_ones(attention_mask.shape[0], 1)], -1) if attention_mask is not None else None- logits = outputs.logits[:, -1, :]- if do_sample: logits = logits / temperature+ logits = outputs.logits[:, -1] / temperature if do_sample else outputs.logits[:, -1]if repetition_penalty != 1.0:for i in range(input_ids.shape[0]):seen = torch.unique(input_ids[i]); score = logits[i, seen]; logits[i, seen] = torch.where(score > 0, score / repetition_penalty, score * repetition_penalty)@@ -279,8 +279,7 @@ class MiniMindForCausalLM(PreTrainedModel, GenerationMixin):if do_sample and top_p < 1.0:sorted_logits, sorted_indices = torch.sort(logits, descending=True)mask = torch.cumsum(torch.softmax(sorted_logits, dim=-1), dim=-1) > top_p- mask[..., 1:], mask[..., 0] = mask[..., :-1].clone(), 0- logits[mask.scatter(1, sorted_indices, mask)] = -float('inf')+ logits[mask.scatter(1, sorted_indices, F.pad(mask[..., :-1], (1, 0)))] = -float('inf')next_token = torch.multinomial(torch.softmax(logits, dim=-1), num_samples=1) if do_sample else torch.argmax(logits, dim=-1, keepdim=True)if eos_token_id is not None: next_token = torch.where(finished.unsqueeze(-1), next_token.new_full((next_token.shape[0], 1), eos_token_id), next_token)input_ids = torch.cat([input_ids, next_token], dim=-1)diff --git a/scripts/serve_openai_api.py b/scripts/serve_openai_api.pyindex 10486cd..f4051af 100644--- a/scripts/serve_openai_api.py+++ b/scripts/serve_openai_api.py@@ -50,7 +50,7 @@ def init_model(args):class ChatRequest(BaseModel):model: strmessages: list- temperature: float = 0.7+ temperature: float = Field(default=0.7, ge=0.0, allow_inf_nan=False)top_p: float = 0.92max_tokens: int = 8192stream: bool = True@@ -115,7 +115,7 @@ def generate_stream_response(messages, temperature, top_p, max_tokens, tools=Nonmodel.generate(inputs.input_ids,max_new_tokens=max_tokens,- do_sample=True,+ do_sample=temperature > 0,temperature=temperature,top_p=top_p,attention_mask=inputs.attention_mask,@@ -226,7 +226,7 @@ def chat_completions(request: ChatRequest):generated_ids = model.generate(inputs["input_ids"],max_new_tokens=request.max_tokens,- do_sample=True,+ do_sample=request.temperature > 0,attention_mask=inputs["attention_mask"],pad_token_id=tokenizer.pad_token_id,eos_token_id=tokenizer.eos_token_id,diff --git a/tests/test_api_inference_event_loop.py b/tests/test_api_inference_event_loop.pydeleted file mode 100644index e408b3d..0000000--- a/tests/test_api_inference_event_loop.py+++ /dev/null@@ -1,41 +0,0 @@-import asyncio-import threading-import unittest-from unittest.mock import patch-import httpx-import torch-from scripts import serve_openai_api as server--class Inputs(dict):- def to(self, device):- return self--class Tokenizer:- pad_token_id = 0- eos_token_id = 2- def apply_chat_template(self, *args, **kwargs):- return 'prompt'- def __call__(self, *args, **kwargs):- return Inputs(input_ids=torch.tensor([[3]]), attention_mask=torch.tensor([[1]]))- def decode(self, *args, **kwargs):- return 'answer'--class EventLoopResponsivenessTests(unittest.IsolatedAsyncioTestCase):- async def test_other_coroutines_run_during_nonstream_generation(self):- release = threading.Event()- observed = []- class Model:- def generate(self, *args, **kwargs):- observed.append(release.wait(timeout=2))- return torch.tensor([[3, 4]])- async def release_from_event_loop():- await asyncio.sleep(0.05)- release.set()- with patch.object(server, 'tokenizer', Tokenizer(), create=True), patch.object(server, 'model', Model(), create=True), patch.object(server, 'device', 'cpu', create=True):- async with httpx.AsyncClient(transport=httpx.ASGITransport(app=server.app), base_url='http://test') as client:- response, _ = await asyncio.gather(- client.post('/v1/chat/completions', json={'model': 'minimind', 'messages': [{'role': 'user', 'content': 'Hi'}], 'stream': False, 'max_tokens': 1}),- release_from_event_loop())- self.assertEqual(response.status_code, 200, response.text)- self.assertEqual(response.json()['choices'][0]['message']['content'], 'answer')- self.assertEqual(observed, [True], 'generation blocked the coroutine needed to release it')diff --git a/tests/test_attention_bool_mask.py b/tests/test_attention_bool_mask.pydeleted file mode 100644index 54546c3..0000000--- a/tests/test_attention_bool_mask.py+++ /dev/null@@ -1,29 +0,0 @@-import unittest-from unittest.mock import patch-import torch-from model.model_minimind import MiniMindConfig, MiniMindForCausalLM--def tiny_model(vocab_size=64):- torch.manual_seed(17)- return MiniMindForCausalLM(MiniMindConfig(- hidden_size=16, num_hidden_layers=1, num_attention_heads=2,- num_key_value_heads=1, head_dim=8, intermediate_size=32,- vocab_size=vocab_size, max_position_embeddings=128)).eval()--class BooleanAttentionMaskTests(unittest.TestCase):- def test_boolean_mask_matches_integer_mask(self):- model = tiny_model()- ids = torch.tensor([[0, 0, 3, 4], [0, 5, 6, 7]])- mask = ids.ne(0)- for cache in (False, True):- with self.subTest(cache=cache), torch.no_grad():- expected = model(ids, attention_mask=mask.long(), use_cache=cache)- actual = model(ids, attention_mask=mask, use_cache=cache)- torch.testing.assert_close(actual.logits, expected.logits)- self.assertTrue(bool(actual.logits.isfinite().all()))- if cache:- token = torch.tensor([[8], [9]])- extended = torch.cat((mask, torch.ones(2, 1, dtype=torch.bool)), 1)- actual_next = model(token, attention_mask=extended, past_key_values=actual.past_key_values)- expected_next = model(token, attention_mask=extended.long(), past_key_values=expected.past_key_values)- torch.testing.assert_close(actual_next.logits, expected_next.logits)diff --git a/tests/test_distillation_empty_supervision.py b/tests/test_distillation_empty_supervision.pydeleted file mode 100644index 78b51bd..0000000--- a/tests/test_distillation_empty_supervision.py+++ /dev/null@@ -1,21 +0,0 @@-import unittest-import torch-from trainer.train_distillation import distillation_loss--class EmptyDistillationTests(unittest.TestCase):- def test_all_ignored_labels_produce_differentiable_zero(self):- student = torch.randn(2, 4, 8, requires_grad=True)- teacher = torch.randn_like(student)- valid = torch.zeros(2, 4, dtype=torch.bool)- loss = distillation_loss(student[valid], teacher[valid], temperature=1.5)- self.assertTrue(bool(loss.isfinite()))- self.assertEqual(loss.item(), 0.0)- loss.backward()- self.assertTrue(torch.equal(student.grad, torch.zeros_like(student)))-- def test_nonempty_identical_logits_keep_zero_loss(self):- student = torch.tensor([[0.1, 0.2, 0.3]], requires_grad=True)- loss = distillation_loss(student, student.detach().clone(), temperature=1.5)- self.assertAlmostEqual(loss.item(), 0.0, places=6)- loss.backward()- self.assertTrue(bool(student.grad.isfinite().all()))diff --git a/tests/test_generation_greedy_temperature.py b/tests/test_generation_greedy_temperature.pydeleted file mode 100644index efad6da..0000000--- a/tests/test_generation_greedy_temperature.py+++ /dev/null@@ -1,34 +0,0 @@-import unittest-from unittest.mock import patch-import torch-from model.model_minimind import MiniMindConfig, MiniMindForCausalLM--def tiny_model(vocab_size=64):- torch.manual_seed(17)- return MiniMindForCausalLM(MiniMindConfig(- hidden_size=16, num_hidden_layers=1, num_attention_heads=2,- num_key_value_heads=1, head_dim=8, intermediate_size=32,- vocab_size=vocab_size, max_position_embeddings=128)).eval()--from types import SimpleNamespace--class GreedyTemperatureTests(unittest.TestCase):- def test_greedy_ignores_temperature_and_sampling_filters(self):- model = tiny_model()- logits = torch.zeros(1, 1, 64)- logits[0, 0, 3] = 1.0- logits[0, 0, 7] = 2.0- for temperature in (0.0, -1.0, 0.7):- with self.subTest(temperature=temperature), patch.object(model, 'forward', return_value=SimpleNamespace(logits=logits.clone(), past_key_values=None)):- result = model.generate(torch.tensor([[4]]), max_new_tokens=1, do_sample=False,- temperature=temperature, top_k=1000, top_p=0.0)- self.assertEqual(result[0, -1].item(), 7)-- def test_repetition_penalty_still_applies_to_greedy(self):- model = tiny_model()- logits = torch.zeros(1, 1, 64)- logits[0, 0, 3], logits[0, 0, 7] = 1.5, 2.0- with patch.object(model, 'forward', return_value=SimpleNamespace(logits=logits, past_key_values=None)):- result = model.generate(torch.tensor([[7]]), max_new_tokens=1, do_sample=False,- temperature=0.0, repetition_penalty=2.0)- self.assertEqual(result[0, -1].item(), 3)diff --git a/tests/test_lora_merge_fp32.py b/tests/test_lora_merge_fp32.pydeleted file mode 100644index 99b0382..0000000--- a/tests/test_lora_merge_fp32.py+++ /dev/null@@ -1,31 +0,0 @@-import unittest-import tempfile-from pathlib import Path-import torch-from model.model_lora import apply_lora, save_lora, merge_lora--class LinearModel(torch.nn.Module):- def __init__(self):- super().__init__()- self.projection = torch.nn.Linear(4, 4, bias=False)- @property- def device(self):- return self.projection.weight.device--class LoRAMergePrecisionTests(unittest.TestCase):- def test_merge_rounds_once_after_adding_delta(self):- model = LinearModel()- model.projection.weight.data.fill_(0.9997)- apply_lora(model, rank=1)- model.projection.lora.A.weight.data.fill_(1.0)- model.projection.lora.B.weight.data.fill_(0.0002)- with tempfile.TemporaryDirectory() as directory:- adapter, merged = Path(directory) / 'adapter.pth', Path(directory) / 'merged.pth'- save_lora(model, adapter)- state = torch.load(adapter, weights_only=True)- expected = (model.projection.weight.float() + state['projection.lora.B.weight'].float() @ state['projection.lora.A.weight'].float()).half()- merge_lora(model, adapter, merged)- actual = torch.load(merged, weights_only=True)- torch.testing.assert_close(actual['projection.weight'], expected, rtol=0, atol=0)- self.assertEqual(actual['projection.weight'].dtype, torch.float16)- self.assertFalse(any('.lora.' in key for key in actual))