🧠 两小时从 0 训出你的 64M 大模型。Train your own 64M LLM from scratch in 2 hours.
update generate args
gongjy19b388cd8718
diff --git a/model/model.py b/model/model.pyindex 3f49236..f6005b3 100644--- a/model/model.py+++ b/model/model.py@@ -329,13 +329,15 @@ class MiniMindLM(PreTrainedModel):stream=False, rp=1., use_cache=True, pad_token_id=0, **args):# 流式生成if stream:- return self._generate_stream(input_ids, eos_token_id, max_new_tokens, temperature, top_p, rp, use_cache)+ return self._generate_stream(input_ids, eos_token_id, max_new_tokens, temperature, top_p, rp, use_cache,+ **args)# 直接生成generated = []for i in range(input_ids.size(0)):non_pad = input_ids[i][input_ids[i] != pad_token_id].unsqueeze(0)- out = self._generate_stream(non_pad, eos_token_id, max_new_tokens, temperature, top_p, rp, use_cache)+ out = self._generate_stream(non_pad, eos_token_id, max_new_tokens, temperature, top_p, rp, use_cache,+ **args)tokens_list = [tokens[:, -1:] for tokens in out]gen = torch.cat(tokens_list, dim=-1) if tokens_list else non_padfull_sequence = torch.cat([non_pad, gen], dim=-1)@@ -353,10 +355,10 @@ class MiniMindLM(PreTrainedModel):start, first_seq, past_kvs = input_ids.shape[1], True, Nonewhile input_ids.shape[1] < max_new_tokens - 1:if first_seq or not use_cache:- out, first_seq = self(input_ids, past_key_values=past_kvs, use_cache=use_cache), False+ out, first_seq = self(input_ids, past_key_values=past_kvs, use_cache=use_cache, **args), Falseelse:out = self(input_ids[:, -1:], past_key_values=past_kvs, use_cache=use_cache,- start_pos=input_ids.shape[1] - 1)+ start_pos=input_ids.shape[1] - 1, **args)logits, past_kvs = out.logits[:, -1, :], out.past_key_valueslogits[:, list(set(input_ids.tolist()[0]))] /= rplogits /= (temperature + 1e-9)