🧠 两小时从 0 训出你的 64M 大模型。Train your own 64M LLM from scratch in 2 hours.
[fix] max length
jingyaogong3a5aba82dbec
diff --git a/dataset/lm_dataset.py b/dataset/lm_dataset.pyindex 1a57ce9..1dd1558 100644--- a/dataset/lm_dataset.py+++ b/dataset/lm_dataset.py@@ -16,8 +16,8 @@ class PretrainDataset(Dataset):def __getitem__(self, index):sample = self.samples[index]- tokens = self.tokenizer(str(sample['text']), add_special_tokens=False).input_ids- tokens = [self.tokenizer.bos_token_id] + tokens[:self.max_length - 2] + [self.tokenizer.eos_token_id]+ tokens = self.tokenizer(str(sample['text']), add_special_tokens=False, max_length=self.max_length - 2, truncation=True).input_ids+ tokens = [self.tokenizer.bos_token_id] + tokens + [self.tokenizer.eos_token_id]input_ids = tokens + [self.tokenizer.pad_token_id] * (self.max_length - len(tokens))input_ids = torch.tensor(input_ids, dtype=torch.long)labels = input_ids.clone()