🧠 两小时从 0 训出你的 64M 大模型。Train your own 64M LLM from scratch in 2 hours.
[update] mask log
jingyaogongf55d4c32a08a
diff --git a/dataset/lm_dataset.py b/dataset/lm_dataset.pyindex 1fa0397..a958303 100644--- a/dataset/lm_dataset.py+++ b/dataset/lm_dataset.py@@ -86,12 +86,9 @@ class SFTDataset(Dataset):Y = torch.tensor(input_ids[1:], dtype=torch.long)loss_mask = torch.tensor(loss_mask[1:], dtype=torch.long) # 对齐预测位置# # === 打印每个token的掩码情况 ===- # print(f"\n--- Sample {index} Token Loss Mask (length: {len(input_ids)}) ---")- # for i, (token_id, mask) in enumerate(zip(input_ids, loss_mask)):- # token_str = self.tokenizer.decode([token_id], skip_special_tokens=False)- # token_str = token_str.replace('\n', '\\n').replace('\t', '\\t') # 处理换行等不可见字符- # print(f"Token {i:3d}: {token_id:5d} -> '{token_str:10s}' | mask: {mask}")- # print(f"--- End of Sample {index} ---")+ # print(f"\n--- Sample {index} ---")+ # for i, (x, y, m) in enumerate(zip(X, Y, loss_mask)):+ # print(f"{i:3d}: X={self.tokenizer.decode([x])!r:16s} ---> Y={self.tokenizer.decode([y])!r:16s} mask={m}")# # ================================return X, Y, loss_mask