🧠 两小时从 0 训出你的 64M 大模型。Train your own 64M LLM from scratch in 2 hours.
Update data preprocessing methods
gongjya8ae34277530
diff --git a/0-eval_pretrain.py b/0-eval_pretrain.pyindex edb3ad8..570d42d 100644--- a/0-eval_pretrain.py+++ b/0-eval_pretrain.py@@ -69,18 +69,11 @@ if __name__ == "__main__":max_seq_len = 512lm_config = LMConfig()lm_config.max_seq_len = max_seq_len- # 对话是否携带历史对话(当前模型太弱,增大历史上下文,基本导致胡言乱语)- contain_history_chat = False# -----------------------------------------------------------------------------model, tokenizer = init_model(lm_config)-model = model.eval()- # 推送到huggingface- # model.push_to_hub("minimind")- # tokenizer.push_to_hub("minimind")-- # answer_way = int(input('输入0自动测试,输入1问题测试:'))+ # int(input('输入0自动测试,输入1问题测试:'))answer_way = 0stream = True@@ -101,15 +94,9 @@ if __name__ == "__main__":'江苏省的最好的大学',]- messages_origin = []- messages = messages_origin-qa_index = 0while True:start = time.time()- if not contain_history_chat:- messages = messages_origin.copy()-if answer_way == 1:# run generationprompt = input('用户:')@@ -120,20 +107,9 @@ if __name__ == "__main__":print('问题:', prompt)qa_index += 1- messages.append({"role": "user", "content": prompt})-- # print(messages)- new_prompt = tokenizer.apply_chat_template(- messages,- tokenize=False,- add_generation_prompt=True- )[-(max_seq_len - 1):]-x = tokenizer(prompt).data['input_ids']x = (torch.tensor(x, dtype=torch.long, device=device)[None, ...])- answer = new_prompt-with torch.no_grad():res_y = model.generate(x, tokenizer.eos_token_id, max_new_tokens=max_seq_len, temperature=temperature,top_k=top_k, stream=stream)@@ -172,8 +148,5 @@ if __name__ == "__main__":print('\n')- if contain_history_chat:- assistant_answer = answer.replace(new_prompt, "")- messages.append({"role": "assistant", "content": assistant_answer})end = time.time()- print(end - start,'s')+ print(end - start, 's')diff --git a/1-pretrain.py b/1-pretrain.pyindex 6cce4f0..20fdf42 100644--- a/1-pretrain.py+++ b/1-pretrain.py@@ -4,6 +4,8 @@ import argparseimport timeimport mathimport warnings++import pandas as pdimport torchimport torch.distributed as distfrom torch import optim@@ -11,6 +13,9 @@ from torch.nn.parallel import DistributedDataParallelfrom torch.optim.lr_scheduler import CosineAnnealingLRfrom torch.utils.data import DataLoader, DistributedSamplerfrom contextlib import nullcontext++from transformers import AutoTokenizer+from model.model import Transformerfrom model.LMConfig import LMConfigfrom model.dataset import PretrainDataset@@ -98,11 +103,13 @@ def init_model():def count_parameters(model):return sum(p.numel() for p in model.parameters() if p.requires_grad)+ tokenizer = AutoTokenizer.from_pretrained('./model/minimind_tokenizer')+model = Transformer(lm_config).to(args.device)- moe_path = '_moe' if lm_config.use_moe else ''+ # moe_path = '_moe' if lm_config.use_moe else ''Logger(f'LLM总参数量:{count_parameters(model) / 1e6:.3f} 百万')- return model+ return model, tokenizerdef init_distributed_mode():@@ -122,14 +129,15 @@ if __name__ == "__main__":parser = argparse.ArgumentParser(description="MiniMind Pretraining")parser.add_argument("--out_dir", type=str, default="out", help="Output directory")parser.add_argument("--epochs", type=int, default=20, help="Number of epochs")- parser.add_argument("--batch_size", type=int, default=32, help="Batch size")+ parser.add_argument("--batch_size", type=int, default=48, help="Batch size")parser.add_argument("--learning_rate", type=float, default=2e-4, help="Learning rate")- parser.add_argument("--device", type=str, default="cuda:0" if torch.cuda.is_available() else "cpu", help="Device to use")+ parser.add_argument("--device", type=str, default="cuda:0" if torch.cuda.is_available() else "cpu",+ help="Device to use")parser.add_argument("--dtype", type=str, default="bfloat16", help="Data type")parser.add_argument("--use_wandb", action="store_true", help="Use Weights & Biases")parser.add_argument("--wandb_project", type=str, default="MiniMind-Pretrain", help="Weights & Biases project name")parser.add_argument("--num_workers", type=int, default=8, help="Number of workers for data loading")- parser.add_argument("--data_path", type=str, default="./dataset/pretrain_data.bin", help="Path to training data")+ parser.add_argument("--data_path", type=str, default="./dataset/pretrain_data.csv", help="Path to training data")parser.add_argument("--ddp", action="store_true", help="Use DistributedDataParallel")parser.add_argument("--accumulation_steps", type=int, default=8, help="Gradient accumulation steps")parser.add_argument("--grad_clip", type=float, default=1.0, help="Gradient clipping threshold")@@ -160,12 +168,15 @@ if __name__ == "__main__":if args.use_wandb and (not ddp or ddp_local_rank == 0):import wandb+wandb.init(project=args.wandb_project, name=args.wandb_run_name)else:wandb = None- data_path_list = [args.data_path]- train_ds = PretrainDataset(data_path_list, max_length=max_seq_len, memmap=True)+ model, tokenizer = init_model()+ df = pd.read_csv(args.data_path)+ df = df.sample(frac=1.0)+ train_ds = PretrainDataset(df, tokenizer, max_length=max_seq_len)train_sampler = DistributedSampler(train_ds) if ddp else Nonetrain_loader = DataLoader(train_ds,@@ -177,8 +188,6 @@ if __name__ == "__main__":sampler=train_sampler)- model = init_model()-scaler = torch.cuda.amp.GradScaler(enabled=(args.dtype in ['float16', 'bfloat16']))optimizer = optim.Adam(model.parameters(), lr=args.learning_rate)diff --git a/README.md b/README.mdindex 265b5a0..d57023a 100644--- a/README.md+++ b/README.md@@ -78,7 +78,18 @@ https://github.com/user-attachments/assets/88b98128-636e-43bc-a419-b1b1403c2055### 👉**最近更新**<details close>-<summary> <b>2024-09-17 (new🎉)</b> </summary>+<summary> <b>2024-09-27</b> </summary>++- 09-27更新pretrain数据集的预处理方式,为了保证文本完整性,放弃预处理成.bin训练的形式(轻微牺牲训练速度)。++- 目前pretrain预处理后的文件命名为:pretrain_data.csv。++- 删除了一些冗余的代码。++</details>++<details close>+<summary> <b>2024-09-17</b> </summary>- 更新minimind-v1-moe模型diff --git a/README_en.md b/README_en.mdindex a0fa5a4..fd0f86f 100644--- a/README_en.md+++ b/README_en.md@@ -84,6 +84,17 @@ We hope this open-source project helps LLM beginners get started quickly!### 👉**Recent Updates**+<details close>+<summary> <b>2024-09-27</b> </summary>++- Updated the preprocessing method for the pretrain dataset on 09-27 to ensure text integrity, opting to abandon the preprocessing into .bin training format (slightly sacrificing training speed).++- The current filename for the pretrain data after preprocessing is: pretrain_data.csv.++- Removed some redundant code.++</details>+<details close><summary> <b>2024-09-17 (new🎉)</b> </summary>diff --git a/data_process.py b/data_process.pyindex 047ff0e..bea9371 100644--- a/data_process.py+++ b/data_process.py@@ -1,3 +1,4 @@+import csvimport itertoolsimport reimport json@@ -12,110 +13,31 @@ from datasets import load_datasetbos_token = "<s>"eos_token = "</s>"-# pretrain-def process_wiki_clean():- with open('./dataset/clean-wikipedia-cn.json', 'r', encoding='utf-8') as f_read:- data = [ujson.loads(line) for line in f_read]- data_len = len(data)- doc_ids = []- for idx, line in enumerate(data):- text = line['response']- text_id = tokenizer(f'{bos_token}{text}{eos_token}').data['input_ids']- if len(text_id) > 5:- doc_ids += text_id- if idx % (int(data_len / 20)) == 0:- print(f"[{idx}/{data_len}] {text}")- arr = np.array(doc_ids, dtype=np.uint16)- with open('./dataset/clean-wikipedia-cn.bin', 'wb') as f:- f.write(arr.tobytes())---# pretrain-def process_other():- data = []- with open('./dataset/alpaca_gpt4_data_zh.json', 'r', encoding='utf-8') as f:- data_ = json.load(f)- data += data_-- with open('./dataset/alpaca_data_zh_51k.json', 'r', encoding='utf-8') as f:- data_ = json.load(f)- data += data_-- doc_ids = []- for idx, per in enumerate(data):- q = per['instruction']- i = per['input']- a = per['output']- q = q + i- if len(q) < 10 or len(a) < 5:- continue- if len(q) > 256 or len(a) > 256:- continue- text_id = tokenizer(f'{bos_token}{q},{a}{eos_token}').data['input_ids']- if len(text_id) > 5:- doc_ids += text_id- if idx % 50000 == 0:- print(idx, len(data))-- arr = np.array(doc_ids, dtype=np.uint16)- with open('./dataset/clean_other.bin', 'wb') as f:- f.write(arr.tobytes())---def process_seq_monkey(chunk_size=50000):- doc_ids = []+def pretrain_process(chunk_size=50000):chunk_idx = 0with jsonlines.open('./dataset/mobvoi_seq_monkey_general_open_corpus.jsonl') as reader:- while True:- chunk = list(itertools.islice(reader, chunk_size))- if not chunk:- break-- for idx, obj in enumerate(chunk):- try:- content = obj.get('text', '')- if len(content) > 512:+ with open('./dataset/pretrain_data.csv', 'w', newline='', encoding='utf-8') as csvfile:+ writer = csv.writer(csvfile)+ writer.writerow(['text'])++ while True:+ chunk = list(itertools.islice(reader, chunk_size))+ if not chunk:+ break++ for idx, obj in enumerate(chunk):+ try:+ content = obj.get('text', '')+ if len(content) > 512:+ continue+ writer.writerow([content])+ except UnicodeDecodeError as e:+ print(f"Skipping invalid line {chunk_idx * chunk_size + idx + 1}: {e}")continue- text_id = tokenizer(f'{bos_token}{content}{eos_token}').data['input_ids']- doc_ids += text_id- except UnicodeDecodeError as e:- print(f"Skipping invalid line {chunk_idx * chunk_size + idx + 1}: {e}")- continue-- chunk_idx += 1- print(f"Processed chunk {chunk_idx} with {chunk_size} lines")-- if len(doc_ids) > 1000000:- arr = np.array(doc_ids, dtype=np.uint16)- with open(f'./dataset/clean_seq_monkey.bin', 'ab') as f:- f.write(arr.tobytes())- doc_ids = []-- if doc_ids:- arr = np.array(doc_ids, dtype=np.uint16)- with open(f'./dataset/clean_seq_monkey.bin', 'ab') as f:- f.write(arr.tobytes())---def pretrain_process():- # process_wiki_clean()- process_seq_monkey()-- data_path_list = [- # './dataset/clean-wikipedia-cn.bin',- './dataset/clean_seq_monkey.bin'- ]- data_lst = []- for data_path in data_path_list:- with open(data_path, 'rb') as f:- data = np.fromfile(f, dtype=np.uint16)- data_lst.append(data)- arr = np.concatenate(data_lst)- print(arr.shape)- with open('./dataset/pretrain_data.bin', 'wb') as f:- f.write(arr.tobytes())+ chunk_idx += 1+ print('chunk:', ((chunk_idx - 1) * chunk_size, chunk_idx * chunk_size), 'process end')def sft_process(contain_history=False):@@ -186,6 +108,7 @@ def sft_process(contain_history=False):process_and_write_data(data)data = []+def rl_process():################# Datasetdiff --git a/model/dataset.py b/model/dataset.pyindex ef58956..2e417f6 100644--- a/model/dataset.py+++ b/model/dataset.py@@ -13,37 +13,28 @@ os.environ["TOKENIZERS_PARALLELISM"] = "false"class PretrainDataset(Dataset):- def __init__(self, data_path_lst, max_length=512, memmap=False):+ def __init__(self, df, tokenizer, max_length=512):super().__init__()- #- if memmap:- with open(data_path_lst[0], 'r') as f:- nbytes = f.seek(0, 2)- flen = f.tell() // np.dtype('uint16').itemsize- self.data = np.memmap(data_path_lst[0], dtype=np.dtype('uint16'), shape=(flen // max_length, max_length))- else:- data_lst = []- for data_path in data_path_lst:- with open(data_path, 'rb') as f:- data = np.fromfile(f, dtype=np.uint16)- data_lst.append(data)- data = np.concatenate(data_lst)- data = data[:max_length * int(len(data) / max_length)]- # np.random.shuffle(data)- self.data = data.reshape(-1, max_length)- #- print("memmap:{} train data.shape:{}".format(memmap, self.data.shape))- print("downloading finished.....")+ self.df = df+ self.tokenizer = tokenizer+ self.max_length = max_length+ self.padding = 0def __len__(self):- return self.data.shape[0]+ return self.df.shape[0]def __getitem__(self, index: int):#- sample = self.data[index]- X = np.array(sample[:-1]).astype(np.int64)- Y = np.array(sample[1:]).astype(np.int64)+ sample = self.df.iloc[index]+ text = f"{self.tokenizer.bos_token}{str(sample['text'])}{self.tokenizer.eos_token}"+ input_id = self.tokenizer(text).data['input_ids'][:self.max_length]+ # 没满最大长度的剩余部分+ padding_len = self.max_length - len(input_id)+ input_id = input_id + [self.padding] * padding_len+ input_id = np.array(input_id)+ X = np.array(input_id[:-1]).astype(np.int64)+ Y = np.array(input_id[1:]).astype(np.int64)return torch.from_numpy(X), torch.from_numpy(Y)@@ -56,7 +47,7 @@ class SFTDataset(Dataset):self.answer_max_len = answer_max_len#self.tokenizer = tokenizer- self.padding = 0 # self.tokenizer.special_tokens['<pad>']+ self.padding = 0self.bos_id = self.tokenizer('<s>assistant').data['input_ids']def __len__(self):