🧠 两小时从 0 训出你的 64M 大模型。Train your own 64M LLM from scratch in 2 hours.
update data_process & full_sft
gongjy56c6139896ed
diff --git a/3-full_sft.py b/3-full_sft.pyindex 459c244..1c76954 100644--- a/3-full_sft.py+++ b/3-full_sft.py@@ -109,7 +109,7 @@ def init_model(lm_config):if model_from == 1:moe_path = '_moe' if lm_config.use_moe else ''- ckp = f'./out/single_chat/full_sft_{lm_config.dim}{moe_path}.pth'+ ckp = f'./out/pretrain_{lm_config.dim}{moe_path}.pth'model = Transformer(lm_config)state_dict = torch.load(ckp, map_location=device)@@ -148,8 +148,8 @@ if __name__ == "__main__":out_dir = 'out'epochs = 19gradient_accumulation_steps = 1- batch_size = 48- learning_rate = 2e-8+ batch_size = 80+ learning_rate = 2e-4device = 'cuda:0'dtype = 'bfloat16'# dtype = 'float16'@@ -175,7 +175,7 @@ if __name__ == "__main__":model, tokenizer = init_model(lm_config)# -----init dataloader------- df = pd.read_csv('./dataset/sft_data.csv')+ df = pd.read_csv('./dataset/sft_data_single.csv')df = df.sample(frac=1.0)train_ds = SFTDataset(df, tokenizer, max_length=max_seq_len)train_sampler = DistributedSampler(train_ds) if ddp else Nonediff --git a/data_process.py b/data_process.pyindex 7595186..102b4a8 100644--- a/data_process.py+++ b/data_process.py@@ -68,13 +68,17 @@ def process_seq_monkey():doc_ids = []with jsonlines.open('./dataset/mobvoi_seq_monkey_general_open_corpus.jsonl') as reader:for idx, obj in enumerate(reader):- content = obj.get('text', '')- if len(content) > 512:+ try:+ content = obj.get('text', '')+ if len(content) > 512:+ continue+ text_id = tokenizer(f'{bos_token}{content}{eos_token}').data['input_ids']+ doc_ids += text_id+ if idx % 50000 == 0:+ print(f"seq_monkey: [{idx}]")+ except UnicodeDecodeError as e:+ print(f"Skipping invalid line {idx + 1}: {e}")continue- text_id = tokenizer(f'{bos_token}{content}{eos_token}').data['input_ids']- doc_ids += text_id- if idx % 50000 == 0:- print(f"seq_monkey: [{idx}]")arr = np.array(doc_ids, dtype=np.uint16)with open('./dataset/clean_seq_monkey.bin', 'wb') as f:@@ -150,15 +154,19 @@ def sft_process(contain_history=False):for path in sft_datasets:with jsonlines.open(path) as reader:for idx, obj in enumerate(reader):- data.append({- 'history': obj.get('history', ''),- 'q': obj.get('input', '') + obj.get('q', ''),- 'a': obj.get('output', '') + obj.get('a', '')- })-- if len(data) >= chunk_size:- process_and_write_data(data)- data = []+ try:+ data.append({+ 'history': obj.get('history', ''),+ 'q': obj.get('input', '') + obj.get('q', ''),+ 'a': obj.get('output', '') + obj.get('a', '')+ })++ if len(data) >= chunk_size:+ process_and_write_data(data)+ data = []+ except jsonlines.InvalidLineError as e:+ print(f"Skipping invalid JSON line {idx + 1}: {e}")+ continueif data:process_and_write_data(data)@@ -191,7 +199,7 @@ def rl_process():if __name__ == "__main__":- tokenizer = AutoTokenizer.from_pretrained('./model', use_fast=False)+ tokenizer = AutoTokenizer.from_pretrained('./model/minimind_tokenizer', use_fast=False)print('tokenizer词表大小:', len(tokenizer))################@@ -199,7 +207,7 @@ if __name__ == "__main__":# 2: sft# 3: RL################- process_type = 2+ process_type = 1if process_type == 1:pretrain_process()