🧠 两小时从 0 训出你的 64M 大模型。Train your own 64M LLM from scratch in 2 hours.
update data_process Chunk_size Read
gongjy7115a9845368
diff --git a/data_process.py b/data_process.pyindex 102b4a8..32162ec 100644--- a/data_process.py+++ b/data_process.py@@ -1,3 +1,4 @@+import itertoolsimport reimport jsonimport jsonlines@@ -11,7 +12,6 @@ from datasets import load_datasetbos_token = "<s>"eos_token = "</s>"-# pretraindef process_wiki_clean():with open('./dataset/clean-wikipedia-cn.json', 'r', encoding='utf-8') as f_read:@@ -63,26 +63,40 @@ def process_other():f.write(arr.tobytes())-# pretrain-def process_seq_monkey():+def process_seq_monkey(chunk_size=50000):doc_ids = []+ chunk_idx = 0+with jsonlines.open('./dataset/mobvoi_seq_monkey_general_open_corpus.jsonl') as reader:- for idx, obj in enumerate(reader):- try:- content = obj.get('text', '')- if len(content) > 512:+ while True:+ chunk = list(itertools.islice(reader, chunk_size))+ if not chunk:+ break++ for idx, obj in enumerate(chunk):+ try:+ content = obj.get('text', '')+ if len(content) > 512:+ continue+ text_id = tokenizer(f'{bos_token}{content}{eos_token}').data['input_ids']+ doc_ids += text_id+ except UnicodeDecodeError as e:+ print(f"Skipping invalid line {chunk_idx * chunk_size + idx + 1}: {e}")continue- text_id = tokenizer(f'{bos_token}{content}{eos_token}').data['input_ids']- doc_ids += text_id- if idx % 50000 == 0:- print(f"seq_monkey: [{idx}]")- except UnicodeDecodeError as e:- print(f"Skipping invalid line {idx + 1}: {e}")- continue- arr = np.array(doc_ids, dtype=np.uint16)- with open('./dataset/clean_seq_monkey.bin', 'wb') as f:- f.write(arr.tobytes())+ chunk_idx += 1+ print(f"Processed chunk {chunk_idx} with {chunk_size} lines")++ if len(doc_ids) > 1000000:+ arr = np.array(doc_ids, dtype=np.uint16)+ with open(f'./dataset/clean_seq_monkey.bin', 'wb') as f:+ f.write(arr.tobytes())+ doc_ids = []++ if doc_ids:+ arr = np.array(doc_ids, dtype=np.uint16)+ with open(f'./dataset/clean_seq_monkey.bin', 'wb') as f:+ f.write(arr.tobytes())def pretrain_process():@@ -172,7 +186,6 @@ def sft_process(contain_history=False):process_and_write_data(data)data = []-def rl_process():################# Dataset@@ -212,6 +225,6 @@ if __name__ == "__main__":if process_type == 1:pretrain_process()if process_type == 2:- sft_process(contain_history=True)+ sft_process(contain_history=False)if process_type == 3:rl_process()