Skip to content

🧠 两小时从 0 训出你的 64M 大模型。Train your own 64M LLM from scratch in 2 hours.

[update] remove expert args

jingyaogong6fc918beb68a
diff --git a/eval_llm.py b/eval_llm.py
index c7a11a8..9359ca8 100755
--- a/eval_llm.py
+++ b/eval_llm.py
@@ -16,8 +16,6 @@ def init_model(args):
hidden_size=args.hidden_size,
num_hidden_layers=args.num_hidden_layers,
use_moe=bool(args.use_moe),
- num_experts=args.num_experts,
- num_experts_per_tok=args.num_experts_per_tok,
inference_rope_scaling=args.inference_rope_scaling
))
moe_suffix = '_moe' if args.use_moe else ''
@@ -40,8 +38,6 @@ def main():
parser.add_argument('--hidden_size', default=768, type=int, help="隐藏层维度")
parser.add_argument('--num_hidden_layers', default=8, type=int, help="隐藏层数量")
parser.add_argument('--use_moe', default=0, type=int, choices=[0, 1], help="是否使用MoE架构(0=否,1=是)")
- parser.add_argument('--num_experts', default=4, type=int, help="MoE专家数量(仅use_moe=1时生效,需与训练时一致)")
- parser.add_argument('--num_experts_per_tok', default=1, type=int, help="每个token路由到的专家数(仅use_moe=1时生效,需与训练时一致)")
parser.add_argument('--inference_rope_scaling', default=False, action='store_true', help="启用RoPE位置编码外推(4倍,仅解决位置编码问题)")
parser.add_argument('--max_new_tokens', default=8192, type=int, help="最大生成长度(注意:并非模型实际长文本能力)")
parser.add_argument('--temperature', default=0.85, type=float, help="生成温度,控制随机性(0-1,越大越随机)")
diff --git a/trainer/train_full_sft.py b/trainer/train_full_sft.py
index b080aef..c22eb80 100644
--- a/trainer/train_full_sft.py
+++ b/trainer/train_full_sft.py
@@ -99,8 +99,6 @@ if __name__ == "__main__":
parser.add_argument('--num_hidden_layers', default=8, type=int, help="隐藏层数量")
parser.add_argument('--max_seq_len', default=768, type=int, help="训练的最大截断长度(中文1token≈1.5~1.7字符)")
parser.add_argument('--use_moe', default=0, type=int, choices=[0, 1], help="是否使用MoE架构(0=否,1=是)")
- parser.add_argument('--num_experts', default=4, type=int, help="MoE专家数量(仅use_moe=1时生效)")
- parser.add_argument('--num_experts_per_tok', default=1, type=int, help="每个token路由到的专家数(仅use_moe=1时生效)")
parser.add_argument('--seed', default=42, type=int, help="随机种子(DDP下每个rank为seed+rank,每轮为seed+epoch)")
parser.add_argument("--data_path", type=str, default="../dataset/sft_t2t_mini.jsonl", help="训练数据路径")
parser.add_argument('--from_weight', default='pretrain', type=str, help="基于哪个权重训练,为none则不基于任何权重训练")
@@ -117,8 +115,7 @@ if __name__ == "__main__":
# ========== 2. 配置目录、模型参数、检查ckp ==========
os.makedirs(args.save_dir, exist_ok=True)
- lm_config = MiniMindConfig(hidden_size=args.hidden_size, num_hidden_layers=args.num_hidden_layers, use_moe=bool(args.use_moe),
- num_experts=args.num_experts, num_experts_per_tok=args.num_experts_per_tok)
+ lm_config = MiniMindConfig(hidden_size=args.hidden_size, num_hidden_layers=args.num_hidden_layers, use_moe=bool(args.use_moe))
ckp_data = lm_checkpoint(lm_config, weight=args.save_weight, save_dir='../checkpoints') if args.from_resume==1 else None
# ========== 3. 设置混合精度 ==========
diff --git a/trainer/train_pretrain.py b/trainer/train_pretrain.py
index 42384c8..83a0382 100644
--- a/trainer/train_pretrain.py
+++ b/trainer/train_pretrain.py
@@ -98,8 +98,6 @@ if __name__ == "__main__":
parser.add_argument('--num_hidden_layers', default=8, type=int, help="隐藏层数量")
parser.add_argument('--max_seq_len', default=340, type=int, help="训练的最大截断长度(中文1token≈1.5~1.7字符)")
parser.add_argument('--use_moe', default=0, type=int, choices=[0, 1], help="是否使用MoE架构(0=否,1=是)")
- parser.add_argument('--num_experts', default=4, type=int, help="MoE专家数量(仅use_moe=1时生效)")
- parser.add_argument('--num_experts_per_tok', default=1, type=int, help="每个token路由到的专家数(仅use_moe=1时生效)")
parser.add_argument('--seed', default=42, type=int, help="随机种子(DDP下每个rank为seed+rank,每轮为seed+epoch)")
parser.add_argument("--data_path", type=str, default="../dataset/pretrain_t2t_mini.jsonl", help="预训练数据路径")
parser.add_argument('--from_weight', default='none', type=str, help="基于哪个权重训练,为none则从头开始")
@@ -116,8 +114,7 @@ if __name__ == "__main__":
# ========== 2. 配置目录、模型参数、检查ckp ==========
os.makedirs(args.save_dir, exist_ok=True)
- lm_config = MiniMindConfig(hidden_size=args.hidden_size, num_hidden_layers=args.num_hidden_layers, use_moe=bool(args.use_moe),
- num_experts=args.num_experts, num_experts_per_tok=args.num_experts_per_tok)
+ lm_config = MiniMindConfig(hidden_size=args.hidden_size, num_hidden_layers=args.num_hidden_layers, use_moe=bool(args.use_moe))
ckp_data = lm_checkpoint(lm_config, weight=args.save_weight, save_dir='../checkpoints') if args.from_resume==1 else None
# ========== 3. 设置混合精度 ==========