LLM inference in C/C++
C++ ggml
feat: add GLM5Next MTP, optimize (#29928)
Piotr Wilkin (ilintar)b9acf138a1e2
diff --git a/src/llama-memory-recurrent.cpp b/src/llama-memory-recurrent.cppindex 53130e8689..2893d819e5 100644--- a/src/llama-memory-recurrent.cpp+++ b/src/llama-memory-recurrent.cpp@@ -199,6 +199,10 @@ bool llama_memory_recurrent::seq_rm(llama_seq_id seq_id, llama_pos p0, llama_pos// partial rollback via per-token snapshot index (bounded by n_rs_seq)if (0 < p0 && p0 <= cell.pos && p1 > cell.pos) {+ // a cell shared by several sequences cannot move back for only one of them+ if (cell.seq_id.size() > 1) {+ return false;+ }// the filter kept no layer (e.g. an MTP draft context), so only the position moves backif (is_empty()) {cell.pos = p0 - 1;diff --git a/src/models/glm5-next.cpp b/src/models/glm5-next.cppindex 955405c75e..4343e6ac67 100644--- a/src/models/glm5-next.cpp+++ b/src/models/glm5-next.cpp@@ -64,9 +64,13 @@ void llama_model_glm5_next::load_arch_tensors(llama_model_loader & ml) {const int64_t hc = hparams.dsv4_hc_mult;const int64_t hc_mix_dim = (2 + hc)*hc;- // the NextN block is loaded but only used by the MTP graph.- // Separated trunk_only/mtp_only handling TODO with DECODER_MTP graph in the MTP follow up- int mtp_flags = 0;+ // TODO: consolidate the trunk_only/mtp_only detection and flags shared with glm4-moe and qwen35moe+ const bool mtp_only = (hparams.n_layer_nextn > 0) && (ml.get_weight("blk.0.attn_norm.weight") == nullptr);+ const std::string mtp_probe = "blk." + std::to_string(n_layer) + ".nextn.eh_proj.weight";+ const bool trunk_only = (hparams.n_layer_nextn > 0) && (ml.get_weight(mtp_probe.c_str()) == nullptr);+ const int trunk_flags = mtp_only ? TENSOR_NOT_REQUIRED : 0;+ int mtp_flags = trunk_only ? TENSOR_NOT_REQUIRED : 0;+if (!ml.load_mtp) {mtp_flags |= TENSOR_SKIP;}@@ -79,18 +83,19 @@ void llama_model_glm5_next::load_arch_tensors(llama_model_loader & ml) {for (int i = 0; i < n_layer_all; ++i) {auto & layer = layers[i];- const int flags = (i >= n_layer) ? mtp_flags : 0;+ const bool is_nextn = i >= n_layer;+ const int flags = is_nextn ? mtp_flags : trunk_flags;layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}, flags);layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd}, flags);if (i < n_layer) {- layer.hc_attn_fn = create_tensor(tn(LLM_TENSOR_HC_ATTN_FN, "weight", i), {hc*n_embd, hc_mix_dim}, 0);- layer.hc_attn_base = create_tensor(tn(LLM_TENSOR_HC_ATTN_BASE, "weight", i), {hc_mix_dim}, 0);- layer.hc_attn_scale = create_tensor(tn(LLM_TENSOR_HC_ATTN_SCALE, "weight", i), {3}, 0);- layer.hc_ffn_fn = create_tensor(tn(LLM_TENSOR_HC_FFN_FN, "weight", i), {hc*n_embd, hc_mix_dim}, 0);- layer.hc_ffn_base = create_tensor(tn(LLM_TENSOR_HC_FFN_BASE, "weight", i), {hc_mix_dim}, 0);- layer.hc_ffn_scale = create_tensor(tn(LLM_TENSOR_HC_FFN_SCALE, "weight", i), {3}, 0);+ layer.hc_attn_fn = create_tensor(tn(LLM_TENSOR_HC_ATTN_FN, "weight", i), {hc*n_embd, hc_mix_dim}, flags);+ layer.hc_attn_base = create_tensor(tn(LLM_TENSOR_HC_ATTN_BASE, "weight", i), {hc_mix_dim}, flags);+ layer.hc_attn_scale = create_tensor(tn(LLM_TENSOR_HC_ATTN_SCALE, "weight", i), {3}, flags);+ layer.hc_ffn_fn = create_tensor(tn(LLM_TENSOR_HC_FFN_FN, "weight", i), {hc*n_embd, hc_mix_dim}, flags);+ layer.hc_ffn_base = create_tensor(tn(LLM_TENSOR_HC_FFN_BASE, "weight", i), {hc_mix_dim}, flags);+ layer.hc_ffn_scale = create_tensor(tn(LLM_TENSOR_HC_FFN_SCALE, "weight", i), {3}, flags);}const int64_t head_dim = hparams.n_embd_head_kda;@@ -100,25 +105,25 @@ void llama_model_glm5_next::load_arch_tensors(llama_model_loader & ml) {if (hparams.is_recr(i)) {auto conv = [&](llm_tensor tid) {ggml_tensor * t = create_tensor(tn(tid, "weight", i), {d_conv, 1, d_inner, 1}, TENSOR_NOT_REQUIRED);- return t ? t : create_tensor(tn(tid, "weight", i), {d_conv, 1, d_inner}, 0);+ return t ? t : create_tensor(tn(tid, "weight", i), {d_conv, 1, d_inner}, flags);};layer.ssm_q_conv = conv(LLM_TENSOR_SSM_CONV1D_Q);layer.ssm_k_conv = conv(LLM_TENSOR_SSM_CONV1D_K);layer.ssm_v_conv = conv(LLM_TENSOR_SSM_CONV1D_V);- create_tensor_qkv(layer, i, n_embd, d_inner, d_inner, d_inner, 0);+ create_tensor_qkv(layer, i, n_embd, d_inner, d_inner, d_inner, flags);- layer.ssm_f_a = create_tensor(tn(LLM_TENSOR_SSM_F_A, "weight", i), {n_embd, head_dim}, 0);- layer.ssm_f_b = create_tensor(tn(LLM_TENSOR_SSM_F_B, "weight", i), {head_dim, d_inner}, 0);- layer.ssm_beta = create_tensor(tn(LLM_TENSOR_SSM_BETA, "weight", i), {n_embd, n_head}, 0);+ layer.ssm_f_a = create_tensor(tn(LLM_TENSOR_SSM_F_A, "weight", i), {n_embd, head_dim}, flags);+ layer.ssm_f_b = create_tensor(tn(LLM_TENSOR_SSM_F_B, "weight", i), {head_dim, d_inner}, flags);+ layer.ssm_beta = create_tensor(tn(LLM_TENSOR_SSM_BETA, "weight", i), {n_embd, n_head}, flags);- layer.ssm_a = create_tensor(tn(LLM_TENSOR_SSM_A_NOSCAN, i), {n_head}, 0);- layer.ssm_dt_b = create_tensor(tn(LLM_TENSOR_SSM_DT, "bias", i), {d_inner}, 0);+ layer.ssm_a = create_tensor(tn(LLM_TENSOR_SSM_A_NOSCAN, i), {n_head}, flags);+ layer.ssm_dt_b = create_tensor(tn(LLM_TENSOR_SSM_DT, "bias", i), {d_inner}, flags);- layer.ssm_g_a = create_tensor(tn(LLM_TENSOR_SSM_G_A, "weight", i), {n_embd, head_dim}, 0);- layer.ssm_g_b = create_tensor(tn(LLM_TENSOR_SSM_G_B, "weight", i), {head_dim, d_inner}, 0);- layer.ssm_o_norm = create_tensor(tn(LLM_TENSOR_SSM_NORM, "weight", i), {head_dim}, 0);- layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), {d_inner, n_embd}, 0);+ layer.ssm_g_a = create_tensor(tn(LLM_TENSOR_SSM_G_A, "weight", i), {n_embd, head_dim}, flags);+ layer.ssm_g_b = create_tensor(tn(LLM_TENSOR_SSM_G_B, "weight", i), {head_dim, d_inner}, flags);+ layer.ssm_o_norm = create_tensor(tn(LLM_TENSOR_SSM_NORM, "weight", i), {head_dim}, flags);+ layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), {d_inner, n_embd}, flags);} else {const int64_t q_lora_rank = hparams.n_lora_q;const int64_t kv_lora_rank = hparams.n_lora_kv;@@ -155,9 +160,9 @@ void llama_model_glm5_next::load_arch_tensors(llama_model_loader & ml) {}if (i < (int) hparams.n_layer_dense_lead) {- layer.ffn_gate = create_tensor(tn(LLM_TENSOR_FFN_GATE, "weight", i), {n_embd, n_ff}, 0);- layer.ffn_down = create_tensor(tn(LLM_TENSOR_FFN_DOWN, "weight", i), {n_ff, n_embd}, 0);- layer.ffn_up = create_tensor(tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff}, 0);+ layer.ffn_gate = create_tensor(tn(LLM_TENSOR_FFN_GATE, "weight", i), {n_embd, n_ff}, flags);+ layer.ffn_down = create_tensor(tn(LLM_TENSOR_FFN_DOWN, "weight", i), {n_ff, n_embd}, flags);+ layer.ffn_up = create_tensor(tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff}, flags);} else {const int64_t n_ff_exp = hparams.n_ff_exp(i);const int64_t n_expert_shared = hparams.n_expert_shared;@@ -187,7 +192,7 @@ void llama_model_glm5_next::load_arch_tensors(llama_model_loader & ml) {std::unique_ptr<llm_graph_context> llama_model_glm5_next::build_arch_graph(const llm_graph_params & params) const {if (params.gtype == LLM_GRAPH_TYPE_DECODER_MTP) {- throw std::runtime_error("GLM5-Next NextN graph not implemented yet");+ return std::make_unique<graph_mtp>(*this, params);}return std::make_unique<graph>(*this, params);}@@ -537,6 +542,136 @@ ggml_tensor * llama_model_glm5_next::graph::build_hc_post(return out;}+// construct the graph helpers without building the trunk, so graph_mtp can reuse them+llama_model_glm5_next::graph::graph(const llama_model & model, const llm_graph_params & params, no_build) :+ llm_build_delta_net_base(params), model(model) {+}++llama_model_glm5_next::graph_mtp::graph_mtp(const llama_model & model, const llm_graph_params & params) :+ graph(model, params, no_build{}) {++ GGML_ASSERT(hparams.n_layer_nextn > 0 && "GLM5-Next MTP requires n_layer_nextn > 0");+ GGML_ASSERT(hparams.n_layer_nextn == 1 && "GLM5-Next MTP currently supports a single NextN block");++ const int il = hparams.n_layer() + cparams.nextn_layer_offset;+ GGML_ASSERT(cparams.nextn_layer_offset >= 0 &&+ cparams.nextn_layer_offset < (int) hparams.n_layer_nextn &&+ "nextn_layer_offset out of range [0, n_layer_nextn)");++ const auto & layer = model.layers[il];++ GGML_ASSERT(layer.nextn.eh_proj && layer.nextn.enorm && layer.nextn.hnorm &&+ "GLM5-Next MTP block is missing - load the model with MTP enabled");++ GGML_ASSERT(hparams.n_embd_out() == (uint32_t) n_embd && "GLM5-Next MTP hidden width mismatch");++ const auto * mctx_hyb = static_cast<const llama_memory_hybrid_idx_context *>(mctx);++ auto * inp_hyb = build_inp_mem_hybrid_k();+ auto * inp_attn = inp_hyb->get_attn();+ auto * inp_kpool = build_inp_kpool(mctx_hyb);++ ggml_build_forward_expand(gf, inp_hyb->get_recr()->s_copy);++ ggml_tensor * inp_out_ids = build_inp_out_ids();++ auto inp = std::make_unique<llm_graph_input_embd_h>(n_embd);++ inp->tokens = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_tokens);+ ggml_set_input(inp->tokens);++ inp->embd = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_embd, n_tokens);+ ggml_set_input(inp->embd);++ inp->h = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_embd, n_tokens);+ ggml_set_input(inp->h);+ ggml_set_name(inp->h, "mtp_h_input");++ ggml_tensor * tok_embd = ggml_get_rows(ctx0,+ layer.nextn.embed_tokens ? layer.nextn.embed_tokens : model.tok_embd, inp->tokens);+ cb(tok_embd, "mtp_tok_embd", il);++ ggml_tensor * h = inp->h;++ res->add_input(std::move(inp));++ ggml_tensor * e_norm = build_norm(tok_embd, layer.nextn.enorm, nullptr, LLM_NORM_RMS, il);+ cb(e_norm, "mtp_enorm", il);++ ggml_tensor * h_norm = build_norm(h, layer.nextn.hnorm, nullptr, LLM_NORM_RMS, il);+ cb(h_norm, "mtp_hnorm", il);++ ggml_tensor * cur = build_lora_mm(layer.nextn.eh_proj, ggml_concat(ctx0, e_norm, h_norm, 0), layer.nextn.eh_proj_s);+ cb(cur, "mtp_eh_proj", il);++ ggml_tensor * inpSA = cur;++ cur = build_norm(cur, layer.attn_norm, nullptr, LLM_NORM_RMS, il);+ cb(cur, "mtp_attn_norm", il);++ ggml_tensor * prev_sel = nullptr;+ cur = build_dsa_layer(cur, layer, mctx_hyb, inp_attn, inp_kpool, &prev_sel, il);+ cb(cur, "mtp_attn_out", il);++ // narrow to the output tokens before the position-wise FFN; unmasked nextn embeddings need all rows+ if (crop_before_nextn(inp_out_ids)) {+ cur = ggml_get_rows(ctx0, cur, inp_out_ids);+ inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids);+ }++ ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA);+ cb(ffn_inp, "mtp_ffn_inp", il);++ cur = build_norm(ffn_inp, layer.ffn_norm, nullptr, LLM_NORM_RMS, il);+ cb(cur, "mtp_ffn_norm", il);++ ggml_tensor * moe_out = build_moe_ffn(cur,+ layer.ffn_gate_inp,+ layer.ffn_up_exps,+ layer.ffn_gate_exps,+ layer.ffn_down_exps,+ layer.ffn_exp_probs_b,+ n_expert, n_expert_used,+ LLM_FFN_SILU, hparams.expert_weights_norm,+ hparams.expert_weights_scale,+ (llama_expert_gating_func_type) hparams.expert_gating_func,+ il);+ cb(moe_out, "mtp_ffn_moe_out", il);++ ggml_tensor * ffn_shexp = build_ffn(cur,+ layer.ffn_up_shexp, nullptr, nullptr,+ layer.ffn_gate_shexp, nullptr, nullptr,+ layer.ffn_down_shexp, nullptr, nullptr,+ nullptr, LLM_FFN_SILU, LLM_FFN_PAR, il);+ cb(ffn_shexp, "mtp_ffn_shexp", il);++ cur = ggml_add(ctx0, moe_out, ffn_shexp);+ cb(cur, "mtp_ffn_out", il);++ cur = ggml_add(ctx0, cur, ffn_inp);+ cb(cur, "mtp_post_ffn", il);++ ggml_tensor * head_norm = layer.nextn.shared_head_norm ? layer.nextn.shared_head_norm : model.output_norm;+ GGML_ASSERT(head_norm && "GLM5-Next MTP: missing both nextn.shared_head_norm and output_norm");+ cur = build_norm(cur, head_norm, nullptr, LLM_NORM_RMS, -1);+ cb(cur, "h_nextn", -1);+ res->t_h_nextn = cur;++ if (crop_after_nextn(inp_out_ids)) {+ cur = ggml_get_rows(ctx0, cur, inp_out_ids);+ }+ cb(cur, "mtp_shared_head_norm", -1);++ ggml_tensor * head_w = layer.nextn.shared_head_head ? layer.nextn.shared_head_head : model.output;+ ggml_tensor * head_s = layer.nextn.shared_head_head ? layer.nextn.shared_head_head_s : model.output_s;+ GGML_ASSERT(head_w && "GLM5-Next MTP: missing both nextn.shared_head_head and output");+ cur = build_lora_mm(head_w, cur, head_s);+ cb(cur, "result_output", -1);++ res->t_logits = cur;+ ggml_build_forward_expand(gf, cur);+}+llama_model_glm5_next::graph::graph(const llama_model & model, const llm_graph_params & params) :llm_build_delta_net_base(params), model(model) {diff --git a/src/models/models.h b/src/models/models.hindex 023ed3021e..1ef0c51564 100644--- a/src/models/models.h+++ b/src/models/models.h@@ -2741,6 +2741,10 @@ struct llama_model_glm5_next : public llama_model_base {struct graph : public llm_build_delta_net_base {graph(const llama_model & model, const llm_graph_params & params);+ // build the helpers without the trunk, so graph_mtp can reuse them+ struct no_build {};+ graph(const llama_model & model, const llm_graph_params & params, no_build);+// collapse the hc streams with per-stream weightsggml_tensor * build_hc_pre(ggml_tensor * x,@@ -2789,6 +2793,11 @@ struct llama_model_glm5_next : public llama_model_base {};+ // the NextN / MTP draft head: one DSA block appended after the trunk+ struct graph_mtp : public graph {+ graph_mtp(const llama_model & model, const llm_graph_params & params);+ };+std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;};