LLM inference in C/C++
C++ ggml
llama : load MTP tensors only if they are really used (#26296)
fairydreaming82dbc4f017a7
diff --git a/common/common.cpp b/common/common.cppindex 1a0aeef4ec..ff27d392fb 100644--- a/common/common.cpp+++ b/common/common.cpp@@ -1620,6 +1620,7 @@ struct llama_model_params common_model_params_to_llama(common_params & params) {mparams.progress_callback = params.load_progress_callback;mparams.progress_callback_user_data = params.load_progress_callback_user_data;mparams.no_alloc = params.no_alloc;+ mparams.load_mtp = std::find(params.speculative.types.begin(), params.speculative.types.end(), COMMON_SPECULATIVE_TYPE_DRAFT_MTP) != params.speculative.types.end();return mparams;}diff --git a/include/llama.h b/include/llama.hindex a3ceecf5ef..6e53e22972 100644--- a/include/llama.h+++ b/include/llama.h@@ -337,6 +337,7 @@ extern "C" {bool use_extra_bufts; // use extra buffer types (used for weight repacking)bool no_host; // bypass host buffer allowing extra buffers to be usedbool no_alloc; // only load metadata and simulate memory allocations+ bool load_mtp; // whether to load MTP layers};struct llama_sampler_seq_config {diff --git a/src/llama-model-loader.cpp b/src/llama-model-loader.cppindex 510586e96c..df8313e819 100644--- a/src/llama-model-loader.cpp+++ b/src/llama-model-loader.cpp@@ -526,6 +526,7 @@ llama_model_loader::llama_model_loader(llama_load_mode load_mode,bool check_tensors,bool no_alloc,+ bool load_mtp,const llama_model_kv_override * param_overrides_p,const llama_model_tensor_buft_override * param_tensor_buft_overrides_p): metadata(meta), set_tensor_data(set_tensor_data), set_tensor_data_ud(set_tensor_data_ud) {@@ -812,6 +813,7 @@ llama_model_loader::llama_model_loader(this->check_tensors = check_tensors;this->no_alloc = no_alloc;+ this->load_mtp = load_mtp;}std::string llama_model_loader::get_arch_name() const {diff --git a/src/llama-model-loader.h b/src/llama-model-loader.hindex 75a3652d06..7ad3807822 100644--- a/src/llama-model-loader.h+++ b/src/llama-model-loader.h@@ -79,6 +79,7 @@ struct llama_model_loader {bool use_direct_io = false;bool check_tensors;bool no_alloc;+ bool load_mtp;llama_files files;llama_ftype ftype;@@ -129,6 +130,7 @@ struct llama_model_loader {llama_load_mode load_mode,bool check_tensors,bool no_alloc,+ bool load_mtp,const llama_model_kv_override * param_overrides_p,const llama_model_tensor_buft_override * param_tensor_buft_overrides_p);diff --git a/src/llama-model.cpp b/src/llama-model.cppindex 5e501f983b..91c2fc9c00 100644--- a/src/llama-model.cpp+++ b/src/llama-model.cpp@@ -2390,6 +2390,7 @@ llama_model_params llama_model_default_params() {/*.use_extra_bufts =*/ true,/*.no_host =*/ false,/*.no_alloc =*/ false,+ /*.load_mtp =*/ false,};return result;diff --git a/src/llama-quant.cpp b/src/llama-quant.cppindex 92ebc11b99..fd6e787bd7 100644--- a/src/llama-quant.cpp+++ b/src/llama-quant.cpp@@ -893,7 +893,7 @@ static void llama_model_quantize_impl(const std::string & fname_inp, const std::const llama_model_kv_override * kv_overrides = params->kv_overrides;std::vector<std::string> splits = {};llama_model_loader ml(/*metadata*/ nullptr, /*set_tensor_data*/ nullptr, /*set_tensor_data_ud*/ nullptr,- fname_inp, splits, /*file*/ nullptr, /*load_mode*/ load_mode, /*check_tensors*/ true, /*no_alloc*/ false, kv_overrides, nullptr);+ fname_inp, splits, /*file*/ nullptr, /*load_mode*/ load_mode, /*check_tensors*/ true, /*no_alloc*/ false, /*load_mtp*/ true, kv_overrides, nullptr);ml.init_mappings(false); // no prefetchingauto mparams = llama_model_default_params();diff --git a/src/llama.cpp b/src/llama.cppindex d22e4c81a9..d6e0bbfefa 100644--- a/src/llama.cpp+++ b/src/llama.cpp@@ -305,7 +305,7 @@ static std::pair<int, llama_model *> llama_model_load(struct gguf_context * metaconst std::string & fname, std::vector<std::string> & splits, FILE * file, llama_model_params & params) {try {llama_model_loader ml(metadata, set_tensor_data, set_tensor_data_ud, fname, splits, file, params.load_mode,- params.check_tensors, params.no_alloc, params.kv_overrides, params.tensor_buft_overrides);+ params.check_tensors, params.no_alloc, params.load_mtp, params.kv_overrides, params.tensor_buft_overrides);ml.print_info();std::unique_ptr<llama_model> model_ptr(llama_model_create(ml, params));diff --git a/src/models/cohere2moe.cpp b/src/models/cohere2moe.cppindex 499c73a1c4..3acb7e77af 100644--- a/src/models/cohere2moe.cpp+++ b/src/models/cohere2moe.cpp@@ -55,7 +55,11 @@ void llama_model_cohere2moe::load_arch_tensors(llama_model_loader & ml) {const std::string mtp_probe = "blk." + std::to_string(n_layer) + ".nextn.eh_proj.weight";const bool trunk_only = (hparams.n_layer_nextn > 0) && (ml.get_weight(mtp_probe.c_str()) == nullptr);const int trunk_flags = mtp_only ? TENSOR_NOT_REQUIRED : 0;- const int mtp_flags = trunk_only ? TENSOR_NOT_REQUIRED : 0;+ int mtp_flags = trunk_only ? TENSOR_NOT_REQUIRED : 0;++ if (!ml.load_mtp) {+ mtp_flags |= TENSOR_SKIP;+ }tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), { n_embd, n_vocab }, 0);diff --git a/src/models/glm-dsa.cpp b/src/models/glm-dsa.cppindex bd1c4df212..360c2ee773 100644--- a/src/models/glm-dsa.cpp+++ b/src/models/glm-dsa.cpp@@ -91,7 +91,11 @@ void llama_model_glm_dsa::load_arch_tensors(llama_model_loader & ml) {const std::string mtp_probe = "blk." + std::to_string(n_layer) + ".nextn.eh_proj.weight";const bool trunk_only = (hparams.n_layer_nextn > 0) && (ml.get_weight(mtp_probe.c_str()) == nullptr);const int trunk_flags = mtp_only ? TENSOR_NOT_REQUIRED : 0;- const int mtp_flags = trunk_only ? TENSOR_NOT_REQUIRED : 0;+ int mtp_flags = trunk_only ? TENSOR_NOT_REQUIRED : 0;++ if (!ml.load_mtp) {+ mtp_flags |= TENSOR_SKIP;+ }const bool is_mla = hparams.is_mla();if (!is_mla) {diff --git a/src/models/hy-v3.cpp b/src/models/hy-v3.cppindex 47a0beaf21..61db93af85 100644--- a/src/models/hy-v3.cpp+++ b/src/models/hy-v3.cpp@@ -33,7 +33,11 @@ void llama_model_hy_v3::load_arch_tensors(llama_model_loader & ml) {const std::string mtp_probe = "blk." + std::to_string(n_layer) + ".nextn.eh_proj.weight";const bool trunk_only = (hparams.n_layer_nextn > 0) && (ml.get_weight(mtp_probe.c_str()) == nullptr);const int trunk_flags = mtp_only ? TENSOR_NOT_REQUIRED : 0;- const int mtp_flags = trunk_only ? TENSOR_NOT_REQUIRED : 0;+ int mtp_flags = trunk_only ? TENSOR_NOT_REQUIRED : 0;++ if (!ml.load_mtp) {+ mtp_flags |= TENSOR_SKIP;+ }tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, 0);diff --git a/src/models/qwen35.cpp b/src/models/qwen35.cppindex d8ffe43ae7..309dd43244 100644--- a/src/models/qwen35.cpp+++ b/src/models/qwen35.cpp@@ -39,6 +39,7 @@ void llama_model_qwen35::load_arch_tensors(llama_model_loader & ml) {const bool mtp_only = (hparams.n_layer_nextn > 0) && (ml.get_weight("blk.0.attn_norm.weight") == nullptr);const int trunk_flags = mtp_only ? TENSOR_NOT_REQUIRED : 0;+ int mtp_flags = !ml.load_mtp ? TENSOR_SKIP : 0;tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), { n_embd, n_vocab }, 0);@@ -97,25 +98,25 @@ void llama_model_qwen35::load_arch_tensors(llama_model_loader & ml) {auto & layer = layers[il];// MTP block looks like a full-attention Qwen3.5 decoder block.- layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "weight", il), { n_embd }, 0);- layer.attn_post_norm = create_tensor(tn(LLM_TENSOR_ATTN_POST_NORM, "weight", il), { n_embd }, 0);+ layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "weight", il), { n_embd }, mtp_flags);+ layer.attn_post_norm = create_tensor(tn(LLM_TENSOR_ATTN_POST_NORM, "weight", il), { n_embd }, mtp_flags);- create_tensor_qkv(layer, il, n_embd, n_embd_head_k * n_head * 2, n_embd_k_gqa, n_embd_v_gqa, 0);- layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", il), { n_embd_head_k * n_head, n_embd }, 0);- layer.attn_q_norm = create_tensor(tn(LLM_TENSOR_ATTN_Q_NORM, "weight", il), { n_embd_head_k }, 0);- layer.attn_k_norm = create_tensor(tn(LLM_TENSOR_ATTN_K_NORM, "weight", il), { n_embd_head_k }, 0);+ create_tensor_qkv(layer, il, n_embd, n_embd_head_k * n_head * 2, n_embd_k_gqa, n_embd_v_gqa, mtp_flags);+ layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", il), { n_embd_head_k * n_head, n_embd }, mtp_flags);+ layer.attn_q_norm = create_tensor(tn(LLM_TENSOR_ATTN_Q_NORM, "weight", il), { n_embd_head_k }, mtp_flags);+ layer.attn_k_norm = create_tensor(tn(LLM_TENSOR_ATTN_K_NORM, "weight", il), { n_embd_head_k }, mtp_flags);- layer.ffn_gate = create_tensor(tn(LLM_TENSOR_FFN_GATE, "weight", il), {n_embd, n_ff}, 0);- layer.ffn_down = create_tensor(tn(LLM_TENSOR_FFN_DOWN, "weight", il), { n_ff, n_embd}, 0);- layer.ffn_up = create_tensor(tn(LLM_TENSOR_FFN_UP, "weight", il), {n_embd, n_ff}, 0);+ layer.ffn_gate = create_tensor(tn(LLM_TENSOR_FFN_GATE, "weight", il), {n_embd, n_ff}, mtp_flags);+ layer.ffn_down = create_tensor(tn(LLM_TENSOR_FFN_DOWN, "weight", il), { n_ff, n_embd}, mtp_flags);+ layer.ffn_up = create_tensor(tn(LLM_TENSOR_FFN_UP, "weight", il), {n_embd, n_ff}, mtp_flags);// NextN-specific tensors that define the MTP block.- layer.nextn.eh_proj = create_tensor(tn(LLM_TENSOR_NEXTN_EH_PROJ, "weight", il), { 2 * n_embd, n_embd }, 0);- layer.nextn.enorm = create_tensor(tn(LLM_TENSOR_NEXTN_ENORM, "weight", il), { n_embd }, 0);- layer.nextn.hnorm = create_tensor(tn(LLM_TENSOR_NEXTN_HNORM, "weight", il), { n_embd }, 0);- layer.nextn.embed_tokens = create_tensor(tn(LLM_TENSOR_NEXTN_EMBED_TOKENS, "weight", il), { n_embd, n_vocab }, TENSOR_NOT_REQUIRED);- layer.nextn.shared_head_head = create_tensor(tn(LLM_TENSOR_NEXTN_SHARED_HEAD_HEAD, "weight", il), { n_embd, n_vocab }, TENSOR_NOT_REQUIRED);- layer.nextn.shared_head_norm = create_tensor(tn(LLM_TENSOR_NEXTN_SHARED_HEAD_NORM, "weight", il), { n_embd }, TENSOR_NOT_REQUIRED);+ layer.nextn.eh_proj = create_tensor(tn(LLM_TENSOR_NEXTN_EH_PROJ, "weight", il), { 2 * n_embd, n_embd }, mtp_flags);+ layer.nextn.enorm = create_tensor(tn(LLM_TENSOR_NEXTN_ENORM, "weight", il), { n_embd }, mtp_flags);+ layer.nextn.hnorm = create_tensor(tn(LLM_TENSOR_NEXTN_HNORM, "weight", il), { n_embd }, mtp_flags);+ layer.nextn.embed_tokens = create_tensor(tn(LLM_TENSOR_NEXTN_EMBED_TOKENS, "weight", il), { n_embd, n_vocab }, mtp_flags|TENSOR_NOT_REQUIRED);+ layer.nextn.shared_head_head = create_tensor(tn(LLM_TENSOR_NEXTN_SHARED_HEAD_HEAD, "weight", il), { n_embd, n_vocab }, mtp_flags|TENSOR_NOT_REQUIRED);+ layer.nextn.shared_head_norm = create_tensor(tn(LLM_TENSOR_NEXTN_SHARED_HEAD_NORM, "weight", il), { n_embd }, mtp_flags|TENSOR_NOT_REQUIRED);};for (int i = 0; i < n_layer; ++i) {diff --git a/src/models/qwen35moe.cpp b/src/models/qwen35moe.cppindex 7b0876cbb0..38f2a57985 100644--- a/src/models/qwen35moe.cpp+++ b/src/models/qwen35moe.cpp@@ -42,6 +42,7 @@ void llama_model_qwen35moe::load_arch_tensors(llama_model_loader & ml) {const bool mtp_only = (hparams.n_layer_nextn > 0) && (ml.get_weight("blk.0.attn_norm.weight") == nullptr);const int trunk_flags = mtp_only ? TENSOR_NOT_REQUIRED : 0;+ int mtp_flags = !ml.load_mtp ? TENSOR_SKIP : 0;tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), { n_embd, n_vocab }, 0);@@ -113,32 +114,32 @@ void llama_model_qwen35moe::load_arch_tensors(llama_model_loader & ml) {const int64_t n_ff_shexp = hparams.n_ff_shexp ? hparams.n_ff_shexp : n_ff;// MTP block looks like a full-attention Qwen3.5 decoder block with MoE FFN.- layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "weight", il), { n_embd }, 0);- layer.attn_post_norm = create_tensor(tn(LLM_TENSOR_ATTN_POST_NORM, "weight", il), { n_embd }, 0);+ layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "weight", il), { n_embd }, mtp_flags);+ layer.attn_post_norm = create_tensor(tn(LLM_TENSOR_ATTN_POST_NORM, "weight", il), { n_embd }, mtp_flags);- create_tensor_qkv(layer, il, n_embd, n_embd_head_k * n_head * 2, n_embd_k_gqa, n_embd_v_gqa, 0);- layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", il), { n_embd_head_k * n_head, n_embd }, 0);- layer.attn_q_norm = create_tensor(tn(LLM_TENSOR_ATTN_Q_NORM, "weight", il), { n_embd_head_k }, 0);- layer.attn_k_norm = create_tensor(tn(LLM_TENSOR_ATTN_K_NORM, "weight", il), { n_embd_head_k }, 0);+ create_tensor_qkv(layer, il, n_embd, n_embd_head_k * n_head * 2, n_embd_k_gqa, n_embd_v_gqa, mtp_flags);+ layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", il), { n_embd_head_k * n_head, n_embd }, mtp_flags);+ layer.attn_q_norm = create_tensor(tn(LLM_TENSOR_ATTN_Q_NORM, "weight", il), { n_embd_head_k }, mtp_flags);+ layer.attn_k_norm = create_tensor(tn(LLM_TENSOR_ATTN_K_NORM, "weight", il), { n_embd_head_k }, mtp_flags);// Routed experts- layer.ffn_gate_inp = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP, "weight", il), { n_embd, n_expert }, 0);- layer.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, "weight", il), { n_ff_exp, n_embd, n_expert }, 0);- create_tensor_gate_up_exps(layer, il, n_embd, n_ff_exp, n_expert, 0);+ layer.ffn_gate_inp = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP, "weight", il), { n_embd, n_expert }, mtp_flags);+ layer.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, "weight", il), { n_ff_exp, n_embd, n_expert }, mtp_flags);+ create_tensor_gate_up_exps(layer, il, n_embd, n_ff_exp, n_expert, mtp_flags);// Shared experts- layer.ffn_gate_inp_shexp = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP_SHEXP, "weight", il), { n_embd }, 0);- layer.ffn_gate_shexp = create_tensor(tn(LLM_TENSOR_FFN_GATE_SHEXP, "weight", il), { n_embd, n_ff_shexp }, 0);- layer.ffn_up_shexp = create_tensor(tn(LLM_TENSOR_FFN_UP_SHEXP, "weight", il), { n_embd, n_ff_shexp }, 0);- layer.ffn_down_shexp = create_tensor(tn(LLM_TENSOR_FFN_DOWN_SHEXP, "weight", il), { n_ff_shexp, n_embd }, 0);+ layer.ffn_gate_inp_shexp = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP_SHEXP, "weight", il), { n_embd }, mtp_flags);+ layer.ffn_gate_shexp = create_tensor(tn(LLM_TENSOR_FFN_GATE_SHEXP, "weight", il), { n_embd, n_ff_shexp }, mtp_flags);+ layer.ffn_up_shexp = create_tensor(tn(LLM_TENSOR_FFN_UP_SHEXP, "weight", il), { n_embd, n_ff_shexp }, mtp_flags);+ layer.ffn_down_shexp = create_tensor(tn(LLM_TENSOR_FFN_DOWN_SHEXP, "weight", il), { n_ff_shexp, n_embd }, mtp_flags);// NextN-specific tensors that define the MTP block.- layer.nextn.eh_proj = create_tensor(tn(LLM_TENSOR_NEXTN_EH_PROJ, "weight", il), { 2 * n_embd, n_embd }, 0);- layer.nextn.enorm = create_tensor(tn(LLM_TENSOR_NEXTN_ENORM, "weight", il), { n_embd }, 0);- layer.nextn.hnorm = create_tensor(tn(LLM_TENSOR_NEXTN_HNORM, "weight", il), { n_embd }, 0);- layer.nextn.embed_tokens = create_tensor(tn(LLM_TENSOR_NEXTN_EMBED_TOKENS, "weight", il), { n_embd, n_vocab }, TENSOR_NOT_REQUIRED);- layer.nextn.shared_head_head = create_tensor(tn(LLM_TENSOR_NEXTN_SHARED_HEAD_HEAD, "weight", il), { n_embd, n_vocab }, TENSOR_NOT_REQUIRED);- layer.nextn.shared_head_norm = create_tensor(tn(LLM_TENSOR_NEXTN_SHARED_HEAD_NORM, "weight", il), { n_embd }, TENSOR_NOT_REQUIRED);+ layer.nextn.eh_proj = create_tensor(tn(LLM_TENSOR_NEXTN_EH_PROJ, "weight", il), { 2 * n_embd, n_embd }, mtp_flags);+ layer.nextn.enorm = create_tensor(tn(LLM_TENSOR_NEXTN_ENORM, "weight", il), { n_embd }, mtp_flags);+ layer.nextn.hnorm = create_tensor(tn(LLM_TENSOR_NEXTN_HNORM, "weight", il), { n_embd }, mtp_flags);+ layer.nextn.embed_tokens = create_tensor(tn(LLM_TENSOR_NEXTN_EMBED_TOKENS, "weight", il), { n_embd, n_vocab }, mtp_flags|TENSOR_NOT_REQUIRED);+ layer.nextn.shared_head_head = create_tensor(tn(LLM_TENSOR_NEXTN_SHARED_HEAD_HEAD, "weight", il), { n_embd, n_vocab }, mtp_flags|TENSOR_NOT_REQUIRED);+ layer.nextn.shared_head_norm = create_tensor(tn(LLM_TENSOR_NEXTN_SHARED_HEAD_NORM, "weight", il), { n_embd }, mtp_flags|TENSOR_NOT_REQUIRED);};for (int i = 0; i < n_layer; ++i) {diff --git a/src/models/step35.cpp b/src/models/step35.cppindex 9b7b18a367..5b1d902581 100644--- a/src/models/step35.cpp+++ b/src/models/step35.cpp@@ -48,7 +48,11 @@ void llama_model_step35::load_arch_tensors(llama_model_loader & ml) {const std::string mtp_probe = "blk." + std::to_string(n_layer) + ".nextn.eh_proj.weight";const bool trunk_only = (hparams.n_layer_nextn > 0) && (ml.get_weight(mtp_probe.c_str()) == nullptr);const int trunk_flags = mtp_only ? TENSOR_NOT_REQUIRED : 0;- const int mtp_flags = trunk_only ? TENSOR_NOT_REQUIRED : 0;+ int mtp_flags = trunk_only ? TENSOR_NOT_REQUIRED : 0;++ if (!ml.load_mtp) {+ mtp_flags |= TENSOR_SKIP;+ }tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, 0);