LLM inference in C/C++
C++ ggml
llama: share the nextn tensor flags between models (#30097)
Pascal448147d42a71
diff --git a/src/llama-model.cpp b/src/llama-model.cppindex 1f3b80b08d..9f05a66e45 100644--- a/src/llama-model.cpp+++ b/src/llama-model.cpp@@ -3461,6 +3461,27 @@ void llama_model_base::create_tensor_qkv(llama_layer & layer, int bid,}}+llama_model_base::nextn_flags_t llama_model_base::nextn_flags(llama_model_loader & ml, llm_tensor trunk_probe) const {+ int trunk = 0;+ int mtp = 0;++ // a file without the first trunk layer is MTP-only, a file without the first NextN layer is trunk-only+ if (hparams.n_layer_nextn > 0) {+ if (ml.get_weight(tn(trunk_probe, "weight", 0).str().c_str()) == nullptr) {+ trunk = TENSOR_NOT_REQUIRED;+ }+ if (ml.get_weight(tn(LLM_TENSOR_NEXTN_EH_PROJ, "weight", hparams.n_layer()).str().c_str()) == nullptr) {+ mtp = TENSOR_NOT_REQUIRED;+ }+ }++ if (!ml.load_mtp) {+ mtp |= TENSOR_SKIP;+ }++ return { trunk, mtp };+}+void llama_model_base::load_swa_pattern(llama_model_loader & ml, uint32_t n_pattern, bool dense_first) {if (ml.get_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, hparams.is_swa_impl, false)) {return;diff --git a/src/llama-model.h b/src/llama-model.hindex a86cc56935..82124471f0 100644--- a/src/llama-model.h+++ b/src/llama-model.h@@ -849,6 +849,13 @@ struct llama_model_base : public llama_model {int64_t n_embd_, int64_t n_embd_q_, int64_t n_embd_k_, int64_t n_embd_v_,int flags);+ // helper: tensor flags for a file that holds the trunk, the NextN layers, or both+ struct nextn_flags_t {+ int trunk; // TENSOR_NOT_REQUIRED when the file holds only the NextN layers+ int mtp; // TENSOR_NOT_REQUIRED when the file holds only the trunk, TENSOR_SKIP when MTP is not loaded+ };+ nextn_flags_t nextn_flags(llama_model_loader & ml, llm_tensor trunk_probe = LLM_TENSOR_ATTN_NORM) const;+// helper: read the SWA pattern as one flag per layer, or as a period expanded by set_swa_patternvoid load_swa_pattern(llama_model_loader & ml, uint32_t n_pattern, bool dense_first = false);diff --git a/src/models/bailingmoe3.cpp b/src/models/bailingmoe3.cppindex b04b58449f..2458b1c1a0 100644--- a/src/models/bailingmoe3.cpp+++ b/src/models/bailingmoe3.cpp@@ -64,15 +64,9 @@ void llama_model_bailingmoe3::load_arch_tensors(llama_model_loader & ml) {const int64_t qk_head_dim = hparams.n_embd_head_k_mla();const int64_t v_head_dim = hparams.n_embd_head_v_mla();- const bool mtp_only = (hparams.n_layer_nextn > 0) && (ml.get_weight("blk.0.attn_norm.weight") == nullptr);- const std::string mtp_probe = "blk." + std::to_string(n_layer) + ".nextn.eh_proj.weight";- const bool trunk_only = (hparams.n_layer_nextn > 0) && (ml.get_weight(mtp_probe.c_str()) == nullptr);- const int trunk_flags = mtp_only ? TENSOR_NOT_REQUIRED : 0;- int mtp_flags = trunk_only ? TENSOR_NOT_REQUIRED : 0;-- if (!ml.load_mtp) {- mtp_flags |= TENSOR_SKIP;- }+ const auto nf = nextn_flags(ml);+ const int trunk_flags = nf.trunk;+ const int mtp_flags = nf.mtp;for (int il = 0; il < n_layer; ++il) {auto & layer = layers[il];diff --git a/src/models/cohere2moe.cpp b/src/models/cohere2moe.cppindex 3baf3950c8..a379e2f60e 100644--- a/src/models/cohere2moe.cpp+++ b/src/models/cohere2moe.cpp@@ -40,18 +40,9 @@ void llama_model_cohere2moe::load_arch_hparams(llama_model_loader & ml) {void llama_model_cohere2moe::load_arch_tensors(llama_model_loader & ml) {LLAMA_LOAD_LOCALS;- const bool mtp_only = (hparams.n_layer_nextn > 0) && (ml.get_weight("blk.0.attn_norm.weight") == nullptr);- // Trunk-only: the GGUF declares MTP layers in metadata but the actual MTP- // tensors live in a separate file. Mark MTP tensors NOT_REQUIRED so the- // trunk loads cleanly.- const std::string mtp_probe = "blk." + std::to_string(n_layer) + ".nextn.eh_proj.weight";- const bool trunk_only = (hparams.n_layer_nextn > 0) && (ml.get_weight(mtp_probe.c_str()) == nullptr);- const int trunk_flags = mtp_only ? TENSOR_NOT_REQUIRED : 0;- int mtp_flags = trunk_only ? TENSOR_NOT_REQUIRED : 0;-- if (!ml.load_mtp) {- mtp_flags |= TENSOR_SKIP;- }+ const auto nf = nextn_flags(ml);+ const int trunk_flags = nf.trunk;+ const int mtp_flags = nf.mtp;tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), { n_embd, n_vocab }, 0);diff --git a/src/models/deepseek2.cpp b/src/models/deepseek2.cppindex 47fc2f1ac9..f1067a5165 100644--- a/src/models/deepseek2.cpp+++ b/src/models/deepseek2.cpp@@ -56,15 +56,9 @@ void llama_model_deepseek2::load_arch_tensors(llama_model_loader & ml) {LLAMA_LOAD_LOCALS;const int64_t n_expert_shared = hparams.n_expert_shared;- const bool mtp_only = (hparams.n_layer_nextn > 0) && (ml.get_weight("blk.0.attn_norm.weight") == nullptr);- const std::string mtp_probe = "blk." + std::to_string(n_layer) + ".nextn.eh_proj.weight";- const bool trunk_only = (hparams.n_layer_nextn > 0) && (ml.get_weight(mtp_probe.c_str()) == nullptr);- const int trunk_flags = mtp_only ? TENSOR_NOT_REQUIRED : 0;- int mtp_flags = trunk_only ? TENSOR_NOT_REQUIRED : 0;-- if (!ml.load_mtp) {- mtp_flags |= TENSOR_SKIP;- }+ const auto nf = nextn_flags(ml);+ const int trunk_flags = nf.trunk;+ const int mtp_flags = nf.mtp;const bool is_mla = hparams.is_mla();diff --git a/src/models/deepseek32.cpp b/src/models/deepseek32.cppindex ade9b9ed9f..76f763c767 100644--- a/src/models/deepseek32.cpp+++ b/src/models/deepseek32.cpp@@ -46,15 +46,9 @@ void llama_model_deepseek32::load_arch_hparams(llama_model_loader & ml) {void llama_model_deepseek32::load_arch_tensors(llama_model_loader & ml) {LLAMA_LOAD_LOCALS;- const bool mtp_only = (hparams.n_layer_nextn > 0) && (ml.get_weight("blk.0.attn_norm.weight") == nullptr);- const std::string mtp_probe = "blk." + std::to_string(n_layer) + ".nextn.eh_proj.weight";- const bool trunk_only = (hparams.n_layer_nextn > 0) && (ml.get_weight(mtp_probe.c_str()) == nullptr);- const int trunk_flags = mtp_only ? TENSOR_NOT_REQUIRED : 0;- int mtp_flags = trunk_only ? TENSOR_NOT_REQUIRED : 0;-- if (!ml.load_mtp) {- mtp_flags |= TENSOR_SKIP;- }+ const auto nf = nextn_flags(ml);+ const int trunk_flags = nf.trunk;+ const int mtp_flags = nf.mtp;const bool is_mla = hparams.is_mla();if (!is_mla) {diff --git a/src/models/deepseek4.cpp b/src/models/deepseek4.cppindex 4edf0c2b46..0974b63b3c 100644--- a/src/models/deepseek4.cpp+++ b/src/models/deepseek4.cpp@@ -93,9 +93,9 @@ void llama_model_deepseek4::load_arch_tensors(llama_model_loader & ml) {const int64_t hc_dim = hc_mult * n_embd;const int64_t hc_mix_dim = (2 + hc_mult) * hc_mult;- const bool mtp_only = (n_layer_nextn > 0) && (ml.get_weight("blk.0.attn_norm.weight") == nullptr);- const int trunk_flags = mtp_only ? TENSOR_NOT_REQUIRED : 0;- const int mtp_flags = ml.load_mtp ? 0 : TENSOR_SKIP;+ const auto nf = nextn_flags(ml);+ const int trunk_flags = nf.trunk;+ const int mtp_flags = nf.mtp;tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, 0);diff --git a/src/models/glm-dsa.cpp b/src/models/glm-dsa.cppindex 32581f8403..3e28ef271c 100644--- a/src/models/glm-dsa.cpp+++ b/src/models/glm-dsa.cpp@@ -75,19 +75,9 @@ void llama_model_glm_dsa::load_arch_tensors(llama_model_loader & ml) {LLAMA_LOAD_LOCALS;const int64_t n_expert_shared = hparams.n_expert_shared;- // MTP-only: the GGUF carries only the NextN/MTP block(s) (user split target/draft).- const bool mtp_only = (hparams.n_layer_nextn > 0) && (ml.get_weight("blk.0.attn_norm.weight") == nullptr);- // Trunk-only: the GGUF declares MTP layers in metadata but the actual MTP- // tensors live in a separate file (or were stripped at conversion). Mark- // MTP tensors NOT_REQUIRED so the trunk loads cleanly.- const std::string mtp_probe = "blk." + std::to_string(n_layer) + ".nextn.eh_proj.weight";- const bool trunk_only = (hparams.n_layer_nextn > 0) && (ml.get_weight(mtp_probe.c_str()) == nullptr);- const int trunk_flags = mtp_only ? TENSOR_NOT_REQUIRED : 0;- int mtp_flags = trunk_only ? TENSOR_NOT_REQUIRED : 0;-- if (!ml.load_mtp) {- mtp_flags |= TENSOR_SKIP;- }+ const auto nf = nextn_flags(ml);+ const int trunk_flags = nf.trunk;+ const int mtp_flags = nf.mtp;const bool is_mla = hparams.is_mla();if (!is_mla) {diff --git a/src/models/glm4-moe.cpp b/src/models/glm4-moe.cppindex d65cace1fe..4b41b59580 100644--- a/src/models/glm4-moe.cpp+++ b/src/models/glm4-moe.cpp@@ -29,15 +29,9 @@ void llama_model_glm4_moe::load_arch_tensors(llama_model_loader & ml) {LLAMA_LOAD_LOCALS;const int64_t n_expert_shared = hparams.n_expert_shared;- const bool mtp_only = (hparams.n_layer_nextn > 0) && (ml.get_weight("blk.0.attn_norm.weight") == nullptr);- const std::string mtp_probe = "blk." + std::to_string(n_layer) + ".nextn.eh_proj.weight";- const bool trunk_only = (hparams.n_layer_nextn > 0) && (ml.get_weight(mtp_probe.c_str()) == nullptr);- const int trunk_flags = mtp_only ? TENSOR_NOT_REQUIRED : 0;- int mtp_flags = trunk_only ? TENSOR_NOT_REQUIRED : 0;-- if (!ml.load_mtp) {- mtp_flags |= TENSOR_SKIP;- }+ const auto nf = nextn_flags(ml);+ const int trunk_flags = nf.trunk;+ const int mtp_flags = nf.mtp;GGML_ASSERT(hparams.n_expert > 0 && "n_expert must be > 0 for GLM4_MOE MoE layers");GGML_ASSERT(hparams.n_expert_used() > 0 && "n_expert_used must be > 0 for GLM4_MOE MoE layers");diff --git a/src/models/glm5-next.cpp b/src/models/glm5-next.cppindex 4343e6ac67..48b07f1e25 100644--- a/src/models/glm5-next.cpp+++ b/src/models/glm5-next.cpp@@ -64,16 +64,9 @@ void llama_model_glm5_next::load_arch_tensors(llama_model_loader & ml) {const int64_t hc = hparams.dsv4_hc_mult;const int64_t hc_mix_dim = (2 + hc)*hc;- // TODO: consolidate the trunk_only/mtp_only detection and flags shared with glm4-moe and qwen35moe- const bool mtp_only = (hparams.n_layer_nextn > 0) && (ml.get_weight("blk.0.attn_norm.weight") == nullptr);- const std::string mtp_probe = "blk." + std::to_string(n_layer) + ".nextn.eh_proj.weight";- const bool trunk_only = (hparams.n_layer_nextn > 0) && (ml.get_weight(mtp_probe.c_str()) == nullptr);- const int trunk_flags = mtp_only ? TENSOR_NOT_REQUIRED : 0;- int mtp_flags = trunk_only ? TENSOR_NOT_REQUIRED : 0;-- if (!ml.load_mtp) {- mtp_flags |= TENSOR_SKIP;- }+ const auto nf = nextn_flags(ml);+ const int trunk_flags = nf.trunk;+ const int mtp_flags = nf.mtp;tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, 0);diff --git a/src/models/hy-v3.cpp b/src/models/hy-v3.cppindex bb0e0a2b8a..c4c55c3f9b 100644--- a/src/models/hy-v3.cpp+++ b/src/models/hy-v3.cpp@@ -22,18 +22,9 @@ void llama_model_hy_v3::load_arch_hparams(llama_model_loader & ml) {void llama_model_hy_v3::load_arch_tensors(llama_model_loader & ml) {LLAMA_LOAD_LOCALS;- const bool mtp_only = (hparams.n_layer_nextn > 0) && (ml.get_weight("blk.0.attn_norm.weight") == nullptr);- // Trunk-only: the GGUF declares MTP layers in metadata but the actual MTP- // tensors live in a separate file (e.g. user split target/draft). Mark- // MTP tensors NOT_REQUIRED so the trunk loads cleanly.- const std::string mtp_probe = "blk." + std::to_string(n_layer) + ".nextn.eh_proj.weight";- const bool trunk_only = (hparams.n_layer_nextn > 0) && (ml.get_weight(mtp_probe.c_str()) == nullptr);- const int trunk_flags = mtp_only ? TENSOR_NOT_REQUIRED : 0;- int mtp_flags = trunk_only ? TENSOR_NOT_REQUIRED : 0;-- if (!ml.load_mtp) {- mtp_flags |= TENSOR_SKIP;- }+ const auto nf = nextn_flags(ml);+ const int trunk_flags = nf.trunk;+ const int mtp_flags = nf.mtp;tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, 0);diff --git a/src/models/mimo2.cpp b/src/models/mimo2.cppindex 866959c839..a70350191b 100644--- a/src/models/mimo2.cpp+++ b/src/models/mimo2.cpp@@ -25,15 +25,9 @@ void llama_model_mimo2::load_arch_hparams(llama_model_loader & ml) {void llama_model_mimo2::load_arch_tensors(llama_model_loader & ml) {LLAMA_LOAD_LOCALS;- const bool mtp_only = (hparams.n_layer_nextn > 0) && (ml.get_weight("blk.0.attn_norm.weight") == nullptr);- const std::string mtp_probe = "blk." + std::to_string(n_layer) + ".nextn.eh_proj.weight";- const bool trunk_only = (hparams.n_layer_nextn > 0) && (ml.get_weight(mtp_probe.c_str()) == nullptr);- const int trunk_flags = mtp_only ? TENSOR_NOT_REQUIRED : 0;- int mtp_flags = trunk_only ? TENSOR_NOT_REQUIRED : 0;-- if (!ml.load_mtp) {- mtp_flags |= TENSOR_SKIP;- }+ const auto nf = nextn_flags(ml);+ const int trunk_flags = nf.trunk;+ const int mtp_flags = nf.mtp;tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, 0);diff --git a/src/models/nemotron-h.cpp b/src/models/nemotron-h.cppindex cd90e24d2c..cf78ebfbe3 100644--- a/src/models/nemotron-h.cpp+++ b/src/models/nemotron-h.cpp@@ -49,9 +49,9 @@ void llama_model_nemotron_h::load_arch_hparams(llama_model_loader & ml) {void llama_model_nemotron_h::load_arch_tensors(llama_model_loader & ml) {LLAMA_LOAD_LOCALS;- const bool mtp_only = hparams.n_layer_nextn > 0 && ml.get_weight("blk.0.attn_norm.weight") == nullptr;- const int trunk_flags = mtp_only ? TENSOR_NOT_REQUIRED : 0;- const int mtp_flags = !ml.load_mtp ? TENSOR_SKIP : 0;+ const auto nf = nextn_flags(ml);+ const int trunk_flags = nf.trunk;+ const int mtp_flags = nf.mtp;// mamba2 Mixer SSM params// NOTE: int64_t for tensor dimensionsdiff --git a/src/models/qwen35.cpp b/src/models/qwen35.cppindex e484850dad..350cea8083 100644--- a/src/models/qwen35.cpp+++ b/src/models/qwen35.cpp@@ -33,9 +33,9 @@ void llama_model_qwen35::load_arch_hparams(llama_model_loader & ml) {void llama_model_qwen35::load_arch_tensors(llama_model_loader & ml) {LLAMA_LOAD_LOCALS;- const bool mtp_only = (hparams.n_layer_nextn > 0) && (ml.get_weight("blk.0.attn_norm.weight") == nullptr);- const int trunk_flags = mtp_only ? TENSOR_NOT_REQUIRED : 0;- int mtp_flags = !ml.load_mtp ? TENSOR_SKIP : 0;+ const auto nf = nextn_flags(ml);+ const int trunk_flags = nf.trunk;+ const int mtp_flags = nf.mtp;tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), { n_embd, n_vocab }, 0);diff --git a/src/models/qwen35moe.cpp b/src/models/qwen35moe.cppindex 6e28028a89..f9cc8a2c6c 100644--- a/src/models/qwen35moe.cpp+++ b/src/models/qwen35moe.cpp@@ -36,9 +36,9 @@ void llama_model_qwen35moe::load_arch_hparams(llama_model_loader & ml) {void llama_model_qwen35moe::load_arch_tensors(llama_model_loader & ml) {LLAMA_LOAD_LOCALS;- const bool mtp_only = (hparams.n_layer_nextn > 0) && (ml.get_weight("blk.0.attn_norm.weight") == nullptr);- const int trunk_flags = mtp_only ? TENSOR_NOT_REQUIRED : 0;- int mtp_flags = !ml.load_mtp ? TENSOR_SKIP : 0;+ const auto nf = nextn_flags(ml);+ const int trunk_flags = nf.trunk;+ const int mtp_flags = nf.mtp;tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), { n_embd, n_vocab }, 0);diff --git a/src/models/qwen3next.cpp b/src/models/qwen3next.cppindex 2d56c38ca1..620161ac73 100644--- a/src/models/qwen3next.cpp+++ b/src/models/qwen3next.cpp@@ -35,9 +35,9 @@ void llama_model_qwen3next::load_arch_tensors(llama_model_loader & ml) {throw std::runtime_error(arch_name() + " model cannot have zero experts");}- const bool mtp_only = (hparams.n_layer_nextn > 0) && (ml.get_weight("blk.0.attn_norm.weight") == nullptr);- const int trunk_flags = mtp_only ? TENSOR_NOT_REQUIRED : 0;- int mtp_flags = !ml.load_mtp ? TENSOR_SKIP : 0;+ const auto nf = nextn_flags(ml);+ const int trunk_flags = nf.trunk;+ const int mtp_flags = nf.mtp;tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), { n_embd, n_vocab }, 0);diff --git a/src/models/qwen4exp.cpp b/src/models/qwen4exp.cppindex d05f72fc3c..88e2e88914 100644--- a/src/models/qwen4exp.cpp+++ b/src/models/qwen4exp.cpp@@ -175,10 +175,9 @@ void llama_model_qwen4exp::load_arch_tensors(llama_model_loader & ml) {const int64_t hc_dim = hc * n_embd;const int64_t hc_lr = hparams.hc_low_rank;- // an MTP-only file carries the MTP block, the embeddings and the LM head, but no trunk- const bool mtp_only = n_layer_nextn > 0 && ml.get_weight(tn(LLM_TENSOR_HC_ATTN_NORM, "weight", 0).str().c_str()) == nullptr;- const int trunk_flags = mtp_only ? TENSOR_NOT_REQUIRED : 0;- const int mtp_flags = ml.load_mtp ? 0 : TENSOR_SKIP;+ const auto nf = nextn_flags(ml, LLM_TENSOR_HC_ATTN_NORM);+ const int trunk_flags = nf.trunk;+ const int mtp_flags = nf.mtp;tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), { n_embd, n_vocab }, 0);diff --git a/src/models/step35.cpp b/src/models/step35.cppindex f4660bf196..bfa80fab3f 100644--- a/src/models/step35.cpp+++ b/src/models/step35.cpp@@ -37,18 +37,9 @@ void llama_model_step35::load_arch_hparams(llama_model_loader & ml) {void llama_model_step35::load_arch_tensors(llama_model_loader & ml) {LLAMA_LOAD_LOCALS;- const bool mtp_only = (hparams.n_layer_nextn > 0) && (ml.get_weight("blk.0.attn_norm.weight") == nullptr);- // Trunk-only: the GGUF declares MTP layers in metadata but the actual MTP- // tensors live in a separate file (e.g. user split target/draft). Mark- // MTP tensors NOT_REQUIRED so the trunk loads cleanly.- const std::string mtp_probe = "blk." + std::to_string(n_layer) + ".nextn.eh_proj.weight";- const bool trunk_only = (hparams.n_layer_nextn > 0) && (ml.get_weight(mtp_probe.c_str()) == nullptr);- const int trunk_flags = mtp_only ? TENSOR_NOT_REQUIRED : 0;- int mtp_flags = trunk_only ? TENSOR_NOT_REQUIRED : 0;-- if (!ml.load_mtp) {- mtp_flags |= TENSOR_SKIP;- }+ const auto nf = nextn_flags(ml);+ const int trunk_flags = nf.trunk;+ const int mtp_flags = nf.mtp;tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, 0);