Skip to content

LLM inference in C/C++

C++ ggml

model : load hparams.n_layer_nextn before n_layer() calls (#28159)

Sigbjørn Skjæret9d817213a097
diff --git a/src/llama-model.cpp b/src/llama-model.cpp
index e679b24e87..ec91b7c6d0 100644
--- a/src/llama-model.cpp
+++ b/src/llama-model.cpp
@@ -1223,6 +1223,8 @@ void llama_model_base::load_hparams(llama_model_loader & ml) {
ml.get_key(LLM_KV_POOLING_TYPE, hparams.pooling_type, false);
ml.get_key(LLM_KV_BLOCK_COUNT, hparams.n_layer_all);
GGML_ASSERT(hparams.n_layer_all > 0 && hparams.n_layer_all <= LLAMA_MAX_LAYERS);
+ ml.get_key(LLM_KV_NEXTN_PREDICT_LAYERS, hparams.n_layer_nextn, false);
+ GGML_ASSERT(hparams.n_layer_nextn < hparams.n_layer_all);
ml.get_key(LLM_KV_EXPERT_COUNT, hparams.n_expert, false);
ml.get_key(LLM_KV_EXPERT_USED_COUNT, hparams.n_expert_used, false);
ml.get_key(LLM_KV_EXPERT_GROUP_COUNT, hparams.n_expert_groups, false);
diff --git a/src/models/bailingmoe2.cpp b/src/models/bailingmoe2.cpp
index 5000e9c6db..8fc0ea752b 100644
--- a/src/models/bailingmoe2.cpp
+++ b/src/models/bailingmoe2.cpp
@@ -9,9 +9,6 @@ void llama_model_bailingmoe2::load_arch_hparams(llama_model_loader & ml) {
ml.get_key(LLM_KV_EXPERT_WEIGHTS_SCALE, hparams.expert_weights_scale, false);
ml.get_key(LLM_KV_EXPERT_WEIGHTS_NORM, hparams.expert_weights_norm, false);
ml.get_key(LLM_KV_EXPERT_GATING_FUNC, hparams.expert_gating_func);
- ml.get_key(LLM_KV_NEXTN_PREDICT_LAYERS, hparams.n_layer_nextn, false);
-
- GGML_ASSERT(hparams.n_layer_nextn < hparams.n_layer_all && "n_layer_nextn must be < n_layer_impl");
switch (hparams.n_layer()) {
case 20: type = LLM_TYPE_16B_A1B; break;
diff --git a/src/models/bailingmoe3.cpp b/src/models/bailingmoe3.cpp
index 0637931cc0..1583d9be34 100644
--- a/src/models/bailingmoe3.cpp
+++ b/src/models/bailingmoe3.cpp
@@ -22,7 +22,6 @@ void llama_model_bailingmoe3::load_arch_hparams(llama_model_loader & ml) {
ml.get_key(LLM_KV_EXPERT_WEIGHTS_SCALE, hparams.expert_weights_scale, false);
ml.get_key(LLM_KV_EXPERT_WEIGHTS_NORM, hparams.expert_weights_norm, false);
ml.get_key(LLM_KV_EXPERT_GATING_FUNC, hparams.expert_gating_func);
- ml.get_key(LLM_KV_NEXTN_PREDICT_LAYERS, hparams.n_layer_nextn, false);
ml.get_key_or_arr(LLM_KV_SWIGLU_CLAMP_EXP, hparams.swiglu_clamp_exp, hparams.n_layer_all, false);
ml.get_key_or_arr(LLM_KV_SWIGLU_CLAMP_SHEXP, hparams.swiglu_clamp_shexp, hparams.n_layer_all, false);
diff --git a/src/models/cohere2moe.cpp b/src/models/cohere2moe.cpp
index 3acb7e77af..c50910edc5 100644
--- a/src/models/cohere2moe.cpp
+++ b/src/models/cohere2moe.cpp
@@ -20,9 +20,6 @@ void llama_model_cohere2moe::load_arch_hparams(llama_model_loader & ml) {
ml.get_key(LLM_KV_EXPERT_WEIGHTS_SCALE, hparams.expert_weights_scale, false);
ml.get_key(LLM_KV_EXPERT_GATING_FUNC, hparams.expert_gating_func, false);
- ml.get_key(LLM_KV_NEXTN_PREDICT_LAYERS, hparams.n_layer_nextn, false);
- GGML_ASSERT(hparams.n_layer_nextn < hparams.n_layer_all && "n_layer_nextn must be < n_layer");
-
if (hparams.expert_gating_func == LLAMA_EXPERT_GATING_FUNC_TYPE_NONE) {
hparams.expert_gating_func = LLAMA_EXPERT_GATING_FUNC_TYPE_SIGMOID;
}
diff --git a/src/models/deepseek2.cpp b/src/models/deepseek2.cpp
index e0e537e005..3a76187aa5 100644
--- a/src/models/deepseek2.cpp
+++ b/src/models/deepseek2.cpp
@@ -37,11 +37,6 @@ void llama_model_deepseek2::load_arch_hparams(llama_model_loader & ml) {
hparams.rope_yarn_log_mul /= 0.1f;
}
- // NextN/MTP
- ml.get_key(LLM_KV_NEXTN_PREDICT_LAYERS, hparams.n_layer_nextn, false);
- GGML_ASSERT(hparams.n_layer_nextn == 0 ||
- hparams.n_layer() + hparams.n_layer_nextn == hparams.n_layer_all);
-
// (optional) temperature tuning - used by mistral-large
ml.get_key(LLM_KV_ATTENTION_TEMPERATURE_SCALE, hparams.f_attn_temp_scale, false);
ml.get_key(LLM_KV_ATTENTION_TEMPERATURE_LENGTH, hparams.n_attn_temp_floor_scale, false); // FIXME why not use temperature_length?
diff --git a/src/models/deepseek32.cpp b/src/models/deepseek32.cpp
index 2b82a780c4..079bdfc302 100644
--- a/src/models/deepseek32.cpp
+++ b/src/models/deepseek32.cpp
@@ -37,10 +37,6 @@ void llama_model_deepseek32::load_arch_hparams(llama_model_loader & ml) {
hparams.rope_yarn_log_mul /= 0.1f;
}
- // NextN/MTP parameters
- ml.get_key(LLM_KV_NEXTN_PREDICT_LAYERS, hparams.n_layer_nextn, false);
- GGML_ASSERT(hparams.n_layer_nextn < hparams.n_layer_all && "n_layer_nextn must be < n_layer");
-
switch (hparams.n_layer()) {
case 61: type = LLM_TYPE_685B_A37B; break;
default: type = LLM_TYPE_UNKNOWN;
diff --git a/src/models/deepseek4.cpp b/src/models/deepseek4.cpp
index fc816e2aeb..0157ce7051 100644
--- a/src/models/deepseek4.cpp
+++ b/src/models/deepseek4.cpp
@@ -17,15 +17,13 @@ static float dsv4_rope_attn_factor(float freq_scale, float ext_factor) {
}
void llama_model_deepseek4::load_arch_hparams(llama_model_loader & ml) {
- ml.get_key(LLM_KV_NEXTN_PREDICT_LAYERS, hparams.n_layer_nextn, false);
- if (hparams.n_layer_nextn > 0 && hparams.n_layer_nextn < hparams.n_layer_all) {
+ if (hparams.n_layer_nextn > 0) {
const uint32_t n_layer_main = hparams.n_layer_all - hparams.n_layer_nextn;
const std::string mtp_probe = "blk." + std::to_string(n_layer_main) + ".nextn.eh_proj.weight";
if (ml.get_weight(mtp_probe.c_str()) == nullptr) {
hparams.n_layer_nextn = 0;
}
}
- GGML_ASSERT(hparams.n_layer_nextn < hparams.n_layer_all && "n_layer_nextn must be < block_count");
ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);
ml.get_key(LLM_KV_ATTENTION_Q_LORA_RANK, hparams.n_lora_q);
diff --git a/src/models/dots3note.cpp b/src/models/dots3note.cpp
index 00a008c2c9..7656562b05 100644
--- a/src/models/dots3note.cpp
+++ b/src/models/dots3note.cpp
@@ -9,10 +9,6 @@ void llama_model_dots3note::load_arch_hparams(llama_model_loader & ml) {
ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);
hparams.f_norm_eps = 1e-6; // eps for the indexer k_norm layer norm
- // TODO: use MTP layer
- ml.get_key(LLM_KV_NEXTN_PREDICT_LAYERS, hparams.n_layer_nextn, false);
- GGML_ASSERT(hparams.n_layer_nextn < hparams.n_layer_all && "n_layer_nextn must be < n_layer_all");
-
// MoE parameters
ml.get_key(LLM_KV_EXPERT_SHARED_COUNT, hparams.n_expert_shared);
ml.get_key(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp);
diff --git a/src/models/exaone-moe.cpp b/src/models/exaone-moe.cpp
index 5aed937940..86e5a3a984 100644
--- a/src/models/exaone-moe.cpp
+++ b/src/models/exaone-moe.cpp
@@ -20,9 +20,6 @@ void llama_model_exaone_moe::load_arch_hparams(llama_model_loader & ml) {
ml.get_key(LLM_KV_EXPERT_WEIGHTS_NORM, hparams.expert_weights_norm, false);
ml.get_key(LLM_KV_LEADING_DENSE_BLOCK_COUNT, hparams.n_layer_dense_lead, false);
- ml.get_key(LLM_KV_NEXTN_PREDICT_LAYERS, hparams.n_layer_nextn, false);
- GGML_ASSERT(hparams.n_layer_nextn < hparams.n_layer_all && "n_layer_nextn must be < n_layer_impl");
-
switch (hparams.n_layer()) {
case 32: type = LLM_TYPE_30B_A3B; break;
case 48: type = LLM_TYPE_235B_A22B; break;
diff --git a/src/models/exaone4.cpp b/src/models/exaone4.cpp
index a06819a67c..9ba978956d 100644
--- a/src/models/exaone4.cpp
+++ b/src/models/exaone4.cpp
@@ -1,9 +1,6 @@
#include "models.h"
void llama_model_exaone4::load_arch_hparams(llama_model_loader & ml) {
- ml.get_key(LLM_KV_NEXTN_PREDICT_LAYERS, hparams.n_layer_nextn, false);
- GGML_ASSERT(hparams.n_layer_nextn < hparams.n_layer_all && "n_layer_nextn must be < n_layer");
-
if (hparams.n_layer() == 64) { // 32B
hparams.swa_type = LLAMA_SWA_TYPE_STANDARD;
hparams.n_swa = 4096;
diff --git a/src/models/gemma4-assistant.cpp b/src/models/gemma4-assistant.cpp
index 6378130e79..989fa42c85 100644
--- a/src/models/gemma4-assistant.cpp
+++ b/src/models/gemma4-assistant.cpp
@@ -11,9 +11,6 @@ void llama_model_gemma4_assistant::load_arch_hparams(llama_model_loader & ml) {
hparams.f_attention_scale = 1.0f;
- ml.get_key(LLM_KV_NEXTN_PREDICT_LAYERS, hparams.n_layer_nextn, false);
- GGML_ASSERT(hparams.n_layer_nextn == hparams.n_layer_all && "n_layer_nextn must be == n_layer_impl");
-
ml.get_key(LLM_KV_ROPE_FREQ_BASE_SWA, hparams.rope_freq_base_train_swa, false);
ml.get_key(LLM_KV_ATTENTION_SLIDING_WINDOW, hparams.n_swa);
ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);
diff --git a/src/models/glm-dsa.cpp b/src/models/glm-dsa.cpp
index 93a1448b46..543b15cf35 100644
--- a/src/models/glm-dsa.cpp
+++ b/src/models/glm-dsa.cpp
@@ -56,10 +56,6 @@ void llama_model_glm_dsa::load_arch_hparams(llama_model_loader & ml) {
hparams.expert_gating_func = LLAMA_EXPERT_GATING_FUNC_TYPE_SIGMOID;
}
- // NextN/MTP parameters
- ml.get_key(LLM_KV_NEXTN_PREDICT_LAYERS, hparams.n_layer_nextn, false);
- GGML_ASSERT(hparams.n_layer_nextn < hparams.n_layer_all && "n_layer_nextn must be < n_layer_all");
-
// BC for GLM 5, 5.1 (full indexers) without indexer_types metadata
const bool is_pre_5_2 = hparams.n_ctx_train < 1048576;
if (is_pre_5_2) {
@@ -70,9 +66,7 @@ void llama_model_glm_dsa::load_arch_hparams(llama_model_loader & ml) {
ml.get_key_or_arr(LLM_KV_ATTENTION_INDEXER_TYPES, hparams.is_indexer_full_impl, hparams.n_layer(), false);
switch (hparams.n_layer()) {
- case 78: // GGUF with NextN/MTP metadata: n_layer() excludes the nextn layer
- case 79:
- type = LLM_TYPE_744B_A40B; break;
+ case 78: type = LLM_TYPE_744B_A40B; break;
default: type = LLM_TYPE_UNKNOWN;
}
}
diff --git a/src/models/glm4-moe.cpp b/src/models/glm4-moe.cpp
index 83ea7f8ac6..1d2ac65fd7 100644
--- a/src/models/glm4-moe.cpp
+++ b/src/models/glm4-moe.cpp
@@ -17,10 +17,6 @@ void llama_model_glm4_moe::load_arch_hparams(llama_model_loader & ml) {
hparams.expert_gating_func = LLAMA_EXPERT_GATING_FUNC_TYPE_SIGMOID;
}
- // NextN/MTP parameters
- ml.get_key(LLM_KV_NEXTN_PREDICT_LAYERS, hparams.n_layer_nextn, false);
- GGML_ASSERT(hparams.n_layer_nextn < hparams.n_layer_all && "n_layer_nextn must be < n_layer_impl");
-
switch (hparams.n_layer()) {
case 46: type = LLM_TYPE_106B_A12B; break; // GLM-4.5-Air
case 48: type = LLM_TYPE_102B_A12B; break; // Solar Open
diff --git a/src/models/glm4.cpp b/src/models/glm4.cpp
index b4326c5f21..463be809d8 100644
--- a/src/models/glm4.cpp
+++ b/src/models/glm4.cpp
@@ -4,10 +4,6 @@ void llama_model_glm4::load_arch_hparams(llama_model_loader & ml) {
ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);
ml.get_key_or_arr(LLM_KV_ROPE_DIMENSION_SECTIONS, hparams.rope_sections, 4, false);
- // NextN/MTP parameters (GLM-OCR)
- ml.get_key(LLM_KV_NEXTN_PREDICT_LAYERS, hparams.n_layer_nextn, false);
- GGML_ASSERT(hparams.n_layer_nextn < hparams.n_layer_all && "n_layer_nextn must be < n_layer_impl");
-
switch (hparams.n_layer()) {
case 17: type = LLM_TYPE_1B; break; // GLM-OCR
case 40: type = LLM_TYPE_9B; break;
diff --git a/src/models/hy-v3.cpp b/src/models/hy-v3.cpp
index 61db93af85..3c45331b11 100644
--- a/src/models/hy-v3.cpp
+++ b/src/models/hy-v3.cpp
@@ -13,10 +13,6 @@ void llama_model_hy_v3::load_arch_hparams(llama_model_loader & ml) {
hparams.expert_gating_func = LLAMA_EXPERT_GATING_FUNC_TYPE_SIGMOID;
}
- // NextN/MTP (HY V3): extra decoder block(s) appended beyond the main stack
- ml.get_key(LLM_KV_NEXTN_PREDICT_LAYERS, hparams.n_layer_nextn, false);
- GGML_ASSERT(hparams.n_layer_nextn < hparams.n_layer_all && "n_layer_nextn must be < n_layer_all");
-
switch (hparams.n_layer()) {
case 48: type = LLM_TYPE_30B_A3B; break;
default: type = LLM_TYPE_UNKNOWN;
diff --git a/src/models/mimo2.cpp b/src/models/mimo2.cpp
index d50e186cce..1dc554220d 100644
--- a/src/models/mimo2.cpp
+++ b/src/models/mimo2.cpp
@@ -16,9 +16,6 @@ void llama_model_mimo2::load_arch_hparams(llama_model_loader & ml) {
hparams.f_attn_value_scale = value_scale;
}
- ml.get_key(LLM_KV_NEXTN_PREDICT_LAYERS, hparams.n_layer_nextn, false);
- GGML_ASSERT(hparams.n_layer_nextn < hparams.n_layer_all && "n_layer_nextn must be < n_layer_impl");
-
switch (hparams.n_layer()) {
case 48: type = LLM_TYPE_310B_A15B; break;
default: type = LLM_TYPE_UNKNOWN;
diff --git a/src/models/nemotron-h.cpp b/src/models/nemotron-h.cpp
index f02674c646..55640c996a 100644
--- a/src/models/nemotron-h.cpp
+++ b/src/models/nemotron-h.cpp
@@ -7,10 +7,6 @@ void llama_model_nemotron_h::load_arch_hparams(llama_model_loader & ml) {
ml.get_key(LLM_KV_SSM_TIME_STEP_RANK, hparams.ssm_dt_rank);
ml.get_key(LLM_KV_SSM_GROUP_COUNT, hparams.ssm_n_group);
- // NextN/MTP: optional draft head appended as extra trailing block(s)
- ml.get_key(LLM_KV_NEXTN_PREDICT_LAYERS, hparams.n_layer_nextn, false);
- GGML_ASSERT(hparams.n_layer_nextn < hparams.n_layer_all && "n_layer_nextn must be < n_layer_all");
-
// A layer is recurrent IFF the n_head_kv value is set to 0 and
// the n_ff value is set to 0. Appended MTP blocks are dense (non-recurrent)
for (uint32_t i = 0; i < hparams.n_layer_all; ++i) {
diff --git a/src/models/qwen35.cpp b/src/models/qwen35.cpp
index 309dd43244..0b9210981d 100644
--- a/src/models/qwen35.cpp
+++ b/src/models/qwen35.cpp
@@ -12,10 +12,6 @@ void llama_model_qwen35::load_arch_hparams(llama_model_loader & ml) {
ml.get_key(LLM_KV_SSM_TIME_STEP_RANK, hparams.ssm_dt_rank);
ml.get_key(LLM_KV_SSM_GROUP_COUNT, hparams.ssm_n_group);
- // NextN/MTP (Qwen3.5/3.6): extra decoder block appended beyond the main stack
- ml.get_key(LLM_KV_NEXTN_PREDICT_LAYERS, hparams.n_layer_nextn, false);
- GGML_ASSERT(hparams.n_layer_nextn < hparams.n_layer_all && "n_layer_nextn must be < n_layer_impl");
-
// Mark recurrent layers (linear attention layers). MTP layers are dense
// attention-only and must be flagged non-recurrent.
if (!ml.get_key_or_arr(LLM_KV_ATTENTION_RECURRENT_LAYERS, hparams.is_recr_impl, hparams.n_layer_all, false)) {
diff --git a/src/models/qwen35moe.cpp b/src/models/qwen35moe.cpp
index 38f2a57985..9bf4ea4328 100644
--- a/src/models/qwen35moe.cpp
+++ b/src/models/qwen35moe.cpp
@@ -15,10 +15,6 @@ void llama_model_qwen35moe::load_arch_hparams(llama_model_loader & ml) {
ml.get_key(LLM_KV_SSM_TIME_STEP_RANK, hparams.ssm_dt_rank);
ml.get_key(LLM_KV_SSM_GROUP_COUNT, hparams.ssm_n_group);
- // NextN/MTP (Qwen3.5/3.6): extra decoder block appended beyond the main stack
- ml.get_key(LLM_KV_NEXTN_PREDICT_LAYERS, hparams.n_layer_nextn, false);
- GGML_ASSERT(hparams.n_layer_nextn < hparams.n_layer_all && "n_layer_nextn must be < n_layer_impl");
-
// Mark recurrent layers (linear attention layers). MTP layers are dense
// attention-only and must be flagged non-recurrent.
if (!ml.get_key_or_arr(LLM_KV_ATTENTION_RECURRENT_LAYERS, hparams.is_recr_impl, hparams.n_layer_all, false)) {
diff --git a/src/models/qwen3next.cpp b/src/models/qwen3next.cpp
index 0808fd87aa..b2b8809c7f 100644
--- a/src/models/qwen3next.cpp
+++ b/src/models/qwen3next.cpp
@@ -13,10 +13,6 @@ void llama_model_qwen3next::load_arch_hparams(llama_model_loader & ml) {
ml.get_key(LLM_KV_SSM_TIME_STEP_RANK, hparams.ssm_dt_rank);
ml.get_key(LLM_KV_SSM_GROUP_COUNT, hparams.ssm_n_group);
- // NextN/MTP: extra decoder block appended beyond the main stack
- ml.get_key(LLM_KV_NEXTN_PREDICT_LAYERS, hparams.n_layer_nextn, false);
- GGML_ASSERT(hparams.n_layer_nextn < hparams.n_layer_all && "n_layer_nextn must be < n_layer_all");
-
// Mark recurrent layers (linear attention layers).
if (!ml.get_key_or_arr(LLM_KV_ATTENTION_RECURRENT_LAYERS, hparams.is_recr_impl, hparams.n_layer_all, false)) {
uint32_t full_attn_interval = 4;
diff --git a/src/models/step35.cpp b/src/models/step35.cpp
index 5b1d902581..d101d115ef 100644
--- a/src/models/step35.cpp
+++ b/src/models/step35.cpp
@@ -23,14 +23,10 @@ void llama_model_step35::load_arch_hparams(llama_model_loader & ml) {
ml.get_key(LLM_KV_ATTENTION_SLIDING_WINDOW, hparams.n_swa);
ml.get_key(LLM_KV_ROPE_FREQ_BASE_SWA, hparams.rope_freq_base_train_swa, false);
- ml.get_key_or_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, hparams.is_swa_impl, hparams.n_layer());
+ ml.get_key_or_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, hparams.is_swa_impl, hparams.n_layer_all);
- ml.get_key_or_arr(LLM_KV_SWIGLU_CLAMP_EXP, hparams.swiglu_clamp_exp, hparams.n_layer(), false);
- ml.get_key_or_arr(LLM_KV_SWIGLU_CLAMP_SHEXP, hparams.swiglu_clamp_shexp, hparams.n_layer(), false);
-
- // NextN/MTP (Step3p5): extra decoder block appended beyond the main stack.
- ml.get_key(LLM_KV_NEXTN_PREDICT_LAYERS, hparams.n_layer_nextn, false);
- GGML_ASSERT(hparams.n_layer_nextn < hparams.n_layer_all && "n_layer_nextn must be < n_layer_impl");
+ ml.get_key_or_arr(LLM_KV_SWIGLU_CLAMP_EXP, hparams.swiglu_clamp_exp, hparams.n_layer_all, false);
+ ml.get_key_or_arr(LLM_KV_SWIGLU_CLAMP_SHEXP, hparams.swiglu_clamp_shexp, hparams.n_layer_all, false);
switch (hparams.n_layer()) {
case 45: type = LLM_TYPE_196B_A11B; break;