LLM inference in C/C++
C++ ggml
model : load hparams.n_layer_nextn before n_layer() calls (#28159)
Sigbjørn Skjæret9d817213a097
diff --git a/src/llama-model.cpp b/src/llama-model.cppindex e679b24e87..ec91b7c6d0 100644--- a/src/llama-model.cpp+++ b/src/llama-model.cpp@@ -1223,6 +1223,8 @@ void llama_model_base::load_hparams(llama_model_loader & ml) {ml.get_key(LLM_KV_POOLING_TYPE, hparams.pooling_type, false);ml.get_key(LLM_KV_BLOCK_COUNT, hparams.n_layer_all);GGML_ASSERT(hparams.n_layer_all > 0 && hparams.n_layer_all <= LLAMA_MAX_LAYERS);+ ml.get_key(LLM_KV_NEXTN_PREDICT_LAYERS, hparams.n_layer_nextn, false);+ GGML_ASSERT(hparams.n_layer_nextn < hparams.n_layer_all);ml.get_key(LLM_KV_EXPERT_COUNT, hparams.n_expert, false);ml.get_key(LLM_KV_EXPERT_USED_COUNT, hparams.n_expert_used, false);ml.get_key(LLM_KV_EXPERT_GROUP_COUNT, hparams.n_expert_groups, false);diff --git a/src/models/bailingmoe2.cpp b/src/models/bailingmoe2.cppindex 5000e9c6db..8fc0ea752b 100644--- a/src/models/bailingmoe2.cpp+++ b/src/models/bailingmoe2.cpp@@ -9,9 +9,6 @@ void llama_model_bailingmoe2::load_arch_hparams(llama_model_loader & ml) {ml.get_key(LLM_KV_EXPERT_WEIGHTS_SCALE, hparams.expert_weights_scale, false);ml.get_key(LLM_KV_EXPERT_WEIGHTS_NORM, hparams.expert_weights_norm, false);ml.get_key(LLM_KV_EXPERT_GATING_FUNC, hparams.expert_gating_func);- ml.get_key(LLM_KV_NEXTN_PREDICT_LAYERS, hparams.n_layer_nextn, false);-- GGML_ASSERT(hparams.n_layer_nextn < hparams.n_layer_all && "n_layer_nextn must be < n_layer_impl");switch (hparams.n_layer()) {case 20: type = LLM_TYPE_16B_A1B; break;diff --git a/src/models/bailingmoe3.cpp b/src/models/bailingmoe3.cppindex 0637931cc0..1583d9be34 100644--- a/src/models/bailingmoe3.cpp+++ b/src/models/bailingmoe3.cpp@@ -22,7 +22,6 @@ void llama_model_bailingmoe3::load_arch_hparams(llama_model_loader & ml) {ml.get_key(LLM_KV_EXPERT_WEIGHTS_SCALE, hparams.expert_weights_scale, false);ml.get_key(LLM_KV_EXPERT_WEIGHTS_NORM, hparams.expert_weights_norm, false);ml.get_key(LLM_KV_EXPERT_GATING_FUNC, hparams.expert_gating_func);- ml.get_key(LLM_KV_NEXTN_PREDICT_LAYERS, hparams.n_layer_nextn, false);ml.get_key_or_arr(LLM_KV_SWIGLU_CLAMP_EXP, hparams.swiglu_clamp_exp, hparams.n_layer_all, false);ml.get_key_or_arr(LLM_KV_SWIGLU_CLAMP_SHEXP, hparams.swiglu_clamp_shexp, hparams.n_layer_all, false);diff --git a/src/models/cohere2moe.cpp b/src/models/cohere2moe.cppindex 3acb7e77af..c50910edc5 100644--- a/src/models/cohere2moe.cpp+++ b/src/models/cohere2moe.cpp@@ -20,9 +20,6 @@ void llama_model_cohere2moe::load_arch_hparams(llama_model_loader & ml) {ml.get_key(LLM_KV_EXPERT_WEIGHTS_SCALE, hparams.expert_weights_scale, false);ml.get_key(LLM_KV_EXPERT_GATING_FUNC, hparams.expert_gating_func, false);- ml.get_key(LLM_KV_NEXTN_PREDICT_LAYERS, hparams.n_layer_nextn, false);- GGML_ASSERT(hparams.n_layer_nextn < hparams.n_layer_all && "n_layer_nextn must be < n_layer");-if (hparams.expert_gating_func == LLAMA_EXPERT_GATING_FUNC_TYPE_NONE) {hparams.expert_gating_func = LLAMA_EXPERT_GATING_FUNC_TYPE_SIGMOID;}diff --git a/src/models/deepseek2.cpp b/src/models/deepseek2.cppindex e0e537e005..3a76187aa5 100644--- a/src/models/deepseek2.cpp+++ b/src/models/deepseek2.cpp@@ -37,11 +37,6 @@ void llama_model_deepseek2::load_arch_hparams(llama_model_loader & ml) {hparams.rope_yarn_log_mul /= 0.1f;}- // NextN/MTP- ml.get_key(LLM_KV_NEXTN_PREDICT_LAYERS, hparams.n_layer_nextn, false);- GGML_ASSERT(hparams.n_layer_nextn == 0 ||- hparams.n_layer() + hparams.n_layer_nextn == hparams.n_layer_all);-// (optional) temperature tuning - used by mistral-largeml.get_key(LLM_KV_ATTENTION_TEMPERATURE_SCALE, hparams.f_attn_temp_scale, false);ml.get_key(LLM_KV_ATTENTION_TEMPERATURE_LENGTH, hparams.n_attn_temp_floor_scale, false); // FIXME why not use temperature_length?diff --git a/src/models/deepseek32.cpp b/src/models/deepseek32.cppindex 2b82a780c4..079bdfc302 100644--- a/src/models/deepseek32.cpp+++ b/src/models/deepseek32.cpp@@ -37,10 +37,6 @@ void llama_model_deepseek32::load_arch_hparams(llama_model_loader & ml) {hparams.rope_yarn_log_mul /= 0.1f;}- // NextN/MTP parameters- ml.get_key(LLM_KV_NEXTN_PREDICT_LAYERS, hparams.n_layer_nextn, false);- GGML_ASSERT(hparams.n_layer_nextn < hparams.n_layer_all && "n_layer_nextn must be < n_layer");-switch (hparams.n_layer()) {case 61: type = LLM_TYPE_685B_A37B; break;default: type = LLM_TYPE_UNKNOWN;diff --git a/src/models/deepseek4.cpp b/src/models/deepseek4.cppindex fc816e2aeb..0157ce7051 100644--- a/src/models/deepseek4.cpp+++ b/src/models/deepseek4.cpp@@ -17,15 +17,13 @@ static float dsv4_rope_attn_factor(float freq_scale, float ext_factor) {}void llama_model_deepseek4::load_arch_hparams(llama_model_loader & ml) {- ml.get_key(LLM_KV_NEXTN_PREDICT_LAYERS, hparams.n_layer_nextn, false);- if (hparams.n_layer_nextn > 0 && hparams.n_layer_nextn < hparams.n_layer_all) {+ if (hparams.n_layer_nextn > 0) {const uint32_t n_layer_main = hparams.n_layer_all - hparams.n_layer_nextn;const std::string mtp_probe = "blk." + std::to_string(n_layer_main) + ".nextn.eh_proj.weight";if (ml.get_weight(mtp_probe.c_str()) == nullptr) {hparams.n_layer_nextn = 0;}}- GGML_ASSERT(hparams.n_layer_nextn < hparams.n_layer_all && "n_layer_nextn must be < block_count");ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);ml.get_key(LLM_KV_ATTENTION_Q_LORA_RANK, hparams.n_lora_q);diff --git a/src/models/dots3note.cpp b/src/models/dots3note.cppindex 00a008c2c9..7656562b05 100644--- a/src/models/dots3note.cpp+++ b/src/models/dots3note.cpp@@ -9,10 +9,6 @@ void llama_model_dots3note::load_arch_hparams(llama_model_loader & ml) {ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);hparams.f_norm_eps = 1e-6; // eps for the indexer k_norm layer norm- // TODO: use MTP layer- ml.get_key(LLM_KV_NEXTN_PREDICT_LAYERS, hparams.n_layer_nextn, false);- GGML_ASSERT(hparams.n_layer_nextn < hparams.n_layer_all && "n_layer_nextn must be < n_layer_all");-// MoE parametersml.get_key(LLM_KV_EXPERT_SHARED_COUNT, hparams.n_expert_shared);ml.get_key(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp);diff --git a/src/models/exaone-moe.cpp b/src/models/exaone-moe.cppindex 5aed937940..86e5a3a984 100644--- a/src/models/exaone-moe.cpp+++ b/src/models/exaone-moe.cpp@@ -20,9 +20,6 @@ void llama_model_exaone_moe::load_arch_hparams(llama_model_loader & ml) {ml.get_key(LLM_KV_EXPERT_WEIGHTS_NORM, hparams.expert_weights_norm, false);ml.get_key(LLM_KV_LEADING_DENSE_BLOCK_COUNT, hparams.n_layer_dense_lead, false);- ml.get_key(LLM_KV_NEXTN_PREDICT_LAYERS, hparams.n_layer_nextn, false);- GGML_ASSERT(hparams.n_layer_nextn < hparams.n_layer_all && "n_layer_nextn must be < n_layer_impl");-switch (hparams.n_layer()) {case 32: type = LLM_TYPE_30B_A3B; break;case 48: type = LLM_TYPE_235B_A22B; break;diff --git a/src/models/exaone4.cpp b/src/models/exaone4.cppindex a06819a67c..9ba978956d 100644--- a/src/models/exaone4.cpp+++ b/src/models/exaone4.cpp@@ -1,9 +1,6 @@#include "models.h"void llama_model_exaone4::load_arch_hparams(llama_model_loader & ml) {- ml.get_key(LLM_KV_NEXTN_PREDICT_LAYERS, hparams.n_layer_nextn, false);- GGML_ASSERT(hparams.n_layer_nextn < hparams.n_layer_all && "n_layer_nextn must be < n_layer");-if (hparams.n_layer() == 64) { // 32Bhparams.swa_type = LLAMA_SWA_TYPE_STANDARD;hparams.n_swa = 4096;diff --git a/src/models/gemma4-assistant.cpp b/src/models/gemma4-assistant.cppindex 6378130e79..989fa42c85 100644--- a/src/models/gemma4-assistant.cpp+++ b/src/models/gemma4-assistant.cpp@@ -11,9 +11,6 @@ void llama_model_gemma4_assistant::load_arch_hparams(llama_model_loader & ml) {hparams.f_attention_scale = 1.0f;- ml.get_key(LLM_KV_NEXTN_PREDICT_LAYERS, hparams.n_layer_nextn, false);- GGML_ASSERT(hparams.n_layer_nextn == hparams.n_layer_all && "n_layer_nextn must be == n_layer_impl");-ml.get_key(LLM_KV_ROPE_FREQ_BASE_SWA, hparams.rope_freq_base_train_swa, false);ml.get_key(LLM_KV_ATTENTION_SLIDING_WINDOW, hparams.n_swa);ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);diff --git a/src/models/glm-dsa.cpp b/src/models/glm-dsa.cppindex 93a1448b46..543b15cf35 100644--- a/src/models/glm-dsa.cpp+++ b/src/models/glm-dsa.cpp@@ -56,10 +56,6 @@ void llama_model_glm_dsa::load_arch_hparams(llama_model_loader & ml) {hparams.expert_gating_func = LLAMA_EXPERT_GATING_FUNC_TYPE_SIGMOID;}- // NextN/MTP parameters- ml.get_key(LLM_KV_NEXTN_PREDICT_LAYERS, hparams.n_layer_nextn, false);- GGML_ASSERT(hparams.n_layer_nextn < hparams.n_layer_all && "n_layer_nextn must be < n_layer_all");-// BC for GLM 5, 5.1 (full indexers) without indexer_types metadataconst bool is_pre_5_2 = hparams.n_ctx_train < 1048576;if (is_pre_5_2) {@@ -70,9 +66,7 @@ void llama_model_glm_dsa::load_arch_hparams(llama_model_loader & ml) {ml.get_key_or_arr(LLM_KV_ATTENTION_INDEXER_TYPES, hparams.is_indexer_full_impl, hparams.n_layer(), false);switch (hparams.n_layer()) {- case 78: // GGUF with NextN/MTP metadata: n_layer() excludes the nextn layer- case 79:- type = LLM_TYPE_744B_A40B; break;+ case 78: type = LLM_TYPE_744B_A40B; break;default: type = LLM_TYPE_UNKNOWN;}}diff --git a/src/models/glm4-moe.cpp b/src/models/glm4-moe.cppindex 83ea7f8ac6..1d2ac65fd7 100644--- a/src/models/glm4-moe.cpp+++ b/src/models/glm4-moe.cpp@@ -17,10 +17,6 @@ void llama_model_glm4_moe::load_arch_hparams(llama_model_loader & ml) {hparams.expert_gating_func = LLAMA_EXPERT_GATING_FUNC_TYPE_SIGMOID;}- // NextN/MTP parameters- ml.get_key(LLM_KV_NEXTN_PREDICT_LAYERS, hparams.n_layer_nextn, false);- GGML_ASSERT(hparams.n_layer_nextn < hparams.n_layer_all && "n_layer_nextn must be < n_layer_impl");-switch (hparams.n_layer()) {case 46: type = LLM_TYPE_106B_A12B; break; // GLM-4.5-Aircase 48: type = LLM_TYPE_102B_A12B; break; // Solar Opendiff --git a/src/models/glm4.cpp b/src/models/glm4.cppindex b4326c5f21..463be809d8 100644--- a/src/models/glm4.cpp+++ b/src/models/glm4.cpp@@ -4,10 +4,6 @@ void llama_model_glm4::load_arch_hparams(llama_model_loader & ml) {ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);ml.get_key_or_arr(LLM_KV_ROPE_DIMENSION_SECTIONS, hparams.rope_sections, 4, false);- // NextN/MTP parameters (GLM-OCR)- ml.get_key(LLM_KV_NEXTN_PREDICT_LAYERS, hparams.n_layer_nextn, false);- GGML_ASSERT(hparams.n_layer_nextn < hparams.n_layer_all && "n_layer_nextn must be < n_layer_impl");-switch (hparams.n_layer()) {case 17: type = LLM_TYPE_1B; break; // GLM-OCRcase 40: type = LLM_TYPE_9B; break;diff --git a/src/models/hy-v3.cpp b/src/models/hy-v3.cppindex 61db93af85..3c45331b11 100644--- a/src/models/hy-v3.cpp+++ b/src/models/hy-v3.cpp@@ -13,10 +13,6 @@ void llama_model_hy_v3::load_arch_hparams(llama_model_loader & ml) {hparams.expert_gating_func = LLAMA_EXPERT_GATING_FUNC_TYPE_SIGMOID;}- // NextN/MTP (HY V3): extra decoder block(s) appended beyond the main stack- ml.get_key(LLM_KV_NEXTN_PREDICT_LAYERS, hparams.n_layer_nextn, false);- GGML_ASSERT(hparams.n_layer_nextn < hparams.n_layer_all && "n_layer_nextn must be < n_layer_all");-switch (hparams.n_layer()) {case 48: type = LLM_TYPE_30B_A3B; break;default: type = LLM_TYPE_UNKNOWN;diff --git a/src/models/mimo2.cpp b/src/models/mimo2.cppindex d50e186cce..1dc554220d 100644--- a/src/models/mimo2.cpp+++ b/src/models/mimo2.cpp@@ -16,9 +16,6 @@ void llama_model_mimo2::load_arch_hparams(llama_model_loader & ml) {hparams.f_attn_value_scale = value_scale;}- ml.get_key(LLM_KV_NEXTN_PREDICT_LAYERS, hparams.n_layer_nextn, false);- GGML_ASSERT(hparams.n_layer_nextn < hparams.n_layer_all && "n_layer_nextn must be < n_layer_impl");-switch (hparams.n_layer()) {case 48: type = LLM_TYPE_310B_A15B; break;default: type = LLM_TYPE_UNKNOWN;diff --git a/src/models/nemotron-h.cpp b/src/models/nemotron-h.cppindex f02674c646..55640c996a 100644--- a/src/models/nemotron-h.cpp+++ b/src/models/nemotron-h.cpp@@ -7,10 +7,6 @@ void llama_model_nemotron_h::load_arch_hparams(llama_model_loader & ml) {ml.get_key(LLM_KV_SSM_TIME_STEP_RANK, hparams.ssm_dt_rank);ml.get_key(LLM_KV_SSM_GROUP_COUNT, hparams.ssm_n_group);- // NextN/MTP: optional draft head appended as extra trailing block(s)- ml.get_key(LLM_KV_NEXTN_PREDICT_LAYERS, hparams.n_layer_nextn, false);- GGML_ASSERT(hparams.n_layer_nextn < hparams.n_layer_all && "n_layer_nextn must be < n_layer_all");-// A layer is recurrent IFF the n_head_kv value is set to 0 and// the n_ff value is set to 0. Appended MTP blocks are dense (non-recurrent)for (uint32_t i = 0; i < hparams.n_layer_all; ++i) {diff --git a/src/models/qwen35.cpp b/src/models/qwen35.cppindex 309dd43244..0b9210981d 100644--- a/src/models/qwen35.cpp+++ b/src/models/qwen35.cpp@@ -12,10 +12,6 @@ void llama_model_qwen35::load_arch_hparams(llama_model_loader & ml) {ml.get_key(LLM_KV_SSM_TIME_STEP_RANK, hparams.ssm_dt_rank);ml.get_key(LLM_KV_SSM_GROUP_COUNT, hparams.ssm_n_group);- // NextN/MTP (Qwen3.5/3.6): extra decoder block appended beyond the main stack- ml.get_key(LLM_KV_NEXTN_PREDICT_LAYERS, hparams.n_layer_nextn, false);- GGML_ASSERT(hparams.n_layer_nextn < hparams.n_layer_all && "n_layer_nextn must be < n_layer_impl");-// Mark recurrent layers (linear attention layers). MTP layers are dense// attention-only and must be flagged non-recurrent.if (!ml.get_key_or_arr(LLM_KV_ATTENTION_RECURRENT_LAYERS, hparams.is_recr_impl, hparams.n_layer_all, false)) {diff --git a/src/models/qwen35moe.cpp b/src/models/qwen35moe.cppindex 38f2a57985..9bf4ea4328 100644--- a/src/models/qwen35moe.cpp+++ b/src/models/qwen35moe.cpp@@ -15,10 +15,6 @@ void llama_model_qwen35moe::load_arch_hparams(llama_model_loader & ml) {ml.get_key(LLM_KV_SSM_TIME_STEP_RANK, hparams.ssm_dt_rank);ml.get_key(LLM_KV_SSM_GROUP_COUNT, hparams.ssm_n_group);- // NextN/MTP (Qwen3.5/3.6): extra decoder block appended beyond the main stack- ml.get_key(LLM_KV_NEXTN_PREDICT_LAYERS, hparams.n_layer_nextn, false);- GGML_ASSERT(hparams.n_layer_nextn < hparams.n_layer_all && "n_layer_nextn must be < n_layer_impl");-// Mark recurrent layers (linear attention layers). MTP layers are dense// attention-only and must be flagged non-recurrent.if (!ml.get_key_or_arr(LLM_KV_ATTENTION_RECURRENT_LAYERS, hparams.is_recr_impl, hparams.n_layer_all, false)) {diff --git a/src/models/qwen3next.cpp b/src/models/qwen3next.cppindex 0808fd87aa..b2b8809c7f 100644--- a/src/models/qwen3next.cpp+++ b/src/models/qwen3next.cpp@@ -13,10 +13,6 @@ void llama_model_qwen3next::load_arch_hparams(llama_model_loader & ml) {ml.get_key(LLM_KV_SSM_TIME_STEP_RANK, hparams.ssm_dt_rank);ml.get_key(LLM_KV_SSM_GROUP_COUNT, hparams.ssm_n_group);- // NextN/MTP: extra decoder block appended beyond the main stack- ml.get_key(LLM_KV_NEXTN_PREDICT_LAYERS, hparams.n_layer_nextn, false);- GGML_ASSERT(hparams.n_layer_nextn < hparams.n_layer_all && "n_layer_nextn must be < n_layer_all");-// Mark recurrent layers (linear attention layers).if (!ml.get_key_or_arr(LLM_KV_ATTENTION_RECURRENT_LAYERS, hparams.is_recr_impl, hparams.n_layer_all, false)) {uint32_t full_attn_interval = 4;diff --git a/src/models/step35.cpp b/src/models/step35.cppindex 5b1d902581..d101d115ef 100644--- a/src/models/step35.cpp+++ b/src/models/step35.cpp@@ -23,14 +23,10 @@ void llama_model_step35::load_arch_hparams(llama_model_loader & ml) {ml.get_key(LLM_KV_ATTENTION_SLIDING_WINDOW, hparams.n_swa);ml.get_key(LLM_KV_ROPE_FREQ_BASE_SWA, hparams.rope_freq_base_train_swa, false);- ml.get_key_or_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, hparams.is_swa_impl, hparams.n_layer());+ ml.get_key_or_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, hparams.is_swa_impl, hparams.n_layer_all);- ml.get_key_or_arr(LLM_KV_SWIGLU_CLAMP_EXP, hparams.swiglu_clamp_exp, hparams.n_layer(), false);- ml.get_key_or_arr(LLM_KV_SWIGLU_CLAMP_SHEXP, hparams.swiglu_clamp_shexp, hparams.n_layer(), false);-- // NextN/MTP (Step3p5): extra decoder block appended beyond the main stack.- ml.get_key(LLM_KV_NEXTN_PREDICT_LAYERS, hparams.n_layer_nextn, false);- GGML_ASSERT(hparams.n_layer_nextn < hparams.n_layer_all && "n_layer_nextn must be < n_layer_impl");+ ml.get_key_or_arr(LLM_KV_SWIGLU_CLAMP_EXP, hparams.swiglu_clamp_exp, hparams.n_layer_all, false);+ ml.get_key_or_arr(LLM_KV_SWIGLU_CLAMP_SHEXP, hparams.swiglu_clamp_shexp, hparams.n_layer_all, false);switch (hparams.n_layer()) {case 45: type = LLM_TYPE_196B_A11B; break;