LLM inference in C/C++
C++ ggml
model : fix MTP context kv cache allocation for deepseek2, glm4moe, … (#28630)
Logan Chu5cdd3d1dad5c
diff --git a/src/llama-model.cpp b/src/llama-model.cppindex d10b60afd9..f9e9a8bcb0 100644--- a/src/llama-model.cpp+++ b/src/llama-model.cpp@@ -2644,9 +2644,9 @@ llama_memory_i * llama_model::create_memory(const llama_memory_params & params,filter = [&](uint32_t il) { return il >= hparams.n_layer(); };}- if ((arch == LLM_ARCH_STEP35 || arch == LLM_ARCH_HY_V3 || arch == LLM_ARCH_GLM_DSA ||- arch == LLM_ARCH_MIMO2 || arch == LLM_ARCH_DEEPSEEK32) &&- hparams.n_layer_nextn > 0) {+ // don't filter when n_layer_nextn is repurposed for a router layer the trunk attends+ // or when a model is entirely n_layer_nextn layers and has no trunk+ if (hparams.n_layer_nextn > 0 && hparams.n_layer() > 0 && hparams.router_layer < 0) {if (params.ctx_type == LLAMA_CONTEXT_TYPE_MTP) {filter = [&](uint32_t il) { return il >= hparams.n_layer(); };} else {