Skip to content

LLM inference in C/C++

C++ ggml

model : fix MTP context kv cache allocation for deepseek2, glm4moe, … (#28630)

Logan Chu5cdd3d1dad5c
diff --git a/src/llama-model.cpp b/src/llama-model.cpp
index d10b60afd9..f9e9a8bcb0 100644
--- a/src/llama-model.cpp
+++ b/src/llama-model.cpp
@@ -2644,9 +2644,9 @@ llama_memory_i * llama_model::create_memory(const llama_memory_params & params,
filter = [&](uint32_t il) { return il >= hparams.n_layer(); };
}
- if ((arch == LLM_ARCH_STEP35 || arch == LLM_ARCH_HY_V3 || arch == LLM_ARCH_GLM_DSA ||
- arch == LLM_ARCH_MIMO2 || arch == LLM_ARCH_DEEPSEEK32) &&
- hparams.n_layer_nextn > 0) {
+ // don't filter when n_layer_nextn is repurposed for a router layer the trunk attends
+ // or when a model is entirely n_layer_nextn layers and has no trunk
+ if (hparams.n_layer_nextn > 0 && hparams.n_layer() > 0 && hparams.router_layer < 0) {
if (params.ctx_type == LLAMA_CONTEXT_TYPE_MTP) {
filter = [&](uint32_t il) { return il >= hparams.n_layer(); };
} else {