Skip to content

LLM inference in C/C++

C++ ggml

memory : avoid allocating V cache for indexer (it's not used) (#28330)

fairydreaming311d4211bf16
diff --git a/src/llama-memory-hybrid-idx.cpp b/src/llama-memory-hybrid-idx.cpp
index 93b468784a..3972ce9ce2 100644
--- a/src/llama-memory-hybrid-idx.cpp
+++ b/src/llama-memory-hybrid-idx.cpp
@@ -55,6 +55,10 @@ llama_memory_hybrid_idx::llama_memory_hybrid_idx(
// K-shift must not rotate them while the stream copies in the same update still apply
hparams_idx.rope_type = LLAMA_ROPE_TYPE_NONE;
+ // fool llama_kv_cache into thinking this is a MLA cache, so it won't cache V tensors
+ hparams_idx.n_embd_head_k_mla_impl = model.hparams.indexer_head_size;
+ hparams_idx.n_embd_head_v_mla_impl = model.hparams.indexer_head_size;
+
LLAMA_LOG_INFO("%s: creating indexer KV cache, size = %u cells\n", __func__, kv_size);
return new llama_kv_cache(