LLM inference in C/C++
C++ ggml
memory : avoid allocating V cache for indexer (it's not used) (#28330)
fairydreaming311d4211bf16
diff --git a/src/llama-memory-hybrid-idx.cpp b/src/llama-memory-hybrid-idx.cppindex 93b468784a..3972ce9ce2 100644--- a/src/llama-memory-hybrid-idx.cpp+++ b/src/llama-memory-hybrid-idx.cpp@@ -55,6 +55,10 @@ llama_memory_hybrid_idx::llama_memory_hybrid_idx(// K-shift must not rotate them while the stream copies in the same update still applyhparams_idx.rope_type = LLAMA_ROPE_TYPE_NONE;+ // fool llama_kv_cache into thinking this is a MLA cache, so it won't cache V tensors+ hparams_idx.n_embd_head_k_mla_impl = model.hparams.indexer_head_size;+ hparams_idx.n_embd_head_v_mla_impl = model.hparams.indexer_head_size;+LLAMA_LOG_INFO("%s: creating indexer KV cache, size = %u cells\n", __func__, kv_size);return new llama_kv_cache(