LLM inference in C/C++
C++ ggml
llama : enforce the same K and V cache types for DeepSeek V4; enable FA if V cache is quantized (#25871)
fairydreaming69e62fc77c91
diff --git a/src/llama-context.cpp b/src/llama-context.cppindex 3e0af3f317..5ef7becf6f 100644--- a/src/llama-context.cpp+++ b/src/llama-context.cpp@@ -3553,6 +3553,22 @@ llama_context * llama_init_from_model(}}+ if ((model->hparams.is_mla() || model->arch == LLM_ARCH_DEEPSEEK4) && params.type_k != params.type_v) {+ LLAMA_LOG_ERROR("%s: model does not support different K (%s) and V (%s) cache types\n", __func__, ggml_type_name(params.type_k), ggml_type_name(params.type_v));+ return nullptr;+ }++ if (ggml_is_quantized(params.type_v) && params.flash_attn_type != LLAMA_FLASH_ATTN_TYPE_ENABLED) {+ if (params.flash_attn_type == LLAMA_FLASH_ATTN_TYPE_AUTO) {+ LLAMA_LOG_INFO("%s: enabling flash_attn since it is required for quantized V cache\n", __func__);+ params.flash_attn_type = LLAMA_FLASH_ATTN_TYPE_ENABLED;+ }+ if (params.flash_attn_type == LLAMA_FLASH_ATTN_TYPE_DISABLED) {+ LLAMA_LOG_ERROR("%s: quantized V cache requires flash_attn to be enabled\n", __func__);+ return nullptr;+ }+ }+if (params.flash_attn_type != LLAMA_FLASH_ATTN_TYPE_DISABLED && ggml_is_quantized(params.type_k)) {const uint32_t blck_size = ggml_blck_size(params.type_k);for (uint32_t il = 0; il < model->hparams.n_layer(); ++il) {@@ -3575,11 +3591,6 @@ llama_context * llama_init_from_model(}}- if (ggml_is_quantized(params.type_v) && params.flash_attn_type == LLAMA_FLASH_ATTN_TYPE_DISABLED) {- LLAMA_LOG_ERROR("%s: V cache quantization requires flash_attn\n", __func__);- return nullptr;- }-if (params.pooling_type != LLAMA_POOLING_TYPE_UNSPECIFIED &¶ms.pooling_type != model->hparams.pooling_type) {//user-specified pooling-type is different from the model default