LLM inference in C/C++
C++ ggml
metal : dequant kv cache only for large batches (#27438)
Georgi Gerganov63b64a50a376
diff --git a/ggml/src/ggml-metal/ggml-metal-ops.cpp b/ggml/src/ggml-metal/ggml-metal-ops.cppindex 2dde14d8dc..8311544b39 100644--- a/ggml/src/ggml-metal/ggml-metal-ops.cpp+++ b/ggml/src/ggml-metal/ggml-metal-ops.cpp@@ -2806,6 +2806,13 @@ bool ggml_metal_op_flash_attn_ext_use_vec(const ggml_tensor * op) {static bool ggml_metal_op_flash_attn_ext_use_kv_f16(const ggml_tensor * op) {assert(op->op == GGML_OP_FLASH_ATTN_EXT);+ // depending on compute/bandwidth ratio, dequant to f16 kv is not always beneficial+ // ref: https://github.com/ggml-org/llama.cpp/pull/27390#issuecomment-5355152767+ // TODO: tune per device+ if (op->src[0]->ne[1] < 32) {+ return false;+ }+switch (op->src[1]->type) {case GGML_TYPE_Q4_0:case GGML_TYPE_Q4_1:@@ -2968,9 +2975,10 @@ size_t ggml_metal_op_flash_attn_ext_extra_tmp(const ggml_tensor * op) {size_t ggml_metal_op_flash_attn_ext_extra_kv_f16(const ggml_tensor * op) {assert(op->op == GGML_OP_FLASH_ATTN_EXT);- if (!ggml_metal_op_flash_attn_ext_use_kv_f16(op)) {- return 0;- }+ // note: always reserve the temp buffer to avoid graph reallocations+ //if (!ggml_metal_op_flash_attn_ext_use_kv_f16(op)) {+ // return 0;+ //}GGML_TENSOR_LOCALS( int32_t, ne2, op->src[2], ne);