LLM inference in C/C++
C++ ggml
metal : dequantize quantized KV to F16 before flash attention (#27390)
Georgi Gerganov70aff2525007
diff --git a/.pi/gg/SYSTEM.md b/.pi/gg/SYSTEM.mdindex d39afbe033..6a757c8694 100644--- a/.pi/gg/SYSTEM.md+++ b/.pi/gg/SYSTEM.md@@ -9,6 +9,7 @@ General:Coding:- When in doubt, always refer to the CONTRIBUTING.md file of the project+- In `test-backend-ops.cpp`, do not mention specific backends (e.g. Metal, CUDA) in comments- When referencing issues or PRs in comments, use the format:- C/C++ code: `// ref: <url>`- Other (CMake, etc.): `# ref: <url>`diff --git a/ggml/src/ggml-metal/ggml-metal-device.cpp b/ggml/src/ggml-metal/ggml-metal-device.cppindex 953c757558..52043696eb 100644--- a/ggml/src/ggml-metal/ggml-metal-device.cpp+++ b/ggml/src/ggml-metal/ggml-metal-device.cpp@@ -1409,6 +1409,23 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_flash_attn_ext_preturn res;}+ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_flash_attn_ext_kv_f16(+ ggml_metal_library_t lib,+ const ggml_tensor * op) {+ assert(op->op == GGML_OP_FLASH_ATTN_EXT);++ char base[256];++ snprintf(base, 256, "kernel_flash_attn_ext_kv_%s_f16", ggml_type_name(op->src[1]->type));++ ggml_metal_pipeline_with_params res = ggml_metal_library_get_pipeline(lib, base);+ if (!res.pipeline) {+ res = ggml_metal_library_compile_pipeline(lib, base, base, nullptr);+ }++ return res;+}+ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_flash_attn_ext_blk(ggml_metal_library_t lib,const struct ggml_tensor * op,@@ -1460,7 +1477,10 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_flash_attn_ext(bool has_bias,bool has_scap,bool has_kvpad,- int32_t nsg) {+ int32_t nsg,+ bool use_kv_f16,+ int32_t ns10,+ int32_t ns20) {assert(op->op == GGML_OP_FLASH_ATTN_EXT);char base[256];@@ -1469,15 +1489,14 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_flash_attn_ext(const int32_t dk = (int32_t) op->src[1]->ne[0];const int32_t dv = (int32_t) op->src[2]->ne[0];- const int32_t ns10 = op->src[1]->nb[1]/op->src[1]->nb[0];- const int32_t ns20 = op->src[2]->nb[1]/op->src[2]->nb[0];+ const char * type = use_kv_f16 ? "f16" : ggml_type_name(op->src[1]->type);// do bounds checks for the mask?const bool bc_mask = op->src[3] && (op->src[3]->ne[1] % 8 != 0);snprintf(base, 256, "kernel_%s_%s_dk%d_dv%d","flash_attn_ext",- ggml_type_name(op->src[1]->type),+ type,dk,dv);@@ -1526,7 +1545,10 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_flash_attn_ext_vbool has_scap,bool has_kvpad,int32_t nsg,- int32_t nwg) {+ int32_t nwg,+ bool use_kv_f16,+ int32_t ns10,+ int32_t ns20) {assert(op->op == GGML_OP_FLASH_ATTN_EXT);char base[256];@@ -1535,12 +1557,11 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_flash_attn_ext_vconst int32_t dk = (int32_t) op->src[1]->ne[0];const int32_t dv = (int32_t) op->src[2]->ne[0];- const int32_t ns10 = op->src[1]->nb[1]/op->src[1]->nb[0];- const int32_t ns20 = op->src[2]->nb[1]/op->src[2]->nb[0];+ const char * type = use_kv_f16 ? "f16" : ggml_type_name(op->src[1]->type);snprintf(base, 256, "kernel_%s_%s_dk%d_dv%d","flash_attn_ext_vec",- ggml_type_name(op->src[1]->type),+ type,dk,dv);diff --git a/ggml/src/ggml-metal/ggml-metal-device.h b/ggml/src/ggml-metal/ggml-metal-device.hindex 7e1deeaa21..b7d4660588 100644--- a/ggml/src/ggml-metal/ggml-metal-device.h+++ b/ggml/src/ggml-metal/ggml-metal-device.h@@ -176,6 +176,10 @@ struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_flash_attbool has_mask,int32_t ncpsg);+struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_flash_attn_ext_kv_f16(+ ggml_metal_library_t lib,+ const struct ggml_tensor * op);+struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_flash_attn_ext_blk(ggml_metal_library_t lib,const struct ggml_tensor * op,@@ -190,7 +194,10 @@ struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_flash_attbool has_bias,bool has_scap,bool has_kvpad,- int32_t nsg);+ int32_t nsg,+ bool use_kv_f16,+ int32_t ns10,+ int32_t ns20);struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_flash_attn_ext_vec(ggml_metal_library_t lib,@@ -201,7 +208,10 @@ struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_flash_attbool has_scap,bool has_kvpad,int32_t nsg,- int32_t nwg);+ int32_t nwg,+ bool use_kv_f16,+ int32_t ns10,+ int32_t ns20);struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_flash_attn_ext_vec_reduce(ggml_metal_library_t lib,diff --git a/ggml/src/ggml-metal/ggml-metal-impl.h b/ggml/src/ggml-metal/ggml-metal-impl.hindex 05ea7470ee..f0b7799791 100644--- a/ggml/src/ggml-metal/ggml-metal-impl.h+++ b/ggml/src/ggml-metal/ggml-metal-impl.h@@ -345,6 +345,18 @@ typedef struct {bool inplace;} ggml_metal_kargs_rope;+typedef struct {+ int32_t ne0;+ int32_t ne1;+ int32_t ne2;+ int32_t ne3;+ uint64_t nb0;+ uint64_t nb1;+ uint64_t nb2;+ uint64_t nb3;+ int32_t nblocks;+} ggml_metal_kargs_flash_attn_ext_kv_f16;+typedef struct {int32_t ne11;int32_t ne_12_2; // assume K and V are same shapediff --git a/ggml/src/ggml-metal/ggml-metal-ops.cpp b/ggml/src/ggml-metal/ggml-metal-ops.cppindex d8435e9577..2dde14d8dc 100644--- a/ggml/src/ggml-metal/ggml-metal-ops.cpp+++ b/ggml/src/ggml-metal/ggml-metal-ops.cpp@@ -2801,6 +2801,44 @@ bool ggml_metal_op_flash_attn_ext_use_vec(const ggml_tensor * op) {return (ne01 < 20) && (ne00 % 32 == 0);}+// ref: https://github.com/ggml-org/llama.cpp/pull/27390+// dequantize the quantized KV cache to F16 before running the F16 flash attention kernels+static bool ggml_metal_op_flash_attn_ext_use_kv_f16(const ggml_tensor * op) {+ assert(op->op == GGML_OP_FLASH_ATTN_EXT);++ switch (op->src[1]->type) {+ case GGML_TYPE_Q4_0:+ case GGML_TYPE_Q4_1:+ case GGML_TYPE_Q5_0:+ case GGML_TYPE_Q5_1:+ case GGML_TYPE_Q8_0:+ return true;+ default:+ return false;+ }+}++// in some models (e.g. MLA-based), V is a view of K (the first ne20 elements of each K row);+// the dequantized V is then a view of the dequantized K and does not need its own dequant or scratch+// - ref: https://github.com/ggml-org/llama.cpp/pull/13435+static bool ggml_metal_op_flash_attn_ext_v_is_view_of_k(const ggml_tensor * op) {+ assert(op->op == GGML_OP_FLASH_ATTN_EXT);++ const ggml_tensor * K = op->src[1];+ const ggml_tensor * V = op->src[2];++ return V->view_src && (V->view_src == K || (V->view_src == K->view_src && V->view_offs == K->view_offs));+}++// size of the F16 dequantized K tensor; the dequantized V tensor follows it in the same scratch buffer+static size_t ggml_metal_op_flash_attn_ext_kv_f16_k_size(const ggml_tensor * op) {+ assert(op->op == GGML_OP_FLASH_ATTN_EXT);++ GGML_TENSOR_LOCALS( int32_t, ne1, op->src[1], ne);++ return GGML_PAD(sizeof(ggml_fp16_t)*(size_t) ne10*ne11*ne12*ne13, 16);+}+size_t ggml_metal_op_flash_attn_ext_extra_pad(const ggml_tensor * op) {assert(op->op == GGML_OP_FLASH_ATTN_EXT);@@ -2816,6 +2854,18 @@ size_t ggml_metal_op_flash_attn_ext_extra_pad(const ggml_tensor * op) {size_t res = 0;const bool has_mask = op->src[3] != nullptr;+ const bool use_kv_f16 = ggml_metal_op_flash_attn_ext_use_kv_f16(op);++ // when the KV is dequantized to F16, the pad kernel copies the tail chunk from the F16 scratch buffer+ // note: when V is a view of K, the dequantized V is read from the dequantized K with K's row stride+ const bool v_is_view_of_k = use_kv_f16 && ggml_metal_op_flash_attn_ext_v_is_view_of_k(op);+ uint64_t nb11_pad = nb11;+ uint64_t nb21_pad = nb21;++ if (use_kv_f16) {+ nb11_pad = sizeof(ggml_fp16_t)*ne10;+ nb21_pad = sizeof(ggml_fp16_t)*(v_is_view_of_k ? ne10 : ne20);+ }// note: the non-vec kernel requires more extra memory, so always reserve for itGGML_ASSERT(OP_FLASH_ATTN_EXT_NCPSG >= OP_FLASH_ATTN_EXT_VEC_NCPSG);@@ -2828,8 +2878,8 @@ size_t ggml_metal_op_flash_attn_ext_extra_pad(const ggml_tensor * op) {if (has_kvpad) {res += OP_FLASH_ATTN_EXT_VEC_NCPSG*(- nb11*ne12*ne13 +- nb21*ne22*ne23 ++ nb11_pad*ne12*ne13 ++ nb21_pad*ne22*ne23 +(has_mask ? ggml_type_size(GGML_TYPE_F16)*ne31*ne32*ne33 : 0));}} else {@@ -2838,8 +2888,8 @@ size_t ggml_metal_op_flash_attn_ext_extra_pad(const ggml_tensor * op) {if (has_kvpad) {res += OP_FLASH_ATTN_EXT_NCPSG*(- nb11*ne12*ne13 +- nb21*ne22*ne23 ++ nb11_pad*ne12*ne13 ++ nb21_pad*ne22*ne23 +(has_mask ? ggml_type_size(GGML_TYPE_F16)*ne31*ne32*ne33 : 0));}}@@ -2915,6 +2965,28 @@ size_t ggml_metal_op_flash_attn_ext_extra_tmp(const ggml_tensor * op) {return res;}+size_t ggml_metal_op_flash_attn_ext_extra_kv_f16(const ggml_tensor * op) {+ assert(op->op == GGML_OP_FLASH_ATTN_EXT);++ if (!ggml_metal_op_flash_attn_ext_use_kv_f16(op)) {+ return 0;+ }++ GGML_TENSOR_LOCALS( int32_t, ne2, op->src[2], ne);++ const size_t k_size = ggml_metal_op_flash_attn_ext_kv_f16_k_size(op);++ // when V is a view of K, the dequantized V is a view of the dequantized K+ const bool v_is_view_of_k = ggml_metal_op_flash_attn_ext_v_is_view_of_k(op);+ if (v_is_view_of_k) {+ return k_size;+ }++ const size_t v_size = GGML_PAD(sizeof(ggml_fp16_t)*(size_t) ne20*ne21*ne22*ne23, 16);++ return k_size + v_size;+}+int ggml_metal_op_flash_attn_ext(ggml_metal_op_t ctx, int idx) {ggml_tensor * op = ctx->node(idx);@@ -2989,6 +3061,111 @@ int ggml_metal_op_flash_attn_ext(ggml_metal_op_t ctx, int idx) {ggml_metal_buffer_id bid_tmp = bid_blk;bid_tmp.offs += ggml_metal_op_flash_attn_ext_extra_blk(op);+ ggml_metal_buffer_id bid_kv_f16 = bid_tmp;+ bid_kv_f16.offs += ggml_metal_op_flash_attn_ext_extra_tmp(op);++ const bool use_kv_f16 = ggml_metal_op_flash_attn_ext_use_kv_f16(op);++ ggml_metal_buffer_id bid_k = bid_src1;+ ggml_metal_buffer_id bid_v = bid_src2;++ uint64_t nb10_attn = nb10;+ uint64_t nb11_attn = nb11;+ uint64_t nb12_attn = nb12;+ uint64_t nb13_attn = nb13;+ uint64_t nb20_attn = nb20;+ uint64_t nb21_attn = nb21;+ uint64_t nb22_attn = nb22;+ uint64_t nb23_attn = nb23;++ if (use_kv_f16) {+ assert(ggml_metal_op_flash_attn_ext_extra_kv_f16(op) != 0);++ const bool v_is_view_of_k = ggml_metal_op_flash_attn_ext_v_is_view_of_k(op);++ const int64_t nblocks1_64 = (ne10/ggml_blck_size(op->src[1]->type))*(int64_t) ne11*ne12*ne13;+ GGML_ASSERT(nblocks1_64 <= INT32_MAX);+ const int32_t nblocks1 = nblocks1_64;++ ggml_metal_buffer_id bid_v_f16 = bid_kv_f16;+ bid_v_f16.offs += ggml_metal_op_flash_attn_ext_kv_f16_k_size(op);++ auto pipeline0 = ggml_metal_library_get_pipeline_flash_attn_ext_kv_f16(lib, op);+ const int nth = std::min(ggml_metal_pipeline_max_theads_per_threadgroup(pipeline0), 256);++ // K+ ggml_metal_kargs_flash_attn_ext_kv_f16 args_k = {+ /*.ne0 =*/ ne10,+ /*.ne1 =*/ ne11,+ /*.ne2 =*/ ne12,+ /*.ne3 =*/ ne13,+ /*.nb0 =*/ nb10,+ /*.nb1 =*/ nb11,+ /*.nb2 =*/ nb12,+ /*.nb3 =*/ nb13,+ /*.nblocks =*/ nblocks1,+ };++ ggml_metal_encoder_set_pipeline(enc, pipeline0);+ ggml_metal_encoder_set_bytes (enc, &args_k, sizeof(args_k), 0);+ ggml_metal_encoder_set_buffer (enc, bid_src1, 1);+ ggml_metal_encoder_set_buffer (enc, bid_kv_f16, 2);++ ggml_metal_encoder_dispatch_threadgroups(enc, (nblocks1 + nth - 1)/nth, 1, 1, nth, 1, 1);++ // V (skip when V is a view of K: the dequantized V is a view of the dequantized K)+ if (!v_is_view_of_k) {+ const int64_t nblocks2_64 = (ne20/ggml_blck_size(op->src[2]->type))*(int64_t) ne21*ne22*ne23;+ GGML_ASSERT(nblocks2_64 <= INT32_MAX);+ const int32_t nblocks2 = nblocks2_64;++ ggml_metal_kargs_flash_attn_ext_kv_f16 args_v = {+ /*.ne0 =*/ ne20,+ /*.ne1 =*/ ne21,+ /*.ne2 =*/ ne22,+ /*.ne3 =*/ ne23,+ /*.nb0 =*/ nb20,+ /*.nb1 =*/ nb21,+ /*.nb2 =*/ nb22,+ /*.nb3 =*/ nb23,+ /*.nblocks =*/ nblocks2,+ };++ ggml_metal_encoder_set_pipeline(enc, pipeline0);+ ggml_metal_encoder_set_bytes (enc, &args_v, sizeof(args_v), 0);+ ggml_metal_encoder_set_buffer (enc, bid_src2, 1);+ ggml_metal_encoder_set_buffer (enc, bid_v_f16, 2);++ ggml_metal_encoder_dispatch_threadgroups(enc, (nblocks2 + nth - 1)/nth, 1, 1, nth, 1, 1);+ }++ // the pad and attention kernels read the dequantized KV+ ggml_metal_op_concurrency_reset(ctx);++ bid_k = bid_kv_f16;+ bid_v = v_is_view_of_k ? bid_k : bid_v_f16;++ // contiguous F16 layout of the dequantized K+ nb10_attn = sizeof(ggml_fp16_t);+ nb11_attn = nb10_attn*ne10;+ nb12_attn = nb11_attn*ne11;+ nb13_attn = nb12_attn*ne12;++ // if V is a view of K, the dequantized V is read from the dequantized K with K's strides+ if (v_is_view_of_k) {+ nb20_attn = nb10_attn;+ nb21_attn = nb11_attn;+ nb22_attn = nb12_attn;+ nb23_attn = nb13_attn;+ } else {+ // contiguous F16 layout of the dequantized V+ nb20_attn = sizeof(ggml_fp16_t);+ nb21_attn = nb20_attn*ne20;+ nb22_attn = nb21_attn*ne21;+ nb23_attn = nb22_attn*ne22;+ }+ }+if (!ggml_metal_op_flash_attn_ext_use_vec(op)) {// half8x8 kernelconst int nqptg = OP_FLASH_ATTN_EXT_NQPSG; // queries per threadgroup@@ -3009,12 +3186,12 @@ int ggml_metal_op_flash_attn_ext(ggml_metal_op_t ctx, int idx) {/*.ne11 =*/ne11,/*.ne_12_2 =*/ne12,/*.ne_12_3 =*/ne13,- /*.nb11 =*/nb11,- /*.nb12 =*/nb12,- /*.nb13 =*/nb13,- /*.nb21 =*/nb21,- /*.nb22 =*/nb22,- /*.nb23 =*/nb23,+ /*.nb11 =*/nb11_attn,+ /*.nb12 =*/nb12_attn,+ /*.nb13 =*/nb13_attn,+ /*.nb21 =*/nb21_attn,+ /*.nb22 =*/nb22_attn,+ /*.nb23 =*/nb23_attn,/*.ne31 =*/ne31,/*.ne32 =*/ne32,/*.ne33 =*/ne33,@@ -3027,8 +3204,8 @@ int ggml_metal_op_flash_attn_ext(ggml_metal_op_t ctx, int idx) {ggml_metal_encoder_set_pipeline(enc, pipeline0);ggml_metal_encoder_set_bytes (enc, &args0, sizeof(args0), 0);- ggml_metal_encoder_set_buffer (enc, bid_src1, 1);- ggml_metal_encoder_set_buffer (enc, bid_src2, 2);+ ggml_metal_encoder_set_buffer (enc, bid_k, 1);+ ggml_metal_encoder_set_buffer (enc, bid_v, 2);ggml_metal_encoder_set_buffer (enc, bid_src3, 3);ggml_metal_encoder_set_buffer (enc, bid_pad, 4);@@ -3073,7 +3250,7 @@ int ggml_metal_op_flash_attn_ext(ggml_metal_op_t ctx, int idx) {ggml_metal_op_concurrency_reset(ctx);}- const int is_q = ggml_is_quantized(op->src[1]->type) ? 1 : 0;+ const int is_q = !use_kv_f16 && ggml_is_quantized(op->src[1]->type) ? 1 : 0;// 2*(2*ncpsg)// ncpsg soft_max values + ncpsg mask values@@ -3104,6 +3281,9 @@ int ggml_metal_op_flash_attn_ext(ggml_metal_op_t ctx, int idx) {const size_t smem = FATTN_SMEM(nsg);+ const int32_t ns10 = nb11_attn/nb10_attn;+ const int32_t ns20 = nb21_attn/nb20_attn;+ggml_metal_kargs_flash_attn_ext args = {/*.ne01 =*/ ne01,/*.ne02 =*/ ne02,@@ -3114,14 +3294,14 @@ int ggml_metal_op_flash_attn_ext(ggml_metal_op_t ctx, int idx) {/*.ne11 =*/ ne11,/*.ne_12_2 =*/ ne12,/*.ne_12_3 =*/ ne13,- /*.ns10 =*/ int32_t(nb11/nb10),- /*.nb11 =*/ nb11,- /*.nb12 =*/ nb12,- /*.nb13 =*/ nb13,- /*.ns20 =*/ int32_t(nb21/nb20),- /*.nb21 =*/ nb21,- /*.nb22 =*/ nb22,- /*.nb23 =*/ nb23,+ /*.ns10 =*/ ns10,+ /*.nb11 =*/ nb11_attn,+ /*.nb12 =*/ nb12_attn,+ /*.nb13 =*/ nb13_attn,+ /*.ns20 =*/ ns20,+ /*.nb21 =*/ nb21_attn,+ /*.nb22 =*/ nb22_attn,+ /*.nb23 =*/ nb23_attn,/*.ne31 =*/ ne31,/*.ne32 =*/ ne32,/*.ne33 =*/ ne33,@@ -3139,13 +3319,13 @@ int ggml_metal_op_flash_attn_ext(ggml_metal_op_t ctx, int idx) {/*.logit_softcap =*/ logit_softcap,};- auto pipeline = ggml_metal_library_get_pipeline_flash_attn_ext(lib, op, has_mask, has_sinks, has_bias, has_scap, has_kvpad, nsg);+ auto pipeline = ggml_metal_library_get_pipeline_flash_attn_ext(lib, op, has_mask, has_sinks, has_bias, has_scap, has_kvpad, nsg, use_kv_f16, ns10, ns20);ggml_metal_encoder_set_pipeline(enc, pipeline);ggml_metal_encoder_set_bytes (enc, &args, sizeof(args), 0);ggml_metal_encoder_set_buffer (enc, bid_src0, 1);- ggml_metal_encoder_set_buffer (enc, bid_src1, 2);- ggml_metal_encoder_set_buffer (enc, bid_src2, 3);+ ggml_metal_encoder_set_buffer (enc, bid_k, 2);+ ggml_metal_encoder_set_buffer (enc, bid_v, 3);ggml_metal_encoder_set_buffer (enc, bid_src3, 4);ggml_metal_encoder_set_buffer (enc, bid_src4, 5);ggml_metal_encoder_set_buffer (enc, bid_pad, 6);@@ -3177,12 +3357,12 @@ int ggml_metal_op_flash_attn_ext(ggml_metal_op_t ctx, int idx) {/*.ne11 =*/ne11,/*.ne_12_2 =*/ne12,/*.ne_12_3 =*/ne13,- /*.nb11 =*/nb11,- /*.nb12 =*/nb12,- /*.nb13 =*/nb13,- /*.nb21 =*/nb21,- /*.nb22 =*/nb22,- /*.nb23 =*/nb23,+ /*.nb11 =*/nb11_attn,+ /*.nb12 =*/nb12_attn,+ /*.nb13 =*/nb13_attn,+ /*.nb21 =*/nb21_attn,+ /*.nb22 =*/nb22_attn,+ /*.nb23 =*/nb23_attn,/*.ne31 =*/ne31,/*.ne32 =*/ne32,/*.ne33 =*/ne33,@@ -3195,8 +3375,8 @@ int ggml_metal_op_flash_attn_ext(ggml_metal_op_t ctx, int idx) {ggml_metal_encoder_set_pipeline(enc, pipeline0);ggml_metal_encoder_set_bytes (enc, &args0, sizeof(args0), 0);- ggml_metal_encoder_set_buffer (enc, bid_src1, 1);- ggml_metal_encoder_set_buffer (enc, bid_src2, 2);+ ggml_metal_encoder_set_buffer (enc, bid_k, 1);+ ggml_metal_encoder_set_buffer (enc, bid_v, 2);ggml_metal_encoder_set_buffer (enc, bid_src3, 3);ggml_metal_encoder_set_buffer (enc, bid_pad, 4);@@ -3242,6 +3422,9 @@ int ggml_metal_op_flash_attn_ext(ggml_metal_op_t ctx, int idx) {}}+ const int32_t ns10 = nb11_attn/nb10_attn;+ const int32_t ns20 = nb21_attn/nb20_attn;+ggml_metal_kargs_flash_attn_ext_vec args = {/*.ne01 =*/ ne01,/*.ne02 =*/ ne02,@@ -3252,14 +3435,14 @@ int ggml_metal_op_flash_attn_ext(ggml_metal_op_t ctx, int idx) {/*.ne11 =*/ ne11,/*.ne_12_2 =*/ ne12,/*.ne_12_3 =*/ ne13,- /*.ns10 =*/ int32_t(nb11/nb10),- /*.nb11 =*/ nb11,- /*.nb12 =*/ nb12,- /*.nb13 =*/ nb13,- /*.ns20 =*/ int32_t(nb21/nb20),- /*.nb21 =*/ nb21,- /*.nb22 =*/ nb22,- /*.nb23 =*/ nb23,+ /*.ns10 =*/ ns10,+ /*.nb11 =*/ nb11_attn,+ /*.nb12 =*/ nb12_attn,+ /*.nb13 =*/ nb13_attn,+ /*.ns20 =*/ ns20,+ /*.nb21 =*/ nb21_attn,+ /*.nb22 =*/ nb22_attn,+ /*.nb23 =*/ nb23_attn,/*.ne31 =*/ ne31,/*.ne32 =*/ ne32,/*.ne33 =*/ ne33,@@ -3277,15 +3460,15 @@ int ggml_metal_op_flash_attn_ext(ggml_metal_op_t ctx, int idx) {/*.logit_softcap =*/ logit_softcap,};- auto pipeline = ggml_metal_library_get_pipeline_flash_attn_ext_vec(lib, op, has_mask, has_sinks, has_bias, has_scap, has_kvpad, nsg, nwg);+ auto pipeline = ggml_metal_library_get_pipeline_flash_attn_ext_vec(lib, op, has_mask, has_sinks, has_bias, has_scap, has_kvpad, nsg, nwg, use_kv_f16, ns10, ns20);GGML_ASSERT(nsg*32 <= ggml_metal_pipeline_max_theads_per_threadgroup(pipeline));ggml_metal_encoder_set_pipeline(enc, pipeline);ggml_metal_encoder_set_bytes (enc, &args, sizeof(args), 0);ggml_metal_encoder_set_buffer (enc, bid_src0, 1);- ggml_metal_encoder_set_buffer (enc, bid_src1, 2);- ggml_metal_encoder_set_buffer (enc, bid_src2, 3);+ ggml_metal_encoder_set_buffer (enc, bid_k, 2);+ ggml_metal_encoder_set_buffer (enc, bid_v, 3);ggml_metal_encoder_set_buffer (enc, bid_src3, 4);ggml_metal_encoder_set_buffer (enc, bid_src4, 5);diff --git a/ggml/src/ggml-metal/ggml-metal-ops.h b/ggml/src/ggml-metal/ggml-metal-ops.hindex b03b59e0bd..159a628d04 100644--- a/ggml/src/ggml-metal/ggml-metal-ops.h+++ b/ggml/src/ggml-metal/ggml-metal-ops.h@@ -42,6 +42,7 @@ bool ggml_metal_op_flash_attn_ext_use_vec(const struct ggml_tensor * op);size_t ggml_metal_op_flash_attn_ext_extra_pad(const struct ggml_tensor * op);size_t ggml_metal_op_flash_attn_ext_extra_blk(const struct ggml_tensor * op);size_t ggml_metal_op_flash_attn_ext_extra_tmp(const struct ggml_tensor * op);+size_t ggml_metal_op_flash_attn_ext_extra_kv_f16(const struct ggml_tensor * op);int ggml_metal_op_concat (ggml_metal_op_t ctx, int idx);int ggml_metal_op_repeat (ggml_metal_op_t ctx, int idx);diff --git a/ggml/src/ggml-metal/ggml-metal.cpp b/ggml/src/ggml-metal/ggml-metal.cppindex ef3c92f271..0e8d409e0b 100644--- a/ggml/src/ggml-metal/ggml-metal.cpp+++ b/ggml/src/ggml-metal/ggml-metal.cpp@@ -225,6 +225,7 @@ static size_t ggml_backend_metal_buffer_type_get_alloc_size(ggml_backend_buffer_res += ggml_metal_op_flash_attn_ext_extra_pad(tensor);res += ggml_metal_op_flash_attn_ext_extra_blk(tensor);res += ggml_metal_op_flash_attn_ext_extra_tmp(tensor);+ res += ggml_metal_op_flash_attn_ext_extra_kv_f16(tensor);} break;case GGML_OP_CUMSUM:case GGML_OP_ARGSORT:diff --git a/ggml/src/ggml-metal/ggml-metal.metal b/ggml/src/ggml-metal/ggml-metal.metalindex 0537fa4cf8..949931c8dc 100644--- a/ggml/src/ggml-metal/ggml-metal.metal+++ b/ggml/src/ggml-metal/ggml-metal.metal@@ -6318,6 +6318,53 @@ template [[host_name("kernel_fwht_f32_128")]] kernel kernel_fwht_t kernel_fwht_ftemplate [[host_name("kernel_fwht_f32_256")]] kernel kernel_fwht_t kernel_fwht_f32<256>;template [[host_name("kernel_fwht_f32_512")]] kernel kernel_fwht_t kernel_fwht_f32<512>;+// dequantize a quantized KV cache tensor to contiguous F16 before running the F16 flash attention kernels+// - one thread per block; dispatched separately for K and V+// - ref: https://github.com/ggml-org/llama.cpp/pull/27390+template <+ typename block_t,+ short QK,+ void (*deq_t4x4)(device const block_t *, short, thread float4x4 &)>+kernel void kernel_flash_attn_ext_kv_f16(+ constant ggml_metal_kargs_flash_attn_ext_kv_f16 & args,+ device const char * x,+ device half * x_dst,+ uint gid [[thread_position_in_grid]]) {+ if (gid >= (uint) args.nblocks) {+ return;+ }++ const uint nb = args.ne0/QK;+ const uint i0 = gid%nb;+ uint ib = gid/nb;+ const uint i1 = ib%args.ne1;+ ib /= args.ne1;+ const uint i2 = ib%args.ne2;+ const uint i3 = ib/args.ne2;++ const uint64_t offs = i0*args.nb0 + i1*args.nb1 + i2*args.nb2 + i3*args.nb3;++ device const block_t * src = (device const block_t *) (x + offs);+ device half4 * dst = (device half4 *) x_dst + (QK/4)*gid;++ for (short i = 0; i < QK/16; ++i) {+ float4x4 reg;+ deq_t4x4(src, i, reg);+ dst[4*i + 0] = (half4) reg[0];+ dst[4*i + 1] = (half4) reg[1];+ dst[4*i + 2] = (half4) reg[2];+ dst[4*i + 3] = (half4) reg[3];+ }+}++typedef decltype(kernel_flash_attn_ext_kv_f16<block_q8_0, 32, dequantize_q8_0>) kernel_flash_attn_ext_kv_f16_t;++template [[host_name("kernel_flash_attn_ext_kv_q4_0_f16")]] kernel kernel_flash_attn_ext_kv_f16_t kernel_flash_attn_ext_kv_f16<block_q4_0, 32, dequantize_q4_0>;+template [[host_name("kernel_flash_attn_ext_kv_q4_1_f16")]] kernel kernel_flash_attn_ext_kv_f16_t kernel_flash_attn_ext_kv_f16<block_q4_1, 32, dequantize_q4_1>;+template [[host_name("kernel_flash_attn_ext_kv_q5_0_f16")]] kernel kernel_flash_attn_ext_kv_f16_t kernel_flash_attn_ext_kv_f16<block_q5_0, 32, dequantize_q5_0>;+template [[host_name("kernel_flash_attn_ext_kv_q5_1_f16")]] kernel kernel_flash_attn_ext_kv_f16_t kernel_flash_attn_ext_kv_f16<block_q5_1, 32, dequantize_q5_1>;+template [[host_name("kernel_flash_attn_ext_kv_q8_0_f16")]] kernel kernel_flash_attn_ext_kv_f16_t kernel_flash_attn_ext_kv_f16<block_q8_0, 32, dequantize_q8_0>;+constant bool FC_flash_attn_ext_pad_has_mask [[function_constant(FC_FLASH_ATTN_EXT_PAD + 0)]];constant int32_t FC_flash_attn_ext_pad_ncpsg [[function_constant(FC_FLASH_ATTN_EXT_PAD + 25)]];diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cppindex c9946c7ae4..17098825bc 100644--- a/tests/test-backend-ops.cpp+++ b/tests/test-backend-ops.cpp@@ -9934,6 +9934,20 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {test_cases.emplace_back(new test_flash_attn_ext(64, 128, 4, {1, 1}, 128, 2, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q4_0, GGML_TYPE_Q2_0));test_cases.emplace_back(new test_flash_attn_ext(128, 64, 4, {1, 1}, 64, 2, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q2_0, GGML_TYPE_F16));+ // q8_0 KV cases: decode and prompt batches, KV pad, permuted KV, feature flags, and long context+ test_cases.emplace_back(new test_flash_attn_ext(256, 256, 2, {16, 1}, 113, 1, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q8_0, GGML_TYPE_Q8_0));+ test_cases.emplace_back(new test_flash_attn_ext(256, 256, 2, {16, 1}, 1024, 1, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q8_0, GGML_TYPE_Q8_0));+ test_cases.emplace_back(new test_flash_attn_ext(256, 256, 2, {16, 1}, 1024, 1, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q8_0, GGML_TYPE_Q8_0, {0, 2, 1, 3}));+ test_cases.emplace_back(new test_flash_attn_ext(256, 256, 2, {16, 2}, 1025, 1, true, true, 8, 30, GGML_PREC_F32, GGML_TYPE_Q8_0, GGML_TYPE_Q8_0));+ test_cases.emplace_back(new test_flash_attn_ext(256, 256, 2, {16, 1}, 1025, 64, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q8_0, GGML_TYPE_Q8_0, {0, 2, 1, 3}));+ test_cases.emplace_back(new test_flash_attn_ext(256, 256, 2, {16, 1}, 16384, 1, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q8_0, GGML_TYPE_Q8_0));++ // MLA shape (V is a view of K) with quantized KV+ // (the test harness builds V as a view of K for this shape; see build_graph)+ test_cases.emplace_back(new test_flash_attn_ext(576, 512, 1, {20, 1}, 113, 1, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q8_0, GGML_TYPE_Q8_0));+ test_cases.emplace_back(new test_flash_attn_ext(576, 512, 1, {20, 1}, 1024, 1, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q8_0, GGML_TYPE_Q8_0));+ test_cases.emplace_back(new test_flash_attn_ext(576, 512, 1, {20, 1}, 1024, 64, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q8_0, GGML_TYPE_Q8_0));+// large-KV F16 cases (Qwen3.6-27B geometry and a llama-class control): the upstream matrix// stops at kv=1024, blind to long-context FA bugs (e.g. the oneDNN SDPA ordering race on BMG).for (int64_t kv : { 4096, 16384 }) {@@ -10325,6 +10339,21 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_perf() {test_cases.emplace_back(new test_flash_attn_ext(64, 64, 8, {8, 1}, 7680, 1, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q8_0, GGML_TYPE_Q8_0));test_cases.emplace_back(new test_flash_attn_ext(64, 64, 8, {8, 1}, 7680, 512, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q8_0, GGML_TYPE_Q8_0));+ // q8_0 KV cases with long context (decode and prompt)+ test_cases.emplace_back(new test_flash_attn_ext(256, 256, 2, {16, 1}, 128, 1, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q8_0, GGML_TYPE_Q8_0));+ test_cases.emplace_back(new test_flash_attn_ext(256, 256, 2, {16, 1}, 512, 1, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q8_0, GGML_TYPE_Q8_0));+ test_cases.emplace_back(new test_flash_attn_ext(256, 256, 2, {16, 1}, 1024, 1, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q8_0, GGML_TYPE_Q8_0));+ test_cases.emplace_back(new test_flash_attn_ext(256, 256, 2, {16, 1}, 2048, 1, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q8_0, GGML_TYPE_Q8_0));+ test_cases.emplace_back(new test_flash_attn_ext(256, 256, 2, {16, 1}, 4096, 1, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q8_0, GGML_TYPE_Q8_0));+ test_cases.emplace_back(new test_flash_attn_ext(256, 256, 2, {16, 1}, 10000, 1, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q8_0, GGML_TYPE_Q8_0));+ test_cases.emplace_back(new test_flash_attn_ext(256, 256, 2, {16, 1}, 20000, 1, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q8_0, GGML_TYPE_Q8_0));+ test_cases.emplace_back(new test_flash_attn_ext(256, 256, 2, {16, 1}, 10000, 1, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16));+ test_cases.emplace_back(new test_flash_attn_ext(256, 256, 2, {16, 1}, 20000, 1, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16));+ test_cases.emplace_back(new test_flash_attn_ext(256, 256, 2, {16, 1}, 10000, 512, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q8_0, GGML_TYPE_Q8_0));+ test_cases.emplace_back(new test_flash_attn_ext(256, 256, 2, {16, 1}, 20000, 512, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q8_0, GGML_TYPE_Q8_0));+ test_cases.emplace_back(new test_flash_attn_ext(256, 256, 2, {16, 1}, 10000, 512, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16));+ test_cases.emplace_back(new test_flash_attn_ext(256, 256, 2, {16, 1}, 20000, 512, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16));+for (int kv : { 4096, 8192, 16384, }) {for (int hs : { 64, 128, }) {for (int nr : { 1, 4, }) {