LLM inference in C/C++
C++ ggml
metal : add sparse FA (#28098)
Georgi Gerganov7bb0fc18f61a
diff --git a/ggml/src/ggml-metal/ggml-metal-device.cpp b/ggml/src/ggml-metal/ggml-metal-device.cppindex b8d2ef9ce2..c296d17b15 100644--- a/ggml/src/ggml-metal/ggml-metal-device.cpp+++ b/ggml/src/ggml-metal/ggml-metal-device.cpp@@ -1577,6 +1577,26 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_flash_attn_ext(return res;}+ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_flash_attn_ext_vec_idx(+ ggml_metal_library_t lib,+ const ggml_tensor * op) {+ assert(op->op == GGML_OP_FLASH_ATTN_EXT);+ assert(op->src[3]);++ char name[256];++ snprintf(name, 256, "kernel_flash_attn_ext_vec_idx");++ ggml_metal_pipeline_with_params res = ggml_metal_library_get_pipeline(lib, name);+ if (!res.pipeline) {+ res = ggml_metal_library_compile_pipeline(lib, name, name, nullptr);+ }++ GGML_UNUSED(op);++ return res;+}+ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_flash_attn_ext_vec(ggml_metal_library_t lib,const ggml_tensor * op,@@ -1585,6 +1605,7 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_flash_attn_ext_vbool has_bias,bool has_scap,bool has_kvpad,+ bool has_sparse,int32_t nqpsg,int32_t ne,int32_t nsg,@@ -1614,13 +1635,14 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_flash_attn_ext_vdv,qne_suffix);- snprintf(name, 256, "%s_mask=%d_sink=%d_bias=%d_scap=%d_kvpad=%d_ns10=%d_ns20=%d_nsg=%d_nwg=%d",+ snprintf(name, 256, "%s_mask=%d_sink=%d_bias=%d_scap=%d_kvpad=%d_sparse=%d_ns10=%d_ns20=%d_nsg=%d_nwg=%d",base,has_mask,has_sinks,has_bias,has_scap,has_kvpad,+ has_sparse,ns10,ns20,nsg, nwg);@@ -1633,7 +1655,8 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_flash_attn_ext_vggml_metal_cv_set_bool(cv, has_sinks, FC_FLASH_ATTN_EXT_VEC + 1);ggml_metal_cv_set_bool(cv, has_bias, FC_FLASH_ATTN_EXT_VEC + 2);ggml_metal_cv_set_bool(cv, has_scap, FC_FLASH_ATTN_EXT_VEC + 3);- ggml_metal_cv_set_bool(cv, has_kvpad, FC_FLASH_ATTN_EXT_VEC + 4);+ ggml_metal_cv_set_bool(cv, has_kvpad, FC_FLASH_ATTN_EXT_VEC + 4);+ ggml_metal_cv_set_bool(cv, has_sparse, FC_FLASH_ATTN_EXT_VEC + 5);ggml_metal_cv_set_int32(cv, ns10, FC_FLASH_ATTN_EXT_VEC + 20);ggml_metal_cv_set_int32(cv, ns20, FC_FLASH_ATTN_EXT_VEC + 21);diff --git a/ggml/src/ggml-metal/ggml-metal-device.h b/ggml/src/ggml-metal/ggml-metal-device.hindex ae4871d358..31fc07d44d 100644--- a/ggml/src/ggml-metal/ggml-metal-device.h+++ b/ggml/src/ggml-metal/ggml-metal-device.h@@ -201,6 +201,10 @@ struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_flash_attint32_t ns10,int32_t ns20);+struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_flash_attn_ext_vec_idx(+ ggml_metal_library_t lib,+ const struct ggml_tensor * op);+struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_flash_attn_ext_vec(ggml_metal_library_t lib,const struct ggml_tensor * op,@@ -209,6 +213,7 @@ struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_flash_attbool has_bias,bool has_scap,bool has_kvpad,+ bool has_sparse,int32_t nqpsg,int32_t ne,int32_t nsg,diff --git a/ggml/src/ggml-metal/ggml-metal-impl.h b/ggml/src/ggml-metal/ggml-metal-impl.hindex bdcd9c9e3d..30e40f527f 100644--- a/ggml/src/ggml-metal/ggml-metal-impl.h+++ b/ggml/src/ggml-metal/ggml-metal-impl.h@@ -458,8 +458,21 @@ typedef struct {float m1;int32_t n_head_log2;float logit_softcap;+ int32_t n_kv_max_padded;} ggml_metal_kargs_flash_attn_ext_vec;+typedef struct {+ int32_t ne30;+ int32_t ne31;+ int32_t ne32;+ int32_t ne33;+ uint64_t nb31;+ uint64_t nb32;+ uint64_t nb33;+ int32_t n_kv_max;+ int32_t n_kv_max_padded;+} ggml_metal_kargs_flash_attn_ext_vec_idx;+typedef struct {int32_t nrows;} ggml_metal_kargs_flash_attn_ext_vec_reduce;diff --git a/ggml/src/ggml-metal/ggml-metal-ops.cpp b/ggml/src/ggml-metal/ggml-metal-ops.cppindex c5c4ec46bc..3db8bca437 100644--- a/ggml/src/ggml-metal/ggml-metal-ops.cpp+++ b/ggml/src/ggml-metal/ggml-metal-ops.cpp@@ -2857,6 +2857,65 @@ static bool ggml_metal_op_flash_attn_ext_use_kv_f16(const ggml_tensor * op) {}}+// returns the n_kv_max hint if the sparse path is available for this op, or 0 otherwise+// the mask (src[3]) remains the single source of truth: finite entries are the valid KV positions,+// n_kv_max is only an upper bound on their number per mask row, used to size the index lists+static int ggml_metal_op_flash_attn_ext_n_kv_max_sparse(const ggml_tensor * op) {+ assert(op->op == GGML_OP_FLASH_ATTN_EXT);++ int32_t n_kv_max = 0;+ memcpy(&n_kv_max, ((const int32_t *) op->op_params) + 4, sizeof(n_kv_max));++ if (n_kv_max <= 0) {+ return 0;+ }++ // the sparse indices are gathered from the mask+ if (!op->src[3]) {+ return 0;+ }++ // bound the size of the index lists+ if (n_kv_max > 4096) {+ return 0;+ }++ // vec kernel instantiations exist for these (type, dk, dv) combinations only+ const int64_t dk = op->src[1]->ne[0];+ const int64_t dv = op->src[2]->ne[0];++ const bool dk_dv_ok = (dk == 32 && dv == 32) ||+ (dk == 64 && dv == 64) ||+ (dk == 96 && dv == 96) ||+ (dk == 128 && dv == 128) ||+ (dk == 192 && dv == 128) ||+ (dk == 192 && dv == 192) ||+ (dk == 256 && dv == 256) ||+ (dk == 320 && dv == 256) ||+ (dk == 512 && dv == 512) ||+ (dk == 576 && dv == 512);++ if (!dk_dv_ok) {+ return 0;+ }++ switch (op->src[1]->type) {+ case GGML_TYPE_F16:+ case GGML_TYPE_BF16:+ case GGML_TYPE_F32:+ case GGML_TYPE_Q4_0:+ case GGML_TYPE_Q4_1:+ case GGML_TYPE_Q5_0:+ case GGML_TYPE_Q5_1:+ case GGML_TYPE_Q8_0:+ break;+ default:+ return 0;+ }++ return n_kv_max;+}+// in some models (e.g. MLA-based), V is a view of K (the first ne20 elements of each K row);// the dequantized V is then a view of the dequantized K and does not need its own dequant or scratch// - ref: https://github.com/ggml-org/llama.cpp/pull/13435@@ -3027,6 +3086,24 @@ size_t ggml_metal_op_flash_attn_ext_extra_kv_f16(const ggml_tensor * op) {return k_size + v_size;}+// size of the sparse index lists: one list of KV indices per mask row,+// padded with -1 up to a multiple of OP_FLASH_ATTN_EXT_VEC_NCPSG+size_t ggml_metal_op_flash_attn_ext_extra_idx(const ggml_tensor * op) {+ assert(op->op == GGML_OP_FLASH_ATTN_EXT);++ GGML_TENSOR_LOCALS( int32_t, ne3, op->src[3], ne);++ const int n_kv_max = ggml_metal_op_flash_attn_ext_n_kv_max_sparse(op);++ if (n_kv_max <= 0) {+ return 0;+ }++ const int n_kv_max_padded = GGML_PAD(n_kv_max, OP_FLASH_ATTN_EXT_VEC_NCPSG);++ return GGML_PAD(sizeof(int32_t)*(size_t) n_kv_max_padded*ne31*ne32*ne33, 16);+}+int ggml_metal_op_flash_attn_ext(ggml_metal_op_t ctx, int idx) {ggml_tensor * op = ctx->node(idx);@@ -3104,7 +3181,16 @@ int ggml_metal_op_flash_attn_ext(ggml_metal_op_t ctx, int idx) {ggml_metal_buffer_id bid_kv_f16 = bid_tmp;bid_kv_f16.offs += ggml_metal_op_flash_attn_ext_extra_tmp(op);- const bool use_kv_f16 = ggml_metal_op_flash_attn_ext_use_kv_f16(op);+ // sparse path: gather the finite mask entries into index lists and run the vec kernels over them+ const int n_kv_max_sparse = ggml_metal_op_flash_attn_ext_n_kv_max_sparse(op);+ const bool use_sparse = n_kv_max_sparse > 0;+ const int n_kv_max_padded = use_sparse ? GGML_PAD(n_kv_max_sparse, OP_FLASH_ATTN_EXT_VEC_NCPSG) : 0;++ // the vec kernels dequantize the KV inline; no need for the F16 dequant pass in the sparse path+ const bool use_kv_f16 = !use_sparse && ggml_metal_op_flash_attn_ext_use_kv_f16(op);++ ggml_metal_buffer_id bid_idx = bid_kv_f16;+ bid_idx.offs += ggml_metal_op_flash_attn_ext_extra_kv_f16(op);ggml_metal_buffer_id bid_k = bid_src1;ggml_metal_buffer_id bid_v = bid_src2;@@ -3206,7 +3292,7 @@ int ggml_metal_op_flash_attn_ext(ggml_metal_op_t ctx, int idx) {}}- if (!ggml_metal_op_flash_attn_ext_use_vec(op)) {+ if (!use_sparse && !ggml_metal_op_flash_attn_ext_use_vec(op)) {// half8x8 kernelconst int nqptg = OP_FLASH_ATTN_EXT_NQPSG; // queries per threadgroupconst int ncpsg = OP_FLASH_ATTN_EXT_NCPSG; // cache values per simdgroup@@ -3378,13 +3464,18 @@ int ggml_metal_op_flash_attn_ext(ggml_metal_op_t ctx, int idx) {#undef FATTN_SMEM} else {// half4x4 kernel- auto cfg = ggml_metal_tuning::fa_vec_pick(- props_dev->device_id,- props_dev->gpu_family,- (int) op->src[1]->type,- (int) ne00, (int) ne20, // dk, dv (ne00 == dk for FA)- ne11, ne01);- int nqptg = cfg.Q; // queries per threadgroup+ // sparse: the index lists are per query row, so a threadgroup can share KV with Q == 1 only+ auto cfg = use_sparse+ ? ggml_metal_tuning::fa_vec_baseline_cfg((int) ne00, (int) ne20)+ : ggml_metal_tuning::fa_vec_pick(+ props_dev->device_id,+ props_dev->gpu_family,+ (int) op->src[1]->type,+ (int) ne00, (int) ne20, // dk, dv (ne00 == dk for FA)+ ne11, ne01);++ int nqptg = cfg.Q; // queries per threadgroup+const int ncpsg = OP_FLASH_ATTN_EXT_VEC_NCPSG; // cache values per simdgroup !! sync with kernel template arguments !!const int nhptg = 1; // heads per threadgroup@@ -3394,7 +3485,39 @@ int ggml_metal_op_flash_attn_ext(ggml_metal_op_t ctx, int idx) {bool need_sync = false;- const bool has_kvpad = ne11 % ncpsg != 0;+ const bool has_kvpad = !use_sparse && ne11 % ncpsg != 0;++ if (use_sparse) {+ assert(ggml_metal_op_flash_attn_ext_extra_idx(op) != 0);++ GGML_ASSERT(ne30 == ne11);++ ggml_metal_kargs_flash_attn_ext_vec_idx args0 = {+ /*.ne30 =*/ ne30,+ /*.ne31 =*/ ne31,+ /*.ne32 =*/ ne32,+ /*.ne33 =*/ ne33,+ /*.nb31 =*/ nb31,+ /*.nb32 =*/ nb32,+ /*.nb33 =*/ nb33,+ /*.n_kv_max =*/ n_kv_max_sparse,+ /*.n_kv_max_padded =*/ n_kv_max_padded,+ };++ auto pipeline0 = ggml_metal_library_get_pipeline_flash_attn_ext_vec_idx(lib, op);++ ggml_metal_encoder_set_pipeline(enc, pipeline0);+ ggml_metal_encoder_set_bytes (enc, &args0, sizeof(args0), 0);+ ggml_metal_encoder_set_buffer (enc, bid_src3, 1);+ ggml_metal_encoder_set_buffer (enc, bid_idx, 2);++ int nth = std::min(ggml_metal_pipeline_max_theads_per_threadgroup(pipeline0), 256);+ nth = std::max(32, (nth/32)*32);++ ggml_metal_encoder_dispatch_threadgroups(enc, ne31, ne32, ne33, nth, 1, 1);++ need_sync = true;+ }if (has_kvpad) {assert(ggml_metal_op_flash_attn_ext_extra_pad(op) != 0);@@ -3455,11 +3578,26 @@ int ggml_metal_op_flash_attn_ext(ggml_metal_op_t ctx, int idx) {// workgroups// each workgroup handles nsg*nkpsg cache valuesint32_t nwg = 1;- if (false) {- // for small KV caches, we could launch a single workgroup and write the results directly to dst/- // however, this does not lead to significant improvement, so disabled- nwg = 1;- nsg = 4;+ if (use_sparse) {+ if (ne01 > 32) {+ // large sparse batch+ nwg = 1;+ nsg = 1;+ if (n_kv_max_padded == 640) {+ nsg = 4; // 640 % (4*32) == 0+ } else {+ while (2*nwg*nsg*ncpsg < n_kv_max_padded && nsg < 4) {+ nsg *= 2;+ }+ }+ } else {+ // small sparse batch+ nwg = 32;+ nsg = 1;+ while (2*nwg*nsg*ncpsg < n_kv_max_padded && nsg < 4) {+ nsg *= 2;+ }+ }} else {nwg = 32;nsg = 1;@@ -3484,7 +3622,7 @@ int ggml_metal_op_flash_attn_ext(ggml_metal_op_t ctx, int idx) {/*.nb01 =*/ nb01,/*.nb02 =*/ nb02,/*.nb03 =*/ nb03,- /*.ne11 =*/ ne11,+ /*.ne11 =*/ use_sparse ? n_kv_max_padded : ne11,/*.ne_12_2 =*/ ne12,/*.ne_12_3 =*/ ne13,/*.ns10 =*/ ns10,@@ -3510,9 +3648,10 @@ int ggml_metal_op_flash_attn_ext(ggml_metal_op_t ctx, int idx) {/*.m1 =*/ m1,/*.n_head_log2 =*/ n_head_log2,/*.logit_softcap =*/ logit_softcap,+ /*.n_kv_max_padded =*/ n_kv_max_padded,};- auto pipeline = ggml_metal_library_get_pipeline_flash_attn_ext_vec(lib, op, has_mask, has_sinks, has_bias, has_scap, has_kvpad, nqptg, cfg.NE, nsg, nwg, use_kv_f16, ns10, ns20);+ auto pipeline = ggml_metal_library_get_pipeline_flash_attn_ext_vec(lib, op, has_mask, has_sinks, has_bias, has_scap, has_kvpad, use_sparse, nqptg, cfg.NE, nsg, nwg, use_kv_f16, ns10, ns20);GGML_ASSERT(nsg*32 <= ggml_metal_pipeline_max_theads_per_threadgroup(pipeline));@@ -3523,6 +3662,7 @@ int ggml_metal_op_flash_attn_ext(ggml_metal_op_t ctx, int idx) {ggml_metal_encoder_set_buffer (enc, bid_v, 3);ggml_metal_encoder_set_buffer (enc, bid_src3, 4);ggml_metal_encoder_set_buffer (enc, bid_src4, 5);+ ggml_metal_encoder_set_buffer (enc, use_sparse ? bid_idx : bid_src0, 8);const size_t smem = FATTN_SMEM(nsg);@@ -3530,8 +3670,6 @@ int ggml_metal_op_flash_attn_ext(ggml_metal_op_t ctx, int idx) {GGML_ASSERT(smem <= props_dev->max_theadgroup_memory_size);if (nwg == 1) {- assert(ggml_metal_op_flash_attn_ext_extra_tmp(op) == 0);-// using 1 workgroup -> write the result directly into dstggml_metal_encoder_set_buffer(enc, bid_pad, 6);ggml_metal_encoder_set_buffer(enc, bid_dst, 7);diff --git a/ggml/src/ggml-metal/ggml-metal-ops.h b/ggml/src/ggml-metal/ggml-metal-ops.hindex 159a628d04..f8fe50b468 100644--- a/ggml/src/ggml-metal/ggml-metal-ops.h+++ b/ggml/src/ggml-metal/ggml-metal-ops.h@@ -43,6 +43,7 @@ size_t ggml_metal_op_flash_attn_ext_extra_pad(const struct ggml_tensor * op);size_t ggml_metal_op_flash_attn_ext_extra_blk(const struct ggml_tensor * op);size_t ggml_metal_op_flash_attn_ext_extra_tmp(const struct ggml_tensor * op);size_t ggml_metal_op_flash_attn_ext_extra_kv_f16(const struct ggml_tensor * op);+size_t ggml_metal_op_flash_attn_ext_extra_idx(const struct ggml_tensor * op);int ggml_metal_op_concat (ggml_metal_op_t ctx, int idx);int ggml_metal_op_repeat (ggml_metal_op_t ctx, int idx);diff --git a/ggml/src/ggml-metal/ggml-metal.cpp b/ggml/src/ggml-metal/ggml-metal.cppindex 4d58dc821c..3bd6abd06f 100644--- a/ggml/src/ggml-metal/ggml-metal.cpp+++ b/ggml/src/ggml-metal/ggml-metal.cpp@@ -232,6 +232,7 @@ static size_t ggml_backend_metal_buffer_type_get_alloc_size(ggml_backend_buffer_res += ggml_metal_op_flash_attn_ext_extra_blk(tensor);res += ggml_metal_op_flash_attn_ext_extra_tmp(tensor);res += ggml_metal_op_flash_attn_ext_extra_kv_f16(tensor);+ res += ggml_metal_op_flash_attn_ext_extra_idx(tensor);} break;case GGML_OP_CUMSUM:case GGML_OP_ARGSORT:diff --git a/ggml/src/ggml-metal/kernels/fa.metal b/ggml/src/ggml-metal/kernels/fa.metalindex e95dec258a..d0e928d732 100644--- a/ggml/src/ggml-metal/kernels/fa.metal+++ b/ggml/src/ggml-metal/kernels/fa.metal@@ -1071,6 +1071,112 @@ constant int32_t FC_flash_attn_ext_vec_ns10 [[function_constant(FC_FLASH_ATTN_EXconstant int32_t FC_flash_attn_ext_vec_ns20 [[function_constant(FC_FLASH_ATTN_EXT_VEC + 21)]];constant int32_t FC_flash_attn_ext_vec_nsg [[function_constant(FC_FLASH_ATTN_EXT_VEC + 22)]];constant int32_t FC_flash_attn_ext_vec_nwg [[function_constant(FC_FLASH_ATTN_EXT_VEC + 23)]];+constant bool FC_flash_attn_ext_vec_has_sparse [[function_constant(FC_FLASH_ATTN_EXT_VEC + 5)]];++// compress the finite entries of each KQ mask row into a list of KV indices (ascending order),+// padded with -1 up to n_kv_max_padded (a multiple of OP_FLASH_ATTN_EXT_VEC_NCPSG)+// one threadgroup per mask row; the mask remains the single source of truth for the values+kernel void kernel_flash_attn_ext_vec_idx(+ constant ggml_metal_kargs_flash_attn_ext_vec_idx & args,+ device const half * mask,+ device int * idx,+ uint3 tgpig[[threadgroup_position_in_grid]],+ ushort tiitg[[thread_index_in_threadgroup]],+ ushort3 ntg[[threads_per_threadgroup]]) {+ constexpr short NW = N_SIMDWIDTH;+ constexpr short NLOCAL = 32; // max finite positions kept in registers per thread++ const int i1 = tgpig[0];+ const int i2 = tgpig[1];+ const int i3 = tgpig[2];++ device const half * pm = (device const half *) ((device const char *) mask + i1*args.nb31 + i2*args.nb32 + i3*args.nb33);+ device int * pidx = idx + (((int64_t)i3*args.ne32 + i2)*args.ne31 + i1)*args.n_kv_max_padded;++ const int n = args.ne30;+ const int q = n/ntg.x;+ const int r = n%ntg.x;++ // each thread handles a contiguous slice of the mask row+ const int r0 = q*tiitg + min((int) tiitg, r);+ const int r1 = r0 + q + (tiitg < r ? 1 : 0);++ // count the finite entries in the slice and keep their positions in registers (single mask read)+ int cnt = 0; // total finite entries in the slice+ int nloc = 0; // finite entries kept in registers+ int local[NLOCAL];+ for (int i = r0; i < r1; ++i) {+ if (isfinite((float) pm[i])) {+ if (nloc < NLOCAL) {+ local[nloc] = i;+ nloc++;+ }+ cnt++;+ }+ }++ const short sgitg = tiitg/NW;+ const short tiisg = tiitg%NW;++ threadgroup int tcount[8];++ // simd_sum is a collective: all lanes must evaluate it+ const int sg_sum = simd_sum(cnt);+ if (tiisg == 0) {+ tcount[sgitg] = sg_sum;+ }++ threadgroup_barrier(mem_flags::mem_threadgroup);++ int total = 0;+ for (short s = 0; s < ntg.x/NW; ++s) {+ total += tcount[s];+ }++ // base offset of this thread's slice in the output list (exclusive scan within the simdgroup)+ int sg_base = 0;+ for (short s = 0; s < sgitg; ++s) {+ sg_base += tcount[s];+ }++ // exclusive prefix scan of the per-thread counts within the simdgroup+ int incl = cnt;+ for (int d = 1; d < NW; d <<= 1) {+ const int v = simd_shuffle_up(incl, d);+ if (tiisg >= d) {+ incl += v;+ }+ }+ const int base = sg_base + (incl - cnt);++ // write the finite positions in order; if the hint is violated, keep only the first n_kv_max entries+ int j = 0;+ for (; j < nloc && base + j < args.n_kv_max; ++j) {+ pidx[base + j] = local[j];+ }++ // a dense mask may have more than NLOCAL finite entries in a slice; re-read the mask to write the rest+ if (cnt > nloc && base + nloc < args.n_kv_max) {+ int j2 = 0;+ for (int i = r0; i < r1; ++i) {+ if (isfinite((float) pm[i])) {+ if (j2 >= nloc) {+ pidx[base + j2] = i;+ }+ j2++;+ if (base + j2 >= args.n_kv_max) {+ break;+ }+ }+ }+ }++ // pad the tail of the list with -1+ const int count = min(total, args.n_kv_max);+ for (int i = count + tiitg; i < args.n_kv_max_padded; i += ntg.x) {+ pidx[i] = -1;+ }+}template<typename q4_t, // query types in shared memory@@ -1091,6 +1197,7 @@ template<short NE = 4, // head elements per threadshort Q = OP_FLASH_ATTN_EXT_VEC_NQPSG, // queries per threadgroupshort C = OP_FLASH_ATTN_EXT_VEC_NCPSG> // cache items per threadgroup+kernel void kernel_flash_attn_ext_vec(constant ggml_metal_kargs_flash_attn_ext_vec & args,device const char * q,@@ -1100,6 +1207,7 @@ kernel void kernel_flash_attn_ext_vec(device const char * sinks,device const char * pad,device char * dst,+ device const char * idx,threadgroup half * shmem_f16 [[threadgroup(0)]],uint3 tgpig[[threadgroup_position_in_grid]],ushort tiisg[[thread_index_in_simdgroup]],@@ -1137,8 +1245,8 @@ kernel void kernel_flash_attn_ext_vec(//const short T = PK + NSG*SH; // shared memory size per query in (half)- //threadgroup q_t * sq = (threadgroup q_t *) (shmem_f16 + 0*PK); // holds the query data- threadgroup q4_t * sq4 = (threadgroup q4_t *) (shmem_f16 + 0*PK); // same as above but in q4_t+ //threadgroup q_t * sq = (threadgroup q_t *) (shmem_f16 + 0*PK); // holds the query data+ threadgroup q4_t * sq4 = (threadgroup q4_t *) (shmem_f16 + 0*PK); // same as above but in q4_tthreadgroup s_t * ss = (threadgroup s_t *) (shmem_f16 + sgitg*SH + Q*NSG*PK); // scratch buffer for attentionthreadgroup s4_t * ss4 = (threadgroup s4_t *) (shmem_f16 + sgitg*SH + Q*NSG*PK); // same as above but in s4_tthreadgroup half * sm = (threadgroup half *) (shmem_f16 + sgitg*SH + 2*Q*C + Q*NSG*PK); // scratch buffer for mask@@ -1207,6 +1315,14 @@ kernel void kernel_flash_attn_ext_vec(// pointer to the maskdevice const half * pm_base = (device const half *) (mask + iq1*Q*args.nb31 + (iq2%args.ne32)*args.nb32 + (iq3%args.ne33)*args.nb33);+ // sparse indices: the list of finite mask entries per query row+ // the sparse path requires Q == 1 (enforced by the host)+ device const int * pidx = nullptr;+ if (FC_flash_attn_ext_vec_has_sparse) {+ pidx = (device const int *) idx ++ ((int64_t)(iq3%args.ne33)*args.ne32 + (iq2%args.ne32))*args.ne31*args.n_kv_max_padded + (iq1%args.ne31)*args.n_kv_max_padded;+ }+float slope = 1.0f;// ALiBi@@ -1265,11 +1381,22 @@ kernel void kernel_flash_attn_ext_vec(}if (FC_flash_attn_ext_vec_has_mask) {- FOR_UNROLL (short qq = 0; qq < Q; ++qq) {- if ((iq1*Q + qq) < args.ne01) {- sm[qq*C + tiisg] = pm[qq][ic + tiisg];- } else {- sm[qq*C + tiisg] = -MAXHALF;+ if (FC_flash_attn_ext_vec_has_sparse) {+ FOR_UNROLL (short qq = 0; qq < Q; ++qq) {+ const int i11 = pidx[ic + tiisg];+ if ((iq1*Q + qq) < args.ne01 && i11 >= 0) {+ sm[qq*C + tiisg] = pm[qq][i11];+ } else {+ sm[qq*C + tiisg] = -MAXHALF;+ }+ }+ } else {+ FOR_UNROLL (short qq = 0; qq < Q; ++qq) {+ if ((iq1*Q + qq) < args.ne01) {+ sm[qq*C + tiisg] = pm[qq][ic + tiisg];+ } else {+ sm[qq*C + tiisg] = -MAXHALF;+ }}}} else {@@ -1280,6 +1407,7 @@ kernel void kernel_flash_attn_ext_vec(}}+ // skip -INF mask{bool any_finite = false;FOR_UNROLL (short qq = 0; qq < Q; ++qq) {@@ -1294,9 +1422,13 @@ kernel void kernel_flash_attn_ext_vec(// Q*K^T{- device const k4_t * pk4 = (device const k4_t *) (k + ic*args.nb11);+ device const k4_t * pk4 = nullptr;++ if (!FC_flash_attn_ext_vec_has_sparse) {+ pk4 = (device const k4_t *) (k + ic*args.nb11);- pk4 += ty*NS10/4 + tx;+ pk4 += ty*NS10/4 + tx;+ }qk_t mqk[Q][C/NE];FOR_UNROLL (short qq = 0; qq < Q; ++qq) {@@ -1307,7 +1439,35 @@ kernel void kernel_flash_attn_ext_vec(// each simdgroup processes Q queries and NE (NW/NL) cache elementsFOR_UNROLL (short cc = 0; cc < C/NE; ++cc) {- if (is_same<kd4_t, k4_t>::value) {+ if (FC_flash_attn_ext_vec_has_sparse) {+ // the KV rows are gathered from the index list; -1 entries are padding+ const int i11 = pidx[ic + NE*cc + ty];+ if (i11 >= 0) {+ if (is_same<kd4_t, k4_t>::value) {+ device const k4_t * pk4s = (device const k4_t *) (k + i11*args.nb11) + tx;+ FOR_UNROLL (short ii = 0; ii < DK4/NL; ++ii) {+ const k4_t k_elem = pk4s[ii*NL];+ FOR_UNROLL (short qq = 0; qq < Q; ++qq) {+ mqk[qq][cc] += dot((float4) k_elem, (float4) sq4[qq*PK4 + ii*NL + tx]);+ }+ }+ } else {+ device const kd4_t * pk = (device const kd4_t *) (k + i11*args.nb11);++ k4_t mk;++ FOR_UNROLL (short ii = 0; ii < DK4/NL; ++ii) {+ const short i = ii*NL + tx;++ deq_k_t4(pk + i/nl_k, i%nl_k, mk);++ FOR_UNROLL (short qq = 0; qq < Q; ++qq) {+ mqk[qq][cc] += dot((float4) mk, (float4) sq4[qq*PK4 + i]);+ }+ }+ }+ }+ } else if (is_same<kd4_t, k4_t>::value) {FOR_UNROLL (short ii = 0; ii < DK4/NL; ++ii) {const k4_t k_elem = pk4[cc*NE*NS10/4 + ii*NL];FOR_UNROLL (short qq = 0; qq < Q; ++qq) {@@ -1422,7 +1582,40 @@ kernel void kernel_flash_attn_ext_vec(}}- if (is_same<vd4_t, v4_t>::value) {+ if (FC_flash_attn_ext_vec_has_sparse) {+ FOR_UNROLL (short cc = 0; cc < C/NE; ++cc) {+ // the KV rows are gathered from the index list; -1 entries are padding+ const int i11 = pidx[ic + NE*cc + ty];+ if (i11 >= 0) {+ if (is_same<vd4_t, v4_t>::value) {+ device const v4_t * pv4 = (device const v4_t *) (v + i11*args.nb21);++ pv4 += tx;++ FOR_UNROLL (short ii = 0; ii < DV4/NL; ++ii) {+ const v4_t v_elem = pv4[ii*NL];+ FOR_UNROLL (short qq = 0; qq < Q; ++qq) {+ lo[qq][ii] += o4_t(float4(v_elem)*float4(ss[qq*C + cc*NE + ty]));+ }+ }+ } else {+ device const vd4_t * pv4 = (device const vd4_t *) (v + i11*args.nb21);++ FOR_UNROLL (short ii = 0; ii < DV4/NL; ++ii) {+ const short i = ii*NL + tx;++ v4_t mv;++ deq_v_t4(pv4 + i/nl_v, i%nl_v, mv);++ FOR_UNROLL (short qq = 0; qq < Q; ++qq) {+ lo[qq][ii] += o4_t(float4(mv)*float4(ss[qq*C + cc*NE + ty]));+ }+ }+ }+ }+ }+ } else if (is_same<vd4_t, v4_t>::value) {device const v4_t * pv4 = (device const v4_t *) (v + ic*args.nb21);pv4 += ty*NS20/4 + tx;diff --git a/src/models/qwen4exp.cpp b/src/models/qwen4exp.cppindex 773204a5a1..1484c9b07b 100644--- a/src/models/qwen4exp.cpp+++ b/src/models/qwen4exp.cpp@@ -744,6 +744,9 @@ ggml_tensor * llama_model_qwen4exp::graph::build_attn_qsa(ggml_tensor * k = mctx_cur->get_k(ctx0, il);ggml_tensor * v = mctx_cur->get_v(ctx0, il);+ // TODO: enable sparse attention when we are ready+ // ref: https://github.com/ggml-org/llama.cpp/pull/27970+ //ggml_tensor * cur = build_attn_mha(q, k, v, nullptr, kq_mask_top_k, nullptr, nullptr, top_k->ne[0], kq_scale, il);ggml_tensor * cur = build_attn_mha(q, k, v, nullptr, kq_mask_top_k, nullptr, nullptr, 0, kq_scale, il);cb(cur, "kqv_out", il);diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cppindex 59051a84ac..eff63575f2 100644--- a/tests/test-backend-ops.cpp+++ b/tests/test-backend-ops.cpp@@ -191,7 +191,7 @@ static void init_tensor_kq_mask(ggml_tensor * tensor, float min = -1.0f, float mstatic void init_tensor_kq_mask_sparse(ggml_tensor * tensor, int64_t n_kv_max) {GGML_ASSERT(tensor->type == GGML_TYPE_F16);- GGML_ASSERT(n_kv_max > 1 && n_kv_max <= tensor->ne[0]);+ GGML_ASSERT(n_kv_max > 0 && n_kv_max <= tensor->ne[0]);const int64_t ne0 = tensor->ne[0];const int64_t nrows = ggml_nrows(tensor);@@ -10295,6 +10295,22 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {test_cases.emplace_back(new test_flash_attn_ext(576, 512, 1, {16, 2}, 4096, 2, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16, {0, 1, 2, 3}, true, true, 768));test_cases.emplace_back(new test_flash_attn_ext(512, 512, 1, { 8, 1}, 4096, 1, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16, {0, 1, 2, 3}, true, false, 2304));test_cases.emplace_back(new test_flash_attn_ext(256, 256, 1, { 8, 1}, 4096, 1, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16, {0, 1, 2, 3}, true, false, 512));+ test_cases.emplace_back(new test_flash_attn_ext(128, 128, 1, { 8, 1}, 4096, 1, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16, {0, 1, 2, 3}, true, false, 512));++ test_cases.emplace_back(new test_flash_attn_ext(128, 128, 1, { 8, 1}, 4096, 4, true, false, 8.0f, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16, {0, 1, 2, 3}, true, false, 512));++ // sparse mask with large batch size+ test_cases.emplace_back(new test_flash_attn_ext(512, 512, 1, { 8, 1}, 4096, 64, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16, {0, 1, 2, 3}, true, false, 512));+ test_cases.emplace_back(new test_flash_attn_ext(512, 512, 1, { 8, 1}, 4096, 64, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16, {0, 1, 2, 3}, true, false, 2048));+ test_cases.emplace_back(new test_flash_attn_ext(576, 512, 1, {16, 1}, 4096, 64, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16, {0, 1, 2, 3}, true, true, 512));+ test_cases.emplace_back(new test_flash_attn_ext(576, 512, 1, {16, 1}, 4096, 64, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16, {0, 1, 2, 3}, true, true, 2048));+ test_cases.emplace_back(new test_flash_attn_ext(256, 256, 1, { 8, 1}, 4096, 64, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16, {0, 1, 2, 3}, true, false, 512));+ test_cases.emplace_back(new test_flash_attn_ext(128, 128, 1, { 8, 1}, 4096, 64, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16, {0, 1, 2, 3}, true, false, 512));+ test_cases.emplace_back(new test_flash_attn_ext(128, 128, 1, { 8, 1}, 4096, 64, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16, {0, 1, 2, 3}, true, false, 2048));++ // sparse mask + quantized cache+ test_cases.emplace_back(new test_flash_attn_ext(128, 128, 1, { 8, 1}, 4096, 1, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q8_0, GGML_TYPE_Q8_0, {0, 1, 2, 3}, true, false, 512));+ test_cases.emplace_back(new test_flash_attn_ext(128, 128, 1, { 8, 1}, 4096, 64, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q8_0, GGML_TYPE_Q8_0, {0, 1, 2, 3}, true, false, 512));// more V-is-sub-view-of-K cases: other head shapes, and full views with equal head sizestest_cases.emplace_back(new test_flash_attn_ext(320, 256, 1, {32, 1}, 512, 1, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16, {0, 1, 2, 3}, true, true));