LLM inference in C/C++
C++ ggml
sycl: accelerate GLM MLA prefill with MKL flash attention (#29171)
Anant Shrivastavad2a79e604649
diff --git a/ggml/src/ggml-sycl/fattn-mkl.cpp b/ggml/src/ggml-sycl/fattn-mkl.cppindex 27a8bce8be..5a5cf69fde 100644--- a/ggml/src/ggml-sycl/fattn-mkl.cpp+++ b/ggml/src/ggml-sycl/fattn-mkl.cpp@@ -282,8 +282,10 @@ static mkl_fa_kv_desc mkl_fa_make_desc(const ggml_tensor * T, bool interleaved,d.ts = (int64_t)ggml_type_size(T->type);if (T->type == GGML_TYPE_F16) {- d.mode = interleaved ? MKL_FA_KV_MODE_F16_INTERLEAVED- : MKL_FA_KV_MODE_F16_DENSE;+ // MLA's V cache is a 512-wide view of 576-wide K rows. Treat any+ // padded row stride as strided even when there is only one KV head.+ d.mode = interleaved || d.nb1 != d.D * (int64_t)sizeof(sycl::half)+ ? MKL_FA_KV_MODE_F16_INTERLEAVED : MKL_FA_KV_MODE_F16_DENSE;} else if (ggml_is_contiguously_allocated(T) && !interleaved) {d.mode = MKL_FA_KV_MODE_QUANT_CONTIG;} else {@@ -412,7 +414,9 @@ void ggml_sycl_flash_attn_ext_mkl(ggml_backend_sycl_context & ctx, ggml_tensor *const int64_t q_row_stride = Q->nb[1] / sizeof(float);const int64_t q_head_stride = Q->nb[2] / sizeof(float);- const bool V_is_K_view = V->view_src+ // Alias the dequantized buffers only when K and V expose the same values.+ // MLA V is a narrower view of K and needs its own strided dequantization.+ const bool V_is_K_view = V->ne[0] == K->ne[0] && V->view_src&& (V->view_src == K || (V->view_src == K->view_src&& V->view_offs == K->view_offs));diff --git a/ggml/src/ggml-sycl/fattn.cpp b/ggml/src/ggml-sycl/fattn.cppindex 541ae8a82f..8bd946e095 100644--- a/ggml/src/ggml-sycl/fattn.cpp+++ b/ggml/src/ggml-sycl/fattn.cpp@@ -146,15 +146,17 @@ static best_fattn_kernel ggml_sycl_get_best_fattn_kernel(const int device, const// Set GGML_SYCL_ENABLE_MKL_FA=0 to force TILE/VEC path for A/B testing.// Example: GGML_SYCL_ENABLE_MKL_FA=0 llama-cli -m model.gguf -fa -ngl 99 ...// Note: MKL GEMM calls are incompatible with SYCL graph capture replay.- // MKL is validated for the mainstream GQA envelope: grouped-query- // (gqa_ratio >= 2), head_dim a multiple of 64 in [64,512] with matching- // K/V head size, mask, no sinks/ALiBi/softcap. Gemma's global layers use- // head_dim 512, so the cap must include it. Head sizes not a multiple of- // 64 (72/80/96), MHA (gqa_ratio == 1), and MLA (DKQ != DV, e.g. 576/512)- // fall through to TILE/VEC; see follow-up work.+ const bool standard_shape = Q->ne[0] >= 64 && Q->ne[0] <= 512 &&+ Q->ne[0] % 64 == 0 && Q->ne[0] == V->ne[0];+ // GLM-4.7 Flash's MLA shape is already expressible by the MKL pipeline:+ // KQ GEMM uses DKQ=576 while VKQ and output use DV=512. Keep this narrow+ // until other mismatched K/V shapes have independent correctness data.+ const bool glm_mla_shape = Q->ne[0] == 576 && K->ne[0] == 576 &&+ V->ne[0] == 512 && gqa_ratio == 20 &&+ K->type == GGML_TYPE_F16 && V->type == GGML_TYPE_F16;+if (g_ggml_sycl_enable_mkl_fa == 1 && mask && !sinks && gqa_ratio >= 2 &&- Q->ne[0] >= 64 && Q->ne[0] <= 512 && Q->ne[0] % 64 == 0 &&- Q->ne[0] == V->ne[0] &&+ (standard_shape || glm_mla_shape) &&Q->ne[1] >= 32 && K->ne[1] >= 1024 &&max_bias == 0.0f && logit_softcap == 0.0f &&(Q->ne[3] == K->ne[3] || K->ne[3] == 1)) {@@ -164,7 +166,10 @@ static best_fattn_kernel ggml_sycl_get_best_fattn_kernel(const int device, const// nb1=75 for ne0=40 fall through to TILE.bool kv_strides_ok = true;for (const ggml_tensor * t : {K, V}) {- if (t->type == GGML_TYPE_F16 && t->nb[1] % (t->ne[0] * 2) != 0) {+ const bool glm_v_stride = glm_mla_shape && t == V &&+ V->view_src && V->nb[1] == K->nb[1];+ if (!glm_v_stride && t->type == GGML_TYPE_F16 &&+ t->nb[1] % (t->ne[0] * 2) != 0) {kv_strides_ok = false;break;}