LLM inference in C/C++
C++ ggml
opencl: improve fa, allow dk512 for gemma-4, improve dk64 (#30266)
lhezf2cc7282ce00
diff --git a/ggml/src/ggml-opencl/ggml-opencl.cpp b/ggml/src/ggml-opencl/ggml-opencl.cppindex 625b12f547..2abc0bc583 100644--- a/ggml/src/ggml-opencl/ggml-opencl.cpp+++ b/ggml/src/ggml-opencl/ggml-opencl.cpp@@ -48,6 +48,7 @@ typedef const void * (*get_adreno_bin_kernel_func_t)(#include <mutex>#include <regex>#include <set>+#include <tuple>#include <unordered_set>#undef MIN@@ -494,6 +495,13 @@ struct ggml_opencl_fa_kernels {std::map<std::pair<int, int>, cl_kernel> f32_f16_q1_split; // flash-decoding K-split// vec decodestd::map<std::pair<int, int>, cl_kernel> f32_f16_q1_vec;+ bool f32_f16_vec_512_attempted = false;+ std::map<std::tuple<int, int, int>, cl_kernel> f32_f16_mq_decode;+ std::map<std::tuple<int, int, int>, size_t> f32_f16_mq_decode_wg;+ std::map<std::tuple<int, int, int>, int> f32_f16_mq_decode_hs;+ std::set<std::tuple<int, int, int>> f32_f16_mq_decode_attempted;+ ggml_cl_buffer fd_partial;+ cl_uint compute_units = 0;// kv-head-coalesced vec decodestd::map<std::pair<int, int>, cl_kernel> f32_f16_q1_vec_mq;// kv-head-coalesced + flash-decoding split@@ -1289,6 +1297,11 @@ struct ggml_backend_opencl_context {ref_count--;if (ref_count == 0) {+ if (fa.fd_partial.buffer) {+ CL_CHECK(clReleaseMemObject(fa.fd_partial.buffer));+ fa.fd_partial.buffer = nullptr;+ fa.fd_partial.size = 0;+ }#ifdef GGML_OPENCL_PROFILINGflush_profiling_batch();write_profiling_info();@@ -1473,14 +1486,7 @@ static bool use_adreno_bin_kernels(ggml_backend_opencl_context * backend_ctx) {#endif // GGML_OPENCL_USE_ADRENO_BIN_KERNELS}-static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) {- if (backend_ctx->kernels_loaded) {- return;- }-- cl_int err;-- // compiler options for general kernels+static std::string ggml_opencl_make_compile_opts(const ggml_backend_opencl_context * backend_ctx) {auto opencl_c_std =std::string("CL") + std::to_string(backend_ctx->opencl_c_version.major) + "." + std::to_string(backend_ctx->opencl_c_version.minor);std::string compile_opts = std::string("-cl-std=") + opencl_c_std +@@ -1491,7 +1497,18 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) {compile_opts += " -qcom-enable-large-buffer ";}- backend_ctx->kernel_compile_opts = compile_opts;+ return compile_opts;+}++static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) {+ if (backend_ctx->kernels_loaded) {+ return;+ }++ cl_int err;+ const std::string & compile_opts = backend_ctx->kernel_compile_opts;+ const std::string opencl_c_std = "CL" + std::to_string(backend_ctx->opencl_c_version.major) ++ "." + std::to_string(backend_ctx->opencl_c_version.minor);GGML_LOG_INFO("ggml_opencl: loading OpenCL kernels");@@ -5606,6 +5623,127 @@ static void ggml_opencl_ensure_fa_pre_kernels(ggml_backend_opencl_context * backclReleaseProgram(prog_pre_f16);}+static bool ggml_opencl_ensure_fa_f32_f16_vec_512(ggml_backend_opencl_context * backend_ctx) {+ const std::pair<int, int> key = {512, 512};+ auto & fa = backend_ctx->fa;+ if (fa.f32_f16_q1_vec.count(key) > 0) {+ return true;+ }+ if (fa.f32_f16_vec_512_attempted || backend_ctx->kernel_compile_opts.empty()) {+ return false;+ }+ fa.f32_f16_vec_512_attempted = true;++ const ggml_opencl_fa_dim * cfg = nullptr;+ for (const auto & d : g_opencl_fa_dims) {+ if (d.dk == 512 && d.dv == 512) {+ cfg = &d;+ break;+ }+ }+ if (cfg == nullptr) {+ return false;+ }++ // Compile only vec decode and merge to stay within the Adreno compiler's memory limit.+ const std::string opts = ggml_opencl_fa_compile_opts(backend_ctx, cfg, FA_VARIANT_F32_F16) ++ " -D FA_DECODE_ONLY -D FA_VEC_ONLY";+ cl_program prog = build_program_from_source_ex(+ backend_ctx->context, backend_ctx->device,+ ggml_opencl_fa_kernel_src(FA_VARIANT_F32_F16).c_str(), opts,+ /*fatal=*/false, "fa f32_f16 decode512 vec", backend_ctx->queue);+ if (!prog) {+ return false;+ }+ cl_int err;+ cl_kernel vec = clCreateKernel(prog, "flash_attn_f32_f16_q1_vec", &err);+ if (err != CL_SUCCESS) {+ clReleaseProgram(prog);+ return false;+ }+ cl_kernel merge = clCreateKernel(prog, "flash_attn_f32_merge", &err);+ clReleaseProgram(prog);+ if (err != CL_SUCCESS) {+ clReleaseKernel(vec);+ return false;+ }+ if (!ggml_opencl_fa_kernel_fits_wg(backend_ctx, vec, 256, "flash_attn_f32_f16_q1_vec", 512, 512) ||+ !ggml_opencl_fa_kernel_fits_wg(backend_ctx, merge, 128, "flash_attn_f32_merge", 512, 512)) {+ clReleaseKernel(vec);+ clReleaseKernel(merge);+ return false;+ }+ fa.f32_f16_q1_vec[key] = vec;+ if (fa.f32_merge.count(key) > 0) {+ clReleaseKernel(merge);+ } else {+ fa.f32_merge[key] = merge;+ }+ return true;+}++static void ggml_opencl_ensure_fa_f32_f16_mq_decode(ggml_backend_opencl_context * backend_ctx, int dk, int dv, int gqa) {+ const std::tuple<int, int, int> key = {dk, dv, gqa};+ auto & fa = backend_ctx->fa;+ if (fa.f32_f16_mq_decode.count(key) > 0 || fa.f32_f16_mq_decode_attempted.count(key) > 0 ||+ backend_ctx->kernel_compile_opts.empty()) {+ return;+ }+ if (gqa == 4 && dk != 128 && fa.f32_f16_q1_vec_mq_split.count({dk, dv}) > 0) {+ fa.f32_f16_mq_decode[key] = fa.f32_f16_q1_vec_mq_split.at({dk, dv});+ fa.f32_f16_mq_decode_wg[key] = 256;+ fa.f32_f16_mq_decode_hs[key] = 1;+ return;+ }+ fa.f32_f16_mq_decode_attempted.insert(key);+ const ggml_opencl_fa_dim * cfg = nullptr;+ for (const auto & d : g_opencl_fa_dims) {+ if (d.dk == dk && d.dv == dv) {+ cfg = &d;+ break;+ }+ }+ if (cfg == nullptr) {+ return;+ }++ const bool cluster = dk == 64 || dk == 128;+ const int head_sub = cluster ? 2 : (gqa == 8 ? (dk == 512 ? 4 : 2) : 1);+ const int nsg_max = dk == 64 ? 1 : (dk == 128 || (gqa == 8 && dk == 256) ? 2 : 4);+ const char * kernel_name = cluster ? "flash_attn_f32_f16_q1_vec_mq_split_c8" : "flash_attn_f32_f16_q1_vec_mq_split";+ const std::string src = ggml_opencl_fa_kernel_src(FA_VARIANT_F32_F16);+ const std::string opts = ggml_opencl_fa_compile_opts(backend_ctx, cfg, FA_VARIANT_F32_F16) ++ " -D FA_MQ_ONLY -D MQ_GQA=" + std::to_string(gqa / head_sub) ++ " -D FA_HEAD_SUB=" + std::to_string(head_sub) ++ (cluster ? " -D MQ_NSG=" + std::to_string(nsg_max) + " -D FA_CL_C=16" : " -D FA_MQ_SPLIT_ONLY") ++ (dk == 64 ? " -D FA_CL_MHRED -D FA_CL_MASK_BCAST" : "") ++ (gqa == 8 && dk == 256 ? " -D FA_Q1_Q_REG" : "");+ for (int nsg = nsg_max; nsg >= 1; nsg /= 2) {+ const size_t wg = 64 * nsg;+ cl_program prog = build_program_from_source_ex(+ backend_ctx->context, backend_ctx->device, src.c_str(),+ opts + " -D MQ_NSG_SPLIT=" + std::to_string(nsg),+ /*fatal=*/false, "fa f32_f16 mq decode", backend_ctx->queue);+ if (!prog) {+ continue;+ }+ cl_int err;+ cl_kernel kernel = clCreateKernel(prog, kernel_name, &err);+ clReleaseProgram(prog);+ if (err != CL_SUCCESS) {+ continue;+ }+ if (!ggml_opencl_fa_kernel_fits_wg(backend_ctx, kernel, wg, kernel_name, dk, dv)) {+ clReleaseKernel(kernel);+ continue;+ }+ fa.f32_f16_mq_decode[key] = kernel;+ fa.f32_f16_mq_decode_wg[key] = wg;+ fa.f32_f16_mq_decode_hs[key] = head_sub;+ return;+ }+}+// DK=512 prefill BM-tilestatic bool ggml_opencl_ensure_fa_f32_f16_prefill_512(ggml_backend_opencl_context * backend_ctx, bool split) {const int dk = 512, dv = 512;@@ -6834,6 +6972,10 @@ static ggml_backend_opencl_context * ggml_cl_init(ggml_backend_dev_t dev) {backend_ctx->adreno_use_large_buffer = getenv("GGML_OPENCL_ADRENO_USE_LARGE_BUFFER") != nullptr &&backend_ctx->gpu_family == GPU_FAMILY::ADRENO;+ backend_ctx->kernel_compile_opts = ggml_opencl_make_compile_opts(backend_ctx.get());+ CL_CHECK(clGetDeviceInfo(device, CL_DEVICE_MAX_COMPUTE_UNITS,+ sizeof(backend_ctx->fa.compute_units), &backend_ctx->fa.compute_units, NULL));+// ragged moe, unspecified or non-zero means enabled, set to 0 to disablestatic const char * ragged_fp16_env = getenv("GGML_OPENCL_MOE_RAGGED_FP16");backend_ctx->adreno_use_moe_ragged = (ragged_fp16_env == NULL) ? 1 : (atoi(ragged_fp16_env) != 0);@@ -9347,10 +9489,13 @@ static bool ggml_opencl_supports_op(ggml_backend_dev_t dev, const struct ggml_tereturn false;}if (q->ne[1] == 1) {- // DK=512 decode is bandwidth-bound and slower on the GPU- // than on the CPU; decline it here so it runs on the CPU.- // Prefill (n_q > 1) stays on the GPU.- return false;+ const char * decode_env = getenv("GGML_OPENCL_FA_DK512_DECODE");+ if ((decode_env && decode_env[0] == '0') ||+ backend_ctx->gpu_family != ADRENO || k->ne[2] <= 0 ||+ (q->ne[2] / k->ne[2] != 4 && q->ne[2] / k->ne[2] != 8) || q->ne[2] % k->ne[2] != 0 ||+ !ggml_opencl_ensure_fa_f32_f16_vec_512(backend_ctx)) {+ return false;+ }} else {// prefill, BM-tile in its own FA_PREFILL_ONLY programif (!ggml_opencl_ensure_fa_f32_f16_prefill_512(backend_ctx, /*split=*/false)) {@@ -17938,10 +18083,7 @@ static void ggml_cl_flash_attn(ggml_backend_t backend, const ggml_tensor * q, co}#endif- // DK=512 (Gemma-4 global layers) runs decode-only (q1 / q1_split) on- // Adreno - it never uses the BM-tile path, and the prepass + split-tile- // programs OOM the compiler at DK=512; supports_op only admits- // n_q==1 here and prefill goes to CPU+ // Compile DK512 decode separately from the prefill programs.const bool fa_decode_only_512 = (d_head_q == 512);// per-variant lazy compile for this (dk, dv)@@ -17970,7 +18112,11 @@ static void ggml_cl_flash_attn(ggml_backend_t backend, const ggml_tensor * q, coif (is_f16) {ggml_opencl_ensure_fa_variant(backend_ctx, d_head_q, d_head_v, FA_VARIANT_F16);} else if (is_mixed) {- ggml_opencl_ensure_fa_variant(backend_ctx, d_head_q, d_head_v, FA_VARIANT_F32_F16);+ if (fa_decode_only_512 && n_q == 1) {+ GGML_ASSERT(ggml_opencl_ensure_fa_f32_f16_vec_512(backend_ctx));+ } else {+ ggml_opencl_ensure_fa_variant(backend_ctx, d_head_q, d_head_v, FA_VARIANT_F32_F16);+ }if (fa_decode_only_512) {// DK=512: the BM-tile prefill kernels are specifically compiled from// FA_PREFILL_ONLY@@ -18004,6 +18150,16 @@ static void ggml_cl_flash_attn(ggml_backend_t backend, const ggml_tensor * q, co}const std::pair<int, int> dk_dv = {d_head_q, d_head_v};+ const int mq_gqa = n_head_kv > 0 ? n_head / n_head_kv : 0;+ const std::tuple<int, int, int> mq_decode_key = {d_head_q, d_head_v, mq_gqa};+ const bool mq_decode_shape = backend_ctx->gpu_family == ADRENO && is_mixed && n_q == 1 &&+ d_head_q == d_head_v && n_head_kv > 0 && n_head % n_head_kv == 0 &&+ ((((d_head_q == 64 && mq_gqa == 8) || (d_head_q == 128 && mq_gqa == 4)) &&+ backend_ctx->has_subgroup_shuffle) ||+ ((d_head_q == 256 || d_head_q == 512) && (mq_gqa == 4 || mq_gqa == 8)));+ if (mq_decode_shape && n_kv >= 32) {+ ggml_opencl_ensure_fa_f32_f16_mq_decode(backend_ctx, d_head_q, d_head_v, mq_gqa);+ }const bool use_native_q8_0_q1 = is_q8_0 && n_q == 1 &&backend_ctx->fa.f32_q8_0_q1.count(dk_dv) > 0;// Native q8_0 prefill — reads q8_0 directly, wg_size = cfg->bm.@@ -18262,6 +18418,8 @@ static void ggml_cl_flash_attn(ggml_backend_t backend, const ggml_tensor * q, coconst int fd_max_n_q = (d_head_q <= FD_MAX_DK_MULTI) ? FD_MAX_N_Q_MULTI : 1;cl_kernel fd_k_split = NULL;bool use_fd_mq = false;+ bool use_fd_mq_decode = false;+ int fd_head_sub = 1;size_t fd_mq_wg = 256; // MQ_GQA=4 kernel: Q1_WG_SIZE(64) * MQ_NSG_SPLIT(4)bool use_fa_k_img = false; // K bound as image1d_buffer_t instead of (buf, offset)@@ -18294,9 +18452,15 @@ static void ggml_cl_flash_attn(ggml_backend_t backend, const ggml_tensor * q, coif (mq_enabled && mq_kv_ok && nq_in_vec_range && !is_causal &&backend_ctx->gpu_family != INTEL &&!use_local_tile &&- n_kv >= FD_MIN_N_KV &&+ n_kv >= (mq_decode_shape ? 32 : FD_MIN_N_KV) &&backend_ctx->fa.f32_merge.count(dk_dv) > 0) {- if (nq1_only && lmq_on && is_mixed && d_head_q == 128 && d_head_v == 128 &&+ if (mq_decode_shape && backend_ctx->fa.f32_f16_mq_decode.count(mq_decode_key) > 0) {+ fd_k_split = backend_ctx->fa.f32_f16_mq_decode.at(mq_decode_key);+ fd_mq_wg = backend_ctx->fa.f32_f16_mq_decode_wg.at(mq_decode_key);+ fd_head_sub = backend_ctx->fa.f32_f16_mq_decode_hs.at(mq_decode_key);+ use_fd_mq = true;+ use_fd_mq_decode = true;+ } else if (nq1_only && lmq_on && is_mixed && d_head_q == 128 && d_head_v == 128 &&gqa_ratio_dispatch == 8 &&backend_ctx->fa.f32_f16_q1_local_mq_split_g8.count(dk_dv) > 0) {fd_k_split = backend_ctx->fa.f32_f16_q1_local_mq_split_g8.at(dk_dv);@@ -18575,6 +18739,14 @@ static void ggml_cl_flash_attn(ggml_backend_t backend, const ggml_tensor * q, coint n_splits = (n_kv + fd_kv_per_split - 1) / fd_kv_per_split;if (n_splits < FD_MIN_SPLITS) { n_splits = FD_MIN_SPLITS; }if (n_splits > fd_max_splits) { n_splits = fd_max_splits; }+ if (use_fd_mq_decode) {+ const size_t wg_per_split = (size_t) n_head_kv * n_batch;+ const size_t wg_target = 4 * (size_t) backend_ctx->fa.compute_units;+ while (wg_per_split * n_splits < wg_target && n_splits < fd_max_splits &&+ n_kv / (n_splits + 1) >= 32) {+ n_splits++;+ }+ }const int kv_per_split = (n_kv + n_splits - 1) / n_splits;const int fa_partial_floats = 2 + d_head_v;@@ -18582,15 +18754,26 @@ static void ggml_cl_flash_attn(ggml_backend_t backend, const ggml_tensor * q, co(size_t) n_batch * n_head * n_q * n_splits * fa_partial_floats * sizeof(float);ggml_cl_flash_attn_temp_buffer temp_partial;+ cl_mem partial_buffer;cl_int err;- temp_partial.data = clCreateBuffer(backend_ctx->context, CL_MEM_READ_WRITE,- partial_size_bytes, NULL, &err);- if (err != CL_SUCCESS) {- CL_CHECK(clFinish(backend_ctx->queue));+ if (use_fd_mq_decode) {+ auto & pool = backend_ctx->fa.fd_partial;+ if (partial_size_bytes > pool.size) {+ CL_CHECK(clFinish(backend_ctx->queue));+ pool.allocate(backend_ctx->context, partial_size_bytes);+ }+ partial_buffer = pool.buffer;+ } else {temp_partial.data = clCreateBuffer(backend_ctx->context, CL_MEM_READ_WRITE,partial_size_bytes, NULL, &err);+ if (err != CL_SUCCESS) {+ CL_CHECK(clFinish(backend_ctx->queue));+ temp_partial.data = clCreateBuffer(backend_ctx->context, CL_MEM_READ_WRITE,+ partial_size_bytes, NULL, &err);+ }+ CL_CHECK(err);+ partial_buffer = temp_partial.data;}- CL_CHECK(err);cl_kernel k_split = fd_k_split;int argi = 0;@@ -18658,7 +18841,7 @@ static void ggml_cl_flash_attn(ggml_backend_t backend, const ggml_tensor * q, coCL_CHECK(clSetKernelArg(k_split, argi++, sizeof(cl_ulong), &mask_nb3));CL_CHECK(clSetKernelArg(k_split, argi++, sizeof(int), &mask_ne2));CL_CHECK(clSetKernelArg(k_split, argi++, sizeof(int), &mask_ne3));- CL_CHECK(clSetKernelArg(k_split, argi++, sizeof(cl_mem), &temp_partial.data));+ CL_CHECK(clSetKernelArg(k_split, argi++, sizeof(cl_mem), &partial_buffer));CL_CHECK(clSetKernelArg(k_split, argi++, sizeof(int), &n_splits));CL_CHECK(clSetKernelArg(k_split, argi++, sizeof(int), &kv_per_split));@@ -18666,7 +18849,7 @@ static void ggml_cl_flash_attn(ggml_backend_t backend, const ggml_tensor * q, co// matches Q1_WG_SIZE * NSG (MQ_GQA=4 -> 256; MQ_GQA=8 -> 192)const size_t fd_wg = use_fd_mq ? fd_mq_wg : 64;const size_t fd_head_dim = use_fd_mq- ? (size_t)(n_head_kv * n_batch)+ ? (size_t)(n_head_kv * fd_head_sub * n_batch): (size_t)(n_head * n_batch);size_t fd_lws[3] = { fd_wg, 1, 1 };// gid(2) packs q_idx * n_splits + split_idx.@@ -18675,7 +18858,7 @@ static void ggml_cl_flash_attn(ggml_backend_t backend, const ggml_tensor * q, cocl_kernel k_merge = backend_ctx->fa.f32_merge.at(dk_dv);argi = 0;- CL_CHECK(clSetKernelArg(k_merge, argi++, sizeof(cl_mem), &temp_partial.data));+ CL_CHECK(clSetKernelArg(k_merge, argi++, sizeof(cl_mem), &partial_buffer));CL_CHECK(clSetKernelArg(k_merge, argi++, sizeof(cl_mem), &extra_o->data_device));CL_CHECK(clSetKernelArg(k_merge, argi++, sizeof(cl_ulong), &offset_o));CL_CHECK(clSetKernelArg(k_merge, argi++, sizeof(int), &n_head));diff --git a/ggml/src/ggml-opencl/kernels/flash_attn_f32_f16.cl b/ggml/src/ggml-opencl/kernels/flash_attn_f32_f16.clindex bf7695a2c1..27553c729a 100644--- a/ggml/src/ggml-opencl/kernels/flash_attn_f32_f16.cl+++ b/ggml/src/ggml-opencl/kernels/flash_attn_f32_f16.cl@@ -665,7 +665,7 @@ __kernel void FA_TILE_NAME(// allow bypassing decode kernels to avoid compiler crash for DK=512 on Adreno GPUs#ifndef FA_PREFILL_ONLY-#ifndef FA_MQ_ONLY // q1 excluded from the MQ-only (g8) program+#if !defined(FA_MQ_ONLY) && !defined(FA_VEC_ONLY)REQD_FA_SG__kernel void flash_attn_f32_f16_q1(const global void * q_void, ulong q_offset,@@ -932,14 +932,14 @@ __kernel void flash_attn_f32_f16_q1_vec(}ACC_TYPE dot_partial = dot4.s0 + dot4.s1 + dot4.s2 + dot4.s3;ACC_TYPE score = sub_group_reduce_add(dot_partial) * scale;+ if (logit_softcap > 0.0f) {+ score = logit_softcap * tanh(score / logit_softcap);+ }if (mask_base != NULL) {const global MASK_DATA_TYPE * mask_ptr = (const global MASK_DATA_TYPE *) mask_base;score += slope * (ACC_TYPE) mask_ptr[k_idx];}- if (logit_softcap > 0.0f) {- score = logit_softcap * tanh(score / logit_softcap);- }// FA-2 online update. All threads in the subgroup see the same score,// so m_i and l_i evolve identically across lanes within the subgroup.@@ -1385,6 +1385,7 @@ __kernel void flash_attn_f32_f16_q1_local_mq_split(#endif#define MQ_WG_SIZE (Q1_WG_SIZE * MQ_NSG)+#ifndef FA_MQ_SPLIT_ONLYREQD_SUBGROUP_SIZE_64__kernel void flash_attn_f32_f16_q1_vec_mq(const global void * q_void, ulong q_offset,@@ -1606,6 +1607,8 @@ __kernel void flash_attn_f32_f16_q1_vec_mq(}}+#endif // !FA_MQ_SPLIT_ONLY+#ifndef MQ_NSG_SPLIT#define MQ_NSG_SPLIT 4#endif@@ -1615,6 +1618,10 @@ __kernel void flash_attn_f32_f16_q1_vec_mq(#define FA_PARTIAL_FLOATS (2 + DV)#endif+#ifndef FA_HEAD_SUB+#define FA_HEAD_SUB 1+#endif+REQD_SUBGROUP_SIZE_64__kernel void flash_attn_f32_f16_q1_vec_mq_split(const global void * q_void, ulong q_offset,@@ -1652,8 +1659,12 @@ __kernel void flash_attn_f32_f16_q1_vec_mq_split(const int split_idx = split_q_idx % n_splits;const int q_idx = split_q_idx / n_splits;- const int batch_idx = kvhead_batch_idx / n_head_kv;- const int head_kv_idx = kvhead_batch_idx % n_head_kv;+ const int hgroups = n_head_kv * FA_HEAD_SUB;+ const int batch_idx = kvhead_batch_idx / hgroups;+ const int hg = kvhead_batch_idx % hgroups;+ const int head_kv_idx = hg / FA_HEAD_SUB;+ const int head_sub = hg % FA_HEAD_SUB;+#define FA_MQS_HEAD_IDX(h) (head_kv_idx * (MQ_GQA * FA_HEAD_SUB) + head_sub * MQ_GQA + (h))const int kv_start = split_idx * kv_per_split;const int kv_end = min(kv_start + kv_per_split, n_kv);@@ -1666,7 +1677,7 @@ __kernel void flash_attn_f32_f16_q1_vec_mq_split(if (tid == 0) {#pragma unrollfor (int h = 0; h < MQ_GQA; ++h) {- const int head_idx = head_kv_idx * MQ_GQA + h;+ const int head_idx = FA_MQS_HEAD_IDX(h);const ulong rec_idx = ((((ulong) batch_idx * n_head + head_idx) * n_q + q_idx)* n_splits + split_idx);global float * rec = partial_void + rec_idx * record_stride;@@ -1681,22 +1692,33 @@ __kernel void flash_attn_f32_f16_q1_vec_mq_split(const global char * k_base = (const global char *) k_void + k_offset;const global char * v_base = (const global char *) v_void + v_offset;+#ifdef FA_Q1_Q_REG+ ACC_TYPE4 q_reg[MQ_GQA];+ #pragma unroll+ for (int h = 0; h < MQ_GQA; ++h) {+ const int head_idx = FA_MQS_HEAD_IDX(h);+ const ulong q_row_offset = batch_idx * q_nb3 + head_idx * q_nb2 + (ulong) q_idx * q_nb1;+ const global Q_DATA_TYPE4 * q_ptr = (const global Q_DATA_TYPE4 *) (q_base + q_row_offset);+ q_reg[h] = (tid_sg < DK_VEC) ? CONVERT_Q_ACC4(q_ptr[tid_sg]) : (ACC_TYPE4)(0.0f);+ }+#else// stage MQ_GQA Q rows in __local once (uniform across WG)__local ACC_TYPE4 q_shared[MQ_GQA * DK_VEC];for (int i = tid; i < MQ_GQA * DK_VEC; i += MQ_SPLIT_WG_SIZE) {const int h = i / DK_VEC;const int k = i % DK_VEC;- const int head_idx = head_kv_idx * MQ_GQA + h;+ const int head_idx = FA_MQS_HEAD_IDX(h);const ulong q_row_offset = batch_idx * q_nb3 + head_idx * q_nb2 + (ulong) q_idx * q_nb1;const global Q_DATA_TYPE4 * q_ptr = (const global Q_DATA_TYPE4 *) (q_base + q_row_offset);q_shared[h * DK_VEC + k] = CONVERT_Q_ACC4(q_ptr[k]);}barrier(CLK_LOCAL_MEM_FENCE);+#endiffloat slope[MQ_GQA];#pragma unrollfor (int h = 0; h < MQ_GQA; ++h) {- slope[h] = get_alibi_slope(max_bias, head_kv_idx * MQ_GQA + h, n_head_log2, m0, m1);+ slope[h] = get_alibi_slope(max_bias, FA_MQS_HEAD_IDX(h), n_head_log2, m0, m1);}const global char * mask_base[MQ_GQA];@@ -1707,7 +1729,7 @@ __kernel void flash_attn_f32_f16_q1_vec_mq_split((ulong) q_idx * mask_nb1;#pragma unrollfor (int h = 0; h < MQ_GQA; ++h) {- const int head_idx = head_kv_idx * MQ_GQA + h;+ const int head_idx = FA_MQS_HEAD_IDX(h);const int mask_head_idx = head_idx % mask_ne2;mask_base[h] = mask_base_b + mask_head_idx * mask_nb2;}@@ -1742,6 +1764,15 @@ __kernel void flash_attn_f32_f16_q1_vec_mq_split(ACC_TYPE4 dot4[MQ_GQA];#pragma unrollfor (int h = 0; h < MQ_GQA; ++h) dot4[h] = (ACC_TYPE4)(0.0f);+#ifdef FA_Q1_Q_REG+ if (tid_sg < DK_VEC) {+ const ACC_TYPE4 k_vec = CONVERT_KV_ACC4(k_ptr[tid_sg]);+ #pragma unroll+ for (int h = 0; h < MQ_GQA; ++h) {+ dot4[h] = mad(q_reg[h], k_vec, dot4[h]);+ }+ }+#elsefor (int k = tid_sg; k < DK_VEC; k += Q1_WG_SIZE) {const ACC_TYPE4 k_vec = CONVERT_KV_ACC4(k_ptr[k]);#pragma unroll@@ -1749,19 +1780,20 @@ __kernel void flash_attn_f32_f16_q1_vec_mq_split(dot4[h] = mad(q_shared[h * DK_VEC + k], k_vec, dot4[h]);}}+#endifACC_TYPE score[MQ_GQA];#pragma unrollfor (int h = 0; h < MQ_GQA; ++h) {const ACC_TYPE dot_partial = dot4[h].s0 + dot4[h].s1 + dot4[h].s2 + dot4[h].s3;ACC_TYPE s = sub_group_reduce_add(dot_partial) * scale;+ if (logit_softcap > 0.0f) {+ s = logit_softcap * tanh(s / logit_softcap);+ }if (mask_base[h] != NULL) {const global MASK_DATA_TYPE * mask_ptr = (const global MASK_DATA_TYPE *) mask_base[h];s += slope[h] * (ACC_TYPE) mask_ptr[k_idx];}- if (logit_softcap > 0.0f) {- s = logit_softcap * tanh(s / logit_softcap);- }score[h] = s;}@@ -1810,7 +1842,7 @@ __kernel void flash_attn_f32_f16_q1_vec_mq_split(barrier(CLK_LOCAL_MEM_FENCE);if (sgid == 0) {- const int head_idx = head_kv_idx * MQ_GQA + h;+ const int head_idx = FA_MQS_HEAD_IDX(h);// fold per-subgroup (m, l) into split-level (m_c, l_c)ACC_TYPE m_c = sg_m[h][0];@@ -1848,6 +1880,9 @@ __kernel void flash_attn_f32_f16_q1_vec_mq_split(}}+#undef FA_MQS_HEAD_IDX++#ifndef FA_MQ_SPLIT_ONLY// Cluster-parallel variant of _q1_vec_mq_split//// Tthe baseline keeps one 256B K row in flight per subgroup (32 lanes cooperate@@ -1936,8 +1971,12 @@ __kernel void flash_attn_f32_f16_q1_vec_mq_split_c8(const int split_idx = split_q_idx % n_splits;const int q_idx = split_q_idx / n_splits;- const int batch_idx = kvhead_batch_idx / n_head_kv;- const int head_kv_idx = kvhead_batch_idx % n_head_kv;+ const int hgroups = n_head_kv * FA_HEAD_SUB;+ const int batch_idx = kvhead_batch_idx / hgroups;+ const int hg = kvhead_batch_idx % hgroups;+ const int head_kv_idx = hg / FA_HEAD_SUB;+ const int head_sub = hg % FA_HEAD_SUB;+#define FA_HEAD_IDX(h) (head_kv_idx * (MQ_GQA * FA_HEAD_SUB) + head_sub * MQ_GQA + (h))const int kv_start = split_idx * kv_per_split;const int kv_end = min(kv_start + kv_per_split, n_kv);@@ -1948,7 +1987,7 @@ __kernel void flash_attn_f32_f16_q1_vec_mq_split_c8(if (tid == 0) {#pragma unrollfor (int h = 0; h < MQ_GQA; ++h) {- const int head_idx = head_kv_idx * MQ_GQA + h;+ const int head_idx = FA_HEAD_IDX(h);const ulong rec_idx = ((((ulong) batch_idx * n_head + head_idx) * n_q + q_idx)* n_splits + split_idx);global float * rec = partial_void + rec_idx * record_stride;@@ -1968,7 +2007,7 @@ __kernel void flash_attn_f32_f16_q1_vec_mq_split_c8(for (int i = tid; i < MQ_GQA * DK_VEC; i += MQ_SPLIT_WG_SIZE) {const int h = i / DK_VEC;const int k = i % DK_VEC;- const int head_idx = head_kv_idx * MQ_GQA + h;+ const int head_idx = FA_HEAD_IDX(h);const ulong q_row_offset = batch_idx * q_nb3 + head_idx * q_nb2 + (ulong) q_idx * q_nb1;const global Q_DATA_TYPE4 * q_ptr = (const global Q_DATA_TYPE4 *) (q_base + q_row_offset);q_shared[h * DK_VEC + k] = CONVERT_Q_ACC4(q_ptr[k]);@@ -1978,9 +2017,17 @@ __kernel void flash_attn_f32_f16_q1_vec_mq_split_c8(float slope[MQ_GQA];#pragma unrollfor (int h = 0; h < MQ_GQA; ++h) {- slope[h] = get_alibi_slope(max_bias, head_kv_idx * MQ_GQA + h, n_head_log2, m0, m1);+ slope[h] = get_alibi_slope(max_bias, FA_HEAD_IDX(h), n_head_log2, m0, m1);}+#ifdef FA_CL_MASK_BCAST+ const global char * mask_base_b = NULL;+ if (mask_void != NULL) {+ mask_base_b = (const global char *) mask_void + mask_offset ++ (batch_idx % mask_ne3) * mask_nb3 + (ulong) q_idx * mask_nb1;+ }+ const int mask_bcast = mask_base_b != NULL && mask_ne2 == 1;+#elseconst global char * mask_base[MQ_GQA];if (mask_void != NULL) {const int mask_batch_idx = batch_idx % mask_ne3;@@ -1989,7 +2036,7 @@ __kernel void flash_attn_f32_f16_q1_vec_mq_split_c8((ulong) q_idx * mask_nb1;#pragma unrollfor (int h = 0; h < MQ_GQA; ++h) {- const int head_idx = head_kv_idx * MQ_GQA + h;+ const int head_idx = FA_HEAD_IDX(h);const int mask_head_idx = head_idx % mask_ne2;mask_base[h] = mask_base_b + mask_head_idx * mask_nb2;}@@ -1997,6 +2044,7 @@ __kernel void flash_attn_f32_f16_q1_vec_mq_split_c8(#pragma unrollfor (int h = 0; h < MQ_GQA; ++h) mask_base[h] = NULL;}+#endif// Per-CLUSTER online-softmax state (uniform across the cluster's lanes);// o_acc holds this lane's DV slice {lic + FA_CL_C*i}.@@ -2031,6 +2079,73 @@ __kernel void flash_attn_f32_f16_q1_vec_mq_split_c8(const global KV_DATA_TYPE4 * k_ptr = (const global KV_DATA_TYPE4 *) (k_base + kv_row_base + (ulong) k_safe * k_nb1);const global KV_DATA_TYPE4 * v_ptr = (const global KV_DATA_TYPE4 *) (v_base + v_row_base + (ulong) k_safe * v_nb1);+#if defined(FA_CL_MHRED) && MQ_GQA == 4 && FA_CL_C == 16 && FA_CL_DK == 1 && FA_CL_DV == 1+ ACC_TYPE mask_val = 0.0f;+ if (mask_bcast) {+ mask_val = (ACC_TYPE) ((const global MASK_DATA_TYPE *) mask_base_b)[k_safe];+ }+ const ACC_TYPE4 k_vec_1 = CONVERT_KV_ACC4(k_ptr[lic]);+ const ACC_TYPE4 v_vec_1 = CONVERT_KV_ACC4(v_ptr[lic]);++ // Reduce four heads with eight shuffles and keep each head's summation order.+ const int mh_b0 = lic & 1;+ const int mh_b1 = lic & 2;++ ACC_TYPE mh_p[MQ_GQA];+ #pragma unroll+ for (int h = 0; h < MQ_GQA; ++h) {+ const ACC_TYPE4 d4 = mad(q_shared[h * DK_VEC + lic], k_vec_1, (ACC_TYPE4)(0.0f));+ mh_p[h] = d4.s0 + d4.s1 + d4.s2 + d4.s3;+ }++ ACC_TYPE mh_r2[2];+ #pragma unroll+ for (int j = 0; j < 2; ++j) {+ const ACC_TYPE keep = mh_b0 ? mh_p[j + 2] : mh_p[j];+ const ACC_TYPE send = mh_b0 ? mh_p[j] : mh_p[j + 2];+ mh_r2[j] = keep + sub_group_shuffle_xor(send, 1);+ }+ ACC_TYPE mh_r1 = (mh_b1 ? mh_r2[1] : mh_r2[0]) ++ sub_group_shuffle_xor(mh_b1 ? mh_r2[0] : mh_r2[1], 2);+ mh_r1 += sub_group_shuffle_xor(mh_r1, 4);+ mh_r1 += sub_group_shuffle_xor(mh_r1, 8);++ ACC_TYPE mh_e2[2];+ {+ const ACC_TYPE other = sub_group_shuffle_xor(mh_r1, 2);+ mh_e2[0] = mh_b1 ? other : mh_r1;+ mh_e2[1] = mh_b1 ? mh_r1 : other;+ }+ ACC_TYPE mh_s[MQ_GQA];+ #pragma unroll+ for (int j = 0; j < 2; ++j) {+ const ACC_TYPE other = sub_group_shuffle_xor(mh_e2[j], 1);+ mh_s[j] = mh_b0 ? other : mh_e2[j];+ mh_s[j + 2] = mh_b0 ? mh_e2[j] : other;+ }++ #pragma unroll+ for (int h = 0; h < MQ_GQA; ++h) {+ ACC_TYPE s = mh_s[h] * scale;+ if (logit_softcap > 0.0f) {+ s = logit_softcap * tanh(s / logit_softcap);+ }+ if (mask_bcast) {+ s += slope[h] * mask_val;+ } else if (mask_base_b != NULL) {+ const int mask_head_idx = FA_HEAD_IDX(h) % mask_ne2;+ const global MASK_DATA_TYPE * mask_ptr = (const global MASK_DATA_TYPE *) (mask_base_b + mask_head_idx * mask_nb2);+ s += slope[h] * (ACC_TYPE) mask_ptr[k_safe];+ }+ const ACC_TYPE sc = valid ? s : FA_M_INIT;+ const ACC_TYPE m_new = max(m_i[h], sc);+ const ACC_TYPE sp = native_exp(m_i[h] - m_new);+ const ACC_TYPE p = native_exp(sc - m_new);+ l_i[h] = l_i[h] * sp + p;+ m_i[h] = m_new;+ o_acc[h][0] = mad(p, v_vec_1, o_acc[h][0] * sp);+ }+#else// Dot: this lane covers DK elements {lic + FA_CL_C*i} of the cluster's row.ACC_TYPE4 dot4[MQ_GQA];#pragma unroll@@ -2055,13 +2170,13 @@ __kernel void flash_attn_f32_f16_q1_vec_mq_split_c8(s += sub_group_shuffle_xor(s, step);}s *= scale;+ if (logit_softcap > 0.0f) {+ s = logit_softcap * tanh(s / logit_softcap);+ }if (mask_base[h] != NULL) {const global MASK_DATA_TYPE * mask_ptr = (const global MASK_DATA_TYPE *) mask_base[h];s += slope[h] * (ACC_TYPE) mask_ptr[k_safe];}- if (logit_softcap > 0.0f) {- s = logit_softcap * tanh(s / logit_softcap);- }score[h] = valid ? s : FA_M_INIT;}@@ -2087,6 +2202,7 @@ __kernel void flash_attn_f32_f16_q1_vec_mq_split_c8(o_acc[h][i] = mad(p_h[h], v_vec, o_acc[h][i] * sp_h[h]);}}+#endif}// Merge stage 1: fold the FA_CL_NCL cluster partials inside the subgroup.@@ -2148,7 +2264,7 @@ __kernel void flash_attn_f32_f16_q1_vec_mq_split_c8(barrier(CLK_LOCAL_MEM_FENCE);if (sgid == 0) {- const int head_idx = head_kv_idx * MQ_GQA + h;+ const int head_idx = FA_HEAD_IDX(h);ACC_TYPE m_c = sg_m[h][0];#pragma unroll@@ -2184,6 +2300,8 @@ __kernel void flash_attn_f32_f16_q1_vec_mq_split_c8(}}+#undef FA_HEAD_IDX+#endif // DK_VEC/DV_VEC divisible by FA_CL_C#endif // HAS_SUBGROUP_SHUFFLE (q1_vec_mq_split_c8)@@ -2419,9 +2537,11 @@ __kernel void flash_attn_f32_f16_q1_vec_mq_split_k_img(barrier(CLK_LOCAL_MEM_FENCE);}}+#endif // !FA_MQ_SPLIT_ONLY#endif // !FA_DECODE_ONLY#ifndef FA_MQ_ONLY // q1_split + merge excluded from the MQ-only (g8) program+#ifndef FA_VEC_ONLY__kernel void flash_attn_f32_f16_q1_split(const global void * q_void, ulong q_offset,const global void * k_void, ulong k_offset,@@ -2578,6 +2698,8 @@ __kernel void flash_attn_f32_f16_q1_split(}}+#endif // !FA_VEC_ONLY+// FD Pass 2: merge per-split partials into final O// empty splits drop via exp(-INF)=0.__kernel void flash_attn_f32_merge(