LLM inference in C/C++
C++ ggml
opencl: fix WAR race in the generic FA tile kernels when the WG spans subgroups (#26434)
Hongqiang Wang60addddf3c56
diff --git a/ggml/src/ggml-opencl/kernels/flash_attn_f16.cl b/ggml/src/ggml-opencl/kernels/flash_attn_f16.clindex fc58a22ecc..f9797d3460 100644--- a/ggml/src/ggml-opencl/kernels/flash_attn_f16.cl+++ b/ggml/src/ggml-opencl/kernels/flash_attn_f16.cl@@ -118,6 +118,17 @@ __kernel void flash_attn_f16(__local DATA_TYPE4 l_v[BLOCK_N][DV_VEC];for (int k_start = 0; k_start < n_kv; k_start += BLOCK_N) {+#if WG_SIZE > FA_SG+ // WAR on l_k/l_v: a thread that finishes the compute below early — either+ // it skipped it (my_query_row >= n_q, the continue) or its subgroup simply+ // ran ahead — wraps around and reloads the tiles while another subgroup is+ // still reading them. Any WG that is exactly one lockstep subgroup+ // (WG_SIZE == FA_SG) cannot diverge and hides this; a WG spanning multiple+ // subgroups (Intel sg=32, or BLOCK_M > 64 on Adreno) corrupts the result.+ // All threads reach this each iteration (no-op on the first), so it does+ // not diverge with the continue. Compiled out when WG == one subgroup.+ barrier(CLK_LOCAL_MEM_FENCE);+#endiffor (int i = tid; i < BLOCK_N * DK_VEC; i += WG_SIZE) {const int row = i / DK_VEC;const int col = i % DK_VEC;diff --git a/ggml/src/ggml-opencl/kernels/flash_attn_f32.cl b/ggml/src/ggml-opencl/kernels/flash_attn_f32.clindex 599877bdba..5911524e15 100644--- a/ggml/src/ggml-opencl/kernels/flash_attn_f32.cl+++ b/ggml/src/ggml-opencl/kernels/flash_attn_f32.cl@@ -119,13 +119,15 @@ __kernel void flash_attn_f32(__local DATA_TYPE4 l_v[BLOCK_N][DV_VEC];for (int k_start = 0; k_start < n_kv; k_start += BLOCK_N) {-#if FA_SG < 64- // WAR on l_k/l_v: threads with my_query_row >= n_q skip the compute below- // (continue) and would race ahead to reload the tiles while active threads- // still read them. A single 64-wide Adreno subgroup (WG == sg) runs lockstep- // and hides this; a WG that spans multiple narrower subgroups (Intel sg=32)- // corrupts the result. All threads reach this each iteration (no-op on the- // first), so it does not diverge with the continue. Compiled out at sg=64.+#if WG_SIZE > FA_SG+ // WAR on l_k/l_v: a thread that finishes the compute below early — either+ // it skipped it (my_query_row >= n_q, the continue) or its subgroup simply+ // ran ahead — wraps around and reloads the tiles while another subgroup is+ // still reading them. Any WG that is exactly one lockstep subgroup+ // (WG_SIZE == FA_SG) cannot diverge and hides this; a WG spanning multiple+ // subgroups (Intel sg=32, or BLOCK_M > 64 on Adreno) corrupts the result.+ // All threads reach this each iteration (no-op on the first), so it does+ // not diverge with the continue. Compiled out when WG == one subgroup.barrier(CLK_LOCAL_MEM_FENCE);#endiffor (int i = tid; i < BLOCK_N * DK_VEC; i += WG_SIZE) {