LLM inference in C/C++
C++ ggml
opencl: fix two issues on flash attention for Adreno a7x (#25697)
Hongqiang Wangc3d47e696b11
diff --git a/ggml/src/ggml-opencl/ggml-opencl.cpp b/ggml/src/ggml-opencl/ggml-opencl.cppindex b14ea8133b..16e851bbdb 100644--- a/ggml/src/ggml-opencl/ggml-opencl.cpp+++ b/ggml/src/ggml-opencl/ggml-opencl.cpp@@ -7319,6 +7319,14 @@ static bool ggml_opencl_supports_op(ggml_backend_dev_t dev, const struct ggml_tereturn false;}+ // Some compilers for A7x (Adreno 740, compiler E031.41) crashes when+ // building FA kernels with mixed or quant types (f32_f16, f32_q8_0, f32_q4_0)+ // Here we skip all A7x for these kernels to avoid crash+ if (backend_ctx->adreno_gen == ADRENO_GPU_GEN::A7X &&+ (is_f32_f16 || is_f32_q8_0 || is_f32_q4_0)) {+ return false;+ }+if (dk == 512) {if (backend_ctx->gpu_family == INTEL) {return false;diff --git a/ggml/src/ggml-opencl/kernels/flash_attn_f32_f16.cl b/ggml/src/ggml-opencl/kernels/flash_attn_f32_f16.clindex 1cc0cc8c34..6e43ee81e7 100644--- a/ggml/src/ggml-opencl/kernels/flash_attn_f32_f16.cl+++ b/ggml/src/ggml-opencl/kernels/flash_attn_f32_f16.cl@@ -30,6 +30,10 @@#elif defined(cl_qcom_subgroup_shuffle)#pragma OPENCL EXTENSION cl_qcom_subgroup_shuffle : enable#define HAS_SUBGROUP_SHUFFLE 1+// Adreno compilers that expose only cl_qcom_subgroup_shuffle do not declare the KHR+// name, so calling it is an implicit declaration and the program fails to build.+// Route it to the qcom builtin.+#define sub_group_shuffle_xor(val, mask) qcom_sub_group_shuffle_xor((val), (mask), CLK_SUB_GROUP_SHUFFLE_WIDTH_WAVE_SIZE_QCOM, 0.0f)#endif#define ACC_TYPE floatdiff --git a/ggml/src/ggml-opencl/kernels/flash_attn_f32_q4_0.cl b/ggml/src/ggml-opencl/kernels/flash_attn_f32_q4_0.clindex de09a1eaae..95d215971e 100644--- a/ggml/src/ggml-opencl/kernels/flash_attn_f32_q4_0.cl+++ b/ggml/src/ggml-opencl/kernels/flash_attn_f32_q4_0.cl@@ -10,6 +10,10 @@#elif defined(cl_qcom_subgroup_shuffle)#pragma OPENCL EXTENSION cl_qcom_subgroup_shuffle : enable#define HAS_SUBGROUP_SHUFFLE 1+// Adreno compilers that expose only cl_qcom_subgroup_shuffle do not declare the KHR+// name, so calling it is an implicit declaration and the program fails to build.+// Route it to the qcom builtin.+#define sub_group_shuffle_xor(val, mask) qcom_sub_group_shuffle_xor((val), (mask), CLK_SUB_GROUP_SHUFFLE_WIDTH_WAVE_SIZE_QCOM, 0.0f)#endif// Flash attention: Q=f32, K=q4_0, V=q4_0.diff --git a/ggml/src/ggml-opencl/kernels/flash_attn_f32_q8_0.cl b/ggml/src/ggml-opencl/kernels/flash_attn_f32_q8_0.clindex 46bc4bc9d9..7e89ed0bd8 100644--- a/ggml/src/ggml-opencl/kernels/flash_attn_f32_q8_0.cl+++ b/ggml/src/ggml-opencl/kernels/flash_attn_f32_q8_0.cl@@ -10,6 +10,10 @@#elif defined(cl_qcom_subgroup_shuffle)#pragma OPENCL EXTENSION cl_qcom_subgroup_shuffle : enable#define HAS_SUBGROUP_SHUFFLE 1+// Adreno compilers that expose only cl_qcom_subgroup_shuffle do not declare the KHR+// name, so calling it is an implicit declaration and the program fails to build.+// Route it to the qcom builtin.+#define sub_group_shuffle_xor(val, mask) qcom_sub_group_shuffle_xor((val), (mask), CLK_SUB_GROUP_SHUFFLE_WIDTH_WAVE_SIZE_QCOM, 0.0f)#endif// Flash attention: Q=f32, K=q8_0, V=q8_0.diff --git a/ggml/src/ggml-opencl/kernels/mul_mv_f16_f32_l4.cl b/ggml/src/ggml-opencl/kernels/mul_mv_f16_f32_l4.clindex da2e14ae99..97148d370f 100644--- a/ggml/src/ggml-opencl/kernels/mul_mv_f16_f32_l4.cl+++ b/ggml/src/ggml-opencl/kernels/mul_mv_f16_f32_l4.cl@@ -24,6 +24,10 @@#elif defined(cl_qcom_subgroup_shuffle)#pragma OPENCL EXTENSION cl_qcom_subgroup_shuffle : enable#define HAS_SUBGROUP_SHUFFLE 1+// Adreno compilers that expose only cl_qcom_subgroup_shuffle do not declare the KHR+// name, so calling it is an implicit declaration and the program fails to build.+// Route it to the qcom builtin.+#define sub_group_shuffle_xor(val, mask) qcom_sub_group_shuffle_xor((val), (mask), CLK_SUB_GROUP_SHUFFLE_WIDTH_WAVE_SIZE_QCOM, 0.0f)#endif// Assumes row size (ne00) is a multiple of 4