LLM inference in C/C++
C++ ggml
opencl: disable FA and MoE weights repack to work around compiler issues for Adreno 850 GPU (#25745)
Hongqiang Wang3dafb585f834
diff --git a/ggml/src/ggml-opencl/ggml-opencl.cpp b/ggml/src/ggml-opencl/ggml-opencl.cppindex 76c5e0cafb..2a7411c019 100644--- a/ggml/src/ggml-opencl/ggml-opencl.cpp+++ b/ggml/src/ggml-opencl/ggml-opencl.cpp@@ -123,6 +123,7 @@ enum ADRENO_GPU_GEN {enum ADRENO_CL_COMPILER_TYPE {E031,+ E17,DX,};@@ -264,7 +265,8 @@ static ADRENO_GPU_GEN get_adreno_gpu_gen(const char *device_name) {}if (strstr(device_name, "830") ||- strstr(device_name, "840")) {+ strstr(device_name, "840") ||+ strstr(device_name, "850")) {return ADRENO_GPU_GEN::A8X;}@@ -288,6 +290,17 @@ static ggml_cl_compiler_version get_adreno_cl_compiler_version(const char *drivesize_t compiler_minor_offset = 8;size_t compiler_patch_offset = 11;+ if (compiler_ver_pos == std::string::npos) {+ compiler_ver_pos = driver_ver_str.find("E17");+ if (compiler_ver_pos != std::string::npos) {+ type = ADRENO_CL_COMPILER_TYPE::E17;+ compiler_ver_len = 12;+ compiler_major_offset = 4;+ compiler_minor_offset = 7;+ compiler_patch_offset = 10;+ }+ }+if (compiler_ver_pos == std::string::npos) {compiler_ver_pos = driver_ver_str.find("DX");if (compiler_ver_pos == std::string::npos) {@@ -296,6 +309,8 @@ static ggml_cl_compiler_version get_adreno_cl_compiler_version(const char *drivetype = ADRENO_CL_COMPILER_TYPE::DX;compiler_ver_len = 11;compiler_major_offset = 3;+ compiler_minor_offset = 6;+ compiler_patch_offset = 9;}std::string compiler_ver_str = driver_ver_str.substr(compiler_ver_pos, compiler_ver_len);@@ -1655,6 +1670,7 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) {// those compiler versions since it is anyway not used for Adreno.if (backend_ctx->gpu_family != ADRENO ||backend_ctx->adreno_cl_compiler_version.newer_than_or_same(E031, 38, 11, 0) ||+ backend_ctx->adreno_cl_compiler_version.type == E17 ||backend_ctx->adreno_cl_compiler_version.type == DX) {#ifdef GGML_OPENCL_EMBED_KERNELSconst std::string kernel_src {@@ -6946,6 +6962,15 @@ inline bool use_adreno_kernels(const ggml_backend_opencl_context *backend_ctx, creturn threashold_ok;}+static bool adreno_e17_compiler_quirks(const ggml_backend_opencl_context *backend_ctx) {+ if (!backend_ctx || backend_ctx->gpu_family != GPU_FAMILY::ADRENO ||+ backend_ctx->adreno_cl_compiler_version.type != ADRENO_CL_COMPILER_TYPE::E17) {+ return false;+ }+ const char * env = getenv("GGML_OPENCL_ADRENO_E17_QUIRKS");+ return !(env && env[0] == '0');+}+inline bool use_adreno_moe_kernels(const ggml_backend_opencl_context *backend_ctx, const ggml_tensor *tensor) {// The moe weight repack kernels *_trans4_ns alias a private ushort8 through a uchar*.// Certain compilers (found with some A7x and A6x) miscompiles this, corrupting the weights.@@ -6957,6 +6982,11 @@ inline bool use_adreno_moe_kernels(const ggml_backend_opencl_context *backend_ctbackend_ctx->adreno_gen == ADRENO_GPU_GEN::ADRENO_UNKNOWN)) {return false;}++ if (adreno_e17_compiler_quirks(backend_ctx)) {+ return false;+ }+int ne01 = tensor->ne[1];return (((strstr(tensor->name, "ffn") != NULL) && (strstr(tensor->name, "exps") != NULL)) || (strstr(tensor->name, "as") != NULL)) && (ne01 % 32 == 0);}@@ -7289,6 +7319,10 @@ static bool ggml_opencl_supports_op(ggml_backend_dev_t dev, const struct ggml_tecase GGML_OP_MEAN:return op->src[0]->type == GGML_TYPE_F32;case GGML_OP_FLASH_ATTN_EXT: {+ // The E17 compilers segfault while building FA kernels, skip E17 for now+ if (adreno_e17_compiler_quirks(backend_ctx)) {+ return false;+ }const ggml_tensor * q = op->src[0];const ggml_tensor * k = op->src[1];const ggml_tensor * v = op->src[2];diff --git a/ggml/src/ggml-opencl/kernels/gemm_moe_mxfp4_f32_ns.cl b/ggml/src/ggml-opencl/kernels/gemm_moe_mxfp4_f32_ns.clindex 834050a4f9..10c8855c1e 100644--- a/ggml/src/ggml-opencl/kernels/gemm_moe_mxfp4_f32_ns.cl+++ b/ggml/src/ggml-opencl/kernels/gemm_moe_mxfp4_f32_ns.cl@@ -274,8 +274,9 @@ kernel void kernel_gemm_moe_mxfp4_f32_ns(shared_b[b_local_offset.y] = bx8_f16.hi;// Dequantization- reg_a.lo = mxfp4_to_fp16_packed8(as_ushort2(mxfp4x16.lo)) * s;- reg_a.hi = mxfp4_to_fp16_packed8(as_ushort2(mxfp4x16.hi)) * s;+ // Cast the e8m0 scale to half to satisfy E17 compilers+ reg_a.lo = mxfp4_to_fp16_packed8(as_ushort2(mxfp4x16.lo)) * (half)s;+ reg_a.hi = mxfp4_to_fp16_packed8(as_ushort2(mxfp4x16.hi)) * (half)s;sub_group_barrier(CLK_LOCAL_MEM_FENCE);@@ -304,8 +305,9 @@ kernel void kernel_gemm_moe_mxfp4_f32_ns(shared_b[b_local_offset.y] = bx8_f16.hi;// Dequantization- reg_a.lo = mxfp4_to_fp16_packed8(as_ushort2(mxfp4x16.lo)) * s;- reg_a.hi = mxfp4_to_fp16_packed8(as_ushort2(mxfp4x16.hi)) * s;+ // Cast the e8m0 scale to half to satisfy E17 compilers+ reg_a.lo = mxfp4_to_fp16_packed8(as_ushort2(mxfp4x16.lo)) * (half)s;+ reg_a.hi = mxfp4_to_fp16_packed8(as_ushort2(mxfp4x16.hi)) * (half)s;sub_group_barrier(CLK_LOCAL_MEM_FENCE);diff --git a/ggml/src/ggml-opencl/kernels/mul_mv_q4_k_f32.cl b/ggml/src/ggml-opencl/kernels/mul_mv_q4_k_f32.clindex 71ab989821..4c3d5f00c7 100644--- a/ggml/src/ggml-opencl/kernels/mul_mv_q4_k_f32.cl+++ b/ggml/src/ggml-opencl/kernels/mul_mv_q4_k_f32.cl@@ -1,3 +1,5 @@+#pragma OPENCL EXTENSION cl_khr_fp16 : enable+#ifdef cl_intel_required_subgroup_size#pragma OPENCL EXTENSION cl_intel_required_subgroup_size : enable#define INTEL_GPU 1