Skip to content

LLM inference in C/C++

C++ ggml

opencl: use a better matmul path on two Adreno GPU generations (#27640)

Hongqiang Wangc841aeeb8bb2
diff --git a/ggml/src/ggml-opencl/ggml-opencl.cpp b/ggml/src/ggml-opencl/ggml-opencl.cpp
index 6ae83449b0..426aac5231 100644
--- a/ggml/src/ggml-opencl/ggml-opencl.cpp
+++ b/ggml/src/ggml-opencl/ggml-opencl.cpp
@@ -6059,9 +6059,13 @@ static ggml_backend_opencl_context * ggml_cl_init(ggml_backend_dev_t dev) {
}
#ifdef GGML_OPENCL_USE_ADRENO_KERNELS
- // determine whether to use Adreno xmem GEMM
- backend_ctx->adreno_xmem_gemm_enabled = getenv("GGML_OPENCL_ADRENO_XMEM_GEMM") != nullptr &&
- backend_ctx->gpu_family == GPU_FAMILY::ADRENO;
+ // Adreno xmem F16xF32 GEMM, default on adreno, opt out with GGML_OPENCL_ADRENO_XMEM_GEMM=0.
+ // This helps models with f16 attention weights, e.g., gpt-oss-20b-f16
+ {
+ const char * xmem_env = getenv("GGML_OPENCL_ADRENO_XMEM_GEMM");
+ backend_ctx->adreno_xmem_gemm_enabled = backend_ctx->gpu_family == GPU_FAMILY::ADRENO &&
+ (xmem_env ? atoi(xmem_env) != 0 : true);
+ }
#endif
// determine whether to use large buffer for Adreno
@@ -19534,9 +19538,18 @@ static void ggml_cl_mul_mat(ggml_backend_t backend, const ggml_tensor * src0, co
// GEMM using local memory
// Current BK = 16, so ne00 % 16 == 0
+ //
+ // Certain A7X compiler (E031.41) executes kernel_mul_mm_f32_f32_l4_lm poorly;
+ // matrices with ne11 <= 8 appears OK.
+ // Fallback to the MV style kernels for A7x and ne11 > 8.
+ // Override with GGML_OPENCL_A7X_F32_LM_BYPASS=0.
+ static const char * a7x_f32lm_env = getenv("GGML_OPENCL_A7X_F32_LM_BYPASS");
+ static const bool a7x_f32lm_bypass = (a7x_f32lm_env == nullptr || a7x_f32lm_env[0] != '0');
if (src1t == GGML_TYPE_F32 &&
ne00 % 16 == 0 &&
- ne11 > 1) {
+ ne11 > 1 &&
+ !(a7x_f32lm_bypass && src0t == GGML_TYPE_F32 && ne11 > 8 &&
+ backend_ctx->adreno_gen == ADRENO_GPU_GEN::A7X)) {
switch(src0t) {
case GGML_TYPE_F32: {
kernel = backend_ctx->kernel_mul_mm_f32_f32_l4_lm;