LLM inference in C/C++
C++ ggml
opencl: route large q6_K lm_head to the flat GEMV (#26427)
Hongqiang Wang39eab74a05d3
diff --git a/ggml/src/ggml-opencl/ggml-opencl.cpp b/ggml/src/ggml-opencl/ggml-opencl.cppindex 915c8e7b90..fc0fce0d78 100644--- a/ggml/src/ggml-opencl/ggml-opencl.cpp+++ b/ggml/src/ggml-opencl/ggml-opencl.cpp@@ -7065,7 +7065,7 @@ static inline bool use_flat_gemv_for_large_m_q4_K(const ggml_tensor *tensor) {return tensor->ne[1] >= 32768 && tensor->ne[2] == 1 && tensor->ne[3] == 1;}-static inline bool use_flat_gemv_for_large_m_q6_K(const ggml_tensor *tensor) {+static inline bool use_flat_gemv_for_large_m_q6_K(const ggml_backend_opencl_context *backend_ctx, const ggml_tensor *tensor) {// gemv_noshuffle variant perf drops for large M, use flat variant for large M.// threshold is well above typical hidden/FFN dims, but below typical vocab sizes.// q6_K flat gemv is worse for smaller K; 2048 seems to be a reasonable threshold.@@ -7083,7 +7083,15 @@ static inline bool use_flat_gemv_for_large_m_q6_K(const ggml_tensor *tensor) {if ((tensor->ne[1] % 128 != 0) && tensor->ne[2] == 1 && tensor->ne[3] == 1) {return true;}- return tensor->ne[1] >= 32768 && tensor->ne[0] >= 2048 && tensor->ne[2] == 1 && tensor->ne[3] == 1;++ // The gemv_noshuffle slowdown tracks TOTAL weight size, not ne0 alone; ne0 >= 2048 is a+ // proxy for "large weight" that misses a narrow-hidden vocab-scale lm_head.+ // Add a direct size escape so such weights also take the flat path, without changing+ // which weights ne0 >= 2048 already routes there.+ // The size escape is not taken on the A7X since its compiler miscompiles the flat K-quant GEMV+ return tensor->ne[1] >= 32768+ && (tensor->ne[0] >= 2048 || (backend_ctx->adreno_gen != ADRENO_GPU_GEN::A7X && ggml_nbytes(tensor) >= (256ull << 20)))+ && tensor->ne[2] == 1 && tensor->ne[3] == 1;}static bool ggml_opencl_supports_op(ggml_backend_dev_t dev, const struct ggml_tensor * op) {@@ -9403,7 +9411,7 @@ static void ggml_backend_opencl_buffer_set_tensor(ggml_backend_buffer_t buffer,cl_kernel kernel;#ifdef GGML_OPENCL_USE_ADRENO_KERNELSkernel = backend_ctx->kernel_convert_block_q6_K;- if (use_adreno_kernels(backend_ctx, tensor) && !use_flat_gemv_for_large_m_q6_K(tensor)) {+ if (use_adreno_kernels(backend_ctx, tensor) && !use_flat_gemv_for_large_m_q6_K(backend_ctx, tensor)) {kernel = backend_ctx->kernel_convert_block_q6_K_noshuffle;}#else@@ -9436,7 +9444,7 @@ static void ggml_backend_opencl_buffer_set_tensor(ggml_backend_buffer_t buffer,tensor->extra = extra;#ifdef GGML_OPENCL_USE_ADRENO_KERNELS- if (use_adreno_kernels(backend_ctx, tensor) && !use_flat_gemv_for_large_m_q6_K(tensor)) {+ if (use_adreno_kernels(backend_ctx, tensor) && !use_flat_gemv_for_large_m_q6_K(backend_ctx, tensor)) {cl_int M = tensor->ne[1]; // ne01cl_int K = tensor->ne[0]; // ne00@@ -10473,7 +10481,7 @@ static void ggml_backend_opencl_buffer_get_tensor(ggml_backend_buffer_t buffer,CL_CHECK(clReleaseMemObject(data_device));return;}- if (use_adreno_kernels(backend_ctx, tensor) && !use_flat_gemv_for_large_m_q6_K(tensor)) {+ if (use_adreno_kernels(backend_ctx, tensor) && !use_flat_gemv_for_large_m_q6_K(backend_ctx, tensor)) {static ggml_cl_buffer buf_trans_ql;static ggml_cl_buffer buf_trans_qh;static ggml_cl_buffer buf_trans_s;@@ -18895,7 +18903,7 @@ static void ggml_cl_mul_mat(ggml_backend_t backend, const ggml_tensor * src0, co}// q6_K x fp32- if (src0t == GGML_TYPE_Q6_K && src1t == GGML_TYPE_F32 && !use_flat_gemv_for_large_m_q6_K(src0)) {+ if (src0t == GGML_TYPE_Q6_K && src1t == GGML_TYPE_F32 && !use_flat_gemv_for_large_m_q6_K(backend_ctx, src0)) {ggml_cl_mul_mat_q6_K_f32_adreno(backend, src0, src1, dst);return;}