LLM inference in C/C++
C++ ggml
opencl: use flat mv q5_k when weight exceeds image1d_buffer_t limit (#26880)
lhezcb27fe9c35d9
diff --git a/ggml/src/ggml-opencl/ggml-opencl.cpp b/ggml/src/ggml-opencl/ggml-opencl.cppindex 19dca4ced0..5d0324dad0 100644--- a/ggml/src/ggml-opencl/ggml-opencl.cpp+++ b/ggml/src/ggml-opencl/ggml-opencl.cpp@@ -7076,6 +7076,19 @@ inline bool enable_adreno_trans_weight(const ggml_backend_opencl_context *backenreturn ((elem_num < 128 * 1024 * 1024) && adreno_kernel && shape_ok); // max element num: 2**27}+inline bool enable_adreno_trans_weight_q5_K(const ggml_backend_opencl_context *backend_ctx, const ggml_tensor *tensor) {+ if (!use_adreno_kernels(backend_ctx, tensor)) {+ return false;+ }++ const size_t elem_num = ggml_nelements(tensor);+ const size_t q_img_width = elem_num / 8;+ const size_t qh_img_width = elem_num / 16;++ return q_img_width <= backend_ctx->image_max_buffer_size &&+ qh_img_width <= backend_ctx->image_max_buffer_size;+}+static inline bool use_flat_gemv_for_large_m_q4_K(const ggml_tensor *tensor) {// gemv_noshuffle variant perf drops for large M, use flat variant for large M.// threshold is well above typical hidden/FFN dims, but below typical vocab sizes.@@ -9255,7 +9268,7 @@ static void ggml_backend_opencl_buffer_set_tensor(ggml_backend_buffer_t buffer,#ifdef GGML_OPENCL_USE_ADRENO_KERNELScl_kernel kernel = backend_ctx->kernel_convert_block_q5_K;- if (use_adreno_kernels(backend_ctx, tensor)) {+ if (enable_adreno_trans_weight_q5_K(backend_ctx, tensor)) {kernel = backend_ctx->kernel_convert_block_q5_K_noshuffle;}#else@@ -9290,7 +9303,7 @@ static void ggml_backend_opencl_buffer_set_tensor(ggml_backend_buffer_t buffer,tensor->extra = extra;#ifdef GGML_OPENCL_USE_ADRENO_KERNELS- if (use_adreno_kernels(backend_ctx, tensor)) {+ if (enable_adreno_trans_weight_q5_K(backend_ctx, tensor)) {int M = tensor->ne[1];int K = tensor->ne[0];@@ -10388,7 +10401,7 @@ static void ggml_backend_opencl_buffer_get_tensor(ggml_backend_buffer_t buffer,CL_CHECK(clReleaseMemObject(data_device));return;}- if (use_adreno_kernels(backend_ctx, tensor)) {+ if (enable_adreno_trans_weight_q5_K(backend_ctx, tensor)) {int M = tensor->ne[1];int K = tensor->ne[0];@@ -18928,7 +18941,8 @@ static void ggml_cl_mul_mat(ggml_backend_t backend, const ggml_tensor * src0, co}// q5_K x fp32- if (src0t == GGML_TYPE_Q5_K && src1t == GGML_TYPE_F32) {+ if (src0t == GGML_TYPE_Q5_K && src1t == GGML_TYPE_F32 &&+ enable_adreno_trans_weight_q5_K(backend_ctx, src0)) {ggml_cl_mul_mat_q5_K_f32_adreno(backend, src0, src1, dst);return;}