Skip to content

LLM inference in C/C++

C++ ggml

opencl: use flat mv q5_k when weight exceeds image1d_buffer_t limit (#26880)

lhezcb27fe9c35d9
diff --git a/ggml/src/ggml-opencl/ggml-opencl.cpp b/ggml/src/ggml-opencl/ggml-opencl.cpp
index 19dca4ced0..5d0324dad0 100644
--- a/ggml/src/ggml-opencl/ggml-opencl.cpp
+++ b/ggml/src/ggml-opencl/ggml-opencl.cpp
@@ -7076,6 +7076,19 @@ inline bool enable_adreno_trans_weight(const ggml_backend_opencl_context *backen
return ((elem_num < 128 * 1024 * 1024) && adreno_kernel && shape_ok); // max element num: 2**27
}
+inline bool enable_adreno_trans_weight_q5_K(const ggml_backend_opencl_context *backend_ctx, const ggml_tensor *tensor) {
+ if (!use_adreno_kernels(backend_ctx, tensor)) {
+ return false;
+ }
+
+ const size_t elem_num = ggml_nelements(tensor);
+ const size_t q_img_width = elem_num / 8;
+ const size_t qh_img_width = elem_num / 16;
+
+ return q_img_width <= backend_ctx->image_max_buffer_size &&
+ qh_img_width <= backend_ctx->image_max_buffer_size;
+}
+
static inline bool use_flat_gemv_for_large_m_q4_K(const ggml_tensor *tensor) {
// gemv_noshuffle variant perf drops for large M, use flat variant for large M.
// threshold is well above typical hidden/FFN dims, but below typical vocab sizes.
@@ -9255,7 +9268,7 @@ static void ggml_backend_opencl_buffer_set_tensor(ggml_backend_buffer_t buffer,
#ifdef GGML_OPENCL_USE_ADRENO_KERNELS
cl_kernel kernel = backend_ctx->kernel_convert_block_q5_K;
- if (use_adreno_kernels(backend_ctx, tensor)) {
+ if (enable_adreno_trans_weight_q5_K(backend_ctx, tensor)) {
kernel = backend_ctx->kernel_convert_block_q5_K_noshuffle;
}
#else
@@ -9290,7 +9303,7 @@ static void ggml_backend_opencl_buffer_set_tensor(ggml_backend_buffer_t buffer,
tensor->extra = extra;
#ifdef GGML_OPENCL_USE_ADRENO_KERNELS
- if (use_adreno_kernels(backend_ctx, tensor)) {
+ if (enable_adreno_trans_weight_q5_K(backend_ctx, tensor)) {
int M = tensor->ne[1];
int K = tensor->ne[0];
@@ -10388,7 +10401,7 @@ static void ggml_backend_opencl_buffer_get_tensor(ggml_backend_buffer_t buffer,
CL_CHECK(clReleaseMemObject(data_device));
return;
}
- if (use_adreno_kernels(backend_ctx, tensor)) {
+ if (enable_adreno_trans_weight_q5_K(backend_ctx, tensor)) {
int M = tensor->ne[1];
int K = tensor->ne[0];
@@ -18928,7 +18941,8 @@ static void ggml_cl_mul_mat(ggml_backend_t backend, const ggml_tensor * src0, co
}
// q5_K x fp32
- if (src0t == GGML_TYPE_Q5_K && src1t == GGML_TYPE_F32) {
+ if (src0t == GGML_TYPE_Q5_K && src1t == GGML_TYPE_F32 &&
+ enable_adreno_trans_weight_q5_K(backend_ctx, src0)) {
ggml_cl_mul_mat_q5_K_f32_adreno(backend, src0, src1, dst);
return;
}