LLM inference in C/C++
C++ ggml
opencl: fix get_tensor for q5_K adreno gemm_nonshuffle kernel (#29555)
lhez6dbbac442944
diff --git a/ggml/src/ggml-opencl/ggml-opencl.cpp b/ggml/src/ggml-opencl/ggml-opencl.cppindex e93d73344d..4dcc9e9574 100644--- a/ggml/src/ggml-opencl/ggml-opencl.cpp+++ b/ggml/src/ggml-opencl/ggml-opencl.cpp@@ -12454,9 +12454,13 @@ static void ggml_backend_opencl_buffer_get_tensor(ggml_backend_buffer_t buffer,buf_trans_d.allocate(backend_ctx->context, size_d);buf_trans_dm.allocate(backend_ctx->context, size_dm);+ // bin kernel transposes s but src kernel does not+ cl_mem buf_s = extra->s;+if (use_q5_k_bin_kernels(backend_ctx, tensor)) {transpose_2d_as_32b(backend_ctx, extra->q, buf_trans_q.buffer, size_q, M, K/8);transpose_2d_as_8b (backend_ctx, extra->s, buf_trans_s.buffer, size_s, M, K/256*12, true, true);+ buf_s = buf_trans_s.buffer;} else {transpose_2d_as_16b(backend_ctx, extra->q, buf_trans_q.buffer, size_q, M, K/4);}@@ -12467,7 +12471,7 @@ static void ggml_backend_opencl_buffer_get_tensor(ggml_backend_buffer_t buffer,cl_kernel kernel = backend_ctx->kernel_restore_block_q5_K_noshuffle;CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &buf_trans_q.buffer));CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &buf_trans_qh.buffer));- CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &buf_trans_s.buffer));+ CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &buf_s));CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &buf_trans_d.buffer));CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &buf_trans_dm.buffer));CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_mem), &data_device));