LLM inference in C/C++
C++ ggml
opencl: add bin kernel `kernel_gemm_noshuffle_q4_k_q8_1_dp4a_ila_a8_bin` (#29056)
shaofeiqid5f66492e661
diff --git a/ggml/src/ggml-opencl/ggml-opencl.cpp b/ggml/src/ggml-opencl/ggml-opencl.cppindex db5d510a69..91cd1a0edd 100644--- a/ggml/src/ggml-opencl/ggml-opencl.cpp+++ b/ggml/src/ggml-opencl/ggml-opencl.cpp@@ -1238,6 +1238,7 @@ struct ggml_backend_opencl_context {cl_kernel kernel_gemv_noshuffle_q4_k_f32_mc3; // multi-column (N=3) verify GEMVcl_kernel kernel_gemm_noshuffle_q4_k_f32;cl_kernel kernel_gemm_noshuffle_q4_k_f32_32b_trans_ila_a8_bin;+ cl_kernel kernel_gemm_noshuffle_q4_k_q8_1_dp4a_ila_a8_bin;cl_kernel kernel_gemv_noshuffle_q4_k_f32_32b_trans;cl_kernel kernel_gemm_noshuffle_q4_k_q8_1_dp4a = nullptr; // dp4a (int8) dense prefill GEMMcl_kernel kernel_gemm_noshuffle_q4_k_q8_1_dp4a_wimg = nullptr; // dp4a dense prefill GEMM, weights via texture (X1 opt-in)@@ -4357,6 +4358,7 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) {backend_ctx->kernel_gemv_noshuffle_q4_k_f32_32b_trans = nullptr;backend_ctx->kernel_gemm_noshuffle_q4_k_f32_32b_trans_ila_a8_bin = nullptr;+ backend_ctx->kernel_gemm_noshuffle_q4_k_q8_1_dp4a_ila_a8_bin = nullptr;if (backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E) {{std::string opts = std::string("-cl-std=") + opencl_c_std +@@ -4389,6 +4391,17 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) {CL_CHECK(clReleaseProgram(bin_prog));GGML_LOG_CONT(".");}++ kernel_bin = (const char *)backend_ctx->get_adreno_bin_kernel("gemm_noshuffle_q4_k_q8_1_dp4a_ila_a8", &bin_size);+ if (kernel_bin && bin_size > 0) {+ cl_program bin_prog =+ build_program_from_binary(backend_ctx->context, backend_ctx->device, kernel_bin, "", bin_size);++ CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q4_k_q8_1_dp4a_ila_a8_bin =+ clCreateKernel(bin_prog, "kernel_gemm_noshuffle_q4_k_q8_1_dp4a_ila_a8", &err), err));+ CL_CHECK(clReleaseProgram(bin_prog));+ GGML_LOG_CONT(".");+ }}}@@ -19340,14 +19353,14 @@ static void ggml_cl_mul_mat_q4_0_f32_adreno_ila(ggml_backend_t backend, const ggcl_mem s_img = extra0_q4_0->d_img;GGML_ASSERT(a_img && s_img && "ILA Q4_0 weight images missing; set_tensor should have built them");- static const char * q4_0_ila_dp4a_env = getenv("GGML_OPENCL_Q4_0_ILA_DP4A");- bool q4_0_ila_dp4a_on = q4_0_ila_dp4a_env- ? (atoi(q4_0_ila_dp4a_env) != 0)+ static const char * q4_0_bin_dp4a_env = getenv("GGML_OPENCL_Q4_0_BIN_DP4A");+ bool q4_0_bin_dp4a_on = q4_0_bin_dp4a_env+ ? (atoi(q4_0_bin_dp4a_env) != 0): true;// dot prod has to be available- q4_0_ila_dp4a_on = backend_ctx->has_integer_dot && q4_0_ila_dp4a_on;+ q4_0_bin_dp4a_on = backend_ctx->has_integer_dot && q4_0_bin_dp4a_on;- if (q4_0_ila_dp4a_on && backend_ctx->kernel_gemm_noshuffle_q4_0_q8_1_dp4a_ila_a8_bin) {+ if (q4_0_bin_dp4a_on && backend_ctx->kernel_gemm_noshuffle_q4_0_q8_1_dp4a_ila_a8_bin) {const int dp4a_N_pad = CEIL_DIV(N, 32) * 32;const size_t n_blocks = (size_t)dp4a_N_pad * (K / 32);@@ -21172,6 +21185,75 @@ static void ggml_cl_mul_mat_q4_k_f32_adreno_ila(ggml_backend_t backend, const ggconst int gemm_tile_n = 64;int N_pad = CEIL_DIV(N, gemm_tile_n) * gemm_tile_n;+ static const char * q4_k_bin_dp4a_env = getenv("GGML_OPENCL_Q4_K_BIN_DP4A");+ bool q4_k_bin_dp4a_on = q4_k_bin_dp4a_env+ ? (atoi(q4_k_bin_dp4a_env) != 0)+ : true;+ // dot prod has to be available+ q4_k_bin_dp4a_on = backend_ctx->has_integer_dot && q4_k_bin_dp4a_on;++ if (q4_k_bin_dp4a_on && backend_ctx->kernel_gemm_noshuffle_q4_k_q8_1_dp4a_ila_a8_bin) {+ const int dp4a_N_pad = CEIL_DIV(N, 32) * 32;+ const size_t n_blocks = (size_t)dp4a_N_pad * (K / 32);++ backend_ctx->prealloc_moe_qa.allocate(context, (size_t)dp4a_N_pad * K * sizeof(cl_char));+ backend_ctx->prealloc_moe_da.allocate(context, n_blocks * sizeof(cl_half));+ backend_ctx->prealloc_moe_sa.allocate(context, n_blocks * sizeof(cl_half));++ cl_mem b_sub = nullptr;+ region.origin = offset1;+ region.size = (size_t)K * N * sizeof(float);+ CL_CHECK((b_sub = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err));++ cl_int tb = (cl_int)((size_t)N * (K / 32));+ cl_kernel qk = backend_ctx->kernel_quant_a_q8_1;+ CL_CHECK(clSetKernelArg(qk, 0, sizeof(cl_mem), &b_sub));+ CL_CHECK(clSetKernelArg(qk, 1, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer));+ CL_CHECK(clSetKernelArg(qk, 2, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer));+ CL_CHECK(clSetKernelArg(qk, 3, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer));+ CL_CHECK(clSetKernelArg(qk, 4, sizeof(cl_int), &tb));+ size_t q_local[1] = { 64 };+ size_t q_global[1] = { (size_t)CEIL_DIV(tb, 64) * 64 };+ backend_ctx->enqueue_ndrange_kernel(qk, 1, q_global, q_local, dst);++ cl_mem d_sub = nullptr;+ cl_mem d_img = nullptr;+ region.origin = offsetd;+ region.size = (size_t)M * N * sizeof(float);+ CL_CHECK((d_sub = clCreateSubBuffer(extrad->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err));++ img_fmt = { CL_R, CL_FLOAT };+ memset(&img_desc, 0, sizeof(img_desc));+ img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER;+ img_desc.image_width = (size_t)M * N;+ img_desc.buffer = d_sub;+ CL_CHECK((d_img = clCreateImage(context, CL_MEM_WRITE_ONLY, &img_fmt, &img_desc, NULL, &err), err));++ kernel = backend_ctx->kernel_gemm_noshuffle_q4_k_q8_1_dp4a_ila_a8_bin;++ cl_uint k_arg = 0;+ CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &extra0_q4_k->q_img));+ CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &extra0_q4_k->d));+ CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &extra0_q4_k->dm));+ CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &extra0_q4_k->s));+ CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer));+ CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer));+ CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer));+ CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &d_img));+ CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_uint), &ne00));+ CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_uint), &ne01));+ CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_int), &N));++ size_t local_work_size[3] = { 64, 1, 1 };+ size_t global_work_size[3] = { 64, (size_t)(M / 64), (size_t)(dp4a_N_pad / 32) };+ backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst);++ CL_CHECK(clReleaseMemObject(b_sub));+ CL_CHECK(clReleaseMemObject(d_img));+ CL_CHECK(clReleaseMemObject(d_sub));+ return;+ }+cl_mem b_sub_buf = nullptr;cl_mem b_padded = nullptr;cl_mem b_buf = nullptr;