LLM inference in C/C++
C++ ggml
opencl: load and use `kernel_gemm_moe_q6_k_f32_ns` from bin kernel lib (#25797)
lhez86a9c79f8667
diff --git a/ggml/src/ggml-opencl/ggml-opencl.cpp b/ggml/src/ggml-opencl/ggml-opencl.cppindex 472cee29a0..3563f40c4f 100644--- a/ggml/src/ggml-opencl/ggml-opencl.cpp+++ b/ggml/src/ggml-opencl/ggml-opencl.cpp@@ -876,7 +876,7 @@ struct ggml_backend_opencl_context {cl_kernel kernel_gemm_moe_q8_1_dp4a_q5k = nullptr; // generic dp4a MoE GEMM (MOE_QT=5, q5_K), opt-incl_kernel kernel_moe_expand_scale_q5_K = nullptr; // q5_K 6-bit s[] -> uniform scale[16]/min[8]cl_kernel kernel_gemv_moe_q5_k_f32_ns, kernel_gemm_moe_q5_k_f32_ns;- cl_kernel kernel_gemv_moe_q6_k_f32_ns, kernel_gemm_moe_q6_k_f32_ns;+ cl_kernel kernel_gemv_moe_q6_k_f32_ns, kernel_gemm_moe_q6_k_f32_ns, kernel_gemm_moe_q6_k_f32_ns_bin;cl_kernel kernel_gemm_moe_q6_k_q8_1_dp4a = nullptr; // dp4a (int8) q6_K MoE prefill GEMMcl_kernel kernel_gemv_moe_mxfp4_f32, kernel_gemm_moe_mxfp4_f32;cl_kernel kernel_gemv_moe_mxfp4_f32_ns, kernel_gemm_moe_mxfp4_f32_ns, kernel_gemm_moe_mxfp4_f32_ns_bin;@@ -4310,6 +4310,24 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) {GGML_LOG_CONT(".");}+ // gemm_moe_q6_k_f32_ns_bin+ {+ size_t bin_size = 0;+ backend_ctx->kernel_gemm_moe_q6_k_f32_ns_bin = nullptr;++ if (use_adreno_bin_kernels(backend_ctx)) {+ const char * kernel_bin = (const char *)backend_ctx->get_adreno_bin_kernel("gemm_moe_q6_k_f32_ns_ila", &bin_size);+ if (kernel_bin && bin_size > 0) {+ cl_program prog =+ build_program_from_binary(backend_ctx->context, backend_ctx->device, kernel_bin, CL_moe_compile_opts, bin_size);++ CL_CHECK((backend_ctx->kernel_gemm_moe_q6_k_f32_ns_bin = clCreateKernel(prog, "kernel_gemm_moe_q6_k_f32_ns_ila", &err), err));+ CL_CHECK(clReleaseProgram(prog));+ GGML_LOG_CONT(".");+ }+ }+ }+// gemm_moe_q6_k_q8_1_dp4a (dp4a q6_K MoE prefill GEMM)if (backend_ctx->has_integer_dot) {#ifdef GGML_OPENCL_EMBED_KERNELS@@ -22291,8 +22309,8 @@ static void ggml_cl_mul_mat_id(ggml_backend_t backend, const ggml_tensor * src0,bool use_moe_dp4a = q5kmdp4a_on&& backend_ctx->kernel_gemm_moe_q8_1_dp4a_q5k != nullptr&& extra0_q5_K->scale != nullptr;- // bin kernel takes precedence- use_moe_dp4a = use_moe_dp4a && backend_ctx->kernel_gemm_moe_q4_k_f32_ns_bin == nullptr;+ // dot prod has to be available+ use_moe_dp4a = backend_ctx->has_integer_dot && use_moe_dp4a;if (use_moe_dp4a) {const size_t tok_slots = (size_t)max_post_router_tile * n_tile_size;@@ -22510,6 +22528,9 @@ static void ggml_cl_mul_mat_id(ggml_backend_t backend, const ggml_tensor * src0,} else { // for gemmkernel = backend_ctx->kernel_gemm_moe_q6_k_f32_ns;+ if (backend_ctx->kernel_gemm_moe_q6_k_f32_ns_bin) {+ kernel = backend_ctx->kernel_gemm_moe_q6_k_f32_ns_bin;+ }// Reorder router if called from test-backend-ops or when new router is generated.// Otherwise reuse the reordered result from previous mul_mat_id call.@@ -22530,6 +22551,8 @@ static void ggml_cl_mul_mat_id(ggml_backend_t backend, const ggml_tensor * src0,|| backend_ctx->adreno_gen == ADRENO_GPU_GEN::X1E);// dot prod has to be availableuse_moe_dp4a = backend_ctx->has_integer_dot && use_moe_dp4a;+ // bin kernel takes precedence+ use_moe_dp4a = use_moe_dp4a && backend_ctx->kernel_gemm_moe_q6_k_f32_ns_bin == nullptr;cl_buffer_region region;region.origin = 0;@@ -22566,6 +22589,11 @@ static void ggml_cl_mul_mat_id(ggml_backend_t backend, const ggml_tensor * src0,CL_CHECK(status);cl_image_format image_format_buf_src1 = {CL_RGBA, CL_FLOAT};cl_image_desc image_desc_buf_src1 = {CL_MEM_OBJECT_IMAGE1D_BUFFER, static_cast<size_t>(ne00 * max_post_router_tile * n_tile_size / 4), 0,0,0,0,0,0,0, {buf_src1_reordered}};+ if (backend_ctx->kernel_gemm_moe_q6_k_f32_ns_bin) {+ // bin kernel uses slightly different image format+ image_format_buf_src1 = {CL_R, CL_FLOAT};+ image_desc_buf_src1.image_width = static_cast<size_t>(ne00 * max_post_router_tile * n_tile_size);+ }image_src1_reordered = clCreateImage(backend_ctx->context, CL_MEM_READ_ONLY, &image_format_buf_src1, &image_desc_buf_src1, NULL, &status);CL_CHECK(status);