LLM inference in C/C++
C++ ggml
opencl: add bin kernels kernel_gemm_moe_q4_k_q8_1_dp4a_bin, kernel_gemm_moe_q6_k_q8_1_dp4a_bin (#30187)
Shawn Guf2918cabbffe
diff --git a/ggml/src/ggml-opencl/ggml-opencl.cpp b/ggml/src/ggml-opencl/ggml-opencl.cppindex 45b791edc4..ea85d99bcf 100644--- a/ggml/src/ggml-opencl/ggml-opencl.cpp+++ b/ggml/src/ggml-opencl/ggml-opencl.cpp@@ -1024,6 +1024,8 @@ struct ggml_backend_opencl_context {cl_kernel kernel_gemm_moe_q4_0_q8_1_dp4a = nullptr; // dp4a (int8) q4_0 MoE prefill GEMMcl_kernel kernel_gemm_moe_mxfp4_q8_1_dp4a_bin = nullptr; // binary dp4a (int8) mxfp4 MoE prefill GEMMcl_kernel kernel_gemm_moe_q4_0_q8_1_dp4a_bin = nullptr; // binary dp4a (int8) q4_0 MoE prefill GEMM+ cl_kernel kernel_gemm_moe_q4_k_q8_1_dp4a_bin = nullptr; // binary dp4a (int8) q4_k MoE prefill GEMM+ cl_kernel kernel_gemm_moe_q6_k_q8_1_dp4a_bin = nullptr; // binary dp4a (int8) q6_k MoE prefill GEMMcl_kernel kernel_moe_reorder_b;cl_kernel kernel_moe_histogram, kernel_moe_scan, kernel_moe_fill, kernel_moe_scatter;cl_kernel kernel_moe_scatter_stable = nullptr; // deterministic slot assignment@@ -4830,6 +4832,24 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) {GGML_LOG_CONT(".");}+ // gemm_moe_q4_k_q8_1_dp4a_bin (dp4a prefill GEMM)+ if (backend_ctx->has_integer_dot) {+ size_t bin_size = 0;+ backend_ctx->kernel_gemm_moe_q4_k_q8_1_dp4a_bin = nullptr;++ if (use_adreno_bin_kernels(backend_ctx)) {+ const char * kernel_bin = (const char *)backend_ctx->get_adreno_bin_kernel("gemm_moe_q4_k_q8_1_dp4a_ila", &bin_size);+ if (kernel_bin && bin_size > 0) {+ cl_program prog =+ build_program_from_binary(backend_ctx->context, backend_ctx->device, kernel_bin, CL_moe_compile_opts, bin_size);++ CL_CHECK((backend_ctx->kernel_gemm_moe_q4_k_q8_1_dp4a_bin = clCreateKernel(prog, "kernel_gemm_moe_q4_k_q8_1_dp4a_ila", &err), err));+ CL_CHECK(clReleaseProgram(prog));+ GGML_LOG_CONT(".");+ }+ }+ }+// gemm_moe_mxfp4_q8_1_dp4a (dp4a prefill GEMM)if (backend_ctx->has_integer_dot) {#ifdef GGML_OPENCL_EMBED_KERNELS@@ -5049,6 +5069,24 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) {GGML_LOG_CONT(".");}+ // gemm_moe_q6_k_q8_1_dp4a_bin (dp4a prefill GEMM)+ if (backend_ctx->has_integer_dot) {+ size_t bin_size = 0;+ backend_ctx->kernel_gemm_moe_q6_k_q8_1_dp4a_bin = nullptr;++ if (use_adreno_bin_kernels(backend_ctx)) {+ const char * kernel_bin = (const char *)backend_ctx->get_adreno_bin_kernel("gemm_moe_q6_k_q8_1_dp4a_ila", &bin_size);+ if (kernel_bin && bin_size > 0) {+ cl_program prog =+ build_program_from_binary(backend_ctx->context, backend_ctx->device, kernel_bin, CL_moe_compile_opts, bin_size);++ CL_CHECK((backend_ctx->kernel_gemm_moe_q6_k_q8_1_dp4a_bin = clCreateKernel(prog, "kernel_gemm_moe_q6_k_q8_1_dp4a_ila", &err), err));+ CL_CHECK(clReleaseProgram(prog));+ GGML_LOG_CONT(".");+ }+ }+ }+// gemv_moe_mxfp4_f32_ns{#ifdef GGML_OPENCL_EMBED_KERNELS@@ -27185,8 +27223,10 @@ static void ggml_cl_mul_mat_id(ggml_backend_t backend, const ggml_tensor * src0,: (backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E || backend_ctx->adreno_gen == ADRENO_GPU_GEN::X1E);// dot prod has to be availableuse_moe_dp4a = backend_ctx->has_integer_dot && use_moe_dp4a;- // bin kernel takes precedence- use_moe_dp4a = use_moe_dp4a && backend_ctx->kernel_gemm_moe_q4_k_f32_ns_bin == nullptr;+ // bin kernel takes precedence, dp4a bin kernel has higher priority than normal bin kernel+ if (backend_ctx->kernel_gemm_moe_q4_k_q8_1_dp4a_bin == nullptr) {+ use_moe_dp4a = use_moe_dp4a && backend_ctx->kernel_gemm_moe_q4_k_f32_ns_bin == nullptr;+ }cl_buffer_region region;region.origin = 0;@@ -27288,6 +27328,10 @@ static void ggml_cl_mul_mat_id(ggml_backend_t backend, const ggml_tensor * src0,// dp4a GEMMcl_kernel dk = backend_ctx->kernel_gemm_moe_q4_k_q8_1_dp4a;+ if (backend_ctx->kernel_gemm_moe_q4_k_q8_1_dp4a_bin) {+ dk = backend_ctx->kernel_gemm_moe_q4_k_q8_1_dp4a_bin;+ }+int aidx = 0;CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &extra0_q4_K->q_img));CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &extra0_q4_K->d));@@ -27695,8 +27739,10 @@ static void ggml_cl_mul_mat_id(ggml_backend_t backend, const ggml_tensor * src0,|| backend_ctx->adreno_gen == ADRENO_GPU_GEN::X1E);// dot prod has to be availableuse_moe_dp4a = backend_ctx->has_integer_dot && use_moe_dp4a;- // bin kernel takes precedence- use_moe_dp4a = use_moe_dp4a && backend_ctx->kernel_gemm_moe_q6_k_f32_ns_bin == nullptr;+ // bin kernel takes precedence, dp4a bin kernel has higher priority than normal bin kernel+ if (backend_ctx->kernel_gemm_moe_q6_k_q8_1_dp4a_bin == nullptr) {+ use_moe_dp4a = use_moe_dp4a && backend_ctx->kernel_gemm_moe_q6_k_f32_ns_bin == nullptr;+ }cl_buffer_region region;region.origin = 0;@@ -27798,6 +27844,10 @@ static void ggml_cl_mul_mat_id(ggml_backend_t backend, const ggml_tensor * src0,backend_ctx->enqueue_ndrange_kernel(rq, 2, rq_global, rq_local, dst);cl_kernel dk = backend_ctx->kernel_gemm_moe_q6_k_q8_1_dp4a;+ if (backend_ctx->kernel_gemm_moe_q6_k_q8_1_dp4a_bin) {+ dk = backend_ctx->kernel_gemm_moe_q6_k_q8_1_dp4a_bin;+ }+int qi = 0;CL_CHECK(clSetKernelArg(dk, qi++, sizeof(cl_mem), &extra0_q6_K->ql_img));CL_CHECK(clSetKernelArg(dk, qi++, sizeof(cl_mem), &extra0_q6_K->qh));