LLM inference in C/C++
C++ ggml
opencl: add bin kernels `kernel_gemm_moe_q4_0_q8_1_dp4a_bin`, `kernel_gemm_moe_mxfp4_q8_1_dp4a_bin` (#27768)
Shawn Gu58546250cfaa
diff --git a/ggml/src/ggml-opencl/ggml-opencl.cpp b/ggml/src/ggml-opencl/ggml-opencl.cppindex 64f3325b2a..6ae83449b0 100644--- a/ggml/src/ggml-opencl/ggml-opencl.cpp+++ b/ggml/src/ggml-opencl/ggml-opencl.cpp@@ -903,6 +903,8 @@ struct ggml_backend_opencl_context {cl_kernel kernel_gemv_moe_mxfp4_f32_ns_wimg = nullptr; // weight-as-texture MoE decode GEMVcl_kernel kernel_gemm_moe_mxfp4_q8_1_dp4a = nullptr; // dp4a (int8) mxfp4 MoE prefill GEMMcl_kernel kernel_gemm_moe_q4_0_q8_1_dp4a = nullptr; // dp4a (int8) q4_0 MoE prefill GEMM+ cl_kernel kernel_gemm_moe_mxfp4_q8_1_dp4a_bin = nullptr; // binary dp4a (int8) mxfp4 MoE prefill GEMM+ cl_kernel kernel_gemm_moe_q4_0_q8_1_dp4a_bin = nullptr; // binary dp4a (int8) q4_0 MoE prefill GEMMcl_kernel kernel_moe_reorder_b;cl_kernel kernel_moe_histogram, kernel_moe_scan, kernel_moe_fill, kernel_moe_scatter;cl_kernel kernel_moe_scatter_stable = nullptr; // deterministic slot assignment@@ -4248,6 +4250,24 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) {GGML_LOG_CONT(".");}+ // gemm_moe_mxfp4_q8_1_dp4a_bin (dp4a prefill GEMM)+ if (backend_ctx->has_integer_dot) {+ size_t bin_size = 0;+ backend_ctx->kernel_gemm_moe_mxfp4_q8_1_dp4a_bin = nullptr;++ if (use_adreno_bin_kernels(backend_ctx)) {+ const char * kernel_bin = (const char *)backend_ctx->get_adreno_bin_kernel("gemm_moe_mxfp4_q8_1_dp4a_ila", &bin_size);+ if (kernel_bin && bin_size > 0) {+ cl_program prog =+ build_program_from_binary(backend_ctx->context, backend_ctx->device, kernel_bin, CL_moe_compile_opts, bin_size);++ CL_CHECK((backend_ctx->kernel_gemm_moe_mxfp4_q8_1_dp4a_bin = clCreateKernel(prog, "kernel_gemm_moe_mxfp4_q8_1_dp4a_ila", &err), err));+ CL_CHECK(clReleaseProgram(prog));+ GGML_LOG_CONT(".");+ }+ }+ }+// gemm_moe_q4_0_q8_1_dp4a (dp4a prefill GEMM)if (backend_ctx->has_integer_dot) {#ifdef GGML_OPENCL_EMBED_KERNELS@@ -4265,6 +4285,24 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) {GGML_LOG_CONT(".");}+ // gemm_moe_q4_0_q8_1_dp4a_bin (dp4a prefill GEMM)+ if (backend_ctx->has_integer_dot) {+ size_t bin_size = 0;+ backend_ctx->kernel_gemm_moe_q4_0_q8_1_dp4a_bin = nullptr;++ if (use_adreno_bin_kernels(backend_ctx)) {+ const char * kernel_bin = (const char *)backend_ctx->get_adreno_bin_kernel("gemm_moe_q4_0_q8_1_dp4a_ila", &bin_size);+ if (kernel_bin && bin_size > 0) {+ cl_program prog =+ build_program_from_binary(backend_ctx->context, backend_ctx->device, kernel_bin, CL_moe_compile_opts, bin_size);++ CL_CHECK((backend_ctx->kernel_gemm_moe_q4_0_q8_1_dp4a_bin = clCreateKernel(prog, "kernel_gemm_moe_q4_0_q8_1_dp4a_ila", &err), err));+ CL_CHECK(clReleaseProgram(prog));+ GGML_LOG_CONT(".");+ }+ }+ }+// gemm_moe_q8_1_dp4a (generic dp4a MoE GEMM; MOE_QT=80 -> q8_0 expert variant)if (backend_ctx->has_integer_dot) {#ifdef GGML_OPENCL_EMBED_KERNELS@@ -21519,7 +21557,9 @@ static void ggml_cl_mul_mat_id(ggml_backend_t backend, const ggml_tensor * src0,// dot prod has to be availableuse_moe_dp4a = backend_ctx->has_integer_dot && use_moe_dp4a;// bin kernel takes precedence- use_moe_dp4a = use_moe_dp4a && backend_ctx->kernel_gemm_moe_q4_0_f32_ns_bin == nullptr;+ if (backend_ctx->kernel_gemm_moe_q4_0_q8_1_dp4a_bin == nullptr) {+ use_moe_dp4a = use_moe_dp4a && backend_ctx->kernel_gemm_moe_q4_0_f32_ns_bin == nullptr;+ }cl_buffer_region region;region.origin = 0;@@ -21625,6 +21665,10 @@ static void ggml_cl_mul_mat_id(ggml_backend_t backend, const ggml_tensor * src0,// dp4a GEMMcl_kernel dk = backend_ctx->kernel_gemm_moe_q4_0_q8_1_dp4a;+ if (backend_ctx->kernel_gemm_moe_q4_0_q8_1_dp4a_bin) {+ dk = backend_ctx->kernel_gemm_moe_q4_0_q8_1_dp4a_bin;+ }+int aidx = 0;CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &extra0_q4_0->q_img));CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &extra0_q4_0->d));@@ -23463,8 +23507,10 @@ static void ggml_cl_mul_mat_id(ggml_backend_t backend, const ggml_tensor * src0,: (backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E);// dot prod has to be availableuse_moe_dp4a = backend_ctx->has_integer_dot && use_moe_dp4a;- // bin kernel takes precedence- use_moe_dp4a = use_moe_dp4a && backend_ctx->kernel_gemm_moe_mxfp4_f32_ns_bin == nullptr;+ // bin kernel takes precedence, dp4a bin kernel has higher priority than normal bin kernel+ if (backend_ctx->kernel_gemm_moe_mxfp4_q8_1_dp4a_bin == nullptr) {+ use_moe_dp4a = use_moe_dp4a && backend_ctx->kernel_gemm_moe_mxfp4_f32_ns_bin == nullptr;+ }cl_buffer_region region;region.origin = 0;@@ -23573,6 +23619,10 @@ static void ggml_cl_mul_mat_id(ggml_backend_t backend, const ggml_tensor * src0,// dp4a GEMMcl_kernel dk = backend_ctx->kernel_gemm_moe_mxfp4_q8_1_dp4a;+ if (backend_ctx->kernel_gemm_moe_mxfp4_q8_1_dp4a_bin) {+ dk = backend_ctx->kernel_gemm_moe_mxfp4_q8_1_dp4a_bin;+ }+int aidx = 0;CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &extra0_mxfp4->q_img));CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &extra0_mxfp4->e));