Skip to content

LLM inference in C/C++

C++ ggml

opencl: add int8 dp4 dense and MoE prefill optimization for Adreno GPUs (#25537)

Hongqiang Wang1d1d9a9ed7a4
diff --git a/ggml/src/ggml-opencl/CMakeLists.txt b/ggml/src/ggml-opencl/CMakeLists.txt
index 9ec3268b7f..ff3ad7e34d 100644
--- a/ggml/src/ggml-opencl/CMakeLists.txt
+++ b/ggml/src/ggml-opencl/CMakeLists.txt
@@ -114,7 +114,9 @@ set(GGML_OPENCL_KERNELS
mul_mv_id_mxfp4_f32
mul_mv_id_mxfp4_f32_flat
gemm_moe_q4_0_f32_ns
+ gemm_moe_q4_0_q8_1_dp4a
gemv_moe_q4_0_f32_ns
+ gemm_moe_q8_0_f32_ns
gemm_moe_q4_1_f32_ns
gemv_moe_q4_1_f32_ns
gemm_moe_q5_0_f32_ns
@@ -122,6 +124,18 @@ set(GGML_OPENCL_KERNELS
gemm_moe_q5_1_f32_ns
gemv_moe_q5_1_f32_ns
gemm_moe_q4_k_f32_ns
+ gemm_moe_q4_k_q8_1_dp4a
+ gemm_moe_q6_k_q8_1_dp4a
+ gemm_moe_q8_1_dp4a
+ moe_reorder_quant_a_q8_1
+ gemm_noshuffle_q4_k_q8_1_dp4a
+ gemm_noshuffle_q5_k_q8_1_dp4a
+ gemm_noshuffle_q6_k_q8_1_dp4a
+ gemm_noshuffle_q8_0_q8_1_dp4a
+ gemm_noshuffle_q5_0_q8_1_dp4a
+ gemm_noshuffle_iq4_nl_q8_1_dp4a
+ gemm_noshuffle_q4_0_q8_1_dp4a
+ quant_a_q8_1
gemv_moe_q4_k_f32_ns
gemm_moe_q5_k_f32_ns
gemv_moe_q5_k_f32_ns
@@ -130,8 +144,10 @@ set(GGML_OPENCL_KERNELS
gemm_moe_mxfp4_f32
gemv_moe_mxfp4_f32
gemm_moe_mxfp4_f32_ns
+ gemm_moe_mxfp4_q8_1_dp4a
gemv_moe_mxfp4_f32_ns
moe_reorder_b
+ moe_combine
moe_sort_by_expert
mul_mm_f32_f32_l4_lm
mul_mm_f16_f32_l4_lm
diff --git a/ggml/src/ggml-opencl/ggml-opencl.cpp b/ggml/src/ggml-opencl/ggml-opencl.cpp
index 5c96b9a9f6..f283f65690 100644
--- a/ggml/src/ggml-opencl/ggml-opencl.cpp
+++ b/ggml/src/ggml-opencl/ggml-opencl.cpp
@@ -538,6 +538,10 @@ struct ggml_backend_opencl_context {
// ragged moe, use int to directly pass to kernel
cl_uint adreno_use_moe_ragged;
cl_uint adreno_moe_ragged_skip_gran;
+ cl_uint adreno_use_moe_ragged_dp4;
+
+ // whether fuse moe combine
+ cl_uint fuse_moe_combine;
bool adreno_has_large_buffer;
bool adreno_use_large_buffer;
@@ -563,6 +567,12 @@ struct ggml_backend_opencl_context {
ggml_cl_buffer prealloc_quant_trans;
ggml_cl_buffer prealloc_scales_trans;
ggml_cl_buffer prealloc_act_trans;
+ // q8_1-quantized reordered MoE activations for the dp4a prefill GEMM.
+ ggml_cl_buffer prealloc_moe_qa; // int8 quants [tok_slots * ne00]
+ ggml_cl_buffer prealloc_moe_da; // per-block d [tok_slots * ne00/32] (half)
+ ggml_cl_buffer prealloc_moe_sa; // per-block s [tok_slots * ne00/32] (half)
+ // scratch copy of the router weights to avoid dst aliasing
+ ggml_cl_buffer prealloc_moe_combine_w;
// pool of persistent image1d_buffer views over kv-cache layers, keyed by
// (parent buffer, offset within parent)
@@ -816,19 +826,33 @@ struct ggml_backend_opencl_context {
// [size_idx][kda][tgpp] where size_idx: 0=S_V=16, 1=32, 2=64, 3=128; kda: 0 or 1.
// tgpp 0 = TG variant (COLS_PER_LANE_GROUP=1), tgpp 1 = prefill variant (COLS_PER_LANE_GROUP=4).
cl_kernel kernel_gated_delta_net_f32[4][2][2] = {};
-
cl_kernel kernel_timestep_embedding;
cl_kernel kernel_gemv_moe_q4_0_f32_ns, kernel_gemm_moe_q4_0_f32_ns, kernel_gemm_moe_q4_0_f32_ns_bin;
+ cl_kernel kernel_gemm_moe_q8_0_f32_ns;
cl_kernel kernel_gemv_moe_q4_1_f32_ns, kernel_gemm_moe_q4_1_f32_ns, kernel_gemm_moe_q4_1_f32_ns_bin;
cl_kernel kernel_gemv_moe_q5_0_f32_ns, kernel_gemm_moe_q5_0_f32_ns;
cl_kernel kernel_gemv_moe_q5_1_f32_ns, kernel_gemm_moe_q5_1_f32_ns;
cl_kernel kernel_gemv_moe_q4_k_f32_ns, kernel_gemm_moe_q4_k_f32_ns, kernel_gemm_moe_q4_k_f32_ns_bin;
+ cl_kernel kernel_gemv_moe_q4_k_f32_ns_wimg = nullptr; // weight-as-texture MoE decode GEMV (opt-in)
+ cl_kernel kernel_gemm_moe_q4_k_q8_1_dp4a; // dp4a (int8) prefill GEMM variant
+ cl_kernel kernel_moe_reorder_quant_a_q8_1; // fused reorder + q8_1 quant for the dp4a GEMM
+ cl_kernel kernel_gemm_moe_q8_1_dp4a_q80 = nullptr; // generic dp4a MoE GEMM (MOE_QT=80), opt-in
+ cl_kernel kernel_moe_expand_scale_q8_0 = nullptr; // q8_0 per-block d -> uniform scale[16]
+ cl_kernel kernel_gemm_moe_q8_1_dp4a_q50 = nullptr; // generic dp4a MoE GEMM (MOE_QT=50, q5_0), opt-in
+ cl_kernel kernel_moe_expand_scale_q5_0 = nullptr; // q5_0 d -> uniform scale[2]/min[1] per 32-block
+ cl_kernel kernel_gemm_moe_q8_1_dp4a_q5k = nullptr; // generic dp4a MoE GEMM (MOE_QT=5, q5_K), opt-in
+ cl_kernel kernel_moe_expand_scale_q5_K = nullptr; // q5_K 6-bit s[] -> uniform scale[16]/min[8]
cl_kernel kernel_gemv_moe_q5_k_f32_ns, kernel_gemm_moe_q5_k_f32_ns;
cl_kernel kernel_gemv_moe_q6_k_f32_ns, kernel_gemm_moe_q6_k_f32_ns;
+ cl_kernel kernel_gemm_moe_q6_k_q8_1_dp4a; // dp4a (int8) q6_K MoE prefill GEMM
cl_kernel kernel_gemv_moe_mxfp4_f32, kernel_gemm_moe_mxfp4_f32;
cl_kernel kernel_gemv_moe_mxfp4_f32_ns, kernel_gemm_moe_mxfp4_f32_ns, kernel_gemm_moe_mxfp4_f32_ns_bin;
+ cl_kernel kernel_gemv_moe_mxfp4_f32_ns_wimg = nullptr; // weight-as-texture MoE decode GEMV
+ cl_kernel kernel_gemm_moe_mxfp4_q8_1_dp4a; // dp4a (int8) mxfp4 MoE prefill GEMM
+ cl_kernel kernel_gemm_moe_q4_0_q8_1_dp4a; // dp4a (int8) q4_0 MoE prefill GEMM
cl_kernel kernel_moe_reorder_b;
cl_kernel kernel_moe_histogram, kernel_moe_scan, kernel_moe_fill, kernel_moe_scatter;
+ cl_kernel kernel_moe_combine_f32 = nullptr; // fused router-weight mul + cross-expert sum
cl_kernel kernel_mul_mv_id_q4_0_f32_8x_flat;
cl_kernel kernel_mul_mv_id_q8_0_f32, kernel_mul_mv_id_q8_0_f32_flat;
cl_kernel kernel_mul_mv_id_mxfp4_f32;
@@ -1006,21 +1030,32 @@ struct ggml_backend_opencl_context {
cl_kernel kernel_gemv_noshuffle_q4_1_f32;
cl_kernel kernel_gemm_noshuffle_q4_1_f32;
cl_kernel kernel_gemm_noshuffle_q8_0_f32, kernel_gemm_noshuffle_q8_0_f32_bin;
+ cl_kernel kernel_gemm_noshuffle_q8_0_q8_1_dp4a = nullptr; // dp4a (int8) dense q8_0 prefill GEMM (opt-in)
+ cl_kernel kernel_gemm_noshuffle_q8_0_q8_1_dp4a_wimg = nullptr; // q8_0 dense dp4a, weights via texture (opt-in)
cl_kernel kernel_gemv_noshuffle_q8_0_f32;
cl_kernel kernel_gemm_noshuffle_q1_0_f32;
cl_kernel kernel_gemv_noshuffle_q1_0_f32;
cl_kernel kernel_gemv_noshuffle_q4_k_f32;
cl_kernel kernel_gemm_noshuffle_q4_k_f32;
+ cl_kernel kernel_gemm_noshuffle_q4_k_q8_1_dp4a; // dp4a (int8) dense prefill GEMM
+ cl_kernel kernel_gemm_noshuffle_q4_k_q8_1_dp4a_wimg; // dp4a dense prefill GEMM, weights via texture (X1 opt-in)
+ cl_kernel kernel_gemm_noshuffle_q5_k_q8_1_dp4a; // dp4a (int8) dense q5_K prefill GEMM
+ cl_kernel kernel_gemm_noshuffle_q6_k_q8_1_dp4a; // dp4a (int8) dense q6_K prefill GEMM
+ cl_kernel kernel_quant_a_q8_1; // plain activation q8_1 pre-pass
cl_kernel kernel_gemv_noshuffle_q6_K_f32;
cl_kernel kernel_gemm_noshuffle_q6_K_f32;
cl_kernel kernel_gemv_noshuffle_q5_k_f32;
cl_kernel kernel_gemm_noshuffle_q5_k_f32;
cl_kernel kernel_gemv_noshuffle_q5_0_f32;
cl_kernel kernel_gemm_noshuffle_q5_0_f32;
+ cl_kernel kernel_gemm_noshuffle_q5_0_q8_1_dp4a = nullptr; // dp4a (int8) dense q5_0 prefill GEMM
+ cl_kernel kernel_gemm_noshuffle_q5_0_q8_1_dp4a_wimg = nullptr; // q5_0 dense dp4a, qs plane via texture (opt-in)
cl_kernel kernel_gemv_noshuffle_q5_1_f32;
cl_kernel kernel_gemm_noshuffle_q5_1_f32;
cl_kernel kernel_gemv_noshuffle_iq4_nl_f32;
cl_kernel kernel_gemm_noshuffle_iq4_nl_f32;
+ cl_kernel kernel_gemm_noshuffle_iq4_nl_q8_1_dp4a = nullptr; // dp4a (int8) dense IQ4_NL prefill GEMM
+ cl_kernel kernel_gemm_noshuffle_q4_0_q8_1_dp4a = nullptr; // dp4a (int8) dense q4_0 prefill GEMM
#endif // GGML_OPENCL_USE_ADRENO_KERNELS
void free() {
@@ -1396,6 +1431,11 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) {
CL_CHECK((backend_ctx->kernel_restore_block_iq4_nl_noshuffle = clCreateKernel(backend_ctx->program_cvt, "kernel_restore_block_iq4_nl_noshuffle", &err), err));
CL_CHECK((backend_ctx->kernel_convert_bf16_to_f16 = clCreateKernel(backend_ctx->program_cvt, "kernel_convert_bf16_to_f16", &err), err));
CL_CHECK((backend_ctx->kernel_convert_f16_to_bf16 = clCreateKernel(backend_ctx->program_cvt, "kernel_convert_f16_to_bf16", &err), err));
+#ifdef GGML_OPENCL_USE_ADRENO_KERNELS
+ CL_CHECK((backend_ctx->kernel_moe_expand_scale_q8_0 = clCreateKernel(backend_ctx->program_cvt, "kernel_moe_expand_scale_q8_0", &err), err));
+ CL_CHECK((backend_ctx->kernel_moe_expand_scale_q5_0 = clCreateKernel(backend_ctx->program_cvt, "kernel_moe_expand_scale_q5_0", &err), err));
+ CL_CHECK((backend_ctx->kernel_moe_expand_scale_q5_K = clCreateKernel(backend_ctx->program_cvt, "kernel_moe_expand_scale_q5_K", &err), err));
+#endif
GGML_LOG_CONT(".");
}
@@ -3115,6 +3155,23 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) {
GGML_LOG_CONT(".");
}
+ // moe_combine (fused router-weight mul + cross-expert sum)
+ {
+ #ifdef GGML_OPENCL_EMBED_KERNELS
+ const std::string kernel_src {
+ #include "moe_combine.cl.h"
+ };
+ #else
+ const std::string kernel_src = read_file("moe_combine.cl");
+ #endif
+ cl_program prog = build_program_from_source(
+ backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts);
+ CL_CHECK((backend_ctx->kernel_moe_combine_f32 =
+ clCreateKernel(prog, "kernel_moe_combine_f32", &err), err));
+ CL_CHECK(clReleaseProgram(prog));
+ GGML_LOG_CONT(".");
+ }
+
// mul_mv_id_q4_0_f32_8x_flat
{
#ifdef GGML_OPENCL_EMBED_KERNELS
@@ -3432,6 +3489,22 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) {
GGML_LOG_CONT(".");
}
+ // gemm_noshuffle_q5_0_q8_1_dp4a (dp4a dense q5_0 prefill GEMM)
+ {
+#ifdef GGML_OPENCL_EMBED_KERNELS
+ const std::string kernel_src {
+ #include "gemm_noshuffle_q5_0_q8_1_dp4a.cl.h"
+ };
+#else
+ const std::string kernel_src = read_file("gemm_noshuffle_q5_0_q8_1_dp4a.cl");
+#endif
+ cl_program prog = build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts);
+ CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q5_0_q8_1_dp4a = clCreateKernel(prog, "kernel_gemm_noshuffle_q5_0_q8_1_dp4a", &err), err));
+ CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q5_0_q8_1_dp4a_wimg = clCreateKernel(prog, "kernel_gemm_noshuffle_q5_0_q8_1_dp4a_wimg", &err), err));
+ CL_CHECK(clReleaseProgram(prog));
+ GGML_LOG_CONT(".");
+ }
+
// gemv_noshuffle_q5_0_f32
{
std::string CL_gemv_compile_opts = std::string("-cl-std=") + opencl_c_std +
@@ -3506,6 +3579,36 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) {
GGML_LOG_CONT(".");
}
+ // gemm_noshuffle_iq4_nl_q8_1_dp4a (dp4a dense IQ4_NL prefill GEMM)
+ {
+#ifdef GGML_OPENCL_EMBED_KERNELS
+ const std::string kernel_src {
+ #include "gemm_noshuffle_iq4_nl_q8_1_dp4a.cl.h"
+ };
+#else
+ const std::string kernel_src = read_file("gemm_noshuffle_iq4_nl_q8_1_dp4a.cl");
+#endif
+ cl_program prog = build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts);
+ CL_CHECK((backend_ctx->kernel_gemm_noshuffle_iq4_nl_q8_1_dp4a = clCreateKernel(prog, "kernel_gemm_noshuffle_iq4_nl_q8_1_dp4a", &err), err));
+ CL_CHECK(clReleaseProgram(prog));
+ GGML_LOG_CONT(".");
+ }
+
+ // gemm_noshuffle_q4_0_q8_1_dp4a (dp4a dense q4_0 prefill GEMM)
+ {
+#ifdef GGML_OPENCL_EMBED_KERNELS
+ const std::string kernel_src {
+ #include "gemm_noshuffle_q4_0_q8_1_dp4a.cl.h"
+ };
+#else
+ const std::string kernel_src = read_file("gemm_noshuffle_q4_0_q8_1_dp4a.cl");
+#endif
+ cl_program prog = build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts);
+ CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q4_0_q8_1_dp4a = clCreateKernel(prog, "kernel_gemm_noshuffle_q4_0_q8_1_dp4a", &err), err));
+ CL_CHECK(clReleaseProgram(prog));
+ GGML_LOG_CONT(".");
+ }
+
// gemv_noshuffle_iq4_nl_f32
{
std::string CL_gemv_compile_opts = std::string("-cl-std=") + opencl_c_std +
@@ -3604,6 +3707,89 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) {
GGML_LOG_CONT(".");
}
+ // gemm_noshuffle_q4_k_q8_1_dp4a (dp4a dense prefill GEMM)
+ {
+#ifdef GGML_OPENCL_EMBED_KERNELS
+ const std::string kernel_src {
+ #include "gemm_noshuffle_q4_k_q8_1_dp4a.cl.h"
+ };
+#else
+ const std::string kernel_src = read_file("gemm_noshuffle_q4_k_q8_1_dp4a.cl");
+#endif
+ // Per-device dp4a dense tile. The X2-tuned TILESIZE_N=32 over-occupies LDS on
+ // X1 (1152 B/WG -> few resident WGs); TILESIZE_N=8 (288 B) lifts occupancy on
+ // X1, byte-identical. X2E keeps 32. Env override wins.
+ int q4k_dp4a_ts = (backend_ctx->adreno_gen == ADRENO_GPU_GEN::X1E) ? 8 : 32;
+ if (const char * e = getenv("GGML_OPENCL_Q4K_DP4A_TS")) q4k_dp4a_ts = atoi(e);
+ std::string dp4a_opts = compile_opts + " -DTILESIZE_N=" + std::to_string(q4k_dp4a_ts);
+ cl_program prog = build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), dp4a_opts);
+ CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q4_k_q8_1_dp4a = clCreateKernel(prog, "kernel_gemm_noshuffle_q4_k_q8_1_dp4a", &err), err));
+ CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q4_k_q8_1_dp4a_wimg = clCreateKernel(prog, "kernel_gemm_noshuffle_q4_k_q8_1_dp4a_wimg", &err), err));
+ CL_CHECK(clReleaseProgram(prog));
+ GGML_LOG_CONT(".");
+ }
+
+ // gemm_noshuffle_q8_0_q8_1_dp4a (dp4a dense q8_0 prefill GEMM)
+ {
+#ifdef GGML_OPENCL_EMBED_KERNELS
+ const std::string kernel_src {
+ #include "gemm_noshuffle_q8_0_q8_1_dp4a.cl.h"
+ };
+#else
+ const std::string kernel_src = read_file("gemm_noshuffle_q8_0_q8_1_dp4a.cl");
+#endif
+ cl_program prog = build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts);
+ CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q8_0_q8_1_dp4a = clCreateKernel(prog, "kernel_gemm_noshuffle_q8_0_q8_1_dp4a", &err), err));
+ CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q8_0_q8_1_dp4a_wimg = clCreateKernel(prog, "kernel_gemm_noshuffle_q8_0_q8_1_dp4a_wimg", &err), err));
+ CL_CHECK(clReleaseProgram(prog));
+ GGML_LOG_CONT(".");
+ }
+
+ // gemm_noshuffle_q5_k_q8_1_dp4a (dp4a dense prefill GEMM for q5_K)
+ {
+#ifdef GGML_OPENCL_EMBED_KERNELS
+ const std::string kernel_src {
+ #include "gemm_noshuffle_q5_k_q8_1_dp4a.cl.h"
+ };
+#else
+ const std::string kernel_src = read_file("gemm_noshuffle_q5_k_q8_1_dp4a.cl");
+#endif
+ cl_program prog = build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts);
+ CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q5_k_q8_1_dp4a = clCreateKernel(prog, "kernel_gemm_noshuffle_q5_k_q8_1_dp4a", &err), err));
+ CL_CHECK(clReleaseProgram(prog));
+ GGML_LOG_CONT(".");
+ }
+
+ // gemm_noshuffle_q6_k_q8_1_dp4a (dp4a dense prefill GEMM for q6_K ffn_down/output)
+ {
+#ifdef GGML_OPENCL_EMBED_KERNELS
+ const std::string kernel_src {
+ #include "gemm_noshuffle_q6_k_q8_1_dp4a.cl.h"
+ };
+#else
+ const std::string kernel_src = read_file("gemm_noshuffle_q6_k_q8_1_dp4a.cl");
+#endif
+ cl_program prog = build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts);
+ CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q6_k_q8_1_dp4a = clCreateKernel(prog, "kernel_gemm_noshuffle_q6_k_q8_1_dp4a", &err), err));
+ CL_CHECK(clReleaseProgram(prog));
+ GGML_LOG_CONT(".");
+ }
+
+ // quant_a_q8_1 (plain activation q8_1 pre-pass for the dense dp4a GEMM)
+ {
+#ifdef GGML_OPENCL_EMBED_KERNELS
+ const std::string kernel_src {
+ #include "quant_a_q8_1.cl.h"
+ };
+#else
+ const std::string kernel_src = read_file("quant_a_q8_1.cl");
+#endif
+ cl_program prog = build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts);
+ CL_CHECK((backend_ctx->kernel_quant_a_q8_1 = clCreateKernel(prog, "kernel_quant_a_q8_1", &err), err));
+ CL_CHECK(clReleaseProgram(prog));
+ GGML_LOG_CONT(".");
+ }
+
// gemv_noshuffle_q4_k_f32
{
std::string CL_gemv_compile_opts = std::string("-cl-std=") + opencl_c_std +
@@ -3766,6 +3952,23 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) {
}
}
+ // gemm_moe_q8_0_f32_ns
+ {
+#ifdef GGML_OPENCL_EMBED_KERNELS
+ const std::string kernel_src {
+ #include "gemm_moe_q8_0_f32_ns.cl.h"
+ };
+#else
+ const std::string kernel_src = read_file("gemm_moe_q8_0_f32_ns.cl");
+#endif
+ cl_program prog =
+ build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), CL_moe_compile_opts);
+
+ CL_CHECK((backend_ctx->kernel_gemm_moe_q8_0_f32_ns = clCreateKernel(prog, "kernel_gemm_moe_q8_0_f32_ns", &err), err));
+ CL_CHECK(clReleaseProgram(prog));
+ GGML_LOG_CONT(".");
+ }
+
// gemv_moe_q5_0_f32_ns
{
#ifdef GGML_OPENCL_EMBED_KERNELS
@@ -3847,6 +4050,7 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) {
build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), CL_moe_compile_opts);
CL_CHECK((backend_ctx->kernel_gemv_moe_q4_k_f32_ns = clCreateKernel(prog, "kernel_gemv_moe_q4_k_f32_ns", &err), err));
+ CL_CHECK((backend_ctx->kernel_gemv_moe_q4_k_f32_ns_wimg = clCreateKernel(prog, "kernel_gemv_moe_q4_k_f32_ns_wimg", &err), err));
CL_CHECK(clReleaseProgram(prog));
GGML_LOG_CONT(".");
}
@@ -3886,6 +4090,103 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) {
}
}
+ // gemm_moe_q4_k_q8_1_dp4a (dp4a prefill GEMM)
+ {
+#ifdef GGML_OPENCL_EMBED_KERNELS
+ const std::string kernel_src {
+ #include "gemm_moe_q4_k_q8_1_dp4a.cl.h"
+ };
+#else
+ const std::string kernel_src = read_file("gemm_moe_q4_k_q8_1_dp4a.cl");
+#endif
+ cl_program prog =
+ build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), CL_moe_compile_opts);
+
+ CL_CHECK((backend_ctx->kernel_gemm_moe_q4_k_q8_1_dp4a = clCreateKernel(prog, "kernel_gemm_moe_q4_k_q8_1_dp4a", &err), err));
+ CL_CHECK(clReleaseProgram(prog));
+ GGML_LOG_CONT(".");
+ }
+
+ // gemm_moe_mxfp4_q8_1_dp4a (dp4a prefill GEMM)
+ {
+#ifdef GGML_OPENCL_EMBED_KERNELS
+ const std::string kernel_src {
+ #include "gemm_moe_mxfp4_q8_1_dp4a.cl.h"
+ };
+#else
+ const std::string kernel_src = read_file("gemm_moe_mxfp4_q8_1_dp4a.cl");
+#endif
+ cl_program prog =
+ build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), CL_moe_compile_opts);
+
+ CL_CHECK((backend_ctx->kernel_gemm_moe_mxfp4_q8_1_dp4a = clCreateKernel(prog, "kernel_gemm_moe_mxfp4_q8_1_dp4a", &err), err));
+ CL_CHECK(clReleaseProgram(prog));
+ GGML_LOG_CONT(".");
+ }
+
+ // gemm_moe_q4_0_q8_1_dp4a (dp4a prefill GEMM)
+ {
+#ifdef GGML_OPENCL_EMBED_KERNELS
+ const std::string kernel_src {
+ #include "gemm_moe_q4_0_q8_1_dp4a.cl.h"
+ };
+#else
+ const std::string kernel_src = read_file("gemm_moe_q4_0_q8_1_dp4a.cl");
+#endif
+ cl_program prog =
+ build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), CL_moe_compile_opts);
+
+ CL_CHECK((backend_ctx->kernel_gemm_moe_q4_0_q8_1_dp4a = clCreateKernel(prog, "kernel_gemm_moe_q4_0_q8_1_dp4a", &err), err));
+ CL_CHECK(clReleaseProgram(prog));
+ GGML_LOG_CONT(".");
+ }
+
+ // gemm_moe_q8_1_dp4a (generic dp4a MoE GEMM; MOE_QT=80 -> q8_0 expert variant)
+ {
+#ifdef GGML_OPENCL_EMBED_KERNELS
+ const std::string kernel_src {
+ #include "gemm_moe_q8_1_dp4a.cl.h"
+ };
+#else
+ const std::string kernel_src = read_file("gemm_moe_q8_1_dp4a.cl");
+#endif
+ const std::string opts80 = CL_moe_compile_opts + " -DMOE_QT=80";
+ cl_program prog =
+ build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), opts80.c_str());
+ CL_CHECK((backend_ctx->kernel_gemm_moe_q8_1_dp4a_q80 = clCreateKernel(prog, "kernel_gemm_moe_q8_1_dp4a", &err), err));
+ CL_CHECK(clReleaseProgram(prog));
+
+ const std::string opts50 = CL_moe_compile_opts + " -DMOE_QT=50";
+ cl_program prog50 =
+ build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), opts50.c_str());
+ CL_CHECK((backend_ctx->kernel_gemm_moe_q8_1_dp4a_q50 = clCreateKernel(prog50, "kernel_gemm_moe_q8_1_dp4a", &err), err));
+ CL_CHECK(clReleaseProgram(prog50));
+
+ const std::string opts5 = CL_moe_compile_opts + " -DMOE_QT=5";
+ cl_program prog5 =
+ build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), opts5.c_str());
+ CL_CHECK((backend_ctx->kernel_gemm_moe_q8_1_dp4a_q5k = clCreateKernel(prog5, "kernel_gemm_moe_q8_1_dp4a", &err), err));
+ CL_CHECK(clReleaseProgram(prog5));
+ GGML_LOG_CONT(".");
+ }
+
+ // moe_reorder_quant_a_q8_1 (fused reorder + q8_1 quant)
+ {
+#ifdef GGML_OPENCL_EMBED_KERNELS
+ const std::string kernel_src {
+ #include "moe_reorder_quant_a_q8_1.cl.h"
+ };
+#else
+ const std::string kernel_src = read_file("moe_reorder_quant_a_q8_1.cl");
+#endif
+ cl_program prog =
+ build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), CL_moe_compile_opts);
+
+ CL_CHECK((backend_ctx->kernel_moe_reorder_quant_a_q8_1 = clCreateKernel(prog, "kernel_moe_reorder_quant_a_q8_1", &err), err));
+ CL_CHECK(clReleaseProgram(prog));
+ GGML_LOG_CONT(".");
+ }
+
// gemv_moe_q5_k_f32_ns
{
#ifdef GGML_OPENCL_EMBED_KERNELS
@@ -3954,6 +4255,23 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) {
GGML_LOG_CONT(".");
}
+ // gemm_moe_q6_k_q8_1_dp4a (dp4a q6_K MoE prefill GEMM)
+ {
+#ifdef GGML_OPENCL_EMBED_KERNELS
+ const std::string kernel_src {
+ #include "gemm_moe_q6_k_q8_1_dp4a.cl.h"
+ };
+#else
+ const std::string kernel_src = read_file("gemm_moe_q6_k_q8_1_dp4a.cl");
+#endif
+ cl_program prog =
+ build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), CL_moe_compile_opts);
+
+ CL_CHECK((backend_ctx->kernel_gemm_moe_q6_k_q8_1_dp4a = clCreateKernel(prog, "kernel_gemm_moe_q6_k_q8_1_dp4a", &err), err));
+ CL_CHECK(clReleaseProgram(prog));
+ GGML_LOG_CONT(".");
+ }
+
// gemv_moe_mxfp4_f32_ns
{
#ifdef GGML_OPENCL_EMBED_KERNELS
@@ -3967,6 +4285,7 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) {
build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), CL_moe_compile_opts);
CL_CHECK((backend_ctx->kernel_gemv_moe_mxfp4_f32_ns = clCreateKernel(prog, "kernel_gemv_moe_mxfp4_f32_ns", &err), err));
+ CL_CHECK((backend_ctx->kernel_gemv_moe_mxfp4_f32_ns_wimg = clCreateKernel(prog, "kernel_gemv_moe_mxfp4_f32_ns_wimg", &err), err));
CL_CHECK(clReleaseProgram(prog));
GGML_LOG_CONT(".");
}
@@ -5484,6 +5803,7 @@ static ggml_backend_opencl_context * ggml_cl_init(ggml_backend_dev_t dev) {
// check Adreno large buffer support
backend_ctx->adreno_has_large_buffer = strstr(ext_buffer, "cl_qcom_large_buffer") != NULL;
+
// subgroup shuffle support (N_SPLIT>1 FA kernel)
backend_ctx->has_qcom_subgroup_shuffle = strstr(ext_buffer, "cl_qcom_subgroup_shuffle") != NULL;
backend_ctx->has_subgroup_shuffle =
@@ -5537,6 +5857,14 @@ static ggml_backend_opencl_context * ggml_cl_init(ggml_backend_dev_t dev) {
static const char * ragged_gran_env = getenv("GGML_OPENCL_MOE_RAGGED_GRAN");
backend_ctx->adreno_moe_ragged_skip_gran = (ragged_gran_env != NULL) ? atoi(ragged_gran_env) : 8;
+ // whether fuse moe combine
+ static const char * fuse_moe_combine_env = getenv("GGML_OPENCL_FUSE_MOE_COMBINE");
+ backend_ctx->fuse_moe_combine = fuse_moe_combine_env == NULL ? 1 : (atoi(fuse_moe_combine_env) != 0);
+
+ // ragged moe dp4 variant
+ static const char * ragged_dp4_env = getenv("GGML_OPENCL_MOE_RAGGED");
+ backend_ctx->adreno_use_moe_ragged_dp4 = ragged_dp4_env == NULL ? 1 : (atoi(ragged_dp4_env) != 0);
+
#ifdef GGML_OPENCL_USE_ADRENO_BIN_KERNELS
// try loading adreno binary kernels if enabled
// if fails to load, builtin kernels will be used
@@ -5863,6 +6191,10 @@ struct ggml_tensor_extra_cl_q5_0 {
cl_mem d = nullptr;
// Scales in image1d_buffer_t.
cl_mem d_img = nullptr;
+ // Uniform per-32-block scale (2/block) + min (1/block, = d*16 for the -16 centering)
+ // for the generic dp4a MoE GEMM. Built from d.
+ cl_mem scale = nullptr;
+ cl_mem min = nullptr;
// Size of quantized values.
size_t size_qs = 0;
// Size of 5-th bit values.
@@ -5891,6 +6223,14 @@ struct ggml_tensor_extra_cl_q5_0 {
CL_CHECK(clReleaseMemObject(qs_img));
qs_img = nullptr;
}
+ if (scale != nullptr) {
+ CL_CHECK(clReleaseMemObject(scale));
+ scale = nullptr;
+ }
+ if (min != nullptr) {
+ CL_CHECK(clReleaseMemObject(min));
+ min = nullptr;
+ }
qh_img = nullptr;
d_img = nullptr;
@@ -6015,6 +6355,11 @@ struct ggml_tensor_extra_cl_q8_0 {
cl_mem d = nullptr;
cl_mem d_img = nullptr;
+ // Uniform per-16-segment scale (16/superblock) for the generic dp4a MoE GEMM.
+ // Expanded from d at set_tensor; the int8 codes are reused from q.
+ // q8_0 is symmetric so no min buffer (has_min=0).
+ cl_mem scale = nullptr;
+
size_t size_q = 0;
size_t size_d = 0;
@@ -6034,6 +6379,10 @@ struct ggml_tensor_extra_cl_q8_0 {
CL_CHECK(clReleaseMemObject(d));
d = nullptr;
}
+ if (scale != nullptr) {
+ CL_CHECK(clReleaseMemObject(scale));
+ scale = nullptr;
+ }
// Currently, q_img and d_img are not used. They can be image1d_buffer_t
// that wraps around q and d to utilize image access path.
q_img = nullptr;
@@ -6120,6 +6469,11 @@ struct ggml_tensor_extra_cl_q5_K {
cl_mem d = nullptr;
// Min for each super block.
cl_mem dm = nullptr;
+ // Uniform per-32-block scale (2/block) + min (1/block, = dm*mn) decoded from the
+ // 6-bit packed s[] for the generic dp4a MoE GEMM kernel_gemm_moe_q8_1_dp4a.
+ // Built from s/d/dm at set_tensor; q/qh are reused as-is.
+ cl_mem scale = nullptr;
+ cl_mem min = nullptr;
size_t size_q = 0;
size_t size_qh = 0;
@@ -6156,6 +6510,14 @@ struct ggml_tensor_extra_cl_q5_K {
CL_CHECK(clReleaseMemObject(q_img));
q_img = nullptr;
}
+ if (scale != nullptr) {
+ CL_CHECK(clReleaseMemObject(scale));
+ scale = nullptr;
+ }
+ if (min != nullptr) {
+ CL_CHECK(clReleaseMemObject(min));
+ min = nullptr;
+ }
size_q = 0;
size_qh = 0;
@@ -6298,6 +6660,122 @@ static void sync_with_other_backends(ggml_backend_t backend) {
sync_with_other_backends(backend_ctx);
}
+// True if two tensors share a device buffer with overlapping byte ranges. The pool
+// allocator may place a fused op's output over a sequentially-dead input (safe for the
+// original separate kernels, but a read/write race inside one fused kernel).
+static bool ggml_cl_tensors_overlap(const ggml_tensor * x, const ggml_tensor * y) {
+ ggml_tensor_extra_cl * ex = (ggml_tensor_extra_cl *)x->extra;
+ ggml_tensor_extra_cl * ey = (ggml_tensor_extra_cl *)y->extra;
+ if (!ex || !ey || ex->data_device != ey->data_device) { return false; }
+ const cl_ulong xo = ex->offset + x->view_offs, xe = xo + ggml_nbytes(x);
+ const cl_ulong yo = ey->offset + y->view_offs, ye = yo + ggml_nbytes(y);
+ return xo < ye && yo < xe;
+}
+
+// Detect the MoE combine epilogue: router-weight MUL ([n_embd,k,nt] * [1,k,nt]) followed
+// by k VIEWs of it and a (k-1)-long ADD reduction chain producing [n_embd, nt]. When it
+// matches (and the output does not alias the inputs), the whole subgraph collapses to one
+// weighted-sum-across-experts kernel.
+static bool ggml_opencl_can_fuse_moe_combine(const struct ggml_cgraph * cgraph, int node_idx,
+ const ggml_tensor ** out_final_add) {
+ const ggml_tensor * mul = cgraph->nodes[node_idx];
+ if (mul->op != GGML_OP_MUL) { return false; }
+ const ggml_tensor * experts = mul->src[0];
+ const ggml_tensor * weights = mul->src[1];
+ if (!experts || !weights) { return false; }
+ if (experts->type != GGML_TYPE_F32 || weights->type != GGML_TYPE_F32 || mul->type != GGML_TYPE_F32) { return false; }
+
+ const int64_t n_embd = experts->ne[0];
+ const int64_t k = experts->ne[1];
+ const int64_t nt = experts->ne[2];
+ if (k < 2 || k > 64 || experts->ne[3] != 1 || n_embd % 4 != 0) { return false; }
+ if (weights->ne[0] != 1 || weights->ne[1] != k || weights->ne[2] != nt || weights->ne[3] != 1) { return false; }
+ if (mul->ne[0] != n_embd || mul->ne[1] != k || mul->ne[2] != nt) { return false; }
+ // the fused kernel needs contiguous experts/weights and a contiguous 2D dst
+ if (!ggml_is_contiguous(experts) || !ggml_is_contiguous(weights)) { return false; }
+
+ const int n_nodes = 1 + (int)k + (int)(k - 1); // MUL + k*VIEW + (k-1)*ADD
+ if (n_nodes >= 32) { return false; }
+ if (node_idx + n_nodes > cgraph->n_nodes) { return false; }
+
+ enum ggml_op ops[1 + 64 + 63];
+ int n = 0;
+ ops[n++] = GGML_OP_MUL;
+ for (int j = 0; j < (int)k; ++j) { ops[n++] = GGML_OP_VIEW; }
+ for (int j = 0; j < (int)k - 1; ++j) { ops[n++] = GGML_OP_ADD; }
+ const int outs[] = { node_idx + n_nodes - 1 };
+ if (!ggml_can_fuse_subgraph(cgraph, node_idx, n_nodes, ops, outs, 1)) { return false; }
+
+ for (int j = 0; j < (int)k; ++j) {
+ const ggml_tensor * vw = cgraph->nodes[node_idx + 1 + j];
+ if (vw->op != GGML_OP_VIEW || vw->src[0] != mul || vw->ne[0] != n_embd || vw->ne[1] != nt) { return false; }
+ }
+ const ggml_tensor * final_add = cgraph->nodes[node_idx + n_nodes - 1];
+ if (final_add->op != GGML_OP_ADD || final_add->type != GGML_TYPE_F32 ||
+ final_add->ne[0] != n_embd || final_add->ne[1] != nt || final_add->ne[2] != 1) { return false; }
+ if (!ggml_is_contiguous(final_add)) { return false; }
+ // the fused kernel reads experts + writes final_add in one pass; bail if the
+ // pool allocator overlapped the output with the (large) experts input -- would race.
+ // The small weights input is copied to a private scratch in the dispatch, so its own
+ // aliasing with the output is handled there and does not block the fusion.
+ if (ggml_cl_tensors_overlap(experts, final_add)) { return false; }
+
+ *out_final_add = final_add;
+ return true;
+}
+
+static void ggml_cl_moe_combine_fused(ggml_backend_t backend, const ggml_tensor * mul, const ggml_tensor * dst) {
+ ggml_backend_opencl_context * backend_ctx = (ggml_backend_opencl_context *)backend->context;
+ const ggml_tensor * experts = mul->src[0];
+ const ggml_tensor * weights = mul->src[1];
+
+ ggml_tensor_extra_cl * ee = (ggml_tensor_extra_cl *)experts->extra;
+ ggml_tensor_extra_cl * ew = (ggml_tensor_extra_cl *)weights->extra;
+ ggml_tensor_extra_cl * ed = (ggml_tensor_extra_cl *)dst->extra;
+ cl_ulong off_e = ee->offset + experts->view_offs;
+ cl_ulong off_w = ew->offset + weights->view_offs;
+ cl_ulong off_d = ed->offset + dst->view_offs;
+
+ const int n_embd4 = (int)(experts->ne[0] / 4);
+ const int k = (int)experts->ne[1];
+ const int nt = (int)experts->ne[2];
+ const cl_uint e1 = (cl_uint)(experts->nb[1] / sizeof(float));
+ const cl_uint e2 = (cl_uint)(experts->nb[2] / sizeof(float));
+ const cl_uint w1 = (cl_uint)(weights->nb[1] / sizeof(float));
+ const cl_uint w2 = (cl_uint)(weights->nb[2] / sizeof(float));
+ const cl_uint d1 = (cl_uint)(dst->nb[1] / sizeof(float));
+
+ // The router weights are tiny ([1,k,nt]) and may share a pool buffer with the output;
+ // copy them into a private scratch so the fused kernel never reads aliased memory.
+ const size_t w_bytes = ggml_nbytes(weights);
+ backend_ctx->prealloc_moe_combine_w.allocate(backend_ctx->context, w_bytes);
+ CL_CHECK(clEnqueueCopyBuffer(backend_ctx->queue, ew->data_device, backend_ctx->prealloc_moe_combine_w.buffer,
+ off_w, 0, w_bytes, 0, NULL, NULL));
+ cl_mem w_dev = backend_ctx->prealloc_moe_combine_w.buffer;
+ cl_ulong w_off = 0;
+
+ cl_kernel kernel = backend_ctx->kernel_moe_combine_f32;
+ int a = 0;
+ CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &ee->data_device));
+ CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_ulong), &off_e));
+ CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &w_dev));
+ CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_ulong), &w_off));
+ CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &ed->data_device));
+ CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_ulong), &off_d));
+ CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &n_embd4));
+ CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &k));
+ CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &nt));
+ CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_uint), &e1));
+ CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_uint), &e2));
+ CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_uint), &w1));
+ CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_uint), &w2));
+ CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_uint), &d1));
+
+ size_t lws[2] = { 64, 1 };
+ size_t gws[2] = { (size_t)(((n_embd4 + 63) / 64) * 64), (size_t)nt };
+ backend_ctx->enqueue_ndrange_kernel(kernel, 2, gws, lws, dst);
+}
+
static bool ggml_opencl_can_fuse(const struct ggml_cgraph * cgraph, int node_idx, std::initializer_list<enum ggml_op> ops) {
if (!ggml_can_fuse(cgraph, node_idx, ops)) {
return false;
@@ -6398,6 +6876,17 @@ static ggml_status ggml_backend_opencl_graph_compute(ggml_backend_t backend, ggm
i += 2;
continue;
}
+ // Fuse the MoE combine: router-weight mul + cross-expert add chain ->
+ // one weighted-sum-across-experts kernel.
+ if (backend_ctx->fuse_moe_combine && !backend_ctx->disable_fusion) {
+ const ggml_tensor * combine_out = nullptr;
+ if (ggml_opencl_can_fuse_moe_combine(cgraph, i, &combine_out)) {
+ ggml_cl_moe_combine_fused(backend, node, combine_out);
+ i += 2 * (int)node->ne[1] - 1; // skip the k VIEWs + (k-1) ADDs
+ continue;
+ }
+ }
+
if (!backend_ctx->disable_fusion && ggml_opencl_can_fuse(cgraph, i, { GGML_OP_RMS_NORM, GGML_OP_MUL })) {
ggml_opencl_op_rms_norm_fused(backend, node, cgraph->nodes[i+1]);
i++;
@@ -6446,7 +6935,12 @@ inline bool enable_adreno_trans_weight(const ggml_backend_opencl_context *backen
size_t elem_num = tensor->ne[0] * tensor->ne[1] * tensor->ne[2] * tensor->ne[3];
- return ((elem_num < 128 * 1024 * 1024) && adreno_kernel); // max element num: 2**27
+ // The 2D weight transpose (transpose_2d_as_*) tiles rows by 4 over a 2D matrix,
+ // so it requires K(ne0)%32==0, M(ne1)%4==0 and ne2==ne3==1.
+ const bool shape_ok = (tensor->ne[0] % 32 == 0) && (tensor->ne[1] % 4 == 0) &&
+ (tensor->ne[2] == 1) && (tensor->ne[3] == 1);
+
+ return ((elem_num < 128 * 1024 * 1024) && adreno_kernel && shape_ok); // max element num: 2**27
}
static inline bool use_flat_gemv_for_large_m_q4_K(const ggml_tensor *tensor) {
@@ -7785,6 +8279,31 @@ static void ggml_backend_opencl_buffer_set_tensor(ggml_backend_buffer_t buffer,
extra->qs_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_format_qs, &img_desc_qs, NULL, &err);
tensor->extra = extra;
+ // Generic dp4a MoE path
+ {
+ static const char * q5dp4a_env = getenv("GGML_OPENCL_Q5_MOE_DP4A");
+ const bool q5dp4a = q5dp4a_env ? (atoi(q5dp4a_env) != 0)
+ : (backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E);
+ if (q5dp4a && ne02 > 1 && (ne00 % 32 == 0)) {
+ size_t nb32 = (size_t)ne00 / 32;
+ size_t sc_elems = (size_t)ne02 * ne01 * nb32 * 2;
+ size_t mn_elems = (size_t)ne02 * ne01 * nb32;
+ extra->scale = clCreateBuffer(context, CL_MEM_READ_WRITE, sc_elems * sizeof(cl_half), NULL, &err); CL_CHECK(err);
+ extra->min = clCreateBuffer(context, CL_MEM_READ_WRITE, mn_elems * sizeof(cl_half), NULL, &err); CL_CHECK(err);
+ cl_kernel ek = backend_ctx->kernel_moe_expand_scale_q5_0;
+ CL_CHECK(clSetKernelArg(ek, 0, sizeof(cl_mem), &extra->d));
+ CL_CHECK(clSetKernelArg(ek, 1, sizeof(cl_mem), &extra->scale));
+ CL_CHECK(clSetKernelArg(ek, 2, sizeof(cl_mem), &extra->min));
+ CL_CHECK(clSetKernelArg(ek, 3, sizeof(int), &ne00));
+ CL_CHECK(clSetKernelArg(ek, 4, sizeof(int), &ne01));
+ size_t eg[3] = { (size_t)(((ne01 + 63) / 64) * 64), nb32, (size_t)ne02 };
+ size_t el[3] = { 64, 1, 1 };
+ cl_event evt;
+ CL_CHECK(clEnqueueNDRangeKernel(queue, ek, 3, NULL, eg, el, 0, NULL, &evt));
+ CL_CHECK(clWaitForEvents(1, &evt));
+ }
+ }
+
return;
}
#endif // GGML_OPENCL_USE_ADRENO_KERNELS
@@ -8164,6 +8683,34 @@ static void ggml_backend_opencl_buffer_set_tensor(ggml_backend_buffer_t buffer,
tensor->extra = extra;
ctx->q8_0_soa_tensors.insert(tensor);
+ // Generic dp4a MoE path (opt-in GGML_OPENCL_Q8_MOE_DP4A)
+#ifdef GGML_OPENCL_USE_ADRENO_KERNELS
+ {
+ static const char * q8dp4a_env = getenv("GGML_OPENCL_Q8_MOE_DP4A");
+ const bool q8dp4a = q8dp4a_env ? (atoi(q8dp4a_env) != 0)
+ : (backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E);
+ if (q8dp4a && tensor->ne[2] > 1 && (tensor->ne[0] % 32 == 0)) {
+ int ne00 = (int)tensor->ne[0];
+ int ne01 = (int)tensor->ne[1];
+ int ne02 = (int)tensor->ne[2];
+ size_t nb32 = (size_t)ne00 / 32;
+ size_t scale_elems = (size_t)ne02 * ne01 * nb32 * 2; // 2 per-16-seg scales / 32-block
+ extra->scale = clCreateBuffer(context, CL_MEM_READ_WRITE, scale_elems * sizeof(cl_half), NULL, &err);
+ CL_CHECK(err);
+ cl_kernel ek = backend_ctx->kernel_moe_expand_scale_q8_0;
+ CL_CHECK(clSetKernelArg(ek, 0, sizeof(cl_mem), &extra->d));
+ CL_CHECK(clSetKernelArg(ek, 1, sizeof(cl_mem), &extra->scale));
+ CL_CHECK(clSetKernelArg(ek, 2, sizeof(int), &ne00));
+ CL_CHECK(clSetKernelArg(ek, 3, sizeof(int), &ne01));
+ size_t eg[3] = { (size_t)(((ne01 + 63) / 64) * 64), nb32, (size_t)ne02 };
+ size_t el[3] = { 64, 1, 1 };
+ cl_event evt;
+ CL_CHECK(clEnqueueNDRangeKernel(queue, ek, 3, NULL, eg, el, 0, NULL, &evt));
+ CL_CHECK(clWaitForEvents(1, &evt));
+ }
+ }
+#endif
+
// Transpose the weights and scales
#ifdef GGML_OPENCL_USE_ADRENO_KERNELS
if (enable_adreno_trans_weight(backend_ctx, tensor)) {
@@ -8516,6 +9063,33 @@ static void ggml_backend_opencl_buffer_set_tensor(ggml_backend_buffer_t buffer,
CL_CHECK(err);
tensor->extra = extra;
+ // Generic dp4a MoE path
+ {
+ static const char * q5kdp4a_env = getenv("GGML_OPENCL_Q5K_MOE_DP4A");
+ const bool q5kdp4a = q5kdp4a_env ? (atoi(q5kdp4a_env) != 0)
+ : (backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E);
+ if (q5kdp4a && ne02 > 1 && (ne00 % 256 == 0)) {
+ size_t nb32 = (size_t)ne00 / 32;
+ size_t sc_elems = (size_t)ne02 * ne01 * nb32 * 2;
+ size_t mn_elems = (size_t)ne02 * ne01 * nb32;
+ extra->scale = clCreateBuffer(context, CL_MEM_READ_WRITE, sc_elems * sizeof(cl_half), NULL, &err); CL_CHECK(err);
+ extra->min = clCreateBuffer(context, CL_MEM_READ_WRITE, mn_elems * sizeof(cl_half), NULL, &err); CL_CHECK(err);
+ cl_kernel ek = backend_ctx->kernel_moe_expand_scale_q5_K;
+ CL_CHECK(clSetKernelArg(ek, 0, sizeof(cl_mem), &extra->s));
+ CL_CHECK(clSetKernelArg(ek, 1, sizeof(cl_mem), &extra->d));
+ CL_CHECK(clSetKernelArg(ek, 2, sizeof(cl_mem), &extra->dm));
+ CL_CHECK(clSetKernelArg(ek, 3, sizeof(cl_mem), &extra->scale));
+ CL_CHECK(clSetKernelArg(ek, 4, sizeof(cl_mem), &extra->min));
+ CL_CHECK(clSetKernelArg(ek, 5, sizeof(int), &ne00));
+ CL_CHECK(clSetKernelArg(ek, 6, sizeof(int), &ne01));
+ size_t eg[3] = { (size_t)(((ne01 + 63) / 64) * 64), (size_t)(ne00 / 256), (size_t)ne02 };
+ size_t el[3] = { 64, 1, 1 };
+ cl_event evt;
+ CL_CHECK(clEnqueueNDRangeKernel(queue, ek, 3, NULL, eg, el, 0, NULL, &evt));
+ CL_CHECK(clWaitForEvents(1, &evt));
+ }
+ }
+
return;
}
#endif // GGML_OPENCL_USE_ADRENO_KERNELS
@@ -15245,6 +15819,61 @@ static void ggml_cl_mul_mat_q4_0_f32_adreno(ggml_backend_t backend, const ggml_t
CL_CHECK(clReleaseMemObject(b_sub_buf));
CL_CHECK(clReleaseMemObject(b_img));
} else {
+ // dp4a (int8) dense prefill GEMM: quant activations to q8_1, then the int8
+ // dp4a inner-loop GEMM, in place of the transpose + f16 half-dot kernel.
+ // q4_0 = d*(q-8); mirrors the IQ4_NL/q8_0 dense dp4a paths (+ the sum term).
+ // OPT-IN / DEFAULT OFF: correct, but neutral on X2E. q4_0's dequant
+ // ((q-8)*scale) is already trivial so the f16 GEMM is weight-BW-bound and the
+ // int8 ALU win has nothing to beat -- same as q5_0 dense (unlike IQ4_NL, whose
+ // codebook dequant is expensive enough for dp4a to help). Kept for A/B; force
+ // on with GGML_OPENCL_Q4_0_DENSE_DP4A=1. Needs N>8, K%32==0, M%64==0.
+ static const char * q4_0_dense_dp4a_env = getenv("GGML_OPENCL_Q4_0_DENSE_DP4A");
+ const bool q4_0_dense_dp4a_on = q4_0_dense_dp4a_env
+ ? (atoi(q4_0_dense_dp4a_env) != 0)
+ : false;
+ if (q4_0_dense_dp4a_on && backend_ctx->kernel_gemm_noshuffle_q4_0_q8_1_dp4a
+ && N > 8 && (K % 32 == 0) && (M % 64 == 0)) {
+ cl_mem a_sub = nullptr;
+ region.origin = offset1;
+ region.size = (size_t)K * N * sizeof(float);
+ CL_CHECK((a_sub = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, &region, &err), err));
+
+ const size_t n_blocks = (size_t)N * (K / 32);
+ backend_ctx->prealloc_moe_qa.allocate(context, (size_t)N * K * sizeof(cl_char));
+ backend_ctx->prealloc_moe_da.allocate(context, n_blocks * sizeof(cl_half));
+ backend_ctx->prealloc_moe_sa.allocate(context, n_blocks * sizeof(cl_half));
+
+ cl_int tb = (cl_int)n_blocks;
+ cl_kernel qk = backend_ctx->kernel_quant_a_q8_1;
+ CL_CHECK(clSetKernelArg(qk, 0, sizeof(cl_mem), &a_sub));
+ CL_CHECK(clSetKernelArg(qk, 1, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer));
+ CL_CHECK(clSetKernelArg(qk, 2, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer));
+ CL_CHECK(clSetKernelArg(qk, 3, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer));
+ CL_CHECK(clSetKernelArg(qk, 4, sizeof(cl_int), &tb));
+ size_t q_local[1] = { 64 };
+ size_t q_global[1] = { (size_t)(((n_blocks + 63) / 64) * 64) };
+ backend_ctx->enqueue_ndrange_kernel(qk, 1, q_global, q_local, dst);
+
+ cl_kernel dk = backend_ctx->kernel_gemm_noshuffle_q4_0_q8_1_dp4a;
+ int ai = 0;
+ CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extra0_q4_0->q));
+ CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extra0_q4_0->d));
+ CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer));
+ CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer));
+ CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer));
+ CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extrad->data_device));
+ CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_ulong), &offsetd));
+ CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_int), &M));
+ CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_int), &N));
+ CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_int), &K));
+ size_t d_local[3] = { 64, 1, 1 };
+ size_t d_global[3] = { 64, (size_t)(M / 64), (size_t)CEIL_DIV(N, 32) };
+ backend_ctx->enqueue_ndrange_kernel(dk, 3, d_global, d_local, dst);
+
+ CL_CHECK(clReleaseMemObject(a_sub));
+ return;
+ }
+
cl_mem b_sub_buf = nullptr;
cl_mem b_sub_buf_trans = nullptr;
cl_mem b_img = nullptr;
@@ -15624,6 +16253,98 @@ static void ggml_cl_mul_mat_q5_0_f32_adreno(ggml_backend_t backend, const ggml_t
CL_CHECK(clReleaseMemObject(b_sub_buf));
CL_CHECK(clReleaseMemObject(b_img));
} else {
+ // dp4a (int8) dense q5_0 prefill GEMM. Quantizes the [N,K] activations to
+ // q8_1 and runs the int8 dot instead of the f16 half-dot. Large-batch
+ // (ne1>8) only. q5_0 weight = (x-16)*d (x = nibble | hi<<4); x packed as a
+ // 0..31 byte (dp4a), the -16 centering folded into a single min term
+ // (d*16) via the q8_1 block sum. Reads the qs/qh/d buffers byte-identically
+ // to the f16 kernel (greedy byte-identical, MUL_MAT NMSE-OK).
+ //
+ // OPT-IN / DEFAULT OFF. Unlike q8_0/q4_K dense, dp4a is not a win for q5_0 on
+ // X2E: the q5_0 model is bottlenecked elsewhere, so the dense-GEMM int8 win
+ // has nothing to surface and the q8_1 prepass slightly hurts. Kept correct +
+ // opt-in for the X1 A/B (different texture-cache dynamic) and the
+ // weight-texture variant. Env: GGML_OPENCL_Q5_DENSE_DP4A=1.
+ // Weight-as-texture variant (X1 lever): routes the dominant qs nibble plane
+ // through an image1d_buffer (qh stays a buffer). Opt-in
+ // GGML_OPENCL_Q5_DENSE_DP4A_WIMG; when set it also forces the dp4a path on.
+ static const char * q5_dense_dp4a_env = getenv("GGML_OPENCL_Q5_DENSE_DP4A");
+ static const char * q5_dense_wimg_env = getenv("GGML_OPENCL_Q5_DENSE_DP4A_WIMG");
+ const bool q5_dense_wimg_on = q5_dense_wimg_env && (atoi(q5_dense_wimg_env) != 0);
+ const bool q5_dense_dp4a_on = q5_dense_wimg_on
+ ? true
+ : (q5_dense_dp4a_env && (atoi(q5_dense_dp4a_env) != 0));
+ if (q5_dense_dp4a_on && backend_ctx->kernel_gemm_noshuffle_q5_0_q8_1_dp4a
+ && N > 8 && (K % 32 == 0) && (M % 64 == 0)) {
+ cl_mem a_sub = nullptr;
+ region.origin = offset1;
+ region.size = (size_t)K * N * sizeof(float);
+ CL_CHECK((a_sub = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, &region, &err), err));
+
+ const size_t n_blocks = (size_t)N * (K / 32);
+ backend_ctx->prealloc_moe_qa.allocate(context, (size_t)N * K * sizeof(cl_char));
+ backend_ctx->prealloc_moe_da.allocate(context, n_blocks * sizeof(cl_half));
+ backend_ctx->prealloc_moe_sa.allocate(context, n_blocks * sizeof(cl_half));
+
+ cl_int tb = (cl_int)n_blocks;
+ cl_kernel qk = backend_ctx->kernel_quant_a_q8_1;
+ CL_CHECK(clSetKernelArg(qk, 0, sizeof(cl_mem), &a_sub));
+ CL_CHECK(clSetKernelArg(qk, 1, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer));
+ CL_CHECK(clSetKernelArg(qk, 2, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer));
+ CL_CHECK(clSetKernelArg(qk, 3, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer));
+ CL_CHECK(clSetKernelArg(qk, 4, sizeof(cl_int), &tb));
+ size_t q_local[1] = { 64 };
+ size_t q_global[1] = { (size_t)(((n_blocks + 63) / 64) * 64) };
+ backend_ctx->enqueue_ndrange_kernel(qk, 1, q_global, q_local, dst);
+
+ // optional qs texture (image1d_buffer over the nibble plane; the same
+ // CL_R/UINT32 view, width M*K/8, the GEMV path builds).
+ cl_mem q5_qs_img = nullptr;
+ bool use_wimg = q5_dense_wimg_on;
+ if (use_wimg) {
+ const size_t tex = (size_t)M * (size_t)K / 8; // uint32 texels (2 ushorts/texel)
+ if (tex == 0 || tex > backend_ctx->image_max_buffer_size) {
+ use_wimg = false;
+ } else {
+ img_fmt = { CL_R, CL_UNSIGNED_INT32 };
+ memset(&img_desc, 0, sizeof(img_desc));
+ img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER;
+ img_desc.image_width = tex;
+ img_desc.buffer = extra0_q5_0->qs;
+ q5_qs_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err);
+ if (err != CL_SUCCESS || q5_qs_img == nullptr) { use_wimg = false; q5_qs_img = nullptr; }
+ }
+ }
+
+ cl_kernel dk = use_wimg ? backend_ctx->kernel_gemm_noshuffle_q5_0_q8_1_dp4a_wimg
+ : backend_ctx->kernel_gemm_noshuffle_q5_0_q8_1_dp4a;
+ int ai = 0;
+ if (use_wimg) {
+ CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &q5_qs_img));
+ } else {
+ CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extra0_q5_0->qs));
+ }
+ CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extra0_q5_0->qh));
+ CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extra0_q5_0->d));
+ CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer));
+ CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer));
+ CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer));
+ CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extrad->data_device));
+ CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_ulong), &offsetd));
+ CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_int), &M));
+ CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_int), &N));
+ CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_int), &K));
+ size_t d_local[3] = { 64, 1, 1 };
+ size_t d_global[3] = { 64, (size_t)(M / 64), (size_t)CEIL_DIV(N, 32) };
+ backend_ctx->enqueue_ndrange_kernel(dk, 3, d_global, d_local, dst);
+
+ if (q5_qs_img != nullptr) {
+ CL_CHECK(clReleaseMemObject(q5_qs_img));
+ }
+ CL_CHECK(clReleaseMemObject(a_sub));
+ return;
+ }
+
cl_mem b_sub_buf = nullptr;
cl_mem b_sub_buf_trans = nullptr;
cl_mem b_img = nullptr;
@@ -15985,6 +16706,59 @@ static void ggml_cl_mul_mat_iq4_nl_f32_adreno(ggml_backend_t backend, const ggml
CL_CHECK(clReleaseMemObject(b_sub_buf));
CL_CHECK(clReleaseMemObject(b_img));
} else {
+ // dp4a (int8) dense IQ4_NL prefill GEMM. Quantizes the [N,K] activations to
+ // q8_1 and runs the int8 dot instead of the f16 half-dot. Large-batch
+ // (ne1>8) only. IQ4_NL weight = kvalues[nibble]*d; the codebook value IS the
+ // int8 (no min term), so this is the q8_0 dense case plus a nibble->int8 LUT
+ // unpack. Reads the q/d buffers byte-identically to the f16 kernel. No bin
+ // kernel for IQ4_NL -> baseline is f16, default ON for X2E (like q4_K/q6_K
+ // dense dp4a). X1 stays on f16. Env: GGML_OPENCL_IQ4NL_DENSE_DP4A.
+ static const char * iq4nl_dense_dp4a_env = getenv("GGML_OPENCL_IQ4NL_DENSE_DP4A");
+ const bool iq4nl_dense_dp4a_on = iq4nl_dense_dp4a_env
+ ? (atoi(iq4nl_dense_dp4a_env) != 0)
+ : (backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E);
+ if (iq4nl_dense_dp4a_on && backend_ctx->kernel_gemm_noshuffle_iq4_nl_q8_1_dp4a
+ && N > 8 && (K % 32 == 0) && (M % 64 == 0)) {
+ cl_mem a_sub = nullptr;
+ region.origin = offset1;
+ region.size = (size_t)K * N * sizeof(float);
+ CL_CHECK((a_sub = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, &region, &err), err));
+
+ const size_t n_blocks = (size_t)N * (K / 32);
+ backend_ctx->prealloc_moe_qa.allocate(context, (size_t)N * K * sizeof(cl_char));
+ backend_ctx->prealloc_moe_da.allocate(context, n_blocks * sizeof(cl_half));
+ backend_ctx->prealloc_moe_sa.allocate(context, n_blocks * sizeof(cl_half));
+
+ cl_int tb = (cl_int)n_blocks;
+ cl_kernel qk = backend_ctx->kernel_quant_a_q8_1;
+ CL_CHECK(clSetKernelArg(qk, 0, sizeof(cl_mem), &a_sub));
+ CL_CHECK(clSetKernelArg(qk, 1, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer));
+ CL_CHECK(clSetKernelArg(qk, 2, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer));
+ CL_CHECK(clSetKernelArg(qk, 3, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer));
+ CL_CHECK(clSetKernelArg(qk, 4, sizeof(cl_int), &tb));
+ size_t q_local[1] = { 64 };
+ size_t q_global[1] = { (size_t)(((n_blocks + 63) / 64) * 64) };
+ backend_ctx->enqueue_ndrange_kernel(qk, 1, q_global, q_local, dst);
+
+ cl_kernel dk = backend_ctx->kernel_gemm_noshuffle_iq4_nl_q8_1_dp4a;
+ int ai = 0;
+ CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extra0_iq4_nl->q));
+ CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extra0_iq4_nl->d));
+ CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer));
+ CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer));
+ CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extrad->data_device));
+ CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_ulong), &offsetd));
+ CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_int), &M));
+ CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_int), &N));
+ CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_int), &K));
+ size_t d_local[3] = { 64, 1, 1 };
+ size_t d_global[3] = { 64, (size_t)(M / 64), (size_t)CEIL_DIV(N, 32) };
+ backend_ctx->enqueue_ndrange_kernel(dk, 3, d_global, d_local, dst);
+
+ CL_CHECK(clReleaseMemObject(a_sub));
+ return;
+ }
+
cl_mem b_sub_buf = nullptr;
cl_mem b_sub_buf_trans = nullptr;
cl_mem b_img = nullptr;
@@ -16184,6 +16958,90 @@ static void ggml_cl_mul_mat_q8_0_f32_adreno(ggml_backend_t backend, const ggml_t
CL_CHECK(clReleaseMemObject(b_img));
CL_CHECK(clReleaseMemObject(b_sub_buf));
} else {
+ // dp4a dense q8_0 prefill GEMM. Quantizes the [N,K] activations to
+ // q8_1 and runs the int8 dot instead of the f16 half-dot. Large-batch
+ // (ne1>8) only; q8_0 weights are already int8 (no requant) and symmetric
+ // (no min term)
+ static const char * q8_dense_dp4a_env = getenv("GGML_OPENCL_Q8_DENSE_DP4A");
+ static const char * q8_dense_wimg_env = getenv("GGML_OPENCL_Q8_DENSE_DP4A_WIMG");
+ const bool q8_dense_wimg_on = q8_dense_wimg_env && (atoi(q8_dense_wimg_env) != 0);
+
+ const bool q8_bin_loaded = (backend_ctx->kernel_gemm_noshuffle_q8_0_f32_bin != nullptr);
+ // bin kernel takes precedence
+ const bool q8_dense_dp4a_on = q8_dense_wimg_on
+ ? true
+ : q8_dense_dp4a_env
+ ? (atoi(q8_dense_dp4a_env) != 0)
+ : (backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E && !q8_bin_loaded);
+
+ if (q8_dense_dp4a_on && backend_ctx->kernel_gemm_noshuffle_q8_0_q8_1_dp4a
+ && N > 8 && (K % 32 == 0) && (M % 64 == 0)) {
+ cl_mem a_sub = nullptr;
+ region.origin = offset1;
+ region.size = (size_t)K * N * sizeof(float);
+ CL_CHECK((a_sub = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, &region, &err), err));
+
+ const size_t n_blocks = (size_t)N * (K / 32);
+ backend_ctx->prealloc_moe_qa.allocate(context, (size_t)N * K * sizeof(cl_char));
+ backend_ctx->prealloc_moe_da.allocate(context, n_blocks * sizeof(cl_half));
+ backend_ctx->prealloc_moe_sa.allocate(context, n_blocks * sizeof(cl_half));
+
+ cl_int tb = (cl_int)n_blocks;
+ cl_kernel qk = backend_ctx->kernel_quant_a_q8_1;
+ CL_CHECK(clSetKernelArg(qk, 0, sizeof(cl_mem), &a_sub));
+ CL_CHECK(clSetKernelArg(qk, 1, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer));
+ CL_CHECK(clSetKernelArg(qk, 2, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer));
+ CL_CHECK(clSetKernelArg(qk, 3, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer));
+ CL_CHECK(clSetKernelArg(qk, 4, sizeof(cl_int), &tb));
+ size_t q_local[1] = { 64 };
+ size_t q_global[1] = { (size_t)(((n_blocks + 63) / 64) * 64) };
+ backend_ctx->enqueue_ndrange_kernel(qk, 1, q_global, q_local, dst);
+
+ // optional weight texture, the same CL_R/UINT32 view, width M*K/4
+ cl_mem q8_q_img = nullptr;
+ bool use_wimg = q8_dense_wimg_on;
+ if (use_wimg) {
+ const size_t tex = (size_t)M * (size_t)K / 4; // uint32 texels
+ if (tex == 0 || tex > backend_ctx->image_max_buffer_size) {
+ use_wimg = false;
+ } else {
+ img_fmt = { CL_R, CL_UNSIGNED_INT32 };
+ memset(&img_desc, 0, sizeof(img_desc));
+ img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER;
+ img_desc.image_width = tex;
+ img_desc.buffer = extra0_q8_0->q;
+ q8_q_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err);
+ if (err != CL_SUCCESS || q8_q_img == nullptr) { use_wimg = false; q8_q_img = nullptr; }
+ }
+ }
+
+ cl_kernel dk = use_wimg ? backend_ctx->kernel_gemm_noshuffle_q8_0_q8_1_dp4a_wimg
+ : backend_ctx->kernel_gemm_noshuffle_q8_0_q8_1_dp4a;
+ int ai = 0;
+ if (use_wimg) {
+ CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &q8_q_img));
+ } else {
+ CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extra0_q8_0->q));
+ }
+ CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extra0_q8_0->d));
+ CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer));
+ CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer));
+ CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extrad->data_device));
+ CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_ulong), &offsetd));
+ CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_int), &M));
+ CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_int), &N));
+ CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_int), &K));
+ size_t d_local[3] = { 64, 1, 1 };
+ size_t d_global[3] = { 64, (size_t)(M / 64), (size_t)CEIL_DIV(N, 32) };
+ backend_ctx->enqueue_ndrange_kernel(dk, 3, d_global, d_local, dst);
+
+ if (q8_q_img != nullptr) {
+ CL_CHECK(clReleaseMemObject(q8_q_img));
+ }
+ CL_CHECK(clReleaseMemObject(a_sub));
+ return;
+ }
+
// use bin kernel if available
if (backend_ctx->kernel_gemm_noshuffle_q8_0_f32_bin) {
int K_pad = K;
@@ -16517,6 +17375,99 @@ static void ggml_cl_mul_mat_q4_k_f32_adreno(ggml_backend_t backend, const ggml_t
size_t global_work_size_t[2] = { (size_t)width_B, (size_t)padded_height_B };
backend_ctx->enqueue_ndrange_kernel(kernel, 2, global_work_size_t, local_work_size_t, dst);
+ // dp4a (int8) dense prefill GEMM and weight via texture
+ static const char * q4k_dense_dp4a_env = getenv("GGML_OPENCL_Q4K_DENSE_DP4A");
+ static const char * q4k_dense_wimg_env = getenv("GGML_OPENCL_Q4K_DENSE_DP4A_WIMG");
+
+ const bool q4k_dense_wimg_on = q4k_dense_wimg_env && (atoi(q4k_dense_wimg_env) != 0);
+ const bool q4k_dense_dp4a_on = q4k_dense_wimg_on
+ ? true
+ : q4k_dense_dp4a_env
+ ? (atoi(q4k_dense_dp4a_env) != 0)
+ : (backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E);
+
+ // Min N for the dp4a prefill GEMM, default 9, i.e., ne1 > 8
+ static const char * q4k_dp4a_minn_env = getenv("GGML_OPENCL_Q4K_DP4A_MINN");
+ const int q4k_dp4a_minn = q4k_dp4a_minn_env ? atoi(q4k_dp4a_minn_env) : 9;
+
+ if (q4k_dense_dp4a_on && N >= q4k_dp4a_minn && (K % 32 == 0) && (M % 64 == 0)) {
+ const size_t n_blocks = (size_t)N * (K / 32);
+ backend_ctx->prealloc_moe_qa.allocate(context, (size_t)N * K * sizeof(cl_char));
+ backend_ctx->prealloc_moe_da.allocate(context, n_blocks * sizeof(cl_half));
+ backend_ctx->prealloc_moe_sa.allocate(context, n_blocks * sizeof(cl_half));
+
+ cl_int tb = (cl_int)n_blocks;
+ cl_kernel qk = backend_ctx->kernel_quant_a_q8_1;
+ CL_CHECK(clSetKernelArg(qk, 0, sizeof(cl_mem), &b_sub_buf));
+ CL_CHECK(clSetKernelArg(qk, 1, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer));
+ CL_CHECK(clSetKernelArg(qk, 2, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer));
+ CL_CHECK(clSetKernelArg(qk, 3, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer));
+ CL_CHECK(clSetKernelArg(qk, 4, sizeof(cl_int), &tb));
+ size_t q_local[1] = { 64 };
+ size_t q_global[1] = { (size_t)(((n_blocks + 63) / 64) * 64) };
+ backend_ctx->enqueue_ndrange_kernel(qk, 1, q_global, q_local, dst);
+
+ // check if weights go through texture
+ cl_mem q4k_q_img = nullptr;
+ bool use_wimg = q4k_dense_wimg_on;
+ if (use_wimg) {
+ const size_t tex = (size_t)M * (size_t)K / 8; // uint32 texels = bytes/4
+ if (tex == 0 || tex > backend_ctx->image_max_buffer_size) {
+ use_wimg = false;
+ } else {
+ img_fmt = { CL_R, CL_UNSIGNED_INT32 };
+ memset(&img_desc, 0, sizeof(img_desc));
+ img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER;
+ img_desc.image_width = tex;
+ img_desc.buffer = extra0_q4_k->q;
+ q4k_q_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err);
+ if (err != CL_SUCCESS || q4k_q_img == nullptr) {
+ use_wimg = false;
+ q4k_q_img = nullptr;
+ }
+ }
+ }
+
+ cl_kernel dk = use_wimg ? backend_ctx->kernel_gemm_noshuffle_q4_k_q8_1_dp4a_wimg
+ : backend_ctx->kernel_gemm_noshuffle_q4_k_q8_1_dp4a;
+ int ai = 0;
+ if (use_wimg) {
+ CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &q4k_q_img));
+ } else {
+ CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extra0_q4_k->q));
+ }
+ CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extra0_q4_k->s));
+ CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extra0_q4_k->d));
+ CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extra0_q4_k->dm));
+ CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer));
+ CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer));
+ CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer));
+ CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extrad->data_device));
+ CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_ulong), &offsetd));
+ CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_int), &M));
+ CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_int), &N));
+ CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_int), &K));
+ CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_uchar), &mask_d6));
+ CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_uchar), &mask_d4));
+ CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_uchar), &mask_hi2));
+ // Must match the compile-time TILESIZE_N chosen at program build (per-device,
+ // X1E=8 else 32; env override). Same inputs -> same value.
+ int q4k_dp4a_ts = (backend_ctx->adreno_gen == ADRENO_GPU_GEN::X1E) ? 8 : 32;
+ if (const char * e = getenv("GGML_OPENCL_Q4K_DP4A_TS")) q4k_dp4a_ts = atoi(e);
+ size_t d_local[3] = { 64, 1, 1 };
+ size_t d_global[3] = { 64, (size_t)(M / 64), (size_t)CEIL_DIV(N, q4k_dp4a_ts) };
+ backend_ctx->enqueue_ndrange_kernel(dk, 3, d_global, d_local, dst);
+
+ if (q4k_q_img != nullptr) {
+ CL_CHECK(clReleaseMemObject(q4k_q_img));
+ }
+ CL_CHECK(clReleaseMemObject(b_sub_buf));
+ CL_CHECK(clReleaseMemObject(b_sub_buf_trans));
+ CL_CHECK(clReleaseMemObject(b_img));
+ CL_CHECK(clReleaseMemObject(b_img_trans));
+ return;
+ }
+
// gemm
kernel = backend_ctx->kernel_gemm_noshuffle_q4_k_f32;
int padded_N = N + padding;
@@ -16655,6 +17606,54 @@ static void ggml_cl_mul_mat_q6_K_f32_adreno(ggml_backend_t backend, const ggml_t
region.size = ne00 * ne1 * sizeof(float);
CL_CHECK((b_sub_buf = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, &region, &err), err));
+ // dp4a (int8) dense q6_K prefill GEMM
+ static const char * q6k_dense_dp4a_env = getenv("GGML_OPENCL_Q6K_DENSE_DP4A");
+ static const bool q6k_dense_dp4a_on = (q6k_dense_dp4a_env != nullptr)
+ ? (atoi(q6k_dense_dp4a_env) != 0)
+ : (backend_ctx->adreno_gen != ADRENO_GPU_GEN::X1E);
+
+ const bool is_output_w_dp4a = strncmp(src0->name, "output", 6) == 0 ||
+ strncmp(src0->name, "token_embd", 10) == 0;
+
+ if (q6k_dense_dp4a_on && !is_output_w_dp4a && ne1 > 8 && (ne00 % 32 == 0) && (ne01 % 64 == 0)) {
+ const int M = ne01, N = ne1, K = ne00;
+ const size_t n_blocks = (size_t)N * (K / 32);
+ backend_ctx->prealloc_moe_qa.allocate(context, (size_t)N * K * sizeof(cl_char));
+ backend_ctx->prealloc_moe_da.allocate(context, n_blocks * sizeof(cl_half));
+ backend_ctx->prealloc_moe_sa.allocate(context, n_blocks * sizeof(cl_half));
+
+ cl_int tb = (cl_int)n_blocks;
+ cl_kernel qk = backend_ctx->kernel_quant_a_q8_1;
+ CL_CHECK(clSetKernelArg(qk, 0, sizeof(cl_mem), &b_sub_buf));
+ CL_CHECK(clSetKernelArg(qk, 1, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer));
+ CL_CHECK(clSetKernelArg(qk, 2, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer));
+ CL_CHECK(clSetKernelArg(qk, 3, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer));
+ CL_CHECK(clSetKernelArg(qk, 4, sizeof(cl_int), &tb));
+ size_t q_local[1] = { 64 };
+ size_t q_global[1] = { (size_t)(((n_blocks + 63) / 64) * 64) };
+ backend_ctx->enqueue_ndrange_kernel(qk, 1, q_global, q_local, dst);
+
+ cl_kernel dk = backend_ctx->kernel_gemm_noshuffle_q6_k_q8_1_dp4a;
+ int ai = 0;
+ CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extra0_q6_K->ql));
+ CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extra0_q6_K->qh));
+ CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extra0_q6_K->s));
+ CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extra0_q6_K->d));
+ CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer));
+ CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer));
+ CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extrad->data_device));
+ CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_ulong), &offsetd));
+ CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_int), &M));
+ CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_int), &N));
+ CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_int), &K));
+ size_t d_local[3] = { 64, 1, 1 };
+ size_t d_global[3] = { 64, (size_t)(M / 64), (size_t)CEIL_DIV(N, 32) };
+ backend_ctx->enqueue_ndrange_kernel(dk, 3, d_global, d_local, dst);
+
+ CL_CHECK(clReleaseMemObject(b_sub_buf));
+ return;
+ }
+
// image for activation
img_fmt.image_channel_order = CL_RGBA;
img_fmt.image_channel_data_type = CL_FLOAT;
@@ -16900,6 +17899,59 @@ static void ggml_cl_mul_mat_q5_K_f32_adreno(ggml_backend_t backend, const ggml_t
size_t global_work_size_t[2] = {(size_t)width_B, (size_t)padded_height_B};
backend_ctx->enqueue_ndrange_kernel(kernel, 2, global_work_size_t, local_work_size_t, dst);
+ // dp4a (int8) dense q5_K prefill GEMM
+ static const char * q5k_dense_dp4a_env = getenv("GGML_OPENCL_Q5K_DENSE_DP4A");
+ const bool q5k_dense_dp4a_on = q5k_dense_dp4a_env
+ ? (atoi(q5k_dense_dp4a_env) != 0)
+ : (backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E);
+
+ if (q5k_dense_dp4a_on && ne1 > 8 && (ne00 % 32 == 0) && (ne01 % 64 == 0)) {
+ const int Mm = ne01, Nn = ne1, Kk = ne00;
+ const size_t n_blocks = (size_t)Nn * (Kk / 32);
+ backend_ctx->prealloc_moe_qa.allocate(context, (size_t)Nn * Kk * sizeof(cl_char));
+ backend_ctx->prealloc_moe_da.allocate(context, n_blocks * sizeof(cl_half));
+ backend_ctx->prealloc_moe_sa.allocate(context, n_blocks * sizeof(cl_half));
+
+ cl_int tb = (cl_int)n_blocks;
+ cl_kernel qk = backend_ctx->kernel_quant_a_q8_1;
+ CL_CHECK(clSetKernelArg(qk, 0, sizeof(cl_mem), &b_sub_buf));
+ CL_CHECK(clSetKernelArg(qk, 1, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer));
+ CL_CHECK(clSetKernelArg(qk, 2, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer));
+ CL_CHECK(clSetKernelArg(qk, 3, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer));
+ CL_CHECK(clSetKernelArg(qk, 4, sizeof(cl_int), &tb));
+ size_t q_local[1] = { 64 };
+ size_t q_global[1] = { (size_t)(((n_blocks + 63) / 64) * 64) };
+ backend_ctx->enqueue_ndrange_kernel(qk, 1, q_global, q_local, dst);
+
+ cl_kernel dk = backend_ctx->kernel_gemm_noshuffle_q5_k_q8_1_dp4a;
+ int ai = 0;
+ CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extra0_q5_k->q));
+ CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extra0_q5_k->qh));
+ CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extra0_q5_k->s));
+ CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extra0_q5_k->d));
+ CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extra0_q5_k->dm));
+ CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer));
+ CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer));
+ CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer));
+ CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extrad->data_device));
+ CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_ulong), &offsetd));
+ CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_int), &Mm));
+ CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_int), &Nn));
+ CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_int), &Kk));
+ CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_uchar), &mask_d6));
+ CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_uchar), &mask_d4));
+ CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_uchar), &mask_hi2));
+ size_t d_local[3] = { 64, 1, 1 };
+ size_t d_global[3] = { 64, (size_t)(Mm / 64), (size_t)CEIL_DIV(Nn, 32) };
+ backend_ctx->enqueue_ndrange_kernel(dk, 3, d_global, d_local, dst);
+
+ CL_CHECK(clReleaseMemObject(b_sub_buf));
+ CL_CHECK(clReleaseMemObject(b_sub_buf_trans));
+ CL_CHECK(clReleaseMemObject(b_img));
+ CL_CHECK(clReleaseMemObject(b_img_trans));
+ return;
+ }
+
// gemm
kernel = backend_ctx->kernel_gemm_noshuffle_q5_k_f32;
int padded_N = N + padding;
@@ -19380,6 +20432,23 @@ static void moe_router_reoerder(ggml_backend_t backend, const ggml_tensor * src,
backend_ctx->enqueue_ndrange_kernel(kernel, 3, histogram_global_size, histogram_local_size, src);
+ // [MOE_TILES] env-gated padding probe: read back total_tiles (= Sum_e
+ // ceil(k_e/n_tile_size)) and compare to the ideal tile count for the real
+ // routing count. Quantifies the per-expert tile-padding waste. Blocking
+ // readback perturbs timing -> diagnostic only.
+ if (getenv("GGML_OPENCL_MOE_TILES_DEBUG")) {
+ int h_total = 0;
+ clFinish(backend_ctx->queue);
+ CL_CHECK(clEnqueueReadBuffer(backend_ctx->queue, total_tiles_buf, CL_TRUE, 0, sizeof(int), &h_total, 0, NULL, NULL));
+ const int routings = ne20 * ne21;
+ const int ideal = (routings + n_tile_size - 1) / n_tile_size;
+ const int slots = h_total * n_tile_size;
+ fprintf(stderr, "[MOE_TILES] routings=%d (ne20=%d ne21=%d nexp=%d) total_tiles=%d ideal=%d slots=%d pad=%.1f%%\n",
+ routings, ne20, ne21, ne02, h_total, ideal, slots,
+ routings > 0 ? 100.0 * (slots - routings) / routings : 0.0);
+ fflush(stderr);
+ }
+
CL_CHECK(clReleaseMemObject(original_router_buf));
CL_CHECK(clReleaseMemObject(hist_buf));
CL_CHECK(clReleaseMemObject(tile_offset_buf));
@@ -19562,9 +20631,18 @@ static void ggml_cl_mul_mat_id(ggml_backend_t backend, const ggml_tensor * src0,
backend_ctx->toggle_reorder = false;
}
- cl_mem sub_buf_src1_pre, buf_src1_reordered, image_src1_reordered, sub_buf_dst, buf_dst_image;
+ cl_mem sub_buf_src1_pre, sub_buf_dst, buf_dst_image;
+ cl_mem buf_src1_reordered = nullptr, image_src1_reordered = nullptr;
cl_mem buf_src2, buf_src2_emap;
+ // dp4a (int8) prefill GEMM variant
+ static const char * q4_0_moe_dp4a_env = getenv("GGML_OPENCL_Q4_0_MOE_DP4A");
+ bool use_moe_dp4a = q4_0_moe_dp4a_env
+ ? (atoi(q4_0_moe_dp4a_env) != 0)
+ : (backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E);
+ // bin kernel takes precedence
+ use_moe_dp4a = use_moe_dp4a && backend_ctx->kernel_gemm_moe_q4_0_f32_ns_bin == nullptr;
+
cl_buffer_region region;
region.origin = 0;
region.size = sizeof(int) * max_post_router_tile * n_tile_size;
@@ -19583,45 +20661,48 @@ static void ggml_cl_mul_mat_id(ggml_backend_t backend, const ggml_tensor * src0,
sub_buf_src1_pre = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, &region, &status);
CL_CHECK(status);
- // Create image for reordered src1
- // Use pre-allocated placeholder
- region.origin = 0;
- region.size = ne00 * max_post_router_tile * n_tile_size * sizeof(float);
- backend_ctx->prealloc_act_trans.allocate(backend_ctx->context, region.size);
- buf_src1_reordered = clCreateSubBuffer(
- backend_ctx->prealloc_act_trans.buffer,
- 0,
- CL_BUFFER_CREATE_TYPE_REGION,
- &region,
- &status);
- CL_CHECK(status);
- cl_image_format image_format_buf_src1;
- cl_image_desc image_desc_buf_src1;
- image_format_buf_src1 = {CL_RGBA, CL_FLOAT};
- image_desc_buf_src1 = {CL_MEM_OBJECT_IMAGE1D_BUFFER, static_cast<size_t>(ne00 * max_post_router_tile * n_tile_size / 4), 0,0,0,0,0,0,0, {buf_src1_reordered}};
- if (backend_ctx->kernel_gemm_moe_q4_0_f32_ns_bin) {
- // bin kernel uses slightly different image format
- image_format_buf_src1 = {CL_R, CL_FLOAT};
- image_desc_buf_src1.image_width = static_cast<size_t>(ne00 * max_post_router_tile * n_tile_size);
- }
- image_src1_reordered = clCreateImage(backend_ctx->context, CL_MEM_READ_ONLY, &image_format_buf_src1, &image_desc_buf_src1, NULL, &status);
- CL_CHECK(status);
-
unsigned short map_ratio = ne20 / ne11;
GGML_ASSERT(((map_ratio == 1) || (map_ratio == ne20)) && "Map ratio not supported\n");
- CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 0, sizeof(cl_mem), &sub_buf_src1_pre));
- CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 1, sizeof(cl_mem), &buf_src2));
- CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 2, sizeof(cl_mem), &buf_src1_reordered));
- CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 3, sizeof(cl_mem), &(backend_ctx->prealloc_total_tiles.buffer)));
- CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 4, sizeof(unsigned int), &ne00));
- CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 5, sizeof(unsigned short), &map_ratio));
- CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 6, sizeof(unsigned int), &n_tile_size));
-
- size_t reorder_b_local_size[3] = {256, 1, 1};
- size_t reorder_b_global_size[3] = {static_cast<size_t>(((ne00 / 4) + 255) / 256 * 256), static_cast<size_t>(max_post_router_tile * n_tile_size), 1};
- // Dispatch reorder kernel
- backend_ctx->enqueue_ndrange_kernel(backend_ctx->kernel_moe_reorder_b, 3, reorder_b_global_size, reorder_b_local_size, dst);
+ if (!use_moe_dp4a) {
+ // Create image for reordered src1
+ // Use pre-allocated placeholder
+ region.origin = 0;
+ region.size = ne00 * max_post_router_tile * n_tile_size * sizeof(float);
+ backend_ctx->prealloc_act_trans.allocate(backend_ctx->context, region.size);
+ buf_src1_reordered = clCreateSubBuffer(
+ backend_ctx->prealloc_act_trans.buffer,
+ 0,
+ CL_BUFFER_CREATE_TYPE_REGION,
+ &region,
+ &status);
+ CL_CHECK(status);
+ cl_image_format image_format_buf_src1;
+ cl_image_desc image_desc_buf_src1;
+ image_format_buf_src1 = {CL_RGBA, CL_FLOAT};
+ image_desc_buf_src1 = {CL_MEM_OBJECT_IMAGE1D_BUFFER, static_cast<size_t>(ne00 * max_post_router_tile * n_tile_size / 4), 0,0,0,0,0,0,0, {buf_src1_reordered}};
+ if (backend_ctx->kernel_gemm_moe_q4_0_f32_ns_bin) {
+ // bin kernel uses slightly different image format
+ image_format_buf_src1 = {CL_R, CL_FLOAT};
+ image_desc_buf_src1.image_width = static_cast<size_t>(ne00 * max_post_router_tile * n_tile_size);
+ }
+ image_src1_reordered = clCreateImage(backend_ctx->context, CL_MEM_READ_ONLY, &image_format_buf_src1, &image_desc_buf_src1, NULL, &status);
+ CL_CHECK(status);
+
+ CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 0, sizeof(cl_mem), &sub_buf_src1_pre));
+ CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 1, sizeof(cl_mem), &buf_src2));
+ CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 2, sizeof(cl_mem), &buf_src1_reordered));
+ CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 3, sizeof(cl_mem), &(backend_ctx->prealloc_total_tiles.buffer)));
+ CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 4, sizeof(unsigned int), &ne00));
+ CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 5, sizeof(unsigned short), &map_ratio));
+ CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 6, sizeof(unsigned int), &n_tile_size));
+
+ size_t reorder_b_local_size[3] = {256, 1, 1};
+ size_t reorder_b_global_size[3] = {static_cast<size_t>(((ne00 / 4) + 255) / 256 * 256), static_cast<size_t>(max_post_router_tile * n_tile_size), 1};
+
+ // Dispatch reorder kernel
+ backend_ctx->enqueue_ndrange_kernel(backend_ctx->kernel_moe_reorder_b, 3, reorder_b_global_size, reorder_b_local_size, dst);
+ }
// MoE kernel prepare
// Create sub buffer for dst
@@ -19640,6 +20721,58 @@ static void ggml_cl_mul_mat_id(ggml_backend_t backend, const ggml_tensor * src0,
buf_dst_image = clCreateImage(backend_ctx->context, CL_MEM_WRITE_ONLY, &image_format_buf_dst, &image_desc_buf_dst, NULL, &status);
CL_CHECK(status);
+ if (use_moe_dp4a) {
+ const size_t tok_slots = (size_t)max_post_router_tile * n_tile_size;
+ const size_t n_blocks = tok_slots * (ne00 / 32);
+ backend_ctx->prealloc_moe_qa.allocate(backend_ctx->context, tok_slots * ne00 * sizeof(cl_char));
+ backend_ctx->prealloc_moe_da.allocate(backend_ctx->context, n_blocks * sizeof(cl_half));
+ backend_ctx->prealloc_moe_sa.allocate(backend_ctx->context, n_blocks * sizeof(cl_half));
+
+ // fused reorder + q8_1 quant straight from the original activations
+ const cl_uint n_kblocks = (cl_uint)(ne00 / 32);
+ cl_kernel rq = backend_ctx->kernel_moe_reorder_quant_a_q8_1;
+ CL_CHECK(clSetKernelArg(rq, 0, sizeof(cl_mem), &sub_buf_src1_pre));
+ CL_CHECK(clSetKernelArg(rq, 1, sizeof(cl_mem), &buf_src2));
+ CL_CHECK(clSetKernelArg(rq, 2, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer));
+ CL_CHECK(clSetKernelArg(rq, 3, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer));
+ CL_CHECK(clSetKernelArg(rq, 4, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer));
+ CL_CHECK(clSetKernelArg(rq, 5, sizeof(cl_mem), &(backend_ctx->prealloc_total_tiles.buffer)));
+ CL_CHECK(clSetKernelArg(rq, 6, sizeof(cl_uint), &ne00));
+ CL_CHECK(clSetKernelArg(rq, 7, sizeof(unsigned short), &map_ratio));
+ CL_CHECK(clSetKernelArg(rq, 8, sizeof(cl_uint), &n_tile_size));
+ CL_CHECK(clSetKernelArg(rq, 9, sizeof(cl_uint), &n_kblocks));
+ size_t rq_local[2] = { 32, 1 };
+ size_t rq_global[2] = { (size_t)(((n_kblocks + 31) / 32) * 32), tok_slots };
+ backend_ctx->enqueue_ndrange_kernel(rq, 2, rq_global, rq_local, dst);
+
+ // dp4a GEMM
+ cl_kernel dk = backend_ctx->kernel_gemm_moe_q4_0_q8_1_dp4a;
+ int aidx = 0;
+ CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &extra0_q4_0->q_img));
+ CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &extra0_q4_0->d));
+ CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer));
+ CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer));
+ CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer));
+ CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &buf_src2));
+ CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &buf_src2_emap));
+ CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &buf_dst_image));
+ CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &(backend_ctx->prealloc_total_tiles.buffer)));
+ CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(int), &ne00));
+ CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(int), &ne01));
+ CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(int), &backend_ctx->adreno_use_moe_ragged_dp4));
+
+ size_t dp_global[3] = { 64, (size_t)((ne01 + 63) / 64), (size_t)max_post_router_tile };
+ size_t dp_local[3] = { 64, 1, 1 };
+ backend_ctx->enqueue_ndrange_kernel(dk, 3, dp_global, dp_local, dst);
+
+ clReleaseMemObject(sub_buf_src1_pre);
+ clReleaseMemObject(buf_src2);
+ clReleaseMemObject(buf_src2_emap);
+ clReleaseMemObject(sub_buf_dst);
+ clReleaseMemObject(buf_dst_image);
+ return;
+ }
+
// Set kernel args
int arg_idx = 0;
CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &extra0_q4_0->q_img));
@@ -19993,6 +21126,81 @@ static void ggml_cl_mul_mat_id(ggml_backend_t backend, const ggml_tensor * src0,
sub_buf_src1_pre = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, &region, &status);
CL_CHECK(status);
+ // Generic dp4a MoE GEMM
+ {
+ static const char * q5mdp4a_env = getenv("GGML_OPENCL_Q5_MOE_DP4A");
+ const bool q5mdp4a_on = q5mdp4a_env ? (atoi(q5mdp4a_env) != 0)
+ : (backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E);
+ const bool use_q5_moe_dp4a = q5mdp4a_on
+ && backend_ctx->kernel_gemm_moe_q8_1_dp4a_q50 != nullptr
+ && extra0_q5_0->scale != nullptr;
+
+ if (use_q5_moe_dp4a) {
+ const size_t tok_slots = (size_t)max_post_router_tile * n_tile_size;
+ const size_t n_blocks = tok_slots * (ne00 / 32);
+ backend_ctx->prealloc_moe_qa.allocate(backend_ctx->context, tok_slots * ne00 * sizeof(cl_char));
+ backend_ctx->prealloc_moe_da.allocate(backend_ctx->context, n_blocks * sizeof(cl_half));
+ backend_ctx->prealloc_moe_sa.allocate(backend_ctx->context, n_blocks * sizeof(cl_half));
+
+ const cl_uint n_kblocks = (cl_uint)(ne00 / 32);
+ unsigned short map_ratio_q5 = ne20 / ne11;
+ cl_kernel rq = backend_ctx->kernel_moe_reorder_quant_a_q8_1;
+ CL_CHECK(clSetKernelArg(rq, 0, sizeof(cl_mem), &sub_buf_src1_pre));
+ CL_CHECK(clSetKernelArg(rq, 1, sizeof(cl_mem), &buf_src2));
+ CL_CHECK(clSetKernelArg(rq, 2, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer));
+ CL_CHECK(clSetKernelArg(rq, 3, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer));
+ CL_CHECK(clSetKernelArg(rq, 4, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer));
+ CL_CHECK(clSetKernelArg(rq, 5, sizeof(cl_mem), &(backend_ctx->prealloc_total_tiles.buffer)));
+ CL_CHECK(clSetKernelArg(rq, 6, sizeof(cl_uint), &ne00));
+ CL_CHECK(clSetKernelArg(rq, 7, sizeof(unsigned short), &map_ratio_q5));
+ CL_CHECK(clSetKernelArg(rq, 8, sizeof(cl_uint), &n_tile_size));
+ CL_CHECK(clSetKernelArg(rq, 9, sizeof(cl_uint), &n_kblocks));
+ size_t rq_local[2] = { 32, 1 };
+ size_t rq_global[2] = { (size_t)(((n_kblocks + 31) / 32) * 32), tok_slots };
+ backend_ctx->enqueue_ndrange_kernel(rq, 2, rq_global, rq_local, dst);
+
+ region.origin = offsetd;
+ region.size = ne0 * ne1 * ne2 * sizeof(float);
+ cl_mem dp_sub_buf_dst = clCreateSubBuffer(extrad->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, &region, &status);
+ CL_CHECK(status);
+ cl_image_format dp_ifd = {CL_R, CL_FLOAT};
+ cl_image_desc dp_idd = {CL_MEM_OBJECT_IMAGE1D_BUFFER, static_cast<size_t>(ne0 * ne1 * ne2), 0,0,0,0,0,0,0, {dp_sub_buf_dst}};
+ cl_mem dp_buf_dst_image = clCreateImage(backend_ctx->context, CL_MEM_WRITE_ONLY, &dp_ifd, &dp_idd, NULL, &status);
+ CL_CHECK(status);
+
+ int ne00i = (int)ne00, ne01i = (int)ne01;
+ cl_kernel dk = backend_ctx->kernel_gemm_moe_q8_1_dp4a_q50;
+ int has_min_q5 = 1;
+ int aidx = 0;
+ CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &extra0_q5_0->qs_img));
+ CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &extra0_q5_0->qh));
+ CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &extra0_q5_0->scale));
+ CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &extra0_q5_0->min));
+ CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer));
+ CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer));
+ CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer));
+ CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &buf_src2));
+ CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &buf_src2_emap));
+ CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &dp_buf_dst_image));
+ CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &(backend_ctx->prealloc_total_tiles.buffer)));
+ CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(int), &ne00i));
+ CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(int), &ne01i));
+ CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(int), &backend_ctx->adreno_use_moe_ragged_dp4));
+ CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(int), &has_min_q5));
+
+ size_t dp_global[3] = { 64, (size_t)((ne01 + 63) / 64), (size_t)max_post_router_tile };
+ size_t dp_local[3] = { 64, 1, 1 };
+ backend_ctx->enqueue_ndrange_kernel(dk, 3, dp_global, dp_local, dst);
+
+ clReleaseMemObject(sub_buf_src1_pre);
+ clReleaseMemObject(buf_src2);
+ clReleaseMemObject(buf_src2_emap);
+ clReleaseMemObject(dp_sub_buf_dst);
+ clReleaseMemObject(dp_buf_dst_image);
+ return;
+ }
+ }
+
// Create image for reordered src1
// Use pre-allocated placeholder
region.origin = 0;
@@ -20261,6 +21469,183 @@ static void ggml_cl_mul_mat_id(ggml_backend_t backend, const ggml_tensor * src0,
#endif //GGML_OPENCL_USE_ADRENO_KERNELS
}
case GGML_TYPE_Q8_0: {
+#ifdef GGML_OPENCL_USE_ADRENO_KERNELS
+ // MoE GEMM for q8_0 at prefill (ne12>1)
+ // There is no corresponding gemv_moe, so the code path is different here
+ static const char * moe_gemm_q8_env = getenv("GGML_OPENCL_MOE_GEMM_Q8");
+ const bool moe_gemm_q8 = moe_gemm_q8_env
+ ? (atoi(moe_gemm_q8_env) != 0)
+ : (backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E);
+ if (moe_gemm_q8 && use_adreno_moe_kernels(backend_ctx, src0) && ne12 > 1) {
+ cl_int status;
+
+ size_t local_size[3] = {64, 2, 1};
+ size_t global_size[3] = {64, 2, 1};
+
+ kernel = backend_ctx->kernel_gemm_moe_q8_0_f32_ns;
+
+ if ((strstr(src0->name, "as") != NULL) || backend_ctx->toggle_reorder) {
+ moe_router_reoerder(backend, src2, ne20);
+ backend_ctx->toggle_reorder = false;
+ }
+
+ cl_mem sub_buf_src1_pre, buf_src1_reordered, image_src1_reordered, sub_buf_dst, buf_dst_image;
+ cl_mem buf_src2, buf_src2_emap;
+
+ cl_buffer_region region;
+ region.origin = 0;
+ region.size = sizeof(int) * max_post_router_tile * n_tile_size;
+ buf_src2 = clCreateSubBuffer(backend_ctx->prealloc_post_router.buffer, 0, CL_BUFFER_CREATE_TYPE_REGION, &region, &status);
+ CL_CHECK(status);
+
+ region.origin = 0;
+ region.size = sizeof(short) * max_post_router_tile;
+ buf_src2_emap = clCreateSubBuffer(backend_ctx->prealloc_emap.buffer, 0, CL_BUFFER_CREATE_TYPE_REGION, &region, &status);
+ CL_CHECK(status);
+
+ // Reorder activations (group tokens by expert into tiles of 32)
+ region.origin = offset1;
+ region.size = ne10 * ne11 * ne12 * sizeof(float);
+ sub_buf_src1_pre = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, &region, &status);
+ CL_CHECK(status);
+
+ // Generic dp4a MoE GEMM
+ {
+ static const char * q8mdp4a_env = getenv("GGML_OPENCL_Q8_MOE_DP4A");
+ const bool q8mdp4a_on = q8mdp4a_env ? (atoi(q8mdp4a_env) != 0)
+ : (backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E);
+ const bool use_q8_moe_dp4a = q8mdp4a_on
+ && backend_ctx->kernel_gemm_moe_q8_1_dp4a_q80 != nullptr
+ && extra0_q8_0->scale != nullptr;
+ if (use_q8_moe_dp4a) {
+ const size_t tok_slots = (size_t)max_post_router_tile * n_tile_size;
+ const size_t n_blocks = tok_slots * (ne00 / 32);
+ backend_ctx->prealloc_moe_qa.allocate(backend_ctx->context, tok_slots * ne00 * sizeof(cl_char));
+ backend_ctx->prealloc_moe_da.allocate(backend_ctx->context, n_blocks * sizeof(cl_half));
+ backend_ctx->prealloc_moe_sa.allocate(backend_ctx->context, n_blocks * sizeof(cl_half));
+
+ const cl_uint n_kblocks = (cl_uint)(ne00 / 32);
+ unsigned short map_ratio_q8 = ne20 / ne11;
+ cl_kernel rq = backend_ctx->kernel_moe_reorder_quant_a_q8_1;
+ CL_CHECK(clSetKernelArg(rq, 0, sizeof(cl_mem), &sub_buf_src1_pre));
+ CL_CHECK(clSetKernelArg(rq, 1, sizeof(cl_mem), &buf_src2));
+ CL_CHECK(clSetKernelArg(rq, 2, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer));
+ CL_CHECK(clSetKernelArg(rq, 3, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer));
+ CL_CHECK(clSetKernelArg(rq, 4, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer));
+ CL_CHECK(clSetKernelArg(rq, 5, sizeof(cl_mem), &(backend_ctx->prealloc_total_tiles.buffer)));
+ CL_CHECK(clSetKernelArg(rq, 6, sizeof(cl_uint), &ne00));
+ CL_CHECK(clSetKernelArg(rq, 7, sizeof(unsigned short), &map_ratio_q8));
+ CL_CHECK(clSetKernelArg(rq, 8, sizeof(cl_uint), &n_tile_size));
+ CL_CHECK(clSetKernelArg(rq, 9, sizeof(cl_uint), &n_kblocks));
+ size_t rq_local[2] = { 32, 1 };
+ size_t rq_global[2] = { (size_t)(((n_kblocks + 31) / 32) * 32), tok_slots };
+ backend_ctx->enqueue_ndrange_kernel(rq, 2, rq_global, rq_local, dst);
+
+ // dst image
+ region.origin = offsetd;
+ region.size = ne0 * ne1 * ne2 * sizeof(float);
+ cl_mem dp_sub_buf_dst = clCreateSubBuffer(extrad->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, &region, &status);
+ CL_CHECK(status);
+ cl_image_format dp_ifd = {CL_R, CL_FLOAT};
+ cl_image_desc dp_idd = {CL_MEM_OBJECT_IMAGE1D_BUFFER, static_cast<size_t>(ne0 * ne1 * ne2), 0,0,0,0,0,0,0, {dp_sub_buf_dst}};
+ cl_mem dp_buf_dst_image = clCreateImage(backend_ctx->context, CL_MEM_WRITE_ONLY, &dp_ifd, &dp_idd, NULL, &status);
+ CL_CHECK(status);
+
+ int ne00i = (int)ne00, ne01i = (int)ne01;
+ cl_kernel dk = backend_ctx->kernel_gemm_moe_q8_1_dp4a_q80;
+ int has_min_q8 = 0;
+ int aidx = 0;
+ CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &extra0_q8_0->q)); // flat int8 codes [expert][row][K]
+ CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &extra0_q8_0->scale)); // uniform scale[16]
+ CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &extra0_q8_0->scale)); // dummy min (has_min=0, unread)
+ CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer));
+ CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer));
+ CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer));
+ CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &buf_src2));
+ CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &buf_src2_emap));
+ CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &dp_buf_dst_image));
+ CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &(backend_ctx->prealloc_total_tiles.buffer)));
+ CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(int), &ne00i));
+ CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(int), &ne01i));
+ CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(int), &backend_ctx->adreno_use_moe_ragged_dp4));
+ CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(int), &has_min_q8));
+
+ size_t dp_global[3] = { 64, (size_t)((ne01 + 63) / 64), (size_t)max_post_router_tile };
+ size_t dp_local[3] = { 64, 1, 1 };
+ backend_ctx->enqueue_ndrange_kernel(dk, 3, dp_global, dp_local, dst);
+
+ clReleaseMemObject(sub_buf_src1_pre);
+ clReleaseMemObject(buf_src2);
+ clReleaseMemObject(buf_src2_emap);
+ clReleaseMemObject(dp_sub_buf_dst);
+ clReleaseMemObject(dp_buf_dst_image);
+ return;
+ }
+ }
+
+ region.origin = 0;
+ region.size = ne00 * max_post_router_tile * n_tile_size * sizeof(float);
+ backend_ctx->prealloc_act_trans.allocate(backend_ctx->context, region.size);
+ buf_src1_reordered = clCreateSubBuffer(
+ backend_ctx->prealloc_act_trans.buffer, 0, CL_BUFFER_CREATE_TYPE_REGION, &region, &status);
+ CL_CHECK(status);
+ cl_image_format image_format_buf_src1 = {CL_RGBA, CL_FLOAT};
+ cl_image_desc image_desc_buf_src1 = {CL_MEM_OBJECT_IMAGE1D_BUFFER, static_cast<size_t>(ne00 * max_post_router_tile * n_tile_size / 4), 0,0,0,0,0,0,0, {buf_src1_reordered}};
+ image_src1_reordered = clCreateImage(backend_ctx->context, CL_MEM_READ_ONLY, &image_format_buf_src1, &image_desc_buf_src1, NULL, &status);
+ CL_CHECK(status);
+
+ unsigned short map_ratio = ne20 / ne11;
+ GGML_ASSERT(((map_ratio == 1) || (map_ratio == ne20)) && "Map ratio not supported\n");
+ CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 0, sizeof(cl_mem), &sub_buf_src1_pre));
+ CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 1, sizeof(cl_mem), &buf_src2));
+ CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 2, sizeof(cl_mem), &buf_src1_reordered));
+ CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 3, sizeof(cl_mem), &(backend_ctx->prealloc_total_tiles.buffer)));
+ CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 4, sizeof(unsigned int), &ne00));
+ CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 5, sizeof(unsigned short), &map_ratio));
+ CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 6, sizeof(unsigned int), &n_tile_size));
+
+ size_t reorder_b_local_size[3] = {256, 1, 1};
+ size_t reorder_b_global_size[3] = {static_cast<size_t>(((ne00 / 4) + 255) / 256 * 256), static_cast<size_t>(max_post_router_tile * n_tile_size), 1};
+ backend_ctx->enqueue_ndrange_kernel(backend_ctx->kernel_moe_reorder_b, 3, reorder_b_global_size, reorder_b_local_size, dst);
+
+ // dst image
+ region.origin = offsetd;
+ region.size = ne0 * ne1 * ne2 * sizeof(float);
+ sub_buf_dst = clCreateSubBuffer(extrad->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, &region, &status);
+ CL_CHECK(status);
+ cl_image_format image_format_buf_dst = {CL_R, CL_FLOAT};
+ cl_image_desc image_desc_buf_dst = {CL_MEM_OBJECT_IMAGE1D_BUFFER, static_cast<size_t>(ne0 * ne1 * ne2), 0,0,0,0,0,0,0, {sub_buf_dst}};
+ buf_dst_image = clCreateImage(backend_ctx->context, CL_MEM_WRITE_ONLY, &image_format_buf_dst, &image_desc_buf_dst, NULL, &status);
+ CL_CHECK(status);
+
+ int arg_idx = 0;
+ CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &extra0_q8_0->q)); // flat q8_0 quants
+ CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &extra0_q8_0->d)); // flat q8_0 scales
+ CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &image_src1_reordered));
+ CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &buf_src2));
+ CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &buf_src2_emap));
+ CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &buf_dst_image));
+ CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &(backend_ctx->prealloc_total_tiles.buffer)));
+ CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(int), &ne00));
+ CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(int), &ne01));
+
+ global_size[1] = static_cast<size_t>((ne01 + 63) / 64);
+ global_size[2] = static_cast<size_t>(max_post_router_tile);
+ local_size[1] = 1;
+ local_size[2] = 1;
+
+ backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_size, local_size, dst);
+
+ clReleaseMemObject(sub_buf_src1_pre);
+ clReleaseMemObject(buf_src1_reordered);
+ clReleaseMemObject(image_src1_reordered);
+ clReleaseMemObject(buf_src2);
+ clReleaseMemObject(buf_src2_emap);
+ clReleaseMemObject(sub_buf_dst);
+ clReleaseMemObject(buf_dst_image);
+ return;
+ }
+#endif // GGML_OPENCL_USE_ADRENO_KERNELS
#ifdef GGML_OPENCL_SOA_Q
kernel = backend_ctx->kernel_mul_mv_id_q8_0_f32_flat;
@@ -20347,6 +21732,18 @@ static void ggml_cl_mul_mat_id(ggml_backend_t backend, const ggml_tensor * src0,
if (ne12 == 1) { // for gemv
kernel = backend_ctx->kernel_gemv_moe_q4_k_f32_ns;
+ // Weight-as-texture MoE decode GEMV
+ static const char * moe_decode_wimg_env = getenv("GGML_OPENCL_MOE_DECODE_WIMG");
+ const bool moe_decode_wimg_on = moe_decode_wimg_env
+ ? (atoi(moe_decode_wimg_env) != 0)
+ : (backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E);
+ const bool use_moe_decode_wimg = moe_decode_wimg_on
+ && backend_ctx->kernel_gemv_moe_q4_k_f32_ns_wimg != nullptr
+ && extra0_q4_K->q_img != nullptr;
+ if (use_moe_decode_wimg) {
+ kernel = backend_ctx->kernel_gemv_moe_q4_k_f32_ns_wimg;
+ }
+
cl_mem src1_sub_buffer, buf_src1_image, buf_src2;
// create a sub_buffer for src2
@@ -20376,7 +21773,7 @@ static void ggml_cl_mul_mat_id(ggml_backend_t backend, const ggml_tensor * src0,
// Set kernel args
int arg_idx = 0;
- CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &extra0_q4_K->q));
+ CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), use_moe_decode_wimg ? &extra0_q4_K->q_img : &extra0_q4_K->q));
CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &extra0_q4_K->d));
CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &extra0_q4_K->dm));
CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &extra0_q4_K->s));
@@ -20409,9 +21806,18 @@ static void ggml_cl_mul_mat_id(ggml_backend_t backend, const ggml_tensor * src0,
backend_ctx->toggle_reorder = false;
}
- cl_mem sub_buf_src1_pre, buf_src1_reordered, image_src1_reordered, sub_buf_dst, buf_dst_image;
+ cl_mem sub_buf_src1_pre, sub_buf_dst, buf_dst_image;
+ cl_mem buf_src1_reordered = nullptr, image_src1_reordered = nullptr;
cl_mem buf_src2, buf_src2_emap;
+ // dp4a (int8) prefill GEMM variant
+ static const char * q4k_moe_dp4a_env = getenv("GGML_OPENCL_Q4K_MOE_DP4A");
+ bool use_moe_dp4a = (q4k_moe_dp4a_env != nullptr)
+ ? (atoi(q4k_moe_dp4a_env) != 0)
+ : (backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E || backend_ctx->adreno_gen == ADRENO_GPU_GEN::X1E);
+ // bin kernel takes precedence
+ use_moe_dp4a = use_moe_dp4a && backend_ctx->kernel_gemm_moe_q4_k_f32_ns_bin == nullptr;
+
cl_buffer_region region;
region.origin = 0;
region.size = sizeof(int) * max_post_router_tile * n_tile_size;
@@ -20429,42 +21835,45 @@ static void ggml_cl_mul_mat_id(ggml_backend_t backend, const ggml_tensor * src0,
sub_buf_src1_pre = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, &region, &status);
CL_CHECK(status);
- // Create image for reordered src1
- region.origin = 0;
- region.size = ne00 * max_post_router_tile * n_tile_size * sizeof(float);
- backend_ctx->prealloc_act_trans.allocate(backend_ctx->context, region.size);
- buf_src1_reordered = clCreateSubBuffer(
- backend_ctx->prealloc_act_trans.buffer,
- 0,
- CL_BUFFER_CREATE_TYPE_REGION,
- &region,
- &status);
- CL_CHECK(status);
- cl_image_format image_format_buf_src1 = {CL_RGBA, CL_FLOAT};
- cl_image_desc image_desc_buf_src1 = {CL_MEM_OBJECT_IMAGE1D_BUFFER, static_cast<size_t>(ne00 * max_post_router_tile * n_tile_size / 4), 0,0,0,0,0,0,0, {buf_src1_reordered}};
- if (backend_ctx->kernel_gemm_moe_q4_k_f32_ns_bin) {
- // bin kernel uses slightly different image format
- image_format_buf_src1 = {CL_R, CL_FLOAT};
- image_desc_buf_src1.image_width = static_cast<size_t>(ne00 * max_post_router_tile * n_tile_size);
- }
- image_src1_reordered = clCreateImage(backend_ctx->context, CL_MEM_READ_ONLY, &image_format_buf_src1, &image_desc_buf_src1, NULL, &status);
- CL_CHECK(status);
-
unsigned short map_ratio = ne20 / ne11;
GGML_ASSERT(((map_ratio == 1) || (map_ratio == ne20)) && "Map ratio not supported\n");
- CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 0, sizeof(cl_mem), &sub_buf_src1_pre));
- CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 1, sizeof(cl_mem), &buf_src2));
- CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 2, sizeof(cl_mem), &buf_src1_reordered));
- CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 3, sizeof(cl_mem), &(backend_ctx->prealloc_total_tiles.buffer)));
- CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 4, sizeof(unsigned int), &ne00));
- CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 5, sizeof(unsigned short), &map_ratio));
- CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 6, sizeof(unsigned int), &n_tile_size));
-
- size_t reorder_b_local_size[3] = {256, 1, 1};
- size_t reorder_b_global_size[3] = {static_cast<size_t>(((ne00 / 4) + 255) / 256 * 256), static_cast<size_t>(max_post_router_tile * n_tile_size), 1};
- // Dispatch reorder kernel
- backend_ctx->enqueue_ndrange_kernel(backend_ctx->kernel_moe_reorder_b, 3, reorder_b_global_size, reorder_b_local_size, dst);
+ if (!use_moe_dp4a) {
+ // Create image for reordered src1
+ region.origin = 0;
+ region.size = ne00 * max_post_router_tile * n_tile_size * sizeof(float);
+ backend_ctx->prealloc_act_trans.allocate(backend_ctx->context, region.size);
+ buf_src1_reordered = clCreateSubBuffer(
+ backend_ctx->prealloc_act_trans.buffer,
+ 0,
+ CL_BUFFER_CREATE_TYPE_REGION,
+ &region,
+ &status);
+ CL_CHECK(status);
+ cl_image_format image_format_buf_src1 = {CL_RGBA, CL_FLOAT};
+ cl_image_desc image_desc_buf_src1 = {CL_MEM_OBJECT_IMAGE1D_BUFFER, static_cast<size_t>(ne00 * max_post_router_tile * n_tile_size / 4), 0,0,0,0,0,0,0, {buf_src1_reordered}};
+ if (backend_ctx->kernel_gemm_moe_q4_k_f32_ns_bin) {
+ // bin kernel uses slightly different image format
+ image_format_buf_src1 = {CL_R, CL_FLOAT};
+ image_desc_buf_src1.image_width = static_cast<size_t>(ne00 * max_post_router_tile * n_tile_size);
+ }
+ image_src1_reordered = clCreateImage(backend_ctx->context, CL_MEM_READ_ONLY, &image_format_buf_src1, &image_desc_buf_src1, NULL, &status);
+ CL_CHECK(status);
+
+ CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 0, sizeof(cl_mem), &sub_buf_src1_pre));
+ CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 1, sizeof(cl_mem), &buf_src2));
+ CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 2, sizeof(cl_mem), &buf_src1_reordered));
+ CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 3, sizeof(cl_mem), &(backend_ctx->prealloc_total_tiles.buffer)));
+ CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 4, sizeof(unsigned int), &ne00));
+ CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 5, sizeof(unsigned short), &map_ratio));
+ CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 6, sizeof(unsigned int), &n_tile_size));
+
+ size_t reorder_b_local_size[3] = {256, 1, 1};
+ size_t reorder_b_global_size[3] = {static_cast<size_t>(((ne00 / 4) + 255) / 256 * 256), static_cast<size_t>(max_post_router_tile * n_tile_size), 1};
+
+ // Dispatch reorder kernel
+ backend_ctx->enqueue_ndrange_kernel(backend_ctx->kernel_moe_reorder_b, 3, reorder_b_global_size, reorder_b_local_size, dst);
+ }
// MoE kernel prepare
region.origin = offsetd;
@@ -20482,6 +21891,61 @@ static void ggml_cl_mul_mat_id(ggml_backend_t backend, const ggml_tensor * src0,
buf_dst_image = clCreateImage(backend_ctx->context, CL_MEM_WRITE_ONLY, &image_format_buf_dst, &image_desc_buf_dst, NULL, &status);
CL_CHECK(status);
+ if (use_moe_dp4a) {
+ const size_t tok_slots = (size_t)max_post_router_tile * n_tile_size;
+ const size_t n_blocks = tok_slots * (ne00 / 32);
+ backend_ctx->prealloc_moe_qa.allocate(backend_ctx->context, tok_slots * ne00 * sizeof(cl_char));
+ backend_ctx->prealloc_moe_da.allocate(backend_ctx->context, n_blocks * sizeof(cl_half));
+ backend_ctx->prealloc_moe_sa.allocate(backend_ctx->context, n_blocks * sizeof(cl_half));
+
+ // fused reorder + q8_1 quant straight from the original
+ // activations (no intermediate f32 reorder buffer)
+ const cl_uint n_kblocks = (cl_uint)(ne00 / 32);
+ cl_kernel rq = backend_ctx->kernel_moe_reorder_quant_a_q8_1;
+ CL_CHECK(clSetKernelArg(rq, 0, sizeof(cl_mem), &sub_buf_src1_pre));
+ CL_CHECK(clSetKernelArg(rq, 1, sizeof(cl_mem), &buf_src2));
+ CL_CHECK(clSetKernelArg(rq, 2, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer));
+ CL_CHECK(clSetKernelArg(rq, 3, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer));
+ CL_CHECK(clSetKernelArg(rq, 4, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer));
+ CL_CHECK(clSetKernelArg(rq, 5, sizeof(cl_mem), &(backend_ctx->prealloc_total_tiles.buffer)));
+ CL_CHECK(clSetKernelArg(rq, 6, sizeof(cl_uint), &ne00));
+ CL_CHECK(clSetKernelArg(rq, 7, sizeof(unsigned short), &map_ratio));
+ CL_CHECK(clSetKernelArg(rq, 8, sizeof(cl_uint), &n_tile_size));
+ CL_CHECK(clSetKernelArg(rq, 9, sizeof(cl_uint), &n_kblocks));
+ size_t rq_local[2] = { 32, 1 };
+ size_t rq_global[2] = { (size_t)(((n_kblocks + 31) / 32) * 32), tok_slots };
+ backend_ctx->enqueue_ndrange_kernel(rq, 2, rq_global, rq_local, dst);
+
+ // dp4a GEMM
+ cl_kernel dk = backend_ctx->kernel_gemm_moe_q4_k_q8_1_dp4a;
+ int aidx = 0;
+ CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &extra0_q4_K->q_img));
+ CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &extra0_q4_K->d));
+ CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &extra0_q4_K->dm));
+ CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &extra0_q4_K->s));
+ CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer));
+ CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer));
+ CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer));
+ CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &buf_src2));
+ CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &buf_src2_emap));
+ CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &buf_dst_image));
+ CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &(backend_ctx->prealloc_total_tiles.buffer)));
+ CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(int), &ne00));
+ CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(int), &ne01));
+ CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(int), &backend_ctx->adreno_use_moe_ragged_dp4));
+
+ size_t dp_global[3] = { 64, (size_t)((ne01 + 63) / 64), (size_t)max_post_router_tile };
+ size_t dp_local[3] = { 64, 1, 1 };
+ backend_ctx->enqueue_ndrange_kernel(dk, 3, dp_global, dp_local, dst);
+
+ clReleaseMemObject(sub_buf_src1_pre);
+ clReleaseMemObject(buf_src2);
+ clReleaseMemObject(buf_src2_emap);
+ clReleaseMemObject(sub_buf_dst);
+ clReleaseMemObject(buf_dst_image);
+ return;
+ }
+
// Set kernel args
int arg_idx = 0;
CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &extra0_q4_K->q_img));
@@ -20611,6 +22075,83 @@ static void ggml_cl_mul_mat_id(ggml_backend_t backend, const ggml_tensor * src0,
sub_buf_src1_pre = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, &region, &status);
CL_CHECK(status);
+ // Generic dp4a MoE GEMM
+ {
+ static const char * q5kmdp4a_env = getenv("GGML_OPENCL_Q5K_MOE_DP4A");
+ const bool q5kmdp4a_on = q5kmdp4a_env ? (atoi(q5kmdp4a_env) != 0)
+ : (backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E);
+ bool use_moe_dp4a = q5kmdp4a_on
+ && backend_ctx->kernel_gemm_moe_q8_1_dp4a_q5k != nullptr
+ && extra0_q5_K->scale != nullptr;
+ // bin kernel takes precedence
+ use_moe_dp4a = use_moe_dp4a && backend_ctx->kernel_gemm_moe_q4_k_f32_ns_bin == nullptr;
+
+ if (use_moe_dp4a) {
+ const size_t tok_slots = (size_t)max_post_router_tile * n_tile_size;
+ const size_t n_blocks = tok_slots * (ne00 / 32);
+ backend_ctx->prealloc_moe_qa.allocate(backend_ctx->context, tok_slots * ne00 * sizeof(cl_char));
+ backend_ctx->prealloc_moe_da.allocate(backend_ctx->context, n_blocks * sizeof(cl_half));
+ backend_ctx->prealloc_moe_sa.allocate(backend_ctx->context, n_blocks * sizeof(cl_half));
+
+ const cl_uint n_kblocks = (cl_uint)(ne00 / 32);
+ unsigned short map_ratio_q5k = ne20 / ne11;
+ cl_kernel rq = backend_ctx->kernel_moe_reorder_quant_a_q8_1;
+ CL_CHECK(clSetKernelArg(rq, 0, sizeof(cl_mem), &sub_buf_src1_pre));
+ CL_CHECK(clSetKernelArg(rq, 1, sizeof(cl_mem), &buf_src2));
+ CL_CHECK(clSetKernelArg(rq, 2, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer));
+ CL_CHECK(clSetKernelArg(rq, 3, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer));
+ CL_CHECK(clSetKernelArg(rq, 4, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer));
+ CL_CHECK(clSetKernelArg(rq, 5, sizeof(cl_mem), &(backend_ctx->prealloc_total_tiles.buffer)));
+ CL_CHECK(clSetKernelArg(rq, 6, sizeof(cl_uint), &ne00));
+ CL_CHECK(clSetKernelArg(rq, 7, sizeof(unsigned short), &map_ratio_q5k));
+ CL_CHECK(clSetKernelArg(rq, 8, sizeof(cl_uint), &n_tile_size));
+ CL_CHECK(clSetKernelArg(rq, 9, sizeof(cl_uint), &n_kblocks));
+ size_t rq_local[2] = { 32, 1 };
+ size_t rq_global[2] = { (size_t)(((n_kblocks + 31) / 32) * 32), tok_slots };
+ backend_ctx->enqueue_ndrange_kernel(rq, 2, rq_global, rq_local, dst);
+
+ region.origin = offsetd;
+ region.size = ne0 * ne1 * ne2 * sizeof(float);
+ cl_mem dp_sub_buf_dst = clCreateSubBuffer(extrad->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, &region, &status);
+ CL_CHECK(status);
+ cl_image_format dp_ifd = {CL_R, CL_FLOAT};
+ cl_image_desc dp_idd = {CL_MEM_OBJECT_IMAGE1D_BUFFER, static_cast<size_t>(ne0 * ne1 * ne2), 0,0,0,0,0,0,0, {dp_sub_buf_dst}};
+ cl_mem dp_buf_dst_image = clCreateImage(backend_ctx->context, CL_MEM_WRITE_ONLY, &dp_ifd, &dp_idd, NULL, &status);
+ CL_CHECK(status);
+
+ int ne00i = (int)ne00, ne01i = (int)ne01;
+ cl_kernel dk = backend_ctx->kernel_gemm_moe_q8_1_dp4a_q5k;
+ int has_min_q5k = 1;
+ int aidx = 0;
+ CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &extra0_q5_K->q_img));
+ CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &extra0_q5_K->qh));
+ CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &extra0_q5_K->scale));
+ CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &extra0_q5_K->min));
+ CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer));
+ CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer));
+ CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer));
+ CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &buf_src2));
+ CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &buf_src2_emap));
+ CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &dp_buf_dst_image));
+ CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &(backend_ctx->prealloc_total_tiles.buffer)));
+ CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(int), &ne00i));
+ CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(int), &ne01i));
+ CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(int), &backend_ctx->adreno_use_moe_ragged_dp4));
+ CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(int), &has_min_q5k));
+
+ size_t dp_global[3] = { 64, (size_t)((ne01 + 63) / 64), (size_t)max_post_router_tile };
+ size_t dp_local[3] = { 64, 1, 1 };
+ backend_ctx->enqueue_ndrange_kernel(dk, 3, dp_global, dp_local, dst);
+
+ clReleaseMemObject(sub_buf_src1_pre);
+ clReleaseMemObject(buf_src2);
+ clReleaseMemObject(buf_src2_emap);
+ clReleaseMemObject(dp_sub_buf_dst);
+ clReleaseMemObject(dp_buf_dst_image);
+ return;
+ }
+ }
+
// Create image for reordered src1
// Use pre-allocated placeholder
region.origin = 0;
@@ -20769,9 +22310,17 @@ static void ggml_cl_mul_mat_id(ggml_backend_t backend, const ggml_tensor * src0,
backend_ctx->toggle_reorder = false;
}
- cl_mem sub_buf_src1_pre, buf_src1_reordered, image_src1_reordered, sub_buf_dst, buf_dst_image;
+ cl_mem sub_buf_src1_pre, sub_buf_dst, buf_dst_image;
+ cl_mem buf_src1_reordered = nullptr, image_src1_reordered = nullptr;
cl_mem buf_src2, buf_src2_emap;
+ // dp4a (int8) q6_K MoE prefill GEMM
+ static const char * q6k_moe_dp4a_env = getenv("GGML_OPENCL_Q6K_MOE_DP4A");
+ static const bool use_moe_dp4a = (q6k_moe_dp4a_env != nullptr)
+ ? (atoi(q6k_moe_dp4a_env) != 0)
+ : (backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E
+ || backend_ctx->adreno_gen == ADRENO_GPU_GEN::X1E);
+
cl_buffer_region region;
region.origin = 0;
region.size = sizeof(int) * max_post_router_tile * n_tile_size;
@@ -20790,37 +22339,40 @@ static void ggml_cl_mul_mat_id(ggml_backend_t backend, const ggml_tensor * src0,
sub_buf_src1_pre = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, &region, &status);
CL_CHECK(status);
- // Create image for reordered src1
- region.origin = 0;
- region.size = ne00 * max_post_router_tile * n_tile_size * sizeof(float);
- backend_ctx->prealloc_act_trans.allocate(backend_ctx->context, region.size);
- buf_src1_reordered = clCreateSubBuffer(
- backend_ctx->prealloc_act_trans.buffer,
- 0,
- CL_BUFFER_CREATE_TYPE_REGION,
- &region,
- &status);
- CL_CHECK(status);
- cl_image_format image_format_buf_src1 = {CL_RGBA, CL_FLOAT};
- cl_image_desc image_desc_buf_src1 = {CL_MEM_OBJECT_IMAGE1D_BUFFER, static_cast<size_t>(ne00 * max_post_router_tile * n_tile_size / 4), 0,0,0,0,0,0,0, {buf_src1_reordered}};
- image_src1_reordered = clCreateImage(backend_ctx->context, CL_MEM_READ_ONLY, &image_format_buf_src1, &image_desc_buf_src1, NULL, &status);
- CL_CHECK(status);
-
unsigned short map_ratio = ne20 / ne11;
GGML_ASSERT(((map_ratio == 1) || (map_ratio == ne20)) && "Map ratio not supported\n");
- CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 0, sizeof(cl_mem), &sub_buf_src1_pre));
- CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 1, sizeof(cl_mem), &buf_src2));
- CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 2, sizeof(cl_mem), &buf_src1_reordered));
- CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 3, sizeof(cl_mem), &(backend_ctx->prealloc_total_tiles.buffer)));
- CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 4, sizeof(unsigned int), &ne00));
- CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 5, sizeof(unsigned short), &map_ratio));
- CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 6, sizeof(unsigned int), &n_tile_size));
- size_t reorder_b_local_size[3] = {256, 1, 1};
- size_t reorder_b_global_size[3] = {static_cast<size_t>(((ne00 / 4) + 255) / 256 * 256), static_cast<size_t>(max_post_router_tile * n_tile_size), 1};
-
- // Dispatch reorder kernel
- backend_ctx->enqueue_ndrange_kernel(backend_ctx->kernel_moe_reorder_b, 3, reorder_b_global_size, reorder_b_local_size, dst);
+ if (!use_moe_dp4a) {
+ // Create image for reordered src1
+ region.origin = 0;
+ region.size = ne00 * max_post_router_tile * n_tile_size * sizeof(float);
+ backend_ctx->prealloc_act_trans.allocate(backend_ctx->context, region.size);
+ buf_src1_reordered = clCreateSubBuffer(
+ backend_ctx->prealloc_act_trans.buffer,
+ 0,
+ CL_BUFFER_CREATE_TYPE_REGION,
+ &region,
+ &status);
+ CL_CHECK(status);
+ cl_image_format image_format_buf_src1 = {CL_RGBA, CL_FLOAT};
+ cl_image_desc image_desc_buf_src1 = {CL_MEM_OBJECT_IMAGE1D_BUFFER, static_cast<size_t>(ne00 * max_post_router_tile * n_tile_size / 4), 0,0,0,0,0,0,0, {buf_src1_reordered}};
+ image_src1_reordered = clCreateImage(backend_ctx->context, CL_MEM_READ_ONLY, &image_format_buf_src1, &image_desc_buf_src1, NULL, &status);
+ CL_CHECK(status);
+
+ CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 0, sizeof(cl_mem), &sub_buf_src1_pre));
+ CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 1, sizeof(cl_mem), &buf_src2));
+ CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 2, sizeof(cl_mem), &buf_src1_reordered));
+ CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 3, sizeof(cl_mem), &(backend_ctx->prealloc_total_tiles.buffer)));
+ CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 4, sizeof(unsigned int), &ne00));
+ CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 5, sizeof(unsigned short), &map_ratio));
+ CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 6, sizeof(unsigned int), &n_tile_size));
+
+ size_t reorder_b_local_size[3] = {256, 1, 1};
+ size_t reorder_b_global_size[3] = {static_cast<size_t>(((ne00 / 4) + 255) / 256 * 256), static_cast<size_t>(max_post_router_tile * n_tile_size), 1};
+
+ // Dispatch reorder kernel
+ backend_ctx->enqueue_ndrange_kernel(backend_ctx->kernel_moe_reorder_b, 3, reorder_b_global_size, reorder_b_local_size, dst);
+ }
// MoE kernel prepare
// Create sub buffer for dst
@@ -20839,6 +22391,58 @@ static void ggml_cl_mul_mat_id(ggml_backend_t backend, const ggml_tensor * src0,
buf_dst_image = clCreateImage(backend_ctx->context, CL_MEM_WRITE_ONLY, &image_format_buf_dst, &image_desc_buf_dst, NULL, &status);
CL_CHECK(status);
+ if (use_moe_dp4a) {
+ const size_t tok_slots = (size_t)max_post_router_tile * n_tile_size;
+ const size_t n_blocks = tok_slots * (ne00 / 32);
+ backend_ctx->prealloc_moe_qa.allocate(backend_ctx->context, tok_slots * ne00 * sizeof(cl_char));
+ backend_ctx->prealloc_moe_da.allocate(backend_ctx->context, n_blocks * sizeof(cl_half));
+ backend_ctx->prealloc_moe_sa.allocate(backend_ctx->context, n_blocks * sizeof(cl_half));
+
+ // fused reorder + q8_1 quant from the original activations
+ const cl_uint n_kblocks = (cl_uint)(ne00 / 32);
+ cl_kernel rq = backend_ctx->kernel_moe_reorder_quant_a_q8_1;
+ CL_CHECK(clSetKernelArg(rq, 0, sizeof(cl_mem), &sub_buf_src1_pre));
+ CL_CHECK(clSetKernelArg(rq, 1, sizeof(cl_mem), &buf_src2));
+ CL_CHECK(clSetKernelArg(rq, 2, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer));
+ CL_CHECK(clSetKernelArg(rq, 3, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer));
+ CL_CHECK(clSetKernelArg(rq, 4, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer));
+ CL_CHECK(clSetKernelArg(rq, 5, sizeof(cl_mem), &(backend_ctx->prealloc_total_tiles.buffer)));
+ CL_CHECK(clSetKernelArg(rq, 6, sizeof(cl_uint), &ne00));
+ CL_CHECK(clSetKernelArg(rq, 7, sizeof(unsigned short), &map_ratio));
+ CL_CHECK(clSetKernelArg(rq, 8, sizeof(cl_uint), &n_tile_size));
+ CL_CHECK(clSetKernelArg(rq, 9, sizeof(cl_uint), &n_kblocks));
+ size_t rq_local[2] = { 32, 1 };
+ size_t rq_global[2] = { (size_t)(((n_kblocks + 31) / 32) * 32), tok_slots };
+ backend_ctx->enqueue_ndrange_kernel(rq, 2, rq_global, rq_local, dst);
+
+ cl_kernel dk = backend_ctx->kernel_gemm_moe_q6_k_q8_1_dp4a;
+ int qi = 0;
+ CL_CHECK(clSetKernelArg(dk, qi++, sizeof(cl_mem), &extra0_q6_K->ql_img));
+ CL_CHECK(clSetKernelArg(dk, qi++, sizeof(cl_mem), &extra0_q6_K->qh));
+ CL_CHECK(clSetKernelArg(dk, qi++, sizeof(cl_mem), &extra0_q6_K->s));
+ CL_CHECK(clSetKernelArg(dk, qi++, sizeof(cl_mem), &extra0_q6_K->d));
+ CL_CHECK(clSetKernelArg(dk, qi++, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer));
+ CL_CHECK(clSetKernelArg(dk, qi++, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer));
+ CL_CHECK(clSetKernelArg(dk, qi++, sizeof(cl_mem), &buf_src2));
+ CL_CHECK(clSetKernelArg(dk, qi++, sizeof(cl_mem), &buf_src2_emap));
+ CL_CHECK(clSetKernelArg(dk, qi++, sizeof(cl_mem), &buf_dst_image));
+ CL_CHECK(clSetKernelArg(dk, qi++, sizeof(cl_mem), &(backend_ctx->prealloc_total_tiles.buffer)));
+ CL_CHECK(clSetKernelArg(dk, qi++, sizeof(int), &ne00));
+ CL_CHECK(clSetKernelArg(dk, qi++, sizeof(int), &ne01));
+ CL_CHECK(clSetKernelArg(dk, qi++, sizeof(int), &backend_ctx->adreno_use_moe_ragged_dp4));
+
+ size_t dp_global[3] = { 64, (size_t)((ne01 + 63) / 64), (size_t)max_post_router_tile };
+ size_t dp_local[3] = { 64, 1, 1 };
+ backend_ctx->enqueue_ndrange_kernel(dk, 3, dp_global, dp_local, dst);
+
+ clReleaseMemObject(sub_buf_src1_pre);
+ clReleaseMemObject(buf_src2);
+ clReleaseMemObject(buf_src2_emap);
+ clReleaseMemObject(sub_buf_dst);
+ clReleaseMemObject(buf_dst_image);
+ return;
+ }
+
// Set kernel args
int arg_idx = 0;
CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &extra0_q6_K->ql_img));
@@ -20887,6 +22491,15 @@ static void ggml_cl_mul_mat_id(ggml_backend_t backend, const ggml_tensor * src0,
if (ne12 == 1) { // for gemv
kernel = backend_ctx->kernel_gemv_moe_mxfp4_f32_ns;
+ // Weight-as-texture MoE decode GEMV (see q4_K _wimg)
+ static const char * moe_decode_wimg_env = getenv("GGML_OPENCL_MOE_DECODE_WIMG");
+ const bool use_moe_decode_wimg = (moe_decode_wimg_env && (atoi(moe_decode_wimg_env) != 0))
+ && backend_ctx->kernel_gemv_moe_mxfp4_f32_ns_wimg != nullptr
+ && extra0_mxfp4->q_img != nullptr;
+ if (use_moe_decode_wimg) {
+ kernel = backend_ctx->kernel_gemv_moe_mxfp4_f32_ns_wimg;
+ }
+
cl_mem src1_sub_buffer, buf_src1_image, buf_src2;
// create a sub_buffer for src2
@@ -20916,7 +22529,7 @@ static void ggml_cl_mul_mat_id(ggml_backend_t backend, const ggml_tensor * src0,
// Set kernel args
int arg_idx = 0;
- CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &extra0_mxfp4->q));
+ CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), use_moe_decode_wimg ? &extra0_mxfp4->q_img : &extra0_mxfp4->q));
CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &extra0_mxfp4->e));
CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &buf_src1_image));
CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &buf_src2));
@@ -20947,9 +22560,18 @@ static void ggml_cl_mul_mat_id(ggml_backend_t backend, const ggml_tensor * src0,
backend_ctx->toggle_reorder = false;
}
- cl_mem sub_buf_src1_pre, buf_src1_reordered, image_src1_reordered, sub_buf_dst, buf_dst_image;
+ cl_mem sub_buf_src1_pre, sub_buf_dst, buf_dst_image;
+ cl_mem buf_src1_reordered = nullptr, image_src1_reordered = nullptr;
cl_mem buf_src2, buf_src2_emap;
+ // dp4a (int8) prefill GEMM variant
+ static const char * mxfp4_moe_dp4a_env = getenv("GGML_OPENCL_MXFP4_MOE_DP4A");
+ bool use_moe_dp4a = mxfp4_moe_dp4a_env
+ ? (atoi(mxfp4_moe_dp4a_env) != 0)
+ : (backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E);
+ // bin kernel takes precedence
+ use_moe_dp4a = use_moe_dp4a && backend_ctx->kernel_gemm_moe_mxfp4_f32_ns_bin == nullptr;
+
cl_buffer_region region;
region.origin = 0;
region.size = sizeof(int) * max_post_router_tile * n_tile_size;
@@ -20969,45 +22591,48 @@ static void ggml_cl_mul_mat_id(ggml_backend_t backend, const ggml_tensor * src0,
sub_buf_src1_pre = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, &region, &status);
CL_CHECK(status);
- // Create image for reordered src1
- // Use pre-allocated placeholder
- region.origin = 0;
- region.size = ne00 * max_post_router_tile * n_tile_size * sizeof(float);
- backend_ctx->prealloc_act_trans.allocate(backend_ctx->context, region.size);
- buf_src1_reordered = clCreateSubBuffer(
- backend_ctx->prealloc_act_trans.buffer,
- 0,
- CL_BUFFER_CREATE_TYPE_REGION,
- &region,
- &status);
- CL_CHECK(status);
- cl_image_format image_format_buf_src1;
- cl_image_desc image_desc_buf_src1;
- image_format_buf_src1 = {CL_RGBA, CL_FLOAT};
- image_desc_buf_src1 = {CL_MEM_OBJECT_IMAGE1D_BUFFER, static_cast<size_t>(ne00 * max_post_router_tile * n_tile_size / 4), 0,0,0,0,0,0,0, {buf_src1_reordered}};
- if (backend_ctx->kernel_gemm_moe_mxfp4_f32_ns_bin) {
- // bin kernel uses slightly different image format
- image_format_buf_src1 = {CL_R, CL_FLOAT};
- image_desc_buf_src1.image_width = static_cast<size_t>(ne00 * max_post_router_tile * n_tile_size);
- }
- image_src1_reordered = clCreateImage(backend_ctx->context, CL_MEM_READ_ONLY, &image_format_buf_src1, &image_desc_buf_src1, NULL, &status);
- CL_CHECK(status);
-
unsigned short map_ratio = ne20 / ne11;
GGML_ASSERT(((map_ratio == 1) || (map_ratio == ne20)) && "Map ratio not supported\n");
- CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 0, sizeof(cl_mem), &sub_buf_src1_pre));
- CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 1, sizeof(cl_mem), &buf_src2));
- CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 2, sizeof(cl_mem), &buf_src1_reordered));
- CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 3, sizeof(cl_mem), &(backend_ctx->prealloc_total_tiles.buffer)));
- CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 4, sizeof(unsigned int), &ne00));
- CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 5, sizeof(unsigned short), &map_ratio));
- CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 6, sizeof(unsigned int), &n_tile_size));
-
- size_t reorder_b_local_size[3] = {256, 1, 1};
- size_t reorder_b_global_size[3] = {static_cast<size_t>(((ne00 / 4) + 255) / 256 * 256), static_cast<size_t>(max_post_router_tile * n_tile_size), 1};
- // Dispatch reorder kernel
- backend_ctx->enqueue_ndrange_kernel(backend_ctx->kernel_moe_reorder_b, 3, reorder_b_global_size, reorder_b_local_size, dst);
+ if (!use_moe_dp4a) {
+ // Create image for reordered src1
+ // Use pre-allocated placeholder
+ region.origin = 0;
+ region.size = ne00 * max_post_router_tile * n_tile_size * sizeof(float);
+ backend_ctx->prealloc_act_trans.allocate(backend_ctx->context, region.size);
+ buf_src1_reordered = clCreateSubBuffer(
+ backend_ctx->prealloc_act_trans.buffer,
+ 0,
+ CL_BUFFER_CREATE_TYPE_REGION,
+ &region,
+ &status);
+ CL_CHECK(status);
+ cl_image_format image_format_buf_src1;
+ cl_image_desc image_desc_buf_src1;
+ image_format_buf_src1 = {CL_RGBA, CL_FLOAT};
+ image_desc_buf_src1 = {CL_MEM_OBJECT_IMAGE1D_BUFFER, static_cast<size_t>(ne00 * max_post_router_tile * n_tile_size / 4), 0,0,0,0,0,0,0, {buf_src1_reordered}};
+ if (backend_ctx->kernel_gemm_moe_mxfp4_f32_ns_bin) {
+ // bin kernel uses slightly different image format
+ image_format_buf_src1 = {CL_R, CL_FLOAT};
+ image_desc_buf_src1.image_width = static_cast<size_t>(ne00 * max_post_router_tile * n_tile_size);
+ }
+ image_src1_reordered = clCreateImage(backend_ctx->context, CL_MEM_READ_ONLY, &image_format_buf_src1, &image_desc_buf_src1, NULL, &status);
+ CL_CHECK(status);
+
+ CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 0, sizeof(cl_mem), &sub_buf_src1_pre));
+ CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 1, sizeof(cl_mem), &buf_src2));
+ CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 2, sizeof(cl_mem), &buf_src1_reordered));
+ CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 3, sizeof(cl_mem), &(backend_ctx->prealloc_total_tiles.buffer)));
+ CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 4, sizeof(unsigned int), &ne00));
+ CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 5, sizeof(unsigned short), &map_ratio));
+ CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 6, sizeof(unsigned int), &n_tile_size));
+
+ size_t reorder_b_local_size[3] = {256, 1, 1};
+ size_t reorder_b_global_size[3] = {static_cast<size_t>(((ne00 / 4) + 255) / 256 * 256), static_cast<size_t>(max_post_router_tile * n_tile_size), 1};
+
+ // Dispatch reorder kernel
+ backend_ctx->enqueue_ndrange_kernel(backend_ctx->kernel_moe_reorder_b, 3, reorder_b_global_size, reorder_b_local_size, dst);
+ }
// MoE kernel prepare
// Create sub buffer for dst
@@ -21026,6 +22651,59 @@ static void ggml_cl_mul_mat_id(ggml_backend_t backend, const ggml_tensor * src0,
buf_dst_image = clCreateImage(backend_ctx->context, CL_MEM_WRITE_ONLY, &image_format_buf_dst, &image_desc_buf_dst, NULL, &status);
CL_CHECK(status);
+ if (use_moe_dp4a) {
+ const size_t tok_slots = (size_t)max_post_router_tile * n_tile_size;
+ const size_t n_blocks = tok_slots * (ne00 / 32);
+ backend_ctx->prealloc_moe_qa.allocate(backend_ctx->context, tok_slots * ne00 * sizeof(cl_char));
+ backend_ctx->prealloc_moe_da.allocate(backend_ctx->context, n_blocks * sizeof(cl_half));
+ backend_ctx->prealloc_moe_sa.allocate(backend_ctx->context, n_blocks * sizeof(cl_half));
+
+ // fused reorder + q8_1 quant straight from the original
+ // activations (no intermediate f32 reorder buffer). mxfp4 has no
+ // min term so the GEMM ignores sa, but reorder_quant still writes it.
+ const cl_uint n_kblocks = (cl_uint)(ne00 / 32);
+ cl_kernel rq = backend_ctx->kernel_moe_reorder_quant_a_q8_1;
+ CL_CHECK(clSetKernelArg(rq, 0, sizeof(cl_mem), &sub_buf_src1_pre));
+ CL_CHECK(clSetKernelArg(rq, 1, sizeof(cl_mem), &buf_src2));
+ CL_CHECK(clSetKernelArg(rq, 2, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer));
+ CL_CHECK(clSetKernelArg(rq, 3, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer));
+ CL_CHECK(clSetKernelArg(rq, 4, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer));
+ CL_CHECK(clSetKernelArg(rq, 5, sizeof(cl_mem), &(backend_ctx->prealloc_total_tiles.buffer)));
+ CL_CHECK(clSetKernelArg(rq, 6, sizeof(cl_uint), &ne00));
+ CL_CHECK(clSetKernelArg(rq, 7, sizeof(unsigned short), &map_ratio));
+ CL_CHECK(clSetKernelArg(rq, 8, sizeof(cl_uint), &n_tile_size));
+ CL_CHECK(clSetKernelArg(rq, 9, sizeof(cl_uint), &n_kblocks));
+ size_t rq_local[2] = { 32, 1 };
+ size_t rq_global[2] = { (size_t)(((n_kblocks + 31) / 32) * 32), tok_slots };
+ backend_ctx->enqueue_ndrange_kernel(rq, 2, rq_global, rq_local, dst);
+
+ // dp4a GEMM
+ cl_kernel dk = backend_ctx->kernel_gemm_moe_mxfp4_q8_1_dp4a;
+ int aidx = 0;
+ CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &extra0_mxfp4->q_img));
+ CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &extra0_mxfp4->e));
+ CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer));
+ CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer));
+ CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &buf_src2));
+ CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &buf_src2_emap));
+ CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &buf_dst_image));
+ CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &(backend_ctx->prealloc_total_tiles.buffer)));
+ CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(int), &ne00));
+ CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(int), &ne01));
+ CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(int), &backend_ctx->adreno_use_moe_ragged_dp4));
+
+ size_t dp_global[3] = { 64, (size_t)((ne01 + 63) / 64), (size_t)max_post_router_tile };
+ size_t dp_local[3] = { 64, 1, 1 };
+ backend_ctx->enqueue_ndrange_kernel(dk, 3, dp_global, dp_local, dst);
+
+ clReleaseMemObject(sub_buf_src1_pre);
+ clReleaseMemObject(buf_src2);
+ clReleaseMemObject(buf_src2_emap);
+ clReleaseMemObject(sub_buf_dst);
+ clReleaseMemObject(buf_dst_image);
+ return;
+ }
+
// Set kernel args
int arg_idx = 0;
CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &extra0_mxfp4->q_img));
diff --git a/ggml/src/ggml-opencl/kernels/cvt.cl b/ggml/src/ggml-opencl/kernels/cvt.cl
index bf0335a698..3d6cff7cff 100644
--- a/ggml/src/ggml-opencl/kernels/cvt.cl
+++ b/ggml/src/ggml-opencl/kernels/cvt.cl
@@ -2372,3 +2372,121 @@ kernel void kernel_restore_block_iq4_nl_noshuffle(
b->qs[2*i + 1] = convert_uchar(((x0 & mask_F0) >> 4) | (x1 & mask_F0));
}
}
+
+// ---------------------------------------------------------------------------
+// kernel_moe_expand_scale_q8_0
+//
+// Expand the q8_0 per-32-block scale d (one half/block, [expert][row][block]) into
+// the UNIFORM scale[16] format the generic dp4a MoE GEMM (kernel_gemm_moe_q8_1_dp4a,
+// MOE_QT=80) consumes: 16 f16 per 256-superblock (per-16-element segment), where the
+// two segments of each 32-block share the block's d. q8_0 is symmetric -> no min
+// buffer (the GEMM runs with has_min=0). The int8 weight codes are reused verbatim
+// from the existing flat q8_0 weight buffer (extra0_q8_0->q), so only the scale is
+// rebuilt here. One work-item per (row, superblock, expert).
+// ---------------------------------------------------------------------------
+kernel void kernel_moe_expand_scale_q8_0(
+ global const half * src_d, // [expert][row][block], one scale per 32-block
+ global half * dst_scale, // [expert][row][block][2] (FLAT per-32-block)
+ int ne00,
+ int ne01
+) {
+ int row = get_global_id(0);
+ int blk = get_global_id(1); // 32-block index along K
+ int e = get_global_id(2);
+ if (row >= ne01) { return; }
+
+ long nb = ne00 / 32; // 32-blocks per row (K only needs % 32 == 0)
+ half d = src_d[((long)e*ne01 + row)*nb + blk];
+ long b = (((long)e*ne01 + row)*nb + blk) * 2;
+ dst_scale[b + 0] = d;
+ dst_scale[b + 1] = d;
+}
+
+// ---------------------------------------------------------------------------
+// kernel_moe_expand_scale_q5_0
+//
+// q5_0 = symmetric, value = d*(code-16), code = nibble | (hi<<4) in 0..31. The
+// generic dp4a MoE GEMM keeps the unsigned code and centers via the min term:
+// scale*dp4a(code,a) - min*sum(a), scale = d, min = d*16.
+// Reads the existing q5_0 d ([expert][block][row], one half/32-block, from the
+// trans4 convert) and writes the FLAT per-32-block uniform scale[2]/min[1] in
+// [expert][row][block] order (a transpose). One work-item per (row, block, expert).
+// ---------------------------------------------------------------------------
+kernel void kernel_moe_expand_scale_q5_0(
+ global const half * src_d, // [expert][block][row]
+ global half * dst_scale, // [expert][row][block][2]
+ global half * dst_min, // [expert][row][block]
+ int ne00,
+ int ne01
+) {
+ int row = get_global_id(0);
+ int blk = get_global_id(1);
+ int e = get_global_id(2);
+ if (row >= ne01) { return; }
+
+ long nb = ne00 / 32;
+ half d = src_d[(long)e*nb*ne01 + (long)blk*ne01 + row]; // [expert][block][row]
+ long sb = (((long)e*ne01 + row)*nb + blk) * 2;
+ long mb = ((long)e*ne01 + row)*nb + blk;
+ dst_scale[sb + 0] = d;
+ dst_scale[sb + 1] = d;
+ dst_min[mb] = (half)((float)d * 16.0f);
+}
+
+// ---------------------------------------------------------------------------
+// kernel_moe_expand_scale_q5_K
+//
+// q5_K value = d*sv*code + (-dm*mn), with the 6-bit packed per-sub-block scale sv
+// and min mn (8 sub-blocks of 32 per 256-superblock, decoded by get_scale_min_k4
+// from the 12-byte s[]). The generic dp4a MoE GEMM (kernel_gemm_moe_q8_1_dp4a,
+// MOE_QT=5) keeps the unsigned 5-bit code and applies scale/min via the uniform
+// per-32-block buffers:
+// acc += sc0*a_d*raw1 + sc1*a_d*raw2 - mn_u*a_s,
+// sc0 = sc1 = d*sv (both per-16 segments of a 32-block share the sub-block scale),
+// mn_u = dm*mn (positive; the GEMM subtracts it -> the -dm*mn min term).
+// q5_K's q_img (low nibbles) + qh (hi-bit plane) are already in the layout the GEMM
+// reads (same trans4_ns convert that feeds gemm_moe_q5_k_f32_ns), so only the scale
+// is rebuilt here.
+//
+// One work-item per (row, superblock, expert); each emits 8 sub-blocks.
+// ---------------------------------------------------------------------------
+kernel void kernel_moe_expand_scale_q5_K(
+ global const uchar * src_s, // [expert][row][superblock][12]
+ global const half * src_d, // [expert][superblock][row]
+ global const half * src_dm, // [expert][superblock][row]
+ global half * dst_scale, // [expert][row][32block][2]
+ global half * dst_min, // [expert][row][32block]
+ int ne00,
+ int ne01
+) {
+ int row = get_global_id(0);
+ int sb = get_global_id(1); // superblock index along K
+ int e = get_global_id(2);
+ if (row >= ne01) { return; }
+
+ long nsb = ne00 / 256; // superblocks per row
+ long nblk32 = ne00 / 32; // 32-blocks per row
+
+ float d = (float)src_d [((long)e*nsb + sb)*ne01 + row];
+ float dm = (float)src_dm[((long)e*nsb + sb)*ne01 + row];
+
+ __global const uchar * sc = src_s + ((long)e*ne01 + row)*nsb*12 + (long)sb*12;
+
+ for (int j = 0; j < 8; ++j) {
+ uchar sv, mn;
+ // get_scale_min_k4 (6-bit packed scale/min for sub-block j of 8)
+ if (j < 4) {
+ sv = sc[j] & 63;
+ mn = sc[j+4] & 63;
+ } else {
+ sv = (sc[j+4] & 0x0F) | ((sc[j-4] & 0xC0) >> 2);
+ mn = ((sc[j+4] >> 4) & 0x0F) | ((sc[j] & 0xC0) >> 2);
+ }
+ long sub = (long)sb*8 + j;
+ long sbase = (((long)e*ne01 + row)*nblk32 + sub) * 2;
+ half s_val = (half)(d * (float)sv);
+ dst_scale[sbase + 0] = s_val;
+ dst_scale[sbase + 1] = s_val;
+ dst_min[((long)e*ne01 + row)*nblk32 + sub] = (half)(dm * (float)mn);
+ }
+}
diff --git a/ggml/src/ggml-opencl/kernels/gemm_moe_mxfp4_q8_1_dp4a.cl b/ggml/src/ggml-opencl/kernels/gemm_moe_mxfp4_q8_1_dp4a.cl
new file mode 100644
index 0000000000..95d0638134
--- /dev/null
+++ b/ggml/src/ggml-opencl/kernels/gemm_moe_mxfp4_q8_1_dp4a.cl
@@ -0,0 +1,186 @@
+#pragma OPENCL EXTENSION cl_khr_fp16 : enable
+#pragma OPENCL EXTENSION cl_khr_subgroups : enable
+#ifdef cl_khr_integer_dot_product
+#pragma OPENCL EXTENSION cl_khr_integer_dot_product : enable
+#endif
+
+#define TILESIZE_M 64
+#define TILESIZE_N 32
+
+// 2*mxfp4_value as signed int8, packed 4 codes per uint. Divergent nibble
+// lookups read a __constant *uint* array + shift, never a byte array
+// (byte-indexed __constant loads serialize on Adreno and are far slower).
+// idx 0-3: 0, 1, 2, 3 = 0x03020100
+// idx 4-7: 4, 6, 8, 12 = 0x0C080604
+// idx 8-11: 0, -1, -2, -3 = 0xFDFEFF00 (-1=0xFF,-2=0xFE,-3=0xFD)
+// idx 12-15:-4, -6, -8,-12 = 0xF4F8FAFC (-4=0xFC,-6=0xFA,-8=0xF8,-12=0xF4)
+__constant uint mxfp4_i8x4[4] = {
+ 0x03020100u, 0x0C080604u, 0xFDFEFF00u, 0xF4F8FAFCu
+};
+inline uint mxfp4_code(uint n) {
+ return (mxfp4_i8x4[n >> 2] >> ((n & 3u) * 8u)) & 0xFFu;
+}
+// 4 nibbles in the low 16 bits of u -> 4 codebook int8, packed for dp4a.
+inline uint mxfp4_pack(ushort u) {
+ return mxfp4_code((uint)( u & 0xF))
+ | (mxfp4_code((uint)((u >> 4) & 0xF)) << 8)
+ | (mxfp4_code((uint)((u >> 8) & 0xF)) << 16)
+ | (mxfp4_code((uint)((u >> 12) & 0xF)) << 24);
+}
+
+static inline float e8m0_to_fp32(uchar x) {
+ int bits;
+ bits = (x == 0) ? 0x00400000 : ((uint) x << 23);
+ return as_float(bits);
+}
+
+// One token's dp4a dot (8 uints = 32 K elems) + mxfp4 block-scale epilogue.
+// blk_scale already carries the 0.5 factor (== 0.5 * 2^e).
+#define MOE_MXFP4_DP4A_T(t) do { \
+ int raw = 0; \
+ raw = dot_acc_sat_4x8packed_ss_int(qw[0], sh_qa[t][0], raw); \
+ raw = dot_acc_sat_4x8packed_ss_int(qw[1], sh_qa[t][1], raw); \
+ raw = dot_acc_sat_4x8packed_ss_int(qw[2], sh_qa[t][2], raw); \
+ raw = dot_acc_sat_4x8packed_ss_int(qw[3], sh_qa[t][3], raw); \
+ raw = dot_acc_sat_4x8packed_ss_int(qw[4], sh_qa[t][4], raw); \
+ raw = dot_acc_sat_4x8packed_ss_int(qw[5], sh_qa[t][5], raw); \
+ raw = dot_acc_sat_4x8packed_ss_int(qw[6], sh_qa[t][6], raw); \
+ raw = dot_acc_sat_4x8packed_ss_int(qw[7], sh_qa[t][7], raw); \
+ acc[t] += blk_scale * (float)sh_d[t] * (float)raw; \
+ } while (0)
+
+__attribute__((qcom_wave_pair_mode(1)))
+kernel void kernel_gemm_moe_mxfp4_q8_1_dp4a(
+ __read_only image1d_buffer_t src0_q, // mxfp4 codes (transposed, packed nibbles)
+ __global uchar * src0_e, // e8m0 per-32-block scale
+ __global uint * src1_qa, // q8_1 activations: int8 quants (as uint, 4/elem)
+ __global half * src1_da, // q8_1 per-block scale [tok_slot * ne00/32]
+ __global uint * src2, // post-router (orig out positions)
+ __global ushort * src2_emap, // tile -> expert id
+ __write_only image1d_buffer_t dst,
+ __global int * total_tiles,
+ uint ne00,
+ uint ne01,
+ int is_ragged // 1: compute only real tokens per tile
+) {
+ const uint block_id_m = get_global_id(1); // m_tile
+ const uint block_id_n = get_global_id(2); // n_tile
+
+ if (block_id_n >= total_tiles[0]) {
+ return;
+ }
+
+ const uint lid = get_local_id(0); // 0..63, == this WI's output row in the M-tile
+
+ const ushort expert_id = src2_emap[block_id_n];
+ const uint row = block_id_m * TILESIZE_M;
+ const uint col = block_id_n * TILESIZE_N;
+
+ const uint num_blocks = ne00 >> 5; // blocks-of-32 per token
+ const uint row_idx = row + lid;
+
+ const uint ne00_u = ne00 >> 2; // ne00 in uint (int8x4) units
+
+ __local uint sh_qa[TILESIZE_N][8]; // 32 tokens x 8 uints (32 int8) = 1 KiB
+ __local half sh_d[TILESIZE_N];
+
+ // Real token count for this tile.
+ // Real tokens are packed contiguously at the tile start; padded slots hold
+ // 0xFFFFFFFF (only the last tile of each expert is partial). is_ragged skips
+ // the dp4a/staging/scatter for padded slots; is_ragged==0 forces n_real=32.
+ __local uint sh_src2[TILESIZE_N];
+ __local int sh_nreal;
+ if (lid < TILESIZE_N) {
+ sh_src2[lid] = src2[col + lid];
+ }
+ barrier(CLK_LOCAL_MEM_FENCE);
+ if (lid == 0) {
+ int nr = TILESIZE_N;
+ if (is_ragged) {
+ nr = 0;
+ #pragma unroll
+ for (int t = 0; t < TILESIZE_N; ++t) {
+ if (sh_src2[t] != 0xFFFFFFFFu) ++nr;
+ }
+ }
+ sh_nreal = nr;
+ }
+ barrier(CLK_LOCAL_MEM_FENCE);
+ const int n_real = sh_nreal;
+
+ float acc[TILESIZE_N];
+ #pragma unroll
+ for (int t = 0; t < TILESIZE_N; ++t) acc[t] = 0.0f;
+
+ for (uint step = 0; step < ne00; step += 32) {
+ const uint sub = step >> 5; // 32-block index along K
+
+ // e8m0 block scale for this WI's row, this 32-block (folded x0.5)
+ const uint e_offset = row_idx + sub * ne01 + expert_id * num_blocks * ne01;
+ const float blk_scale = 0.5f * e8m0_to_fp32(src0_e[e_offset]);
+
+ // repack this WI's 32 weight nibbles into 8 dp4a uints
+ const uint qoff0 = row + ((ne01 * step) >> 3) + ((expert_id * ne00 * ne01) >> 3);
+ const uint qoff1 = row + ((ne01 * (step + 16)) >> 3) + ((expert_id * ne00 * ne01) >> 3);
+ const uint r0 = read_imageui(src0_q, qoff0 + lid).x;
+ const uint r1 = read_imageui(src0_q, qoff0 + lid + ne01).x;
+ const uint r2 = read_imageui(src0_q, qoff1 + lid).x;
+ const uint r3 = read_imageui(src0_q, qoff1 + lid + ne01).x;
+ uint qw[8];
+ qw[0] = mxfp4_pack((ushort)(r0)); qw[1] = mxfp4_pack((ushort)(r0 >> 16));
+ qw[2] = mxfp4_pack((ushort)(r1)); qw[3] = mxfp4_pack((ushort)(r1 >> 16));
+ qw[4] = mxfp4_pack((ushort)(r2)); qw[5] = mxfp4_pack((ushort)(r2 >> 16));
+ qw[6] = mxfp4_pack((ushort)(r3)); qw[7] = mxfp4_pack((ushort)(r3 >> 16));
+
+ // cooperatively stage the n_real-token x 32-K int8 activations
+ const uint stage_lim = (uint)n_real * 8;
+ for (uint idx = lid; idx < stage_lim; idx += 64) {
+ const uint t = idx >> 3;
+ const uint u = idx & 7;
+ sh_qa[t][u] = src1_qa[(col + t) * ne00_u + (step >> 2) + u];
+ }
+ if (lid < (uint)n_real) {
+ sh_d[lid] = src1_da[(col + lid) * num_blocks + sub];
+ }
+ barrier(CLK_LOCAL_MEM_FENCE);
+
+ // Full tiles keep the fully-unrolled 32-wide loop; partial tiles run only n_real
+ if (n_real == TILESIZE_N) {
+ #pragma unroll
+ for (int t = 0; t < TILESIZE_N; ++t) { MOE_MXFP4_DP4A_T(t); }
+ } else {
+ #pragma unroll 4
+ for (int t = 0; t < n_real; ++t) { MOE_MXFP4_DP4A_T(t); }
+ }
+ barrier(CLK_LOCAL_MEM_FENCE);
+ }
+
+ if (row_idx >= ne01) {
+ return;
+ }
+
+ // scatter results to original output rows (reuse sh_src2 from the top)
+ __local uint out_idx[TILESIZE_N];
+ if (lid < TILESIZE_N) {
+ uint idx = sh_src2[lid];
+ if (idx == 0xFFFFFFFF) {
+ idx = sh_src2[0];
+ }
+ out_idx[lid] = idx * ne01;
+ }
+ barrier(CLK_LOCAL_MEM_FENCE);
+
+ const uint m_offset = row + lid;
+ if (n_real == TILESIZE_N) {
+ #pragma unroll
+ for (int t = 1; t < TILESIZE_N; ++t) {
+ write_imagef(dst, out_idx[t] + m_offset, acc[t]);
+ }
+ barrier(CLK_GLOBAL_MEM_FENCE);
+ write_imagef(dst, out_idx[0] + m_offset, acc[0]);
+ } else {
+ for (int t = 0; t < n_real; ++t) {
+ write_imagef(dst, out_idx[t] + m_offset, acc[t]);
+ }
+ }
+}
diff --git a/ggml/src/ggml-opencl/kernels/gemm_moe_q4_0_q8_1_dp4a.cl b/ggml/src/ggml-opencl/kernels/gemm_moe_q4_0_q8_1_dp4a.cl
new file mode 100644
index 0000000000..86ff943c51
--- /dev/null
+++ b/ggml/src/ggml-opencl/kernels/gemm_moe_q4_0_q8_1_dp4a.cl
@@ -0,0 +1,165 @@
+#pragma OPENCL EXTENSION cl_khr_fp16 : enable
+#pragma OPENCL EXTENSION cl_khr_subgroups : enable
+#ifdef cl_khr_integer_dot_product
+#pragma OPENCL EXTENSION cl_khr_integer_dot_product : enable
+#endif
+
+#define TILESIZE_M 64
+#define TILESIZE_N 32
+
+// Expand the 4 nibbles held in the low 16 bits of `u` into 4 bytes (one nibble
+// per byte, value 0..15), packed for the int8 dp4a. The -8 zero-point is applied
+// in the epilogue via the activation sum term (cheaper than biasing every byte).
+#define EXP4(u) ( ((uint)((u) & 0x000Fu)) | \
+ (((uint)((u) & 0x00F0u)) << 4) | \
+ (((uint)((u) & 0x0F00u)) << 8) | \
+ (((uint)((u) & 0xF000u)) << 12) )
+
+// One token's dp4a dot (8 uints = 32 K elems) + q4_0 scale/zero-point epilogue.
+#define MOE_Q40_DP4A_T(t) do { \
+ int raw = 0; \
+ raw = dot_acc_sat_4x8packed_ss_int(qw[0], sh_qa[t][0], raw); \
+ raw = dot_acc_sat_4x8packed_ss_int(qw[1], sh_qa[t][1], raw); \
+ raw = dot_acc_sat_4x8packed_ss_int(qw[2], sh_qa[t][2], raw); \
+ raw = dot_acc_sat_4x8packed_ss_int(qw[3], sh_qa[t][3], raw); \
+ raw = dot_acc_sat_4x8packed_ss_int(qw[4], sh_qa[t][4], raw); \
+ raw = dot_acc_sat_4x8packed_ss_int(qw[5], sh_qa[t][5], raw); \
+ raw = dot_acc_sat_4x8packed_ss_int(qw[6], sh_qa[t][6], raw); \
+ raw = dot_acc_sat_4x8packed_ss_int(qw[7], sh_qa[t][7], raw); \
+ acc[t] += d_val * ((float)sh_d[t] * (float)raw - 8.0f * (float)sh_s[t]); \
+ } while (0)
+
+__attribute__((qcom_wave_pair_mode(1)))
+kernel void kernel_gemm_moe_q4_0_q8_1_dp4a(
+ __read_only image1d_buffer_t src0_q, // q4_0 weights (transposed, packed nibbles)
+ __global half * src0_d, // per-32-block scale
+ __global uint * src1_qa, // q8_1 activations: int8 quants (as uint, 4/elem)
+ __global half * src1_da, // q8_1 per-block scale [tok_slot * ne00/32]
+ __global half * src1_sa, // q8_1 per-block sum*d [tok_slot * ne00/32]
+ __global uint * src2, // post-router (orig out positions)
+ __global ushort * src2_emap,// tile -> expert id
+ __write_only image1d_buffer_t dst,
+ __global int * total_tiles,
+ uint ne00,
+ uint ne01,
+ int is_ragged // 1: compute only real tokens per tile
+) {
+ const uint block_id_m = get_global_id(1); // m_tile
+ const uint block_id_n = get_global_id(2); // n_tile
+
+ if (block_id_n >= total_tiles[0]) {
+ return;
+ }
+
+ const uint lid = get_local_id(0); // 0..63, == this WI's output row in the M-tile
+
+ const ushort expert_id = src2_emap[block_id_n];
+ const uint row = block_id_m * TILESIZE_M;
+ const uint col = block_id_n * TILESIZE_N;
+
+ const uint num_blocks = ne00 >> 5; // blocks-of-32 per token
+ const uint row_idx = row + lid;
+
+ const uint ne00_u = ne00 >> 2; // ne00 in uint (int8x4) units
+
+ __local uint sh_qa[TILESIZE_N][8]; // 32 tokens x 8 uints (32 int8) = 1 KiB
+ __local half sh_d[TILESIZE_N];
+ __local half sh_s[TILESIZE_N];
+
+ // Real-token count for this tile
+ __local uint sh_src2[TILESIZE_N];
+ __local int sh_nreal;
+ if (lid < TILESIZE_N) {
+ sh_src2[lid] = src2[col + lid];
+ }
+ barrier(CLK_LOCAL_MEM_FENCE);
+ if (lid == 0) {
+ int nr = TILESIZE_N;
+ if (is_ragged) {
+ nr = 0;
+ #pragma unroll
+ for (int t = 0; t < TILESIZE_N; ++t) {
+ if (sh_src2[t] != 0xFFFFFFFFu) ++nr;
+ }
+ }
+ sh_nreal = nr;
+ }
+ barrier(CLK_LOCAL_MEM_FENCE);
+ const int n_real = sh_nreal;
+
+ float acc[TILESIZE_N];
+ #pragma unroll
+ for (int t = 0; t < TILESIZE_N; ++t) acc[t] = 0.0f;
+
+ for (uint step = 0; step < ne00; step += 32) {
+ const uint sub = step >> 5; // 32-block index along K
+
+ // per-32-block scale for this WI's row
+ const uint d_offset = row_idx + sub * ne01 + expert_id * num_blocks * ne01;
+ const float d_val = (float)src0_d[d_offset];
+
+ // repack this WI's 32 weight nibbles into 8 dp4a uints
+ const uint qoff0 = row + ((ne01 * step) >> 3) + ((expert_id * ne00 * ne01) >> 3);
+ const uint qoff1 = row + ((ne01 * (step + 16)) >> 3) + ((expert_id * ne00 * ne01) >> 3);
+ const uint r0 = read_imageui(src0_q, qoff0 + lid).x;
+ const uint r1 = read_imageui(src0_q, qoff0 + lid + ne01).x;
+ const uint r2 = read_imageui(src0_q, qoff1 + lid).x;
+ const uint r3 = read_imageui(src0_q, qoff1 + lid + ne01).x;
+ uint qw[8];
+ qw[0] = EXP4(r0); qw[1] = EXP4(r0 >> 16);
+ qw[2] = EXP4(r1); qw[3] = EXP4(r1 >> 16);
+ qw[4] = EXP4(r2); qw[5] = EXP4(r2 >> 16);
+ qw[6] = EXP4(r3); qw[7] = EXP4(r3 >> 16);
+
+ // cooperatively stage the n_real-token x 32-K int8 activations
+ const uint stage_lim = (uint)n_real * 8;
+ for (uint idx = lid; idx < stage_lim; idx += 64) {
+ const uint t = idx >> 3;
+ const uint u = idx & 7;
+ sh_qa[t][u] = src1_qa[(col + t) * ne00_u + (step >> 2) + u];
+ }
+ if (lid < (uint)n_real) {
+ sh_d[lid] = src1_da[(col + lid) * num_blocks + sub];
+ sh_s[lid] = src1_sa[(col + lid) * num_blocks + sub];
+ }
+ barrier(CLK_LOCAL_MEM_FENCE);
+
+ if (n_real == TILESIZE_N) {
+ #pragma unroll
+ for (int t = 0; t < TILESIZE_N; ++t) { MOE_Q40_DP4A_T(t); }
+ } else {
+ #pragma unroll 4
+ for (int t = 0; t < n_real; ++t) { MOE_Q40_DP4A_T(t); }
+ }
+ barrier(CLK_LOCAL_MEM_FENCE);
+ }
+
+ if (row_idx >= ne01) {
+ return;
+ }
+
+ // scatter results to original output rows (reuse sh_src2 from the top)
+ __local uint out_idx[TILESIZE_N];
+ if (lid < TILESIZE_N) {
+ uint idx = sh_src2[lid];
+ if (idx == 0xFFFFFFFF) {
+ idx = sh_src2[0];
+ }
+ out_idx[lid] = idx * ne01;
+ }
+ barrier(CLK_LOCAL_MEM_FENCE);
+
+ const uint m_offset = row + lid;
+ if (n_real == TILESIZE_N) {
+ #pragma unroll
+ for (int t = 1; t < TILESIZE_N; ++t) {
+ write_imagef(dst, out_idx[t] + m_offset, acc[t]);
+ }
+ barrier(CLK_GLOBAL_MEM_FENCE);
+ write_imagef(dst, out_idx[0] + m_offset, acc[0]);
+ } else {
+ for (int t = 0; t < n_real; ++t) {
+ write_imagef(dst, out_idx[t] + m_offset, acc[t]);
+ }
+ }
+}
diff --git a/ggml/src/ggml-opencl/kernels/gemm_moe_q4_k_q8_1_dp4a.cl b/ggml/src/ggml-opencl/kernels/gemm_moe_q4_k_q8_1_dp4a.cl
new file mode 100644
index 0000000000..5408975440
--- /dev/null
+++ b/ggml/src/ggml-opencl/kernels/gemm_moe_q4_k_q8_1_dp4a.cl
@@ -0,0 +1,202 @@
+#pragma OPENCL EXTENSION cl_khr_fp16 : enable
+#pragma OPENCL EXTENSION cl_khr_subgroups : enable
+#ifdef cl_khr_integer_dot_product
+#pragma OPENCL EXTENSION cl_khr_integer_dot_product : enable
+#endif
+
+// q4_K subblock (32 elems): w_i = scale*q_i - minv, q_i in [0,15], scale =
+// d_super*sv6, minv = dmin_super*mn6. With activation block (a_d, a_s, qa[32]):
+// Sum_i w_i * a_i = scale * a_d * dp4a(q, qa) - minv * a_s
+// where a_s = a_d * Sum(qa) (the q8_1 "s" field)
+
+#define TILESIZE_M 64
+#define TILESIZE_N 32
+#define QK_K 256
+#define K_SCALE_SIZE 12
+
+inline void get_scale_min_k4(
+ int j,
+ global const uchar * q,
+ uchar * d,
+ uchar * m
+) {
+ if (j < 4) {
+ *d = q[j] & 63;
+ *m = q[j+4] & 63;
+ } else {
+ *d = (q[j+4] & 0x0F) | ((q[j-4] & 0xC0) >> 2);
+ *m = ((q[j+4] >> 4) & 0x0F) | ((q[j] & 0xC0) >> 2);
+ }
+}
+
+// Expand the 4 nibbles held in the low 16 bits of `u` into 4 bytes (one nibble
+// per byte, value 0..15), packed for the int8 dp4a.
+#define EXP4(u) ( ((uint)((u) & 0x000Fu)) | \
+ (((uint)((u) & 0x00F0u)) << 4) | \
+ (((uint)((u) & 0x0F00u)) << 8) | \
+ (((uint)((u) & 0xF000u)) << 12) )
+
+// One token's dp4a dot (8 uints = 32 K elems) + q4_K scale/min epilogue into acc[t].
+#define MOE_Q4K_DP4A_T(t) do { \
+ int raw = 0; \
+ raw = dot_acc_sat_4x8packed_ss_int(qw[0], sh_qa[t][0], raw); \
+ raw = dot_acc_sat_4x8packed_ss_int(qw[1], sh_qa[t][1], raw); \
+ raw = dot_acc_sat_4x8packed_ss_int(qw[2], sh_qa[t][2], raw); \
+ raw = dot_acc_sat_4x8packed_ss_int(qw[3], sh_qa[t][3], raw); \
+ raw = dot_acc_sat_4x8packed_ss_int(qw[4], sh_qa[t][4], raw); \
+ raw = dot_acc_sat_4x8packed_ss_int(qw[5], sh_qa[t][5], raw); \
+ raw = dot_acc_sat_4x8packed_ss_int(qw[6], sh_qa[t][6], raw); \
+ raw = dot_acc_sat_4x8packed_ss_int(qw[7], sh_qa[t][7], raw); \
+ acc[t] += scale * (float)sh_d[t] * (float)raw - minv * (float)sh_s[t]; \
+ } while (0)
+
+__attribute__((qcom_wave_pair_mode(1)))
+kernel void kernel_gemm_moe_q4_k_q8_1_dp4a(
+ __read_only image1d_buffer_t src0_q, // q4_K weights (transposed, packed nibbles)
+ __global half * src0_d, // per-superblock scale
+ __global half * src0_dm, // per-superblock min
+ __global uchar * src0_s, // 6-bit scale/min codes
+ __global uint * src1_qa, // q8_1 activations: int8 quants (as uint, 4/elem)
+ __global half * src1_da, // q8_1 per-block scale [tok_slot * ne00/32]
+ __global half * src1_sa, // q8_1 per-block sum*d [tok_slot * ne00/32]
+ __global uint * src2, // post-router (orig out positions)
+ __global ushort * src2_emap,// tile -> expert id
+ __write_only image1d_buffer_t dst,
+ __global int * total_tiles,
+ uint ne00,
+ uint ne01,
+ int is_ragged // 1: compute only real tokens per tile
+) {
+ const uint block_id_m = get_global_id(1); // m_tile
+ const uint block_id_n = get_global_id(2); // n_tile
+
+ if (block_id_n >= total_tiles[0]) {
+ return;
+ }
+
+ const uint lid = get_local_id(0); // 0..63, == this WI's output row in the M-tile
+
+ const ushort expert_id = src2_emap[block_id_n];
+ const uint row = block_id_m * TILESIZE_M;
+ const uint col = block_id_n * TILESIZE_N;
+
+ const uint num_superblocks = ne00 / QK_K;
+ const uint scales_per_row = num_superblocks * K_SCALE_SIZE;
+ const uint row_idx = row + lid;
+
+ const uint ne00_u = ne00 >> 2; // ne00 in uint (int8x4) units
+ const uint ne00_b = ne00 >> 5; // blocks-of-32 per token
+
+ __local uint sh_qa[TILESIZE_N][8]; // 32 tokens x 8 uints (32 int8) = 1 KiB
+ __local half sh_d[TILESIZE_N];
+ __local half sh_s[TILESIZE_N];
+
+ // Real token count for this tile
+ __local uint sh_src2[TILESIZE_N];
+ __local int sh_nreal;
+ if (lid < TILESIZE_N) {
+ sh_src2[lid] = src2[col + lid];
+ }
+ barrier(CLK_LOCAL_MEM_FENCE);
+ if (lid == 0) {
+ int nr = TILESIZE_N;
+ if (is_ragged) {
+ nr = 0;
+ #pragma unroll
+ for (int t = 0; t < TILESIZE_N; ++t) {
+ if (sh_src2[t] != 0xFFFFFFFFu) ++nr;
+ }
+ }
+ sh_nreal = nr;
+ }
+ barrier(CLK_LOCAL_MEM_FENCE);
+ const int n_real = sh_nreal;
+
+ float acc[TILESIZE_N];
+ #pragma unroll
+ for (int t = 0; t < TILESIZE_N; ++t) acc[t] = 0.0f;
+
+ for (uint step = 0; step < ne00; step += 32) {
+ const uint sub = step >> 5; // subblock index along K
+ const uint sb = sub >> 3; // superblock index
+ const uint j = sub & 7; // subblock within superblock
+
+ // --- weight scale / min for this WI's row, this subblock ---
+ const uint d_offset = row + sb * ne01 + expert_id * num_superblocks * ne01 + lid;
+ const float d_val = (float)src0_d[d_offset];
+ const float dm_val = (float)src0_dm[d_offset];
+
+ global const uchar * sc = src0_s + (expert_id * ne01 + row_idx) * scales_per_row + sb * K_SCALE_SIZE;
+ uchar sv, mn;
+ get_scale_min_k4(j, sc, &sv, &mn);
+ const float scale = d_val * (float)sv;
+ const float minv = dm_val * (float)mn;
+
+ // --- repack this WI's 32 weight nibbles into 8 dp4a uints ---
+ const uint qoff0 = row + ((ne01 * step) >> 3) + ((expert_id * ne00 * ne01) >> 3);
+ const uint qoff1 = row + ((ne01 * (step + 16)) >> 3) + ((expert_id * ne00 * ne01) >> 3);
+ const uint r0 = read_imageui(src0_q, qoff0 + lid).x;
+ const uint r1 = read_imageui(src0_q, qoff0 + lid + ne01).x;
+ const uint r2 = read_imageui(src0_q, qoff1 + lid).x;
+ const uint r3 = read_imageui(src0_q, qoff1 + lid + ne01).x;
+ uint qw[8];
+ qw[0] = EXP4(r0); qw[1] = EXP4(r0 >> 16);
+ qw[2] = EXP4(r1); qw[3] = EXP4(r1 >> 16);
+ qw[4] = EXP4(r2); qw[5] = EXP4(r2 >> 16);
+ qw[6] = EXP4(r3); qw[7] = EXP4(r3 >> 16);
+
+ // --- cooperatively stage the n_real-token x 32-K int8 activations to LDS ---
+ const uint stage_lim = (uint)n_real * 8;
+ for (uint idx = lid; idx < stage_lim; idx += 64) {
+ const uint t = idx >> 3;
+ const uint u = idx & 7;
+ sh_qa[t][u] = src1_qa[(col + t) * ne00_u + (step >> 2) + u];
+ }
+ if (lid < (uint)n_real) {
+ sh_d[lid] = src1_da[(col + lid) * ne00_b + sub];
+ sh_s[lid] = src1_sa[(col + lid) * ne00_b + sub];
+ }
+ barrier(CLK_LOCAL_MEM_FENCE);
+
+ // dp4a - each real token sum over 8 uints (32 K), then scale/min
+ // Full tiles keep the fully-unrolled 32-wide loop;
+ // partial tiles run only n_real (saves the padded-slot dp4a + staging).
+ if (n_real == TILESIZE_N) {
+ #pragma unroll
+ for (int t = 0; t < TILESIZE_N; ++t) { MOE_Q4K_DP4A_T(t); }
+ } else {
+ #pragma unroll 4
+ for (int t = 0; t < n_real; ++t) { MOE_Q4K_DP4A_T(t); }
+ }
+ barrier(CLK_LOCAL_MEM_FENCE);
+ }
+
+ if (row_idx >= ne01) {
+ return;
+ }
+
+ // scatter results to original output rows
+ __local uint out_idx[TILESIZE_N];
+ if (lid < TILESIZE_N) {
+ uint idx = sh_src2[lid];
+ if (idx == 0xFFFFFFFF) {
+ idx = sh_src2[0];
+ }
+ out_idx[lid] = idx * ne01;
+ }
+ barrier(CLK_LOCAL_MEM_FENCE);
+
+ const uint m_offset = row + lid;
+ if (n_real == TILESIZE_N) {
+ #pragma unroll
+ for (int t = 1; t < TILESIZE_N; ++t) {
+ write_imagef(dst, out_idx[t] + m_offset, acc[t]);
+ }
+ barrier(CLK_GLOBAL_MEM_FENCE);
+ write_imagef(dst, out_idx[0] + m_offset, acc[0]);
+ } else {
+ for (int t = 0; t < n_real; ++t) {
+ write_imagef(dst, out_idx[t] + m_offset, acc[t]);
+ }
+ }
+}
diff --git a/ggml/src/ggml-opencl/kernels/gemm_moe_q6_k_q8_1_dp4a.cl b/ggml/src/ggml-opencl/kernels/gemm_moe_q6_k_q8_1_dp4a.cl
new file mode 100644
index 0000000000..35e63dcab0
--- /dev/null
+++ b/ggml/src/ggml-opencl/kernels/gemm_moe_q6_k_q8_1_dp4a.cl
@@ -0,0 +1,196 @@
+#pragma OPENCL EXTENSION cl_khr_fp16 : enable
+#pragma OPENCL EXTENSION cl_khr_subgroups : enable
+#ifdef cl_khr_integer_dot_product
+#pragma OPENCL EXTENSION cl_khr_integer_dot_product : enable
+#endif
+
+#define TILESIZE_N 32
+#define QK_K 256
+
+// 4 nibbles in the low 16 bits of `u` -> 4 bytes (value 0..15, in bits 0-3).
+#define EXP4(u) ( ((uint)((u) & 0x000Fu)) | \
+ (((uint)((u) & 0x00F0u)) << 4) | \
+ (((uint)((u) & 0x0F00u)) << 8) | \
+ (((uint)((u) & 0xF000u)) << 12) )
+
+// 4 2-bit highs in byte `b` (8 bits) -> 4 bytes, value 0..3 in bits 4-5
+// (pre-multiplied by 16 so it ORs with the EXP4 nibble to form q6 in 0..63).
+#define EXP2(b) ( (((uint)((b) & 0x03u)) << 4) | \
+ (((uint)((b) & 0x0Cu)) << 10) | \
+ (((uint)((b) & 0x30u)) << 16) | \
+ (((uint)((b) & 0xC0u)) << 22) )
+
+// q6 (0..63, bits 0-5 of each byte) -> (q6-32) as a signed int8 per byte.
+// Flipping bit5 subtracts 32 in 6-bit two's complement; then replicate bit5
+// into bits 6-7 to sign-extend to int8. Per-byte, no inter-byte carry.
+inline uint SIGN6(uint q6p) {
+ uint x = q6p ^ 0x20202020u;
+ uint s = x & 0x20202020u;
+ return x | (s << 1) | (s << 2);
+}
+
+inline int dp4a_q6(uint qw0, uint qw1, uint qw2, uint qw3,
+ uint a0, uint a1, uint a2, uint a3) {
+ int raw = 0;
+ raw = dot_acc_sat_4x8packed_ss_int(qw0, a0, raw);
+ raw = dot_acc_sat_4x8packed_ss_int(qw1, a1, raw);
+ raw = dot_acc_sat_4x8packed_ss_int(qw2, a2, raw);
+ raw = dot_acc_sat_4x8packed_ss_int(qw3, a3, raw);
+ return raw;
+}
+
+// One token's q6_K dp4a dot (two halves, per-16 scales) + epilogue into acc[t].
+#define MOE_Q6K_DP4A_T(t) do { \
+ const int raw1 = dp4a_q6(qw[0], qw[1], qw[2], qw[3], sh_qa[t][0], sh_qa[t][1], sh_qa[t][2], sh_qa[t][3]); \
+ const int raw2 = dp4a_q6(qw[4], qw[5], qw[6], qw[7], sh_qa[t][4], sh_qa[t][5], sh_qa[t][6], sh_qa[t][7]); \
+ const float a_d = (float)sh_d[t]; \
+ acc[t] += scale0 * a_d * (float)raw1 + scale1 * a_d * (float)raw2; \
+ } while (0)
+
+__attribute__((qcom_wave_pair_mode(1)))
+kernel void kernel_gemm_moe_q6_k_q8_1_dp4a(
+ __read_only image1d_buffer_t src0_ql, // q6_K low nibbles (image, q4_K-style layout)
+ __global uint * src0_qh, // q6_K high 2-bit (16 elems/uint)
+ __global char * src0_s, // int8 scales (one per 16 elems)
+ __global half * src0_d, // per-superblock scale
+ __global uint * src1_qa, // q8_1 activations int8 (as uint, 4/elem)
+ __global half * src1_da, // q8_1 per-block scale [tok_slot * ne00/32]
+ __global uint * src2, // post-router (orig out positions)
+ __global ushort * src2_emap, // tile -> expert id
+ __write_only image1d_buffer_t dst,
+ __global int * total_tiles,
+ uint ne00,
+ uint ne01,
+ int is_ragged // 1: compute only real tokens per tile
+) {
+ const uint block_id_m = get_global_id(1);
+ const uint block_id_n = get_global_id(2);
+
+ if (block_id_n >= total_tiles[0]) {
+ return;
+ }
+
+ const uint lid = get_local_id(0); // 0..63 -> row within M-tile
+
+ const ushort expert_id = src2_emap[block_id_n];
+ const uint row = block_id_m * 64;
+ const uint col = block_id_n * TILESIZE_N;
+
+ const uint num_superblocks = ne00 / QK_K;
+ const uint scales_per_row = num_superblocks * 16;
+ const uint row_idx = row + lid;
+
+ const uint ne00_u = ne00 >> 2;
+ const uint ne00_b = ne00 >> 5;
+
+ __local uint sh_qa[TILESIZE_N][8];
+ __local half sh_d[TILESIZE_N];
+
+ // Real token count for this tile
+ __local uint sh_src2[TILESIZE_N];
+ __local int sh_nreal;
+ if (lid < TILESIZE_N) {
+ sh_src2[lid] = src2[col + lid];
+ }
+ barrier(CLK_LOCAL_MEM_FENCE);
+ if (lid == 0) {
+ int nr = TILESIZE_N;
+ if (is_ragged) {
+ nr = 0;
+ #pragma unroll
+ for (int t = 0; t < TILESIZE_N; ++t) {
+ if (sh_src2[t] != 0xFFFFFFFFu) ++nr;
+ }
+ }
+ sh_nreal = nr;
+ }
+ barrier(CLK_LOCAL_MEM_FENCE);
+ const int n_real = sh_nreal;
+
+ float acc[TILESIZE_N];
+ #pragma unroll
+ for (int t = 0; t < TILESIZE_N; ++t) acc[t] = 0.0f;
+
+ for (uint step = 0; step < ne00; step += 32) {
+ const uint sub = step >> 5;
+ const uint sb = sub >> 3;
+ const uint j = sub & 7;
+
+ const float d_val = (float)src0_d[row + sb * ne01 + expert_id * num_superblocks * ne01 + lid];
+ global const char * sc = src0_s + (expert_id * ne01 + row_idx) * scales_per_row + sb * 16;
+ const float scale0 = d_val * (float)sc[j * 2];
+ const float scale1 = d_val * (float)sc[j * 2 + 1];
+
+ // high bits: one uint covers 16 elems; first/second 16 of this 32-block
+ const uint qh_base = row + (sub * 2) * ne01 + expert_id * (num_superblocks * 16) * ne01 + lid;
+ const uint qh1 = src0_qh[qh_base];
+ const uint qh2 = src0_qh[qh_base + ne01];
+
+ // low nibbles: same image layout as q4_K (8 ushorts over the 32 K)
+ const uint qoff0 = row + ((ne01 * step) >> 3) + ((expert_id * ne00 * ne01) >> 3);
+ const uint qoff1 = row + ((ne01 * (step + 16)) >> 3) + ((expert_id * ne00 * ne01) >> 3);
+ const uint r0 = read_imageui(src0_ql, qoff0 + lid).x;
+ const uint r1 = read_imageui(src0_ql, qoff0 + lid + ne01).x;
+ const uint r2 = read_imageui(src0_ql, qoff1 + lid).x;
+ const uint r3 = read_imageui(src0_ql, qoff1 + lid + ne01).x;
+
+ uint qw[8];
+ qw[0] = SIGN6(EXP4(r0) | EXP2((qh1) & 0xFFu));
+ qw[1] = SIGN6(EXP4(r0 >> 16) | EXP2((qh1 >> 8) & 0xFFu));
+ qw[2] = SIGN6(EXP4(r1) | EXP2((qh1 >> 16) & 0xFFu));
+ qw[3] = SIGN6(EXP4(r1 >> 16) | EXP2((qh1 >> 24) & 0xFFu));
+ qw[4] = SIGN6(EXP4(r2) | EXP2((qh2) & 0xFFu));
+ qw[5] = SIGN6(EXP4(r2 >> 16) | EXP2((qh2 >> 8) & 0xFFu));
+ qw[6] = SIGN6(EXP4(r3) | EXP2((qh2 >> 16) & 0xFFu));
+ qw[7] = SIGN6(EXP4(r3 >> 16) | EXP2((qh2 >> 24) & 0xFFu));
+
+ const uint stage_lim = (uint)n_real * 8;
+ for (uint idx = lid; idx < stage_lim; idx += 64) {
+ const uint t = idx >> 3;
+ const uint u = idx & 7;
+ sh_qa[t][u] = src1_qa[(col + t) * ne00_u + (step >> 2) + u];
+ }
+ if (lid < (uint)n_real) {
+ sh_d[lid] = src1_da[(col + lid) * ne00_b + sub];
+ }
+ barrier(CLK_LOCAL_MEM_FENCE);
+
+ // Full tiles keep the fully-unrolled 32-wide loop; partial tiles run n_real.
+ if (n_real == TILESIZE_N) {
+ #pragma unroll
+ for (int t = 0; t < TILESIZE_N; ++t) { MOE_Q6K_DP4A_T(t); }
+ } else {
+ #pragma unroll 4
+ for (int t = 0; t < n_real; ++t) { MOE_Q6K_DP4A_T(t); }
+ }
+ barrier(CLK_LOCAL_MEM_FENCE);
+ }
+
+ if (row_idx >= ne01) {
+ return;
+ }
+
+ __local uint out_idx[TILESIZE_N];
+ if (lid < TILESIZE_N) {
+ uint idx = sh_src2[lid];
+ if (idx == 0xFFFFFFFF) {
+ idx = sh_src2[0];
+ }
+ out_idx[lid] = idx * ne01;
+ }
+ barrier(CLK_LOCAL_MEM_FENCE);
+
+ const uint m_offset = row + lid;
+ if (n_real == TILESIZE_N) {
+ #pragma unroll
+ for (int t = 1; t < TILESIZE_N; ++t) {
+ write_imagef(dst, out_idx[t] + m_offset, acc[t]);
+ }
+ barrier(CLK_GLOBAL_MEM_FENCE);
+ write_imagef(dst, out_idx[0] + m_offset, acc[0]);
+ } else {
+ for (int t = 0; t < n_real; ++t) {
+ write_imagef(dst, out_idx[t] + m_offset, acc[t]);
+ }
+ }
+}
diff --git a/ggml/src/ggml-opencl/kernels/gemm_moe_q8_0_f32_ns.cl b/ggml/src/ggml-opencl/kernels/gemm_moe_q8_0_f32_ns.cl
new file mode 100644
index 0000000000..dc0f0ed86c
--- /dev/null
+++ b/ggml/src/ggml-opencl/kernels/gemm_moe_q8_0_f32_ns.cl
@@ -0,0 +1,221 @@
+#pragma OPENCL EXTENSION cl_khr_fp16 : enable
+#pragma OPENCL EXTENSION cl_khr_subgroups : enable
+#pragma OPENCL EXTENSION cl_qcom_subgroup_uniform_load: enable
+#pragma OPENCL EXTENSION cl_qcom_subgroup_constant_load: enable
+#pragma OPENCL EXTENSION cl_qcom_extra_vector_types : enable
+
+#define TILESIZE_K 16
+#define TILESIZE_M 64
+#define TILESIZE_N 32
+
+// q8_0: 16 signed int8 weights (one uint4 = 16 chars) -> half16, scaled.
+#define dequantize_q8_0(q4, a_f16, scale) \
+ a_f16 = convert_half16(as_char16(q4)) * scale;
+
+#define dotx16_reduce8(a_reg, b_lm, c_reg, lm_offset) \
+ acc.s0 = dot(a_reg.s0123, b_lm[lm_offset + 0]); \
+ acc.s1 = dot(a_reg.s0123, b_lm[lm_offset + 1]); \
+ acc.s2 = dot(a_reg.s0123, b_lm[lm_offset + 2]); \
+ acc.s3 = dot(a_reg.s0123, b_lm[lm_offset + 3]); \
+ acc.s4 = dot(a_reg.s0123, b_lm[lm_offset + 4]); \
+ acc.s5 = dot(a_reg.s0123, b_lm[lm_offset + 5]); \
+ acc.s6 = dot(a_reg.s0123, b_lm[lm_offset + 6]); \
+ acc.s7 = dot(a_reg.s0123, b_lm[lm_offset + 7]); \
+ acc.s8 = dot(a_reg.s0123, b_lm[lm_offset + 8]); \
+ acc.s9 = dot(a_reg.s0123, b_lm[lm_offset + 9]); \
+ acc.sa = dot(a_reg.s0123, b_lm[lm_offset + 10]); \
+ acc.sb = dot(a_reg.s0123, b_lm[lm_offset + 11]); \
+ acc.sc = dot(a_reg.s0123, b_lm[lm_offset + 12]); \
+ acc.sd = dot(a_reg.s0123, b_lm[lm_offset + 13]); \
+ acc.se = dot(a_reg.s0123, b_lm[lm_offset + 14]); \
+ acc.sf = dot(a_reg.s0123, b_lm[lm_offset + 15]); \
+ acc.s0 += dot(a_reg.s4567, b_lm[lm_offset + 32]); \
+ acc.s1 += dot(a_reg.s4567, b_lm[lm_offset + 33]); \
+ acc.s2 += dot(a_reg.s4567, b_lm[lm_offset + 34]); \
+ acc.s3 += dot(a_reg.s4567, b_lm[lm_offset + 35]); \
+ acc.s4 += dot(a_reg.s4567, b_lm[lm_offset + 36]); \
+ acc.s5 += dot(a_reg.s4567, b_lm[lm_offset + 37]); \
+ acc.s6 += dot(a_reg.s4567, b_lm[lm_offset + 38]); \
+ acc.s7 += dot(a_reg.s4567, b_lm[lm_offset + 39]); \
+ acc.s8 += dot(a_reg.s4567, b_lm[lm_offset + 40]); \
+ acc.s9 += dot(a_reg.s4567, b_lm[lm_offset + 41]); \
+ acc.sa += dot(a_reg.s4567, b_lm[lm_offset + 42]); \
+ acc.sb += dot(a_reg.s4567, b_lm[lm_offset + 43]); \
+ acc.sc += dot(a_reg.s4567, b_lm[lm_offset + 44]); \
+ acc.sd += dot(a_reg.s4567, b_lm[lm_offset + 45]); \
+ acc.se += dot(a_reg.s4567, b_lm[lm_offset + 46]); \
+ acc.sf += dot(a_reg.s4567, b_lm[lm_offset + 47]); \
+ c_reg.lo += convert_float8(acc.lo); \
+ c_reg.hi += convert_float8(acc.hi); \
+ acc.s0 = dot(a_reg.s89ab, b_lm[lm_offset + 64]); \
+ acc.s1 = dot(a_reg.s89ab, b_lm[lm_offset + 65]); \
+ acc.s2 = dot(a_reg.s89ab, b_lm[lm_offset + 66]); \
+ acc.s3 = dot(a_reg.s89ab, b_lm[lm_offset + 67]); \
+ acc.s4 = dot(a_reg.s89ab, b_lm[lm_offset + 68]); \
+ acc.s5 = dot(a_reg.s89ab, b_lm[lm_offset + 69]); \
+ acc.s6 = dot(a_reg.s89ab, b_lm[lm_offset + 70]); \
+ acc.s7 = dot(a_reg.s89ab, b_lm[lm_offset + 71]); \
+ acc.s8 = dot(a_reg.s89ab, b_lm[lm_offset + 72]); \
+ acc.s9 = dot(a_reg.s89ab, b_lm[lm_offset + 73]); \
+ acc.sa = dot(a_reg.s89ab, b_lm[lm_offset + 74]); \
+ acc.sb = dot(a_reg.s89ab, b_lm[lm_offset + 75]); \
+ acc.sc = dot(a_reg.s89ab, b_lm[lm_offset + 76]); \
+ acc.sd = dot(a_reg.s89ab, b_lm[lm_offset + 77]); \
+ acc.se = dot(a_reg.s89ab, b_lm[lm_offset + 78]); \
+ acc.sf = dot(a_reg.s89ab, b_lm[lm_offset + 79]); \
+ acc.s0 += dot(a_reg.scdef, b_lm[lm_offset + 96]); \
+ acc.s1 += dot(a_reg.scdef, b_lm[lm_offset + 97]); \
+ acc.s2 += dot(a_reg.scdef, b_lm[lm_offset + 98]); \
+ acc.s3 += dot(a_reg.scdef, b_lm[lm_offset + 99]); \
+ acc.s4 += dot(a_reg.scdef, b_lm[lm_offset + 100]); \
+ acc.s5 += dot(a_reg.scdef, b_lm[lm_offset + 101]); \
+ acc.s6 += dot(a_reg.scdef, b_lm[lm_offset + 102]); \
+ acc.s7 += dot(a_reg.scdef, b_lm[lm_offset + 103]); \
+ acc.s8 += dot(a_reg.scdef, b_lm[lm_offset + 104]); \
+ acc.s9 += dot(a_reg.scdef, b_lm[lm_offset + 105]); \
+ acc.sa += dot(a_reg.scdef, b_lm[lm_offset + 106]); \
+ acc.sb += dot(a_reg.scdef, b_lm[lm_offset + 107]); \
+ acc.sc += dot(a_reg.scdef, b_lm[lm_offset + 108]); \
+ acc.sd += dot(a_reg.scdef, b_lm[lm_offset + 109]); \
+ acc.se += dot(a_reg.scdef, b_lm[lm_offset + 110]); \
+ acc.sf += dot(a_reg.scdef, b_lm[lm_offset + 111]); \
+ c_reg.lo += convert_float8(acc.lo); \
+ c_reg.hi += convert_float8(acc.hi); \
+
+
+__attribute__((qcom_wave_pair_mode(1)))
+kernel void kernel_gemm_moe_q8_0_f32_ns(
+ __global char * src0_q, // flat q8_0 quants [n_expert*ne01*ne00]
+ __global half * src0_d, // flat q8_0 scales [n_expert*ne01*nb]
+ __read_only image1d_buffer_t src1, // reordered activations (f32)
+ __global uint * src2, // post-router out indices
+ __global ushort * src2_emap,// expert per tile
+ __write_only image1d_buffer_t dst,
+ __global int * total_tiles,
+ uint ne00,
+ uint ne01
+) {
+ uint block_id_m = get_global_id(1); // m_tile
+ uint block_id_n = get_global_id(2); // n_tile
+
+ if (block_id_n >= total_tiles[0]) {
+ return;
+ }
+
+ __private half16 reg_a;
+ __private float32 reg_c = (float32)(0);
+ __local half4 shared_b[128];
+
+ const ushort expert_id = src2_emap[block_id_n];
+
+ const uint row = block_id_m * TILESIZE_M;
+ const uint col = block_id_n * TILESIZE_N;
+
+ const uint nb = ne00 >> 5; // blocks per row (ne00/32)
+ const uint w_row = expert_id * ne01 + row + get_local_id(0); // this lane's output row
+ __global char * w_q = src0_q + (ulong)w_row * ne00; // char base for the row
+ __global half * w_d = src0_d + (ulong)w_row * nb; // scale base for the row
+
+ uint sub_block_id_m = get_local_id(0);
+ uint2 b_global_offset;
+ b_global_offset.x = ((sub_block_id_m & 3) << 2) + (sub_block_id_m >> 2) * ne00;
+ b_global_offset.y = b_global_offset.x + (16 * ne00);
+ uint2 b_local_offset;
+ b_local_offset.x = (sub_block_id_m & 3) * 32 + (sub_block_id_m >> 2);
+ b_local_offset.y = b_local_offset.x + 16;
+
+ // Loop along K axis, 32 elements per iteration, split into 2 sub-blocks.
+ for (uint step = 0; step < ne00; step += TILESIZE_K * 2) {
+ half s = w_d[step >> 5]; // one q8_0 scale per 32-element block
+
+ // First sub-block: 16 weights (16 chars = one uint4) at K=step
+ uint4 q8x16 = *((__global uint4 *)(w_q + step));
+
+ uint b_sub_offset = col * ne00 + step;
+ float8 bx8_f32;
+ bx8_f32.lo = read_imagef(src1, (b_sub_offset + b_global_offset.x) / 4);
+ bx8_f32.hi = read_imagef(src1, (b_sub_offset + b_global_offset.y) / 4);
+ half8 bx8_f16 = convert_half8(bx8_f32);
+ shared_b[b_local_offset.x] = bx8_f16.lo;
+ shared_b[b_local_offset.y] = bx8_f16.hi;
+
+ dequantize_q8_0(q8x16, reg_a, s);
+
+ sub_group_barrier(CLK_LOCAL_MEM_FENCE);
+
+ half16 acc;
+ dotx16_reduce8(reg_a, shared_b, reg_c.lo, 0);
+ dotx16_reduce8(reg_a, shared_b, reg_c.hi, 16);
+
+ // Second sub-block: next 16 weights at K=step+16
+ uint half_step = step + TILESIZE_K;
+ q8x16 = *((__global uint4 *)(w_q + half_step));
+ b_sub_offset = col * ne00 + half_step;
+
+ bx8_f32.lo = read_imagef(src1, (b_sub_offset + b_global_offset.x) / 4);
+ bx8_f32.hi = read_imagef(src1, (b_sub_offset + b_global_offset.y) / 4);
+ bx8_f16 = convert_half8(bx8_f32);
+ shared_b[b_local_offset.x] = bx8_f16.lo;
+ shared_b[b_local_offset.y] = bx8_f16.hi;
+
+ dequantize_q8_0(q8x16, reg_a, s);
+
+ sub_group_barrier(CLK_LOCAL_MEM_FENCE);
+
+ dotx16_reduce8(reg_a, shared_b, reg_c.lo, 0);
+ dotx16_reduce8(reg_a, shared_b, reg_c.hi, 16);
+ }
+
+ if ((get_global_id(0) + block_id_m * TILESIZE_M) >= ne01) {
+ return;
+ }
+
+ __local uint out_idx[TILESIZE_N];
+
+ if (get_local_id(0) < TILESIZE_N) {
+ uint idx = src2[block_id_n * TILESIZE_N + get_local_id(0)];
+ if (idx == 0xFFFFFFFF) {
+ idx = src2[block_id_n * TILESIZE_N + 0];
+ }
+ out_idx[get_local_id(0)] = idx * ne01;
+ }
+
+ barrier(CLK_LOCAL_MEM_FENCE);
+
+ uint m_offset = row + get_local_id(0);
+
+ write_imagef(dst, out_idx[1] + m_offset, (reg_c.s1));
+ write_imagef(dst, out_idx[2] + m_offset, (reg_c.s2));
+ write_imagef(dst, out_idx[3] + m_offset, (reg_c.s3));
+ write_imagef(dst, out_idx[4] + m_offset, (reg_c.s4));
+ write_imagef(dst, out_idx[5] + m_offset, (reg_c.s5));
+ write_imagef(dst, out_idx[6] + m_offset, (reg_c.s6));
+ write_imagef(dst, out_idx[7] + m_offset, (reg_c.s7));
+ write_imagef(dst, out_idx[8] + m_offset, (reg_c.s8));
+ write_imagef(dst, out_idx[9] + m_offset, (reg_c.s9));
+ write_imagef(dst, out_idx[10] + m_offset, (reg_c.sa));
+ write_imagef(dst, out_idx[11] + m_offset, (reg_c.sb));
+ write_imagef(dst, out_idx[12] + m_offset, (reg_c.sc));
+ write_imagef(dst, out_idx[13] + m_offset, (reg_c.sd));
+ write_imagef(dst, out_idx[14] + m_offset, (reg_c.se));
+ write_imagef(dst, out_idx[15] + m_offset, (reg_c.sf));
+ write_imagef(dst, out_idx[16] + m_offset, (reg_c.sg));
+ write_imagef(dst, out_idx[17] + m_offset, (reg_c.sh));
+ write_imagef(dst, out_idx[18] + m_offset, (reg_c.si));
+ write_imagef(dst, out_idx[19] + m_offset, (reg_c.sj));
+ write_imagef(dst, out_idx[20] + m_offset, (reg_c.sk));
+ write_imagef(dst, out_idx[21] + m_offset, (reg_c.sl));
+ write_imagef(dst, out_idx[22] + m_offset, (reg_c.sm));
+ write_imagef(dst, out_idx[23] + m_offset, (reg_c.sn));
+ write_imagef(dst, out_idx[24] + m_offset, (reg_c.so));
+ write_imagef(dst, out_idx[25] + m_offset, (reg_c.sp));
+ write_imagef(dst, out_idx[26] + m_offset, (reg_c.sq));
+ write_imagef(dst, out_idx[27] + m_offset, (reg_c.sr));
+ write_imagef(dst, out_idx[28] + m_offset, (reg_c.ss));
+ write_imagef(dst, out_idx[29] + m_offset, (reg_c.st));
+ write_imagef(dst, out_idx[30] + m_offset, (reg_c.su));
+ write_imagef(dst, out_idx[31] + m_offset, (reg_c.sv));
+
+ barrier(CLK_GLOBAL_MEM_FENCE);
+ write_imagef(dst, out_idx[0] + m_offset, (reg_c.s0));
+}
diff --git a/ggml/src/ggml-opencl/kernels/gemm_moe_q8_1_dp4a.cl b/ggml/src/ggml-opencl/kernels/gemm_moe_q8_1_dp4a.cl
new file mode 100644
index 0000000000..39bf5d8321
--- /dev/null
+++ b/ggml/src/ggml-opencl/kernels/gemm_moe_q8_1_dp4a.cl
@@ -0,0 +1,221 @@
+#pragma OPENCL EXTENSION cl_khr_fp16 : enable
+#pragma OPENCL EXTENSION cl_khr_subgroups : enable
+#ifdef cl_khr_integer_dot_product
+#pragma OPENCL EXTENSION cl_khr_integer_dot_product : enable
+#endif
+
+// Generic int8 dp4a MoE GEMM, specialized versions also exist
+// MOE_QT:
+// 4 (q4_K)/41(q4_1)/40(q4_0) NIBBLE image low nibbles -> EXP4
+// 5 (q5_K)/51(q5_1)/50(q5_0) NIBBLE+HI image nibbles + qh high-bit plane
+// 6 (q6_K) Q6 image nibbles + qh 2-bit -> SIGN6((nibble|hi2))
+// 80(q8_0)/82(mxfp4) INT8 global int8 codes (mxfp4: convert applies kvalues LUT)
+
+#define TILESIZE_M 64
+#define TILESIZE_N 32
+#define QK_K 256
+
+#ifndef MOE_QT
+#define MOE_QT 4
+#endif
+
+// 4 nibbles in low 16 bits of u -> 4 bytes (value 0..15)
+#define EXP4(u) ( ((uint)((u) & 0x000Fu)) | \
+ (((uint)((u) & 0x00F0u)) << 4) | \
+ (((uint)((u) & 0x0F00u)) << 8) | \
+ (((uint)((u) & 0xF000u)) << 12) )
+// 4 2-bit highs in byte b -> 4 bytes, bits 4-5 (q6_K)
+#define EXP2(b) ( (((uint)((b) & 0x03u)) << 4) | \
+ (((uint)((b) & 0x0Cu)) << 10) | \
+ (((uint)((b) & 0x30u)) << 16) | \
+ (((uint)((b) & 0xC0u)) << 22) )
+
+// q6 (0..63) -> (q6-32) signed int8/byte (no inter-byte carry)
+inline uint SIGN6(uint q6p){ uint x=q6p^0x20202020u; uint s=x&0x20202020u; return x|(s<<1)|(s<<2); }
+
+// 4 high bits (one per element, in bits 0..3 of h) -> bit4 of each of 4 bytes (5-bit hi)
+#define EXP1(h) ( (((uint)((h) & 0x1u)) << 4) | \
+ (((uint)((h) & 0x2u)) << 11) | \
+ (((uint)((h) & 0x4u)) << 18) | \
+ (((uint)((h) & 0x8u)) << 25) )
+
+// per-type weight params + per-32-step unpack into qw[8] (8 int8 uints)
+#if MOE_QT == 4 || MOE_QT == 41 || MOE_QT == 40
+ #define WEIGHT_PARAMS __read_only image1d_buffer_t src0_q,
+ #define LOAD_QW(step, sub) \
+ uint qw[8]; { \
+ const uint qoff0 = row + ((ne01*(step))>>3) + ((expert_id*ne00*ne01)>>3); \
+ const uint qoff1 = row + ((ne01*((step)+16))>>3) + ((expert_id*ne00*ne01)>>3); \
+ const uint r0=read_imageui(src0_q,qoff0+lid).x, r1=read_imageui(src0_q,qoff0+lid+ne01).x; \
+ const uint r2=read_imageui(src0_q,qoff1+lid).x, r3=read_imageui(src0_q,qoff1+lid+ne01).x; \
+ qw[0]=EXP4(r0); qw[1]=EXP4(r0>>16); qw[2]=EXP4(r1); qw[3]=EXP4(r1>>16); \
+ qw[4]=EXP4(r2); qw[5]=EXP4(r2>>16); qw[6]=EXP4(r3); qw[7]=EXP4(r3>>16); }
+
+#elif MOE_QT == 5 || MOE_QT == 51 || MOE_QT == 50
+ // low nibbles via image (q4_K layout) + high-bit plane src0_qh: 1 uint per 32-block
+ // (bit i = high bit of element i). qh laid out [expert][block][row] to match the
+ // existing q5_0 trans4 convert
+ #define WEIGHT_PARAMS __read_only image1d_buffer_t src0_q, __global uint * src0_qh,
+ #define LOAD_QW(step, sub) \
+ uint qw[8]; { \
+ const uint qoff0 = row + ((ne01*(step))>>3) + ((expert_id*ne00*ne01)>>3); \
+ const uint qoff1 = row + ((ne01*((step)+16))>>3) + ((expert_id*ne00*ne01)>>3); \
+ const uint r0=read_imageui(src0_q,qoff0+lid).x, r1=read_imageui(src0_q,qoff0+lid+ne01).x; \
+ const uint r2=read_imageui(src0_q,qoff1+lid).x, r3=read_imageui(src0_q,qoff1+lid+ne01).x; \
+ const uint h = src0_qh[row_idx + (sub)*ne01 + expert_id*(ne00>>5)*ne01]; \
+ qw[0]=EXP4(r0)|EXP1(h); qw[1]=EXP4(r0>>16)|EXP1(h>>4); \
+ qw[2]=EXP4(r1)|EXP1(h>>8); qw[3]=EXP4(r1>>16)|EXP1(h>>12); \
+ qw[4]=EXP4(r2)|EXP1(h>>16); qw[5]=EXP4(r2>>16)|EXP1(h>>20); \
+ qw[6]=EXP4(r3)|EXP1(h>>24); qw[7]=EXP4(r3>>16)|EXP1(h>>28); }
+
+#elif MOE_QT == 6
+ #define WEIGHT_PARAMS __read_only image1d_buffer_t src0_ql, __global uint * src0_qh,
+ #define LOAD_QW(step, sub) \
+ uint qw[8]; { \
+ const uint qoff0 = row + ((ne01*(step))>>3) + ((expert_id*ne00*ne01)>>3); \
+ const uint qoff1 = row + ((ne01*((step)+16))>>3) + ((expert_id*ne00*ne01)>>3); \
+ const uint r0=read_imageui(src0_ql,qoff0+lid).x, r1=read_imageui(src0_ql,qoff0+lid+ne01).x; \
+ const uint r2=read_imageui(src0_ql,qoff1+lid).x, r3=read_imageui(src0_ql,qoff1+lid+ne01).x; \
+ const uint qhb = row + ((sub)*2)*ne01 + expert_id*((ne00>>5)*2)*ne01 + lid; \
+ const uint qh1=src0_qh[qhb], qh2=src0_qh[qhb+ne01]; \
+ qw[0]=SIGN6(EXP4(r0)|EXP2(qh1&0xFFu)); qw[1]=SIGN6(EXP4(r0>>16)|EXP2((qh1>>8)&0xFFu)); \
+ qw[2]=SIGN6(EXP4(r1)|EXP2((qh1>>16)&0xFFu)); qw[3]=SIGN6(EXP4(r1>>16)|EXP2((qh1>>24)&0xFFu)); \
+ qw[4]=SIGN6(EXP4(r2)|EXP2(qh2&0xFFu)); qw[5]=SIGN6(EXP4(r2>>16)|EXP2((qh2>>8)&0xFFu)); \
+ qw[6]=SIGN6(EXP4(r3)|EXP2((qh2>>16)&0xFFu)); qw[7]=SIGN6(EXP4(r3>>16)|EXP2((qh2>>24)&0xFFu)); }
+
+#elif MOE_QT == 80 || MOE_QT == 82
+ // 8-bit direct: int8 codes 8 uints / 32-block, [expert][row][8*sub]. mxfp4: the
+ // convert resolves kvalues_mxfp4[nibble] -> int8 and stores the e8m0_half scale.
+ #define WEIGHT_PARAMS __global uint * src0_q8,
+ #define LOAD_QW(step, sub) \
+ uint qw[8]; { \
+ const uint qb = (expert_id*ne01 + row_idx)*(ne00>>2) + (sub)*8; \
+ qw[0]=src0_q8[qb+0]; qw[1]=src0_q8[qb+1]; qw[2]=src0_q8[qb+2]; qw[3]=src0_q8[qb+3]; \
+ qw[4]=src0_q8[qb+4]; qw[5]=src0_q8[qb+5]; qw[6]=src0_q8[qb+6]; qw[7]=src0_q8[qb+7]; }
+#else
+ #error "unknown MOE_QT"
+#endif
+
+inline int dp4a4(uint w0,uint w1,uint w2,uint w3,uint a0,uint a1,uint a2,uint a3){
+ int r=0; r=dot_acc_sat_4x8packed_ss_int(w0,a0,r); r=dot_acc_sat_4x8packed_ss_int(w1,a1,r);
+ r=dot_acc_sat_4x8packed_ss_int(w2,a2,r); r=dot_acc_sat_4x8packed_ss_int(w3,a3,r); return r; }
+
+// One token's two-half dp4a + uniform scale/min epilogue into acc[t].
+#define MOE_DP4A_T(t) do { \
+ const int raw1 = dp4a4(qw[0],qw[1],qw[2],qw[3], sh_qa[t][0],sh_qa[t][1],sh_qa[t][2],sh_qa[t][3]); \
+ const int raw2 = dp4a4(qw[4],qw[5],qw[6],qw[7], sh_qa[t][4],sh_qa[t][5],sh_qa[t][6],sh_qa[t][7]); \
+ const float a_d = (float)sh_d[t]; \
+ acc[t] += sc0*a_d*(float)raw1 + sc1*a_d*(float)raw2 - mn*(float)sh_s[t]; \
+ } while (0)
+
+__attribute__((qcom_wave_pair_mode(1)))
+kernel void kernel_gemm_moe_q8_1_dp4a(
+ WEIGHT_PARAMS // per-type native weight buffer(s)
+ __global half * src0_scale,// uniform f16 16/superblock (per-16), [expert,row]
+ __global half * src0_min, // uniform f16 8/superblock (per-32), [expert,row]
+ __global uint * src1_qa, // q8_1 activations int8 (as uint, 4/elem)
+ __global half * src1_da, // q8_1 per-block scale [tok_slot * ne00/32]
+ __global half * src1_sa, // q8_1 per-block sum*d [tok_slot * ne00/32]
+ __global uint * src2, // post-router (orig out positions)
+ __global ushort * src2_emap, // tile -> expert id
+ __write_only image1d_buffer_t dst,
+ __global int * total_tiles,
+ uint ne00,
+ uint ne01,
+ int is_ragged,
+ int has_min // 0 for symmetric types (q8_0/q6_K/q4_0/...): skip min read
+) {
+ const uint block_id_m = get_global_id(1);
+ const uint block_id_n = get_global_id(2);
+ if (block_id_n >= total_tiles[0]) return;
+
+ const uint lid = get_local_id(0); // 0..63 -> output row within M-tile
+ const ushort expert_id = src2_emap[block_id_n];
+ const uint row = block_id_m * TILESIZE_M;
+ const uint col = block_id_n * TILESIZE_N;
+ const uint row_idx = row + lid;
+
+ // Scale/min are laid out FLAT per-32-block (2 per-16-segment scales + 1 min per
+ // 32-block), so K only needs to be a multiple of 32 — works for the 32-block
+ // types (q8_0/q5_0/q4_0/...) as well as the K-quants (K%256==0, same bytes).
+ const uint nblk32 = ne00 / 32;
+ const uint sc_per_row = nblk32 * 2;
+ const uint mn_per_row = nblk32;
+ const uint ne00_u = ne00 >> 2;
+ const uint ne00_b = ne00 >> 5;
+
+ __local uint sh_qa[TILESIZE_N][8];
+ __local half sh_d[TILESIZE_N];
+ __local half sh_s[TILESIZE_N];
+
+ __local uint sh_src2[TILESIZE_N];
+ __local int sh_nreal;
+ if (lid < TILESIZE_N) sh_src2[lid] = src2[col + lid];
+ barrier(CLK_LOCAL_MEM_FENCE);
+ if (lid == 0) {
+ int nr = TILESIZE_N;
+ if (is_ragged) { nr = 0;
+ #pragma unroll
+ for (int t = 0; t < TILESIZE_N; ++t) if (sh_src2[t] != 0xFFFFFFFFu) ++nr; }
+ sh_nreal = nr;
+ }
+ barrier(CLK_LOCAL_MEM_FENCE);
+ const int n_real = sh_nreal;
+
+ float acc[TILESIZE_N];
+ #pragma unroll
+ for (int t = 0; t < TILESIZE_N; ++t) acc[t] = 0.0f;
+
+ for (uint step = 0; step < ne00; step += 32) {
+ const uint sub = step >> 5; // 32-block index along K
+
+ // uniform pre-decoded scale (2 per-16-seg) + min (1) for this row, this 32-block
+ __global half * scl = src0_scale + (expert_id*ne01 + row_idx)*sc_per_row + sub*2;
+ const float sc0 = (float)scl[0];
+ const float sc1 = (float)scl[1];
+ float mn = 0.0f;
+ if (has_min) mn = (float)src0_min[(expert_id*ne01 + row_idx)*mn_per_row + sub];
+
+ LOAD_QW(step, sub)
+
+ const uint stage_lim = (uint)n_real * 8;
+ for (uint idx = lid; idx < stage_lim; idx += 64) {
+ const uint t = idx >> 3, u = idx & 7;
+ sh_qa[t][u] = src1_qa[(col + t) * ne00_u + (step >> 2) + u];
+ }
+ if (lid < (uint)n_real) {
+ sh_d[lid] = src1_da[(col + lid) * ne00_b + sub];
+ sh_s[lid] = src1_sa[(col + lid) * ne00_b + sub];
+ }
+ barrier(CLK_LOCAL_MEM_FENCE);
+
+ if (n_real == TILESIZE_N) {
+ #pragma unroll
+ for (int t = 0; t < TILESIZE_N; ++t) { MOE_DP4A_T(t); }
+ } else {
+ #pragma unroll 4
+ for (int t = 0; t < n_real; ++t) { MOE_DP4A_T(t); }
+ }
+ barrier(CLK_LOCAL_MEM_FENCE);
+ }
+
+ if (row_idx >= ne01) return;
+
+ __local uint out_idx[TILESIZE_N];
+ if (lid < TILESIZE_N) {
+ uint idx = sh_src2[lid];
+ if (idx == 0xFFFFFFFF) idx = sh_src2[0];
+ out_idx[lid] = idx * ne01;
+ }
+ barrier(CLK_LOCAL_MEM_FENCE);
+
+ const uint m_offset = row + lid;
+ if (n_real == TILESIZE_N) {
+ #pragma unroll
+ for (int t = 1; t < TILESIZE_N; ++t) write_imagef(dst, out_idx[t] + m_offset, acc[t]);
+ barrier(CLK_GLOBAL_MEM_FENCE);
+ write_imagef(dst, out_idx[0] + m_offset, acc[0]);
+ } else {
+ for (int t = 0; t < n_real; ++t) write_imagef(dst, out_idx[t] + m_offset, acc[t]);
+ }
+}
diff --git a/ggml/src/ggml-opencl/kernels/gemm_noshuffle_iq4_nl_q8_1_dp4a.cl b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_iq4_nl_q8_1_dp4a.cl
new file mode 100644
index 0000000000..2941289ddf
--- /dev/null
+++ b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_iq4_nl_q8_1_dp4a.cl
@@ -0,0 +1,143 @@
+#pragma OPENCL EXTENSION cl_khr_fp16 : enable
+#pragma OPENCL EXTENSION cl_khr_subgroups : enable
+#ifdef cl_khr_integer_dot_product
+#pragma OPENCL EXTENSION cl_khr_integer_dot_product : enable
+#endif
+
+// Weight layout, feature-major:
+// src0_q[row + (k/4)*m] ushort = 4 nibbles (K = 4*grp .. +3)
+// src0_d[row + (k/32)*m] half = per-32-block scale
+
+#define TILESIZE_N 32
+
+// IQ4_NL non-linear codebook as signed int8, packed 4 codes per uint.
+// divergent nibble lookups read a small __constant uint array + shift,
+// never a byte array because byte-indexed __constant loads serialize on Adreno and tank perf
+// idx 0-3: -127,-104,-83,-65 = 0x81,0x98,0xAD,0xBF
+// idx 4-7: -49,-35,-22,-10 = 0xCF,0xDD,0xEA,0xF6
+// idx 8-11: 1, 13, 25, 38 = 0x01,0x0D,0x19,0x26
+// idx 12-15: 53, 69, 89,113 = 0x35,0x45,0x59,0x71
+__constant uint kvalues_iq4nl_i8x4[4] = {
+ 0xBFAD9881u, 0xF6EADDCFu, 0x26190D01u, 0x71594535u
+};
+
+// nibble (0..15) -> its codebook byte in the low 8 bits.
+inline uint iq4nl_code(uint n) {
+ return (kvalues_iq4nl_i8x4[n >> 2] >> ((n & 3u) * 8u)) & 0xFFu;
+}
+
+// 4 nibbles in low 16 bits of u -> 4 codebook int8, packed for dp4a.
+inline uint iq4nl_pack(ushort u) {
+ return iq4nl_code((uint)( u & 0xF))
+ | (iq4nl_code((uint)((u >> 4) & 0xF)) << 8)
+ | (iq4nl_code((uint)((u >> 8) & 0xF)) << 16)
+ | (iq4nl_code((uint)((u >> 12) & 0xF)) << 24);
+}
+
+inline int dot8_q8a(uint8 qw, __local const uint * a) {
+ int r = 0;
+ r = dot_acc_sat_4x8packed_ss_int(qw.s0, a[0], r);
+ r = dot_acc_sat_4x8packed_ss_int(qw.s1, a[1], r);
+ r = dot_acc_sat_4x8packed_ss_int(qw.s2, a[2], r);
+ r = dot_acc_sat_4x8packed_ss_int(qw.s3, a[3], r);
+ r = dot_acc_sat_4x8packed_ss_int(qw.s4, a[4], r);
+ r = dot_acc_sat_4x8packed_ss_int(qw.s5, a[5], r);
+ r = dot_acc_sat_4x8packed_ss_int(qw.s6, a[6], r);
+ r = dot_acc_sat_4x8packed_ss_int(qw.s7, a[7], r);
+ return r;
+}
+
+__attribute__((qcom_wave_pair_mode(1)))
+kernel void kernel_gemm_noshuffle_iq4_nl_q8_1_dp4a(
+ __global const ushort * src0_q, // IQ4_NL nibbles (4/ushort, feature-major)
+ __global const half * src0_d, // per-32-block scale, feature-major
+ __global const uint * src1_qa, // q8_1 activations int8 (as uint, 4/elem) [N, K]
+ __global const half * src1_da, // q8_1 per-block scale [N, K/32]
+ __global float * dst,
+ ulong offsetd,
+ int m, // output features (rows)
+ int n_no_padding, // tokens (cols)
+ int k // K (== ne00)
+) {
+ dst = (global float *)((global char *)dst + offsetd);
+
+ const uint lid = get_local_id(0); // 0..63 -> row within the M-tile
+ const uint block_id_m = get_global_id(1);
+ const uint block_id_n = get_global_id(2);
+
+ const uint row = block_id_m * 64 + lid;
+ const uint col_base = block_id_n * TILESIZE_N;
+ const bool row_valid = row < (uint)m;
+ const uint rrow = row_valid ? row : 0; // clamp OOB rows; their writes are masked
+
+ const uint k_u = (uint)k >> 2; // K in uint (int8x4) units
+ const uint k_b = (uint)k >> 5; // blocks-of-32 along K
+
+ __local uint sh_qa[TILESIZE_N][8];
+ __local half sh_d[TILESIZE_N];
+
+#define NGROUPS (TILESIZE_N / 4)
+ float4 acc[NGROUPS];
+ #pragma unroll
+ for (int g = 0; g < NGROUPS; ++g) acc[g] = (float4)(0.0f);
+
+ for (uint step = 0; step < (uint)k; step += 32) {
+ const uint sub = step >> 5;
+
+ const float d_w = (float)src0_d[rrow + sub * (uint)m];
+
+ // 8 weight uints (32 codebook int8) for this row, this 32-block.
+ const uint qsbase = rrow + (step >> 2) * (uint)m;
+ uint8 qw;
+ qw.s0 = iq4nl_pack(src0_q[qsbase + 0 * m]);
+ qw.s1 = iq4nl_pack(src0_q[qsbase + 1 * m]);
+ qw.s2 = iq4nl_pack(src0_q[qsbase + 2 * m]);
+ qw.s3 = iq4nl_pack(src0_q[qsbase + 3 * m]);
+ qw.s4 = iq4nl_pack(src0_q[qsbase + 4 * m]);
+ qw.s5 = iq4nl_pack(src0_q[qsbase + 5 * m]);
+ qw.s6 = iq4nl_pack(src0_q[qsbase + 6 * m]);
+ qw.s7 = iq4nl_pack(src0_q[qsbase + 7 * m]);
+
+ // cooperatively stage the 32-token x 32-K int8 activations to lm
+ for (uint idx = lid; idx < TILESIZE_N * 8; idx += 64) {
+ const uint t = idx >> 3;
+ const uint u = idx & 7;
+ const uint c = col_base + t;
+ sh_qa[t][u] = (c < (uint)n_no_padding) ? src1_qa[c * k_u + (step >> 2) + u] : 0u;
+ }
+ if (lid < TILESIZE_N) {
+ const uint c = col_base + lid;
+ sh_d[lid] = (c < (uint)n_no_padding) ? src1_da[c * k_b + sub] : (half)0;
+ }
+ barrier(CLK_LOCAL_MEM_FENCE);
+
+#define LD4(arr, b) ((float4)((float)arr[(b)+0], (float)arr[(b)+1], (float)arr[(b)+2], (float)arr[(b)+3]))
+ #pragma unroll
+ for (int g = 0; g < NGROUPS; ++g) {
+ const int b = g * 4;
+ float4 rf;
+ rf.s0 = (float)dot8_q8a(qw, sh_qa[b+0]); rf.s1 = (float)dot8_q8a(qw, sh_qa[b+1]);
+ rf.s2 = (float)dot8_q8a(qw, sh_qa[b+2]); rf.s3 = (float)dot8_q8a(qw, sh_qa[b+3]);
+ acc[g] += d_w * LD4(sh_d, b) * rf;
+ }
+#undef LD4
+ barrier(CLK_LOCAL_MEM_FENCE);
+ }
+
+ if (!row_valid) {
+ return;
+ }
+
+ // dst is [token, feature] row-major (stride m): dst[col*m + row].
+ #pragma unroll
+ for (int g = 0; g < NGROUPS; ++g) {
+ const uint b = (uint)(g * 4);
+ const float4 a = acc[g];
+ const uint c0 = col_base + b;
+ if (c0 + 0 < (uint)n_no_padding) dst[(c0 + 0) * (uint)m + row] = a.s0;
+ if (c0 + 1 < (uint)n_no_padding) dst[(c0 + 1) * (uint)m + row] = a.s1;
+ if (c0 + 2 < (uint)n_no_padding) dst[(c0 + 2) * (uint)m + row] = a.s2;
+ if (c0 + 3 < (uint)n_no_padding) dst[(c0 + 3) * (uint)m + row] = a.s3;
+ }
+#undef NGROUPS
+}
diff --git a/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q4_0_q8_1_dp4a.cl b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q4_0_q8_1_dp4a.cl
new file mode 100644
index 0000000000..446a8eb184
--- /dev/null
+++ b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q4_0_q8_1_dp4a.cl
@@ -0,0 +1,127 @@
+#pragma OPENCL EXTENSION cl_khr_fp16 : enable
+#pragma OPENCL EXTENSION cl_khr_subgroups : enable
+#ifdef cl_khr_integer_dot_product
+#pragma OPENCL EXTENSION cl_khr_integer_dot_product : enable
+#endif
+
+#define TILESIZE_N 32
+
+// Expand the 4 nibbles in the low 16 bits of u into 4 bytes (value 0..15),
+// packed for the int8 dp4a. The -8 zero-point is applied via the sum term.
+#define EXP4(u) ( ((uint)((u) & 0x000Fu)) | \
+ (((uint)((u) & 0x00F0u)) << 4) | \
+ (((uint)((u) & 0x0F00u)) << 8) | \
+ (((uint)((u) & 0xF000u)) << 12) )
+
+inline int dot8_q8a(uint8 qw, __local const uint * a) {
+ int r = 0;
+ r = dot_acc_sat_4x8packed_ss_int(qw.s0, a[0], r);
+ r = dot_acc_sat_4x8packed_ss_int(qw.s1, a[1], r);
+ r = dot_acc_sat_4x8packed_ss_int(qw.s2, a[2], r);
+ r = dot_acc_sat_4x8packed_ss_int(qw.s3, a[3], r);
+ r = dot_acc_sat_4x8packed_ss_int(qw.s4, a[4], r);
+ r = dot_acc_sat_4x8packed_ss_int(qw.s5, a[5], r);
+ r = dot_acc_sat_4x8packed_ss_int(qw.s6, a[6], r);
+ r = dot_acc_sat_4x8packed_ss_int(qw.s7, a[7], r);
+ return r;
+}
+
+__attribute__((qcom_wave_pair_mode(1)))
+kernel void kernel_gemm_noshuffle_q4_0_q8_1_dp4a(
+ __global const ushort * src0_q, // q4_0 nibbles (4/ushort, feature-major)
+ __global const half * src0_d, // per-32-block scale, feature-major
+ __global const uint * src1_qa, // q8_1 activations int8 (as uint, 4/elem) [N, K]
+ __global const half * src1_da, // q8_1 per-block scale [N, K/32]
+ __global const half * src1_sa, // q8_1 per-block sum*d [N, K/32]
+ __global float * dst,
+ ulong offsetd,
+ int m, // output features (rows)
+ int n_no_padding, // tokens (cols)
+ int k // K (== ne00)
+) {
+ dst = (global float *)((global char *)dst + offsetd);
+
+ const uint lid = get_local_id(0); // 0..63 -> row within the M-tile
+ const uint block_id_m = get_global_id(1);
+ const uint block_id_n = get_global_id(2);
+
+ const uint row = block_id_m * 64 + lid;
+ const uint col_base = block_id_n * TILESIZE_N;
+ const bool row_valid = row < (uint)m;
+ const uint rrow = row_valid ? row : 0; // clamp OOB rows; their writes are masked
+
+ const uint k_u = (uint)k >> 2; // K in uint (int8x4) units
+ const uint k_b = (uint)k >> 5; // blocks-of-32 along K
+
+ __local uint sh_qa[TILESIZE_N][8];
+ __local half sh_d[TILESIZE_N];
+ __local half sh_s[TILESIZE_N];
+
+#define NGROUPS (TILESIZE_N / 4)
+ float4 acc[NGROUPS];
+ #pragma unroll
+ for (int g = 0; g < NGROUPS; ++g) acc[g] = (float4)(0.0f);
+
+ for (uint step = 0; step < (uint)k; step += 32) {
+ const uint sub = step >> 5;
+
+ const float d_w = (float)src0_d[rrow + sub * (uint)m];
+
+ // 8 weight uints (32 nibbles) for this row, this 32-block. Feature-major:
+ // src0_q[row + (k/4 + u)*m], k/4 = step/4 (= step>>2). EXP4 -> dp4a int8.
+ const uint qsbase = rrow + (step >> 2) * (uint)m;
+ uint8 qw;
+ qw.s0 = EXP4(src0_q[qsbase + 0 * m]);
+ qw.s1 = EXP4(src0_q[qsbase + 1 * m]);
+ qw.s2 = EXP4(src0_q[qsbase + 2 * m]);
+ qw.s3 = EXP4(src0_q[qsbase + 3 * m]);
+ qw.s4 = EXP4(src0_q[qsbase + 4 * m]);
+ qw.s5 = EXP4(src0_q[qsbase + 5 * m]);
+ qw.s6 = EXP4(src0_q[qsbase + 6 * m]);
+ qw.s7 = EXP4(src0_q[qsbase + 7 * m]);
+
+ // cooperatively stage the 32-token x 32-K int8 activations to LDS
+ for (uint idx = lid; idx < TILESIZE_N * 8; idx += 64) {
+ const uint t = idx >> 3;
+ const uint u = idx & 7;
+ const uint c = col_base + t;
+ sh_qa[t][u] = (c < (uint)n_no_padding) ? src1_qa[c * k_u + (step >> 2) + u] : 0u;
+ }
+ if (lid < TILESIZE_N) {
+ const uint c = col_base + lid;
+ sh_d[lid] = (c < (uint)n_no_padding) ? src1_da[c * k_b + sub] : (half)0;
+ sh_s[lid] = (c < (uint)n_no_padding) ? src1_sa[c * k_b + sub] : (half)0;
+ }
+ barrier(CLK_LOCAL_MEM_FENCE);
+
+#define LD4(arr, b) ((float4)((float)arr[(b)+0], (float)arr[(b)+1], (float)arr[(b)+2], (float)arr[(b)+3]))
+ #pragma unroll
+ for (int g = 0; g < NGROUPS; ++g) {
+ const int b = g * 4;
+ float4 rf;
+ rf.s0 = (float)dot8_q8a(qw, sh_qa[b+0]); rf.s1 = (float)dot8_q8a(qw, sh_qa[b+1]);
+ rf.s2 = (float)dot8_q8a(qw, sh_qa[b+2]); rf.s3 = (float)dot8_q8a(qw, sh_qa[b+3]);
+ // q4_0: w = d*(q-8) -> d_w * (a_d * dp4a(q,qa) - 8 * a_s)
+ acc[g] += d_w * (LD4(sh_d, b) * rf - 8.0f * LD4(sh_s, b));
+ }
+#undef LD4
+ barrier(CLK_LOCAL_MEM_FENCE);
+ }
+
+ if (!row_valid) {
+ return;
+ }
+
+ // dst is [token, feature] row-major (stride m): dst[col*m + row].
+ #pragma unroll
+ for (int g = 0; g < NGROUPS; ++g) {
+ const uint b = (uint)(g * 4);
+ const float4 a = acc[g];
+ const uint c0 = col_base + b;
+ if (c0 + 0 < (uint)n_no_padding) dst[(c0 + 0) * (uint)m + row] = a.s0;
+ if (c0 + 1 < (uint)n_no_padding) dst[(c0 + 1) * (uint)m + row] = a.s1;
+ if (c0 + 2 < (uint)n_no_padding) dst[(c0 + 2) * (uint)m + row] = a.s2;
+ if (c0 + 3 < (uint)n_no_padding) dst[(c0 + 3) * (uint)m + row] = a.s3;
+ }
+#undef NGROUPS
+}
diff --git a/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q4_k_q8_1_dp4a.cl b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q4_k_q8_1_dp4a.cl
new file mode 100644
index 0000000000..8656180149
--- /dev/null
+++ b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q4_k_q8_1_dp4a.cl
@@ -0,0 +1,281 @@
+#pragma OPENCL EXTENSION cl_khr_fp16 : enable
+#pragma OPENCL EXTENSION cl_khr_subgroups : enable
+#ifdef cl_khr_integer_dot_product
+#pragma OPENCL EXTENSION cl_khr_integer_dot_product : enable
+#endif
+
+#ifndef TILESIZE_N
+#define TILESIZE_N 32
+#endif
+#define QK_K 256
+#define K_SCALE_SIZE 12
+
+inline void get_scale_min_k4(
+ int j,
+ global const uchar * q,
+ uchar * d,
+ uchar * m,
+ uchar mask_d6,
+ uchar mask_d4,
+ uchar mask_hi2
+) {
+ if (j < 4) {
+ *d = q[j] & mask_d6;
+ *m = q[j+4] & mask_d6;
+ } else {
+ *d = (q[j+4] & mask_d4) | ((q[j-4] & mask_hi2) >> 2);
+ *m = ((q[j+4] >> 4) & mask_d4) | ((q[j] & mask_hi2) >> 2);
+ }
+}
+
+// Expand the 4 nibbles in the low 16 bits of `u` into 4 bytes (one nibble per
+// byte, value 0..15), packed for the int8 dp4a.
+#define EXP4(u) ( ((uint)((u) & 0x000Fu)) | \
+ (((uint)((u) & 0x00F0u)) << 4) | \
+ (((uint)((u) & 0x0F00u)) << 8) | \
+ (((uint)((u) & 0xF000u)) << 12) )
+
+// 32-K dp4a dot of one token's int8 activations (8 packed uints in lm) against the
+// row's 8 packed weight uints. qw passed by value as a uint8 (register), not an array.
+inline int dot8_q8a(uint8 qw, __local const uint * a) {
+ int r = 0;
+ r = dot_acc_sat_4x8packed_ss_int(qw.s0, a[0], r);
+ r = dot_acc_sat_4x8packed_ss_int(qw.s1, a[1], r);
+ r = dot_acc_sat_4x8packed_ss_int(qw.s2, a[2], r);
+ r = dot_acc_sat_4x8packed_ss_int(qw.s3, a[3], r);
+ r = dot_acc_sat_4x8packed_ss_int(qw.s4, a[4], r);
+ r = dot_acc_sat_4x8packed_ss_int(qw.s5, a[5], r);
+ r = dot_acc_sat_4x8packed_ss_int(qw.s6, a[6], r);
+ r = dot_acc_sat_4x8packed_ss_int(qw.s7, a[7], r);
+ return r;
+}
+
+__attribute__((qcom_wave_pair_mode(1)))
+kernel void kernel_gemm_noshuffle_q4_k_q8_1_dp4a(
+ __global const ushort * src0_q, // q4_K weights (noshuffle, packed nibbles)
+ __global const uchar * src0_s, // 6-bit scale/min codes
+ __global const half * src0_d, // per-superblock scale
+ __global const half * src0_dm, // per-superblock min
+ __global const uint * src1_qa, // q8_1 activations int8 (as uint, 4/elem) [N, K]
+ __global const half * src1_da, // q8_1 per-block scale [N, K/32]
+ __global const half * src1_sa, // q8_1 per-block sum*d [N, K/32]
+ __global float * dst,
+ ulong offsetd,
+ int m, // output features (rows)
+ int n_no_padding, // tokens (cols)
+ int k, // K (== ne00)
+ uchar mask_d6,
+ uchar mask_d4,
+ uchar mask_hi2
+) {
+ dst = (global float *)((global char *)dst + offsetd);
+
+ const uint lid = get_local_id(0); // 0..63 -> row within the M-tile
+ const uint block_id_m = get_global_id(1);
+ const uint block_id_n = get_global_id(2);
+
+ const uint row = block_id_m * 64 + lid;
+ const uint col_base = block_id_n * TILESIZE_N;
+ const bool row_valid = row < (uint)m;
+ const uint rrow = row_valid ? row : 0; // clamp OOB rows; their writes are masked
+
+ const uint num_superblocks = (uint)k / QK_K;
+ const uint k_u = (uint)k >> 2; // K in uint (int8x4) units
+ const uint k_b = (uint)k >> 5; // blocks-of-32 along K
+
+ __local uint sh_qa[TILESIZE_N][8];
+ __local half sh_d[TILESIZE_N];
+ __local half sh_s[TILESIZE_N];
+
+ // One float4 vector-register accumulator per group of 4 tokens (NGROUPS = TILESIZE_N/4).
+#define NGROUPS (TILESIZE_N / 4)
+ float4 acc[NGROUPS];
+ #pragma unroll
+ for (int g = 0; g < NGROUPS; ++g) { acc[g] = (float4)(0.0f); }
+
+ for (uint step = 0; step < (uint)k; step += 32) {
+ const uint sub = step >> 5;
+ const uint sb_idx = step / QK_K;
+ const uint sub_idx = sub & 7;
+
+ // weight scale/min for this WI's row, this subblock
+ const float dd = (float)src0_d [rrow + sb_idx * m];
+ const float dmm = (float)src0_dm[rrow + sb_idx * m];
+ global const uchar * sc = src0_s + rrow * num_superblocks * K_SCALE_SIZE + sb_idx * K_SCALE_SIZE;
+ uchar sv, mn;
+ get_scale_min_k4(sub_idx, sc, &sv, &mn, mask_d6, mask_d4, mask_hi2);
+ const float scale = dd * (float)sv;
+ const float minv = dmm * (float)mn;
+
+ // repack this row's 32 weight nibbles into 8 dp4a uints. The packed q4_K
+ // layout stores one ushort = 4 consecutive-K nibbles for a row at
+ // src0_q[row + (K_group)*m], K_group = step/4 + u.
+ const uint wbase = rrow + (step >> 2) * (uint)m;
+ uint8 qw;
+ qw.s0 = EXP4(src0_q[wbase + 0 * m]);
+ qw.s1 = EXP4(src0_q[wbase + 1 * m]);
+ qw.s2 = EXP4(src0_q[wbase + 2 * m]);
+ qw.s3 = EXP4(src0_q[wbase + 3 * m]);
+ qw.s4 = EXP4(src0_q[wbase + 4 * m]);
+ qw.s5 = EXP4(src0_q[wbase + 5 * m]);
+ qw.s6 = EXP4(src0_q[wbase + 6 * m]);
+ qw.s7 = EXP4(src0_q[wbase + 7 * m]);
+
+ // cooperatively stage the 32-token x 32-K int8 activations to lm
+ for (uint idx = lid; idx < TILESIZE_N * 8; idx += 64) {
+ const uint t = idx >> 3;
+ const uint u = idx & 7;
+ const uint c = col_base + t;
+ sh_qa[t][u] = (c < (uint)n_no_padding) ? src1_qa[c * k_u + (step >> 2) + u] : 0u;
+ }
+ if (lid < TILESIZE_N) {
+ const uint c = col_base + lid;
+ sh_d[lid] = (c < (uint)n_no_padding) ? src1_da[c * k_b + sub] : (half)0;
+ sh_s[lid] = (c < (uint)n_no_padding) ? src1_sa[c * k_b + sub] : (half)0;
+ }
+ barrier(CLK_LOCAL_MEM_FENCE);
+
+#define LD4(arr, b) ((float4)((float)arr[(b)+0], (float)arr[(b)+1], (float)arr[(b)+2], (float)arr[(b)+3]))
+ #pragma unroll
+ for (int g = 0; g < NGROUPS; ++g) {
+ const int b = g * 4;
+ float4 rf;
+ rf.s0 = (float)dot8_q8a(qw, sh_qa[b+0]); rf.s1 = (float)dot8_q8a(qw, sh_qa[b+1]);
+ rf.s2 = (float)dot8_q8a(qw, sh_qa[b+2]); rf.s3 = (float)dot8_q8a(qw, sh_qa[b+3]);
+ acc[g] += scale * LD4(sh_d, b) * rf - minv * LD4(sh_s, b);
+ }
+#undef LD4
+ barrier(CLK_LOCAL_MEM_FENCE);
+ }
+
+ if (!row_valid) {
+ return;
+ }
+
+ // dst is [token, feature] row-major (stride m): dst[col*m + row]. Scatter each
+ // lane with a per-token padding guard (dst is non-contiguous in token).
+ #pragma unroll
+ for (int g = 0; g < NGROUPS; ++g) {
+ const uint b = (uint)(g * 4);
+ const float4 a = acc[g];
+ const uint c0 = col_base + b;
+ if (c0 + 0 < (uint)n_no_padding) dst[(c0 + 0) * (uint)m + row] = a.s0;
+ if (c0 + 1 < (uint)n_no_padding) dst[(c0 + 1) * (uint)m + row] = a.s1;
+ if (c0 + 2 < (uint)n_no_padding) dst[(c0 + 2) * (uint)m + row] = a.s2;
+ if (c0 + 3 < (uint)n_no_padding) dst[(c0 + 3) * (uint)m + row] = a.s3;
+ }
+#undef NGROUPS
+}
+
+__attribute__((qcom_wave_pair_mode(1)))
+kernel void kernel_gemm_noshuffle_q4_k_q8_1_dp4a_wimg(
+ __read_only image1d_buffer_t src0_q_img, // q4_K weights as uint32 texels (2 ushorts/texel)
+ __global const uchar * src0_s, // 6-bit scale/min codes
+ __global const half * src0_d, // per-superblock scale
+ __global const half * src0_dm, // per-superblock min
+ __global const uint * src1_qa, // q8_1 activations int8 (as uint, 4/elem) [N, K]
+ __global const half * src1_da, // q8_1 per-block scale [N, K/32]
+ __global const half * src1_sa, // q8_1 per-block sum*d [N, K/32]
+ __global float * dst,
+ ulong offsetd,
+ int m, // output features (rows)
+ int n_no_padding, // tokens (cols)
+ int k, // K (== ne00)
+ uchar mask_d6,
+ uchar mask_d4,
+ uchar mask_hi2
+) {
+ dst = (global float *)((global char *)dst + offsetd);
+
+ const uint lid = get_local_id(0); // 0..63 -> row within the M-tile
+ const uint block_id_m = get_global_id(1);
+ const uint block_id_n = get_global_id(2);
+
+ const uint row = block_id_m * 64 + lid;
+ const uint col_base = block_id_n * TILESIZE_N;
+ const bool row_valid = row < (uint)m;
+ const uint rrow = row_valid ? row : 0; // clamp OOB rows; their writes are masked
+
+ // Constant per WI: the ushort the row needs always sits in the same half of
+ // its uint32 texel (m even => index parity == rrow parity). Hoist the shift.
+ const uint sel = (rrow & 1u) * 16u;
+
+ const uint k_u = (uint)k >> 2; // K in uint (int8x4) units
+ const uint k_b = (uint)k >> 5; // blocks-of-32 along K
+ const uint num_superblocks = (uint)k / QK_K;
+
+ __local uint sh_qa[TILESIZE_N][8];
+ __local half sh_d[TILESIZE_N];
+ __local half sh_s[TILESIZE_N];
+
+#define NGROUPS (TILESIZE_N / 4)
+ float4 acc[NGROUPS];
+ #pragma unroll
+ for (int g = 0; g < NGROUPS; ++g) acc[g] = (float4)(0.0f);
+
+ for (uint step = 0; step < (uint)k; step += 32) {
+ const uint sub = step >> 5;
+ const uint sb_idx = step / QK_K;
+ const uint sub_idx = sub & 7;
+
+ const float dd = (float)src0_d [rrow + sb_idx * m];
+ const float dmm = (float)src0_dm[rrow + sb_idx * m];
+ global const uchar * sc = src0_s + rrow * num_superblocks * K_SCALE_SIZE + sb_idx * K_SCALE_SIZE;
+ uchar sv, mn;
+ get_scale_min_k4(sub_idx, sc, &sv, &mn, mask_d6, mask_d4, mask_hi2);
+ const float scale = dd * (float)sv;
+ const float minv = dmm * (float)mn;
+
+ const uint wbase = rrow + (step >> 2) * (uint)m;
+ uint8 qw;
+ qw.s0 = EXP4(read_imageui(src0_q_img, (int)((wbase + 0 * m) >> 1)).x >> sel);
+ qw.s1 = EXP4(read_imageui(src0_q_img, (int)((wbase + 1 * m) >> 1)).x >> sel);
+ qw.s2 = EXP4(read_imageui(src0_q_img, (int)((wbase + 2 * m) >> 1)).x >> sel);
+ qw.s3 = EXP4(read_imageui(src0_q_img, (int)((wbase + 3 * m) >> 1)).x >> sel);
+ qw.s4 = EXP4(read_imageui(src0_q_img, (int)((wbase + 4 * m) >> 1)).x >> sel);
+ qw.s5 = EXP4(read_imageui(src0_q_img, (int)((wbase + 5 * m) >> 1)).x >> sel);
+ qw.s6 = EXP4(read_imageui(src0_q_img, (int)((wbase + 6 * m) >> 1)).x >> sel);
+ qw.s7 = EXP4(read_imageui(src0_q_img, (int)((wbase + 7 * m) >> 1)).x >> sel);
+
+ for (uint idx = lid; idx < TILESIZE_N * 8; idx += 64) {
+ const uint t = idx >> 3;
+ const uint u = idx & 7;
+ const uint c = col_base + t;
+ sh_qa[t][u] = (c < (uint)n_no_padding) ? src1_qa[c * k_u + (step >> 2) + u] : 0u;
+ }
+ if (lid < TILESIZE_N) {
+ const uint c = col_base + lid;
+ sh_d[lid] = (c < (uint)n_no_padding) ? src1_da[c * k_b + sub] : (half)0;
+ sh_s[lid] = (c < (uint)n_no_padding) ? src1_sa[c * k_b + sub] : (half)0;
+ }
+ barrier(CLK_LOCAL_MEM_FENCE);
+
+#define LD4(arr, b) ((float4)((float)arr[(b)+0], (float)arr[(b)+1], (float)arr[(b)+2], (float)arr[(b)+3]))
+ #pragma unroll
+ for (int g = 0; g < NGROUPS; ++g) {
+ const int b = g * 4;
+ float4 rf;
+ rf.s0 = (float)dot8_q8a(qw, sh_qa[b+0]); rf.s1 = (float)dot8_q8a(qw, sh_qa[b+1]);
+ rf.s2 = (float)dot8_q8a(qw, sh_qa[b+2]); rf.s3 = (float)dot8_q8a(qw, sh_qa[b+3]);
+ acc[g] += scale * LD4(sh_d, b) * rf - minv * LD4(sh_s, b);
+ }
+#undef LD4
+ barrier(CLK_LOCAL_MEM_FENCE);
+ }
+
+ if (!row_valid) {
+ return;
+ }
+
+ #pragma unroll
+ for (int g = 0; g < NGROUPS; ++g) {
+ const uint b = (uint)(g * 4);
+ const float4 a = acc[g];
+ const uint c0 = col_base + b;
+ if (c0 + 0 < (uint)n_no_padding) dst[(c0 + 0) * (uint)m + row] = a.s0;
+ if (c0 + 1 < (uint)n_no_padding) dst[(c0 + 1) * (uint)m + row] = a.s1;
+ if (c0 + 2 < (uint)n_no_padding) dst[(c0 + 2) * (uint)m + row] = a.s2;
+ if (c0 + 3 < (uint)n_no_padding) dst[(c0 + 3) * (uint)m + row] = a.s3;
+ }
+#undef NGROUPS
+}
diff --git a/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q5_0_q8_1_dp4a.cl b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q5_0_q8_1_dp4a.cl
new file mode 100644
index 0000000000..4d1c6bdbcb
--- /dev/null
+++ b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q5_0_q8_1_dp4a.cl
@@ -0,0 +1,235 @@
+#pragma OPENCL EXTENSION cl_khr_fp16 : enable
+#pragma OPENCL EXTENSION cl_khr_subgroups : enable
+#ifdef cl_khr_integer_dot_product
+#pragma OPENCL EXTENSION cl_khr_integer_dot_product : enable
+#endif
+
+// Weight layout
+// src0_qs[row + (k/4)*m] ushort = 4 low nibbles (K = 4*grp .. +3)
+// src0_qh[row + (k/8)*m] uchar = 8 high bits (one per element)
+// src0_d [row + (k/32)*m] half = per-32-block scale
+
+#define TILESIZE_N 32
+
+// 4 nibbles in low 16 bits of u -> 4 bytes (value 0..15)
+#define EXP4(u) ( ((uint)((u) & 0x000Fu)) | \
+ (((uint)((u) & 0x00F0u)) << 4) | \
+ (((uint)((u) & 0x0F00u)) << 8) | \
+ (((uint)((u) & 0xF000u)) << 12) )
+// 4 high bits (one per element, in bits 0..3 of h) -> bit4 of each of 4 bytes
+#define EXP1(h) ( (((uint)((h) & 0x1u)) << 4) | \
+ (((uint)((h) & 0x2u)) << 11) | \
+ (((uint)((h) & 0x4u)) << 18) | \
+ (((uint)((h) & 0x8u)) << 25) )
+
+inline int dot8_q8a(uint8 qw, __local const uint * a) {
+ int r = 0;
+ r = dot_acc_sat_4x8packed_ss_int(qw.s0, a[0], r);
+ r = dot_acc_sat_4x8packed_ss_int(qw.s1, a[1], r);
+ r = dot_acc_sat_4x8packed_ss_int(qw.s2, a[2], r);
+ r = dot_acc_sat_4x8packed_ss_int(qw.s3, a[3], r);
+ r = dot_acc_sat_4x8packed_ss_int(qw.s4, a[4], r);
+ r = dot_acc_sat_4x8packed_ss_int(qw.s5, a[5], r);
+ r = dot_acc_sat_4x8packed_ss_int(qw.s6, a[6], r);
+ r = dot_acc_sat_4x8packed_ss_int(qw.s7, a[7], r);
+ return r;
+}
+
+__attribute__((qcom_wave_pair_mode(1)))
+kernel void kernel_gemm_noshuffle_q5_0_q8_1_dp4a(
+ __global const ushort * src0_qs, // q5_0 low nibbles (4/ushort, feature-major)
+ __global const uchar * src0_qh, // q5_0 high-bit plane (8/uchar, feature-major)
+ __global const half * src0_d, // per-32-block scale, feature-major
+ __global const uint * src1_qa, // q8_1 activations int8 (as uint, 4/elem) [N, K]
+ __global const half * src1_da, // q8_1 per-block scale [N, K/32]
+ __global const half * src1_sa, // q8_1 per-block sum*d [N, K/32]
+ __global float * dst,
+ ulong offsetd,
+ int m, // output features (rows)
+ int n_no_padding, // tokens (cols)
+ int k // K (== ne00)
+) {
+ dst = (global float *)((global char *)dst + offsetd);
+
+ const uint lid = get_local_id(0); // 0..63 -> row within the M-tile
+ const uint block_id_m = get_global_id(1);
+ const uint block_id_n = get_global_id(2);
+
+ const uint row = block_id_m * 64 + lid;
+ const uint col_base = block_id_n * TILESIZE_N;
+ const bool row_valid = row < (uint)m;
+ const uint rrow = row_valid ? row : 0; // clamp OOB rows; their writes are masked
+
+ const uint k_u = (uint)k >> 2; // K in uint (int8x4) units
+ const uint k_b = (uint)k >> 5; // blocks-of-32 along K
+
+ __local uint sh_qa[TILESIZE_N][8];
+ __local half sh_d[TILESIZE_N];
+ __local half sh_s[TILESIZE_N];
+
+#define NGROUPS (TILESIZE_N / 4)
+ float4 acc[NGROUPS];
+ #pragma unroll
+ for (int g = 0; g < NGROUPS; ++g) acc[g] = (float4)(0.0f);
+
+ for (uint step = 0; step < (uint)k; step += 32) {
+ const uint sub = step >> 5;
+
+ const float d_w = (float)src0_d[rrow + sub * (uint)m];
+ const float minv = d_w * 16.0f; // -16 centering -> subtract via q8_1 sum
+
+ // 8 weight uints (32 elements) for this row, this 32-block.
+ // nibbles: src0_qs[row + (step/4 + u)*m]; high bits: src0_qh[row + (step/8 + u/2)*m],
+ // 4-bit group selected by (u&1)*4.
+ const uint qsbase = rrow + (step >> 2) * (uint)m;
+ const uint qhbase = rrow + (step >> 3) * (uint)m;
+ uint8 qw;
+ #define QW(u) (EXP4(src0_qs[qsbase + (u) * m]) | \
+ EXP1((uint)(src0_qh[qhbase + ((u) >> 1) * m] >> (((u) & 1u) * 4u)) & 0xFu))
+ qw.s0 = QW(0); qw.s1 = QW(1); qw.s2 = QW(2); qw.s3 = QW(3);
+ qw.s4 = QW(4); qw.s5 = QW(5); qw.s6 = QW(6); qw.s7 = QW(7);
+ #undef QW
+
+ // cooperatively stage the 32-token x 32-K int8 activations to lm
+ for (uint idx = lid; idx < TILESIZE_N * 8; idx += 64) {
+ const uint t = idx >> 3;
+ const uint u = idx & 7;
+ const uint c = col_base + t;
+ sh_qa[t][u] = (c < (uint)n_no_padding) ? src1_qa[c * k_u + (step >> 2) + u] : 0u;
+ }
+ if (lid < TILESIZE_N) {
+ const uint c = col_base + lid;
+ sh_d[lid] = (c < (uint)n_no_padding) ? src1_da[c * k_b + sub] : (half)0;
+ sh_s[lid] = (c < (uint)n_no_padding) ? src1_sa[c * k_b + sub] : (half)0;
+ }
+ barrier(CLK_LOCAL_MEM_FENCE);
+
+#define LD4(arr, b) ((float4)((float)arr[(b)+0], (float)arr[(b)+1], (float)arr[(b)+2], (float)arr[(b)+3]))
+ #pragma unroll
+ for (int g = 0; g < NGROUPS; ++g) {
+ const int b = g * 4;
+ float4 rf;
+ rf.s0 = (float)dot8_q8a(qw, sh_qa[b+0]); rf.s1 = (float)dot8_q8a(qw, sh_qa[b+1]);
+ rf.s2 = (float)dot8_q8a(qw, sh_qa[b+2]); rf.s3 = (float)dot8_q8a(qw, sh_qa[b+3]);
+ acc[g] += d_w * LD4(sh_d, b) * rf - minv * LD4(sh_s, b);
+ }
+#undef LD4
+ barrier(CLK_LOCAL_MEM_FENCE);
+ }
+
+ if (!row_valid) {
+ return;
+ }
+
+ #pragma unroll
+ for (int g = 0; g < NGROUPS; ++g) {
+ const uint b = (uint)(g * 4);
+ const float4 a = acc[g];
+ const uint c0 = col_base + b;
+ if (c0 + 0 < (uint)n_no_padding) dst[(c0 + 0) * (uint)m + row] = a.s0;
+ if (c0 + 1 < (uint)n_no_padding) dst[(c0 + 1) * (uint)m + row] = a.s1;
+ if (c0 + 2 < (uint)n_no_padding) dst[(c0 + 2) * (uint)m + row] = a.s2;
+ if (c0 + 3 < (uint)n_no_padding) dst[(c0 + 3) * (uint)m + row] = a.s3;
+ }
+#undef NGROUPS
+}
+
+__attribute__((qcom_wave_pair_mode(1)))
+kernel void kernel_gemm_noshuffle_q5_0_q8_1_dp4a_wimg(
+ __read_only image1d_buffer_t src0_qs_img, // q5_0 low nibbles as uint32 texels (2 ushorts/texel)
+ __global const uchar * src0_qh,
+ __global const half * src0_d,
+ __global const uint * src1_qa,
+ __global const half * src1_da,
+ __global const half * src1_sa,
+ __global float * dst,
+ ulong offsetd,
+ int m,
+ int n_no_padding,
+ int k
+) {
+ dst = (global float *)((global char *)dst + offsetd);
+
+ const uint lid = get_local_id(0);
+ const uint block_id_m = get_global_id(1);
+ const uint block_id_n = get_global_id(2);
+
+ const uint row = block_id_m * 64 + lid;
+ const uint col_base = block_id_n * TILESIZE_N;
+ const bool row_valid = row < (uint)m;
+ const uint rrow = row_valid ? row : 0;
+
+ const uint sel = (rrow & 1u) * 16u; // constant per WI: qs ushort half in its uint32 texel
+
+ const uint k_u = (uint)k >> 2;
+ const uint k_b = (uint)k >> 5;
+
+ __local uint sh_qa[TILESIZE_N][8];
+ __local half sh_d[TILESIZE_N];
+ __local half sh_s[TILESIZE_N];
+
+#define NGROUPS (TILESIZE_N / 4)
+ float4 acc[NGROUPS];
+ #pragma unroll
+ for (int g = 0; g < NGROUPS; ++g) acc[g] = (float4)(0.0f);
+
+ for (uint step = 0; step < (uint)k; step += 32) {
+ const uint sub = step >> 5;
+
+ const float d_w = (float)src0_d[rrow + sub * (uint)m];
+ const float minv = d_w * 16.0f;
+
+ const uint qsbase = rrow + (step >> 2) * (uint)m; // ushort index
+ const uint qhbase = rrow + (step >> 3) * (uint)m;
+ uint8 qw;
+ // qs ushort via texture: uint32 texel = ushort_index>>1, half = sel.
+ #define QSU(u) ((read_imageui(src0_qs_img, (int)((qsbase + (u) * m) >> 1)).x >> sel) & 0xFFFFu)
+ #define QW(u) (EXP4(QSU(u)) | \
+ EXP1((uint)(src0_qh[qhbase + ((u) >> 1) * m] >> (((u) & 1u) * 4u)) & 0xFu))
+ qw.s0 = QW(0); qw.s1 = QW(1); qw.s2 = QW(2); qw.s3 = QW(3);
+ qw.s4 = QW(4); qw.s5 = QW(5); qw.s6 = QW(6); qw.s7 = QW(7);
+ #undef QW
+ #undef QSU
+
+ for (uint idx = lid; idx < TILESIZE_N * 8; idx += 64) {
+ const uint t = idx >> 3;
+ const uint u = idx & 7;
+ const uint c = col_base + t;
+ sh_qa[t][u] = (c < (uint)n_no_padding) ? src1_qa[c * k_u + (step >> 2) + u] : 0u;
+ }
+ if (lid < TILESIZE_N) {
+ const uint c = col_base + lid;
+ sh_d[lid] = (c < (uint)n_no_padding) ? src1_da[c * k_b + sub] : (half)0;
+ sh_s[lid] = (c < (uint)n_no_padding) ? src1_sa[c * k_b + sub] : (half)0;
+ }
+ barrier(CLK_LOCAL_MEM_FENCE);
+
+#define LD4(arr, b) ((float4)((float)arr[(b)+0], (float)arr[(b)+1], (float)arr[(b)+2], (float)arr[(b)+3]))
+ #pragma unroll
+ for (int g = 0; g < NGROUPS; ++g) {
+ const int b = g * 4;
+ float4 rf;
+ rf.s0 = (float)dot8_q8a(qw, sh_qa[b+0]); rf.s1 = (float)dot8_q8a(qw, sh_qa[b+1]);
+ rf.s2 = (float)dot8_q8a(qw, sh_qa[b+2]); rf.s3 = (float)dot8_q8a(qw, sh_qa[b+3]);
+ acc[g] += d_w * LD4(sh_d, b) * rf - minv * LD4(sh_s, b);
+ }
+#undef LD4
+ barrier(CLK_LOCAL_MEM_FENCE);
+ }
+
+ if (!row_valid) {
+ return;
+ }
+
+ #pragma unroll
+ for (int g = 0; g < NGROUPS; ++g) {
+ const uint b = (uint)(g * 4);
+ const float4 a = acc[g];
+ const uint c0 = col_base + b;
+ if (c0 + 0 < (uint)n_no_padding) dst[(c0 + 0) * (uint)m + row] = a.s0;
+ if (c0 + 1 < (uint)n_no_padding) dst[(c0 + 1) * (uint)m + row] = a.s1;
+ if (c0 + 2 < (uint)n_no_padding) dst[(c0 + 2) * (uint)m + row] = a.s2;
+ if (c0 + 3 < (uint)n_no_padding) dst[(c0 + 3) * (uint)m + row] = a.s3;
+ }
+#undef NGROUPS
+}
diff --git a/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q5_k_q8_1_dp4a.cl b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q5_k_q8_1_dp4a.cl
new file mode 100644
index 0000000000..aaeed68f61
--- /dev/null
+++ b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q5_k_q8_1_dp4a.cl
@@ -0,0 +1,164 @@
+#pragma OPENCL EXTENSION cl_khr_fp16 : enable
+#pragma OPENCL EXTENSION cl_khr_subgroups : enable
+#ifdef cl_khr_integer_dot_product
+#pragma OPENCL EXTENSION cl_khr_integer_dot_product : enable
+#endif
+
+#define TILESIZE_N 32
+#define QK_K 256
+#define K_SCALE_SIZE 12
+
+inline void get_scale_min_k4(
+ int j,
+ global const uchar * q,
+ uchar * d,
+ uchar * m,
+ uchar mask_d6,
+ uchar mask_d4,
+ uchar mask_hi2
+) {
+ if (j < 4) {
+ *d = q[j] & mask_d6;
+ *m = q[j+4] & mask_d6;
+ } else {
+ *d = (q[j+4] & mask_d4) | ((q[j-4] & mask_hi2) >> 2);
+ *m = ((q[j+4] >> 4) & mask_d4) | ((q[j] & mask_hi2) >> 2);
+ }
+}
+
+// 4 nibbles in the low 16 bits of `u` -> 4 bytes (value 0..15, bits 0-3).
+#define EXP4(u) ( ((uint)((u) & 0x000Fu)) | \
+ (((uint)((u) & 0x00F0u)) << 4) | \
+ (((uint)((u) & 0x0F00u)) << 8) | \
+ (((uint)((u) & 0xF000u)) << 12) )
+
+// 4 high bits (one per element, in bits 0-3 of h) -> bit 4 of each of 4 bytes,
+// so OR with EXP4 forms the 5-bit q5_K code 0..31.
+#define EXP1(h) ( (((uint)((h) & 0x1u)) << 4) | \
+ (((uint)((h) & 0x2u)) << 11) | \
+ (((uint)((h) & 0x4u)) << 18) | \
+ (((uint)((h) & 0x8u)) << 25) )
+
+inline int dot8_q8a(uint8 qw, __local const uint * a) {
+ int r = 0;
+ r = dot_acc_sat_4x8packed_ss_int(qw.s0, a[0], r);
+ r = dot_acc_sat_4x8packed_ss_int(qw.s1, a[1], r);
+ r = dot_acc_sat_4x8packed_ss_int(qw.s2, a[2], r);
+ r = dot_acc_sat_4x8packed_ss_int(qw.s3, a[3], r);
+ r = dot_acc_sat_4x8packed_ss_int(qw.s4, a[4], r);
+ r = dot_acc_sat_4x8packed_ss_int(qw.s5, a[5], r);
+ r = dot_acc_sat_4x8packed_ss_int(qw.s6, a[6], r);
+ r = dot_acc_sat_4x8packed_ss_int(qw.s7, a[7], r);
+ return r;
+}
+
+__attribute__((qcom_wave_pair_mode(1)))
+kernel void kernel_gemm_noshuffle_q5_k_q8_1_dp4a(
+ __global const ushort * src0_q, // q5_K low nibbles (transposed, ushort = 4 nibbles)
+ __global const uchar * src0_qh, // q5_K high bits (transposed, uchar = 8 elems/byte)
+ __global const uchar * src0_s, // 6-bit scale/min codes [row][superblock][12]
+ __global const half * src0_d, // per-superblock scale (transposed)
+ __global const half * src0_dm, // per-superblock min (transposed)
+ __global const uint * src1_qa, // q8_1 activations int8 (as uint, 4/elem) [N, K]
+ __global const half * src1_da, // q8_1 per-block scale [N, K/32]
+ __global const half * src1_sa, // q8_1 per-block sum*d [N, K/32]
+ __global float * dst,
+ ulong offsetd,
+ int m, // output features (rows)
+ int n_no_padding, // tokens (cols)
+ int k, // K (== ne00)
+ uchar mask_d6,
+ uchar mask_d4,
+ uchar mask_hi2
+) {
+ dst = (global float *)((global char *)dst + offsetd);
+
+ const uint lid = get_local_id(0); // 0..63 -> row within the M-tile
+ const uint block_id_m = get_global_id(1);
+ const uint block_id_n = get_global_id(2);
+
+ const uint row = block_id_m * 64 + lid;
+ const uint col_base = block_id_n * TILESIZE_N;
+ const bool row_valid = row < (uint)m;
+ const uint rrow = row_valid ? row : 0;
+
+ const uint num_superblocks = (uint)k / QK_K;
+ const uint k_u = (uint)k >> 2;
+ const uint k_b = (uint)k >> 5;
+
+ __local uint sh_qa[TILESIZE_N][8];
+ __local half sh_d[TILESIZE_N];
+ __local half sh_s[TILESIZE_N];
+
+#define NGROUPS (TILESIZE_N / 4)
+ float4 acc[NGROUPS];
+ #pragma unroll
+ for (int g = 0; g < NGROUPS; ++g) acc[g] = (float4)(0.0f);
+
+ for (uint step = 0; step < (uint)k; step += 32) {
+ const uint sub = step >> 5;
+ const uint sb_idx = step / QK_K;
+ const uint sub_idx = sub & 7;
+
+ const float dd = (float)src0_d [rrow + sb_idx * m];
+ const float dmm = (float)src0_dm[rrow + sb_idx * m];
+ global const uchar * sc = src0_s + rrow * num_superblocks * K_SCALE_SIZE + sb_idx * K_SCALE_SIZE;
+ uchar sv, mn;
+ get_scale_min_k4(sub_idx, sc, &sv, &mn, mask_d6, mask_d4, mask_hi2);
+ const float scale = dd * (float)sv;
+ const float minv = dmm * (float)mn;
+
+ // repack this row's 32 weights (nibble | high-bit) into 8 dp4a uints.
+ // ushort u -> 4 elements at K = step + u*4; its 4 high bits are nibble
+ // (u&1) of qh byte (step/8 + u/2).
+ const uint wbase = rrow + (step >> 2) * (uint)m;
+ const uint qhbase = rrow + (step >> 3) * (uint)m;
+ uint8 qw;
+#define QWU(u) ( EXP4((uint)src0_q[wbase + (uint)(u) * m]) \
+ | EXP1( (uint)((src0_qh[qhbase + (uint)((u) >> 1) * m] >> (((u) & 1) * 4)) & 0x0Fu) ) )
+ qw.s0 = QWU(0); qw.s1 = QWU(1); qw.s2 = QWU(2); qw.s3 = QWU(3);
+ qw.s4 = QWU(4); qw.s5 = QWU(5); qw.s6 = QWU(6); qw.s7 = QWU(7);
+#undef QWU
+
+ for (uint idx = lid; idx < TILESIZE_N * 8; idx += 64) {
+ const uint t = idx >> 3;
+ const uint u = idx & 7;
+ const uint c = col_base + t;
+ sh_qa[t][u] = (c < (uint)n_no_padding) ? src1_qa[c * k_u + (step >> 2) + u] : 0u;
+ }
+ if (lid < TILESIZE_N) {
+ const uint c = col_base + lid;
+ sh_d[lid] = (c < (uint)n_no_padding) ? src1_da[c * k_b + sub] : (half)0;
+ sh_s[lid] = (c < (uint)n_no_padding) ? src1_sa[c * k_b + sub] : (half)0;
+ }
+ barrier(CLK_LOCAL_MEM_FENCE);
+
+#define LD4(arr, b) ((float4)((float)arr[(b)+0], (float)arr[(b)+1], (float)arr[(b)+2], (float)arr[(b)+3]))
+ #pragma unroll
+ for (int g = 0; g < NGROUPS; ++g) {
+ const int b = g * 4;
+ float4 rf;
+ rf.s0 = (float)dot8_q8a(qw, sh_qa[b+0]); rf.s1 = (float)dot8_q8a(qw, sh_qa[b+1]);
+ rf.s2 = (float)dot8_q8a(qw, sh_qa[b+2]); rf.s3 = (float)dot8_q8a(qw, sh_qa[b+3]);
+ acc[g] += scale * LD4(sh_d, b) * rf - minv * LD4(sh_s, b);
+ }
+#undef LD4
+ barrier(CLK_LOCAL_MEM_FENCE);
+ }
+
+ if (!row_valid) {
+ return;
+ }
+
+ #pragma unroll
+ for (int g = 0; g < NGROUPS; ++g) {
+ const uint b = (uint)(g * 4);
+ const float4 a = acc[g];
+ const uint c0 = col_base + b;
+ if (c0 + 0 < (uint)n_no_padding) dst[(c0 + 0) * (uint)m + row] = a.s0;
+ if (c0 + 1 < (uint)n_no_padding) dst[(c0 + 1) * (uint)m + row] = a.s1;
+ if (c0 + 2 < (uint)n_no_padding) dst[(c0 + 2) * (uint)m + row] = a.s2;
+ if (c0 + 3 < (uint)n_no_padding) dst[(c0 + 3) * (uint)m + row] = a.s3;
+ }
+#undef NGROUPS
+}
diff --git a/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q6_k_q8_1_dp4a.cl b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q6_k_q8_1_dp4a.cl
new file mode 100644
index 0000000000..382d79fdda
--- /dev/null
+++ b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q6_k_q8_1_dp4a.cl
@@ -0,0 +1,144 @@
+#pragma OPENCL EXTENSION cl_khr_fp16 : enable
+#pragma OPENCL EXTENSION cl_khr_subgroups : enable
+#ifdef cl_khr_integer_dot_product
+#pragma OPENCL EXTENSION cl_khr_integer_dot_product : enable
+#endif
+
+#define TILESIZE_N 32
+#define QK_K 256
+
+// 4 nibbles in the low 16 bits of `u` -> 4 bytes (value 0..15, in bits 0-3).
+#define EXP4(u) ( ((uint)((u) & 0x000Fu)) | \
+ (((uint)((u) & 0x00F0u)) << 4) | \
+ (((uint)((u) & 0x0F00u)) << 8) | \
+ (((uint)((u) & 0xF000u)) << 12) )
+
+// 4 2-bit highs in byte `b` -> 4 bytes, value 0..3 in bits 4-5 (pre-multiplied
+// by 16 so it ORs with the EXP4 nibble to form q6 in 0..63).
+#define EXP2(b) ( (((uint)((b) & 0x03u)) << 4) | \
+ (((uint)((b) & 0x0Cu)) << 10) | \
+ (((uint)((b) & 0x30u)) << 16) | \
+ (((uint)((b) & 0xC0u)) << 22) )
+
+// q6 (0..63, bits 0-5 of each byte) -> (q6-32) as a signed int8 per byte.
+inline uint SIGN6(uint q6p) {
+ uint x = q6p ^ 0x20202020u;
+ uint s = x & 0x20202020u;
+ return x | (s << 1) | (s << 2);
+}
+
+// 16-K dp4a dot: 4 packed weight uints against 4 packed int8 activation uints.
+inline int dot4_q8a(uint w0, uint w1, uint w2, uint w3,
+ uint a0, uint a1, uint a2, uint a3) {
+ int r = 0;
+ r = dot_acc_sat_4x8packed_ss_int(w0, a0, r);
+ r = dot_acc_sat_4x8packed_ss_int(w1, a1, r);
+ r = dot_acc_sat_4x8packed_ss_int(w2, a2, r);
+ r = dot_acc_sat_4x8packed_ss_int(w3, a3, r);
+ return r;
+}
+
+__attribute__((qcom_wave_pair_mode(1)))
+kernel void kernel_gemm_noshuffle_q6_k_q8_1_dp4a(
+ __global const ushort * src0_ql, // q6_K low nibbles (noshuffle)
+ __global const uchar * src0_qh, // q6_K high 2-bit (uchar, 4 highs/elem)
+ __global const ushort * src0_s, // int8 scale codes (2 chars/ushort, per 16)
+ __global const half * src0_d, // per-superblock scale
+ __global const uint * src1_qa, // q8_1 activations int8 (as uint, 4/elem) [N, K]
+ __global const half * src1_da, // q8_1 per-block scale [N, K/32]
+ __global float * dst,
+ ulong offsetd,
+ int m, // output features (rows)
+ int n_no_padding, // tokens (cols)
+ int k // K (== ne00)
+) {
+ dst = (global float *)((global char *)dst + offsetd);
+
+ const uint lid = get_local_id(0); // 0..63 -> row within the M-tile
+ const uint block_id_m = get_global_id(1);
+ const uint block_id_n = get_global_id(2);
+
+ const uint row = block_id_m * 64 + lid;
+ const uint col_base = block_id_n * TILESIZE_N;
+ const bool row_valid = row < (uint)m;
+ const uint rrow = row_valid ? row : 0; // clamp OOB rows; their writes are masked
+
+ const uint k_u = (uint)k >> 2; // K in uint (int8x4) units
+ const uint k_b = (uint)k >> 5; // blocks-of-32 along K
+
+ __local uint sh_qa[TILESIZE_N][8];
+ __local half sh_d[TILESIZE_N];
+
+#define NGROUPS (TILESIZE_N / 4)
+ float4 acc[NGROUPS];
+ #pragma unroll
+ for (int g = 0; g < NGROUPS; ++g) acc[g] = (float4)(0.0f);
+
+ for (uint step = 0; step < (uint)k; step += 32) {
+ const uint sub = step >> 5; // 32-block index along K
+ const uint sb_idx = step / QK_K; // superblock index
+
+ // q6_K superblock scale + the two int8 sub-scales spanning this 32-block
+ const float dd = (float)src0_d[rrow + sb_idx * m];
+ const char2 sc = as_char2(src0_s[rrow + sub * m]);
+ const float scale0 = dd * (float)sc.s0; // K step..step+15
+ const float scale1 = dd * (float)sc.s1; // K step+16..step+31
+
+ // repack this row's 32 weights into 8 dp4a uints (4 K each). ql ushort +
+ // qh uchar are co-located at src0_*[row + (step/4 + u)*m].
+ const uint wbase = rrow + (step >> 2) * (uint)m;
+ uint qw[8];
+ #pragma unroll
+ for (int u = 0; u < 8; ++u) {
+ const uint o = wbase + (uint)u * (uint)m;
+ qw[u] = SIGN6(EXP4((uint)src0_ql[o]) | EXP2((uint)src0_qh[o]));
+ }
+
+ // cooperatively stage the 32-token x 32-K int8 activations + scale
+ for (uint idx = lid; idx < TILESIZE_N * 8; idx += 64) {
+ const uint t = idx >> 3;
+ const uint u = idx & 7;
+ const uint c = col_base + t;
+ sh_qa[t][u] = (c < (uint)n_no_padding) ? src1_qa[c * k_u + (step >> 2) + u] : 0u;
+ }
+ if (lid < TILESIZE_N) {
+ const uint c = col_base + lid;
+ sh_d[lid] = (c < (uint)n_no_padding) ? src1_da[c * k_b + sub] : (half)0;
+ }
+ barrier(CLK_LOCAL_MEM_FENCE);
+
+ #pragma unroll
+ for (int g = 0; g < NGROUPS; ++g) {
+ const int b = g * 4;
+ float4 rf;
+ #define DOT_TOK(j) { \
+ __local const uint * a = sh_qa[b + (j)]; \
+ const int raw1 = dot4_q8a(qw[0], qw[1], qw[2], qw[3], a[0], a[1], a[2], a[3]); \
+ const int raw2 = dot4_q8a(qw[4], qw[5], qw[6], qw[7], a[4], a[5], a[6], a[7]); \
+ rf.s##j = scale0 * (float)raw1 + scale1 * (float)raw2; \
+ }
+ DOT_TOK(0); DOT_TOK(1); DOT_TOK(2); DOT_TOK(3);
+ #undef DOT_TOK
+ const float4 ad = (float4)((float)sh_d[b+0], (float)sh_d[b+1], (float)sh_d[b+2], (float)sh_d[b+3]);
+ acc[g] += ad * rf;
+ }
+ barrier(CLK_LOCAL_MEM_FENCE);
+ }
+
+ if (!row_valid) {
+ return;
+ }
+
+ // dst is [token, feature] row-major (stride m): dst[col*m + row].
+ #pragma unroll
+ for (int g = 0; g < NGROUPS; ++g) {
+ const uint b = (uint)(g * 4);
+ const float4 a = acc[g];
+ const uint c0 = col_base + b;
+ if (c0 + 0 < (uint)n_no_padding) dst[(c0 + 0) * (uint)m + row] = a.s0;
+ if (c0 + 1 < (uint)n_no_padding) dst[(c0 + 1) * (uint)m + row] = a.s1;
+ if (c0 + 2 < (uint)n_no_padding) dst[(c0 + 2) * (uint)m + row] = a.s2;
+ if (c0 + 3 < (uint)n_no_padding) dst[(c0 + 3) * (uint)m + row] = a.s3;
+ }
+#undef NGROUPS
+}
diff --git a/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q8_0_q8_1_dp4a.cl b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q8_0_q8_1_dp4a.cl
new file mode 100644
index 0000000000..a481636c23
--- /dev/null
+++ b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q8_0_q8_1_dp4a.cl
@@ -0,0 +1,212 @@
+#pragma OPENCL EXTENSION cl_khr_fp16 : enable
+#pragma OPENCL EXTENSION cl_khr_subgroups : enable
+#ifdef cl_khr_integer_dot_product
+#pragma OPENCL EXTENSION cl_khr_integer_dot_product : enable
+#endif
+
+// ne1<=8 keeps the f16 / bin small-batch path.
+
+#define TILESIZE_N 32
+
+// 32-K dp4a dot of one token's int8 activations (8 packed uints in lm) against
+// 8 packed weight uints. q8_0 weights are already dp4a-format signed int8.
+inline int dot8_q8a(uint8 qw, __local const uint * a) {
+ int r = 0;
+ r = dot_acc_sat_4x8packed_ss_int(qw.s0, a[0], r);
+ r = dot_acc_sat_4x8packed_ss_int(qw.s1, a[1], r);
+ r = dot_acc_sat_4x8packed_ss_int(qw.s2, a[2], r);
+ r = dot_acc_sat_4x8packed_ss_int(qw.s3, a[3], r);
+ r = dot_acc_sat_4x8packed_ss_int(qw.s4, a[4], r);
+ r = dot_acc_sat_4x8packed_ss_int(qw.s5, a[5], r);
+ r = dot_acc_sat_4x8packed_ss_int(qw.s6, a[6], r);
+ r = dot_acc_sat_4x8packed_ss_int(qw.s7, a[7], r);
+ return r;
+}
+
+__attribute__((qcom_wave_pair_mode(1)))
+kernel void kernel_gemm_noshuffle_q8_0_q8_1_dp4a(
+ __global const uint * src0_q, // q8_0 weights: signed int8, 4/uint, feature-major
+ __global const half * src0_d, // per-32-block scale, feature-major [row + (k/32)*m]
+ __global const uint * src1_qa, // q8_1 activations int8 (as uint, 4/elem) [N, K]
+ __global const half * src1_da, // q8_1 per-block scale [N, K/32]
+ __global float * dst,
+ ulong offsetd,
+ int m, // output features (rows)
+ int n_no_padding, // tokens (cols)
+ int k // K (== ne00)
+) {
+ dst = (global float *)((global char *)dst + offsetd);
+
+ const uint lid = get_local_id(0); // 0..63 -> row within the M-tile
+ const uint block_id_m = get_global_id(1);
+ const uint block_id_n = get_global_id(2);
+
+ const uint row = block_id_m * 64 + lid;
+ const uint col_base = block_id_n * TILESIZE_N;
+ const bool row_valid = row < (uint)m;
+ const uint rrow = row_valid ? row : 0; // clamp OOB rows; their writes are masked
+
+ const uint k_u = (uint)k >> 2; // K in uint (int8x4) units
+ const uint k_b = (uint)k >> 5; // blocks-of-32 along K
+
+ __local uint sh_qa[TILESIZE_N][8];
+ __local half sh_d[TILESIZE_N];
+
+#define NGROUPS (TILESIZE_N / 4)
+ float4 acc[NGROUPS];
+ #pragma unroll
+ for (int g = 0; g < NGROUPS; ++g) acc[g] = (float4)(0.0f);
+
+ for (uint step = 0; step < (uint)k; step += 32) {
+ const uint sub = step >> 5;
+
+ const float d_w = (float)src0_d[rrow + sub * (uint)m];
+
+ // 8 weight uints (32 int8) for this row, this 32-block. Feature-major:
+ // src0_q[row + (k/4 + u)*m], k/4 = step/4 (= step>>2).
+ const uint wbase = rrow + (step >> 2) * (uint)m;
+ uint8 qw;
+ qw.s0 = src0_q[wbase + 0 * m];
+ qw.s1 = src0_q[wbase + 1 * m];
+ qw.s2 = src0_q[wbase + 2 * m];
+ qw.s3 = src0_q[wbase + 3 * m];
+ qw.s4 = src0_q[wbase + 4 * m];
+ qw.s5 = src0_q[wbase + 5 * m];
+ qw.s6 = src0_q[wbase + 6 * m];
+ qw.s7 = src0_q[wbase + 7 * m];
+
+ // cooperatively stage the 32-token x 32-K int8 activations to LDS
+ for (uint idx = lid; idx < TILESIZE_N * 8; idx += 64) {
+ const uint t = idx >> 3;
+ const uint u = idx & 7;
+ const uint c = col_base + t;
+ sh_qa[t][u] = (c < (uint)n_no_padding) ? src1_qa[c * k_u + (step >> 2) + u] : 0u;
+ }
+ if (lid < TILESIZE_N) {
+ const uint c = col_base + lid;
+ sh_d[lid] = (c < (uint)n_no_padding) ? src1_da[c * k_b + sub] : (half)0;
+ }
+ barrier(CLK_LOCAL_MEM_FENCE);
+
+#define LD4(arr, b) ((float4)((float)arr[(b)+0], (float)arr[(b)+1], (float)arr[(b)+2], (float)arr[(b)+3]))
+ #pragma unroll
+ for (int g = 0; g < NGROUPS; ++g) {
+ const int b = g * 4;
+ float4 rf;
+ rf.s0 = (float)dot8_q8a(qw, sh_qa[b+0]); rf.s1 = (float)dot8_q8a(qw, sh_qa[b+1]);
+ rf.s2 = (float)dot8_q8a(qw, sh_qa[b+2]); rf.s3 = (float)dot8_q8a(qw, sh_qa[b+3]);
+ acc[g] += d_w * LD4(sh_d, b) * rf;
+ }
+#undef LD4
+ barrier(CLK_LOCAL_MEM_FENCE);
+ }
+
+ if (!row_valid) {
+ return;
+ }
+
+ // dst is [token, feature] row-major (stride m): dst[col*m + row].
+ #pragma unroll
+ for (int g = 0; g < NGROUPS; ++g) {
+ const uint b = (uint)(g * 4);
+ const float4 a = acc[g];
+ const uint c0 = col_base + b;
+ if (c0 + 0 < (uint)n_no_padding) dst[(c0 + 0) * (uint)m + row] = a.s0;
+ if (c0 + 1 < (uint)n_no_padding) dst[(c0 + 1) * (uint)m + row] = a.s1;
+ if (c0 + 2 < (uint)n_no_padding) dst[(c0 + 2) * (uint)m + row] = a.s2;
+ if (c0 + 3 < (uint)n_no_padding) dst[(c0 + 3) * (uint)m + row] = a.s3;
+ }
+#undef NGROUPS
+}
+
+__attribute__((qcom_wave_pair_mode(1)))
+kernel void kernel_gemm_noshuffle_q8_0_q8_1_dp4a_wimg(
+ __read_only image1d_buffer_t src0_q_img, // q8_0 weights as uint32 texels (4 int8/texel)
+ __global const half * src0_d,
+ __global const uint * src1_qa,
+ __global const half * src1_da,
+ __global float * dst,
+ ulong offsetd,
+ int m,
+ int n_no_padding,
+ int k
+) {
+ dst = (global float *)((global char *)dst + offsetd);
+
+ const uint lid = get_local_id(0);
+ const uint block_id_m = get_global_id(1);
+ const uint block_id_n = get_global_id(2);
+
+ const uint row = block_id_m * 64 + lid;
+ const uint col_base = block_id_n * TILESIZE_N;
+ const bool row_valid = row < (uint)m;
+ const uint rrow = row_valid ? row : 0;
+
+ const uint k_u = (uint)k >> 2;
+ const uint k_b = (uint)k >> 5;
+
+ __local uint sh_qa[TILESIZE_N][8];
+ __local half sh_d[TILESIZE_N];
+
+#define NGROUPS (TILESIZE_N / 4)
+ float4 acc[NGROUPS];
+ #pragma unroll
+ for (int g = 0; g < NGROUPS; ++g) acc[g] = (float4)(0.0f);
+
+ for (uint step = 0; step < (uint)k; step += 32) {
+ const uint sub = step >> 5;
+
+ const float d_w = (float)src0_d[rrow + sub * (uint)m];
+
+ const uint wbase = rrow + (step >> 2) * (uint)m;
+ uint8 qw;
+ qw.s0 = read_imageui(src0_q_img, (int)(wbase + 0 * m)).x;
+ qw.s1 = read_imageui(src0_q_img, (int)(wbase + 1 * m)).x;
+ qw.s2 = read_imageui(src0_q_img, (int)(wbase + 2 * m)).x;
+ qw.s3 = read_imageui(src0_q_img, (int)(wbase + 3 * m)).x;
+ qw.s4 = read_imageui(src0_q_img, (int)(wbase + 4 * m)).x;
+ qw.s5 = read_imageui(src0_q_img, (int)(wbase + 5 * m)).x;
+ qw.s6 = read_imageui(src0_q_img, (int)(wbase + 6 * m)).x;
+ qw.s7 = read_imageui(src0_q_img, (int)(wbase + 7 * m)).x;
+
+ for (uint idx = lid; idx < TILESIZE_N * 8; idx += 64) {
+ const uint t = idx >> 3;
+ const uint u = idx & 7;
+ const uint c = col_base + t;
+ sh_qa[t][u] = (c < (uint)n_no_padding) ? src1_qa[c * k_u + (step >> 2) + u] : 0u;
+ }
+ if (lid < TILESIZE_N) {
+ const uint c = col_base + lid;
+ sh_d[lid] = (c < (uint)n_no_padding) ? src1_da[c * k_b + sub] : (half)0;
+ }
+ barrier(CLK_LOCAL_MEM_FENCE);
+
+#define LD4(arr, b) ((float4)((float)arr[(b)+0], (float)arr[(b)+1], (float)arr[(b)+2], (float)arr[(b)+3]))
+ #pragma unroll
+ for (int g = 0; g < NGROUPS; ++g) {
+ const int b = g * 4;
+ float4 rf;
+ rf.s0 = (float)dot8_q8a(qw, sh_qa[b+0]); rf.s1 = (float)dot8_q8a(qw, sh_qa[b+1]);
+ rf.s2 = (float)dot8_q8a(qw, sh_qa[b+2]); rf.s3 = (float)dot8_q8a(qw, sh_qa[b+3]);
+ acc[g] += d_w * LD4(sh_d, b) * rf;
+ }
+#undef LD4
+ barrier(CLK_LOCAL_MEM_FENCE);
+ }
+
+ if (!row_valid) {
+ return;
+ }
+
+ #pragma unroll
+ for (int g = 0; g < NGROUPS; ++g) {
+ const uint b = (uint)(g * 4);
+ const float4 a = acc[g];
+ const uint c0 = col_base + b;
+ if (c0 + 0 < (uint)n_no_padding) dst[(c0 + 0) * (uint)m + row] = a.s0;
+ if (c0 + 1 < (uint)n_no_padding) dst[(c0 + 1) * (uint)m + row] = a.s1;
+ if (c0 + 2 < (uint)n_no_padding) dst[(c0 + 2) * (uint)m + row] = a.s2;
+ if (c0 + 3 < (uint)n_no_padding) dst[(c0 + 3) * (uint)m + row] = a.s3;
+ }
+#undef NGROUPS
+}
diff --git a/ggml/src/ggml-opencl/kernels/gemv_moe_mxfp4_f32_ns.cl b/ggml/src/ggml-opencl/kernels/gemv_moe_mxfp4_f32_ns.cl
index 75129e20c6..ee8b94f446 100644
--- a/ggml/src/ggml-opencl/kernels/gemv_moe_mxfp4_f32_ns.cl
+++ b/ggml/src/ggml-opencl/kernels/gemv_moe_mxfp4_f32_ns.cl
@@ -163,3 +163,95 @@ __kernel void kernel_gemv_moe_mxfp4_f32_ns(
}
}
+
+__attribute__((qcom_reqd_sub_group_size("half")))
+__kernel void kernel_gemv_moe_mxfp4_f32_ns_wimg(
+ __read_only image1d_buffer_t src0_q,
+ __global uchar * src0_e,
+ __read_only image1d_buffer_t src1,
+ __global uint * src2,
+ __global float * dst,
+ ulong offsetd,
+ int ne00,
+ int ne01,
+ int ne11
+) {
+ uint i01 = get_global_id(0);
+ uint i20 = get_global_id(2);
+ uint sgid = get_local_id(1);
+ uint slid = get_sub_group_local_id();
+
+ if (i01 >= ne01) {
+ return;
+ }
+
+ uint i11 = i20 % ne11;
+
+ uint expert_id = src2[i20];
+ uint expert_offset = expert_id * ne00 * ne01 / 32;
+
+ __private float sum = 0.0f;
+
+ for (uint ib00 = sgid; ib00 < (ne00 / QK_MXFP4); ib00 += N_SIMDGROUP) {
+
+ uint4 regQ;
+ uint block_offset = expert_offset * 4 + ib00 * ne01 * 4 + i01;
+
+ regQ.s0 = read_imageui(src0_q, (int)(block_offset)).x;
+ regQ.s1 = read_imageui(src0_q, (int)(block_offset + ne01)).x;
+ regQ.s2 = read_imageui(src0_q, (int)(block_offset + ne01 * 2)).x;
+ regQ.s3 = read_imageui(src0_q, (int)(block_offset + ne01 * 3)).x;
+
+ uint offset = i11 * ne00 / 4 + ib00 * 8;
+
+ half8 fp16x8 = mxfp4_to_fp16_packed8(as_ushort2(regQ.s0));
+
+ float4 shared_y4;
+ shared_y4 = read_imagef(src1, (offset + 0));
+ float4 acc = shared_y4 * convert_float4(fp16x8.lo);
+
+ shared_y4 = read_imagef(src1, (offset + 1));
+ acc += shared_y4 * convert_float4(fp16x8.hi);
+
+ fp16x8 = mxfp4_to_fp16_packed8(as_ushort2(regQ.s1));
+
+ shared_y4 = read_imagef(src1, (offset + 2));
+ acc += shared_y4 * convert_float4(fp16x8.lo);
+
+ shared_y4 = read_imagef(src1, (offset + 3));
+ acc += shared_y4 * convert_float4(fp16x8.hi);
+
+ fp16x8 = mxfp4_to_fp16_packed8(as_ushort2(regQ.s2));
+
+ shared_y4 = read_imagef(src1, (offset + 4));
+ acc += shared_y4 * convert_float4(fp16x8.lo);
+
+ shared_y4 = read_imagef(src1, (offset + 5));
+ acc += shared_y4 * convert_float4(fp16x8.hi);
+
+ fp16x8 = mxfp4_to_fp16_packed8(as_ushort2(regQ.s3));
+
+ shared_y4 = read_imagef(src1, (offset + 6));
+ acc += shared_y4 * convert_float4(fp16x8.lo);
+
+ shared_y4 = read_imagef(src1, (offset + 7));
+ acc += shared_y4 * convert_float4(fp16x8.hi);
+
+ uchar regE = src0_e[ib00 * ne01 + i01 + expert_offset];
+ sum += e8m0_to_fp32(regE) * ((acc.s0 + acc.s1) + (acc.s2 + acc.s3));
+ }
+
+ __local float reduceLM[SIMDGROUP_WIDTH * (N_SIMDGROUP - 1)];
+ if (sgid == 1) reduceLM[SIMDGROUP_WIDTH * 0 + slid] = sum;
+ if (sgid == 2) reduceLM[SIMDGROUP_WIDTH * 1 + slid] = sum;
+ if (sgid == 3) reduceLM[SIMDGROUP_WIDTH * 2 + slid] = sum;
+ barrier(CLK_LOCAL_MEM_FENCE);
+ if (sgid == 0) sum += reduceLM[SIMDGROUP_WIDTH * 0 + slid];
+ if (sgid == 0) sum += reduceLM[SIMDGROUP_WIDTH * 1 + slid];
+ if (sgid == 0) sum += reduceLM[SIMDGROUP_WIDTH * 2 + slid];
+
+ if (sgid == 0) {
+ dst = dst + (offsetd >> 2);
+ dst[i01 + i20 * ne01] = sum;
+ }
+}
diff --git a/ggml/src/ggml-opencl/kernels/gemv_moe_q4_k_f32_ns.cl b/ggml/src/ggml-opencl/kernels/gemv_moe_q4_k_f32_ns.cl
index 12464e9826..d3a3c7db87 100644
--- a/ggml/src/ggml-opencl/kernels/gemv_moe_q4_k_f32_ns.cl
+++ b/ggml/src/ggml-opencl/kernels/gemv_moe_q4_k_f32_ns.cl
@@ -153,3 +153,114 @@ __kernel void kernel_gemv_moe_q4_k_f32_ns(
dst[i01 + i20 * ne01] = sum;
}
}
+
+__attribute__((qcom_reqd_sub_group_size("half")))
+__kernel void kernel_gemv_moe_q4_k_f32_ns_wimg(
+ __read_only image1d_buffer_t src0_q,
+ __global half * src0_d,
+ __global half * src0_dm,
+ __global uchar * src0_s,
+ __read_only image1d_buffer_t src1,
+ __global uint * src2,
+ __global float * dst,
+ ulong offsetd,
+ int ne00,
+ int ne01,
+ int ne11
+) {
+ uint i01 = get_global_id(0);
+ uint i20 = get_global_id(2);
+ uint sgid = get_local_id(1);
+ uint slid = get_sub_group_local_id();
+
+ if (i01 >= ne01) {
+ return;
+ }
+
+ uint i11 = i20 % ne11;
+
+ uint expert_id = src2[i20];
+
+ int num_superblocks = ne00 / QK_K;
+ int num_subblocks = ne00 / 32;
+ int scales_per_row = num_superblocks * K_SCALE_SIZE;
+
+ uint expert_q_offset = expert_id * (ne00 / 8) * ne01;
+ uint expert_d_offset = expert_id * num_superblocks * ne01;
+
+ __private float sum = 0.0f;
+
+ for (uint ib = sgid; ib < num_subblocks; ib += N_SIMDGROUP) {
+ uint sb = ib / 8;
+ uint j = ib % 8;
+
+ half d_val = src0_d[expert_d_offset + sb * ne01 + i01];
+ half dm_val = src0_dm[expert_d_offset + sb * ne01 + i01];
+
+ global const uchar * sc = src0_s + (expert_id * ne01 + i01) * scales_per_row + sb * K_SCALE_SIZE;
+ uchar sv, mn;
+ get_scale_min_k4(j, sc, &sv, &mn);
+
+ float scale = (float)d_val * (float)sv;
+ float minv = (float)dm_val * (float)mn;
+
+ uint q_base = expert_q_offset + ib * ne01 * 4 + i01;
+
+ uint4 regQ;
+ regQ.s0 = read_imageui(src0_q, (int)(q_base)).x;
+ regQ.s1 = read_imageui(src0_q, (int)(q_base + ne01)).x;
+ regQ.s2 = read_imageui(src0_q, (int)(q_base + ne01 * 2)).x;
+ regQ.s3 = read_imageui(src0_q, (int)(q_base + ne01 * 3)).x;
+
+ uint y_offset = i11 * ne00 / 4 + ib * 8;
+
+ float8 fp32x8 = q4_k_to_fp32_packed8(as_ushort2(regQ.s0), scale, minv);
+
+ float4 shared_y4;
+ shared_y4 = read_imagef(src1, (y_offset + 0));
+ float4 acc = shared_y4 * fp32x8.lo;
+
+ shared_y4 = read_imagef(src1, (y_offset + 1));
+ acc += shared_y4 * fp32x8.hi;
+
+ fp32x8 = q4_k_to_fp32_packed8(as_ushort2(regQ.s1), scale, minv);
+
+ shared_y4 = read_imagef(src1, (y_offset + 2));
+ acc += shared_y4 * fp32x8.lo;
+
+ shared_y4 = read_imagef(src1, (y_offset + 3));
+ acc += shared_y4 * fp32x8.hi;
+
+ fp32x8 = q4_k_to_fp32_packed8(as_ushort2(regQ.s2), scale, minv);
+
+ shared_y4 = read_imagef(src1, (y_offset + 4));
+ acc += shared_y4 * fp32x8.lo;
+
+ shared_y4 = read_imagef(src1, (y_offset + 5));
+ acc += shared_y4 * fp32x8.hi;
+
+ fp32x8 = q4_k_to_fp32_packed8(as_ushort2(regQ.s3), scale, minv);
+
+ shared_y4 = read_imagef(src1, (y_offset + 6));
+ acc += shared_y4 * fp32x8.lo;
+
+ shared_y4 = read_imagef(src1, (y_offset + 7));
+ acc += shared_y4 * fp32x8.hi;
+
+ sum += ((acc.s0 + acc.s1) + (acc.s2 + acc.s3));
+ }
+
+ __local float reduceLM[SIMDGROUP_WIDTH * (N_SIMDGROUP - 1)];
+ if (sgid == 1) reduceLM[SIMDGROUP_WIDTH * 0 + slid] = sum;
+ if (sgid == 2) reduceLM[SIMDGROUP_WIDTH * 1 + slid] = sum;
+ if (sgid == 3) reduceLM[SIMDGROUP_WIDTH * 2 + slid] = sum;
+ barrier(CLK_LOCAL_MEM_FENCE);
+ if (sgid == 0) sum += reduceLM[SIMDGROUP_WIDTH * 0 + slid];
+ if (sgid == 0) sum += reduceLM[SIMDGROUP_WIDTH * 1 + slid];
+ if (sgid == 0) sum += reduceLM[SIMDGROUP_WIDTH * 2 + slid];
+
+ if (sgid == 0) {
+ dst = dst + (offsetd >> 2);
+ dst[i01 + i20 * ne01] = sum;
+ }
+}
diff --git a/ggml/src/ggml-opencl/kernels/moe_combine.cl b/ggml/src/ggml-opencl/kernels/moe_combine.cl
new file mode 100644
index 0000000000..c195f14728
--- /dev/null
+++ b/ggml/src/ggml-opencl/kernels/moe_combine.cl
@@ -0,0 +1,36 @@
+// Fused MoE combine epilogue: replaces the router-weight MUL + the (n_expert_used-1)
+// cross-expert ADD chain with ONE weighted-sum-across-experts pass.
+// dst[row, tok] = sum_e experts[row, e, tok] * weights[0, e, tok]
+// experts: [n_embd, n_expert_used, n_tokens] f32 (contiguous after down-proj GEMM)
+// weights: [1, n_expert_used, n_tokens] f32
+// dst: [n_embd, n_tokens] f32
+// One read of experts + one write of dst (eliminates the intermediate weighted
+// buffer and the k-1 elementwise add round-trips). Vectorized float4 over rows.
+// strides e1/e2/w1/w2/d1 are in ELEMENTS (floats).
+
+__kernel void kernel_moe_combine_f32(
+ __global const char * e_buf, ulong off_e,
+ __global const char * w_buf, ulong off_w,
+ __global char * d_buf, ulong off_d,
+ int n_embd4, // n_embd / 4
+ int k, // n_expert_used
+ int n_tokens,
+ uint e1, uint e2, // experts strides (elements): per-expert, per-token
+ uint w1, uint w2, // weights strides (elements)
+ uint d1) // dst per-token stride (elements)
+{
+ const uint r4 = get_global_id(0);
+ const uint tok = get_global_id(1);
+ if (r4 >= (uint)n_embd4 || tok >= (uint)n_tokens) return;
+
+ __global const float * E = (__global const float *)(e_buf + off_e) + tok*e2 + r4*4u;
+ __global const float * W = (__global const float *)(w_buf + off_w) + tok*w2;
+
+ float4 acc = (float4)(0.0f);
+ for (int e = 0; e < k; ++e) {
+ acc = mad(vload4(0, E + (uint)e*e1), (float4)(W[(uint)e*w1]), acc);
+ }
+
+ __global float * D = (__global float *)(d_buf + off_d) + tok*d1 + r4*4u;
+ vstore4(acc, 0, D);
+}
diff --git a/ggml/src/ggml-opencl/kernels/moe_reorder_quant_a_q8_1.cl b/ggml/src/ggml-opencl/kernels/moe_reorder_quant_a_q8_1.cl
new file mode 100644
index 0000000000..0d16f3abdb
--- /dev/null
+++ b/ggml/src/ggml-opencl/kernels/moe_reorder_quant_a_q8_1.cl
@@ -0,0 +1,64 @@
+#pragma OPENCL EXTENSION cl_khr_fp16 : enable
+
+// Fused MoE activation reorder + q8_1 quantization for the dp4a prefill GEMM.
+// Combines kernel_moe_reorder_b (gather src1 rows per the post-router map) with
+// the q8_1 quant pre-pass, so the f32 reordered-activation tile buffer is never
+// materialised (saves a full write + read of [tok_slots * ne00] floats).
+//
+// One work-item per (token_slot, 32-block). Padding lanes (router 0xFFFFFFFF)
+// emit d=0,s=0,qs=0 so they contribute nothing to the GEMM, exactly as the
+// reorder zero-fill did. Output layout matches kernel_moe_quant_a_q8_1:
+// qa[token_slot*K + blk*32 + i], da/sa[token_slot*(K/32) + blk].
+__kernel void kernel_moe_reorder_quant_a_q8_1(
+ __global const float * src, // original activations (offset applied)
+ __global const uint * router, // post-router indices [tok_slots]
+ __global char * qa,
+ __global half * da,
+ __global half * sa,
+ __global const int * total_tiles,
+ uint K,
+ ushort map_ratio,
+ uint tile_size,
+ uint n_kblocks // K / 32
+) {
+ const uint blk = get_global_id(0); // 32-block along K
+ const uint tok = get_global_id(1); // token slot (post_router_idx)
+
+ if (blk >= n_kblocks || tok >= (uint)total_tiles[0] * tile_size) {
+ return;
+ }
+
+ const uint out_base = tok * K + blk * 32;
+ const uint bidx = tok * n_kblocks + blk;
+
+ const uint router_idx = router[tok];
+
+ float v[32];
+ float amax = 0.0f;
+ if (router_idx == 0xFFFFFFFF) {
+ #pragma unroll
+ for (int i = 0; i < 32; ++i) v[i] = 0.0f;
+ } else {
+ const uint act_idx = router_idx / map_ratio;
+ const uint in_base = act_idx * K + blk * 32;
+ #pragma unroll
+ for (int i = 0; i < 32; ++i) {
+ v[i] = src[in_base + i];
+ amax = fmax(amax, fabs(v[i]));
+ }
+ }
+
+ const float d = amax / 127.0f;
+ const float id = (amax > 0.0f) ? (127.0f / amax) : 0.0f;
+
+ int sum = 0;
+ #pragma unroll
+ for (int i = 0; i < 32; ++i) {
+ const int q = (int)rint(v[i] * id);
+ qa[out_base + i] = (char)q;
+ sum += q;
+ }
+
+ da[bidx] = (half)d;
+ sa[bidx] = (half)(d * (float)sum);
+}
diff --git a/ggml/src/ggml-opencl/kernels/quant_a_q8_1.cl b/ggml/src/ggml-opencl/kernels/quant_a_q8_1.cl
new file mode 100644
index 0000000000..3ce06305a3
--- /dev/null
+++ b/ggml/src/ggml-opencl/kernels/quant_a_q8_1.cl
@@ -0,0 +1,42 @@
+#pragma OPENCL EXTENSION cl_khr_fp16 : enable
+
+// Quantize a contiguous [N, K] f32 activation buffer (token-major, K contiguous
+// per token) into q8_1 blocks of 32: int8 quants + per-block scale d + per-block
+// sum s (= d * Sum(qs)). Consumed by kernel_gemm_noshuffle_q4_k_q8_1_dp4a for the
+// dp4a (int8) dense q4_K prefill GEMM. One work-item per 32-element block.
+__kernel void kernel_quant_a_q8_1(
+ __global const float * src, // [N * K]
+ __global char * qa, // [N * K]
+ __global half * da, // [N * (K/32)]
+ __global half * sa, // [N * (K/32)]
+ int total_blocks // N * (K/32)
+) {
+ const int blk = get_global_id(0);
+ if (blk >= total_blocks) {
+ return;
+ }
+
+ const int base = blk * 32;
+
+ float v[32];
+ float amax = 0.0f;
+ #pragma unroll
+ for (int i = 0; i < 32; ++i) {
+ v[i] = src[base + i];
+ amax = fmax(amax, fabs(v[i]));
+ }
+
+ const float d = amax / 127.0f;
+ const float id = (amax > 0.0f) ? (127.0f / amax) : 0.0f;
+
+ int sum = 0;
+ #pragma unroll
+ for (int i = 0; i < 32; ++i) {
+ const int q = (int)rint(v[i] * id);
+ qa[base + i] = (char)q;
+ sum += q;
+ }
+
+ da[blk] = (half)d;
+ sa[blk] = (half)(d * (float)sum);
+}