LLM inference in C/C++
C++ ggml
opencl: add int8 dp4 dense and MoE prefill optimization for Adreno GPUs (#25537)
Hongqiang Wang1d1d9a9ed7a4
diff --git a/ggml/src/ggml-opencl/CMakeLists.txt b/ggml/src/ggml-opencl/CMakeLists.txtindex 9ec3268b7f..ff3ad7e34d 100644--- a/ggml/src/ggml-opencl/CMakeLists.txt+++ b/ggml/src/ggml-opencl/CMakeLists.txt@@ -114,7 +114,9 @@ set(GGML_OPENCL_KERNELSmul_mv_id_mxfp4_f32mul_mv_id_mxfp4_f32_flatgemm_moe_q4_0_f32_ns+ gemm_moe_q4_0_q8_1_dp4agemv_moe_q4_0_f32_ns+ gemm_moe_q8_0_f32_nsgemm_moe_q4_1_f32_nsgemv_moe_q4_1_f32_nsgemm_moe_q5_0_f32_ns@@ -122,6 +124,18 @@ set(GGML_OPENCL_KERNELSgemm_moe_q5_1_f32_nsgemv_moe_q5_1_f32_nsgemm_moe_q4_k_f32_ns+ gemm_moe_q4_k_q8_1_dp4a+ gemm_moe_q6_k_q8_1_dp4a+ gemm_moe_q8_1_dp4a+ moe_reorder_quant_a_q8_1+ gemm_noshuffle_q4_k_q8_1_dp4a+ gemm_noshuffle_q5_k_q8_1_dp4a+ gemm_noshuffle_q6_k_q8_1_dp4a+ gemm_noshuffle_q8_0_q8_1_dp4a+ gemm_noshuffle_q5_0_q8_1_dp4a+ gemm_noshuffle_iq4_nl_q8_1_dp4a+ gemm_noshuffle_q4_0_q8_1_dp4a+ quant_a_q8_1gemv_moe_q4_k_f32_nsgemm_moe_q5_k_f32_nsgemv_moe_q5_k_f32_ns@@ -130,8 +144,10 @@ set(GGML_OPENCL_KERNELSgemm_moe_mxfp4_f32gemv_moe_mxfp4_f32gemm_moe_mxfp4_f32_ns+ gemm_moe_mxfp4_q8_1_dp4agemv_moe_mxfp4_f32_nsmoe_reorder_b+ moe_combinemoe_sort_by_expertmul_mm_f32_f32_l4_lmmul_mm_f16_f32_l4_lmdiff --git a/ggml/src/ggml-opencl/ggml-opencl.cpp b/ggml/src/ggml-opencl/ggml-opencl.cppindex 5c96b9a9f6..f283f65690 100644--- a/ggml/src/ggml-opencl/ggml-opencl.cpp+++ b/ggml/src/ggml-opencl/ggml-opencl.cpp@@ -538,6 +538,10 @@ struct ggml_backend_opencl_context {// ragged moe, use int to directly pass to kernelcl_uint adreno_use_moe_ragged;cl_uint adreno_moe_ragged_skip_gran;+ cl_uint adreno_use_moe_ragged_dp4;++ // whether fuse moe combine+ cl_uint fuse_moe_combine;bool adreno_has_large_buffer;bool adreno_use_large_buffer;@@ -563,6 +567,12 @@ struct ggml_backend_opencl_context {ggml_cl_buffer prealloc_quant_trans;ggml_cl_buffer prealloc_scales_trans;ggml_cl_buffer prealloc_act_trans;+ // q8_1-quantized reordered MoE activations for the dp4a prefill GEMM.+ ggml_cl_buffer prealloc_moe_qa; // int8 quants [tok_slots * ne00]+ ggml_cl_buffer prealloc_moe_da; // per-block d [tok_slots * ne00/32] (half)+ ggml_cl_buffer prealloc_moe_sa; // per-block s [tok_slots * ne00/32] (half)+ // scratch copy of the router weights to avoid dst aliasing+ ggml_cl_buffer prealloc_moe_combine_w;// pool of persistent image1d_buffer views over kv-cache layers, keyed by// (parent buffer, offset within parent)@@ -816,19 +826,33 @@ struct ggml_backend_opencl_context {// [size_idx][kda][tgpp] where size_idx: 0=S_V=16, 1=32, 2=64, 3=128; kda: 0 or 1.// tgpp 0 = TG variant (COLS_PER_LANE_GROUP=1), tgpp 1 = prefill variant (COLS_PER_LANE_GROUP=4).cl_kernel kernel_gated_delta_net_f32[4][2][2] = {};-cl_kernel kernel_timestep_embedding;cl_kernel kernel_gemv_moe_q4_0_f32_ns, kernel_gemm_moe_q4_0_f32_ns, kernel_gemm_moe_q4_0_f32_ns_bin;+ cl_kernel kernel_gemm_moe_q8_0_f32_ns;cl_kernel kernel_gemv_moe_q4_1_f32_ns, kernel_gemm_moe_q4_1_f32_ns, kernel_gemm_moe_q4_1_f32_ns_bin;cl_kernel kernel_gemv_moe_q5_0_f32_ns, kernel_gemm_moe_q5_0_f32_ns;cl_kernel kernel_gemv_moe_q5_1_f32_ns, kernel_gemm_moe_q5_1_f32_ns;cl_kernel kernel_gemv_moe_q4_k_f32_ns, kernel_gemm_moe_q4_k_f32_ns, kernel_gemm_moe_q4_k_f32_ns_bin;+ cl_kernel kernel_gemv_moe_q4_k_f32_ns_wimg = nullptr; // weight-as-texture MoE decode GEMV (opt-in)+ cl_kernel kernel_gemm_moe_q4_k_q8_1_dp4a; // dp4a (int8) prefill GEMM variant+ cl_kernel kernel_moe_reorder_quant_a_q8_1; // fused reorder + q8_1 quant for the dp4a GEMM+ cl_kernel kernel_gemm_moe_q8_1_dp4a_q80 = nullptr; // generic dp4a MoE GEMM (MOE_QT=80), opt-in+ cl_kernel kernel_moe_expand_scale_q8_0 = nullptr; // q8_0 per-block d -> uniform scale[16]+ cl_kernel kernel_gemm_moe_q8_1_dp4a_q50 = nullptr; // generic dp4a MoE GEMM (MOE_QT=50, q5_0), opt-in+ cl_kernel kernel_moe_expand_scale_q5_0 = nullptr; // q5_0 d -> uniform scale[2]/min[1] per 32-block+ cl_kernel kernel_gemm_moe_q8_1_dp4a_q5k = nullptr; // generic dp4a MoE GEMM (MOE_QT=5, q5_K), opt-in+ cl_kernel kernel_moe_expand_scale_q5_K = nullptr; // q5_K 6-bit s[] -> uniform scale[16]/min[8]cl_kernel kernel_gemv_moe_q5_k_f32_ns, kernel_gemm_moe_q5_k_f32_ns;cl_kernel kernel_gemv_moe_q6_k_f32_ns, kernel_gemm_moe_q6_k_f32_ns;+ cl_kernel kernel_gemm_moe_q6_k_q8_1_dp4a; // dp4a (int8) q6_K MoE prefill GEMMcl_kernel kernel_gemv_moe_mxfp4_f32, kernel_gemm_moe_mxfp4_f32;cl_kernel kernel_gemv_moe_mxfp4_f32_ns, kernel_gemm_moe_mxfp4_f32_ns, kernel_gemm_moe_mxfp4_f32_ns_bin;+ cl_kernel kernel_gemv_moe_mxfp4_f32_ns_wimg = nullptr; // weight-as-texture MoE decode GEMV+ cl_kernel kernel_gemm_moe_mxfp4_q8_1_dp4a; // dp4a (int8) mxfp4 MoE prefill GEMM+ cl_kernel kernel_gemm_moe_q4_0_q8_1_dp4a; // dp4a (int8) q4_0 MoE prefill GEMMcl_kernel kernel_moe_reorder_b;cl_kernel kernel_moe_histogram, kernel_moe_scan, kernel_moe_fill, kernel_moe_scatter;+ cl_kernel kernel_moe_combine_f32 = nullptr; // fused router-weight mul + cross-expert sumcl_kernel kernel_mul_mv_id_q4_0_f32_8x_flat;cl_kernel kernel_mul_mv_id_q8_0_f32, kernel_mul_mv_id_q8_0_f32_flat;cl_kernel kernel_mul_mv_id_mxfp4_f32;@@ -1006,21 +1030,32 @@ struct ggml_backend_opencl_context {cl_kernel kernel_gemv_noshuffle_q4_1_f32;cl_kernel kernel_gemm_noshuffle_q4_1_f32;cl_kernel kernel_gemm_noshuffle_q8_0_f32, kernel_gemm_noshuffle_q8_0_f32_bin;+ cl_kernel kernel_gemm_noshuffle_q8_0_q8_1_dp4a = nullptr; // dp4a (int8) dense q8_0 prefill GEMM (opt-in)+ cl_kernel kernel_gemm_noshuffle_q8_0_q8_1_dp4a_wimg = nullptr; // q8_0 dense dp4a, weights via texture (opt-in)cl_kernel kernel_gemv_noshuffle_q8_0_f32;cl_kernel kernel_gemm_noshuffle_q1_0_f32;cl_kernel kernel_gemv_noshuffle_q1_0_f32;cl_kernel kernel_gemv_noshuffle_q4_k_f32;cl_kernel kernel_gemm_noshuffle_q4_k_f32;+ cl_kernel kernel_gemm_noshuffle_q4_k_q8_1_dp4a; // dp4a (int8) dense prefill GEMM+ cl_kernel kernel_gemm_noshuffle_q4_k_q8_1_dp4a_wimg; // dp4a dense prefill GEMM, weights via texture (X1 opt-in)+ cl_kernel kernel_gemm_noshuffle_q5_k_q8_1_dp4a; // dp4a (int8) dense q5_K prefill GEMM+ cl_kernel kernel_gemm_noshuffle_q6_k_q8_1_dp4a; // dp4a (int8) dense q6_K prefill GEMM+ cl_kernel kernel_quant_a_q8_1; // plain activation q8_1 pre-passcl_kernel kernel_gemv_noshuffle_q6_K_f32;cl_kernel kernel_gemm_noshuffle_q6_K_f32;cl_kernel kernel_gemv_noshuffle_q5_k_f32;cl_kernel kernel_gemm_noshuffle_q5_k_f32;cl_kernel kernel_gemv_noshuffle_q5_0_f32;cl_kernel kernel_gemm_noshuffle_q5_0_f32;+ cl_kernel kernel_gemm_noshuffle_q5_0_q8_1_dp4a = nullptr; // dp4a (int8) dense q5_0 prefill GEMM+ cl_kernel kernel_gemm_noshuffle_q5_0_q8_1_dp4a_wimg = nullptr; // q5_0 dense dp4a, qs plane via texture (opt-in)cl_kernel kernel_gemv_noshuffle_q5_1_f32;cl_kernel kernel_gemm_noshuffle_q5_1_f32;cl_kernel kernel_gemv_noshuffle_iq4_nl_f32;cl_kernel kernel_gemm_noshuffle_iq4_nl_f32;+ cl_kernel kernel_gemm_noshuffle_iq4_nl_q8_1_dp4a = nullptr; // dp4a (int8) dense IQ4_NL prefill GEMM+ cl_kernel kernel_gemm_noshuffle_q4_0_q8_1_dp4a = nullptr; // dp4a (int8) dense q4_0 prefill GEMM#endif // GGML_OPENCL_USE_ADRENO_KERNELSvoid free() {@@ -1396,6 +1431,11 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) {CL_CHECK((backend_ctx->kernel_restore_block_iq4_nl_noshuffle = clCreateKernel(backend_ctx->program_cvt, "kernel_restore_block_iq4_nl_noshuffle", &err), err));CL_CHECK((backend_ctx->kernel_convert_bf16_to_f16 = clCreateKernel(backend_ctx->program_cvt, "kernel_convert_bf16_to_f16", &err), err));CL_CHECK((backend_ctx->kernel_convert_f16_to_bf16 = clCreateKernel(backend_ctx->program_cvt, "kernel_convert_f16_to_bf16", &err), err));+#ifdef GGML_OPENCL_USE_ADRENO_KERNELS+ CL_CHECK((backend_ctx->kernel_moe_expand_scale_q8_0 = clCreateKernel(backend_ctx->program_cvt, "kernel_moe_expand_scale_q8_0", &err), err));+ CL_CHECK((backend_ctx->kernel_moe_expand_scale_q5_0 = clCreateKernel(backend_ctx->program_cvt, "kernel_moe_expand_scale_q5_0", &err), err));+ CL_CHECK((backend_ctx->kernel_moe_expand_scale_q5_K = clCreateKernel(backend_ctx->program_cvt, "kernel_moe_expand_scale_q5_K", &err), err));+#endifGGML_LOG_CONT(".");}@@ -3115,6 +3155,23 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) {GGML_LOG_CONT(".");}+ // moe_combine (fused router-weight mul + cross-expert sum)+ {+ #ifdef GGML_OPENCL_EMBED_KERNELS+ const std::string kernel_src {+ #include "moe_combine.cl.h"+ };+ #else+ const std::string kernel_src = read_file("moe_combine.cl");+ #endif+ cl_program prog = build_program_from_source(+ backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts);+ CL_CHECK((backend_ctx->kernel_moe_combine_f32 =+ clCreateKernel(prog, "kernel_moe_combine_f32", &err), err));+ CL_CHECK(clReleaseProgram(prog));+ GGML_LOG_CONT(".");+ }+// mul_mv_id_q4_0_f32_8x_flat{#ifdef GGML_OPENCL_EMBED_KERNELS@@ -3432,6 +3489,22 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) {GGML_LOG_CONT(".");}+ // gemm_noshuffle_q5_0_q8_1_dp4a (dp4a dense q5_0 prefill GEMM)+ {+#ifdef GGML_OPENCL_EMBED_KERNELS+ const std::string kernel_src {+ #include "gemm_noshuffle_q5_0_q8_1_dp4a.cl.h"+ };+#else+ const std::string kernel_src = read_file("gemm_noshuffle_q5_0_q8_1_dp4a.cl");+#endif+ cl_program prog = build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts);+ CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q5_0_q8_1_dp4a = clCreateKernel(prog, "kernel_gemm_noshuffle_q5_0_q8_1_dp4a", &err), err));+ CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q5_0_q8_1_dp4a_wimg = clCreateKernel(prog, "kernel_gemm_noshuffle_q5_0_q8_1_dp4a_wimg", &err), err));+ CL_CHECK(clReleaseProgram(prog));+ GGML_LOG_CONT(".");+ }+// gemv_noshuffle_q5_0_f32{std::string CL_gemv_compile_opts = std::string("-cl-std=") + opencl_c_std +@@ -3506,6 +3579,36 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) {GGML_LOG_CONT(".");}+ // gemm_noshuffle_iq4_nl_q8_1_dp4a (dp4a dense IQ4_NL prefill GEMM)+ {+#ifdef GGML_OPENCL_EMBED_KERNELS+ const std::string kernel_src {+ #include "gemm_noshuffle_iq4_nl_q8_1_dp4a.cl.h"+ };+#else+ const std::string kernel_src = read_file("gemm_noshuffle_iq4_nl_q8_1_dp4a.cl");+#endif+ cl_program prog = build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts);+ CL_CHECK((backend_ctx->kernel_gemm_noshuffle_iq4_nl_q8_1_dp4a = clCreateKernel(prog, "kernel_gemm_noshuffle_iq4_nl_q8_1_dp4a", &err), err));+ CL_CHECK(clReleaseProgram(prog));+ GGML_LOG_CONT(".");+ }++ // gemm_noshuffle_q4_0_q8_1_dp4a (dp4a dense q4_0 prefill GEMM)+ {+#ifdef GGML_OPENCL_EMBED_KERNELS+ const std::string kernel_src {+ #include "gemm_noshuffle_q4_0_q8_1_dp4a.cl.h"+ };+#else+ const std::string kernel_src = read_file("gemm_noshuffle_q4_0_q8_1_dp4a.cl");+#endif+ cl_program prog = build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts);+ CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q4_0_q8_1_dp4a = clCreateKernel(prog, "kernel_gemm_noshuffle_q4_0_q8_1_dp4a", &err), err));+ CL_CHECK(clReleaseProgram(prog));+ GGML_LOG_CONT(".");+ }+// gemv_noshuffle_iq4_nl_f32{std::string CL_gemv_compile_opts = std::string("-cl-std=") + opencl_c_std +@@ -3604,6 +3707,89 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) {GGML_LOG_CONT(".");}+ // gemm_noshuffle_q4_k_q8_1_dp4a (dp4a dense prefill GEMM)+ {+#ifdef GGML_OPENCL_EMBED_KERNELS+ const std::string kernel_src {+ #include "gemm_noshuffle_q4_k_q8_1_dp4a.cl.h"+ };+#else+ const std::string kernel_src = read_file("gemm_noshuffle_q4_k_q8_1_dp4a.cl");+#endif+ // Per-device dp4a dense tile. The X2-tuned TILESIZE_N=32 over-occupies LDS on+ // X1 (1152 B/WG -> few resident WGs); TILESIZE_N=8 (288 B) lifts occupancy on+ // X1, byte-identical. X2E keeps 32. Env override wins.+ int q4k_dp4a_ts = (backend_ctx->adreno_gen == ADRENO_GPU_GEN::X1E) ? 8 : 32;+ if (const char * e = getenv("GGML_OPENCL_Q4K_DP4A_TS")) q4k_dp4a_ts = atoi(e);+ std::string dp4a_opts = compile_opts + " -DTILESIZE_N=" + std::to_string(q4k_dp4a_ts);+ cl_program prog = build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), dp4a_opts);+ CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q4_k_q8_1_dp4a = clCreateKernel(prog, "kernel_gemm_noshuffle_q4_k_q8_1_dp4a", &err), err));+ CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q4_k_q8_1_dp4a_wimg = clCreateKernel(prog, "kernel_gemm_noshuffle_q4_k_q8_1_dp4a_wimg", &err), err));+ CL_CHECK(clReleaseProgram(prog));+ GGML_LOG_CONT(".");+ }++ // gemm_noshuffle_q8_0_q8_1_dp4a (dp4a dense q8_0 prefill GEMM)+ {+#ifdef GGML_OPENCL_EMBED_KERNELS+ const std::string kernel_src {+ #include "gemm_noshuffle_q8_0_q8_1_dp4a.cl.h"+ };+#else+ const std::string kernel_src = read_file("gemm_noshuffle_q8_0_q8_1_dp4a.cl");+#endif+ cl_program prog = build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts);+ CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q8_0_q8_1_dp4a = clCreateKernel(prog, "kernel_gemm_noshuffle_q8_0_q8_1_dp4a", &err), err));+ CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q8_0_q8_1_dp4a_wimg = clCreateKernel(prog, "kernel_gemm_noshuffle_q8_0_q8_1_dp4a_wimg", &err), err));+ CL_CHECK(clReleaseProgram(prog));+ GGML_LOG_CONT(".");+ }++ // gemm_noshuffle_q5_k_q8_1_dp4a (dp4a dense prefill GEMM for q5_K)+ {+#ifdef GGML_OPENCL_EMBED_KERNELS+ const std::string kernel_src {+ #include "gemm_noshuffle_q5_k_q8_1_dp4a.cl.h"+ };+#else+ const std::string kernel_src = read_file("gemm_noshuffle_q5_k_q8_1_dp4a.cl");+#endif+ cl_program prog = build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts);+ CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q5_k_q8_1_dp4a = clCreateKernel(prog, "kernel_gemm_noshuffle_q5_k_q8_1_dp4a", &err), err));+ CL_CHECK(clReleaseProgram(prog));+ GGML_LOG_CONT(".");+ }++ // gemm_noshuffle_q6_k_q8_1_dp4a (dp4a dense prefill GEMM for q6_K ffn_down/output)+ {+#ifdef GGML_OPENCL_EMBED_KERNELS+ const std::string kernel_src {+ #include "gemm_noshuffle_q6_k_q8_1_dp4a.cl.h"+ };+#else+ const std::string kernel_src = read_file("gemm_noshuffle_q6_k_q8_1_dp4a.cl");+#endif+ cl_program prog = build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts);+ CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q6_k_q8_1_dp4a = clCreateKernel(prog, "kernel_gemm_noshuffle_q6_k_q8_1_dp4a", &err), err));+ CL_CHECK(clReleaseProgram(prog));+ GGML_LOG_CONT(".");+ }++ // quant_a_q8_1 (plain activation q8_1 pre-pass for the dense dp4a GEMM)+ {+#ifdef GGML_OPENCL_EMBED_KERNELS+ const std::string kernel_src {+ #include "quant_a_q8_1.cl.h"+ };+#else+ const std::string kernel_src = read_file("quant_a_q8_1.cl");+#endif+ cl_program prog = build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts);+ CL_CHECK((backend_ctx->kernel_quant_a_q8_1 = clCreateKernel(prog, "kernel_quant_a_q8_1", &err), err));+ CL_CHECK(clReleaseProgram(prog));+ GGML_LOG_CONT(".");+ }+// gemv_noshuffle_q4_k_f32{std::string CL_gemv_compile_opts = std::string("-cl-std=") + opencl_c_std +@@ -3766,6 +3952,23 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) {}}+ // gemm_moe_q8_0_f32_ns+ {+#ifdef GGML_OPENCL_EMBED_KERNELS+ const std::string kernel_src {+ #include "gemm_moe_q8_0_f32_ns.cl.h"+ };+#else+ const std::string kernel_src = read_file("gemm_moe_q8_0_f32_ns.cl");+#endif+ cl_program prog =+ build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), CL_moe_compile_opts);++ CL_CHECK((backend_ctx->kernel_gemm_moe_q8_0_f32_ns = clCreateKernel(prog, "kernel_gemm_moe_q8_0_f32_ns", &err), err));+ CL_CHECK(clReleaseProgram(prog));+ GGML_LOG_CONT(".");+ }+// gemv_moe_q5_0_f32_ns{#ifdef GGML_OPENCL_EMBED_KERNELS@@ -3847,6 +4050,7 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) {build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), CL_moe_compile_opts);CL_CHECK((backend_ctx->kernel_gemv_moe_q4_k_f32_ns = clCreateKernel(prog, "kernel_gemv_moe_q4_k_f32_ns", &err), err));+ CL_CHECK((backend_ctx->kernel_gemv_moe_q4_k_f32_ns_wimg = clCreateKernel(prog, "kernel_gemv_moe_q4_k_f32_ns_wimg", &err), err));CL_CHECK(clReleaseProgram(prog));GGML_LOG_CONT(".");}@@ -3886,6 +4090,103 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) {}}+ // gemm_moe_q4_k_q8_1_dp4a (dp4a prefill GEMM)+ {+#ifdef GGML_OPENCL_EMBED_KERNELS+ const std::string kernel_src {+ #include "gemm_moe_q4_k_q8_1_dp4a.cl.h"+ };+#else+ const std::string kernel_src = read_file("gemm_moe_q4_k_q8_1_dp4a.cl");+#endif+ cl_program prog =+ build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), CL_moe_compile_opts);++ CL_CHECK((backend_ctx->kernel_gemm_moe_q4_k_q8_1_dp4a = clCreateKernel(prog, "kernel_gemm_moe_q4_k_q8_1_dp4a", &err), err));+ CL_CHECK(clReleaseProgram(prog));+ GGML_LOG_CONT(".");+ }++ // gemm_moe_mxfp4_q8_1_dp4a (dp4a prefill GEMM)+ {+#ifdef GGML_OPENCL_EMBED_KERNELS+ const std::string kernel_src {+ #include "gemm_moe_mxfp4_q8_1_dp4a.cl.h"+ };+#else+ const std::string kernel_src = read_file("gemm_moe_mxfp4_q8_1_dp4a.cl");+#endif+ cl_program prog =+ build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), CL_moe_compile_opts);++ CL_CHECK((backend_ctx->kernel_gemm_moe_mxfp4_q8_1_dp4a = clCreateKernel(prog, "kernel_gemm_moe_mxfp4_q8_1_dp4a", &err), err));+ CL_CHECK(clReleaseProgram(prog));+ GGML_LOG_CONT(".");+ }++ // gemm_moe_q4_0_q8_1_dp4a (dp4a prefill GEMM)+ {+#ifdef GGML_OPENCL_EMBED_KERNELS+ const std::string kernel_src {+ #include "gemm_moe_q4_0_q8_1_dp4a.cl.h"+ };+#else+ const std::string kernel_src = read_file("gemm_moe_q4_0_q8_1_dp4a.cl");+#endif+ cl_program prog =+ build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), CL_moe_compile_opts);++ CL_CHECK((backend_ctx->kernel_gemm_moe_q4_0_q8_1_dp4a = clCreateKernel(prog, "kernel_gemm_moe_q4_0_q8_1_dp4a", &err), err));+ CL_CHECK(clReleaseProgram(prog));+ GGML_LOG_CONT(".");+ }++ // gemm_moe_q8_1_dp4a (generic dp4a MoE GEMM; MOE_QT=80 -> q8_0 expert variant)+ {+#ifdef GGML_OPENCL_EMBED_KERNELS+ const std::string kernel_src {+ #include "gemm_moe_q8_1_dp4a.cl.h"+ };+#else+ const std::string kernel_src = read_file("gemm_moe_q8_1_dp4a.cl");+#endif+ const std::string opts80 = CL_moe_compile_opts + " -DMOE_QT=80";+ cl_program prog =+ build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), opts80.c_str());+ CL_CHECK((backend_ctx->kernel_gemm_moe_q8_1_dp4a_q80 = clCreateKernel(prog, "kernel_gemm_moe_q8_1_dp4a", &err), err));+ CL_CHECK(clReleaseProgram(prog));++ const std::string opts50 = CL_moe_compile_opts + " -DMOE_QT=50";+ cl_program prog50 =+ build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), opts50.c_str());+ CL_CHECK((backend_ctx->kernel_gemm_moe_q8_1_dp4a_q50 = clCreateKernel(prog50, "kernel_gemm_moe_q8_1_dp4a", &err), err));+ CL_CHECK(clReleaseProgram(prog50));++ const std::string opts5 = CL_moe_compile_opts + " -DMOE_QT=5";+ cl_program prog5 =+ build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), opts5.c_str());+ CL_CHECK((backend_ctx->kernel_gemm_moe_q8_1_dp4a_q5k = clCreateKernel(prog5, "kernel_gemm_moe_q8_1_dp4a", &err), err));+ CL_CHECK(clReleaseProgram(prog5));+ GGML_LOG_CONT(".");+ }++ // moe_reorder_quant_a_q8_1 (fused reorder + q8_1 quant)+ {+#ifdef GGML_OPENCL_EMBED_KERNELS+ const std::string kernel_src {+ #include "moe_reorder_quant_a_q8_1.cl.h"+ };+#else+ const std::string kernel_src = read_file("moe_reorder_quant_a_q8_1.cl");+#endif+ cl_program prog =+ build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), CL_moe_compile_opts);++ CL_CHECK((backend_ctx->kernel_moe_reorder_quant_a_q8_1 = clCreateKernel(prog, "kernel_moe_reorder_quant_a_q8_1", &err), err));+ CL_CHECK(clReleaseProgram(prog));+ GGML_LOG_CONT(".");+ }+// gemv_moe_q5_k_f32_ns{#ifdef GGML_OPENCL_EMBED_KERNELS@@ -3954,6 +4255,23 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) {GGML_LOG_CONT(".");}+ // gemm_moe_q6_k_q8_1_dp4a (dp4a q6_K MoE prefill GEMM)+ {+#ifdef GGML_OPENCL_EMBED_KERNELS+ const std::string kernel_src {+ #include "gemm_moe_q6_k_q8_1_dp4a.cl.h"+ };+#else+ const std::string kernel_src = read_file("gemm_moe_q6_k_q8_1_dp4a.cl");+#endif+ cl_program prog =+ build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), CL_moe_compile_opts);++ CL_CHECK((backend_ctx->kernel_gemm_moe_q6_k_q8_1_dp4a = clCreateKernel(prog, "kernel_gemm_moe_q6_k_q8_1_dp4a", &err), err));+ CL_CHECK(clReleaseProgram(prog));+ GGML_LOG_CONT(".");+ }+// gemv_moe_mxfp4_f32_ns{#ifdef GGML_OPENCL_EMBED_KERNELS@@ -3967,6 +4285,7 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) {build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), CL_moe_compile_opts);CL_CHECK((backend_ctx->kernel_gemv_moe_mxfp4_f32_ns = clCreateKernel(prog, "kernel_gemv_moe_mxfp4_f32_ns", &err), err));+ CL_CHECK((backend_ctx->kernel_gemv_moe_mxfp4_f32_ns_wimg = clCreateKernel(prog, "kernel_gemv_moe_mxfp4_f32_ns_wimg", &err), err));CL_CHECK(clReleaseProgram(prog));GGML_LOG_CONT(".");}@@ -5484,6 +5803,7 @@ static ggml_backend_opencl_context * ggml_cl_init(ggml_backend_dev_t dev) {// check Adreno large buffer supportbackend_ctx->adreno_has_large_buffer = strstr(ext_buffer, "cl_qcom_large_buffer") != NULL;+// subgroup shuffle support (N_SPLIT>1 FA kernel)backend_ctx->has_qcom_subgroup_shuffle = strstr(ext_buffer, "cl_qcom_subgroup_shuffle") != NULL;backend_ctx->has_subgroup_shuffle =@@ -5537,6 +5857,14 @@ static ggml_backend_opencl_context * ggml_cl_init(ggml_backend_dev_t dev) {static const char * ragged_gran_env = getenv("GGML_OPENCL_MOE_RAGGED_GRAN");backend_ctx->adreno_moe_ragged_skip_gran = (ragged_gran_env != NULL) ? atoi(ragged_gran_env) : 8;+ // whether fuse moe combine+ static const char * fuse_moe_combine_env = getenv("GGML_OPENCL_FUSE_MOE_COMBINE");+ backend_ctx->fuse_moe_combine = fuse_moe_combine_env == NULL ? 1 : (atoi(fuse_moe_combine_env) != 0);++ // ragged moe dp4 variant+ static const char * ragged_dp4_env = getenv("GGML_OPENCL_MOE_RAGGED");+ backend_ctx->adreno_use_moe_ragged_dp4 = ragged_dp4_env == NULL ? 1 : (atoi(ragged_dp4_env) != 0);+#ifdef GGML_OPENCL_USE_ADRENO_BIN_KERNELS// try loading adreno binary kernels if enabled// if fails to load, builtin kernels will be used@@ -5863,6 +6191,10 @@ struct ggml_tensor_extra_cl_q5_0 {cl_mem d = nullptr;// Scales in image1d_buffer_t.cl_mem d_img = nullptr;+ // Uniform per-32-block scale (2/block) + min (1/block, = d*16 for the -16 centering)+ // for the generic dp4a MoE GEMM. Built from d.+ cl_mem scale = nullptr;+ cl_mem min = nullptr;// Size of quantized values.size_t size_qs = 0;// Size of 5-th bit values.@@ -5891,6 +6223,14 @@ struct ggml_tensor_extra_cl_q5_0 {CL_CHECK(clReleaseMemObject(qs_img));qs_img = nullptr;}+ if (scale != nullptr) {+ CL_CHECK(clReleaseMemObject(scale));+ scale = nullptr;+ }+ if (min != nullptr) {+ CL_CHECK(clReleaseMemObject(min));+ min = nullptr;+ }qh_img = nullptr;d_img = nullptr;@@ -6015,6 +6355,11 @@ struct ggml_tensor_extra_cl_q8_0 {cl_mem d = nullptr;cl_mem d_img = nullptr;+ // Uniform per-16-segment scale (16/superblock) for the generic dp4a MoE GEMM.+ // Expanded from d at set_tensor; the int8 codes are reused from q.+ // q8_0 is symmetric so no min buffer (has_min=0).+ cl_mem scale = nullptr;+size_t size_q = 0;size_t size_d = 0;@@ -6034,6 +6379,10 @@ struct ggml_tensor_extra_cl_q8_0 {CL_CHECK(clReleaseMemObject(d));d = nullptr;}+ if (scale != nullptr) {+ CL_CHECK(clReleaseMemObject(scale));+ scale = nullptr;+ }// Currently, q_img and d_img are not used. They can be image1d_buffer_t// that wraps around q and d to utilize image access path.q_img = nullptr;@@ -6120,6 +6469,11 @@ struct ggml_tensor_extra_cl_q5_K {cl_mem d = nullptr;// Min for each super block.cl_mem dm = nullptr;+ // Uniform per-32-block scale (2/block) + min (1/block, = dm*mn) decoded from the+ // 6-bit packed s[] for the generic dp4a MoE GEMM kernel_gemm_moe_q8_1_dp4a.+ // Built from s/d/dm at set_tensor; q/qh are reused as-is.+ cl_mem scale = nullptr;+ cl_mem min = nullptr;size_t size_q = 0;size_t size_qh = 0;@@ -6156,6 +6510,14 @@ struct ggml_tensor_extra_cl_q5_K {CL_CHECK(clReleaseMemObject(q_img));q_img = nullptr;}+ if (scale != nullptr) {+ CL_CHECK(clReleaseMemObject(scale));+ scale = nullptr;+ }+ if (min != nullptr) {+ CL_CHECK(clReleaseMemObject(min));+ min = nullptr;+ }size_q = 0;size_qh = 0;@@ -6298,6 +6660,122 @@ static void sync_with_other_backends(ggml_backend_t backend) {sync_with_other_backends(backend_ctx);}+// True if two tensors share a device buffer with overlapping byte ranges. The pool+// allocator may place a fused op's output over a sequentially-dead input (safe for the+// original separate kernels, but a read/write race inside one fused kernel).+static bool ggml_cl_tensors_overlap(const ggml_tensor * x, const ggml_tensor * y) {+ ggml_tensor_extra_cl * ex = (ggml_tensor_extra_cl *)x->extra;+ ggml_tensor_extra_cl * ey = (ggml_tensor_extra_cl *)y->extra;+ if (!ex || !ey || ex->data_device != ey->data_device) { return false; }+ const cl_ulong xo = ex->offset + x->view_offs, xe = xo + ggml_nbytes(x);+ const cl_ulong yo = ey->offset + y->view_offs, ye = yo + ggml_nbytes(y);+ return xo < ye && yo < xe;+}++// Detect the MoE combine epilogue: router-weight MUL ([n_embd,k,nt] * [1,k,nt]) followed+// by k VIEWs of it and a (k-1)-long ADD reduction chain producing [n_embd, nt]. When it+// matches (and the output does not alias the inputs), the whole subgraph collapses to one+// weighted-sum-across-experts kernel.+static bool ggml_opencl_can_fuse_moe_combine(const struct ggml_cgraph * cgraph, int node_idx,+ const ggml_tensor ** out_final_add) {+ const ggml_tensor * mul = cgraph->nodes[node_idx];+ if (mul->op != GGML_OP_MUL) { return false; }+ const ggml_tensor * experts = mul->src[0];+ const ggml_tensor * weights = mul->src[1];+ if (!experts || !weights) { return false; }+ if (experts->type != GGML_TYPE_F32 || weights->type != GGML_TYPE_F32 || mul->type != GGML_TYPE_F32) { return false; }++ const int64_t n_embd = experts->ne[0];+ const int64_t k = experts->ne[1];+ const int64_t nt = experts->ne[2];+ if (k < 2 || k > 64 || experts->ne[3] != 1 || n_embd % 4 != 0) { return false; }+ if (weights->ne[0] != 1 || weights->ne[1] != k || weights->ne[2] != nt || weights->ne[3] != 1) { return false; }+ if (mul->ne[0] != n_embd || mul->ne[1] != k || mul->ne[2] != nt) { return false; }+ // the fused kernel needs contiguous experts/weights and a contiguous 2D dst+ if (!ggml_is_contiguous(experts) || !ggml_is_contiguous(weights)) { return false; }++ const int n_nodes = 1 + (int)k + (int)(k - 1); // MUL + k*VIEW + (k-1)*ADD+ if (n_nodes >= 32) { return false; }+ if (node_idx + n_nodes > cgraph->n_nodes) { return false; }++ enum ggml_op ops[1 + 64 + 63];+ int n = 0;+ ops[n++] = GGML_OP_MUL;+ for (int j = 0; j < (int)k; ++j) { ops[n++] = GGML_OP_VIEW; }+ for (int j = 0; j < (int)k - 1; ++j) { ops[n++] = GGML_OP_ADD; }+ const int outs[] = { node_idx + n_nodes - 1 };+ if (!ggml_can_fuse_subgraph(cgraph, node_idx, n_nodes, ops, outs, 1)) { return false; }++ for (int j = 0; j < (int)k; ++j) {+ const ggml_tensor * vw = cgraph->nodes[node_idx + 1 + j];+ if (vw->op != GGML_OP_VIEW || vw->src[0] != mul || vw->ne[0] != n_embd || vw->ne[1] != nt) { return false; }+ }+ const ggml_tensor * final_add = cgraph->nodes[node_idx + n_nodes - 1];+ if (final_add->op != GGML_OP_ADD || final_add->type != GGML_TYPE_F32 ||+ final_add->ne[0] != n_embd || final_add->ne[1] != nt || final_add->ne[2] != 1) { return false; }+ if (!ggml_is_contiguous(final_add)) { return false; }+ // the fused kernel reads experts + writes final_add in one pass; bail if the+ // pool allocator overlapped the output with the (large) experts input -- would race.+ // The small weights input is copied to a private scratch in the dispatch, so its own+ // aliasing with the output is handled there and does not block the fusion.+ if (ggml_cl_tensors_overlap(experts, final_add)) { return false; }++ *out_final_add = final_add;+ return true;+}++static void ggml_cl_moe_combine_fused(ggml_backend_t backend, const ggml_tensor * mul, const ggml_tensor * dst) {+ ggml_backend_opencl_context * backend_ctx = (ggml_backend_opencl_context *)backend->context;+ const ggml_tensor * experts = mul->src[0];+ const ggml_tensor * weights = mul->src[1];++ ggml_tensor_extra_cl * ee = (ggml_tensor_extra_cl *)experts->extra;+ ggml_tensor_extra_cl * ew = (ggml_tensor_extra_cl *)weights->extra;+ ggml_tensor_extra_cl * ed = (ggml_tensor_extra_cl *)dst->extra;+ cl_ulong off_e = ee->offset + experts->view_offs;+ cl_ulong off_w = ew->offset + weights->view_offs;+ cl_ulong off_d = ed->offset + dst->view_offs;++ const int n_embd4 = (int)(experts->ne[0] / 4);+ const int k = (int)experts->ne[1];+ const int nt = (int)experts->ne[2];+ const cl_uint e1 = (cl_uint)(experts->nb[1] / sizeof(float));+ const cl_uint e2 = (cl_uint)(experts->nb[2] / sizeof(float));+ const cl_uint w1 = (cl_uint)(weights->nb[1] / sizeof(float));+ const cl_uint w2 = (cl_uint)(weights->nb[2] / sizeof(float));+ const cl_uint d1 = (cl_uint)(dst->nb[1] / sizeof(float));++ // The router weights are tiny ([1,k,nt]) and may share a pool buffer with the output;+ // copy them into a private scratch so the fused kernel never reads aliased memory.+ const size_t w_bytes = ggml_nbytes(weights);+ backend_ctx->prealloc_moe_combine_w.allocate(backend_ctx->context, w_bytes);+ CL_CHECK(clEnqueueCopyBuffer(backend_ctx->queue, ew->data_device, backend_ctx->prealloc_moe_combine_w.buffer,+ off_w, 0, w_bytes, 0, NULL, NULL));+ cl_mem w_dev = backend_ctx->prealloc_moe_combine_w.buffer;+ cl_ulong w_off = 0;++ cl_kernel kernel = backend_ctx->kernel_moe_combine_f32;+ int a = 0;+ CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &ee->data_device));+ CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_ulong), &off_e));+ CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &w_dev));+ CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_ulong), &w_off));+ CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &ed->data_device));+ CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_ulong), &off_d));+ CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &n_embd4));+ CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &k));+ CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &nt));+ CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_uint), &e1));+ CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_uint), &e2));+ CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_uint), &w1));+ CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_uint), &w2));+ CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_uint), &d1));++ size_t lws[2] = { 64, 1 };+ size_t gws[2] = { (size_t)(((n_embd4 + 63) / 64) * 64), (size_t)nt };+ backend_ctx->enqueue_ndrange_kernel(kernel, 2, gws, lws, dst);+}+static bool ggml_opencl_can_fuse(const struct ggml_cgraph * cgraph, int node_idx, std::initializer_list<enum ggml_op> ops) {if (!ggml_can_fuse(cgraph, node_idx, ops)) {return false;@@ -6398,6 +6876,17 @@ static ggml_status ggml_backend_opencl_graph_compute(ggml_backend_t backend, ggmi += 2;continue;}+ // Fuse the MoE combine: router-weight mul + cross-expert add chain ->+ // one weighted-sum-across-experts kernel.+ if (backend_ctx->fuse_moe_combine && !backend_ctx->disable_fusion) {+ const ggml_tensor * combine_out = nullptr;+ if (ggml_opencl_can_fuse_moe_combine(cgraph, i, &combine_out)) {+ ggml_cl_moe_combine_fused(backend, node, combine_out);+ i += 2 * (int)node->ne[1] - 1; // skip the k VIEWs + (k-1) ADDs+ continue;+ }+ }+if (!backend_ctx->disable_fusion && ggml_opencl_can_fuse(cgraph, i, { GGML_OP_RMS_NORM, GGML_OP_MUL })) {ggml_opencl_op_rms_norm_fused(backend, node, cgraph->nodes[i+1]);i++;@@ -6446,7 +6935,12 @@ inline bool enable_adreno_trans_weight(const ggml_backend_opencl_context *backensize_t elem_num = tensor->ne[0] * tensor->ne[1] * tensor->ne[2] * tensor->ne[3];- return ((elem_num < 128 * 1024 * 1024) && adreno_kernel); // max element num: 2**27+ // The 2D weight transpose (transpose_2d_as_*) tiles rows by 4 over a 2D matrix,+ // so it requires K(ne0)%32==0, M(ne1)%4==0 and ne2==ne3==1.+ const bool shape_ok = (tensor->ne[0] % 32 == 0) && (tensor->ne[1] % 4 == 0) &&+ (tensor->ne[2] == 1) && (tensor->ne[3] == 1);++ return ((elem_num < 128 * 1024 * 1024) && adreno_kernel && shape_ok); // max element num: 2**27}static inline bool use_flat_gemv_for_large_m_q4_K(const ggml_tensor *tensor) {@@ -7785,6 +8279,31 @@ static void ggml_backend_opencl_buffer_set_tensor(ggml_backend_buffer_t buffer,extra->qs_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_format_qs, &img_desc_qs, NULL, &err);tensor->extra = extra;+ // Generic dp4a MoE path+ {+ static const char * q5dp4a_env = getenv("GGML_OPENCL_Q5_MOE_DP4A");+ const bool q5dp4a = q5dp4a_env ? (atoi(q5dp4a_env) != 0)+ : (backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E);+ if (q5dp4a && ne02 > 1 && (ne00 % 32 == 0)) {+ size_t nb32 = (size_t)ne00 / 32;+ size_t sc_elems = (size_t)ne02 * ne01 * nb32 * 2;+ size_t mn_elems = (size_t)ne02 * ne01 * nb32;+ extra->scale = clCreateBuffer(context, CL_MEM_READ_WRITE, sc_elems * sizeof(cl_half), NULL, &err); CL_CHECK(err);+ extra->min = clCreateBuffer(context, CL_MEM_READ_WRITE, mn_elems * sizeof(cl_half), NULL, &err); CL_CHECK(err);+ cl_kernel ek = backend_ctx->kernel_moe_expand_scale_q5_0;+ CL_CHECK(clSetKernelArg(ek, 0, sizeof(cl_mem), &extra->d));+ CL_CHECK(clSetKernelArg(ek, 1, sizeof(cl_mem), &extra->scale));+ CL_CHECK(clSetKernelArg(ek, 2, sizeof(cl_mem), &extra->min));+ CL_CHECK(clSetKernelArg(ek, 3, sizeof(int), &ne00));+ CL_CHECK(clSetKernelArg(ek, 4, sizeof(int), &ne01));+ size_t eg[3] = { (size_t)(((ne01 + 63) / 64) * 64), nb32, (size_t)ne02 };+ size_t el[3] = { 64, 1, 1 };+ cl_event evt;+ CL_CHECK(clEnqueueNDRangeKernel(queue, ek, 3, NULL, eg, el, 0, NULL, &evt));+ CL_CHECK(clWaitForEvents(1, &evt));+ }+ }+return;}#endif // GGML_OPENCL_USE_ADRENO_KERNELS@@ -8164,6 +8683,34 @@ static void ggml_backend_opencl_buffer_set_tensor(ggml_backend_buffer_t buffer,tensor->extra = extra;ctx->q8_0_soa_tensors.insert(tensor);+ // Generic dp4a MoE path (opt-in GGML_OPENCL_Q8_MOE_DP4A)+#ifdef GGML_OPENCL_USE_ADRENO_KERNELS+ {+ static const char * q8dp4a_env = getenv("GGML_OPENCL_Q8_MOE_DP4A");+ const bool q8dp4a = q8dp4a_env ? (atoi(q8dp4a_env) != 0)+ : (backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E);+ if (q8dp4a && tensor->ne[2] > 1 && (tensor->ne[0] % 32 == 0)) {+ int ne00 = (int)tensor->ne[0];+ int ne01 = (int)tensor->ne[1];+ int ne02 = (int)tensor->ne[2];+ size_t nb32 = (size_t)ne00 / 32;+ size_t scale_elems = (size_t)ne02 * ne01 * nb32 * 2; // 2 per-16-seg scales / 32-block+ extra->scale = clCreateBuffer(context, CL_MEM_READ_WRITE, scale_elems * sizeof(cl_half), NULL, &err);+ CL_CHECK(err);+ cl_kernel ek = backend_ctx->kernel_moe_expand_scale_q8_0;+ CL_CHECK(clSetKernelArg(ek, 0, sizeof(cl_mem), &extra->d));+ CL_CHECK(clSetKernelArg(ek, 1, sizeof(cl_mem), &extra->scale));+ CL_CHECK(clSetKernelArg(ek, 2, sizeof(int), &ne00));+ CL_CHECK(clSetKernelArg(ek, 3, sizeof(int), &ne01));+ size_t eg[3] = { (size_t)(((ne01 + 63) / 64) * 64), nb32, (size_t)ne02 };+ size_t el[3] = { 64, 1, 1 };+ cl_event evt;+ CL_CHECK(clEnqueueNDRangeKernel(queue, ek, 3, NULL, eg, el, 0, NULL, &evt));+ CL_CHECK(clWaitForEvents(1, &evt));+ }+ }+#endif+// Transpose the weights and scales#ifdef GGML_OPENCL_USE_ADRENO_KERNELSif (enable_adreno_trans_weight(backend_ctx, tensor)) {@@ -8516,6 +9063,33 @@ static void ggml_backend_opencl_buffer_set_tensor(ggml_backend_buffer_t buffer,CL_CHECK(err);tensor->extra = extra;+ // Generic dp4a MoE path+ {+ static const char * q5kdp4a_env = getenv("GGML_OPENCL_Q5K_MOE_DP4A");+ const bool q5kdp4a = q5kdp4a_env ? (atoi(q5kdp4a_env) != 0)+ : (backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E);+ if (q5kdp4a && ne02 > 1 && (ne00 % 256 == 0)) {+ size_t nb32 = (size_t)ne00 / 32;+ size_t sc_elems = (size_t)ne02 * ne01 * nb32 * 2;+ size_t mn_elems = (size_t)ne02 * ne01 * nb32;+ extra->scale = clCreateBuffer(context, CL_MEM_READ_WRITE, sc_elems * sizeof(cl_half), NULL, &err); CL_CHECK(err);+ extra->min = clCreateBuffer(context, CL_MEM_READ_WRITE, mn_elems * sizeof(cl_half), NULL, &err); CL_CHECK(err);+ cl_kernel ek = backend_ctx->kernel_moe_expand_scale_q5_K;+ CL_CHECK(clSetKernelArg(ek, 0, sizeof(cl_mem), &extra->s));+ CL_CHECK(clSetKernelArg(ek, 1, sizeof(cl_mem), &extra->d));+ CL_CHECK(clSetKernelArg(ek, 2, sizeof(cl_mem), &extra->dm));+ CL_CHECK(clSetKernelArg(ek, 3, sizeof(cl_mem), &extra->scale));+ CL_CHECK(clSetKernelArg(ek, 4, sizeof(cl_mem), &extra->min));+ CL_CHECK(clSetKernelArg(ek, 5, sizeof(int), &ne00));+ CL_CHECK(clSetKernelArg(ek, 6, sizeof(int), &ne01));+ size_t eg[3] = { (size_t)(((ne01 + 63) / 64) * 64), (size_t)(ne00 / 256), (size_t)ne02 };+ size_t el[3] = { 64, 1, 1 };+ cl_event evt;+ CL_CHECK(clEnqueueNDRangeKernel(queue, ek, 3, NULL, eg, el, 0, NULL, &evt));+ CL_CHECK(clWaitForEvents(1, &evt));+ }+ }+return;}#endif // GGML_OPENCL_USE_ADRENO_KERNELS@@ -15245,6 +15819,61 @@ static void ggml_cl_mul_mat_q4_0_f32_adreno(ggml_backend_t backend, const ggml_tCL_CHECK(clReleaseMemObject(b_sub_buf));CL_CHECK(clReleaseMemObject(b_img));} else {+ // dp4a (int8) dense prefill GEMM: quant activations to q8_1, then the int8+ // dp4a inner-loop GEMM, in place of the transpose + f16 half-dot kernel.+ // q4_0 = d*(q-8); mirrors the IQ4_NL/q8_0 dense dp4a paths (+ the sum term).+ // OPT-IN / DEFAULT OFF: correct, but neutral on X2E. q4_0's dequant+ // ((q-8)*scale) is already trivial so the f16 GEMM is weight-BW-bound and the+ // int8 ALU win has nothing to beat -- same as q5_0 dense (unlike IQ4_NL, whose+ // codebook dequant is expensive enough for dp4a to help). Kept for A/B; force+ // on with GGML_OPENCL_Q4_0_DENSE_DP4A=1. Needs N>8, K%32==0, M%64==0.+ static const char * q4_0_dense_dp4a_env = getenv("GGML_OPENCL_Q4_0_DENSE_DP4A");+ const bool q4_0_dense_dp4a_on = q4_0_dense_dp4a_env+ ? (atoi(q4_0_dense_dp4a_env) != 0)+ : false;+ if (q4_0_dense_dp4a_on && backend_ctx->kernel_gemm_noshuffle_q4_0_q8_1_dp4a+ && N > 8 && (K % 32 == 0) && (M % 64 == 0)) {+ cl_mem a_sub = nullptr;+ region.origin = offset1;+ region.size = (size_t)K * N * sizeof(float);+ CL_CHECK((a_sub = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err));++ const size_t n_blocks = (size_t)N * (K / 32);+ backend_ctx->prealloc_moe_qa.allocate(context, (size_t)N * K * sizeof(cl_char));+ backend_ctx->prealloc_moe_da.allocate(context, n_blocks * sizeof(cl_half));+ backend_ctx->prealloc_moe_sa.allocate(context, n_blocks * sizeof(cl_half));++ cl_int tb = (cl_int)n_blocks;+ cl_kernel qk = backend_ctx->kernel_quant_a_q8_1;+ CL_CHECK(clSetKernelArg(qk, 0, sizeof(cl_mem), &a_sub));+ CL_CHECK(clSetKernelArg(qk, 1, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer));+ CL_CHECK(clSetKernelArg(qk, 2, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer));+ CL_CHECK(clSetKernelArg(qk, 3, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer));+ CL_CHECK(clSetKernelArg(qk, 4, sizeof(cl_int), &tb));+ size_t q_local[1] = { 64 };+ size_t q_global[1] = { (size_t)(((n_blocks + 63) / 64) * 64) };+ backend_ctx->enqueue_ndrange_kernel(qk, 1, q_global, q_local, dst);++ cl_kernel dk = backend_ctx->kernel_gemm_noshuffle_q4_0_q8_1_dp4a;+ int ai = 0;+ CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extra0_q4_0->q));+ CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extra0_q4_0->d));+ CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer));+ CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer));+ CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer));+ CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extrad->data_device));+ CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_ulong), &offsetd));+ CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_int), &M));+ CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_int), &N));+ CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_int), &K));+ size_t d_local[3] = { 64, 1, 1 };+ size_t d_global[3] = { 64, (size_t)(M / 64), (size_t)CEIL_DIV(N, 32) };+ backend_ctx->enqueue_ndrange_kernel(dk, 3, d_global, d_local, dst);++ CL_CHECK(clReleaseMemObject(a_sub));+ return;+ }+cl_mem b_sub_buf = nullptr;cl_mem b_sub_buf_trans = nullptr;cl_mem b_img = nullptr;@@ -15624,6 +16253,98 @@ static void ggml_cl_mul_mat_q5_0_f32_adreno(ggml_backend_t backend, const ggml_tCL_CHECK(clReleaseMemObject(b_sub_buf));CL_CHECK(clReleaseMemObject(b_img));} else {+ // dp4a (int8) dense q5_0 prefill GEMM. Quantizes the [N,K] activations to+ // q8_1 and runs the int8 dot instead of the f16 half-dot. Large-batch+ // (ne1>8) only. q5_0 weight = (x-16)*d (x = nibble | hi<<4); x packed as a+ // 0..31 byte (dp4a), the -16 centering folded into a single min term+ // (d*16) via the q8_1 block sum. Reads the qs/qh/d buffers byte-identically+ // to the f16 kernel (greedy byte-identical, MUL_MAT NMSE-OK).+ //+ // OPT-IN / DEFAULT OFF. Unlike q8_0/q4_K dense, dp4a is not a win for q5_0 on+ // X2E: the q5_0 model is bottlenecked elsewhere, so the dense-GEMM int8 win+ // has nothing to surface and the q8_1 prepass slightly hurts. Kept correct ++ // opt-in for the X1 A/B (different texture-cache dynamic) and the+ // weight-texture variant. Env: GGML_OPENCL_Q5_DENSE_DP4A=1.+ // Weight-as-texture variant (X1 lever): routes the dominant qs nibble plane+ // through an image1d_buffer (qh stays a buffer). Opt-in+ // GGML_OPENCL_Q5_DENSE_DP4A_WIMG; when set it also forces the dp4a path on.+ static const char * q5_dense_dp4a_env = getenv("GGML_OPENCL_Q5_DENSE_DP4A");+ static const char * q5_dense_wimg_env = getenv("GGML_OPENCL_Q5_DENSE_DP4A_WIMG");+ const bool q5_dense_wimg_on = q5_dense_wimg_env && (atoi(q5_dense_wimg_env) != 0);+ const bool q5_dense_dp4a_on = q5_dense_wimg_on+ ? true+ : (q5_dense_dp4a_env && (atoi(q5_dense_dp4a_env) != 0));+ if (q5_dense_dp4a_on && backend_ctx->kernel_gemm_noshuffle_q5_0_q8_1_dp4a+ && N > 8 && (K % 32 == 0) && (M % 64 == 0)) {+ cl_mem a_sub = nullptr;+ region.origin = offset1;+ region.size = (size_t)K * N * sizeof(float);+ CL_CHECK((a_sub = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err));++ const size_t n_blocks = (size_t)N * (K / 32);+ backend_ctx->prealloc_moe_qa.allocate(context, (size_t)N * K * sizeof(cl_char));+ backend_ctx->prealloc_moe_da.allocate(context, n_blocks * sizeof(cl_half));+ backend_ctx->prealloc_moe_sa.allocate(context, n_blocks * sizeof(cl_half));++ cl_int tb = (cl_int)n_blocks;+ cl_kernel qk = backend_ctx->kernel_quant_a_q8_1;+ CL_CHECK(clSetKernelArg(qk, 0, sizeof(cl_mem), &a_sub));+ CL_CHECK(clSetKernelArg(qk, 1, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer));+ CL_CHECK(clSetKernelArg(qk, 2, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer));+ CL_CHECK(clSetKernelArg(qk, 3, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer));+ CL_CHECK(clSetKernelArg(qk, 4, sizeof(cl_int), &tb));+ size_t q_local[1] = { 64 };+ size_t q_global[1] = { (size_t)(((n_blocks + 63) / 64) * 64) };+ backend_ctx->enqueue_ndrange_kernel(qk, 1, q_global, q_local, dst);++ // optional qs texture (image1d_buffer over the nibble plane; the same+ // CL_R/UINT32 view, width M*K/8, the GEMV path builds).+ cl_mem q5_qs_img = nullptr;+ bool use_wimg = q5_dense_wimg_on;+ if (use_wimg) {+ const size_t tex = (size_t)M * (size_t)K / 8; // uint32 texels (2 ushorts/texel)+ if (tex == 0 || tex > backend_ctx->image_max_buffer_size) {+ use_wimg = false;+ } else {+ img_fmt = { CL_R, CL_UNSIGNED_INT32 };+ memset(&img_desc, 0, sizeof(img_desc));+ img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER;+ img_desc.image_width = tex;+ img_desc.buffer = extra0_q5_0->qs;+ q5_qs_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err);+ if (err != CL_SUCCESS || q5_qs_img == nullptr) { use_wimg = false; q5_qs_img = nullptr; }+ }+ }++ cl_kernel dk = use_wimg ? backend_ctx->kernel_gemm_noshuffle_q5_0_q8_1_dp4a_wimg+ : backend_ctx->kernel_gemm_noshuffle_q5_0_q8_1_dp4a;+ int ai = 0;+ if (use_wimg) {+ CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &q5_qs_img));+ } else {+ CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extra0_q5_0->qs));+ }+ CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extra0_q5_0->qh));+ CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extra0_q5_0->d));+ CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer));+ CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer));+ CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer));+ CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extrad->data_device));+ CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_ulong), &offsetd));+ CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_int), &M));+ CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_int), &N));+ CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_int), &K));+ size_t d_local[3] = { 64, 1, 1 };+ size_t d_global[3] = { 64, (size_t)(M / 64), (size_t)CEIL_DIV(N, 32) };+ backend_ctx->enqueue_ndrange_kernel(dk, 3, d_global, d_local, dst);++ if (q5_qs_img != nullptr) {+ CL_CHECK(clReleaseMemObject(q5_qs_img));+ }+ CL_CHECK(clReleaseMemObject(a_sub));+ return;+ }+cl_mem b_sub_buf = nullptr;cl_mem b_sub_buf_trans = nullptr;cl_mem b_img = nullptr;@@ -15985,6 +16706,59 @@ static void ggml_cl_mul_mat_iq4_nl_f32_adreno(ggml_backend_t backend, const ggmlCL_CHECK(clReleaseMemObject(b_sub_buf));CL_CHECK(clReleaseMemObject(b_img));} else {+ // dp4a (int8) dense IQ4_NL prefill GEMM. Quantizes the [N,K] activations to+ // q8_1 and runs the int8 dot instead of the f16 half-dot. Large-batch+ // (ne1>8) only. IQ4_NL weight = kvalues[nibble]*d; the codebook value IS the+ // int8 (no min term), so this is the q8_0 dense case plus a nibble->int8 LUT+ // unpack. Reads the q/d buffers byte-identically to the f16 kernel. No bin+ // kernel for IQ4_NL -> baseline is f16, default ON for X2E (like q4_K/q6_K+ // dense dp4a). X1 stays on f16. Env: GGML_OPENCL_IQ4NL_DENSE_DP4A.+ static const char * iq4nl_dense_dp4a_env = getenv("GGML_OPENCL_IQ4NL_DENSE_DP4A");+ const bool iq4nl_dense_dp4a_on = iq4nl_dense_dp4a_env+ ? (atoi(iq4nl_dense_dp4a_env) != 0)+ : (backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E);+ if (iq4nl_dense_dp4a_on && backend_ctx->kernel_gemm_noshuffle_iq4_nl_q8_1_dp4a+ && N > 8 && (K % 32 == 0) && (M % 64 == 0)) {+ cl_mem a_sub = nullptr;+ region.origin = offset1;+ region.size = (size_t)K * N * sizeof(float);+ CL_CHECK((a_sub = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err));++ const size_t n_blocks = (size_t)N * (K / 32);+ backend_ctx->prealloc_moe_qa.allocate(context, (size_t)N * K * sizeof(cl_char));+ backend_ctx->prealloc_moe_da.allocate(context, n_blocks * sizeof(cl_half));+ backend_ctx->prealloc_moe_sa.allocate(context, n_blocks * sizeof(cl_half));++ cl_int tb = (cl_int)n_blocks;+ cl_kernel qk = backend_ctx->kernel_quant_a_q8_1;+ CL_CHECK(clSetKernelArg(qk, 0, sizeof(cl_mem), &a_sub));+ CL_CHECK(clSetKernelArg(qk, 1, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer));+ CL_CHECK(clSetKernelArg(qk, 2, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer));+ CL_CHECK(clSetKernelArg(qk, 3, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer));+ CL_CHECK(clSetKernelArg(qk, 4, sizeof(cl_int), &tb));+ size_t q_local[1] = { 64 };+ size_t q_global[1] = { (size_t)(((n_blocks + 63) / 64) * 64) };+ backend_ctx->enqueue_ndrange_kernel(qk, 1, q_global, q_local, dst);++ cl_kernel dk = backend_ctx->kernel_gemm_noshuffle_iq4_nl_q8_1_dp4a;+ int ai = 0;+ CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extra0_iq4_nl->q));+ CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extra0_iq4_nl->d));+ CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer));+ CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer));+ CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extrad->data_device));+ CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_ulong), &offsetd));+ CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_int), &M));+ CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_int), &N));+ CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_int), &K));+ size_t d_local[3] = { 64, 1, 1 };+ size_t d_global[3] = { 64, (size_t)(M / 64), (size_t)CEIL_DIV(N, 32) };+ backend_ctx->enqueue_ndrange_kernel(dk, 3, d_global, d_local, dst);++ CL_CHECK(clReleaseMemObject(a_sub));+ return;+ }+cl_mem b_sub_buf = nullptr;cl_mem b_sub_buf_trans = nullptr;cl_mem b_img = nullptr;@@ -16184,6 +16958,90 @@ static void ggml_cl_mul_mat_q8_0_f32_adreno(ggml_backend_t backend, const ggml_tCL_CHECK(clReleaseMemObject(b_img));CL_CHECK(clReleaseMemObject(b_sub_buf));} else {+ // dp4a dense q8_0 prefill GEMM. Quantizes the [N,K] activations to+ // q8_1 and runs the int8 dot instead of the f16 half-dot. Large-batch+ // (ne1>8) only; q8_0 weights are already int8 (no requant) and symmetric+ // (no min term)+ static const char * q8_dense_dp4a_env = getenv("GGML_OPENCL_Q8_DENSE_DP4A");+ static const char * q8_dense_wimg_env = getenv("GGML_OPENCL_Q8_DENSE_DP4A_WIMG");+ const bool q8_dense_wimg_on = q8_dense_wimg_env && (atoi(q8_dense_wimg_env) != 0);++ const bool q8_bin_loaded = (backend_ctx->kernel_gemm_noshuffle_q8_0_f32_bin != nullptr);+ // bin kernel takes precedence+ const bool q8_dense_dp4a_on = q8_dense_wimg_on+ ? true+ : q8_dense_dp4a_env+ ? (atoi(q8_dense_dp4a_env) != 0)+ : (backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E && !q8_bin_loaded);++ if (q8_dense_dp4a_on && backend_ctx->kernel_gemm_noshuffle_q8_0_q8_1_dp4a+ && N > 8 && (K % 32 == 0) && (M % 64 == 0)) {+ cl_mem a_sub = nullptr;+ region.origin = offset1;+ region.size = (size_t)K * N * sizeof(float);+ CL_CHECK((a_sub = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err));++ const size_t n_blocks = (size_t)N * (K / 32);+ backend_ctx->prealloc_moe_qa.allocate(context, (size_t)N * K * sizeof(cl_char));+ backend_ctx->prealloc_moe_da.allocate(context, n_blocks * sizeof(cl_half));+ backend_ctx->prealloc_moe_sa.allocate(context, n_blocks * sizeof(cl_half));++ cl_int tb = (cl_int)n_blocks;+ cl_kernel qk = backend_ctx->kernel_quant_a_q8_1;+ CL_CHECK(clSetKernelArg(qk, 0, sizeof(cl_mem), &a_sub));+ CL_CHECK(clSetKernelArg(qk, 1, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer));+ CL_CHECK(clSetKernelArg(qk, 2, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer));+ CL_CHECK(clSetKernelArg(qk, 3, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer));+ CL_CHECK(clSetKernelArg(qk, 4, sizeof(cl_int), &tb));+ size_t q_local[1] = { 64 };+ size_t q_global[1] = { (size_t)(((n_blocks + 63) / 64) * 64) };+ backend_ctx->enqueue_ndrange_kernel(qk, 1, q_global, q_local, dst);++ // optional weight texture, the same CL_R/UINT32 view, width M*K/4+ cl_mem q8_q_img = nullptr;+ bool use_wimg = q8_dense_wimg_on;+ if (use_wimg) {+ const size_t tex = (size_t)M * (size_t)K / 4; // uint32 texels+ if (tex == 0 || tex > backend_ctx->image_max_buffer_size) {+ use_wimg = false;+ } else {+ img_fmt = { CL_R, CL_UNSIGNED_INT32 };+ memset(&img_desc, 0, sizeof(img_desc));+ img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER;+ img_desc.image_width = tex;+ img_desc.buffer = extra0_q8_0->q;+ q8_q_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err);+ if (err != CL_SUCCESS || q8_q_img == nullptr) { use_wimg = false; q8_q_img = nullptr; }+ }+ }++ cl_kernel dk = use_wimg ? backend_ctx->kernel_gemm_noshuffle_q8_0_q8_1_dp4a_wimg+ : backend_ctx->kernel_gemm_noshuffle_q8_0_q8_1_dp4a;+ int ai = 0;+ if (use_wimg) {+ CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &q8_q_img));+ } else {+ CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extra0_q8_0->q));+ }+ CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extra0_q8_0->d));+ CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer));+ CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer));+ CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extrad->data_device));+ CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_ulong), &offsetd));+ CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_int), &M));+ CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_int), &N));+ CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_int), &K));+ size_t d_local[3] = { 64, 1, 1 };+ size_t d_global[3] = { 64, (size_t)(M / 64), (size_t)CEIL_DIV(N, 32) };+ backend_ctx->enqueue_ndrange_kernel(dk, 3, d_global, d_local, dst);++ if (q8_q_img != nullptr) {+ CL_CHECK(clReleaseMemObject(q8_q_img));+ }+ CL_CHECK(clReleaseMemObject(a_sub));+ return;+ }+// use bin kernel if availableif (backend_ctx->kernel_gemm_noshuffle_q8_0_f32_bin) {int K_pad = K;@@ -16517,6 +17375,99 @@ static void ggml_cl_mul_mat_q4_k_f32_adreno(ggml_backend_t backend, const ggml_tsize_t global_work_size_t[2] = { (size_t)width_B, (size_t)padded_height_B };backend_ctx->enqueue_ndrange_kernel(kernel, 2, global_work_size_t, local_work_size_t, dst);+ // dp4a (int8) dense prefill GEMM and weight via texture+ static const char * q4k_dense_dp4a_env = getenv("GGML_OPENCL_Q4K_DENSE_DP4A");+ static const char * q4k_dense_wimg_env = getenv("GGML_OPENCL_Q4K_DENSE_DP4A_WIMG");++ const bool q4k_dense_wimg_on = q4k_dense_wimg_env && (atoi(q4k_dense_wimg_env) != 0);+ const bool q4k_dense_dp4a_on = q4k_dense_wimg_on+ ? true+ : q4k_dense_dp4a_env+ ? (atoi(q4k_dense_dp4a_env) != 0)+ : (backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E);++ // Min N for the dp4a prefill GEMM, default 9, i.e., ne1 > 8+ static const char * q4k_dp4a_minn_env = getenv("GGML_OPENCL_Q4K_DP4A_MINN");+ const int q4k_dp4a_minn = q4k_dp4a_minn_env ? atoi(q4k_dp4a_minn_env) : 9;++ if (q4k_dense_dp4a_on && N >= q4k_dp4a_minn && (K % 32 == 0) && (M % 64 == 0)) {+ const size_t n_blocks = (size_t)N * (K / 32);+ backend_ctx->prealloc_moe_qa.allocate(context, (size_t)N * K * sizeof(cl_char));+ backend_ctx->prealloc_moe_da.allocate(context, n_blocks * sizeof(cl_half));+ backend_ctx->prealloc_moe_sa.allocate(context, n_blocks * sizeof(cl_half));++ cl_int tb = (cl_int)n_blocks;+ cl_kernel qk = backend_ctx->kernel_quant_a_q8_1;+ CL_CHECK(clSetKernelArg(qk, 0, sizeof(cl_mem), &b_sub_buf));+ CL_CHECK(clSetKernelArg(qk, 1, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer));+ CL_CHECK(clSetKernelArg(qk, 2, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer));+ CL_CHECK(clSetKernelArg(qk, 3, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer));+ CL_CHECK(clSetKernelArg(qk, 4, sizeof(cl_int), &tb));+ size_t q_local[1] = { 64 };+ size_t q_global[1] = { (size_t)(((n_blocks + 63) / 64) * 64) };+ backend_ctx->enqueue_ndrange_kernel(qk, 1, q_global, q_local, dst);++ // check if weights go through texture+ cl_mem q4k_q_img = nullptr;+ bool use_wimg = q4k_dense_wimg_on;+ if (use_wimg) {+ const size_t tex = (size_t)M * (size_t)K / 8; // uint32 texels = bytes/4+ if (tex == 0 || tex > backend_ctx->image_max_buffer_size) {+ use_wimg = false;+ } else {+ img_fmt = { CL_R, CL_UNSIGNED_INT32 };+ memset(&img_desc, 0, sizeof(img_desc));+ img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER;+ img_desc.image_width = tex;+ img_desc.buffer = extra0_q4_k->q;+ q4k_q_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err);+ if (err != CL_SUCCESS || q4k_q_img == nullptr) {+ use_wimg = false;+ q4k_q_img = nullptr;+ }+ }+ }++ cl_kernel dk = use_wimg ? backend_ctx->kernel_gemm_noshuffle_q4_k_q8_1_dp4a_wimg+ : backend_ctx->kernel_gemm_noshuffle_q4_k_q8_1_dp4a;+ int ai = 0;+ if (use_wimg) {+ CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &q4k_q_img));+ } else {+ CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extra0_q4_k->q));+ }+ CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extra0_q4_k->s));+ CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extra0_q4_k->d));+ CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extra0_q4_k->dm));+ CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer));+ CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer));+ CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer));+ CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extrad->data_device));+ CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_ulong), &offsetd));+ CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_int), &M));+ CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_int), &N));+ CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_int), &K));+ CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_uchar), &mask_d6));+ CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_uchar), &mask_d4));+ CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_uchar), &mask_hi2));+ // Must match the compile-time TILESIZE_N chosen at program build (per-device,+ // X1E=8 else 32; env override). Same inputs -> same value.+ int q4k_dp4a_ts = (backend_ctx->adreno_gen == ADRENO_GPU_GEN::X1E) ? 8 : 32;+ if (const char * e = getenv("GGML_OPENCL_Q4K_DP4A_TS")) q4k_dp4a_ts = atoi(e);+ size_t d_local[3] = { 64, 1, 1 };+ size_t d_global[3] = { 64, (size_t)(M / 64), (size_t)CEIL_DIV(N, q4k_dp4a_ts) };+ backend_ctx->enqueue_ndrange_kernel(dk, 3, d_global, d_local, dst);++ if (q4k_q_img != nullptr) {+ CL_CHECK(clReleaseMemObject(q4k_q_img));+ }+ CL_CHECK(clReleaseMemObject(b_sub_buf));+ CL_CHECK(clReleaseMemObject(b_sub_buf_trans));+ CL_CHECK(clReleaseMemObject(b_img));+ CL_CHECK(clReleaseMemObject(b_img_trans));+ return;+ }+// gemmkernel = backend_ctx->kernel_gemm_noshuffle_q4_k_f32;int padded_N = N + padding;@@ -16655,6 +17606,54 @@ static void ggml_cl_mul_mat_q6_K_f32_adreno(ggml_backend_t backend, const ggml_tregion.size = ne00 * ne1 * sizeof(float);CL_CHECK((b_sub_buf = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err));+ // dp4a (int8) dense q6_K prefill GEMM+ static const char * q6k_dense_dp4a_env = getenv("GGML_OPENCL_Q6K_DENSE_DP4A");+ static const bool q6k_dense_dp4a_on = (q6k_dense_dp4a_env != nullptr)+ ? (atoi(q6k_dense_dp4a_env) != 0)+ : (backend_ctx->adreno_gen != ADRENO_GPU_GEN::X1E);++ const bool is_output_w_dp4a = strncmp(src0->name, "output", 6) == 0 ||+ strncmp(src0->name, "token_embd", 10) == 0;++ if (q6k_dense_dp4a_on && !is_output_w_dp4a && ne1 > 8 && (ne00 % 32 == 0) && (ne01 % 64 == 0)) {+ const int M = ne01, N = ne1, K = ne00;+ const size_t n_blocks = (size_t)N * (K / 32);+ backend_ctx->prealloc_moe_qa.allocate(context, (size_t)N * K * sizeof(cl_char));+ backend_ctx->prealloc_moe_da.allocate(context, n_blocks * sizeof(cl_half));+ backend_ctx->prealloc_moe_sa.allocate(context, n_blocks * sizeof(cl_half));++ cl_int tb = (cl_int)n_blocks;+ cl_kernel qk = backend_ctx->kernel_quant_a_q8_1;+ CL_CHECK(clSetKernelArg(qk, 0, sizeof(cl_mem), &b_sub_buf));+ CL_CHECK(clSetKernelArg(qk, 1, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer));+ CL_CHECK(clSetKernelArg(qk, 2, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer));+ CL_CHECK(clSetKernelArg(qk, 3, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer));+ CL_CHECK(clSetKernelArg(qk, 4, sizeof(cl_int), &tb));+ size_t q_local[1] = { 64 };+ size_t q_global[1] = { (size_t)(((n_blocks + 63) / 64) * 64) };+ backend_ctx->enqueue_ndrange_kernel(qk, 1, q_global, q_local, dst);++ cl_kernel dk = backend_ctx->kernel_gemm_noshuffle_q6_k_q8_1_dp4a;+ int ai = 0;+ CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extra0_q6_K->ql));+ CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extra0_q6_K->qh));+ CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extra0_q6_K->s));+ CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extra0_q6_K->d));+ CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer));+ CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer));+ CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extrad->data_device));+ CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_ulong), &offsetd));+ CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_int), &M));+ CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_int), &N));+ CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_int), &K));+ size_t d_local[3] = { 64, 1, 1 };+ size_t d_global[3] = { 64, (size_t)(M / 64), (size_t)CEIL_DIV(N, 32) };+ backend_ctx->enqueue_ndrange_kernel(dk, 3, d_global, d_local, dst);++ CL_CHECK(clReleaseMemObject(b_sub_buf));+ return;+ }+// image for activationimg_fmt.image_channel_order = CL_RGBA;img_fmt.image_channel_data_type = CL_FLOAT;@@ -16900,6 +17899,59 @@ static void ggml_cl_mul_mat_q5_K_f32_adreno(ggml_backend_t backend, const ggml_tsize_t global_work_size_t[2] = {(size_t)width_B, (size_t)padded_height_B};backend_ctx->enqueue_ndrange_kernel(kernel, 2, global_work_size_t, local_work_size_t, dst);+ // dp4a (int8) dense q5_K prefill GEMM+ static const char * q5k_dense_dp4a_env = getenv("GGML_OPENCL_Q5K_DENSE_DP4A");+ const bool q5k_dense_dp4a_on = q5k_dense_dp4a_env+ ? (atoi(q5k_dense_dp4a_env) != 0)+ : (backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E);++ if (q5k_dense_dp4a_on && ne1 > 8 && (ne00 % 32 == 0) && (ne01 % 64 == 0)) {+ const int Mm = ne01, Nn = ne1, Kk = ne00;+ const size_t n_blocks = (size_t)Nn * (Kk / 32);+ backend_ctx->prealloc_moe_qa.allocate(context, (size_t)Nn * Kk * sizeof(cl_char));+ backend_ctx->prealloc_moe_da.allocate(context, n_blocks * sizeof(cl_half));+ backend_ctx->prealloc_moe_sa.allocate(context, n_blocks * sizeof(cl_half));++ cl_int tb = (cl_int)n_blocks;+ cl_kernel qk = backend_ctx->kernel_quant_a_q8_1;+ CL_CHECK(clSetKernelArg(qk, 0, sizeof(cl_mem), &b_sub_buf));+ CL_CHECK(clSetKernelArg(qk, 1, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer));+ CL_CHECK(clSetKernelArg(qk, 2, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer));+ CL_CHECK(clSetKernelArg(qk, 3, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer));+ CL_CHECK(clSetKernelArg(qk, 4, sizeof(cl_int), &tb));+ size_t q_local[1] = { 64 };+ size_t q_global[1] = { (size_t)(((n_blocks + 63) / 64) * 64) };+ backend_ctx->enqueue_ndrange_kernel(qk, 1, q_global, q_local, dst);++ cl_kernel dk = backend_ctx->kernel_gemm_noshuffle_q5_k_q8_1_dp4a;+ int ai = 0;+ CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extra0_q5_k->q));+ CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extra0_q5_k->qh));+ CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extra0_q5_k->s));+ CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extra0_q5_k->d));+ CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extra0_q5_k->dm));+ CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer));+ CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer));+ CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer));+ CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extrad->data_device));+ CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_ulong), &offsetd));+ CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_int), &Mm));+ CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_int), &Nn));+ CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_int), &Kk));+ CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_uchar), &mask_d6));+ CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_uchar), &mask_d4));+ CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_uchar), &mask_hi2));+ size_t d_local[3] = { 64, 1, 1 };+ size_t d_global[3] = { 64, (size_t)(Mm / 64), (size_t)CEIL_DIV(Nn, 32) };+ backend_ctx->enqueue_ndrange_kernel(dk, 3, d_global, d_local, dst);++ CL_CHECK(clReleaseMemObject(b_sub_buf));+ CL_CHECK(clReleaseMemObject(b_sub_buf_trans));+ CL_CHECK(clReleaseMemObject(b_img));+ CL_CHECK(clReleaseMemObject(b_img_trans));+ return;+ }+// gemmkernel = backend_ctx->kernel_gemm_noshuffle_q5_k_f32;int padded_N = N + padding;@@ -19380,6 +20432,23 @@ static void moe_router_reoerder(ggml_backend_t backend, const ggml_tensor * src,backend_ctx->enqueue_ndrange_kernel(kernel, 3, histogram_global_size, histogram_local_size, src);+ // [MOE_TILES] env-gated padding probe: read back total_tiles (= Sum_e+ // ceil(k_e/n_tile_size)) and compare to the ideal tile count for the real+ // routing count. Quantifies the per-expert tile-padding waste. Blocking+ // readback perturbs timing -> diagnostic only.+ if (getenv("GGML_OPENCL_MOE_TILES_DEBUG")) {+ int h_total = 0;+ clFinish(backend_ctx->queue);+ CL_CHECK(clEnqueueReadBuffer(backend_ctx->queue, total_tiles_buf, CL_TRUE, 0, sizeof(int), &h_total, 0, NULL, NULL));+ const int routings = ne20 * ne21;+ const int ideal = (routings + n_tile_size - 1) / n_tile_size;+ const int slots = h_total * n_tile_size;+ fprintf(stderr, "[MOE_TILES] routings=%d (ne20=%d ne21=%d nexp=%d) total_tiles=%d ideal=%d slots=%d pad=%.1f%%\n",+ routings, ne20, ne21, ne02, h_total, ideal, slots,+ routings > 0 ? 100.0 * (slots - routings) / routings : 0.0);+ fflush(stderr);+ }+CL_CHECK(clReleaseMemObject(original_router_buf));CL_CHECK(clReleaseMemObject(hist_buf));CL_CHECK(clReleaseMemObject(tile_offset_buf));@@ -19562,9 +20631,18 @@ static void ggml_cl_mul_mat_id(ggml_backend_t backend, const ggml_tensor * src0,backend_ctx->toggle_reorder = false;}- cl_mem sub_buf_src1_pre, buf_src1_reordered, image_src1_reordered, sub_buf_dst, buf_dst_image;+ cl_mem sub_buf_src1_pre, sub_buf_dst, buf_dst_image;+ cl_mem buf_src1_reordered = nullptr, image_src1_reordered = nullptr;cl_mem buf_src2, buf_src2_emap;+ // dp4a (int8) prefill GEMM variant+ static const char * q4_0_moe_dp4a_env = getenv("GGML_OPENCL_Q4_0_MOE_DP4A");+ bool use_moe_dp4a = q4_0_moe_dp4a_env+ ? (atoi(q4_0_moe_dp4a_env) != 0)+ : (backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E);+ // bin kernel takes precedence+ use_moe_dp4a = use_moe_dp4a && backend_ctx->kernel_gemm_moe_q4_0_f32_ns_bin == nullptr;+cl_buffer_region region;region.origin = 0;region.size = sizeof(int) * max_post_router_tile * n_tile_size;@@ -19583,45 +20661,48 @@ static void ggml_cl_mul_mat_id(ggml_backend_t backend, const ggml_tensor * src0,sub_buf_src1_pre = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &status);CL_CHECK(status);- // Create image for reordered src1- // Use pre-allocated placeholder- region.origin = 0;- region.size = ne00 * max_post_router_tile * n_tile_size * sizeof(float);- backend_ctx->prealloc_act_trans.allocate(backend_ctx->context, region.size);- buf_src1_reordered = clCreateSubBuffer(- backend_ctx->prealloc_act_trans.buffer,- 0,- CL_BUFFER_CREATE_TYPE_REGION,- ®ion,- &status);- CL_CHECK(status);- cl_image_format image_format_buf_src1;- cl_image_desc image_desc_buf_src1;- image_format_buf_src1 = {CL_RGBA, CL_FLOAT};- image_desc_buf_src1 = {CL_MEM_OBJECT_IMAGE1D_BUFFER, static_cast<size_t>(ne00 * max_post_router_tile * n_tile_size / 4), 0,0,0,0,0,0,0, {buf_src1_reordered}};- if (backend_ctx->kernel_gemm_moe_q4_0_f32_ns_bin) {- // bin kernel uses slightly different image format- image_format_buf_src1 = {CL_R, CL_FLOAT};- image_desc_buf_src1.image_width = static_cast<size_t>(ne00 * max_post_router_tile * n_tile_size);- }- image_src1_reordered = clCreateImage(backend_ctx->context, CL_MEM_READ_ONLY, &image_format_buf_src1, &image_desc_buf_src1, NULL, &status);- CL_CHECK(status);-unsigned short map_ratio = ne20 / ne11;GGML_ASSERT(((map_ratio == 1) || (map_ratio == ne20)) && "Map ratio not supported\n");- CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 0, sizeof(cl_mem), &sub_buf_src1_pre));- CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 1, sizeof(cl_mem), &buf_src2));- CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 2, sizeof(cl_mem), &buf_src1_reordered));- CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 3, sizeof(cl_mem), &(backend_ctx->prealloc_total_tiles.buffer)));- CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 4, sizeof(unsigned int), &ne00));- CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 5, sizeof(unsigned short), &map_ratio));- CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 6, sizeof(unsigned int), &n_tile_size));-- size_t reorder_b_local_size[3] = {256, 1, 1};- size_t reorder_b_global_size[3] = {static_cast<size_t>(((ne00 / 4) + 255) / 256 * 256), static_cast<size_t>(max_post_router_tile * n_tile_size), 1};- // Dispatch reorder kernel- backend_ctx->enqueue_ndrange_kernel(backend_ctx->kernel_moe_reorder_b, 3, reorder_b_global_size, reorder_b_local_size, dst);+ if (!use_moe_dp4a) {+ // Create image for reordered src1+ // Use pre-allocated placeholder+ region.origin = 0;+ region.size = ne00 * max_post_router_tile * n_tile_size * sizeof(float);+ backend_ctx->prealloc_act_trans.allocate(backend_ctx->context, region.size);+ buf_src1_reordered = clCreateSubBuffer(+ backend_ctx->prealloc_act_trans.buffer,+ 0,+ CL_BUFFER_CREATE_TYPE_REGION,+ ®ion,+ &status);+ CL_CHECK(status);+ cl_image_format image_format_buf_src1;+ cl_image_desc image_desc_buf_src1;+ image_format_buf_src1 = {CL_RGBA, CL_FLOAT};+ image_desc_buf_src1 = {CL_MEM_OBJECT_IMAGE1D_BUFFER, static_cast<size_t>(ne00 * max_post_router_tile * n_tile_size / 4), 0,0,0,0,0,0,0, {buf_src1_reordered}};+ if (backend_ctx->kernel_gemm_moe_q4_0_f32_ns_bin) {+ // bin kernel uses slightly different image format+ image_format_buf_src1 = {CL_R, CL_FLOAT};+ image_desc_buf_src1.image_width = static_cast<size_t>(ne00 * max_post_router_tile * n_tile_size);+ }+ image_src1_reordered = clCreateImage(backend_ctx->context, CL_MEM_READ_ONLY, &image_format_buf_src1, &image_desc_buf_src1, NULL, &status);+ CL_CHECK(status);++ CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 0, sizeof(cl_mem), &sub_buf_src1_pre));+ CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 1, sizeof(cl_mem), &buf_src2));+ CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 2, sizeof(cl_mem), &buf_src1_reordered));+ CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 3, sizeof(cl_mem), &(backend_ctx->prealloc_total_tiles.buffer)));+ CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 4, sizeof(unsigned int), &ne00));+ CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 5, sizeof(unsigned short), &map_ratio));+ CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 6, sizeof(unsigned int), &n_tile_size));++ size_t reorder_b_local_size[3] = {256, 1, 1};+ size_t reorder_b_global_size[3] = {static_cast<size_t>(((ne00 / 4) + 255) / 256 * 256), static_cast<size_t>(max_post_router_tile * n_tile_size), 1};++ // Dispatch reorder kernel+ backend_ctx->enqueue_ndrange_kernel(backend_ctx->kernel_moe_reorder_b, 3, reorder_b_global_size, reorder_b_local_size, dst);+ }// MoE kernel prepare// Create sub buffer for dst@@ -19640,6 +20721,58 @@ static void ggml_cl_mul_mat_id(ggml_backend_t backend, const ggml_tensor * src0,buf_dst_image = clCreateImage(backend_ctx->context, CL_MEM_WRITE_ONLY, &image_format_buf_dst, &image_desc_buf_dst, NULL, &status);CL_CHECK(status);+ if (use_moe_dp4a) {+ const size_t tok_slots = (size_t)max_post_router_tile * n_tile_size;+ const size_t n_blocks = tok_slots * (ne00 / 32);+ backend_ctx->prealloc_moe_qa.allocate(backend_ctx->context, tok_slots * ne00 * sizeof(cl_char));+ backend_ctx->prealloc_moe_da.allocate(backend_ctx->context, n_blocks * sizeof(cl_half));+ backend_ctx->prealloc_moe_sa.allocate(backend_ctx->context, n_blocks * sizeof(cl_half));++ // fused reorder + q8_1 quant straight from the original activations+ const cl_uint n_kblocks = (cl_uint)(ne00 / 32);+ cl_kernel rq = backend_ctx->kernel_moe_reorder_quant_a_q8_1;+ CL_CHECK(clSetKernelArg(rq, 0, sizeof(cl_mem), &sub_buf_src1_pre));+ CL_CHECK(clSetKernelArg(rq, 1, sizeof(cl_mem), &buf_src2));+ CL_CHECK(clSetKernelArg(rq, 2, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer));+ CL_CHECK(clSetKernelArg(rq, 3, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer));+ CL_CHECK(clSetKernelArg(rq, 4, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer));+ CL_CHECK(clSetKernelArg(rq, 5, sizeof(cl_mem), &(backend_ctx->prealloc_total_tiles.buffer)));+ CL_CHECK(clSetKernelArg(rq, 6, sizeof(cl_uint), &ne00));+ CL_CHECK(clSetKernelArg(rq, 7, sizeof(unsigned short), &map_ratio));+ CL_CHECK(clSetKernelArg(rq, 8, sizeof(cl_uint), &n_tile_size));+ CL_CHECK(clSetKernelArg(rq, 9, sizeof(cl_uint), &n_kblocks));+ size_t rq_local[2] = { 32, 1 };+ size_t rq_global[2] = { (size_t)(((n_kblocks + 31) / 32) * 32), tok_slots };+ backend_ctx->enqueue_ndrange_kernel(rq, 2, rq_global, rq_local, dst);++ // dp4a GEMM+ cl_kernel dk = backend_ctx->kernel_gemm_moe_q4_0_q8_1_dp4a;+ int aidx = 0;+ CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &extra0_q4_0->q_img));+ CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &extra0_q4_0->d));+ CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer));+ CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer));+ CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer));+ CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &buf_src2));+ CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &buf_src2_emap));+ CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &buf_dst_image));+ CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &(backend_ctx->prealloc_total_tiles.buffer)));+ CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(int), &ne00));+ CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(int), &ne01));+ CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(int), &backend_ctx->adreno_use_moe_ragged_dp4));++ size_t dp_global[3] = { 64, (size_t)((ne01 + 63) / 64), (size_t)max_post_router_tile };+ size_t dp_local[3] = { 64, 1, 1 };+ backend_ctx->enqueue_ndrange_kernel(dk, 3, dp_global, dp_local, dst);++ clReleaseMemObject(sub_buf_src1_pre);+ clReleaseMemObject(buf_src2);+ clReleaseMemObject(buf_src2_emap);+ clReleaseMemObject(sub_buf_dst);+ clReleaseMemObject(buf_dst_image);+ return;+ }+// Set kernel argsint arg_idx = 0;CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &extra0_q4_0->q_img));@@ -19993,6 +21126,81 @@ static void ggml_cl_mul_mat_id(ggml_backend_t backend, const ggml_tensor * src0,sub_buf_src1_pre = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &status);CL_CHECK(status);+ // Generic dp4a MoE GEMM+ {+ static const char * q5mdp4a_env = getenv("GGML_OPENCL_Q5_MOE_DP4A");+ const bool q5mdp4a_on = q5mdp4a_env ? (atoi(q5mdp4a_env) != 0)+ : (backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E);+ const bool use_q5_moe_dp4a = q5mdp4a_on+ && backend_ctx->kernel_gemm_moe_q8_1_dp4a_q50 != nullptr+ && extra0_q5_0->scale != nullptr;++ if (use_q5_moe_dp4a) {+ const size_t tok_slots = (size_t)max_post_router_tile * n_tile_size;+ const size_t n_blocks = tok_slots * (ne00 / 32);+ backend_ctx->prealloc_moe_qa.allocate(backend_ctx->context, tok_slots * ne00 * sizeof(cl_char));+ backend_ctx->prealloc_moe_da.allocate(backend_ctx->context, n_blocks * sizeof(cl_half));+ backend_ctx->prealloc_moe_sa.allocate(backend_ctx->context, n_blocks * sizeof(cl_half));++ const cl_uint n_kblocks = (cl_uint)(ne00 / 32);+ unsigned short map_ratio_q5 = ne20 / ne11;+ cl_kernel rq = backend_ctx->kernel_moe_reorder_quant_a_q8_1;+ CL_CHECK(clSetKernelArg(rq, 0, sizeof(cl_mem), &sub_buf_src1_pre));+ CL_CHECK(clSetKernelArg(rq, 1, sizeof(cl_mem), &buf_src2));+ CL_CHECK(clSetKernelArg(rq, 2, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer));+ CL_CHECK(clSetKernelArg(rq, 3, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer));+ CL_CHECK(clSetKernelArg(rq, 4, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer));+ CL_CHECK(clSetKernelArg(rq, 5, sizeof(cl_mem), &(backend_ctx->prealloc_total_tiles.buffer)));+ CL_CHECK(clSetKernelArg(rq, 6, sizeof(cl_uint), &ne00));+ CL_CHECK(clSetKernelArg(rq, 7, sizeof(unsigned short), &map_ratio_q5));+ CL_CHECK(clSetKernelArg(rq, 8, sizeof(cl_uint), &n_tile_size));+ CL_CHECK(clSetKernelArg(rq, 9, sizeof(cl_uint), &n_kblocks));+ size_t rq_local[2] = { 32, 1 };+ size_t rq_global[2] = { (size_t)(((n_kblocks + 31) / 32) * 32), tok_slots };+ backend_ctx->enqueue_ndrange_kernel(rq, 2, rq_global, rq_local, dst);++ region.origin = offsetd;+ region.size = ne0 * ne1 * ne2 * sizeof(float);+ cl_mem dp_sub_buf_dst = clCreateSubBuffer(extrad->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &status);+ CL_CHECK(status);+ cl_image_format dp_ifd = {CL_R, CL_FLOAT};+ cl_image_desc dp_idd = {CL_MEM_OBJECT_IMAGE1D_BUFFER, static_cast<size_t>(ne0 * ne1 * ne2), 0,0,0,0,0,0,0, {dp_sub_buf_dst}};+ cl_mem dp_buf_dst_image = clCreateImage(backend_ctx->context, CL_MEM_WRITE_ONLY, &dp_ifd, &dp_idd, NULL, &status);+ CL_CHECK(status);++ int ne00i = (int)ne00, ne01i = (int)ne01;+ cl_kernel dk = backend_ctx->kernel_gemm_moe_q8_1_dp4a_q50;+ int has_min_q5 = 1;+ int aidx = 0;+ CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &extra0_q5_0->qs_img));+ CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &extra0_q5_0->qh));+ CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &extra0_q5_0->scale));+ CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &extra0_q5_0->min));+ CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer));+ CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer));+ CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer));+ CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &buf_src2));+ CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &buf_src2_emap));+ CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &dp_buf_dst_image));+ CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &(backend_ctx->prealloc_total_tiles.buffer)));+ CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(int), &ne00i));+ CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(int), &ne01i));+ CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(int), &backend_ctx->adreno_use_moe_ragged_dp4));+ CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(int), &has_min_q5));++ size_t dp_global[3] = { 64, (size_t)((ne01 + 63) / 64), (size_t)max_post_router_tile };+ size_t dp_local[3] = { 64, 1, 1 };+ backend_ctx->enqueue_ndrange_kernel(dk, 3, dp_global, dp_local, dst);++ clReleaseMemObject(sub_buf_src1_pre);+ clReleaseMemObject(buf_src2);+ clReleaseMemObject(buf_src2_emap);+ clReleaseMemObject(dp_sub_buf_dst);+ clReleaseMemObject(dp_buf_dst_image);+ return;+ }+ }+// Create image for reordered src1// Use pre-allocated placeholderregion.origin = 0;@@ -20261,6 +21469,183 @@ static void ggml_cl_mul_mat_id(ggml_backend_t backend, const ggml_tensor * src0,#endif //GGML_OPENCL_USE_ADRENO_KERNELS}case GGML_TYPE_Q8_0: {+#ifdef GGML_OPENCL_USE_ADRENO_KERNELS+ // MoE GEMM for q8_0 at prefill (ne12>1)+ // There is no corresponding gemv_moe, so the code path is different here+ static const char * moe_gemm_q8_env = getenv("GGML_OPENCL_MOE_GEMM_Q8");+ const bool moe_gemm_q8 = moe_gemm_q8_env+ ? (atoi(moe_gemm_q8_env) != 0)+ : (backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E);+ if (moe_gemm_q8 && use_adreno_moe_kernels(backend_ctx, src0) && ne12 > 1) {+ cl_int status;++ size_t local_size[3] = {64, 2, 1};+ size_t global_size[3] = {64, 2, 1};++ kernel = backend_ctx->kernel_gemm_moe_q8_0_f32_ns;++ if ((strstr(src0->name, "as") != NULL) || backend_ctx->toggle_reorder) {+ moe_router_reoerder(backend, src2, ne20);+ backend_ctx->toggle_reorder = false;+ }++ cl_mem sub_buf_src1_pre, buf_src1_reordered, image_src1_reordered, sub_buf_dst, buf_dst_image;+ cl_mem buf_src2, buf_src2_emap;++ cl_buffer_region region;+ region.origin = 0;+ region.size = sizeof(int) * max_post_router_tile * n_tile_size;+ buf_src2 = clCreateSubBuffer(backend_ctx->prealloc_post_router.buffer, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &status);+ CL_CHECK(status);++ region.origin = 0;+ region.size = sizeof(short) * max_post_router_tile;+ buf_src2_emap = clCreateSubBuffer(backend_ctx->prealloc_emap.buffer, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &status);+ CL_CHECK(status);++ // Reorder activations (group tokens by expert into tiles of 32)+ region.origin = offset1;+ region.size = ne10 * ne11 * ne12 * sizeof(float);+ sub_buf_src1_pre = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &status);+ CL_CHECK(status);++ // Generic dp4a MoE GEMM+ {+ static const char * q8mdp4a_env = getenv("GGML_OPENCL_Q8_MOE_DP4A");+ const bool q8mdp4a_on = q8mdp4a_env ? (atoi(q8mdp4a_env) != 0)+ : (backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E);+ const bool use_q8_moe_dp4a = q8mdp4a_on+ && backend_ctx->kernel_gemm_moe_q8_1_dp4a_q80 != nullptr+ && extra0_q8_0->scale != nullptr;+ if (use_q8_moe_dp4a) {+ const size_t tok_slots = (size_t)max_post_router_tile * n_tile_size;+ const size_t n_blocks = tok_slots * (ne00 / 32);+ backend_ctx->prealloc_moe_qa.allocate(backend_ctx->context, tok_slots * ne00 * sizeof(cl_char));+ backend_ctx->prealloc_moe_da.allocate(backend_ctx->context, n_blocks * sizeof(cl_half));+ backend_ctx->prealloc_moe_sa.allocate(backend_ctx->context, n_blocks * sizeof(cl_half));++ const cl_uint n_kblocks = (cl_uint)(ne00 / 32);+ unsigned short map_ratio_q8 = ne20 / ne11;+ cl_kernel rq = backend_ctx->kernel_moe_reorder_quant_a_q8_1;+ CL_CHECK(clSetKernelArg(rq, 0, sizeof(cl_mem), &sub_buf_src1_pre));+ CL_CHECK(clSetKernelArg(rq, 1, sizeof(cl_mem), &buf_src2));+ CL_CHECK(clSetKernelArg(rq, 2, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer));+ CL_CHECK(clSetKernelArg(rq, 3, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer));+ CL_CHECK(clSetKernelArg(rq, 4, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer));+ CL_CHECK(clSetKernelArg(rq, 5, sizeof(cl_mem), &(backend_ctx->prealloc_total_tiles.buffer)));+ CL_CHECK(clSetKernelArg(rq, 6, sizeof(cl_uint), &ne00));+ CL_CHECK(clSetKernelArg(rq, 7, sizeof(unsigned short), &map_ratio_q8));+ CL_CHECK(clSetKernelArg(rq, 8, sizeof(cl_uint), &n_tile_size));+ CL_CHECK(clSetKernelArg(rq, 9, sizeof(cl_uint), &n_kblocks));+ size_t rq_local[2] = { 32, 1 };+ size_t rq_global[2] = { (size_t)(((n_kblocks + 31) / 32) * 32), tok_slots };+ backend_ctx->enqueue_ndrange_kernel(rq, 2, rq_global, rq_local, dst);++ // dst image+ region.origin = offsetd;+ region.size = ne0 * ne1 * ne2 * sizeof(float);+ cl_mem dp_sub_buf_dst = clCreateSubBuffer(extrad->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &status);+ CL_CHECK(status);+ cl_image_format dp_ifd = {CL_R, CL_FLOAT};+ cl_image_desc dp_idd = {CL_MEM_OBJECT_IMAGE1D_BUFFER, static_cast<size_t>(ne0 * ne1 * ne2), 0,0,0,0,0,0,0, {dp_sub_buf_dst}};+ cl_mem dp_buf_dst_image = clCreateImage(backend_ctx->context, CL_MEM_WRITE_ONLY, &dp_ifd, &dp_idd, NULL, &status);+ CL_CHECK(status);++ int ne00i = (int)ne00, ne01i = (int)ne01;+ cl_kernel dk = backend_ctx->kernel_gemm_moe_q8_1_dp4a_q80;+ int has_min_q8 = 0;+ int aidx = 0;+ CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &extra0_q8_0->q)); // flat int8 codes [expert][row][K]+ CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &extra0_q8_0->scale)); // uniform scale[16]+ CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &extra0_q8_0->scale)); // dummy min (has_min=0, unread)+ CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer));+ CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer));+ CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer));+ CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &buf_src2));+ CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &buf_src2_emap));+ CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &dp_buf_dst_image));+ CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &(backend_ctx->prealloc_total_tiles.buffer)));+ CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(int), &ne00i));+ CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(int), &ne01i));+ CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(int), &backend_ctx->adreno_use_moe_ragged_dp4));+ CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(int), &has_min_q8));++ size_t dp_global[3] = { 64, (size_t)((ne01 + 63) / 64), (size_t)max_post_router_tile };+ size_t dp_local[3] = { 64, 1, 1 };+ backend_ctx->enqueue_ndrange_kernel(dk, 3, dp_global, dp_local, dst);++ clReleaseMemObject(sub_buf_src1_pre);+ clReleaseMemObject(buf_src2);+ clReleaseMemObject(buf_src2_emap);+ clReleaseMemObject(dp_sub_buf_dst);+ clReleaseMemObject(dp_buf_dst_image);+ return;+ }+ }++ region.origin = 0;+ region.size = ne00 * max_post_router_tile * n_tile_size * sizeof(float);+ backend_ctx->prealloc_act_trans.allocate(backend_ctx->context, region.size);+ buf_src1_reordered = clCreateSubBuffer(+ backend_ctx->prealloc_act_trans.buffer, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &status);+ CL_CHECK(status);+ cl_image_format image_format_buf_src1 = {CL_RGBA, CL_FLOAT};+ cl_image_desc image_desc_buf_src1 = {CL_MEM_OBJECT_IMAGE1D_BUFFER, static_cast<size_t>(ne00 * max_post_router_tile * n_tile_size / 4), 0,0,0,0,0,0,0, {buf_src1_reordered}};+ image_src1_reordered = clCreateImage(backend_ctx->context, CL_MEM_READ_ONLY, &image_format_buf_src1, &image_desc_buf_src1, NULL, &status);+ CL_CHECK(status);++ unsigned short map_ratio = ne20 / ne11;+ GGML_ASSERT(((map_ratio == 1) || (map_ratio == ne20)) && "Map ratio not supported\n");+ CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 0, sizeof(cl_mem), &sub_buf_src1_pre));+ CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 1, sizeof(cl_mem), &buf_src2));+ CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 2, sizeof(cl_mem), &buf_src1_reordered));+ CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 3, sizeof(cl_mem), &(backend_ctx->prealloc_total_tiles.buffer)));+ CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 4, sizeof(unsigned int), &ne00));+ CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 5, sizeof(unsigned short), &map_ratio));+ CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 6, sizeof(unsigned int), &n_tile_size));++ size_t reorder_b_local_size[3] = {256, 1, 1};+ size_t reorder_b_global_size[3] = {static_cast<size_t>(((ne00 / 4) + 255) / 256 * 256), static_cast<size_t>(max_post_router_tile * n_tile_size), 1};+ backend_ctx->enqueue_ndrange_kernel(backend_ctx->kernel_moe_reorder_b, 3, reorder_b_global_size, reorder_b_local_size, dst);++ // dst image+ region.origin = offsetd;+ region.size = ne0 * ne1 * ne2 * sizeof(float);+ sub_buf_dst = clCreateSubBuffer(extrad->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &status);+ CL_CHECK(status);+ cl_image_format image_format_buf_dst = {CL_R, CL_FLOAT};+ cl_image_desc image_desc_buf_dst = {CL_MEM_OBJECT_IMAGE1D_BUFFER, static_cast<size_t>(ne0 * ne1 * ne2), 0,0,0,0,0,0,0, {sub_buf_dst}};+ buf_dst_image = clCreateImage(backend_ctx->context, CL_MEM_WRITE_ONLY, &image_format_buf_dst, &image_desc_buf_dst, NULL, &status);+ CL_CHECK(status);++ int arg_idx = 0;+ CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &extra0_q8_0->q)); // flat q8_0 quants+ CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &extra0_q8_0->d)); // flat q8_0 scales+ CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &image_src1_reordered));+ CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &buf_src2));+ CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &buf_src2_emap));+ CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &buf_dst_image));+ CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &(backend_ctx->prealloc_total_tiles.buffer)));+ CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(int), &ne00));+ CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(int), &ne01));++ global_size[1] = static_cast<size_t>((ne01 + 63) / 64);+ global_size[2] = static_cast<size_t>(max_post_router_tile);+ local_size[1] = 1;+ local_size[2] = 1;++ backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_size, local_size, dst);++ clReleaseMemObject(sub_buf_src1_pre);+ clReleaseMemObject(buf_src1_reordered);+ clReleaseMemObject(image_src1_reordered);+ clReleaseMemObject(buf_src2);+ clReleaseMemObject(buf_src2_emap);+ clReleaseMemObject(sub_buf_dst);+ clReleaseMemObject(buf_dst_image);+ return;+ }+#endif // GGML_OPENCL_USE_ADRENO_KERNELS#ifdef GGML_OPENCL_SOA_Qkernel = backend_ctx->kernel_mul_mv_id_q8_0_f32_flat;@@ -20347,6 +21732,18 @@ static void ggml_cl_mul_mat_id(ggml_backend_t backend, const ggml_tensor * src0,if (ne12 == 1) { // for gemvkernel = backend_ctx->kernel_gemv_moe_q4_k_f32_ns;+ // Weight-as-texture MoE decode GEMV+ static const char * moe_decode_wimg_env = getenv("GGML_OPENCL_MOE_DECODE_WIMG");+ const bool moe_decode_wimg_on = moe_decode_wimg_env+ ? (atoi(moe_decode_wimg_env) != 0)+ : (backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E);+ const bool use_moe_decode_wimg = moe_decode_wimg_on+ && backend_ctx->kernel_gemv_moe_q4_k_f32_ns_wimg != nullptr+ && extra0_q4_K->q_img != nullptr;+ if (use_moe_decode_wimg) {+ kernel = backend_ctx->kernel_gemv_moe_q4_k_f32_ns_wimg;+ }+cl_mem src1_sub_buffer, buf_src1_image, buf_src2;// create a sub_buffer for src2@@ -20376,7 +21773,7 @@ static void ggml_cl_mul_mat_id(ggml_backend_t backend, const ggml_tensor * src0,// Set kernel argsint arg_idx = 0;- CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &extra0_q4_K->q));+ CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), use_moe_decode_wimg ? &extra0_q4_K->q_img : &extra0_q4_K->q));CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &extra0_q4_K->d));CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &extra0_q4_K->dm));CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &extra0_q4_K->s));@@ -20409,9 +21806,18 @@ static void ggml_cl_mul_mat_id(ggml_backend_t backend, const ggml_tensor * src0,backend_ctx->toggle_reorder = false;}- cl_mem sub_buf_src1_pre, buf_src1_reordered, image_src1_reordered, sub_buf_dst, buf_dst_image;+ cl_mem sub_buf_src1_pre, sub_buf_dst, buf_dst_image;+ cl_mem buf_src1_reordered = nullptr, image_src1_reordered = nullptr;cl_mem buf_src2, buf_src2_emap;+ // dp4a (int8) prefill GEMM variant+ static const char * q4k_moe_dp4a_env = getenv("GGML_OPENCL_Q4K_MOE_DP4A");+ bool use_moe_dp4a = (q4k_moe_dp4a_env != nullptr)+ ? (atoi(q4k_moe_dp4a_env) != 0)+ : (backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E || backend_ctx->adreno_gen == ADRENO_GPU_GEN::X1E);+ // bin kernel takes precedence+ use_moe_dp4a = use_moe_dp4a && backend_ctx->kernel_gemm_moe_q4_k_f32_ns_bin == nullptr;+cl_buffer_region region;region.origin = 0;region.size = sizeof(int) * max_post_router_tile * n_tile_size;@@ -20429,42 +21835,45 @@ static void ggml_cl_mul_mat_id(ggml_backend_t backend, const ggml_tensor * src0,sub_buf_src1_pre = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &status);CL_CHECK(status);- // Create image for reordered src1- region.origin = 0;- region.size = ne00 * max_post_router_tile * n_tile_size * sizeof(float);- backend_ctx->prealloc_act_trans.allocate(backend_ctx->context, region.size);- buf_src1_reordered = clCreateSubBuffer(- backend_ctx->prealloc_act_trans.buffer,- 0,- CL_BUFFER_CREATE_TYPE_REGION,- ®ion,- &status);- CL_CHECK(status);- cl_image_format image_format_buf_src1 = {CL_RGBA, CL_FLOAT};- cl_image_desc image_desc_buf_src1 = {CL_MEM_OBJECT_IMAGE1D_BUFFER, static_cast<size_t>(ne00 * max_post_router_tile * n_tile_size / 4), 0,0,0,0,0,0,0, {buf_src1_reordered}};- if (backend_ctx->kernel_gemm_moe_q4_k_f32_ns_bin) {- // bin kernel uses slightly different image format- image_format_buf_src1 = {CL_R, CL_FLOAT};- image_desc_buf_src1.image_width = static_cast<size_t>(ne00 * max_post_router_tile * n_tile_size);- }- image_src1_reordered = clCreateImage(backend_ctx->context, CL_MEM_READ_ONLY, &image_format_buf_src1, &image_desc_buf_src1, NULL, &status);- CL_CHECK(status);-unsigned short map_ratio = ne20 / ne11;GGML_ASSERT(((map_ratio == 1) || (map_ratio == ne20)) && "Map ratio not supported\n");- CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 0, sizeof(cl_mem), &sub_buf_src1_pre));- CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 1, sizeof(cl_mem), &buf_src2));- CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 2, sizeof(cl_mem), &buf_src1_reordered));- CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 3, sizeof(cl_mem), &(backend_ctx->prealloc_total_tiles.buffer)));- CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 4, sizeof(unsigned int), &ne00));- CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 5, sizeof(unsigned short), &map_ratio));- CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 6, sizeof(unsigned int), &n_tile_size));-- size_t reorder_b_local_size[3] = {256, 1, 1};- size_t reorder_b_global_size[3] = {static_cast<size_t>(((ne00 / 4) + 255) / 256 * 256), static_cast<size_t>(max_post_router_tile * n_tile_size), 1};- // Dispatch reorder kernel- backend_ctx->enqueue_ndrange_kernel(backend_ctx->kernel_moe_reorder_b, 3, reorder_b_global_size, reorder_b_local_size, dst);+ if (!use_moe_dp4a) {+ // Create image for reordered src1+ region.origin = 0;+ region.size = ne00 * max_post_router_tile * n_tile_size * sizeof(float);+ backend_ctx->prealloc_act_trans.allocate(backend_ctx->context, region.size);+ buf_src1_reordered = clCreateSubBuffer(+ backend_ctx->prealloc_act_trans.buffer,+ 0,+ CL_BUFFER_CREATE_TYPE_REGION,+ ®ion,+ &status);+ CL_CHECK(status);+ cl_image_format image_format_buf_src1 = {CL_RGBA, CL_FLOAT};+ cl_image_desc image_desc_buf_src1 = {CL_MEM_OBJECT_IMAGE1D_BUFFER, static_cast<size_t>(ne00 * max_post_router_tile * n_tile_size / 4), 0,0,0,0,0,0,0, {buf_src1_reordered}};+ if (backend_ctx->kernel_gemm_moe_q4_k_f32_ns_bin) {+ // bin kernel uses slightly different image format+ image_format_buf_src1 = {CL_R, CL_FLOAT};+ image_desc_buf_src1.image_width = static_cast<size_t>(ne00 * max_post_router_tile * n_tile_size);+ }+ image_src1_reordered = clCreateImage(backend_ctx->context, CL_MEM_READ_ONLY, &image_format_buf_src1, &image_desc_buf_src1, NULL, &status);+ CL_CHECK(status);++ CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 0, sizeof(cl_mem), &sub_buf_src1_pre));+ CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 1, sizeof(cl_mem), &buf_src2));+ CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 2, sizeof(cl_mem), &buf_src1_reordered));+ CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 3, sizeof(cl_mem), &(backend_ctx->prealloc_total_tiles.buffer)));+ CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 4, sizeof(unsigned int), &ne00));+ CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 5, sizeof(unsigned short), &map_ratio));+ CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 6, sizeof(unsigned int), &n_tile_size));++ size_t reorder_b_local_size[3] = {256, 1, 1};+ size_t reorder_b_global_size[3] = {static_cast<size_t>(((ne00 / 4) + 255) / 256 * 256), static_cast<size_t>(max_post_router_tile * n_tile_size), 1};++ // Dispatch reorder kernel+ backend_ctx->enqueue_ndrange_kernel(backend_ctx->kernel_moe_reorder_b, 3, reorder_b_global_size, reorder_b_local_size, dst);+ }// MoE kernel prepareregion.origin = offsetd;@@ -20482,6 +21891,61 @@ static void ggml_cl_mul_mat_id(ggml_backend_t backend, const ggml_tensor * src0,buf_dst_image = clCreateImage(backend_ctx->context, CL_MEM_WRITE_ONLY, &image_format_buf_dst, &image_desc_buf_dst, NULL, &status);CL_CHECK(status);+ if (use_moe_dp4a) {+ const size_t tok_slots = (size_t)max_post_router_tile * n_tile_size;+ const size_t n_blocks = tok_slots * (ne00 / 32);+ backend_ctx->prealloc_moe_qa.allocate(backend_ctx->context, tok_slots * ne00 * sizeof(cl_char));+ backend_ctx->prealloc_moe_da.allocate(backend_ctx->context, n_blocks * sizeof(cl_half));+ backend_ctx->prealloc_moe_sa.allocate(backend_ctx->context, n_blocks * sizeof(cl_half));++ // fused reorder + q8_1 quant straight from the original+ // activations (no intermediate f32 reorder buffer)+ const cl_uint n_kblocks = (cl_uint)(ne00 / 32);+ cl_kernel rq = backend_ctx->kernel_moe_reorder_quant_a_q8_1;+ CL_CHECK(clSetKernelArg(rq, 0, sizeof(cl_mem), &sub_buf_src1_pre));+ CL_CHECK(clSetKernelArg(rq, 1, sizeof(cl_mem), &buf_src2));+ CL_CHECK(clSetKernelArg(rq, 2, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer));+ CL_CHECK(clSetKernelArg(rq, 3, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer));+ CL_CHECK(clSetKernelArg(rq, 4, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer));+ CL_CHECK(clSetKernelArg(rq, 5, sizeof(cl_mem), &(backend_ctx->prealloc_total_tiles.buffer)));+ CL_CHECK(clSetKernelArg(rq, 6, sizeof(cl_uint), &ne00));+ CL_CHECK(clSetKernelArg(rq, 7, sizeof(unsigned short), &map_ratio));+ CL_CHECK(clSetKernelArg(rq, 8, sizeof(cl_uint), &n_tile_size));+ CL_CHECK(clSetKernelArg(rq, 9, sizeof(cl_uint), &n_kblocks));+ size_t rq_local[2] = { 32, 1 };+ size_t rq_global[2] = { (size_t)(((n_kblocks + 31) / 32) * 32), tok_slots };+ backend_ctx->enqueue_ndrange_kernel(rq, 2, rq_global, rq_local, dst);++ // dp4a GEMM+ cl_kernel dk = backend_ctx->kernel_gemm_moe_q4_k_q8_1_dp4a;+ int aidx = 0;+ CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &extra0_q4_K->q_img));+ CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &extra0_q4_K->d));+ CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &extra0_q4_K->dm));+ CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &extra0_q4_K->s));+ CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer));+ CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer));+ CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer));+ CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &buf_src2));+ CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &buf_src2_emap));+ CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &buf_dst_image));+ CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &(backend_ctx->prealloc_total_tiles.buffer)));+ CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(int), &ne00));+ CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(int), &ne01));+ CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(int), &backend_ctx->adreno_use_moe_ragged_dp4));++ size_t dp_global[3] = { 64, (size_t)((ne01 + 63) / 64), (size_t)max_post_router_tile };+ size_t dp_local[3] = { 64, 1, 1 };+ backend_ctx->enqueue_ndrange_kernel(dk, 3, dp_global, dp_local, dst);++ clReleaseMemObject(sub_buf_src1_pre);+ clReleaseMemObject(buf_src2);+ clReleaseMemObject(buf_src2_emap);+ clReleaseMemObject(sub_buf_dst);+ clReleaseMemObject(buf_dst_image);+ return;+ }+// Set kernel argsint arg_idx = 0;CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &extra0_q4_K->q_img));@@ -20611,6 +22075,83 @@ static void ggml_cl_mul_mat_id(ggml_backend_t backend, const ggml_tensor * src0,sub_buf_src1_pre = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &status);CL_CHECK(status);+ // Generic dp4a MoE GEMM+ {+ static const char * q5kmdp4a_env = getenv("GGML_OPENCL_Q5K_MOE_DP4A");+ const bool q5kmdp4a_on = q5kmdp4a_env ? (atoi(q5kmdp4a_env) != 0)+ : (backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E);+ bool use_moe_dp4a = q5kmdp4a_on+ && backend_ctx->kernel_gemm_moe_q8_1_dp4a_q5k != nullptr+ && extra0_q5_K->scale != nullptr;+ // bin kernel takes precedence+ use_moe_dp4a = use_moe_dp4a && backend_ctx->kernel_gemm_moe_q4_k_f32_ns_bin == nullptr;++ if (use_moe_dp4a) {+ const size_t tok_slots = (size_t)max_post_router_tile * n_tile_size;+ const size_t n_blocks = tok_slots * (ne00 / 32);+ backend_ctx->prealloc_moe_qa.allocate(backend_ctx->context, tok_slots * ne00 * sizeof(cl_char));+ backend_ctx->prealloc_moe_da.allocate(backend_ctx->context, n_blocks * sizeof(cl_half));+ backend_ctx->prealloc_moe_sa.allocate(backend_ctx->context, n_blocks * sizeof(cl_half));++ const cl_uint n_kblocks = (cl_uint)(ne00 / 32);+ unsigned short map_ratio_q5k = ne20 / ne11;+ cl_kernel rq = backend_ctx->kernel_moe_reorder_quant_a_q8_1;+ CL_CHECK(clSetKernelArg(rq, 0, sizeof(cl_mem), &sub_buf_src1_pre));+ CL_CHECK(clSetKernelArg(rq, 1, sizeof(cl_mem), &buf_src2));+ CL_CHECK(clSetKernelArg(rq, 2, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer));+ CL_CHECK(clSetKernelArg(rq, 3, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer));+ CL_CHECK(clSetKernelArg(rq, 4, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer));+ CL_CHECK(clSetKernelArg(rq, 5, sizeof(cl_mem), &(backend_ctx->prealloc_total_tiles.buffer)));+ CL_CHECK(clSetKernelArg(rq, 6, sizeof(cl_uint), &ne00));+ CL_CHECK(clSetKernelArg(rq, 7, sizeof(unsigned short), &map_ratio_q5k));+ CL_CHECK(clSetKernelArg(rq, 8, sizeof(cl_uint), &n_tile_size));+ CL_CHECK(clSetKernelArg(rq, 9, sizeof(cl_uint), &n_kblocks));+ size_t rq_local[2] = { 32, 1 };+ size_t rq_global[2] = { (size_t)(((n_kblocks + 31) / 32) * 32), tok_slots };+ backend_ctx->enqueue_ndrange_kernel(rq, 2, rq_global, rq_local, dst);++ region.origin = offsetd;+ region.size = ne0 * ne1 * ne2 * sizeof(float);+ cl_mem dp_sub_buf_dst = clCreateSubBuffer(extrad->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &status);+ CL_CHECK(status);+ cl_image_format dp_ifd = {CL_R, CL_FLOAT};+ cl_image_desc dp_idd = {CL_MEM_OBJECT_IMAGE1D_BUFFER, static_cast<size_t>(ne0 * ne1 * ne2), 0,0,0,0,0,0,0, {dp_sub_buf_dst}};+ cl_mem dp_buf_dst_image = clCreateImage(backend_ctx->context, CL_MEM_WRITE_ONLY, &dp_ifd, &dp_idd, NULL, &status);+ CL_CHECK(status);++ int ne00i = (int)ne00, ne01i = (int)ne01;+ cl_kernel dk = backend_ctx->kernel_gemm_moe_q8_1_dp4a_q5k;+ int has_min_q5k = 1;+ int aidx = 0;+ CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &extra0_q5_K->q_img));+ CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &extra0_q5_K->qh));+ CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &extra0_q5_K->scale));+ CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &extra0_q5_K->min));+ CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer));+ CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer));+ CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer));+ CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &buf_src2));+ CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &buf_src2_emap));+ CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &dp_buf_dst_image));+ CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &(backend_ctx->prealloc_total_tiles.buffer)));+ CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(int), &ne00i));+ CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(int), &ne01i));+ CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(int), &backend_ctx->adreno_use_moe_ragged_dp4));+ CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(int), &has_min_q5k));++ size_t dp_global[3] = { 64, (size_t)((ne01 + 63) / 64), (size_t)max_post_router_tile };+ size_t dp_local[3] = { 64, 1, 1 };+ backend_ctx->enqueue_ndrange_kernel(dk, 3, dp_global, dp_local, dst);++ clReleaseMemObject(sub_buf_src1_pre);+ clReleaseMemObject(buf_src2);+ clReleaseMemObject(buf_src2_emap);+ clReleaseMemObject(dp_sub_buf_dst);+ clReleaseMemObject(dp_buf_dst_image);+ return;+ }+ }+// Create image for reordered src1// Use pre-allocated placeholderregion.origin = 0;@@ -20769,9 +22310,17 @@ static void ggml_cl_mul_mat_id(ggml_backend_t backend, const ggml_tensor * src0,backend_ctx->toggle_reorder = false;}- cl_mem sub_buf_src1_pre, buf_src1_reordered, image_src1_reordered, sub_buf_dst, buf_dst_image;+ cl_mem sub_buf_src1_pre, sub_buf_dst, buf_dst_image;+ cl_mem buf_src1_reordered = nullptr, image_src1_reordered = nullptr;cl_mem buf_src2, buf_src2_emap;+ // dp4a (int8) q6_K MoE prefill GEMM+ static const char * q6k_moe_dp4a_env = getenv("GGML_OPENCL_Q6K_MOE_DP4A");+ static const bool use_moe_dp4a = (q6k_moe_dp4a_env != nullptr)+ ? (atoi(q6k_moe_dp4a_env) != 0)+ : (backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E+ || backend_ctx->adreno_gen == ADRENO_GPU_GEN::X1E);+cl_buffer_region region;region.origin = 0;region.size = sizeof(int) * max_post_router_tile * n_tile_size;@@ -20790,37 +22339,40 @@ static void ggml_cl_mul_mat_id(ggml_backend_t backend, const ggml_tensor * src0,sub_buf_src1_pre = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &status);CL_CHECK(status);- // Create image for reordered src1- region.origin = 0;- region.size = ne00 * max_post_router_tile * n_tile_size * sizeof(float);- backend_ctx->prealloc_act_trans.allocate(backend_ctx->context, region.size);- buf_src1_reordered = clCreateSubBuffer(- backend_ctx->prealloc_act_trans.buffer,- 0,- CL_BUFFER_CREATE_TYPE_REGION,- ®ion,- &status);- CL_CHECK(status);- cl_image_format image_format_buf_src1 = {CL_RGBA, CL_FLOAT};- cl_image_desc image_desc_buf_src1 = {CL_MEM_OBJECT_IMAGE1D_BUFFER, static_cast<size_t>(ne00 * max_post_router_tile * n_tile_size / 4), 0,0,0,0,0,0,0, {buf_src1_reordered}};- image_src1_reordered = clCreateImage(backend_ctx->context, CL_MEM_READ_ONLY, &image_format_buf_src1, &image_desc_buf_src1, NULL, &status);- CL_CHECK(status);-unsigned short map_ratio = ne20 / ne11;GGML_ASSERT(((map_ratio == 1) || (map_ratio == ne20)) && "Map ratio not supported\n");- CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 0, sizeof(cl_mem), &sub_buf_src1_pre));- CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 1, sizeof(cl_mem), &buf_src2));- CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 2, sizeof(cl_mem), &buf_src1_reordered));- CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 3, sizeof(cl_mem), &(backend_ctx->prealloc_total_tiles.buffer)));- CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 4, sizeof(unsigned int), &ne00));- CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 5, sizeof(unsigned short), &map_ratio));- CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 6, sizeof(unsigned int), &n_tile_size));- size_t reorder_b_local_size[3] = {256, 1, 1};- size_t reorder_b_global_size[3] = {static_cast<size_t>(((ne00 / 4) + 255) / 256 * 256), static_cast<size_t>(max_post_router_tile * n_tile_size), 1};-- // Dispatch reorder kernel- backend_ctx->enqueue_ndrange_kernel(backend_ctx->kernel_moe_reorder_b, 3, reorder_b_global_size, reorder_b_local_size, dst);+ if (!use_moe_dp4a) {+ // Create image for reordered src1+ region.origin = 0;+ region.size = ne00 * max_post_router_tile * n_tile_size * sizeof(float);+ backend_ctx->prealloc_act_trans.allocate(backend_ctx->context, region.size);+ buf_src1_reordered = clCreateSubBuffer(+ backend_ctx->prealloc_act_trans.buffer,+ 0,+ CL_BUFFER_CREATE_TYPE_REGION,+ ®ion,+ &status);+ CL_CHECK(status);+ cl_image_format image_format_buf_src1 = {CL_RGBA, CL_FLOAT};+ cl_image_desc image_desc_buf_src1 = {CL_MEM_OBJECT_IMAGE1D_BUFFER, static_cast<size_t>(ne00 * max_post_router_tile * n_tile_size / 4), 0,0,0,0,0,0,0, {buf_src1_reordered}};+ image_src1_reordered = clCreateImage(backend_ctx->context, CL_MEM_READ_ONLY, &image_format_buf_src1, &image_desc_buf_src1, NULL, &status);+ CL_CHECK(status);++ CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 0, sizeof(cl_mem), &sub_buf_src1_pre));+ CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 1, sizeof(cl_mem), &buf_src2));+ CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 2, sizeof(cl_mem), &buf_src1_reordered));+ CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 3, sizeof(cl_mem), &(backend_ctx->prealloc_total_tiles.buffer)));+ CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 4, sizeof(unsigned int), &ne00));+ CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 5, sizeof(unsigned short), &map_ratio));+ CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 6, sizeof(unsigned int), &n_tile_size));++ size_t reorder_b_local_size[3] = {256, 1, 1};+ size_t reorder_b_global_size[3] = {static_cast<size_t>(((ne00 / 4) + 255) / 256 * 256), static_cast<size_t>(max_post_router_tile * n_tile_size), 1};++ // Dispatch reorder kernel+ backend_ctx->enqueue_ndrange_kernel(backend_ctx->kernel_moe_reorder_b, 3, reorder_b_global_size, reorder_b_local_size, dst);+ }// MoE kernel prepare// Create sub buffer for dst@@ -20839,6 +22391,58 @@ static void ggml_cl_mul_mat_id(ggml_backend_t backend, const ggml_tensor * src0,buf_dst_image = clCreateImage(backend_ctx->context, CL_MEM_WRITE_ONLY, &image_format_buf_dst, &image_desc_buf_dst, NULL, &status);CL_CHECK(status);+ if (use_moe_dp4a) {+ const size_t tok_slots = (size_t)max_post_router_tile * n_tile_size;+ const size_t n_blocks = tok_slots * (ne00 / 32);+ backend_ctx->prealloc_moe_qa.allocate(backend_ctx->context, tok_slots * ne00 * sizeof(cl_char));+ backend_ctx->prealloc_moe_da.allocate(backend_ctx->context, n_blocks * sizeof(cl_half));+ backend_ctx->prealloc_moe_sa.allocate(backend_ctx->context, n_blocks * sizeof(cl_half));++ // fused reorder + q8_1 quant from the original activations+ const cl_uint n_kblocks = (cl_uint)(ne00 / 32);+ cl_kernel rq = backend_ctx->kernel_moe_reorder_quant_a_q8_1;+ CL_CHECK(clSetKernelArg(rq, 0, sizeof(cl_mem), &sub_buf_src1_pre));+ CL_CHECK(clSetKernelArg(rq, 1, sizeof(cl_mem), &buf_src2));+ CL_CHECK(clSetKernelArg(rq, 2, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer));+ CL_CHECK(clSetKernelArg(rq, 3, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer));+ CL_CHECK(clSetKernelArg(rq, 4, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer));+ CL_CHECK(clSetKernelArg(rq, 5, sizeof(cl_mem), &(backend_ctx->prealloc_total_tiles.buffer)));+ CL_CHECK(clSetKernelArg(rq, 6, sizeof(cl_uint), &ne00));+ CL_CHECK(clSetKernelArg(rq, 7, sizeof(unsigned short), &map_ratio));+ CL_CHECK(clSetKernelArg(rq, 8, sizeof(cl_uint), &n_tile_size));+ CL_CHECK(clSetKernelArg(rq, 9, sizeof(cl_uint), &n_kblocks));+ size_t rq_local[2] = { 32, 1 };+ size_t rq_global[2] = { (size_t)(((n_kblocks + 31) / 32) * 32), tok_slots };+ backend_ctx->enqueue_ndrange_kernel(rq, 2, rq_global, rq_local, dst);++ cl_kernel dk = backend_ctx->kernel_gemm_moe_q6_k_q8_1_dp4a;+ int qi = 0;+ CL_CHECK(clSetKernelArg(dk, qi++, sizeof(cl_mem), &extra0_q6_K->ql_img));+ CL_CHECK(clSetKernelArg(dk, qi++, sizeof(cl_mem), &extra0_q6_K->qh));+ CL_CHECK(clSetKernelArg(dk, qi++, sizeof(cl_mem), &extra0_q6_K->s));+ CL_CHECK(clSetKernelArg(dk, qi++, sizeof(cl_mem), &extra0_q6_K->d));+ CL_CHECK(clSetKernelArg(dk, qi++, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer));+ CL_CHECK(clSetKernelArg(dk, qi++, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer));+ CL_CHECK(clSetKernelArg(dk, qi++, sizeof(cl_mem), &buf_src2));+ CL_CHECK(clSetKernelArg(dk, qi++, sizeof(cl_mem), &buf_src2_emap));+ CL_CHECK(clSetKernelArg(dk, qi++, sizeof(cl_mem), &buf_dst_image));+ CL_CHECK(clSetKernelArg(dk, qi++, sizeof(cl_mem), &(backend_ctx->prealloc_total_tiles.buffer)));+ CL_CHECK(clSetKernelArg(dk, qi++, sizeof(int), &ne00));+ CL_CHECK(clSetKernelArg(dk, qi++, sizeof(int), &ne01));+ CL_CHECK(clSetKernelArg(dk, qi++, sizeof(int), &backend_ctx->adreno_use_moe_ragged_dp4));++ size_t dp_global[3] = { 64, (size_t)((ne01 + 63) / 64), (size_t)max_post_router_tile };+ size_t dp_local[3] = { 64, 1, 1 };+ backend_ctx->enqueue_ndrange_kernel(dk, 3, dp_global, dp_local, dst);++ clReleaseMemObject(sub_buf_src1_pre);+ clReleaseMemObject(buf_src2);+ clReleaseMemObject(buf_src2_emap);+ clReleaseMemObject(sub_buf_dst);+ clReleaseMemObject(buf_dst_image);+ return;+ }+// Set kernel argsint arg_idx = 0;CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &extra0_q6_K->ql_img));@@ -20887,6 +22491,15 @@ static void ggml_cl_mul_mat_id(ggml_backend_t backend, const ggml_tensor * src0,if (ne12 == 1) { // for gemvkernel = backend_ctx->kernel_gemv_moe_mxfp4_f32_ns;+ // Weight-as-texture MoE decode GEMV (see q4_K _wimg)+ static const char * moe_decode_wimg_env = getenv("GGML_OPENCL_MOE_DECODE_WIMG");+ const bool use_moe_decode_wimg = (moe_decode_wimg_env && (atoi(moe_decode_wimg_env) != 0))+ && backend_ctx->kernel_gemv_moe_mxfp4_f32_ns_wimg != nullptr+ && extra0_mxfp4->q_img != nullptr;+ if (use_moe_decode_wimg) {+ kernel = backend_ctx->kernel_gemv_moe_mxfp4_f32_ns_wimg;+ }+cl_mem src1_sub_buffer, buf_src1_image, buf_src2;// create a sub_buffer for src2@@ -20916,7 +22529,7 @@ static void ggml_cl_mul_mat_id(ggml_backend_t backend, const ggml_tensor * src0,// Set kernel argsint arg_idx = 0;- CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &extra0_mxfp4->q));+ CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), use_moe_decode_wimg ? &extra0_mxfp4->q_img : &extra0_mxfp4->q));CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &extra0_mxfp4->e));CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &buf_src1_image));CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &buf_src2));@@ -20947,9 +22560,18 @@ static void ggml_cl_mul_mat_id(ggml_backend_t backend, const ggml_tensor * src0,backend_ctx->toggle_reorder = false;}- cl_mem sub_buf_src1_pre, buf_src1_reordered, image_src1_reordered, sub_buf_dst, buf_dst_image;+ cl_mem sub_buf_src1_pre, sub_buf_dst, buf_dst_image;+ cl_mem buf_src1_reordered = nullptr, image_src1_reordered = nullptr;cl_mem buf_src2, buf_src2_emap;+ // dp4a (int8) prefill GEMM variant+ static const char * mxfp4_moe_dp4a_env = getenv("GGML_OPENCL_MXFP4_MOE_DP4A");+ bool use_moe_dp4a = mxfp4_moe_dp4a_env+ ? (atoi(mxfp4_moe_dp4a_env) != 0)+ : (backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E);+ // bin kernel takes precedence+ use_moe_dp4a = use_moe_dp4a && backend_ctx->kernel_gemm_moe_mxfp4_f32_ns_bin == nullptr;+cl_buffer_region region;region.origin = 0;region.size = sizeof(int) * max_post_router_tile * n_tile_size;@@ -20969,45 +22591,48 @@ static void ggml_cl_mul_mat_id(ggml_backend_t backend, const ggml_tensor * src0,sub_buf_src1_pre = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &status);CL_CHECK(status);- // Create image for reordered src1- // Use pre-allocated placeholder- region.origin = 0;- region.size = ne00 * max_post_router_tile * n_tile_size * sizeof(float);- backend_ctx->prealloc_act_trans.allocate(backend_ctx->context, region.size);- buf_src1_reordered = clCreateSubBuffer(- backend_ctx->prealloc_act_trans.buffer,- 0,- CL_BUFFER_CREATE_TYPE_REGION,- ®ion,- &status);- CL_CHECK(status);- cl_image_format image_format_buf_src1;- cl_image_desc image_desc_buf_src1;- image_format_buf_src1 = {CL_RGBA, CL_FLOAT};- image_desc_buf_src1 = {CL_MEM_OBJECT_IMAGE1D_BUFFER, static_cast<size_t>(ne00 * max_post_router_tile * n_tile_size / 4), 0,0,0,0,0,0,0, {buf_src1_reordered}};- if (backend_ctx->kernel_gemm_moe_mxfp4_f32_ns_bin) {- // bin kernel uses slightly different image format- image_format_buf_src1 = {CL_R, CL_FLOAT};- image_desc_buf_src1.image_width = static_cast<size_t>(ne00 * max_post_router_tile * n_tile_size);- }- image_src1_reordered = clCreateImage(backend_ctx->context, CL_MEM_READ_ONLY, &image_format_buf_src1, &image_desc_buf_src1, NULL, &status);- CL_CHECK(status);-unsigned short map_ratio = ne20 / ne11;GGML_ASSERT(((map_ratio == 1) || (map_ratio == ne20)) && "Map ratio not supported\n");- CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 0, sizeof(cl_mem), &sub_buf_src1_pre));- CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 1, sizeof(cl_mem), &buf_src2));- CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 2, sizeof(cl_mem), &buf_src1_reordered));- CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 3, sizeof(cl_mem), &(backend_ctx->prealloc_total_tiles.buffer)));- CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 4, sizeof(unsigned int), &ne00));- CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 5, sizeof(unsigned short), &map_ratio));- CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 6, sizeof(unsigned int), &n_tile_size));-- size_t reorder_b_local_size[3] = {256, 1, 1};- size_t reorder_b_global_size[3] = {static_cast<size_t>(((ne00 / 4) + 255) / 256 * 256), static_cast<size_t>(max_post_router_tile * n_tile_size), 1};- // Dispatch reorder kernel- backend_ctx->enqueue_ndrange_kernel(backend_ctx->kernel_moe_reorder_b, 3, reorder_b_global_size, reorder_b_local_size, dst);+ if (!use_moe_dp4a) {+ // Create image for reordered src1+ // Use pre-allocated placeholder+ region.origin = 0;+ region.size = ne00 * max_post_router_tile * n_tile_size * sizeof(float);+ backend_ctx->prealloc_act_trans.allocate(backend_ctx->context, region.size);+ buf_src1_reordered = clCreateSubBuffer(+ backend_ctx->prealloc_act_trans.buffer,+ 0,+ CL_BUFFER_CREATE_TYPE_REGION,+ ®ion,+ &status);+ CL_CHECK(status);+ cl_image_format image_format_buf_src1;+ cl_image_desc image_desc_buf_src1;+ image_format_buf_src1 = {CL_RGBA, CL_FLOAT};+ image_desc_buf_src1 = {CL_MEM_OBJECT_IMAGE1D_BUFFER, static_cast<size_t>(ne00 * max_post_router_tile * n_tile_size / 4), 0,0,0,0,0,0,0, {buf_src1_reordered}};+ if (backend_ctx->kernel_gemm_moe_mxfp4_f32_ns_bin) {+ // bin kernel uses slightly different image format+ image_format_buf_src1 = {CL_R, CL_FLOAT};+ image_desc_buf_src1.image_width = static_cast<size_t>(ne00 * max_post_router_tile * n_tile_size);+ }+ image_src1_reordered = clCreateImage(backend_ctx->context, CL_MEM_READ_ONLY, &image_format_buf_src1, &image_desc_buf_src1, NULL, &status);+ CL_CHECK(status);++ CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 0, sizeof(cl_mem), &sub_buf_src1_pre));+ CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 1, sizeof(cl_mem), &buf_src2));+ CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 2, sizeof(cl_mem), &buf_src1_reordered));+ CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 3, sizeof(cl_mem), &(backend_ctx->prealloc_total_tiles.buffer)));+ CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 4, sizeof(unsigned int), &ne00));+ CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 5, sizeof(unsigned short), &map_ratio));+ CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 6, sizeof(unsigned int), &n_tile_size));++ size_t reorder_b_local_size[3] = {256, 1, 1};+ size_t reorder_b_global_size[3] = {static_cast<size_t>(((ne00 / 4) + 255) / 256 * 256), static_cast<size_t>(max_post_router_tile * n_tile_size), 1};++ // Dispatch reorder kernel+ backend_ctx->enqueue_ndrange_kernel(backend_ctx->kernel_moe_reorder_b, 3, reorder_b_global_size, reorder_b_local_size, dst);+ }// MoE kernel prepare// Create sub buffer for dst@@ -21026,6 +22651,59 @@ static void ggml_cl_mul_mat_id(ggml_backend_t backend, const ggml_tensor * src0,buf_dst_image = clCreateImage(backend_ctx->context, CL_MEM_WRITE_ONLY, &image_format_buf_dst, &image_desc_buf_dst, NULL, &status);CL_CHECK(status);+ if (use_moe_dp4a) {+ const size_t tok_slots = (size_t)max_post_router_tile * n_tile_size;+ const size_t n_blocks = tok_slots * (ne00 / 32);+ backend_ctx->prealloc_moe_qa.allocate(backend_ctx->context, tok_slots * ne00 * sizeof(cl_char));+ backend_ctx->prealloc_moe_da.allocate(backend_ctx->context, n_blocks * sizeof(cl_half));+ backend_ctx->prealloc_moe_sa.allocate(backend_ctx->context, n_blocks * sizeof(cl_half));++ // fused reorder + q8_1 quant straight from the original+ // activations (no intermediate f32 reorder buffer). mxfp4 has no+ // min term so the GEMM ignores sa, but reorder_quant still writes it.+ const cl_uint n_kblocks = (cl_uint)(ne00 / 32);+ cl_kernel rq = backend_ctx->kernel_moe_reorder_quant_a_q8_1;+ CL_CHECK(clSetKernelArg(rq, 0, sizeof(cl_mem), &sub_buf_src1_pre));+ CL_CHECK(clSetKernelArg(rq, 1, sizeof(cl_mem), &buf_src2));+ CL_CHECK(clSetKernelArg(rq, 2, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer));+ CL_CHECK(clSetKernelArg(rq, 3, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer));+ CL_CHECK(clSetKernelArg(rq, 4, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer));+ CL_CHECK(clSetKernelArg(rq, 5, sizeof(cl_mem), &(backend_ctx->prealloc_total_tiles.buffer)));+ CL_CHECK(clSetKernelArg(rq, 6, sizeof(cl_uint), &ne00));+ CL_CHECK(clSetKernelArg(rq, 7, sizeof(unsigned short), &map_ratio));+ CL_CHECK(clSetKernelArg(rq, 8, sizeof(cl_uint), &n_tile_size));+ CL_CHECK(clSetKernelArg(rq, 9, sizeof(cl_uint), &n_kblocks));+ size_t rq_local[2] = { 32, 1 };+ size_t rq_global[2] = { (size_t)(((n_kblocks + 31) / 32) * 32), tok_slots };+ backend_ctx->enqueue_ndrange_kernel(rq, 2, rq_global, rq_local, dst);++ // dp4a GEMM+ cl_kernel dk = backend_ctx->kernel_gemm_moe_mxfp4_q8_1_dp4a;+ int aidx = 0;+ CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &extra0_mxfp4->q_img));+ CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &extra0_mxfp4->e));+ CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer));+ CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer));+ CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &buf_src2));+ CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &buf_src2_emap));+ CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &buf_dst_image));+ CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &(backend_ctx->prealloc_total_tiles.buffer)));+ CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(int), &ne00));+ CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(int), &ne01));+ CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(int), &backend_ctx->adreno_use_moe_ragged_dp4));++ size_t dp_global[3] = { 64, (size_t)((ne01 + 63) / 64), (size_t)max_post_router_tile };+ size_t dp_local[3] = { 64, 1, 1 };+ backend_ctx->enqueue_ndrange_kernel(dk, 3, dp_global, dp_local, dst);++ clReleaseMemObject(sub_buf_src1_pre);+ clReleaseMemObject(buf_src2);+ clReleaseMemObject(buf_src2_emap);+ clReleaseMemObject(sub_buf_dst);+ clReleaseMemObject(buf_dst_image);+ return;+ }+// Set kernel argsint arg_idx = 0;CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &extra0_mxfp4->q_img));diff --git a/ggml/src/ggml-opencl/kernels/cvt.cl b/ggml/src/ggml-opencl/kernels/cvt.clindex bf0335a698..3d6cff7cff 100644--- a/ggml/src/ggml-opencl/kernels/cvt.cl+++ b/ggml/src/ggml-opencl/kernels/cvt.cl@@ -2372,3 +2372,121 @@ kernel void kernel_restore_block_iq4_nl_noshuffle(b->qs[2*i + 1] = convert_uchar(((x0 & mask_F0) >> 4) | (x1 & mask_F0));}}++// ---------------------------------------------------------------------------+// kernel_moe_expand_scale_q8_0+//+// Expand the q8_0 per-32-block scale d (one half/block, [expert][row][block]) into+// the UNIFORM scale[16] format the generic dp4a MoE GEMM (kernel_gemm_moe_q8_1_dp4a,+// MOE_QT=80) consumes: 16 f16 per 256-superblock (per-16-element segment), where the+// two segments of each 32-block share the block's d. q8_0 is symmetric -> no min+// buffer (the GEMM runs with has_min=0). The int8 weight codes are reused verbatim+// from the existing flat q8_0 weight buffer (extra0_q8_0->q), so only the scale is+// rebuilt here. One work-item per (row, superblock, expert).+// ---------------------------------------------------------------------------+kernel void kernel_moe_expand_scale_q8_0(+ global const half * src_d, // [expert][row][block], one scale per 32-block+ global half * dst_scale, // [expert][row][block][2] (FLAT per-32-block)+ int ne00,+ int ne01+) {+ int row = get_global_id(0);+ int blk = get_global_id(1); // 32-block index along K+ int e = get_global_id(2);+ if (row >= ne01) { return; }++ long nb = ne00 / 32; // 32-blocks per row (K only needs % 32 == 0)+ half d = src_d[((long)e*ne01 + row)*nb + blk];+ long b = (((long)e*ne01 + row)*nb + blk) * 2;+ dst_scale[b + 0] = d;+ dst_scale[b + 1] = d;+}++// ---------------------------------------------------------------------------+// kernel_moe_expand_scale_q5_0+//+// q5_0 = symmetric, value = d*(code-16), code = nibble | (hi<<4) in 0..31. The+// generic dp4a MoE GEMM keeps the unsigned code and centers via the min term:+// scale*dp4a(code,a) - min*sum(a), scale = d, min = d*16.+// Reads the existing q5_0 d ([expert][block][row], one half/32-block, from the+// trans4 convert) and writes the FLAT per-32-block uniform scale[2]/min[1] in+// [expert][row][block] order (a transpose). One work-item per (row, block, expert).+// ---------------------------------------------------------------------------+kernel void kernel_moe_expand_scale_q5_0(+ global const half * src_d, // [expert][block][row]+ global half * dst_scale, // [expert][row][block][2]+ global half * dst_min, // [expert][row][block]+ int ne00,+ int ne01+) {+ int row = get_global_id(0);+ int blk = get_global_id(1);+ int e = get_global_id(2);+ if (row >= ne01) { return; }++ long nb = ne00 / 32;+ half d = src_d[(long)e*nb*ne01 + (long)blk*ne01 + row]; // [expert][block][row]+ long sb = (((long)e*ne01 + row)*nb + blk) * 2;+ long mb = ((long)e*ne01 + row)*nb + blk;+ dst_scale[sb + 0] = d;+ dst_scale[sb + 1] = d;+ dst_min[mb] = (half)((float)d * 16.0f);+}++// ---------------------------------------------------------------------------+// kernel_moe_expand_scale_q5_K+//+// q5_K value = d*sv*code + (-dm*mn), with the 6-bit packed per-sub-block scale sv+// and min mn (8 sub-blocks of 32 per 256-superblock, decoded by get_scale_min_k4+// from the 12-byte s[]). The generic dp4a MoE GEMM (kernel_gemm_moe_q8_1_dp4a,+// MOE_QT=5) keeps the unsigned 5-bit code and applies scale/min via the uniform+// per-32-block buffers:+// acc += sc0*a_d*raw1 + sc1*a_d*raw2 - mn_u*a_s,+// sc0 = sc1 = d*sv (both per-16 segments of a 32-block share the sub-block scale),+// mn_u = dm*mn (positive; the GEMM subtracts it -> the -dm*mn min term).+// q5_K's q_img (low nibbles) + qh (hi-bit plane) are already in the layout the GEMM+// reads (same trans4_ns convert that feeds gemm_moe_q5_k_f32_ns), so only the scale+// is rebuilt here.+//+// One work-item per (row, superblock, expert); each emits 8 sub-blocks.+// ---------------------------------------------------------------------------+kernel void kernel_moe_expand_scale_q5_K(+ global const uchar * src_s, // [expert][row][superblock][12]+ global const half * src_d, // [expert][superblock][row]+ global const half * src_dm, // [expert][superblock][row]+ global half * dst_scale, // [expert][row][32block][2]+ global half * dst_min, // [expert][row][32block]+ int ne00,+ int ne01+) {+ int row = get_global_id(0);+ int sb = get_global_id(1); // superblock index along K+ int e = get_global_id(2);+ if (row >= ne01) { return; }++ long nsb = ne00 / 256; // superblocks per row+ long nblk32 = ne00 / 32; // 32-blocks per row++ float d = (float)src_d [((long)e*nsb + sb)*ne01 + row];+ float dm = (float)src_dm[((long)e*nsb + sb)*ne01 + row];++ __global const uchar * sc = src_s + ((long)e*ne01 + row)*nsb*12 + (long)sb*12;++ for (int j = 0; j < 8; ++j) {+ uchar sv, mn;+ // get_scale_min_k4 (6-bit packed scale/min for sub-block j of 8)+ if (j < 4) {+ sv = sc[j] & 63;+ mn = sc[j+4] & 63;+ } else {+ sv = (sc[j+4] & 0x0F) | ((sc[j-4] & 0xC0) >> 2);+ mn = ((sc[j+4] >> 4) & 0x0F) | ((sc[j] & 0xC0) >> 2);+ }+ long sub = (long)sb*8 + j;+ long sbase = (((long)e*ne01 + row)*nblk32 + sub) * 2;+ half s_val = (half)(d * (float)sv);+ dst_scale[sbase + 0] = s_val;+ dst_scale[sbase + 1] = s_val;+ dst_min[((long)e*ne01 + row)*nblk32 + sub] = (half)(dm * (float)mn);+ }+}diff --git a/ggml/src/ggml-opencl/kernels/gemm_moe_mxfp4_q8_1_dp4a.cl b/ggml/src/ggml-opencl/kernels/gemm_moe_mxfp4_q8_1_dp4a.clnew file mode 100644index 0000000000..95d0638134--- /dev/null+++ b/ggml/src/ggml-opencl/kernels/gemm_moe_mxfp4_q8_1_dp4a.cl@@ -0,0 +1,186 @@+#pragma OPENCL EXTENSION cl_khr_fp16 : enable+#pragma OPENCL EXTENSION cl_khr_subgroups : enable+#ifdef cl_khr_integer_dot_product+#pragma OPENCL EXTENSION cl_khr_integer_dot_product : enable+#endif++#define TILESIZE_M 64+#define TILESIZE_N 32++// 2*mxfp4_value as signed int8, packed 4 codes per uint. Divergent nibble+// lookups read a __constant *uint* array + shift, never a byte array+// (byte-indexed __constant loads serialize on Adreno and are far slower).+// idx 0-3: 0, 1, 2, 3 = 0x03020100+// idx 4-7: 4, 6, 8, 12 = 0x0C080604+// idx 8-11: 0, -1, -2, -3 = 0xFDFEFF00 (-1=0xFF,-2=0xFE,-3=0xFD)+// idx 12-15:-4, -6, -8,-12 = 0xF4F8FAFC (-4=0xFC,-6=0xFA,-8=0xF8,-12=0xF4)+__constant uint mxfp4_i8x4[4] = {+ 0x03020100u, 0x0C080604u, 0xFDFEFF00u, 0xF4F8FAFCu+};+inline uint mxfp4_code(uint n) {+ return (mxfp4_i8x4[n >> 2] >> ((n & 3u) * 8u)) & 0xFFu;+}+// 4 nibbles in the low 16 bits of u -> 4 codebook int8, packed for dp4a.+inline uint mxfp4_pack(ushort u) {+ return mxfp4_code((uint)( u & 0xF))+ | (mxfp4_code((uint)((u >> 4) & 0xF)) << 8)+ | (mxfp4_code((uint)((u >> 8) & 0xF)) << 16)+ | (mxfp4_code((uint)((u >> 12) & 0xF)) << 24);+}++static inline float e8m0_to_fp32(uchar x) {+ int bits;+ bits = (x == 0) ? 0x00400000 : ((uint) x << 23);+ return as_float(bits);+}++// One token's dp4a dot (8 uints = 32 K elems) + mxfp4 block-scale epilogue.+// blk_scale already carries the 0.5 factor (== 0.5 * 2^e).+#define MOE_MXFP4_DP4A_T(t) do { \+ int raw = 0; \+ raw = dot_acc_sat_4x8packed_ss_int(qw[0], sh_qa[t][0], raw); \+ raw = dot_acc_sat_4x8packed_ss_int(qw[1], sh_qa[t][1], raw); \+ raw = dot_acc_sat_4x8packed_ss_int(qw[2], sh_qa[t][2], raw); \+ raw = dot_acc_sat_4x8packed_ss_int(qw[3], sh_qa[t][3], raw); \+ raw = dot_acc_sat_4x8packed_ss_int(qw[4], sh_qa[t][4], raw); \+ raw = dot_acc_sat_4x8packed_ss_int(qw[5], sh_qa[t][5], raw); \+ raw = dot_acc_sat_4x8packed_ss_int(qw[6], sh_qa[t][6], raw); \+ raw = dot_acc_sat_4x8packed_ss_int(qw[7], sh_qa[t][7], raw); \+ acc[t] += blk_scale * (float)sh_d[t] * (float)raw; \+ } while (0)++__attribute__((qcom_wave_pair_mode(1)))+kernel void kernel_gemm_moe_mxfp4_q8_1_dp4a(+ __read_only image1d_buffer_t src0_q, // mxfp4 codes (transposed, packed nibbles)+ __global uchar * src0_e, // e8m0 per-32-block scale+ __global uint * src1_qa, // q8_1 activations: int8 quants (as uint, 4/elem)+ __global half * src1_da, // q8_1 per-block scale [tok_slot * ne00/32]+ __global uint * src2, // post-router (orig out positions)+ __global ushort * src2_emap, // tile -> expert id+ __write_only image1d_buffer_t dst,+ __global int * total_tiles,+ uint ne00,+ uint ne01,+ int is_ragged // 1: compute only real tokens per tile+) {+ const uint block_id_m = get_global_id(1); // m_tile+ const uint block_id_n = get_global_id(2); // n_tile++ if (block_id_n >= total_tiles[0]) {+ return;+ }++ const uint lid = get_local_id(0); // 0..63, == this WI's output row in the M-tile++ const ushort expert_id = src2_emap[block_id_n];+ const uint row = block_id_m * TILESIZE_M;+ const uint col = block_id_n * TILESIZE_N;++ const uint num_blocks = ne00 >> 5; // blocks-of-32 per token+ const uint row_idx = row + lid;++ const uint ne00_u = ne00 >> 2; // ne00 in uint (int8x4) units++ __local uint sh_qa[TILESIZE_N][8]; // 32 tokens x 8 uints (32 int8) = 1 KiB+ __local half sh_d[TILESIZE_N];++ // Real token count for this tile.+ // Real tokens are packed contiguously at the tile start; padded slots hold+ // 0xFFFFFFFF (only the last tile of each expert is partial). is_ragged skips+ // the dp4a/staging/scatter for padded slots; is_ragged==0 forces n_real=32.+ __local uint sh_src2[TILESIZE_N];+ __local int sh_nreal;+ if (lid < TILESIZE_N) {+ sh_src2[lid] = src2[col + lid];+ }+ barrier(CLK_LOCAL_MEM_FENCE);+ if (lid == 0) {+ int nr = TILESIZE_N;+ if (is_ragged) {+ nr = 0;+ #pragma unroll+ for (int t = 0; t < TILESIZE_N; ++t) {+ if (sh_src2[t] != 0xFFFFFFFFu) ++nr;+ }+ }+ sh_nreal = nr;+ }+ barrier(CLK_LOCAL_MEM_FENCE);+ const int n_real = sh_nreal;++ float acc[TILESIZE_N];+ #pragma unroll+ for (int t = 0; t < TILESIZE_N; ++t) acc[t] = 0.0f;++ for (uint step = 0; step < ne00; step += 32) {+ const uint sub = step >> 5; // 32-block index along K++ // e8m0 block scale for this WI's row, this 32-block (folded x0.5)+ const uint e_offset = row_idx + sub * ne01 + expert_id * num_blocks * ne01;+ const float blk_scale = 0.5f * e8m0_to_fp32(src0_e[e_offset]);++ // repack this WI's 32 weight nibbles into 8 dp4a uints+ const uint qoff0 = row + ((ne01 * step) >> 3) + ((expert_id * ne00 * ne01) >> 3);+ const uint qoff1 = row + ((ne01 * (step + 16)) >> 3) + ((expert_id * ne00 * ne01) >> 3);+ const uint r0 = read_imageui(src0_q, qoff0 + lid).x;+ const uint r1 = read_imageui(src0_q, qoff0 + lid + ne01).x;+ const uint r2 = read_imageui(src0_q, qoff1 + lid).x;+ const uint r3 = read_imageui(src0_q, qoff1 + lid + ne01).x;+ uint qw[8];+ qw[0] = mxfp4_pack((ushort)(r0)); qw[1] = mxfp4_pack((ushort)(r0 >> 16));+ qw[2] = mxfp4_pack((ushort)(r1)); qw[3] = mxfp4_pack((ushort)(r1 >> 16));+ qw[4] = mxfp4_pack((ushort)(r2)); qw[5] = mxfp4_pack((ushort)(r2 >> 16));+ qw[6] = mxfp4_pack((ushort)(r3)); qw[7] = mxfp4_pack((ushort)(r3 >> 16));++ // cooperatively stage the n_real-token x 32-K int8 activations+ const uint stage_lim = (uint)n_real * 8;+ for (uint idx = lid; idx < stage_lim; idx += 64) {+ const uint t = idx >> 3;+ const uint u = idx & 7;+ sh_qa[t][u] = src1_qa[(col + t) * ne00_u + (step >> 2) + u];+ }+ if (lid < (uint)n_real) {+ sh_d[lid] = src1_da[(col + lid) * num_blocks + sub];+ }+ barrier(CLK_LOCAL_MEM_FENCE);++ // Full tiles keep the fully-unrolled 32-wide loop; partial tiles run only n_real+ if (n_real == TILESIZE_N) {+ #pragma unroll+ for (int t = 0; t < TILESIZE_N; ++t) { MOE_MXFP4_DP4A_T(t); }+ } else {+ #pragma unroll 4+ for (int t = 0; t < n_real; ++t) { MOE_MXFP4_DP4A_T(t); }+ }+ barrier(CLK_LOCAL_MEM_FENCE);+ }++ if (row_idx >= ne01) {+ return;+ }++ // scatter results to original output rows (reuse sh_src2 from the top)+ __local uint out_idx[TILESIZE_N];+ if (lid < TILESIZE_N) {+ uint idx = sh_src2[lid];+ if (idx == 0xFFFFFFFF) {+ idx = sh_src2[0];+ }+ out_idx[lid] = idx * ne01;+ }+ barrier(CLK_LOCAL_MEM_FENCE);++ const uint m_offset = row + lid;+ if (n_real == TILESIZE_N) {+ #pragma unroll+ for (int t = 1; t < TILESIZE_N; ++t) {+ write_imagef(dst, out_idx[t] + m_offset, acc[t]);+ }+ barrier(CLK_GLOBAL_MEM_FENCE);+ write_imagef(dst, out_idx[0] + m_offset, acc[0]);+ } else {+ for (int t = 0; t < n_real; ++t) {+ write_imagef(dst, out_idx[t] + m_offset, acc[t]);+ }+ }+}diff --git a/ggml/src/ggml-opencl/kernels/gemm_moe_q4_0_q8_1_dp4a.cl b/ggml/src/ggml-opencl/kernels/gemm_moe_q4_0_q8_1_dp4a.clnew file mode 100644index 0000000000..86ff943c51--- /dev/null+++ b/ggml/src/ggml-opencl/kernels/gemm_moe_q4_0_q8_1_dp4a.cl@@ -0,0 +1,165 @@+#pragma OPENCL EXTENSION cl_khr_fp16 : enable+#pragma OPENCL EXTENSION cl_khr_subgroups : enable+#ifdef cl_khr_integer_dot_product+#pragma OPENCL EXTENSION cl_khr_integer_dot_product : enable+#endif++#define TILESIZE_M 64+#define TILESIZE_N 32++// Expand the 4 nibbles held in the low 16 bits of `u` into 4 bytes (one nibble+// per byte, value 0..15), packed for the int8 dp4a. The -8 zero-point is applied+// in the epilogue via the activation sum term (cheaper than biasing every byte).+#define EXP4(u) ( ((uint)((u) & 0x000Fu)) | \+ (((uint)((u) & 0x00F0u)) << 4) | \+ (((uint)((u) & 0x0F00u)) << 8) | \+ (((uint)((u) & 0xF000u)) << 12) )++// One token's dp4a dot (8 uints = 32 K elems) + q4_0 scale/zero-point epilogue.+#define MOE_Q40_DP4A_T(t) do { \+ int raw = 0; \+ raw = dot_acc_sat_4x8packed_ss_int(qw[0], sh_qa[t][0], raw); \+ raw = dot_acc_sat_4x8packed_ss_int(qw[1], sh_qa[t][1], raw); \+ raw = dot_acc_sat_4x8packed_ss_int(qw[2], sh_qa[t][2], raw); \+ raw = dot_acc_sat_4x8packed_ss_int(qw[3], sh_qa[t][3], raw); \+ raw = dot_acc_sat_4x8packed_ss_int(qw[4], sh_qa[t][4], raw); \+ raw = dot_acc_sat_4x8packed_ss_int(qw[5], sh_qa[t][5], raw); \+ raw = dot_acc_sat_4x8packed_ss_int(qw[6], sh_qa[t][6], raw); \+ raw = dot_acc_sat_4x8packed_ss_int(qw[7], sh_qa[t][7], raw); \+ acc[t] += d_val * ((float)sh_d[t] * (float)raw - 8.0f * (float)sh_s[t]); \+ } while (0)++__attribute__((qcom_wave_pair_mode(1)))+kernel void kernel_gemm_moe_q4_0_q8_1_dp4a(+ __read_only image1d_buffer_t src0_q, // q4_0 weights (transposed, packed nibbles)+ __global half * src0_d, // per-32-block scale+ __global uint * src1_qa, // q8_1 activations: int8 quants (as uint, 4/elem)+ __global half * src1_da, // q8_1 per-block scale [tok_slot * ne00/32]+ __global half * src1_sa, // q8_1 per-block sum*d [tok_slot * ne00/32]+ __global uint * src2, // post-router (orig out positions)+ __global ushort * src2_emap,// tile -> expert id+ __write_only image1d_buffer_t dst,+ __global int * total_tiles,+ uint ne00,+ uint ne01,+ int is_ragged // 1: compute only real tokens per tile+) {+ const uint block_id_m = get_global_id(1); // m_tile+ const uint block_id_n = get_global_id(2); // n_tile++ if (block_id_n >= total_tiles[0]) {+ return;+ }++ const uint lid = get_local_id(0); // 0..63, == this WI's output row in the M-tile++ const ushort expert_id = src2_emap[block_id_n];+ const uint row = block_id_m * TILESIZE_M;+ const uint col = block_id_n * TILESIZE_N;++ const uint num_blocks = ne00 >> 5; // blocks-of-32 per token+ const uint row_idx = row + lid;++ const uint ne00_u = ne00 >> 2; // ne00 in uint (int8x4) units++ __local uint sh_qa[TILESIZE_N][8]; // 32 tokens x 8 uints (32 int8) = 1 KiB+ __local half sh_d[TILESIZE_N];+ __local half sh_s[TILESIZE_N];++ // Real-token count for this tile+ __local uint sh_src2[TILESIZE_N];+ __local int sh_nreal;+ if (lid < TILESIZE_N) {+ sh_src2[lid] = src2[col + lid];+ }+ barrier(CLK_LOCAL_MEM_FENCE);+ if (lid == 0) {+ int nr = TILESIZE_N;+ if (is_ragged) {+ nr = 0;+ #pragma unroll+ for (int t = 0; t < TILESIZE_N; ++t) {+ if (sh_src2[t] != 0xFFFFFFFFu) ++nr;+ }+ }+ sh_nreal = nr;+ }+ barrier(CLK_LOCAL_MEM_FENCE);+ const int n_real = sh_nreal;++ float acc[TILESIZE_N];+ #pragma unroll+ for (int t = 0; t < TILESIZE_N; ++t) acc[t] = 0.0f;++ for (uint step = 0; step < ne00; step += 32) {+ const uint sub = step >> 5; // 32-block index along K++ // per-32-block scale for this WI's row+ const uint d_offset = row_idx + sub * ne01 + expert_id * num_blocks * ne01;+ const float d_val = (float)src0_d[d_offset];++ // repack this WI's 32 weight nibbles into 8 dp4a uints+ const uint qoff0 = row + ((ne01 * step) >> 3) + ((expert_id * ne00 * ne01) >> 3);+ const uint qoff1 = row + ((ne01 * (step + 16)) >> 3) + ((expert_id * ne00 * ne01) >> 3);+ const uint r0 = read_imageui(src0_q, qoff0 + lid).x;+ const uint r1 = read_imageui(src0_q, qoff0 + lid + ne01).x;+ const uint r2 = read_imageui(src0_q, qoff1 + lid).x;+ const uint r3 = read_imageui(src0_q, qoff1 + lid + ne01).x;+ uint qw[8];+ qw[0] = EXP4(r0); qw[1] = EXP4(r0 >> 16);+ qw[2] = EXP4(r1); qw[3] = EXP4(r1 >> 16);+ qw[4] = EXP4(r2); qw[5] = EXP4(r2 >> 16);+ qw[6] = EXP4(r3); qw[7] = EXP4(r3 >> 16);++ // cooperatively stage the n_real-token x 32-K int8 activations+ const uint stage_lim = (uint)n_real * 8;+ for (uint idx = lid; idx < stage_lim; idx += 64) {+ const uint t = idx >> 3;+ const uint u = idx & 7;+ sh_qa[t][u] = src1_qa[(col + t) * ne00_u + (step >> 2) + u];+ }+ if (lid < (uint)n_real) {+ sh_d[lid] = src1_da[(col + lid) * num_blocks + sub];+ sh_s[lid] = src1_sa[(col + lid) * num_blocks + sub];+ }+ barrier(CLK_LOCAL_MEM_FENCE);++ if (n_real == TILESIZE_N) {+ #pragma unroll+ for (int t = 0; t < TILESIZE_N; ++t) { MOE_Q40_DP4A_T(t); }+ } else {+ #pragma unroll 4+ for (int t = 0; t < n_real; ++t) { MOE_Q40_DP4A_T(t); }+ }+ barrier(CLK_LOCAL_MEM_FENCE);+ }++ if (row_idx >= ne01) {+ return;+ }++ // scatter results to original output rows (reuse sh_src2 from the top)+ __local uint out_idx[TILESIZE_N];+ if (lid < TILESIZE_N) {+ uint idx = sh_src2[lid];+ if (idx == 0xFFFFFFFF) {+ idx = sh_src2[0];+ }+ out_idx[lid] = idx * ne01;+ }+ barrier(CLK_LOCAL_MEM_FENCE);++ const uint m_offset = row + lid;+ if (n_real == TILESIZE_N) {+ #pragma unroll+ for (int t = 1; t < TILESIZE_N; ++t) {+ write_imagef(dst, out_idx[t] + m_offset, acc[t]);+ }+ barrier(CLK_GLOBAL_MEM_FENCE);+ write_imagef(dst, out_idx[0] + m_offset, acc[0]);+ } else {+ for (int t = 0; t < n_real; ++t) {+ write_imagef(dst, out_idx[t] + m_offset, acc[t]);+ }+ }+}diff --git a/ggml/src/ggml-opencl/kernels/gemm_moe_q4_k_q8_1_dp4a.cl b/ggml/src/ggml-opencl/kernels/gemm_moe_q4_k_q8_1_dp4a.clnew file mode 100644index 0000000000..5408975440--- /dev/null+++ b/ggml/src/ggml-opencl/kernels/gemm_moe_q4_k_q8_1_dp4a.cl@@ -0,0 +1,202 @@+#pragma OPENCL EXTENSION cl_khr_fp16 : enable+#pragma OPENCL EXTENSION cl_khr_subgroups : enable+#ifdef cl_khr_integer_dot_product+#pragma OPENCL EXTENSION cl_khr_integer_dot_product : enable+#endif++// q4_K subblock (32 elems): w_i = scale*q_i - minv, q_i in [0,15], scale =+// d_super*sv6, minv = dmin_super*mn6. With activation block (a_d, a_s, qa[32]):+// Sum_i w_i * a_i = scale * a_d * dp4a(q, qa) - minv * a_s+// where a_s = a_d * Sum(qa) (the q8_1 "s" field)++#define TILESIZE_M 64+#define TILESIZE_N 32+#define QK_K 256+#define K_SCALE_SIZE 12++inline void get_scale_min_k4(+ int j,+ global const uchar * q,+ uchar * d,+ uchar * m+) {+ if (j < 4) {+ *d = q[j] & 63;+ *m = q[j+4] & 63;+ } else {+ *d = (q[j+4] & 0x0F) | ((q[j-4] & 0xC0) >> 2);+ *m = ((q[j+4] >> 4) & 0x0F) | ((q[j] & 0xC0) >> 2);+ }+}++// Expand the 4 nibbles held in the low 16 bits of `u` into 4 bytes (one nibble+// per byte, value 0..15), packed for the int8 dp4a.+#define EXP4(u) ( ((uint)((u) & 0x000Fu)) | \+ (((uint)((u) & 0x00F0u)) << 4) | \+ (((uint)((u) & 0x0F00u)) << 8) | \+ (((uint)((u) & 0xF000u)) << 12) )++// One token's dp4a dot (8 uints = 32 K elems) + q4_K scale/min epilogue into acc[t].+#define MOE_Q4K_DP4A_T(t) do { \+ int raw = 0; \+ raw = dot_acc_sat_4x8packed_ss_int(qw[0], sh_qa[t][0], raw); \+ raw = dot_acc_sat_4x8packed_ss_int(qw[1], sh_qa[t][1], raw); \+ raw = dot_acc_sat_4x8packed_ss_int(qw[2], sh_qa[t][2], raw); \+ raw = dot_acc_sat_4x8packed_ss_int(qw[3], sh_qa[t][3], raw); \+ raw = dot_acc_sat_4x8packed_ss_int(qw[4], sh_qa[t][4], raw); \+ raw = dot_acc_sat_4x8packed_ss_int(qw[5], sh_qa[t][5], raw); \+ raw = dot_acc_sat_4x8packed_ss_int(qw[6], sh_qa[t][6], raw); \+ raw = dot_acc_sat_4x8packed_ss_int(qw[7], sh_qa[t][7], raw); \+ acc[t] += scale * (float)sh_d[t] * (float)raw - minv * (float)sh_s[t]; \+ } while (0)++__attribute__((qcom_wave_pair_mode(1)))+kernel void kernel_gemm_moe_q4_k_q8_1_dp4a(+ __read_only image1d_buffer_t src0_q, // q4_K weights (transposed, packed nibbles)+ __global half * src0_d, // per-superblock scale+ __global half * src0_dm, // per-superblock min+ __global uchar * src0_s, // 6-bit scale/min codes+ __global uint * src1_qa, // q8_1 activations: int8 quants (as uint, 4/elem)+ __global half * src1_da, // q8_1 per-block scale [tok_slot * ne00/32]+ __global half * src1_sa, // q8_1 per-block sum*d [tok_slot * ne00/32]+ __global uint * src2, // post-router (orig out positions)+ __global ushort * src2_emap,// tile -> expert id+ __write_only image1d_buffer_t dst,+ __global int * total_tiles,+ uint ne00,+ uint ne01,+ int is_ragged // 1: compute only real tokens per tile+) {+ const uint block_id_m = get_global_id(1); // m_tile+ const uint block_id_n = get_global_id(2); // n_tile++ if (block_id_n >= total_tiles[0]) {+ return;+ }++ const uint lid = get_local_id(0); // 0..63, == this WI's output row in the M-tile++ const ushort expert_id = src2_emap[block_id_n];+ const uint row = block_id_m * TILESIZE_M;+ const uint col = block_id_n * TILESIZE_N;++ const uint num_superblocks = ne00 / QK_K;+ const uint scales_per_row = num_superblocks * K_SCALE_SIZE;+ const uint row_idx = row + lid;++ const uint ne00_u = ne00 >> 2; // ne00 in uint (int8x4) units+ const uint ne00_b = ne00 >> 5; // blocks-of-32 per token++ __local uint sh_qa[TILESIZE_N][8]; // 32 tokens x 8 uints (32 int8) = 1 KiB+ __local half sh_d[TILESIZE_N];+ __local half sh_s[TILESIZE_N];++ // Real token count for this tile+ __local uint sh_src2[TILESIZE_N];+ __local int sh_nreal;+ if (lid < TILESIZE_N) {+ sh_src2[lid] = src2[col + lid];+ }+ barrier(CLK_LOCAL_MEM_FENCE);+ if (lid == 0) {+ int nr = TILESIZE_N;+ if (is_ragged) {+ nr = 0;+ #pragma unroll+ for (int t = 0; t < TILESIZE_N; ++t) {+ if (sh_src2[t] != 0xFFFFFFFFu) ++nr;+ }+ }+ sh_nreal = nr;+ }+ barrier(CLK_LOCAL_MEM_FENCE);+ const int n_real = sh_nreal;++ float acc[TILESIZE_N];+ #pragma unroll+ for (int t = 0; t < TILESIZE_N; ++t) acc[t] = 0.0f;++ for (uint step = 0; step < ne00; step += 32) {+ const uint sub = step >> 5; // subblock index along K+ const uint sb = sub >> 3; // superblock index+ const uint j = sub & 7; // subblock within superblock++ // --- weight scale / min for this WI's row, this subblock ---+ const uint d_offset = row + sb * ne01 + expert_id * num_superblocks * ne01 + lid;+ const float d_val = (float)src0_d[d_offset];+ const float dm_val = (float)src0_dm[d_offset];++ global const uchar * sc = src0_s + (expert_id * ne01 + row_idx) * scales_per_row + sb * K_SCALE_SIZE;+ uchar sv, mn;+ get_scale_min_k4(j, sc, &sv, &mn);+ const float scale = d_val * (float)sv;+ const float minv = dm_val * (float)mn;++ // --- repack this WI's 32 weight nibbles into 8 dp4a uints ---+ const uint qoff0 = row + ((ne01 * step) >> 3) + ((expert_id * ne00 * ne01) >> 3);+ const uint qoff1 = row + ((ne01 * (step + 16)) >> 3) + ((expert_id * ne00 * ne01) >> 3);+ const uint r0 = read_imageui(src0_q, qoff0 + lid).x;+ const uint r1 = read_imageui(src0_q, qoff0 + lid + ne01).x;+ const uint r2 = read_imageui(src0_q, qoff1 + lid).x;+ const uint r3 = read_imageui(src0_q, qoff1 + lid + ne01).x;+ uint qw[8];+ qw[0] = EXP4(r0); qw[1] = EXP4(r0 >> 16);+ qw[2] = EXP4(r1); qw[3] = EXP4(r1 >> 16);+ qw[4] = EXP4(r2); qw[5] = EXP4(r2 >> 16);+ qw[6] = EXP4(r3); qw[7] = EXP4(r3 >> 16);++ // --- cooperatively stage the n_real-token x 32-K int8 activations to LDS ---+ const uint stage_lim = (uint)n_real * 8;+ for (uint idx = lid; idx < stage_lim; idx += 64) {+ const uint t = idx >> 3;+ const uint u = idx & 7;+ sh_qa[t][u] = src1_qa[(col + t) * ne00_u + (step >> 2) + u];+ }+ if (lid < (uint)n_real) {+ sh_d[lid] = src1_da[(col + lid) * ne00_b + sub];+ sh_s[lid] = src1_sa[(col + lid) * ne00_b + sub];+ }+ barrier(CLK_LOCAL_MEM_FENCE);++ // dp4a - each real token sum over 8 uints (32 K), then scale/min+ // Full tiles keep the fully-unrolled 32-wide loop;+ // partial tiles run only n_real (saves the padded-slot dp4a + staging).+ if (n_real == TILESIZE_N) {+ #pragma unroll+ for (int t = 0; t < TILESIZE_N; ++t) { MOE_Q4K_DP4A_T(t); }+ } else {+ #pragma unroll 4+ for (int t = 0; t < n_real; ++t) { MOE_Q4K_DP4A_T(t); }+ }+ barrier(CLK_LOCAL_MEM_FENCE);+ }++ if (row_idx >= ne01) {+ return;+ }++ // scatter results to original output rows+ __local uint out_idx[TILESIZE_N];+ if (lid < TILESIZE_N) {+ uint idx = sh_src2[lid];+ if (idx == 0xFFFFFFFF) {+ idx = sh_src2[0];+ }+ out_idx[lid] = idx * ne01;+ }+ barrier(CLK_LOCAL_MEM_FENCE);++ const uint m_offset = row + lid;+ if (n_real == TILESIZE_N) {+ #pragma unroll+ for (int t = 1; t < TILESIZE_N; ++t) {+ write_imagef(dst, out_idx[t] + m_offset, acc[t]);+ }+ barrier(CLK_GLOBAL_MEM_FENCE);+ write_imagef(dst, out_idx[0] + m_offset, acc[0]);+ } else {+ for (int t = 0; t < n_real; ++t) {+ write_imagef(dst, out_idx[t] + m_offset, acc[t]);+ }+ }+}diff --git a/ggml/src/ggml-opencl/kernels/gemm_moe_q6_k_q8_1_dp4a.cl b/ggml/src/ggml-opencl/kernels/gemm_moe_q6_k_q8_1_dp4a.clnew file mode 100644index 0000000000..35e63dcab0--- /dev/null+++ b/ggml/src/ggml-opencl/kernels/gemm_moe_q6_k_q8_1_dp4a.cl@@ -0,0 +1,196 @@+#pragma OPENCL EXTENSION cl_khr_fp16 : enable+#pragma OPENCL EXTENSION cl_khr_subgroups : enable+#ifdef cl_khr_integer_dot_product+#pragma OPENCL EXTENSION cl_khr_integer_dot_product : enable+#endif++#define TILESIZE_N 32+#define QK_K 256++// 4 nibbles in the low 16 bits of `u` -> 4 bytes (value 0..15, in bits 0-3).+#define EXP4(u) ( ((uint)((u) & 0x000Fu)) | \+ (((uint)((u) & 0x00F0u)) << 4) | \+ (((uint)((u) & 0x0F00u)) << 8) | \+ (((uint)((u) & 0xF000u)) << 12) )++// 4 2-bit highs in byte `b` (8 bits) -> 4 bytes, value 0..3 in bits 4-5+// (pre-multiplied by 16 so it ORs with the EXP4 nibble to form q6 in 0..63).+#define EXP2(b) ( (((uint)((b) & 0x03u)) << 4) | \+ (((uint)((b) & 0x0Cu)) << 10) | \+ (((uint)((b) & 0x30u)) << 16) | \+ (((uint)((b) & 0xC0u)) << 22) )++// q6 (0..63, bits 0-5 of each byte) -> (q6-32) as a signed int8 per byte.+// Flipping bit5 subtracts 32 in 6-bit two's complement; then replicate bit5+// into bits 6-7 to sign-extend to int8. Per-byte, no inter-byte carry.+inline uint SIGN6(uint q6p) {+ uint x = q6p ^ 0x20202020u;+ uint s = x & 0x20202020u;+ return x | (s << 1) | (s << 2);+}++inline int dp4a_q6(uint qw0, uint qw1, uint qw2, uint qw3,+ uint a0, uint a1, uint a2, uint a3) {+ int raw = 0;+ raw = dot_acc_sat_4x8packed_ss_int(qw0, a0, raw);+ raw = dot_acc_sat_4x8packed_ss_int(qw1, a1, raw);+ raw = dot_acc_sat_4x8packed_ss_int(qw2, a2, raw);+ raw = dot_acc_sat_4x8packed_ss_int(qw3, a3, raw);+ return raw;+}++// One token's q6_K dp4a dot (two halves, per-16 scales) + epilogue into acc[t].+#define MOE_Q6K_DP4A_T(t) do { \+ const int raw1 = dp4a_q6(qw[0], qw[1], qw[2], qw[3], sh_qa[t][0], sh_qa[t][1], sh_qa[t][2], sh_qa[t][3]); \+ const int raw2 = dp4a_q6(qw[4], qw[5], qw[6], qw[7], sh_qa[t][4], sh_qa[t][5], sh_qa[t][6], sh_qa[t][7]); \+ const float a_d = (float)sh_d[t]; \+ acc[t] += scale0 * a_d * (float)raw1 + scale1 * a_d * (float)raw2; \+ } while (0)++__attribute__((qcom_wave_pair_mode(1)))+kernel void kernel_gemm_moe_q6_k_q8_1_dp4a(+ __read_only image1d_buffer_t src0_ql, // q6_K low nibbles (image, q4_K-style layout)+ __global uint * src0_qh, // q6_K high 2-bit (16 elems/uint)+ __global char * src0_s, // int8 scales (one per 16 elems)+ __global half * src0_d, // per-superblock scale+ __global uint * src1_qa, // q8_1 activations int8 (as uint, 4/elem)+ __global half * src1_da, // q8_1 per-block scale [tok_slot * ne00/32]+ __global uint * src2, // post-router (orig out positions)+ __global ushort * src2_emap, // tile -> expert id+ __write_only image1d_buffer_t dst,+ __global int * total_tiles,+ uint ne00,+ uint ne01,+ int is_ragged // 1: compute only real tokens per tile+) {+ const uint block_id_m = get_global_id(1);+ const uint block_id_n = get_global_id(2);++ if (block_id_n >= total_tiles[0]) {+ return;+ }++ const uint lid = get_local_id(0); // 0..63 -> row within M-tile++ const ushort expert_id = src2_emap[block_id_n];+ const uint row = block_id_m * 64;+ const uint col = block_id_n * TILESIZE_N;++ const uint num_superblocks = ne00 / QK_K;+ const uint scales_per_row = num_superblocks * 16;+ const uint row_idx = row + lid;++ const uint ne00_u = ne00 >> 2;+ const uint ne00_b = ne00 >> 5;++ __local uint sh_qa[TILESIZE_N][8];+ __local half sh_d[TILESIZE_N];++ // Real token count for this tile+ __local uint sh_src2[TILESIZE_N];+ __local int sh_nreal;+ if (lid < TILESIZE_N) {+ sh_src2[lid] = src2[col + lid];+ }+ barrier(CLK_LOCAL_MEM_FENCE);+ if (lid == 0) {+ int nr = TILESIZE_N;+ if (is_ragged) {+ nr = 0;+ #pragma unroll+ for (int t = 0; t < TILESIZE_N; ++t) {+ if (sh_src2[t] != 0xFFFFFFFFu) ++nr;+ }+ }+ sh_nreal = nr;+ }+ barrier(CLK_LOCAL_MEM_FENCE);+ const int n_real = sh_nreal;++ float acc[TILESIZE_N];+ #pragma unroll+ for (int t = 0; t < TILESIZE_N; ++t) acc[t] = 0.0f;++ for (uint step = 0; step < ne00; step += 32) {+ const uint sub = step >> 5;+ const uint sb = sub >> 3;+ const uint j = sub & 7;++ const float d_val = (float)src0_d[row + sb * ne01 + expert_id * num_superblocks * ne01 + lid];+ global const char * sc = src0_s + (expert_id * ne01 + row_idx) * scales_per_row + sb * 16;+ const float scale0 = d_val * (float)sc[j * 2];+ const float scale1 = d_val * (float)sc[j * 2 + 1];++ // high bits: one uint covers 16 elems; first/second 16 of this 32-block+ const uint qh_base = row + (sub * 2) * ne01 + expert_id * (num_superblocks * 16) * ne01 + lid;+ const uint qh1 = src0_qh[qh_base];+ const uint qh2 = src0_qh[qh_base + ne01];++ // low nibbles: same image layout as q4_K (8 ushorts over the 32 K)+ const uint qoff0 = row + ((ne01 * step) >> 3) + ((expert_id * ne00 * ne01) >> 3);+ const uint qoff1 = row + ((ne01 * (step + 16)) >> 3) + ((expert_id * ne00 * ne01) >> 3);+ const uint r0 = read_imageui(src0_ql, qoff0 + lid).x;+ const uint r1 = read_imageui(src0_ql, qoff0 + lid + ne01).x;+ const uint r2 = read_imageui(src0_ql, qoff1 + lid).x;+ const uint r3 = read_imageui(src0_ql, qoff1 + lid + ne01).x;++ uint qw[8];+ qw[0] = SIGN6(EXP4(r0) | EXP2((qh1) & 0xFFu));+ qw[1] = SIGN6(EXP4(r0 >> 16) | EXP2((qh1 >> 8) & 0xFFu));+ qw[2] = SIGN6(EXP4(r1) | EXP2((qh1 >> 16) & 0xFFu));+ qw[3] = SIGN6(EXP4(r1 >> 16) | EXP2((qh1 >> 24) & 0xFFu));+ qw[4] = SIGN6(EXP4(r2) | EXP2((qh2) & 0xFFu));+ qw[5] = SIGN6(EXP4(r2 >> 16) | EXP2((qh2 >> 8) & 0xFFu));+ qw[6] = SIGN6(EXP4(r3) | EXP2((qh2 >> 16) & 0xFFu));+ qw[7] = SIGN6(EXP4(r3 >> 16) | EXP2((qh2 >> 24) & 0xFFu));++ const uint stage_lim = (uint)n_real * 8;+ for (uint idx = lid; idx < stage_lim; idx += 64) {+ const uint t = idx >> 3;+ const uint u = idx & 7;+ sh_qa[t][u] = src1_qa[(col + t) * ne00_u + (step >> 2) + u];+ }+ if (lid < (uint)n_real) {+ sh_d[lid] = src1_da[(col + lid) * ne00_b + sub];+ }+ barrier(CLK_LOCAL_MEM_FENCE);++ // Full tiles keep the fully-unrolled 32-wide loop; partial tiles run n_real.+ if (n_real == TILESIZE_N) {+ #pragma unroll+ for (int t = 0; t < TILESIZE_N; ++t) { MOE_Q6K_DP4A_T(t); }+ } else {+ #pragma unroll 4+ for (int t = 0; t < n_real; ++t) { MOE_Q6K_DP4A_T(t); }+ }+ barrier(CLK_LOCAL_MEM_FENCE);+ }++ if (row_idx >= ne01) {+ return;+ }++ __local uint out_idx[TILESIZE_N];+ if (lid < TILESIZE_N) {+ uint idx = sh_src2[lid];+ if (idx == 0xFFFFFFFF) {+ idx = sh_src2[0];+ }+ out_idx[lid] = idx * ne01;+ }+ barrier(CLK_LOCAL_MEM_FENCE);++ const uint m_offset = row + lid;+ if (n_real == TILESIZE_N) {+ #pragma unroll+ for (int t = 1; t < TILESIZE_N; ++t) {+ write_imagef(dst, out_idx[t] + m_offset, acc[t]);+ }+ barrier(CLK_GLOBAL_MEM_FENCE);+ write_imagef(dst, out_idx[0] + m_offset, acc[0]);+ } else {+ for (int t = 0; t < n_real; ++t) {+ write_imagef(dst, out_idx[t] + m_offset, acc[t]);+ }+ }+}diff --git a/ggml/src/ggml-opencl/kernels/gemm_moe_q8_0_f32_ns.cl b/ggml/src/ggml-opencl/kernels/gemm_moe_q8_0_f32_ns.clnew file mode 100644index 0000000000..dc0f0ed86c--- /dev/null+++ b/ggml/src/ggml-opencl/kernels/gemm_moe_q8_0_f32_ns.cl@@ -0,0 +1,221 @@+#pragma OPENCL EXTENSION cl_khr_fp16 : enable+#pragma OPENCL EXTENSION cl_khr_subgroups : enable+#pragma OPENCL EXTENSION cl_qcom_subgroup_uniform_load: enable+#pragma OPENCL EXTENSION cl_qcom_subgroup_constant_load: enable+#pragma OPENCL EXTENSION cl_qcom_extra_vector_types : enable++#define TILESIZE_K 16+#define TILESIZE_M 64+#define TILESIZE_N 32++// q8_0: 16 signed int8 weights (one uint4 = 16 chars) -> half16, scaled.+#define dequantize_q8_0(q4, a_f16, scale) \+ a_f16 = convert_half16(as_char16(q4)) * scale;++#define dotx16_reduce8(a_reg, b_lm, c_reg, lm_offset) \+ acc.s0 = dot(a_reg.s0123, b_lm[lm_offset + 0]); \+ acc.s1 = dot(a_reg.s0123, b_lm[lm_offset + 1]); \+ acc.s2 = dot(a_reg.s0123, b_lm[lm_offset + 2]); \+ acc.s3 = dot(a_reg.s0123, b_lm[lm_offset + 3]); \+ acc.s4 = dot(a_reg.s0123, b_lm[lm_offset + 4]); \+ acc.s5 = dot(a_reg.s0123, b_lm[lm_offset + 5]); \+ acc.s6 = dot(a_reg.s0123, b_lm[lm_offset + 6]); \+ acc.s7 = dot(a_reg.s0123, b_lm[lm_offset + 7]); \+ acc.s8 = dot(a_reg.s0123, b_lm[lm_offset + 8]); \+ acc.s9 = dot(a_reg.s0123, b_lm[lm_offset + 9]); \+ acc.sa = dot(a_reg.s0123, b_lm[lm_offset + 10]); \+ acc.sb = dot(a_reg.s0123, b_lm[lm_offset + 11]); \+ acc.sc = dot(a_reg.s0123, b_lm[lm_offset + 12]); \+ acc.sd = dot(a_reg.s0123, b_lm[lm_offset + 13]); \+ acc.se = dot(a_reg.s0123, b_lm[lm_offset + 14]); \+ acc.sf = dot(a_reg.s0123, b_lm[lm_offset + 15]); \+ acc.s0 += dot(a_reg.s4567, b_lm[lm_offset + 32]); \+ acc.s1 += dot(a_reg.s4567, b_lm[lm_offset + 33]); \+ acc.s2 += dot(a_reg.s4567, b_lm[lm_offset + 34]); \+ acc.s3 += dot(a_reg.s4567, b_lm[lm_offset + 35]); \+ acc.s4 += dot(a_reg.s4567, b_lm[lm_offset + 36]); \+ acc.s5 += dot(a_reg.s4567, b_lm[lm_offset + 37]); \+ acc.s6 += dot(a_reg.s4567, b_lm[lm_offset + 38]); \+ acc.s7 += dot(a_reg.s4567, b_lm[lm_offset + 39]); \+ acc.s8 += dot(a_reg.s4567, b_lm[lm_offset + 40]); \+ acc.s9 += dot(a_reg.s4567, b_lm[lm_offset + 41]); \+ acc.sa += dot(a_reg.s4567, b_lm[lm_offset + 42]); \+ acc.sb += dot(a_reg.s4567, b_lm[lm_offset + 43]); \+ acc.sc += dot(a_reg.s4567, b_lm[lm_offset + 44]); \+ acc.sd += dot(a_reg.s4567, b_lm[lm_offset + 45]); \+ acc.se += dot(a_reg.s4567, b_lm[lm_offset + 46]); \+ acc.sf += dot(a_reg.s4567, b_lm[lm_offset + 47]); \+ c_reg.lo += convert_float8(acc.lo); \+ c_reg.hi += convert_float8(acc.hi); \+ acc.s0 = dot(a_reg.s89ab, b_lm[lm_offset + 64]); \+ acc.s1 = dot(a_reg.s89ab, b_lm[lm_offset + 65]); \+ acc.s2 = dot(a_reg.s89ab, b_lm[lm_offset + 66]); \+ acc.s3 = dot(a_reg.s89ab, b_lm[lm_offset + 67]); \+ acc.s4 = dot(a_reg.s89ab, b_lm[lm_offset + 68]); \+ acc.s5 = dot(a_reg.s89ab, b_lm[lm_offset + 69]); \+ acc.s6 = dot(a_reg.s89ab, b_lm[lm_offset + 70]); \+ acc.s7 = dot(a_reg.s89ab, b_lm[lm_offset + 71]); \+ acc.s8 = dot(a_reg.s89ab, b_lm[lm_offset + 72]); \+ acc.s9 = dot(a_reg.s89ab, b_lm[lm_offset + 73]); \+ acc.sa = dot(a_reg.s89ab, b_lm[lm_offset + 74]); \+ acc.sb = dot(a_reg.s89ab, b_lm[lm_offset + 75]); \+ acc.sc = dot(a_reg.s89ab, b_lm[lm_offset + 76]); \+ acc.sd = dot(a_reg.s89ab, b_lm[lm_offset + 77]); \+ acc.se = dot(a_reg.s89ab, b_lm[lm_offset + 78]); \+ acc.sf = dot(a_reg.s89ab, b_lm[lm_offset + 79]); \+ acc.s0 += dot(a_reg.scdef, b_lm[lm_offset + 96]); \+ acc.s1 += dot(a_reg.scdef, b_lm[lm_offset + 97]); \+ acc.s2 += dot(a_reg.scdef, b_lm[lm_offset + 98]); \+ acc.s3 += dot(a_reg.scdef, b_lm[lm_offset + 99]); \+ acc.s4 += dot(a_reg.scdef, b_lm[lm_offset + 100]); \+ acc.s5 += dot(a_reg.scdef, b_lm[lm_offset + 101]); \+ acc.s6 += dot(a_reg.scdef, b_lm[lm_offset + 102]); \+ acc.s7 += dot(a_reg.scdef, b_lm[lm_offset + 103]); \+ acc.s8 += dot(a_reg.scdef, b_lm[lm_offset + 104]); \+ acc.s9 += dot(a_reg.scdef, b_lm[lm_offset + 105]); \+ acc.sa += dot(a_reg.scdef, b_lm[lm_offset + 106]); \+ acc.sb += dot(a_reg.scdef, b_lm[lm_offset + 107]); \+ acc.sc += dot(a_reg.scdef, b_lm[lm_offset + 108]); \+ acc.sd += dot(a_reg.scdef, b_lm[lm_offset + 109]); \+ acc.se += dot(a_reg.scdef, b_lm[lm_offset + 110]); \+ acc.sf += dot(a_reg.scdef, b_lm[lm_offset + 111]); \+ c_reg.lo += convert_float8(acc.lo); \+ c_reg.hi += convert_float8(acc.hi); \+++__attribute__((qcom_wave_pair_mode(1)))+kernel void kernel_gemm_moe_q8_0_f32_ns(+ __global char * src0_q, // flat q8_0 quants [n_expert*ne01*ne00]+ __global half * src0_d, // flat q8_0 scales [n_expert*ne01*nb]+ __read_only image1d_buffer_t src1, // reordered activations (f32)+ __global uint * src2, // post-router out indices+ __global ushort * src2_emap,// expert per tile+ __write_only image1d_buffer_t dst,+ __global int * total_tiles,+ uint ne00,+ uint ne01+) {+ uint block_id_m = get_global_id(1); // m_tile+ uint block_id_n = get_global_id(2); // n_tile++ if (block_id_n >= total_tiles[0]) {+ return;+ }++ __private half16 reg_a;+ __private float32 reg_c = (float32)(0);+ __local half4 shared_b[128];++ const ushort expert_id = src2_emap[block_id_n];++ const uint row = block_id_m * TILESIZE_M;+ const uint col = block_id_n * TILESIZE_N;++ const uint nb = ne00 >> 5; // blocks per row (ne00/32)+ const uint w_row = expert_id * ne01 + row + get_local_id(0); // this lane's output row+ __global char * w_q = src0_q + (ulong)w_row * ne00; // char base for the row+ __global half * w_d = src0_d + (ulong)w_row * nb; // scale base for the row++ uint sub_block_id_m = get_local_id(0);+ uint2 b_global_offset;+ b_global_offset.x = ((sub_block_id_m & 3) << 2) + (sub_block_id_m >> 2) * ne00;+ b_global_offset.y = b_global_offset.x + (16 * ne00);+ uint2 b_local_offset;+ b_local_offset.x = (sub_block_id_m & 3) * 32 + (sub_block_id_m >> 2);+ b_local_offset.y = b_local_offset.x + 16;++ // Loop along K axis, 32 elements per iteration, split into 2 sub-blocks.+ for (uint step = 0; step < ne00; step += TILESIZE_K * 2) {+ half s = w_d[step >> 5]; // one q8_0 scale per 32-element block++ // First sub-block: 16 weights (16 chars = one uint4) at K=step+ uint4 q8x16 = *((__global uint4 *)(w_q + step));++ uint b_sub_offset = col * ne00 + step;+ float8 bx8_f32;+ bx8_f32.lo = read_imagef(src1, (b_sub_offset + b_global_offset.x) / 4);+ bx8_f32.hi = read_imagef(src1, (b_sub_offset + b_global_offset.y) / 4);+ half8 bx8_f16 = convert_half8(bx8_f32);+ shared_b[b_local_offset.x] = bx8_f16.lo;+ shared_b[b_local_offset.y] = bx8_f16.hi;++ dequantize_q8_0(q8x16, reg_a, s);++ sub_group_barrier(CLK_LOCAL_MEM_FENCE);++ half16 acc;+ dotx16_reduce8(reg_a, shared_b, reg_c.lo, 0);+ dotx16_reduce8(reg_a, shared_b, reg_c.hi, 16);++ // Second sub-block: next 16 weights at K=step+16+ uint half_step = step + TILESIZE_K;+ q8x16 = *((__global uint4 *)(w_q + half_step));+ b_sub_offset = col * ne00 + half_step;++ bx8_f32.lo = read_imagef(src1, (b_sub_offset + b_global_offset.x) / 4);+ bx8_f32.hi = read_imagef(src1, (b_sub_offset + b_global_offset.y) / 4);+ bx8_f16 = convert_half8(bx8_f32);+ shared_b[b_local_offset.x] = bx8_f16.lo;+ shared_b[b_local_offset.y] = bx8_f16.hi;++ dequantize_q8_0(q8x16, reg_a, s);++ sub_group_barrier(CLK_LOCAL_MEM_FENCE);++ dotx16_reduce8(reg_a, shared_b, reg_c.lo, 0);+ dotx16_reduce8(reg_a, shared_b, reg_c.hi, 16);+ }++ if ((get_global_id(0) + block_id_m * TILESIZE_M) >= ne01) {+ return;+ }++ __local uint out_idx[TILESIZE_N];++ if (get_local_id(0) < TILESIZE_N) {+ uint idx = src2[block_id_n * TILESIZE_N + get_local_id(0)];+ if (idx == 0xFFFFFFFF) {+ idx = src2[block_id_n * TILESIZE_N + 0];+ }+ out_idx[get_local_id(0)] = idx * ne01;+ }++ barrier(CLK_LOCAL_MEM_FENCE);++ uint m_offset = row + get_local_id(0);++ write_imagef(dst, out_idx[1] + m_offset, (reg_c.s1));+ write_imagef(dst, out_idx[2] + m_offset, (reg_c.s2));+ write_imagef(dst, out_idx[3] + m_offset, (reg_c.s3));+ write_imagef(dst, out_idx[4] + m_offset, (reg_c.s4));+ write_imagef(dst, out_idx[5] + m_offset, (reg_c.s5));+ write_imagef(dst, out_idx[6] + m_offset, (reg_c.s6));+ write_imagef(dst, out_idx[7] + m_offset, (reg_c.s7));+ write_imagef(dst, out_idx[8] + m_offset, (reg_c.s8));+ write_imagef(dst, out_idx[9] + m_offset, (reg_c.s9));+ write_imagef(dst, out_idx[10] + m_offset, (reg_c.sa));+ write_imagef(dst, out_idx[11] + m_offset, (reg_c.sb));+ write_imagef(dst, out_idx[12] + m_offset, (reg_c.sc));+ write_imagef(dst, out_idx[13] + m_offset, (reg_c.sd));+ write_imagef(dst, out_idx[14] + m_offset, (reg_c.se));+ write_imagef(dst, out_idx[15] + m_offset, (reg_c.sf));+ write_imagef(dst, out_idx[16] + m_offset, (reg_c.sg));+ write_imagef(dst, out_idx[17] + m_offset, (reg_c.sh));+ write_imagef(dst, out_idx[18] + m_offset, (reg_c.si));+ write_imagef(dst, out_idx[19] + m_offset, (reg_c.sj));+ write_imagef(dst, out_idx[20] + m_offset, (reg_c.sk));+ write_imagef(dst, out_idx[21] + m_offset, (reg_c.sl));+ write_imagef(dst, out_idx[22] + m_offset, (reg_c.sm));+ write_imagef(dst, out_idx[23] + m_offset, (reg_c.sn));+ write_imagef(dst, out_idx[24] + m_offset, (reg_c.so));+ write_imagef(dst, out_idx[25] + m_offset, (reg_c.sp));+ write_imagef(dst, out_idx[26] + m_offset, (reg_c.sq));+ write_imagef(dst, out_idx[27] + m_offset, (reg_c.sr));+ write_imagef(dst, out_idx[28] + m_offset, (reg_c.ss));+ write_imagef(dst, out_idx[29] + m_offset, (reg_c.st));+ write_imagef(dst, out_idx[30] + m_offset, (reg_c.su));+ write_imagef(dst, out_idx[31] + m_offset, (reg_c.sv));++ barrier(CLK_GLOBAL_MEM_FENCE);+ write_imagef(dst, out_idx[0] + m_offset, (reg_c.s0));+}diff --git a/ggml/src/ggml-opencl/kernels/gemm_moe_q8_1_dp4a.cl b/ggml/src/ggml-opencl/kernels/gemm_moe_q8_1_dp4a.clnew file mode 100644index 0000000000..39bf5d8321--- /dev/null+++ b/ggml/src/ggml-opencl/kernels/gemm_moe_q8_1_dp4a.cl@@ -0,0 +1,221 @@+#pragma OPENCL EXTENSION cl_khr_fp16 : enable+#pragma OPENCL EXTENSION cl_khr_subgroups : enable+#ifdef cl_khr_integer_dot_product+#pragma OPENCL EXTENSION cl_khr_integer_dot_product : enable+#endif++// Generic int8 dp4a MoE GEMM, specialized versions also exist+// MOE_QT:+// 4 (q4_K)/41(q4_1)/40(q4_0) NIBBLE image low nibbles -> EXP4+// 5 (q5_K)/51(q5_1)/50(q5_0) NIBBLE+HI image nibbles + qh high-bit plane+// 6 (q6_K) Q6 image nibbles + qh 2-bit -> SIGN6((nibble|hi2))+// 80(q8_0)/82(mxfp4) INT8 global int8 codes (mxfp4: convert applies kvalues LUT)++#define TILESIZE_M 64+#define TILESIZE_N 32+#define QK_K 256++#ifndef MOE_QT+#define MOE_QT 4+#endif++// 4 nibbles in low 16 bits of u -> 4 bytes (value 0..15)+#define EXP4(u) ( ((uint)((u) & 0x000Fu)) | \+ (((uint)((u) & 0x00F0u)) << 4) | \+ (((uint)((u) & 0x0F00u)) << 8) | \+ (((uint)((u) & 0xF000u)) << 12) )+// 4 2-bit highs in byte b -> 4 bytes, bits 4-5 (q6_K)+#define EXP2(b) ( (((uint)((b) & 0x03u)) << 4) | \+ (((uint)((b) & 0x0Cu)) << 10) | \+ (((uint)((b) & 0x30u)) << 16) | \+ (((uint)((b) & 0xC0u)) << 22) )++// q6 (0..63) -> (q6-32) signed int8/byte (no inter-byte carry)+inline uint SIGN6(uint q6p){ uint x=q6p^0x20202020u; uint s=x&0x20202020u; return x|(s<<1)|(s<<2); }++// 4 high bits (one per element, in bits 0..3 of h) -> bit4 of each of 4 bytes (5-bit hi)+#define EXP1(h) ( (((uint)((h) & 0x1u)) << 4) | \+ (((uint)((h) & 0x2u)) << 11) | \+ (((uint)((h) & 0x4u)) << 18) | \+ (((uint)((h) & 0x8u)) << 25) )++// per-type weight params + per-32-step unpack into qw[8] (8 int8 uints)+#if MOE_QT == 4 || MOE_QT == 41 || MOE_QT == 40+ #define WEIGHT_PARAMS __read_only image1d_buffer_t src0_q,+ #define LOAD_QW(step, sub) \+ uint qw[8]; { \+ const uint qoff0 = row + ((ne01*(step))>>3) + ((expert_id*ne00*ne01)>>3); \+ const uint qoff1 = row + ((ne01*((step)+16))>>3) + ((expert_id*ne00*ne01)>>3); \+ const uint r0=read_imageui(src0_q,qoff0+lid).x, r1=read_imageui(src0_q,qoff0+lid+ne01).x; \+ const uint r2=read_imageui(src0_q,qoff1+lid).x, r3=read_imageui(src0_q,qoff1+lid+ne01).x; \+ qw[0]=EXP4(r0); qw[1]=EXP4(r0>>16); qw[2]=EXP4(r1); qw[3]=EXP4(r1>>16); \+ qw[4]=EXP4(r2); qw[5]=EXP4(r2>>16); qw[6]=EXP4(r3); qw[7]=EXP4(r3>>16); }++#elif MOE_QT == 5 || MOE_QT == 51 || MOE_QT == 50+ // low nibbles via image (q4_K layout) + high-bit plane src0_qh: 1 uint per 32-block+ // (bit i = high bit of element i). qh laid out [expert][block][row] to match the+ // existing q5_0 trans4 convert+ #define WEIGHT_PARAMS __read_only image1d_buffer_t src0_q, __global uint * src0_qh,+ #define LOAD_QW(step, sub) \+ uint qw[8]; { \+ const uint qoff0 = row + ((ne01*(step))>>3) + ((expert_id*ne00*ne01)>>3); \+ const uint qoff1 = row + ((ne01*((step)+16))>>3) + ((expert_id*ne00*ne01)>>3); \+ const uint r0=read_imageui(src0_q,qoff0+lid).x, r1=read_imageui(src0_q,qoff0+lid+ne01).x; \+ const uint r2=read_imageui(src0_q,qoff1+lid).x, r3=read_imageui(src0_q,qoff1+lid+ne01).x; \+ const uint h = src0_qh[row_idx + (sub)*ne01 + expert_id*(ne00>>5)*ne01]; \+ qw[0]=EXP4(r0)|EXP1(h); qw[1]=EXP4(r0>>16)|EXP1(h>>4); \+ qw[2]=EXP4(r1)|EXP1(h>>8); qw[3]=EXP4(r1>>16)|EXP1(h>>12); \+ qw[4]=EXP4(r2)|EXP1(h>>16); qw[5]=EXP4(r2>>16)|EXP1(h>>20); \+ qw[6]=EXP4(r3)|EXP1(h>>24); qw[7]=EXP4(r3>>16)|EXP1(h>>28); }++#elif MOE_QT == 6+ #define WEIGHT_PARAMS __read_only image1d_buffer_t src0_ql, __global uint * src0_qh,+ #define LOAD_QW(step, sub) \+ uint qw[8]; { \+ const uint qoff0 = row + ((ne01*(step))>>3) + ((expert_id*ne00*ne01)>>3); \+ const uint qoff1 = row + ((ne01*((step)+16))>>3) + ((expert_id*ne00*ne01)>>3); \+ const uint r0=read_imageui(src0_ql,qoff0+lid).x, r1=read_imageui(src0_ql,qoff0+lid+ne01).x; \+ const uint r2=read_imageui(src0_ql,qoff1+lid).x, r3=read_imageui(src0_ql,qoff1+lid+ne01).x; \+ const uint qhb = row + ((sub)*2)*ne01 + expert_id*((ne00>>5)*2)*ne01 + lid; \+ const uint qh1=src0_qh[qhb], qh2=src0_qh[qhb+ne01]; \+ qw[0]=SIGN6(EXP4(r0)|EXP2(qh1&0xFFu)); qw[1]=SIGN6(EXP4(r0>>16)|EXP2((qh1>>8)&0xFFu)); \+ qw[2]=SIGN6(EXP4(r1)|EXP2((qh1>>16)&0xFFu)); qw[3]=SIGN6(EXP4(r1>>16)|EXP2((qh1>>24)&0xFFu)); \+ qw[4]=SIGN6(EXP4(r2)|EXP2(qh2&0xFFu)); qw[5]=SIGN6(EXP4(r2>>16)|EXP2((qh2>>8)&0xFFu)); \+ qw[6]=SIGN6(EXP4(r3)|EXP2((qh2>>16)&0xFFu)); qw[7]=SIGN6(EXP4(r3>>16)|EXP2((qh2>>24)&0xFFu)); }++#elif MOE_QT == 80 || MOE_QT == 82+ // 8-bit direct: int8 codes 8 uints / 32-block, [expert][row][8*sub]. mxfp4: the+ // convert resolves kvalues_mxfp4[nibble] -> int8 and stores the e8m0_half scale.+ #define WEIGHT_PARAMS __global uint * src0_q8,+ #define LOAD_QW(step, sub) \+ uint qw[8]; { \+ const uint qb = (expert_id*ne01 + row_idx)*(ne00>>2) + (sub)*8; \+ qw[0]=src0_q8[qb+0]; qw[1]=src0_q8[qb+1]; qw[2]=src0_q8[qb+2]; qw[3]=src0_q8[qb+3]; \+ qw[4]=src0_q8[qb+4]; qw[5]=src0_q8[qb+5]; qw[6]=src0_q8[qb+6]; qw[7]=src0_q8[qb+7]; }+#else+ #error "unknown MOE_QT"+#endif++inline int dp4a4(uint w0,uint w1,uint w2,uint w3,uint a0,uint a1,uint a2,uint a3){+ int r=0; r=dot_acc_sat_4x8packed_ss_int(w0,a0,r); r=dot_acc_sat_4x8packed_ss_int(w1,a1,r);+ r=dot_acc_sat_4x8packed_ss_int(w2,a2,r); r=dot_acc_sat_4x8packed_ss_int(w3,a3,r); return r; }++// One token's two-half dp4a + uniform scale/min epilogue into acc[t].+#define MOE_DP4A_T(t) do { \+ const int raw1 = dp4a4(qw[0],qw[1],qw[2],qw[3], sh_qa[t][0],sh_qa[t][1],sh_qa[t][2],sh_qa[t][3]); \+ const int raw2 = dp4a4(qw[4],qw[5],qw[6],qw[7], sh_qa[t][4],sh_qa[t][5],sh_qa[t][6],sh_qa[t][7]); \+ const float a_d = (float)sh_d[t]; \+ acc[t] += sc0*a_d*(float)raw1 + sc1*a_d*(float)raw2 - mn*(float)sh_s[t]; \+ } while (0)++__attribute__((qcom_wave_pair_mode(1)))+kernel void kernel_gemm_moe_q8_1_dp4a(+ WEIGHT_PARAMS // per-type native weight buffer(s)+ __global half * src0_scale,// uniform f16 16/superblock (per-16), [expert,row]+ __global half * src0_min, // uniform f16 8/superblock (per-32), [expert,row]+ __global uint * src1_qa, // q8_1 activations int8 (as uint, 4/elem)+ __global half * src1_da, // q8_1 per-block scale [tok_slot * ne00/32]+ __global half * src1_sa, // q8_1 per-block sum*d [tok_slot * ne00/32]+ __global uint * src2, // post-router (orig out positions)+ __global ushort * src2_emap, // tile -> expert id+ __write_only image1d_buffer_t dst,+ __global int * total_tiles,+ uint ne00,+ uint ne01,+ int is_ragged,+ int has_min // 0 for symmetric types (q8_0/q6_K/q4_0/...): skip min read+) {+ const uint block_id_m = get_global_id(1);+ const uint block_id_n = get_global_id(2);+ if (block_id_n >= total_tiles[0]) return;++ const uint lid = get_local_id(0); // 0..63 -> output row within M-tile+ const ushort expert_id = src2_emap[block_id_n];+ const uint row = block_id_m * TILESIZE_M;+ const uint col = block_id_n * TILESIZE_N;+ const uint row_idx = row + lid;++ // Scale/min are laid out FLAT per-32-block (2 per-16-segment scales + 1 min per+ // 32-block), so K only needs to be a multiple of 32 — works for the 32-block+ // types (q8_0/q5_0/q4_0/...) as well as the K-quants (K%256==0, same bytes).+ const uint nblk32 = ne00 / 32;+ const uint sc_per_row = nblk32 * 2;+ const uint mn_per_row = nblk32;+ const uint ne00_u = ne00 >> 2;+ const uint ne00_b = ne00 >> 5;++ __local uint sh_qa[TILESIZE_N][8];+ __local half sh_d[TILESIZE_N];+ __local half sh_s[TILESIZE_N];++ __local uint sh_src2[TILESIZE_N];+ __local int sh_nreal;+ if (lid < TILESIZE_N) sh_src2[lid] = src2[col + lid];+ barrier(CLK_LOCAL_MEM_FENCE);+ if (lid == 0) {+ int nr = TILESIZE_N;+ if (is_ragged) { nr = 0;+ #pragma unroll+ for (int t = 0; t < TILESIZE_N; ++t) if (sh_src2[t] != 0xFFFFFFFFu) ++nr; }+ sh_nreal = nr;+ }+ barrier(CLK_LOCAL_MEM_FENCE);+ const int n_real = sh_nreal;++ float acc[TILESIZE_N];+ #pragma unroll+ for (int t = 0; t < TILESIZE_N; ++t) acc[t] = 0.0f;++ for (uint step = 0; step < ne00; step += 32) {+ const uint sub = step >> 5; // 32-block index along K++ // uniform pre-decoded scale (2 per-16-seg) + min (1) for this row, this 32-block+ __global half * scl = src0_scale + (expert_id*ne01 + row_idx)*sc_per_row + sub*2;+ const float sc0 = (float)scl[0];+ const float sc1 = (float)scl[1];+ float mn = 0.0f;+ if (has_min) mn = (float)src0_min[(expert_id*ne01 + row_idx)*mn_per_row + sub];++ LOAD_QW(step, sub)++ const uint stage_lim = (uint)n_real * 8;+ for (uint idx = lid; idx < stage_lim; idx += 64) {+ const uint t = idx >> 3, u = idx & 7;+ sh_qa[t][u] = src1_qa[(col + t) * ne00_u + (step >> 2) + u];+ }+ if (lid < (uint)n_real) {+ sh_d[lid] = src1_da[(col + lid) * ne00_b + sub];+ sh_s[lid] = src1_sa[(col + lid) * ne00_b + sub];+ }+ barrier(CLK_LOCAL_MEM_FENCE);++ if (n_real == TILESIZE_N) {+ #pragma unroll+ for (int t = 0; t < TILESIZE_N; ++t) { MOE_DP4A_T(t); }+ } else {+ #pragma unroll 4+ for (int t = 0; t < n_real; ++t) { MOE_DP4A_T(t); }+ }+ barrier(CLK_LOCAL_MEM_FENCE);+ }++ if (row_idx >= ne01) return;++ __local uint out_idx[TILESIZE_N];+ if (lid < TILESIZE_N) {+ uint idx = sh_src2[lid];+ if (idx == 0xFFFFFFFF) idx = sh_src2[0];+ out_idx[lid] = idx * ne01;+ }+ barrier(CLK_LOCAL_MEM_FENCE);++ const uint m_offset = row + lid;+ if (n_real == TILESIZE_N) {+ #pragma unroll+ for (int t = 1; t < TILESIZE_N; ++t) write_imagef(dst, out_idx[t] + m_offset, acc[t]);+ barrier(CLK_GLOBAL_MEM_FENCE);+ write_imagef(dst, out_idx[0] + m_offset, acc[0]);+ } else {+ for (int t = 0; t < n_real; ++t) write_imagef(dst, out_idx[t] + m_offset, acc[t]);+ }+}diff --git a/ggml/src/ggml-opencl/kernels/gemm_noshuffle_iq4_nl_q8_1_dp4a.cl b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_iq4_nl_q8_1_dp4a.clnew file mode 100644index 0000000000..2941289ddf--- /dev/null+++ b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_iq4_nl_q8_1_dp4a.cl@@ -0,0 +1,143 @@+#pragma OPENCL EXTENSION cl_khr_fp16 : enable+#pragma OPENCL EXTENSION cl_khr_subgroups : enable+#ifdef cl_khr_integer_dot_product+#pragma OPENCL EXTENSION cl_khr_integer_dot_product : enable+#endif++// Weight layout, feature-major:+// src0_q[row + (k/4)*m] ushort = 4 nibbles (K = 4*grp .. +3)+// src0_d[row + (k/32)*m] half = per-32-block scale++#define TILESIZE_N 32++// IQ4_NL non-linear codebook as signed int8, packed 4 codes per uint.+// divergent nibble lookups read a small __constant uint array + shift,+// never a byte array because byte-indexed __constant loads serialize on Adreno and tank perf+// idx 0-3: -127,-104,-83,-65 = 0x81,0x98,0xAD,0xBF+// idx 4-7: -49,-35,-22,-10 = 0xCF,0xDD,0xEA,0xF6+// idx 8-11: 1, 13, 25, 38 = 0x01,0x0D,0x19,0x26+// idx 12-15: 53, 69, 89,113 = 0x35,0x45,0x59,0x71+__constant uint kvalues_iq4nl_i8x4[4] = {+ 0xBFAD9881u, 0xF6EADDCFu, 0x26190D01u, 0x71594535u+};++// nibble (0..15) -> its codebook byte in the low 8 bits.+inline uint iq4nl_code(uint n) {+ return (kvalues_iq4nl_i8x4[n >> 2] >> ((n & 3u) * 8u)) & 0xFFu;+}++// 4 nibbles in low 16 bits of u -> 4 codebook int8, packed for dp4a.+inline uint iq4nl_pack(ushort u) {+ return iq4nl_code((uint)( u & 0xF))+ | (iq4nl_code((uint)((u >> 4) & 0xF)) << 8)+ | (iq4nl_code((uint)((u >> 8) & 0xF)) << 16)+ | (iq4nl_code((uint)((u >> 12) & 0xF)) << 24);+}++inline int dot8_q8a(uint8 qw, __local const uint * a) {+ int r = 0;+ r = dot_acc_sat_4x8packed_ss_int(qw.s0, a[0], r);+ r = dot_acc_sat_4x8packed_ss_int(qw.s1, a[1], r);+ r = dot_acc_sat_4x8packed_ss_int(qw.s2, a[2], r);+ r = dot_acc_sat_4x8packed_ss_int(qw.s3, a[3], r);+ r = dot_acc_sat_4x8packed_ss_int(qw.s4, a[4], r);+ r = dot_acc_sat_4x8packed_ss_int(qw.s5, a[5], r);+ r = dot_acc_sat_4x8packed_ss_int(qw.s6, a[6], r);+ r = dot_acc_sat_4x8packed_ss_int(qw.s7, a[7], r);+ return r;+}++__attribute__((qcom_wave_pair_mode(1)))+kernel void kernel_gemm_noshuffle_iq4_nl_q8_1_dp4a(+ __global const ushort * src0_q, // IQ4_NL nibbles (4/ushort, feature-major)+ __global const half * src0_d, // per-32-block scale, feature-major+ __global const uint * src1_qa, // q8_1 activations int8 (as uint, 4/elem) [N, K]+ __global const half * src1_da, // q8_1 per-block scale [N, K/32]+ __global float * dst,+ ulong offsetd,+ int m, // output features (rows)+ int n_no_padding, // tokens (cols)+ int k // K (== ne00)+) {+ dst = (global float *)((global char *)dst + offsetd);++ const uint lid = get_local_id(0); // 0..63 -> row within the M-tile+ const uint block_id_m = get_global_id(1);+ const uint block_id_n = get_global_id(2);++ const uint row = block_id_m * 64 + lid;+ const uint col_base = block_id_n * TILESIZE_N;+ const bool row_valid = row < (uint)m;+ const uint rrow = row_valid ? row : 0; // clamp OOB rows; their writes are masked++ const uint k_u = (uint)k >> 2; // K in uint (int8x4) units+ const uint k_b = (uint)k >> 5; // blocks-of-32 along K++ __local uint sh_qa[TILESIZE_N][8];+ __local half sh_d[TILESIZE_N];++#define NGROUPS (TILESIZE_N / 4)+ float4 acc[NGROUPS];+ #pragma unroll+ for (int g = 0; g < NGROUPS; ++g) acc[g] = (float4)(0.0f);++ for (uint step = 0; step < (uint)k; step += 32) {+ const uint sub = step >> 5;++ const float d_w = (float)src0_d[rrow + sub * (uint)m];++ // 8 weight uints (32 codebook int8) for this row, this 32-block.+ const uint qsbase = rrow + (step >> 2) * (uint)m;+ uint8 qw;+ qw.s0 = iq4nl_pack(src0_q[qsbase + 0 * m]);+ qw.s1 = iq4nl_pack(src0_q[qsbase + 1 * m]);+ qw.s2 = iq4nl_pack(src0_q[qsbase + 2 * m]);+ qw.s3 = iq4nl_pack(src0_q[qsbase + 3 * m]);+ qw.s4 = iq4nl_pack(src0_q[qsbase + 4 * m]);+ qw.s5 = iq4nl_pack(src0_q[qsbase + 5 * m]);+ qw.s6 = iq4nl_pack(src0_q[qsbase + 6 * m]);+ qw.s7 = iq4nl_pack(src0_q[qsbase + 7 * m]);++ // cooperatively stage the 32-token x 32-K int8 activations to lm+ for (uint idx = lid; idx < TILESIZE_N * 8; idx += 64) {+ const uint t = idx >> 3;+ const uint u = idx & 7;+ const uint c = col_base + t;+ sh_qa[t][u] = (c < (uint)n_no_padding) ? src1_qa[c * k_u + (step >> 2) + u] : 0u;+ }+ if (lid < TILESIZE_N) {+ const uint c = col_base + lid;+ sh_d[lid] = (c < (uint)n_no_padding) ? src1_da[c * k_b + sub] : (half)0;+ }+ barrier(CLK_LOCAL_MEM_FENCE);++#define LD4(arr, b) ((float4)((float)arr[(b)+0], (float)arr[(b)+1], (float)arr[(b)+2], (float)arr[(b)+3]))+ #pragma unroll+ for (int g = 0; g < NGROUPS; ++g) {+ const int b = g * 4;+ float4 rf;+ rf.s0 = (float)dot8_q8a(qw, sh_qa[b+0]); rf.s1 = (float)dot8_q8a(qw, sh_qa[b+1]);+ rf.s2 = (float)dot8_q8a(qw, sh_qa[b+2]); rf.s3 = (float)dot8_q8a(qw, sh_qa[b+3]);+ acc[g] += d_w * LD4(sh_d, b) * rf;+ }+#undef LD4+ barrier(CLK_LOCAL_MEM_FENCE);+ }++ if (!row_valid) {+ return;+ }++ // dst is [token, feature] row-major (stride m): dst[col*m + row].+ #pragma unroll+ for (int g = 0; g < NGROUPS; ++g) {+ const uint b = (uint)(g * 4);+ const float4 a = acc[g];+ const uint c0 = col_base + b;+ if (c0 + 0 < (uint)n_no_padding) dst[(c0 + 0) * (uint)m + row] = a.s0;+ if (c0 + 1 < (uint)n_no_padding) dst[(c0 + 1) * (uint)m + row] = a.s1;+ if (c0 + 2 < (uint)n_no_padding) dst[(c0 + 2) * (uint)m + row] = a.s2;+ if (c0 + 3 < (uint)n_no_padding) dst[(c0 + 3) * (uint)m + row] = a.s3;+ }+#undef NGROUPS+}diff --git a/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q4_0_q8_1_dp4a.cl b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q4_0_q8_1_dp4a.clnew file mode 100644index 0000000000..446a8eb184--- /dev/null+++ b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q4_0_q8_1_dp4a.cl@@ -0,0 +1,127 @@+#pragma OPENCL EXTENSION cl_khr_fp16 : enable+#pragma OPENCL EXTENSION cl_khr_subgroups : enable+#ifdef cl_khr_integer_dot_product+#pragma OPENCL EXTENSION cl_khr_integer_dot_product : enable+#endif++#define TILESIZE_N 32++// Expand the 4 nibbles in the low 16 bits of u into 4 bytes (value 0..15),+// packed for the int8 dp4a. The -8 zero-point is applied via the sum term.+#define EXP4(u) ( ((uint)((u) & 0x000Fu)) | \+ (((uint)((u) & 0x00F0u)) << 4) | \+ (((uint)((u) & 0x0F00u)) << 8) | \+ (((uint)((u) & 0xF000u)) << 12) )++inline int dot8_q8a(uint8 qw, __local const uint * a) {+ int r = 0;+ r = dot_acc_sat_4x8packed_ss_int(qw.s0, a[0], r);+ r = dot_acc_sat_4x8packed_ss_int(qw.s1, a[1], r);+ r = dot_acc_sat_4x8packed_ss_int(qw.s2, a[2], r);+ r = dot_acc_sat_4x8packed_ss_int(qw.s3, a[3], r);+ r = dot_acc_sat_4x8packed_ss_int(qw.s4, a[4], r);+ r = dot_acc_sat_4x8packed_ss_int(qw.s5, a[5], r);+ r = dot_acc_sat_4x8packed_ss_int(qw.s6, a[6], r);+ r = dot_acc_sat_4x8packed_ss_int(qw.s7, a[7], r);+ return r;+}++__attribute__((qcom_wave_pair_mode(1)))+kernel void kernel_gemm_noshuffle_q4_0_q8_1_dp4a(+ __global const ushort * src0_q, // q4_0 nibbles (4/ushort, feature-major)+ __global const half * src0_d, // per-32-block scale, feature-major+ __global const uint * src1_qa, // q8_1 activations int8 (as uint, 4/elem) [N, K]+ __global const half * src1_da, // q8_1 per-block scale [N, K/32]+ __global const half * src1_sa, // q8_1 per-block sum*d [N, K/32]+ __global float * dst,+ ulong offsetd,+ int m, // output features (rows)+ int n_no_padding, // tokens (cols)+ int k // K (== ne00)+) {+ dst = (global float *)((global char *)dst + offsetd);++ const uint lid = get_local_id(0); // 0..63 -> row within the M-tile+ const uint block_id_m = get_global_id(1);+ const uint block_id_n = get_global_id(2);++ const uint row = block_id_m * 64 + lid;+ const uint col_base = block_id_n * TILESIZE_N;+ const bool row_valid = row < (uint)m;+ const uint rrow = row_valid ? row : 0; // clamp OOB rows; their writes are masked++ const uint k_u = (uint)k >> 2; // K in uint (int8x4) units+ const uint k_b = (uint)k >> 5; // blocks-of-32 along K++ __local uint sh_qa[TILESIZE_N][8];+ __local half sh_d[TILESIZE_N];+ __local half sh_s[TILESIZE_N];++#define NGROUPS (TILESIZE_N / 4)+ float4 acc[NGROUPS];+ #pragma unroll+ for (int g = 0; g < NGROUPS; ++g) acc[g] = (float4)(0.0f);++ for (uint step = 0; step < (uint)k; step += 32) {+ const uint sub = step >> 5;++ const float d_w = (float)src0_d[rrow + sub * (uint)m];++ // 8 weight uints (32 nibbles) for this row, this 32-block. Feature-major:+ // src0_q[row + (k/4 + u)*m], k/4 = step/4 (= step>>2). EXP4 -> dp4a int8.+ const uint qsbase = rrow + (step >> 2) * (uint)m;+ uint8 qw;+ qw.s0 = EXP4(src0_q[qsbase + 0 * m]);+ qw.s1 = EXP4(src0_q[qsbase + 1 * m]);+ qw.s2 = EXP4(src0_q[qsbase + 2 * m]);+ qw.s3 = EXP4(src0_q[qsbase + 3 * m]);+ qw.s4 = EXP4(src0_q[qsbase + 4 * m]);+ qw.s5 = EXP4(src0_q[qsbase + 5 * m]);+ qw.s6 = EXP4(src0_q[qsbase + 6 * m]);+ qw.s7 = EXP4(src0_q[qsbase + 7 * m]);++ // cooperatively stage the 32-token x 32-K int8 activations to LDS+ for (uint idx = lid; idx < TILESIZE_N * 8; idx += 64) {+ const uint t = idx >> 3;+ const uint u = idx & 7;+ const uint c = col_base + t;+ sh_qa[t][u] = (c < (uint)n_no_padding) ? src1_qa[c * k_u + (step >> 2) + u] : 0u;+ }+ if (lid < TILESIZE_N) {+ const uint c = col_base + lid;+ sh_d[lid] = (c < (uint)n_no_padding) ? src1_da[c * k_b + sub] : (half)0;+ sh_s[lid] = (c < (uint)n_no_padding) ? src1_sa[c * k_b + sub] : (half)0;+ }+ barrier(CLK_LOCAL_MEM_FENCE);++#define LD4(arr, b) ((float4)((float)arr[(b)+0], (float)arr[(b)+1], (float)arr[(b)+2], (float)arr[(b)+3]))+ #pragma unroll+ for (int g = 0; g < NGROUPS; ++g) {+ const int b = g * 4;+ float4 rf;+ rf.s0 = (float)dot8_q8a(qw, sh_qa[b+0]); rf.s1 = (float)dot8_q8a(qw, sh_qa[b+1]);+ rf.s2 = (float)dot8_q8a(qw, sh_qa[b+2]); rf.s3 = (float)dot8_q8a(qw, sh_qa[b+3]);+ // q4_0: w = d*(q-8) -> d_w * (a_d * dp4a(q,qa) - 8 * a_s)+ acc[g] += d_w * (LD4(sh_d, b) * rf - 8.0f * LD4(sh_s, b));+ }+#undef LD4+ barrier(CLK_LOCAL_MEM_FENCE);+ }++ if (!row_valid) {+ return;+ }++ // dst is [token, feature] row-major (stride m): dst[col*m + row].+ #pragma unroll+ for (int g = 0; g < NGROUPS; ++g) {+ const uint b = (uint)(g * 4);+ const float4 a = acc[g];+ const uint c0 = col_base + b;+ if (c0 + 0 < (uint)n_no_padding) dst[(c0 + 0) * (uint)m + row] = a.s0;+ if (c0 + 1 < (uint)n_no_padding) dst[(c0 + 1) * (uint)m + row] = a.s1;+ if (c0 + 2 < (uint)n_no_padding) dst[(c0 + 2) * (uint)m + row] = a.s2;+ if (c0 + 3 < (uint)n_no_padding) dst[(c0 + 3) * (uint)m + row] = a.s3;+ }+#undef NGROUPS+}diff --git a/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q4_k_q8_1_dp4a.cl b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q4_k_q8_1_dp4a.clnew file mode 100644index 0000000000..8656180149--- /dev/null+++ b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q4_k_q8_1_dp4a.cl@@ -0,0 +1,281 @@+#pragma OPENCL EXTENSION cl_khr_fp16 : enable+#pragma OPENCL EXTENSION cl_khr_subgroups : enable+#ifdef cl_khr_integer_dot_product+#pragma OPENCL EXTENSION cl_khr_integer_dot_product : enable+#endif++#ifndef TILESIZE_N+#define TILESIZE_N 32+#endif+#define QK_K 256+#define K_SCALE_SIZE 12++inline void get_scale_min_k4(+ int j,+ global const uchar * q,+ uchar * d,+ uchar * m,+ uchar mask_d6,+ uchar mask_d4,+ uchar mask_hi2+) {+ if (j < 4) {+ *d = q[j] & mask_d6;+ *m = q[j+4] & mask_d6;+ } else {+ *d = (q[j+4] & mask_d4) | ((q[j-4] & mask_hi2) >> 2);+ *m = ((q[j+4] >> 4) & mask_d4) | ((q[j] & mask_hi2) >> 2);+ }+}++// Expand the 4 nibbles in the low 16 bits of `u` into 4 bytes (one nibble per+// byte, value 0..15), packed for the int8 dp4a.+#define EXP4(u) ( ((uint)((u) & 0x000Fu)) | \+ (((uint)((u) & 0x00F0u)) << 4) | \+ (((uint)((u) & 0x0F00u)) << 8) | \+ (((uint)((u) & 0xF000u)) << 12) )++// 32-K dp4a dot of one token's int8 activations (8 packed uints in lm) against the+// row's 8 packed weight uints. qw passed by value as a uint8 (register), not an array.+inline int dot8_q8a(uint8 qw, __local const uint * a) {+ int r = 0;+ r = dot_acc_sat_4x8packed_ss_int(qw.s0, a[0], r);+ r = dot_acc_sat_4x8packed_ss_int(qw.s1, a[1], r);+ r = dot_acc_sat_4x8packed_ss_int(qw.s2, a[2], r);+ r = dot_acc_sat_4x8packed_ss_int(qw.s3, a[3], r);+ r = dot_acc_sat_4x8packed_ss_int(qw.s4, a[4], r);+ r = dot_acc_sat_4x8packed_ss_int(qw.s5, a[5], r);+ r = dot_acc_sat_4x8packed_ss_int(qw.s6, a[6], r);+ r = dot_acc_sat_4x8packed_ss_int(qw.s7, a[7], r);+ return r;+}++__attribute__((qcom_wave_pair_mode(1)))+kernel void kernel_gemm_noshuffle_q4_k_q8_1_dp4a(+ __global const ushort * src0_q, // q4_K weights (noshuffle, packed nibbles)+ __global const uchar * src0_s, // 6-bit scale/min codes+ __global const half * src0_d, // per-superblock scale+ __global const half * src0_dm, // per-superblock min+ __global const uint * src1_qa, // q8_1 activations int8 (as uint, 4/elem) [N, K]+ __global const half * src1_da, // q8_1 per-block scale [N, K/32]+ __global const half * src1_sa, // q8_1 per-block sum*d [N, K/32]+ __global float * dst,+ ulong offsetd,+ int m, // output features (rows)+ int n_no_padding, // tokens (cols)+ int k, // K (== ne00)+ uchar mask_d6,+ uchar mask_d4,+ uchar mask_hi2+) {+ dst = (global float *)((global char *)dst + offsetd);++ const uint lid = get_local_id(0); // 0..63 -> row within the M-tile+ const uint block_id_m = get_global_id(1);+ const uint block_id_n = get_global_id(2);++ const uint row = block_id_m * 64 + lid;+ const uint col_base = block_id_n * TILESIZE_N;+ const bool row_valid = row < (uint)m;+ const uint rrow = row_valid ? row : 0; // clamp OOB rows; their writes are masked++ const uint num_superblocks = (uint)k / QK_K;+ const uint k_u = (uint)k >> 2; // K in uint (int8x4) units+ const uint k_b = (uint)k >> 5; // blocks-of-32 along K++ __local uint sh_qa[TILESIZE_N][8];+ __local half sh_d[TILESIZE_N];+ __local half sh_s[TILESIZE_N];++ // One float4 vector-register accumulator per group of 4 tokens (NGROUPS = TILESIZE_N/4).+#define NGROUPS (TILESIZE_N / 4)+ float4 acc[NGROUPS];+ #pragma unroll+ for (int g = 0; g < NGROUPS; ++g) { acc[g] = (float4)(0.0f); }++ for (uint step = 0; step < (uint)k; step += 32) {+ const uint sub = step >> 5;+ const uint sb_idx = step / QK_K;+ const uint sub_idx = sub & 7;++ // weight scale/min for this WI's row, this subblock+ const float dd = (float)src0_d [rrow + sb_idx * m];+ const float dmm = (float)src0_dm[rrow + sb_idx * m];+ global const uchar * sc = src0_s + rrow * num_superblocks * K_SCALE_SIZE + sb_idx * K_SCALE_SIZE;+ uchar sv, mn;+ get_scale_min_k4(sub_idx, sc, &sv, &mn, mask_d6, mask_d4, mask_hi2);+ const float scale = dd * (float)sv;+ const float minv = dmm * (float)mn;++ // repack this row's 32 weight nibbles into 8 dp4a uints. The packed q4_K+ // layout stores one ushort = 4 consecutive-K nibbles for a row at+ // src0_q[row + (K_group)*m], K_group = step/4 + u.+ const uint wbase = rrow + (step >> 2) * (uint)m;+ uint8 qw;+ qw.s0 = EXP4(src0_q[wbase + 0 * m]);+ qw.s1 = EXP4(src0_q[wbase + 1 * m]);+ qw.s2 = EXP4(src0_q[wbase + 2 * m]);+ qw.s3 = EXP4(src0_q[wbase + 3 * m]);+ qw.s4 = EXP4(src0_q[wbase + 4 * m]);+ qw.s5 = EXP4(src0_q[wbase + 5 * m]);+ qw.s6 = EXP4(src0_q[wbase + 6 * m]);+ qw.s7 = EXP4(src0_q[wbase + 7 * m]);++ // cooperatively stage the 32-token x 32-K int8 activations to lm+ for (uint idx = lid; idx < TILESIZE_N * 8; idx += 64) {+ const uint t = idx >> 3;+ const uint u = idx & 7;+ const uint c = col_base + t;+ sh_qa[t][u] = (c < (uint)n_no_padding) ? src1_qa[c * k_u + (step >> 2) + u] : 0u;+ }+ if (lid < TILESIZE_N) {+ const uint c = col_base + lid;+ sh_d[lid] = (c < (uint)n_no_padding) ? src1_da[c * k_b + sub] : (half)0;+ sh_s[lid] = (c < (uint)n_no_padding) ? src1_sa[c * k_b + sub] : (half)0;+ }+ barrier(CLK_LOCAL_MEM_FENCE);++#define LD4(arr, b) ((float4)((float)arr[(b)+0], (float)arr[(b)+1], (float)arr[(b)+2], (float)arr[(b)+3]))+ #pragma unroll+ for (int g = 0; g < NGROUPS; ++g) {+ const int b = g * 4;+ float4 rf;+ rf.s0 = (float)dot8_q8a(qw, sh_qa[b+0]); rf.s1 = (float)dot8_q8a(qw, sh_qa[b+1]);+ rf.s2 = (float)dot8_q8a(qw, sh_qa[b+2]); rf.s3 = (float)dot8_q8a(qw, sh_qa[b+3]);+ acc[g] += scale * LD4(sh_d, b) * rf - minv * LD4(sh_s, b);+ }+#undef LD4+ barrier(CLK_LOCAL_MEM_FENCE);+ }++ if (!row_valid) {+ return;+ }++ // dst is [token, feature] row-major (stride m): dst[col*m + row]. Scatter each+ // lane with a per-token padding guard (dst is non-contiguous in token).+ #pragma unroll+ for (int g = 0; g < NGROUPS; ++g) {+ const uint b = (uint)(g * 4);+ const float4 a = acc[g];+ const uint c0 = col_base + b;+ if (c0 + 0 < (uint)n_no_padding) dst[(c0 + 0) * (uint)m + row] = a.s0;+ if (c0 + 1 < (uint)n_no_padding) dst[(c0 + 1) * (uint)m + row] = a.s1;+ if (c0 + 2 < (uint)n_no_padding) dst[(c0 + 2) * (uint)m + row] = a.s2;+ if (c0 + 3 < (uint)n_no_padding) dst[(c0 + 3) * (uint)m + row] = a.s3;+ }+#undef NGROUPS+}++__attribute__((qcom_wave_pair_mode(1)))+kernel void kernel_gemm_noshuffle_q4_k_q8_1_dp4a_wimg(+ __read_only image1d_buffer_t src0_q_img, // q4_K weights as uint32 texels (2 ushorts/texel)+ __global const uchar * src0_s, // 6-bit scale/min codes+ __global const half * src0_d, // per-superblock scale+ __global const half * src0_dm, // per-superblock min+ __global const uint * src1_qa, // q8_1 activations int8 (as uint, 4/elem) [N, K]+ __global const half * src1_da, // q8_1 per-block scale [N, K/32]+ __global const half * src1_sa, // q8_1 per-block sum*d [N, K/32]+ __global float * dst,+ ulong offsetd,+ int m, // output features (rows)+ int n_no_padding, // tokens (cols)+ int k, // K (== ne00)+ uchar mask_d6,+ uchar mask_d4,+ uchar mask_hi2+) {+ dst = (global float *)((global char *)dst + offsetd);++ const uint lid = get_local_id(0); // 0..63 -> row within the M-tile+ const uint block_id_m = get_global_id(1);+ const uint block_id_n = get_global_id(2);++ const uint row = block_id_m * 64 + lid;+ const uint col_base = block_id_n * TILESIZE_N;+ const bool row_valid = row < (uint)m;+ const uint rrow = row_valid ? row : 0; // clamp OOB rows; their writes are masked++ // Constant per WI: the ushort the row needs always sits in the same half of+ // its uint32 texel (m even => index parity == rrow parity). Hoist the shift.+ const uint sel = (rrow & 1u) * 16u;++ const uint k_u = (uint)k >> 2; // K in uint (int8x4) units+ const uint k_b = (uint)k >> 5; // blocks-of-32 along K+ const uint num_superblocks = (uint)k / QK_K;++ __local uint sh_qa[TILESIZE_N][8];+ __local half sh_d[TILESIZE_N];+ __local half sh_s[TILESIZE_N];++#define NGROUPS (TILESIZE_N / 4)+ float4 acc[NGROUPS];+ #pragma unroll+ for (int g = 0; g < NGROUPS; ++g) acc[g] = (float4)(0.0f);++ for (uint step = 0; step < (uint)k; step += 32) {+ const uint sub = step >> 5;+ const uint sb_idx = step / QK_K;+ const uint sub_idx = sub & 7;++ const float dd = (float)src0_d [rrow + sb_idx * m];+ const float dmm = (float)src0_dm[rrow + sb_idx * m];+ global const uchar * sc = src0_s + rrow * num_superblocks * K_SCALE_SIZE + sb_idx * K_SCALE_SIZE;+ uchar sv, mn;+ get_scale_min_k4(sub_idx, sc, &sv, &mn, mask_d6, mask_d4, mask_hi2);+ const float scale = dd * (float)sv;+ const float minv = dmm * (float)mn;++ const uint wbase = rrow + (step >> 2) * (uint)m;+ uint8 qw;+ qw.s0 = EXP4(read_imageui(src0_q_img, (int)((wbase + 0 * m) >> 1)).x >> sel);+ qw.s1 = EXP4(read_imageui(src0_q_img, (int)((wbase + 1 * m) >> 1)).x >> sel);+ qw.s2 = EXP4(read_imageui(src0_q_img, (int)((wbase + 2 * m) >> 1)).x >> sel);+ qw.s3 = EXP4(read_imageui(src0_q_img, (int)((wbase + 3 * m) >> 1)).x >> sel);+ qw.s4 = EXP4(read_imageui(src0_q_img, (int)((wbase + 4 * m) >> 1)).x >> sel);+ qw.s5 = EXP4(read_imageui(src0_q_img, (int)((wbase + 5 * m) >> 1)).x >> sel);+ qw.s6 = EXP4(read_imageui(src0_q_img, (int)((wbase + 6 * m) >> 1)).x >> sel);+ qw.s7 = EXP4(read_imageui(src0_q_img, (int)((wbase + 7 * m) >> 1)).x >> sel);++ for (uint idx = lid; idx < TILESIZE_N * 8; idx += 64) {+ const uint t = idx >> 3;+ const uint u = idx & 7;+ const uint c = col_base + t;+ sh_qa[t][u] = (c < (uint)n_no_padding) ? src1_qa[c * k_u + (step >> 2) + u] : 0u;+ }+ if (lid < TILESIZE_N) {+ const uint c = col_base + lid;+ sh_d[lid] = (c < (uint)n_no_padding) ? src1_da[c * k_b + sub] : (half)0;+ sh_s[lid] = (c < (uint)n_no_padding) ? src1_sa[c * k_b + sub] : (half)0;+ }+ barrier(CLK_LOCAL_MEM_FENCE);++#define LD4(arr, b) ((float4)((float)arr[(b)+0], (float)arr[(b)+1], (float)arr[(b)+2], (float)arr[(b)+3]))+ #pragma unroll+ for (int g = 0; g < NGROUPS; ++g) {+ const int b = g * 4;+ float4 rf;+ rf.s0 = (float)dot8_q8a(qw, sh_qa[b+0]); rf.s1 = (float)dot8_q8a(qw, sh_qa[b+1]);+ rf.s2 = (float)dot8_q8a(qw, sh_qa[b+2]); rf.s3 = (float)dot8_q8a(qw, sh_qa[b+3]);+ acc[g] += scale * LD4(sh_d, b) * rf - minv * LD4(sh_s, b);+ }+#undef LD4+ barrier(CLK_LOCAL_MEM_FENCE);+ }++ if (!row_valid) {+ return;+ }++ #pragma unroll+ for (int g = 0; g < NGROUPS; ++g) {+ const uint b = (uint)(g * 4);+ const float4 a = acc[g];+ const uint c0 = col_base + b;+ if (c0 + 0 < (uint)n_no_padding) dst[(c0 + 0) * (uint)m + row] = a.s0;+ if (c0 + 1 < (uint)n_no_padding) dst[(c0 + 1) * (uint)m + row] = a.s1;+ if (c0 + 2 < (uint)n_no_padding) dst[(c0 + 2) * (uint)m + row] = a.s2;+ if (c0 + 3 < (uint)n_no_padding) dst[(c0 + 3) * (uint)m + row] = a.s3;+ }+#undef NGROUPS+}diff --git a/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q5_0_q8_1_dp4a.cl b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q5_0_q8_1_dp4a.clnew file mode 100644index 0000000000..4d1c6bdbcb--- /dev/null+++ b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q5_0_q8_1_dp4a.cl@@ -0,0 +1,235 @@+#pragma OPENCL EXTENSION cl_khr_fp16 : enable+#pragma OPENCL EXTENSION cl_khr_subgroups : enable+#ifdef cl_khr_integer_dot_product+#pragma OPENCL EXTENSION cl_khr_integer_dot_product : enable+#endif++// Weight layout+// src0_qs[row + (k/4)*m] ushort = 4 low nibbles (K = 4*grp .. +3)+// src0_qh[row + (k/8)*m] uchar = 8 high bits (one per element)+// src0_d [row + (k/32)*m] half = per-32-block scale++#define TILESIZE_N 32++// 4 nibbles in low 16 bits of u -> 4 bytes (value 0..15)+#define EXP4(u) ( ((uint)((u) & 0x000Fu)) | \+ (((uint)((u) & 0x00F0u)) << 4) | \+ (((uint)((u) & 0x0F00u)) << 8) | \+ (((uint)((u) & 0xF000u)) << 12) )+// 4 high bits (one per element, in bits 0..3 of h) -> bit4 of each of 4 bytes+#define EXP1(h) ( (((uint)((h) & 0x1u)) << 4) | \+ (((uint)((h) & 0x2u)) << 11) | \+ (((uint)((h) & 0x4u)) << 18) | \+ (((uint)((h) & 0x8u)) << 25) )++inline int dot8_q8a(uint8 qw, __local const uint * a) {+ int r = 0;+ r = dot_acc_sat_4x8packed_ss_int(qw.s0, a[0], r);+ r = dot_acc_sat_4x8packed_ss_int(qw.s1, a[1], r);+ r = dot_acc_sat_4x8packed_ss_int(qw.s2, a[2], r);+ r = dot_acc_sat_4x8packed_ss_int(qw.s3, a[3], r);+ r = dot_acc_sat_4x8packed_ss_int(qw.s4, a[4], r);+ r = dot_acc_sat_4x8packed_ss_int(qw.s5, a[5], r);+ r = dot_acc_sat_4x8packed_ss_int(qw.s6, a[6], r);+ r = dot_acc_sat_4x8packed_ss_int(qw.s7, a[7], r);+ return r;+}++__attribute__((qcom_wave_pair_mode(1)))+kernel void kernel_gemm_noshuffle_q5_0_q8_1_dp4a(+ __global const ushort * src0_qs, // q5_0 low nibbles (4/ushort, feature-major)+ __global const uchar * src0_qh, // q5_0 high-bit plane (8/uchar, feature-major)+ __global const half * src0_d, // per-32-block scale, feature-major+ __global const uint * src1_qa, // q8_1 activations int8 (as uint, 4/elem) [N, K]+ __global const half * src1_da, // q8_1 per-block scale [N, K/32]+ __global const half * src1_sa, // q8_1 per-block sum*d [N, K/32]+ __global float * dst,+ ulong offsetd,+ int m, // output features (rows)+ int n_no_padding, // tokens (cols)+ int k // K (== ne00)+) {+ dst = (global float *)((global char *)dst + offsetd);++ const uint lid = get_local_id(0); // 0..63 -> row within the M-tile+ const uint block_id_m = get_global_id(1);+ const uint block_id_n = get_global_id(2);++ const uint row = block_id_m * 64 + lid;+ const uint col_base = block_id_n * TILESIZE_N;+ const bool row_valid = row < (uint)m;+ const uint rrow = row_valid ? row : 0; // clamp OOB rows; their writes are masked++ const uint k_u = (uint)k >> 2; // K in uint (int8x4) units+ const uint k_b = (uint)k >> 5; // blocks-of-32 along K++ __local uint sh_qa[TILESIZE_N][8];+ __local half sh_d[TILESIZE_N];+ __local half sh_s[TILESIZE_N];++#define NGROUPS (TILESIZE_N / 4)+ float4 acc[NGROUPS];+ #pragma unroll+ for (int g = 0; g < NGROUPS; ++g) acc[g] = (float4)(0.0f);++ for (uint step = 0; step < (uint)k; step += 32) {+ const uint sub = step >> 5;++ const float d_w = (float)src0_d[rrow + sub * (uint)m];+ const float minv = d_w * 16.0f; // -16 centering -> subtract via q8_1 sum++ // 8 weight uints (32 elements) for this row, this 32-block.+ // nibbles: src0_qs[row + (step/4 + u)*m]; high bits: src0_qh[row + (step/8 + u/2)*m],+ // 4-bit group selected by (u&1)*4.+ const uint qsbase = rrow + (step >> 2) * (uint)m;+ const uint qhbase = rrow + (step >> 3) * (uint)m;+ uint8 qw;+ #define QW(u) (EXP4(src0_qs[qsbase + (u) * m]) | \+ EXP1((uint)(src0_qh[qhbase + ((u) >> 1) * m] >> (((u) & 1u) * 4u)) & 0xFu))+ qw.s0 = QW(0); qw.s1 = QW(1); qw.s2 = QW(2); qw.s3 = QW(3);+ qw.s4 = QW(4); qw.s5 = QW(5); qw.s6 = QW(6); qw.s7 = QW(7);+ #undef QW++ // cooperatively stage the 32-token x 32-K int8 activations to lm+ for (uint idx = lid; idx < TILESIZE_N * 8; idx += 64) {+ const uint t = idx >> 3;+ const uint u = idx & 7;+ const uint c = col_base + t;+ sh_qa[t][u] = (c < (uint)n_no_padding) ? src1_qa[c * k_u + (step >> 2) + u] : 0u;+ }+ if (lid < TILESIZE_N) {+ const uint c = col_base + lid;+ sh_d[lid] = (c < (uint)n_no_padding) ? src1_da[c * k_b + sub] : (half)0;+ sh_s[lid] = (c < (uint)n_no_padding) ? src1_sa[c * k_b + sub] : (half)0;+ }+ barrier(CLK_LOCAL_MEM_FENCE);++#define LD4(arr, b) ((float4)((float)arr[(b)+0], (float)arr[(b)+1], (float)arr[(b)+2], (float)arr[(b)+3]))+ #pragma unroll+ for (int g = 0; g < NGROUPS; ++g) {+ const int b = g * 4;+ float4 rf;+ rf.s0 = (float)dot8_q8a(qw, sh_qa[b+0]); rf.s1 = (float)dot8_q8a(qw, sh_qa[b+1]);+ rf.s2 = (float)dot8_q8a(qw, sh_qa[b+2]); rf.s3 = (float)dot8_q8a(qw, sh_qa[b+3]);+ acc[g] += d_w * LD4(sh_d, b) * rf - minv * LD4(sh_s, b);+ }+#undef LD4+ barrier(CLK_LOCAL_MEM_FENCE);+ }++ if (!row_valid) {+ return;+ }++ #pragma unroll+ for (int g = 0; g < NGROUPS; ++g) {+ const uint b = (uint)(g * 4);+ const float4 a = acc[g];+ const uint c0 = col_base + b;+ if (c0 + 0 < (uint)n_no_padding) dst[(c0 + 0) * (uint)m + row] = a.s0;+ if (c0 + 1 < (uint)n_no_padding) dst[(c0 + 1) * (uint)m + row] = a.s1;+ if (c0 + 2 < (uint)n_no_padding) dst[(c0 + 2) * (uint)m + row] = a.s2;+ if (c0 + 3 < (uint)n_no_padding) dst[(c0 + 3) * (uint)m + row] = a.s3;+ }+#undef NGROUPS+}++__attribute__((qcom_wave_pair_mode(1)))+kernel void kernel_gemm_noshuffle_q5_0_q8_1_dp4a_wimg(+ __read_only image1d_buffer_t src0_qs_img, // q5_0 low nibbles as uint32 texels (2 ushorts/texel)+ __global const uchar * src0_qh,+ __global const half * src0_d,+ __global const uint * src1_qa,+ __global const half * src1_da,+ __global const half * src1_sa,+ __global float * dst,+ ulong offsetd,+ int m,+ int n_no_padding,+ int k+) {+ dst = (global float *)((global char *)dst + offsetd);++ const uint lid = get_local_id(0);+ const uint block_id_m = get_global_id(1);+ const uint block_id_n = get_global_id(2);++ const uint row = block_id_m * 64 + lid;+ const uint col_base = block_id_n * TILESIZE_N;+ const bool row_valid = row < (uint)m;+ const uint rrow = row_valid ? row : 0;++ const uint sel = (rrow & 1u) * 16u; // constant per WI: qs ushort half in its uint32 texel++ const uint k_u = (uint)k >> 2;+ const uint k_b = (uint)k >> 5;++ __local uint sh_qa[TILESIZE_N][8];+ __local half sh_d[TILESIZE_N];+ __local half sh_s[TILESIZE_N];++#define NGROUPS (TILESIZE_N / 4)+ float4 acc[NGROUPS];+ #pragma unroll+ for (int g = 0; g < NGROUPS; ++g) acc[g] = (float4)(0.0f);++ for (uint step = 0; step < (uint)k; step += 32) {+ const uint sub = step >> 5;++ const float d_w = (float)src0_d[rrow + sub * (uint)m];+ const float minv = d_w * 16.0f;++ const uint qsbase = rrow + (step >> 2) * (uint)m; // ushort index+ const uint qhbase = rrow + (step >> 3) * (uint)m;+ uint8 qw;+ // qs ushort via texture: uint32 texel = ushort_index>>1, half = sel.+ #define QSU(u) ((read_imageui(src0_qs_img, (int)((qsbase + (u) * m) >> 1)).x >> sel) & 0xFFFFu)+ #define QW(u) (EXP4(QSU(u)) | \+ EXP1((uint)(src0_qh[qhbase + ((u) >> 1) * m] >> (((u) & 1u) * 4u)) & 0xFu))+ qw.s0 = QW(0); qw.s1 = QW(1); qw.s2 = QW(2); qw.s3 = QW(3);+ qw.s4 = QW(4); qw.s5 = QW(5); qw.s6 = QW(6); qw.s7 = QW(7);+ #undef QW+ #undef QSU++ for (uint idx = lid; idx < TILESIZE_N * 8; idx += 64) {+ const uint t = idx >> 3;+ const uint u = idx & 7;+ const uint c = col_base + t;+ sh_qa[t][u] = (c < (uint)n_no_padding) ? src1_qa[c * k_u + (step >> 2) + u] : 0u;+ }+ if (lid < TILESIZE_N) {+ const uint c = col_base + lid;+ sh_d[lid] = (c < (uint)n_no_padding) ? src1_da[c * k_b + sub] : (half)0;+ sh_s[lid] = (c < (uint)n_no_padding) ? src1_sa[c * k_b + sub] : (half)0;+ }+ barrier(CLK_LOCAL_MEM_FENCE);++#define LD4(arr, b) ((float4)((float)arr[(b)+0], (float)arr[(b)+1], (float)arr[(b)+2], (float)arr[(b)+3]))+ #pragma unroll+ for (int g = 0; g < NGROUPS; ++g) {+ const int b = g * 4;+ float4 rf;+ rf.s0 = (float)dot8_q8a(qw, sh_qa[b+0]); rf.s1 = (float)dot8_q8a(qw, sh_qa[b+1]);+ rf.s2 = (float)dot8_q8a(qw, sh_qa[b+2]); rf.s3 = (float)dot8_q8a(qw, sh_qa[b+3]);+ acc[g] += d_w * LD4(sh_d, b) * rf - minv * LD4(sh_s, b);+ }+#undef LD4+ barrier(CLK_LOCAL_MEM_FENCE);+ }++ if (!row_valid) {+ return;+ }++ #pragma unroll+ for (int g = 0; g < NGROUPS; ++g) {+ const uint b = (uint)(g * 4);+ const float4 a = acc[g];+ const uint c0 = col_base + b;+ if (c0 + 0 < (uint)n_no_padding) dst[(c0 + 0) * (uint)m + row] = a.s0;+ if (c0 + 1 < (uint)n_no_padding) dst[(c0 + 1) * (uint)m + row] = a.s1;+ if (c0 + 2 < (uint)n_no_padding) dst[(c0 + 2) * (uint)m + row] = a.s2;+ if (c0 + 3 < (uint)n_no_padding) dst[(c0 + 3) * (uint)m + row] = a.s3;+ }+#undef NGROUPS+}diff --git a/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q5_k_q8_1_dp4a.cl b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q5_k_q8_1_dp4a.clnew file mode 100644index 0000000000..aaeed68f61--- /dev/null+++ b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q5_k_q8_1_dp4a.cl@@ -0,0 +1,164 @@+#pragma OPENCL EXTENSION cl_khr_fp16 : enable+#pragma OPENCL EXTENSION cl_khr_subgroups : enable+#ifdef cl_khr_integer_dot_product+#pragma OPENCL EXTENSION cl_khr_integer_dot_product : enable+#endif++#define TILESIZE_N 32+#define QK_K 256+#define K_SCALE_SIZE 12++inline void get_scale_min_k4(+ int j,+ global const uchar * q,+ uchar * d,+ uchar * m,+ uchar mask_d6,+ uchar mask_d4,+ uchar mask_hi2+) {+ if (j < 4) {+ *d = q[j] & mask_d6;+ *m = q[j+4] & mask_d6;+ } else {+ *d = (q[j+4] & mask_d4) | ((q[j-4] & mask_hi2) >> 2);+ *m = ((q[j+4] >> 4) & mask_d4) | ((q[j] & mask_hi2) >> 2);+ }+}++// 4 nibbles in the low 16 bits of `u` -> 4 bytes (value 0..15, bits 0-3).+#define EXP4(u) ( ((uint)((u) & 0x000Fu)) | \+ (((uint)((u) & 0x00F0u)) << 4) | \+ (((uint)((u) & 0x0F00u)) << 8) | \+ (((uint)((u) & 0xF000u)) << 12) )++// 4 high bits (one per element, in bits 0-3 of h) -> bit 4 of each of 4 bytes,+// so OR with EXP4 forms the 5-bit q5_K code 0..31.+#define EXP1(h) ( (((uint)((h) & 0x1u)) << 4) | \+ (((uint)((h) & 0x2u)) << 11) | \+ (((uint)((h) & 0x4u)) << 18) | \+ (((uint)((h) & 0x8u)) << 25) )++inline int dot8_q8a(uint8 qw, __local const uint * a) {+ int r = 0;+ r = dot_acc_sat_4x8packed_ss_int(qw.s0, a[0], r);+ r = dot_acc_sat_4x8packed_ss_int(qw.s1, a[1], r);+ r = dot_acc_sat_4x8packed_ss_int(qw.s2, a[2], r);+ r = dot_acc_sat_4x8packed_ss_int(qw.s3, a[3], r);+ r = dot_acc_sat_4x8packed_ss_int(qw.s4, a[4], r);+ r = dot_acc_sat_4x8packed_ss_int(qw.s5, a[5], r);+ r = dot_acc_sat_4x8packed_ss_int(qw.s6, a[6], r);+ r = dot_acc_sat_4x8packed_ss_int(qw.s7, a[7], r);+ return r;+}++__attribute__((qcom_wave_pair_mode(1)))+kernel void kernel_gemm_noshuffle_q5_k_q8_1_dp4a(+ __global const ushort * src0_q, // q5_K low nibbles (transposed, ushort = 4 nibbles)+ __global const uchar * src0_qh, // q5_K high bits (transposed, uchar = 8 elems/byte)+ __global const uchar * src0_s, // 6-bit scale/min codes [row][superblock][12]+ __global const half * src0_d, // per-superblock scale (transposed)+ __global const half * src0_dm, // per-superblock min (transposed)+ __global const uint * src1_qa, // q8_1 activations int8 (as uint, 4/elem) [N, K]+ __global const half * src1_da, // q8_1 per-block scale [N, K/32]+ __global const half * src1_sa, // q8_1 per-block sum*d [N, K/32]+ __global float * dst,+ ulong offsetd,+ int m, // output features (rows)+ int n_no_padding, // tokens (cols)+ int k, // K (== ne00)+ uchar mask_d6,+ uchar mask_d4,+ uchar mask_hi2+) {+ dst = (global float *)((global char *)dst + offsetd);++ const uint lid = get_local_id(0); // 0..63 -> row within the M-tile+ const uint block_id_m = get_global_id(1);+ const uint block_id_n = get_global_id(2);++ const uint row = block_id_m * 64 + lid;+ const uint col_base = block_id_n * TILESIZE_N;+ const bool row_valid = row < (uint)m;+ const uint rrow = row_valid ? row : 0;++ const uint num_superblocks = (uint)k / QK_K;+ const uint k_u = (uint)k >> 2;+ const uint k_b = (uint)k >> 5;++ __local uint sh_qa[TILESIZE_N][8];+ __local half sh_d[TILESIZE_N];+ __local half sh_s[TILESIZE_N];++#define NGROUPS (TILESIZE_N / 4)+ float4 acc[NGROUPS];+ #pragma unroll+ for (int g = 0; g < NGROUPS; ++g) acc[g] = (float4)(0.0f);++ for (uint step = 0; step < (uint)k; step += 32) {+ const uint sub = step >> 5;+ const uint sb_idx = step / QK_K;+ const uint sub_idx = sub & 7;++ const float dd = (float)src0_d [rrow + sb_idx * m];+ const float dmm = (float)src0_dm[rrow + sb_idx * m];+ global const uchar * sc = src0_s + rrow * num_superblocks * K_SCALE_SIZE + sb_idx * K_SCALE_SIZE;+ uchar sv, mn;+ get_scale_min_k4(sub_idx, sc, &sv, &mn, mask_d6, mask_d4, mask_hi2);+ const float scale = dd * (float)sv;+ const float minv = dmm * (float)mn;++ // repack this row's 32 weights (nibble | high-bit) into 8 dp4a uints.+ // ushort u -> 4 elements at K = step + u*4; its 4 high bits are nibble+ // (u&1) of qh byte (step/8 + u/2).+ const uint wbase = rrow + (step >> 2) * (uint)m;+ const uint qhbase = rrow + (step >> 3) * (uint)m;+ uint8 qw;+#define QWU(u) ( EXP4((uint)src0_q[wbase + (uint)(u) * m]) \+ | EXP1( (uint)((src0_qh[qhbase + (uint)((u) >> 1) * m] >> (((u) & 1) * 4)) & 0x0Fu) ) )+ qw.s0 = QWU(0); qw.s1 = QWU(1); qw.s2 = QWU(2); qw.s3 = QWU(3);+ qw.s4 = QWU(4); qw.s5 = QWU(5); qw.s6 = QWU(6); qw.s7 = QWU(7);+#undef QWU++ for (uint idx = lid; idx < TILESIZE_N * 8; idx += 64) {+ const uint t = idx >> 3;+ const uint u = idx & 7;+ const uint c = col_base + t;+ sh_qa[t][u] = (c < (uint)n_no_padding) ? src1_qa[c * k_u + (step >> 2) + u] : 0u;+ }+ if (lid < TILESIZE_N) {+ const uint c = col_base + lid;+ sh_d[lid] = (c < (uint)n_no_padding) ? src1_da[c * k_b + sub] : (half)0;+ sh_s[lid] = (c < (uint)n_no_padding) ? src1_sa[c * k_b + sub] : (half)0;+ }+ barrier(CLK_LOCAL_MEM_FENCE);++#define LD4(arr, b) ((float4)((float)arr[(b)+0], (float)arr[(b)+1], (float)arr[(b)+2], (float)arr[(b)+3]))+ #pragma unroll+ for (int g = 0; g < NGROUPS; ++g) {+ const int b = g * 4;+ float4 rf;+ rf.s0 = (float)dot8_q8a(qw, sh_qa[b+0]); rf.s1 = (float)dot8_q8a(qw, sh_qa[b+1]);+ rf.s2 = (float)dot8_q8a(qw, sh_qa[b+2]); rf.s3 = (float)dot8_q8a(qw, sh_qa[b+3]);+ acc[g] += scale * LD4(sh_d, b) * rf - minv * LD4(sh_s, b);+ }+#undef LD4+ barrier(CLK_LOCAL_MEM_FENCE);+ }++ if (!row_valid) {+ return;+ }++ #pragma unroll+ for (int g = 0; g < NGROUPS; ++g) {+ const uint b = (uint)(g * 4);+ const float4 a = acc[g];+ const uint c0 = col_base + b;+ if (c0 + 0 < (uint)n_no_padding) dst[(c0 + 0) * (uint)m + row] = a.s0;+ if (c0 + 1 < (uint)n_no_padding) dst[(c0 + 1) * (uint)m + row] = a.s1;+ if (c0 + 2 < (uint)n_no_padding) dst[(c0 + 2) * (uint)m + row] = a.s2;+ if (c0 + 3 < (uint)n_no_padding) dst[(c0 + 3) * (uint)m + row] = a.s3;+ }+#undef NGROUPS+}diff --git a/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q6_k_q8_1_dp4a.cl b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q6_k_q8_1_dp4a.clnew file mode 100644index 0000000000..382d79fdda--- /dev/null+++ b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q6_k_q8_1_dp4a.cl@@ -0,0 +1,144 @@+#pragma OPENCL EXTENSION cl_khr_fp16 : enable+#pragma OPENCL EXTENSION cl_khr_subgroups : enable+#ifdef cl_khr_integer_dot_product+#pragma OPENCL EXTENSION cl_khr_integer_dot_product : enable+#endif++#define TILESIZE_N 32+#define QK_K 256++// 4 nibbles in the low 16 bits of `u` -> 4 bytes (value 0..15, in bits 0-3).+#define EXP4(u) ( ((uint)((u) & 0x000Fu)) | \+ (((uint)((u) & 0x00F0u)) << 4) | \+ (((uint)((u) & 0x0F00u)) << 8) | \+ (((uint)((u) & 0xF000u)) << 12) )++// 4 2-bit highs in byte `b` -> 4 bytes, value 0..3 in bits 4-5 (pre-multiplied+// by 16 so it ORs with the EXP4 nibble to form q6 in 0..63).+#define EXP2(b) ( (((uint)((b) & 0x03u)) << 4) | \+ (((uint)((b) & 0x0Cu)) << 10) | \+ (((uint)((b) & 0x30u)) << 16) | \+ (((uint)((b) & 0xC0u)) << 22) )++// q6 (0..63, bits 0-5 of each byte) -> (q6-32) as a signed int8 per byte.+inline uint SIGN6(uint q6p) {+ uint x = q6p ^ 0x20202020u;+ uint s = x & 0x20202020u;+ return x | (s << 1) | (s << 2);+}++// 16-K dp4a dot: 4 packed weight uints against 4 packed int8 activation uints.+inline int dot4_q8a(uint w0, uint w1, uint w2, uint w3,+ uint a0, uint a1, uint a2, uint a3) {+ int r = 0;+ r = dot_acc_sat_4x8packed_ss_int(w0, a0, r);+ r = dot_acc_sat_4x8packed_ss_int(w1, a1, r);+ r = dot_acc_sat_4x8packed_ss_int(w2, a2, r);+ r = dot_acc_sat_4x8packed_ss_int(w3, a3, r);+ return r;+}++__attribute__((qcom_wave_pair_mode(1)))+kernel void kernel_gemm_noshuffle_q6_k_q8_1_dp4a(+ __global const ushort * src0_ql, // q6_K low nibbles (noshuffle)+ __global const uchar * src0_qh, // q6_K high 2-bit (uchar, 4 highs/elem)+ __global const ushort * src0_s, // int8 scale codes (2 chars/ushort, per 16)+ __global const half * src0_d, // per-superblock scale+ __global const uint * src1_qa, // q8_1 activations int8 (as uint, 4/elem) [N, K]+ __global const half * src1_da, // q8_1 per-block scale [N, K/32]+ __global float * dst,+ ulong offsetd,+ int m, // output features (rows)+ int n_no_padding, // tokens (cols)+ int k // K (== ne00)+) {+ dst = (global float *)((global char *)dst + offsetd);++ const uint lid = get_local_id(0); // 0..63 -> row within the M-tile+ const uint block_id_m = get_global_id(1);+ const uint block_id_n = get_global_id(2);++ const uint row = block_id_m * 64 + lid;+ const uint col_base = block_id_n * TILESIZE_N;+ const bool row_valid = row < (uint)m;+ const uint rrow = row_valid ? row : 0; // clamp OOB rows; their writes are masked++ const uint k_u = (uint)k >> 2; // K in uint (int8x4) units+ const uint k_b = (uint)k >> 5; // blocks-of-32 along K++ __local uint sh_qa[TILESIZE_N][8];+ __local half sh_d[TILESIZE_N];++#define NGROUPS (TILESIZE_N / 4)+ float4 acc[NGROUPS];+ #pragma unroll+ for (int g = 0; g < NGROUPS; ++g) acc[g] = (float4)(0.0f);++ for (uint step = 0; step < (uint)k; step += 32) {+ const uint sub = step >> 5; // 32-block index along K+ const uint sb_idx = step / QK_K; // superblock index++ // q6_K superblock scale + the two int8 sub-scales spanning this 32-block+ const float dd = (float)src0_d[rrow + sb_idx * m];+ const char2 sc = as_char2(src0_s[rrow + sub * m]);+ const float scale0 = dd * (float)sc.s0; // K step..step+15+ const float scale1 = dd * (float)sc.s1; // K step+16..step+31++ // repack this row's 32 weights into 8 dp4a uints (4 K each). ql ushort ++ // qh uchar are co-located at src0_*[row + (step/4 + u)*m].+ const uint wbase = rrow + (step >> 2) * (uint)m;+ uint qw[8];+ #pragma unroll+ for (int u = 0; u < 8; ++u) {+ const uint o = wbase + (uint)u * (uint)m;+ qw[u] = SIGN6(EXP4((uint)src0_ql[o]) | EXP2((uint)src0_qh[o]));+ }++ // cooperatively stage the 32-token x 32-K int8 activations + scale+ for (uint idx = lid; idx < TILESIZE_N * 8; idx += 64) {+ const uint t = idx >> 3;+ const uint u = idx & 7;+ const uint c = col_base + t;+ sh_qa[t][u] = (c < (uint)n_no_padding) ? src1_qa[c * k_u + (step >> 2) + u] : 0u;+ }+ if (lid < TILESIZE_N) {+ const uint c = col_base + lid;+ sh_d[lid] = (c < (uint)n_no_padding) ? src1_da[c * k_b + sub] : (half)0;+ }+ barrier(CLK_LOCAL_MEM_FENCE);++ #pragma unroll+ for (int g = 0; g < NGROUPS; ++g) {+ const int b = g * 4;+ float4 rf;+ #define DOT_TOK(j) { \+ __local const uint * a = sh_qa[b + (j)]; \+ const int raw1 = dot4_q8a(qw[0], qw[1], qw[2], qw[3], a[0], a[1], a[2], a[3]); \+ const int raw2 = dot4_q8a(qw[4], qw[5], qw[6], qw[7], a[4], a[5], a[6], a[7]); \+ rf.s##j = scale0 * (float)raw1 + scale1 * (float)raw2; \+ }+ DOT_TOK(0); DOT_TOK(1); DOT_TOK(2); DOT_TOK(3);+ #undef DOT_TOK+ const float4 ad = (float4)((float)sh_d[b+0], (float)sh_d[b+1], (float)sh_d[b+2], (float)sh_d[b+3]);+ acc[g] += ad * rf;+ }+ barrier(CLK_LOCAL_MEM_FENCE);+ }++ if (!row_valid) {+ return;+ }++ // dst is [token, feature] row-major (stride m): dst[col*m + row].+ #pragma unroll+ for (int g = 0; g < NGROUPS; ++g) {+ const uint b = (uint)(g * 4);+ const float4 a = acc[g];+ const uint c0 = col_base + b;+ if (c0 + 0 < (uint)n_no_padding) dst[(c0 + 0) * (uint)m + row] = a.s0;+ if (c0 + 1 < (uint)n_no_padding) dst[(c0 + 1) * (uint)m + row] = a.s1;+ if (c0 + 2 < (uint)n_no_padding) dst[(c0 + 2) * (uint)m + row] = a.s2;+ if (c0 + 3 < (uint)n_no_padding) dst[(c0 + 3) * (uint)m + row] = a.s3;+ }+#undef NGROUPS+}diff --git a/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q8_0_q8_1_dp4a.cl b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q8_0_q8_1_dp4a.clnew file mode 100644index 0000000000..a481636c23--- /dev/null+++ b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q8_0_q8_1_dp4a.cl@@ -0,0 +1,212 @@+#pragma OPENCL EXTENSION cl_khr_fp16 : enable+#pragma OPENCL EXTENSION cl_khr_subgroups : enable+#ifdef cl_khr_integer_dot_product+#pragma OPENCL EXTENSION cl_khr_integer_dot_product : enable+#endif++// ne1<=8 keeps the f16 / bin small-batch path.++#define TILESIZE_N 32++// 32-K dp4a dot of one token's int8 activations (8 packed uints in lm) against+// 8 packed weight uints. q8_0 weights are already dp4a-format signed int8.+inline int dot8_q8a(uint8 qw, __local const uint * a) {+ int r = 0;+ r = dot_acc_sat_4x8packed_ss_int(qw.s0, a[0], r);+ r = dot_acc_sat_4x8packed_ss_int(qw.s1, a[1], r);+ r = dot_acc_sat_4x8packed_ss_int(qw.s2, a[2], r);+ r = dot_acc_sat_4x8packed_ss_int(qw.s3, a[3], r);+ r = dot_acc_sat_4x8packed_ss_int(qw.s4, a[4], r);+ r = dot_acc_sat_4x8packed_ss_int(qw.s5, a[5], r);+ r = dot_acc_sat_4x8packed_ss_int(qw.s6, a[6], r);+ r = dot_acc_sat_4x8packed_ss_int(qw.s7, a[7], r);+ return r;+}++__attribute__((qcom_wave_pair_mode(1)))+kernel void kernel_gemm_noshuffle_q8_0_q8_1_dp4a(+ __global const uint * src0_q, // q8_0 weights: signed int8, 4/uint, feature-major+ __global const half * src0_d, // per-32-block scale, feature-major [row + (k/32)*m]+ __global const uint * src1_qa, // q8_1 activations int8 (as uint, 4/elem) [N, K]+ __global const half * src1_da, // q8_1 per-block scale [N, K/32]+ __global float * dst,+ ulong offsetd,+ int m, // output features (rows)+ int n_no_padding, // tokens (cols)+ int k // K (== ne00)+) {+ dst = (global float *)((global char *)dst + offsetd);++ const uint lid = get_local_id(0); // 0..63 -> row within the M-tile+ const uint block_id_m = get_global_id(1);+ const uint block_id_n = get_global_id(2);++ const uint row = block_id_m * 64 + lid;+ const uint col_base = block_id_n * TILESIZE_N;+ const bool row_valid = row < (uint)m;+ const uint rrow = row_valid ? row : 0; // clamp OOB rows; their writes are masked++ const uint k_u = (uint)k >> 2; // K in uint (int8x4) units+ const uint k_b = (uint)k >> 5; // blocks-of-32 along K++ __local uint sh_qa[TILESIZE_N][8];+ __local half sh_d[TILESIZE_N];++#define NGROUPS (TILESIZE_N / 4)+ float4 acc[NGROUPS];+ #pragma unroll+ for (int g = 0; g < NGROUPS; ++g) acc[g] = (float4)(0.0f);++ for (uint step = 0; step < (uint)k; step += 32) {+ const uint sub = step >> 5;++ const float d_w = (float)src0_d[rrow + sub * (uint)m];++ // 8 weight uints (32 int8) for this row, this 32-block. Feature-major:+ // src0_q[row + (k/4 + u)*m], k/4 = step/4 (= step>>2).+ const uint wbase = rrow + (step >> 2) * (uint)m;+ uint8 qw;+ qw.s0 = src0_q[wbase + 0 * m];+ qw.s1 = src0_q[wbase + 1 * m];+ qw.s2 = src0_q[wbase + 2 * m];+ qw.s3 = src0_q[wbase + 3 * m];+ qw.s4 = src0_q[wbase + 4 * m];+ qw.s5 = src0_q[wbase + 5 * m];+ qw.s6 = src0_q[wbase + 6 * m];+ qw.s7 = src0_q[wbase + 7 * m];++ // cooperatively stage the 32-token x 32-K int8 activations to LDS+ for (uint idx = lid; idx < TILESIZE_N * 8; idx += 64) {+ const uint t = idx >> 3;+ const uint u = idx & 7;+ const uint c = col_base + t;+ sh_qa[t][u] = (c < (uint)n_no_padding) ? src1_qa[c * k_u + (step >> 2) + u] : 0u;+ }+ if (lid < TILESIZE_N) {+ const uint c = col_base + lid;+ sh_d[lid] = (c < (uint)n_no_padding) ? src1_da[c * k_b + sub] : (half)0;+ }+ barrier(CLK_LOCAL_MEM_FENCE);++#define LD4(arr, b) ((float4)((float)arr[(b)+0], (float)arr[(b)+1], (float)arr[(b)+2], (float)arr[(b)+3]))+ #pragma unroll+ for (int g = 0; g < NGROUPS; ++g) {+ const int b = g * 4;+ float4 rf;+ rf.s0 = (float)dot8_q8a(qw, sh_qa[b+0]); rf.s1 = (float)dot8_q8a(qw, sh_qa[b+1]);+ rf.s2 = (float)dot8_q8a(qw, sh_qa[b+2]); rf.s3 = (float)dot8_q8a(qw, sh_qa[b+3]);+ acc[g] += d_w * LD4(sh_d, b) * rf;+ }+#undef LD4+ barrier(CLK_LOCAL_MEM_FENCE);+ }++ if (!row_valid) {+ return;+ }++ // dst is [token, feature] row-major (stride m): dst[col*m + row].+ #pragma unroll+ for (int g = 0; g < NGROUPS; ++g) {+ const uint b = (uint)(g * 4);+ const float4 a = acc[g];+ const uint c0 = col_base + b;+ if (c0 + 0 < (uint)n_no_padding) dst[(c0 + 0) * (uint)m + row] = a.s0;+ if (c0 + 1 < (uint)n_no_padding) dst[(c0 + 1) * (uint)m + row] = a.s1;+ if (c0 + 2 < (uint)n_no_padding) dst[(c0 + 2) * (uint)m + row] = a.s2;+ if (c0 + 3 < (uint)n_no_padding) dst[(c0 + 3) * (uint)m + row] = a.s3;+ }+#undef NGROUPS+}++__attribute__((qcom_wave_pair_mode(1)))+kernel void kernel_gemm_noshuffle_q8_0_q8_1_dp4a_wimg(+ __read_only image1d_buffer_t src0_q_img, // q8_0 weights as uint32 texels (4 int8/texel)+ __global const half * src0_d,+ __global const uint * src1_qa,+ __global const half * src1_da,+ __global float * dst,+ ulong offsetd,+ int m,+ int n_no_padding,+ int k+) {+ dst = (global float *)((global char *)dst + offsetd);++ const uint lid = get_local_id(0);+ const uint block_id_m = get_global_id(1);+ const uint block_id_n = get_global_id(2);++ const uint row = block_id_m * 64 + lid;+ const uint col_base = block_id_n * TILESIZE_N;+ const bool row_valid = row < (uint)m;+ const uint rrow = row_valid ? row : 0;++ const uint k_u = (uint)k >> 2;+ const uint k_b = (uint)k >> 5;++ __local uint sh_qa[TILESIZE_N][8];+ __local half sh_d[TILESIZE_N];++#define NGROUPS (TILESIZE_N / 4)+ float4 acc[NGROUPS];+ #pragma unroll+ for (int g = 0; g < NGROUPS; ++g) acc[g] = (float4)(0.0f);++ for (uint step = 0; step < (uint)k; step += 32) {+ const uint sub = step >> 5;++ const float d_w = (float)src0_d[rrow + sub * (uint)m];++ const uint wbase = rrow + (step >> 2) * (uint)m;+ uint8 qw;+ qw.s0 = read_imageui(src0_q_img, (int)(wbase + 0 * m)).x;+ qw.s1 = read_imageui(src0_q_img, (int)(wbase + 1 * m)).x;+ qw.s2 = read_imageui(src0_q_img, (int)(wbase + 2 * m)).x;+ qw.s3 = read_imageui(src0_q_img, (int)(wbase + 3 * m)).x;+ qw.s4 = read_imageui(src0_q_img, (int)(wbase + 4 * m)).x;+ qw.s5 = read_imageui(src0_q_img, (int)(wbase + 5 * m)).x;+ qw.s6 = read_imageui(src0_q_img, (int)(wbase + 6 * m)).x;+ qw.s7 = read_imageui(src0_q_img, (int)(wbase + 7 * m)).x;++ for (uint idx = lid; idx < TILESIZE_N * 8; idx += 64) {+ const uint t = idx >> 3;+ const uint u = idx & 7;+ const uint c = col_base + t;+ sh_qa[t][u] = (c < (uint)n_no_padding) ? src1_qa[c * k_u + (step >> 2) + u] : 0u;+ }+ if (lid < TILESIZE_N) {+ const uint c = col_base + lid;+ sh_d[lid] = (c < (uint)n_no_padding) ? src1_da[c * k_b + sub] : (half)0;+ }+ barrier(CLK_LOCAL_MEM_FENCE);++#define LD4(arr, b) ((float4)((float)arr[(b)+0], (float)arr[(b)+1], (float)arr[(b)+2], (float)arr[(b)+3]))+ #pragma unroll+ for (int g = 0; g < NGROUPS; ++g) {+ const int b = g * 4;+ float4 rf;+ rf.s0 = (float)dot8_q8a(qw, sh_qa[b+0]); rf.s1 = (float)dot8_q8a(qw, sh_qa[b+1]);+ rf.s2 = (float)dot8_q8a(qw, sh_qa[b+2]); rf.s3 = (float)dot8_q8a(qw, sh_qa[b+3]);+ acc[g] += d_w * LD4(sh_d, b) * rf;+ }+#undef LD4+ barrier(CLK_LOCAL_MEM_FENCE);+ }++ if (!row_valid) {+ return;+ }++ #pragma unroll+ for (int g = 0; g < NGROUPS; ++g) {+ const uint b = (uint)(g * 4);+ const float4 a = acc[g];+ const uint c0 = col_base + b;+ if (c0 + 0 < (uint)n_no_padding) dst[(c0 + 0) * (uint)m + row] = a.s0;+ if (c0 + 1 < (uint)n_no_padding) dst[(c0 + 1) * (uint)m + row] = a.s1;+ if (c0 + 2 < (uint)n_no_padding) dst[(c0 + 2) * (uint)m + row] = a.s2;+ if (c0 + 3 < (uint)n_no_padding) dst[(c0 + 3) * (uint)m + row] = a.s3;+ }+#undef NGROUPS+}diff --git a/ggml/src/ggml-opencl/kernels/gemv_moe_mxfp4_f32_ns.cl b/ggml/src/ggml-opencl/kernels/gemv_moe_mxfp4_f32_ns.clindex 75129e20c6..ee8b94f446 100644--- a/ggml/src/ggml-opencl/kernels/gemv_moe_mxfp4_f32_ns.cl+++ b/ggml/src/ggml-opencl/kernels/gemv_moe_mxfp4_f32_ns.cl@@ -163,3 +163,95 @@ __kernel void kernel_gemv_moe_mxfp4_f32_ns(}}++__attribute__((qcom_reqd_sub_group_size("half")))+__kernel void kernel_gemv_moe_mxfp4_f32_ns_wimg(+ __read_only image1d_buffer_t src0_q,+ __global uchar * src0_e,+ __read_only image1d_buffer_t src1,+ __global uint * src2,+ __global float * dst,+ ulong offsetd,+ int ne00,+ int ne01,+ int ne11+) {+ uint i01 = get_global_id(0);+ uint i20 = get_global_id(2);+ uint sgid = get_local_id(1);+ uint slid = get_sub_group_local_id();++ if (i01 >= ne01) {+ return;+ }++ uint i11 = i20 % ne11;++ uint expert_id = src2[i20];+ uint expert_offset = expert_id * ne00 * ne01 / 32;++ __private float sum = 0.0f;++ for (uint ib00 = sgid; ib00 < (ne00 / QK_MXFP4); ib00 += N_SIMDGROUP) {++ uint4 regQ;+ uint block_offset = expert_offset * 4 + ib00 * ne01 * 4 + i01;++ regQ.s0 = read_imageui(src0_q, (int)(block_offset)).x;+ regQ.s1 = read_imageui(src0_q, (int)(block_offset + ne01)).x;+ regQ.s2 = read_imageui(src0_q, (int)(block_offset + ne01 * 2)).x;+ regQ.s3 = read_imageui(src0_q, (int)(block_offset + ne01 * 3)).x;++ uint offset = i11 * ne00 / 4 + ib00 * 8;++ half8 fp16x8 = mxfp4_to_fp16_packed8(as_ushort2(regQ.s0));++ float4 shared_y4;+ shared_y4 = read_imagef(src1, (offset + 0));+ float4 acc = shared_y4 * convert_float4(fp16x8.lo);++ shared_y4 = read_imagef(src1, (offset + 1));+ acc += shared_y4 * convert_float4(fp16x8.hi);++ fp16x8 = mxfp4_to_fp16_packed8(as_ushort2(regQ.s1));++ shared_y4 = read_imagef(src1, (offset + 2));+ acc += shared_y4 * convert_float4(fp16x8.lo);++ shared_y4 = read_imagef(src1, (offset + 3));+ acc += shared_y4 * convert_float4(fp16x8.hi);++ fp16x8 = mxfp4_to_fp16_packed8(as_ushort2(regQ.s2));++ shared_y4 = read_imagef(src1, (offset + 4));+ acc += shared_y4 * convert_float4(fp16x8.lo);++ shared_y4 = read_imagef(src1, (offset + 5));+ acc += shared_y4 * convert_float4(fp16x8.hi);++ fp16x8 = mxfp4_to_fp16_packed8(as_ushort2(regQ.s3));++ shared_y4 = read_imagef(src1, (offset + 6));+ acc += shared_y4 * convert_float4(fp16x8.lo);++ shared_y4 = read_imagef(src1, (offset + 7));+ acc += shared_y4 * convert_float4(fp16x8.hi);++ uchar regE = src0_e[ib00 * ne01 + i01 + expert_offset];+ sum += e8m0_to_fp32(regE) * ((acc.s0 + acc.s1) + (acc.s2 + acc.s3));+ }++ __local float reduceLM[SIMDGROUP_WIDTH * (N_SIMDGROUP - 1)];+ if (sgid == 1) reduceLM[SIMDGROUP_WIDTH * 0 + slid] = sum;+ if (sgid == 2) reduceLM[SIMDGROUP_WIDTH * 1 + slid] = sum;+ if (sgid == 3) reduceLM[SIMDGROUP_WIDTH * 2 + slid] = sum;+ barrier(CLK_LOCAL_MEM_FENCE);+ if (sgid == 0) sum += reduceLM[SIMDGROUP_WIDTH * 0 + slid];+ if (sgid == 0) sum += reduceLM[SIMDGROUP_WIDTH * 1 + slid];+ if (sgid == 0) sum += reduceLM[SIMDGROUP_WIDTH * 2 + slid];++ if (sgid == 0) {+ dst = dst + (offsetd >> 2);+ dst[i01 + i20 * ne01] = sum;+ }+}diff --git a/ggml/src/ggml-opencl/kernels/gemv_moe_q4_k_f32_ns.cl b/ggml/src/ggml-opencl/kernels/gemv_moe_q4_k_f32_ns.clindex 12464e9826..d3a3c7db87 100644--- a/ggml/src/ggml-opencl/kernels/gemv_moe_q4_k_f32_ns.cl+++ b/ggml/src/ggml-opencl/kernels/gemv_moe_q4_k_f32_ns.cl@@ -153,3 +153,114 @@ __kernel void kernel_gemv_moe_q4_k_f32_ns(dst[i01 + i20 * ne01] = sum;}}++__attribute__((qcom_reqd_sub_group_size("half")))+__kernel void kernel_gemv_moe_q4_k_f32_ns_wimg(+ __read_only image1d_buffer_t src0_q,+ __global half * src0_d,+ __global half * src0_dm,+ __global uchar * src0_s,+ __read_only image1d_buffer_t src1,+ __global uint * src2,+ __global float * dst,+ ulong offsetd,+ int ne00,+ int ne01,+ int ne11+) {+ uint i01 = get_global_id(0);+ uint i20 = get_global_id(2);+ uint sgid = get_local_id(1);+ uint slid = get_sub_group_local_id();++ if (i01 >= ne01) {+ return;+ }++ uint i11 = i20 % ne11;++ uint expert_id = src2[i20];++ int num_superblocks = ne00 / QK_K;+ int num_subblocks = ne00 / 32;+ int scales_per_row = num_superblocks * K_SCALE_SIZE;++ uint expert_q_offset = expert_id * (ne00 / 8) * ne01;+ uint expert_d_offset = expert_id * num_superblocks * ne01;++ __private float sum = 0.0f;++ for (uint ib = sgid; ib < num_subblocks; ib += N_SIMDGROUP) {+ uint sb = ib / 8;+ uint j = ib % 8;++ half d_val = src0_d[expert_d_offset + sb * ne01 + i01];+ half dm_val = src0_dm[expert_d_offset + sb * ne01 + i01];++ global const uchar * sc = src0_s + (expert_id * ne01 + i01) * scales_per_row + sb * K_SCALE_SIZE;+ uchar sv, mn;+ get_scale_min_k4(j, sc, &sv, &mn);++ float scale = (float)d_val * (float)sv;+ float minv = (float)dm_val * (float)mn;++ uint q_base = expert_q_offset + ib * ne01 * 4 + i01;++ uint4 regQ;+ regQ.s0 = read_imageui(src0_q, (int)(q_base)).x;+ regQ.s1 = read_imageui(src0_q, (int)(q_base + ne01)).x;+ regQ.s2 = read_imageui(src0_q, (int)(q_base + ne01 * 2)).x;+ regQ.s3 = read_imageui(src0_q, (int)(q_base + ne01 * 3)).x;++ uint y_offset = i11 * ne00 / 4 + ib * 8;++ float8 fp32x8 = q4_k_to_fp32_packed8(as_ushort2(regQ.s0), scale, minv);++ float4 shared_y4;+ shared_y4 = read_imagef(src1, (y_offset + 0));+ float4 acc = shared_y4 * fp32x8.lo;++ shared_y4 = read_imagef(src1, (y_offset + 1));+ acc += shared_y4 * fp32x8.hi;++ fp32x8 = q4_k_to_fp32_packed8(as_ushort2(regQ.s1), scale, minv);++ shared_y4 = read_imagef(src1, (y_offset + 2));+ acc += shared_y4 * fp32x8.lo;++ shared_y4 = read_imagef(src1, (y_offset + 3));+ acc += shared_y4 * fp32x8.hi;++ fp32x8 = q4_k_to_fp32_packed8(as_ushort2(regQ.s2), scale, minv);++ shared_y4 = read_imagef(src1, (y_offset + 4));+ acc += shared_y4 * fp32x8.lo;++ shared_y4 = read_imagef(src1, (y_offset + 5));+ acc += shared_y4 * fp32x8.hi;++ fp32x8 = q4_k_to_fp32_packed8(as_ushort2(regQ.s3), scale, minv);++ shared_y4 = read_imagef(src1, (y_offset + 6));+ acc += shared_y4 * fp32x8.lo;++ shared_y4 = read_imagef(src1, (y_offset + 7));+ acc += shared_y4 * fp32x8.hi;++ sum += ((acc.s0 + acc.s1) + (acc.s2 + acc.s3));+ }++ __local float reduceLM[SIMDGROUP_WIDTH * (N_SIMDGROUP - 1)];+ if (sgid == 1) reduceLM[SIMDGROUP_WIDTH * 0 + slid] = sum;+ if (sgid == 2) reduceLM[SIMDGROUP_WIDTH * 1 + slid] = sum;+ if (sgid == 3) reduceLM[SIMDGROUP_WIDTH * 2 + slid] = sum;+ barrier(CLK_LOCAL_MEM_FENCE);+ if (sgid == 0) sum += reduceLM[SIMDGROUP_WIDTH * 0 + slid];+ if (sgid == 0) sum += reduceLM[SIMDGROUP_WIDTH * 1 + slid];+ if (sgid == 0) sum += reduceLM[SIMDGROUP_WIDTH * 2 + slid];++ if (sgid == 0) {+ dst = dst + (offsetd >> 2);+ dst[i01 + i20 * ne01] = sum;+ }+}diff --git a/ggml/src/ggml-opencl/kernels/moe_combine.cl b/ggml/src/ggml-opencl/kernels/moe_combine.clnew file mode 100644index 0000000000..c195f14728--- /dev/null+++ b/ggml/src/ggml-opencl/kernels/moe_combine.cl@@ -0,0 +1,36 @@+// Fused MoE combine epilogue: replaces the router-weight MUL + the (n_expert_used-1)+// cross-expert ADD chain with ONE weighted-sum-across-experts pass.+// dst[row, tok] = sum_e experts[row, e, tok] * weights[0, e, tok]+// experts: [n_embd, n_expert_used, n_tokens] f32 (contiguous after down-proj GEMM)+// weights: [1, n_expert_used, n_tokens] f32+// dst: [n_embd, n_tokens] f32+// One read of experts + one write of dst (eliminates the intermediate weighted+// buffer and the k-1 elementwise add round-trips). Vectorized float4 over rows.+// strides e1/e2/w1/w2/d1 are in ELEMENTS (floats).++__kernel void kernel_moe_combine_f32(+ __global const char * e_buf, ulong off_e,+ __global const char * w_buf, ulong off_w,+ __global char * d_buf, ulong off_d,+ int n_embd4, // n_embd / 4+ int k, // n_expert_used+ int n_tokens,+ uint e1, uint e2, // experts strides (elements): per-expert, per-token+ uint w1, uint w2, // weights strides (elements)+ uint d1) // dst per-token stride (elements)+{+ const uint r4 = get_global_id(0);+ const uint tok = get_global_id(1);+ if (r4 >= (uint)n_embd4 || tok >= (uint)n_tokens) return;++ __global const float * E = (__global const float *)(e_buf + off_e) + tok*e2 + r4*4u;+ __global const float * W = (__global const float *)(w_buf + off_w) + tok*w2;++ float4 acc = (float4)(0.0f);+ for (int e = 0; e < k; ++e) {+ acc = mad(vload4(0, E + (uint)e*e1), (float4)(W[(uint)e*w1]), acc);+ }++ __global float * D = (__global float *)(d_buf + off_d) + tok*d1 + r4*4u;+ vstore4(acc, 0, D);+}diff --git a/ggml/src/ggml-opencl/kernels/moe_reorder_quant_a_q8_1.cl b/ggml/src/ggml-opencl/kernels/moe_reorder_quant_a_q8_1.clnew file mode 100644index 0000000000..0d16f3abdb--- /dev/null+++ b/ggml/src/ggml-opencl/kernels/moe_reorder_quant_a_q8_1.cl@@ -0,0 +1,64 @@+#pragma OPENCL EXTENSION cl_khr_fp16 : enable++// Fused MoE activation reorder + q8_1 quantization for the dp4a prefill GEMM.+// Combines kernel_moe_reorder_b (gather src1 rows per the post-router map) with+// the q8_1 quant pre-pass, so the f32 reordered-activation tile buffer is never+// materialised (saves a full write + read of [tok_slots * ne00] floats).+//+// One work-item per (token_slot, 32-block). Padding lanes (router 0xFFFFFFFF)+// emit d=0,s=0,qs=0 so they contribute nothing to the GEMM, exactly as the+// reorder zero-fill did. Output layout matches kernel_moe_quant_a_q8_1:+// qa[token_slot*K + blk*32 + i], da/sa[token_slot*(K/32) + blk].+__kernel void kernel_moe_reorder_quant_a_q8_1(+ __global const float * src, // original activations (offset applied)+ __global const uint * router, // post-router indices [tok_slots]+ __global char * qa,+ __global half * da,+ __global half * sa,+ __global const int * total_tiles,+ uint K,+ ushort map_ratio,+ uint tile_size,+ uint n_kblocks // K / 32+) {+ const uint blk = get_global_id(0); // 32-block along K+ const uint tok = get_global_id(1); // token slot (post_router_idx)++ if (blk >= n_kblocks || tok >= (uint)total_tiles[0] * tile_size) {+ return;+ }++ const uint out_base = tok * K + blk * 32;+ const uint bidx = tok * n_kblocks + blk;++ const uint router_idx = router[tok];++ float v[32];+ float amax = 0.0f;+ if (router_idx == 0xFFFFFFFF) {+ #pragma unroll+ for (int i = 0; i < 32; ++i) v[i] = 0.0f;+ } else {+ const uint act_idx = router_idx / map_ratio;+ const uint in_base = act_idx * K + blk * 32;+ #pragma unroll+ for (int i = 0; i < 32; ++i) {+ v[i] = src[in_base + i];+ amax = fmax(amax, fabs(v[i]));+ }+ }++ const float d = amax / 127.0f;+ const float id = (amax > 0.0f) ? (127.0f / amax) : 0.0f;++ int sum = 0;+ #pragma unroll+ for (int i = 0; i < 32; ++i) {+ const int q = (int)rint(v[i] * id);+ qa[out_base + i] = (char)q;+ sum += q;+ }++ da[bidx] = (half)d;+ sa[bidx] = (half)(d * (float)sum);+}diff --git a/ggml/src/ggml-opencl/kernels/quant_a_q8_1.cl b/ggml/src/ggml-opencl/kernels/quant_a_q8_1.clnew file mode 100644index 0000000000..3ce06305a3--- /dev/null+++ b/ggml/src/ggml-opencl/kernels/quant_a_q8_1.cl@@ -0,0 +1,42 @@+#pragma OPENCL EXTENSION cl_khr_fp16 : enable++// Quantize a contiguous [N, K] f32 activation buffer (token-major, K contiguous+// per token) into q8_1 blocks of 32: int8 quants + per-block scale d + per-block+// sum s (= d * Sum(qs)). Consumed by kernel_gemm_noshuffle_q4_k_q8_1_dp4a for the+// dp4a (int8) dense q4_K prefill GEMM. One work-item per 32-element block.+__kernel void kernel_quant_a_q8_1(+ __global const float * src, // [N * K]+ __global char * qa, // [N * K]+ __global half * da, // [N * (K/32)]+ __global half * sa, // [N * (K/32)]+ int total_blocks // N * (K/32)+) {+ const int blk = get_global_id(0);+ if (blk >= total_blocks) {+ return;+ }++ const int base = blk * 32;++ float v[32];+ float amax = 0.0f;+ #pragma unroll+ for (int i = 0; i < 32; ++i) {+ v[i] = src[base + i];+ amax = fmax(amax, fabs(v[i]));+ }++ const float d = amax / 127.0f;+ const float id = (amax > 0.0f) ? (127.0f / amax) : 0.0f;++ int sum = 0;+ #pragma unroll+ for (int i = 0; i < 32; ++i) {+ const int q = (int)rint(v[i] * id);+ qa[base + i] = (char)q;+ sum += q;+ }++ da[blk] = (half)d;+ sa[blk] = (half)(d * (float)sum);+}