Skip to content

LLM inference in C/C++

C++ ggml

opencl: quant lm_head / decode GEMV and medium-batch GEMM optimizations (speculative decoding/MTP) (#26477)

Hongqiang Wang95ef7fc16054
diff --git a/ggml/src/ggml-opencl/CMakeLists.txt b/ggml/src/ggml-opencl/CMakeLists.txt
index 1f62ce1c6a..8a1b6b964a 100644
--- a/ggml/src/ggml-opencl/CMakeLists.txt
+++ b/ggml/src/ggml-opencl/CMakeLists.txt
@@ -85,6 +85,7 @@ set(GGML_OPENCL_KERNELS
mul_mv_f16_f32_1row
mul_mv_f16_f32_l4
mul_mv_f16_f32
+ mul_mv_f16_f32_mrow
mul_mv_f32_f32
mul_mv_q1_0_f32
mul_mv_q1_0_f32_flat
@@ -180,9 +181,14 @@ set(GGML_OPENCL_KERNELS
gemv_noshuffle_q8_0_f32
gemm_noshuffle_q8_0_f32
gemv_noshuffle_q4_k_f32
+ gemv_noshuffle_q4_k_f32_o4
+ gemv_noshuffle_q4_k_f32_tiled
gemm_noshuffle_q4_k_f32
gemv_noshuffle_q6_k_f32
+ gemv_noshuffle_q6_k_f32_o4
+ gemv_noshuffle_q6_k_f32_tiled
gemm_noshuffle_q6_k_f32
+ gemm_noshuffle_q6_k_f32_tiled
gemv_noshuffle_q5_k_f32
gemm_noshuffle_q5_k_f32
mul
diff --git a/ggml/src/ggml-opencl/ggml-opencl.cpp b/ggml/src/ggml-opencl/ggml-opencl.cpp
index d95123eb15..12465a517d 100644
--- a/ggml/src/ggml-opencl/ggml-opencl.cpp
+++ b/ggml/src/ggml-opencl/ggml-opencl.cpp
@@ -568,6 +568,10 @@ struct ggml_backend_opencl_context {
bool has_integer_dot = false; // cl_khr_integer_dot_product or cl_qcom_dot_product8
bool has_qcom_subgroup_shuffle = false; // specifically cl_qcom_subgroup_shuffle
bool disable_fusion;
+ bool fuse_mm_glu = true; // opt-out GGML_OPENCL_FUSE_MM_GLU=0 (byte-identical gate+up GEMV + GLU, q4_K FFN)
+ bool fuse_rms_add = true; // opt-out GGML_OPENCL_FUSE_RMS_ADD=0 (fused rms_norm*w + residual)
+ bool f16_mrow = true; // opt-out GGML_OPENCL_F16_MROW=0 (multi-row-per-WG f16 decode GEMV for attn proj + lm_head)
+ int f16_mrow_rpt = 1; // GGML_OPENCL_F16_MROW_RPT={1,2,4,8,16} rows-per-subgroup register blocking
// ragged moe, use int to directly pass to kernel
cl_uint adreno_use_moe_ragged;
@@ -619,6 +623,7 @@ struct ggml_backend_opencl_context {
ggml_cl_buffer prealloc_moe_sa; // per-block s [tok_slots * ne00/32] (half)
// scratch copy of the router weights to avoid dst aliasing
ggml_cl_buffer prealloc_moe_combine_w;
+ ggml_cl_buffer prealloc_splitk_partial; // [ksplit * M] partials for split-K GEMV
// pool of persistent image1d_buffer views over kv-cache layers, keyed by
// (parent buffer, offset within parent)
@@ -749,6 +754,7 @@ struct ggml_backend_opencl_context {
kernel_geglu_erf_f16, kernel_geglu_quick_f16;
cl_kernel kernel_norm, kernel_norm_mul_add;
cl_kernel kernel_rms_norm, kernel_rms_norm_mul;
+ cl_kernel kernel_rms_norm_mul_add = nullptr; // fused rms_norm(x)*w + b (residual)
cl_kernel kernel_l2_norm_f32;
cl_kernel kernel_group_norm, kernel_group_norm_mul_add;
cl_kernel kernel_diag_mask_inf, kernel_diag_mask_inf_8;
@@ -768,6 +774,12 @@ struct ggml_backend_opencl_context {
cl_kernel kernel_mul_mat_f32_f32;
cl_kernel kernel_mul_mat_f16_f16;
cl_kernel kernel_mul_mat_f16_f32_1row;
+ cl_program program_mul_mv_f16_f32_mrow;
+ cl_kernel kernel_mul_mat_f16_f32_mrow = nullptr; // multi-row decode GEMV (attn proj + lm_head)
+ cl_kernel kernel_mul_mat_f16_f32_mrow_r2 = nullptr;
+ cl_kernel kernel_mul_mat_f16_f32_mrow_r4 = nullptr;
+ cl_kernel kernel_mul_mat_f16_f32_mrow_h8 = nullptr;
+ cl_kernel kernel_mul_mat_f16_f32_mrow_h8r2 = nullptr;
cl_kernel kernel_mul_mat_f16_f32;
cl_kernel kernel_mul_mat_f16_f32_l4;
cl_kernel kernel_mul_mat_f16_f32_l4_dr;
@@ -916,6 +928,7 @@ struct ggml_backend_opencl_context {
cl_kernel kernel_mul_mv_id_mxfp4_f32;
cl_kernel kernel_mul_mv_id_mxfp4_f32_flat;
cl_kernel kernel_mul_mm_f32_f32_l4_lm;
+ cl_kernel kernel_gemv_f32_f32_mc; // multi-column (small-N) f32 GEMV for spec/MTP verify
cl_kernel kernel_mul_mm_f16_f32_l4_lm;
cl_kernel kernel_mul_mm_q1_0_f32_l4_lm;
cl_kernel kernel_mul_mm_q4_0_f32_l4_lm;
@@ -1081,28 +1094,50 @@ struct ggml_backend_opencl_context {
// Gemm and Gemv related programs, kernels, etc
cl_kernel kernel_gemm_noshuffle_q4_0_f32;
cl_kernel kernel_gemv_noshuffle_q4_0_f32;
+ cl_kernel kernel_gemv_noshuffle_q4_0_f32_mc3; // multi-column (N=3) verify GEMV (spec/MTP)
cl_kernel kernel_gemv_noshuffle_q4_0_f32_4096_1_11008;
cl_kernel kernel_gemv_noshuffle_q4_0_f32_4096_1_4096;
cl_kernel kernel_gemv_noshuffle_q4_0_f32_11008_1_4096;
cl_kernel kernel_gemv_noshuffle_q4_0_f32_32000_1_4096;
cl_kernel kernel_gemv_noshuffle_q4_1_f32;
+ cl_kernel kernel_gemv_noshuffle_q4_1_f32_mc3; // multi-column (N=3) verify GEMV (spec/MTP)
cl_kernel kernel_gemm_noshuffle_q4_1_f32;
cl_kernel kernel_gemm_noshuffle_q8_0_f32, kernel_gemm_noshuffle_q8_0_f32_bin;
cl_kernel kernel_gemm_noshuffle_q8_0_q8_1_dp4a = nullptr; // dp4a (int8) dense q8_0 prefill GEMM (opt-in)
cl_kernel kernel_gemm_noshuffle_q8_0_q8_1_dp4a_wimg = nullptr; // q8_0 dense dp4a, weights via texture (opt-in)
cl_kernel kernel_gemv_noshuffle_q8_0_f32;
+ cl_kernel kernel_gemv_noshuffle_q8_0_f32_splitk; // split-K across WGs (small-M decode)
cl_kernel kernel_gemm_noshuffle_q1_0_f32;
cl_kernel kernel_gemv_noshuffle_q1_0_f32;
cl_kernel kernel_gemv_noshuffle_q4_k_f32;
+ cl_kernel kernel_gemv_noshuffle_q4_k_f32_o4; // 4-output-per-WI, long-vocab lm_head
+ cl_kernel kernel_gemv_noshuffle_q4_k_f32_tiled; // tiled-wide layout (opt-in)
+ cl_kernel kernel_gemv_noshuffle_q4_k_f32_splitk; // split-K across WGs (small-M decode)
+ cl_kernel kernel_gemv_splitk_reduce_f32; // sums split-K per-slice partials
+ cl_kernel kernel_gemv_noshuffle_q4_k_f32_glu; // fused gate+up GEMV + GLU (FFN)
+ cl_kernel kernel_convert_block_q4_k_tiled_ns; // tiled-wide convert (opt-in)
+ cl_kernel kernel_gemv_noshuffle_q4_k_f32_mc3; // multi-column (N=3) verify GEMV
cl_kernel kernel_gemm_noshuffle_q4_k_f32;
cl_kernel kernel_gemm_noshuffle_q4_k_q8_1_dp4a = nullptr; // dp4a (int8) dense prefill GEMM
cl_kernel kernel_gemm_noshuffle_q4_k_q8_1_dp4a_wimg = nullptr; // dp4a dense prefill GEMM, weights via texture (X1 opt-in)
cl_kernel kernel_gemm_noshuffle_q5_k_q8_1_dp4a = nullptr; // dp4a (int8) dense q5_K prefill GEMM
cl_kernel kernel_gemm_noshuffle_q6_k_q8_1_dp4a = nullptr; // dp4a (int8) dense q6_K prefill GEMM
cl_kernel kernel_quant_a_q8_1; // plain activation q8_1 pre-pass
+ cl_kernel kernel_gemm_noshuffle_q4_k_f32_r1;
+ cl_kernel kernel_gemm_noshuffle_q4_k_f32_kimg;
+ cl_kernel kernel_gemm_noshuffle_q4_k_f32_cok;
cl_kernel kernel_gemv_noshuffle_q6_K_f32;
+ cl_kernel kernel_gemv_noshuffle_q6_K_f32_o4;
+ cl_kernel kernel_gemv_noshuffle_q6_K_f32_o4_global; // weights via __global (opt-in)
+ cl_kernel kernel_gemv_noshuffle_q6_K_f32_tiled; // tiled-wide layout (opt-in)
+ cl_kernel kernel_gemv_noshuffle_q6_K_f32_tiled_mc3; // tiled multi-column (N=3) verify lm_head
+ cl_kernel kernel_gemm_noshuffle_q6_K_f32_tiled; // batched (N>1) over the tiled layout
+ cl_kernel kernel_convert_block_q6_k_tiled_ns; // tiled-wide convert (opt-in)
+ cl_kernel kernel_gemv_noshuffle_q6_K_f32_mc3; // multi-column (N=3) verify GEMV
cl_kernel kernel_gemm_noshuffle_q6_K_f32;
+ cl_kernel kernel_gemm_noshuffle_q6_K_f32_cok;
cl_kernel kernel_gemv_noshuffle_q5_k_f32;
+ cl_kernel kernel_gemv_noshuffle_q5_k_f32_mc3; // multi-column (N=3) verify GEMV (spec/MTP)
cl_kernel kernel_gemm_noshuffle_q5_k_f32;
cl_kernel kernel_gemv_noshuffle_q5_0_f32;
cl_kernel kernel_gemm_noshuffle_q5_0_f32;
@@ -1488,10 +1523,16 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) {
CL_CHECK((backend_ctx->kernel_restore_block_q5_1_trans4_ns = clCreateKernel(backend_ctx->program_cvt, "kernel_restore_block_q5_1_trans4_ns", &err), err));
CL_CHECK((backend_ctx->kernel_convert_block_q4_k_trans4_ns = clCreateKernel(backend_ctx->program_cvt, "kernel_convert_block_q4_k_trans4_ns", &err), err));
CL_CHECK((backend_ctx->kernel_restore_block_q4_k_trans4_ns = clCreateKernel(backend_ctx->program_cvt, "kernel_restore_block_q4_k_trans4_ns", &err), err));
+#ifdef GGML_OPENCL_USE_ADRENO_KERNELS
+ CL_CHECK((backend_ctx->kernel_convert_block_q4_k_tiled_ns = clCreateKernel(backend_ctx->program_cvt, "kernel_convert_block_q4_k_tiled_ns", &err), err));
+#endif
CL_CHECK((backend_ctx->kernel_convert_block_q5_k_trans4_ns = clCreateKernel(backend_ctx->program_cvt, "kernel_convert_block_q5_k_trans4_ns", &err), err));
CL_CHECK((backend_ctx->kernel_restore_block_q5_k_trans4_ns = clCreateKernel(backend_ctx->program_cvt, "kernel_restore_block_q5_k_trans4_ns", &err), err));
CL_CHECK((backend_ctx->kernel_convert_block_q6_k_trans4_ns = clCreateKernel(backend_ctx->program_cvt, "kernel_convert_block_q6_k_trans4_ns", &err), err));
CL_CHECK((backend_ctx->kernel_restore_block_q6_k_trans4_ns = clCreateKernel(backend_ctx->program_cvt, "kernel_restore_block_q6_k_trans4_ns", &err), err));
+#ifdef GGML_OPENCL_USE_ADRENO_KERNELS
+ CL_CHECK((backend_ctx->kernel_convert_block_q6_k_tiled_ns = clCreateKernel(backend_ctx->program_cvt, "kernel_convert_block_q6_k_tiled_ns", &err), err));
+#endif
CL_CHECK((backend_ctx->kernel_convert_block_mxfp4 = clCreateKernel(backend_ctx->program_cvt, "kernel_convert_block_mxfp4", &err), err));
CL_CHECK((backend_ctx->kernel_convert_block_mxfp4_trans = clCreateKernel(backend_ctx->program_cvt, "kernel_convert_block_mxfp4_trans", &err), err));
CL_CHECK((backend_ctx->kernel_convert_block_mxfp4_trans4_ns = clCreateKernel(backend_ctx->program_cvt, "kernel_convert_block_mxfp4_trans4_ns", &err), err));
@@ -2142,6 +2183,26 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) {
GGML_LOG_CONT(".");
}
+ // mul_mv_f16_f32_mrow (multi-row decode GEMV)
+ {
+#ifdef GGML_OPENCL_EMBED_KERNELS
+ const std::string kernel_src {
+ #include "mul_mv_f16_f32_mrow.cl.h"
+ };
+#else
+ const std::string kernel_src = read_file("mul_mv_f16_f32_mrow.cl");
+#endif
+ backend_ctx->program_mul_mv_f16_f32_mrow =
+ build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts);
+
+ CL_CHECK((backend_ctx->kernel_mul_mat_f16_f32_mrow = clCreateKernel(backend_ctx->program_mul_mv_f16_f32_mrow, "kernel_mul_mat_f16_f32_mrow", &err), err));
+ CL_CHECK((backend_ctx->kernel_mul_mat_f16_f32_mrow_r2 = clCreateKernel(backend_ctx->program_mul_mv_f16_f32_mrow, "kernel_mul_mat_f16_f32_mrow_r2", &err), err));
+ CL_CHECK((backend_ctx->kernel_mul_mat_f16_f32_mrow_r4 = clCreateKernel(backend_ctx->program_mul_mv_f16_f32_mrow, "kernel_mul_mat_f16_f32_mrow_r4", &err), err));
+ CL_CHECK((backend_ctx->kernel_mul_mat_f16_f32_mrow_h8 = clCreateKernel(backend_ctx->program_mul_mv_f16_f32_mrow, "kernel_mul_mat_f16_f32_mrow_h8", &err), err));
+ CL_CHECK((backend_ctx->kernel_mul_mat_f16_f32_mrow_h8r2 = clCreateKernel(backend_ctx->program_mul_mv_f16_f32_mrow, "kernel_mul_mat_f16_f32_mrow_h8r2", &err), err));
+ GGML_LOG_CONT(".");
+ }
+
// mul_mv_f16_f32_l4
{
#ifdef GGML_OPENCL_EMBED_KERNELS
@@ -2295,6 +2356,7 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) {
build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts);
CL_CHECK((backend_ctx->kernel_mul_mm_f32_f32_l4_lm = clCreateKernel(backend_ctx->program_mul_mm_f32_f32_l4_lm, "kernel_mul_mm_f32_f32_l4_lm", &err), err));
+ CL_CHECK((backend_ctx->kernel_gemv_f32_f32_mc = clCreateKernel(backend_ctx->program_mul_mm_f32_f32_l4_lm, "kernel_gemv_f32_f32_mc", &err), err));
GGML_LOG_CONT(".");
}
@@ -2564,6 +2626,7 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) {
CL_CHECK((backend_ctx->kernel_rms_norm = clCreateKernel(backend_ctx->program_rms_norm, "kernel_rms_norm", &err), err));
CL_CHECK((backend_ctx->kernel_rms_norm_mul = clCreateKernel(backend_ctx->program_rms_norm, "kernel_rms_norm_mul", &err), err));
+ CL_CHECK((backend_ctx->kernel_rms_norm_mul_add = clCreateKernel(backend_ctx->program_rms_norm, "kernel_rms_norm_mul_add", &err), err));
GGML_LOG_CONT(".");
}
@@ -3475,6 +3538,7 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) {
cl_program prog = build_program_from_source(backend_ctx, kernel_src_CL_gemv_general.c_str(), CL_gemv_compile_opts);
CL_CHECK((backend_ctx->kernel_gemv_noshuffle_q4_0_f32 = clCreateKernel(prog, "kernel_gemv_noshuffle_q4_0_f32", &err), err));
+ CL_CHECK((backend_ctx->kernel_gemv_noshuffle_q4_0_f32_mc3 = clCreateKernel(prog, "kernel_gemv_noshuffle_q4_0_f32_mc3", &err), err));
CL_CHECK(clReleaseProgram(prog));
GGML_LOG_CONT(".");
}
@@ -3604,6 +3668,7 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) {
cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), CL_gemv_compile_opts);
CL_CHECK((backend_ctx->kernel_gemv_noshuffle_q4_1_f32 = clCreateKernel(prog, "kernel_gemv_noshuffle_q4_1_f32", &err), err));
+ CL_CHECK((backend_ctx->kernel_gemv_noshuffle_q4_1_f32_mc3 = clCreateKernel(prog, "kernel_gemv_noshuffle_q4_1_f32_mc3", &err), err));
CL_CHECK(clReleaseProgram(prog));
GGML_LOG_CONT(".");
}
@@ -3818,6 +3883,7 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) {
cl_program prog = build_program_from_source(backend_ctx, kernel_src_CL_gemv_general.c_str(), CL_gemv_compile_opts);
CL_CHECK((backend_ctx->kernel_gemv_noshuffle_q8_0_f32 = clCreateKernel(prog, "kernel_gemv_noshuffle_q8_0_f32", &err), err));
+ CL_CHECK((backend_ctx->kernel_gemv_noshuffle_q8_0_f32_splitk = clCreateKernel(prog, "kernel_gemv_noshuffle_q8_0_f32_splitk", &err), err));
CL_CHECK(clReleaseProgram(prog));
GGML_LOG_CONT(".");
}
@@ -3833,6 +3899,9 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) {
#endif
cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts);
CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q4_k_f32 = clCreateKernel(prog, "kernel_gemm_noshuffle_q4_k_f32", &err), err));
+ CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q4_k_f32_r1 = clCreateKernel(prog, "kernel_gemm_noshuffle_q4_k_f32_r1", &err), err));
+ CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q4_k_f32_kimg = clCreateKernel(prog, "kernel_gemm_noshuffle_q4_k_f32_kimg", &err), err));
+ CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q4_k_f32_cok = clCreateKernel(prog, "kernel_gemm_noshuffle_q4_k_f32_cok", &err), err));
CL_CHECK(clReleaseProgram(prog));
GGML_LOG_CONT(".");
}
@@ -3927,6 +3996,18 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) {
if (backend_ctx->has_vector_subgroup_broadcast) {
CL_gemv_compile_opts += " -DVECTOR_SUB_GROUP_BROADCAST ";
}
+ // Opt-in: dequant-once-per-block mc3 verify GEMV (factors q4_K dequant
+ // out of the 3-column loop; byte-identical, lower spill). A/B vs the
+ // shipped inline mc3 in the same binary.
+ if (getenv("GGML_OPENCL_Q4K_MC3_DQ")) {
+ CL_gemv_compile_opts += " -DQ4K_MC3_DEQUANT_ONCE ";
+ }
+ // Opt-in: LDS-staged dequant mc3 verify GEMV (stages the dequantized
+ // q4_K weights in __local instead of private regs that spill to slow
+ // global on Adreno; byte-identical). A/B vs inline + dequant-once.
+ if (getenv("GGML_OPENCL_Q4K_MC3_LDS")) {
+ CL_gemv_compile_opts += " -DQ4K_MC3_DEQUANT_LDS ";
+ }
#ifdef GGML_OPENCL_EMBED_KERNELS
const std::string kernel_src {
@@ -3939,6 +4020,50 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) {
cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), CL_gemv_compile_opts);
CL_CHECK((backend_ctx->kernel_gemv_noshuffle_q4_k_f32 = clCreateKernel(prog, "kernel_gemv_noshuffle_q4_k_f32", &err), err));
+ CL_CHECK((backend_ctx->kernel_gemv_noshuffle_q4_k_f32_mc3 = clCreateKernel(prog, "kernel_gemv_noshuffle_q4_k_f32_mc3", &err), err));
+ CL_CHECK((backend_ctx->kernel_gemv_noshuffle_q4_k_f32_splitk = clCreateKernel(prog, "kernel_gemv_noshuffle_q4_k_f32_splitk", &err), err));
+ CL_CHECK((backend_ctx->kernel_gemv_splitk_reduce_f32 = clCreateKernel(prog, "kernel_gemv_splitk_reduce_f32", &err), err));
+ CL_CHECK((backend_ctx->kernel_gemv_noshuffle_q4_k_f32_glu = clCreateKernel(prog, "kernel_gemv_noshuffle_q4_k_f32_glu", &err), err));
+ CL_CHECK(clReleaseProgram(prog));
+ GGML_LOG_CONT(".");
+ }
+
+ // gemv_noshuffle_q4_k_f32_o4 — 4-output-per-WI variant for the long-vocab
+ // q4_K lm_head/embed GEMV (shares one activation read across 4 output rows).
+ {
+#ifdef GGML_OPENCL_EMBED_KERNELS
+ const std::string kernel_src {
+ #include "gemv_noshuffle_q4_k_f32_o4.cl.h"
+ };
+#else
+ const std::string kernel_src = read_file("gemv_noshuffle_q4_k_f32_o4.cl");
+#endif
+ std::string CL_gemv_compile_opts = std::string("-cl-std=") + opencl_c_std + " -cl-mad-enable ";
+ if (backend_ctx->has_vector_subgroup_broadcast) {
+ CL_gemv_compile_opts += " -DVECTOR_SUB_GROUP_BROADCAST ";
+ }
+ cl_program prog = build_program_from_source(
+ backend_ctx, kernel_src.c_str(), CL_gemv_compile_opts);
+ CL_CHECK((backend_ctx->kernel_gemv_noshuffle_q4_k_f32_o4 = clCreateKernel(prog, "kernel_gemv_noshuffle_q4_k_f32_o4", &err), err));
+ CL_CHECK(clReleaseProgram(prog));
+ GGML_LOG_CONT(".");
+ }
+
+ // gemv_noshuffle_q4_k_f32_tiled — tiled-wide canonical layout, default ON
+ // (opt out: GGML_OPENCL_Q4K_GEMV_TILED=0; separate convert + GEMV; weights via __global).
+ {
+#ifdef GGML_OPENCL_EMBED_KERNELS
+ const std::string kernel_src {
+ #include "gemv_noshuffle_q4_k_f32_tiled.cl.h"
+ };
+#else
+ const std::string kernel_src = read_file("gemv_noshuffle_q4_k_f32_tiled.cl");
+#endif
+ std::string compile_opts = std::string("-cl-std=") + opencl_c_std + " -cl-mad-enable ";
+ cl_program prog =
+ build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts);
+ CL_CHECK((backend_ctx->kernel_gemv_noshuffle_q4_k_f32_tiled =
+ clCreateKernel(prog, "kernel_gemv_noshuffle_q4_k_f32_tiled", &err), err));
CL_CHECK(clReleaseProgram(prog));
GGML_LOG_CONT(".");
}
@@ -4566,6 +4691,91 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) {
build_program_from_source(backend_ctx, kernel_src.c_str(), CL_gemv_compile_opts);
CL_CHECK((backend_ctx->kernel_gemv_noshuffle_q6_K_f32 = clCreateKernel(prog, "kernel_gemv_noshuffle_q6_K_f32", &err), err));
+ CL_CHECK((backend_ctx->kernel_gemv_noshuffle_q6_K_f32_mc3 = clCreateKernel(prog, "kernel_gemv_noshuffle_q6_K_f32_mc3", &err), err));
+ if (getenv("GGML_OPENCL_MC3_PROBE")) {
+ cl_ulong pm6 = 0, pm4 = 0; size_t wg6 = 0, wg4 = 0, mult = 0;
+ clGetKernelWorkGroupInfo(backend_ctx->kernel_gemv_noshuffle_q6_K_f32_mc3, backend_ctx->device, CL_KERNEL_PRIVATE_MEM_SIZE, sizeof(pm6), &pm6, NULL);
+ clGetKernelWorkGroupInfo(backend_ctx->kernel_gemv_noshuffle_q6_K_f32_mc3, backend_ctx->device, CL_KERNEL_WORK_GROUP_SIZE, sizeof(wg6), &wg6, NULL);
+ clGetKernelWorkGroupInfo(backend_ctx->kernel_gemv_noshuffle_q4_k_f32_mc3, backend_ctx->device, CL_KERNEL_PRIVATE_MEM_SIZE, sizeof(pm4), &pm4, NULL);
+ clGetKernelWorkGroupInfo(backend_ctx->kernel_gemv_noshuffle_q4_k_f32_mc3, backend_ctx->device, CL_KERNEL_WORK_GROUP_SIZE, sizeof(wg4), &wg4, NULL);
+ clGetKernelWorkGroupInfo(backend_ctx->kernel_gemv_noshuffle_q6_K_f32_mc3, backend_ctx->device, CL_KERNEL_PREFERRED_WORK_GROUP_SIZE_MULTIPLE, sizeof(mult), &mult, NULL);
+ fprintf(stderr, "[MC3-PROBE] q4K_mc3 private=%llu wg_cap=%zu | q6K_mc3 private=%llu wg_cap=%zu | pref_mult=%zu\n",
+ (unsigned long long)pm4, wg4, (unsigned long long)pm6, wg6, mult);
+ fflush(stderr);
+ }
+ GGML_LOG_CONT(".");
+ }
+
+ // gemv_noshuffle_q6_k_f32_o4 — 4-output-per-WI variant, opt-in via
+ // GGML_OPENCL_Q6K_GEMV_O4=1 (~3x fewer dispatches on long-vocab lm_head).
+ {
+#ifdef GGML_OPENCL_EMBED_KERNELS
+ const std::string kernel_src {
+ #include "gemv_noshuffle_q6_k_f32_o4.cl.h"
+ };
+#else
+ const std::string kernel_src = read_file("gemv_noshuffle_q6_k_f32_o4.cl");
+#endif
+
+ std::string CL_gemv_compile_opts = std::string("-cl-std=") + opencl_c_std +
+ " -cl-mad-enable ";
+ if (backend_ctx->has_vector_subgroup_broadcast) {
+ CL_gemv_compile_opts += " -DVECTOR_SUB_GROUP_BROADCAT ";
+ }
+
+ cl_program prog =
+ build_program_from_source(backend_ctx, kernel_src.c_str(), CL_gemv_compile_opts);
+
+ CL_CHECK((backend_ctx->kernel_gemv_noshuffle_q6_K_f32_o4 = clCreateKernel(prog, "kernel_gemv_noshuffle_q6_K_f32_o4", &err), err));
+ CL_CHECK(clReleaseProgram(prog));
+
+ // Global-read variant: weights read from __global coalesced instead of
+ // image1d_buffer (the texture cache caps the streaming lm_head read
+ // bandwidth). Opt-in via GGML_OPENCL_Q6K_GEMV_O4_GLOBAL.
+ cl_program prog_g = build_program_from_source(backend_ctx, kernel_src.c_str(), CL_gemv_compile_opts + " -DQ6K_O4_GLOBAL");
+ CL_CHECK((backend_ctx->kernel_gemv_noshuffle_q6_K_f32_o4_global =
+ clCreateKernel(prog_g, "kernel_gemv_noshuffle_q6_K_f32_o4_global", &err), err));
+ CL_CHECK(clReleaseProgram(prog_g));
+ GGML_LOG_CONT(".");
+ }
+
+ // gemv_noshuffle_q6_k_f32_tiled — tiled-wide canonical layout, default ON
+ // (opt out: GGML_OPENCL_Q6K_GEMV_TILED=0; separate convert + GEMV; weights via __global).
+ {
+#ifdef GGML_OPENCL_EMBED_KERNELS
+ const std::string kernel_src {
+ #include "gemv_noshuffle_q6_k_f32_tiled.cl.h"
+ };
+#else
+ const std::string kernel_src = read_file("gemv_noshuffle_q6_k_f32_tiled.cl");
+#endif
+ std::string compile_opts = std::string("-cl-std=") + opencl_c_std + " -cl-mad-enable ";
+ cl_program prog =
+ build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts);
+ CL_CHECK((backend_ctx->kernel_gemv_noshuffle_q6_K_f32_tiled =
+ clCreateKernel(prog, "kernel_gemv_noshuffle_q6_K_f32_tiled", &err), err));
+ CL_CHECK((backend_ctx->kernel_gemv_noshuffle_q6_K_f32_tiled_mc3 =
+ clCreateKernel(prog, "kernel_gemv_noshuffle_q6_K_f32_tiled_mc3", &err), err));
+ CL_CHECK(clReleaseProgram(prog));
+ GGML_LOG_CONT(".");
+ }
+
+ // gemm_noshuffle_q6_k_f32_tiled — batched (N>1) GEMM over the same tiled-wide
+ // canonical layout, so batched lm_head/embed stays correct + on GPU.
+ {
+#ifdef GGML_OPENCL_EMBED_KERNELS
+ const std::string kernel_src {
+ #include "gemm_noshuffle_q6_k_f32_tiled.cl.h"
+ };
+#else
+ const std::string kernel_src = read_file("gemm_noshuffle_q6_k_f32_tiled.cl");
+#endif
+ std::string compile_opts = std::string("-cl-std=") + opencl_c_std + " -cl-mad-enable ";
+ cl_program prog =
+ build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts);
+ CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q6_K_f32_tiled =
+ clCreateKernel(prog, "kernel_gemm_noshuffle_q6_K_f32_tiled", &err), err));
+ CL_CHECK(clReleaseProgram(prog));
GGML_LOG_CONT(".");
}
@@ -4582,6 +4792,7 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) {
build_program_from_source(backend_ctx, kernel_src.c_str(), CL_moe_compile_opts);
CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q6_K_f32 = clCreateKernel(prog, "kernel_gemm_noshuffle_q6_K_f32", &err), err));
+ CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q6_K_f32_cok = clCreateKernel(prog, "kernel_gemm_noshuffle_q6_K_f32_cok", &err), err));
GGML_LOG_CONT(".");
}
@@ -4604,6 +4815,7 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) {
cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), CL_gemv_compile_opts);
CL_CHECK((backend_ctx->kernel_gemv_noshuffle_q5_k_f32 = clCreateKernel(prog, "kernel_gemv_noshuffle_q5_k_f32", &err), err));
+ CL_CHECK((backend_ctx->kernel_gemv_noshuffle_q5_k_f32_mc3 = clCreateKernel(prog, "kernel_gemv_noshuffle_q5_k_f32_mc3", &err), err));
CL_CHECK(clReleaseProgram(prog));
GGML_LOG_CONT(".");
}
@@ -6166,6 +6378,19 @@ static ggml_backend_opencl_context * ggml_cl_init(ggml_backend_dev_t dev) {
#endif // GGML_OPENCL_USE_ADRENO_KERNELS
backend_ctx->disable_fusion = getenv("GGML_OPENCL_DISABLE_FUSION") != nullptr;
+ if (const char * env = getenv("GGML_OPENCL_FUSE_MM_GLU")) {
+ backend_ctx->fuse_mm_glu = atoi(env) != 0;
+ }
+ if (const char * env = getenv("GGML_OPENCL_FUSE_RMS_ADD")) {
+ backend_ctx->fuse_rms_add = atoi(env) != 0;
+ }
+ if (const char * env = getenv("GGML_OPENCL_F16_MROW")) {
+ backend_ctx->f16_mrow = atoi(env) != 0;
+ }
+ if (const char * env = getenv("GGML_OPENCL_F16_MROW_RPT")) {
+ const int v = atoi(env);
+ backend_ctx->f16_mrow_rpt = (v == 2 || v == 4 || v == 8 || v == 16) ? v : 1;
+ }
dev_ctx->backend_ctx = backend_ctx.release();
return dev_ctx->backend_ctx;
@@ -7305,7 +7530,73 @@ static void ggml_cl_moe_combine_fused(ggml_backend_t backend, const ggml_tensor
backend_ctx->enqueue_ndrange_kernel(kernel, 2, gws, lws, dst);
}
-static bool ggml_opencl_can_fuse(const struct ggml_cgraph * cgraph, int node_idx, std::initializer_list<enum ggml_op> ops) {
+inline bool use_q4k_tiled(const ggml_backend_opencl_context *backend_ctx, const ggml_tensor *tensor); // defined below (used by the GLU-subgraph fuse check)
+inline bool use_adreno_kernels(const ggml_backend_opencl_context *backend_ctx, const ggml_tensor *tensor); // defined below
+
+static bool ggml_opencl_can_fuse(const ggml_backend_opencl_context * backend_ctx, const struct ggml_cgraph * cgraph, int node_idx, std::initializer_list<enum ggml_op> ops) {
+
+ // glu(mul_mat(Wg,x), mul_mat(Wu,x)) — the FFN gate/up GEMVs + GLU. This is a
+ // non-linear subgraph (up does NOT consume gate), so the contiguous
+ // ggml_can_fuse below rejects it; use ggml_can_fuse_subgraph with the glu as
+ // the sole output and validate the edges explicitly. q4_K decode only;
+ // byte-identical to the per-op path.
+ if (ops.size() == 3 && ops.begin()[0] == GGML_OP_MUL_MAT &&
+ ops.begin()[1] == GGML_OP_MUL_MAT && ops.begin()[2] == GGML_OP_GLU) {
+ const enum ggml_op glu_ops[] = { GGML_OP_MUL_MAT, GGML_OP_MUL_MAT, GGML_OP_GLU };
+ const int glu_out[] = { node_idx + 2 };
+ if (!ggml_can_fuse_subgraph(cgraph, node_idx, 3, glu_ops, glu_out, 1)) {
+ return false;
+ }
+
+ const ggml_tensor *gate = cgraph->nodes[node_idx];
+ const ggml_tensor *up = cgraph->nodes[node_idx+1];
+ const ggml_tensor *glu = cgraph->nodes[node_idx+2];
+
+ // decode GEMV path only (single token); prefill GEMM is separate
+ if (gate->ne[1] != 1 || up->ne[1] != 1) {
+ return false;
+ }
+ // both projections must be q4_K weights, f32 activation/output
+ if (gate->src[0]->type != GGML_TYPE_Q4_K || up->src[0]->type != GGML_TYPE_Q4_K ||
+ gate->src[1]->type != GGML_TYPE_F32 || up->src[1]->type != GGML_TYPE_F32 ||
+ gate->type != GGML_TYPE_F32 || up->type != GGML_TYPE_F32 || glu->type != GGML_TYPE_F32) {
+ return false;
+ }
+ // gate and up must share the same activation and have matching shape/stride
+ if (gate->src[1] != up->src[1] ||
+ !ggml_are_same_shape(gate->src[0], up->src[0]) ||
+ !ggml_are_same_stride(gate->src[0], up->src[0])) {
+ return false;
+ }
+ // GLU must read gate as src[0] and up as src[1], no swap (the fused
+ // epilogue applies the activation to gate, multiplies by up)
+ if (glu->src[0] != gate || glu->src[1] != up) {
+ return false;
+ }
+ if (ggml_get_op_params_i32(glu, 1) /* swapped */) {
+ return false;
+ }
+ // SWIGLU_OAI carries extra alpha/limit params -> not handled by the fused kernel
+ if (ggml_get_glu_op(glu) == GGML_GLU_OP_SWIGLU_OAI) {
+ return false;
+ }
+ // the fused kernel reads the standard noshuffle image layout; the tiled
+ // layout packs weights differently -> defer those to the per-op path
+ if (use_q4k_tiled(backend_ctx, gate->src[0]) || use_q4k_tiled(backend_ctx, up->src[0])) {
+ return false;
+ }
+ // that noshuffle layout is only produced at set_tensor time when
+ // use_adreno_kernels() accepts the weight (ne0 >= 512 && ne1 >= 512).
+ // Smaller weights stay in the plain q4_K layout, which this kernel would
+ // misread -> defer them to the per-op path. Real FFN gate/up weights are
+ // far above the threshold, so production dispatch is unchanged.
+ if (!use_adreno_kernels(backend_ctx, gate->src[0]) ||
+ !use_adreno_kernels(backend_ctx, up->src[0])) {
+ return false;
+ }
+ return true;
+ }
+
if (!ggml_can_fuse(cgraph, node_idx, ops)) {
return false;
}
@@ -7353,6 +7644,38 @@ static bool ggml_opencl_can_fuse(const struct ggml_cgraph * cgraph, int node_idx
if (!ggml_is_contiguous(norm->src[0]) || !ggml_is_contiguous(w) || !ggml_is_contiguous(b)) {
return false;
}
+ } else if (ops.size() == 3 && ops.begin()[0] == GGML_OP_RMS_NORM && ops.begin()[1] == GGML_OP_MUL && ops.begin()[2] == GGML_OP_ADD) {
+ // rms_norm(x) * w + b, fused (residual). Mirrors the RMS_NORM+MUL gate
+ // plus the residual-add operand's constraints.
+ const ggml_tensor *rms_norm = cgraph->nodes[node_idx];
+ const ggml_tensor *mul = cgraph->nodes[node_idx+1];
+ const ggml_tensor *add = cgraph->nodes[node_idx+2];
+ const ggml_tensor *w = mul->src[0] == rms_norm ? mul->src[1] : mul->src[0];
+ const ggml_tensor *b = add->src[0] == mul ? add->src[1] : add->src[0];
+
+ GGML_ASSERT(rms_norm->src[0]->type == GGML_TYPE_F32);
+ GGML_ASSERT(rms_norm->type == GGML_TYPE_F32);
+
+ if (w->type != GGML_TYPE_F32 || mul->type != GGML_TYPE_F32 ||
+ b->type != GGML_TYPE_F32 || add->type != GGML_TYPE_F32) {
+ return false;
+ }
+ if (rms_norm->src[0]->ne[0] % 4 != 0) {
+ return false;
+ }
+ // if rms_norm is the B operand of mul, broadcast is not handled
+ if (rms_norm == mul->src[1] && !ggml_are_same_shape(mul->src[0], rms_norm)) {
+ return false;
+ }
+ // the residual must match the normed output shape (no add broadcast)
+ if (!ggml_are_same_shape(b, add)) {
+ return false;
+ }
+ // rms_norm assumes contiguous rows
+ if (!ggml_is_contiguous_rows(mul->src[0]) || !ggml_is_contiguous_rows(mul->src[1]) ||
+ !ggml_is_contiguous_rows(b)) {
+ return false;
+ }
} else if (ops.size() == 3 && ops.begin()[0] == GGML_OP_GROUP_NORM && ops.begin()[1] == GGML_OP_MUL && ops.begin()[2] == GGML_OP_ADD) {
const ggml_tensor *gn = cgraph->nodes[node_idx];
const ggml_tensor *mul = cgraph->nodes[node_idx+1];
@@ -7376,6 +7699,216 @@ static void ggml_opencl_op_rms_norm_fused(ggml_backend_t backend, ggml_tensor *
static void ggml_opencl_op_norm_fused(ggml_backend_t backend, ggml_tensor * norm_tensor, ggml_tensor * mul_tensor, ggml_tensor * add_tensor);
static void ggml_opencl_op_group_norm_fused(ggml_backend_t backend, ggml_tensor * gn_tensor, ggml_tensor * mul_tensor, ggml_tensor * add_tensor);
+static void ggml_cl_mul_mat_q4_k_glu_fused(ggml_backend_t backend, ggml_tensor * gate_tensor, ggml_tensor * up_tensor, ggml_tensor * glu_tensor) {
+#ifdef GGML_OPENCL_USE_ADRENO_KERNELS
+ GGML_ASSERT(gate_tensor && up_tensor && glu_tensor);
+
+ const ggml_tensor * Wg = gate_tensor->src[0];
+ const ggml_tensor * Wu = up_tensor->src[0];
+ const ggml_tensor * src1 = gate_tensor->src[1]; // == up_tensor->src[1]
+ const ggml_tensor * dst = glu_tensor;
+
+ GGML_ASSERT(Wg && Wg->extra);
+ GGML_ASSERT(Wu && Wu->extra);
+ GGML_ASSERT(src1 && src1->extra);
+ GGML_ASSERT(dst && dst->extra);
+
+ ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context;
+
+ ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra;
+ ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra;
+ ggml_tensor_extra_cl_q4_K * extra_g = (ggml_tensor_extra_cl_q4_K *)Wg->extra;
+ ggml_tensor_extra_cl_q4_K * extra_u = (ggml_tensor_extra_cl_q4_K *)Wu->extra;
+
+ cl_ulong offset1 = extra1->offset + src1->view_offs;
+ cl_ulong offsetd = extrad->offset + dst->view_offs;
+
+ const int K = Wg->ne[0]; // ne00
+ const int M = Wg->ne[1]; // ne01 (= ffn intermediate width)
+ const int N = 1; // decode GEMV
+
+ const cl_uchar mask_d6 = 0x3F, mask_d4 = 0x0F, mask_hi2 = 0xC0;
+ const int glu_op = (int)ggml_get_glu_op(dst);
+
+ cl_context context = backend_ctx->context;
+ cl_int err;
+ cl_image_format img_fmt;
+ cl_image_desc img_desc;
+ cl_buffer_region region;
+
+ // q images for the two weight matrices (standard noshuffle layout)
+ img_fmt = { CL_R, CL_UNSIGNED_INT32 };
+ memset(&img_desc, 0, sizeof(img_desc));
+ img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER;
+ img_desc.image_width = (size_t)M * K / 2 / 4;
+ img_desc.buffer = extra_g->q;
+ cl_mem qg_img = nullptr, qu_img = nullptr;
+ CL_CHECK((qg_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err));
+ img_desc.buffer = extra_u->q;
+ CL_CHECK((qu_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err));
+
+ // shared activation image (one column at decode)
+ region.origin = offset1;
+ region.size = (size_t)K * N * sizeof(float);
+ cl_mem b_sub_buf = nullptr, b_img = nullptr;
+ CL_CHECK((b_sub_buf = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, &region, &err), err));
+ img_fmt = { CL_RGBA, CL_FLOAT };
+ memset(&img_desc, 0, sizeof(img_desc));
+ img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER;
+ img_desc.image_width = (size_t)K * N / 4;
+ img_desc.buffer = b_sub_buf;
+ CL_CHECK((b_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err));
+
+ cl_kernel kernel = backend_ctx->kernel_gemv_noshuffle_q4_k_f32_glu;
+ CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &qg_img));
+ CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra_g->d));
+ CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra_g->dm));
+ CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra_g->s));
+ CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &qu_img));
+ CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_mem), &extra_u->d));
+ CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_mem), &extra_u->dm));
+ CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_mem), &extra_u->s));
+ CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_mem), &b_img));
+ CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_mem), &extrad->data_device));
+ CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &offsetd));
+ CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_int), &K));
+ CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_int), &M));
+ CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_int), &glu_op));
+ CL_CHECK(clSetKernelArg(kernel, 14, sizeof(cl_uchar), &mask_d6));
+ CL_CHECK(clSetKernelArg(kernel, 15, sizeof(cl_uchar), &mask_d4));
+ CL_CHECK(clSetKernelArg(kernel, 16, sizeof(cl_uchar), &mask_hi2));
+
+ // K-split = nsg_y subgroups. HARD-CAP at 8 (512 work-items): the fused
+ // kernel's cross-subgroup reduce uses a float4 reduceLM (gate+up packed) =
+ // 2x the LDS of the base GEMV's float2 reduce, so 16 co-resident subgroups
+ // exceed the per-CU LDS budget on X2 and the WG barrier DEADLOCKS -> GPU TDR
+ // (reproduced on upstream gemma-4 E4B decode, K=2560 M=10240). This used to
+ // be masked: get_kernel_workgroup_size reported 896 for this kernel (so the
+ // cap loop fell to 8), but it now returns 1024 and the Adreno per-kernel WG
+ // query is unreliable (over-reports), so cap explicitly instead of trusting
+ // it. nsg_y < 16 also means the cross-subgroup accumulation grouping differs
+ // from the standalone wide (nsg=16) GEMV, so the output is coherent but NOT
+ // byte-identical to the per-op path. Keep the maxwg query as a further floor
+ // for any driver that reports < 512.
+ size_t maxwg = backend_ctx->get_kernel_workgroup_size(kernel);
+ size_t nsg_y = 8;
+ while (nsg_y > 1 && 64 * nsg_y > maxwg) { nsg_y >>= 1; }
+ size_t local_work_size[3] = { 64, nsg_y, 1 };
+ size_t global_work_size[3] = { (size_t)CEIL_DIV(M / 2, 64) * 64, nsg_y, 1 };
+
+ if (getenv("GGML_OPENCL_FUSE_DEBUG")) {
+ static int dbg = 0;
+ if (dbg < 3) { fprintf(stderr, "[FUSE_MM_GLU] fired #%d K=%d M=%d glu_op=%d nsg=%zu maxwg=%zu\n", ++dbg, K, M, glu_op, nsg_y, maxwg); fflush(stderr); }
+ }
+
+ backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst);
+
+ CL_CHECK(clReleaseMemObject(qg_img));
+ CL_CHECK(clReleaseMemObject(qu_img));
+ CL_CHECK(clReleaseMemObject(b_img));
+ CL_CHECK(clReleaseMemObject(b_sub_buf));
+#else
+ GGML_UNUSED(backend);
+ GGML_UNUSED(gate_tensor);
+ GGML_UNUSED(up_tensor);
+ GGML_UNUSED(glu_tensor);
+ GGML_ABORT("q4_K GLU fusion requires GGML_OPENCL_USE_ADRENO_KERNELS");
+#endif
+}
+
+
+static void ggml_opencl_op_rms_norm_mul_add_fused(ggml_backend_t backend, ggml_tensor * rms_norm_tensor, ggml_tensor * mul_tensor, ggml_tensor * add_tensor) {
+ GGML_ASSERT(rms_norm_tensor && mul_tensor && add_tensor);
+
+ const ggml_tensor * src0 = rms_norm_tensor->src[0];
+ const ggml_tensor * src1 = mul_tensor->src[0] == rms_norm_tensor ? mul_tensor->src[1] : mul_tensor->src[0];
+ const ggml_tensor * src2 = add_tensor->src[0] == mul_tensor ? add_tensor->src[1] : add_tensor->src[0];
+ const ggml_tensor * dst = add_tensor;
+
+ GGML_ASSERT(src0 && src0->extra);
+ GGML_ASSERT(src1 && src1->extra);
+ GGML_ASSERT(src2 && src2->extra);
+ GGML_ASSERT(dst && dst->extra);
+
+ ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra;
+ ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra;
+ ggml_tensor_extra_cl * extra2 = (ggml_tensor_extra_cl *)src2->extra;
+ ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra;
+
+ cl_ulong offset0 = extra0->offset + src0->view_offs;
+ cl_ulong offset1 = extra1->offset + src1->view_offs;
+ cl_ulong offset2 = extra2->offset + src2->view_offs;
+ cl_ulong offsetd = extrad->offset + dst->view_offs;
+
+ ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context;
+
+ float eps;
+ memcpy(&eps, rms_norm_tensor->op_params, sizeof(float));
+
+ const int ne00 = src0->ne[0], ne01 = src0->ne[1], ne02 = src0->ne[2], ne03 = src0->ne[3];
+ const cl_ulong nb01 = src0->nb[1], nb02 = src0->nb[2], nb03 = src0->nb[3];
+ const int ne10 = src1->ne[0], ne11 = src1->ne[1], ne12 = src1->ne[2], ne13 = src1->ne[3];
+ const cl_ulong nb11 = src1->nb[1], nb12 = src1->nb[2], nb13 = src1->nb[3];
+ const int ne20 = src2->ne[0], ne21 = src2->ne[1], ne22 = src2->ne[2], ne23 = src2->ne[3];
+ const cl_ulong nb21 = src2->nb[1], nb22 = src2->nb[2], nb23 = src2->nb[3];
+ const cl_ulong nb1 = dst->nb[1], nb2 = dst->nb[2], nb3 = dst->nb[3];
+
+ GGML_ASSERT(ne00 % 4 == 0);
+
+ size_t sgs;
+ if (backend_ctx->gpu_family == ADRENO) sgs = 64;
+ else if (backend_ctx->gpu_family == INTEL) sgs = 32;
+ else GGML_ASSERT(false && "Unsupported GPU");
+
+ cl_kernel kernel = backend_ctx->kernel_rms_norm_mul_add;
+
+ int nth = sgs;
+ int max_workgroup_size = backend_ctx->get_kernel_workgroup_size(kernel);
+ while (nth < ne00 && nth < max_workgroup_size) nth *= 2;
+ nth = MIN(nth, max_workgroup_size);
+ nth = MIN(nth, ne00);
+
+ size_t global_work_size[] = {(size_t)ne01*nth, (size_t)ne02, (size_t)ne03};
+ size_t local_work_size[] = {(size_t)nth, 1, 1};
+
+ CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device));
+ CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0));
+ CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device));
+ CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1));
+ CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extra2->data_device));
+ CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offset2));
+ CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_mem), &extrad->data_device));
+ CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &offsetd));
+ CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne00));
+ CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne01));
+ CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne02));
+ CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne03));
+ CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb01));
+ CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_ulong), &nb02));
+ CL_CHECK(clSetKernelArg(kernel, 14, sizeof(cl_ulong), &nb03));
+ CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &ne10));
+ CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &ne11));
+ CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &ne12));
+ CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &ne13));
+ CL_CHECK(clSetKernelArg(kernel, 19, sizeof(cl_ulong), &nb11));
+ CL_CHECK(clSetKernelArg(kernel, 20, sizeof(cl_ulong), &nb12));
+ CL_CHECK(clSetKernelArg(kernel, 21, sizeof(cl_ulong), &nb13));
+ CL_CHECK(clSetKernelArg(kernel, 22, sizeof(int), &ne20));
+ CL_CHECK(clSetKernelArg(kernel, 23, sizeof(int), &ne21));
+ CL_CHECK(clSetKernelArg(kernel, 24, sizeof(int), &ne22));
+ CL_CHECK(clSetKernelArg(kernel, 25, sizeof(int), &ne23));
+ CL_CHECK(clSetKernelArg(kernel, 26, sizeof(cl_ulong), &nb21));
+ CL_CHECK(clSetKernelArg(kernel, 27, sizeof(cl_ulong), &nb22));
+ CL_CHECK(clSetKernelArg(kernel, 28, sizeof(cl_ulong), &nb23));
+ CL_CHECK(clSetKernelArg(kernel, 29, sizeof(cl_ulong), &nb1));
+ CL_CHECK(clSetKernelArg(kernel, 30, sizeof(cl_ulong), &nb2));
+ CL_CHECK(clSetKernelArg(kernel, 31, sizeof(cl_ulong), &nb3));
+ CL_CHECK(clSetKernelArg(kernel, 32, sizeof(float), &eps));
+ CL_CHECK(clSetKernelArg(kernel, 33, sizeof(float)*sgs, NULL));
+
+ backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst);
+}
+
static ggml_status ggml_backend_opencl_graph_compute(ggml_backend_t backend, ggml_cgraph * cgraph) {
ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context;
@@ -7395,12 +7928,12 @@ static ggml_status ggml_backend_opencl_graph_compute(ggml_backend_t backend, ggm
continue;
}
- if (!backend_ctx->disable_fusion && ggml_opencl_can_fuse(cgraph, i, { GGML_OP_NORM, GGML_OP_MUL, GGML_OP_ADD })) {
+ if (!backend_ctx->disable_fusion && ggml_opencl_can_fuse(backend_ctx, cgraph, i, { GGML_OP_NORM, GGML_OP_MUL, GGML_OP_ADD })) {
ggml_opencl_op_norm_fused(backend, node, cgraph->nodes[i+1], cgraph->nodes[i+2]);
i += 2;
continue;
}
- if (!backend_ctx->disable_fusion && ggml_opencl_can_fuse(cgraph, i, { GGML_OP_GROUP_NORM, GGML_OP_MUL, GGML_OP_ADD })) {
+ if (!backend_ctx->disable_fusion && ggml_opencl_can_fuse(backend_ctx, cgraph, i, { GGML_OP_GROUP_NORM, GGML_OP_MUL, GGML_OP_ADD })) {
ggml_opencl_op_group_norm_fused(backend, node, cgraph->nodes[i+1], cgraph->nodes[i+2]);
i += 2;
continue;
@@ -7441,11 +7974,35 @@ static ggml_status ggml_backend_opencl_graph_compute(ggml_backend_t backend, ggm
}
}
- if (!backend_ctx->disable_fusion && ggml_opencl_can_fuse(cgraph, i, { GGML_OP_RMS_NORM, GGML_OP_MUL })) {
+ // Fuse rms_norm + mul(weight) + add(residual). Checked before the
+ // rms_norm+mul fuse so the 3-op pattern wins over its 2-op prefix.
+ // Default on, opt-out GGML_OPENCL_FUSE_RMS_ADD=0.
+ if (!backend_ctx->disable_fusion && backend_ctx->fuse_rms_add &&
+ ggml_opencl_can_fuse(backend_ctx, cgraph, i, { GGML_OP_RMS_NORM, GGML_OP_MUL, GGML_OP_ADD })) {
+ ggml_opencl_op_rms_norm_mul_add_fused(backend, node, cgraph->nodes[i+1], cgraph->nodes[i+2]);
+ i += 2;
+ continue;
+ }
+ if (!backend_ctx->disable_fusion && ggml_opencl_can_fuse(backend_ctx, cgraph, i, { GGML_OP_RMS_NORM, GGML_OP_MUL })) {
ggml_opencl_op_rms_norm_fused(backend, node, cgraph->nodes[i+1]);
i++;
continue;
}
+ // Fuse mul_mat(Wg,x) + mul_mat(Wu,x) + glu — fold the FFN's two decode
+ // GEMVs and the GLU into one dispatch. q4_K only (guarded below); the
+ // fused kernel uses the same accumulation/reduction order and the same
+ // scalar GLU formula -> coherent. Default on, opt-out GGML_OPENCL_FUSE_MM_GLU=0.
+#ifdef GGML_OPENCL_USE_ADRENO_KERNELS
+ // The fused executor (ggml_cl_mul_mat_q4_k_glu_fused) is image-path /
+ // Adreno-only (GGML_ABORT on the non-Adreno #else); gate the dispatch to
+ // match so the FFN GLU subgraph stays dormant on Intel/other drivers.
+ if (backend_ctx->fuse_mm_glu && !backend_ctx->disable_fusion &&
+ ggml_opencl_can_fuse(backend_ctx, cgraph, i, { GGML_OP_MUL_MAT, GGML_OP_MUL_MAT, GGML_OP_GLU })) {
+ ggml_cl_mul_mat_q4_k_glu_fused(backend, node, cgraph->nodes[i+1], cgraph->nodes[i+2]);
+ i += 2;
+ continue;
+ }
+#endif
bool ok = ggml_cl_compute_forward(backend, node);
if (!ok) {
@@ -7506,6 +8063,72 @@ inline bool use_adreno_moe_kernels(const ggml_backend_opencl_context *backend_ct
return (((strstr(tensor->name, "ffn") != NULL) && (strstr(tensor->name, "exps") != NULL)) || (strstr(tensor->name, "as") != NULL)) && (ne01 % 32 == 0);
}
+// Device default for the tiled-wide lm_head/embed GEMV layout: ON for X2E and A8X.
+//
+// These kernels were previously off everywhere on the grounds that they compute
+// wrong values at multi-superblock K. They do not: that NMSE ~2 came from the
+// backend having no get_tensor restore path for the tiled layout, so
+// test-backend-ops (which builds its CPU reference by copying the weights back
+// out of the backend) compared a correct GPU result against a reference
+// dequantized from tiled bytes. With the restore path added, MUL_MAT passes with
+// the tiled kernels on, unmodified, on both devices.
+//
+// Perf, Qwen3-4B-Q4_K_M (q6_K lm_head 151936x2560), tg128, matched pairs with
+// alternating lead, tiled vs o4:
+//
+// A8X +11.9% 6/6 pairs positive, order bias -0.06% (16.93 vs 15.14 tok/s)
+// X2E +6.9% 4/4 pairs positive, order bias -0.03% (35.24 vs 32.87 tok/s)
+//
+// Measure this one on a COLD device. These kernels are far more clock-sensitive
+// than the o4 route they replace: on a heat-soaked A8X (CPU cap at 1.5-1.9 GHz)
+// tiled pins at ~14.2 tok/s while o4 still makes ~14.9, which reads as a 4-5%
+// LOSS and inverts the ranking. The same box, after a reboot and a gate that
+// waits for policy6 to return to 4396800, reports the +11.9% above with no
+// order bias. A7X regresses hard on this layout and stays off.
+// GGML_OPENCL_{Q4K,Q6K}_GEMV_TILED forces either way (=0 off, any other value on).
+inline bool tiled_gemv_default_on(const ggml_backend_opencl_context *backend_ctx) {
+ return backend_ctx && (backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E ||
+ backend_ctx->adreno_gen == ADRENO_GPU_GEN::A8X);
+}
+
+// Tiled-wide q6_K GEMV (default OFF; GGML_OPENCL_Q6K_GEMV_TILED forces either
+// way: =0 off everywhere, any other value on everywhere).
+// Both the convert (set_tensor) and the GEMV dispatch must agree on this so the
+// buffer layout matches the kernel.
+inline bool q6k_gemv_tiled_enabled(const ggml_backend_opencl_context *backend_ctx) {
+ static const char * e = std::getenv("GGML_OPENCL_Q6K_GEMV_TILED");
+ if (e && e[0] != '\0') {
+ return e[0] != '0';
+ }
+ return tiled_gemv_default_on(backend_ctx);
+}
+
+// Only the long-vocab lm_head/embed shapes use the tiled layout; ne01 % 64 == 0
+// is required by the 64-row tiling (no row padding in the buffers).
+// use_adreno_kernels is required: only the Adreno GEMV path can read the tiled
+// layout, so converting a weight it would decline (e.g. ne00 < 512) leaves the
+// generic kernel reading tiled bytes as plain SOA.
+inline bool use_q6k_tiled(const ggml_backend_opencl_context *backend_ctx, const ggml_tensor *tensor) {
+ return q6k_gemv_tiled_enabled(backend_ctx) && tensor->type == GGML_TYPE_Q6_K &&
+ tensor->ne[1] >= 32768 && tensor->ne[1] % 64 == 0 &&
+ use_adreno_kernels(backend_ctx, tensor);
+}
+
+// q4_K analog of the tiled-wide lm_head/embed GEMV (default OFF;
+// GGML_OPENCL_Q4K_GEMV_TILED forces either way: =0 off, else on). Same gate.
+inline bool q4k_gemv_tiled_enabled(const ggml_backend_opencl_context *backend_ctx) {
+ static const char * e = std::getenv("GGML_OPENCL_Q4K_GEMV_TILED");
+ if (e && e[0] != '\0') {
+ return e[0] != '0';
+ }
+ return tiled_gemv_default_on(backend_ctx);
+}
+inline bool use_q4k_tiled(const ggml_backend_opencl_context *backend_ctx, const ggml_tensor *tensor) {
+ return q4k_gemv_tiled_enabled(backend_ctx) && tensor->type == GGML_TYPE_Q4_K &&
+ tensor->ne[1] >= 32768 && tensor->ne[1] % 64 == 0 &&
+ use_adreno_kernels(backend_ctx, tensor);
+}
+
inline bool enable_adreno_trans_weight(const ggml_backend_opencl_context *backend_ctx, const ggml_tensor *tensor) {
bool adreno_kernel = use_adreno_kernels(backend_ctx, tensor);
@@ -7533,18 +8156,53 @@ inline bool enable_adreno_trans_weight_q5_K(const ggml_backend_opencl_context *b
qh_img_width <= backend_ctx->image_max_buffer_size;
}
-static inline bool use_flat_gemv_for_large_m_q4_K(const ggml_tensor *tensor) {
+// The flat-GEMV large-m escape is OPT-IN (GGML_OPENCL_FLAT_LARGE_M=1) because it
+// is SLOWER than the route it replaces, not because it is unsafe. It was first
+// parked on the theory that it out-of-bounds-writes at vocab-scale shapes; that
+// was a misattribution (the test-backend-ops dst sentinel was tripped by the o4
+// GEMV's unguarded tail store, fixed separately - and at the shape it was blamed
+// for, k=1536, this predicate returns false anyway, so the flat route never ran).
+//
+// The escape's original rationale, "gemv_noshuffle perf drops for large M",
+// predates the o4 kernel, which now covers the same long-vocab shapes and beats
+// this route on every device measured (Qwen3-4B-Q4_K_M, q6_K lm_head
+// 151936x2560, tg128, matched pairs vs o4): A8X -10.3% (0/3 pairs), X2E -3.7%
+// (0/3). Keep it reachable for shapes o4 declines, but do not default it on.
+static inline bool flat_large_m_enabled() {
+ static const char * e = getenv("GGML_OPENCL_FLAT_LARGE_M");
+ static const bool en = e != nullptr && atoi(e) != 0;
+ return en;
+}
+
+static inline bool use_flat_gemv_for_large_m_q4_K(const ggml_backend_opencl_context *backend_ctx, const ggml_tensor *tensor) {
+ if (!flat_large_m_enabled()) {
+ return false;
+ }
// gemv_noshuffle variant perf drops for large M, use flat variant for large M.
// threshold is well above typical hidden/FFN dims, but below typical vocab sizes.
// note that this forces large M weights to use LM GEMM.
- return tensor->ne[1] >= 32768 && tensor->ne[2] == 1 && tensor->ne[3] == 1;
+ // EXCEPT when this branch's tiled-canonical lm_head/embed layout is active: the
+ // weight is converted to the 64-row tiled layout, which the flat gemv would
+ // misread as garbage. use_q4k_tiled owns these large-M weights, so defer to it.
+ return tensor->ne[1] >= 32768 && tensor->ne[2] == 1 && tensor->ne[3] == 1
+ && !use_q4k_tiled(backend_ctx, tensor);
}
static inline bool use_flat_gemv_for_large_m_q6_K(const ggml_backend_opencl_context *backend_ctx, const ggml_tensor *tensor) {
+ // NOTE on ordering: the ne01 % 128 escape below is a CORRECTNESS guard, not a
+ // performance one, so it must be reachable regardless of flat_large_m_enabled().
+ // The opt-in gate therefore sits after it, and after the tiled deferral.
// gemv_noshuffle variant perf drops for large M, use flat variant for large M.
// threshold is well above typical hidden/FFN dims, but below typical vocab sizes.
// q6_K flat gemv is worse for smaller K; 2048 seems to be a reasonable threshold.
// note that this forces large M weights to use LM GEMM.
+ // When this branch's tiled-canonical lm_head/embed layout is active, the weight is
+ // converted to the 64-row tiled layout, which the flat gemv would misread as
+ // garbage. use_q6k_tiled owns these large-M weights (it requires ne01 % 64 == 0,
+ // so it never claims an odd-vocab weight), so defer to it first.
+ if (use_q6k_tiled(backend_ctx, tensor)) {
+ return false;
+ }
// The noshuffle (transposed-weight) layout packs 2 rows per 32-bit texel and the
// gemv reads it with a ne01/2 texel stride and an exact-cover dispatch of
// ceil(ne01/2 / 64)*64 work-items with no store guard; the gemm uses 4-row tiles.
@@ -7559,6 +8217,10 @@ static inline bool use_flat_gemv_for_large_m_q6_K(const ggml_backend_opencl_cont
return true;
}
+ if (!flat_large_m_enabled()) {
+ return false;
+ }
+
// The gemv_noshuffle slowdown tracks TOTAL weight size, not ne0 alone; ne0 >= 2048 is a
// proxy for "large weight" that misses a narrow-hidden vocab-scale lm_head.
// Add a direct size escape so such weights also take the flat path, without changing
@@ -7805,8 +8467,29 @@ static bool ggml_opencl_supports_op(ggml_backend_dev_t dev, const struct ggml_te
op->src[0]->ne[1] >= 32768) { // vocab-scale weight; no FFN/attn weight is this tall
return false;
}
+ // The generic mul_mv (GEMV) kernels are wrong for large-batch prefill on
+ // Adreno. A quant mul_mat only avoids the GEMV when it reaches the Adreno
+ // trans-weight GEMM, which needs both a GEMM kernel for the type and
+ // use_adreno_kernels(). Decline the large-N shapes that would otherwise
+ // fall through to the GEMV.
+ {
+ const ggml_type t = op->src[0]->type;
+ const bool type_has_gemm = (t == GGML_TYPE_Q4_0 || t == GGML_TYPE_Q4_1 ||
+ t == GGML_TYPE_IQ4_NL || t == GGML_TYPE_Q8_0 ||
+ t == GGML_TYPE_Q4_K || t == GGML_TYPE_Q5_K ||
+ t == GGML_TYPE_Q6_K);
+ const bool uses_gemm = type_has_gemm && use_adreno_kernels(backend_ctx, op->src[0]);
+ if (!uses_gemm && op->src[1]->ne[1] >= 512) {
+ return false;
+ }
+ }
return op->src[1]->type == GGML_TYPE_F32 && ggml_is_contiguous(op->src[0]) && ggml_is_contiguous(op->src[1]);
} else if (op->src[0]->type == GGML_TYPE_Q8_0) {
+ // ggml_cl_mul_mat_q8_0_f32_adreno now honors src1/dst view_offs (the
+ // activation sub-buffer starts at offset1 and the kernels take offsetd),
+ // so a broadcast q8_0 matmul (src1 batch > src0 batch, e.g. Qwen3.5-9B-UD
+ // / Qwen3.6-35B q8_0 GDN ssm_out) runs on GPU via the per-slice broadcast
+ // iteration in ggml_cl_mul_mat. No special-casing needed.
return op->src[1]->type == GGML_TYPE_F32;
}
return false;
@@ -9574,8 +10257,41 @@ static void ggml_backend_opencl_buffer_set_tensor(ggml_backend_buffer_t buffer,
#endif // GGML_OPENCL_USE_ADRENO_KERNELS
#ifdef GGML_OPENCL_USE_ADRENO_KERNELS
+ // Tiled-wide convert for the long-vocab lm_head/embed (opt-in). The embed/
+ // output q4_K weight (token_embd.weight, ne1=vocab) is NOT matched by
+ // use_adreno_moe_kernels, so it lands here in the general branch. Produce
+ // the final 64-row-tiled canonical layout directly into q/d/dm/s (buffer
+ // sizes already match), read back by kernel_gemv_noshuffle_q4_k_f32_tiled.
+ if (use_q4k_tiled(backend_ctx, tensor)) {
+ cl_kernel tk = backend_ctx->kernel_convert_block_q4_k_tiled_ns;
+
+ int ne00 = tensor->ne[0];
+ int ne01 = tensor->ne[1];
+ int ne02 = tensor->ne[2];
+
+ CL_CHECK(clSetKernelArg(tk, 0, sizeof(cl_mem), &data_device));
+ CL_CHECK(clSetKernelArg(tk, 1, sizeof(cl_mem), &extra->q));
+ CL_CHECK(clSetKernelArg(tk, 2, sizeof(cl_mem), &extra->d));
+ CL_CHECK(clSetKernelArg(tk, 3, sizeof(cl_mem), &extra->dm));
+ CL_CHECK(clSetKernelArg(tk, 4, sizeof(cl_mem), &extra->s));
+ CL_CHECK(clSetKernelArg(tk, 5, sizeof(int), &ne00));
+ CL_CHECK(clSetKernelArg(tk, 6, sizeof(int), &ne01));
+
+ size_t gws[] = {static_cast<size_t>(((ne01 + 63) / 64) * 64), static_cast<size_t>(ne00 / 256), static_cast<size_t>(ne02)};
+ size_t lws[] = {64, 1, 1};
+
+ cl_event tevt;
+ CL_CHECK(clEnqueueNDRangeKernel(queue, tk, 3, NULL, gws, lws, 0, NULL, &tevt));
+ CL_CHECK(clWaitForEvents(1, &tevt));
+ CL_CHECK(clReleaseMemObject(data_device));
+
+ extra->q_img = nullptr;
+ tensor->extra = extra;
+ return;
+ }
+
cl_kernel kernel = backend_ctx->kernel_convert_block_q4_K;
- if (use_adreno_kernels(backend_ctx, tensor) && !use_flat_gemv_for_large_m_q4_K(tensor)) {
+ if (use_adreno_kernels(backend_ctx, tensor) && !use_flat_gemv_for_large_m_q4_K(backend_ctx, tensor)) {
kernel = backend_ctx->kernel_convert_block_q4_K_noshuffle;
}
#else
@@ -9603,7 +10319,7 @@ static void ggml_backend_opencl_buffer_set_tensor(ggml_backend_buffer_t buffer,
tensor->extra = extra;
#ifdef GGML_OPENCL_USE_ADRENO_KERNELS
- if (use_adreno_kernels(backend_ctx, tensor) && !use_flat_gemv_for_large_m_q4_K(tensor)) {
+ if (use_adreno_kernels(backend_ctx, tensor) && !use_flat_gemv_for_large_m_q4_K(backend_ctx, tensor)) {
int M = tensor->ne[1];
int K = tensor->ne[0];
@@ -9929,6 +10645,45 @@ static void ggml_backend_opencl_buffer_set_tensor(ggml_backend_buffer_t buffer,
CL_CHECK((extra->d = clCreateSubBuffer(extra_orig->data_device, CL_MEM_READ_WRITE, CL_BUFFER_CREATE_TYPE_REGION, &region, &err), err));
previous_origin = region.origin;
+#ifdef GGML_OPENCL_USE_ADRENO_KERNELS
+ // Tiled-wide convert for the long-vocab lm_head/embed (opt-in). The embed
+ // /output q6_K weight (e.g. token_embd.weight, ne1=vocab) is NOT matched by
+ // use_adreno_moe_kernels, so it lands here in the general branch. Produce
+ // the final 64-row-tiled canonical layout directly into ql/qh/s/d (buffer
+ // sizes already match), read back by kernel_gemv_noshuffle_q6_K_f32_tiled.
+ // Bypasses the plain-SOA convert + per-array transpose below.
+ if (use_q6k_tiled(backend_ctx, tensor)) {
+ cl_kernel kernel = backend_ctx->kernel_convert_block_q6_k_tiled_ns;
+
+ int ne00 = tensor->ne[0];
+ int ne01 = tensor->ne[1];
+ int ne02 = tensor->ne[2];
+
+ CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &data_device));
+ CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->ql));
+ CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra->qh));
+ CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra->d));
+ CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extra->s));
+ CL_CHECK(clSetKernelArg(kernel, 5, sizeof(int), &ne00));
+ CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne01));
+
+ size_t global_work_size[] = {static_cast<size_t>(((ne01 + 63) / 64) * 64), static_cast<size_t>(ne00 / 256), static_cast<size_t>(ne02)};
+ size_t local_work_size[] = {64, 1, 1};
+
+ cl_event evt;
+ CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, global_work_size, local_work_size, 0, NULL, &evt));
+ CL_CHECK(clWaitForEvents(1, &evt));
+ CL_CHECK(clReleaseMemObject(data_device));
+
+ extra->size_ql = size_ql;
+ extra->size_qh = size_qh;
+ extra->size_s = size_s;
+ extra->size_d = size_d;
+ tensor->extra = extra;
+ return;
+ }
+#endif // GGML_OPENCL_USE_ADRENO_KERNELS
+
// Flatten the weights
cl_kernel kernel;
#ifdef GGML_OPENCL_USE_ADRENO_KERNELS
@@ -10744,6 +11499,54 @@ static void ggml_backend_opencl_buffer_get_tensor(ggml_backend_buffer_t buffer,
cl_uchar mask_F0 = 0xF0;
#ifdef GGML_OPENCL_USE_ADRENO_KERNELS
+ // Undo the 64-row-tiled canonical pack (kernel_convert_block_q4_k_tiled_ns).
+ // Without this, a read-back of a tiled weight returns the tiled bytes
+ // reinterpreted as block_q4_K -- which is how test-backend-ops builds its
+ // CPU reference (ggml_backend_graph_copy -> tensor_get), so the tiled path
+ // "failed" the suite while computing the correct product.
+ if (use_q4k_tiled(backend_ctx, tensor)) {
+ const int ne00v = tensor->ne[0];
+ const int ne01v = tensor->ne[1];
+ const int nbv = ne00v / 256;
+ const size_t n_blk = (size_t)nbv * ne01v;
+
+ std::vector<uint32_t> tq(n_blk*32);
+ std::vector<uint16_t> td(n_blk), tdm(n_blk);
+ std::vector<uint8_t> ts(n_blk*12);
+ CL_CHECK(clEnqueueReadBuffer(queue, extra->q, CL_TRUE, 0, tq.size()*4, tq.data(), 0, NULL, NULL));
+ CL_CHECK(clEnqueueReadBuffer(queue, extra->d, CL_TRUE, 0, td.size()*2, td.data(), 0, NULL, NULL));
+ CL_CHECK(clEnqueueReadBuffer(queue, extra->dm, CL_TRUE, 0, tdm.size()*2, tdm.data(), 0, NULL, NULL));
+ CL_CHECK(clEnqueueReadBuffer(queue, extra->s, CL_TRUE, 0, ts.size(), ts.data(), 0, NULL, NULL));
+
+ std::vector<uint8_t> rebuilt(ggml_nbytes(tensor), 0);
+ for (int i01 = 0; i01 < ne01v; ++i01) {
+ const int rt = i01/64, rit = i01%64;
+ for (int i00 = 0; i00 < nbv; ++i00) {
+ uint8_t * b = rebuilt.data() + ((size_t)i00 + (size_t)i01*nbv)*144;
+ const int tb = rt*nbv + i00;
+ const size_t si = (size_t)tb*64 + rit;
+
+ memcpy(b + 0, &td [si], 2);
+ memcpy(b + 2, &tdm[si], 2);
+ memcpy(b + 4, &ts[si*12], 12);
+
+ uint32_t qw[32];
+ for (int gr = 0; gr < 8; ++gr) {
+ const size_t base = ((size_t)tb*8 + gr)*64 + rit;
+ for (int j = 0; j < 4; ++j) qw[gr*4 + j] = tq[base*4 + j];
+ }
+ uint8_t * q = b + 16;
+ for (int e = 0; e < 256; ++e) {
+ const int g = e>>6, w = e&63, h = w>>5, l = w&31;
+ const uint32_t code = (qw[e>>3] >> ((e&7)*4)) & 0xF;
+ q[g*32 + l] |= (uint8_t)(h ? (code << 4) : code);
+ }
+ }
+ }
+ memcpy(data, rebuilt.data() + offset, size);
+ CL_CHECK(clReleaseMemObject(data_device));
+ return;
+ }
if (use_adreno_moe_kernels(backend_ctx, tensor)) {
cl_int err;
cl_mem data_device = clCreateBuffer(context, CL_MEM_READ_WRITE,
@@ -10778,7 +11581,7 @@ static void ggml_backend_opencl_buffer_get_tensor(ggml_backend_buffer_t buffer,
CL_CHECK(clReleaseMemObject(data_device));
return;
}
- if (use_adreno_kernels(backend_ctx, tensor) && !use_flat_gemv_for_large_m_q4_K(tensor)) {
+ if (use_adreno_kernels(backend_ctx, tensor) && !use_flat_gemv_for_large_m_q4_K(backend_ctx, tensor)) {
int M = tensor->ne[1];
int K = tensor->ne[0];
@@ -10966,6 +11769,61 @@ static void ggml_backend_opencl_buffer_get_tensor(ggml_backend_buffer_t buffer,
ggml_tensor_extra_cl_q6_K * extra = (ggml_tensor_extra_cl_q6_K *)tensor->extra;
#ifdef GGML_OPENCL_USE_ADRENO_KERNELS
+ // Undo the 64-row-tiled canonical pack (kernel_convert_block_q6_k_tiled_ns).
+ // See the q4_K tiled restore above for why a read-back path is required.
+ if (use_q6k_tiled(backend_ctx, tensor)) {
+ const int ne00v = tensor->ne[0];
+ const int ne01v = tensor->ne[1];
+ const int nbv = ne00v / 256;
+ const size_t n_blk = (size_t)nbv * ne01v;
+
+ std::vector<uint32_t> tql(n_blk*32), tqh(n_blk*16);
+ std::vector<uint8_t> ts(n_blk*16);
+ std::vector<uint16_t> td(n_blk);
+ CL_CHECK(clEnqueueReadBuffer(queue, extra->ql, CL_TRUE, 0, tql.size()*4, tql.data(), 0, NULL, NULL));
+ CL_CHECK(clEnqueueReadBuffer(queue, extra->qh, CL_TRUE, 0, tqh.size()*4, tqh.data(), 0, NULL, NULL));
+ CL_CHECK(clEnqueueReadBuffer(queue, extra->s, CL_TRUE, 0, ts.size(), ts.data(), 0, NULL, NULL));
+ CL_CHECK(clEnqueueReadBuffer(queue, extra->d, CL_TRUE, 0, td.size()*2, td.data(), 0, NULL, NULL));
+
+ std::vector<uint8_t> rebuilt(ggml_nbytes(tensor), 0);
+ for (int i01 = 0; i01 < ne01v; ++i01) {
+ const int rt = i01/64, rit = i01%64;
+ for (int i00 = 0; i00 < nbv; ++i00) {
+ uint8_t * b = rebuilt.data() + ((size_t)i00 + (size_t)i01*nbv)*210;
+ const int tb = rt*nbv + i00;
+ const size_t si = (size_t)tb*64 + rit;
+
+ uint32_t qlw[32], qhw[16];
+ for (int g = 0; g < 8; ++g) {
+ const size_t base = ((size_t)tb*8 + g)*64 + rit;
+ for (int j = 0; j < 4; ++j) qlw[g*4 + j] = tql[base*4 + j];
+ }
+ for (int g = 0; g < 4; ++g) {
+ const size_t base = ((size_t)tb*4 + g)*64 + rit;
+ for (int j = 0; j < 4; ++j) qhw[g*4 + j] = tqh[base*4 + j];
+ }
+
+ uint8_t * ql = b;
+ uint8_t * qh = b + 128;
+ for (int e = 0; e < 256; ++e) {
+ const int n = (e >= 128) ? 1 : 0;
+ const int within = e - n*128, q = within/32, l = within%32;
+ const int off_ql = n*64, off_qh = n*32;
+ const uint8_t low4 = (qlw[e>>3] >> ((e&7)*4)) & 0xF;
+ const uint8_t hi2 = (qhw[e>>4] >> ((e&15)*2)) & 0x3;
+ if (q == 0) ql[off_ql + l] |= low4;
+ else if (q == 1) ql[off_ql + l + 32] |= low4;
+ else if (q == 2) ql[off_ql + l] |= (uint8_t)(low4 << 4);
+ else ql[off_ql + l + 32] |= (uint8_t)(low4 << 4);
+ qh[off_qh + l] |= (uint8_t)(hi2 << (q*2));
+ }
+ memcpy(b + 192, &ts[si*16], 16);
+ memcpy(b + 208, &td[si], 2);
+ }
+ }
+ memcpy(data, rebuilt.data() + offset, size);
+ return;
+ }
if (use_adreno_moe_kernels(backend_ctx, tensor)) {
cl_int err;
cl_mem data_device = clCreateBuffer(context, CL_MEM_READ_WRITE,
@@ -16634,7 +17492,17 @@ static void ggml_cl_mul_mat_q4_0_f32_adreno(ggml_backend_t backend, const ggml_t
int N = ne1;
int K = ne00;
- if (ne1 == 1) {
+ // Multi-column (N=3) verify GEMV for q4_0: route the spec/MTP verify batch
+ // (ne1==3) onto the efficient GEMV path instead of the transposed-GEMM dead-
+ // zone (gemm_noshuffle_q4_0 is ~50% of MTP decode on a Q4_0 model since q4_0
+ // weights have no cok/mc3, unlike q4_K). Reuses the ne1==1 GEMV image setup
+ // (activation image already sized by N=ne1). Byte-identical. Opt-in via
+ // GGML_OPENCL_Q40_MC3=1. Per-layer only (ne01 < 32768); q4_0 lm_head doesn't
+ // occur (token_embd/output stay Q6_K), guard kept for parity with q4_K mc3.
+ static const bool q40_mc3 = (getenv("GGML_OPENCL_Q40_MC3") != nullptr);
+ const bool use_q40_mc3 = q40_mc3 && (ne1 >= 2 && ne1 <= 4) && (ne01 < 32768);
+
+ if (ne1 == 1 || use_q40_mc3) {
cl_mem q_img = nullptr;
cl_mem b_sub_buf = nullptr;
cl_mem b_img = nullptr;
@@ -16660,38 +17528,56 @@ static void ggml_cl_mul_mat_q4_0_f32_adreno(ggml_backend_t backend, const ggml_t
img_desc.buffer = b_sub_buf;
CL_CHECK((b_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err));
- kernel = backend_ctx->kernel_gemv_noshuffle_q4_0_f32;
- if (M == 4096 && K == 4096) {
- kernel = backend_ctx->kernel_gemv_noshuffle_q4_0_f32_4096_1_4096;
- } else if (M == 4096 && K == 11008) {
- kernel = backend_ctx->kernel_gemv_noshuffle_q4_0_f32_4096_1_11008;
- } else if (M == 11008 && K == 4096) {
- kernel = backend_ctx->kernel_gemv_noshuffle_q4_0_f32_11008_1_4096;
- } else if (M == 32000 && K == 4096) {
- kernel = backend_ctx->kernel_gemv_noshuffle_q4_0_f32_32000_1_4096;
- }
+ if (use_q40_mc3) {
+ kernel = backend_ctx->kernel_gemv_noshuffle_q4_0_f32_mc3;
+ CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &q_img));
+ CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q4_0->d));
+ CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &b_img));
+ CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extrad->data_device));
+ CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_ulong), &offsetd));
+ CL_CHECK(clSetKernelArg(kernel, 5, sizeof(int), &ne00));
+ CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne01));
+ CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne1));
+ } else {
+ kernel = backend_ctx->kernel_gemv_noshuffle_q4_0_f32;
+ if (M == 4096 && K == 4096) {
+ kernel = backend_ctx->kernel_gemv_noshuffle_q4_0_f32_4096_1_4096;
+ } else if (M == 4096 && K == 11008) {
+ kernel = backend_ctx->kernel_gemv_noshuffle_q4_0_f32_4096_1_11008;
+ } else if (M == 11008 && K == 4096) {
+ kernel = backend_ctx->kernel_gemv_noshuffle_q4_0_f32_11008_1_4096;
+ } else if (M == 32000 && K == 4096) {
+ kernel = backend_ctx->kernel_gemv_noshuffle_q4_0_f32_32000_1_4096;
+ }
- int r2 = 1;
- int r3 = 1;
+ int r2 = 1;
+ int r3 = 1;
- CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &q_img));
- CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q4_0->d));
- CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &b_img));
- CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1));
- CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device));
- CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd));
- CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne00));
- CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne01));
- CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne02));
- CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne10));
- CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne12));
- CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne0));
- CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne1));
- CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &r2));
- CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &r3));
+ CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &q_img));
+ CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q4_0->d));
+ CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &b_img));
+ CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1));
+ CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device));
+ CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd));
+ CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne00));
+ CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne01));
+ CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne02));
+ CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne10));
+ CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne12));
+ CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne0));
+ CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne1));
+ CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &r2));
+ CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &r3));
+ }
- size_t local_work_size[3] = {64, 4, 1};
- size_t global_work_size[3] = {(size_t)CEIL_DIV(ne01/2, 64)*64, 4, 1};
+ // Small-M mc3 verify is occupancy/latency-bound (too few WGs at small M, so
+ // its bandwidth falls well short of the FFN matmuls'). Use 8 subgroups (512-WI WGs, half the
+ // per-lane K-walk) for small M. Layout stride is fixed (4 uints/block), so only
+ // the K-split count changes; the mc3 kernel reads it via get_local_size(1). The
+ // ne1==1 base kernel hardcodes N_SIMDGROUP=4, so it always stays at 4.
+ const int mc3_nsg = (use_q40_mc3 && ne01 < 4096) ? 8 : 4;
+ size_t local_work_size[3] = {64, (size_t)mc3_nsg, 1};
+ size_t global_work_size[3] = {(size_t)CEIL_DIV(ne01/2, 64)*64, (size_t)mc3_nsg, 1};
backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst);
@@ -16909,7 +17795,14 @@ static void ggml_cl_mul_mat_q4_1_f32_adreno(ggml_backend_t backend, const ggml_t
int N = ne1;
int K = ne00;
- if (ne1 == 1) {
+ // Multi-column (N=3) verify GEMV for q4_1: route the spec/MTP verify batch
+ // (ne1==3) onto the efficient GEMV path instead of the transposed-GEMM dead-
+ // zone (gemm_noshuffle_q4_1). Reuses the ne1==1 GEMV image setup. Opt-in via
+ // GGML_OPENCL_Q41_MC3=1. Per-layer only (ne01 < 32768).
+ static const bool q41_mc3 = (getenv("GGML_OPENCL_Q41_MC3") != nullptr);
+ const bool use_q41_mc3 = q41_mc3 && (ne1 >= 2 && ne1 <= 4) && (ne01 < 32768);
+
+ if (ne1 == 1 || use_q41_mc3) {
cl_mem q_img = nullptr;
cl_mem b_sub_buf = nullptr;
cl_mem b_img = nullptr;
@@ -16935,7 +17828,8 @@ static void ggml_cl_mul_mat_q4_1_f32_adreno(ggml_backend_t backend, const ggml_t
img_desc.buffer = b_sub_buf;
CL_CHECK((b_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err));
- kernel = backend_ctx->kernel_gemv_noshuffle_q4_1_f32;
+ kernel = use_q41_mc3 ? backend_ctx->kernel_gemv_noshuffle_q4_1_f32_mc3
+ : backend_ctx->kernel_gemv_noshuffle_q4_1_f32;
CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &q_img));
CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q4_1->d));
@@ -16945,6 +17839,9 @@ static void ggml_cl_mul_mat_q4_1_f32_adreno(ggml_backend_t backend, const ggml_t
CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd));
CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_int), &ne00));
CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_int), &ne01));
+ if (use_q41_mc3) {
+ CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_int), &ne1)); // n_cols
+ }
size_t local_work_size[3] = {64, 4, 1};
size_t global_work_size[3] = {(size_t)CEIL_DIV(ne01/2, 64)*64, 4, 1};
@@ -17788,6 +18685,66 @@ static void ggml_cl_mul_mat_q8_0_f32_adreno(ggml_backend_t backend, const ggml_t
img_desc.buffer = b_sub_buf;
CL_CHECK((b_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err));
+ // Split-K for small-M decode GEMVs. The base kernel puts one output row
+ // per lane and splits K only inside one workgroup, so M is the sole source
+ // of workgroup parallelism: gpt-oss's K and V projections are M=512 = 8
+ // workgroups on a 16-CU X2, and the kernel measures 48 GB/s where the
+ // M=2880/4096 projections in the same decode graph reach 122-123. Mirrors
+ // the q4_0/q4_K split-K above and reuses their reduce kernel.
+ //
+ // Enabled where it is measured to win, like the q4_K gate: X2-90 +2.8%
+ // tg32 @d4096 on gpt-oss; Adreno 840 (12 CU) NEUTRAL on Llama-3.2-3B-Q8_0
+ // (0.0% @d4096 -- its K/V proj is M=1024 = 16 workgroups, which already
+ // fills 12 CUs). Unmeasured on X1E/A7X/A6X and the q4_K split-K measured
+ // -0.7% on X1E, so the default is not widened on absence of evidence.
+ static const bool q8_splitk_env_set = []{
+ const char * e = std::getenv("GGML_OPENCL_Q8_GEMV_SPLITK");
+ return e && e[0] != '\0';
+ }();
+ static const bool q8_splitk_env_on = []{
+ const char * e = std::getenv("GGML_OPENCL_Q8_GEMV_SPLITK");
+ return !(e && e[0] == '0');
+ }();
+ const bool q8_splitk_on = q8_splitk_env_set
+ ? q8_splitk_env_on
+ : (backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E);
+ if (q8_splitk_on && backend_ctx->kernel_gemv_noshuffle_q8_0_f32_splitk &&
+ ne01 <= 1024 && ne01 % 64 == 0) {
+ const int nsg = 8;
+ const int ksplit = 8; // -> 8 * M/64 workgroups
+ const size_t gx = (size_t) CEIL_DIV(ne01, 64) * 64;
+
+ backend_ctx->prealloc_splitk_partial.allocate(
+ backend_ctx->context, (size_t) ksplit * ne01 * sizeof(float));
+ cl_mem partial = backend_ctx->prealloc_splitk_partial.buffer;
+
+ cl_kernel ks = backend_ctx->kernel_gemv_noshuffle_q8_0_f32_splitk;
+ CL_CHECK(clSetKernelArg(ks, 0, sizeof(cl_mem), &q_img));
+ CL_CHECK(clSetKernelArg(ks, 1, sizeof(cl_mem), &extra0_q8_0->d));
+ CL_CHECK(clSetKernelArg(ks, 2, sizeof(cl_mem), &b_img));
+ CL_CHECK(clSetKernelArg(ks, 3, sizeof(cl_mem), &partial));
+ CL_CHECK(clSetKernelArg(ks, 4, sizeof(cl_int), &ne00));
+ CL_CHECK(clSetKernelArg(ks, 5, sizeof(cl_int), &ne01));
+ size_t lsk[3] = { 64, (size_t) nsg, 1 };
+ size_t gsk[3] = { gx, (size_t) (nsg * ksplit), 1 };
+ backend_ctx->enqueue_ndrange_kernel(ks, 3, gsk, lsk, dst);
+
+ cl_kernel kr = backend_ctx->kernel_gemv_splitk_reduce_f32;
+ CL_CHECK(clSetKernelArg(kr, 0, sizeof(cl_mem), &partial));
+ CL_CHECK(clSetKernelArg(kr, 1, sizeof(cl_mem), &extrad->data_device));
+ CL_CHECK(clSetKernelArg(kr, 2, sizeof(cl_ulong), &offsetd));
+ CL_CHECK(clSetKernelArg(kr, 3, sizeof(cl_int), &ne01));
+ CL_CHECK(clSetKernelArg(kr, 4, sizeof(cl_int), &ksplit));
+ size_t lr[3] = { 64, 1, 1 };
+ size_t gr[3] = { (size_t) CEIL_DIV(ne01, 64) * 64, 1, 1 };
+ backend_ctx->enqueue_ndrange_kernel(kr, 3, gr, lr, dst);
+
+ CL_CHECK(clReleaseMemObject(q_img));
+ CL_CHECK(clReleaseMemObject(b_img));
+ CL_CHECK(clReleaseMemObject(b_sub_buf));
+ return;
+ }
+
kernel = backend_ctx->kernel_gemv_noshuffle_q8_0_f32;
int r2 = 1;
@@ -18129,18 +19086,33 @@ static void ggml_cl_mul_mat_q4_k_f32_adreno(ggml_backend_t backend, const ggml_t
cl_uchar mask_d4 = 0x0F;
cl_uchar mask_hi2 = 0xC0;
- if (ne1 == 1) {
+ // Multi-column verify GEMV: route the spec/MTP verify batch (ne1==3 = 2
+ // drafts + 1 bonus) onto the efficient GEMV path (subgroup-broadcast, no
+ // transpose) instead of the transposed-GEMM dead-zone. Reuses the ne1==1
+ // GEMV setup (the activation image is already sized by N=ne1). Byte-
+ // identical. Opt-in via GGML_OPENCL_Q4K_MC3=1 while validating.
+ static const bool q4k_mc3 = (getenv("GGML_OPENCL_Q4K_MC3") != nullptr);
+ // Per-layer only (ne01 < 32768): the batched large-vocab lm_head at ne1==3
+ // is left to the existing routing (corrupts on the Adreno GEMV path; x2-
+ // unified routes batched Q6_K lm_head to CPU). Per-layer mc3 is byte-identical.
+ const bool use_mc3 = q4k_mc3 && (ne1 == 3) && (ne01 < 32768);
+
+ if (ne1 == 1 || use_mc3) {
cl_mem q_img = nullptr;
cl_mem b_sub_buf = nullptr;
cl_mem b_img = nullptr;
- // image for q
- img_fmt = { CL_R, CL_UNSIGNED_INT32};
- memset(&img_desc, 0, sizeof(img_desc));
- img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER;
- img_desc.image_width = M * K / 2 / 4;
- img_desc.buffer = extra0_q4_k->q;
- CL_CHECK((q_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err));
+ const bool use_tiled = !use_mc3 && use_q4k_tiled(backend_ctx, src0);
+
+ // image for q (not needed for the tiled path, which reads __global)
+ if (!use_tiled) {
+ img_fmt = { CL_R, CL_UNSIGNED_INT32};
+ memset(&img_desc, 0, sizeof(img_desc));
+ img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER;
+ img_desc.image_width = M * K / 2 / 4;
+ img_desc.buffer = extra0_q4_k->q;
+ CL_CHECK((q_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err));
+ }
// subbuffer for activations
region.origin = offset1;
@@ -18155,27 +19127,173 @@ static void ggml_cl_mul_mat_q4_k_f32_adreno(ggml_backend_t backend, const ggml_t
img_desc.buffer = b_sub_buf;
CL_CHECK((b_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err));
- kernel = backend_ctx->kernel_gemv_noshuffle_q4_k_f32;
-
- CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &q_img));
- CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q4_k->d));
- CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_q4_k->dm));
- CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra0_q4_k->s));
- CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &b_img));
- CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_mem), &extrad->data_device));
- CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_ulong), &offsetd));
- CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_int), &ne00));
- CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_int), &ne01));
- CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_uchar), &mask_d6));
- CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_uchar), &mask_d4));
- CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_uchar), &mask_hi2));
+ // 4-output-per-WI o4 variant for the long-vocab lm_head/embed GEMV
+ // (ne01 = vocab ~256K on Gemma): shares one activation read across 4
+ // output rows. Gated to large ne01 (lm_head/embed). Default on; opt-out
+ // GGML_OPENCL_Q4K_GEMV_O4=0. (Skipped when mc3 handles the ne1==3 verify.)
+ static const bool q4k_o4_env = []{
+ const char * e = std::getenv("GGML_OPENCL_Q4K_GEMV_O4");
+ return !e || e[0] == '\0' || e[0] != '0';
+ }();
+ const bool use_q4k_o4 = !use_tiled && !use_mc3 && q4k_o4_env && (ne01 % 4 == 0) && (ne01 >= 32768);
+ // Split-K across workgroups for small-M decode GEMVs. A single-token GEMV
+ // makes only CEIL_DIV(M/2,64) workgroups; even with the wide intra-WG split
+ // (16 subgroups) those all land on ONE CU, so small-M matmuls under-fill the
+ // 16 CUs and their bandwidth falls well short of what the large-M FFN matmuls
+ // reach. Adding a `ksplit` second grid dim that spreads K across WGs (+ a
+ // reduce pass) fills the CUs. Gate is M<=2560: the tiny M<=1024 ones only
+ // break even (the reduce dispatch eats the kernel win), but the big-K M=2560
+ // cases (ffn_down, attn_output) make the per-call win dwarf the reduce, and
+ // are byte-identical. ffn_gate/up (large M) fill the CUs already and are excluded.
+ //
+ // DEVICE-GATED. Split-K buys GPU time by spending an extra kernel LAUNCH (the
+ // reduce), so it only pays where launches are cheap. That is a per-device
+ // property and it does not travel from the X2-90 this was tuned on. Measured
+ // with one binary, env A/B (tg32, GGML_OPENCL_Q4K_GEMV_SPLITK=0/1):
+ //
+ // X2-90 +3.36% gemma-4 E4B (the number this gate was built on)
+ // 840 -1.3% Qwen3.5-4B-Q4_K_M 14.00 -> 13.85
+ // 850 -20.0% Qwen3-1.7B-Q4_K_M 6.97 -> 5.58 (6 interleaved reps)
+ //
+ // The kernel is not the problem. On the 850 split-K makes the GPU strictly
+ // faster -- total busy 537 -> 485 ms, this GEMV 43.7 -> 34.0 us/call (-22%) --
+ // and still costs a fifth of decode, because the +3696 reduce dispatches cost
+ // ~550 us of HOST round-trip each against 2.7 us of GPU work (~200x; that part
+ // is ~95% host-bound at decode). The 840 pays the same tax at ~42 us/dispatch.
+ // Break-even needs launch cost below the ~9.7 us/call the split actually saves,
+ // so this is not a "the 850 is slow" adjustment that a faster part would fix --
+ // the 840 is 13x cheaper per launch and still loses.
+ //
+ // Enabled where it is measured to win, i.e. X2E only. The X1-85 was measured
+ // afterwards and is NOT a win either: Qwen3.5-4B-Q4_K_M tg32, split-K off
+ // 17.98/18.10/18.19 vs on 18.03/17.91/17.97 = -0.7%, so X1E stays excluded on
+ // evidence rather than on absence of it. Do not widen this without a NEW
+ // measurement. The env still forces either way so every device stays measurable.
+ static const bool splitk_env_set = []{
+ const char * e = std::getenv("GGML_OPENCL_Q4K_GEMV_SPLITK");
+ return e && e[0] != '\0';
+ }();
+ static const bool splitk_env_on = []{
+ const char * e = std::getenv("GGML_OPENCL_Q4K_GEMV_SPLITK");
+ return !(e && e[0] == '0');
+ }();
+ const bool splitk_wg_env = splitk_env_set
+ ? splitk_env_on
+ : (backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E);
+ // Gate: small-M decode GEMVs that under-fill the 16 CUs even with the wide
+ // intra-WG split (all 16 subgroups land on one CU). M<=2560 covers Kcur/Vcur
+ // (M=1024), Qcur (2048), attn_output + ffn_down (2560). The tiny ones
+ // (M<=1024) only break even (reduce dispatch eats the kernel win), but the
+ // big-K M=2560 cases (ffn_down K=10240 @182us, attn_output @42us) have a
+ // large per-call win that dwarfs the ~5us reduce, so extending to 2560 nets
+ // positive end-to-end. ffn_gate/up (M=10240) already fill the CUs -> excluded.
+ const bool use_splitk = splitk_wg_env && !use_tiled && !use_q4k_o4 && !use_mc3 && ne01 <= 2560;
+
+ if (use_splitk) {
+ const int nsg = 8;
+ const int ksplit = (ne01 <= 512) ? 8 : 4; // -> ~32 total WGs
+ const size_t gx = (size_t)CEIL_DIV(ne01/2, 64) * 64;
+
+ backend_ctx->prealloc_splitk_partial.allocate(
+ backend_ctx->context, (size_t)ksplit * ne01 * sizeof(float));
+ cl_mem partial = backend_ctx->prealloc_splitk_partial.buffer;
+
+ cl_kernel ks = backend_ctx->kernel_gemv_noshuffle_q4_k_f32_splitk;
+ CL_CHECK(clSetKernelArg(ks, 0, sizeof(cl_mem), &q_img));
+ CL_CHECK(clSetKernelArg(ks, 1, sizeof(cl_mem), &extra0_q4_k->d));
+ CL_CHECK(clSetKernelArg(ks, 2, sizeof(cl_mem), &extra0_q4_k->dm));
+ CL_CHECK(clSetKernelArg(ks, 3, sizeof(cl_mem), &extra0_q4_k->s));
+ CL_CHECK(clSetKernelArg(ks, 4, sizeof(cl_mem), &b_img));
+ CL_CHECK(clSetKernelArg(ks, 5, sizeof(cl_mem), &partial));
+ CL_CHECK(clSetKernelArg(ks, 6, sizeof(cl_int), &ne00));
+ CL_CHECK(clSetKernelArg(ks, 7, sizeof(cl_int), &ne01));
+ CL_CHECK(clSetKernelArg(ks, 8, sizeof(cl_uchar), &mask_d6));
+ CL_CHECK(clSetKernelArg(ks, 9, sizeof(cl_uchar), &mask_d4));
+ CL_CHECK(clSetKernelArg(ks, 10, sizeof(cl_uchar), &mask_hi2));
+ size_t lsk[3] = {64, (size_t)nsg, 1};
+ size_t gsk[3] = {gx, (size_t)(nsg * ksplit), 1};
+ backend_ctx->enqueue_ndrange_kernel(ks, 3, gsk, lsk, dst);
+
+ cl_kernel kr = backend_ctx->kernel_gemv_splitk_reduce_f32;
+ CL_CHECK(clSetKernelArg(kr, 0, sizeof(cl_mem), &partial));
+ CL_CHECK(clSetKernelArg(kr, 1, sizeof(cl_mem), &extrad->data_device));
+ CL_CHECK(clSetKernelArg(kr, 2, sizeof(cl_ulong), &offsetd));
+ CL_CHECK(clSetKernelArg(kr, 3, sizeof(cl_int), &ne01));
+ CL_CHECK(clSetKernelArg(kr, 4, sizeof(cl_int), &ksplit));
+ size_t lr[3] = {64, 1, 1};
+ size_t gr[3] = {(size_t)CEIL_DIV(ne01, 64) * 64, 1, 1};
+ backend_ctx->enqueue_ndrange_kernel(kr, 3, gr, lr, dst);
+
+ if (q_img) CL_CHECK(clReleaseMemObject(q_img));
+ CL_CHECK(clReleaseMemObject(b_sub_buf));
+ CL_CHECK(clReleaseMemObject(b_img));
+ return;
+ }
- size_t local_work_size[3] = {64, 4, 1};
- size_t global_work_size[3] = {(size_t)CEIL_DIV(ne01/2, 64)*64, 4, 1};
+ kernel = use_mc3 ? backend_ctx->kernel_gemv_noshuffle_q4_k_f32_mc3
+ : use_tiled ? backend_ctx->kernel_gemv_noshuffle_q4_k_f32_tiled
+ : use_q4k_o4 ? backend_ctx->kernel_gemv_noshuffle_q4_k_f32_o4
+ : backend_ctx->kernel_gemv_noshuffle_q4_k_f32;
+
+ if (use_tiled) {
+ CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q4_k->q));
+ CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q4_k->d));
+ CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_q4_k->dm));
+ CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra0_q4_k->s));
+ CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &b_img));
+ CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_mem), &extrad->data_device));
+ CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_ulong), &offsetd));
+ CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_int), &ne00));
+ CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_int), &ne01));
+ } else {
+ CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &q_img));
+ CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q4_k->d));
+ CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_q4_k->dm));
+ CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra0_q4_k->s));
+ CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &b_img));
+ CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_mem), &extrad->data_device));
+ CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_ulong), &offsetd));
+ CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_int), &ne00));
+ CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_int), &ne01));
+ CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_uchar), &mask_d6));
+ CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_uchar), &mask_d4));
+ CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_uchar), &mask_hi2));
+ }
+
+ // Wide K-split for the decode GEMV: the default 4-subgroup K-split leaves
+ // each Adreno SP with only ~4 waves, too few to hide LPDDR weight-load
+ // latency, so even the large FFN matmuls run well below the achievable
+ // bandwidth. Widen to 16 subgroups/WG (= the 1024-lane Adreno WG max) so
+ // each SP holds enough in-flight memory requests. Prefill is unaffected (the
+ // GEMM path is separate) and coherence-identical (greedy output unchanged).
+ // Applies to the plain base
+ // GEMV only; tiled/o4/mc3 keep 4 (their reductions are hard-coded to 4).
+ // Layout-safe: the base kernel derives its K-split from get_local_size(1)
+ // and the packed block stride is a physical constant (independent of it).
+ // Opt-out: GGML_OPENCL_Q4K_GEMV_WIDE=0.
+ static const bool splitk_wide_env = []{
+ const char * e = std::getenv("GGML_OPENCL_Q4K_GEMV_WIDE");
+ return !e || e[0] == '\0' || e[0] != '0';
+ }();
+ const bool splitk_wide = splitk_wide_env && !use_tiled && !use_q4k_o4 && !use_mc3;
+ size_t nsg_y = splitk_wide ? 16 : 4;
+ // Cap the wide K-split by the kernel's real max WG. X1-class drivers cap
+ // this GEMV at 768 (< 64*16 = 1024), so an uncapped lws aborts the
+ // dispatch with CL_INVALID_WORK_GROUP_SIZE (-54) and breaks ALL q4_K
+ // decode for M>2560. nsg_y is a pure K-split (the base kernel reads it
+ // from get_local_size(1); the packed block stride is a physical constant),
+ // so halving it stays coherent — just a narrower split. X2 keeps 16
+ // (maxwg 1024); X1 falls to 8.
+ if (splitk_wide) {
+ const size_t maxwg = backend_ctx->get_kernel_workgroup_size(kernel);
+ while (nsg_y > 4 && 64 * nsg_y > maxwg) { nsg_y >>= 1; }
+ }
+ size_t local_work_size[3] = {64, nsg_y, 1};
+ size_t global_work_size[3] = {(size_t)CEIL_DIV(use_tiled ? ne01 : (use_q4k_o4 ? ne01/4 : ne01/2), 64)*64, nsg_y, 1};
backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst);
- CL_CHECK(clReleaseMemObject(q_img));
+ if (q_img) CL_CHECK(clReleaseMemObject(q_img));
CL_CHECK(clReleaseMemObject(b_sub_buf));
CL_CHECK(clReleaseMemObject(b_img));
} else {
@@ -18335,10 +19453,44 @@ static void ggml_cl_mul_mat_q4_k_f32_adreno(ggml_backend_t backend, const ggml_t
}
// gemm
- kernel = backend_ctx->kernel_gemm_noshuffle_q4_k_f32;
+ // Small-batch (medium n_q) occupancy fix: at ne1<=8 the 2x8 grid is
+ // (1, ceil(M/2)) -> ~M/256 workgroups, which under-occupies the SP and
+ // makes the GEMM much slower than the ne1==1 GEMV at the same weight
+ // traffic. The _r1 (1-row) kernel doubles the M-axis workgroup count
+ // and removes the accumulator spill. Opt-in via env while validating.
+ static const bool q4k_gemm_r1 = (getenv("GGML_OPENCL_Q4K_GEMM_R1") != nullptr);
+ static const bool q4k_gemm_kimg = (getenv("GGML_OPENCL_Q4K_GEMM_KIMG") != nullptr);
+ // Cooperative-K (intra-WG K-split + reduction) for the small-batch
+ // (n_q in [2..8]) path: DEFAULT ON, opt out with GGML_OPENCL_Q4K_GEMM_COK=0.
+ // Byte-identical greedy output; large-batch (ne1>8) untouched.
+ static const char * q4k_cok_env = getenv("GGML_OPENCL_Q4K_GEMM_COK");
+ static const bool q4k_gemm_cok = (q4k_cok_env == nullptr) || (atoi(q4k_cok_env) != 0);
+ const bool use_cok = q4k_gemm_cok && (ne1 <= 8);
+ const bool use_r1 = !use_cok && q4k_gemm_r1 && (ne1 <= 8);
+ // Weights-as-image (L1/TPL1) for the small-batch weight-read-bound path.
+ const bool use_kimg = !use_cok && !use_r1 && q4k_gemm_kimg && (ne1 <= 8);
+
+ cl_mem q_img = nullptr;
+ if (use_kimg) {
+ img_fmt = { CL_R, CL_UNSIGNED_INT32 };
+ memset(&img_desc, 0, sizeof(img_desc));
+ img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER;
+ img_desc.image_width = M * K / 2 / 4;
+ img_desc.buffer = extra0_q4_k->q;
+ CL_CHECK((q_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err));
+ }
+
+ kernel = use_cok ? backend_ctx->kernel_gemm_noshuffle_q4_k_f32_cok
+ : use_r1 ? backend_ctx->kernel_gemm_noshuffle_q4_k_f32_r1
+ : use_kimg ? backend_ctx->kernel_gemm_noshuffle_q4_k_f32_kimg
+ : backend_ctx->kernel_gemm_noshuffle_q4_k_f32;
int padded_N = N + padding;
- CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q4_k->q));
+ if (use_kimg) {
+ CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &q_img));
+ } else {
+ CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q4_k->q));
+ }
CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q4_k->s));
CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_q4_k->d));
CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra0_q4_k->dm));
@@ -18353,10 +19505,45 @@ static void ggml_cl_mul_mat_q4_k_f32_adreno(ggml_backend_t backend, const ggml_t
CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_uchar), &mask_d4));
CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_uchar), &mask_hi2));
- size_t global_work_size[3] = {(size_t)CEIL_DIV(ne1, 8), (size_t)CEIL_DIV(ne01, 4), 1};
- size_t local_work_size[3] = {1, 128, 1};
+ size_t global_work_size[3];
+ size_t local_work_size[3];
+ if (use_cok) {
+ // (COK_SG lanes x COK_NSG subgroups): one row per lane, K split
+ // across the COK_NSG subgroups. ne01 is a multiple of 64.
+ global_work_size[0] = (size_t)ne01; // rows
+ global_work_size[1] = 8; // COK_NSG
+ global_work_size[2] = 1;
+ local_work_size[0] = 64; // COK_SG
+ local_work_size[1] = 8; // COK_NSG
+ local_work_size[2] = 1;
+ } else if (use_r1) {
+ // 1 row per WI (opt-in occupancy experiment).
+ global_work_size[0] = (size_t)CEIL_DIV(ne1, 8);
+ global_work_size[1] = (size_t)ne01;
+ global_work_size[2] = 1;
+ local_work_size[0] = 1;
+ local_work_size[1] = 128;
+ local_work_size[2] = 1;
+ } else if (use_kimg) {
+ // kimg is a 2-row tile (opt-in weights-as-image experiment).
+ global_work_size[0] = (size_t)CEIL_DIV(ne1, 8);
+ global_work_size[1] = (size_t)CEIL_DIV(ne01, 2);
+ global_work_size[2] = 1;
+ local_work_size[0] = 1;
+ local_work_size[1] = 128;
+ local_work_size[2] = 1;
+ } else {
+ // Default: x2-unified base kernel is the 4-row (gx<<2) tile.
+ global_work_size[0] = (size_t)CEIL_DIV(ne1, 8);
+ global_work_size[1] = (size_t)CEIL_DIV(ne01, 4);
+ global_work_size[2] = 1;
+ local_work_size[0] = 1;
+ local_work_size[1] = 128;
+ local_work_size[2] = 1;
+ }
backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst);
+ if (q_img) CL_CHECK(clReleaseMemObject(q_img));
CL_CHECK(clReleaseMemObject(b_sub_buf));
CL_CHECK(clReleaseMemObject(b_sub_buf_trans));
CL_CHECK(clReleaseMemObject(b_img));
@@ -18404,29 +19591,65 @@ static void ggml_cl_mul_mat_q6_K_f32_adreno(ggml_backend_t backend, const ggml_t
cl_image_desc img_desc;
// subbuffer and image for activation
- if (ne1 == 1) {
+ // Multi-column verify GEMV: route the spec/MTP verify q6_K matmuls (ne1==3)
+ // onto the efficient GEMV path instead of the transposed-GEMM dead-zone.
+ // Reuses the ne1==1 image setup (activation image sized by N=ne1). Byte-
+ // identical. Opt-in via GGML_OPENCL_Q6K_MC3=1 while validating.
+ static const bool q6k_mc3 = (getenv("GGML_OPENCL_Q6K_MC3") != nullptr);
+ // Per-layer only (ne01 < 32768): batched large-vocab lm_head stays on the
+ // existing path (x2-unified routes batched Q6_K lm_head to CPU; the Adreno
+ // GEMV corrupts it). Per-layer mc3 is byte-identical.
+ const bool use_q6k_mc3 = q6k_mc3 && (ne1 == 3) && (ne01 < 32768);
+ // Batched verify lm_head/embed (ne1==3, tiled layout): multi-column tiled
+ // GEMV — streams the large lm_head weight once across the 3 verify columns
+ // (the #1 MTP bottleneck; mc3 above can't, it reads the noshuffle layout).
+ const bool use_q6k_tiled_mc = q6k_mc3 && (ne1 == 3) && (ne01 >= 32768) && use_q6k_tiled(backend_ctx, src0);
+
+ if (ne1 == 1 || use_q6k_mc3 || use_q6k_tiled_mc) {
cl_mem ql_img = nullptr;
cl_mem qh_img = nullptr;
cl_mem b_sub_buffer = nullptr;
cl_mem b_img = nullptr;
- // image for ql
- img_fmt.image_channel_order = CL_R;
- img_fmt.image_channel_data_type = CL_FLOAT;
- memset(&img_desc, 0, sizeof(img_desc));
- img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER;
- img_desc.image_width = ne01 * ne00 / 8;
- img_desc.buffer = extra0_q6_K->ql;
- CL_CHECK((ql_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err));
+ // o4 = 4-output-per-WI variant for long-vocab lm_head/embed; gated to
+ // ne01 >= 32768 so per-layer q6_K (ne01=hidden 2-8K) keeps the 2-output
+ // kernel (o4 regresses there). o4_global reads the weights from __global
+ // coalesced instead of image1d_buffer -- the texture cache caps the
+ // read-once-per-token lm_head bandwidth, while __global reaches the higher
+ // rate the rest of the model gets. Both default ON; opt out via
+ // GGML_OPENCL_Q6K_GEMV_O4 / GGML_OPENCL_Q6K_GEMV_O4_GLOBAL = 0.
+ static const bool gemv_o4_env = []{
+ const char * e = std::getenv("GGML_OPENCL_Q6K_GEMV_O4");
+ return !e || e[0] == '\0' || e[0] != '0';
+ }();
+ static const bool o4_global_env = []{
+ const char * e = std::getenv("GGML_OPENCL_Q6K_GEMV_O4_GLOBAL");
+ return !e || e[0] == '\0' || e[0] != '0';
+ }();
+ const bool use_tiled = !use_q6k_mc3 && use_q6k_tiled(backend_ctx, src0);
+ const bool use_o4 = !use_tiled && !use_q6k_mc3 && gemv_o4_env && (ne01 % 4 == 0) && (ne01 >= 32768);
+ const bool use_o4_global = use_o4 && o4_global_env;
+
+ // ql/qh image views are only needed when NOT reading weights from global.
+ if (!use_o4_global && !use_tiled) {
+ // image for ql
+ img_fmt.image_channel_order = CL_R;
+ img_fmt.image_channel_data_type = CL_FLOAT;
+ memset(&img_desc, 0, sizeof(img_desc));
+ img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER;
+ img_desc.image_width = ne01 * ne00 / 8;
+ img_desc.buffer = extra0_q6_K->ql;
+ CL_CHECK((ql_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err));
- // image for qh
- img_fmt.image_channel_order = CL_R;
- img_fmt.image_channel_data_type = CL_HALF_FLOAT;
- memset(&img_desc, 0, sizeof(img_desc));
- img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER;
- img_desc.image_width = ne01 * ne00 / 8;
- img_desc.buffer = extra0_q6_K->qh;
- CL_CHECK((qh_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err));
+ // image for qh
+ img_fmt.image_channel_order = CL_R;
+ img_fmt.image_channel_data_type = CL_HALF_FLOAT;
+ memset(&img_desc, 0, sizeof(img_desc));
+ img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER;
+ img_desc.image_width = ne01 * ne00 / 8;
+ img_desc.buffer = extra0_q6_K->qh;
+ CL_CHECK((qh_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err));
+ }
region.origin = offset1;
region.size = ne00 * ne1 * sizeof(float);
@@ -18440,10 +19663,20 @@ static void ggml_cl_mul_mat_q6_K_f32_adreno(ggml_backend_t backend, const ggml_t
img_desc.buffer = b_sub_buffer;
CL_CHECK((b_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err));
- kernel = backend_ctx->kernel_gemv_noshuffle_q6_K_f32;
+ kernel = use_q6k_mc3 ? backend_ctx->kernel_gemv_noshuffle_q6_K_f32_mc3
+ : use_q6k_tiled_mc ? backend_ctx->kernel_gemv_noshuffle_q6_K_f32_tiled_mc3
+ : use_tiled ? backend_ctx->kernel_gemv_noshuffle_q6_K_f32_tiled
+ : use_o4_global ? backend_ctx->kernel_gemv_noshuffle_q6_K_f32_o4_global
+ : use_o4 ? backend_ctx->kernel_gemv_noshuffle_q6_K_f32_o4
+ : backend_ctx->kernel_gemv_noshuffle_q6_K_f32;
- CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &ql_img));
- CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &qh_img));
+ if (use_o4_global || use_tiled) {
+ CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q6_K->ql));
+ CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q6_K->qh));
+ } else {
+ CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &ql_img));
+ CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &qh_img));
+ }
CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_q6_K->s));
CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra0_q6_K->d));
CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &b_img));
@@ -18452,16 +19685,67 @@ static void ggml_cl_mul_mat_q6_K_f32_adreno(ggml_backend_t backend, const ggml_t
CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_int), &ne00));
CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_int), &ne01));
- size_t local_work_size[3] = {64, 4, 1};
- size_t global_work_size[3] = {(size_t)CEIL_DIV(ne01/2, 64)*64, 4, 1};
+ const size_t gws_x = use_tiled
+ ? (size_t) CEIL_DIV(ne01, 64) * 64
+ : use_o4
+ ? (size_t) CEIL_DIV(ne01/4, 64) * 64
+ : (size_t) CEIL_DIV(ne01/2, 64) * 64;
+ size_t local_work_size[3] = {64, 4, 1};
+ size_t global_work_size[3] = {gws_x, 4, 1};
backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst);
- CL_CHECK(clReleaseMemObject(ql_img));
- CL_CHECK(clReleaseMemObject(qh_img));
+ if (ql_img) CL_CHECK(clReleaseMemObject(ql_img));
+ if (qh_img) CL_CHECK(clReleaseMemObject(qh_img));
CL_CHECK(clReleaseMemObject(b_sub_buffer));
CL_CHECK(clReleaseMemObject(b_img));
} else {
+ // Tiled-layout batched GEMM. When the weight was converted to the 64-row
+ // tiled canonical layout (use_q6k_tiled — the default for lm_head/embed),
+ // the plain noshuffle GEMM below reads it as plain-transposed and produces
+ // garbage. Use the batched GEMM that matches the decode tiled GEMV's
+ // layout; it reads the f32 activation directly (column-major, no transpose).
+ if (use_q6k_tiled(backend_ctx, src0)) {
+ cl_mem b_sub_buf_t = nullptr;
+ cl_mem b_img_t = nullptr;
+
+ region.origin = offset1;
+ region.size = ne00 * ne1 * sizeof(float);
+ CL_CHECK((b_sub_buf_t = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, &region, &err), err));
+
+ img_fmt.image_channel_order = CL_RGBA;
+ img_fmt.image_channel_data_type = CL_FLOAT;
+ memset(&img_desc, 0, sizeof(img_desc));
+ img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER;
+ img_desc.image_width = ne00 * ne1 / 4;
+ img_desc.buffer = b_sub_buf_t;
+ CL_CHECK((b_img_t = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err));
+
+ cl_kernel kt = backend_ctx->kernel_gemm_noshuffle_q6_K_f32_tiled;
+ CL_CHECK(clSetKernelArg(kt, 0, sizeof(cl_mem), &extra0_q6_K->ql));
+ CL_CHECK(clSetKernelArg(kt, 1, sizeof(cl_mem), &extra0_q6_K->qh));
+ CL_CHECK(clSetKernelArg(kt, 2, sizeof(cl_mem), &extra0_q6_K->s));
+ CL_CHECK(clSetKernelArg(kt, 3, sizeof(cl_mem), &extra0_q6_K->d));
+ CL_CHECK(clSetKernelArg(kt, 4, sizeof(cl_mem), &b_img_t));
+ CL_CHECK(clSetKernelArg(kt, 5, sizeof(cl_mem), &extrad->data_device));
+ CL_CHECK(clSetKernelArg(kt, 6, sizeof(cl_ulong), &offsetd));
+ CL_CHECK(clSetKernelArg(kt, 7, sizeof(int), &ne00));
+ CL_CHECK(clSetKernelArg(kt, 8, sizeof(int), &ne01));
+ CL_CHECK(clSetKernelArg(kt, 9, sizeof(int), &ne1));
+
+ // Must match the kernel: NTILES=4 64-row tiles per work-group (256 rows),
+ // BN=8 output columns per work-group.
+ const int BN_T = 16;
+ const int WROWS = 4 * 64; // NTILES * TILE_ROWS
+ size_t local_work_size[3] = {64, 4, 1};
+ size_t global_work_size[3] = {(size_t)CEIL_DIV(ne01, WROWS) * 64, 4, (size_t)CEIL_DIV(ne1, BN_T)};
+ backend_ctx->enqueue_ndrange_kernel(kt, 3, global_work_size, local_work_size, dst);
+
+ CL_CHECK(clReleaseMemObject(b_img_t));
+ CL_CHECK(clReleaseMemObject(b_sub_buf_t));
+ return;
+ }
+
cl_mem b_sub_buf;
cl_mem b_buf_trans;
cl_mem b_img;
@@ -18573,7 +19857,19 @@ static void ggml_cl_mul_mat_q6_K_f32_adreno(ggml_backend_t backend, const ggml_t
backend_ctx->enqueue_ndrange_kernel(kernel, 2, global_size_t, local_size_t, dst);
// gemm
- kernel = backend_ctx->kernel_gemm_noshuffle_q6_K_f32;
+ // Cooperative-K small-batch (n_q in [2..8]) path: intra-WG K-split,
+ // mirrors the q4_K _cok path (batched serving). OPT-IN
+ // (GGML_OPENCL_Q6K_GEMM_COK=1), DEFAULT OFF: q6_K is the tied lm_head/
+ // output projection, so the K-reassociation perturbs final logits and
+ // greedy is NOT byte-identical (op-tests pass, output coherent, but not
+ // bit-exact). It is also NEUTRAL on end-to-end MTP (q4_K cok already
+ // captured that; the MTP bottleneck moved off the GEMMs). Keep opt-in
+ // for batched serving until PPL-validated on a non-GDN q6_K model.
+ static const char * q6k_cok_env = getenv("GGML_OPENCL_Q6K_GEMM_COK");
+ static const bool q6k_gemm_cok = (q6k_cok_env != nullptr) && (atoi(q6k_cok_env) != 0);
+ const bool use_q6k_cok = q6k_gemm_cok && (ne1 <= 8);
+ kernel = use_q6k_cok ? backend_ctx->kernel_gemm_noshuffle_q6_K_f32_cok
+ : backend_ctx->kernel_gemm_noshuffle_q6_K_f32;
int padded_N = ne1 + padding;
cl_ushort mask_f000 = 0xF000;
@@ -18593,8 +19889,23 @@ static void ggml_cl_mul_mat_q6_K_f32_adreno(ggml_backend_t backend, const ggml_t
CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ushort),&mask_f000));
CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_uchar), &mask_c0));
- size_t global_work_size[3] = {(size_t)CEIL_DIV(ne1, 8), (size_t)CEIL_DIV(ne01, 4), 1};
- size_t local_work_size[3] = {2, 128, 1};
+ size_t global_work_size[3];
+ size_t local_work_size[3];
+ if (use_q6k_cok) {
+ global_work_size[0] = (size_t)ne01; // rows (1 per lane)
+ global_work_size[1] = 8; // COK_NSG
+ global_work_size[2] = 1;
+ local_work_size[0] = 64; // COK_SG
+ local_work_size[1] = 8; // COK_NSG
+ local_work_size[2] = 1;
+ } else {
+ global_work_size[0] = (size_t)CEIL_DIV(ne1, 8);
+ global_work_size[1] = (size_t)CEIL_DIV(ne01, 4);
+ global_work_size[2] = 1;
+ local_work_size[0] = 2;
+ local_work_size[1] = 128;
+ local_work_size[2] = 1;
+ }
backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst);
CL_CHECK(clReleaseMemObject(b_sub_buf));
@@ -18650,7 +19961,15 @@ static void ggml_cl_mul_mat_q5_K_f32_adreno(ggml_backend_t backend, const ggml_t
cl_uchar mask_d4 = 0x0F;
cl_uchar mask_hi2 = 0xC0;
- if (ne1 == 1) {
+ // Multi-column (N=3) verify GEMV for q5_K: route the spec/MTP verify batch
+ // (ne1==3) onto the efficient GEMV path instead of the transposed-GEMM dead-
+ // zone (gemm_noshuffle_q5_k, the #2 chunk of MTP decode on a Q4_0-mix model
+ // after q4_0 mc3). Reuses the ne1==1 GEMV image setup (q + qh + activations).
+ // Opt-in via GGML_OPENCL_Q5K_MC3=1. Per-layer only (ne01 < 32768).
+ static const bool q5k_mc3 = (getenv("GGML_OPENCL_Q5K_MC3") != nullptr);
+ const bool use_q5k_mc3 = q5k_mc3 && (ne1 >= 2 && ne1 <= 4) && (ne01 < 32768);
+
+ if (ne1 == 1 || use_q5k_mc3) {
cl_mem q_img = nullptr;
cl_mem qh_img = nullptr;
cl_mem b_sub_buf = nullptr;
@@ -18685,7 +20004,8 @@ static void ggml_cl_mul_mat_q5_K_f32_adreno(ggml_backend_t backend, const ggml_t
img_desc.buffer = b_sub_buf;
CL_CHECK((b_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err));
- kernel = backend_ctx->kernel_gemv_noshuffle_q5_k_f32;
+ kernel = use_q5k_mc3 ? backend_ctx->kernel_gemv_noshuffle_q5_k_f32_mc3
+ : backend_ctx->kernel_gemv_noshuffle_q5_k_f32;
CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &q_img));
CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &qh_img));
@@ -18700,6 +20020,9 @@ static void ggml_cl_mul_mat_q5_K_f32_adreno(ggml_backend_t backend, const ggml_t
CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_uchar), &mask_d6));
CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_uchar), &mask_d4));
CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_uchar), &mask_hi2));
+ if (use_q5k_mc3) {
+ CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_int), &ne1)); // n_cols
+ }
size_t local_work_size[3] = {64, 4, 1};
size_t global_work_size[3] = {(size_t)CEIL_DIV(ne01/2, 64)*64, 4, 1};
@@ -19571,7 +20894,7 @@ static void ggml_cl_mul_mat(ggml_backend_t backend, const ggml_tensor * src0, co
}
// q4_k x fp32
- if (src0t == GGML_TYPE_Q4_K && src1t == GGML_TYPE_F32 && !use_flat_gemv_for_large_m_q4_K(src0)) {
+ if (src0t == GGML_TYPE_Q4_K && src1t == GGML_TYPE_F32 && !use_flat_gemv_for_large_m_q4_K(backend_ctx, src0)) {
ggml_cl_mul_mat_q4_k_f32_adreno(backend, src0, src1, dst);
return;
}
@@ -19607,6 +20930,35 @@ static void ggml_cl_mul_mat(ggml_backend_t backend, const ggml_tensor * src0, co
backend_ctx->adreno_gen == ADRENO_GPU_GEN::A7X)) {
switch(src0t) {
case GGML_TYPE_F32: {
+ // Small-N f32 GEMV for the spec/MTP verify batch: the tiled GEMM
+ // below always computes a full 64x64 tile, so at ne11=3 with a
+ // skinny f32 weight (GDN ssm_alpha/ssm_beta, M=32) it launches one
+ // under-occupied WG at ~2.3% tile utilization. Route to a per-output
+ // (m,n) GEMV (64-thread WG, K-split + __local reduce) instead.
+ // Opt-in GGML_OPENCL_F32_MC=1; 2D contiguous, small N + skinny M only.
+ static const bool f32_mc = (getenv("GGML_OPENCL_F32_MC") != nullptr);
+ if (f32_mc && ne11 >= 2 && ne11 <= 8 && ne01 <= 512 && (ne00 % 4 == 0) &&
+ ne02 == 1 && ne12 == 1 && ne13 == 1 &&
+ ggml_is_contiguous(src0) && ggml_is_contiguous(src1)) {
+ cl_kernel kmc = backend_ctx->kernel_gemv_f32_f32_mc;
+ int stride_a = ne00, stride_b = ne00, stride_d = ne01;
+ CL_CHECK(clSetKernelArg(kmc, 0, sizeof(cl_mem), &extra0->data_device));
+ CL_CHECK(clSetKernelArg(kmc, 1, sizeof(cl_ulong), &offset0));
+ CL_CHECK(clSetKernelArg(kmc, 2, sizeof(cl_mem), &extra1->data_device));
+ CL_CHECK(clSetKernelArg(kmc, 3, sizeof(cl_ulong), &offset1));
+ CL_CHECK(clSetKernelArg(kmc, 4, sizeof(cl_mem), &extrad->data_device));
+ CL_CHECK(clSetKernelArg(kmc, 5, sizeof(cl_ulong), &offsetd));
+ CL_CHECK(clSetKernelArg(kmc, 6, sizeof(int), &ne00));
+ CL_CHECK(clSetKernelArg(kmc, 7, sizeof(int), &ne01));
+ CL_CHECK(clSetKernelArg(kmc, 8, sizeof(int), &ne11));
+ CL_CHECK(clSetKernelArg(kmc, 9, sizeof(int), &stride_a));
+ CL_CHECK(clSetKernelArg(kmc, 10, sizeof(int), &stride_b));
+ CL_CHECK(clSetKernelArg(kmc, 11, sizeof(int), &stride_d));
+ size_t gws[3] = {64, (size_t)ne01 * (size_t)ne11, 1};
+ size_t lws[3] = {64, 1, 1};
+ backend_ctx->enqueue_ndrange_kernel(kmc, 3, gws, lws, dst);
+ return;
+ }
kernel = backend_ctx->kernel_mul_mm_f32_f32_l4_lm;
nth0 = 128; // calculated as (BM*BN)/(TM*TN)
@@ -20263,6 +21615,7 @@ static void ggml_cl_mul_mat(ggml_backend_t backend, const ggml_tensor * src0, co
}
// use custom matrix x vector kernel
+ bool use_f16_mrow = false;
switch (src0t) {
case GGML_TYPE_F32:
//GGML_ASSERT(ne02 == ne12);
@@ -20328,7 +21681,46 @@ static void ggml_cl_mul_mat(ggml_backend_t backend, const ggml_tensor * src0, co
(ne12 % r2) == 0;
if (ne11 * ne12 < 4) {
- kernel = backend_ctx->kernel_mul_mat_f16_f32_1row;
+ // Decode (single token): the legacy _1row runs one 64-lane
+ // subgroup per WG (one output row), under-utilizing BW. Route the
+ // wide f16 weight matmuls (attn proj + lm_head) to the multi-row
+ // variant: MROW rows per WG -> more loads in flight + activation
+ // staged once in __local. ne00<=8192 bounds the LDS. The mrow WG
+ // is 64 x MROW = 1024 work-items (> Intel's 512 max) and reduces
+ // within a 64-wide subgroup, so skip on Intel.
+ if (backend_ctx->f16_mrow && backend_ctx->gpu_family != INTEL &&
+ backend_ctx->kernel_mul_mat_f16_f32_mrow != nullptr &&
+ ne00 >= 128 && ne01 >= 8 && ne00 % 4 == 0 && ne00 <= 8192) {
+ // The register-blocked / half8 variants cast the src0 row pointer to
+ // half4 / half8 (8- and 16-byte loads) with no scalar fallback inside
+ // the kernel. ne00 % 4 == 0 constrains the element count per row, NOT
+ // the byte stride between rows: a permuted or strided src0 (or a view
+ // at an odd offset) can leave nb01/nb02/nb03 unaligned. Only take them
+ // when every row this dispatch touches is aligned; the base mrow kernel
+ // re-checks per row and falls back to its scalar loop.
+ const cl_ulong row_addr_bits = offset0 | nb01 | nb02 | nb03;
+ const bool aligned8 = (row_addr_bits & 7) == 0;
+ const bool aligned16 = (row_addr_bits & 15) == 0;
+
+ // Register-blocked variants: each subgroup does RPT rows (more
+ // weight loads in flight per lane). 8/16 use half8 (128-bit)
+ // loads, gated on ne00 % 8 == 0.
+ const int rpt = backend_ctx->f16_mrow_rpt;
+ if (rpt == 16 && ne00 % 8 == 0 && aligned16 && backend_ctx->kernel_mul_mat_f16_f32_mrow_h8r2 != nullptr) {
+ kernel = backend_ctx->kernel_mul_mat_f16_f32_mrow_h8r2;
+ } else if (rpt == 8 && ne00 % 8 == 0 && aligned16 && backend_ctx->kernel_mul_mat_f16_f32_mrow_h8 != nullptr) {
+ kernel = backend_ctx->kernel_mul_mat_f16_f32_mrow_h8;
+ } else if (rpt == 4 && aligned8 && backend_ctx->kernel_mul_mat_f16_f32_mrow_r4 != nullptr) {
+ kernel = backend_ctx->kernel_mul_mat_f16_f32_mrow_r4;
+ } else if (rpt == 2 && aligned8 && backend_ctx->kernel_mul_mat_f16_f32_mrow_r2 != nullptr) {
+ kernel = backend_ctx->kernel_mul_mat_f16_f32_mrow_r2;
+ } else {
+ kernel = backend_ctx->kernel_mul_mat_f16_f32_mrow;
+ }
+ use_f16_mrow = true;
+ } else {
+ kernel = backend_ctx->kernel_mul_mat_f16_f32_1row;
+ }
} else if (adreno_use_lane_split && ne00 >= 64 && ne00 <= 128) {
kernel = backend_ctx->kernel_mul_mat_f16_f32_l4_dr_lq;
nrows = 1;
@@ -20414,6 +21806,23 @@ static void ggml_cl_mul_mat(ggml_backend_t backend, const ggml_tensor * src0, co
CL_CHECK(clSetKernelArg(kernel, 21, sizeof(int), &ne1));
CL_CHECK(clSetKernelArg(kernel, 22, sizeof(int), &r2));
CL_CHECK(clSetKernelArg(kernel, 23, sizeof(int), &r3));
+ if (use_f16_mrow) {
+ const int MROW = 16; // must match MROW in mul_mv_f16_f32_mrow.cl
+ // rows-per-subgroup multiplier for the selected variant:
+ // 1/2/4 -> half4 register blocking; 8 -> half8(1 row); 16 -> half8(2 rows)
+ const int rpt = backend_ctx->f16_mrow_rpt;
+ int rmul;
+ if (rpt == 16) rmul = (ne00 % 8 == 0) ? 2 : 1;
+ else if (rpt == 8) rmul = 1;
+ else rmul = rpt; // 1,2,4
+ const int rows_per_wg = MROW * rmul;
+ // __local activation buffer: ne00 floats, rounded up for float4 access
+ CL_CHECK(clSetKernelArg(kernel, 24, sizeof(float) * ((ne00 + 3) / 4 * 4), nullptr));
+ size_t mrow_global[] = { (size_t)((ne01 + rows_per_wg - 1) / rows_per_wg) * 64, (size_t)ne11 * MROW, (size_t)ne12 * ne13 };
+ size_t mrow_local[] = { 64, (size_t)MROW, 1 };
+ backend_ctx->enqueue_ndrange_kernel(kernel, 3, mrow_global, mrow_local, dst);
+ return;
+ }
break;
case GGML_TYPE_Q1_0: {
#ifdef GGML_OPENCL_SOA_Q
diff --git a/ggml/src/ggml-opencl/kernels/cvt.cl b/ggml/src/ggml-opencl/kernels/cvt.cl
index 3d6cff7cff..acc8f98076 100644
--- a/ggml/src/ggml-opencl/kernels/cvt.cl
+++ b/ggml/src/ggml-opencl/kernels/cvt.cl
@@ -1110,6 +1110,78 @@ kernel void kernel_restore_block_q4_k_trans4_ns(
}
}
+//------------------------------------------------------------------------------
+// kernel_convert_block_q4_k_tiled_ns
+//
+// Tiled-wide layout for the long-vocab q4_K lm_head/embed GEMV (decode path).
+// Mirror of kernel_convert_block_q6_k_tiled_ns: recovers each weight's 4-bit
+// code in CANONICAL ggml element order (e in [0,256)) and re-packs into 32 uints
+// (8 codes/uint), stored TILED by 64 output rows so the matching GEMV
+// (gemv_noshuffle_q4_k_f32_tiled) coalesces every weight load. The 12-byte
+// packed scale block `s` and d/dm are stored per (row, K-block) tiled; the GEMV
+// re-derives the 8 (scale,min) pairs via get_scale_min_k4, exactly like the o4
+// kernel. Both ends owned here -> correct by construction vs the reference q4_K
+// dequant. Requires ne01 % 64 == 0 (gated host-side). Buffer sizes identical to
+// the trans4_ns layout.
+//
+// q uint4 granule g of (row r, K-block sb): idx = ((rt*ne00_blk+sb)*8 + g)*64 + rit
+// s (12 bytes) of (r, sb): idx = (rt*ne00_blk+sb)*64 + rit, *12
+// d/dm (half) of (r, sb): idx = (rt*ne00_blk+sb)*64 + rit
+// where rt = r/64, rit = r%64.
+//------------------------------------------------------------------------------
+kernel void kernel_convert_block_q4_k_tiled_ns(
+ __global struct block_q4_K * src0,
+ __global uint * dst_q, // 32 uints / superblock (4-bit codes, 8 codes/uint)
+ __global half * dst_d, // 1 half / superblock
+ __global half * dst_dm, // 1 half / superblock
+ __global uchar * dst_s, // K_SCALE_SIZE (12) bytes / superblock
+ uint ne00,
+ uint ne01
+) {
+ uint i00 = get_global_id(1); // K-block index (superblock along ne00)
+ uint i01 = get_global_id(0); // output row index (along ne01)
+ uint i02 = get_global_id(2); // batch
+
+ uint ne00_blk = ne00 / QK_K;
+
+ uint src_blk_offset = i00 + i01 * ne00_blk + i02 * ne00_blk * ne01;
+ __global struct block_q4_K * b = src0 + src_blk_offset;
+
+ uint rt = i01 / 64;
+ uint rit = i01 % 64;
+ uint tile_blk = (i02 * (ne01 / 64) + rt) * ne00_blk + i00;
+
+ // --- recover canonical 4-bit codes in e-order, pack 8 codes/uint ---
+ uint qw[32] = {0};
+ for (uint e = 0; e < 256; ++e) {
+ uint g = e >> 6; // group 0..3 (q advances 32 bytes/group)
+ uint within = e & 63u;
+ uint hlf = within >> 5; // 0 = low nibble, 1 = high nibble
+ uint l = within & 31u; // 0..31
+ uchar byte = b->q[g * 32u + l];
+ uint code = (hlf == 0u) ? (uint)(byte & 0x0F) : (uint)(byte >> 4);
+ qw[e >> 3] |= code << ((e & 7u) * 4u);
+ }
+
+ for (uint gr = 0; gr < 8; ++gr) {
+ uint base = (tile_blk * 8u + gr) * 64u + rit; // uint4 index
+ dst_q[base * 4u + 0u] = qw[gr * 4u + 0u];
+ dst_q[base * 4u + 1u] = qw[gr * 4u + 1u];
+ dst_q[base * 4u + 2u] = qw[gr * 4u + 2u];
+ dst_q[base * 4u + 3u] = qw[gr * 4u + 3u];
+ }
+
+ // packed scales (12 bytes), tiled per (row, block)
+ __global uchar * s_dst = dst_s + (tile_blk * 64u + rit) * K_SCALE_SIZE;
+ #pragma unroll
+ for (int i = 0; i < K_SCALE_SIZE; ++i) {
+ s_dst[i] = b->s[i];
+ }
+
+ dst_d [tile_blk * 64u + rit] = b->d;
+ dst_dm[tile_blk * 64u + rit] = b->dm;
+}
+
kernel void kernel_convert_block_q5_k_trans4_ns(
__global struct block_q5_K * src0,
__global uint * dst_qs,
@@ -1494,6 +1566,105 @@ kernel void kernel_restore_block_mxfp4_trans(
b->e = src_e[src_blk_offset];
}
+//------------------------------------------------------------------------------
+// kernel_convert_block_q6_k_tiled_ns
+//
+// Tiled-wide layout for the long-vocab q6_K lm_head/embed GEMV (decode path).
+// Unlike *_trans4_ns (which mirrors the bit-interleave the legacy 2-output GEMV
+// consumes), this kernel is correct-by-construction against the CANONICAL ggml
+// q6_K dequant: it recovers each weight's 6-bit code in element order e in
+// [0,256), then re-packs low-4-bits into 32 uints (8 codes/uint) and high-2-bits
+// into 16 uints (16 codes/uint). The matching GEMV (gemv_noshuffle_q6_k_f32_tiled)
+// unpacks the same order, so both ends are owned here.
+//
+// Storage is TILED by 64 output rows so the GEMV's 64-thread tile coalesces:
+// ql uint4 granule g of (row r, K-block sb): idx = ((rt*ne00_blk + sb)*8 + g)*64 + rit
+// qh uint4 granule g: idx = ((rt*ne00_blk + sb)*4 + g)*64 + rit
+// scales (char16) of (r, sb): idx = (rt*ne00_blk + sb)*64 + rit
+// d (half) of (r, sb): idx = (rt*ne00_blk + sb)*64 + rit
+// where rt = r/64, rit = r%64. Requires ne01 % 64 == 0 (gated host-side).
+// Buffer sizes are byte-identical to the trans4_ns layout.
+//------------------------------------------------------------------------------
+kernel void kernel_convert_block_q6_k_tiled_ns(
+ __global struct block_q6_K * src0,
+ __global uint * dst_ql, // 32 uints / superblock (low 4 bits, 8 codes/uint)
+ __global uint * dst_qh, // 16 uints / superblock (high 2 bits, 16 codes/uint)
+ __global half * dst_d, // 1 half / superblock
+ __global char * dst_s, // 16 chars/ superblock
+ uint ne00,
+ uint ne01
+) {
+ uint i00 = get_global_id(1); // K-block index (superblock along ne00)
+ uint i01 = get_global_id(0); // output row index (along ne01)
+ uint i02 = get_global_id(2); // batch
+
+ uint ne00_blk = ne00 / QK_K;
+
+ // Source block: row-major over (i02, i01, i00).
+ uint src_blk_offset = i00 + i01 * ne00_blk + i02 * ne00_blk * ne01;
+ __global struct block_q6_K * b = src0 + src_blk_offset;
+
+ uint rt = i01 / 64;
+ uint rit = i01 % 64;
+ uint tile_blk = (i02 * (ne01 / 64) + rt) * ne00_blk + i00; // tile-major (row-tile, K-block)
+
+ // --- recover canonical 6-bit codes, pack into ql (4b) + qh (2b) in e-order ---
+ // 32 ql-uints (8 low-nibbles each) + 16 qh-uints (16 2-bit slots each).
+ uint qlw[32] = {0};
+ uint qhw[16] = {0};
+
+ for (uint e = 0; e < 256; ++e) {
+ uint n = (e >= 128) ? 1u : 0u; // which 128-half
+ uint within = e - n * 128u;
+ uint q = within / 32u; // quadrant 0..3
+ uint l = within % 32u; // 0..31
+
+ uint off_ql = n * 64u; // raw ql byte base for this half
+ uint off_qh = n * 32u; // raw qh byte base for this half
+
+ uchar low4;
+ uchar qlb0 = b->ql[off_ql + l];
+ uchar qlb1 = b->ql[off_ql + l + 32];
+ if (q == 0) low4 = qlb0 & 0x0F;
+ else if (q == 1) low4 = qlb1 & 0x0F;
+ else if (q == 2) low4 = (qlb0 >> 4) & 0x0F;
+ else low4 = (qlb1 >> 4) & 0x0F;
+
+ uchar hi2 = (b->qh[off_qh + l] >> (q * 2u)) & 0x03;
+
+ // pack low4 (e-order): uint e/8, nibble (e%8)
+ qlw[e >> 3] |= ((uint)low4) << ((e & 7u) * 4u);
+ // pack hi2 (e-order): uint e/16, 2-bit slot (e%16)
+ qhw[e >> 4] |= ((uint)hi2) << ((e & 15u) * 2u);
+ }
+
+ // --- write tiled ---
+ for (uint g = 0; g < 8; ++g) {
+ uint base = (tile_blk * 8u + g) * 64u + rit; // uint4 index
+ dst_ql[base * 4u + 0u] = qlw[g * 4u + 0u];
+ dst_ql[base * 4u + 1u] = qlw[g * 4u + 1u];
+ dst_ql[base * 4u + 2u] = qlw[g * 4u + 2u];
+ dst_ql[base * 4u + 3u] = qlw[g * 4u + 3u];
+ }
+ for (uint g = 0; g < 4; ++g) {
+ uint base = (tile_blk * 4u + g) * 64u + rit; // uint4 index
+ dst_qh[base * 4u + 0u] = qhw[g * 4u + 0u];
+ dst_qh[base * 4u + 1u] = qhw[g * 4u + 1u];
+ dst_qh[base * 4u + 2u] = qhw[g * 4u + 2u];
+ dst_qh[base * 4u + 3u] = qhw[g * 4u + 3u];
+ }
+
+ // scales: 16 chars contiguous per (row, block), tiled
+ __global char * s_dst = dst_s + (tile_blk * 64u + rit) * 16u;
+ #pragma unroll
+ for (int i = 0; i < 16; ++i) {
+ s_dst[i] = b->scales[i];
+ }
+
+ // super-block scale
+ dst_d[tile_blk * 64u + rit] = b->d;
+}
+
kernel void kernel_convert_block_mxfp4_trans4_ns(
global struct block_mxfp4 * src0,
__global uint * dst_q,
diff --git a/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q4_k_f32.cl b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q4_k_f32.cl
index 22b4e91146..c379a9a399 100644
--- a/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q4_k_f32.cl
+++ b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q4_k_f32.cl
@@ -4,6 +4,7 @@
#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable
#define ADRENO_GPU 1
#define REQD_SUBGROUP_SIZE_128 __attribute__((qcom_reqd_sub_group_size("full")))
+#define REQD_SUBGROUP_SIZE_64 __attribute__((qcom_reqd_sub_group_size("half")))
#endif
#define QK_K 256
#define K_SCALE_SIZE 12
@@ -171,3 +172,319 @@ kernel void kernel_gemm_noshuffle_q4_k_f32(
vstore4((float4)(c0.s7, c1.s7, c2.s7, c3.s7), 0, dst + idx);
}
}
+
+// 1x8 per-WI tile (1 output row x 8 output cols). For the small-batch
+// (medium n_q, e.g. MTP/spec verify) path where the 2x8 kernel is starved:
+// at ne1<=8 the grid is (1, ceil(M/2)) -> only ~M/256 workgroups, leaving
+// the SP under-occupied. 1 row per WI doubles the M-axis workgroup count
+// (ceil(M/1)/128 vs ceil(M/2)/128) AND collapses the accumulators to a
+// single half8 (16 regs, no spill), so more waves co-reside. Same weight
+// traffic as 2x8 (rows never share weights); the win is pure occupancy.
+#ifdef ADRENO_GPU
+REQD_SUBGROUP_SIZE_128
+#endif
+kernel void kernel_gemm_noshuffle_q4_k_f32_r1(
+ global const ushort * src0_q,
+ global const uchar * src0_s,
+ global const half * src0_d,
+ global const half * src0_dm,
+ read_only image1d_buffer_t src1,
+ global float * dst,
+ ulong offsetd,
+ int m,
+ int n,
+ int k,
+ int n_no_padding,
+ uchar mask_d6,
+ uchar mask_d4,
+ uchar mask_hi2
+) {
+ dst = (global float *)((global char *)dst + offsetd);
+ int n_4 = n >> 2;
+ int gy = get_global_id(0);
+ int gx = get_global_id(1); // 1 row per WI
+
+ half8 c0 = 0;
+ half8 B;
+ half dq;
+
+ int num_blocks_K = k / QK_K;
+
+ global const ushort * weight_ptr = src0_q + gx;
+ global const half * d_ptr = src0_d + gx;
+ global const half * dm_ptr = src0_dm + gx;
+
+ for (int i = 0; i < k; i += 32) {
+ int sb_idx = i / QK_K;
+ int sub_idx = (i / 32) % 8;
+
+ half dd = d_ptr [sb_idx * m];
+ half dmm = dm_ptr[sb_idx * m];
+
+ global const uchar * sc0 = src0_s + sb_idx * K_SCALE_SIZE * m + gx;
+
+ uchar sv0, mn0;
+ get_scale_min_k4(sub_idx, sc0, m, &sv0, &mn0, mask_d6, mask_d4, mask_hi2);
+
+ half scale = convert_half(convert_float(dd) * (float)sv0);
+ half mval = convert_half(convert_float(dmm) * (float)mn0);
+
+ for (int l = 0; l < 32; l += 4) {
+ int ki = i + l;
+ ushort bits = weight_ptr[(ki/4) * m];
+
+ B.s0123 = read_imageh(src1, gy*2 + (ki+0) * n_4);
+ B.s4567 = read_imageh(src1, gy*2+1 + (ki+0) * n_4);
+ dq = (bits & 0x000F) * scale - mval;
+ c0 += B * dq;
+
+ B.s0123 = read_imageh(src1, gy*2 + (ki+1) * n_4);
+ B.s4567 = read_imageh(src1, gy*2+1 + (ki+1) * n_4);
+ dq = ((bits & 0x00F0) >> 4) * scale - mval;
+ c0 += B * dq;
+
+ B.s0123 = read_imageh(src1, gy*2 + (ki+2) * n_4);
+ B.s4567 = read_imageh(src1, gy*2+1 + (ki+2) * n_4);
+ dq = ((bits & 0x0F00) >> 8) * scale - mval;
+ c0 += B * dq;
+
+ B.s0123 = read_imageh(src1, gy*2 + (ki+3) * n_4);
+ B.s4567 = read_imageh(src1, gy*2+1 + (ki+3) * n_4);
+ dq = ((bits & 0xF000) >> 12) * scale - mval;
+ c0 += B * dq;
+ }
+ }
+
+ // Output: 8 cols, 1 row per col-step. Scalar store, coalesced across
+ // neighbouring WIs (consecutive gx -> consecutive dst addresses).
+ int idx = (gy<<3)*m + gx;
+ if (idx < m*n_no_padding) { dst[idx] = c0.s0; idx += m; }
+ if (idx < m*n_no_padding) { dst[idx] = c0.s1; idx += m; }
+ if (idx < m*n_no_padding) { dst[idx] = c0.s2; idx += m; }
+ if (idx < m*n_no_padding) { dst[idx] = c0.s3; idx += m; }
+ if (idx < m*n_no_padding) { dst[idx] = c0.s4; idx += m; }
+ if (idx < m*n_no_padding) { dst[idx] = c0.s5; idx += m; }
+ if (idx < m*n_no_padding) { dst[idx] = c0.s6; idx += m; }
+ if (idx < m*n_no_padding) { dst[idx] = c0.s7; }
+}
+
+// 2x8 tile, but weights read through an image1d_buffer (CL_R/UINT32 over the
+// same packed-q buffer) instead of a plain global buffer. The ne1==1 GEMV
+// already does this and is much faster per weight byte than this GEMM at
+// small n_q; the structural difference is the image path hits the dedicated
+// TPL1 weight cache (L1) while the global path only reaches L2. At small n_q
+// the forward is weight-read-bound, so L1-cached weights is the lever.
+// The 2 adjacent rows the 2x8 tile reads as a ushort2 are exactly one uint32,
+// so the vload2 becomes a single read_imageui at index gx + (ki/4)*(m/2).
+#ifdef ADRENO_GPU
+REQD_SUBGROUP_SIZE_128
+#endif
+kernel void kernel_gemm_noshuffle_q4_k_f32_kimg(
+ read_only image1d_buffer_t src0_q_img,
+ global const uchar * src0_s,
+ global const half * src0_d,
+ global const half * src0_dm,
+ read_only image1d_buffer_t src1,
+ global float * dst,
+ ulong offsetd,
+ int m,
+ int n,
+ int k,
+ int n_no_padding,
+ uchar mask_d6,
+ uchar mask_d4,
+ uchar mask_hi2
+) {
+ dst = (global float *)((global char *)dst + offsetd);
+ int n_4 = n >> 2;
+ int m_2 = m >> 1;
+ int gy = get_global_id(0);
+ int gx = get_global_id(1);
+ int gx_2 = gx << 1;
+
+ half8 c0 = 0, c1 = 0;
+ half8 B;
+ half2 dequantized_weights;
+
+ int num_blocks_K = k / QK_K;
+
+ global const half * d_ptr = src0_d + gx_2;
+ global const half * dm_ptr = src0_dm + gx_2;
+
+ for (int i = 0; i < k; i += 32) {
+ int sb_idx = i / QK_K;
+ int sub_idx = (i / 32) % 8;
+
+ half2 d = vload2(0, d_ptr + sb_idx * m);
+ half2 dm = vload2(0, dm_ptr + sb_idx * m);
+
+ global const uchar * sc0 = src0_s + sb_idx * K_SCALE_SIZE * m + (gx_2+0);
+ global const uchar * sc1 = sc0 + 1;
+
+ uchar sv0, mn0, sv1, mn1;
+ get_scale_min_k4(sub_idx, sc0, m, &sv0, &mn0, mask_d6, mask_d4, mask_hi2);
+ get_scale_min_k4(sub_idx, sc1, m, &sv1, &mn1, mask_d6, mask_d4, mask_hi2);
+
+ half2 scale = convert_half2(convert_float2(d) * convert_float2((uchar2)(sv0, sv1)));
+ half2 mval = convert_half2(convert_float2(dm) * convert_float2((uchar2)(mn0, mn1)));
+
+ for (int l = 0; l < 32; l += 4) {
+ int ki = i + l;
+ uint wpacked = read_imageui(src0_q_img, gx + (ki/4) * m_2).x;
+ ushort2 bits2 = (ushort2)((ushort)(wpacked & 0xFFFFu), (ushort)(wpacked >> 16));
+
+ // j=0
+ B.s0123 = read_imageh(src1, gy*2 + (ki+0) * n_4);
+ B.s4567 = read_imageh(src1, gy*2+1 + (ki+0) * n_4);
+ dequantized_weights.s0 = (bits2.s0 & 0x000F) * scale.s0 - mval.s0;
+ dequantized_weights.s1 = (bits2.s1 & 0x000F) * scale.s1 - mval.s1;
+ c0 += B * dequantized_weights.s0;
+ c1 += B * dequantized_weights.s1;
+
+ // j=1
+ B.s0123 = read_imageh(src1, gy*2 + (ki+1) * n_4);
+ B.s4567 = read_imageh(src1, gy*2+1 + (ki+1) * n_4);
+ dequantized_weights.s0 = ((bits2.s0 & 0x00F0) >> 4) * scale.s0 - mval.s0;
+ dequantized_weights.s1 = ((bits2.s1 & 0x00F0) >> 4) * scale.s1 - mval.s1;
+ c0 += B * dequantized_weights.s0;
+ c1 += B * dequantized_weights.s1;
+
+ // j=2
+ B.s0123 = read_imageh(src1, gy*2 + (ki+2) * n_4);
+ B.s4567 = read_imageh(src1, gy*2+1 + (ki+2) * n_4);
+ dequantized_weights.s0 = ((bits2.s0 & 0x0F00) >> 8) * scale.s0 - mval.s0;
+ dequantized_weights.s1 = ((bits2.s1 & 0x0F00) >> 8) * scale.s1 - mval.s1;
+ c0 += B * dequantized_weights.s0;
+ c1 += B * dequantized_weights.s1;
+
+ // j=3
+ B.s0123 = read_imageh(src1, gy*2 + (ki+3) * n_4);
+ B.s4567 = read_imageh(src1, gy*2+1 + (ki+3) * n_4);
+ dequantized_weights.s0 = ((bits2.s0 & 0xF000) >> 12) * scale.s0 - mval.s0;
+ dequantized_weights.s1 = ((bits2.s1 & 0xF000) >> 12) * scale.s1 - mval.s1;
+ c0 += B * dequantized_weights.s0;
+ c1 += B * dequantized_weights.s1;
+ }
+ }
+
+ int idx = (gy<<3)*m + (gx<<1);
+ if (idx+1 < m*n_no_padding) { vstore2((float2)(c0.s0, c1.s0), 0, dst + idx); idx += m; }
+ if (idx+1 < m*n_no_padding) { vstore2((float2)(c0.s1, c1.s1), 0, dst + idx); idx += m; }
+ if (idx+1 < m*n_no_padding) { vstore2((float2)(c0.s2, c1.s2), 0, dst + idx); idx += m; }
+ if (idx+1 < m*n_no_padding) { vstore2((float2)(c0.s3, c1.s3), 0, dst + idx); idx += m; }
+ if (idx+1 < m*n_no_padding) { vstore2((float2)(c0.s4, c1.s4), 0, dst + idx); idx += m; }
+ if (idx+1 < m*n_no_padding) { vstore2((float2)(c0.s5, c1.s5), 0, dst + idx); idx += m; }
+ if (idx+1 < m*n_no_padding) { vstore2((float2)(c0.s6, c1.s6), 0, dst + idx); idx += m; }
+ if (idx+1 < m*n_no_padding) { vstore2((float2)(c0.s7, c1.s7), 0, dst + idx); }
+}
+
+// Cooperative-K GEMM for the small-batch (n_q in [2..8]) path. Mirrors the
+// ne1==1 GEMV's structure: a WG is (COK_SG lanes x COK_NSG subgroups); each
+// lane owns ONE output row and computes its 8 (padded) columns, and the
+// COK_NSG subgroups SPLIT the K reduction round-robin, combining via a
+// __local reduction. This is the thing the per-WI GEMM lacked — at small n_q
+// the old kernel had ~M/256 workgroups each walking all of K serially; this
+// has M/64 workgroups AND COK_NSG-way K parallelism. Uses REQD_SUBGROUP_SIZE_64
+// + barrier (same safe reduction pattern as the GEMV; never sub_group_reduce
+// at full width on X2 per the GDN miscompile note).
+#define COK_NSG 8
+#define COK_SG 64
+#ifdef ADRENO_GPU
+REQD_SUBGROUP_SIZE_64
+#endif
+kernel void kernel_gemm_noshuffle_q4_k_f32_cok(
+ global const ushort * src0_q,
+ global const uchar * src0_s,
+ global const half * src0_d,
+ global const half * src0_dm,
+ read_only image1d_buffer_t src1,
+ global float * dst,
+ ulong offsetd,
+ int m,
+ int n,
+ int k,
+ int n_no_padding,
+ uchar mask_d6,
+ uchar mask_d4,
+ uchar mask_hi2
+) {
+ dst = (global float *)((global char *)dst + offsetd);
+ int n_4 = n >> 2;
+ int gx = get_global_id(0); // output row
+ int sg = get_local_id(1); // subgroup index (K-split lane)
+ int lane = get_local_id(0); // lane within subgroup (0..COK_SG-1)
+
+ int num_blocks_K = k / QK_K;
+ int num_32blk = k / 32;
+
+ global const ushort * weight_ptr = src0_q + gx;
+ global const half * d_ptr = src0_d + gx;
+ global const half * dm_ptr = src0_dm + gx;
+
+ half8 acc = 0;
+ half8 B;
+ half dq;
+
+ for (int blk = sg; blk < num_32blk; blk += COK_NSG) {
+ int i = blk << 5; // blk * 32
+ int sb_idx = blk >> 3; // (blk*32) / QK_K (QK_K = 256 = 32*8)
+ int sub_idx = blk & 7; // (i/32) % 8
+
+ half dd = d_ptr [sb_idx * m];
+ half dmm = dm_ptr[sb_idx * m];
+
+ global const uchar * sc0 = src0_s + sb_idx * K_SCALE_SIZE * m + gx;
+ uchar sv0, mn0;
+ get_scale_min_k4(sub_idx, sc0, m, &sv0, &mn0, mask_d6, mask_d4, mask_hi2);
+ half scale = convert_half(convert_float(dd) * (float)sv0);
+ half mval = convert_half(convert_float(dmm) * (float)mn0);
+
+ for (int l = 0; l < 32; l += 4) {
+ int ki = i + l;
+ ushort bits = weight_ptr[(ki>>2) * m];
+
+ B.s0123 = read_imageh(src1, (ki+0) * n_4);
+ B.s4567 = read_imageh(src1, 1 + (ki+0) * n_4);
+ dq = (bits & 0x000F) * scale - mval;
+ acc += B * dq;
+
+ B.s0123 = read_imageh(src1, (ki+1) * n_4);
+ B.s4567 = read_imageh(src1, 1 + (ki+1) * n_4);
+ dq = ((bits & 0x00F0) >> 4) * scale - mval;
+ acc += B * dq;
+
+ B.s0123 = read_imageh(src1, (ki+2) * n_4);
+ B.s4567 = read_imageh(src1, 1 + (ki+2) * n_4);
+ dq = ((bits & 0x0F00) >> 8) * scale - mval;
+ acc += B * dq;
+
+ B.s0123 = read_imageh(src1, (ki+3) * n_4);
+ B.s4567 = read_imageh(src1, 1 + (ki+3) * n_4);
+ dq = ((bits & 0xF000) >> 12) * scale - mval;
+ acc += B * dq;
+ }
+ }
+
+ // cross-subgroup reduction over the K-split (float for accuracy)
+ local float8 reduceLM[COK_SG * (COK_NSG - 1)];
+ if (sg > 0) {
+ reduceLM[(sg - 1) * COK_SG + lane] = convert_float8(acc);
+ }
+ barrier(CLK_LOCAL_MEM_FENCE);
+
+ if (sg == 0) {
+ float8 sum = convert_float8(acc);
+ for (int s = 0; s < COK_NSG - 1; s++) {
+ sum += reduceLM[s * COK_SG + lane];
+ }
+ int idx = gx;
+ if (idx < m*n_no_padding) { dst[idx] = sum.s0; idx += m; }
+ if (idx < m*n_no_padding) { dst[idx] = sum.s1; idx += m; }
+ if (idx < m*n_no_padding) { dst[idx] = sum.s2; idx += m; }
+ if (idx < m*n_no_padding) { dst[idx] = sum.s3; idx += m; }
+ if (idx < m*n_no_padding) { dst[idx] = sum.s4; idx += m; }
+ if (idx < m*n_no_padding) { dst[idx] = sum.s5; idx += m; }
+ if (idx < m*n_no_padding) { dst[idx] = sum.s6; idx += m; }
+ if (idx < m*n_no_padding) { dst[idx] = sum.s7; }
+ }
+}
diff --git a/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q6_k_f32.cl b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q6_k_f32.cl
index 3a9c624508..141f6a2f68 100644
--- a/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q6_k_f32.cl
+++ b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q6_k_f32.cl
@@ -5,6 +5,7 @@
#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable
#define ADRENO_GPU 1
#define REQD_SUBGROUP_SIZE_128 __attribute__((qcom_reqd_sub_group_size("full")))
+#define REQD_SUBGROUP_SIZE_64 __attribute__((qcom_reqd_sub_group_size("half")))
#endif
#ifdef ADRENO_GPU
@@ -138,3 +139,107 @@ kernel void kernel_gemm_noshuffle_q6_K_f32(
vstore4((float4)(c0.s7, c1.s7, c2.s7, c3.s7), 0, dst + idx);
}
}
+
+// Cooperative-K q6_K GEMM for the small-batch (n_q in [2..8]) path. Same idea
+// as the q4_K _cok kernel: WG = (COK_SG lanes x COK_NSG subgroups), each lane
+// owns ONE output row (half8 over the 8 padded cols), and the COK_NSG
+// subgroups split the K iterations round-robin and combine via a __local
+// reduction. Replaces the default 4-row-per-WI tile that walked all of K alone
+// (~M/512 WGs + serial reduction) at small n_q. REQD_SUBGROUP_SIZE_64 +
+// barrier (never sub_group_reduce at full width on X2).
+#define COK_NSG 8
+#define COK_SG 64
+#ifdef ADRENO_GPU
+REQD_SUBGROUP_SIZE_64
+#endif
+kernel void kernel_gemm_noshuffle_q6_K_f32_cok(
+ global const ushort * src0_ql,
+ global const uchar * src0_qh,
+ global const ushort * src0_s,
+ global const half * src0_d,
+ read_only image1d_buffer_t src1,
+ global float * dst,
+ ulong offsetd,
+ int m,
+ int n,
+ int k,
+ int n_no_padding,
+ ushort mask_f000,
+ uchar mask_c0
+) {
+ dst = (global float *)( (global char *)dst + offsetd );
+
+ int n_4 = n >> 2;
+ int gx = get_global_id(0); // output row
+ int sg = get_local_id(1); // subgroup index (K-split)
+ int lane = get_local_id(0); // lane within subgroup
+
+ global const ushort * ptr_ql = src0_ql + gx;
+ global const uchar * ptr_qh = src0_qh + gx;
+ global const ushort * ptr_s = src0_s + gx;
+ global const half * ptr_d = src0_d + gx;
+
+ half8 acc = 0;
+ half8 B;
+ half dq;
+
+ int num_iter = k >> 2; // k/4 iterations, 4 k-values each
+
+ for (int ib = sg; ib < num_iter; ib += COK_NSG) {
+ int i = ib << 2; // ib * 4
+
+ ushort bits4 = ptr_ql[ib * m]; // ql for row gx at this 4-block
+ uchar bits2 = ptr_qh[ib * m]; // qh
+
+ ushort s_packed = ptr_s[(i >> 5) * m]; // (i/16/2) = i/32
+ char2 sc2 = as_char2(s_packed);
+ char scale_s = (((i >> 4) & 1) == 0) ? sc2.s0 : sc2.s1; // (i/16)%2
+ half scale_d = ptr_d[(i >> 8) * m]; // i/256
+
+ // j=0
+ B.s0123 = read_imageh(src1, (i + 0)*n_4 + 0);
+ B.s4567 = read_imageh(src1, (i + 0)*n_4 + 1);
+ dq = (convert_half((bits4 & 0x000F) | ((bits2 & 0x03) << 4)) - 32.f) * scale_s * scale_d;
+ acc += B * dq;
+
+ // j=1
+ B.s0123 = read_imageh(src1, (i + 1)*n_4 + 0);
+ B.s4567 = read_imageh(src1, (i + 1)*n_4 + 1);
+ dq = (convert_half(((bits4 & 0x00F0) >> 4) | ((bits2 & 0x0C) << 2)) - 32.f) * scale_s * scale_d;
+ acc += B * dq;
+
+ // j=2
+ B.s0123 = read_imageh(src1, (i + 2)*n_4 + 0);
+ B.s4567 = read_imageh(src1, (i + 2)*n_4 + 1);
+ dq = (convert_half(((bits4 & 0x0F00) >> 8) | (bits2 & 0x30)) - 32.f) * scale_s * scale_d;
+ acc += B * dq;
+
+ // j=3
+ B.s0123 = read_imageh(src1, (i + 3)*n_4 + 0);
+ B.s4567 = read_imageh(src1, (i + 3)*n_4 + 1);
+ dq = (convert_half(((bits4 & mask_f000) >> 12) | ((bits2 & mask_c0) >> 2)) - 32.f) * scale_s * scale_d;
+ acc += B * dq;
+ }
+
+ local float8 reduceLM[COK_SG * (COK_NSG - 1)];
+ if (sg > 0) {
+ reduceLM[(sg - 1) * COK_SG + lane] = convert_float8(acc);
+ }
+ barrier(CLK_LOCAL_MEM_FENCE);
+
+ if (sg == 0) {
+ float8 sum = convert_float8(acc);
+ for (int s = 0; s < COK_NSG - 1; s++) {
+ sum += reduceLM[s * COK_SG + lane];
+ }
+ int idx = gx;
+ if (idx < m*n_no_padding) { dst[idx] = sum.s0; idx += m; }
+ if (idx < m*n_no_padding) { dst[idx] = sum.s1; idx += m; }
+ if (idx < m*n_no_padding) { dst[idx] = sum.s2; idx += m; }
+ if (idx < m*n_no_padding) { dst[idx] = sum.s3; idx += m; }
+ if (idx < m*n_no_padding) { dst[idx] = sum.s4; idx += m; }
+ if (idx < m*n_no_padding) { dst[idx] = sum.s5; idx += m; }
+ if (idx < m*n_no_padding) { dst[idx] = sum.s6; idx += m; }
+ if (idx < m*n_no_padding) { dst[idx] = sum.s7; }
+ }
+}
diff --git a/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q6_k_f32_tiled.cl b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q6_k_f32_tiled.cl
new file mode 100644
index 0000000000..ffd943a278
--- /dev/null
+++ b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q6_k_f32_tiled.cl
@@ -0,0 +1,136 @@
+// Batched (N>1) q6_K GEMM over the 64-row-TILED canonical layout produced by
+// kernel_convert_block_q6_k_tiled_ns (cvt.cl). Companion to the decode kernel
+// kernel_gemv_noshuffle_q6_K_f32_tiled: SAME pack, SAME canonical e-order
+// dequant (correct by construction vs reference ggml q6_K), extended to N output
+// columns. Makes the batched lm_head/embed (perplexity, spec-decode verify,
+// batched serving) correct on GPU while keeping the tiled convert the fast decode
+// GEMV depends on.
+//
+// One work-item owns one output ROW for a block of BN columns. A work-group is
+// {64 lanes, NTILES subgroups} = NTILES*64 rows; the global z dimension tiles the
+// N columns by BN. Each work-item computes its row's FULL K (no K-split, so no
+// cross-subgroup reduction), which lets the whole work-group share one staged
+// activation block:
+//
+// __local activation staging — the BN columns of the current superblock (BN*256
+// floats) are loaded into __local once per superblock, cooperatively by all
+// NTILES*64 work-items, then every row reads its activation from __local. This
+// removes the ~Nrows-fold redundant image reads of the first version (each lane
+// re-read the activation), which made the batched GEMM ~2x slower than the plain
+// noshuffle GEMM.
+//
+// Weights are read from __global (coalesced) — matching the decode kernel; the
+// lm_head weight is streamed with little reuse where coalesced global beats the
+// Adreno texture cache.
+
+#pragma OPENCL EXTENSION cl_khr_fp16 : enable
+
+#ifdef cl_qcom_reqd_sub_group_size
+#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable
+#define ADRENO_GPU 1
+#define REQD_SUBGROUP_SIZE_64 __attribute__((qcom_reqd_sub_group_size("half")))
+#endif
+
+#define NTILES 4 // 64-row tiles per work-group (NTILES*64 = 256 rows)
+#define TILE_ROWS 64
+#define BN 16 // output columns handled per work-group (global z step)
+#define WG_THREADS (NTILES * TILE_ROWS)
+
+#if defined(ADRENO_GPU)
+REQD_SUBGROUP_SIZE_64
+#endif
+kernel void kernel_gemm_noshuffle_q6_K_f32_tiled(
+ __global uint4 * src0_ql, // tiled: 8 uint4 granules / superblock
+ __global uint4 * src0_qh, // tiled: 4 uint4 granules / superblock
+ __global char * src0_s, // tiled: 16 chars / superblock
+ __global half * src0_d, // tiled: 1 half / superblock
+ read_only image1d_buffer_t src1, // activation [ne00, ne11] f32 (RGBA), column-major
+ global float * dst,
+ ulong offsetd,
+ int ne00,
+ int ne01,
+ int ne11
+) {
+ int rit = get_local_id(0); // 0..63 (lane within a tile; coalesces weight loads)
+ int sg = get_local_id(1); // 0..NTILES-1
+ int lid = sg * TILE_ROWS + rit; // 0..WG_THREADS-1 (flat local id)
+ int row = get_group_id(0) * WG_THREADS + lid;
+ int rt = row / TILE_ROWS; // global 64-row tile index
+ int col0 = get_global_id(2) * BN; // first output column of this block
+
+ int nb = ne00 / 256; // superblocks per row
+ int act_col_stride = ne00 / 4; // activation float4 pixels per column
+
+ const bool row_ok = row < ne01;
+
+ // staged activation: BN columns x 256 elements for the current superblock
+ __local float lact[BN * 256];
+
+ float acc[BN];
+ #pragma unroll
+ for (int j = 0; j < BN; ++j) acc[j] = 0.0f;
+
+ for (int sb = 0; sb < nb; ++sb) {
+ // cooperatively stage BN columns' 256 activation elements (= BN*64 float4)
+ for (int p = lid; p < BN * 64; p += WG_THREADS) {
+ int j = p >> 6; // column within the BN block (p / 64)
+ int e4 = p & 63; // element-quad within the column (p % 64)
+ int c = col0 + j;
+ float4 v = (c < ne11)
+ ? read_imagef(src1, c * act_col_stride + sb * 64 + e4)
+ : (float4)(0.0f);
+ lact[p * 4 + 0] = v.x;
+ lact[p * 4 + 1] = v.y;
+ lact[p * 4 + 2] = v.z;
+ lact[p * 4 + 3] = v.w; // lact[j*256 + e], e = e4*4 + t
+ }
+ barrier(CLK_LOCAL_MEM_FENCE);
+
+ if (row_ok) {
+ int tile_blk = rt * nb + sb; // ne02 == 1 for lm_head/embed
+
+ float dval = (float)src0_d[tile_blk * TILE_ROWS + rit];
+ __global char * sc = src0_s + (tile_blk * TILE_ROWS + rit) * 16;
+
+ uint ql[32];
+ uint qh[16];
+ #pragma unroll
+ for (int g = 0; g < 8; ++g) {
+ uint4 v = src0_ql[(tile_blk * 8 + g) * TILE_ROWS + rit];
+ ql[g*4+0] = v.x; ql[g*4+1] = v.y; ql[g*4+2] = v.z; ql[g*4+3] = v.w;
+ }
+ #pragma unroll
+ for (int g = 0; g < 4; ++g) {
+ uint4 v = src0_qh[(tile_blk * 4 + g) * TILE_ROWS + rit];
+ qh[g*4+0] = v.x; qh[g*4+1] = v.y; qh[g*4+2] = v.z; qh[g*4+3] = v.w;
+ }
+
+ // NOTE: the e loop (256) is deliberately NOT unrolled. Fully unrolling
+ // 256*BN MACs overflows the in-process Adreno compiler (host stack
+ // overflow at clBuildProgram, same class as the FA DK=512 OOM).
+ for (int e = 0; e < 256; ++e) {
+ uint low4 = (ql[e >> 3] >> ((e & 7) * 4)) & 0xF;
+ uint hi2 = (qh[e >> 4] >> ((e & 15) * 2)) & 0x3;
+ int code = (int)(low4 | (hi2 << 4)) - 32;
+ int sidx = ((e >> 7) << 3) + (((e >> 5) & 3) << 1) + ((e >> 4) & 1);
+ float cs = (float)code * (float)sc[sidx] * dval;
+ #pragma unroll
+ for (int j = 0; j < BN; ++j) {
+ acc[j] += cs * lact[j * 256 + e];
+ }
+ }
+ }
+ barrier(CLK_LOCAL_MEM_FENCE);
+ }
+
+ if (row_ok) {
+ dst = (global float*)((global char*)dst + offsetd);
+ #pragma unroll
+ for (int j = 0; j < BN; ++j) {
+ int c = col0 + j;
+ if (c < ne11) {
+ dst[(ulong)c * ne01 + row] = acc[j];
+ }
+ }
+ }
+}
diff --git a/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_0_f32.cl b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_0_f32.cl
index 8de0de1cc3..023e848f73 100644
--- a/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_0_f32.cl
+++ b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_0_f32.cl
@@ -277,3 +277,107 @@ __kernel void kernel_gemv_noshuffle_q4_0_f32(
}
}
+
+// Multi-column (N in [2..4]) variant of the q4_0 decode GEMV, for the speculative
+// / MTP verify batch (n_cols = 2..4 = drafted + bonus positions). Routes the small-
+// batch verify OFF the transposed-GEMM dead-zone (gemm_noshuffle_q4_0) onto the
+// efficient GEMV path. Each K-block's weights (regA hi+lo) are loaded ONCE and
+// reused across the n_cols activation columns. Per-column accumulation is
+// independent and identical to n_cols standalone GEMVs. n_cols==3 is byte-identical
+// to the original mc3 (col3 disabled, slots 6/7 stay zero). Kept the _mc3 name.
+#ifdef VECTOR_SUB_GROUP_BROADCAST
+#define MC_DQ_HI dequantizeBlockAccum_ns_sgbroadcast_8_hi
+#define MC_DQ_LO dequantizeBlockAccum_ns_sgbroadcast_8_lo
+#else
+#define MC_DQ_HI dequantizeBlockAccum_ns_sgbroadcast_1_hi
+#define MC_DQ_LO dequantizeBlockAccum_ns_sgbroadcast_1_lo
+#endif
+// One column c: load this column's activation (own brace scope so the macros'
+// `shared_y` decl is re-scoped), then dequant (hi+lo) against the shared weights.
+#define MC_COL_Q40(ts, c) \
+ { if (slid < 4) { regB.s0123 = read_imagef(src1, (c)*COL_STRIDE + slid*2 + k*8); \
+ regB.s4567 = read_imagef(src1, (c)*COL_STRIDE + 1 + slid*2 + k*8); } \
+ MC_DQ_HI(ts, as_ushort8(regA_hi), regS, regB); \
+ MC_DQ_LO(ts, as_ushort8(regA_lo), regS, regB); }
+
+#ifdef ADRENO_GPU
+REQD_SUBGROUP_SIZE_64
+#endif
+__kernel void kernel_gemv_noshuffle_q4_0_f32_mc3(
+ __read_only image1d_buffer_t src0_q, // quantized A
+ global half2 * src0_d, // A scales
+ __read_only image1d_buffer_t src1, // B (n_cols columns, col-major image)
+ global float * dst, // C (column-major [M x n_cols])
+ ulong offsetd,
+ int ne00, // K
+ int ne01, // M
+ int n_cols) // N (2..4)
+{
+ uint groupId = get_local_id(1);
+ uint gid = get_global_id(0);
+ ushort slid = get_sub_group_local_id();
+
+ uint K = ne00;
+ uint M = ne01;
+
+ uint LINE_STRIDE_A = M / 2;
+ // BLOCK_STRIDE_A is the LAYOUT stride between consecutive K-blocks = 4 uints
+ // per q4_0 block * M (set by the trans4_ns convert). The "4" is uints/block, NOT
+ // the subgroup count — keep it fixed so the K-split count (nsg) can vary.
+ uint BLOCK_STRIDE_A = N_SIMDGROUP * M; // = 4 * M (N_SIMDGROUP is the #define 4)
+ uint COL_STRIDE = K / 4; // float4 pixels per activation column
+ uint nsg = get_local_size(1); // runtime K-split (4 default, 8 small-M)
+
+ __private uint4 regA_hi, regA_lo;
+ __private half2 regS;
+ __private float8 regB;
+
+ __private float2 ts0 = (float2)(0.0f);
+ __private float2 ts1 = (float2)(0.0f);
+ __private float2 ts2 = (float2)(0.0f);
+ __private float2 ts3 = (float2)(0.0f);
+
+ for (uint k = groupId; k < (K / QK4_0); k += nsg) {
+ regS = src0_d[gid + k * LINE_STRIDE_A];
+
+ // weights loaded ONCE, reused across the columns
+ regA_hi.s0 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 0)).x;
+ regA_hi.s1 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 1)).x;
+ regA_hi.s2 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 2)).x;
+ regA_hi.s3 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 3)).x;
+ regA_lo.s0 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 4)).x;
+ regA_lo.s1 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 5)).x;
+ regA_lo.s2 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 6)).x;
+ regA_lo.s3 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 7)).x;
+
+ MC_COL_Q40(ts0, 0);
+ MC_COL_Q40(ts1, 1);
+ if (n_cols > 2) MC_COL_Q40(ts2, 2);
+ if (n_cols > 3) MC_COL_Q40(ts3, 3);
+ }
+
+ // cross-subgroup reduce over nsg subgroups: pack the (up to 4) columns' float2
+ // into a float8. Generalized to runtime nsg (4 default, 8 for small-M). Each
+ // subgroup writes its partial; subgroup 0 sums the rest into its own acc. At
+ // nsg==4 this is byte-identical to the original (sums subgroups 1,2,3 in order).
+ __local float8 reduceLM[SIMDGROUP_WIDTH * 8];
+ float8 acc = (float8)(ts0.s0, ts0.s1, ts1.s0, ts1.s1, ts2.s0, ts2.s1, ts3.s0, ts3.s1);
+ reduceLM[groupId * SIMDGROUP_WIDTH + slid] = acc;
+
+ barrier(CLK_LOCAL_MEM_FENCE);
+
+ if (groupId == 0) {
+ for (uint g = 1; g < nsg; g++) {
+ acc += reduceLM[g * SIMDGROUP_WIDTH + slid];
+ }
+ dst = (global float*)((global char*)dst + offsetd);
+ // dst is column-major [M rows x n_cols cols]: (row, col) at col*M + row
+ vstore2((float2)(acc.s0, acc.s1), 0, &(dst[0 * M + gid * 2]));
+ vstore2((float2)(acc.s2, acc.s3), 0, &(dst[1 * M + gid * 2]));
+ if (n_cols > 2) vstore2((float2)(acc.s4, acc.s5), 0, &(dst[2 * M + gid * 2]));
+ if (n_cols > 3) vstore2((float2)(acc.s6, acc.s7), 0, &(dst[3 * M + gid * 2]));
+ }
+}
+#undef MC_COL_Q40
+#undef MC_DQ_HI
+#undef MC_DQ_LO
diff --git a/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_1_f32.cl b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_1_f32.cl
index 5fa3127806..2ccf4214c0 100644
--- a/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_1_f32.cl
+++ b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_1_f32.cl
@@ -286,3 +286,99 @@ kernel void kernel_gemv_noshuffle_q4_1_f32(
}
}
+
+// Multi-column (N in [2..4]) variant of the q4_1 decode GEMV (spec/MTP verify) =
+// q4_0 mc3 + the q4_1 per-block min (regM; dequant = q*scale + minv). n_cols=2..4;
+// routes the small-batch verify OFF the gemm_noshuffle_q4_1 dead-zone. n_cols==3 is
+// byte-identical to the original mc3. NB: this file spells the vec-broadcast define
+// BROADCAT (no S) — match it so the fast _8 path compiles.
+#ifdef VECTOR_SUB_GROUP_BROADCAT
+#define MC_DQ1_HI dequantizeBlockAccum_ns_sgbroadcast_8_hi
+#define MC_DQ1_LO dequantizeBlockAccum_ns_sgbroadcast_8_lo
+#else
+#define MC_DQ1_HI dequantizeBlockAccum_ns_sgbroadcast_1_hi
+#define MC_DQ1_LO dequantizeBlockAccum_ns_sgbroadcast_1_lo
+#endif
+#define MC_COL_Q41(ts, c) \
+ { if (slid < 4) { regB.s0123 = read_imagef(src1, (c)*COL_STRIDE + slid*2 + k*8); \
+ regB.s4567 = read_imagef(src1, (c)*COL_STRIDE + 1 + slid*2 + k*8); } \
+ MC_DQ1_HI(ts, as_ushort8(regA_hi), regS, regM, regB); \
+ MC_DQ1_LO(ts, as_ushort8(regA_lo), regS, regM, regB); }
+#ifdef ADRENO_GPU
+REQD_SUBGROUP_SIZE_64
+#endif
+kernel void kernel_gemv_noshuffle_q4_1_f32_mc3(
+ read_only image1d_buffer_t src0_q,
+ global half2 * src0_d,
+ global half2 * src0_m,
+ read_only image1d_buffer_t src1,
+ global float * dst,
+ ulong offsetd,
+ int ne00,
+ int ne01,
+ int n_cols)
+{
+ uint groupId = get_local_id(1);
+ uint gid = get_global_id(0);
+ ushort slid = get_sub_group_local_id();
+
+ uint K = ne00;
+ uint M = ne01;
+
+ uint LINE_STRIDE_A = M / 2;
+ uint BLOCK_STRIDE_A = NSUBGROUPS * M;
+ uint COL_STRIDE = K / 4; // float4 pixels per activation column
+
+ private uint4 regA_hi, regA_lo;
+ private half2 regS, regM;
+ private float8 regB;
+
+ private float2 ts0 = (float2)(0.0f);
+ private float2 ts1 = (float2)(0.0f);
+ private float2 ts2 = (float2)(0.0f);
+ private float2 ts3 = (float2)(0.0f);
+
+ for (uint k = groupId; k < (K / QK4_0); k += NSUBGROUPS) {
+ regS = src0_d[gid + k * LINE_STRIDE_A];
+ regM = src0_m[gid + k * LINE_STRIDE_A];
+
+ // weights loaded ONCE, reused across the columns
+ regA_hi.s0 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 0)).x;
+ regA_hi.s1 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 1)).x;
+ regA_hi.s2 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 2)).x;
+ regA_hi.s3 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 3)).x;
+ regA_lo.s0 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 4)).x;
+ regA_lo.s1 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 5)).x;
+ regA_lo.s2 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 6)).x;
+ regA_lo.s3 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 7)).x;
+
+ MC_COL_Q41(ts0, 0);
+ MC_COL_Q41(ts1, 1);
+ if (n_cols > 2) MC_COL_Q41(ts2, 2);
+ if (n_cols > 3) MC_COL_Q41(ts3, 3);
+ }
+
+ // cross-subgroup reduce: pack the (up to 4) columns' float2 into a float8.
+ local float8 reduceLM[SUBGROUP_SIZE * 3];
+ float8 acc = (float8)(ts0.s0, ts0.s1, ts1.s0, ts1.s1, ts2.s0, ts2.s1, ts3.s0, ts3.s1);
+ if (groupId == 1) { reduceLM[SUBGROUP_SIZE * 0 + slid] = acc; }
+ if (groupId == 2) { reduceLM[SUBGROUP_SIZE * 1 + slid] = acc; }
+ if (groupId == 3) { reduceLM[SUBGROUP_SIZE * 2 + slid] = acc; }
+
+ barrier(CLK_LOCAL_MEM_FENCE);
+
+ if (groupId == 0) {
+ acc += reduceLM[SUBGROUP_SIZE * 0 + slid];
+ acc += reduceLM[SUBGROUP_SIZE * 1 + slid];
+ acc += reduceLM[SUBGROUP_SIZE * 2 + slid];
+ dst = (global float*)((global char*)dst + offsetd);
+ // dst is column-major [M rows x n_cols cols]: (row, col) at col*M + row
+ vstore2((float2)(acc.s0, acc.s1), 0, &(dst[0 * M + gid * 2]));
+ vstore2((float2)(acc.s2, acc.s3), 0, &(dst[1 * M + gid * 2]));
+ if (n_cols > 2) vstore2((float2)(acc.s4, acc.s5), 0, &(dst[2 * M + gid * 2]));
+ if (n_cols > 3) vstore2((float2)(acc.s6, acc.s7), 0, &(dst[3 * M + gid * 2]));
+ }
+}
+#undef MC_COL_Q41
+#undef MC_DQ1_HI
+#undef MC_DQ1_LO
diff --git a/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_k_f32.cl b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_k_f32.cl
index 9ab0dee693..c0078131e9 100644
--- a/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_k_f32.cl
+++ b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_k_f32.cl
@@ -228,12 +228,20 @@ kernel void kernel_gemv_noshuffle_q4_k_f32(
uint groupId = get_local_id(1);
uint gid = get_global_id(0);
ushort slid = get_sub_group_local_id();
+ // K-split factor = #subgroups in the WG. Read from the launch (NOT a compile
+ // constant) so small-M projections (Kcur/Vcur/Qcur) can dispatch a wider
+ // K-split (more waves/SP -> latency hiding) while large-M keeps 4. The
+ // physical weight layout stride below is INDEPENDENT of this (see BLOCK_STRIDE_A).
+ uint nsg = get_local_size(1);
uint K = ne00;
uint M = ne01;
uint LINE_STRIDE_A = M / 2;
- uint BLOCK_STRIDE_A = NSUBGROUPS * M;
+ // Physical per-K-block stride in the packed image: 8 uints/block-row-pair *
+ // (M/2) row-pairs = 4*M uints. This is a layout constant, not tied to nsg.
+ uint BLOCK_STRIDE_A = 4 * M;
+ uint scales_per_row = (K / QK_K) * 12;
// The x-grid is padded to CEIL_DIV(ne01/2,64)*64, so when ne01 % 128 != 0 the
// tail lanes hold gid >= ne01/2. The output stores below are guarded, but the
@@ -259,7 +267,7 @@ kernel void kernel_gemv_noshuffle_q4_k_f32(
private float2 totalSum = (float2)(0.0f);
- for (uint k = groupId; k < (K / 32); k += NSUBGROUPS) {
+ for (uint k = groupId; k < (K / 32); k += nsg) {
uint sb = k / 8;
uint j = k % 8;
@@ -303,28 +311,21 @@ kernel void kernel_gemv_noshuffle_q4_k_f32(
#endif // VECTOR_SUB_GROUP_BROADCAST
}
- // reduction in local memory, assumes #wave=4
- local float2 reduceLM[SUBGROUP_SIZE * 3];
- if (groupId == 1) {
- reduceLM[SUBGROUP_SIZE * 0 + slid] = totalSum;
- }
- if (groupId == 2) {
- reduceLM[SUBGROUP_SIZE * 1 + slid] = totalSum;
- }
- if (groupId == 3) {
- reduceLM[SUBGROUP_SIZE * 2 + slid] = totalSum;
+ // Cross-subgroup reduction in local memory. Generalized to nsg subgroups
+ // (was a hard-coded 4-wave unroll). Sized for up to 16 subgroups (the widest
+ // K-split we dispatch for small M). At nsg==4 the accumulation order is
+ // identical to the original unroll -> byte-identical for the large-M path.
+ local float2 reduceLM[SUBGROUP_SIZE * 15];
+ if (groupId > 0) {
+ reduceLM[SUBGROUP_SIZE * (groupId - 1) + slid] = totalSum;
}
barrier(CLK_LOCAL_MEM_FENCE);
if (groupId == 0) {
- totalSum += reduceLM[SUBGROUP_SIZE * 0 + slid];
- }
- if (groupId == 0) {
- totalSum += reduceLM[SUBGROUP_SIZE * 1 + slid];
- }
- if (groupId == 0) {
- totalSum += reduceLM[SUBGROUP_SIZE * 2 + slid];
+ for (uint i = 0; i < nsg - 1; ++i) {
+ totalSum += reduceLM[SUBGROUP_SIZE * i + slid];
+ }
}
// 2 outputs per fiber in wave 0
@@ -339,3 +340,484 @@ kernel void kernel_gemv_noshuffle_q4_k_f32(
}
}
+
+// --- Fused gate+up GEMV + GLU epilogue (FFN) ------------------------------------
+// Folds the FFN's two decode GEMVs (ffn_gate, ffn_up) and the following GLU into a
+// SINGLE dispatch: {MUL_MAT(Wg,x), MUL_MAT(Wu,x), GLU}. Both matmuls share the same
+// activation x (ffn_norm), so the activation image read is issued ONCE per K-block
+// and reused for the gate and up dot products (the per-op path re-reads it twice and
+// also materializes the two full ffn-wide intermediates to global, which the GLU
+// then re-reads). The gate/up partial sums are accumulated in the SAME per-fiber
+// order and reduced in the SAME cross-subgroup order as the standalone GEMV, and the
+// GLU formula is the exact scalar expression from kernels/glu.cl, so the output is
+// BYTE-IDENTICAL to the per-op matmul+matmul+glu path -> safe to default on.
+// glu_op: REGLU=0, GEGLU=1, SWIGLU=2, GEGLU_ERF=4, GEGLU_QUICK=5 (ggml_glu_op).
+// Weights: src0g_* = gate (= GLU src[0]); src0u_* = up (= GLU src[1]).
+#define GLU_GEGLU_COEF_A 0.044715f
+#define GLU_SQRT_2_OVER_PI 0.79788456080286535587989211986876f
+#define GLU_SQRT_2_INV 0.70710678118654752440084436210484f
+#define GLU_QUICK_COEF -1.702f
+
+inline float glu_apply(int glu_op, float g, float u) {
+ float act;
+ if (glu_op == 1) { // GEGLU (tanh-approx gelu)
+ act = 0.5f*g*(1.0f + tanh(GLU_SQRT_2_OVER_PI*g*(1.0f + GLU_GEGLU_COEF_A*g*g)));
+ } else if (glu_op == 2) { // SWIGLU (silu)
+ act = g / (1.0f + exp(-g));
+ } else if (glu_op == 0) { // REGLU
+ return g*u*(g > 0.0f);
+ } else if (glu_op == 4) { // GEGLU_ERF
+ act = 0.5f*g*(1.0f + erf(g*GLU_SQRT_2_INV));
+ } else { // GEGLU_QUICK (glu_op == 5)
+ act = g*(1.0f/(1.0f + exp(GLU_QUICK_COEF*g)));
+ }
+ return act*u;
+}
+
+#ifdef ADRENO_GPU
+REQD_SUBGROUP_SIZE_64
+#endif
+kernel void kernel_gemv_noshuffle_q4_k_f32_glu(
+ read_only image1d_buffer_t src0g_q,
+ global half2 * src0g_d,
+ global half2 * src0g_m,
+ global uchar * src0g_s,
+ read_only image1d_buffer_t src0u_q,
+ global half2 * src0u_d,
+ global half2 * src0u_m,
+ global uchar * src0u_s,
+ read_only image1d_buffer_t src1,
+ global float * dst,
+ ulong offsetd,
+ int ne00,
+ int ne01,
+ int glu_op,
+ uchar mask_d6,
+ uchar mask_d4,
+ uchar mask_hi2)
+{
+ uint groupId = get_local_id(1);
+ uint gid = get_global_id(0);
+ ushort slid = get_sub_group_local_id();
+ uint nsg = get_local_size(1);
+
+ uint K = ne00;
+ uint M = ne01;
+
+ uint LINE_STRIDE_A = M / 2;
+ uint BLOCK_STRIDE_A = 4 * M;
+
+ private uint4 regA;
+ private half2 regS, regM;
+ private float8 regB;
+
+ private float2 gateSum = (float2)(0.0f);
+ private float2 upSum = (float2)(0.0f);
+
+ // Two SEQUENTIAL K-loops (gate fully, then up). Keeping only one weight's
+ // working set live at a time holds the kernel's register footprint at ~the
+ // base single-weight GEMV's, so its max WG stays 1024 (16 subgroups) and the
+ // per-subgroup K-split matches the standalone wide GEMV exactly -> the gate
+ // and up partial sums are BYTE-IDENTICAL to the per-op path. The macro body
+ // is the base kernel's inner loop verbatim, parameterized by weight source.
+#define Q4K_GLU_LOOP(SUM, Q, DD, MM, SS) \
+ for (uint k = groupId; k < (K / 32); k += nsg) { \
+ uint sb = k / 8; \
+ uint j = k % 8; \
+ half2 d = DD[gid + sb * LINE_STRIDE_A]; \
+ half2 dm = MM[gid + sb * LINE_STRIDE_A]; \
+ global const uchar * sc0 = SS + sb * 12 * M + 2 * gid; \
+ global const uchar * sc1 = sc0 + 1; \
+ uchar sv0, mn0, sv1, mn1; \
+ get_scale_min_k4(j, sc0, M, &sv0, &mn0, mask_d6, mask_d4, mask_hi2); \
+ get_scale_min_k4(j, sc1, M, &sv1, &mn1, mask_d6, mask_d4, mask_hi2); \
+ regS = convert_half2(convert_float2(d) * convert_float2((uchar2)(sv0, sv1))); \
+ regM = convert_half2(convert_float2(dm) * convert_float2((uchar2)(mn0, mn1))); \
+ if (slid < 4) { \
+ regB.s0123 = read_imagef(src1, (slid * 2 + k * 8)); \
+ regB.s4567 = read_imagef(src1, (1 + slid * 2 + k * 8)); \
+ } \
+ regA.s0 = read_imageui(Q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 0)).x; \
+ regA.s1 = read_imageui(Q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 1)).x; \
+ regA.s2 = read_imageui(Q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 2)).x; \
+ regA.s3 = read_imageui(Q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 3)).x; \
+ DEQ_HI(SUM, as_ushort8(regA), regS, regM, regB); \
+ regA.s0 = read_imageui(Q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 4)).x; \
+ regA.s1 = read_imageui(Q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 5)).x; \
+ regA.s2 = read_imageui(Q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 6)).x; \
+ regA.s3 = read_imageui(Q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 7)).x; \
+ DEQ_LO(SUM, as_ushort8(regA), regS, regM, regB); \
+ }
+
+#ifdef VECTOR_SUB_GROUP_BROADCAST
+#define DEQ_HI dequantizeBlockAccum_ns_sgbroadcast_8_hi
+#define DEQ_LO dequantizeBlockAccum_ns_sgbroadcast_8_lo
+#else
+#define DEQ_HI dequantizeBlockAccum_ns_sgbroadcast_1_hi
+#define DEQ_LO dequantizeBlockAccum_ns_sgbroadcast_1_lo
+#endif
+
+ Q4K_GLU_LOOP(gateSum, src0g_q, src0g_d, src0g_m, src0g_s)
+ Q4K_GLU_LOOP(upSum, src0u_q, src0u_d, src0u_m, src0u_s)
+
+#undef DEQ_HI
+#undef DEQ_LO
+#undef Q4K_GLU_LOOP
+
+ // Cross-subgroup reduction in local memory. Packs gate (xy) + up (zw) into a
+ // float4 so both reduce in one pass; summation order matches the base GEMV's
+ // per-channel loop -> byte-identical partial sums.
+ local float4 reduceLM[SUBGROUP_SIZE * 15];
+ if (groupId > 0) {
+ reduceLM[SUBGROUP_SIZE * (groupId - 1) + slid] = (float4)(gateSum, upSum);
+ }
+ barrier(CLK_LOCAL_MEM_FENCE);
+ if (groupId == 0) {
+ for (uint i = 0; i < nsg - 1; ++i) {
+ float4 p = reduceLM[SUBGROUP_SIZE * i + slid];
+ gateSum += p.xy;
+ upSum += p.zw;
+ }
+ dst = (global float*)((global char*)dst + offsetd);
+ dst[gid * 2 + 0] = glu_apply(glu_op, gateSum.s0, upSum.s0);
+ dst[gid * 2 + 1] = glu_apply(glu_op, gateSum.s1, upSum.s1);
+ }
+}
+
+// --- Split-K-across-workgroups decode GEMV (small-M projections) ----------------
+// A single-token GEMV makes only ceil(M/2/64) workgroups; a WG runs on one Adreno
+// compute unit, so for small M (Kcur/Vcur, M=512 -> 4 WGs) most of the 16 CUs sit
+// idle and the matmul is bandwidth-starved even with a wide intra-WG K-split. This
+// variant adds a SECOND grid dimension of `ksplit` workgroups that each reduce a
+// disjoint slice of K and write a per-slice partial; kernel_gemv_splitk_reduce_f32
+// then sums the partials into dst. Identical math/layout to the base kernel
+// (physical block stride 4*M, get_scale_min_k4) -> coherent. Gated host-side to
+// M<=1024 (M>=2048
+// already fills the CUs and the extra reduce dispatch only hurts).
+#ifdef ADRENO_GPU
+REQD_SUBGROUP_SIZE_64
+#endif
+kernel void kernel_gemv_noshuffle_q4_k_f32_splitk(
+ read_only image1d_buffer_t src0_q,
+ global half2 * src0_d,
+ global half2 * src0_m,
+ global uchar * src0_s,
+ read_only image1d_buffer_t src1,
+ global float * partial, // [ksplit * M], slice-major
+ int ne00,
+ int ne01,
+ uchar mask_d6,
+ uchar mask_d4,
+ uchar mask_hi2)
+{
+ uint groupId = get_local_id(1);
+ uint gid = get_global_id(0);
+ ushort slid = get_sub_group_local_id();
+ uint nsg = get_local_size(1);
+ uint ksplit = get_num_groups(1);
+ uint kslice = get_group_id(1);
+
+ uint K = ne00;
+ uint M = ne01;
+ uint LINE_STRIDE_A = M / 2;
+ uint BLOCK_STRIDE_A = 4 * M; // physical, independent of the K-split
+
+ private uint4 regA;
+ private half2 regS, regM;
+ private float8 regB;
+ private float2 totalSum = (float2)(0.0f);
+
+ // each (kslice, subgroup) pair owns a disjoint set of K-blocks
+ for (uint k = kslice * nsg + groupId; k < (K / 32); k += ksplit * nsg) {
+ uint sb = k / 8;
+ uint j = k % 8;
+ half2 d = src0_d[gid + sb * LINE_STRIDE_A];
+ half2 dm = src0_m[gid + sb * LINE_STRIDE_A];
+ global const uchar * sc0 = src0_s + sb * 12 * M + 2 * gid;
+ global const uchar * sc1 = sc0 + 1;
+ uchar sv0, mn0, sv1, mn1;
+ get_scale_min_k4(j, sc0, M, &sv0, &mn0, mask_d6, mask_d4, mask_hi2);
+ get_scale_min_k4(j, sc1, M, &sv1, &mn1, mask_d6, mask_d4, mask_hi2);
+ regS = convert_half2(convert_float2(d) * convert_float2((uchar2)(sv0, sv1)));
+ regM = convert_half2(convert_float2(dm) * convert_float2((uchar2)(mn0, mn1)));
+ if (slid < 4) {
+ regB.s0123 = read_imagef(src1, (slid * 2 + k * 8));
+ regB.s4567 = read_imagef(src1, (1 + slid * 2 + k * 8));
+ }
+ regA.s0 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 0)).x;
+ regA.s1 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 1)).x;
+ regA.s2 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 2)).x;
+ regA.s3 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 3)).x;
+#ifdef VECTOR_SUB_GROUP_BROADCAST
+ dequantizeBlockAccum_ns_sgbroadcast_8_hi(totalSum, as_ushort8(regA), regS, regM, regB);
+#else
+ dequantizeBlockAccum_ns_sgbroadcast_1_hi(totalSum, as_ushort8(regA), regS, regM, regB);
+#endif
+ regA.s0 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 4)).x;
+ regA.s1 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 5)).x;
+ regA.s2 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 6)).x;
+ regA.s3 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 7)).x;
+#ifdef VECTOR_SUB_GROUP_BROADCAST
+ dequantizeBlockAccum_ns_sgbroadcast_8_lo(totalSum, as_ushort8(regA), regS, regM, regB);
+#else
+ dequantizeBlockAccum_ns_sgbroadcast_1_lo(totalSum, as_ushort8(regA), regS, regM, regB);
+#endif
+ }
+
+ local float2 reduceLM[SUBGROUP_SIZE * 15];
+ if (groupId > 0) {
+ reduceLM[SUBGROUP_SIZE * (groupId - 1) + slid] = totalSum;
+ }
+ barrier(CLK_LOCAL_MEM_FENCE);
+ if (groupId == 0) {
+ for (uint i = 0; i < nsg - 1; ++i) {
+ totalSum += reduceLM[SUBGROUP_SIZE * i + slid];
+ }
+ vstore2(totalSum, 0, &(partial[kslice * M + gid * 2]));
+ }
+}
+
+// Sum the per-slice partials [ksplit * M] into dst[M]; applies the dst byte offset.
+kernel void kernel_gemv_splitk_reduce_f32(
+ global float * partial,
+ global float * dst,
+ ulong offsetd,
+ int ne01, // M
+ int ksplit)
+{
+ uint r = get_global_id(0);
+ if (r >= (uint)ne01) return;
+ float acc = 0.0f;
+ for (uint s = 0; s < (uint)ksplit; ++s) {
+ acc += partial[s * (uint)ne01 + r];
+ }
+ dst = (global float*)((global char*)dst + offsetd);
+ dst[r] = acc;
+}
+
+
+// --- Dequant-once macros for the mc3 verify GEMV (Q4K_MC3_DEQUANT_ONCE) ---
+// The inline dequantizeBlockAccum_* macros recompute the dequantized weight
+// ((code & mask)>>shift)*scale - minv ONCE PER COLUMN (3x), and the flat
+// 32-FMA unroll spills ~430 B of temporaries. These macros split the work:
+// DEQUANT_Q4K_BLOCK computes the 16 weights/row of one 32-block ONCE into a
+// half2[] (row0 in .s0, row1 in .s1) — stored as half, the exact type the
+// inline expression yields (int*half-half), so no extra rounding. MAC_Q4K_BLOCK
+// then accumulates them against a column's broadcast activation in the SAME
+// per-accumulator order as the inline macro. Each weight value and each
+// accumulator's add-chain is bit-for-bit identical => byte-identical output,
+// while the dequant ALU drops 3x->1x and the live set shrinks. Requires the
+// Qualcomm vector sub_group_broadcast (float8); enabled opt-in on Adreno.
+#define DEQ_Q4K_HALF2(b0, b1, msk, sh, scale, minv) \
+ (half2)( ((b0 & msk) >> sh) * scale.s0 - minv.s0, \
+ ((b1 & msk) >> sh) * scale.s1 - minv.s1 )
+
+#define DEQUANT_Q4K_BLOCK(wq, bits, scale, minv) \
+ wq[0] = DEQ_Q4K_HALF2(bits.s0, bits.s1, 0x000F, 0, scale, minv); \
+ wq[1] = DEQ_Q4K_HALF2(bits.s0, bits.s1, 0x00F0, 4, scale, minv); \
+ wq[2] = DEQ_Q4K_HALF2(bits.s0, bits.s1, 0x0F00, 8, scale, minv); \
+ wq[3] = DEQ_Q4K_HALF2(bits.s0, bits.s1, 0xF000, 12, scale, minv); \
+ wq[4] = DEQ_Q4K_HALF2(bits.s2, bits.s3, 0x000F, 0, scale, minv); \
+ wq[5] = DEQ_Q4K_HALF2(bits.s2, bits.s3, 0x00F0, 4, scale, minv); \
+ wq[6] = DEQ_Q4K_HALF2(bits.s2, bits.s3, 0x0F00, 8, scale, minv); \
+ wq[7] = DEQ_Q4K_HALF2(bits.s2, bits.s3, 0xF000, 12, scale, minv); \
+ wq[8] = DEQ_Q4K_HALF2(bits.s4, bits.s5, 0x000F, 0, scale, minv); \
+ wq[9] = DEQ_Q4K_HALF2(bits.s4, bits.s5, 0x00F0, 4, scale, minv); \
+ wq[10] = DEQ_Q4K_HALF2(bits.s4, bits.s5, 0x0F00, 8, scale, minv); \
+ wq[11] = DEQ_Q4K_HALF2(bits.s4, bits.s5, 0xF000, 12, scale, minv); \
+ wq[12] = DEQ_Q4K_HALF2(bits.s6, bits.s7, 0x000F, 0, scale, minv); \
+ wq[13] = DEQ_Q4K_HALF2(bits.s6, bits.s7, 0x00F0, 4, scale, minv); \
+ wq[14] = DEQ_Q4K_HALF2(bits.s6, bits.s7, 0x0F00, 8, scale, minv); \
+ wq[15] = DEQ_Q4K_HALF2(bits.s6, bits.s7, 0xF000, 12, scale, minv);
+
+// ln0/ln1 = the two source lanes whose activation float8 this block consumes
+// (0,1 for the hi block, 2,3 for the lo block — matching the inline _hi/_lo).
+#define MAC_Q4K_BLOCK(ts, wq, y, ln0, ln1) { \
+ float8 sy = sub_group_broadcast(y, ln0); \
+ ts.s0 += wq[0].s0*sy.s0; ts.s0 += wq[1].s0*sy.s1; ts.s0 += wq[2].s0*sy.s2; ts.s0 += wq[3].s0*sy.s3; \
+ ts.s0 += wq[4].s0*sy.s4; ts.s0 += wq[5].s0*sy.s5; ts.s0 += wq[6].s0*sy.s6; ts.s0 += wq[7].s0*sy.s7; \
+ ts.s1 += wq[0].s1*sy.s0; ts.s1 += wq[1].s1*sy.s1; ts.s1 += wq[2].s1*sy.s2; ts.s1 += wq[3].s1*sy.s3; \
+ ts.s1 += wq[4].s1*sy.s4; ts.s1 += wq[5].s1*sy.s5; ts.s1 += wq[6].s1*sy.s6; ts.s1 += wq[7].s1*sy.s7; \
+ sy = sub_group_broadcast(y, ln1); \
+ ts.s0 += wq[8].s0*sy.s0; ts.s0 += wq[9].s0*sy.s1; ts.s0 += wq[10].s0*sy.s2; ts.s0 += wq[11].s0*sy.s3; \
+ ts.s0 += wq[12].s0*sy.s4; ts.s0 += wq[13].s0*sy.s5; ts.s0 += wq[14].s0*sy.s6; ts.s0 += wq[15].s0*sy.s7; \
+ ts.s1 += wq[8].s1*sy.s0; ts.s1 += wq[9].s1*sy.s1; ts.s1 += wq[10].s1*sy.s2; ts.s1 += wq[11].s1*sy.s3; \
+ ts.s1 += wq[12].s1*sy.s4; ts.s1 += wq[13].s1*sy.s5; ts.s1 += wq[14].s1*sy.s6; ts.s1 += wq[15].s1*sy.s7; \
+}
+
+// Multi-column (N=3) variant of the q4_K decode GEMV, for the speculative /
+// MTP verify batch (ne1=3 = 2 drafts + 1 bonus). Stays on the efficient GEMV
+// path (subgroup-broadcast activation, NSUBGROUPS K-split) instead of the
+// transposed-GEMM dead-zone path. Each K-block's weights (regA_hi/regA_lo) are
+// loaded ONCE and reused across all 3 activation columns — same weight traffic
+// as one decode, ~3x the (cheap) dequant ALU. Per-column accumulation is
+// independent and identical to 3 standalone GEMVs => byte-identical, so it does
+// NOT perturb the lm_head logits / spec accept rate.
+#ifdef ADRENO_GPU
+REQD_SUBGROUP_SIZE_64
+#endif
+kernel void kernel_gemv_noshuffle_q4_k_f32_mc3(
+ read_only image1d_buffer_t src0_q,
+ global half2 * src0_d,
+ global half2 * src0_m,
+ global uchar * src0_s,
+ read_only image1d_buffer_t src1,
+ global float * dst,
+ ulong offsetd,
+ int ne00,
+ int ne01,
+ uchar mask_d6,
+ uchar mask_d4,
+ uchar mask_hi2)
+{
+ uint groupId = get_local_id(1);
+ uint gid = get_global_id(0);
+ ushort slid = get_sub_group_local_id();
+
+ uint K = ne00;
+ uint M = ne01;
+
+ uint LINE_STRIDE_A = M / 2;
+ uint BLOCK_STRIDE_A = NSUBGROUPS * M;
+ uint COL_STRIDE = K / 4; // float4 pixels per activation column
+
+ private uint4 regA_hi, regA_lo;
+ private half2 regS, regM;
+ private float8 regB;
+
+ private float2 ts0 = (float2)(0.0f);
+ private float2 ts1 = (float2)(0.0f);
+ private float2 ts2 = (float2)(0.0f);
+
+#ifdef Q4K_MC3_DEQUANT_LDS
+ // One 16-half2 block buffer per WI (reused hi->lo): forces the dequantized
+ // weights into LDS instead of private arrays (which spill to slow global on
+ // Adreno). 64*NSUBGROUPS WIs * 16 half2 = 16 KB; each WI owns its own slot
+ // range (flat*16) -> no cross-lane sharing, no barrier needed.
+ local half2 wstage[SUBGROUP_SIZE * NSUBGROUPS * 16];
+ local half2 * ws = wstage + (groupId * SUBGROUP_SIZE + slid) * 16;
+#endif
+
+ for (uint k = groupId; k < (K / 32); k += NSUBGROUPS) {
+ uint sb = k / 8;
+ uint j = k % 8;
+
+ half2 d = src0_d[gid + sb * LINE_STRIDE_A];
+ half2 dm = src0_m[gid + sb * LINE_STRIDE_A];
+
+ global const uchar * sc0 = src0_s + sb * 12 * M + 2 * gid;
+ global const uchar * sc1 = sc0 + 1;
+
+ uchar sv0, mn0, sv1, mn1;
+ get_scale_min_k4(j, sc0, M, &sv0, &mn0, mask_d6, mask_d4, mask_hi2);
+ get_scale_min_k4(j, sc1, M, &sv1, &mn1, mask_d6, mask_d4, mask_hi2);
+
+ regS = convert_half2(convert_float2(d) * convert_float2((uchar2)(sv0, sv1)));
+ regM = convert_half2(convert_float2(dm) * convert_float2((uchar2)(mn0, mn1)));
+
+ // weights loaded ONCE, reused across the 3 columns
+ regA_hi.s0 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 0)).x;
+ regA_hi.s1 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 1)).x;
+ regA_hi.s2 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 2)).x;
+ regA_hi.s3 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 3)).x;
+ regA_lo.s0 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 4)).x;
+ regA_lo.s1 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 5)).x;
+ regA_lo.s2 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 6)).x;
+ regA_lo.s3 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 7)).x;
+
+#ifdef Q4K_MC3_DEQUANT_ONCE
+ // Dequant the 32 weights/row (16 hi + 16 lo) ONCE into half2[] (byte-
+ // identical to the inline intermediate), then MAC against each column's
+ // activation. Drops the dequant ALU 3x->1x and the macro-temp spill.
+ half2 wq_hi[16], wq_lo[16];
+ DEQUANT_Q4K_BLOCK(wq_hi, as_ushort8(regA_hi), regS, regM);
+ DEQUANT_Q4K_BLOCK(wq_lo, as_ushort8(regA_lo), regS, regM);
+ { if (slid < 4) { regB.s0123 = read_imagef(src1, 0*COL_STRIDE + slid*2 + k*8);
+ regB.s4567 = read_imagef(src1, 0*COL_STRIDE + 1 + slid*2 + k*8); }
+ MAC_Q4K_BLOCK(ts0, wq_hi, regB, 0, 1); MAC_Q4K_BLOCK(ts0, wq_lo, regB, 2, 3); }
+ { if (slid < 4) { regB.s0123 = read_imagef(src1, 1*COL_STRIDE + slid*2 + k*8);
+ regB.s4567 = read_imagef(src1, 1*COL_STRIDE + 1 + slid*2 + k*8); }
+ MAC_Q4K_BLOCK(ts1, wq_hi, regB, 0, 1); MAC_Q4K_BLOCK(ts1, wq_lo, regB, 2, 3); }
+ { if (slid < 4) { regB.s0123 = read_imagef(src1, 2*COL_STRIDE + slid*2 + k*8);
+ regB.s4567 = read_imagef(src1, 2*COL_STRIDE + 1 + slid*2 + k*8); }
+ MAC_Q4K_BLOCK(ts2, wq_hi, regB, 0, 1); MAC_Q4K_BLOCK(ts2, wq_lo, regB, 2, 3); }
+#elif defined(Q4K_MC3_DEQUANT_LDS)
+ // LDS-staged dequant: dequant a 32-block ONCE into the per-WI LDS slot
+ // (hi pass then lo pass, overwriting), MAC each column from LDS. ts*
+ // receive hi-then-lo in the same order as DEQUANT_ONCE -> byte-identical.
+ // Activations reloaded per pass (cheap, imaged); only one regB + 0 weight
+ // regs live -> the weight working set lives in LDS, not spilled private.
+ DEQUANT_Q4K_BLOCK(ws, as_ushort8(regA_hi), regS, regM);
+ { if (slid < 4) { regB.s0123 = read_imagef(src1, 0*COL_STRIDE + slid*2 + k*8);
+ regB.s4567 = read_imagef(src1, 0*COL_STRIDE + 1 + slid*2 + k*8); }
+ MAC_Q4K_BLOCK(ts0, ws, regB, 0, 1); }
+ { if (slid < 4) { regB.s0123 = read_imagef(src1, 1*COL_STRIDE + slid*2 + k*8);
+ regB.s4567 = read_imagef(src1, 1*COL_STRIDE + 1 + slid*2 + k*8); }
+ MAC_Q4K_BLOCK(ts1, ws, regB, 0, 1); }
+ { if (slid < 4) { regB.s0123 = read_imagef(src1, 2*COL_STRIDE + slid*2 + k*8);
+ regB.s4567 = read_imagef(src1, 2*COL_STRIDE + 1 + slid*2 + k*8); }
+ MAC_Q4K_BLOCK(ts2, ws, regB, 0, 1); }
+ DEQUANT_Q4K_BLOCK(ws, as_ushort8(regA_lo), regS, regM);
+ { if (slid < 4) { regB.s0123 = read_imagef(src1, 0*COL_STRIDE + slid*2 + k*8);
+ regB.s4567 = read_imagef(src1, 0*COL_STRIDE + 1 + slid*2 + k*8); }
+ MAC_Q4K_BLOCK(ts0, ws, regB, 2, 3); }
+ { if (slid < 4) { regB.s0123 = read_imagef(src1, 1*COL_STRIDE + slid*2 + k*8);
+ regB.s4567 = read_imagef(src1, 1*COL_STRIDE + 1 + slid*2 + k*8); }
+ MAC_Q4K_BLOCK(ts1, ws, regB, 2, 3); }
+ { if (slid < 4) { regB.s0123 = read_imagef(src1, 2*COL_STRIDE + slid*2 + k*8);
+ regB.s4567 = read_imagef(src1, 2*COL_STRIDE + 1 + slid*2 + k*8); }
+ MAC_Q4K_BLOCK(ts2, ws, regB, 2, 3); }
+#else
+ // Per-column: load only this column's activation (single regB live at a
+ // time -> 1/3 the activation register pressure vs holding all 3) then
+ // dequant against the shared weights. Cuts the private-mem spill.
+#ifdef VECTOR_SUB_GROUP_BROADCAST
+ { if (slid < 4) { regB.s0123 = read_imagef(src1, 0*COL_STRIDE + slid*2 + k*8);
+ regB.s4567 = read_imagef(src1, 0*COL_STRIDE + 1 + slid*2 + k*8); }
+ dequantizeBlockAccum_ns_sgbroadcast_8_hi(ts0, as_ushort8(regA_hi), regS, regM, regB);
+ dequantizeBlockAccum_ns_sgbroadcast_8_lo(ts0, as_ushort8(regA_lo), regS, regM, regB); }
+ { if (slid < 4) { regB.s0123 = read_imagef(src1, 1*COL_STRIDE + slid*2 + k*8);
+ regB.s4567 = read_imagef(src1, 1*COL_STRIDE + 1 + slid*2 + k*8); }
+ dequantizeBlockAccum_ns_sgbroadcast_8_hi(ts1, as_ushort8(regA_hi), regS, regM, regB);
+ dequantizeBlockAccum_ns_sgbroadcast_8_lo(ts1, as_ushort8(regA_lo), regS, regM, regB); }
+ { if (slid < 4) { regB.s0123 = read_imagef(src1, 2*COL_STRIDE + slid*2 + k*8);
+ regB.s4567 = read_imagef(src1, 2*COL_STRIDE + 1 + slid*2 + k*8); }
+ dequantizeBlockAccum_ns_sgbroadcast_8_hi(ts2, as_ushort8(regA_hi), regS, regM, regB);
+ dequantizeBlockAccum_ns_sgbroadcast_8_lo(ts2, as_ushort8(regA_lo), regS, regM, regB); }
+#else
+ { if (slid < 4) { regB.s0123 = read_imagef(src1, 0*COL_STRIDE + slid*2 + k*8);
+ regB.s4567 = read_imagef(src1, 0*COL_STRIDE + 1 + slid*2 + k*8); }
+ dequantizeBlockAccum_ns_sgbroadcast_1_hi(ts0, as_ushort8(regA_hi), regS, regM, regB);
+ dequantizeBlockAccum_ns_sgbroadcast_1_lo(ts0, as_ushort8(regA_lo), regS, regM, regB); }
+ { if (slid < 4) { regB.s0123 = read_imagef(src1, 1*COL_STRIDE + slid*2 + k*8);
+ regB.s4567 = read_imagef(src1, 1*COL_STRIDE + 1 + slid*2 + k*8); }
+ dequantizeBlockAccum_ns_sgbroadcast_1_hi(ts1, as_ushort8(regA_hi), regS, regM, regB);
+ dequantizeBlockAccum_ns_sgbroadcast_1_lo(ts1, as_ushort8(regA_lo), regS, regM, regB); }
+ { if (slid < 4) { regB.s0123 = read_imagef(src1, 2*COL_STRIDE + slid*2 + k*8);
+ regB.s4567 = read_imagef(src1, 2*COL_STRIDE + 1 + slid*2 + k*8); }
+ dequantizeBlockAccum_ns_sgbroadcast_1_hi(ts2, as_ushort8(regA_hi), regS, regM, regB);
+ dequantizeBlockAccum_ns_sgbroadcast_1_lo(ts2, as_ushort8(regA_lo), regS, regM, regB); }
+#endif
+#endif // Q4K_MC3_DEQUANT_ONCE
+ }
+
+ // cross-subgroup reduce: pack the 3 columns' float2 into a float8 (6 used).
+ local float8 reduceLM[SUBGROUP_SIZE * 3];
+ float8 acc = (float8)(ts0.s0, ts0.s1, ts1.s0, ts1.s1, ts2.s0, ts2.s1, 0.0f, 0.0f);
+ if (groupId == 1) { reduceLM[SUBGROUP_SIZE * 0 + slid] = acc; }
+ if (groupId == 2) { reduceLM[SUBGROUP_SIZE * 1 + slid] = acc; }
+ if (groupId == 3) { reduceLM[SUBGROUP_SIZE * 2 + slid] = acc; }
+
+ barrier(CLK_LOCAL_MEM_FENCE);
+
+ if (groupId == 0) {
+ acc += reduceLM[SUBGROUP_SIZE * 0 + slid];
+ acc += reduceLM[SUBGROUP_SIZE * 1 + slid];
+ acc += reduceLM[SUBGROUP_SIZE * 2 + slid];
+ dst = (global float*)((global char*)dst + offsetd);
+ // dst is column-major [M rows x 3 cols]: (row, col) at col*M + row
+ vstore2((float2)(acc.s0, acc.s1), 0, &(dst[0 * M + gid * 2]));
+ vstore2((float2)(acc.s2, acc.s3), 0, &(dst[1 * M + gid * 2]));
+ vstore2((float2)(acc.s4, acc.s5), 0, &(dst[2 * M + gid * 2]));
+ }
+}
diff --git a/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_k_f32_o4.cl b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_k_f32_o4.cl
new file mode 100644
index 0000000000..02916bb91f
--- /dev/null
+++ b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_k_f32_o4.cl
@@ -0,0 +1,349 @@
+#pragma OPENCL EXTENSION cl_khr_fp16 : enable
+#pragma OPENCL EXTENSION cl_khr_subgroups : enable
+
+#ifdef cl_qcom_reqd_sub_group_size
+#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable
+#define ADRENO_GPU 1
+#define REQD_SUBGROUP_SIZE_64 __attribute__((qcom_reqd_sub_group_size("half")))
+#endif
+
+#define QK_K 256
+#define NSUBGROUPS 4
+#define SUBGROUP_SIZE 64
+
+// scales are transposed: consecutive codes of a row are `stride` apart
+inline void get_scale_min_k4(
+ int j,
+ global const uchar * q,
+ uint stride,
+ uchar * d,
+ uchar * m,
+ uchar mask_d6,
+ uchar mask_d4,
+ uchar mask_hi2
+) {
+ if (j < 4) {
+ *d = q[j*stride] & mask_d6;
+ *m = q[(j+4)*stride] & mask_d6;
+ } else {
+ *d = (q[(j+4)*stride] & mask_d4) | ((q[(j-4)*stride] & mask_hi2) >> 2);
+ *m = ((q[(j+4)*stride] >> 4) & mask_d4) | ((q[j*stride] & mask_hi2) >> 2);
+ }
+}
+
+#define dequantizeBlockAccum_ns_sgbroadcast_1_hi(total_sums, bits4, scale, minv, y) \
+ float shared_y; \
+ shared_y = sub_group_broadcast(y.s0, 0); \
+ total_sums.s0 += ((bits4.s0 & 0x000F) * scale.s0 - minv.s0) * shared_y; \
+ total_sums.s1 += ((bits4.s1 & 0x000F) * scale.s1 - minv.s1) * shared_y; \
+ shared_y = sub_group_broadcast(y.s1, 0); \
+ total_sums.s0 += (((bits4.s0 & 0x00F0) >> 4) * scale.s0 - minv.s0) * shared_y; \
+ total_sums.s1 += (((bits4.s1 & 0x00F0) >> 4) * scale.s1 - minv.s1) * shared_y; \
+ shared_y = sub_group_broadcast(y.s2, 0); \
+ total_sums.s0 += (((bits4.s0 & 0x0F00) >> 8) * scale.s0 - minv.s0) * shared_y; \
+ total_sums.s1 += (((bits4.s1 & 0x0F00) >> 8) * scale.s1 - minv.s1) * shared_y; \
+ shared_y = sub_group_broadcast(y.s3, 0); \
+ total_sums.s0 += (((bits4.s0 & 0xF000) >> 12) * scale.s0 - minv.s0) * shared_y; \
+ total_sums.s1 += (((bits4.s1 & 0xF000) >> 12) * scale.s1 - minv.s1) * shared_y; \
+ shared_y = sub_group_broadcast(y.s4, 0); \
+ total_sums.s0 += ((bits4.s2 & 0x000F) * scale.s0 - minv.s0) * shared_y; \
+ total_sums.s1 += ((bits4.s3 & 0x000F) * scale.s1 - minv.s1) * shared_y; \
+ shared_y = sub_group_broadcast(y.s5, 0); \
+ total_sums.s0 += (((bits4.s2 & 0x00F0) >> 4) * scale.s0 - minv.s0) * shared_y; \
+ total_sums.s1 += (((bits4.s3 & 0x00F0) >> 4) * scale.s1 - minv.s1) * shared_y; \
+ shared_y = sub_group_broadcast(y.s6, 0); \
+ total_sums.s0 += (((bits4.s2 & 0x0F00) >> 8) * scale.s0 - minv.s0) * shared_y; \
+ total_sums.s1 += (((bits4.s3 & 0x0F00) >> 8) * scale.s1 - minv.s1) * shared_y; \
+ shared_y = sub_group_broadcast(y.s7, 0); \
+ total_sums.s0 += (((bits4.s2 & 0xF000) >> 12) * scale.s0 - minv.s0) * shared_y; \
+ total_sums.s1 += (((bits4.s3 & 0xF000) >> 12) * scale.s1 - minv.s1) * shared_y; \
+ shared_y = sub_group_broadcast(y.s0, 1); \
+ total_sums.s0 += ((bits4.s4 & 0x000F) * scale.s0 - minv.s0) * shared_y; \
+ total_sums.s1 += ((bits4.s5 & 0x000F) * scale.s1 - minv.s1) * shared_y; \
+ shared_y = sub_group_broadcast(y.s1, 1); \
+ total_sums.s0 += (((bits4.s4 & 0x00F0) >> 4) * scale.s0 - minv.s0) * shared_y; \
+ total_sums.s1 += (((bits4.s5 & 0x00F0) >> 4) * scale.s1 - minv.s1) * shared_y; \
+ shared_y = sub_group_broadcast(y.s2, 1); \
+ total_sums.s0 += (((bits4.s4 & 0x0F00) >> 8) * scale.s0 - minv.s0) * shared_y; \
+ total_sums.s1 += (((bits4.s5 & 0x0F00) >> 8) * scale.s1 - minv.s1) * shared_y; \
+ shared_y = sub_group_broadcast(y.s3, 1); \
+ total_sums.s0 += (((bits4.s4 & 0xF000) >> 12) * scale.s0 - minv.s0) * shared_y; \
+ total_sums.s1 += (((bits4.s5 & 0xF000) >> 12) * scale.s1 - minv.s1) * shared_y; \
+ shared_y = sub_group_broadcast(y.s4, 1); \
+ total_sums.s0 += ((bits4.s6 & 0x000F) * scale.s0 - minv.s0) * shared_y; \
+ total_sums.s1 += ((bits4.s7 & 0x000F) * scale.s1 - minv.s1) * shared_y; \
+ shared_y = sub_group_broadcast(y.s5, 1); \
+ total_sums.s0 += (((bits4.s6 & 0x00F0) >> 4) * scale.s0 - minv.s0) * shared_y; \
+ total_sums.s1 += (((bits4.s7 & 0x00F0) >> 4) * scale.s1 - minv.s1) * shared_y; \
+ shared_y = sub_group_broadcast(y.s6, 1); \
+ total_sums.s0 += (((bits4.s6 & 0x0F00) >> 8) * scale.s0 - minv.s0) * shared_y; \
+ total_sums.s1 += (((bits4.s7 & 0x0F00) >> 8) * scale.s1 - minv.s1) * shared_y; \
+ shared_y = sub_group_broadcast(y.s7, 1); \
+ total_sums.s0 += (((bits4.s6 & 0xF000) >> 12) * scale.s0 - minv.s0) * shared_y; \
+ total_sums.s1 += (((bits4.s7 & 0xF000) >> 12) * scale.s1 - minv.s1) * shared_y; \
+
+
+#define dequantizeBlockAccum_ns_sgbroadcast_1_lo(total_sums, bits4, scale, minv, y) \
+ shared_y = sub_group_broadcast(y.s0, 2); \
+ total_sums.s0 += ((bits4.s0 & 0x000F) * scale.s0 - minv.s0) * shared_y; \
+ total_sums.s1 += ((bits4.s1 & 0x000F) * scale.s1 - minv.s1) * shared_y; \
+ shared_y = sub_group_broadcast(y.s1, 2); \
+ total_sums.s0 += (((bits4.s0 & 0x00F0) >> 4) * scale.s0 - minv.s0) * shared_y; \
+ total_sums.s1 += (((bits4.s1 & 0x00F0) >> 4) * scale.s1 - minv.s1) * shared_y; \
+ shared_y = sub_group_broadcast(y.s2, 2); \
+ total_sums.s0 += (((bits4.s0 & 0x0F00) >> 8) * scale.s0 - minv.s0) * shared_y; \
+ total_sums.s1 += (((bits4.s1 & 0x0F00) >> 8) * scale.s1 - minv.s1) * shared_y; \
+ shared_y = sub_group_broadcast(y.s3, 2); \
+ total_sums.s0 += (((bits4.s0 & 0xF000) >> 12) * scale.s0 - minv.s0) * shared_y; \
+ total_sums.s1 += (((bits4.s1 & 0xF000) >> 12) * scale.s1 - minv.s1) * shared_y; \
+ shared_y = sub_group_broadcast(y.s4, 2); \
+ total_sums.s0 += ((bits4.s2 & 0x000F) * scale.s0 - minv.s0) * shared_y; \
+ total_sums.s1 += ((bits4.s3 & 0x000F) * scale.s1 - minv.s1) * shared_y; \
+ shared_y = sub_group_broadcast(y.s5, 2); \
+ total_sums.s0 += (((bits4.s2 & 0x00F0) >> 4) * scale.s0 - minv.s0) * shared_y; \
+ total_sums.s1 += (((bits4.s3 & 0x00F0) >> 4) * scale.s1 - minv.s1) * shared_y; \
+ shared_y = sub_group_broadcast(y.s6, 2); \
+ total_sums.s0 += (((bits4.s2 & 0x0F00) >> 8) * scale.s0 - minv.s0) * shared_y; \
+ total_sums.s1 += (((bits4.s3 & 0x0F00) >> 8) * scale.s1 - minv.s1) * shared_y; \
+ shared_y = sub_group_broadcast(y.s7, 2); \
+ total_sums.s0 += (((bits4.s2 & 0xF000) >> 12) * scale.s0 - minv.s0) * shared_y; \
+ total_sums.s1 += (((bits4.s3 & 0xF000) >> 12) * scale.s1 - minv.s1) * shared_y; \
+ shared_y = sub_group_broadcast(y.s0, 3); \
+ total_sums.s0 += ((bits4.s4 & 0x000F) * scale.s0 - minv.s0) * shared_y; \
+ total_sums.s1 += ((bits4.s5 & 0x000F) * scale.s1 - minv.s1) * shared_y; \
+ shared_y = sub_group_broadcast(y.s1, 3); \
+ total_sums.s0 += (((bits4.s4 & 0x00F0) >> 4) * scale.s0 - minv.s0) * shared_y; \
+ total_sums.s1 += (((bits4.s5 & 0x00F0) >> 4) * scale.s1 - minv.s1) * shared_y; \
+ shared_y = sub_group_broadcast(y.s2, 3); \
+ total_sums.s0 += (((bits4.s4 & 0x0F00) >> 8) * scale.s0 - minv.s0) * shared_y; \
+ total_sums.s1 += (((bits4.s5 & 0x0F00) >> 8) * scale.s1 - minv.s1) * shared_y; \
+ shared_y = sub_group_broadcast(y.s3, 3); \
+ total_sums.s0 += (((bits4.s4 & 0xF000) >> 12) * scale.s0 - minv.s0) * shared_y; \
+ total_sums.s1 += (((bits4.s5 & 0xF000) >> 12) * scale.s1 - minv.s1) * shared_y; \
+ shared_y = sub_group_broadcast(y.s4, 3); \
+ total_sums.s0 += ((bits4.s6 & 0x000F) * scale.s0 - minv.s0) * shared_y; \
+ total_sums.s1 += ((bits4.s7 & 0x000F) * scale.s1 - minv.s1) * shared_y; \
+ shared_y = sub_group_broadcast(y.s5, 3); \
+ total_sums.s0 += (((bits4.s6 & 0x00F0) >> 4) * scale.s0 - minv.s0) * shared_y; \
+ total_sums.s1 += (((bits4.s7 & 0x00F0) >> 4) * scale.s1 - minv.s1) * shared_y; \
+ shared_y = sub_group_broadcast(y.s6, 3); \
+ total_sums.s0 += (((bits4.s6 & 0x0F00) >> 8) * scale.s0 - minv.s0) * shared_y; \
+ total_sums.s1 += (((bits4.s7 & 0x0F00) >> 8) * scale.s1 - minv.s1) * shared_y; \
+ shared_y = sub_group_broadcast(y.s7, 3); \
+ total_sums.s0 += (((bits4.s6 & 0xF000) >> 12) * scale.s0 - minv.s0) * shared_y; \
+ total_sums.s1 += (((bits4.s7 & 0xF000) >> 12) * scale.s1 - minv.s1) * shared_y; \
+
+
+#define dequantizeBlockAccum_ns_sgbroadcast_8_hi(total_sums, bits4, scale, minv, y) \
+ float8 shared_y; \
+ shared_y = sub_group_broadcast(y, 0); \
+ total_sums.s0 += ((bits4.s0 & 0x000F) * scale.s0 - minv.s0) * shared_y.s0; \
+ total_sums.s0 += (((bits4.s0 & 0x00F0) >> 4) * scale.s0 - minv.s0) * shared_y.s1; \
+ total_sums.s0 += (((bits4.s0 & 0x0F00) >> 8) * scale.s0 - minv.s0) * shared_y.s2; \
+ total_sums.s0 += (((bits4.s0 & 0xF000) >> 12) * scale.s0 - minv.s0) * shared_y.s3; \
+ total_sums.s0 += ((bits4.s2 & 0x000F) * scale.s0 - minv.s0) * shared_y.s4; \
+ total_sums.s0 += (((bits4.s2 & 0x00F0) >> 4) * scale.s0 - minv.s0) * shared_y.s5; \
+ total_sums.s0 += (((bits4.s2 & 0x0F00) >> 8) * scale.s0 - minv.s0) * shared_y.s6; \
+ total_sums.s0 += (((bits4.s2 & 0xF000) >> 12) * scale.s0 - minv.s0) * shared_y.s7; \
+ total_sums.s1 += ((bits4.s1 & 0x000F) * scale.s1 - minv.s1) * shared_y.s0; \
+ total_sums.s1 += (((bits4.s1 & 0x00F0) >> 4) * scale.s1 - minv.s1) * shared_y.s1; \
+ total_sums.s1 += (((bits4.s1 & 0x0F00) >> 8) * scale.s1 - minv.s1) * shared_y.s2; \
+ total_sums.s1 += (((bits4.s1 & 0xF000) >> 12) * scale.s1 - minv.s1) * shared_y.s3; \
+ total_sums.s1 += ((bits4.s3 & 0x000F) * scale.s1 - minv.s1) * shared_y.s4; \
+ total_sums.s1 += (((bits4.s3 & 0x00F0) >> 4) * scale.s1 - minv.s1) * shared_y.s5; \
+ total_sums.s1 += (((bits4.s3 & 0x0F00) >> 8) * scale.s1 - minv.s1) * shared_y.s6; \
+ total_sums.s1 += (((bits4.s3 & 0xF000) >> 12) * scale.s1 - minv.s1) * shared_y.s7; \
+ shared_y = sub_group_broadcast(y, 1); \
+ total_sums.s0 += ((bits4.s4 & 0x000F) * scale.s0 - minv.s0) * shared_y.s0; \
+ total_sums.s0 += (((bits4.s4 & 0x00F0) >> 4) * scale.s0 - minv.s0) * shared_y.s1; \
+ total_sums.s0 += (((bits4.s4 & 0x0F00) >> 8) * scale.s0 - minv.s0) * shared_y.s2; \
+ total_sums.s0 += (((bits4.s4 & 0xF000) >> 12) * scale.s0 - minv.s0) * shared_y.s3; \
+ total_sums.s0 += ((bits4.s6 & 0x000F) * scale.s0 - minv.s0) * shared_y.s4; \
+ total_sums.s0 += (((bits4.s6 & 0x00F0) >> 4) * scale.s0 - minv.s0) * shared_y.s5; \
+ total_sums.s0 += (((bits4.s6 & 0x0F00) >> 8) * scale.s0 - minv.s0) * shared_y.s6; \
+ total_sums.s0 += (((bits4.s6 & 0xF000) >> 12) * scale.s0 - minv.s0) * shared_y.s7; \
+ total_sums.s1 += ((bits4.s5 & 0x000F) * scale.s1 - minv.s1) * shared_y.s0; \
+ total_sums.s1 += (((bits4.s5 & 0x00F0) >> 4) * scale.s1 - minv.s1) * shared_y.s1; \
+ total_sums.s1 += (((bits4.s5 & 0x0F00) >> 8) * scale.s1 - minv.s1) * shared_y.s2; \
+ total_sums.s1 += (((bits4.s5 & 0xF000) >> 12) * scale.s1 - minv.s1) * shared_y.s3; \
+ total_sums.s1 += ((bits4.s7 & 0x000F) * scale.s1 - minv.s1) * shared_y.s4; \
+ total_sums.s1 += (((bits4.s7 & 0x00F0) >> 4) * scale.s1 - minv.s1) * shared_y.s5; \
+ total_sums.s1 += (((bits4.s7 & 0x0F00) >> 8) * scale.s1 - minv.s1) * shared_y.s6; \
+ total_sums.s1 += (((bits4.s7 & 0xF000) >> 12) * scale.s1 - minv.s1) * shared_y.s7; \
+
+
+#define dequantizeBlockAccum_ns_sgbroadcast_8_lo(total_sums, bits4, scale, minv, y) \
+ shared_y = sub_group_broadcast(y, 2); \
+ total_sums.s0 += ((bits4.s0 & 0x000F) * scale.s0 - minv.s0) * shared_y.s0; \
+ total_sums.s0 += (((bits4.s0 & 0x00F0) >> 4) * scale.s0 - minv.s0) * shared_y.s1; \
+ total_sums.s0 += (((bits4.s0 & 0x0F00) >> 8) * scale.s0 - minv.s0) * shared_y.s2; \
+ total_sums.s0 += (((bits4.s0 & 0xF000) >> 12) * scale.s0 - minv.s0) * shared_y.s3; \
+ total_sums.s0 += ((bits4.s2 & 0x000F) * scale.s0 - minv.s0) * shared_y.s4; \
+ total_sums.s0 += (((bits4.s2 & 0x00F0) >> 4) * scale.s0 - minv.s0) * shared_y.s5; \
+ total_sums.s0 += (((bits4.s2 & 0x0F00) >> 8) * scale.s0 - minv.s0) * shared_y.s6; \
+ total_sums.s0 += (((bits4.s2 & 0xF000) >> 12) * scale.s0 - minv.s0) * shared_y.s7; \
+ total_sums.s1 += ((bits4.s1 & 0x000F) * scale.s1 - minv.s1) * shared_y.s0; \
+ total_sums.s1 += (((bits4.s1 & 0x00F0) >> 4) * scale.s1 - minv.s1) * shared_y.s1; \
+ total_sums.s1 += (((bits4.s1 & 0x0F00) >> 8) * scale.s1 - minv.s1) * shared_y.s2; \
+ total_sums.s1 += (((bits4.s1 & 0xF000) >> 12) * scale.s1 - minv.s1) * shared_y.s3; \
+ total_sums.s1 += ((bits4.s3 & 0x000F) * scale.s1 - minv.s1) * shared_y.s4; \
+ total_sums.s1 += (((bits4.s3 & 0x00F0) >> 4) * scale.s1 - minv.s1) * shared_y.s5; \
+ total_sums.s1 += (((bits4.s3 & 0x0F00) >> 8) * scale.s1 - minv.s1) * shared_y.s6; \
+ total_sums.s1 += (((bits4.s3 & 0xF000) >> 12) * scale.s1 - minv.s1) * shared_y.s7; \
+ shared_y = sub_group_broadcast(y, 3); \
+ total_sums.s0 += ((bits4.s4 & 0x000F) * scale.s0 - minv.s0) * shared_y.s0; \
+ total_sums.s0 += (((bits4.s4 & 0x00F0) >> 4) * scale.s0 - minv.s0) * shared_y.s1; \
+ total_sums.s0 += (((bits4.s4 & 0x0F00) >> 8) * scale.s0 - minv.s0) * shared_y.s2; \
+ total_sums.s0 += (((bits4.s4 & 0xF000) >> 12) * scale.s0 - minv.s0) * shared_y.s3; \
+ total_sums.s0 += ((bits4.s6 & 0x000F) * scale.s0 - minv.s0) * shared_y.s4; \
+ total_sums.s0 += (((bits4.s6 & 0x00F0) >> 4) * scale.s0 - minv.s0) * shared_y.s5; \
+ total_sums.s0 += (((bits4.s6 & 0x0F00) >> 8) * scale.s0 - minv.s0) * shared_y.s6; \
+ total_sums.s0 += (((bits4.s6 & 0xF000) >> 12) * scale.s0 - minv.s0) * shared_y.s7; \
+ total_sums.s1 += ((bits4.s5 & 0x000F) * scale.s1 - minv.s1) * shared_y.s0; \
+ total_sums.s1 += (((bits4.s5 & 0x00F0) >> 4) * scale.s1 - minv.s1) * shared_y.s1; \
+ total_sums.s1 += (((bits4.s5 & 0x0F00) >> 8) * scale.s1 - minv.s1) * shared_y.s2; \
+ total_sums.s1 += (((bits4.s5 & 0xF000) >> 12) * scale.s1 - minv.s1) * shared_y.s3; \
+ total_sums.s1 += ((bits4.s7 & 0x000F) * scale.s1 - minv.s1) * shared_y.s4; \
+ total_sums.s1 += (((bits4.s7 & 0x00F0) >> 4) * scale.s1 - minv.s1) * shared_y.s5; \
+ total_sums.s1 += (((bits4.s7 & 0x0F00) >> 8) * scale.s1 - minv.s1) * shared_y.s6; \
+ total_sums.s1 += (((bits4.s7 & 0xF000) >> 12) * scale.s1 - minv.s1) * shared_y.s7; \
+
+#ifdef ADRENO_GPU
+REQD_SUBGROUP_SIZE_64
+#endif
+kernel void kernel_gemv_noshuffle_q4_k_f32_o4(
+ read_only image1d_buffer_t src0_q,
+ global half2 * src0_d,
+ global half2 * src0_m,
+ global uchar * src0_s,
+ read_only image1d_buffer_t src1,
+ global float * dst,
+ ulong offsetd,
+ int ne00,
+ int ne01,
+ uchar mask_d6,
+ uchar mask_d4,
+ uchar mask_hi2)
+{
+ uint groupId = get_local_id(1);
+ uint gid = get_global_id(0); // 4-output quad index
+ ushort slid = get_sub_group_local_id();
+
+ // Two consecutive pair-indices (each the same access pattern the 2-output
+ // kernel uses); together they cover 4 consecutive output rows.
+ uint gid_a = gid * 2;
+ uint gid_b = gid * 2 + 1;
+
+ uint K = ne00;
+ uint M = ne01;
+
+ uint LINE_STRIDE_A = M / 2;
+ uint BLOCK_STRIDE_A = NSUBGROUPS * M;
+
+ private uint4 regA;
+ private half2 regS_a, regS_b;
+ private half2 regM_a, regM_b;
+ private float8 regB;
+
+ private float2 totalSum_a = (float2)(0.0f);
+ private float2 totalSum_b = (float2)(0.0f);
+
+ for (uint k = groupId; k < (K / 32); k += NSUBGROUPS) {
+ uint sb = k / 8;
+ uint j = k % 8;
+
+ // pair a scales/mins
+ half2 d_a = src0_d[gid_a + sb * LINE_STRIDE_A];
+ half2 dm_a = src0_m[gid_a + sb * LINE_STRIDE_A];
+ global const uchar * sc0a = src0_s + sb * 12 * M + 2 * gid_a;
+ global const uchar * sc1a = sc0a + 1;
+ uchar sv0a, mn0a, sv1a, mn1a;
+ get_scale_min_k4(j, sc0a, M, &sv0a, &mn0a, mask_d6, mask_d4, mask_hi2);
+ get_scale_min_k4(j, sc1a, M, &sv1a, &mn1a, mask_d6, mask_d4, mask_hi2);
+ regS_a = convert_half2(convert_float2(d_a) * convert_float2((uchar2)(sv0a, sv1a)));
+ regM_a = convert_half2(convert_float2(dm_a) * convert_float2((uchar2)(mn0a, mn1a)));
+
+ // pair b scales/mins
+ half2 d_b = src0_d[gid_b + sb * LINE_STRIDE_A];
+ half2 dm_b = src0_m[gid_b + sb * LINE_STRIDE_A];
+ global const uchar * sc0b = src0_s + sb * 12 * M + 2 * gid_b;
+ global const uchar * sc1b = sc0b + 1;
+ uchar sv0b, mn0b, sv1b, mn1b;
+ get_scale_min_k4(j, sc0b, M, &sv0b, &mn0b, mask_d6, mask_d4, mask_hi2);
+ get_scale_min_k4(j, sc1b, M, &sv1b, &mn1b, mask_d6, mask_d4, mask_hi2);
+ regS_b = convert_half2(convert_float2(d_b) * convert_float2((uchar2)(sv0b, sv1b)));
+ regM_b = convert_half2(convert_float2(dm_b) * convert_float2((uchar2)(mn0b, mn1b)));
+
+ // activation: load once, reuse for both pairs
+ if (slid < 4) {
+ regB.s0123 = read_imagef(src1, (slid * 2 + k * 8));
+ regB.s4567 = read_imagef(src1, (1 + slid * 2 + k * 8));
+ }
+
+ // pair a (own block so _lo sees the shared_y declared by _hi)
+ {
+ regA.s0 = read_imageui(src0_q, (gid_a + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 0)).x;
+ regA.s1 = read_imageui(src0_q, (gid_a + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 1)).x;
+ regA.s2 = read_imageui(src0_q, (gid_a + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 2)).x;
+ regA.s3 = read_imageui(src0_q, (gid_a + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 3)).x;
+#ifdef VECTOR_SUB_GROUP_BROADCAST
+ dequantizeBlockAccum_ns_sgbroadcast_8_hi(totalSum_a, as_ushort8(regA), regS_a, regM_a, regB);
+#else
+ dequantizeBlockAccum_ns_sgbroadcast_1_hi(totalSum_a, as_ushort8(regA), regS_a, regM_a, regB);
+#endif
+ regA.s0 = read_imageui(src0_q, (gid_a + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 4)).x;
+ regA.s1 = read_imageui(src0_q, (gid_a + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 5)).x;
+ regA.s2 = read_imageui(src0_q, (gid_a + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 6)).x;
+ regA.s3 = read_imageui(src0_q, (gid_a + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 7)).x;
+#ifdef VECTOR_SUB_GROUP_BROADCAST
+ dequantizeBlockAccum_ns_sgbroadcast_8_lo(totalSum_a, as_ushort8(regA), regS_a, regM_a, regB);
+#else
+ dequantizeBlockAccum_ns_sgbroadcast_1_lo(totalSum_a, as_ushort8(regA), regS_a, regM_a, regB);
+#endif
+ }
+
+ // pair b
+ {
+ regA.s0 = read_imageui(src0_q, (gid_b + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 0)).x;
+ regA.s1 = read_imageui(src0_q, (gid_b + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 1)).x;
+ regA.s2 = read_imageui(src0_q, (gid_b + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 2)).x;
+ regA.s3 = read_imageui(src0_q, (gid_b + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 3)).x;
+#ifdef VECTOR_SUB_GROUP_BROADCAST
+ dequantizeBlockAccum_ns_sgbroadcast_8_hi(totalSum_b, as_ushort8(regA), regS_b, regM_b, regB);
+#else
+ dequantizeBlockAccum_ns_sgbroadcast_1_hi(totalSum_b, as_ushort8(regA), regS_b, regM_b, regB);
+#endif
+ regA.s0 = read_imageui(src0_q, (gid_b + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 4)).x;
+ regA.s1 = read_imageui(src0_q, (gid_b + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 5)).x;
+ regA.s2 = read_imageui(src0_q, (gid_b + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 6)).x;
+ regA.s3 = read_imageui(src0_q, (gid_b + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 7)).x;
+#ifdef VECTOR_SUB_GROUP_BROADCAST
+ dequantizeBlockAccum_ns_sgbroadcast_8_lo(totalSum_b, as_ushort8(regA), regS_b, regM_b, regB);
+#else
+ dequantizeBlockAccum_ns_sgbroadcast_1_lo(totalSum_b, as_ushort8(regA), regS_b, regM_b, regB);
+#endif
+ }
+ }
+
+ // reduce 4 outputs (a.s0, a.s1, b.s0, b.s1) across the 4 subgroups
+ local float4 reduceLM[SUBGROUP_SIZE * 3];
+ float4 acc = (float4)(totalSum_a.s0, totalSum_a.s1, totalSum_b.s0, totalSum_b.s1);
+ if (groupId == 1) { reduceLM[SUBGROUP_SIZE * 0 + slid] = acc; }
+ if (groupId == 2) { reduceLM[SUBGROUP_SIZE * 1 + slid] = acc; }
+ if (groupId == 3) { reduceLM[SUBGROUP_SIZE * 2 + slid] = acc; }
+
+ barrier(CLK_LOCAL_MEM_FENCE);
+
+ if (groupId == 0) {
+ acc += reduceLM[SUBGROUP_SIZE * 0 + slid];
+ acc += reduceLM[SUBGROUP_SIZE * 1 + slid];
+ acc += reduceLM[SUBGROUP_SIZE * 2 + slid];
+ dst = (global float*)((global char*)dst + offsetd);
+ // The dispatch rounds ne01/4 up to the subgroup width, so the tail
+ // quads past the last row must not store (they wrote 128 rows past
+ // dst on every ne01 % 256 == 128 vocab, e.g. 151936).
+ if (gid * 4 + 3 < (uint)ne01) {
+ vstore4(acc, 0, &(dst[gid * 4]));
+ }
+ }
+}
diff --git a/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_k_f32_tiled.cl b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_k_f32_tiled.cl
new file mode 100644
index 0000000000..929538c41d
--- /dev/null
+++ b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_k_f32_tiled.cl
@@ -0,0 +1,118 @@
+// Tiled-wide q4_K GEMV for the long-vocab lm_head/embed (decode path).
+//
+// Pairs with kernel_convert_block_q4_k_tiled_ns (cvt.cl): the weights are laid
+// out CANONICALLY (4-bit code in element order e in [0,256)) and TILED by 64
+// output rows so the 64-thread lane group coalesces every weight load. Both the
+// pack (convert) and the unpack (here) are owned by us -> correct by
+// construction vs the reference ggml q4_K dequant. Same structure as the q6_K
+// tiled GEMV; the only differences are the 4-bit dequant and the q4_K
+// scale/min decode (get_scale_min_k4 from the packed 12-byte block).
+//
+// One work-item produces one output row. WG = {64 lanes, 4 subgroups}: the 64
+// lanes cover the 64 rows of one tile (coalesced uint4 reads), the 4 subgroups
+// split the K-blocks and reduce through __local at the end. Weights read from
+// __global (lm_head is streamed once per token; texture cache caps it below the
+// coalesced-global rate).
+
+#pragma OPENCL EXTENSION cl_khr_fp16 : enable
+
+#ifdef cl_qcom_reqd_sub_group_size
+#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable
+#define ADRENO_GPU 1
+#define REQD_SUBGROUP_SIZE_64 __attribute__((qcom_reqd_sub_group_size("half")))
+#endif
+
+#define QK_K 256
+#define NSUBGROUPS 4
+#define TILE_ROWS 64
+
+// Decode one q4_K sub-block scale + min from the packed 12-byte block.
+// Identical to the o4 kernel's helper (masks hard-coded: d6=0x3F, d4=0x0F, hi2=0xC0).
+inline void q4k_scale_min(int j, __global const uchar * q, uchar * d, uchar * m) {
+ if (j < 4) {
+ *d = q[j] & 0x3F;
+ *m = q[j+4] & 0x3F;
+ } else {
+ *d = (q[j+4] & 0x0F) | ((q[j-4] & 0xC0) >> 2);
+ *m = ((q[j+4] >> 4) & 0x0F) | ((q[j] & 0xC0) >> 2);
+ }
+}
+
+#if defined(ADRENO_GPU)
+REQD_SUBGROUP_SIZE_64
+#endif
+kernel void kernel_gemv_noshuffle_q4_k_f32_tiled(
+ __global uint4 * src0_q, // tiled: 8 uint4 granules / superblock (4-bit codes)
+ __global half * src0_d, // tiled: 1 half / superblock
+ __global half * src0_dm, // tiled: 1 half / superblock
+ __global uchar * src0_s, // tiled: 12 bytes / superblock (packed scales)
+ read_only image1d_buffer_t src1, // activation (RGBA f32)
+ global float * dst,
+ ulong offsetd,
+ int ne00,
+ int ne01
+) {
+ int grp = get_local_id(1); // subgroup index 0..3 (splits K)
+ int row = get_global_id(0); // output row along ne01
+ int rt = row / TILE_ROWS;
+ int rit = row % TILE_ROWS;
+
+ int nb = ne00 / QK_K; // superblocks per row
+
+ float acc = 0.0f;
+
+ for (int sb = grp; sb < nb; sb += NSUBGROUPS) {
+ int tile_blk = rt * nb + sb; // ne02 == 1 for lm_head/embed
+
+ float dval = (float)src0_d [tile_blk * TILE_ROWS + rit];
+ float dmval = (float)src0_dm[tile_blk * TILE_ROWS + rit];
+
+ // decode the 8 sub-block (scale, min) pairs
+ __global uchar * sc = src0_s + (tile_blk * TILE_ROWS + rit) * 12;
+ float scale[8], minv[8];
+ #pragma unroll
+ for (int is = 0; is < 8; ++is) {
+ uchar sd, sm;
+ q4k_scale_min(is, sc, &sd, &sm);
+ scale[is] = dval * (float)sd;
+ minv[is] = dmval * (float)sm;
+ }
+
+ // 32 uints of 4-bit codes (8 codes/uint), e-order
+ uint q[32];
+ #pragma unroll
+ for (int g = 0; g < 8; ++g) {
+ uint4 v = src0_q[(tile_blk * 8 + g) * TILE_ROWS + rit];
+ q[g*4+0] = v.x; q[g*4+1] = v.y; q[g*4+2] = v.z; q[g*4+3] = v.w;
+ }
+
+ // dequant 256 codes in canonical e-order, MAC with activation.
+ int act_base = sb * 64; // activation float4 pixel base (256/4)
+ #pragma unroll
+ for (int e4 = 0; e4 < 64; ++e4) {
+ float4 a = read_imagef(src1, act_base + e4);
+ #pragma unroll
+ for (int t = 0; t < 4; ++t) {
+ int e = e4 * 4 + t;
+ uint code = (q[e >> 3] >> ((e & 7) * 4)) & 0xF;
+ int is = e >> 5; // sub-block index = e/32
+ float av = (t == 0) ? a.x : (t == 1) ? a.y : (t == 2) ? a.z : a.w;
+ acc += ((float)code * scale[is] - minv[is]) * av;
+ }
+ }
+ }
+
+ // reduce across the NSUBGROUPS subgroups (same rit, different K-subset)
+ local float reduce_lm[NSUBGROUPS * TILE_ROWS];
+ reduce_lm[grp * TILE_ROWS + rit] = acc;
+ barrier(CLK_LOCAL_MEM_FENCE);
+
+ if (grp == 0) {
+ float total = reduce_lm[0 * TILE_ROWS + rit]
+ + reduce_lm[1 * TILE_ROWS + rit]
+ + reduce_lm[2 * TILE_ROWS + rit]
+ + reduce_lm[3 * TILE_ROWS + rit];
+ dst = (global float*)((global char*)dst + offsetd);
+ dst[row] = total;
+ }
+}
diff --git a/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q5_k_f32.cl b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q5_k_f32.cl
index 446f465338..ae864b19ba 100644
--- a/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q5_k_f32.cl
+++ b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q5_k_f32.cl
@@ -329,3 +329,125 @@ kernel void kernel_gemv_noshuffle_q5_k_f32(
if (gid * 2 + 1 < M) dst[gid * 2 + 1] = totalSum.s1;
}
}
+
+// Multi-column (N in [2..4]) variant of the q5_K decode GEMV (spec/MTP verify) =
+// q4_K mc3 + the high-bit qh plane (regH). n_cols = 2..4 (drafted + bonus); routes
+// the small-batch verify OFF the gemm_noshuffle_q5_k dead-zone. n_cols==3 is byte-
+// identical to the original mc3 (col3 disabled, float8 slots 6/7 stay zero).
+#ifdef VECTOR_SUB_GROUP_BROADCAST
+#define MC_DQ5_HI dequantizeBlockAccum_ns_sgbroadcast_8_hi
+#define MC_DQ5_LO dequantizeBlockAccum_ns_sgbroadcast_8_lo
+#else
+#define MC_DQ5_HI dequantizeBlockAccum_ns_sgbroadcast_1_hi
+#define MC_DQ5_LO dequantizeBlockAccum_ns_sgbroadcast_1_lo
+#endif
+#define MC_COL_Q5K(ts, c) \
+ { if (slid < 4) { regB.s0123 = read_imagef(src1, (c)*COL_STRIDE + slid*2 + k*8); \
+ regB.s4567 = read_imagef(src1, (c)*COL_STRIDE + 1 + slid*2 + k*8); } \
+ MC_DQ5_HI(ts, as_ushort8(regA_hi), as_uchar8(regH), regS, regM, regB); \
+ MC_DQ5_LO(ts, as_ushort8(regA_lo), as_uchar8(regH), regS, regM, regB); }
+#ifdef ADRENO_GPU
+REQD_SUBGROUP_SIZE_64
+#endif
+kernel void kernel_gemv_noshuffle_q5_k_f32_mc3(
+ read_only image1d_buffer_t src0_q,
+ read_only image1d_buffer_t src0_qh,
+ global half2 * src0_d,
+ global half2 * src0_m,
+ global uchar * src0_s,
+ read_only image1d_buffer_t src1,
+ global float * dst,
+ ulong offsetd,
+ int ne00,
+ int ne01,
+ uchar mask_d6,
+ uchar mask_d4,
+ uchar mask_hi2,
+ int n_cols)
+{
+ uint groupId = get_local_id(1);
+ uint gid = get_global_id(0);
+ ushort slid = get_sub_group_local_id();
+
+ uint K = ne00;
+ uint M = ne01;
+
+ uint LINE_STRIDE_A = M / 2;
+ uint BLOCK_STRIDE_A = NSUBGROUPS * M;
+ uint LINE_STRIDE_A_QH = M / 2;
+ uint BLOCK_STRIDE_A_QH = NSUBGROUPS * M / 2;
+ uint scales_per_row = (K / QK_K) * 12;
+ uint COL_STRIDE = K / 4; // float4 pixels per activation column
+
+ private uint4 regA_hi, regA_lo;
+ private ushort4 regH;
+ private half2 regS, regM;
+ private float8 regB;
+
+ private float2 ts0 = (float2)(0.0f);
+ private float2 ts1 = (float2)(0.0f);
+ private float2 ts2 = (float2)(0.0f);
+ private float2 ts3 = (float2)(0.0f);
+
+ for (uint k = groupId; k < (K / 32); k += NSUBGROUPS) {
+ uint sb = k / 8;
+ uint j = k % 8;
+
+ half2 d = src0_d[gid + sb * LINE_STRIDE_A];
+ half2 dm = src0_m[gid + sb * LINE_STRIDE_A];
+
+ global const uchar * sc0 = src0_s + 2 * gid * scales_per_row + sb * 12;
+ global const uchar * sc1 = src0_s + (2 * gid + 1) * scales_per_row + sb * 12;
+
+ uchar sv0, mn0, sv1, mn1;
+ get_scale_min_k4(j, sc0, &sv0, &mn0, mask_d6, mask_d4, mask_hi2);
+ get_scale_min_k4(j, sc1, &sv1, &mn1, mask_d6, mask_d4, mask_hi2);
+
+ regS = convert_half2(convert_float2(d) * convert_float2((uchar2)(sv0, sv1)));
+ regM = convert_half2(convert_float2(dm) * convert_float2((uchar2)(mn0, mn1)));
+
+ // high-bit plane + weights loaded ONCE, reused across the columns
+ regH.s0 = as_ushort(read_imageh(src0_qh, (gid + k * BLOCK_STRIDE_A_QH + LINE_STRIDE_A_QH * 0)).x);
+ regH.s1 = as_ushort(read_imageh(src0_qh, (gid + k * BLOCK_STRIDE_A_QH + LINE_STRIDE_A_QH * 1)).x);
+ regH.s2 = as_ushort(read_imageh(src0_qh, (gid + k * BLOCK_STRIDE_A_QH + LINE_STRIDE_A_QH * 2)).x);
+ regH.s3 = as_ushort(read_imageh(src0_qh, (gid + k * BLOCK_STRIDE_A_QH + LINE_STRIDE_A_QH * 3)).x);
+
+ regA_hi.s0 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 0)).x;
+ regA_hi.s1 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 1)).x;
+ regA_hi.s2 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 2)).x;
+ regA_hi.s3 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 3)).x;
+ regA_lo.s0 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 4)).x;
+ regA_lo.s1 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 5)).x;
+ regA_lo.s2 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 6)).x;
+ regA_lo.s3 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 7)).x;
+
+ MC_COL_Q5K(ts0, 0);
+ MC_COL_Q5K(ts1, 1);
+ if (n_cols > 2) MC_COL_Q5K(ts2, 2);
+ if (n_cols > 3) MC_COL_Q5K(ts3, 3);
+ }
+
+ // cross-subgroup reduce: pack the (up to 4) columns' float2 into a float8.
+ local float8 reduceLM[SUBGROUP_SIZE * 3];
+ float8 acc = (float8)(ts0.s0, ts0.s1, ts1.s0, ts1.s1, ts2.s0, ts2.s1, ts3.s0, ts3.s1);
+ if (groupId == 1) { reduceLM[SUBGROUP_SIZE * 0 + slid] = acc; }
+ if (groupId == 2) { reduceLM[SUBGROUP_SIZE * 1 + slid] = acc; }
+ if (groupId == 3) { reduceLM[SUBGROUP_SIZE * 2 + slid] = acc; }
+
+ barrier(CLK_LOCAL_MEM_FENCE);
+
+ if (groupId == 0) {
+ acc += reduceLM[SUBGROUP_SIZE * 0 + slid];
+ acc += reduceLM[SUBGROUP_SIZE * 1 + slid];
+ acc += reduceLM[SUBGROUP_SIZE * 2 + slid];
+ dst = (global float*)((global char*)dst + offsetd);
+ // dst is column-major [M rows x n_cols cols]: (row, col) at col*M + row
+ vstore2((float2)(acc.s0, acc.s1), 0, &(dst[0 * M + gid * 2]));
+ vstore2((float2)(acc.s2, acc.s3), 0, &(dst[1 * M + gid * 2]));
+ if (n_cols > 2) vstore2((float2)(acc.s4, acc.s5), 0, &(dst[2 * M + gid * 2]));
+ if (n_cols > 3) vstore2((float2)(acc.s6, acc.s7), 0, &(dst[3 * M + gid * 2]));
+ }
+}
+#undef MC_COL_Q5K
+#undef MC_DQ5_HI
+#undef MC_DQ5_LO
diff --git a/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q6_k_f32.cl b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q6_k_f32.cl
index 51682ecebb..32624ac868 100644
--- a/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q6_k_f32.cl
+++ b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q6_k_f32.cl
@@ -296,3 +296,114 @@ kernel void kernel_gemv_noshuffle_q6_K_f32(
if (gid * 2 + 1 < ne01) dst[gid * 2 + 1] = total_sum.s1;
}
}
+
+// Multi-column (N=3) q6_K decode GEMV for the spec/MTP verify batch. Same idea
+// as the q4_K mc3: stay on the efficient GEMV path (subgroup broadcast, no
+// transpose) instead of the transposed-GEMM dead-zone. Each K-block's weights
+// (ql/qh, hi+lo) are loaded ONCE and reused across all 3 activation columns.
+// Per-column accumulation is independent and identical to 3 standalone GEMVs
+// => byte-identical; does NOT perturb the lm_head logits / spec accept rate.
+#if defined(ADRENO_GPU)
+REQD_SUBGROUP_SIZE_64
+#endif
+kernel void kernel_gemv_noshuffle_q6_K_f32_mc3(
+ read_only image1d_buffer_t src0_ql,
+ read_only image1d_buffer_t src0_qh,
+ global half2 * src0_s,
+ global half2 * src0_d,
+ read_only image1d_buffer_t src1,
+ global float * dst,
+ ulong offsetd,
+ int ne00,
+ int ne01
+) {
+ int grp = get_local_id(1);
+ int gid = get_global_id(0);
+ ushort slid = get_sub_group_local_id();
+
+ int nb = ne00 / 32;
+ int line_stride_a = ne01 / 2;
+ int block_stride_a = NSUBGROUPS * ne01;
+ int COL_STRIDE = ne00 / 4; // float4 pixels per activation column
+
+ uint4 ql_hi, ql_lo;
+ ushort4 qh_hi, qh_lo;
+ half2 reg_d;
+ char4 reg_s;
+ float8 reg_b;
+
+ float2 ts0 = 0.0f, ts1 = 0.0f, ts2 = 0.0f;
+
+ for (int k = grp; k < nb; k += NSUBGROUPS) {
+ reg_d = src0_d[gid + k/8 * line_stride_a];
+ reg_s = as_char4(src0_s[gid + k * line_stride_a]);
+
+ // weights loaded ONCE (hi: blocks 0-3, lo: blocks 4-7), reused x3 cols
+ ql_hi.s0 = read_imageui(src0_ql, gid + k*block_stride_a + line_stride_a*0).x;
+ ql_hi.s1 = read_imageui(src0_ql, gid + k*block_stride_a + line_stride_a*1).x;
+ ql_hi.s2 = read_imageui(src0_ql, gid + k*block_stride_a + line_stride_a*2).x;
+ ql_hi.s3 = read_imageui(src0_ql, gid + k*block_stride_a + line_stride_a*3).x;
+ qh_hi.s0 = as_ushort(read_imageh(src0_qh, gid + k*block_stride_a + line_stride_a*0).x);
+ qh_hi.s1 = as_ushort(read_imageh(src0_qh, gid + k*block_stride_a + line_stride_a*1).x);
+ qh_hi.s2 = as_ushort(read_imageh(src0_qh, gid + k*block_stride_a + line_stride_a*2).x);
+ qh_hi.s3 = as_ushort(read_imageh(src0_qh, gid + k*block_stride_a + line_stride_a*3).x);
+
+ ql_lo.s0 = read_imageui(src0_ql, gid + k*block_stride_a + line_stride_a*4).x;
+ ql_lo.s1 = read_imageui(src0_ql, gid + k*block_stride_a + line_stride_a*5).x;
+ ql_lo.s2 = read_imageui(src0_ql, gid + k*block_stride_a + line_stride_a*6).x;
+ ql_lo.s3 = read_imageui(src0_ql, gid + k*block_stride_a + line_stride_a*7).x;
+ qh_lo.s0 = as_ushort(read_imageh(src0_qh, gid + k*block_stride_a + line_stride_a*4).x);
+ qh_lo.s1 = as_ushort(read_imageh(src0_qh, gid + k*block_stride_a + line_stride_a*5).x);
+ qh_lo.s2 = as_ushort(read_imageh(src0_qh, gid + k*block_stride_a + line_stride_a*6).x);
+ qh_lo.s3 = as_ushort(read_imageh(src0_qh, gid + k*block_stride_a + line_stride_a*7).x);
+
+ // Per-column: load only this column's activation (single reg_b live) ->
+ // 1/3 the activation register pressure, cutting the private-mem spill.
+#ifdef VECTOR_SUB_GROUP_BROADCAT
+ { if (slid < 4) { reg_b.s0123 = read_imagef(src1, 0*COL_STRIDE + 0 + slid*2 + k*8);
+ reg_b.s4567 = read_imagef(src1, 0*COL_STRIDE + 1 + slid*2 + k*8); }
+ dequantize_block_acc_bcast_8_hi(ts0, as_ushort8(ql_hi), as_uchar8(qh_hi), reg_d, reg_s, reg_b);
+ dequantize_block_acc_bcast_8_lo(ts0, as_ushort8(ql_lo), as_uchar8(qh_lo), reg_d, reg_s, reg_b); }
+ { if (slid < 4) { reg_b.s0123 = read_imagef(src1, 1*COL_STRIDE + 0 + slid*2 + k*8);
+ reg_b.s4567 = read_imagef(src1, 1*COL_STRIDE + 1 + slid*2 + k*8); }
+ dequantize_block_acc_bcast_8_hi(ts1, as_ushort8(ql_hi), as_uchar8(qh_hi), reg_d, reg_s, reg_b);
+ dequantize_block_acc_bcast_8_lo(ts1, as_ushort8(ql_lo), as_uchar8(qh_lo), reg_d, reg_s, reg_b); }
+ { if (slid < 4) { reg_b.s0123 = read_imagef(src1, 2*COL_STRIDE + 0 + slid*2 + k*8);
+ reg_b.s4567 = read_imagef(src1, 2*COL_STRIDE + 1 + slid*2 + k*8); }
+ dequantize_block_acc_bcast_8_hi(ts2, as_ushort8(ql_hi), as_uchar8(qh_hi), reg_d, reg_s, reg_b);
+ dequantize_block_acc_bcast_8_lo(ts2, as_ushort8(ql_lo), as_uchar8(qh_lo), reg_d, reg_s, reg_b); }
+#else
+ { if (slid < 4) { reg_b.s0123 = read_imagef(src1, 0*COL_STRIDE + 0 + slid*2 + k*8);
+ reg_b.s4567 = read_imagef(src1, 0*COL_STRIDE + 1 + slid*2 + k*8); }
+ dequantize_block_acc_bcast_1_hi(ts0, as_ushort8(ql_hi), as_uchar8(qh_hi), reg_d, reg_s, reg_b);
+ dequantize_block_acc_bcast_1_lo(ts0, as_ushort8(ql_lo), as_uchar8(qh_lo), reg_d, reg_s, reg_b); }
+ { if (slid < 4) { reg_b.s0123 = read_imagef(src1, 1*COL_STRIDE + 0 + slid*2 + k*8);
+ reg_b.s4567 = read_imagef(src1, 1*COL_STRIDE + 1 + slid*2 + k*8); }
+ dequantize_block_acc_bcast_1_hi(ts1, as_ushort8(ql_hi), as_uchar8(qh_hi), reg_d, reg_s, reg_b);
+ dequantize_block_acc_bcast_1_lo(ts1, as_ushort8(ql_lo), as_uchar8(qh_lo), reg_d, reg_s, reg_b); }
+ { if (slid < 4) { reg_b.s0123 = read_imagef(src1, 2*COL_STRIDE + 0 + slid*2 + k*8);
+ reg_b.s4567 = read_imagef(src1, 2*COL_STRIDE + 1 + slid*2 + k*8); }
+ dequantize_block_acc_bcast_1_hi(ts2, as_ushort8(ql_hi), as_uchar8(qh_hi), reg_d, reg_s, reg_b);
+ dequantize_block_acc_bcast_1_lo(ts2, as_ushort8(ql_lo), as_uchar8(qh_lo), reg_d, reg_s, reg_b); }
+#endif
+ }
+
+ local float8 reduce_lm[SUBGROUP_SIZE * 3];
+ float8 acc = (float8)(ts0.s0, ts0.s1, ts1.s0, ts1.s1, ts2.s0, ts2.s1, 0.0f, 0.0f);
+ if (grp == 1) { reduce_lm[SUBGROUP_SIZE*0 + slid] = acc; }
+ if (grp == 2) { reduce_lm[SUBGROUP_SIZE*1 + slid] = acc; }
+ if (grp == 3) { reduce_lm[SUBGROUP_SIZE*2 + slid] = acc; }
+
+ barrier(CLK_LOCAL_MEM_FENCE);
+
+ if (grp == 0) {
+ acc += reduce_lm[SUBGROUP_SIZE*0 + slid];
+ acc += reduce_lm[SUBGROUP_SIZE*1 + slid];
+ acc += reduce_lm[SUBGROUP_SIZE*2 + slid];
+ dst = (global float*)((global char*)dst + offsetd);
+ // dst column-major [ne01 rows x 3 cols]: (row, col) at col*ne01 + row
+ vstore2((float2)(acc.s0, acc.s1), 0, &(dst[0*ne01 + gid*2]));
+ vstore2((float2)(acc.s2, acc.s3), 0, &(dst[1*ne01 + gid*2]));
+ vstore2((float2)(acc.s4, acc.s5), 0, &(dst[2*ne01 + gid*2]));
+ }
+}
diff --git a/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q6_k_f32_o4.cl b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q6_k_f32_o4.cl
new file mode 100644
index 0000000000..84447e61bb
--- /dev/null
+++ b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q6_k_f32_o4.cl
@@ -0,0 +1,372 @@
+// 4-output-per-WI variant of kernel_gemv_noshuffle_q6_K_f32.
+// Each WI now produces 4 consecutive outputs (output quad). The activation
+// fetch (reg_b) is shared across all 4 outputs, doubling per-WI ALU per
+// activation broadcast and halving the WG count vs the 2-output kernel.
+//
+// Implementation: each K-block we fetch TWO sets of (scales + ql + qh)
+// — one for the low pair (rows 0,1 of the quad) and one for the high pair
+// (rows 2,3) — and invoke the existing 2-output dequant macros twice
+// against the *same* reg_b. Identical data layout to the 2-output kernel,
+// so the host only needs to halve the grid and double the gid-to-output
+// mapping.
+//
+// Opt-in via the host dispatch when GGML_OPENCL_Q6K_GEMV_O4=1.
+
+#pragma OPENCL EXTENSION cl_khr_fp16 : enable
+#pragma OPENCL EXTENSION cl_khr_subgroups : enable
+
+#ifdef cl_intel_required_subgroup_size
+#pragma OPENCL EXTENSION cl_intel_required_subgroup_size : enable
+#define INTEL_GPU 1
+#define REQD_SUBGROUP_SIZE_16 __attribute__((intel_reqd_sub_group_size(16)))
+#define REQD_SUBGROUP_SIZE_32 __attribute__((intel_reqd_sub_group_size(32)))
+#elif defined(cl_qcom_reqd_sub_group_size)
+#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable
+#define ADRENO_GPU 1
+#define REQD_SUBGROUP_SIZE_64 __attribute__((qcom_reqd_sub_group_size("half")))
+#define REQD_SUBGROUP_SIZE_128 __attribute__((qcom_reqd_sub_group_size("full")))
+#endif
+
+#define NSUBGROUPS 4
+#define SUBGROUP_SIZE 64
+
+// Macros are identical to the 2-output kernel — they accept `total_sum` as
+// a parameter so we can call them twice (once per pair) against different
+// accumulators against the same reg_b.
+#define dequantize_block_acc_bcast_8_hi(total_sum, bits4, bits2, cs, y) \
+ float8 shared_y; \
+ shared_y = sub_group_broadcast(y, 0); \
+ total_sum.s0 += ((float)(((bits4.s0 & 0x000F) ) | ((bits2.s0 & 0x03) << 4)) - 32.f) * cs.s0 * shared_y.s0; \
+ total_sum.s0 += ((float)(((bits4.s0 & 0x00F0) >> 4) | ((bits2.s0 & 0x0C) << 2)) - 32.f) * cs.s0 * shared_y.s1; \
+ total_sum.s0 += ((float)(((bits4.s0 & 0x0F00) >> 8) | ((bits2.s0 & 0x30) )) - 32.f) * cs.s0 * shared_y.s2; \
+ total_sum.s0 += ((float)(((bits4.s0 & 0xF000) >> 12) | ((bits2.s0 & 0xC0) >> 2)) - 32.f) * cs.s0 * shared_y.s3; \
+ total_sum.s0 += ((float)(((bits4.s2 & 0x000F) ) | ((bits2.s2 & 0x03) << 4)) - 32.f) * cs.s0 * shared_y.s4; \
+ total_sum.s0 += ((float)(((bits4.s2 & 0x00F0) >> 4) | ((bits2.s2 & 0x0C) << 2)) - 32.f) * cs.s0 * shared_y.s5; \
+ total_sum.s0 += ((float)(((bits4.s2 & 0x0F00) >> 8) | ((bits2.s2 & 0x30) )) - 32.f) * cs.s0 * shared_y.s6; \
+ total_sum.s0 += ((float)(((bits4.s2 & 0xF000) >> 12) | ((bits2.s2 & 0xC0) >> 2)) - 32.f) * cs.s0 * shared_y.s7; \
+ total_sum.s1 += ((float)(((bits4.s1 & 0x000F) ) | ((bits2.s1 & 0x03) << 4)) - 32.f) * cs.s2 * shared_y.s0; \
+ total_sum.s1 += ((float)(((bits4.s1 & 0x00F0) >> 4) | ((bits2.s1 & 0x0C) << 2)) - 32.f) * cs.s2 * shared_y.s1; \
+ total_sum.s1 += ((float)(((bits4.s1 & 0x0F00) >> 8) | ((bits2.s1 & 0x30) )) - 32.f) * cs.s2 * shared_y.s2; \
+ total_sum.s1 += ((float)(((bits4.s1 & 0xF000) >> 12) | ((bits2.s1 & 0xC0) >> 2)) - 32.f) * cs.s2 * shared_y.s3; \
+ total_sum.s1 += ((float)(((bits4.s3 & 0x000F) ) | ((bits2.s3 & 0x03) << 4)) - 32.f) * cs.s2 * shared_y.s4; \
+ total_sum.s1 += ((float)(((bits4.s3 & 0x00F0) >> 4) | ((bits2.s3 & 0x0C) << 2)) - 32.f) * cs.s2 * shared_y.s5; \
+ total_sum.s1 += ((float)(((bits4.s3 & 0x0F00) >> 8) | ((bits2.s3 & 0x30) )) - 32.f) * cs.s2 * shared_y.s6; \
+ total_sum.s1 += ((float)(((bits4.s3 & 0xF000) >> 12) | ((bits2.s3 & 0xC0) >> 2)) - 32.f) * cs.s2 * shared_y.s7; \
+ shared_y = sub_group_broadcast(y, 1); \
+ total_sum.s0 += ((float)(((bits4.s4 & 0x000F) ) | ((bits2.s4 & 0x03) << 4)) - 32.f) * cs.s0 * shared_y.s0; \
+ total_sum.s0 += ((float)(((bits4.s4 & 0x00F0) >> 4) | ((bits2.s4 & 0x0C) << 2)) - 32.f) * cs.s0 * shared_y.s1; \
+ total_sum.s0 += ((float)(((bits4.s4 & 0x0F00) >> 8) | ((bits2.s4 & 0x30) )) - 32.f) * cs.s0 * shared_y.s2; \
+ total_sum.s0 += ((float)(((bits4.s4 & 0xF000) >> 12) | ((bits2.s4 & 0xC0) >> 2)) - 32.f) * cs.s0 * shared_y.s3; \
+ total_sum.s0 += ((float)(((bits4.s6 & 0x000F) ) | ((bits2.s6 & 0x03) << 4)) - 32.f) * cs.s0 * shared_y.s4; \
+ total_sum.s0 += ((float)(((bits4.s6 & 0x00F0) >> 4) | ((bits2.s6 & 0x0C) << 2)) - 32.f) * cs.s0 * shared_y.s5; \
+ total_sum.s0 += ((float)(((bits4.s6 & 0x0F00) >> 8) | ((bits2.s6 & 0x30) )) - 32.f) * cs.s0 * shared_y.s6; \
+ total_sum.s0 += ((float)(((bits4.s6 & 0xF000) >> 12) | ((bits2.s6 & 0xC0) >> 2)) - 32.f) * cs.s0 * shared_y.s7; \
+ total_sum.s1 += ((float)(((bits4.s5 & 0x000F) ) | ((bits2.s5 & 0x03) << 4)) - 32.f) * cs.s2 * shared_y.s0; \
+ total_sum.s1 += ((float)(((bits4.s5 & 0x00F0) >> 4) | ((bits2.s5 & 0x0C) << 2)) - 32.f) * cs.s2 * shared_y.s1; \
+ total_sum.s1 += ((float)(((bits4.s5 & 0x0F00) >> 8) | ((bits2.s5 & 0x30) )) - 32.f) * cs.s2 * shared_y.s2; \
+ total_sum.s1 += ((float)(((bits4.s5 & 0xF000) >> 12) | ((bits2.s5 & 0xC0) >> 2)) - 32.f) * cs.s2 * shared_y.s3; \
+ total_sum.s1 += ((float)(((bits4.s7 & 0x000F) ) | ((bits2.s7 & 0x03) << 4)) - 32.f) * cs.s2 * shared_y.s4; \
+ total_sum.s1 += ((float)(((bits4.s7 & 0x00F0) >> 4) | ((bits2.s7 & 0x0C) << 2)) - 32.f) * cs.s2 * shared_y.s5; \
+ total_sum.s1 += ((float)(((bits4.s7 & 0x0F00) >> 8) | ((bits2.s7 & 0x30) )) - 32.f) * cs.s2 * shared_y.s6; \
+ total_sum.s1 += ((float)(((bits4.s7 & 0xF000) >> 12) | ((bits2.s7 & 0xC0) >> 2)) - 32.f) * cs.s2 * shared_y.s7; \
+
+#define dequantize_block_acc_bcast_8_lo(total_sum, bits4, bits2, cs, y) \
+ shared_y = sub_group_broadcast(y, 2); \
+ total_sum.s0 += ((float)(((bits4.s0 & 0x000F) ) | ((bits2.s0 & 0x03) << 4)) - 32.f) * cs.s1 * shared_y.s0; \
+ total_sum.s0 += ((float)(((bits4.s0 & 0x00F0) >> 4) | ((bits2.s0 & 0x0C) << 2)) - 32.f) * cs.s1 * shared_y.s1; \
+ total_sum.s0 += ((float)(((bits4.s0 & 0x0F00) >> 8) | ((bits2.s0 & 0x30) )) - 32.f) * cs.s1 * shared_y.s2; \
+ total_sum.s0 += ((float)(((bits4.s0 & 0xF000) >> 12) | ((bits2.s0 & 0xC0) >> 2)) - 32.f) * cs.s1 * shared_y.s3; \
+ total_sum.s0 += ((float)(((bits4.s2 & 0x000F) ) | ((bits2.s2 & 0x03) << 4)) - 32.f) * cs.s1 * shared_y.s4; \
+ total_sum.s0 += ((float)(((bits4.s2 & 0x00F0) >> 4) | ((bits2.s2 & 0x0C) << 2)) - 32.f) * cs.s1 * shared_y.s5; \
+ total_sum.s0 += ((float)(((bits4.s2 & 0x0F00) >> 8) | ((bits2.s2 & 0x30) )) - 32.f) * cs.s1 * shared_y.s6; \
+ total_sum.s0 += ((float)(((bits4.s2 & 0xF000) >> 12) | ((bits2.s2 & 0xC0) >> 2)) - 32.f) * cs.s1 * shared_y.s7; \
+ total_sum.s1 += ((float)(((bits4.s1 & 0x000F) ) | ((bits2.s1 & 0x03) << 4)) - 32.f) * cs.s3 * shared_y.s0; \
+ total_sum.s1 += ((float)(((bits4.s1 & 0x00F0) >> 4) | ((bits2.s1 & 0x0C) << 2)) - 32.f) * cs.s3 * shared_y.s1; \
+ total_sum.s1 += ((float)(((bits4.s1 & 0x0F00) >> 8) | ((bits2.s1 & 0x30) )) - 32.f) * cs.s3 * shared_y.s2; \
+ total_sum.s1 += ((float)(((bits4.s1 & 0xF000) >> 12) | ((bits2.s1 & 0xC0) >> 2)) - 32.f) * cs.s3 * shared_y.s3; \
+ total_sum.s1 += ((float)(((bits4.s3 & 0x000F) ) | ((bits2.s3 & 0x03) << 4)) - 32.f) * cs.s3 * shared_y.s4; \
+ total_sum.s1 += ((float)(((bits4.s3 & 0x00F0) >> 4) | ((bits2.s3 & 0x0C) << 2)) - 32.f) * cs.s3 * shared_y.s5; \
+ total_sum.s1 += ((float)(((bits4.s3 & 0x0F00) >> 8) | ((bits2.s3 & 0x30) )) - 32.f) * cs.s3 * shared_y.s6; \
+ total_sum.s1 += ((float)(((bits4.s3 & 0xF000) >> 12) | ((bits2.s3 & 0xC0) >> 2)) - 32.f) * cs.s3 * shared_y.s7; \
+ shared_y = sub_group_broadcast(y, 3); \
+ total_sum.s0 += ((float)(((bits4.s4 & 0x000F) ) | ((bits2.s4 & 0x03) << 4)) - 32.f) * cs.s1 * shared_y.s0; \
+ total_sum.s0 += ((float)(((bits4.s4 & 0x00F0) >> 4) | ((bits2.s4 & 0x0C) << 2)) - 32.f) * cs.s1 * shared_y.s1; \
+ total_sum.s0 += ((float)(((bits4.s4 & 0x0F00) >> 8) | ((bits2.s4 & 0x30) )) - 32.f) * cs.s1 * shared_y.s2; \
+ total_sum.s0 += ((float)(((bits4.s4 & 0xF000) >> 12) | ((bits2.s4 & 0xC0) >> 2)) - 32.f) * cs.s1 * shared_y.s3; \
+ total_sum.s0 += ((float)(((bits4.s6 & 0x000F) ) | ((bits2.s6 & 0x03) << 4)) - 32.f) * cs.s1 * shared_y.s4; \
+ total_sum.s0 += ((float)(((bits4.s6 & 0x00F0) >> 4) | ((bits2.s6 & 0x0C) << 2)) - 32.f) * cs.s1 * shared_y.s5; \
+ total_sum.s0 += ((float)(((bits4.s6 & 0x0F00) >> 8) | ((bits2.s6 & 0x30) )) - 32.f) * cs.s1 * shared_y.s6; \
+ total_sum.s0 += ((float)(((bits4.s6 & 0xF000) >> 12) | ((bits2.s6 & 0xC0) >> 2)) - 32.f) * cs.s1 * shared_y.s7; \
+ total_sum.s1 += ((float)(((bits4.s5 & 0x000F) ) | ((bits2.s5 & 0x03) << 4)) - 32.f) * cs.s3 * shared_y.s0; \
+ total_sum.s1 += ((float)(((bits4.s5 & 0x00F0) >> 4) | ((bits2.s5 & 0x0C) << 2)) - 32.f) * cs.s3 * shared_y.s1; \
+ total_sum.s1 += ((float)(((bits4.s5 & 0x0F00) >> 8) | ((bits2.s5 & 0x30) )) - 32.f) * cs.s3 * shared_y.s2; \
+ total_sum.s1 += ((float)(((bits4.s5 & 0xF000) >> 12) | ((bits2.s5 & 0xC0) >> 2)) - 32.f) * cs.s3 * shared_y.s3; \
+ total_sum.s1 += ((float)(((bits4.s7 & 0x000F) ) | ((bits2.s7 & 0x03) << 4)) - 32.f) * cs.s3 * shared_y.s4; \
+ total_sum.s1 += ((float)(((bits4.s7 & 0x00F0) >> 4) | ((bits2.s7 & 0x0C) << 2)) - 32.f) * cs.s3 * shared_y.s5; \
+ total_sum.s1 += ((float)(((bits4.s7 & 0x0F00) >> 8) | ((bits2.s7 & 0x30) )) - 32.f) * cs.s3 * shared_y.s6; \
+ total_sum.s1 += ((float)(((bits4.s7 & 0xF000) >> 12) | ((bits2.s7 & 0xC0) >> 2)) - 32.f) * cs.s3 * shared_y.s7; \
+
+#define dequantize_block_acc_bcast_1_hi(total_sum, bits4, bits2, cs, y) \
+ float shared_y; \
+ shared_y = sub_group_broadcast(y.s0, 0); \
+ total_sum.s0 += ((float)(((bits4.s0 & 0x000F) ) | ((bits2.s0 & 0x03) << 4)) - 32.f) * cs.s0 * shared_y; \
+ total_sum.s1 += ((float)(((bits4.s1 & 0x000F) ) | ((bits2.s1 & 0x03) << 4)) - 32.f) * cs.s2 * shared_y; \
+ shared_y = sub_group_broadcast(y.s1, 0); \
+ total_sum.s0 += ((float)(((bits4.s0 & 0x00F0) >> 4) | ((bits2.s0 & 0x0C) << 2)) - 32.f) * cs.s0 * shared_y; \
+ total_sum.s1 += ((float)(((bits4.s1 & 0x00F0) >> 4) | ((bits2.s1 & 0x0C) << 2)) - 32.f) * cs.s2 * shared_y; \
+ shared_y = sub_group_broadcast(y.s2, 0); \
+ total_sum.s0 += ((float)(((bits4.s0 & 0x0F00) >> 8) | ((bits2.s0 & 0x30) )) - 32.f) * cs.s0 * shared_y; \
+ total_sum.s1 += ((float)(((bits4.s1 & 0x0F00) >> 8) | ((bits2.s1 & 0x30) )) - 32.f) * cs.s2 * shared_y; \
+ shared_y = sub_group_broadcast(y.s3, 0); \
+ total_sum.s0 += ((float)(((bits4.s0 & 0xF000) >> 12) | ((bits2.s0 & 0xC0) >> 2)) - 32.f) * cs.s0 * shared_y; \
+ total_sum.s1 += ((float)(((bits4.s1 & 0xF000) >> 12) | ((bits2.s1 & 0xC0) >> 2)) - 32.f) * cs.s2 * shared_y; \
+ shared_y = sub_group_broadcast(y.s4, 0); \
+ total_sum.s0 += ((float)(((bits4.s2 & 0x000F) ) | ((bits2.s2 & 0x03) << 4)) - 32.f) * cs.s0 * shared_y; \
+ total_sum.s1 += ((float)(((bits4.s3 & 0x000F) ) | ((bits2.s3 & 0x03) << 4)) - 32.f) * cs.s2 * shared_y; \
+ shared_y = sub_group_broadcast(y.s5, 0); \
+ total_sum.s0 += ((float)(((bits4.s2 & 0x00F0) >> 4) | ((bits2.s2 & 0x0C) << 2)) - 32.f) * cs.s0 * shared_y; \
+ total_sum.s1 += ((float)(((bits4.s3 & 0x00F0) >> 4) | ((bits2.s3 & 0x0C) << 2)) - 32.f) * cs.s2 * shared_y; \
+ shared_y = sub_group_broadcast(y.s6, 0); \
+ total_sum.s0 += ((float)(((bits4.s2 & 0x0F00) >> 8) | ((bits2.s2 & 0x30) )) - 32.f) * cs.s0 * shared_y; \
+ total_sum.s1 += ((float)(((bits4.s3 & 0x0F00) >> 8) | ((bits2.s3 & 0x30) )) - 32.f) * cs.s2 * shared_y; \
+ shared_y = sub_group_broadcast(y.s7, 0); \
+ total_sum.s0 += ((float)(((bits4.s2 & 0xF000) >> 12) | ((bits2.s2 & 0xC0) >> 2)) - 32.f) * cs.s0 * shared_y; \
+ total_sum.s1 += ((float)(((bits4.s3 & 0xF000) >> 12) | ((bits2.s3 & 0xC0) >> 2)) - 32.f) * cs.s2 * shared_y; \
+ shared_y = sub_group_broadcast(y.s0, 1); \
+ total_sum.s0 += ((float)(((bits4.s4 & 0x000F) ) | ((bits2.s4 & 0x03) << 4)) - 32.f) * cs.s0 * shared_y; \
+ total_sum.s1 += ((float)(((bits4.s5 & 0x000F) ) | ((bits2.s5 & 0x03) << 4)) - 32.f) * cs.s2 * shared_y; \
+ shared_y = sub_group_broadcast(y.s1, 1); \
+ total_sum.s0 += ((float)(((bits4.s4 & 0x00F0) >> 4) | ((bits2.s4 & 0x0C) << 2)) - 32.f) * cs.s0 * shared_y; \
+ total_sum.s1 += ((float)(((bits4.s5 & 0x00F0) >> 4) | ((bits2.s5 & 0x0C) << 2)) - 32.f) * cs.s2 * shared_y; \
+ shared_y = sub_group_broadcast(y.s2, 1); \
+ total_sum.s0 += ((float)(((bits4.s4 & 0x0F00) >> 8) | ((bits2.s4 & 0x30) )) - 32.f) * cs.s0 * shared_y; \
+ total_sum.s1 += ((float)(((bits4.s5 & 0x0F00) >> 8) | ((bits2.s5 & 0x30) )) - 32.f) * cs.s2 * shared_y; \
+ shared_y = sub_group_broadcast(y.s3, 1); \
+ total_sum.s0 += ((float)(((bits4.s4 & 0xF000) >> 12) | ((bits2.s4 & 0xC0) >> 2)) - 32.f) * cs.s0 * shared_y; \
+ total_sum.s1 += ((float)(((bits4.s5 & 0xF000) >> 12) | ((bits2.s5 & 0xC0) >> 2)) - 32.f) * cs.s2 * shared_y; \
+ shared_y = sub_group_broadcast(y.s4, 1); \
+ total_sum.s0 += ((float)(((bits4.s6 & 0x000F) ) | ((bits2.s6 & 0x03) << 4)) - 32.f) * cs.s0 * shared_y; \
+ total_sum.s1 += ((float)(((bits4.s7 & 0x000F) ) | ((bits2.s7 & 0x03) << 4)) - 32.f) * cs.s2 * shared_y; \
+ shared_y = sub_group_broadcast(y.s5, 1); \
+ total_sum.s0 += ((float)(((bits4.s6 & 0x00F0) >> 4) | ((bits2.s6 & 0x0C) << 2)) - 32.f) * cs.s0 * shared_y; \
+ total_sum.s1 += ((float)(((bits4.s7 & 0x00F0) >> 4) | ((bits2.s7 & 0x0C) << 2)) - 32.f) * cs.s2 * shared_y; \
+ shared_y = sub_group_broadcast(y.s6, 1); \
+ total_sum.s0 += ((float)(((bits4.s6 & 0x0F00) >> 8) | ((bits2.s6 & 0x30) )) - 32.f) * cs.s0 * shared_y; \
+ total_sum.s1 += ((float)(((bits4.s7 & 0x0F00) >> 8) | ((bits2.s7 & 0x30) )) - 32.f) * cs.s2 * shared_y; \
+ shared_y = sub_group_broadcast(y.s7, 1); \
+ total_sum.s0 += ((float)(((bits4.s6 & 0xF000) >> 12) | ((bits2.s6 & 0xC0) >> 2)) - 32.f) * cs.s0 * shared_y; \
+ total_sum.s1 += ((float)(((bits4.s7 & 0xF000) >> 12) | ((bits2.s7 & 0xC0) >> 2)) - 32.f) * cs.s2 * shared_y; \
+
+#define dequantize_block_acc_bcast_1_lo(total_sum, bits4, bits2, cs, y) \
+ shared_y = sub_group_broadcast(y.s0, 2); \
+ total_sum.s0 += ((float)(((bits4.s0 & 0x000F) ) | ((bits2.s0 & 0x03) << 4)) - 32.f) * cs.s1 * shared_y; \
+ total_sum.s1 += ((float)(((bits4.s1 & 0x000F) ) | ((bits2.s1 & 0x03) << 4)) - 32.f) * cs.s3 * shared_y; \
+ shared_y = sub_group_broadcast(y.s1, 2); \
+ total_sum.s0 += ((float)(((bits4.s0 & 0x00F0) >> 4) | ((bits2.s0 & 0x0C) << 2)) - 32.f) * cs.s1 * shared_y; \
+ total_sum.s1 += ((float)(((bits4.s1 & 0x00F0) >> 4) | ((bits2.s1 & 0x0C) << 2)) - 32.f) * cs.s3 * shared_y; \
+ shared_y = sub_group_broadcast(y.s2, 2); \
+ total_sum.s0 += ((float)(((bits4.s0 & 0x0F00) >> 8) | ((bits2.s0 & 0x30) )) - 32.f) * cs.s1 * shared_y; \
+ total_sum.s1 += ((float)(((bits4.s1 & 0x0F00) >> 8) | ((bits2.s1 & 0x30) )) - 32.f) * cs.s3 * shared_y; \
+ shared_y = sub_group_broadcast(y.s3, 2); \
+ total_sum.s0 += ((float)(((bits4.s0 & 0xF000) >> 12) | ((bits2.s0 & 0xC0) >> 2)) - 32.f) * cs.s1 * shared_y; \
+ total_sum.s1 += ((float)(((bits4.s1 & 0xF000) >> 12) | ((bits2.s1 & 0xC0) >> 2)) - 32.f) * cs.s3 * shared_y; \
+ shared_y = sub_group_broadcast(y.s4, 2); \
+ total_sum.s0 += ((float)(((bits4.s2 & 0x000F) ) | ((bits2.s2 & 0x03) << 4)) - 32.f) * cs.s1 * shared_y; \
+ total_sum.s1 += ((float)(((bits4.s3 & 0x000F) ) | ((bits2.s3 & 0x03) << 4)) - 32.f) * cs.s3 * shared_y; \
+ shared_y = sub_group_broadcast(y.s5, 2); \
+ total_sum.s0 += ((float)(((bits4.s2 & 0x00F0) >> 4) | ((bits2.s2 & 0x0C) << 2)) - 32.f) * cs.s1 * shared_y; \
+ total_sum.s1 += ((float)(((bits4.s3 & 0x00F0) >> 4) | ((bits2.s3 & 0x0C) << 2)) - 32.f) * cs.s3 * shared_y; \
+ shared_y = sub_group_broadcast(y.s6, 2); \
+ total_sum.s0 += ((float)(((bits4.s2 & 0x0F00) >> 8) | ((bits2.s2 & 0x30) )) - 32.f) * cs.s1 * shared_y; \
+ total_sum.s1 += ((float)(((bits4.s3 & 0x0F00) >> 8) | ((bits2.s3 & 0x30) )) - 32.f) * cs.s3 * shared_y; \
+ shared_y = sub_group_broadcast(y.s7, 2); \
+ total_sum.s0 += ((float)(((bits4.s2 & 0xF000) >> 12) | ((bits2.s2 & 0xC0) >> 2)) - 32.f) * cs.s1 * shared_y; \
+ total_sum.s1 += ((float)(((bits4.s3 & 0xF000) >> 12) | ((bits2.s3 & 0xC0) >> 2)) - 32.f) * cs.s3 * shared_y; \
+ shared_y = sub_group_broadcast(y.s0, 3); \
+ total_sum.s0 += ((float)(((bits4.s4 & 0x000F) ) | ((bits2.s4 & 0x03) << 4)) - 32.f) * cs.s1 * shared_y; \
+ total_sum.s1 += ((float)(((bits4.s5 & 0x000F) ) | ((bits2.s5 & 0x03) << 4)) - 32.f) * cs.s3 * shared_y; \
+ shared_y = sub_group_broadcast(y.s1, 3); \
+ total_sum.s0 += ((float)(((bits4.s4 & 0x00F0) >> 4) | ((bits2.s4 & 0x0C) << 2)) - 32.f) * cs.s1 * shared_y; \
+ total_sum.s1 += ((float)(((bits4.s5 & 0x00F0) >> 4) | ((bits2.s5 & 0x0C) << 2)) - 32.f) * cs.s3 * shared_y; \
+ shared_y = sub_group_broadcast(y.s2, 3); \
+ total_sum.s0 += ((float)(((bits4.s4 & 0x0F00) >> 8) | ((bits2.s4 & 0x30) )) - 32.f) * cs.s1 * shared_y; \
+ total_sum.s1 += ((float)(((bits4.s5 & 0x0F00) >> 8) | ((bits2.s5 & 0x30) )) - 32.f) * cs.s3 * shared_y; \
+ shared_y = sub_group_broadcast(y.s3, 3); \
+ total_sum.s0 += ((float)(((bits4.s4 & 0xF000) >> 12) | ((bits2.s4 & 0xC0) >> 2)) - 32.f) * cs.s1 * shared_y; \
+ total_sum.s1 += ((float)(((bits4.s5 & 0xF000) >> 12) | ((bits2.s5 & 0xC0) >> 2)) - 32.f) * cs.s3 * shared_y; \
+ shared_y = sub_group_broadcast(y.s4, 3); \
+ total_sum.s0 += ((float)(((bits4.s6 & 0x000F) ) | ((bits2.s6 & 0x03) << 4)) - 32.f) * cs.s1 * shared_y; \
+ total_sum.s1 += ((float)(((bits4.s7 & 0x000F) ) | ((bits2.s7 & 0x03) << 4)) - 32.f) * cs.s3 * shared_y; \
+ shared_y = sub_group_broadcast(y.s5, 3); \
+ total_sum.s0 += ((float)(((bits4.s6 & 0x00F0) >> 4) | ((bits2.s6 & 0x0C) << 2)) - 32.f) * cs.s1 * shared_y; \
+ total_sum.s1 += ((float)(((bits4.s7 & 0x00F0) >> 4) | ((bits2.s7 & 0x0C) << 2)) - 32.f) * cs.s3 * shared_y; \
+ shared_y = sub_group_broadcast(y.s6, 3); \
+ total_sum.s0 += ((float)(((bits4.s6 & 0x0F00) >> 8) | ((bits2.s6 & 0x30) )) - 32.f) * cs.s1 * shared_y; \
+ total_sum.s1 += ((float)(((bits4.s7 & 0x0F00) >> 8) | ((bits2.s7 & 0x30) )) - 32.f) * cs.s3 * shared_y; \
+ shared_y = sub_group_broadcast(y.s7, 3); \
+ total_sum.s0 += ((float)(((bits4.s6 & 0xF000) >> 12) | ((bits2.s6 & 0xC0) >> 2)) - 32.f) * cs.s1 * shared_y; \
+ total_sum.s1 += ((float)(((bits4.s7 & 0xF000) >> 12) | ((bits2.s7 & 0xC0) >> 2)) - 32.f) * cs.s3 * shared_y; \
+
+#if defined(ADRENO_GPU)
+REQD_SUBGROUP_SIZE_64
+#endif
+// Q6K_O4_GLOBAL: read the (read-once-per-token, no-reuse) lm_head/embed weights
+// from __global coalesced instead of image1d_buffer. The texture cache caps the
+// streaming (no-reuse) lm_head read bandwidth; global coalesced reaches the
+// higher rate the rest of the model gets. src1 (activation) stays an image (it IS reused via
+// the cross-subgroup broadcast).
+#ifdef Q6K_O4_GLOBAL
+#define Q6K_O4_NAME kernel_gemv_noshuffle_q6_K_f32_o4_global
+#define QL_ARG __global uint * src0_ql
+#define QH_ARG __global half * src0_qh
+#define RD_QL(b,i) (b[i])
+#define RD_QH(b,i) as_ushort(b[i])
+#else
+#define Q6K_O4_NAME kernel_gemv_noshuffle_q6_K_f32_o4
+#define QL_ARG read_only image1d_buffer_t src0_ql
+#define QH_ARG read_only image1d_buffer_t src0_qh
+#define RD_QL(b,i) (read_imageui(b,i).x)
+#define RD_QH(b,i) as_ushort(read_imageh(b,i).x)
+#endif
+kernel void Q6K_O4_NAME(
+ QL_ARG,
+ QH_ARG,
+ global half2 * src0_s,
+ global half2 * src0_d,
+ read_only image1d_buffer_t src1,
+ global float * dst,
+ ulong offsetd,
+ int ne00,
+ int ne01
+) {
+ int grp = get_local_id(1);
+ int gid = get_global_id(0); // 4-output-quad index
+ ushort slid = get_sub_group_local_id();
+
+ // Map quad index to the two pair-indices the existing 2-output access
+ // pattern uses (consecutive output pairs along ne01). NB: the two pairs are
+ // kept ADJACENT (gid*2, gid*2+1) on purpose -- a "stride-1" split (pairs
+ // ne01/4 apart) is slower because two distant cache-line streams have worse
+ // locality than the adjacent pair whose reads interleave into the same lines
+ // each iteration.
+ int gid_a = gid * 2;
+ int gid_b = gid * 2 + 1;
+
+ int nb = ne00 / 32;
+
+ uint4 reg_a_l_a, reg_a_l_b;
+ ushort4 reg_a_h_a, reg_a_h_b;
+ half2 reg_d_a, reg_d_b;
+ char4 reg_s_a, reg_s_b;
+ float8 reg_b;
+
+ float2 total_sum_a = 0.0f;
+ float2 total_sum_b = 0.0f;
+
+ int line_stride_a = ne01 / 2;
+ int block_stride_a = NSUBGROUPS * ne01;
+
+ for (int k = grp; k < nb; k += NSUBGROUPS) {
+ reg_d_a = src0_d[gid_a + k/8 * line_stride_a];
+ reg_d_b = src0_d[gid_b + k/8 * line_stride_a];
+ reg_s_a = as_char4(src0_s[gid_a + k * line_stride_a]);
+ reg_s_b = as_char4(src0_s[gid_b + k * line_stride_a]);
+ // Precompute the loop-invariant combined scale (sub-block scale * super-block d)
+ // once per pair instead of re-multiplying it for every one of the 256 elements.
+ float4 cs_a = (float4)((float)reg_s_a.s0*(float)reg_d_a.s0, (float)reg_s_a.s1*(float)reg_d_a.s0,
+ (float)reg_s_a.s2*(float)reg_d_a.s1, (float)reg_s_a.s3*(float)reg_d_a.s1);
+ float4 cs_b = (float4)((float)reg_s_b.s0*(float)reg_d_b.s0, (float)reg_s_b.s1*(float)reg_d_b.s0,
+ (float)reg_s_b.s2*(float)reg_d_b.s1, (float)reg_s_b.s3*(float)reg_d_b.s1);
+
+ if (slid < 4) {
+ reg_b.s0123 = read_imagef(src1, 0 + slid*2 + k*8);
+ reg_b.s4567 = read_imagef(src1, 1 + slid*2 + k*8);
+ }
+
+ // Pair a (output rows gid_a*2, gid_a*2+1): read hi+lo then dequant
+ // both in one block so the `_lo` macro can see the `shared_y` that
+ // `_hi` declared. Pair b follows in its own block — fresh shared_y.
+ {
+ reg_a_l_a.s0 = RD_QL(src0_ql, gid_a + k*block_stride_a + line_stride_a*0);
+ reg_a_l_a.s1 = RD_QL(src0_ql, gid_a + k*block_stride_a + line_stride_a*1);
+ reg_a_l_a.s2 = RD_QL(src0_ql, gid_a + k*block_stride_a + line_stride_a*2);
+ reg_a_l_a.s3 = RD_QL(src0_ql, gid_a + k*block_stride_a + line_stride_a*3);
+ reg_a_h_a.s0 = RD_QH(src0_qh, gid_a + k*block_stride_a + line_stride_a*0);
+ reg_a_h_a.s1 = RD_QH(src0_qh, gid_a + k*block_stride_a + line_stride_a*1);
+ reg_a_h_a.s2 = RD_QH(src0_qh, gid_a + k*block_stride_a + line_stride_a*2);
+ reg_a_h_a.s3 = RD_QH(src0_qh, gid_a + k*block_stride_a + line_stride_a*3);
+#ifdef VECTOR_SUB_GROUP_BROADCAT
+ dequantize_block_acc_bcast_8_hi(total_sum_a, as_ushort8(reg_a_l_a), as_uchar8(reg_a_h_a), cs_a, reg_b);
+#else
+ dequantize_block_acc_bcast_1_hi(total_sum_a, as_ushort8(reg_a_l_a), as_uchar8(reg_a_h_a), cs_a, reg_b);
+#endif
+
+ reg_a_l_a.s0 = RD_QL(src0_ql, gid_a + k*block_stride_a + line_stride_a*4);
+ reg_a_l_a.s1 = RD_QL(src0_ql, gid_a + k*block_stride_a + line_stride_a*5);
+ reg_a_l_a.s2 = RD_QL(src0_ql, gid_a + k*block_stride_a + line_stride_a*6);
+ reg_a_l_a.s3 = RD_QL(src0_ql, gid_a + k*block_stride_a + line_stride_a*7);
+ reg_a_h_a.s0 = RD_QH(src0_qh, gid_a + k*block_stride_a + line_stride_a*4);
+ reg_a_h_a.s1 = RD_QH(src0_qh, gid_a + k*block_stride_a + line_stride_a*5);
+ reg_a_h_a.s2 = RD_QH(src0_qh, gid_a + k*block_stride_a + line_stride_a*6);
+ reg_a_h_a.s3 = RD_QH(src0_qh, gid_a + k*block_stride_a + line_stride_a*7);
+#ifdef VECTOR_SUB_GROUP_BROADCAT
+ dequantize_block_acc_bcast_8_lo(total_sum_a, as_ushort8(reg_a_l_a), as_uchar8(reg_a_h_a), cs_a, reg_b);
+#else
+ dequantize_block_acc_bcast_1_lo(total_sum_a, as_ushort8(reg_a_l_a), as_uchar8(reg_a_h_a), cs_a, reg_b);
+#endif
+ }
+
+ {
+ reg_a_l_b.s0 = RD_QL(src0_ql, gid_b + k*block_stride_a + line_stride_a*0);
+ reg_a_l_b.s1 = RD_QL(src0_ql, gid_b + k*block_stride_a + line_stride_a*1);
+ reg_a_l_b.s2 = RD_QL(src0_ql, gid_b + k*block_stride_a + line_stride_a*2);
+ reg_a_l_b.s3 = RD_QL(src0_ql, gid_b + k*block_stride_a + line_stride_a*3);
+ reg_a_h_b.s0 = RD_QH(src0_qh, gid_b + k*block_stride_a + line_stride_a*0);
+ reg_a_h_b.s1 = RD_QH(src0_qh, gid_b + k*block_stride_a + line_stride_a*1);
+ reg_a_h_b.s2 = RD_QH(src0_qh, gid_b + k*block_stride_a + line_stride_a*2);
+ reg_a_h_b.s3 = RD_QH(src0_qh, gid_b + k*block_stride_a + line_stride_a*3);
+#ifdef VECTOR_SUB_GROUP_BROADCAT
+ dequantize_block_acc_bcast_8_hi(total_sum_b, as_ushort8(reg_a_l_b), as_uchar8(reg_a_h_b), cs_b, reg_b);
+#else
+ dequantize_block_acc_bcast_1_hi(total_sum_b, as_ushort8(reg_a_l_b), as_uchar8(reg_a_h_b), cs_b, reg_b);
+#endif
+
+ reg_a_l_b.s0 = RD_QL(src0_ql, gid_b + k*block_stride_a + line_stride_a*4);
+ reg_a_l_b.s1 = RD_QL(src0_ql, gid_b + k*block_stride_a + line_stride_a*5);
+ reg_a_l_b.s2 = RD_QL(src0_ql, gid_b + k*block_stride_a + line_stride_a*6);
+ reg_a_l_b.s3 = RD_QL(src0_ql, gid_b + k*block_stride_a + line_stride_a*7);
+ reg_a_h_b.s0 = RD_QH(src0_qh, gid_b + k*block_stride_a + line_stride_a*4);
+ reg_a_h_b.s1 = RD_QH(src0_qh, gid_b + k*block_stride_a + line_stride_a*5);
+ reg_a_h_b.s2 = RD_QH(src0_qh, gid_b + k*block_stride_a + line_stride_a*6);
+ reg_a_h_b.s3 = RD_QH(src0_qh, gid_b + k*block_stride_a + line_stride_a*7);
+#ifdef VECTOR_SUB_GROUP_BROADCAT
+ dequantize_block_acc_bcast_8_lo(total_sum_b, as_ushort8(reg_a_l_b), as_uchar8(reg_a_h_b), cs_b, reg_b);
+#else
+ dequantize_block_acc_bcast_1_lo(total_sum_b, as_ushort8(reg_a_l_b), as_uchar8(reg_a_h_b), cs_b, reg_b);
+#endif
+ }
+ }
+
+ // Cross-subgroup reduce. Same shape as the 2-output kernel but with the
+ // pair-a and pair-b accumulators concatenated into a single float4.
+ local float4 reduce_lm[SUBGROUP_SIZE * 3];
+ float4 acc = (float4)(total_sum_a.s0, total_sum_a.s1, total_sum_b.s0, total_sum_b.s1);
+ if (grp == 1) { reduce_lm[SUBGROUP_SIZE*0 + slid] = acc; }
+ if (grp == 2) { reduce_lm[SUBGROUP_SIZE*1 + slid] = acc; }
+ if (grp == 3) { reduce_lm[SUBGROUP_SIZE*2 + slid] = acc; }
+
+ barrier(CLK_LOCAL_MEM_FENCE);
+
+ if (grp == 0) {
+ acc += reduce_lm[SUBGROUP_SIZE*0 + slid];
+ acc += reduce_lm[SUBGROUP_SIZE*1 + slid];
+ acc += reduce_lm[SUBGROUP_SIZE*2 + slid];
+ dst = (global float*)((global char*)dst + offsetd);
+ // The dispatch rounds ne01/4 up to the subgroup width, so the tail
+ // quads past the last row must not store (they wrote 128 rows past
+ // dst on every ne01 % 256 == 128 vocab, e.g. 151936).
+ if (gid * 4 + 3 < (uint)ne01) {
+ vstore4(acc, 0, &(dst[gid * 4]));
+ }
+ }
+}
diff --git a/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q6_k_f32_tiled.cl b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q6_k_f32_tiled.cl
new file mode 100644
index 0000000000..c5049f3964
--- /dev/null
+++ b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q6_k_f32_tiled.cl
@@ -0,0 +1,196 @@
+// Tiled-wide q6_K GEMV for the long-vocab lm_head/embed (decode path).
+//
+// Pairs with kernel_convert_block_q6_k_tiled_ns (cvt.cl): the weights are laid
+// out CANONICALLY (6-bit code in element order e in [0,256)) and TILED by 64
+// output rows so the 64-thread lane group coalesces every weight load. Both the
+// pack (convert) and the unpack (here) are owned by us — correct by construction
+// against the reference ggml q6_K dequant, no bit-interleave reverse-engineering.
+//
+// One work-item produces one output row. A work-group is {64 lanes, 4 subgroups}:
+// the 64 lanes cover the 64 rows of one tile (coalesced reads), the 4 subgroups
+// split the K-blocks and reduce through __local at the end.
+//
+// Weights are read from __global (coalesced) rather than image1d_buffer: the
+// lm_head is read once per token with no reuse, and the Adreno texture cache
+// caps such a streaming read well below the coalesced-global rate
+// (see opencl_q6k_gemv_o4_shipped / x2-90 roofline notes).
+
+#pragma OPENCL EXTENSION cl_khr_fp16 : enable
+
+#ifdef cl_qcom_reqd_sub_group_size
+#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable
+#define ADRENO_GPU 1
+#define REQD_SUBGROUP_SIZE_64 __attribute__((qcom_reqd_sub_group_size("half")))
+#endif
+
+#define NSUBGROUPS 4
+#define TILE_ROWS 64
+
+#if defined(ADRENO_GPU)
+REQD_SUBGROUP_SIZE_64
+#endif
+kernel void kernel_gemv_noshuffle_q6_K_f32_tiled(
+ __global uint4 * src0_ql, // tiled: 8 uint4 granules / superblock
+ __global uint4 * src0_qh, // tiled: 4 uint4 granules / superblock
+ __global char * src0_s, // tiled: 16 chars / superblock
+ __global half * src0_d, // tiled: 1 half / superblock
+ read_only image1d_buffer_t src1, // activation (RGBA f32)
+ global float * dst,
+ ulong offsetd,
+ int ne00,
+ int ne01
+) {
+ int grp = get_local_id(1); // subgroup index 0..3 (splits K)
+ int row = get_global_id(0); // output row along ne01
+ int rt = row / TILE_ROWS;
+ int rit = row % TILE_ROWS;
+
+ int nb = ne00 / 256; // superblocks per row
+
+ float acc = 0.0f;
+
+ for (int sb = grp; sb < nb; sb += NSUBGROUPS) {
+ int tile_blk = rt * nb + sb; // ne02 == 1 for lm_head/embed
+
+ // d + 16 scales for this (row, superblock)
+ float dval = (float)src0_d[tile_blk * TILE_ROWS + rit];
+ __global char * sc = src0_s + (tile_blk * TILE_ROWS + rit) * 16;
+
+ // 32 ql-uints (8 codes/uint) + 16 qh-uints (16 codes/uint)
+ uint ql[32];
+ uint qh[16];
+ #pragma unroll
+ for (int g = 0; g < 8; ++g) {
+ uint4 v = src0_ql[(tile_blk * 8 + g) * TILE_ROWS + rit];
+ ql[g*4+0] = v.x; ql[g*4+1] = v.y; ql[g*4+2] = v.z; ql[g*4+3] = v.w;
+ }
+ #pragma unroll
+ for (int g = 0; g < 4; ++g) {
+ uint4 v = src0_qh[(tile_blk * 4 + g) * TILE_ROWS + rit];
+ qh[g*4+0] = v.x; qh[g*4+1] = v.y; qh[g*4+2] = v.z; qh[g*4+3] = v.w;
+ }
+
+ // dequant 256 codes in canonical e-order, MAC with activation.
+ int act_base = sb * 64; // activation float4 pixel base (256/4)
+ #pragma unroll
+ for (int e4 = 0; e4 < 64; ++e4) {
+ float4 a = read_imagef(src1, act_base + e4);
+ #pragma unroll
+ for (int t = 0; t < 4; ++t) {
+ int e = e4 * 4 + t;
+ uint low4 = (ql[e >> 3] >> ((e & 7) * 4)) & 0xF;
+ uint hi2 = (qh[e >> 4] >> ((e & 15) * 2)) & 0x3;
+ int code = (int)(low4 | (hi2 << 4)) - 32;
+ int sidx = ((e >> 7) << 3) + (((e >> 5) & 3) << 1) + ((e >> 4) & 1);
+ float scale = (float)sc[sidx] * dval;
+ float av = (t == 0) ? a.x : (t == 1) ? a.y : (t == 2) ? a.z : a.w;
+ acc += (float)code * scale * av;
+ }
+ }
+ }
+
+ // reduce across the NSUBGROUPS subgroups (same rit, different K-subset)
+ local float reduce_lm[NSUBGROUPS * TILE_ROWS];
+ reduce_lm[grp * TILE_ROWS + rit] = acc;
+ barrier(CLK_LOCAL_MEM_FENCE);
+
+ if (grp == 0) {
+ float total = reduce_lm[0 * TILE_ROWS + rit]
+ + reduce_lm[1 * TILE_ROWS + rit]
+ + reduce_lm[2 * TILE_ROWS + rit]
+ + reduce_lm[3 * TILE_ROWS + rit];
+ dst = (global float*)((global char*)dst + offsetd);
+ dst[row] = total;
+ }
+}
+
+// Multi-column (N=3) variant of the tiled q6_K decode GEMV, for the speculative/
+// MTP VERIFY lm_head/embed (ne1=3 = 2 drafts + 1 bonus). Identical tiled weight
+// layout + unpack as the ne1=1 kernel above; each WI computes 3 output columns,
+// streaming the (large) lm_head weight ONCE per superblock and reusing it across
+// the 3 verify activation columns (dequant once per code, MAC into 3 accs). This
+// is the lm_head analogue of the per-layer mc3 GEMV; the multiply order matches
+// the ne1=1 kernel, so each column is byte-identical to a standalone tiled GEMV.
+#if defined(ADRENO_GPU)
+REQD_SUBGROUP_SIZE_64
+#endif
+kernel void kernel_gemv_noshuffle_q6_K_f32_tiled_mc3(
+ __global uint4 * src0_ql,
+ __global uint4 * src0_qh,
+ __global char * src0_s,
+ __global half * src0_d,
+ read_only image1d_buffer_t src1,
+ global float * dst,
+ ulong offsetd,
+ int ne00,
+ int ne01
+) {
+ int grp = get_local_id(1);
+ int row = get_global_id(0);
+ int rt = row / TILE_ROWS;
+ int rit = row % TILE_ROWS;
+
+ int nb = ne00 / 256;
+ int col_stride = ne00 / 4; // activation float4 pixels per column
+
+ float acc0 = 0.0f, acc1 = 0.0f, acc2 = 0.0f;
+
+ for (int sb = grp; sb < nb; sb += NSUBGROUPS) {
+ int tile_blk = rt * nb + sb;
+
+ float dval = (float)src0_d[tile_blk * TILE_ROWS + rit];
+ __global char * sc = src0_s + (tile_blk * TILE_ROWS + rit) * 16;
+
+ uint ql[32];
+ uint qh[16];
+ #pragma unroll
+ for (int g = 0; g < 8; ++g) {
+ uint4 v = src0_ql[(tile_blk * 8 + g) * TILE_ROWS + rit];
+ ql[g*4+0] = v.x; ql[g*4+1] = v.y; ql[g*4+2] = v.z; ql[g*4+3] = v.w;
+ }
+ #pragma unroll
+ for (int g = 0; g < 4; ++g) {
+ uint4 v = src0_qh[(tile_blk * 4 + g) * TILE_ROWS + rit];
+ qh[g*4+0] = v.x; qh[g*4+1] = v.y; qh[g*4+2] = v.z; qh[g*4+3] = v.w;
+ }
+
+ int act_base = sb * 64;
+ #pragma unroll
+ for (int e4 = 0; e4 < 64; ++e4) {
+ float4 a0 = read_imagef(src1, 0*col_stride + act_base + e4);
+ float4 a1 = read_imagef(src1, 1*col_stride + act_base + e4);
+ float4 a2 = read_imagef(src1, 2*col_stride + act_base + e4);
+ #pragma unroll
+ for (int t = 0; t < 4; ++t) {
+ int e = e4 * 4 + t;
+ uint low4 = (ql[e >> 3] >> ((e & 7) * 4)) & 0xF;
+ uint hi2 = (qh[e >> 4] >> ((e & 15) * 2)) & 0x3;
+ int code = (int)(low4 | (hi2 << 4)) - 32;
+ int sidx = ((e >> 7) << 3) + (((e >> 5) & 3) << 1) + ((e >> 4) & 1);
+ float w = (float)code * ((float)sc[sidx] * dval); // dequant+scale once
+ float av0 = (t == 0) ? a0.x : (t == 1) ? a0.y : (t == 2) ? a0.z : a0.w;
+ float av1 = (t == 0) ? a1.x : (t == 1) ? a1.y : (t == 2) ? a1.z : a1.w;
+ float av2 = (t == 0) ? a2.x : (t == 1) ? a2.y : (t == 2) ? a2.z : a2.w;
+ acc0 += w * av0;
+ acc1 += w * av1;
+ acc2 += w * av2;
+ }
+ }
+ }
+
+ local float4 reduce_lm[NSUBGROUPS * TILE_ROWS];
+ reduce_lm[grp * TILE_ROWS + rit] = (float4)(acc0, acc1, acc2, 0.0f);
+ barrier(CLK_LOCAL_MEM_FENCE);
+
+ if (grp == 0) {
+ float4 total = reduce_lm[0 * TILE_ROWS + rit]
+ + reduce_lm[1 * TILE_ROWS + rit]
+ + reduce_lm[2 * TILE_ROWS + rit]
+ + reduce_lm[3 * TILE_ROWS + rit];
+ dst = (global float*)((global char*)dst + offsetd);
+ // dst column-major [ne01 rows x 3 cols]: (row, col) at col*ne01 + row
+ dst[0*ne01 + row] = total.x;
+ dst[1*ne01 + row] = total.y;
+ dst[2*ne01 + row] = total.z;
+ }
+}
diff --git a/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q8_0_f32.cl b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q8_0_f32.cl
index 09bae2d555..6f6d7425c6 100644
--- a/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q8_0_f32.cl
+++ b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q8_0_f32.cl
@@ -118,6 +118,87 @@
elem = (char)((bits8.s7 & 0xFF000000) >> 24); \
total_sums += convert_int(elem) * scale * shared_y; \
+// ============================================================================
+// Split-K variant for small-M decode GEMVs.
+// ----------------------------------------------------------------------------
+// The base kernel below puts one output row per lane and splits K only across
+// the N_SIMDGROUP subgroups of a single workgroup, so M=512 yields M/64 = 8
+// workgroups -- half the compute units on a 16-CU X2 sit idle, and the kernel
+// measures ~48 GB/s against the ~122 GB/s the larger projections reach in the
+// same graph. Here each (kslice, subgroup) pair reduces a disjoint set of
+// K-blocks into partial[kslice * M + row]; kernel_gemv_splitk_reduce_f32 (in
+// gemv_noshuffle_q4_k_f32.cl) sums the slices. Same operand order within a
+// slice as the base kernel; only the cross-slice grouping differs.
+//
+// Placed BEFORE the base kernel deliberately: on A6X no kernel may be defined
+// after one that uses a subgroup builtin, or it silently miscompiles.
+// ============================================================================
+#ifdef ADRENO_GPU
+REQD_SUBGROUP_SIZE_64
+#endif
+__kernel void kernel_gemv_noshuffle_q8_0_f32_splitk(
+ __read_only image1d_buffer_t src0_q, // quantized A (weights)
+ global half * src0_d, // A scales
+ __read_only image1d_buffer_t src1, // B (activations)
+ global float * partial, // [ksplit * M], slice-major
+ int ne00, // K
+ int ne01) // M
+{
+ uint groupId = get_local_id(1);
+ uint gid = get_global_id(0);
+ ushort slid = get_sub_group_local_id();
+ uint nsg = get_local_size(1);
+ uint ksplit = get_num_groups(1);
+ uint kslice = get_group_id(1);
+
+ uint K = ne00;
+ uint M = ne01;
+
+ uint LINE_STRIDE_A = M;
+ uint BLOCK_STRIDE_A = 8 * M; // physical, independent of the K-split
+
+ __private uint8 regA;
+ __private half regS;
+ __private float8 regB;
+ __private float totalSum = (float)(0.0f);
+
+ #pragma unroll 1
+ for (uint k = kslice * nsg + groupId; k < (K / QK8_0); k += ksplit * nsg) {
+ regS = src0_d[gid + k * LINE_STRIDE_A];
+ if (slid < 4) {
+ regB.s0123 = read_imagef(src1, (slid * 2 + k * 8));
+ regB.s4567 = read_imagef(src1, (1 + slid * 2 + k * 8));
+ }
+ regA.s0 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 0)).x;
+ regA.s1 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 1)).x;
+ regA.s2 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 2)).x;
+ regA.s3 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 3)).x;
+ regA.s4 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 4)).x;
+ regA.s5 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 5)).x;
+ regA.s6 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 6)).x;
+ regA.s7 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 7)).x;
+
+ dequantizeBlockAccum_ns_sgbroadcast_1(totalSum, regA, convert_float(regS), regB);
+ }
+
+ // Intra-workgroup reduce across this K-slice's subgroups. Sized for
+ // nsg <= 8; the host never dispatches more.
+ __local float reduceLM[SIMDGROUP_WIDTH * 7];
+ if (groupId > 0) {
+ reduceLM[SIMDGROUP_WIDTH * (groupId - 1) + slid] = totalSum;
+ }
+ barrier(CLK_LOCAL_MEM_FENCE);
+ if (groupId == 0) {
+ for (uint i = 0; i < nsg - 1; ++i) {
+ totalSum += reduceLM[SIMDGROUP_WIDTH * i + slid];
+ }
+ // x-grid is padded to CEIL_DIV(M,wave)*wave; guard the tail rows.
+ if (gid < M) {
+ partial[kslice * M + gid] = totalSum;
+ }
+ }
+}
+
#ifdef ADRENO_GPU
REQD_SUBGROUP_SIZE_64
#endif
diff --git a/ggml/src/ggml-opencl/kernels/mul_mm_f32_f32_l4_lm.cl b/ggml/src/ggml-opencl/kernels/mul_mm_f32_f32_l4_lm.cl
index d7d5ba647e..9dc9862bef 100644
--- a/ggml/src/ggml-opencl/kernels/mul_mm_f32_f32_l4_lm.cl
+++ b/ggml/src/ggml-opencl/kernels/mul_mm_f32_f32_l4_lm.cl
@@ -145,3 +145,52 @@ kernel void kernel_mul_mm_f32_f32_l4_lm(
}
}
}
+
+// Multi-column f32 GEMV for the small-N (spec/MTP verify) batch. The tiled GEMM
+// above always computes a full BM x BN = 64 x 64 output tile, so at ne11=3 with a
+// skinny weight (e.g. GDN ssm_alpha/ssm_beta, M=32) it launches ONE under-occupied
+// workgroup at ~2.3% tile utilization. This kernel assigns one 64-thread workgroup
+// per output element (m,n): the 64 threads split the K reduction (float4) and
+// tree-reduce in __local (no subgroup ops -> portable). ne01*ne11 workgroups.
+// Weight row is re-read per column (N small -> negligible). Summation order differs
+// from the tiled GEMM (lane-strided + tree) -> f32-exact-ish, not bit-identical.
+kernel void kernel_gemv_f32_f32_mc(
+ global float * src0, ulong offset0, // weight: row m at m*stride_a (elements)
+ global float * src1, ulong offset1, // activations: col n at n*stride_b
+ global float * dst, ulong offsetd, // dst [M x N] col-major: (m,n) at n*stride_d+m
+ int ne00, // K
+ int ne01, // M
+ int ne11, // N
+ int stride_a, // weight row stride (elements) = K
+ int stride_b, // activation col stride (elements) = K
+ int stride_d) // dst column stride (elements) = M
+{
+ src0 = (global float*)((global char*)src0 + offset0);
+ src1 = (global float*)((global char*)src1 + offset1);
+ dst = (global float*)((global char*)dst + offsetd);
+
+ uint lane = get_local_id(0); // 0..63
+ uint out = get_global_id(1); // 0 .. ne01*ne11 - 1
+ uint m = out % (uint)ne01;
+ uint n = out / (uint)ne01;
+
+ global float4 * wrow = (global float4*)(src0 + (ulong)m * (uint)stride_a);
+ global float4 * xcol = (global float4*)(src1 + (ulong)n * (uint)stride_b);
+ uint k4 = (uint)ne00 >> 2;
+
+ float acc = 0.0f;
+ for (uint k = lane; k < k4; k += 64) {
+ float4 w = wrow[k];
+ float4 x = xcol[k];
+ acc += w.s0*x.s0 + w.s1*x.s1 + w.s2*x.s2 + w.s3*x.s3;
+ }
+
+ local float red[64];
+ red[lane] = acc;
+ barrier(CLK_LOCAL_MEM_FENCE);
+ for (uint s = 32; s > 0; s >>= 1) {
+ if (lane < s) red[lane] += red[lane + s];
+ barrier(CLK_LOCAL_MEM_FENCE);
+ }
+ if (lane == 0) dst[(ulong)n * (uint)stride_d + m] = red[0];
+}
diff --git a/ggml/src/ggml-opencl/kernels/mul_mv_f16_f32_mrow.cl b/ggml/src/ggml-opencl/kernels/mul_mv_f16_f32_mrow.cl
new file mode 100644
index 0000000000..9a7627cf9b
--- /dev/null
+++ b/ggml/src/ggml-opencl/kernels/mul_mv_f16_f32_mrow.cl
@@ -0,0 +1,306 @@
+#pragma OPENCL EXTENSION cl_khr_fp16 : enable
+
+#ifdef cl_intel_subgroups
+#pragma OPENCL EXTENSION cl_intel_subgroups : enable
+#else
+#pragma OPENCL EXTENSION cl_khr_subgroups : enable
+#endif
+
+#ifdef cl_intel_required_subgroup_size
+#pragma OPENCL EXTENSION cl_intel_required_subgroup_size : enable
+#define INTEL_GPU 1
+#define REQD_SUBGROUP_SIZE_16 __attribute__((intel_reqd_sub_group_size(16)))
+#define REQD_SUBGROUP_SIZE_32 __attribute__((intel_reqd_sub_group_size(32)))
+#elif defined(cl_qcom_reqd_sub_group_size)
+#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable
+#define ADRENO_GPU 1
+#define REQD_SUBGROUP_SIZE_64 __attribute__((qcom_reqd_sub_group_size("half")))
+#define REQD_SUBGROUP_SIZE_128 __attribute__((qcom_reqd_sub_group_size("full")))
+#endif
+
+// Multi-row f16xf32 GEMV for the DECODE path (single token, ne11*ne12 small).
+// The legacy kernel_mul_mat_f16_f32_1row runs ONE 64-lane subgroup per workgroup =
+// one output row per WG, which caps memory-level parallelism at roughly half of
+// LPDDR5x peak. This variant packs MROW subgroups per workgroup, each
+// computing a distinct output row, so a WG keeps 64*MROW loads in flight. The
+// activation column y (shared by every output row) is staged into __local ONCE per
+// WG and reused across the MROW rows, cutting redundant activation reads. Used for
+// the f16 attention projections (Q/K/V/O) and lm_head, which dominate decode.
+// Numerically equivalent to _1row (same f16->f32 widening, same float4 partial sums,
+// same subgroup-reduce order), so byte-identical to the per-op path.
+
+#define MROW 16
+
+#ifdef ADRENO_GPU
+REQD_SUBGROUP_SIZE_64
+#endif
+kernel void kernel_mul_mat_f16_f32_mrow(
+ global char * src0,
+ ulong offset0,
+ global char * src1,
+ ulong offset1,
+ global float * dst,
+ ulong offsetd,
+ int ne00,
+ int ne01,
+ int ne02,
+ ulong nb00,
+ ulong nb01,
+ ulong nb02,
+ ulong nb03,
+ int ne10,
+ int ne11,
+ int ne12,
+ ulong nb10,
+ ulong nb11,
+ ulong nb12,
+ ulong nb13,
+ int ne0,
+ int ne1,
+ int r2,
+ int r3,
+ __local float * ysh
+) {
+ src0 = (global char*)((global char*)src0 + offset0);
+ src1 = (global char*)((global char*)src1 + offset1);
+ dst = (global float*)((global char*)dst + offsetd);
+
+ int r0 = get_group_id(0) * MROW + get_local_id(1); // output row
+ int r1 = get_group_id(1); // token (ne11)
+ int im = get_group_id(2);
+ int lid = get_sub_group_local_id(); // 0..63
+ int nsg = get_local_size(1); // == MROW
+
+ int i12 = im % ne12;
+ int i13 = im / ne12;
+
+ ulong offset_src1 = r1*nb11 + (i12)*nb12 + (i13)*nb13;
+ global float * y = (global float *) (src1 + offset_src1);
+
+ // Cooperatively stage the activation column (ne00 floats) into __local once per
+ // WG and reuse across the MROW rows. Staging is the actual win here: dropping it
+ // (each subgroup re-reading y from global) regresses below the 1-row kernel.
+ for (int i = get_local_id(1)*get_sub_group_size() + lid; i < ne00; i += nsg*get_sub_group_size()) {
+ ysh[i] = y[i];
+ }
+ barrier(CLK_LOCAL_MEM_FENCE);
+
+ if (r0 >= ne01) {
+ return;
+ }
+
+ ulong offset_src0 = r0*nb01 + (i12/r2)*nb02 + (i13/r3)*nb03;
+ global half * x = (global half *) (src0 + offset_src0);
+
+ // The vector path below casts the row pointer to half4, which must be 8-byte aligned.
+ // A row address is r0*nb01 + ..., and a permuted or strided src0 leaves nb01/nb02/nb03
+ // unconstrained -- ne00 % 4 == 0 bounds the element count per row, not the byte stride
+ // between rows. Take the vector path only when this work-item's row is actually
+ // aligned; the scalar loop below has no such requirement.
+ const bool row_aligned = (((ulong) x) & 7) == 0;
+
+ float sumf = 0.0f;
+ if (ne00 < 128 || !row_aligned) {
+ for (int i = lid; i < ne00; i += get_sub_group_size()) {
+ sumf += (float) x[i] * ysh[i];
+ }
+ float all_sum = sub_group_reduce_add(sumf);
+ if (lid == 0) {
+ dst[im*ne1*ne0 + r1*ne0 + r0] = all_sum;
+ }
+ } else {
+ global half4 * x4 = (global half4 *) x;
+ __local float4 * ysh4 = (__local float4 *) ysh;
+ for (int i = lid; i < ne00/4; i += get_sub_group_size()) {
+ float4 yv = ysh4[i];
+ sumf += (float) x4[i].s0 * yv.s0;
+ sumf += (float) x4[i].s1 * yv.s1;
+ sumf += (float) x4[i].s2 * yv.s2;
+ sumf += (float) x4[i].s3 * yv.s3;
+ }
+ float all_sum = sub_group_reduce_add(sumf);
+ if (lid == 0) {
+ for (int i = 4*(ne00/4); i < ne00; ++i) {
+ all_sum += (float) x[i] * ysh[i];
+ }
+ dst[im*ne1*ne0 + r1*ne0 + r0] = all_sum;
+ }
+ }
+}
+
+// Register-blocked variant: each 64-lane subgroup accumulates RPT consecutive
+// output rows instead of one. The staged activation is reused across all RPT rows,
+// and each lane keeps RPT independent weight loads in flight per column step ->
+// more memory-level parallelism on the streaming f16 weight read (the BW limiter),
+// plus RPT fewer staging barriers per output row. Per-row reduction order is
+// identical to _mrow, so byte-identical to the per-op path. Dispatch guarantees
+// ne00 >= 128 and ne00 % 4 == 0, so only the half4 path is needed (no tail).
+#define MROW_RB_BODY(RPT) \
+ src0 = (global char*)((global char*)src0 + offset0); \
+ src1 = (global char*)((global char*)src1 + offset1); \
+ dst = (global float*)((global char*)dst + offsetd); \
+ int r0b = (get_group_id(0) * get_local_size(1) + get_local_id(1)) * (RPT); \
+ int r1 = get_group_id(1); \
+ int im = get_group_id(2); \
+ int lid = get_sub_group_local_id(); \
+ int nsg = get_local_size(1); \
+ int i12 = im % ne12; \
+ int i13 = im / ne12; \
+ ulong off_y = r1*nb11 + i12*nb12 + i13*nb13; \
+ global float * y = (global float *) (src1 + off_y); \
+ for (int i = get_local_id(1)*get_sub_group_size() + lid; i < ne00; \
+ i += nsg*get_sub_group_size()) { \
+ ysh[i] = y[i]; \
+ } \
+ barrier(CLK_LOCAL_MEM_FENCE); \
+ __local float4 * ysh4 = (__local float4 *) ysh; \
+ global half4 * xr[RPT]; \
+ _Pragma("unroll") \
+ for (int rr = 0; rr < (RPT); ++rr) { \
+ int row = r0b + rr; \
+ if (row > ne01 - 1) row = ne01 - 1; \
+ xr[rr] = (global half4 *) (src0 + (ulong)row*nb01 + (i12/r2)*nb02 + (i13/r3)*nb03); \
+ } \
+ float sumf[RPT]; \
+ _Pragma("unroll") \
+ for (int rr = 0; rr < (RPT); ++rr) sumf[rr] = 0.0f; \
+ for (int i = lid; i < ne00/4; i += get_sub_group_size()) { \
+ float4 yv = ysh4[i]; \
+ _Pragma("unroll") \
+ for (int rr = 0; rr < (RPT); ++rr) { \
+ half4 xv = xr[rr][i]; \
+ sumf[rr] += (float) xv.s0 * yv.s0 + (float) xv.s1 * yv.s1 \
+ + (float) xv.s2 * yv.s2 + (float) xv.s3 * yv.s3; \
+ } \
+ } \
+ _Pragma("unroll") \
+ for (int rr = 0; rr < (RPT); ++rr) { \
+ float s = sub_group_reduce_add(sumf[rr]); \
+ int row = r0b + rr; \
+ if (lid == 0 && row < ne01) { \
+ dst[im*ne1*ne0 + r1*ne0 + row] = s; \
+ } \
+ }
+
+// half8 (128-bit) load variant: Adreno's load/store unit issues 128-bit
+// transactions, so half4 (64-bit) loads may leave the load path half-idle. This
+// processes 8 weight elements per lane per step via half8. Accumulation groups
+// elements in 8s rather than 4s, so it is NOT bit-identical to _1row (float add is
+// non-associative) -- experimental BW probe, gate on ne00 % 8 == 0.
+#define MROW_H8_BODY(RPT) \
+ src0 = (global char*)((global char*)src0 + offset0); \
+ src1 = (global char*)((global char*)src1 + offset1); \
+ dst = (global float*)((global char*)dst + offsetd); \
+ int r0b = (get_group_id(0) * get_local_size(1) + get_local_id(1)) * (RPT); \
+ int r1 = get_group_id(1); \
+ int im = get_group_id(2); \
+ int lid = get_sub_group_local_id(); \
+ int nsg = get_local_size(1); \
+ int i12 = im % ne12; \
+ int i13 = im / ne12; \
+ ulong off_y = r1*nb11 + i12*nb12 + i13*nb13; \
+ global float * y = (global float *) (src1 + off_y); \
+ for (int i = get_local_id(1)*get_sub_group_size() + lid; i < ne00; \
+ i += nsg*get_sub_group_size()) { \
+ ysh[i] = y[i]; \
+ } \
+ barrier(CLK_LOCAL_MEM_FENCE); \
+ __local float4 * ysh4 = (__local float4 *) ysh; \
+ global half8 * xr[RPT]; \
+ _Pragma("unroll") \
+ for (int rr = 0; rr < (RPT); ++rr) { \
+ int row = r0b + rr; \
+ if (row > ne01 - 1) row = ne01 - 1; \
+ xr[rr] = (global half8 *) (src0 + (ulong)row*nb01 + (i12/r2)*nb02 + (i13/r3)*nb03); \
+ } \
+ float sumf[RPT]; \
+ _Pragma("unroll") \
+ for (int rr = 0; rr < (RPT); ++rr) sumf[rr] = 0.0f; \
+ for (int i = lid; i < ne00/8; i += get_sub_group_size()) { \
+ float4 y0 = ysh4[2*i]; \
+ float4 y1 = ysh4[2*i + 1]; \
+ _Pragma("unroll") \
+ for (int rr = 0; rr < (RPT); ++rr) { \
+ half8 xv = xr[rr][i]; \
+ sumf[rr] += (float) xv.s0 * y0.s0 + (float) xv.s1 * y0.s1 \
+ + (float) xv.s2 * y0.s2 + (float) xv.s3 * y0.s3 \
+ + (float) xv.s4 * y1.s0 + (float) xv.s5 * y1.s1 \
+ + (float) xv.s6 * y1.s2 + (float) xv.s7 * y1.s3; \
+ } \
+ } \
+ _Pragma("unroll") \
+ for (int rr = 0; rr < (RPT); ++rr) { \
+ float s = sub_group_reduce_add(sumf[rr]); \
+ int row = r0b + rr; \
+ if (lid == 0 && row < ne01) { \
+ dst[im*ne1*ne0 + r1*ne0 + row] = s; \
+ } \
+ }
+
+#ifdef ADRENO_GPU
+REQD_SUBGROUP_SIZE_64
+#endif
+kernel void kernel_mul_mat_f16_f32_mrow_h8(
+ global char * src0, ulong offset0,
+ global char * src1, ulong offset1,
+ global float * dst, ulong offsetd,
+ int ne00, int ne01, int ne02,
+ ulong nb00, ulong nb01, ulong nb02, ulong nb03,
+ int ne10, int ne11, int ne12,
+ ulong nb10, ulong nb11, ulong nb12, ulong nb13,
+ int ne0, int ne1, int r2, int r3,
+ __local float * ysh
+) {
+ MROW_H8_BODY(1)
+}
+
+#ifdef ADRENO_GPU
+REQD_SUBGROUP_SIZE_64
+#endif
+kernel void kernel_mul_mat_f16_f32_mrow_h8r2(
+ global char * src0, ulong offset0,
+ global char * src1, ulong offset1,
+ global float * dst, ulong offsetd,
+ int ne00, int ne01, int ne02,
+ ulong nb00, ulong nb01, ulong nb02, ulong nb03,
+ int ne10, int ne11, int ne12,
+ ulong nb10, ulong nb11, ulong nb12, ulong nb13,
+ int ne0, int ne1, int r2, int r3,
+ __local float * ysh
+) {
+ MROW_H8_BODY(2)
+}
+
+#ifdef ADRENO_GPU
+REQD_SUBGROUP_SIZE_64
+#endif
+kernel void kernel_mul_mat_f16_f32_mrow_r2(
+ global char * src0, ulong offset0,
+ global char * src1, ulong offset1,
+ global float * dst, ulong offsetd,
+ int ne00, int ne01, int ne02,
+ ulong nb00, ulong nb01, ulong nb02, ulong nb03,
+ int ne10, int ne11, int ne12,
+ ulong nb10, ulong nb11, ulong nb12, ulong nb13,
+ int ne0, int ne1, int r2, int r3,
+ __local float * ysh
+) {
+ MROW_RB_BODY(2)
+}
+
+#ifdef ADRENO_GPU
+REQD_SUBGROUP_SIZE_64
+#endif
+kernel void kernel_mul_mat_f16_f32_mrow_r4(
+ global char * src0, ulong offset0,
+ global char * src1, ulong offset1,
+ global float * dst, ulong offsetd,
+ int ne00, int ne01, int ne02,
+ ulong nb00, ulong nb01, ulong nb02, ulong nb03,
+ int ne10, int ne11, int ne12,
+ ulong nb10, ulong nb11, ulong nb12, ulong nb13,
+ int ne0, int ne1, int r2, int r3,
+ __local float * ysh
+) {
+ MROW_RB_BODY(4)
+}
diff --git a/ggml/src/ggml-opencl/kernels/rms_norm.cl b/ggml/src/ggml-opencl/kernels/rms_norm.cl
index 4b18d17d6f..99085625a4 100644
--- a/ggml/src/ggml-opencl/kernels/rms_norm.cl
+++ b/ggml/src/ggml-opencl/kernels/rms_norm.cl
@@ -188,3 +188,182 @@ kernel void kernel_rms_norm_mul(
y[i00] = (x[i00] * scale) * f[i00%(ne10/4)];
}
}
+
+//------------------------------------------------------------------------------
+// rms_norm + mul (norm weight) + add (residual), fused. Mirrors
+// kernel_rms_norm_mul with an extra residual operand src2: computes
+// y = (rmsnorm(x) * w) + g
+// in one dispatch, removing one kernel launch + one global round-trip per
+// residual block (the dominant per-layer adjacency on Gemma matformers).
+//------------------------------------------------------------------------------
+kernel void kernel_rms_norm_mul_add(
+ global char * src0,
+ ulong offset0,
+ global char * src1,
+ ulong offset1,
+ global char * src2,
+ ulong offset2,
+ global char * dst,
+ ulong offsetd,
+ int ne00,
+ int ne01,
+ int ne02,
+ int ne03,
+ ulong nb01,
+ ulong nb02,
+ ulong nb03,
+ int ne10,
+ int ne11,
+ int ne12,
+ int ne13,
+ ulong nb11,
+ ulong nb12,
+ ulong nb13,
+ int ne20,
+ int ne21,
+ int ne22,
+ int ne23,
+ ulong nb21,
+ ulong nb22,
+ ulong nb23,
+ ulong nb1,
+ ulong nb2,
+ ulong nb3,
+ float eps,
+ local float * sum
+) {
+ src0 = src0 + offset0;
+ src1 = src1 + offset1;
+ src2 = src2 + offset2;
+ dst = dst + offsetd;
+
+ if (get_sub_group_id() == 0) {
+ sum[get_sub_group_local_id()] = 0.0f;
+ }
+
+ int i03 = get_group_id(2);
+ int i02 = get_group_id(1);
+ int i01 = get_group_id(0);
+
+ global float4 * x = (global float4 *) (src0 + i03*nb03 + i02*nb02 + i01*nb01);
+ global float4 * f = (global float4 *) (src1 + (i03%ne13)*nb13 + (i02%ne12)*nb12 + (i01%ne11)*nb11);
+ global float4 * g = (global float4 *) (src2 + (i03%ne23)*nb23 + (i02%ne22)*nb22 + (i01%ne21)*nb21);
+
+ float sumf = 0;
+
+ for (int i00 = get_local_id(0); i00 < ne00/4; i00 += get_local_size(0)) {
+ sumf += dot(x[i00], x[i00]);
+ }
+ sumf = sub_group_reduce_add(sumf);
+
+ barrier(CLK_LOCAL_MEM_FENCE);
+
+ if (get_sub_group_local_id() == 0) {
+ sum[get_sub_group_id()] = sumf;
+ }
+
+ barrier(CLK_LOCAL_MEM_FENCE);
+
+ sumf = sum[get_sub_group_local_id()];
+ sumf = sub_group_reduce_add(sumf);
+
+ float mean = sumf / ne00;
+ float scale = 1.0f/sqrt(mean + eps);
+
+ global float4 * y = (global float4 *) (dst + i03*nb3 + i02*nb2 + i01*nb1);
+ for (int i00 = get_local_id(0); i00 < ne00/4; i00 += get_local_size(0)) {
+ y[i00] = (x[i00] * scale) * f[i00%(ne10/4)] + g[i00%(ne20/4)];
+ }
+}
+
+//------------------------------------------------------------------------------
+// rms_norm + mul(norm weight) + add(residual) + mul(scalar scale), fused.
+// Computes y = ((rmsnorm(x) * w) + g) * s, where s is a broadcast SCALAR (e.g.
+// Gemma-4 layer_output_scale). Folds the trailing per-layer l_out scale-mul into
+// the residual-norm kernel: one extra dispatch + global round-trip saved per
+// layer. src3 points at the single scale value.
+//------------------------------------------------------------------------------
+kernel void kernel_rms_norm_mul_add_scale(
+ global char * src0,
+ ulong offset0,
+ global char * src1,
+ ulong offset1,
+ global char * src2,
+ ulong offset2,
+ global char * src3,
+ ulong offset3,
+ global char * dst,
+ ulong offsetd,
+ int ne00,
+ int ne01,
+ int ne02,
+ int ne03,
+ ulong nb01,
+ ulong nb02,
+ ulong nb03,
+ int ne10,
+ int ne11,
+ int ne12,
+ int ne13,
+ ulong nb11,
+ ulong nb12,
+ ulong nb13,
+ int ne20,
+ int ne21,
+ int ne22,
+ int ne23,
+ ulong nb21,
+ ulong nb22,
+ ulong nb23,
+ ulong nb1,
+ ulong nb2,
+ ulong nb3,
+ float eps,
+ local float * sum
+) {
+ src0 = src0 + offset0;
+ src1 = src1 + offset1;
+ src2 = src2 + offset2;
+ src3 = src3 + offset3;
+ dst = dst + offsetd;
+
+ const float sc = *((global float *) src3);
+
+ if (get_sub_group_id() == 0) {
+ sum[get_sub_group_local_id()] = 0.0f;
+ }
+
+ int i03 = get_group_id(2);
+ int i02 = get_group_id(1);
+ int i01 = get_group_id(0);
+
+ global float4 * x = (global float4 *) (src0 + i03*nb03 + i02*nb02 + i01*nb01);
+ global float4 * f = (global float4 *) (src1 + (i03%ne13)*nb13 + (i02%ne12)*nb12 + (i01%ne11)*nb11);
+ global float4 * g = (global float4 *) (src2 + (i03%ne23)*nb23 + (i02%ne22)*nb22 + (i01%ne21)*nb21);
+
+ float sumf = 0;
+
+ for (int i00 = get_local_id(0); i00 < ne00/4; i00 += get_local_size(0)) {
+ sumf += dot(x[i00], x[i00]);
+ }
+ sumf = sub_group_reduce_add(sumf);
+
+ barrier(CLK_LOCAL_MEM_FENCE);
+
+ if (get_sub_group_local_id() == 0) {
+ sum[get_sub_group_id()] = sumf;
+ }
+
+ barrier(CLK_LOCAL_MEM_FENCE);
+
+ sumf = sum[get_sub_group_local_id()];
+ sumf = sub_group_reduce_add(sumf);
+
+ float mean = sumf / ne00;
+ float scale = 1.0f/sqrt(mean + eps);
+
+ global float4 * y = (global float4 *) (dst + i03*nb3 + i02*nb2 + i01*nb1);
+ for (int i00 = get_local_id(0); i00 < ne00/4; i00 += get_local_size(0)) {
+ y[i00] = ((x[i00] * scale) * f[i00%(ne10/4)] + g[i00%(ne20/4)]) * sc;
+ }
+}