LLM inference in C/C++
C++ ggml
opencl: add cooperative-k dp4a gemm for q4_0, q4_k and q6_k (#30741)
lhez1b29621bacc6
diff --git a/ggml/src/ggml-opencl/CMakeLists.txt b/ggml/src/ggml-opencl/CMakeLists.txtindex 5ba63dade1..84087162d6 100644--- a/ggml/src/ggml-opencl/CMakeLists.txt+++ b/ggml/src/ggml-opencl/CMakeLists.txt@@ -140,6 +140,9 @@ set(GGML_OPENCL_KERNELSgemm_noshuffle_q5_0_q8_1_dp4agemm_noshuffle_iq4_nl_q8_1_dp4agemm_noshuffle_q4_0_q8_1_dp4a+ gemm_coopk_q4_0_q8_1_dp4a+ gemm_coopk_q4_k_q8_1_dp4a+ gemm_coopk_q6_k_q8_1_dp4aquant_a_q8_1gemv_moe_q4_k_f32_nsgemm_moe_q5_k_f32_nsdiff --git a/ggml/src/ggml-opencl/ggml-opencl.cpp b/ggml/src/ggml-opencl/ggml-opencl.cppindex 96461bcd94..ad65de00ad 100644--- a/ggml/src/ggml-opencl/ggml-opencl.cpp+++ b/ggml/src/ggml-opencl/ggml-opencl.cpp@@ -1296,6 +1296,22 @@ struct ggml_backend_opencl_context {cl_kernel kernel_gemm_noshuffle_iq4_nl_f32;cl_kernel kernel_gemm_noshuffle_iq4_nl_q8_1_dp4a = nullptr; // dp4a (int8) dense IQ4_NL prefill GEMMcl_kernel kernel_gemm_noshuffle_q4_0_q8_1_dp4a = nullptr; // dp4a (int8) dense q4_0 prefill GEMM+ cl_kernel kernel_gemm_coopk_q4_0_q8_1_dp4a_c4 = nullptr;+ cl_kernel kernel_gemm_coopk_q4_0_q8_1_dp4a_c2 = nullptr;+ int nsg_coopk_dp4a_q4_0 = 8;+ int rows_coopk_dp4a_q4_0 = 4;+ cl_kernel kernel_gemm_coopk_q4_k_q8_1_dp4a_c4 = nullptr;+ cl_kernel kernel_gemm_coopk_q4_k_q8_1_dp4a_c2 = nullptr;+ int nsg_coopk_dp4a_q4_k = 8;+ int rows_coopk_dp4a_q4_k = 4;+ cl_kernel kernel_gemm_coopk_q4_k_q8_1_dp4a_bin_c4 = nullptr;+ cl_kernel kernel_gemm_coopk_q4_k_q8_1_dp4a_bin_c2 = nullptr;+ int nsg_coopk_dp4a_q4_k_bin_c4 = 8;+ int nsg_coopk_dp4a_q4_k_bin_c2 = 8;+ cl_kernel kernel_gemm_coopk_q6_k_q8_1_dp4a_c4 = nullptr;+ cl_kernel kernel_gemm_coopk_q6_k_q8_1_dp4a_c2 = nullptr;+ int nsg_coopk_dp4a_q6_k = 8;+ int rows_coopk_dp4a_q6_k = 4;#endif // GGML_OPENCL_USE_ADRENO_KERNELSvoid free() {@@ -1492,6 +1508,180 @@ static bool use_adreno_bin_kernels(ggml_backend_opencl_context * backend_ctx) {#endif // GGML_OPENCL_USE_ADRENO_BIN_KERNELS}+#ifdef GGML_OPENCL_USE_ADRENO_KERNELS+static bool ggml_cl_coopk_dp4a_enabled(const ggml_backend_opencl_context * backend_ctx) {+ static const char * const env = getenv("GGML_OPENCL_COOPK_DP4A");+ if (env && *env && atoi(env) == 0) {+ return false;+ }+ if (!backend_ctx->has_integer_dot) {+ return false;+ }+ if (backend_ctx->gpu_family == GPU_FAMILY::ADRENO) {+ if (backend_ctx->adreno_gen == ADRENO_GPU_GEN::ADRENO_UNKNOWN ||+ backend_ctx->adreno_gen == ADRENO_GPU_GEN::A6X ||+ backend_ctx->adreno_gen == ADRENO_GPU_GEN::A7X) {+ return false;+ }+ if (backend_ctx->adreno_cl_compiler_version.type == ADRENO_CL_COMPILER_TYPE::E17) {+ return false;+ }+ }+ return true;+}++static bool ggml_cl_coopk_dp4a_narrow_on(const ggml_backend_opencl_context * backend_ctx,+ const ggml_tensor * op, int rows) {+ if (!ggml_cl_coopk_dp4a_enabled(backend_ctx)) {+ return false;+ }+ // Each lane handles rows output rows; subgroups split K for the same rows.+ // Full row groups and 32-value K blocks avoid tail checks in the kernels.+ return op->ne[1] >= 2 && op->ne[1] <= 4+ && rows > 0 && (op->src[0]->ne[1] % (64 * rows)) == 0+ && (op->src[0]->ne[0] % 32) == 0;+}++// COOPK_NSG sizes the local reduction buffer, so a smaller workgroup requires a rebuild.+static cl_program ggml_cl_build_program_coopk(ggml_backend_opencl_context * backend_ctx,+ const char * kernel_src, const char * kernel_name,+ const std::string & opts_base, int & nsg) {+ while (nsg >= 2) {+ const std::string opts = opts_base + " -DCOOPK_NSG=" + std::to_string(nsg);+ cl_program program = build_program_from_source(backend_ctx, kernel_src, opts);+ cl_int err;+ cl_kernel kernel = clCreateKernel(program, kernel_name, &err);+ CL_CHECK(err);+ size_t max_workgroup_size = 0;+ CL_CHECK(clGetKernelWorkGroupInfo(kernel, backend_ctx->device, CL_KERNEL_WORK_GROUP_SIZE,+ sizeof(max_workgroup_size), &max_workgroup_size, NULL));+ CL_CHECK(clReleaseKernel(kernel));+ if ((size_t) 64 * nsg <= max_workgroup_size) {+ return program;+ }+ CL_CHECK(clReleaseProgram(program));+ nsg /= 2;+ }+ return nullptr;+}++static void ggml_cl_coopk_build_narrow(ggml_backend_opencl_context * backend_ctx,+ const std::string & kernel_src, const char * kernel_name,+ int rows, int & nsg, cl_kernel & kernel_c4, cl_kernel & kernel_c2) {+ const std::string base_opts = backend_ctx->kernel_compile_opts+ + " -DCOOPK_ROWS=" + std::to_string(rows);+ for (int cols = 4; cols >= 2; cols -= 2) {+ int nsg_eff = nsg;+ cl_program program = ggml_cl_build_program_coopk(+ backend_ctx, kernel_src.c_str(), kernel_name,+ base_opts + " -DCOOPK_COLS=" + std::to_string(cols), nsg_eff);+ if (program == nullptr) {+ return;+ }+ cl_int err;+ cl_kernel kernel = clCreateKernel(program, kernel_name, &err);+ if (err != CL_SUCCESS) { kernel = nullptr; }+ CL_CHECK(clReleaseProgram(program));+ if (cols == 4) {+ kernel_c4 = kernel;+ nsg = nsg_eff;+ if (kernel == nullptr) {+ return;+ }+ } else {+ // Both widths share one stored subgroup count for launch.+ if (kernel != nullptr && nsg_eff != nsg) {+ CL_CHECK(clReleaseKernel(kernel));+ kernel = nullptr;+ }+ kernel_c2 = kernel;+ }+ }+}++static void load_cl_kernels_gemm_coopk(ggml_backend_opencl_context * backend_ctx) {+ if (backend_ctx->gpu_family != GPU_FAMILY::ADRENO || !ggml_cl_coopk_dp4a_enabled(backend_ctx)) {+ return;+ }++ // Q4_0+ {+#ifdef GGML_OPENCL_EMBED_KERNELS+ const std::string kernel_src {+ #include "gemm_coopk_q4_0_q8_1_dp4a.cl.h"+ };+#else+ const std::string kernel_src = read_file("gemm_coopk_q4_0_q8_1_dp4a.cl");+#endif+ ggml_cl_coopk_build_narrow(backend_ctx, kernel_src, "kernel_gemm_coopk_q4_0_q8_1_dp4a",+ backend_ctx->rows_coopk_dp4a_q4_0, backend_ctx->nsg_coopk_dp4a_q4_0,+ backend_ctx->kernel_gemm_coopk_q4_0_q8_1_dp4a_c4,+ backend_ctx->kernel_gemm_coopk_q4_0_q8_1_dp4a_c2);+ GGML_LOG_CONT(".");+ }++ // Q4_K+ {+#ifdef GGML_OPENCL_EMBED_KERNELS+ const std::string kernel_src {+ #include "gemm_coopk_q4_k_q8_1_dp4a.cl.h"+ };+#else+ const std::string kernel_src = read_file("gemm_coopk_q4_k_q8_1_dp4a.cl");+#endif+ ggml_cl_coopk_build_narrow(backend_ctx, kernel_src, "kernel_gemm_coopk_q4_k_q8_1_dp4a",+ backend_ctx->rows_coopk_dp4a_q4_k, backend_ctx->nsg_coopk_dp4a_q4_k,+ backend_ctx->kernel_gemm_coopk_q4_k_q8_1_dp4a_c4,+ backend_ctx->kernel_gemm_coopk_q4_k_q8_1_dp4a_c2);++ const char * env = getenv("GGML_OPENCL_COOPK_Q4_K_BIN");+ if (use_adreno_bin_kernels(backend_ctx) && (!env || !*env || atoi(env) != 0)) {+ const std::string base_opts = backend_ctx->kernel_compile_opts+ + " -DCOOPK_Q_BIN=1 -DCOOPK_ROWS=" + std::to_string(backend_ctx->rows_coopk_dp4a_q4_k);+ for (int cols = 4; cols >= 2; cols -= 2) {+ int & nsg = (cols == 4) ? backend_ctx->nsg_coopk_dp4a_q4_k_bin_c4+ : backend_ctx->nsg_coopk_dp4a_q4_k_bin_c2;+ int nsg_eff = nsg;+ cl_program program = ggml_cl_build_program_coopk(+ backend_ctx, kernel_src.c_str(), "kernel_gemm_coopk_q4_k_q8_1_dp4a",+ base_opts + " -DCOOPK_COLS=" + std::to_string(cols), nsg_eff);+ if (program == nullptr) {+ continue;+ }+ cl_int err;+ cl_kernel kernel = clCreateKernel(program, "kernel_gemm_coopk_q4_k_q8_1_dp4a", &err);+ if (err != CL_SUCCESS) { kernel = nullptr; }+ CL_CHECK(clReleaseProgram(program));+ if (cols == 4) {+ backend_ctx->kernel_gemm_coopk_q4_k_q8_1_dp4a_bin_c4 = kernel;+ } else {+ backend_ctx->kernel_gemm_coopk_q4_k_q8_1_dp4a_bin_c2 = kernel;+ }+ nsg = nsg_eff;+ }+ }+ GGML_LOG_CONT(".");+ }++ // Q6_K+ {+#ifdef GGML_OPENCL_EMBED_KERNELS+ const std::string kernel_src {+ #include "gemm_coopk_q6_k_q8_1_dp4a.cl.h"+ };+#else+ const std::string kernel_src = read_file("gemm_coopk_q6_k_q8_1_dp4a.cl");+#endif+ ggml_cl_coopk_build_narrow(backend_ctx, kernel_src, "kernel_gemm_coopk_q6_k_q8_1_dp4a",+ backend_ctx->rows_coopk_dp4a_q6_k, backend_ctx->nsg_coopk_dp4a_q6_k,+ backend_ctx->kernel_gemm_coopk_q6_k_q8_1_dp4a_c4,+ backend_ctx->kernel_gemm_coopk_q6_k_q8_1_dp4a_c2);+ GGML_LOG_CONT(".");+ }+}++#endif+static std::string ggml_opencl_make_compile_opts(const ggml_backend_opencl_context * backend_ctx) {auto opencl_c_std =std::string("CL") + std::to_string(backend_ctx->opencl_c_version.major) + "." + std::to_string(backend_ctx->opencl_c_version.minor);@@ -5440,6 +5630,7 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) {}}}+ load_cl_kernels_gemm_coopk(backend_ctx);#endif // GGML_OPENCL_USE_ADRENO_KERNELSGGML_LOG_CONT("\n");backend_ctx->kernels_loaded = true;@@ -20052,6 +20243,7 @@ static void ggml_cl_mul_mat_q4_0_f32_adreno(ggml_backend_t backend, const ggml_twarned = true;}}+ // This bin layout is not supported by CoopK.ggml_cl_mul_mat_q4_0_f32_adreno_ila(backend, src0, src1, dst);return;}@@ -20139,6 +20331,71 @@ static void ggml_cl_mul_mat_q4_0_f32_adreno(ggml_backend_t backend, const ggml_tCL_CHECK(clReleaseMemObject(b_sub_buf));CL_CHECK(clReleaseMemObject(b_img));} else {+ // CoopK uses the same untiled weight layout as gemm_noshuffle_q4_0_f32, with K-major Q8_1 activations (K contiguous).+ if (ggml_cl_coopk_dp4a_narrow_on(backend_ctx, dst, backend_ctx->rows_coopk_dp4a_q4_0)+ && backend_ctx->kernel_gemm_coopk_q4_0_q8_1_dp4a_c4 != nullptr+ && ((N <= 2) ? backend_ctx->kernel_gemm_coopk_q4_0_q8_1_dp4a_c2+ : backend_ctx->kernel_gemm_coopk_q4_0_q8_1_dp4a_c4) != nullptr) {+ cl_mem b_sub_buf = nullptr;+ region.origin = offset1;+ region.size = (size_t)K * N * sizeof(float);+ CL_CHECK((b_sub_buf = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err));++ const int cols = (N <= 2) ? 2 : 4;+ const size_t n_blocks_padded = (size_t)cols * (K / 32);+ const size_t size_qa = (size_t)cols * K * sizeof(cl_char);+ const size_t size_qa_prev = backend_ctx->prealloc_moe_qa.size;+ const size_t size_da_prev = backend_ctx->prealloc_moe_da.size;+ backend_ctx->prealloc_moe_qa.allocate(context, size_qa);+ backend_ctx->prealloc_moe_da.allocate(context, n_blocks_padded * sizeof(cl_half));+ backend_ctx->prealloc_moe_sa.allocate(context, n_blocks_padded * sizeof(cl_half));+ if (backend_ctx->prealloc_moe_qa.size != size_qa_prev) {+ const cl_char z8 = 0;+ CL_CHECK(clEnqueueFillBuffer(backend_ctx->queue, backend_ctx->prealloc_moe_qa.buffer,+ &z8, sizeof(z8), 0, backend_ctx->prealloc_moe_qa.size, 0, NULL, NULL));+ }+ if (backend_ctx->prealloc_moe_da.size != size_da_prev) {+ const cl_half z16 = 0;+ CL_CHECK(clEnqueueFillBuffer(backend_ctx->queue, backend_ctx->prealloc_moe_da.buffer,+ &z16, sizeof(z16), 0, backend_ctx->prealloc_moe_da.size, 0, NULL, NULL));+ CL_CHECK(clEnqueueFillBuffer(backend_ctx->queue, backend_ctx->prealloc_moe_sa.buffer,+ &z16, sizeof(z16), 0, backend_ctx->prealloc_moe_sa.size, 0, NULL, NULL));+ }++ cl_int n_blocks = (cl_int)((size_t)N * (K / 32));+ cl_kernel kernel_quant = backend_ctx->kernel_quant_a_q8_1;+ CL_CHECK(clSetKernelArg(kernel_quant, 0, sizeof(cl_mem), &b_sub_buf));+ CL_CHECK(clSetKernelArg(kernel_quant, 1, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer));+ CL_CHECK(clSetKernelArg(kernel_quant, 2, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer));+ CL_CHECK(clSetKernelArg(kernel_quant, 3, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer));+ CL_CHECK(clSetKernelArg(kernel_quant, 4, sizeof(cl_int), &n_blocks));+ size_t local_work_size_quant[1] = { 64 };+ size_t global_work_size_quant[1] = { (size_t)((((size_t)n_blocks + 63) / 64) * 64) };+ backend_ctx->enqueue_ndrange_kernel(kernel_quant, 1, global_work_size_quant, local_work_size_quant, dst);++ cl_kernel kernel_coopk = (N <= 2) ? backend_ctx->kernel_gemm_coopk_q4_0_q8_1_dp4a_c2+ : backend_ctx->kernel_gemm_coopk_q4_0_q8_1_dp4a_c4;+ const cl_int n_padded = cols;+ CL_CHECK(clSetKernelArg(kernel_coopk, 0, sizeof(cl_mem), &extra0_q4_0->q));+ CL_CHECK(clSetKernelArg(kernel_coopk, 1, sizeof(cl_mem), &extra0_q4_0->d));+ CL_CHECK(clSetKernelArg(kernel_coopk, 2, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer));+ CL_CHECK(clSetKernelArg(kernel_coopk, 3, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer));+ CL_CHECK(clSetKernelArg(kernel_coopk, 4, sizeof(cl_mem), &extrad->data_device));+ CL_CHECK(clSetKernelArg(kernel_coopk, 5, sizeof(cl_ulong), &offsetd));+ CL_CHECK(clSetKernelArg(kernel_coopk, 6, sizeof(cl_int), &M));+ CL_CHECK(clSetKernelArg(kernel_coopk, 7, sizeof(cl_int), &n_padded));+ CL_CHECK(clSetKernelArg(kernel_coopk, 8, sizeof(cl_int), &K));+ CL_CHECK(clSetKernelArg(kernel_coopk, 9, sizeof(cl_int), &N));++ const size_t nsg = (size_t)backend_ctx->nsg_coopk_dp4a_q4_0;+ size_t local_work_size[3] = { 64, nsg, 1 };+ size_t global_work_size[3] = { (size_t)(M / backend_ctx->rows_coopk_dp4a_q4_0), nsg, 1 };+ backend_ctx->enqueue_ndrange_kernel(kernel_coopk, 3, global_work_size, local_work_size, dst);++ CL_CHECK(clReleaseMemObject(b_sub_buf));+ return;+ }+// dp4a (int8) dense prefill GEMM, default offstatic const char * q4_0_dense_dp4a_env = getenv("GGML_OPENCL_Q4_0_DENSE_DP4A");bool q4_0_dense_dp4a_on = q4_0_dense_dp4a_env@@ -21711,6 +21968,86 @@ static void ggml_cl_mul_mat_q4_k_f32_adreno_ila(ggml_backend_t backend, const ggint N = ne1;int K = ne00;+ // CoopK handles with _bin use the bin weight layout (uint-transposed q), with K-major Q8_1 activations (K contiguous).+ // This switch disables only this bin-layout path; weight conversion is unchanged.+ static const char * env_coopk_bin = getenv("GGML_OPENCL_COOPK_Q4_K_BIN");+ const bool use_coopk_bin = !env_coopk_bin || !*env_coopk_bin || atoi(env_coopk_bin) != 0;+ if (use_coopk_bin && ne1 >= 2 && ne1 <= 4 &&+ ggml_cl_coopk_dp4a_narrow_on(backend_ctx, dst, backend_ctx->rows_coopk_dp4a_q4_k) &&+ backend_ctx->kernel_gemm_coopk_q4_k_q8_1_dp4a_bin_c4 != nullptr &&+ ((ne1 <= 2) ? backend_ctx->kernel_gemm_coopk_q4_k_q8_1_dp4a_bin_c2+ : backend_ctx->kernel_gemm_coopk_q4_k_q8_1_dp4a_bin_c4) != nullptr) {+ cl_mem b_sub_buf = nullptr;+ region.origin = offset1;+ region.size = (size_t)K * N * sizeof(float);+ CL_CHECK((b_sub_buf = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err));++ // Allocate all kernel columns; padded columns have independent accumulators and are not stored.+ const int cols = (ne1 <= 2) ? 2 : 4;+ const size_t size_qa = (size_t)cols * K * sizeof(cl_char);+ const size_t n_blocks_padded = (size_t)cols * (K / 32);+ const size_t size_qa_prev = backend_ctx->prealloc_moe_qa.size;+ const size_t size_da_prev = backend_ctx->prealloc_moe_da.size;+ backend_ctx->prealloc_moe_qa.allocate(context, size_qa);+ backend_ctx->prealloc_moe_da.allocate(context, n_blocks_padded * sizeof(cl_half));+ backend_ctx->prealloc_moe_sa.allocate(context, n_blocks_padded * sizeof(cl_half));+ if (backend_ctx->prealloc_moe_qa.size != size_qa_prev) {+ const cl_char z8 = 0;+ CL_CHECK(clEnqueueFillBuffer(backend_ctx->queue, backend_ctx->prealloc_moe_qa.buffer,+ &z8, sizeof(z8), 0, backend_ctx->prealloc_moe_qa.size, 0, NULL, NULL));+ }+ if (backend_ctx->prealloc_moe_da.size != size_da_prev) {+ const cl_half z16 = 0;+ CL_CHECK(clEnqueueFillBuffer(backend_ctx->queue, backend_ctx->prealloc_moe_da.buffer,+ &z16, sizeof(z16), 0, backend_ctx->prealloc_moe_da.size, 0, NULL, NULL));+ CL_CHECK(clEnqueueFillBuffer(backend_ctx->queue, backend_ctx->prealloc_moe_sa.buffer,+ &z16, sizeof(z16), 0, backend_ctx->prealloc_moe_sa.size, 0, NULL, NULL));+ }++ cl_int n_blocks = (cl_int)((size_t)N * (K / 32));+ cl_kernel kernel_quant = backend_ctx->kernel_quant_a_q8_1;+ CL_CHECK(clSetKernelArg(kernel_quant, 0, sizeof(cl_mem), &b_sub_buf));+ CL_CHECK(clSetKernelArg(kernel_quant, 1, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer));+ CL_CHECK(clSetKernelArg(kernel_quant, 2, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer));+ CL_CHECK(clSetKernelArg(kernel_quant, 3, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer));+ CL_CHECK(clSetKernelArg(kernel_quant, 4, sizeof(cl_int), &n_blocks));+ size_t local_work_size_quant[1] = { 64 };+ size_t global_work_size_quant[1] = { (size_t)((((size_t)n_blocks + 63) / 64) * 64) };+ backend_ctx->enqueue_ndrange_kernel(kernel_quant, 1, global_work_size_quant, local_work_size_quant, dst);++ cl_kernel kernel_coopk = (ne1 <= 2) ? backend_ctx->kernel_gemm_coopk_q4_k_q8_1_dp4a_bin_c2+ : backend_ctx->kernel_gemm_coopk_q4_k_q8_1_dp4a_bin_c4;+ const int nsg_selected = (ne1 <= 2) ? backend_ctx->nsg_coopk_dp4a_q4_k_bin_c2+ : backend_ctx->nsg_coopk_dp4a_q4_k_bin_c4;+ const cl_uchar mask_d6 = 0x3F, mask_d4 = 0x0F, mask_hi2 = 0xC0;+ const cl_int n_padded = cols;+ CL_CHECK(clSetKernelArg(kernel_coopk, 0, sizeof(cl_mem), &extra0_q4_k->q));+ CL_CHECK(clSetKernelArg(kernel_coopk, 1, sizeof(cl_mem), &extra0_q4_k->s));+ CL_CHECK(clSetKernelArg(kernel_coopk, 2, sizeof(cl_mem), &extra0_q4_k->d));+ CL_CHECK(clSetKernelArg(kernel_coopk, 3, sizeof(cl_mem), &extra0_q4_k->dm));+ CL_CHECK(clSetKernelArg(kernel_coopk, 4, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer));+ CL_CHECK(clSetKernelArg(kernel_coopk, 5, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer));+ CL_CHECK(clSetKernelArg(kernel_coopk, 6, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer));+ CL_CHECK(clSetKernelArg(kernel_coopk, 7, sizeof(cl_mem), &extrad->data_device));+ CL_CHECK(clSetKernelArg(kernel_coopk, 8, sizeof(cl_ulong), &offsetd));+ CL_CHECK(clSetKernelArg(kernel_coopk, 9, sizeof(cl_int), &ne01));+ CL_CHECK(clSetKernelArg(kernel_coopk, 10, sizeof(cl_int), &n_padded));+ CL_CHECK(clSetKernelArg(kernel_coopk, 11, sizeof(cl_int), &ne00));+ CL_CHECK(clSetKernelArg(kernel_coopk, 12, sizeof(cl_int), &ne1));+ CL_CHECK(clSetKernelArg(kernel_coopk, 13, sizeof(cl_uchar), &mask_d6));+ CL_CHECK(clSetKernelArg(kernel_coopk, 14, sizeof(cl_uchar), &mask_d4));+ CL_CHECK(clSetKernelArg(kernel_coopk, 15, sizeof(cl_uchar), &mask_hi2));++ // Each width can compile with a different subgroup count.+ const size_t nsg = (size_t)nsg_selected;+ size_t local_work_size[3] = { 64, nsg, 1 };+ size_t global_work_size[3] = { (size_t)(ne01 / backend_ctx->rows_coopk_dp4a_q4_k), nsg, 1 };+ backend_ctx->enqueue_ndrange_kernel(kernel_coopk, 3, global_work_size, local_work_size, dst);++ CL_CHECK(clReleaseMemObject(b_sub_buf));+ return;+ }+if (ne1 == 1) {cl_mem b_sub_buf = nullptr;cl_mem b_img = nullptr;@@ -22158,6 +22495,74 @@ static void ggml_cl_mul_mat_q4_k_f32_adreno(ggml_backend_t backend, const ggml_tregion.size = K * N * sizeof(float);CL_CHECK((b_sub_buf = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err));+ // CoopK handles without _bin use the same untiled weight layout as gemm_noshuffle_q4_k_f32.+ // Activations use K-major Q8_1 SoA (K contiguous).+ if (ggml_cl_coopk_dp4a_narrow_on(backend_ctx, dst, backend_ctx->rows_coopk_dp4a_q4_k)+ && backend_ctx->kernel_gemm_coopk_q4_k_q8_1_dp4a_c4 != nullptr+ && ((N <= 2) ? backend_ctx->kernel_gemm_coopk_q4_k_q8_1_dp4a_c2+ : backend_ctx->kernel_gemm_coopk_q4_k_q8_1_dp4a_c4) != nullptr) {+ const int cols = (N <= 2) ? 2 : 4;+ const size_t size_qa = (size_t)cols * K * sizeof(cl_char);+ const size_t n_blocks_padded = (size_t)cols * (K / 32);+ const size_t size_qa_prev = backend_ctx->prealloc_moe_qa.size;+ const size_t size_da_prev = backend_ctx->prealloc_moe_da.size;+ backend_ctx->prealloc_moe_qa.allocate(context, size_qa);+ backend_ctx->prealloc_moe_da.allocate(context, n_blocks_padded * sizeof(cl_half));+ backend_ctx->prealloc_moe_sa.allocate(context, n_blocks_padded * sizeof(cl_half));+ if (backend_ctx->prealloc_moe_qa.size != size_qa_prev) {+ const cl_char z8 = 0;+ CL_CHECK(clEnqueueFillBuffer(backend_ctx->queue, backend_ctx->prealloc_moe_qa.buffer,+ &z8, sizeof(z8), 0, backend_ctx->prealloc_moe_qa.size, 0, NULL, NULL));+ }+ if (backend_ctx->prealloc_moe_da.size != size_da_prev) {+ const cl_half z16 = 0;+ CL_CHECK(clEnqueueFillBuffer(backend_ctx->queue, backend_ctx->prealloc_moe_da.buffer,+ &z16, sizeof(z16), 0, backend_ctx->prealloc_moe_da.size, 0, NULL, NULL));+ CL_CHECK(clEnqueueFillBuffer(backend_ctx->queue, backend_ctx->prealloc_moe_sa.buffer,+ &z16, sizeof(z16), 0, backend_ctx->prealloc_moe_sa.size, 0, NULL, NULL));+ }++ cl_int n_blocks = (cl_int)((size_t)N * (K / 32));+ cl_kernel kernel_quant = backend_ctx->kernel_quant_a_q8_1;+ CL_CHECK(clSetKernelArg(kernel_quant, 0, sizeof(cl_mem), &b_sub_buf));+ CL_CHECK(clSetKernelArg(kernel_quant, 1, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer));+ CL_CHECK(clSetKernelArg(kernel_quant, 2, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer));+ CL_CHECK(clSetKernelArg(kernel_quant, 3, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer));+ CL_CHECK(clSetKernelArg(kernel_quant, 4, sizeof(cl_int), &n_blocks));+ size_t local_work_size_quant[1] = { 64 };+ size_t global_work_size_quant[1] = { (size_t)((((size_t)n_blocks + 63) / 64) * 64) };+ backend_ctx->enqueue_ndrange_kernel(kernel_quant, 1, global_work_size_quant, local_work_size_quant, dst);++ cl_kernel kernel_coopk = (N <= 2) ? backend_ctx->kernel_gemm_coopk_q4_k_q8_1_dp4a_c2+ : backend_ctx->kernel_gemm_coopk_q4_k_q8_1_dp4a_c4;+ const cl_uchar mask_d6 = 0x3F, mask_d4 = 0x0F, mask_hi2 = 0xC0;+ const cl_int n_padded = cols;+ CL_CHECK(clSetKernelArg(kernel_coopk, 0, sizeof(cl_mem), &extra0_q4_k->q));+ CL_CHECK(clSetKernelArg(kernel_coopk, 1, sizeof(cl_mem), &extra0_q4_k->s));+ CL_CHECK(clSetKernelArg(kernel_coopk, 2, sizeof(cl_mem), &extra0_q4_k->d));+ CL_CHECK(clSetKernelArg(kernel_coopk, 3, sizeof(cl_mem), &extra0_q4_k->dm));+ CL_CHECK(clSetKernelArg(kernel_coopk, 4, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer));+ CL_CHECK(clSetKernelArg(kernel_coopk, 5, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer));+ CL_CHECK(clSetKernelArg(kernel_coopk, 6, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer));+ CL_CHECK(clSetKernelArg(kernel_coopk, 7, sizeof(cl_mem), &extrad->data_device));+ CL_CHECK(clSetKernelArg(kernel_coopk, 8, sizeof(cl_ulong), &offsetd));+ CL_CHECK(clSetKernelArg(kernel_coopk, 9, sizeof(cl_int), &M));+ CL_CHECK(clSetKernelArg(kernel_coopk, 10, sizeof(cl_int), &n_padded));+ CL_CHECK(clSetKernelArg(kernel_coopk, 11, sizeof(cl_int), &K));+ CL_CHECK(clSetKernelArg(kernel_coopk, 12, sizeof(cl_int), &N));+ CL_CHECK(clSetKernelArg(kernel_coopk, 13, sizeof(cl_uchar), &mask_d6));+ CL_CHECK(clSetKernelArg(kernel_coopk, 14, sizeof(cl_uchar), &mask_d4));+ CL_CHECK(clSetKernelArg(kernel_coopk, 15, sizeof(cl_uchar), &mask_hi2));++ const size_t nsg = (size_t)backend_ctx->nsg_coopk_dp4a_q4_k;+ size_t local_work_size[3] = { 64, nsg, 1 };+ size_t global_work_size[3] = { (size_t)(M / backend_ctx->rows_coopk_dp4a_q4_k), nsg, 1 };+ backend_ctx->enqueue_ndrange_kernel(kernel_coopk, 3, global_work_size, local_work_size, dst);++ CL_CHECK(clReleaseMemObject(b_sub_buf));+ return;+ }+// image for activationsimg_fmt = {CL_RGBA, CL_FLOAT};memset(&img_desc, 0, sizeof(img_desc));@@ -22672,6 +23077,7 @@ static void ggml_cl_mul_mat_q6_K_f32_adreno(ggml_backend_t backend, const ggml_twarned = true;}}+ // This bin layout is not supported by CoopK.ggml_cl_mul_mat_q6_K_f32_adreno_ila(backend, src0, src1, dst);return;}@@ -22827,6 +23233,77 @@ static void ggml_cl_mul_mat_q6_K_f32_adreno(ggml_backend_t backend, const ggml_tregion.size = ne00 * ne1 * sizeof(float);CL_CHECK((b_sub_buf = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err));+ // CoopK uses the same untiled weight layout as gemm_noshuffle_q6_K_f32, with K-major Q8_1 activations (K contiguous).+ // Keep output and embedding weights on the existing path for logit stability.+ if (ggml_cl_coopk_dp4a_narrow_on(backend_ctx, dst, backend_ctx->rows_coopk_dp4a_q6_k)+ && backend_ctx->kernel_gemm_coopk_q6_k_q8_1_dp4a_c4 != nullptr+ && strncmp(src0->name, "output", 6) != 0+ && strncmp(src0->name, "token_embd", 10) != 0+ && ((ne1 <= 2) ? backend_ctx->kernel_gemm_coopk_q6_k_q8_1_dp4a_c2+ : backend_ctx->kernel_gemm_coopk_q6_k_q8_1_dp4a_c4) != nullptr) {+ const int N = (int)ne1, K = (int)ne00;+ const int cols = (N <= 2) ? 2 : 4;+ const size_t n_blocks_padded = (size_t)cols * (K / 32);+ const size_t size_qa = (size_t)cols * K * sizeof(cl_char);+ const size_t size_qa_prev = backend_ctx->prealloc_moe_qa.size;+ const size_t size_da_prev = backend_ctx->prealloc_moe_da.size;+ backend_ctx->prealloc_moe_qa.allocate(context, size_qa);+ backend_ctx->prealloc_moe_da.allocate(context, n_blocks_padded * sizeof(cl_half));+ backend_ctx->prealloc_moe_sa.allocate(context, n_blocks_padded * sizeof(cl_half));+ if (backend_ctx->prealloc_moe_qa.size != size_qa_prev) {+ const cl_char z8 = 0;+ CL_CHECK(clEnqueueFillBuffer(backend_ctx->queue, backend_ctx->prealloc_moe_qa.buffer,+ &z8, sizeof(z8), 0, backend_ctx->prealloc_moe_qa.size, 0, NULL, NULL));+ }+ if (backend_ctx->prealloc_moe_da.size != size_da_prev) {+ const cl_half z16 = 0;+ CL_CHECK(clEnqueueFillBuffer(backend_ctx->queue, backend_ctx->prealloc_moe_da.buffer,+ &z16, sizeof(z16), 0, backend_ctx->prealloc_moe_da.size, 0, NULL, NULL));+ CL_CHECK(clEnqueueFillBuffer(backend_ctx->queue, backend_ctx->prealloc_moe_sa.buffer,+ &z16, sizeof(z16), 0, backend_ctx->prealloc_moe_sa.size, 0, NULL, NULL));+ }++ cl_int n_blocks = (cl_int)((size_t)N * (K / 32));+ cl_kernel kernel_quant = backend_ctx->kernel_quant_a_q8_1;+ CL_CHECK(clSetKernelArg(kernel_quant, 0, sizeof(cl_mem), &b_sub_buf));+ CL_CHECK(clSetKernelArg(kernel_quant, 1, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer));+ CL_CHECK(clSetKernelArg(kernel_quant, 2, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer));+ CL_CHECK(clSetKernelArg(kernel_quant, 3, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer));+ CL_CHECK(clSetKernelArg(kernel_quant, 4, sizeof(cl_int), &n_blocks));+ size_t local_work_size_quant[1] = { 64 };+ size_t global_work_size_quant[1] = { (size_t)((((size_t)n_blocks + 63) / 64) * 64) };+ backend_ctx->enqueue_ndrange_kernel(kernel_quant, 1, global_work_size_quant, local_work_size_quant, dst);++ cl_kernel kernel_coopk = (N <= 2) ? backend_ctx->kernel_gemm_coopk_q6_k_q8_1_dp4a_c2+ : backend_ctx->kernel_gemm_coopk_q6_k_q8_1_dp4a_c4;+ const cl_ushort mask_f000 = 0xF000;+ const cl_uchar mask_c0 = 0xC0;+ const cl_int n_padded = cols;+ const cl_int m = (cl_int)ne01, k = (cl_int)ne00, n = (cl_int)ne1;+ CL_CHECK(clSetKernelArg(kernel_coopk, 0, sizeof(cl_mem), &extra0_q6_K->ql));+ CL_CHECK(clSetKernelArg(kernel_coopk, 1, sizeof(cl_mem), &extra0_q6_K->qh));+ CL_CHECK(clSetKernelArg(kernel_coopk, 2, sizeof(cl_mem), &extra0_q6_K->s));+ CL_CHECK(clSetKernelArg(kernel_coopk, 3, sizeof(cl_mem), &extra0_q6_K->d));+ CL_CHECK(clSetKernelArg(kernel_coopk, 4, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer));+ CL_CHECK(clSetKernelArg(kernel_coopk, 5, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer));+ CL_CHECK(clSetKernelArg(kernel_coopk, 6, sizeof(cl_mem), &extrad->data_device));+ CL_CHECK(clSetKernelArg(kernel_coopk, 7, sizeof(cl_ulong), &offsetd));+ CL_CHECK(clSetKernelArg(kernel_coopk, 8, sizeof(cl_int), &m));+ CL_CHECK(clSetKernelArg(kernel_coopk, 9, sizeof(cl_int), &n_padded));+ CL_CHECK(clSetKernelArg(kernel_coopk, 10, sizeof(cl_int), &k));+ CL_CHECK(clSetKernelArg(kernel_coopk, 11, sizeof(cl_int), &n));+ CL_CHECK(clSetKernelArg(kernel_coopk, 12, sizeof(cl_ushort), &mask_f000));+ CL_CHECK(clSetKernelArg(kernel_coopk, 13, sizeof(cl_uchar), &mask_c0));++ const size_t nsg = (size_t)backend_ctx->nsg_coopk_dp4a_q6_k;+ size_t local_work_size[3] = { 64, nsg, 1 };+ size_t global_work_size[3] = { (size_t)(ne01 / backend_ctx->rows_coopk_dp4a_q6_k), nsg, 1 };+ backend_ctx->enqueue_ndrange_kernel(kernel_coopk, 3, global_work_size, local_work_size, dst);++ CL_CHECK(clReleaseMemObject(b_sub_buf));+ return;+ }+// dp4a (int8) dense q6_K prefill GEMMstatic const char * q6k_dense_dp4a_env = getenv("GGML_OPENCL_Q6K_DENSE_DP4A");bool q6k_dense_dp4a_on = (q6k_dense_dp4a_env != nullptr)diff --git a/ggml/src/ggml-opencl/kernels/gemm_coopk_q4_0_q8_1_dp4a.cl b/ggml/src/ggml-opencl/kernels/gemm_coopk_q4_0_q8_1_dp4a.clnew file mode 100644index 0000000000..a359574099--- /dev/null+++ b/ggml/src/ggml-opencl/kernels/gemm_coopk_q4_0_q8_1_dp4a.cl@@ -0,0 +1,205 @@+// Cooperative-K DP4A GEMM for 2 to 4 columns with Q4_0 weights and Q8_1 activations.+// Named vector components avoid dynamic private-array indexing, which can cause register spills.++#pragma OPENCL EXTENSION cl_khr_fp16 : enable+#ifdef cl_khr_integer_dot_product+#pragma OPENCL EXTENSION cl_khr_integer_dot_product : enable+#endif++#ifndef COOPK_NSG+#define COOPK_NSG 4+#endif+#define COOPK_SG 64++#ifndef COOPK_ROWS+#define COOPK_ROWS 4+#endif+#ifndef COOPK_COLS+#define COOPK_COLS 4+#endif++#if COOPK_COLS == 4+typedef float4 coopk_accv;+typedef int4 coopk_dotv;+#define COOPK_CONVF convert_float4+#else+typedef float2 coopk_accv;+typedef int2 coopk_dotv;+#define COOPK_CONVF convert_float2+#endif++// Pack four Q4_0 weights as signed bytes after subtracting 8.+#define EXP40(l) \+ ( (((uint)((int)(( (l) & 0x000Fu) - 8) & 0xFF))) \+ | (((uint)((int)((((l) & 0x00F0u) >> 4) - 8) & 0xFF)) << 8) \+ | (((uint)((int)((((l) & 0x0F00u) >> 8) - 8) & 0xFF)) << 16) \+ | (((uint)((int)((((l) & 0xF000u) >> 12) - 8) & 0xFF)) << 24) )++#if COOPK_ROWS == 4+#define COOPK_DOT(ci, t) \+ d0.s##ci = dot_acc_sat_4x8packed_ss_int(w0, A##ci.s##t, d0.s##ci); \+ d1.s##ci = dot_acc_sat_4x8packed_ss_int(w1, A##ci.s##t, d1.s##ci); \+ d2.s##ci = dot_acc_sat_4x8packed_ss_int(w2, A##ci.s##t, d2.s##ci); \+ d3.s##ci = dot_acc_sat_4x8packed_ss_int(w3, A##ci.s##t, d3.s##ci);+#else+#define COOPK_DOT(ci, t) \+ d0.s##ci = dot_acc_sat_4x8packed_ss_int(w0, A##ci.s##t, d0.s##ci); \+ d1.s##ci = dot_acc_sat_4x8packed_ss_int(w1, A##ci.s##t, d1.s##ci);+#endif++#if COOPK_COLS == 4+#define COOPK_DOTS_AT(t) COOPK_DOT(0,t) COOPK_DOT(1,t) COOPK_DOT(2,t) COOPK_DOT(3,t)+#define COOPK_FOR_COLS(F) F(0) F(1) F(2) F(3)+#else+#define COOPK_DOTS_AT(t) COOPK_DOT(0,t) COOPK_DOT(1,t)+#define COOPK_FOR_COLS(F) F(0) F(1)+#endif++#if COOPK_ROWS == 4+#define COOPK_KSTEP(t) \+ { \+ ushort4 bl = vload4(0, src0_q + row0 + (ku0 + t) * m); \+ const uint w0 = EXP40(bl.s0); \+ const uint w1 = EXP40(bl.s1); \+ const uint w2 = EXP40(bl.s2); \+ const uint w3 = EXP40(bl.s3); \+ COOPK_DOTS_AT(t) \+ }+#else+#define COOPK_KSTEP(t) \+ { \+ ushort2 bl = vload2(0, src0_q + row0 + (ku0 + t) * m); \+ const uint w0 = EXP40(bl.s0); \+ const uint w1 = EXP40(bl.s1); \+ COOPK_DOTS_AT(t) \+ }+#endif++// Weights use the untiled Adreno noshuffle SoA layout, with the output row contiguous in each plane.+// Q8_1 activation planes are K-major (K contiguous); no activation transpose is used.+kernel void kernel_gemm_coopk_q4_0_q8_1_dp4a(+ global const ushort * src0_q, // four consecutive K nibbles per ushort, [k/4][m]+ global const half * src0_d, // block scales, [k/32][m]+ global const uint * src1_qa, // four signed activation bytes per uint, [n][k/4]+ global const half * src1_da, // activation scales, [n][k/32]+ global float * dst,+ ulong offsetd,+ int m,+ int n,+ int k,+ int n_no_padding+) {+ dst = (global float *)((global char *)dst + offsetd);++ const int gx = get_global_id(0);+ const int sg = get_local_id(1);+ const int lane = get_local_id(0);++ const int row0 = gx * COOPK_ROWS;+ const int num_32blk = k / 32;+ const int k_u = k >> 2;+ const int k_b = k >> 5;++ // The host pads activations to COOPK_COLS; stores mask columns beyond n_no_padding.+ const int c0 = 0, c1 = 1;+#if COOPK_COLS == 4+ const int c2 = 2, c3 = 3;+#endif++ coopk_accv acc0 = (coopk_accv)(0.0f), acc1 = (coopk_accv)(0.0f);+#if COOPK_ROWS == 4+ coopk_accv acc2 = (coopk_accv)(0.0f), acc3 = (coopk_accv)(0.0f);+#endif++ for (int blk = sg; blk < num_32blk; blk += COOPK_NSG) {+#if COOPK_ROWS == 4+ half4 scale = vload4(0, src0_d + row0 + blk * m);+#else+ half2 scale = vload2(0, src0_d + row0 + blk * m);+#endif++ coopk_accv da;+ da.s0 = (float)src1_da[c0*k_b + blk];+ da.s1 = (float)src1_da[c1*k_b + blk];+#if COOPK_COLS == 4+ da.s2 = (float)src1_da[c2*k_b + blk];+ da.s3 = (float)src1_da[c3*k_b + blk];+#endif++ // Q4_0 uses one weight scale per 32 K values.+ coopk_dotv d0 = (coopk_dotv)(0), d1 = (coopk_dotv)(0);+#if COOPK_ROWS == 4+ coopk_dotv d2 = (coopk_dotv)(0), d3 = (coopk_dotv)(0);+#endif+ for (int half_idx = 0; half_idx < 2; ++half_idx) {+ const int ku0 = (blk << 3) + half_idx * 4;++ uint4 A0 = vload4(0, src1_qa + (uint)c0 * k_u + ku0);+ uint4 A1 = vload4(0, src1_qa + (uint)c1 * k_u + ku0);+#if COOPK_COLS == 4+ uint4 A2 = vload4(0, src1_qa + (uint)c2 * k_u + ku0);+ uint4 A3 = vload4(0, src1_qa + (uint)c3 * k_u + ku0);+#endif+ COOPK_KSTEP(0)+ COOPK_KSTEP(1)+ COOPK_KSTEP(2)+ COOPK_KSTEP(3)+ }++ acc0 += (float)scale.s0 * da * COOPK_CONVF(d0);+ acc1 += (float)scale.s1 * da * COOPK_CONVF(d1);+#if COOPK_ROWS == 4+ acc2 += (float)scale.s2 * da * COOPK_CONVF(d2);+ acc3 += (float)scale.s3 * da * COOPK_CONVF(d3);+#endif+ }++ // Reuse one local buffer for all rows; the first barrier protects the previous row's reads.+ local coopk_accv reduceLM[COOPK_SG * (COOPK_NSG - 1)];+ coopk_accv out0 = (coopk_accv)(0.0f), out1 = (coopk_accv)(0.0f);+#if COOPK_ROWS == 4+ coopk_accv out2 = (coopk_accv)(0.0f), out3 = (coopk_accv)(0.0f);+#endif++#define COOPK_REDUCE(accv, outv) \+ barrier(CLK_LOCAL_MEM_FENCE); \+ if (sg > 0) { reduceLM[(sg - 1) * COOPK_SG + lane] = (accv); } \+ barrier(CLK_LOCAL_MEM_FENCE); \+ if (sg == 0) { \+ coopk_accv sum = (accv); \+ for (int s = 0; s < COOPK_NSG - 1; s++) { \+ sum += reduceLM[s * COOPK_SG + lane]; \+ } \+ (outv) = sum; \+ }++ COOPK_REDUCE(acc0, out0)+ COOPK_REDUCE(acc1, out1)+#if COOPK_ROWS == 4+ COOPK_REDUCE(acc2, out2)+ COOPK_REDUCE(acc3, out3)+#endif++#undef COOPK_REDUCE++#if COOPK_ROWS == 4+#define COOPK_STORE_COL(ci) \+ if (idx < m*n_no_padding) { \+ vstore4((float4)(out0.s##ci, out1.s##ci, out2.s##ci, out3.s##ci), 0, dst + idx); \+ idx += m; \+ }+#else+#define COOPK_STORE_COL(ci) \+ if (idx < m*n_no_padding) { \+ vstore2((float2)(out0.s##ci, out1.s##ci), 0, dst + idx); \+ idx += m; \+ }+#endif++ if (sg == 0) {+ int idx = row0;+ COOPK_FOR_COLS(COOPK_STORE_COL)+ }++#undef COOPK_STORE_COL+}diff --git a/ggml/src/ggml-opencl/kernels/gemm_coopk_q4_k_q8_1_dp4a.cl b/ggml/src/ggml-opencl/kernels/gemm_coopk_q4_k_q8_1_dp4a.clnew file mode 100644index 0000000000..31dce626f1--- /dev/null+++ b/ggml/src/ggml-opencl/kernels/gemm_coopk_q4_k_q8_1_dp4a.cl@@ -0,0 +1,264 @@+// Cooperative-K DP4A GEMM for 2 to 4 columns with Q4_K weights and Q8_1 activations.+// Named vector components avoid dynamic private-array indexing, which can cause register spills.++#pragma OPENCL EXTENSION cl_khr_fp16 : enable+#ifdef cl_khr_integer_dot_product+#pragma OPENCL EXTENSION cl_khr_integer_dot_product : enable+#endif++#define QK_K 256+#define K_SCALE_SIZE 12++#ifndef COOPK_NSG+#define COOPK_NSG 4+#endif+#define COOPK_SG 64++#ifndef COOPK_ROWS+#define COOPK_ROWS 4+#endif+#ifndef COOPK_COLS+#define COOPK_COLS 4+#endif++#if COOPK_COLS == 4+typedef float4 coopk_accv;+typedef int4 coopk_dotv;+#define COOPK_CONVF convert_float4+#else+typedef float2 coopk_accv;+typedef int2 coopk_dotv;+#define COOPK_CONVF convert_float2+#endif++// Expand four Q4_K nibbles into bytes for DP4A.+#define EXP4(u) ( ((uint)((u) & 0x000Fu)) | \+ (((uint)((u) & 0x00F0u)) << 4) | \+ (((uint)((u) & 0x0F00u)) << 8) | \+ (((uint)((u) & 0xF000u)) << 12) )++#if COOPK_ROWS == 4+#define COOPK_DOT(ci, t) \+ s0.s##ci = dot_acc_sat_4x8packed_ss_int(w0, A##ci.s##t, s0.s##ci); \+ s1.s##ci = dot_acc_sat_4x8packed_ss_int(w1, A##ci.s##t, s1.s##ci); \+ s2.s##ci = dot_acc_sat_4x8packed_ss_int(w2, A##ci.s##t, s2.s##ci); \+ s3.s##ci = dot_acc_sat_4x8packed_ss_int(w3, A##ci.s##t, s3.s##ci);+#else+#define COOPK_DOT(ci, t) \+ s0.s##ci = dot_acc_sat_4x8packed_ss_int(w0, A##ci.s##t, s0.s##ci); \+ s1.s##ci = dot_acc_sat_4x8packed_ss_int(w1, A##ci.s##t, s1.s##ci);+#endif++#if COOPK_COLS == 4+#define COOPK_DOTS_AT(t) COOPK_DOT(0,t) COOPK_DOT(1,t) COOPK_DOT(2,t) COOPK_DOT(3,t)+#define COOPK_FOR_COLS(F) F(0) F(1) F(2) F(3)+#else+#define COOPK_DOTS_AT(t) COOPK_DOT(0,t) COOPK_DOT(1,t)+#define COOPK_FOR_COLS(F) F(0) F(1)+#endif++// The bin layout packs eight K values in each uint, with four nibbles in each 16-bit half.+// Since ku0 is a multiple of four, t / 2 selects the uint and t & 1 selects its half.+#ifdef COOPK_Q_BIN+#define COOPK_QBITS4(t) convert_ushort4(((t) & 1) ? (vload4(0, src0_q + row0 + ((ku0 >> 1) + ((t) >> 1)) * m) >> 16) \+ : (vload4(0, src0_q + row0 + ((ku0 >> 1) + ((t) >> 1)) * m) & 0xFFFFu))+#define COOPK_QBITS2(t) convert_ushort2(((t) & 1) ? (vload2(0, src0_q + row0 + ((ku0 >> 1) + ((t) >> 1)) * m) >> 16) \+ : (vload2(0, src0_q + row0 + ((ku0 >> 1) + ((t) >> 1)) * m) & 0xFFFFu))+#else+#define COOPK_QBITS4(t) vload4(0, src0_q + row0 + (ku0 + t) * m)+#define COOPK_QBITS2(t) vload2(0, src0_q + row0 + (ku0 + t) * m)+#endif++#if COOPK_ROWS == 4+#define COOPK_KSTEP(t) \+ { \+ ushort4 bits = COOPK_QBITS4(t); \+ const uint w0 = EXP4(bits.s0); \+ const uint w1 = EXP4(bits.s1); \+ const uint w2 = EXP4(bits.s2); \+ const uint w3 = EXP4(bits.s3); \+ COOPK_DOTS_AT(t) \+ }+#else+#define COOPK_KSTEP(t) \+ { \+ ushort2 bits = COOPK_QBITS2(t); \+ const uint w0 = EXP4(bits.s0); \+ const uint w1 = EXP4(bits.s1); \+ COOPK_DOTS_AT(t) \+ }+#endif++inline void get_scale_min_k4_c(int j, global const uchar * q, int stride,+ uchar * d, uchar * m,+ uchar mask_d6, uchar mask_d4, uchar mask_hi2) {+ if (j < 4) {+ *d = q[j*stride] & mask_d6;+ *m = q[(j + 4)*stride] & mask_d6;+ } else {+ *d = (q[(j + 4)*stride] & mask_d4) | ((q[(j - 4)*stride] >> 6) << 4);+ *m = (q[(j + 4)*stride] >> 4) | ((q[(j )*stride] >> 6) << 4);+ }+}++// Weights use the untiled Adreno noshuffle SoA layout; COOPK_Q_BIN selects uint-transposed q instead of ushort-transposed q.+// Q8_1 activation planes are K-major (K contiguous); no activation transpose is used.+kernel void kernel_gemm_coopk_q4_k_q8_1_dp4a(+#ifdef COOPK_Q_BIN+ global const uint * src0_q, // eight consecutive K nibbles per uint, [k/8][m]+#else+ global const ushort * src0_q, // four consecutive K nibbles per ushort, [k/4][m]+#endif+ global const uchar * src0_s, // packed 6-bit scales/mins, [k/256][12][m]+ global const half * src0_d, // super-block scales, [k/256][m]+ global const half * src0_dm, // super-block mins, [k/256][m]+ global const uint * src1_qa, // four signed activation bytes per uint, [n][k/4]+ global const half * src1_da, // activation scales, [n][k/32]+ global const half * src1_sa, // scaled activation sums, [n][k/32]+ global float * dst,+ ulong offsetd,+ int m,+ int n,+ int k,+ int n_no_padding,+ uchar mask_d6,+ uchar mask_d4,+ uchar mask_hi2+) {+ dst = (global float *)((global char *)dst + offsetd);++ const int gx = get_global_id(0); // row group+ const int sg = get_local_id(1); // K-split subgroup+ const int lane = get_local_id(0);++ const int row0 = gx * COOPK_ROWS;+ const int num_32blk = k / 32;+ const int k_u = k >> 2; // K in uint (int8x4) units+ const int k_b = k >> 5; // 32-blocks along K++ // The host pads activations to COOPK_COLS; stores mask columns beyond n_no_padding.+ const int nl = n - 1;+ const int c0 = 0;+ const int c1 = (1 < n) ? 1 : nl;+#if COOPK_COLS >= 4+ const int c2 = (2 < n) ? 2 : nl;+ const int c3 = (3 < n) ? 3 : nl;+#endif++ coopk_accv acc0 = (coopk_accv)(0.0f), acc1 = (coopk_accv)(0.0f);+#if COOPK_ROWS == 4+ coopk_accv acc2 = (coopk_accv)(0.0f), acc3 = (coopk_accv)(0.0f);+#endif++ for (int blk = sg; blk < num_32blk; blk += COOPK_NSG) {+ const int i = blk << 5;+ const int sb_idx = blk >> 3;+ const int sub_idx = blk & 7;++ global const uchar * sc = src0_s + sb_idx * K_SCALE_SIZE * m + row0;+ uchar sv0, mn0, sv1, mn1;+ get_scale_min_k4_c(sub_idx, sc + 0, m, &sv0, &mn0, mask_d6, mask_d4, mask_hi2);+ get_scale_min_k4_c(sub_idx, sc + 1, m, &sv1, &mn1, mask_d6, mask_d4, mask_hi2);+#if COOPK_ROWS == 4+ half4 dd = vload4(0, src0_d + row0 + sb_idx * m);+ half4 dmm = vload4(0, src0_dm + row0 + sb_idx * m);+ uchar sv2, mn2, sv3, mn3;+ get_scale_min_k4_c(sub_idx, sc + 2, m, &sv2, &mn2, mask_d6, mask_d4, mask_hi2);+ get_scale_min_k4_c(sub_idx, sc + 3, m, &sv3, &mn3, mask_d6, mask_d4, mask_hi2);+ const float sc2 = (float)dd.s2 * (float)sv2, mv2 = (float)dmm.s2 * (float)mn2;+ const float sc3 = (float)dd.s3 * (float)sv3, mv3 = (float)dmm.s3 * (float)mn3;+#else+ half2 dd = vload2(0, src0_d + row0 + sb_idx * m);+ half2 dmm = vload2(0, src0_dm + row0 + sb_idx * m);+#endif+ const float sc0 = (float)dd.s0 * (float)sv0, mv0 = (float)dmm.s0 * (float)mn0;+ const float sc1 = (float)dd.s1 * (float)sv1, mv1 = (float)dmm.s1 * (float)mn1;++ // Q4_K scale and min values change every 32 K values.+ coopk_dotv s0 = (coopk_dotv)(0), s1 = (coopk_dotv)(0);+#if COOPK_ROWS == 4+ coopk_dotv s2 = (coopk_dotv)(0), s3 = (coopk_dotv)(0);+#endif++ for (int uq = 0; uq < 2; ++uq) {+ const int ku0 = (i >> 2) + uq * 4;++ uint4 A0 = vload4(0, src1_qa + (uint)c0 * k_u + ku0);+ uint4 A1 = vload4(0, src1_qa + (uint)c1 * k_u + ku0);+#if COOPK_COLS >= 4+ uint4 A2 = vload4(0, src1_qa + (uint)c2 * k_u + ku0);+ uint4 A3 = vload4(0, src1_qa + (uint)c3 * k_u + ku0);+#endif+ COOPK_KSTEP(0)+ COOPK_KSTEP(1)+ COOPK_KSTEP(2)+ COOPK_KSTEP(3)+ }++ // Q4_K stores q * scale - min for each 32-value block.+ // src1_sa already includes the activation scale, so the min term needs no extra factor of da.+ coopk_accv da, sa;+ da.s0 = (float)src1_da[c0*k_b + blk]; sa.s0 = (float)src1_sa[c0*k_b + blk];+ da.s1 = (float)src1_da[c1*k_b + blk]; sa.s1 = (float)src1_sa[c1*k_b + blk];+#if COOPK_COLS >= 4+ da.s2 = (float)src1_da[c2*k_b + blk]; sa.s2 = (float)src1_sa[c2*k_b + blk];+ da.s3 = (float)src1_da[c3*k_b + blk]; sa.s3 = (float)src1_sa[c3*k_b + blk];+#endif++ acc0 += sc0 * da * COOPK_CONVF(s0) - mv0 * sa;+ acc1 += sc1 * da * COOPK_CONVF(s1) - mv1 * sa;+#if COOPK_ROWS == 4+ acc2 += sc2 * da * COOPK_CONVF(s2) - mv2 * sa;+ acc3 += sc3 * da * COOPK_CONVF(s3) - mv3 * sa;+#endif+ }++ // Reuse one local buffer for all rows; the first barrier protects the previous row's reads.+ local coopk_accv reduceLM[COOPK_SG * (COOPK_NSG - 1)];+ coopk_accv out0 = (coopk_accv)(0.0f), out1 = (coopk_accv)(0.0f);+#if COOPK_ROWS == 4+ coopk_accv out2 = (coopk_accv)(0.0f), out3 = (coopk_accv)(0.0f);+#endif++#define COOPK_REDUCE(accv, outv) \+ barrier(CLK_LOCAL_MEM_FENCE); \+ if (sg > 0) { reduceLM[(sg - 1) * COOPK_SG + lane] = (accv); } \+ barrier(CLK_LOCAL_MEM_FENCE); \+ if (sg == 0) { \+ coopk_accv sum = (accv); \+ for (int s = 0; s < COOPK_NSG - 1; s++) { \+ sum += reduceLM[s * COOPK_SG + lane]; \+ } \+ (outv) = sum; \+ }++ COOPK_REDUCE(acc0, out0)+ COOPK_REDUCE(acc1, out1)+#if COOPK_ROWS == 4+ COOPK_REDUCE(acc2, out2)+ COOPK_REDUCE(acc3, out3)+#endif++#undef COOPK_REDUCE++#if COOPK_ROWS == 4+#define COOPK_STORE_COL(ci) \+ if (idx < m*n_no_padding) { \+ vstore4((float4)(out0.s##ci, out1.s##ci, out2.s##ci, out3.s##ci), 0, dst + idx); \+ idx += m; \+ }+#else+#define COOPK_STORE_COL(ci) \+ if (idx < m*n_no_padding) { \+ vstore2((float2)(out0.s##ci, out1.s##ci), 0, dst + idx); \+ idx += m; \+ }+#endif++ if (sg == 0) {+ int idx = row0;+ COOPK_FOR_COLS(COOPK_STORE_COL)+ }++#undef COOPK_STORE_COL+}diff --git a/ggml/src/ggml-opencl/kernels/gemm_coopk_q6_k_q8_1_dp4a.cl b/ggml/src/ggml-opencl/kernels/gemm_coopk_q6_k_q8_1_dp4a.clnew file mode 100644index 0000000000..4410a230d7--- /dev/null+++ b/ggml/src/ggml-opencl/kernels/gemm_coopk_q6_k_q8_1_dp4a.cl@@ -0,0 +1,216 @@+// Cooperative-K DP4A GEMM for 2 to 4 columns with Q6_K weights and Q8_1 activations.+// Named vector components avoid dynamic private-array indexing, which can cause register spills.++#pragma OPENCL EXTENSION cl_khr_fp16 : enable+#ifdef cl_khr_integer_dot_product+#pragma OPENCL EXTENSION cl_khr_integer_dot_product : enable+#endif++#ifndef COOPK_NSG+#define COOPK_NSG 4+#endif+#define COOPK_SG 64++#ifndef COOPK_ROWS+#define COOPK_ROWS 4+#endif+#ifndef COOPK_COLS+#define COOPK_COLS 4+#endif++#if COOPK_COLS == 4+typedef float4 coopk_accv;+typedef int4 coopk_dotv;+#define COOPK_CONVF convert_float4+#else+typedef float2 coopk_accv;+typedef int2 coopk_dotv;+#define COOPK_CONVF convert_float2+#endif++// Combine low nibbles and high bits, subtract 32, and pack four signed bytes for DP4A.+#define EXP6(l, h, mf, mc) \+ ( (((uint)((int)((( (l) & 0x000Fu) | (((h) & 0x03u) << 4)) - 32) & 0xFF))) \+ | (((uint)((int)(((((l) & 0x00F0u) >> 4) | (((h) & 0x0Cu) << 2)) - 32) & 0xFF)) << 8) \+ | (((uint)((int)(((((l) & 0x0F00u) >> 8) | ((h) & 0x30u)) - 32) & 0xFF)) << 16) \+ | (((uint)((int)(((((l) & (mf)) >> 12) | (((h) & (mc)) >> 2)) - 32) & 0xFF)) << 24) )++#if COOPK_ROWS == 4+#define COOPK_DOT(ci, t) \+ d0.s##ci = dot_acc_sat_4x8packed_ss_int(w0, A##ci.s##t, d0.s##ci); \+ d1.s##ci = dot_acc_sat_4x8packed_ss_int(w1, A##ci.s##t, d1.s##ci); \+ d2.s##ci = dot_acc_sat_4x8packed_ss_int(w2, A##ci.s##t, d2.s##ci); \+ d3.s##ci = dot_acc_sat_4x8packed_ss_int(w3, A##ci.s##t, d3.s##ci);+#else+#define COOPK_DOT(ci, t) \+ d0.s##ci = dot_acc_sat_4x8packed_ss_int(w0, A##ci.s##t, d0.s##ci); \+ d1.s##ci = dot_acc_sat_4x8packed_ss_int(w1, A##ci.s##t, d1.s##ci);+#endif++#if COOPK_COLS == 4+#define COOPK_DOTS_AT(t) COOPK_DOT(0,t) COOPK_DOT(1,t) COOPK_DOT(2,t) COOPK_DOT(3,t)+#define COOPK_FOR_COLS(F) F(0) F(1) F(2) F(3)+#else+#define COOPK_DOTS_AT(t) COOPK_DOT(0,t) COOPK_DOT(1,t)+#define COOPK_FOR_COLS(F) F(0) F(1)+#endif++#if COOPK_ROWS == 4+#define COOPK_KSTEP(t) \+ { \+ ushort4 bl = vload4(0, src0_ql + row0 + (ku0 + t) * m); \+ uchar4 bh = vload4(0, src0_qh + row0 + (ku0 + t) * m); \+ const uint w0 = EXP6(bl.s0, bh.s0, mask_f000, mask_c0); \+ const uint w1 = EXP6(bl.s1, bh.s1, mask_f000, mask_c0); \+ const uint w2 = EXP6(bl.s2, bh.s2, mask_f000, mask_c0); \+ const uint w3 = EXP6(bl.s3, bh.s3, mask_f000, mask_c0); \+ COOPK_DOTS_AT(t) \+ }+#else+#define COOPK_KSTEP(t) \+ { \+ ushort2 bl = vload2(0, src0_ql + row0 + (ku0 + t) * m); \+ uchar2 bh = vload2(0, src0_qh + row0 + (ku0 + t) * m); \+ const uint w0 = EXP6(bl.s0, bh.s0, mask_f000, mask_c0); \+ const uint w1 = EXP6(bl.s1, bh.s1, mask_f000, mask_c0); \+ COOPK_DOTS_AT(t) \+ }+#endif++// Weights use the untiled Adreno noshuffle SoA layout, with the output row contiguous in each plane.+// Q8_1 activation planes are K-major (K contiguous); no activation transpose is used.+kernel void kernel_gemm_coopk_q6_k_q8_1_dp4a(+ global const ushort * src0_ql, // four consecutive low nibbles per ushort, [k/4][m]+ global const uchar * src0_qh, // four consecutive high 2-bit codes per uchar, [k/4][m]+ global const ushort * src0_s, // two signed int8 scales per ushort, [k/32][m]+ global const half * src0_d, // super-block scales, [k/256][m]+ global const uint * src1_qa, // four signed activation bytes per uint, [n][k/4]+ global const half * src1_da, // activation scales, [n][k/32]+ global float * dst,+ ulong offsetd,+ int m,+ int n,+ int k,+ int n_no_padding,+ ushort mask_f000,+ uchar mask_c0+) {+ dst = (global float *)((global char *)dst + offsetd);++ const int gx = get_global_id(0);+ const int sg = get_local_id(1);+ const int lane = get_local_id(0);++ const int row0 = gx * COOPK_ROWS;+ const int num_32blk = k / 32;+ const int k_u = k >> 2;+ const int k_b = k >> 5;++ // The host pads activations to COOPK_COLS; stores mask columns beyond n_no_padding.+ const int c0 = 0, c1 = 1;+#if COOPK_COLS == 4+ const int c2 = 2, c3 = 3;+#endif++ coopk_accv acc0 = (coopk_accv)(0.0f), acc1 = (coopk_accv)(0.0f);+#if COOPK_ROWS == 4+ coopk_accv acc2 = (coopk_accv)(0.0f), acc3 = (coopk_accv)(0.0f);+#endif++ for (int blk = sg; blk < num_32blk; blk += COOPK_NSG) {+ const int i = blk << 5;++ // Each ushort packs two signed scale bytes, one for each 16-value half of this 32-value block.+ // The half-precision super-block scale is shared by 256 K values.+ ushort4 spk = vload4(0, src0_s + row0 + blk * m);+ half4 scd = vload4(0, src0_d + row0 + (i >> 8) * m);++ coopk_accv da;+ da.s0 = (float)src1_da[c0*k_b + blk];+ da.s1 = (float)src1_da[c1*k_b + blk];+#if COOPK_COLS == 4+ da.s2 = (float)src1_da[c2*k_b + blk];+ da.s3 = (float)src1_da[c3*k_b + blk];+#endif++ // Apply each signed weight scale to its 16-value dot product.+ for (int half_idx = 0; half_idx < 2; ++half_idx) {+ const int ku0 = (i >> 2) + half_idx * 4;++ coopk_dotv d0 = (coopk_dotv)(0), d1 = (coopk_dotv)(0);+#if COOPK_ROWS == 4+ coopk_dotv d2 = (coopk_dotv)(0), d3 = (coopk_dotv)(0);+#endif+ uint4 A0 = vload4(0, src1_qa + (uint)c0 * k_u + ku0);+ uint4 A1 = vload4(0, src1_qa + (uint)c1 * k_u + ku0);+#if COOPK_COLS == 4+ uint4 A2 = vload4(0, src1_qa + (uint)c2 * k_u + ku0);+ uint4 A3 = vload4(0, src1_qa + (uint)c3 * k_u + ku0);+#endif+ COOPK_KSTEP(0)+ COOPK_KSTEP(1)+ COOPK_KSTEP(2)+ COOPK_KSTEP(3)++ const float s0f = (float)(half_idx ? as_char2(spk.s0).s1 : as_char2(spk.s0).s0) * (float)scd.s0;+ const float s1f = (float)(half_idx ? as_char2(spk.s1).s1 : as_char2(spk.s1).s0) * (float)scd.s1;+ acc0 += s0f * da * COOPK_CONVF(d0);+ acc1 += s1f * da * COOPK_CONVF(d1);+#if COOPK_ROWS == 4+ const float s2f = (float)(half_idx ? as_char2(spk.s2).s1 : as_char2(spk.s2).s0) * (float)scd.s2;+ const float s3f = (float)(half_idx ? as_char2(spk.s3).s1 : as_char2(spk.s3).s0) * (float)scd.s3;+ acc2 += s2f * da * COOPK_CONVF(d2);+ acc3 += s3f * da * COOPK_CONVF(d3);+#endif+ }+ }++ // Reuse one local buffer for all rows; the first barrier protects the previous row's reads.+ local coopk_accv reduceLM[COOPK_SG * (COOPK_NSG - 1)];+ coopk_accv out0 = (coopk_accv)(0.0f), out1 = (coopk_accv)(0.0f);+#if COOPK_ROWS == 4+ coopk_accv out2 = (coopk_accv)(0.0f), out3 = (coopk_accv)(0.0f);+#endif++#define COOPK_REDUCE(accv, outv) \+ barrier(CLK_LOCAL_MEM_FENCE); \+ if (sg > 0) { reduceLM[(sg - 1) * COOPK_SG + lane] = (accv); } \+ barrier(CLK_LOCAL_MEM_FENCE); \+ if (sg == 0) { \+ coopk_accv sum = (accv); \+ for (int s = 0; s < COOPK_NSG - 1; s++) { \+ sum += reduceLM[s * COOPK_SG + lane]; \+ } \+ (outv) = sum; \+ }++ COOPK_REDUCE(acc0, out0)+ COOPK_REDUCE(acc1, out1)+#if COOPK_ROWS == 4+ COOPK_REDUCE(acc2, out2)+ COOPK_REDUCE(acc3, out3)+#endif++#undef COOPK_REDUCE++#if COOPK_ROWS == 4+#define COOPK_STORE_COL(ci) \+ if (idx < m*n_no_padding) { \+ vstore4((float4)(out0.s##ci, out1.s##ci, out2.s##ci, out3.s##ci), 0, dst + idx); \+ idx += m; \+ }+#else+#define COOPK_STORE_COL(ci) \+ if (idx < m*n_no_padding) { \+ vstore2((float2)(out0.s##ci, out1.s##ci), 0, dst + idx); \+ idx += m; \+ }+#endif++ if (sg == 0) {+ int idx = row0;+ COOPK_FOR_COLS(COOPK_STORE_COL)+ }++#undef COOPK_STORE_COL+}diff --git a/ggml/src/ggml-opencl/kernels/quant_a_q8_1.cl b/ggml/src/ggml-opencl/kernels/quant_a_q8_1.clindex 3ce06305a3..5b6dea773e 100644--- a/ggml/src/ggml-opencl/kernels/quant_a_q8_1.cl+++ b/ggml/src/ggml-opencl/kernels/quant_a_q8_1.cl@@ -1,9 +1,7 @@#pragma OPENCL EXTENSION cl_khr_fp16 : enable-// Quantize a contiguous [N, K] f32 activation buffer (token-major, K contiguous-// per token) into q8_1 blocks of 32: int8 quants + per-block scale d + per-block-// sum s (= d * Sum(qs)). Consumed by kernel_gemm_noshuffle_q4_k_q8_1_dp4a for the-// dp4a (int8) dense q4_K prefill GEMM. One work-item per 32-element block.+// Quantize a contiguous K-major [N, K] f32 activation buffer (K contiguous) into 32-value Q8_1 SoA blocks.+// Store int8 quants, per-block scale d, and scaled sum s = d * sum(qs); one work-item handles each block.__kernel void kernel_quant_a_q8_1(__global const float * src, // [N * K]__global char * qa, // [N * K]