LLM inference in C/C++
C++ ggml
sycl: reduce redundant work in Q4_K multi-column MMVQ (#27062)
Eurekatic4aa6ffba259a
diff --git a/ggml/src/ggml-sycl/mmvq.cpp b/ggml/src/ggml-sycl/mmvq.cppindex 220663d5ac..933bc77d2e 100644--- a/ggml/src/ggml-sycl/mmvq.cpp+++ b/ggml/src/ggml-sycl/mmvq.cpp@@ -6,6 +6,24 @@#include "quants.hpp"#include "vecdotq.hpp"+// Minimum weight-row count at which the Q4_K multi-column MMVQ kernel handles two output rows per+// subgroup (rows_per_sg == 2) instead of one, when ncols_dst == 2.+//+// Pairing rows lets a subgroup load each activation block once and apply it to two rows, at the cost+// of halving the number of subgroups in the launch. With only two destination columns there is too+// little work per row to hide that loss of parallelism, so pairing only pays off once there are+// enough rows to keep the device occupied. This is a measured performance crossover, not a+// correctness or hardware limit - both variants compute the same result for any nrows.+//+// Derived on Intel Arc Pro B70 with `test-backend-ops perf -o MUL_MAT` (Q4_K, ncols_dst == 2),+// sweeping nrows over 5120..6912 at ncols 17408 and 19968: one row per subgroup was up to 9% faster+// below the crossover, two rows per subgroup 8-15% faster above it, and the crossover fell inside+// (6144, 6272] for both ncols with no measurable ncols dependence. A later 32-row granularity sweep+// narrowed it to (6144, 6176], so 6272 is a conservative gate rather than the exact crossover.+// ncols_dst >= 3 amortizes the activation loads over more columns and is faster with two rows at+// every row count, so it does not consult this threshold.+static constexpr int Q4_K_MMVQ_ROW_PAIR_MIN_NROWS = 6272;+template <typename reorder_vec_dot_q_sycl>static void mul_mat_vec_q_reorder(const void * __restrict__ vx, const void * __restrict__ vy, float * __restrict__ dst,const int ncols, const int nrows, const sycl::nd_item<3> & nd_item) {@@ -59,7 +77,7 @@ static void mul_mat_vec_q_reorder(const void * __restrict__ vx, const void * __r// With has_fusion, `vgate` is a second weight matrix sharing vx's shape, stride and reorder// layout: one pass computes both row dot products and the epilogue writes glu(gate, up).-template <typename reorder_vec_dot_q_sycl, int ncols_dst, bool has_fusion = false>+template <typename reorder_vec_dot_q_sycl, int ncols_dst, bool has_fusion = false, int rows_per_sg = 1>static void mul_mat_vec_q_reorder_ncols(const void * __restrict__ vx, const void * __restrict__ vgate,const void * __restrict__ vy, float * __restrict__ dst, const int ncols,const int nrows, const int stride_col_y_bytes, const int stride_col_dst,@@ -71,14 +89,17 @@ static void mul_mat_vec_q_reorder_ncols(const void * __restrict__ vx, const voidconst int sg_range = sg.get_group_linear_range();const int workgroup_id = nd_item.get_group_linear_id();const int sg_id = sg.get_group_linear_id();- const int row = workgroup_id * sg_range + sg_id;+ const int row0 = (workgroup_id * sg_range + sg_id) * rows_per_sg;// row is sub-group uniform, so this retires whole sub-groups and the collectives below// stay convergent- if (row >= nrows) {+ if (row0 >= nrows) {return;}+ static_assert(rows_per_sg == 1 ||+ reorder_vec_dot_shared_activations<reorder_vec_dot_q_sycl::gtype>::value);+const int blocks_per_row = ncols / block_traits::qk;constexpr int blocks_per_subgroup = ceil_div(block_traits::vdr_mmvq * WARP_SIZE, block_traits::qi);constexpr int block_elements_per_subgroup = block_traits::qi / block_traits::vdr_mmvq;@@ -87,34 +108,96 @@ static void mul_mat_vec_q_reorder_ncols(const void * __restrict__ vx, const voidstatic_assert(blocks_per_subgroup > 0);static_assert(block_elements_per_subgroup > 0);- float partial_sum[ncols_dst] = { 0.0f };+ float partial_sum[ncols_dst][rows_per_sg] = {};// sized 1 rather than 0 when unused: zero-length arrays are not standard C++, and the// array is dead and eliminated in that case- [[maybe_unused]] float partial_gate[has_fusion ? ncols_dst : 1] = { 0.0f };+ [[maybe_unused]] float partial_gate[has_fusion ? ncols_dst : 1][has_fusion ? rows_per_sg : 1] = {};for (int i = sg.get_local_linear_id() / block_elements_per_subgroup; i < blocks_per_row; i += blocks_per_subgroup) {- const int ibx = row * blocks_per_row + i;-- // the offsets depend only on the block index and the matrix shape, never on the base- // pointer, which is what lets vgate reuse them- const auto bx_offset = block_type::get_block_offset(ibx, nblocks);- const auto d_offset = block_type::get_d_offset(nrows, ncols, ibx);const int iby = i * block_type::block_to_q8_1_ratio();#pragma unrollfor (int elem = 0; elem < block_elements_per_subgroup; elem += WARP_SIZE) {const int iqs = elem + block_traits::vdr_mmvq * (sg.get_local_linear_id() % block_elements_per_subgroup);+ if constexpr (rows_per_sg > 1) {+ typename reorder_vec_dot_q_sycl::weights wx[rows_per_sg];+ [[maybe_unused]] typename reorder_vec_dot_q_sycl::weights wg[rows_per_sg];#pragma unroll- for (int j = 0; j < ncols_dst; ++j) {- const char * vy_j = (const char *) vy + j * stride_col_y_bytes;- const int8_t * q8_1_quant_ptr = (const int8_t *) vy_j + iby * QK8_1;- const sycl::half2 * q8_1_ds_ptr = (const sycl::half2 *) (vy_j + ncols + iby * sizeof(sycl::half2));+ for (int r = 0; r < rows_per_sg; ++r) {+ const int row = sycl::min(row0 + r, nrows - 1);+ const int ibx = row * blocks_per_row + i;+ const auto bx_offset = block_type::get_block_offset(ibx, nblocks);+ const auto d_offset = block_type::get_d_offset(nrows, ncols, ibx);+ wx[r] = reorder_vec_dot_q_sycl::load(vx, bx_offset, d_offset, iqs);+ if constexpr (has_fusion) {+ wg[r] = reorder_vec_dot_q_sycl::load(vgate, bx_offset, d_offset, iqs);+ }+ }+#pragma unroll+ for (int j = 0; j < ncols_dst; ++j) {+ const char * vy_j = (const char *) vy + j * stride_col_y_bytes;+ const int8_t * q8_1_quant_ptr = (const int8_t *) vy_j + iby * QK8_1;+ const sycl::half2 * q8_1_ds_ptr =+ (const sycl::half2 *) (vy_j + ncols + iby * sizeof(sycl::half2));+ const auto a = reorder_vec_dot_q_sycl::load_activations(q8_1_quant_ptr, q8_1_ds_ptr, iqs);+#pragma unroll+ for (int r = 0; r < rows_per_sg; ++r) {+ partial_sum[j][r] += reorder_vec_dot_q_sycl::apply(wx[r], a);+ if constexpr (has_fusion) {+ partial_gate[j][r] += reorder_vec_dot_q_sycl::apply(wg[r], a);+ }+ }+ }+ } else if constexpr (reorder_vec_dot_shared_weights<reorder_vec_dot_q_sycl::gtype>::value) {+ const int ibx = row0 * blocks_per_row + i;+ const auto bx_offset = block_type::get_block_offset(ibx, nblocks);+ const auto d_offset = block_type::get_d_offset(nrows, ncols, ibx);+ const auto wx = reorder_vec_dot_q_sycl::load(vx, bx_offset, d_offset, iqs);+ if constexpr (has_fusion) {+ const auto wg = reorder_vec_dot_q_sycl::load(vgate, bx_offset, d_offset, iqs);- partial_sum[j] += reorder_vec_dot_q_sycl()(vx, bx_offset, d_offset, q8_1_quant_ptr, q8_1_ds_ptr, iqs);+#pragma unroll+ for (int j = 0; j < ncols_dst; ++j) {+ const char * vy_j = (const char *) vy + j * stride_col_y_bytes;+ const int8_t * q8_1_quant_ptr = (const int8_t *) vy_j + iby * QK8_1;+ const sycl::half2 * q8_1_ds_ptr =+ (const sycl::half2 *) (vy_j + ncols + iby * sizeof(sycl::half2));- if constexpr (has_fusion) {- partial_gate[j] +=- reorder_vec_dot_q_sycl()(vgate, bx_offset, d_offset, q8_1_quant_ptr, q8_1_ds_ptr, iqs);+ // up and gate share the activation, so load it once and apply it twice+ const auto a = reorder_vec_dot_q_sycl::load_activations(q8_1_quant_ptr, q8_1_ds_ptr, iqs);++ partial_sum[j][0] += reorder_vec_dot_q_sycl::apply(wx, a);+ partial_gate[j][0] += reorder_vec_dot_q_sycl::apply(wg, a);+ }+ } else {+#pragma unroll+ for (int j = 0; j < ncols_dst; ++j) {+ const char * vy_j = (const char *) vy + j * stride_col_y_bytes;+ const int8_t * q8_1_quant_ptr = (const int8_t *) vy_j + iby * QK8_1;+ const sycl::half2 * q8_1_ds_ptr =+ (const sycl::half2 *) (vy_j + ncols + iby * sizeof(sycl::half2));++ partial_sum[j][0] += reorder_vec_dot_q_sycl::dot(wx, q8_1_quant_ptr, q8_1_ds_ptr, iqs);+ }+ }+ } else {+ const int ibx = row0 * blocks_per_row + i;+ const auto bx_offset = block_type::get_block_offset(ibx, nblocks);+ const auto d_offset = block_type::get_d_offset(nrows, ncols, ibx);+#pragma unroll+ for (int j = 0; j < ncols_dst; ++j) {+ const char * vy_j = (const char *) vy + j * stride_col_y_bytes;+ const int8_t * q8_1_quant_ptr = (const int8_t *) vy_j + iby * QK8_1;+ const sycl::half2 * q8_1_ds_ptr =+ (const sycl::half2 *) (vy_j + ncols + iby * sizeof(sycl::half2));++ partial_sum[j][0] +=+ reorder_vec_dot_q_sycl()(vx, bx_offset, d_offset, q8_1_quant_ptr, q8_1_ds_ptr, iqs);++ if constexpr (has_fusion) {+ partial_gate[j][0] +=+ reorder_vec_dot_q_sycl()(vgate, bx_offset, d_offset, q8_1_quant_ptr, q8_1_ds_ptr, iqs);+ }}}}@@ -122,17 +205,20 @@ static void mul_mat_vec_q_reorder_ncols(const void * __restrict__ vx, const void#pragma unrollfor (int j = 0; j < ncols_dst; ++j) {- float sum = sycl::reduce_over_group(nd_item.get_sub_group(), partial_sum[j], std::plus<>());+#pragma unroll+ for (int r = 0; r < rows_per_sg; ++r) {+ float sum = sycl::reduce_over_group(nd_item.get_sub_group(), partial_sum[j][r], std::plus<>());- if constexpr (has_fusion) {- const float gate = sycl::reduce_over_group(nd_item.get_sub_group(), partial_gate[j], std::plus<>());+ if constexpr (has_fusion) {+ const float gate = sycl::reduce_over_group(nd_item.get_sub_group(), partial_gate[j][r], std::plus<>());- // uniform across the launch; the launcher only instantiates SWIGLU and GEGLU- sum *= glu_op == GGML_GLU_OP_SWIGLU ? op_silu(gate) : op_gelu(gate);- }+ // uniform across the launch; the launcher only instantiates SWIGLU and GEGLU+ sum *= glu_op == GGML_GLU_OP_SWIGLU ? op_silu(gate) : op_gelu(gate);+ }- if (sg.leader()) {- dst[j * stride_col_dst + row] = sum;+ if (sg.leader() && row0 + r < nrows) {+ dst[j * stride_col_dst + row0 + r] = sum;+ }}}}@@ -1671,8 +1757,8 @@ static void reorder_mul_mat_vec_q4_k_q8_1_sycl(const void * vx, const void * vy,});}-template <int ncols_dst>-static void reorder_mul_mat_vec_q4_k_q8_1_sycl_ncols(+template <int ncols_dst, int rows_per_sg>+static void reorder_mul_mat_vec_q4_k_q8_1_sycl_ncols_impl(const void * vx, const void * vy, float * dst,const int ncols, const int nrows,const int stride_col_y_bytes, const int stride_col_dst,@@ -1680,20 +1766,31 @@ static void reorder_mul_mat_vec_q4_k_q8_1_sycl_ncols(GGML_ASSERT(ncols % QK_K == 0);constexpr size_t num_subgroups = WARP_SIZE;- const int block_num_y = ceil_div(nrows, GGML_SYCL_MMV_Y * (int) num_subgroups);+ const int block_num_y = ceil_div(nrows, GGML_SYCL_MMV_Y * (int) num_subgroups * rows_per_sg);const sycl::range<3> block_nums(1, 1, block_num_y);const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, num_subgroups * WARP_SIZE);stream->submit([&](sycl::handler & cgh) {cgh.parallel_for(sycl::nd_range<3>(block_nums * block_dims, block_dims),[=](sycl::nd_item<3> nd_item) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {- mul_mat_vec_q_reorder_ncols<reorder_vec_dot_q_sycl<GGML_TYPE_Q4_K>, ncols_dst>(+ mul_mat_vec_q_reorder_ncols<reorder_vec_dot_q_sycl<GGML_TYPE_Q4_K>, ncols_dst,+ /*has_fusion=*/ false, rows_per_sg>(vx, /*vgate=*/ nullptr, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst,/*glu_op=*/ GGML_GLU_OP_SWIGLU, nd_item);});});}+template <int ncols_dst>+static void reorder_mul_mat_vec_q4_k_q8_1_sycl_ncols(+ const void * vx, const void * vy, float * dst,+ const int ncols, const int nrows,+ const int stride_col_y_bytes, const int stride_col_dst,+ dpct::queue_ptr stream) {+ constexpr int rows_per_sg = ncols_dst >= 3 && ncols_dst <= 4 ? 2 : 1;+ reorder_mul_mat_vec_q4_k_q8_1_sycl_ncols_impl<ncols_dst, rows_per_sg>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream);+}+static void reorder_mul_mat_vec_q4_k_q8_1_sycl_switch_ncols(const void * vx, const void * vy, float * dst,const int ncols, const int nrows, const int ncols_dst,@@ -1701,7 +1798,13 @@ static void reorder_mul_mat_vec_q4_k_q8_1_sycl_switch_ncols(dpct::queue_ptr stream) {switch (ncols_dst) {case 1: reorder_mul_mat_vec_q4_k_q8_1_sycl(vx, vy, dst, ncols, nrows, stream); break;- case 2: reorder_mul_mat_vec_q4_k_q8_1_sycl_ncols<2>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break;+ case 2:+ if (nrows >= Q4_K_MMVQ_ROW_PAIR_MIN_NROWS) {+ reorder_mul_mat_vec_q4_k_q8_1_sycl_ncols_impl<2, 2>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream);+ } else {+ reorder_mul_mat_vec_q4_k_q8_1_sycl_ncols_impl<2, 1>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream);+ }+ break;case 3: reorder_mul_mat_vec_q4_k_q8_1_sycl_ncols<3>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break;case 4: reorder_mul_mat_vec_q4_k_q8_1_sycl_ncols<4>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break;case 5: reorder_mul_mat_vec_q4_k_q8_1_sycl_ncols<5>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break;@@ -2839,8 +2942,8 @@ bool ggml_sycl_mul_mat_vec_q_id_reorder(}}-template <typename reorder_vec_dot_q_sycl, int ncols_dst>-static void launch_mul_mat_vec_q_reorder_glu(const void * vx, const void * vgate, const void * vy, float * dst,+template <typename reorder_vec_dot_q_sycl, int ncols_dst, int rows_per_sg>+static void launch_mul_mat_vec_q_reorder_glu_impl(const void * vx, const void * vgate, const void * vy, float * dst,const int ncols, const int nrows, const int stride_col_y_bytes,const int stride_col_dst, const ggml_glu_op glu_op,dpct::queue_ptr stream) {@@ -2848,20 +2951,33 @@ static void launch_mul_mat_vec_q_reorder_glu(const void * vx, const void * vgateconstexpr size_t num_subgroups = WARP_SIZE;- const int block_num_y = ceil_div(nrows, GGML_SYCL_MMV_Y * (int) num_subgroups);+ const int block_num_y = ceil_div(nrows, GGML_SYCL_MMV_Y * (int) num_subgroups * rows_per_sg);const sycl::range<3> block_nums(1, 1, block_num_y);const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, num_subgroups * WARP_SIZE);stream->submit([&](sycl::handler & cgh) {cgh.parallel_for(sycl::nd_range<3>(block_nums * block_dims, block_dims),[=](sycl::nd_item<3> nd_item) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {- mul_mat_vec_q_reorder_ncols<reorder_vec_dot_q_sycl, ncols_dst, /*has_fusion=*/ true>(+ mul_mat_vec_q_reorder_ncols<reorder_vec_dot_q_sycl, ncols_dst, /*has_fusion=*/ true,+ rows_per_sg>(vx, vgate, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, glu_op,nd_item);});});}+template <typename reorder_vec_dot_q_sycl, int ncols_dst>+static void launch_mul_mat_vec_q_reorder_glu(const void * vx, const void * vgate, const void * vy, float * dst,+ const int ncols, const int nrows, const int stride_col_y_bytes,+ const int stride_col_dst, const ggml_glu_op glu_op,+ dpct::queue_ptr stream) {+ constexpr int rows_per_sg =+ reorder_vec_dot_shared_activations<reorder_vec_dot_q_sycl::gtype>::value && ncols_dst >= 3 && ncols_dst <= 4+ ? 2+ : 1;+ launch_mul_mat_vec_q_reorder_glu_impl<reorder_vec_dot_q_sycl, ncols_dst, rows_per_sg>(vx, vgate, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, glu_op, stream);+}+bool ggml_sycl_mul_mat_vec_q_glu_reorder(enum ggml_type src0_type, enum ggml_glu_op glu_op, const void * vx,const void * vgate, const void * vy, float * dst, int ncols, int nrows,int ncols_dst, int stride_col_y_bytes, int stride_col_dst,@@ -2881,8 +2997,11 @@ bool ggml_sycl_mul_mat_vec_q_glu_reorder(enum ggml_type src0_type, enum ggml_glustride_col_dst, glu_op, stream);return true;case 2:- launch_mul_mat_vec_q_reorder_glu<vec_dot, 2>(vx, vgate, vy, dst, ncols, nrows, stride_col_y_bytes,- stride_col_dst, glu_op, stream);+ if (nrows >= Q4_K_MMVQ_ROW_PAIR_MIN_NROWS) {+ launch_mul_mat_vec_q_reorder_glu_impl<vec_dot, 2, 2>(vx, vgate, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, glu_op, stream);+ } else {+ launch_mul_mat_vec_q_reorder_glu_impl<vec_dot, 2, 1>(vx, vgate, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, glu_op, stream);+ }return true;case 3:launch_mul_mat_vec_q_reorder_glu<vec_dot, 3>(vx, vgate, vy, dst, ncols, nrows, stride_col_y_bytes,diff --git a/ggml/src/ggml-sycl/vecdotq.hpp b/ggml/src/ggml-sycl/vecdotq.hppindex 3ad4cee93a..ed5fd7de89 100644--- a/ggml/src/ggml-sycl/vecdotq.hpp+++ b/ggml/src/ggml-sycl/vecdotq.hpp@@ -351,6 +351,25 @@ template <ggml_type T> struct reorder_vec_dot_q_sycl {static_assert(T != T, "ggml_type for reorder vecdot not implemented");};+// For some types the weight side of the dot product does not depend on the destination column, so a+// multi-column mul_mat_vec can unpack it once per block instead of once per column. Such a type adds+// load() and dot() next to operator() and opts in here. See reorder_vec_dot_q_sycl<GGML_TYPE_Q4_K>.+template <ggml_type T> struct reorder_vec_dot_shared_weights {+ static constexpr bool value = false;+};++template <> struct reorder_vec_dot_shared_weights<GGML_TYPE_Q4_K> {+ static constexpr bool value = true;+};++template <ggml_type T> struct reorder_vec_dot_shared_activations {+ static constexpr bool value = false;+};++template <> struct reorder_vec_dot_shared_activations<GGML_TYPE_Q4_K> {+ static constexpr bool value = true;+};+template <> struct reorder_vec_dot_q_sycl<GGML_TYPE_Q4_0> {static constexpr ggml_type gtype = GGML_TYPE_Q4_0;@@ -540,50 +559,84 @@ template <> struct reorder_vec_dot_q_sycl<GGML_TYPE_Q4_K> {using q4_k_block = ggml_sycl_reordered::block_q_t<GGML_TYPE_Q4_K>;using q4_k_traits = typename q4_k_block::traits;- __dpct_inline__ float operator()(const void * __restrict__ vbq, const std::pair<int, int> ibx_offset,- const std::pair<int, int> d_offset, const int8_t * q8_1_quant_ptr,- const sycl::half2 * q8_1_ds, const int & iqs) {- const uint8_t * base = static_cast<const uint8_t *>(vbq);- const uint8_t * qs = base + ibx_offset.first;- const uint8_t * scs = base + d_offset.first;- const ggml_half2 * dms = reinterpret_cast<const ggml_half2 *>(base + d_offset.second);-- const int bq8_offset = QR4_K * ((iqs / 2) / (QI8_1 / 2));- const int * q4 = (const int *) (qs + 16 * bq8_offset + 4 * ((iqs / 2) % 4));- const uint16_t * scales = (const uint16_t *) scs;+ struct weights {+ int v[2];+ uint16_t aux[2];+ ggml_half2 dm;+ int bq8_offset;+ };- int v[2];+ struct activations {int u[2 * QR4_K];float d8[QR4_K];+ };- v[0] = q4[0];- v[1] = q4[4];+ __dpct_inline__ static weights load(const void * __restrict__ vbq, const std::pair<int, int> ibx_offset,+ const std::pair<int, int> d_offset, const int & iqs) {+ const uint8_t * base = static_cast<const uint8_t *>(vbq);+ const uint8_t * qs = base + ibx_offset.first;+ const uint8_t * scs = base + d_offset.first;+ const ggml_half2 * dms = reinterpret_cast<const ggml_half2 *>(base + d_offset.second);++ weights w;+ w.bq8_offset = QR4_K * ((iqs / 2) / (QI8_1 / 2));++ const int * q4 = (const int *) (qs + 16 * w.bq8_offset + 4 * ((iqs / 2) % 4));+ const uint16_t * scales = (const uint16_t *) scs;++ w.v[0] = q4[0];+ w.v[1] = q4[4];- uint16_t aux[2];const int j = (QR4_K * ((iqs / 2) / (QI8_1 / 2))) / 2;if (j < 2) {- aux[0] = scales[j + 0] & 0x3f3f;- aux[1] = scales[j + 2] & 0x3f3f;+ w.aux[0] = scales[j + 0] & 0x3f3f;+ w.aux[1] = scales[j + 2] & 0x3f3f;} else {- aux[0] = ((scales[j + 2] >> 0) & 0x0f0f) | ((scales[j - 2] & 0xc0c0) >> 2);- aux[1] = ((scales[j + 2] >> 4) & 0x0f0f) | ((scales[j - 0] & 0xc0c0) >> 2);+ w.aux[0] = ((scales[j + 2] >> 0) & 0x0f0f) | ((scales[j - 2] & 0xc0c0) >> 2);+ w.aux[1] = ((scales[j + 2] >> 4) & 0x0f0f) | ((scales[j - 0] & 0xc0c0) >> 2);}- const uint8_t * sc = (const uint8_t *) aux;- const uint8_t * m = sc + 2;+ w.dm = *dms;+ return w;+ }++ __dpct_inline__ static activations load_activations(const int8_t * q8_1_quant_ptr,+ const sycl::half2 * q8_1_ds, const int & iqs) {+ activations a;+ const int bq8_offset = QR4_K * ((iqs / 2) / (QI8_1 / 2));for (int i = 0; i < QR4_K; ++i) {- const int8_t* quant_base_ptr = q8_1_quant_ptr + (bq8_offset + i) * QK8_1;- sycl::half2 ds_values = *(q8_1_ds + bq8_offset + i);+ const int8_t * quant_base_ptr = q8_1_quant_ptr + (bq8_offset + i) * QK8_1;+ sycl::half2 ds_values = *(q8_1_ds + bq8_offset + i);- d8[i] = ds_values[0];+ a.d8[i] = ds_values[0];const int * q8 = (const int *) quant_base_ptr + ((iqs / 2) % 4);- u[2 * i + 0] = q8[0];- u[2 * i + 1] = q8[4];+ a.u[2 * i + 0] = q8[0];+ a.u[2 * i + 1] = q8[4];}- return vec_dot_q4_K_q8_1_impl_vmmq(v, u, sc, m, *dms, d8);+ return a;+ }++ __dpct_inline__ static float apply(const weights & w, const activations & a) {+ const uint8_t * sc = (const uint8_t *) w.aux;+ const uint8_t * m = sc + 2;++ return vec_dot_q4_K_q8_1_impl_vmmq(w.v, a.u, sc, m, w.dm, a.d8);+ }++ __dpct_inline__ static float dot(const weights & w, const int8_t * q8_1_quant_ptr,+ const sycl::half2 * q8_1_ds, const int & iqs) {+ const auto a = load_activations(q8_1_quant_ptr, q8_1_ds, iqs);++ return apply(w, a);+ }++ __dpct_inline__ float operator()(const void * __restrict__ vbq, const std::pair<int, int> ibx_offset,+ const std::pair<int, int> d_offset, const int8_t * q8_1_quant_ptr,+ const sycl::half2 * q8_1_ds, const int & iqs) {+ return dot(load(vbq, ibx_offset, d_offset, iqs), q8_1_quant_ptr, q8_1_ds, iqs);}};diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cppindex 1b0eaca8f0..154146dda4 100644--- a/tests/test-backend-ops.cpp+++ b/tests/test-backend-ops.cpp@@ -9435,6 +9435,21 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {test_cases.emplace_back(new test_mul_mat(type_a, GGML_TYPE_F32, 16, 8, 16*256, { 1, 1}, {1, 1}));}+ // Multi-column MMVQ coverage for the Q4_K weight-reuse path and a Q5_K control.+ for (ggml_type type_a : { GGML_TYPE_Q4_K, GGML_TYPE_Q5_K }) {+ for (int n = 1; n <= 8; ++n) {+ test_cases.emplace_back(new test_mul_mat(type_a, GGML_TYPE_F32, 4096, n, 1024, { 1, 1 }, { 1, 1 }));+ test_cases.emplace_back(new test_mul_mat(type_a, GGML_TYPE_F32, 1023, n, 4096, { 1, 1 }, { 1, 1 }));+ }+ }++ // The SYCL backend picks between one and two output rows per subgroup by row count when there+ // are two destination columns (Q4_K_MMVQ_ROW_PAIR_MIN_NROWS in ggml-sycl/mmvq.cpp). Cover both+ // sides of that boundary, including an odd row count above it for the row-pair tail.+ for (int64_t m : {6271, 6272, 6273}) {+ test_cases.emplace_back(new test_mul_mat(GGML_TYPE_Q4_K, GGML_TYPE_F32, m, 2, 1024, { 1, 1 }, { 1, 1 }));+ }+test_cases.emplace_back(new test_mul_mat(GGML_TYPE_Q4_0, GGML_TYPE_F32, 2880, 32, 2880, {1, 1}, {1, 1}));test_cases.emplace_back(new test_mul_mat(GGML_TYPE_Q8_0, GGML_TYPE_F32, 2880, 32, 2880, {1, 1}, {1, 1}));test_cases.emplace_back(new test_mul_mat(GGML_TYPE_MXFP4, GGML_TYPE_F32, 2880, 32, 2880, {1, 1}, {1, 1}));@@ -10364,6 +10379,22 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {true, 16, 8, b, false, true, false));}+ // Fused row-pair coverage: minimum rows, an even pair, and an odd tail.+ for (ggml_glu_op glu_op : { GGML_GLU_OP_SWIGLU, GGML_GLU_OP_GEGLU }) {+ for (int64_t m_batch : { 2, 3, 4 }) {+ for (int64_t rows : { 1, 2, 3 }) {+ test_cases.emplace_back(new test_mul_mat_vec_fusion(GGML_TYPE_Q4_K, glu_op, m_batch, rows, 256,+ false, 16, 8, false, false, true, false, { 1, 1 }));+ }+ }+ }++ // Both sides of the same row-count boundary as above, on the fused path.+ for (int64_t rows : {6271, 6272, 6273}) {+ test_cases.emplace_back(new test_mul_mat_vec_fusion(GGML_TYPE_Q4_K, GGML_GLU_OP_SWIGLU, 2, rows, 256,+ false, 16, 8, false, false, true, false, { 1, 1 }));+ }+for (auto gate : {GATING_FUNC_SOFTMAX, GATING_FUNC_SIGMOID, GATING_FUNC_SOFTMAX_WEIGHT, GATING_FUNC_SQRT_SOFTPLUS}) {for (bool with_norm : {false, true}) {for (bool bias_probs : {false, true}) {@@ -10651,6 +10682,16 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_perf() {}}+ // Q4_K multi-column mat-vec, at ffn_up/ffn_gate geometry (k = n_embd, m = n_ff): n sweeps the+ // per-column specializations used for short prompts and speculative/MTP verify, and m brackets+ // the row count at which the SYCL backend switches to two output rows per subgroup+ // (Q4_K_MMVQ_ROW_PAIR_MIN_NROWS in ggml-sycl/mmvq.cpp), so both sides of it can be measured.+ for (int64_t m : {4096, 6144, 6272, 14336}) {+ for (int bs : {1, 2, 3, 4, 8}) {+ test_cases.emplace_back(new test_mul_mat(GGML_TYPE_Q4_K, GGML_TYPE_F32, m, bs, 4096, {1, 1}, {1, 1}));+ }+ }+// qwen3-30b-a3bfor (int bs : {1, 4, 8, 32, 64, 128, 256, 512}) {for (ggml_type type_a : {GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_Q4_0, GGML_TYPE_Q8_0, GGML_TYPE_Q4_K, GGML_TYPE_Q6_K, GGML_TYPE_IQ2_XS}) {