LLM inference in C/C++
C++ ggml
sycl : port multi-column MMVQ from CUDA backend (#21845)
Mason Milburn7fe2ae45ab64
diff --git a/ggml/src/ggml-sycl/ggml-sycl.cpp b/ggml/src/ggml-sycl/ggml-sycl.cppindex 96138f57eb..3f246e8672 100644--- a/ggml/src/ggml-sycl/ggml-sycl.cpp+++ b/ggml/src/ggml-sycl/ggml-sycl.cpp@@ -3971,7 +3971,9 @@ static bool should_reorder_tensor(ggml_backend_sycl_context& ctx, const ggml_tenreturn !g_ggml_sycl_disable_optimize && //allow optimize, controlled by $GGML_SYCL_DISABLE_OPTctx.opt_feature.reorder && //allow this device due to good perf, skip the devices with bad perf.dst->op == GGML_OP_MUL_MAT && //limit to some supported cases of Q4_0, to do for more cases.- dst->src[1]->ne[1]==1 && dst->src[1]->ne[2]==1 && dst->src[1]->ne[3]==1;+ // ne[1] <= 8 so multi-column decode (spec / MTP verify) also bootstraps the reorder;+ // all reorderable types have a _switch_ncols kernel.+ dst->src[1]->ne[1] <= 8 && dst->src[1]->ne[2]==1 && dst->src[1]->ne[3]==1;}static void opt_for_reorder(ggml_backend_sycl_context * ctx, const ggml_tensor * src0, const ggml_tensor * /* src1 */,diff --git a/ggml/src/ggml-sycl/mmvq.cpp b/ggml/src/ggml-sycl/mmvq.cppindex abd1e49a70..cf2b59576a 100644--- a/ggml/src/ggml-sycl/mmvq.cpp+++ b/ggml/src/ggml-sycl/mmvq.cpp@@ -56,6 +56,65 @@ static void mul_mat_vec_q_reorder(const void * __restrict__ vx, const void * __r}}+template <typename reorder_vec_dot_q_sycl, int ncols_dst>+static void mul_mat_vec_q_reorder_ncols(const void * __restrict__ vx, const void * __restrict__ vy,+ float * __restrict__ dst, const int ncols, const int nrows,+ const int stride_col_y_bytes, const int stride_col_dst,+ const sycl::nd_item<3> & nd_item) {+ using block_type = ggml_sycl_reordered::block_q_t<reorder_vec_dot_q_sycl::gtype>;+ using block_traits = typename block_type::traits;++ const auto sg = nd_item.get_sub_group();+ const int sg_range = sg.get_group_linear_range();+ const int workgroup_id = nd_item.get_group_linear_id();+ const int sg_id = sg.get_group_linear_id();+ const int row = workgroup_id * sg_range + sg_id;++ if (row >= nrows) {+ return;+ }++ const int blocks_per_row = ncols / block_traits::qk;+ constexpr int blocks_per_subgroup = ceil_div(block_traits::vdr_mmvq * WARP_SIZE, block_traits::qi);+ constexpr int block_elements_per_subgroup = block_traits::qi / block_traits::vdr_mmvq;+ const int nblocks = nrows * (ncols / block_traits::qk);++ static_assert(blocks_per_subgroup > 0);+ static_assert(block_elements_per_subgroup > 0);++ float partial_sum[ncols_dst] = {0.0f};+ for (int i = sg.get_local_linear_id() / block_elements_per_subgroup; i < blocks_per_row; i += blocks_per_subgroup) {+ const int ibx = row * blocks_per_row + i;++ const auto bx_offset = block_type::get_block_offset(ibx, nblocks);+ const auto d_offset = block_type::get_d_offset(nrows, ncols, ibx);+ const int iby = i * block_type::block_to_q8_1_ratio();++#pragma unroll+ for (int elem = 0; elem < block_elements_per_subgroup; elem += WARP_SIZE) {+ const int iqs = elem + block_traits::vdr_mmvq * (sg.get_local_linear_id() % block_elements_per_subgroup);++#pragma unroll+ for (int j = 0; j < ncols_dst; ++j) {+ const char * vy_j = (const char *)vy + j * stride_col_y_bytes;+ const int8_t * q8_1_quant_ptr = (const int8_t *)vy_j + iby * QK8_1;+ const sycl::half2* q8_1_ds_ptr = (const sycl::half2 *)(vy_j + ncols + iby * sizeof(sycl::half2));++ partial_sum[j] += reorder_vec_dot_q_sycl()(vx, bx_offset, d_offset, q8_1_quant_ptr, q8_1_ds_ptr, iqs);+ }+ }+ }++#pragma unroll+ for (int j = 0; j < ncols_dst; ++j) {+ float sum = sycl::reduce_over_group(nd_item.get_sub_group(), partial_sum[j], std::plus<>());++ if (sg.leader()) {+ dst[j * stride_col_dst + row] = sum;+ }+ }+}+template <int qk, int qi, typename block_q_t, int vdr, vec_dot_q_sycl_t vec_dot_q_sycl>static void mul_mat_vec_q(const void * __restrict__ vx, const void * __restrict__ vy, float * __restrict__ dst,const int ncols, const int nrows, const sycl::nd_item<3> & item_ct1) {@@ -100,6 +159,70 @@ static void mul_mat_vec_q(const void * __restrict__ vx, const void * __restrict_}}+template <int qk, int qi, typename block_q_t, int vdr,+ vec_dot_q_sycl_t vec_dot_q_sycl, int ncols_dst>+static void mul_mat_vec_q_ncols(+ const void * __restrict__ vx,+ const void * __restrict__ vy,+ float * __restrict__ dst,+ const int ncols,+ const int nrows,+ const int stride_col_y,+ const int stride_col_dst,+ const sycl::nd_item<3> & item_ct1) {++ const int row = item_ct1.get_group(2) * item_ct1.get_local_range(1)+ + item_ct1.get_local_id(1);++ if (row >= nrows) {+ return;+ }++ const int blocks_per_row = ncols / qk;+ constexpr int blocks_per_warp = (vdr * WARP_SIZE + qi - 1) / qi;++ // partial sums: one per output column+ float tmp[ncols_dst] = {0.0f};++ const block_q_t * x = (const block_q_t *) vx;+ const block_q8_1 * y = (const block_q8_1 *) vy;++ for (int i = item_ct1.get_local_id(2) / (qi / vdr);+ i < blocks_per_row;+ i += blocks_per_warp) {++ const int ibx = row * blocks_per_row + i;+ const int iby = i * (qk / QK8_1);++ // read weight block once, dot against all columns+ for (size_t elem = 0; elem < qi / vdr; elem += WARP_SIZE) {+ const int iqs = elem + vdr * (item_ct1.get_local_id(2) % (qi / vdr));++#pragma unroll+ for (int j = 0; j < ncols_dst; ++j) {+ tmp[j] += vec_dot_q_sycl(&x[ibx], &y[j * stride_col_y + iby], iqs);+ }+ }+ }++ // reduce within subgroup+#pragma unroll+ for (int j = 0; j < ncols_dst; ++j) {+#pragma unroll+ for (int mask = WARP_SIZE / 2; mask > 0; mask >>= 1) {+ tmp[j] += dpct::permute_sub_group_by_xor(+ item_ct1.get_sub_group(), tmp[j], mask);+ }+ }++ if (item_ct1.get_local_id(2) == 0) {+#pragma unroll+ for (int j = 0; j < ncols_dst; ++j) {+ dst[j * stride_col_dst + row] = tmp[j];+ }+ }+}+template <int qk, int qi, typename block_q_t, int vdr>static void mul_mat_vec_q_iq2_xxs_q8_1(const void *__restrict__ vx,const void *__restrict__ vy,@@ -553,6 +676,45 @@ static void reorder_mul_mat_vec_q4_0_q8_1_sycl(const void * vx, const void * vy,});}+template <int ncols_dst>+static void reorder_mul_mat_vec_q4_0_q8_1_sycl_ncols(+ const void * vx, const void * vy, float * dst,+ const int ncols, const int nrows,+ const int stride_col_y_bytes, const int stride_col_dst,+ dpct::queue_ptr stream) {+ GGML_ASSERT(ncols % QK4_0 == 0);+ const int block_num_y = ceil_div(nrows, GGML_SYCL_MMV_Y);+ constexpr size_t num_subgroups = 16;+ GGML_ASSERT(block_num_y % num_subgroups == 0);+ const sycl::range<3> global_size(1, GGML_SYCL_MMV_Y, block_num_y * WARP_SIZE);+ const sycl::range<3> workgroup_size(1, GGML_SYCL_MMV_Y, num_subgroups * WARP_SIZE);+ stream->submit([&](sycl::handler & cgh) {+ cgh.parallel_for(sycl::nd_range<3>(global_size, workgroup_size),+ [=](sycl::nd_item<3> nd_item) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {+ mul_mat_vec_q_reorder_ncols<reorder_vec_dot_q_sycl<GGML_TYPE_Q4_0>, ncols_dst>(+ vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, nd_item);+ });+ });+}++static void reorder_mul_mat_vec_q4_0_q8_1_sycl_switch_ncols(+ const void * vx, const void * vy, float * dst,+ const int ncols, const int nrows, const int ncols_dst,+ const int stride_col_y_bytes, const int stride_col_dst,+ dpct::queue_ptr stream) {+ switch (ncols_dst) {+ case 1: reorder_mul_mat_vec_q4_0_q8_1_sycl(vx, vy, dst, ncols, nrows, stream); break;+ case 2: reorder_mul_mat_vec_q4_0_q8_1_sycl_ncols<2>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break;+ case 3: reorder_mul_mat_vec_q4_0_q8_1_sycl_ncols<3>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break;+ case 4: reorder_mul_mat_vec_q4_0_q8_1_sycl_ncols<4>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break;+ case 5: reorder_mul_mat_vec_q4_0_q8_1_sycl_ncols<5>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break;+ case 6: reorder_mul_mat_vec_q4_0_q8_1_sycl_ncols<6>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break;+ case 7: reorder_mul_mat_vec_q4_0_q8_1_sycl_ncols<7>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break;+ case 8: reorder_mul_mat_vec_q4_0_q8_1_sycl_ncols<8>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break;+ default: GGML_ABORT("unsupported ncols_dst=%d for Q4_0 reorder multi-col MMVQ", ncols_dst);+ }+}+static void mul_mat_vec_q4_0_q8_1_sycl(const void * vx, const void * vy, float * dst, const int ncols, const int nrows,dpct::queue_ptr stream) {GGML_ASSERT(ncols % QK4_0 == 0);@@ -571,6 +733,45 @@ static void mul_mat_vec_q4_0_q8_1_sycl(const void * vx, const void * vy, float *}}+template <int ncols_dst>+static void mul_mat_vec_q4_0_q8_1_sycl_ncols(+ const void * vx, const void * vy, float * dst,+ const int ncols, const int nrows,+ const int stride_col_y, const int stride_col_dst,+ dpct::queue_ptr stream) {+ GGML_ASSERT(ncols % QK4_0 == 0);+ const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y;+ const sycl::range<3> block_nums(1, 1, block_num_y);+ const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE);+ stream->submit([&](sycl::handler & cgh) {+ cgh.parallel_for(+ sycl::nd_range<3>(block_nums * block_dims, block_dims),+ [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {+ mul_mat_vec_q_ncols<QK4_0, QI4_0, block_q4_0,+ VDR_Q4_0_Q8_1_MMVQ, vec_dot_q4_0_q8_1, ncols_dst>(+ vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, item_ct1);+ });+ });+}++static void mul_mat_vec_q4_0_q8_1_sycl_switch_ncols(+ const void * vx, const void * vy, float * dst,+ const int ncols, const int nrows, const int ncols_dst,+ const int stride_col_y, const int stride_col_dst,+ dpct::queue_ptr stream) {+ switch (ncols_dst) {+ case 1: mul_mat_vec_q4_0_q8_1_sycl(vx, vy, dst, ncols, nrows, stream); break;+ case 2: mul_mat_vec_q4_0_q8_1_sycl_ncols<2>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;+ case 3: mul_mat_vec_q4_0_q8_1_sycl_ncols<3>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;+ case 4: mul_mat_vec_q4_0_q8_1_sycl_ncols<4>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;+ case 5: mul_mat_vec_q4_0_q8_1_sycl_ncols<5>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;+ case 6: mul_mat_vec_q4_0_q8_1_sycl_ncols<6>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;+ case 7: mul_mat_vec_q4_0_q8_1_sycl_ncols<7>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;+ case 8: mul_mat_vec_q4_0_q8_1_sycl_ncols<8>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;+ default: GGML_ABORT("unsupported ncols_dst=%d for Q4_0 multi-col MMVQ", ncols_dst);+ }+}+static void mul_mat_vec_q4_1_q8_1_sycl(const void *vx, const void *vy,float *dst, const int ncols,const int nrows,@@ -595,6 +796,45 @@ static void mul_mat_vec_q4_1_q8_1_sycl(const void *vx, const void *vy,}}+template <int ncols_dst>+static void mul_mat_vec_q4_1_q8_1_sycl_ncols(+ const void * vx, const void * vy, float * dst,+ const int ncols, const int nrows,+ const int stride_col_y, const int stride_col_dst,+ dpct::queue_ptr stream) {+ GGML_ASSERT(ncols % QK4_1 == 0);+ const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y;+ const sycl::range<3> block_nums(1, 1, block_num_y);+ const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE);+ stream->submit([&](sycl::handler & cgh) {+ cgh.parallel_for(+ sycl::nd_range<3>(block_nums * block_dims, block_dims),+ [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {+ mul_mat_vec_q_ncols<QK4_0, QI4_1, block_q4_1,+ VDR_Q4_1_Q8_1_MMVQ, vec_dot_q4_1_q8_1, ncols_dst>(+ vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, item_ct1);+ });+ });+}++static void mul_mat_vec_q4_1_q8_1_sycl_switch_ncols(+ const void * vx, const void * vy, float * dst,+ const int ncols, const int nrows, const int ncols_dst,+ const int stride_col_y, const int stride_col_dst,+ dpct::queue_ptr stream) {+ switch (ncols_dst) {+ case 1: mul_mat_vec_q4_1_q8_1_sycl(vx, vy, dst, ncols, nrows, stream); break;+ case 2: mul_mat_vec_q4_1_q8_1_sycl_ncols<2>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;+ case 3: mul_mat_vec_q4_1_q8_1_sycl_ncols<3>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;+ case 4: mul_mat_vec_q4_1_q8_1_sycl_ncols<4>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;+ case 5: mul_mat_vec_q4_1_q8_1_sycl_ncols<5>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;+ case 6: mul_mat_vec_q4_1_q8_1_sycl_ncols<6>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;+ case 7: mul_mat_vec_q4_1_q8_1_sycl_ncols<7>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;+ case 8: mul_mat_vec_q4_1_q8_1_sycl_ncols<8>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;+ default: GGML_ABORT("unsupported ncols_dst=%d for Q4_1 multi-col MMVQ", ncols_dst);+ }+}+static void mul_mat_vec_mxfp4_q8_1_sycl(const void * vx, const void * vy, float * dst, const int ncols, const int nrows,dpct::queue_ptr stream) {GGML_ASSERT(ncols % QK_MXFP4 == 0);@@ -613,6 +853,45 @@ static void mul_mat_vec_mxfp4_q8_1_sycl(const void * vx, const void * vy, float}}+template <int ncols_dst>+static void mul_mat_vec_mxfp4_q8_1_sycl_ncols(+ const void * vx, const void * vy, float * dst,+ const int ncols, const int nrows,+ const int stride_col_y, const int stride_col_dst,+ dpct::queue_ptr stream) {+ GGML_ASSERT(ncols % QK_MXFP4 == 0);+ const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y;+ const sycl::range<3> block_nums(1, 1, block_num_y);+ const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE);+ stream->submit([&](sycl::handler & cgh) {+ cgh.parallel_for(+ sycl::nd_range<3>(block_nums * block_dims, block_dims),+ [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {+ mul_mat_vec_q_ncols<QK_MXFP4, QI_MXFP4, block_mxfp4,+ VDR_MXFP4_Q8_1_MMVQ, vec_dot_mxfp4_q8_1, ncols_dst>(+ vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, item_ct1);+ });+ });+}++static void mul_mat_vec_mxfp4_q8_1_sycl_switch_ncols(+ const void * vx, const void * vy, float * dst,+ const int ncols, const int nrows, const int ncols_dst,+ const int stride_col_y, const int stride_col_dst,+ dpct::queue_ptr stream) {+ switch (ncols_dst) {+ case 1: mul_mat_vec_mxfp4_q8_1_sycl(vx, vy, dst, ncols, nrows, stream); break;+ case 2: mul_mat_vec_mxfp4_q8_1_sycl_ncols<2>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;+ case 3: mul_mat_vec_mxfp4_q8_1_sycl_ncols<3>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;+ case 4: mul_mat_vec_mxfp4_q8_1_sycl_ncols<4>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;+ case 5: mul_mat_vec_mxfp4_q8_1_sycl_ncols<5>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;+ case 6: mul_mat_vec_mxfp4_q8_1_sycl_ncols<6>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;+ case 7: mul_mat_vec_mxfp4_q8_1_sycl_ncols<7>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;+ case 8: mul_mat_vec_mxfp4_q8_1_sycl_ncols<8>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;+ default: GGML_ABORT("unsupported ncols_dst=%d for MXFP4 multi-col MMVQ", ncols_dst);+ }+}+static void mul_mat_vec_nvfp4_q8_1_sycl(const void * vx, const void * vy, float * dst, const int ncols, const int nrows,dpct::queue_ptr stream) {GGML_ASSERT(ncols % QK_NVFP4 == 0);@@ -631,6 +910,45 @@ static void mul_mat_vec_nvfp4_q8_1_sycl(const void * vx, const void * vy, float}}+template <int ncols_dst>+static void mul_mat_vec_nvfp4_q8_1_sycl_ncols(+ const void * vx, const void * vy, float * dst,+ const int ncols, const int nrows,+ const int stride_col_y, const int stride_col_dst,+ dpct::queue_ptr stream) {+ GGML_ASSERT(ncols % QK_NVFP4 == 0);+ const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y;+ const sycl::range<3> block_nums(1, 1, block_num_y);+ const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE);+ stream->submit([&](sycl::handler & cgh) {+ cgh.parallel_for(+ sycl::nd_range<3>(block_nums * block_dims, block_dims),+ [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {+ mul_mat_vec_q_ncols<QK_NVFP4, QI_NVFP4, block_nvfp4,+ VDR_NVFP4_Q8_1_MMVQ, vec_dot_nvfp4_q8_1, ncols_dst>(+ vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, item_ct1);+ });+ });+}++static void mul_mat_vec_nvfp4_q8_1_sycl_switch_ncols(+ const void * vx, const void * vy, float * dst,+ const int ncols, const int nrows, const int ncols_dst,+ const int stride_col_y, const int stride_col_dst,+ dpct::queue_ptr stream) {+ switch (ncols_dst) {+ case 1: mul_mat_vec_nvfp4_q8_1_sycl(vx, vy, dst, ncols, nrows, stream); break;+ case 2: mul_mat_vec_nvfp4_q8_1_sycl_ncols<2>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;+ case 3: mul_mat_vec_nvfp4_q8_1_sycl_ncols<3>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;+ case 4: mul_mat_vec_nvfp4_q8_1_sycl_ncols<4>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;+ case 5: mul_mat_vec_nvfp4_q8_1_sycl_ncols<5>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;+ case 6: mul_mat_vec_nvfp4_q8_1_sycl_ncols<6>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;+ case 7: mul_mat_vec_nvfp4_q8_1_sycl_ncols<7>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;+ case 8: mul_mat_vec_nvfp4_q8_1_sycl_ncols<8>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;+ default: GGML_ABORT("unsupported ncols_dst=%d for NVFP4 multi-col MMVQ", ncols_dst);+ }+}+static void mul_mat_vec_q5_0_q8_1_sycl(const void *vx, const void *vy,float *dst, const int ncols,const int nrows,@@ -655,6 +973,45 @@ static void mul_mat_vec_q5_0_q8_1_sycl(const void *vx, const void *vy,}}+template <int ncols_dst>+static void mul_mat_vec_q5_0_q8_1_sycl_ncols(+ const void * vx, const void * vy, float * dst,+ const int ncols, const int nrows,+ const int stride_col_y, const int stride_col_dst,+ dpct::queue_ptr stream) {+ GGML_ASSERT(ncols % QK5_0 == 0);+ const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y;+ const sycl::range<3> block_nums(1, 1, block_num_y);+ const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE);+ stream->submit([&](sycl::handler & cgh) {+ cgh.parallel_for(+ sycl::nd_range<3>(block_nums * block_dims, block_dims),+ [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {+ mul_mat_vec_q_ncols<QK5_0, QI5_0, block_q5_0,+ VDR_Q5_0_Q8_1_MMVQ, vec_dot_q5_0_q8_1, ncols_dst>(+ vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, item_ct1);+ });+ });+}++static void mul_mat_vec_q5_0_q8_1_sycl_switch_ncols(+ const void * vx, const void * vy, float * dst,+ const int ncols, const int nrows, const int ncols_dst,+ const int stride_col_y, const int stride_col_dst,+ dpct::queue_ptr stream) {+ switch (ncols_dst) {+ case 1: mul_mat_vec_q5_0_q8_1_sycl(vx, vy, dst, ncols, nrows, stream); break;+ case 2: mul_mat_vec_q5_0_q8_1_sycl_ncols<2>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;+ case 3: mul_mat_vec_q5_0_q8_1_sycl_ncols<3>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;+ case 4: mul_mat_vec_q5_0_q8_1_sycl_ncols<4>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;+ case 5: mul_mat_vec_q5_0_q8_1_sycl_ncols<5>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;+ case 6: mul_mat_vec_q5_0_q8_1_sycl_ncols<6>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;+ case 7: mul_mat_vec_q5_0_q8_1_sycl_ncols<7>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;+ case 8: mul_mat_vec_q5_0_q8_1_sycl_ncols<8>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;+ default: GGML_ABORT("unsupported ncols_dst=%d for Q5_0 multi-col MMVQ", ncols_dst);+ }+}+static void mul_mat_vec_q5_1_q8_1_sycl(const void *vx, const void *vy,float *dst, const int ncols,const int nrows,@@ -679,6 +1036,45 @@ static void mul_mat_vec_q5_1_q8_1_sycl(const void *vx, const void *vy,}}+template <int ncols_dst>+static void mul_mat_vec_q5_1_q8_1_sycl_ncols(+ const void * vx, const void * vy, float * dst,+ const int ncols, const int nrows,+ const int stride_col_y, const int stride_col_dst,+ dpct::queue_ptr stream) {+ GGML_ASSERT(ncols % QK5_1 == 0);+ const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y;+ const sycl::range<3> block_nums(1, 1, block_num_y);+ const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE);+ stream->submit([&](sycl::handler & cgh) {+ cgh.parallel_for(+ sycl::nd_range<3>(block_nums * block_dims, block_dims),+ [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {+ mul_mat_vec_q_ncols<QK5_1, QI5_1, block_q5_1,+ VDR_Q5_1_Q8_1_MMVQ, vec_dot_q5_1_q8_1, ncols_dst>(+ vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, item_ct1);+ });+ });+}++static void mul_mat_vec_q5_1_q8_1_sycl_switch_ncols(+ const void * vx, const void * vy, float * dst,+ const int ncols, const int nrows, const int ncols_dst,+ const int stride_col_y, const int stride_col_dst,+ dpct::queue_ptr stream) {+ switch (ncols_dst) {+ case 1: mul_mat_vec_q5_1_q8_1_sycl(vx, vy, dst, ncols, nrows, stream); break;+ case 2: mul_mat_vec_q5_1_q8_1_sycl_ncols<2>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;+ case 3: mul_mat_vec_q5_1_q8_1_sycl_ncols<3>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;+ case 4: mul_mat_vec_q5_1_q8_1_sycl_ncols<4>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;+ case 5: mul_mat_vec_q5_1_q8_1_sycl_ncols<5>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;+ case 6: mul_mat_vec_q5_1_q8_1_sycl_ncols<6>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;+ case 7: mul_mat_vec_q5_1_q8_1_sycl_ncols<7>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;+ case 8: mul_mat_vec_q5_1_q8_1_sycl_ncols<8>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;+ default: GGML_ABORT("unsupported ncols_dst=%d for Q5_1 multi-col MMVQ", ncols_dst);+ }+}+static void reorder_mul_mat_vec_q8_0_q8_1_sycl(const void * vx, const void * vy, float * dst, const int ncols,const int nrows, dpct::queue_ptr stream) {GGML_ASSERT(ncols % QK8_0 == 0);@@ -698,6 +1094,45 @@ static void reorder_mul_mat_vec_q8_0_q8_1_sycl(const void * vx, const void * vy,});}+template <int ncols_dst>+static void reorder_mul_mat_vec_q8_0_q8_1_sycl_ncols(+ const void * vx, const void * vy, float * dst,+ const int ncols, const int nrows,+ const int stride_col_y_bytes, const int stride_col_dst,+ dpct::queue_ptr stream) {+ GGML_ASSERT(ncols % QK8_0 == 0);+ const int block_num_y = ceil_div(nrows, GGML_SYCL_MMV_Y);+ constexpr size_t num_subgroups = 16;+ GGML_ASSERT(block_num_y % num_subgroups == 0);+ const sycl::range<3> global_size(1, GGML_SYCL_MMV_Y, block_num_y * WARP_SIZE);+ const sycl::range<3> workgroup_size(1, GGML_SYCL_MMV_Y, num_subgroups * WARP_SIZE);+ stream->submit([&](sycl::handler & cgh) {+ cgh.parallel_for(sycl::nd_range<3>(global_size, workgroup_size),+ [=](sycl::nd_item<3> nd_item) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {+ mul_mat_vec_q_reorder_ncols<reorder_vec_dot_q_sycl<GGML_TYPE_Q8_0>, ncols_dst>(+ vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, nd_item);+ });+ });+}++static void reorder_mul_mat_vec_q8_0_q8_1_sycl_switch_ncols(+ const void * vx, const void * vy, float * dst,+ const int ncols, const int nrows, const int ncols_dst,+ const int stride_col_y_bytes, const int stride_col_dst,+ dpct::queue_ptr stream) {+ switch (ncols_dst) {+ case 1: reorder_mul_mat_vec_q8_0_q8_1_sycl(vx, vy, dst, ncols, nrows, stream); break;+ case 2: reorder_mul_mat_vec_q8_0_q8_1_sycl_ncols<2>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break;+ case 3: reorder_mul_mat_vec_q8_0_q8_1_sycl_ncols<3>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break;+ case 4: reorder_mul_mat_vec_q8_0_q8_1_sycl_ncols<4>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break;+ case 5: reorder_mul_mat_vec_q8_0_q8_1_sycl_ncols<5>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break;+ case 6: reorder_mul_mat_vec_q8_0_q8_1_sycl_ncols<6>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break;+ case 7: reorder_mul_mat_vec_q8_0_q8_1_sycl_ncols<7>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break;+ case 8: reorder_mul_mat_vec_q8_0_q8_1_sycl_ncols<8>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break;+ default: GGML_ABORT("unsupported ncols_dst=%d for Q8_0 reorder multi-col MMVQ", ncols_dst);+ }+}+static void mul_mat_vec_q8_0_q8_1_sycl(const void *vx, const void *vy,float *dst, const int ncols,const int nrows,@@ -722,6 +1157,45 @@ static void mul_mat_vec_q8_0_q8_1_sycl(const void *vx, const void *vy,}}+template <int ncols_dst>+static void mul_mat_vec_q8_0_q8_1_sycl_ncols(+ const void * vx, const void * vy, float * dst,+ const int ncols, const int nrows,+ const int stride_col_y, const int stride_col_dst,+ dpct::queue_ptr stream) {+ GGML_ASSERT(ncols % QK8_0 == 0);+ const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y;+ const sycl::range<3> block_nums(1, 1, block_num_y);+ const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE);+ stream->submit([&](sycl::handler & cgh) {+ cgh.parallel_for(+ sycl::nd_range<3>(block_nums * block_dims, block_dims),+ [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {+ mul_mat_vec_q_ncols<QK8_0, QI8_0, block_q8_0,+ VDR_Q8_0_Q8_1_MMVQ, vec_dot_q8_0_q8_1, ncols_dst>(+ vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, item_ct1);+ });+ });+}++static void mul_mat_vec_q8_0_q8_1_sycl_switch_ncols(+ const void * vx, const void * vy, float * dst,+ const int ncols, const int nrows, const int ncols_dst,+ const int stride_col_y, const int stride_col_dst,+ dpct::queue_ptr stream) {+ switch (ncols_dst) {+ case 1: mul_mat_vec_q8_0_q8_1_sycl(vx, vy, dst, ncols, nrows, stream); break;+ case 2: mul_mat_vec_q8_0_q8_1_sycl_ncols<2>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;+ case 3: mul_mat_vec_q8_0_q8_1_sycl_ncols<3>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;+ case 4: mul_mat_vec_q8_0_q8_1_sycl_ncols<4>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;+ case 5: mul_mat_vec_q8_0_q8_1_sycl_ncols<5>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;+ case 6: mul_mat_vec_q8_0_q8_1_sycl_ncols<6>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;+ case 7: mul_mat_vec_q8_0_q8_1_sycl_ncols<7>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;+ case 8: mul_mat_vec_q8_0_q8_1_sycl_ncols<8>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;+ default: GGML_ABORT("unsupported ncols_dst=%d for Q8_0 multi-col MMVQ", ncols_dst);+ }+}+static void mul_mat_vec_q2_K_q8_1_sycl(const void *vx, const void *vy,float *dst, const int ncols,const int nrows,@@ -746,6 +1220,45 @@ static void mul_mat_vec_q2_K_q8_1_sycl(const void *vx, const void *vy,}}+template <int ncols_dst>+static void mul_mat_vec_q2_K_q8_1_sycl_ncols(+ const void * vx, const void * vy, float * dst,+ const int ncols, const int nrows,+ const int stride_col_y, const int stride_col_dst,+ dpct::queue_ptr stream) {+ GGML_ASSERT(ncols % QK_K == 0);+ const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y;+ const sycl::range<3> block_nums(1, 1, block_num_y);+ const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE);+ stream->submit([&](sycl::handler & cgh) {+ cgh.parallel_for(+ sycl::nd_range<3>(block_nums * block_dims, block_dims),+ [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {+ mul_mat_vec_q_ncols<QK_K, QI2_K, block_q2_K,+ VDR_Q2_K_Q8_1_MMVQ, vec_dot_q2_K_q8_1, ncols_dst>(+ vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, item_ct1);+ });+ });+}++static void mul_mat_vec_q2_K_q8_1_sycl_switch_ncols(+ const void * vx, const void * vy, float * dst,+ const int ncols, const int nrows, const int ncols_dst,+ const int stride_col_y, const int stride_col_dst,+ dpct::queue_ptr stream) {+ switch (ncols_dst) {+ case 1: mul_mat_vec_q2_K_q8_1_sycl(vx, vy, dst, ncols, nrows, stream); break;+ case 2: mul_mat_vec_q2_K_q8_1_sycl_ncols<2>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;+ case 3: mul_mat_vec_q2_K_q8_1_sycl_ncols<3>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;+ case 4: mul_mat_vec_q2_K_q8_1_sycl_ncols<4>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;+ case 5: mul_mat_vec_q2_K_q8_1_sycl_ncols<5>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;+ case 6: mul_mat_vec_q2_K_q8_1_sycl_ncols<6>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;+ case 7: mul_mat_vec_q2_K_q8_1_sycl_ncols<7>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;+ case 8: mul_mat_vec_q2_K_q8_1_sycl_ncols<8>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;+ default: GGML_ABORT("unsupported ncols_dst=%d for Q2_K multi-col MMVQ", ncols_dst);+ }+}+static void mul_mat_vec_q3_K_q8_1_sycl(const void *vx, const void *vy,float *dst, const int ncols,const int nrows,@@ -790,6 +1303,85 @@ static void reorder_mul_mat_vec_q3_k_q8_1_sycl(const void * vx, const void * vy,});}+template <int ncols_dst>+static void reorder_mul_mat_vec_q3_k_q8_1_sycl_ncols(+ const void * vx, const void * vy, float * dst,+ const int ncols, const int nrows,+ const int stride_col_y_bytes, const int stride_col_dst,+ dpct::queue_ptr stream) {+ GGML_ASSERT(ncols % QK_K == 0);+ const int block_num_y = ceil_div(nrows, GGML_SYCL_MMV_Y);+ constexpr size_t num_subgroups = 16;+ GGML_ASSERT(block_num_y % num_subgroups == 0);+ const sycl::range<3> global_size(1, GGML_SYCL_MMV_Y, block_num_y * WARP_SIZE);+ const sycl::range<3> workgroup_size(1, GGML_SYCL_MMV_Y, num_subgroups * WARP_SIZE);+ stream->submit([&](sycl::handler & cgh) {+ cgh.parallel_for(sycl::nd_range<3>(global_size, workgroup_size),+ [=](sycl::nd_item<3> nd_item) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {+ mul_mat_vec_q_reorder_ncols<reorder_vec_dot_q_sycl<GGML_TYPE_Q3_K>, ncols_dst>(+ vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, nd_item);+ });+ });+}++static void reorder_mul_mat_vec_q3_k_q8_1_sycl_switch_ncols(+ const void * vx, const void * vy, float * dst,+ const int ncols, const int nrows, const int ncols_dst,+ const int stride_col_y_bytes, const int stride_col_dst,+ dpct::queue_ptr stream) {+ switch (ncols_dst) {+ case 1: reorder_mul_mat_vec_q3_k_q8_1_sycl(vx, vy, dst, ncols, nrows, stream); break;+ case 2: reorder_mul_mat_vec_q3_k_q8_1_sycl_ncols<2>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break;+ case 3: reorder_mul_mat_vec_q3_k_q8_1_sycl_ncols<3>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break;+ case 4: reorder_mul_mat_vec_q3_k_q8_1_sycl_ncols<4>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break;+ case 5: reorder_mul_mat_vec_q3_k_q8_1_sycl_ncols<5>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break;+ case 6: reorder_mul_mat_vec_q3_k_q8_1_sycl_ncols<6>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break;+ case 7: reorder_mul_mat_vec_q3_k_q8_1_sycl_ncols<7>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break;+ case 8: reorder_mul_mat_vec_q3_k_q8_1_sycl_ncols<8>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break;+ default: GGML_ABORT("unsupported ncols_dst=%d for Q3_K reorder multi-col MMVQ", ncols_dst);+ }+}++template <int ncols_dst>+static void mul_mat_vec_q3_K_q8_1_sycl_ncols(+ const void * vx, const void * vy, float * dst,+ const int ncols, const int nrows,+ const int stride_col_y, const int stride_col_dst,+ dpct::queue_ptr stream) {+ GGML_ASSERT(ncols % QK_K == 0);+ const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y;+ const sycl::range<3> block_nums(1, 1, block_num_y);+ const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE);+ stream->submit([&](sycl::handler & cgh) {+ cgh.parallel_for(+ sycl::nd_range<3>(block_nums * block_dims, block_dims),+ [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {+ mul_mat_vec_q_ncols<QK_K, QI3_K, block_q3_K,+ VDR_Q3_K_Q8_1_MMVQ, vec_dot_q3_K_q8_1, ncols_dst>(+ vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, item_ct1);+ });+ });+}++static void mul_mat_vec_q3_K_q8_1_sycl_switch_ncols(+ const void * vx, const void * vy, float * dst,+ const int ncols, const int nrows, const int ncols_dst,+ const int stride_col_y, const int stride_col_dst,+ dpct::queue_ptr stream) {+ switch (ncols_dst) {+ case 1: mul_mat_vec_q3_K_q8_1_sycl(vx, vy, dst, ncols, nrows, stream); break;+ case 2: mul_mat_vec_q3_K_q8_1_sycl_ncols<2>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;+ case 3: mul_mat_vec_q3_K_q8_1_sycl_ncols<3>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;+ case 4: mul_mat_vec_q3_K_q8_1_sycl_ncols<4>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;+ case 5: mul_mat_vec_q3_K_q8_1_sycl_ncols<5>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;+ case 6: mul_mat_vec_q3_K_q8_1_sycl_ncols<6>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;+ case 7: mul_mat_vec_q3_K_q8_1_sycl_ncols<7>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;+ case 8: mul_mat_vec_q3_K_q8_1_sycl_ncols<8>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;+ default: GGML_ABORT("unsupported ncols_dst=%d for Q3_K multi-col MMVQ", ncols_dst);+ }+}++static void mul_mat_vec_q4_K_q8_1_sycl(const void *vx, const void *vy,float *dst, const int ncols,const int nrows,@@ -814,6 +1406,51 @@ static void mul_mat_vec_q4_K_q8_1_sycl(const void *vx, const void *vy,}}+template <int ncols_dst>+static void mul_mat_vec_q4_K_q8_1_sycl_ncols(+ const void * vx, const void * vy, float * dst,+ const int ncols, const int nrows,+ const int stride_col_y, const int stride_col_dst,+ dpct::queue_ptr stream) {+ GGML_ASSERT(ncols % QK_K == 0);+ const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y;+ const sycl::range<3> block_nums(1, 1, block_num_y);+ const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE);++ stream->submit([&](sycl::handler & cgh) {+ cgh.parallel_for(+ sycl::nd_range<3>(block_nums * block_dims, block_dims),+ [=](sycl::nd_item<3> item_ct1)+ [[sycl::reqd_sub_group_size(WARP_SIZE)]] {+ mul_mat_vec_q_ncols<QK_K, QI4_K, block_q4_K,+ VDR_Q4_K_Q8_1_MMVQ,+ vec_dot_q4_K_q8_1,+ ncols_dst>(+ vx, vy, dst, ncols, nrows,+ stride_col_y, stride_col_dst, item_ct1);+ });+ });+}++static void mul_mat_vec_q4_K_q8_1_sycl_switch_ncols(+ const void * vx, const void * vy, float * dst,+ const int ncols, const int nrows,+ const int ncols_dst,+ const int stride_col_y, const int stride_col_dst,+ dpct::queue_ptr stream) {+ switch (ncols_dst) {+ case 1: mul_mat_vec_q4_K_q8_1_sycl(vx, vy, dst, ncols, nrows, stream); break;+ case 2: mul_mat_vec_q4_K_q8_1_sycl_ncols<2>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;+ case 3: mul_mat_vec_q4_K_q8_1_sycl_ncols<3>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;+ case 4: mul_mat_vec_q4_K_q8_1_sycl_ncols<4>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;+ case 5: mul_mat_vec_q4_K_q8_1_sycl_ncols<5>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;+ case 6: mul_mat_vec_q4_K_q8_1_sycl_ncols<6>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;+ case 7: mul_mat_vec_q4_K_q8_1_sycl_ncols<7>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;+ case 8: mul_mat_vec_q4_K_q8_1_sycl_ncols<8>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;+ default: GGML_ABORT("unsupported ncols_dst=%d for Q4_K multi-col MMVQ", ncols_dst);+ }+}+static void reorder_mul_mat_vec_q4_k_q8_1_sycl(const void * vx, const void * vy, float * dst, const int ncols,const int nrows, dpct::queue_ptr stream) {GGML_ASSERT(ncols % QK_K == 0);@@ -834,6 +1471,44 @@ static void reorder_mul_mat_vec_q4_k_q8_1_sycl(const void * vx, const void * vy,});}+template <int ncols_dst>+static void reorder_mul_mat_vec_q4_k_q8_1_sycl_ncols(+ const void * vx, const void * vy, float * dst,+ const int ncols, const int nrows,+ const int stride_col_y_bytes, const int stride_col_dst,+ dpct::queue_ptr stream) {+ GGML_ASSERT(ncols % QK_K == 0);+ const int block_num_y = ceil_div(nrows, GGML_SYCL_MMV_Y);+ constexpr size_t num_subgroups = 16;+ GGML_ASSERT(block_num_y % num_subgroups == 0);+ const sycl::range<3> global_size(1, GGML_SYCL_MMV_Y, block_num_y * WARP_SIZE);+ const sycl::range<3> workgroup_size(1, GGML_SYCL_MMV_Y, num_subgroups * WARP_SIZE);+ stream->submit([&](sycl::handler & cgh) {+ cgh.parallel_for(sycl::nd_range<3>(global_size, workgroup_size),+ [=](sycl::nd_item<3> nd_item) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {+ mul_mat_vec_q_reorder_ncols<reorder_vec_dot_q_sycl<GGML_TYPE_Q4_K>, ncols_dst>(+ vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, nd_item);+ });+ });+}++static void reorder_mul_mat_vec_q4_k_q8_1_sycl_switch_ncols(+ const void * vx, const void * vy, float * dst,+ const int ncols, const int nrows, const int ncols_dst,+ const int stride_col_y_bytes, const int stride_col_dst,+ dpct::queue_ptr stream) {+ switch (ncols_dst) {+ case 1: reorder_mul_mat_vec_q4_k_q8_1_sycl(vx, vy, dst, ncols, nrows, stream); break;+ case 2: reorder_mul_mat_vec_q4_k_q8_1_sycl_ncols<2>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break;+ case 3: reorder_mul_mat_vec_q4_k_q8_1_sycl_ncols<3>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break;+ case 4: reorder_mul_mat_vec_q4_k_q8_1_sycl_ncols<4>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break;+ case 5: reorder_mul_mat_vec_q4_k_q8_1_sycl_ncols<5>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break;+ case 6: reorder_mul_mat_vec_q4_k_q8_1_sycl_ncols<6>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break;+ case 7: reorder_mul_mat_vec_q4_k_q8_1_sycl_ncols<7>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break;+ case 8: reorder_mul_mat_vec_q4_k_q8_1_sycl_ncols<8>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break;+ default: GGML_ABORT("unsupported ncols_dst=%d for Q4_K reorder multi-col MMVQ", ncols_dst);+ }+}static void mul_mat_vec_q5_K_q8_1_sycl(const void *vx, const void *vy,float *dst, const int ncols,@@ -859,6 +1534,51 @@ static void mul_mat_vec_q5_K_q8_1_sycl(const void *vx, const void *vy,}}+template <int ncols_dst>+static void mul_mat_vec_q5_K_q8_1_sycl_ncols(+ const void * vx, const void * vy, float * dst,+ const int ncols, const int nrows,+ const int stride_col_y, const int stride_col_dst,+ dpct::queue_ptr stream) {+ GGML_ASSERT(ncols % QK_K == 0);+ const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y;+ const sycl::range<3> block_nums(1, 1, block_num_y);+ const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE);++ stream->submit([&](sycl::handler & cgh) {+ cgh.parallel_for(+ sycl::nd_range<3>(block_nums * block_dims, block_dims),+ [=](sycl::nd_item<3> item_ct1)+ [[sycl::reqd_sub_group_size(WARP_SIZE)]] {+ mul_mat_vec_q_ncols<QK_K, QI5_K, block_q5_K,+ VDR_Q5_K_Q8_1_MMVQ,+ vec_dot_q5_K_q8_1,+ ncols_dst>(+ vx, vy, dst, ncols, nrows,+ stride_col_y, stride_col_dst, item_ct1);+ });+ });+}++static void mul_mat_vec_q5_K_q8_1_sycl_switch_ncols(+ const void * vx, const void * vy, float * dst,+ const int ncols, const int nrows,+ const int ncols_dst,+ const int stride_col_y, const int stride_col_dst,+ dpct::queue_ptr stream) {+ switch (ncols_dst) {+ case 1: mul_mat_vec_q5_K_q8_1_sycl(vx, vy, dst, ncols, nrows, stream); break;+ case 2: mul_mat_vec_q5_K_q8_1_sycl_ncols<2>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;+ case 3: mul_mat_vec_q5_K_q8_1_sycl_ncols<3>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;+ case 4: mul_mat_vec_q5_K_q8_1_sycl_ncols<4>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;+ case 5: mul_mat_vec_q5_K_q8_1_sycl_ncols<5>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;+ case 6: mul_mat_vec_q5_K_q8_1_sycl_ncols<6>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;+ case 7: mul_mat_vec_q5_K_q8_1_sycl_ncols<7>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;+ case 8: mul_mat_vec_q5_K_q8_1_sycl_ncols<8>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;+ default: GGML_ABORT("unsupported ncols_dst=%d for Q5_K multi-col MMVQ", ncols_dst);+ }+}+static void reorder_mul_mat_vec_q5_k_q8_1_sycl(const void * vx, const void * vy, float * dst, const int ncols,const int nrows, dpct::queue_ptr stream) {GGML_ASSERT(ncols % QK_K == 0);@@ -879,6 +1599,45 @@ static void reorder_mul_mat_vec_q5_k_q8_1_sycl(const void * vx, const void * vy,});}+template <int ncols_dst>+static void reorder_mul_mat_vec_q5_k_q8_1_sycl_ncols(+ const void * vx, const void * vy, float * dst,+ const int ncols, const int nrows,+ const int stride_col_y_bytes, const int stride_col_dst,+ dpct::queue_ptr stream) {+ GGML_ASSERT(ncols % QK_K == 0);+ const int block_num_y = ceil_div(nrows, GGML_SYCL_MMV_Y);+ constexpr size_t num_subgroups = 16;+ GGML_ASSERT(block_num_y % num_subgroups == 0);+ const sycl::range<3> global_size(1, GGML_SYCL_MMV_Y, block_num_y * WARP_SIZE);+ const sycl::range<3> workgroup_size(1, GGML_SYCL_MMV_Y, num_subgroups * WARP_SIZE);+ stream->submit([&](sycl::handler & cgh) {+ cgh.parallel_for(sycl::nd_range<3>(global_size, workgroup_size),+ [=](sycl::nd_item<3> nd_item) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {+ mul_mat_vec_q_reorder_ncols<reorder_vec_dot_q_sycl<GGML_TYPE_Q5_K>, ncols_dst>(+ vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, nd_item);+ });+ });+}++static void reorder_mul_mat_vec_q5_k_q8_1_sycl_switch_ncols(+ const void * vx, const void * vy, float * dst,+ const int ncols, const int nrows, const int ncols_dst,+ const int stride_col_y_bytes, const int stride_col_dst,+ dpct::queue_ptr stream) {+ switch (ncols_dst) {+ case 1: reorder_mul_mat_vec_q5_k_q8_1_sycl(vx, vy, dst, ncols, nrows, stream); break;+ case 2: reorder_mul_mat_vec_q5_k_q8_1_sycl_ncols<2>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break;+ case 3: reorder_mul_mat_vec_q5_k_q8_1_sycl_ncols<3>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break;+ case 4: reorder_mul_mat_vec_q5_k_q8_1_sycl_ncols<4>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break;+ case 5: reorder_mul_mat_vec_q5_k_q8_1_sycl_ncols<5>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break;+ case 6: reorder_mul_mat_vec_q5_k_q8_1_sycl_ncols<6>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break;+ case 7: reorder_mul_mat_vec_q5_k_q8_1_sycl_ncols<7>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break;+ case 8: reorder_mul_mat_vec_q5_k_q8_1_sycl_ncols<8>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break;+ default: GGML_ABORT("unsupported ncols_dst=%d for Q5_K reorder multi-col MMVQ", ncols_dst);+ }+}+static void reorder_mul_mat_vec_q6_k_q8_1_sycl(const void * vx, const void * vy, float * dst, const int ncols,const int nrows, dpct::queue_ptr stream) {GGML_ASSERT(ncols % QK_K == 0);@@ -897,6 +1656,46 @@ static void reorder_mul_mat_vec_q6_k_q8_1_sycl(const void * vx, const void * vy,});});}++template <int ncols_dst>+static void reorder_mul_mat_vec_q6_k_q8_1_sycl_ncols(+ const void * vx, const void * vy, float * dst,+ const int ncols, const int nrows,+ const int stride_col_y_bytes, const int stride_col_dst,+ dpct::queue_ptr stream) {+ GGML_ASSERT(ncols % QK_K == 0);+ const int block_num_y = ceil_div(nrows, GGML_SYCL_MMV_Y);+ constexpr size_t num_subgroups = 16;+ GGML_ASSERT(block_num_y % num_subgroups == 0);+ const sycl::range<3> global_size(1, GGML_SYCL_MMV_Y, block_num_y * WARP_SIZE);+ const sycl::range<3> workgroup_size(1, GGML_SYCL_MMV_Y, num_subgroups * WARP_SIZE);+ stream->submit([&](sycl::handler & cgh) {+ cgh.parallel_for(sycl::nd_range<3>(global_size, workgroup_size),+ [=](sycl::nd_item<3> nd_item) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {+ mul_mat_vec_q_reorder_ncols<reorder_vec_dot_q_sycl<GGML_TYPE_Q6_K>, ncols_dst>(+ vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, nd_item);+ });+ });+}++static void reorder_mul_mat_vec_q6_k_q8_1_sycl_switch_ncols(+ const void * vx, const void * vy, float * dst,+ const int ncols, const int nrows, const int ncols_dst,+ const int stride_col_y_bytes, const int stride_col_dst,+ dpct::queue_ptr stream) {+ switch (ncols_dst) {+ case 1: reorder_mul_mat_vec_q6_k_q8_1_sycl(vx, vy, dst, ncols, nrows, stream); break;+ case 2: reorder_mul_mat_vec_q6_k_q8_1_sycl_ncols<2>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break;+ case 3: reorder_mul_mat_vec_q6_k_q8_1_sycl_ncols<3>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break;+ case 4: reorder_mul_mat_vec_q6_k_q8_1_sycl_ncols<4>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break;+ case 5: reorder_mul_mat_vec_q6_k_q8_1_sycl_ncols<5>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break;+ case 6: reorder_mul_mat_vec_q6_k_q8_1_sycl_ncols<6>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break;+ case 7: reorder_mul_mat_vec_q6_k_q8_1_sycl_ncols<7>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break;+ case 8: reorder_mul_mat_vec_q6_k_q8_1_sycl_ncols<8>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break;+ default: GGML_ABORT("unsupported ncols_dst=%d for Q6_K reorder multi-col MMVQ", ncols_dst);+ }+}+static void mul_mat_vec_q6_K_q8_1_sycl(const void *vx, const void *vy,float *dst, const int ncols,const int nrows,@@ -921,6 +1720,51 @@ static void mul_mat_vec_q6_K_q8_1_sycl(const void *vx, const void *vy,}}+template <int ncols_dst>+static void mul_mat_vec_q6_K_q8_1_sycl_ncols(+ const void * vx, const void * vy, float * dst,+ const int ncols, const int nrows,+ const int stride_col_y, const int stride_col_dst,+ dpct::queue_ptr stream) {+ GGML_ASSERT(ncols % QK_K == 0);+ const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y;+ const sycl::range<3> block_nums(1, 1, block_num_y);+ const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE);++ stream->submit([&](sycl::handler & cgh) {+ cgh.parallel_for(+ sycl::nd_range<3>(block_nums * block_dims, block_dims),+ [=](sycl::nd_item<3> item_ct1)+ [[sycl::reqd_sub_group_size(WARP_SIZE)]] {+ mul_mat_vec_q_ncols<QK_K, QI6_K, block_q6_K,+ VDR_Q6_K_Q8_1_MMVQ,+ vec_dot_q6_K_q8_1,+ ncols_dst>(+ vx, vy, dst, ncols, nrows,+ stride_col_y, stride_col_dst, item_ct1);+ });+ });+}++static void mul_mat_vec_q6_K_q8_1_sycl_switch_ncols(+ const void * vx, const void * vy, float * dst,+ const int ncols, const int nrows,+ const int ncols_dst,+ const int stride_col_y, const int stride_col_dst,+ dpct::queue_ptr stream) {+ switch (ncols_dst) {+ case 1: mul_mat_vec_q6_K_q8_1_sycl(vx, vy, dst, ncols, nrows, stream); break;+ case 2: mul_mat_vec_q6_K_q8_1_sycl_ncols<2>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;+ case 3: mul_mat_vec_q6_K_q8_1_sycl_ncols<3>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;+ case 4: mul_mat_vec_q6_K_q8_1_sycl_ncols<4>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;+ case 5: mul_mat_vec_q6_K_q8_1_sycl_ncols<5>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;+ case 6: mul_mat_vec_q6_K_q8_1_sycl_ncols<6>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;+ case 7: mul_mat_vec_q6_K_q8_1_sycl_ncols<7>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;+ case 8: mul_mat_vec_q6_K_q8_1_sycl_ncols<8>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;+ default: GGML_ABORT("unsupported ncols_dst=%d for Q6_K multi-col MMVQ", ncols_dst);+ }+}+static void mul_mat_vec_iq2_xxs_q8_1_sycl(const void *vx, const void *vy,float *dst, const int ncols,@@ -1117,6 +1961,51 @@ static void mul_mat_vec_iq4_xs_q8_1_sycl(const void *vx, const void *vy,}}+template <int ncols_dst>+static void mul_mat_vec_iq4_xs_q8_1_sycl_ncols(+ const void * vx, const void * vy, float * dst,+ const int ncols, const int nrows,+ const int stride_col_y, const int stride_col_dst,+ dpct::queue_ptr stream) {+ GGML_ASSERT(ncols % QK_K == 0);+ const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y;+ const sycl::range<3> block_nums(1, 1, block_num_y);+ const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE);++ stream->submit([&](sycl::handler & cgh) {+ cgh.parallel_for(+ sycl::nd_range<3>(block_nums * block_dims, block_dims),+ [=](sycl::nd_item<3> item_ct1)+ [[sycl::reqd_sub_group_size(WARP_SIZE)]] {+ mul_mat_vec_q_ncols<QK_K, QI4_XS/4, block_iq4_xs,+ 1,+ vec_dot_iq4_xs_q8_1,+ ncols_dst>(+ vx, vy, dst, ncols, nrows,+ stride_col_y, stride_col_dst, item_ct1);+ });+ });+}++static void mul_mat_vec_iq4_xs_q8_1_sycl_switch_ncols(+ const void * vx, const void * vy, float * dst,+ const int ncols, const int nrows,+ const int ncols_dst,+ const int stride_col_y, const int stride_col_dst,+ dpct::queue_ptr stream) {+ switch (ncols_dst) {+ case 1: mul_mat_vec_iq4_xs_q8_1_sycl(vx, vy, dst, ncols, nrows, stream); break;+ case 2: mul_mat_vec_iq4_xs_q8_1_sycl_ncols<2>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;+ case 3: mul_mat_vec_iq4_xs_q8_1_sycl_ncols<3>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;+ case 4: mul_mat_vec_iq4_xs_q8_1_sycl_ncols<4>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;+ case 5: mul_mat_vec_iq4_xs_q8_1_sycl_ncols<5>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;+ case 6: mul_mat_vec_iq4_xs_q8_1_sycl_ncols<6>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;+ case 7: mul_mat_vec_iq4_xs_q8_1_sycl_ncols<7>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;+ case 8: mul_mat_vec_iq4_xs_q8_1_sycl_ncols<8>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;+ default: GGML_ABORT("unsupported ncols_dst=%d for IQ4_XS multi-col MMVQ", ncols_dst);+ }+}+void ggml_sycl_op_mul_mat_vec_q(ggml_backend_sycl_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1,ggml_tensor * dst, const char * src0_dd_i, const float * src1_ddf_i,const char * src1_ddq_i, float * dst_dd_i, const int64_t row_low,@@ -1143,42 +2032,135 @@ void ggml_sycl_op_mul_mat_vec_q(ggml_backend_sycl_context & ctx, const ggml_tenscase GGML_TYPE_Q4_0:if ((ggml_tensor_extra_gpu *) dst->src[0]->extra &&((ggml_tensor_extra_gpu *) dst->src[0]->extra)->optimized_feature.reorder) {- GGML_SYCL_DEBUG("Calling reorder_mul_mat_vec_q4_0_q8_1_sycl\n");- reorder_mul_mat_vec_q4_0_q8_1_sycl(src0_dd_i, src1_ddq_i_bs, dst_dd_i_bs, ne00, row_diff, stream);- } else {+ if (i == 0 && src1_ncols > 1 && src1_ncols <= 8) {+ const int stride_col_y_bytes = src1_padded_col_size * q8_1_ts / q8_1_bs;+ const int stride_col_dst = dst->ne[0];+ GGML_SYCL_DEBUG("Calling reorder_mul_mat_vec_q4_0_q8_1_sycl_switch_ncols ncols=%d\n", (int)src1_ncols);+ reorder_mul_mat_vec_q4_0_q8_1_sycl_switch_ncols(+ src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff,+ src1_ncols, stride_col_y_bytes, stride_col_dst, stream);+ return;+ } else {+ GGML_SYCL_DEBUG("Calling reorder_mul_mat_vec_q4_0_q8_1_sycl\n");+ reorder_mul_mat_vec_q4_0_q8_1_sycl(src0_dd_i, src1_ddq_i_bs, dst_dd_i_bs, ne00, row_diff, stream);+ }+ } else if (i == 0 && src1_ncols > 1 && src1_ncols <= 8) {+ const int stride_col_y = src1_padded_col_size / QK8_1;+ const int stride_col_dst = dst->ne[0];+ GGML_SYCL_DEBUG("Calling mul_mat_vec_q4_0_q8_1_sycl_switch_ncols ncols=%d\n", (int)src1_ncols);+ mul_mat_vec_q4_0_q8_1_sycl_switch_ncols(+ src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff,+ src1_ncols, stride_col_y, stride_col_dst, stream);+ return;+ } else if (i == 0 || src1_ncols == 1) {GGML_SYCL_DEBUG("Calling mul_mat_vec_q4_0_q8_1_sycl\n");mul_mat_vec_q4_0_q8_1_sycl(src0_dd_i, src1_ddq_i_bs, dst_dd_i_bs, ne00, row_diff, stream);}break;case GGML_TYPE_Q4_1:- mul_mat_vec_q4_1_q8_1_sycl(src0_dd_i, src1_ddq_i_bs, dst_dd_i_bs, ne00, row_diff, stream);+ if (i == 0 && src1_ncols > 1 && src1_ncols <= 8) {+ const int stride_col_y = src1_padded_col_size / QK8_1;+ const int stride_col_dst = dst->ne[0];+ GGML_SYCL_DEBUG("Calling mul_mat_vec_q4_1_q8_1_sycl_switch_ncols ncols=%d\n", (int)src1_ncols);+ mul_mat_vec_q4_1_q8_1_sycl_switch_ncols(+ src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff,+ src1_ncols, stride_col_y, stride_col_dst, stream);+ return;+ } else if (i == 0 || src1_ncols == 1) {+ mul_mat_vec_q4_1_q8_1_sycl(src0_dd_i, src1_ddq_i_bs, dst_dd_i_bs, ne00, row_diff, stream);+ }break;case GGML_TYPE_Q5_0:- mul_mat_vec_q5_0_q8_1_sycl(src0_dd_i, src1_ddq_i_bs, dst_dd_i_bs, ne00, row_diff, stream);+ if (i == 0 && src1_ncols > 1 && src1_ncols <= 8) {+ const int stride_col_y = src1_padded_col_size / QK8_1;+ const int stride_col_dst = dst->ne[0];+ GGML_SYCL_DEBUG("Calling mul_mat_vec_q5_0_q8_1_sycl_switch_ncols ncols=%d\n", (int)src1_ncols);+ mul_mat_vec_q5_0_q8_1_sycl_switch_ncols(+ src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff,+ src1_ncols, stride_col_y, stride_col_dst, stream);+ return;+ } else if (i == 0 || src1_ncols == 1) {+ mul_mat_vec_q5_0_q8_1_sycl(src0_dd_i, src1_ddq_i_bs, dst_dd_i_bs, ne00, row_diff, stream);+ }break;case GGML_TYPE_Q5_1:- mul_mat_vec_q5_1_q8_1_sycl(src0_dd_i, src1_ddq_i_bs, dst_dd_i_bs, ne00, row_diff, stream);+ if (i == 0 && src1_ncols > 1 && src1_ncols <= 8) {+ const int stride_col_y = src1_padded_col_size / QK8_1;+ const int stride_col_dst = dst->ne[0];+ GGML_SYCL_DEBUG("Calling mul_mat_vec_q5_1_q8_1_sycl_switch_ncols ncols=%d\n", (int)src1_ncols);+ mul_mat_vec_q5_1_q8_1_sycl_switch_ncols(+ src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff,+ src1_ncols, stride_col_y, stride_col_dst, stream);+ return;+ } else if (i == 0 || src1_ncols == 1) {+ mul_mat_vec_q5_1_q8_1_sycl(src0_dd_i, src1_ddq_i_bs, dst_dd_i_bs, ne00, row_diff, stream);+ }break;case GGML_TYPE_Q8_0:if ((ggml_tensor_extra_gpu *) dst->src[0]->extra &&((ggml_tensor_extra_gpu *) dst->src[0]->extra)->optimized_feature.reorder) {- GGML_SYCL_DEBUG("Calling reorder_mul_mat_vec_q8_0_q8_1_sycl\n");- reorder_mul_mat_vec_q8_0_q8_1_sycl(src0_dd_i, src1_ddq_i_bs, dst_dd_i_bs, ne00, row_diff, stream);- } else {+ if (i == 0 && src1_ncols > 1 && src1_ncols <= 8) {+ const int stride_col_y_bytes = src1_padded_col_size * q8_1_ts / q8_1_bs;+ const int stride_col_dst = dst->ne[0];+ GGML_SYCL_DEBUG("Calling reorder_mul_mat_vec_q8_0_q8_1_sycl_switch_ncols ncols=%d\n", (int)src1_ncols);+ reorder_mul_mat_vec_q8_0_q8_1_sycl_switch_ncols(+ src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff,+ src1_ncols, stride_col_y_bytes, stride_col_dst, stream);+ return;+ } else {+ GGML_SYCL_DEBUG("Calling reorder_mul_mat_vec_q8_0_q8_1_sycl\n");+ reorder_mul_mat_vec_q8_0_q8_1_sycl(src0_dd_i, src1_ddq_i_bs, dst_dd_i_bs, ne00, row_diff, stream);+ }+ } else if (i == 0 && src1_ncols > 1 && src1_ncols <= 8) {+ const int stride_col_y = src1_padded_col_size / QK8_1;+ const int stride_col_dst = dst->ne[0];+ GGML_SYCL_DEBUG("Calling mul_mat_vec_q8_0_q8_1_sycl_switch_ncols ncols=%d\n", (int)src1_ncols);+ mul_mat_vec_q8_0_q8_1_sycl_switch_ncols(+ src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff,+ src1_ncols, stride_col_y, stride_col_dst, stream);+ return;+ } else if (i == 0 || src1_ncols == 1) {GGML_SYCL_DEBUG("Calling mul_mat_vec_q8_0_q8_1_sycl\n");mul_mat_vec_q8_0_q8_1_sycl(src0_dd_i, src1_ddq_i_bs, dst_dd_i_bs, ne00, row_diff, stream);}break;case GGML_TYPE_Q2_K:- mul_mat_vec_q2_K_q8_1_sycl(src0_dd_i, src1_ddq_i_bs, dst_dd_i_bs, ne00, row_diff, stream);+ if (i == 0 && src1_ncols > 1 && src1_ncols <= 8) {+ const int stride_col_y = src1_padded_col_size / QK8_1;+ const int stride_col_dst = dst->ne[0];+ GGML_SYCL_DEBUG("Calling mul_mat_vec_q2_K_q8_1_sycl_switch_ncols ncols=%d\n", (int)src1_ncols);+ mul_mat_vec_q2_K_q8_1_sycl_switch_ncols(+ src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff,+ src1_ncols, stride_col_y, stride_col_dst, stream);+ return;+ } else if (i == 0 || src1_ncols == 1) {+ mul_mat_vec_q2_K_q8_1_sycl(src0_dd_i, src1_ddq_i_bs, dst_dd_i_bs, ne00, row_diff, stream);+ }break;case GGML_TYPE_Q3_K:if ((ggml_tensor_extra_gpu *) dst->src[0]->extra &&((ggml_tensor_extra_gpu *) dst->src[0]->extra)->optimized_feature.reorder) {- GGML_SYCL_DEBUG("Calling reorder_mul_mat_vec_q3_k_q8_1_sycl\n");- reorder_mul_mat_vec_q3_k_q8_1_sycl(src0_dd_i, src1_ddq_i_bs, dst_dd_i_bs, ne00, row_diff,- stream);- } else {+ if (i == 0 && src1_ncols > 1 && src1_ncols <= 8) {+ const int stride_col_y_bytes = src1_padded_col_size * q8_1_ts / q8_1_bs;+ const int stride_col_dst = dst->ne[0];+ GGML_SYCL_DEBUG("Calling reorder_mul_mat_vec_q3_k_q8_1_sycl_switch_ncols ncols=%d\n", (int)src1_ncols);+ reorder_mul_mat_vec_q3_k_q8_1_sycl_switch_ncols(+ src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff,+ src1_ncols, stride_col_y_bytes, stride_col_dst, stream);+ return;+ } else {+ GGML_SYCL_DEBUG("Calling reorder_mul_mat_vec_q3_k_q8_1_sycl\n");+ reorder_mul_mat_vec_q3_k_q8_1_sycl(src0_dd_i, src1_ddq_i_bs, dst_dd_i_bs, ne00, row_diff, stream);+ }+ } else if (i == 0 && src1_ncols > 1 && src1_ncols <= 8) {+ const int stride_col_y = src1_padded_col_size / QK8_1;+ const int stride_col_dst = dst->ne[0];+ GGML_SYCL_DEBUG("Calling mul_mat_vec_q3_K_q8_1_sycl_switch_ncols ncols=%d\n", (int)src1_ncols);+ mul_mat_vec_q3_K_q8_1_sycl_switch_ncols(+ src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff,+ src1_ncols, stride_col_y, stride_col_dst, stream);+ return;+ } else if (i == 0 || src1_ncols == 1) {GGML_SYCL_DEBUG("Calling mul_mat_vec_q3_K_q8_1_sycl\n");mul_mat_vec_q3_K_q8_1_sycl(src0_dd_i, src1_ddq_i_bs, dst_dd_i_bs, ne00, row_diff, stream);}@@ -1186,9 +2168,27 @@ void ggml_sycl_op_mul_mat_vec_q(ggml_backend_sycl_context & ctx, const ggml_tenscase GGML_TYPE_Q4_K:if ((ggml_tensor_extra_gpu *) dst->src[0]->extra &&((ggml_tensor_extra_gpu *) dst->src[0]->extra)->optimized_feature.reorder) {- GGML_SYCL_DEBUG("Calling reorder_mul_mat_vec_q4_k_q8_1_sycl\n");- reorder_mul_mat_vec_q4_k_q8_1_sycl(src0_dd_i, src1_ddq_i_bs, dst_dd_i_bs, ne00, row_diff, stream);- } else {+ if (i == 0 && src1_ncols > 1 && src1_ncols <= 8) {+ const int stride_col_y_bytes = src1_padded_col_size * q8_1_ts / q8_1_bs;+ const int stride_col_dst = dst->ne[0];+ GGML_SYCL_DEBUG("Calling reorder_mul_mat_vec_q4_k_q8_1_sycl_switch_ncols ncols=%d\n", (int)src1_ncols);+ reorder_mul_mat_vec_q4_k_q8_1_sycl_switch_ncols(+ src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff,+ src1_ncols, stride_col_y_bytes, stride_col_dst, stream);+ return;+ } else {+ GGML_SYCL_DEBUG("Calling reorder_mul_mat_vec_q4_k_q8_1_sycl\n");+ reorder_mul_mat_vec_q4_k_q8_1_sycl(src0_dd_i, src1_ddq_i_bs, dst_dd_i_bs, ne00, row_diff, stream);+ }+ } else if (i == 0 && src1_ncols > 1 && src1_ncols <= 8) {+ const int stride_col_y = src1_padded_col_size / QK8_1;+ const int stride_col_dst = dst->ne[0];+ GGML_SYCL_DEBUG("Calling mul_mat_vec_q4_K_q8_1_sycl_switch_ncols ncols=%d\n", (int)src1_ncols);+ mul_mat_vec_q4_K_q8_1_sycl_switch_ncols(+ src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff,+ src1_ncols, stride_col_y, stride_col_dst, stream);+ return;+ } else if (i == 0 || src1_ncols == 1) {GGML_SYCL_DEBUG("Calling mul_mat_vec_q4_K_q8_1_sycl\n");mul_mat_vec_q4_K_q8_1_sycl(src0_dd_i, src1_ddq_i_bs, dst_dd_i_bs, ne00, row_diff, stream);}@@ -1196,9 +2196,27 @@ void ggml_sycl_op_mul_mat_vec_q(ggml_backend_sycl_context & ctx, const ggml_tenscase GGML_TYPE_Q5_K:if ((ggml_tensor_extra_gpu *) dst->src[0]->extra &&((ggml_tensor_extra_gpu *) dst->src[0]->extra)->optimized_feature.reorder) {- GGML_SYCL_DEBUG("Calling reorder_mul_mat_vec_q5_k_q8_1_sycl\n");- reorder_mul_mat_vec_q5_k_q8_1_sycl(src0_dd_i, src1_ddq_i_bs, dst_dd_i_bs, ne00, row_diff, stream);- } else {+ if (i == 0 && src1_ncols > 1 && src1_ncols <= 8) {+ const int stride_col_y_bytes = src1_padded_col_size * q8_1_ts / q8_1_bs;+ const int stride_col_dst = dst->ne[0];+ GGML_SYCL_DEBUG("Calling reorder_mul_mat_vec_q5_k_q8_1_sycl_switch_ncols ncols=%d\n", (int)src1_ncols);+ reorder_mul_mat_vec_q5_k_q8_1_sycl_switch_ncols(+ src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff,+ src1_ncols, stride_col_y_bytes, stride_col_dst, stream);+ return;+ } else {+ GGML_SYCL_DEBUG("Calling reorder_mul_mat_vec_q5_k_q8_1_sycl\n");+ reorder_mul_mat_vec_q5_k_q8_1_sycl(src0_dd_i, src1_ddq_i_bs, dst_dd_i_bs, ne00, row_diff, stream);+ }+ } else if (i == 0 && src1_ncols > 1 && src1_ncols <= 8) {+ const int stride_col_y = src1_padded_col_size / QK8_1;+ const int stride_col_dst = dst->ne[0];+ GGML_SYCL_DEBUG("Calling mul_mat_vec_q5_K_q8_1_sycl_switch_ncols ncols=%d\n", (int)src1_ncols);+ mul_mat_vec_q5_K_q8_1_sycl_switch_ncols(+ src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff,+ src1_ncols, stride_col_y, stride_col_dst, stream);+ return;+ } else if (i == 0 || src1_ncols == 1) {GGML_SYCL_DEBUG("Calling mul_mat_vec_q5_K_q8_1_sycl\n");mul_mat_vec_q5_K_q8_1_sycl(src0_dd_i, src1_ddq_i_bs, dst_dd_i_bs, ne00, row_diff, stream);}@@ -1206,9 +2224,27 @@ void ggml_sycl_op_mul_mat_vec_q(ggml_backend_sycl_context & ctx, const ggml_tenscase GGML_TYPE_Q6_K:if ((ggml_tensor_extra_gpu *) dst->src[0]->extra &&((ggml_tensor_extra_gpu *) dst->src[0]->extra)->optimized_feature.reorder) {- GGML_SYCL_DEBUG("Calling reorder_mul_mat_vec_q6_k_q8_1_sycl\n");- reorder_mul_mat_vec_q6_k_q8_1_sycl(src0_dd_i, src1_ddq_i_bs, dst_dd_i_bs, ne00, row_diff, stream);- } else {+ if (i == 0 && src1_ncols > 1 && src1_ncols <= 8) {+ const int stride_col_y_bytes = src1_padded_col_size * q8_1_ts / q8_1_bs;+ const int stride_col_dst = dst->ne[0];+ GGML_SYCL_DEBUG("Calling reorder_mul_mat_vec_q6_k_q8_1_sycl_switch_ncols ncols=%d\n", (int)src1_ncols);+ reorder_mul_mat_vec_q6_k_q8_1_sycl_switch_ncols(+ src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff,+ src1_ncols, stride_col_y_bytes, stride_col_dst, stream);+ return;+ } else {+ GGML_SYCL_DEBUG("Calling reorder_mul_mat_vec_q6_k_q8_1_sycl\n");+ reorder_mul_mat_vec_q6_k_q8_1_sycl(src0_dd_i, src1_ddq_i_bs, dst_dd_i_bs, ne00, row_diff, stream);+ }+ } else if (i == 0 && src1_ncols > 1 && src1_ncols <= 8) {+ const int stride_col_y = src1_padded_col_size / QK8_1;+ const int stride_col_dst = dst->ne[0];+ GGML_SYCL_DEBUG("Calling mul_mat_vec_q6_K_q8_1_sycl_switch_ncols ncols=%d\n", (int)src1_ncols);+ mul_mat_vec_q6_K_q8_1_sycl_switch_ncols(+ src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff,+ src1_ncols, stride_col_y, stride_col_dst, stream);+ return;+ } else if (i == 0 || src1_ncols == 1) {GGML_SYCL_DEBUG("Calling mul_mat_vec_q6_k_q8_1_sycl\n");mul_mat_vec_q6_K_q8_1_sycl(src0_dd_i, src1_ddq_i_bs, dst_dd_i_bs, ne00, row_diff, stream);}@@ -1238,13 +2274,43 @@ void ggml_sycl_op_mul_mat_vec_q(ggml_backend_sycl_context & ctx, const ggml_tensmul_mat_vec_iq4_nl_q8_1_sycl(src0_dd_i, src1_ddq_i_bs, dst_dd_i_bs, ne00, row_diff, stream);break;case GGML_TYPE_IQ4_XS:- mul_mat_vec_iq4_xs_q8_1_sycl(src0_dd_i, src1_ddq_i_bs, dst_dd_i_bs, ne00, row_diff, stream);+ if (i == 0 && src1_ncols > 1 && src1_ncols <= 8) {+ const int stride_col_y = src1_padded_col_size / QK8_1;+ const int stride_col_dst = dst->ne[0];+ GGML_SYCL_DEBUG("Calling mul_mat_vec_iq4_xs_q8_1_sycl_switch_ncols ncols=%d\n", (int)src1_ncols);+ mul_mat_vec_iq4_xs_q8_1_sycl_switch_ncols(+ src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff,+ src1_ncols, stride_col_y, stride_col_dst, stream);+ return;+ } else if (i == 0 || src1_ncols == 1) {+ mul_mat_vec_iq4_xs_q8_1_sycl(src0_dd_i, src1_ddq_i_bs, dst_dd_i_bs, ne00, row_diff, stream);+ }break;case GGML_TYPE_MXFP4:- mul_mat_vec_mxfp4_q8_1_sycl(src0_dd_i, src1_ddq_i_bs, dst_dd_i_bs, ne00, row_diff, stream);+ if (i == 0 && src1_ncols > 1 && src1_ncols <= 8) {+ const int stride_col_y = src1_padded_col_size / QK8_1;+ const int stride_col_dst = dst->ne[0];+ GGML_SYCL_DEBUG("Calling mul_mat_vec_mxfp4_q8_1_sycl_switch_ncols ncols=%d\n", (int)src1_ncols);+ mul_mat_vec_mxfp4_q8_1_sycl_switch_ncols(+ src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff,+ src1_ncols, stride_col_y, stride_col_dst, stream);+ return;+ } else if (i == 0 || src1_ncols == 1) {+ mul_mat_vec_mxfp4_q8_1_sycl(src0_dd_i, src1_ddq_i_bs, dst_dd_i_bs, ne00, row_diff, stream);+ }break;case GGML_TYPE_NVFP4:- mul_mat_vec_nvfp4_q8_1_sycl(src0_dd_i, src1_ddq_i_bs, dst_dd_i_bs, ne00, row_diff, stream);+ if (i == 0 && src1_ncols > 1 && src1_ncols <= 8) {+ const int stride_col_y = src1_padded_col_size / QK8_1;+ const int stride_col_dst = dst->ne[0];+ GGML_SYCL_DEBUG("Calling mul_mat_vec_nvfp4_q8_1_sycl_switch_ncols ncols=%d\n", (int)src1_ncols);+ mul_mat_vec_nvfp4_q8_1_sycl_switch_ncols(+ src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff,+ src1_ncols, stride_col_y, stride_col_dst, stream);+ return;+ } else if (i == 0 || src1_ncols == 1) {+ mul_mat_vec_nvfp4_q8_1_sycl(src0_dd_i, src1_ddq_i_bs, dst_dd_i_bs, ne00, row_diff, stream);+ }break;default:GGML_ABORT("fatal error: unsupport data type=%s\n", ggml_type_name(src0->type));