LLM inference in C/C++
C++ ggml
sycl: support reordered Q8_0 MoE weights (#29337)
Frosty40e707790c7359
diff --git a/ggml/src/ggml-sycl/ggml-sycl.cpp b/ggml/src/ggml-sycl/ggml-sycl.cppindex 6759af0edc..1cfc841264 100644--- a/ggml/src/ggml-sycl/ggml-sycl.cpp+++ b/ggml/src/ggml-sycl/ggml-sycl.cpp@@ -4442,6 +4442,47 @@ static bool reorder_qw_q8_0(uint8_t * data_device, const int ncols, const int nrreturn true;}+// Reorder each Q8_0 expert slice into a self-contained [qs][d] SoA layout.+static bool reorder_qw_q8_0_moe(uint8_t * data_device, size_t expert_bytes, int64_t n_expert, dpct::queue_ptr stream) {+ GGML_ASSERT(expert_bytes % sizeof(block_q8_0) == 0);+ const int blocks_per_expert = (int) (expert_bytes / sizeof(block_q8_0));+ const size_t total_bytes = expert_bytes * (size_t) n_expert;++ sycl_reorder_temp_buffer tmp(stream, total_bytes);+ if (!tmp) {+ GGML_LOG_WARN("%s: failed to allocate %zu bytes for reorder temp buffer, skipping reorder\n", __func__, total_bytes);+ return false;+ }+ uint8_t * tmp_buf = static_cast<uint8_t *>(tmp.ptr);++ sycl::event copy_event;+ SYCL_CHECK(CHECK_TRY_ERROR(copy_event = stream->memcpy(tmp_buf, data_device, total_bytes)));+ if (!g_ggml_sycl_use_async_mem_op) {+ copy_event.wait();+ }++ const int total_blocks = blocks_per_expert * (int) n_expert;+ auto reorder_event = stream->parallel_for(total_blocks, [=](auto gb_) {+ const int gb = gb_;+ const int e = gb / blocks_per_expert;+ const int ib = gb % blocks_per_expert;+ const block_q8_0 * x = (const block_q8_0 *) (tmp_buf + (size_t) e * expert_bytes);+ uint8_t * base = data_device + (size_t) e * expert_bytes;++ auto * qs_ptr = base;+ auto * d_ptr = (sycl::half *) (qs_ptr + (size_t) blocks_per_expert * QK8_0);++ for (int j = 0; j < QK8_0; ++j) {+ *((int8_t *) qs_ptr + ib * QK8_0 + j) = x[ib].qs[j];+ }+ d_ptr[ib] = x[ib].d;+ });+ if (!g_ggml_sycl_use_async_mem_op) {+ reorder_event.wait_and_throw();+ }+ return true;+}+static bool reorder_qw_q4_k(uint8_t * data_device, size_t size, size_t offset, dpct::queue_ptr stream) {GGML_ASSERT(size % sizeof(block_q4_K) == 0);GGML_ASSERT(offset % sizeof(block_q4_K) == 0);@@ -4966,6 +5007,8 @@ static bool reorder_qw(const ggml_tensor * src0, dpct::queue_ptr stream) {if (src0->ne[2] > 1) {GGML_ASSERT((size_t) size == (size_t) src0->ne[2] * src0->nb[2]);switch (src0->type) {+ case GGML_TYPE_Q8_0:+ return reorder_qw_q8_0_moe(data_device, src0->nb[2], src0->ne[2], stream);case GGML_TYPE_Q4_K:return reorder_qw_q4_k_moe(data_device, src0->nb[2], src0->ne[2], stream);case GGML_TYPE_Q5_K:@@ -5051,8 +5094,8 @@ static void opt_for_reorder_id(ggml_backend_sycl_context * ctx, const ggml_tensoif (!g_ggml_sycl_enable_optimize || !ctx->opt_feature.reorder) {return;}- if (src0->type != GGML_TYPE_Q4_K && src0->type != GGML_TYPE_Q5_K && src0->type != GGML_TYPE_Q6_K &&- src0->type != GGML_TYPE_MXFP4) {+ if (src0->type != GGML_TYPE_Q8_0 && src0->type != GGML_TYPE_Q4_K &&+ src0->type != GGML_TYPE_Q5_K && src0->type != GGML_TYPE_Q6_K && src0->type != GGML_TYPE_MXFP4) {return;}// The MXFP4 reorder kernels use 8-byte vector loads, so every expert slice must stay aligned.@@ -5438,7 +5481,7 @@ static bool ggml_sycl_mul_mat_id_mmvq_fused(const int n_experts_used = (int) n_ids_per_group;const int nrows = (int) src0->ne[1];- // Lazily reorder the (Q4_K) expert weights into a per-expert SoA layout, then run the reorder+ // Lazily reorder supported expert weights into a per-expert SoA layout, then run the reorder// GEMV. Placed after the bail checks so a non-dispatchable op does not pay the reorder cost.opt_for_reorder_id(&ctx, src0);const ggml_tensor_extra_gpu * src0_extra =diff --git a/ggml/src/ggml-sycl/mmvq.cpp b/ggml/src/ggml-sycl/mmvq.cppindex eade8949fb..e9057777cb 100644--- a/ggml/src/ggml-sycl/mmvq.cpp+++ b/ggml/src/ggml-sycl/mmvq.cpp@@ -3202,9 +3202,11 @@ static void mul_mat_vec_q_moe_reorder(const auto sg = item_ct1.get_sub_group();+ constexpr int vdr_mmvq = reorder_vec_dot_q_sycl::gtype == GGML_TYPE_Q8_0 ?+ VDR_Q8_0_Q8_1_MMVQ : block_traits::vdr_mmvq;const int blocks_per_row = ncols / block_traits::qk;- constexpr int blocks_per_subgroup = ceil_div(block_traits::vdr_mmvq * WARP_SIZE, block_traits::qi);- constexpr int block_elements_per_subgroup = block_traits::qi / block_traits::vdr_mmvq;+ constexpr int blocks_per_subgroup = ceil_div(vdr_mmvq * WARP_SIZE, block_traits::qi);+ constexpr int block_elements_per_subgroup = block_traits::qi / vdr_mmvq;const int nblocks = nrows * (ncols / block_traits::qk);static_assert(blocks_per_subgroup > 0);@@ -3223,12 +3225,24 @@ static void mul_mat_vec_q_moe_reorder(#pragma unrollfor (int elem = 0; elem < block_elements_per_subgroup; elem += WARP_SIZE) {- const int iqs = elem + block_traits::vdr_mmvq * (sg.get_local_linear_id() % block_elements_per_subgroup);- partial_sum += reorder_vec_dot_q_sycl()(vx, bx_offset, d_offset, q8_1_quant_ptr, q8_1_ds_ptr, iqs);+ const int iqs = elem + vdr_mmvq * (sg.get_local_linear_id() % block_elements_per_subgroup);+ if constexpr (reorder_vec_dot_q_sycl::gtype == GGML_TYPE_Q8_0) {+ partial_sum += reorder_vec_dot_q_sycl()(vx, bx_offset, d_offset, q8_1_quant_ptr, q8_1_ds_ptr, iqs, vdr_mmvq);+ } else {+ partial_sum += reorder_vec_dot_q_sycl()(vx, bx_offset, d_offset, q8_1_quant_ptr, q8_1_ds_ptr, iqs);+ }}}- auto sum = sycl::reduce_over_group(sg, partial_sum, std::plus<>());+ float sum = partial_sum;+ if constexpr (reorder_vec_dot_q_sycl::gtype == GGML_TYPE_Q8_0) {+ // Keep the accumulation order of the non-reordered MoE kernel.+ for (int mask = WARP_SIZE / 2; mask > 0; mask >>= 1) {+ sum += dpct::permute_sub_group_by_xor(sg, sum, mask);+ }+ } else {+ sum = sycl::reduce_over_group(sg, partial_sum, std::plus<>());+ }if (sg.leader()) {dst[row] = sum;}@@ -3269,6 +3283,11 @@ bool ggml_sycl_mul_mat_vec_q_id_reorder(size_t src1_row_stride,dpct::queue_ptr stream) {switch (src0_type) {+ case GGML_TYPE_Q8_0:+ launch_mul_mat_vec_q_moe_reorder<reorder_vec_dot_q_sycl<GGML_TYPE_Q8_0>>(+ vx_base, vy, ids_dev, dst_base, ncols, nrows, n_experts_used,+ expert_weight_stride, dst_row_stride, src1_row_stride, stream);+ return true;case GGML_TYPE_Q4_K:launch_mul_mat_vec_q_moe_reorder<reorder_vec_dot_q_sycl<GGML_TYPE_Q4_K>>(vx_base, vy, ids_dev, dst_base, ncols, nrows, n_experts_used,diff --git a/ggml/src/ggml-sycl/vecdotq.hpp b/ggml/src/ggml-sycl/vecdotq.hppindex d1b6092bea..57360a004b 100644--- a/ggml/src/ggml-sycl/vecdotq.hpp+++ b/ggml/src/ggml-sycl/vecdotq.hpp@@ -483,7 +483,8 @@ template <> struct reorder_vec_dot_q_sycl<GGML_TYPE_Q8_0> {__dpct_inline__ float operator()(const void * __restrict__ vbq, const std::pair<int, int> ibx_offset,const std::pair<int, int> d_offset, const int8_t * q8_1_quant_ptr,- const sycl::half2 * q8_1_ds, const int & iqs) {+ const sycl::half2 * q8_1_ds, const int & iqs,+ const int vdr = q8_0_traits::vdr_mmvq) {const uint8_t * base = static_cast<const uint8_t *>(vbq);const int8_t * qs = reinterpret_cast<const int8_t *>(base + ibx_offset.first);const ggml_half d = *reinterpret_cast<const ggml_half *>(base + d_offset.first);@@ -492,14 +493,14 @@ template <> struct reorder_vec_dot_q_sycl<GGML_TYPE_Q8_0> {int u[q8_0_traits::vdr_mmvq];#pragma unroll- for (size_t i = 0; i < q8_0_traits::vdr_mmvq; ++i) {+ for (int i = 0; i < vdr; ++i) {v[i] = get_int_from_int8(qs, iqs + i);u[i] = get_int_from_int8_aligned(q8_1_quant_ptr, iqs + i);}int sumi = 0;#pragma unroll- for (size_t i = 0; i < q8_0_traits::vdr_mmvq; ++i) {+ for (int i = 0; i < vdr; ++i) {sumi = dpct::dp4a(v[i], u[i], sumi);}