LLM inference in C/C++
C++ ggml
Restore ESIMD kernels on FFN path (#30239)
Todd Malsbary547208dca250
diff --git a/ggml/src/ggml-sycl/common.hpp b/ggml/src/ggml-sycl/common.hppindex 4755093721..0bf6facec0 100644--- a/ggml/src/ggml-sycl/common.hpp+++ b/ggml/src/ggml-sycl/common.hpp@@ -37,6 +37,10 @@ namespace syclexp = sycl::ext::oneapi::experimental;#endif#endif+#if defined(__INTEL_LLVM_COMPILER)+ #define GGML_SYCL_DMMV_HAS_ESIMD+#endif+#if GGML_SYCL_DNNL#include "dnnl.hpp"#include "dnnl_sycl.hpp"diff --git a/ggml/src/ggml-sycl/dmmv.cpp b/ggml/src/ggml-sycl/dmmv.cppindex 90f9268298..d720bc1f82 100644--- a/ggml/src/ggml-sycl/dmmv.cpp+++ b/ggml/src/ggml-sycl/dmmv.cpp@@ -10,7 +10,6 @@#endif#include <sycl/ext/intel/esimd.hpp>#include "esimd.hpp"- #define GGML_SYCL_DMMV_HAS_ESIMD#endifstatic void convert_f16(const void * vx, const int64_t ib, const int iqs, dfloat2 & v){@@ -1873,11 +1872,67 @@ static void dequantize_mul_mat_vec_q6_K_sycl(const void *vx, const float *y,#ifdef GGML_SYCL_DMMV_HAS_ESIMDusing ggml_sycl_esimd::GGML_SYCL_DMMV_ESIMD_WG_SIZE;-// generic reordered dequantize-matvec: each work-group owns a pair of-// consecutive output rows and updates one 32-wide accumulator per row-template <ggml_type T>+// policies for dequantize_mul_mat_vec_reorder_esimd: row mapping and what to do with the two sums+// plain mat-vec: a work-group owns two consecutive rows of one matrix+struct esimd_dmmv_row_pair_policy {+ const void * vx;+ float * dst;++ ESIMD_INLINE const void * w_a() const { return vx; }+ ESIMD_INLINE const void * w_b() const { return vx; }++ ESIMD_INLINE int row_a(int g) const { return g * 2; }+ ESIMD_INLINE int row_b(int g) const { return g * 2 + 1; }+ ESIMD_INLINE bool has_b(int g, int nrows) const { return row_b(g) < nrows; }++ ESIMD_INLINE void epilogue(int g, int nrows, float sum_a, float sum_b) const {+ dst[row_a(g)] = sum_a;+ if (has_b(g, nrows)) {+ dst[row_b(g)] = sum_b;+ }+ }+};++// fused gate+up+GLU: a work-group owns one row of both matrices; a is the gate, b the up+struct esimd_dmmv_glu_policy {+ const void * vgate;+ const void * vup;+ float * dst;+ ggml_glu_op glu_op;++ ESIMD_INLINE const void * w_a() const { return vgate; }+ ESIMD_INLINE const void * w_b() const { return vup; }++ ESIMD_INLINE int row_a(int g) const { return g; }+ ESIMD_INLINE int row_b(int g) const { return g; }+ ESIMD_INLINE bool has_b(int, int) const { return true; } // up always contributes++ ESIMD_INLINE void epilogue(int g, int, float gate, float up) const {+ using namespace sycl::ext::intel::esimd;++ // SwiGLU: silu(gate)*up = gate*sigmoid(gate)*up. GEGLU's tanh form reduces+ // to the same shape via tanh(y) = 2*sigmoid(2y) - 1, so only the sigmoid+ // argument differs. sigmoid via exp2 (width-1 simd is a degenerate case for+ // the EM pipe, so broadcast to 16 lanes); lane 0 holds the result.+ const float GELU_COEF_A = 0.044715f;+ const float SQRT_2_OVER_PI = 0.79788456080286535587989211986876f;+ const float NEG_LOG2E = -1.44269504088896341f;++ const float arg = (glu_op == GGML_GLU_OP_GEGLU) ?+ 2.0f * SQRT_2_OVER_PI * gate * (1.0f + GELU_COEF_A * gate * gate) :+ gate;++ simd<float, 16> t = arg * NEG_LOG2E;+ simd<float, 16> e = exp2(t);+ dst[g] = gate / (1.0f + e[0]) * up;+ }+};++// generic reordered dequantize-matvec: each work-group owns a pair of rows chosen by the policy+// and updates one 32-wide accumulator per row+template <ggml_type T, typename Policy>ESIMD_INLINE void dequantize_mul_mat_vec_reorder_esimd(- const void * vx, const float * y, float * dst,+ const Policy & policy, const float * y,const int ncols, const int nrows,sycl::local_accessor<float, 1> lmem,const sycl::nd_item<1> & it) {@@ -1886,41 +1941,41 @@ ESIMD_INLINE void dequantize_mul_mat_vec_reorder_esimd(const int num_blocks_per_row = ncols / QK_K;const size_t nb = (size_t) nrows * num_blocks_per_row;- const auto ps = traits::make_ptrs(vx, nb);+ const auto pa = traits::make_ptrs(policy.w_a(), nb);+ const auto pb = traits::make_ptrs(policy.w_b(), nb);- const int tid = it.get_local_id(0);- const int row_pair = it.get_group(0);- const int row0 = row_pair * 2; // two consecutive output rows- const bool has_row1 = row0 + 1 < nrows;+ const int tid = it.get_local_id(0);+ const int g = it.get_group(0);+ const int row_a = policy.row_a(g);+ const int row_b = policy.row_b(g);+ const bool has_b = policy.has_b(g, nrows);- // one 32-wide accumulator per output row (small footprint, no spill)- simd<float, 32> acc0 = 0.0f;- simd<float, 32> acc1 = 0.0f;+ // one 32-wide accumulator per row (small footprint, no spill)+ simd<float, 32> acc_a = 0.0f;+ simd<float, 32> acc_b = 0.0f;for (int ib = tid; ib < num_blocks_per_row; ib += GGML_SYCL_DMMV_ESIMD_WG_SIZE) {simd<float, 256> y_vec = block_load<float, 256>(y + (size_t) ib * QK_K);- const size_t bi0 = (size_t) (row0 + 0) * num_blocks_per_row + ib;- const size_t bi1 = (size_t) (row0 + 1) * num_blocks_per_row + ib;+ const size_t bi_a = (size_t) row_a * num_blocks_per_row + ib;+ const size_t bi_b = (size_t) row_b * num_blocks_per_row + ib;- traits::mac_pair(ps, bi0, ps, bi1, has_row1, y_vec, acc0, acc1);+ traits::mac_pair(pa, bi_a, pb, bi_b, has_b, y_vec, acc_a, acc_b);}- lmem[tid * 2 + 0] = reduce<float>(acc0, std::plus<>{});- lmem[tid * 2 + 1] = reduce<float>(acc1, std::plus<>{});+ lmem[tid * 2 + 0] = reduce<float>(acc_a, std::plus<>{});+ lmem[tid * 2 + 1] = reduce<float>(acc_b, std::plus<>{});it.barrier(sycl::access::fence_space::local_space);if (tid == 0) {- float sum0 = 0.0f;- float sum1 = 0.0f;+ float sum_a = 0.0f;+ float sum_b = 0.0f;for (int p = 0; p < GGML_SYCL_DMMV_ESIMD_WG_SIZE; ++p) {- sum0 += lmem[p * 2 + 0];- sum1 += lmem[p * 2 + 1];- }- dst[row0 + 0] = sum0;- if (has_row1) {- dst[row0 + 1] = sum1;+ sum_a += lmem[p * 2 + 0];+ sum_b += lmem[p * 2 + 1];}++ policy.epilogue(g, nrows, sum_a, sum_b);}}@@ -1936,7 +1991,7 @@ static void dequantize_mul_mat_vec_q2_K_sycl_reorder_esimd(const void *vx, constsycl::nd_range<1>(sycl::range<1>((size_t)workgroups * GGML_SYCL_DMMV_ESIMD_WG_SIZE), sycl::range<1>(GGML_SYCL_DMMV_ESIMD_WG_SIZE)),[=](sycl::nd_item<1> it) [[intel::sycl_explicit_simd]] {dequantize_mul_mat_vec_reorder_esimd<GGML_TYPE_Q2_K>(- vx, y, dst, ncols, nrows, lmem, it);+ esimd_dmmv_row_pair_policy{ vx, dst }, y, ncols, nrows, lmem, it);});});}@@ -1953,7 +2008,7 @@ static void dequantize_mul_mat_vec_q3_K_sycl_reorder_esimd(const void *vx, constsycl::nd_range<1>(sycl::range<1>((size_t)workgroups * GGML_SYCL_DMMV_ESIMD_WG_SIZE), sycl::range<1>(GGML_SYCL_DMMV_ESIMD_WG_SIZE)),[=](sycl::nd_item<1> it) [[intel::sycl_explicit_simd]] {dequantize_mul_mat_vec_reorder_esimd<GGML_TYPE_Q3_K>(- vx, y, dst, ncols, nrows, lmem, it);+ esimd_dmmv_row_pair_policy{ vx, dst }, y, ncols, nrows, lmem, it);});});}@@ -1970,7 +2025,7 @@ static void dequantize_mul_mat_vec_q4_K_sycl_reorder_esimd(const void *vx, constsycl::nd_range<1>(sycl::range<1>((size_t)workgroups * GGML_SYCL_DMMV_ESIMD_WG_SIZE), sycl::range<1>(GGML_SYCL_DMMV_ESIMD_WG_SIZE)),[=](sycl::nd_item<1> it) [[intel::sycl_explicit_simd]] {dequantize_mul_mat_vec_reorder_esimd<GGML_TYPE_Q4_K>(- vx, y, dst, ncols, nrows, lmem, it);+ esimd_dmmv_row_pair_policy{ vx, dst }, y, ncols, nrows, lmem, it);});});}@@ -1987,7 +2042,7 @@ static void dequantize_mul_mat_vec_q5_K_sycl_reorder_esimd(const void *vx, constsycl::nd_range<1>(sycl::range<1>((size_t)workgroups * GGML_SYCL_DMMV_ESIMD_WG_SIZE), sycl::range<1>(GGML_SYCL_DMMV_ESIMD_WG_SIZE)),[=](sycl::nd_item<1> it) [[intel::sycl_explicit_simd]] {dequantize_mul_mat_vec_reorder_esimd<GGML_TYPE_Q5_K>(- vx, y, dst, ncols, nrows, lmem, it);+ esimd_dmmv_row_pair_policy{ vx, dst }, y, ncols, nrows, lmem, it);});});}@@ -2004,7 +2059,7 @@ static void dequantize_mul_mat_vec_q6_K_sycl_reorder_esimd(const void *vx, constsycl::nd_range<1>(sycl::range<1>((size_t)workgroups * GGML_SYCL_DMMV_ESIMD_WG_SIZE), sycl::range<1>(GGML_SYCL_DMMV_ESIMD_WG_SIZE)),[=](sycl::nd_item<1> it) [[intel::sycl_explicit_simd]] {dequantize_mul_mat_vec_reorder_esimd<GGML_TYPE_Q6_K>(- vx, y, dst, ncols, nrows, lmem, it);+ esimd_dmmv_row_pair_policy{ vx, dst }, y, ncols, nrows, lmem, it);});});}@@ -2045,9 +2100,9 @@ ESIMD_INLINE void q8_0_mac_stripe(}}-template <int WG>+template <int WG, typename Policy>ESIMD_INLINE void dequantize_mul_mat_vec_q8_0_reorder_esimd(- const void * vx, const float * y, float * dst,+ const Policy & policy, const float * y,const int ncols, const int nrows,sycl::local_accessor<float, 1> lmem,const sycl::nd_item<1> & it) {@@ -2057,84 +2112,137 @@ ESIMD_INLINE void dequantize_mul_mat_vec_q8_0_reorder_esimd(const int nblk_row = ncols / QK8_0;const size_t nb = (size_t) nrows * nblk_row;- const int8_t * qs = (const int8_t *) vx;- const sycl::half * d = (const sycl::half *) (qs + nb * QK8_0);+ const int8_t * qs_a = (const int8_t *) policy.w_a();+ const int8_t * qs_b = (const int8_t *) policy.w_b();+ const sycl::half * d_a = (const sycl::half *) (qs_a + nb * QK8_0);+ const sycl::half * d_b = (const sycl::half *) (qs_b + nb * QK8_0);- const int tid = it.get_local_id(0);- const int row_pair = it.get_group(0);- const int row0 = row_pair * 2;- const bool has_row1 = row0 + 1 < nrows;+ const int tid = it.get_local_id(0);+ const int g = it.get_group(0);+ const bool has_b = policy.has_b(g, nrows);- const size_t base0 = (size_t) row0 * nblk_row;- const size_t base1 = has_row1 ? (size_t) (row0 + 1) * nblk_row : base0;+ const size_t base_a = (size_t) policy.row_a(g) * nblk_row;+ const size_t base_b = (size_t) policy.row_b(g) * nblk_row;- simd<float, 32> acc0 = 0.0f;- simd<float, 32> acc1 = 0.0f;+ simd<float, 32> acc_a = 0.0f;+ simd<float, 32> acc_b = 0.0f;// Each thread processes one contiguous stripe.int ib = 0;for (; ib + WG * STRIPE <= nblk_row; ib += WG * STRIPE) {const int b = ib + tid * STRIPE;simd<float, 256> y_vec = block_load<float, 256>(y + (size_t) b * QK8_0);- q8_0_mac_stripe<STRIPE>(qs + (base0 + b) * QK8_0, qs + (base1 + b) * QK8_0,- d + base0 + b, d + base1 + b, has_row1, y_vec, acc0, acc1);+ q8_0_mac_stripe<STRIPE>(qs_a + (base_a + b) * QK8_0, qs_b + (base_b + b) * QK8_0,+ d_a + base_a + b, d_b + base_b + b, has_b, y_vec, acc_a, acc_b);}// Distribute remaining blocks across the work-group.for (int b = ib + tid; b < nblk_row; b += WG) {simd<float, 32> y_vec = block_load<float, 32>(y + (size_t) b * QK8_0);- q8_0_mac_stripe<1>(qs + (base0 + b) * QK8_0, qs + (base1 + b) * QK8_0,- d + base0 + b, d + base1 + b, has_row1, y_vec, acc0, acc1);+ q8_0_mac_stripe<1>(qs_a + (base_a + b) * QK8_0, qs_b + (base_b + b) * QK8_0,+ d_a + base_a + b, d_b + base_b + b, has_b, y_vec, acc_a, acc_b);}- lmem[tid * 2 + 0] = reduce<float>(acc0, std::plus<>{});- lmem[tid * 2 + 1] = reduce<float>(acc1, std::plus<>{});+ lmem[tid * 2 + 0] = reduce<float>(acc_a, std::plus<>{});+ lmem[tid * 2 + 1] = reduce<float>(acc_b, std::plus<>{});it.barrier(sycl::access::fence_space::local_space);if (tid == 0) {- float sum0 = 0.0f;- float sum1 = 0.0f;+ float sum_a = 0.0f;+ float sum_b = 0.0f;for (int p = 0; p < WG; ++p) {- sum0 += lmem[p * 2 + 0];- sum1 += lmem[p * 2 + 1];- }- dst[row0 + 0] = sum0;- if (has_row1) {- dst[row0 + 1] = sum1;+ sum_a += lmem[p * 2 + 0];+ sum_b += lmem[p * 2 + 1];}++ policy.epilogue(g, nrows, sum_a, sum_b);}}-template <int WG>-static void q8_0_esimd_launch(const void * vx, const float * y, float * dst, const int ncols,- const int nrows, dpct::queue_ptr stream) {- const int workgroups = (nrows + 1) / 2;+template <int WG, typename Policy>+static void q8_0_esimd_launch(const Policy & policy, const float * y, const int ncols, const int nrows,+ const int workgroups, dpct::queue_ptr stream) {stream->submit([&](sycl::handler & h) {sycl::local_accessor<float, 1> lmem(sycl::range<1>(WG * 2), h);h.parallel_for(sycl::nd_range<1>(sycl::range<1>((size_t) workgroups * WG), sycl::range<1>(WG)),[=](sycl::nd_item<1> it) [[intel::sycl_explicit_simd]] {- dequantize_mul_mat_vec_q8_0_reorder_esimd<WG>(vx, y, dst, ncols, nrows, lmem, it);+ dequantize_mul_mat_vec_q8_0_reorder_esimd<WG>(policy, y, ncols, nrows, lmem, it);});});}-static void dequantize_mul_mat_vec_q8_0_sycl_reorder_esimd(const void *vx, const float *y,- float *dst, const int ncols,- const int nrows,- dpct::queue_ptr stream) {+// Scale the work-group with the number of blocks per row.+template <typename Policy>+static void q8_0_esimd_launch_scaled(const Policy & policy, const float * y, const int ncols, const int nrows,+ const int workgroups, dpct::queue_ptr stream) {GGML_ASSERT(ncols % QK8_0 == 0);- // Scale the work-group with the number of blocks per row.const int nblk_row = ncols / QK8_0;if (nblk_row >= 64) {- q8_0_esimd_launch<8>(vx, y, dst, ncols, nrows, stream);+ q8_0_esimd_launch<8>(policy, y, ncols, nrows, workgroups, stream);} else if (nblk_row >= 32) {- q8_0_esimd_launch<4>(vx, y, dst, ncols, nrows, stream);+ q8_0_esimd_launch<4>(policy, y, ncols, nrows, workgroups, stream);} else if (nblk_row >= 16) {- q8_0_esimd_launch<2>(vx, y, dst, ncols, nrows, stream);+ q8_0_esimd_launch<2>(policy, y, ncols, nrows, workgroups, stream);} else {- q8_0_esimd_launch<1>(vx, y, dst, ncols, nrows, stream);+ q8_0_esimd_launch<1>(policy, y, ncols, nrows, workgroups, stream);+ }+}++static void dequantize_mul_mat_vec_q8_0_sycl_reorder_esimd(const void *vx, const float *y,+ float *dst, const int ncols,+ const int nrows,+ dpct::queue_ptr stream) {+ q8_0_esimd_launch_scaled(esimd_dmmv_row_pair_policy{ vx, dst }, y, ncols, nrows, (nrows + 1) / 2, stream);+}++// ESIMD fused gate+up+GLU on reordered K-quant SOA weights: one output row (gate+// and up) per work-group, two independent 32-wide accumulators per super-block.+// Reuses the DMMV kernels' dequant+MAC primitive (esimd_reorder_q_traits<T>::mac_pair).+template <ggml_type T>+static void dequantize_mul_mat_vec_glu_reorder_esimd(const void * vx, const void * vgate,+ const float * y, float * dst,+ const int ncols, const int nrows,+ const ggml_glu_op glu_op,+ dpct::queue_ptr stream) {+ GGML_ASSERT(ncols % QK_K == 0);+ const int workgroups = nrows; // one output row (gate+up) per work-group+ stream->submit([&](sycl::handler & h) {+ sycl::local_accessor<float, 1> lmem(sycl::range<1>(GGML_SYCL_DMMV_ESIMD_WG_SIZE * 2), h);+ h.parallel_for(+ sycl::nd_range<1>(sycl::range<1>((size_t) workgroups * GGML_SYCL_DMMV_ESIMD_WG_SIZE), sycl::range<1>(GGML_SYCL_DMMV_ESIMD_WG_SIZE)),+ [=](sycl::nd_item<1> it) [[intel::sycl_explicit_simd]] {+ dequantize_mul_mat_vec_reorder_esimd<T>(+ esimd_dmmv_glu_policy{ vgate, vx, dst, glu_op }, y, ncols, nrows, lmem, it);+ });+ });+}++bool ggml_sycl_dequantize_mul_mat_vec_glu_reorder_esimd(enum ggml_type src0_type, enum ggml_glu_op glu_op,+ const void * vx, const void * vgate, const float * y,+ float * dst, int ncols, int nrows, dpct::queue_ptr stream) {+ switch (src0_type) {+ case GGML_TYPE_Q2_K:+ dequantize_mul_mat_vec_glu_reorder_esimd<GGML_TYPE_Q2_K>(vx, vgate, y, dst, ncols, nrows, glu_op, stream);+ return true;+ case GGML_TYPE_Q3_K:+ dequantize_mul_mat_vec_glu_reorder_esimd<GGML_TYPE_Q3_K>(vx, vgate, y, dst, ncols, nrows, glu_op, stream);+ return true;+ case GGML_TYPE_Q4_K:+ dequantize_mul_mat_vec_glu_reorder_esimd<GGML_TYPE_Q4_K>(vx, vgate, y, dst, ncols, nrows, glu_op, stream);+ return true;+ case GGML_TYPE_Q5_K:+ dequantize_mul_mat_vec_glu_reorder_esimd<GGML_TYPE_Q5_K>(vx, vgate, y, dst, ncols, nrows, glu_op, stream);+ return true;+ case GGML_TYPE_Q6_K:+ dequantize_mul_mat_vec_glu_reorder_esimd<GGML_TYPE_Q6_K>(vx, vgate, y, dst, ncols, nrows, glu_op, stream);+ return true;+ case GGML_TYPE_Q8_0:+ q8_0_esimd_launch_scaled(esimd_dmmv_glu_policy{ vgate, vx, dst, glu_op }, y, ncols, nrows, nrows, stream);+ return true;+ default:+ return false;}}diff --git a/ggml/src/ggml-sycl/dmmv.hpp b/ggml/src/ggml-sycl/dmmv.hppindex bd83735641..318e3489da 100644--- a/ggml/src/ggml-sycl/dmmv.hpp+++ b/ggml/src/ggml-sycl/dmmv.hpp@@ -24,4 +24,13 @@ void ggml_sycl_op_dequantize_mul_mat_vec(const int64_t src1_ncols, const int64_t src1_padded_row_size,const dpct::queue_ptr &stream);+// Fused dense-FFN mat-vec: writes glu(gate . y, up . y) for same-type reordered+// weights (Q2_K..Q6_K, Q8_0) and a single F32 activation column, using the ESIMD DMMV kernels.+// vx is the up weight, vgate the gate weight. Only defined when GGML_SYCL_DMMV_HAS_ESIMD;+// returns false if the type is unhandled.+bool ggml_sycl_dequantize_mul_mat_vec_glu_reorder_esimd(+ enum ggml_type src0_type, enum ggml_glu_op glu_op,+ const void * vx, const void * vgate, const float * y,+ float * dst, int ncols, int nrows, dpct::queue_ptr stream);+#endif // GGML_SYCL_DMMV_HPPdiff --git a/ggml/src/ggml-sycl/fusion.cpp b/ggml/src/ggml-sycl/fusion.cppindex 79fe13a1d0..cb4274e796 100644--- a/ggml/src/ggml-sycl/fusion.cpp+++ b/ggml/src/ggml-sycl/fusion.cpp@@ -4,7 +4,7 @@#include <algorithm>// mul_mat(gate) + mul_mat(up) + GLU: graph shape and tensor properties only. Backend state-// (weight layout, split buffers, DMMV) is checked by ggml_sycl_mul_mat_glu_mmvq_fused().+// (weight layout, split buffers, DMMV) is checked by ggml_sycl_op_mul_mat_glu_fused().static bool ggml_sycl_should_fuse_mul_mat_glu(const ggml_tensor * gate, const ggml_tensor * up,const ggml_tensor * glu) {// the fused epilogue implements these two; the rest fall back to the standalone GLU kernels@@ -32,12 +32,22 @@ static bool ggml_sycl_should_fuse_mul_mat_glu(const ggml_tensor * gate, const ggreturn false;}- // fused GEMVs walk whole QK_K super-blocks: the reorder kernel covers same-type+ // fused K-quant GEMVs walk whole QK_K super-blocks: the reorder kernel covers same-type// q4_K, the plain-layout kernel covers q5_K / iq4_xs pairs incl. mixed gate/up typesconst bool reorder_pair = wu->type == GGML_TYPE_Q4_K && wg->type == GGML_TYPE_Q4_K;const bool plain_pair = (wu->type == GGML_TYPE_Q5_K || wu->type == GGML_TYPE_IQ4_XS) &&(wg->type == GGML_TYPE_Q5_K || wg->type == GGML_TYPE_IQ4_XS);- if ((!reorder_pair && !plain_pair) || wu->ne[0] % QK_K != 0) {+ // the ESIMD kernel covers same-type q2_K..q6_K and q8_0, single column only (as unfused DMMV)+#ifdef GGML_SYCL_DMMV_HAS_ESIMD+ const bool esimd_pair = g_ggml_sycl_enable_esimd && wu->type == wg->type && act->ne[1] == 1 &&+ (wu->type == GGML_TYPE_Q2_K || wu->type == GGML_TYPE_Q3_K || wu->type == GGML_TYPE_Q4_K ||+ wu->type == GGML_TYPE_Q5_K || wu->type == GGML_TYPE_Q6_K || wu->type == GGML_TYPE_Q8_0);+#else+ const bool esimd_pair = false;+#endif+ // q8_0 rows are walked in 32-wide blocks, so they only need to be a multiple of QK8_0+ const int64_t row_align = esimd_pair && wu->type == GGML_TYPE_Q8_0 ? QK8_0 : QK_K;+ if ((!reorder_pair && !plain_pair && !esimd_pair) || wu->ne[0] % row_align != 0) {return false;}diff --git a/ggml/src/ggml-sycl/ggml-sycl.cpp b/ggml/src/ggml-sycl/ggml-sycl.cppindex b380cbedfd..3cc565d6ba 100644--- a/ggml/src/ggml-sycl/ggml-sycl.cpp+++ b/ggml/src/ggml-sycl/ggml-sycl.cpp@@ -46,9 +46,6 @@# include <sycl/ext/oneapi/virtual_mem/virtual_mem.hpp># define GGML_SYCL_SUPPORT_VMM#endif-#if defined(__INTEL_LLVM_COMPILER)- #define GGML_SYCL_DMMV_HAS_ESIMD-#endif#include <sycl/half_type.hpp>#include "ggml.h"@@ -5287,6 +5284,12 @@ static bool ggml_sycl_mul_mat_glu_mmvq_plain(ggml_backend_sycl_context & ctx, ggreturn false;}+ // the plain kernel only covers q5_K / iq4_xs pairs: bail before quantizing the activation+ const auto plain_type = [](ggml_type t) { return t == GGML_TYPE_Q5_K || t == GGML_TYPE_IQ4_XS; };+ if (!plain_type(wu->type) || !plain_type(wg->type)) {+ return false;+ }+// log the up mat-mul: glu's own srcs are the two intermediates the fusion never materialisesscope_op_debug_print scope_dbg_print(__func__, up, /*num_src=*/2, " : fused with gate + GLU (plain layout)");@@ -5311,9 +5314,42 @@ static bool ggml_sycl_mul_mat_glu_mmvq_plain(ggml_backend_sycl_context & ctx, ggGGML_UNUSED(gate);}+#ifdef GGML_SYCL_DMMV_HAS_ESIMD+// {mul_mat(gate), mul_mat(up), GLU} on the reorder layout via the ESIMD DMMV kernel, for+// same-type K-quant or Q8_0 pairs and a single activation column. It reads the F32 activation+// directly, so one launch replaces three and no q8_1 quantization is needed.+static bool ggml_sycl_mul_mat_glu_dmmv_esimd(ggml_backend_sycl_context & ctx, ggml_tensor * glu,+ ggml_tensor * gate, ggml_tensor * up, const ggml_tensor * wu,+ const ggml_tensor * wg, const ggml_tensor * act) {+ // the kernel reads the activation while writing the output; glu may reuse the dead activation's memory+ const char * act_begin = (const char *) act->data;+ const char * glu_begin = (const char *) glu->data;+ if (act_begin < glu_begin + ggml_nbytes(glu) && glu_begin < act_begin + ggml_nbytes(act)) {+ return false;+ }++ // install the reorder (SoA) layout the kernel needs, as the unfused DMMV path would+ opt_for_reorder(&ctx, wu, act, up, mul_mat_algo::DMMV);+ opt_for_reorder(&ctx, wg, act, gate, mul_mat_algo::DMMV);++ const auto * extra_u = static_cast<const ggml_tensor_extra_gpu *>(wu->extra);+ const auto * extra_g = static_cast<const ggml_tensor_extra_gpu *>(wg->extra);+ if (!extra_u || !extra_g || !extra_u->optimized_feature.reorder || !extra_g->optimized_feature.reorder) {+ return false;+ }++ // log the up mat-mul: glu's own srcs are the two intermediates the fusion never materialises+ scope_op_debug_print scope_dbg_print(__func__, up, /*num_src=*/2, " : fused with gate + GLU (ESIMD)");++ return ggml_sycl_dequantize_mul_mat_vec_glu_reorder_esimd(wu->type, ggml_get_glu_op(glu), wu->data, wg->data,+ (const float *) act->data, (float *) glu->data,+ (int) wu->ne[0], (int) wu->ne[1], ctx.stream());+}+#endif+// Fused dense-FFN mat-vec for the {mul_mat(gate), mul_mat(up), GLU} subgraph at node_idx.// Returns false if it declined, in which case the caller runs the three nodes normally.-static bool ggml_sycl_mul_mat_glu_mmvq_fused(ggml_backend_sycl_context & ctx, ggml_cgraph * cgraph, int node_idx) {+static bool ggml_sycl_mul_mat_glu_fused(ggml_backend_sycl_context & ctx, ggml_cgraph * cgraph, int node_idx) {if (!ggml_sycl_can_fuse(cgraph, node_idx, { GGML_OP_MUL_MAT, GGML_OP_MUL_MAT, GGML_OP_GLU }, {})) {return false;}@@ -5331,6 +5367,16 @@ static bool ggml_sycl_mul_mat_glu_mmvq_fused(ggml_backend_sycl_context & ctx, ggreturn false;}+#ifdef GGML_SYCL_DMMV_HAS_ESIMD+ // single-column same-type K-quant or Q8_0 pairs take the ESIMD kernel, as the unfused mul-mats+ // would via DMMV; if the reorder layout could not be installed, fall through to mmvq+ if (g_ggml_sycl_enable_esimd && wu->type == wg->type && act->ne[1] == 1 &&+ ggml_sycl_supports_reorder_esimd(wu->type) &&+ ggml_sycl_mul_mat_glu_dmmv_esimd(ctx, glu, gate, up, wu, wg, act)) {+ return true;+ }+#endif+// with DMMV prioritised the unfused path would not have gone through mmvq at allif (g_ggml_sycl_prioritize_dmmv) {return false;@@ -6594,7 +6640,7 @@ static void ggml_backend_sycl_graph_compute_impl(ggml_backend_sycl_context * syccontinue;}- if (node->op == GGML_OP_MUL_MAT && ggml_sycl_mul_mat_glu_mmvq_fused(*sycl_ctx, cgraph, i)) {+ if (node->op == GGML_OP_MUL_MAT && ggml_sycl_mul_mat_glu_fused(*sycl_ctx, cgraph, i)) {i += 2;continue;}diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cppindex 8826b274b9..011f93865d 100644--- a/tests/test-backend-ops.cpp+++ b/tests/test-backend-ops.cpp@@ -11927,6 +11927,13 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {true, 16, 8, b, false, true, false));}+ for (int64_t k : {96, 896, 1024, 2880}) {+ for (ggml_glu_op glu_op : {GGML_GLU_OP_SWIGLU, GGML_GLU_OP_GEGLU}) {+ test_cases.emplace_back(new test_mul_mat_vec_fusion(GGML_TYPE_Q8_0, glu_op, 1, 32, k,+ false, 16, 8, false, false, true, false, {1, 1}));+ }+ }+// Fused row-pair coverage: minimum rows, an even pair, and an odd tail.// TODO: the max_nmse_err() for these cases is not estimated correctly causing sporadic false failures.//for (ggml_glu_op glu_op : { GGML_GLU_OP_SWIGLU, GGML_GLU_OP_GEGLU }) {