LLM inference in C/C++
C++ ggml
Add IQ type handling for MoE (#28476)
cwriter304665fe7ac9
diff --git a/ggml/src/ggml-sycl/mmvq.cpp b/ggml/src/ggml-sycl/mmvq.cppindex 933bc77d2e..32903431be 100644--- a/ggml/src/ggml-sycl/mmvq.cpp+++ b/ggml/src/ggml-sycl/mmvq.cpp@@ -2671,6 +2671,34 @@ void ggml_sycl_op_mul_mat_vec_q(ggml_backend_sycl_context & ctx, const ggml_tensGGML_UNUSED(ctx);}+// vec_dot_q_sycl_t adapters for the IQ vec_dots that take their codebook tables as extra+// arguments: bind the constant tables here (as vec_dot_iq2_s_q8_1 / vec_dot_iq1_m_q8_1 already do+// internally) so they can be used as template arguments of mul_mat_vec_q_moe.+static __dpct_inline__ float vec_dot_iq2_xxs_q8_1_moe(const void * __restrict__ vbq,+ const block_q8_1 * __restrict__ bq8_1, const int & iqs) {+ return vec_dot_iq2_xxs_q8_1(vbq, bq8_1, iqs, iq2xxs_grid, ksigns_iq2xs, kmask_iq2xs);+}++static __dpct_inline__ float vec_dot_iq2_xs_q8_1_moe(const void * __restrict__ vbq,+ const block_q8_1 * __restrict__ bq8_1, const int & iqs) {+ return vec_dot_iq2_xs_q8_1(vbq, bq8_1, iqs, iq2xs_grid, ksigns64);+}++static __dpct_inline__ float vec_dot_iq3_xxs_q8_1_moe(const void * __restrict__ vbq,+ const block_q8_1 * __restrict__ bq8_1, const int & iqs) {+ return vec_dot_iq3_xxs_q8_1(vbq, bq8_1, iqs, iq3xxs_grid, ksigns64);+}++static __dpct_inline__ float vec_dot_iq3_s_q8_1_moe(const void * __restrict__ vbq,+ const block_q8_1 * __restrict__ bq8_1, const int & iqs) {+ return vec_dot_iq3_s_q8_1(vbq, bq8_1, iqs, iq3s_grid);+}++static __dpct_inline__ float vec_dot_iq1_s_q8_1_moe(const void * __restrict__ vbq,+ const block_q8_1 * __restrict__ bq8_1, const int & iqs) {+ return vec_dot_iq1_s_q8_1(vbq, bq8_1, iqs, iq1s_grid_gpu);+}+// src1_row_stride: 0 for shared src1 (gate/up proj), else per-expert stride (down proj).template <int qk, int qi, typename block_q_t, int vdr, vec_dot_q_sycl_t vec_dot_q_sycl>static void mul_mat_vec_q_moe(@@ -2822,6 +2850,51 @@ bool ggml_sycl_mul_mat_vec_q_id(vx_base, vy, ids_dev, dst_base, ncols, nrows, n_experts_used,expert_weight_stride, dst_row_stride, src1_row_stride, stream);return true;+ case GGML_TYPE_IQ2_XXS:+ launch_mul_mat_vec_q_moe<QK_K, QI2_XXS/2, block_iq2_xxs, VDR_IQ2_XXS_Q8_1_MMVQ, vec_dot_iq2_xxs_q8_1_moe>(+ vx_base, vy, ids_dev, dst_base, ncols, nrows, n_experts_used,+ expert_weight_stride, dst_row_stride, src1_row_stride, stream);+ return true;+ case GGML_TYPE_IQ2_XS:+ launch_mul_mat_vec_q_moe<QK_K, QI2_XS/2, block_iq2_xs, VDR_IQ2_XS_Q8_1_MMVQ, vec_dot_iq2_xs_q8_1_moe>(+ vx_base, vy, ids_dev, dst_base, ncols, nrows, n_experts_used,+ expert_weight_stride, dst_row_stride, src1_row_stride, stream);+ return true;+ case GGML_TYPE_IQ2_S:+ launch_mul_mat_vec_q_moe<QK_K, QI2_S/2, block_iq2_s, VDR_IQ2_S_Q8_1_MMVQ, vec_dot_iq2_s_q8_1>(+ vx_base, vy, ids_dev, dst_base, ncols, nrows, n_experts_used,+ expert_weight_stride, dst_row_stride, src1_row_stride, stream);+ return true;+ case GGML_TYPE_IQ3_XXS:+ launch_mul_mat_vec_q_moe<QK_K, QI3_XXS/2, block_iq3_xxs, VDR_IQ3_XXS_Q8_1_MMVQ, vec_dot_iq3_xxs_q8_1_moe>(+ vx_base, vy, ids_dev, dst_base, ncols, nrows, n_experts_used,+ expert_weight_stride, dst_row_stride, src1_row_stride, stream);+ return true;+ case GGML_TYPE_IQ3_S:+ launch_mul_mat_vec_q_moe<QK_K, QI3_S/2, block_iq3_s, VDR_IQ3_S_Q8_1_MMVQ, vec_dot_iq3_s_q8_1_moe>(+ vx_base, vy, ids_dev, dst_base, ncols, nrows, n_experts_used,+ expert_weight_stride, dst_row_stride, src1_row_stride, stream);+ return true;+ case GGML_TYPE_IQ1_S:+ launch_mul_mat_vec_q_moe<QK_K, QI1_S, block_iq1_s, VDR_IQ1_S_Q8_1_MMVQ, vec_dot_iq1_s_q8_1_moe>(+ vx_base, vy, ids_dev, dst_base, ncols, nrows, n_experts_used,+ expert_weight_stride, dst_row_stride, src1_row_stride, stream);+ return true;+ case GGML_TYPE_IQ1_M:+ launch_mul_mat_vec_q_moe<QK_K, QI1_S, block_iq1_m, VDR_IQ1_M_Q8_1_MMVQ, vec_dot_iq1_m_q8_1>(+ vx_base, vy, ids_dev, dst_base, ncols, nrows, n_experts_used,+ expert_weight_stride, dst_row_stride, src1_row_stride, stream);+ return true;+ case GGML_TYPE_IQ4_NL:+ launch_mul_mat_vec_q_moe<QK4_NL, QI4_NL, block_iq4_nl, VDR_IQ4_NL_Q8_1_MMVQ, vec_dot_iq4_nl_q8_1>(+ vx_base, vy, ids_dev, dst_base, ncols, nrows, n_experts_used,+ expert_weight_stride, dst_row_stride, src1_row_stride, stream);+ return true;+ case GGML_TYPE_IQ4_XS:+ launch_mul_mat_vec_q_moe<QK_K, QI4_XS/4, block_iq4_xs, VDR_IQ4_XS_Q8_1_MMVQ, vec_dot_iq4_xs_q8_1>(+ vx_base, vy, ids_dev, dst_base, ncols, nrows, n_experts_used,+ expert_weight_stride, dst_row_stride, src1_row_stride, stream);+ return true;default:return false;}diff --git a/ggml/src/ggml-sycl/vecdotq.hpp b/ggml/src/ggml-sycl/vecdotq.hppindex ed5fd7de89..909f7a7895 100644--- a/ggml/src/ggml-sycl/vecdotq.hpp+++ b/ggml/src/ggml-sycl/vecdotq.hpp@@ -1398,6 +1398,11 @@ vec_dot_q6_K_q8_1(const void *__restrict__ vbq,}+// NOTE: the VDR_IQ*_Q8_1_MMVQ values deliberately differ from the identically named CUDA constants+// (vecdotq.cuh): the SYCL kernels pair them with a halved qi (e.g. QI3_S/2), so the values are not+// interchangeable and must not be copied across backends.+#define VDR_IQ2_XXS_Q8_1_MMVQ 1+static __dpct_inline__ floatvec_dot_iq2_xxs_q8_1(const void *__restrict__ vbq,const block_q8_1 *__restrict__ bq8_1, const int &iqs,@@ -1429,6 +1434,8 @@ vec_dot_iq2_xxs_q8_1(const void *__restrict__ vbq,#endif}+#define VDR_IQ2_XS_Q8_1_MMVQ 1+static __dpct_inline__ floatvec_dot_iq2_xs_q8_1(const void *__restrict__ vbq,const block_q8_1 *__restrict__ bq8_1, const int &iqs,@@ -1479,6 +1486,8 @@ vec_dot_iq2_xs_q8_1(const void *__restrict__ vbq,#endif}+#define VDR_IQ2_S_Q8_1_MMVQ 1+static __dpct_inline__ floatvec_dot_iq2_s_q8_1(const void *__restrict__ vbq,const block_q8_1 *__restrict__ bq8_1, const int &iqs) {@@ -1531,6 +1540,8 @@ vec_dot_iq2_s_q8_1(const void *__restrict__ vbq,#endif}+#define VDR_IQ3_XXS_Q8_1_MMVQ 1+static __dpct_inline__ floatvec_dot_iq3_xxs_q8_1(const void *__restrict__ vbq,const block_q8_1 *__restrict__ bq8_1, const int &iqs,@@ -1571,6 +1582,8 @@ vec_dot_iq3_xxs_q8_1(const void *__restrict__ vbq,#endif}+#define VDR_IQ3_S_Q8_1_MMVQ 1+static __dpct_inline__ floatvec_dot_iq3_s_q8_1(const void *__restrict__ vbq,const block_q8_1 *__restrict__ bq8_1, const int &iqs,@@ -1609,6 +1622,8 @@ vec_dot_iq3_s_q8_1(const void *__restrict__ vbq,#endif}+#define VDR_IQ1_S_Q8_1_MMVQ 1+static __dpct_inline__ floatvec_dot_iq1_s_q8_1(const void *__restrict__ vbq,const block_q8_1 *__restrict__ bq8_1, const int &iqs,@@ -1637,6 +1652,8 @@ vec_dot_iq1_s_q8_1(const void *__restrict__ vbq,#endif}+#define VDR_IQ1_M_Q8_1_MMVQ 1+static __dpct_inline__ floatvec_dot_iq1_m_q8_1(const void *__restrict__ vbq,const block_q8_1 *__restrict__ bq8_1, const int &iqs) {@@ -1671,6 +1688,8 @@ vec_dot_iq1_m_q8_1(const void *__restrict__ vbq,}+#define VDR_IQ4_NL_Q8_1_MMVQ 2+static __dpct_inline__ floatvec_dot_iq4_nl_q8_1(const void *__restrict__ vbq,const block_q8_1 *__restrict__ bq8_1, const int &iqs) {@@ -1696,6 +1715,8 @@ vec_dot_iq4_nl_q8_1(const void *__restrict__ vbq,}+#define VDR_IQ4_XS_Q8_1_MMVQ 1+static __dpct_inline__ floatvec_dot_iq4_xs_q8_1(const void *__restrict__ vbq,const block_q8_1 *__restrict__ bq8_1, const int &iqs) {