LLM inference in C/C++
C++ ggml
sycl: q8 fast dequant (#30775)
92es0756e83fc152
diff --git a/ggml/src/ggml-sycl/convert.cpp b/ggml/src/ggml-sycl/convert.cppindex bc1b74f349..bcd0dd6b34 100644--- a/ggml/src/ggml-sycl/convert.cpp+++ b/ggml/src/ggml-sycl/convert.cpp@@ -177,15 +177,12 @@ static void dequantize_row_q8_0_sycl_reorder(const void *vx, dst_t *y, const intdpct::has_capability_or_fail(stream->get_device(),{sycl::aspect::fp16});+ // flat range<1>, two pairs (4 elements) per item: no nd_range/group math, one vectorized store.+ // k % QK8_0 == 0 implies k % 4 == 0, so the wide launch covers the row exactly (no tail).GGML_ASSERT(k % QK8_0 == 0);- const int64_t n_groups = ceil_div(k / 2, SYCL_DEQUANTIZE_BLOCK_SIZE);- // decrease global range when it exceeds the max int; the kernel loops to cover the rest- const int64_t wg_size = downsample_sycl_global_range(n_groups, SYCL_DEQUANTIZE_BLOCK_SIZE);- stream->parallel_for(sycl::nd_range<3>(sycl::range<3>(1, 1, n_groups) *- sycl::range<3>(1, 1, wg_size),- sycl::range<3>(1, 1, wg_size)),- [=](sycl::nd_item<3> item_ct1) {- dequantize_block_q8_0_reorder(vx, y, k, item_ct1);+ stream->parallel_for(sycl::range<1>((size_t) (k / 4)),+ [=](sycl::id<1> idx) {+ dequantize_block_q8_0_reorder_wide(vx, y, k, idx);});}diff --git a/ggml/src/ggml-sycl/dequantize.hpp b/ggml/src/ggml-sycl/dequantize.hppindex 40f5cd6460..ae709efa1f 100644--- a/ggml/src/ggml-sycl/dequantize.hpp+++ b/ggml/src/ggml-sycl/dequantize.hpp@@ -857,6 +857,26 @@ static void dequantize_block_q8_0_reorder(const void * __restrict__ vx, dst_t *}}+template<typename dst_t>+static void dequantize_block_q8_0_reorder_wide(const void * __restrict__ vx, dst_t * __restrict__ yy, int64_t k,+ const sycl::id<1> idx) {+ const int64_t i = 4 * (int64_t) idx[0];++ const int8_t * qs = (const int8_t *) vx;+ const sycl::half * d_ptr = (const sycl::half *) ((const uint8_t *) vx + k);++ const float d0 = float(d_ptr[i / QK8_0]);+ const float d1 = float(d_ptr[(i + 2) / QK8_0]);++ sycl::vec<float, 4> p;+ p[0] = d0 * (float) qs[i + 0];+ p[1] = d0 * (float) qs[i + 1];+ p[2] = d1 * (float) qs[i + 2];+ p[3] = d1 * (float) qs[i + 3];++ *reinterpret_cast<sycl::vec<dst_t, 4> *>(yy + i) = p.convert<dst_t, sycl::rounding_mode::automatic>();+}+template<typename dst_t>static void dequantize_block_q4_1(const void * __restrict__ vx, dst_t * __restrict__ yy, int64_t nb32,const sycl::nd_item<3> &item_ct1) {