Skip to content

LLM inference in C/C++

C++ ggml

sycl: split long rows in TOP_K instead of one work-group per row (#27847)

Titaniumtown370cb12e8bb7
diff --git a/ggml/src/ggml-sycl/ggml-sycl.cpp b/ggml/src/ggml-sycl/ggml-sycl.cpp
index dc8a174432..d58ffd00da 100644
--- a/ggml/src/ggml-sycl/ggml-sycl.cpp
+++ b/ggml/src/ggml-sycl/ggml-sycl.cpp
@@ -2402,7 +2402,138 @@ static void argsort_f32_i32_sycl(const float *x, int *dst, const int ncols,
}
}
+// Scan and block merge, shared by every launch shape below so a partitioned row uses the
+// same insertion order as an unpartitioned one.
+//
+// src_map != nullptr: report src_map[col] instead of col, so a merge pass can carry the
+// original column index through.
+// out_vals != nullptr: also emit the k winning values, for a later merge pass.
+// swap01: emit in the output order the single-pass path uses.
+static void top_k_scan_merge_f32(
+ const float * src_vals,
+ const int32_t * src_map,
+ const int begin,
+ const int end,
+ const int k,
+ const int block_size,
+ float * shared_vals,
+ int * shared_idx,
+ float * out_vals,
+ int32_t * out_idx,
+ const bool swap01,
+ const sycl::nd_item<1> & item_ct1
+) {
+ const int tid = item_ct1.get_local_id(0);
+
+ // The running top-k lives in SLM (shared local memory) rather than a private array:
+ // an array indexed by a runtime position cannot be register-allocated, so a private
+ // one lands in scratch, i.e. device memory, and insertion is this kernel's dominant
+ // cost.
+ //
+ // Lane-strided (lv[i * block_size]) rather than lane-blocked (lv[i]) so a given i is
+ // contiguous across lanes; a k-strided layout would put every lane of a shift step in
+ // the same SLM bank.
+ float * lv = shared_vals + tid;
+ int * li = shared_idx + tid;
+
+ for (int i = 0; i < k; i++) {
+ lv[i * block_size] = -FLT_MAX;
+ li[i * block_size] = -1;
+ }
+
+ // The k-th best, cached in a register. The reject test is taken for the large
+ // majority of elements scanned, and in that case touches no memory.
+ float kth = -FLT_MAX;
+
+ for (int col = begin + tid; col < end; col += block_size) {
+ float val = src_vals[col];
+
+ if (val > kth) {
+ int pos = k - 1;
+ while (pos > 0 && val > lv[(pos - 1) * block_size]) {
+ pos--;
+ }
+
+ for (int i = k - 1; i > pos; i--) {
+ lv[i * block_size] = lv[(i - 1) * block_size];
+ li[i * block_size] = li[(i - 1) * block_size];
+ }
+ lv[pos * block_size] = val;
+ li[pos * block_size] = src_map ? src_map[col] : col;
+
+ kth = lv[(k - 1) * block_size];
+ }
+ }
+
+ item_ct1.barrier(sycl::access::fence_space::local_space);
+
+ if (tid != 0) {
+ return;
+ }
+
+ // Same treatment for the merge accumulator, past the per-lane region.
+ float * fv = shared_vals + (size_t) k * block_size;
+ int * fi = shared_idx + (size_t) k * block_size;
+
+ for (int i = 0; i < k; i++) {
+ fv[i] = -FLT_MAX;
+ fi[i] = -1;
+ }
+
+ float fkth = -FLT_MAX;
+
+ // Candidates are visited in the same (t, i) order as before, so tie-breaking is
+ // unchanged.
+ for (int t = 0; t < block_size; t++) {
+ for (int i = 0; i < k; i++) {
+ float val = shared_vals[i * block_size + t];
+
+ if (val <= fkth) {
+ // Lane t's list is sorted descending, so once one of its entries loses
+ // to the k-th best, every later entry loses too. fkth only rises, so
+ // that stays true for the rest of the merge. This turns the merge from
+ // block_size*k steps into roughly block_size plus the candidates
+ // accepted.
+ break;
+ }
+
+ int idx = shared_idx[i * block_size + t];
+
+ int pos = k - 1;
+ while (pos > 0 && val > fv[pos - 1]) {
+ pos--;
+ }
+
+ for (int j = k - 1; j > pos; j--) {
+ fv[j] = fv[j - 1];
+ fi[j] = fi[j - 1];
+ }
+ fv[pos] = val;
+ fi[pos] = idx;
+
+ fkth = fv[k - 1];
+ }
+ }
+
+ if (out_vals) {
+ for (int i = 0; i < k; i++) {
+ out_vals[i] = fv[i];
+ }
+ }
+
+ for (int i = 0; i < k; i++) {
+ out_idx[i] = fi[i];
+ }
+
+ if (swap01 && k > 1) {
+ int32_t temp = out_idx[0];
+ out_idx[0] = out_idx[1];
+ out_idx[1] = temp;
+ }
+}
+
static void top_k_f32_sycl(
+ ggml_backend_sycl_context & ctx,
const float * src,
int32_t * dst_indices,
const int64_t ncols,
@@ -2410,98 +2541,107 @@ static void top_k_f32_sycl(
const int k,
dpct::queue_ptr main_stream
) {
- const int block_size = 128;
+ // A row is scanned by exactly one work-group, so a vocabulary-sized row leaves the
+ // rest of the device idle. What the scan is short of is memory requests in flight,
+ // not bandwidth or per-request latency, so lanes in flight is the lever: split the
+ // row across independent work-groups, have each emit its partition's top-k, and
+ // merge those nsplit*k candidates in a second launch.
+ //
+ // split_block trades parallelism against SLM residency. Its cost is
+ // (split_block + 1) * k * 8 bytes of SLM per group, so at the k <= 32 ceiling 128
+ // lanes need about 33 KB, which leaves a single resident group per Xe-core. Revisit
+ // if the supported k ever grows.
+ constexpr int split_block = 128;
+ constexpr int max_splits = 128;
+ constexpr int min_cols = 8192;
- const sycl::range<1> block_dims(block_size);
- const sycl::range<1> grid_dims(nrows);
+ int nsplit = 1;
+ if (ncols >= min_cols) {
+ // A partition is then always >= split_block = 128 columns, hence always more than
+ // the k <= 32 ceiling, so no pass is ever padded with -FLT_MAX sentinels.
+ const int64_t want = ncols / split_block;
+ nsplit = (int) (want > max_splits ? max_splits : want);
+ }
- main_stream->submit([&](sycl::handler &cgh) {
- sycl::local_accessor<float, 1> shared_vals(sycl::range<1>(block_size * k), cgh);
- sycl::local_accessor<int, 1> shared_idx(sycl::range<1>(block_size * k), cgh);
+ if (nsplit > 1) {
+ const int nchunk = (int) ((ncols + nsplit - 1) / nsplit);
+ const size_t ncand = (size_t) nrows * nsplit * k;
- cgh.parallel_for(
- sycl::nd_range<1>(grid_dims * block_dims, block_dims),
- [=](sycl::nd_item<1> item_ct1) {
- const int row = item_ct1.get_group(0);
- const int tid = item_ct1.get_local_id(0);
+ ggml_sycl_pool_alloc<float> part_vals(ctx.pool(), ncand);
+ ggml_sycl_pool_alloc<int32_t> part_idx(ctx.pool(), ncand);
- if (row >= nrows) return;
+ float * pv = part_vals.get();
+ int32_t * pi = part_idx.get();
- const float * src_row = src + row * ncols;
- int32_t * dst_idx_row = dst_indices + row * k;
+ const sycl::range<1> block_dims(split_block);
- float local_vals[32];
- int local_idx[32];
+ main_stream->submit([&](sycl::handler &cgh) {
+ sycl::local_accessor<float, 1> shared_vals(sycl::range<1>((split_block + 1) * k), cgh);
+ sycl::local_accessor<int, 1> shared_idx(sycl::range<1>((split_block + 1) * k), cgh);
- for (int i = 0; i < k; i++) {
- local_vals[i] = -FLT_MAX;
- local_idx[i] = -1;
- }
+ cgh.parallel_for(
+ sycl::nd_range<1>(sycl::range<1>(nrows * nsplit) * block_dims, block_dims),
+ [=](sycl::nd_item<1> item_ct1) {
+ const int grp = item_ct1.get_group(0);
+ const int row = grp / nsplit;
+ const int part = grp % nsplit;
+
+ const int begin = part * nchunk;
+ int end = begin + nchunk;
+ if (end > (int) ncols) {
+ end = (int) ncols;
+ }
- for (int col = tid; col < ncols; col += block_size) {
- float val = src_row[col];
+ top_k_scan_merge_f32(
+ src + (int64_t) row * ncols, nullptr, begin, end, k, split_block,
+ shared_vals.get_multi_ptr<sycl::access::decorated::no>().get(),
+ shared_idx.get_multi_ptr<sycl::access::decorated::no>().get(),
+ pv + (size_t) grp * k, pi + (size_t) grp * k, false, item_ct1);
+ });
+ });
- if (val > local_vals[k-1]) {
- int pos = k - 1;
- while (pos > 0 && val > local_vals[pos - 1]) {
- pos--;
- }
+ main_stream->submit([&](sycl::handler &cgh) {
+ sycl::local_accessor<float, 1> shared_vals(sycl::range<1>((split_block + 1) * k), cgh);
+ sycl::local_accessor<int, 1> shared_idx(sycl::range<1>((split_block + 1) * k), cgh);
- for (int i = k - 1; i > pos; i--) {
- local_vals[i] = local_vals[i - 1];
- local_idx[i] = local_idx[i - 1];
- }
- local_vals[pos] = val;
- local_idx[pos] = col;
- }
- }
+ cgh.parallel_for(
+ sycl::nd_range<1>(sycl::range<1>(nrows) * block_dims, block_dims),
+ [=](sycl::nd_item<1> item_ct1) {
+ const int row = item_ct1.get_group(0);
+ const size_t off = (size_t) row * nsplit * k;
+
+ top_k_scan_merge_f32(
+ pv + off, pi + off, 0, nsplit * k, k, split_block,
+ shared_vals.get_multi_ptr<sycl::access::decorated::no>().get(),
+ shared_idx.get_multi_ptr<sycl::access::decorated::no>().get(),
+ nullptr, dst_indices + (int64_t) row * k, true, item_ct1);
+ });
+ });
- for (int i = 0; i < k; i++) {
- shared_vals[tid * k + i] = local_vals[i];
- shared_idx[tid * k + i] = local_idx[i];
- }
- item_ct1.barrier(sycl::access::fence_space::local_space);
+ return;
+ }
- if (tid == 0) {
- float final_vals[32];
- int final_idx[32];
+ const int block_size = 128;
- for (int i = 0; i < k; i++) {
- final_vals[i] = -FLT_MAX;
- final_idx[i] = -1;
- }
+ const sycl::range<1> block_dims(block_size);
+ const sycl::range<1> grid_dims(nrows);
- for (int t = 0; t < block_size; t++) {
- for (int i = 0; i < k; i++) {
- float val = shared_vals[t * k + i];
- int idx = shared_idx[t * k + i];
-
- if (val > final_vals[k-1]) {
- int pos = k - 1;
- while (pos > 0 && val > final_vals[pos - 1]) {
- pos--;
- }
-
- for (int j = k - 1; j > pos; j--) {
- final_vals[j] = final_vals[j - 1];
- final_idx[j] = final_idx[j - 1];
- }
- final_vals[pos] = val;
- final_idx[pos] = idx;
- }
- }
- }
+ main_stream->submit([&](sycl::handler &cgh) {
+ sycl::local_accessor<float, 1> shared_vals(sycl::range<1>((block_size + 1) * k), cgh);
+ sycl::local_accessor<int, 1> shared_idx(sycl::range<1>((block_size + 1) * k), cgh);
- for (int i = 0; i < k; i++) {
- dst_idx_row[i] = final_idx[i];
- }
+ cgh.parallel_for(
+ sycl::nd_range<1>(grid_dims * block_dims, block_dims),
+ [=](sycl::nd_item<1> item_ct1) {
+ const int row = item_ct1.get_group(0);
- if (k > 1) {
- int32_t temp = dst_idx_row[0];
- dst_idx_row[0] = dst_idx_row[1];
- dst_idx_row[1] = temp;
- }
- }
+ if (row >= nrows) return;
+
+ top_k_scan_merge_f32(
+ src + (int64_t) row * ncols, nullptr, 0, (int) ncols, k, block_size,
+ shared_vals.get_multi_ptr<sycl::access::decorated::no>().get(),
+ shared_idx.get_multi_ptr<sycl::access::decorated::no>().get(),
+ nullptr, dst_indices + (int64_t) row * k, true, item_ct1);
});
});
}
@@ -2902,7 +3042,7 @@ static void ggml_sycl_op_top_k(ggml_backend_sycl_context & ctx, ggml_tensor * ds
GGML_ASSERT(k > 0 && k <= 32);
GGML_ASSERT(k <= ncols);
- top_k_f32_sycl(src0_dd, dst_dd, ncols, nrows, k, main_stream);
+ top_k_f32_sycl(ctx, src0_dd, dst_dd, ncols, nrows, k, main_stream);
}
inline void ggml_sycl_op_argmax(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {