LLM inference in C/C++
C++ ggml
opencl: make the MoE expert scatter deterministic (#26464)
Hongqiang Wang9ee9fc04c136
diff --git a/ggml/src/ggml-opencl/ggml-opencl.cpp b/ggml/src/ggml-opencl/ggml-opencl.cppindex fa4702a855..fbf7dadb90 100644--- a/ggml/src/ggml-opencl/ggml-opencl.cpp+++ b/ggml/src/ggml-opencl/ggml-opencl.cpp@@ -895,6 +895,7 @@ struct ggml_backend_opencl_context {cl_kernel kernel_gemm_moe_q4_0_q8_1_dp4a = nullptr; // dp4a (int8) q4_0 MoE prefill GEMMcl_kernel kernel_moe_reorder_b;cl_kernel kernel_moe_histogram, kernel_moe_scan, kernel_moe_fill, kernel_moe_scatter;+ cl_kernel kernel_moe_scatter_stable = nullptr; // deterministic slot assignmentcl_kernel kernel_moe_combine_f32 = nullptr; // fused router-weight mul + cross-expert sumcl_kernel kernel_mul_mv_id_q4_0_f32_8x_flat;cl_kernel kernel_mul_mv_id_q8_0_f32, kernel_mul_mv_id_q8_0_f32_flat;@@ -4463,6 +4464,7 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) {CL_CHECK((backend_ctx->kernel_moe_scan = clCreateKernel(prog, "kernel_moe_scan", &err), err));CL_CHECK((backend_ctx->kernel_moe_fill = clCreateKernel(prog, "kernel_moe_fill", &err), err));CL_CHECK((backend_ctx->kernel_moe_scatter = clCreateKernel(prog, "kernel_moe_scatter", &err), err));+ CL_CHECK((backend_ctx->kernel_moe_scatter_stable = clCreateKernel(prog, "kernel_moe_scatter_stable", &err), err));CL_CHECK(clReleaseProgram(prog));GGML_LOG_CONT(".");}@@ -20863,18 +20865,42 @@ static void moe_router_reoerder(ggml_backend_t backend, const ggml_tensor * src,size_t fill_local_size[] = {64, 1, 1};backend_ctx->enqueue_ndrange_kernel(kernel, 3, fill_global_size, fill_local_size, src);- // Scatter- kernel = backend_ctx->kernel_moe_scatter;- CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &original_router_buf));- CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &post_router_buf));- CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &emap_buf));- CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &tile_offset_buf));- CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &slot_counter_buf));- CL_CHECK(clSetKernelArg(kernel, 5, sizeof(int), &ne21));- CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne20));- CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne02));+ // Scatter. The deterministic variant is the default: kernel_moe_scatter derives+ // each token's slot from an atomic counter, so the packing inside an expert - and+ // with it the output of the ragged prefill GEMM - changes from run to run. Set+ // GGML_OPENCL_MOE_STABLE_SCATTER=0 to restore the atomic version.+ static const bool stable_scatter = []{+ const char * e = getenv("GGML_OPENCL_MOE_STABLE_SCATTER");+ return !e || e[0] == '\0' || e[0] != '0';+ }();- backend_ctx->enqueue_ndrange_kernel(kernel, 3, histogram_global_size, histogram_local_size, src);+ if (stable_scatter) {+ kernel = backend_ctx->kernel_moe_scatter_stable;+ CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &original_router_buf));+ CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &post_router_buf));+ CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &emap_buf));+ CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &tile_offset_buf));+ CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &ne21));+ CL_CHECK(clSetKernelArg(kernel, 5, sizeof(int), &ne20));+ CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne02));++ // one workgroup (one wave) per expert; each ranks its own tokens+ size_t scatter_global_size[] = {64, (size_t)ne02};+ size_t scatter_local_size[] = {64, 1};+ backend_ctx->enqueue_ndrange_kernel(kernel, 2, scatter_global_size, scatter_local_size, src);+ } else {+ kernel = backend_ctx->kernel_moe_scatter;+ CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &original_router_buf));+ CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &post_router_buf));+ CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &emap_buf));+ CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &tile_offset_buf));+ CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &slot_counter_buf));+ CL_CHECK(clSetKernelArg(kernel, 5, sizeof(int), &ne21));+ CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne20));+ CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne02));++ backend_ctx->enqueue_ndrange_kernel(kernel, 3, histogram_global_size, histogram_local_size, src);+ }// [MOE_TILES] env-gated padding probe: read back total_tiles (= Sum_e// ceil(k_e/n_tile_size)) and compare to the ideal tile count for the realdiff --git a/ggml/src/ggml-opencl/kernels/moe_sort_by_expert.cl b/ggml/src/ggml-opencl/kernels/moe_sort_by_expert.clindex d9703429b1..d52d11aa56 100644--- a/ggml/src/ggml-opencl/kernels/moe_sort_by_expert.cl+++ b/ggml/src/ggml-opencl/kernels/moe_sort_by_expert.cl@@ -68,6 +68,79 @@ __kernel void kernel_moe_scatter(emap[tile_idx] = val;}+// Deterministic replacement for kernel_moe_scatter.+//+// kernel_moe_scatter takes each token's slot from atomic_inc(slot_counter[expert]),+// so the token -> slot packing inside an expert depends on which work-item wins the+// atomic and changes from run to run. The ragged prefill GEMM path is sensitive to+// that packing (the non-ragged path is not, since its padded slots alias slot 0 and+// are overwritten last), which makes MoE prompt processing non-reproducible: the same+// binary on the same prompt returns one of several outputs.+//+// Here the slot is the token's rank in flat (n, k) order among the tokens routed to+// the same expert - a fixed function of the routing input. One workgroup per expert+// walks the flat routing list in blocks of 64 and ranks its own tokens with a+// workgroup scan, carrying a running count between blocks. Cost is one pass over the+// routing list per expert; the list is a few KiB and stays in cache.+__kernel void kernel_moe_scatter_stable(+ __global const int * input,+ __global int * post_router,+ __global ushort * emap,+ __global const int * tile_offset,+ int N,+ int topK,+ uint n_experts+) {+ const int e = get_group_id(1);+ const int lid = get_local_id(0);+ const int M = N * topK;++ __local int scan[64];+ __local int running;++ if (lid == 0) {+ running = 0;+ }+ barrier(CLK_LOCAL_MEM_FENCE);++ for (int base = 0; base < M; base += 64) {+ const int j = base + lid;++ int pred = 0;+ if (j < M) {+ const int n = j / topK;+ const int k = j - n * topK;+ pred = (input[n * (int)n_experts + k] == e) ? 1 : 0;+ }++ scan[lid] = pred;+ barrier(CLK_LOCAL_MEM_FENCE);++ // Hillis-Steele inclusive scan over the 64 lanes+ for (int off = 1; off < 64; off <<= 1) {+ int add = (lid >= off) ? scan[lid - off] : 0;+ barrier(CLK_LOCAL_MEM_FENCE);+ scan[lid] += add;+ barrier(CLK_LOCAL_MEM_FENCE);+ }++ if (pred) {+ const int local_slot = running + (scan[lid] - 1); // exclusive rank+ const int tile_idx = tile_offset[e] + (local_slot >> 5);+ const int lane = local_slot & 31;++ post_router[tile_idx * 32 + lane] = j;+ emap[tile_idx] = (ushort)e;+ }++ barrier(CLK_LOCAL_MEM_FENCE);+ if (lid == 63) {+ running += scan[63];+ }+ barrier(CLK_LOCAL_MEM_FENCE);+ }+}+__kernel void kernel_moe_fill(__global int * post_router,__global int * total_tiles,