LLM inference in C/C++
C++ ggml
[ggml-webgpu] Implement 2D workgroups for scale, binary, and unary ops (#24044)
Nikhil Jain3b3da01dc21d
diff --git a/ggml/src/ggml-webgpu/ggml-webgpu.cpp b/ggml/src/ggml-webgpu/ggml-webgpu.cppindex c6cfb0bbba..94a108dfa7 100644--- a/ggml/src/ggml-webgpu/ggml-webgpu.cpp+++ b/ggml/src/ggml-webgpu/ggml-webgpu.cpp@@ -621,10 +621,11 @@ static void ggml_backend_webgpu_buffer_memset(webgpu_global_context & ctx,uint32_t value,size_t offset,size_t size) {- std::vector<uint32_t> params = { (uint32_t) offset, (uint32_t) size, value };- std::vector<wgpu::BindGroupEntry> entries = { ggml_webgpu_make_bind_group_entry(0, buf, 0, buf.GetSize()) };- size_t bytes_per_wg = ctx->capabilities.limits.maxComputeInvocationsPerWorkgroup * ctx->capabilities.memset_bytes_per_thread;- uint32_t wg_x = CEIL_DIV(size + 3, bytes_per_wg);+ std::vector<uint32_t> params = { (uint32_t) offset, (uint32_t) size, value };+ std::vector<wgpu::BindGroupEntry> entries = { ggml_webgpu_make_bind_group_entry(0, buf, 0, buf.GetSize()) };+ size_t bytes_per_wg =+ ctx->capabilities.limits.maxComputeInvocationsPerWorkgroup * ctx->capabilities.memset_bytes_per_thread;+ uint32_t wg_x = CEIL_DIV(size + 3, bytes_per_wg);ctx->queue.WriteBuffer(ctx->memset_params_buf, 0, params.data(), params.size() * sizeof(uint32_t));@@ -1362,7 +1363,7 @@ static webgpu_encoded_op ggml_webgpu_get_rows(webgpu_context & ctx,shader_lib_ctx.src0 = src;shader_lib_ctx.src1 = nullptr;shader_lib_ctx.dst = dst;- shader_lib_ctx.max_wg_size = ctx->global_ctx->capabilities.limits.maxComputeInvocationsPerWorkgroup;+ shader_lib_ctx.max_wg_size = ctx->global_ctx->capabilities.limits.maxComputeInvocationsPerWorkgroup;webgpu_pipeline pipeline = ctx->shader_lib->get_get_rows_pipeline(shader_lib_ctx);auto * decisions = static_cast<ggml_webgpu_generic_shader_decisions *>(pipeline.context.get());@@ -2169,8 +2170,10 @@ static webgpu_encoded_op ggml_webgpu_unary_op(webgpu_context & ctx, ggml_tensorentries.push_back(ggml_webgpu_make_tensor_bind_group_entry(ctx, 1, dst));}- uint32_t wg_x = CEIL_DIV(ne, decisions->wg_size);- return ggml_backend_webgpu_build(ctx, pipeline, params, entries, wg_x);+ uint32_t wg_x, wg_y;+ uint32_t total_wg = CEIL_DIV(ggml_nelements(dst), decisions->wg_size);+ compute_2d_workgroups(total_wg, ctx->global_ctx->capabilities.limits.maxComputeWorkgroupsPerDimension, wg_x, wg_y);+ return ggml_backend_webgpu_build(ctx, pipeline, params, entries, wg_x, wg_y);}static webgpu_encoded_op ggml_webgpu_binary_op(webgpu_context & ctx,@@ -2244,8 +2247,10 @@ static webgpu_encoded_op ggml_webgpu_binary_op(webgpu_context & ctx,}}- uint32_t wg_x = CEIL_DIV(ne, decisions->wg_size);- return ggml_backend_webgpu_build(ctx, pipeline, params, entries, wg_x);+ uint32_t wg_x, wg_y;+ uint32_t total_wg = CEIL_DIV(ggml_nelements(dst), decisions->wg_size);+ compute_2d_workgroups(total_wg, ctx->global_ctx->capabilities.limits.maxComputeWorkgroupsPerDimension, wg_x, wg_y);+ return ggml_backend_webgpu_build(ctx, pipeline, params, entries, wg_x, wg_y);}static webgpu_encoded_op ggml_webgpu_add_id(webgpu_context & ctx,@@ -2673,8 +2678,10 @@ static webgpu_encoded_op ggml_webgpu_scale(webgpu_context & ctx, ggml_tensor * sentries.push_back(ggml_webgpu_make_tensor_bind_group_entry(ctx, 1, dst));}- uint32_t wg_x = CEIL_DIV(ggml_nelements(dst), decisions->wg_size);- return ggml_backend_webgpu_build(ctx, pipeline, params, entries, wg_x);+ uint32_t wg_x, wg_y;+ uint32_t total_wg = CEIL_DIV(ggml_nelements(dst), decisions->wg_size);+ compute_2d_workgroups(total_wg, ctx->global_ctx->capabilities.limits.maxComputeWorkgroupsPerDimension, wg_x, wg_y);+ return ggml_backend_webgpu_build(ctx, pipeline, params, entries, wg_x, wg_y);}static webgpu_encoded_op ggml_webgpu_soft_max(webgpu_context & ctx,@@ -3751,7 +3758,8 @@ static ggml_guid_t ggml_backend_webgpu_guid(void) {static void ggml_webgpu_init_memset_pipeline(webgpu_global_context & ctx) {// we use the maximum workgroup size for the memset pipeline- size_t max_threads = ctx->capabilities.limits.maxComputeInvocationsPerWorkgroup * ctx->capabilities.limits.maxComputeWorkgroupsPerDimension;+ size_t max_threads = ctx->capabilities.limits.maxComputeInvocationsPerWorkgroup *+ ctx->capabilities.limits.maxComputeWorkgroupsPerDimension;// Size the bytes_per_thread so that the largest buffer size can be handledctx->capabilities.memset_bytes_per_thread =CEIL_DIV(ctx->capabilities.limits.maxStorageBufferBindingSize, max_threads);diff --git a/ggml/src/ggml-webgpu/wgsl-shaders/binary.wgsl b/ggml/src/ggml-webgpu/wgsl-shaders/binary.wgslindex 605de7aa7b..f262c4a8f6 100644--- a/ggml/src/ggml-webgpu/wgsl-shaders/binary.wgsl+++ b/ggml/src/ggml-webgpu/wgsl-shaders/binary.wgsl@@ -130,10 +130,13 @@ fn update(dst_i: u32, src0_i: u32, src1_i: u32) {}@compute @workgroup_size(WG_SIZE)-fn main(@builtin(global_invocation_id) gid: vec3<u32>) {- if (gid.x < params.ne) {- let src0_i = params.offset_src0 + src0_index(gid.x);- let src1_i = params.offset_src1 + src1_index(gid.x);- update(params.offset_dst + gid.x, src0_i, src1_i);+fn main(@builtin(global_invocation_id) gid: vec3<u32>,+ @builtin(num_workgroups) num_wg: vec3<u32>) {+ let threads_per_group = u32(WG_SIZE);+ let i = gid.x + (num_wg.x * threads_per_group) * gid.y;+ if (i < params.ne) {+ let src0_i = params.offset_src0 + src0_index(i);+ let src1_i = params.offset_src1 + src1_index(i);+ update(params.offset_dst + i, src0_i, src1_i);}}diff --git a/ggml/src/ggml-webgpu/wgsl-shaders/scale.wgsl b/ggml/src/ggml-webgpu/wgsl-shaders/scale.wgslindex 3b70a876d7..6c76ed69e4 100644--- a/ggml/src/ggml-webgpu/wgsl-shaders/scale.wgsl+++ b/ggml/src/ggml-webgpu/wgsl-shaders/scale.wgsl@@ -43,12 +43,14 @@ struct Params {var<storage, read_write> src: array<f32>;@compute @workgroup_size(WG_SIZE)-fn main(@builtin(global_invocation_id) gid: vec3<u32>) {- if (gid.x >= params.ne) {+fn main(+ @builtin(global_invocation_id) gid: vec3<u32>,+ @builtin(num_workgroups) num_wg: vec3<u32>) {+ let threads_per_group = u32(WG_SIZE);+ var i = gid.x + (num_wg.x * threads_per_group) * gid.y;+ if (i >= params.ne) {return;}-- var i = gid.x;let i3 = i / (params.ne2 * params.ne1 * params.ne0);i = i % (params.ne2 * params.ne1 * params.ne0);let i2 = i / (params.ne1 * params.ne0);diff --git a/ggml/src/ggml-webgpu/wgsl-shaders/unary.wgsl b/ggml/src/ggml-webgpu/wgsl-shaders/unary.wgslindex 8e34e1c9ca..cb342c4726 100644--- a/ggml/src/ggml-webgpu/wgsl-shaders/unary.wgsl+++ b/ggml/src/ggml-webgpu/wgsl-shaders/unary.wgsl@@ -66,11 +66,14 @@ fn erf_approx(x: TYPE) -> TYPE {}@compute @workgroup_size(WG_SIZE)-fn main(@builtin(global_invocation_id) gid: vec3<u32>) {- if (gid.x >= params.ne) {+fn main(@builtin(global_invocation_id) gid: vec3<u32>,+ @builtin(num_workgroups) num_wg: vec3<u32>) {+ let threads_per_group = u32(WG_SIZE);+ let flat_i = gid.x + (num_wg.x * threads_per_group) * gid.y;+ if (flat_i >= params.ne) {return;}- var i = gid.x;+ var i = flat_i;let ne2 = params.ne2;#ifdef DIAGlet ne1 = params.ne0;@@ -205,6 +208,6 @@ fn main(@builtin(global_invocation_id) gid: vec3<u32>) {#ifdef INPLACEsrc[params.offset_src + src_idx] = res;#else- dst[params.offset_dst + gid.x] = res;+ dst[params.offset_dst + flat_i] = res;#endif}