LLM inference in C/C++
C++ ggml
[ggml-webgpu] Handle buffer overlap / buffer aliasing for concat operator (#24000)
Nikhil Jain1705d434f6b3
diff --git a/ggml/src/ggml-webgpu/ggml-webgpu-shader-lib.hpp b/ggml/src/ggml-webgpu/ggml-webgpu-shader-lib.hppindex a5e7de785b..c75a98a8dd 100644--- a/ggml/src/ggml-webgpu/ggml-webgpu-shader-lib.hpp+++ b/ggml/src/ggml-webgpu/ggml-webgpu-shader-lib.hpp@@ -448,15 +448,19 @@ struct ggml_webgpu_upscale_pipeline_key_hash {/** Concat **/struct ggml_webgpu_concat_pipeline_key {- int type;+ int type;+ bool src_overlap;- bool operator==(const ggml_webgpu_concat_pipeline_key & other) const { return type == other.type; }+ bool operator==(const ggml_webgpu_concat_pipeline_key & other) const {+ return type == other.type && src_overlap == other.src_overlap;+ }};struct ggml_webgpu_concat_pipeline_key_hash {size_t operator()(const ggml_webgpu_concat_pipeline_key & key) const {size_t seed = 0;ggml_webgpu_hash_combine(seed, key.type);+ ggml_webgpu_hash_combine(seed, key.src_overlap);return seed;}};@@ -2634,6 +2638,7 @@ class ggml_webgpu_shader_lib {webgpu_pipeline get_concat_pipeline(const ggml_webgpu_shader_lib_context & context) {ggml_webgpu_concat_pipeline_key key = {};key.type = context.dst->type;+ key.src_overlap = ggml_webgpu_tensor_overlap(context.src0, context.src1);auto it = concat_pipelines.find(key);if (it != concat_pipelines.end()) {@@ -2656,11 +2661,17 @@ class ggml_webgpu_shader_lib {GGML_ABORT("Unsupported type for concat shader");}+ if (key.src_overlap) {+ defines.push_back("SRC_OVERLAP");+ variant += "_src_overlap";+ }+defines.push_back(std::string("WG_SIZE=") + std::to_string(context.max_wg_size));auto processed = preprocessor.preprocess(wgsl_concat, defines);- auto decisions = std::make_shared<ggml_webgpu_generic_shader_decisions>();+ auto decisions = std::make_shared<ggml_webgpu_binary_shader_decisions>();decisions->wg_size = context.max_wg_size;+ decisions->src_overlap = key.src_overlap;webgpu_pipeline pipeline = ggml_webgpu_create_pipeline(device, processed, variant);pipeline.context = decisions;concat_pipelines[key] = pipeline;diff --git a/ggml/src/ggml-webgpu/ggml-webgpu.cpp b/ggml/src/ggml-webgpu/ggml-webgpu.cppindex 94a108dfa7..79d5138029 100644--- a/ggml/src/ggml-webgpu/ggml-webgpu.cpp+++ b/ggml/src/ggml-webgpu/ggml-webgpu.cpp@@ -2310,33 +2310,6 @@ static webgpu_encoded_op ggml_webgpu_concat(webgpu_context & ctx,uint32_t ne = (uint32_t) ggml_nelements(dst);uint32_t dim = (uint32_t) dst->op_params[0];- std::vector<uint32_t> params = {- ne,- (uint32_t) (ggml_webgpu_tensor_misalignment(ctx, src0) / ggml_type_size(src0->type)),- (uint32_t) (ggml_webgpu_tensor_misalignment(ctx, src1) / ggml_type_size(src1->type)),- (uint32_t) (ggml_webgpu_tensor_misalignment(ctx, dst) / ggml_type_size(dst->type)),- (uint32_t) (src0->nb[0] / ggml_type_size(src0->type)),- (uint32_t) (src0->nb[1] / ggml_type_size(src0->type)),- (uint32_t) (src0->nb[2] / ggml_type_size(src0->type)),- (uint32_t) (src0->nb[3] / ggml_type_size(src0->type)),- (uint32_t) (src1->nb[0] / ggml_type_size(src1->type)),- (uint32_t) (src1->nb[1] / ggml_type_size(src1->type)),- (uint32_t) (src1->nb[2] / ggml_type_size(src1->type)),- (uint32_t) (src1->nb[3] / ggml_type_size(src1->type)),- (uint32_t) dst->ne[0],- (uint32_t) dst->ne[1],- (uint32_t) dst->ne[2],- (uint32_t) dst->ne[3],- dim,- (uint32_t) src0->ne[dim]- };-- std::vector<wgpu::BindGroupEntry> entries = {- ggml_webgpu_make_tensor_bind_group_entry(ctx, 0, src0),- ggml_webgpu_make_tensor_bind_group_entry(ctx, 1, src1),- ggml_webgpu_make_tensor_bind_group_entry(ctx, 2, dst),- };-ggml_webgpu_shader_lib_context shader_lib_ctx = {};shader_lib_ctx.src0 = src0;shader_lib_ctx.src1 = src1;@@ -2344,8 +2317,52 @@ static webgpu_encoded_op ggml_webgpu_concat(webgpu_context & ctx,shader_lib_ctx.max_wg_size = ctx->global_ctx->capabilities.limits.maxComputeInvocationsPerWorkgroup;webgpu_pipeline pipeline = ctx->shader_lib->get_concat_pipeline(shader_lib_ctx);- auto * decisions = static_cast<ggml_webgpu_generic_shader_decisions *>(pipeline.context.get());- uint32_t wg_x = CEIL_DIV(ne, decisions->wg_size);+ auto * decisions = static_cast<ggml_webgpu_binary_shader_decisions *>(pipeline.context.get());++ uint32_t offset_src0 = (uint32_t) (ggml_webgpu_tensor_misalignment(ctx, src0) / ggml_type_size(src0->type));+ uint32_t offset_src1 = (uint32_t) (ggml_webgpu_tensor_misalignment(ctx, src1) / ggml_type_size(src1->type));+ size_t merged_offset = 0;+ size_t merged_size = 0;+ if (decisions->src_overlap) {+ const ggml_webgpu_merged_binding_range merged_range =+ ggml_webgpu_tensor_merged_binding_range(ctx, { src0, src1 });+ merged_offset = merged_range.offset;+ merged_size = merged_range.size;+ offset_src0 = ggml_webgpu_tensor_merged_element_offset(src0, merged_range);+ offset_src1 = ggml_webgpu_tensor_merged_element_offset(src1, merged_range);+ }++ std::vector<uint32_t> params = { ne,+ offset_src0,+ offset_src1,+ (uint32_t) (ggml_webgpu_tensor_misalignment(ctx, dst) / ggml_type_size(dst->type)),+ (uint32_t) (src0->nb[0] / ggml_type_size(src0->type)),+ (uint32_t) (src0->nb[1] / ggml_type_size(src0->type)),+ (uint32_t) (src0->nb[2] / ggml_type_size(src0->type)),+ (uint32_t) (src0->nb[3] / ggml_type_size(src0->type)),+ (uint32_t) (src1->nb[0] / ggml_type_size(src1->type)),+ (uint32_t) (src1->nb[1] / ggml_type_size(src1->type)),+ (uint32_t) (src1->nb[2] / ggml_type_size(src1->type)),+ (uint32_t) (src1->nb[3] / ggml_type_size(src1->type)),+ (uint32_t) dst->ne[0],+ (uint32_t) dst->ne[1],+ (uint32_t) dst->ne[2],+ (uint32_t) dst->ne[3],+ dim,+ (uint32_t) src0->ne[dim] };++ std::vector<wgpu::BindGroupEntry> entries = {};+ if (decisions->src_overlap) {+ entries.push_back(+ ggml_webgpu_make_bind_group_entry(0, ggml_webgpu_tensor_buf(src0), merged_offset, merged_size));+ entries.push_back(ggml_webgpu_make_tensor_bind_group_entry(ctx, 1, dst));+ } else {+ entries.push_back(ggml_webgpu_make_tensor_bind_group_entry(ctx, 0, src0));+ entries.push_back(ggml_webgpu_make_tensor_bind_group_entry(ctx, 1, src1));+ entries.push_back(ggml_webgpu_make_tensor_bind_group_entry(ctx, 2, dst));+ }++ uint32_t wg_x = CEIL_DIV(ne, decisions->wg_size);return ggml_backend_webgpu_build(ctx, pipeline, params, entries, wg_x);}diff --git a/ggml/src/ggml-webgpu/wgsl-shaders/concat.wgsl b/ggml/src/ggml-webgpu/wgsl-shaders/concat.wgslindex a22d245d2c..eb901bf054 100644--- a/ggml/src/ggml-webgpu/wgsl-shaders/concat.wgsl+++ b/ggml/src/ggml-webgpu/wgsl-shaders/concat.wgsl@@ -31,6 +31,16 @@ struct Params {#define DataType i32#endif+#ifdef SRC_OVERLAP+@group(0) @binding(0)+var<storage, read_write> merged_src: array<DataType>;++@group(0) @binding(1)+var<storage, read_write> dst: array<DataType>;++@group(0) @binding(2)+var<uniform> params: Params;+#else@group(0) @binding(0)var<storage, read_write> src0: array<DataType>;@@ -42,7 +52,7 @@ var<storage, read_write> dst: array<DataType>;@group(0) @binding(3)var<uniform> params: Params;-+#endif@compute @workgroup_size(WG_SIZE)fn main(@builtin(global_invocation_id) gid: vec3<u32>) {@@ -62,14 +72,22 @@ fn main(@builtin(global_invocation_id) gid: vec3<u32>) {ni[1] * params.stride_src0_1 +ni[2] * params.stride_src0_2 +ni[3] * params.stride_src0_3;+#ifdef SRC_OVERLAP+ dst[params.offset_dst + gid.x] = merged_src[params.offset_src0 + src_i];+#elsedst[params.offset_dst + gid.x] = src0[params.offset_src0 + src_i];+#endif} else {ni[params.dim] -= params.src0_nedim;let src_i = ni[0] * params.stride_src1_0 +ni[1] * params.stride_src1_1 +ni[2] * params.stride_src1_2 +ni[3] * params.stride_src1_3;+#ifdef SRC_OVERLAP+ dst[params.offset_dst + gid.x] = merged_src[params.offset_src1 + src_i];+#elsedst[params.offset_dst + gid.x] = src1[params.offset_src1 + src_i];+#endif}}}