LLM inference in C/C++
C++ ggml
vulkan: handle mul_mat_id duplicates in the prepass rather than looping (#29998)
Jeff Bolz966baae76b57
diff --git a/ggml/src/ggml-vulkan/ggml-vulkan-push-constants.h b/ggml/src/ggml-vulkan/ggml-vulkan-push-constants.hindex f1f66a6282..fdce443a14 100644--- a/ggml/src/ggml-vulkan/ggml-vulkan-push-constants.h+++ b/ggml/src/ggml-vulkan/ggml-vulkan-push-constants.h@@ -63,7 +63,7 @@ struct vk_mat_mat_id_push_constants {uint32_t batch_stride_a; uint32_t batch_stride_b; uint32_t batch_stride_d;uint32_t nei0; uint32_t nei1; uint32_t nbi1; uint32_t ne11;uint32_t n_experts;- uint32_t hoist_row_ids;+ uint32_t row_ids_offset;};struct vk_mat_vec_id_push_constants {@@ -218,9 +218,10 @@ struct vk_op_count_experts_push_constants {uint32_t nb01;uint32_t a_offset;uint32_t n_experts;- uint32_t hoist_row_ids;uint32_t ne00mp;uint32_t ne00L;+ uint32_t row_tile_size;+ uint32_t row_ids_offset;};struct vk_op_glu_push_constants {diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vulkan.cppindex 907ac82ce7..80f4d3acfc 100644--- a/ggml/src/ggml-vulkan/ggml-vulkan.cpp+++ b/ggml/src/ggml-vulkan/ggml-vulkan.cpp@@ -1473,9 +1473,8 @@ static bool ggml_vk_matmul_shmem_support(const vk_device& device, const std::vecconst uint32_t load_bufs = (warptile[1] + warptile[2]) * (warptile[3] + bank_conflict_offset) * type_size;const uint32_t mmid_row_ids = mul_mat_id ? (warptile[2] * 2 * sizeof(uint16_t)) : 0;const uint32_t coopmat_stage = device->coopmat_support ? warptile[7] * warptile[8] / warps * sizeof(float) : 0;- const uint32_t ballots_sh = mul_mat_id ? (warps * 4 * sizeof(uint32_t)) : 0;- const uint32_t total_size = load_bufs + mmid_row_ids + coopmat_stage + lut_size + ballots_sh;+ const uint32_t total_size = load_bufs + mmid_row_ids + coopmat_stage + lut_size;const bool supported = total_size <= device->properties.limits.maxComputeSharedMemorySize;VK_LOG_DEBUG("ggml_vk_matmul_shmem_support(warptile=(" << warptile[0] << "," << warptile[1] << "," << warptile[2] << "), "@@ -1540,10 +1539,7 @@ static bool ggml_vk_matmul_int_shmem_support(const vk_device& device, const std:const uint32_t buf_b_size = BN * BK_STEP * block_b_size;const uint32_t mmid_row_ids = mul_mat_id ? (BN * 2u * (uint32_t)sizeof(uint16_t)) : 0u;- const uint32_t warps = warptile[0] / warptile[10];- const uint32_t ballots_sh = mul_mat_id ? (warps * 4u * (uint32_t)sizeof(uint32_t)) : 0u;-- const uint32_t total_size = buf_a_size + buf_b_size + mmid_row_ids + ballots_sh + lut_size;+ const uint32_t total_size = buf_a_size + buf_b_size + mmid_row_ids + lut_size;const bool supported = total_size <= device->properties.limits.maxComputeSharedMemorySize;VK_LOG_DEBUG("ggml_vk_matmul_int_shmem_support(warptile=(" << warptile[0] << "," << warptile[1] << "," << warptile[2] << "), "@@ -1573,10 +1569,8 @@ static bool ggml_vk_matmul_cm1_int_shmem_support(const vk_device& device, constreturn false;}- const uint32_t BLOCK_SIZE = warptile[0];const uint32_t BM = warptile[1];const uint32_t BN = warptile[2];- const uint32_t WARP = warptile[10];const uint32_t BK = 32;const uint32_t BK_STEP = mul_mat_id ? 2u : 4u;@@ -1600,8 +1594,6 @@ static bool ggml_vk_matmul_cm1_int_shmem_support(const vk_device& device, const}if (mul_mat_id) {total += BN * 2u * (uint32_t)sizeof(uint16_t); // row_ids[BN] (u16vec2)- const uint32_t num_warps = BLOCK_SIZE / std::max(WARP, 1u);- total += num_warps * 4u * (uint32_t)sizeof(uint32_t); // ballots_sh[NUM_WARPS] (uvec4)}const bool supported = total <= device->properties.limits.maxComputeSharedMemorySize;@@ -3595,7 +3587,7 @@ void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) {ggml_vk_create_pipeline(device, device->pipeline_count_equal_i32, "count_equal_i32", count_equal_i32_len, count_equal_i32_data, "main", 3, sizeof(vk_op_push_constants), {512, 1, 1}, { device->subgroup_size }, 1);- if (device->subgroup_arithmetic && device->subgroup_require_full_support) {+ if (device->subgroup_arithmetic && device->subgroup_vote && device->subgroup_require_full_support) {ggml_vk_create_pipeline(device, device->pipeline_count_experts, "count_experts", count_experts_subgroup_len, count_experts_subgroup_data, "main", 2, sizeof(vk_op_count_experts_push_constants), {1, 1, 1}, {}, 1, true, true);} else {ggml_vk_create_pipeline(device, device->pipeline_count_experts, "count_experts", count_experts_len, count_experts_data, "main", 2, sizeof(vk_op_count_experts_push_constants), {1, 1, 1}, {}, 1, true);@@ -6014,20 +6006,27 @@ static uint32_t ggml_vk_guess_matmul_pipeline_align_map(ggml_backend_vk_contextreturn configs[idx].align;}+static uint64_t ggml_vk_mul_mat_id_max_tiles(uint64_t n_experts, uint64_t n_rows, uint32_t bn) {+ // Each nonempty expert can add at most one partial tile to the combined grid.+ return std::min(n_rows, CEIL_DIV(n_rows, bn) + std::min(n_experts, n_rows) - 1);+}+static void ggml_vk_matmul_id(ggml_backend_vk_context * ctx, vk_context& subctx, vk_pipeline& pipeline,vk_subbuffer&& a, vk_subbuffer&& b, vk_subbuffer&& d, vk_subbuffer&& ids, const vk_subbuffer & expert_count_buf,uint32_t m, uint32_t n, uint32_t k, uint32_t stride_a, uint32_t stride_b, uint32_t stride_d,uint32_t batch_stride_a, uint32_t batch_stride_b, uint32_t batch_stride_d,uint32_t n_as, uint32_t nei0, uint32_t nei1, uint32_t nbi1, uint32_t ne11,- bool hoist_row_ids) {+ uint32_t row_ids_offset, uint32_t max_tiles) {VK_LOG_DEBUG("ggml_vk_matmul_id(a: (" << a.buffer->buffer << ", " << a.offset << ", " << a.size << "), b: (" << b.buffer->buffer << ", " << b.offset << ", " << b.size << "), d: (" << d.buffer->buffer << ", " << d.offset << ", " << d.size << "), ids: (" << ids.buffer->buffer << ", " << ids.offset << ", " << ids.size << "), expert_count: (" << expert_count_buf.buffer->buffer << ", " << expert_count_buf.offset << ", " << expert_count_buf.size << "), " <<"m: " << m << ", n: " << n << ", k: " << k << ", stride_a: " << stride_a << ", stride_b: " << stride_b << ", stride_d: " << stride_d << ", " <<"batch_stride_a: " << batch_stride_a << ", batch_stride_b: " << batch_stride_b << ", batch_stride_d: " << batch_stride_d << ", " <<"n_as: " << n_as << ", nei0: " << nei0 << ", nei1: " << nei1 << ", nbi1: " << nbi1 << ", ne11: " << ne11 << ")");const vk_mat_mat_id_push_constants pc = { m, n, k, stride_a, stride_b, stride_d, batch_stride_a, batch_stride_b, batch_stride_d,- nei0, nei1, nbi1, ne11, n_as, uint32_t(hoist_row_ids) };- ggml_vk_dispatch_pipeline(ctx, subctx, pipeline, { a, b, d, ids, expert_count_buf }, pc, { m, nei1, n_as });+ nei0, nei1, nbi1, ne11, n_as, row_ids_offset };+ const uint32_t tiles_y = std::min(max_tiles, ctx->device->properties.limits.maxComputeWorkGroupCount[1]);+ const std::array<uint32_t, 3> elements = { m, tiles_y * pipeline->wg_denoms[1], uint32_t(CEIL_DIV(max_tiles, tiles_y)) };+ ggml_vk_dispatch_pipeline(ctx, subctx, pipeline, { a, b, d, ids, expert_count_buf }, pc, elements);}bool ggml_vk_dim01_contiguous(const ggml_tensor * tensor) {@@ -7336,7 +7335,6 @@ static void ggml_vk_mul_mat_id_q_f16(ggml_backend_vk_context * ctx, vk_context&const uint32_t nbi0 = ids->nb[0];const uint32_t nbi1 = ids->nb[1];- const uint32_t nbi2 = ids->nb[2];const uint64_t ne20 = dst->ne[0];const uint64_t ne21 = dst->ne[1];@@ -7344,38 +7342,24 @@ static void ggml_vk_mul_mat_id_q_f16(ggml_backend_vk_context * ctx, vk_context&// const uint64_t ne23 = dst->ne[3];const uint64_t n_as = ne02;- // n_as counts, n_as offsets, one total, then one packed row id per (expert, token).- // Hoisting requires 16-bit indices for the packing and a table that fits one binding.- const uint64_t hoisted_row_id_words = 2 * n_as + 1 + nei0 * nei1;- // 1024 matches MAX_EXPERTS in count_experts.comp and LLAMA_MAX_EXPERTS. It costs- // 3 * 1024 * 4 = 12 KiB of shared memory, within the 16 KiB Vulkan guarantees.- const bool hoist_row_ids = n_as <= 1024 && nei0 <= 0xffff && nei1 <= 0xffff &&- hoisted_row_id_words * sizeof(uint32_t) <=- ctx->device->properties.limits.maxStorageBufferRange;ggml_backend_vk_buffer_context * dst_buf_ctx = (ggml_backend_vk_buffer_context *)dst->buffer->context;ggml_backend_vk_buffer_context * src0_buf_ctx = (ggml_backend_vk_buffer_context *)src0->buffer->context;ggml_backend_vk_buffer_context * src1_buf_ctx = (ggml_backend_vk_buffer_context *)src1->buffer->context;- ggml_backend_vk_buffer_context * ids_buf_ctx = (ggml_backend_vk_buffer_context *)ids->buffer->context;vk_buffer d_Qx = nullptr;size_t qx_buf_offset = 0;vk_buffer d_Qy = nullptr;size_t qy_buf_offset = 0;- vk_buffer d_ids = nullptr;- size_t ids_buf_offset = 0;bool src0_uma = false;bool src1_uma = false;- bool ids_uma = false;if (ctx->device->uma) {ggml_vk_host_get(ctx->device, src0->data, d_Qx, qx_buf_offset);ggml_vk_host_get(ctx->device, src1->data, d_Qy, qy_buf_offset);- ggml_vk_host_get(ctx->device, ids->data, d_ids, ids_buf_offset);src0_uma = d_Qx != nullptr;src1_uma = d_Qy != nullptr;- ids_uma = d_ids != nullptr;}// Reformat and convert to fp16 if non-contiguous, or for coopmat2 for better perf@@ -7445,6 +7429,11 @@ static void ggml_vk_mul_mat_id_q_f16(ggml_backend_vk_context * ctx, vk_context&vk_pipeline pipeline = ggml_vk_guess_matmul_pipeline_map(ctx, *mmp_map, ne01, nei1, aligned, true);+ const uint32_t bn = pipeline->wg_denoms[1];+ const uint64_t max_tiles = ggml_vk_mul_mat_id_max_tiles(n_as, nei0 * nei1, bn);+ const uint64_t row_ids_offset = 1 + 2 * max_tiles;+ const uint64_t row_map_words = row_ids_offset + max_tiles * bn;+if (ggml_nbytes(src0) > ctx->device->properties.limits.maxStorageBufferRange) {pipeline = ggml_vk_get_64b_indexing_pipeline(ctx, pipeline);}@@ -7456,7 +7445,6 @@ static void ggml_vk_mul_mat_id_q_f16(ggml_backend_vk_context * ctx, vk_context&const uint64_t qy_sz = ggml_type_size(src1->type) * ggml_nelements(src1) / ggml_blck_size(src1->type);const uint64_t x_sz = !qx_needs_dequant ? qx_sz : sizeof(ggml_fp16_t) * x_ne;const uint64_t y_sz = quantize_y ? (ggml_vk_align_size(y_ne, 128) * ggml_type_size(GGML_TYPE_Q8_1) / ggml_blck_size(GGML_TYPE_Q8_1)) : (y_f32_kernel ? sizeof(float) * y_ne : sizeof(ggml_fp16_t) * y_ne);- const uint64_t ids_sz = nbi2;const uint64_t d_sz = sizeof(float) * d_ne;vk_pipeline to_fp16_vk_0 = nullptr;@@ -7498,8 +7486,7 @@ static void ggml_vk_mul_mat_id_q_f16(ggml_backend_vk_context * ctx, vk_context&}vk_pipeline count_experts = ctx->device->pipeline_count_experts;- const size_t expert_data_size = sizeof(uint32_t) *- (hoist_row_ids ? hoisted_row_id_words : n_as);+ const size_t expert_data_size = sizeof(uint32_t) * row_map_words;{if (@@ -7551,11 +7538,6 @@ static void ggml_vk_mul_mat_id_q_f16(ggml_backend_vk_context * ctx, vk_context&qy_buf_offset = vk_tensor_offset(src1) + src1->view_offs;GGML_ASSERT(d_Qy != nullptr);}- if (!ids_uma) {- d_ids = ids_buf_ctx->dev_buffer;- ids_buf_offset = vk_tensor_offset(ids) + ids->view_offs;- GGML_ASSERT(d_ids != nullptr);- }if (qx_needs_dequant) {d_X = ctx->prealloc_x;GGML_ASSERT(d_X->size >= x_sz);@@ -7581,6 +7563,8 @@ static void ggml_vk_mul_mat_id_q_f16(ggml_backend_vk_context * ctx, vk_context&ggml_vk_sync_buffers(ctx, subctx);}}+ vk_subbuffer d_ids = ggml_vk_tensor_subbuffer(ctx, ids, true);+// Count how many times each expert is usedvk_subbuffer expert_count_buf = { ctx->prealloc_split_k, 0, expert_data_size };if (ctx->prealloc_split_k_need_sync) {@@ -7593,12 +7577,11 @@ static void ggml_vk_mul_mat_id_q_f16(ggml_backend_vk_context * ctx, vk_context&(uint32_t)(nbi1 / ggml_type_size(ids->type)),(uint32_t)(get_misalign_bytes(ctx, ids) / ggml_type_size(ids->type)),(uint32_t)n_as,- uint32_t(hoist_row_ids),- 0, 0 };+ 0, 0, bn, (uint32_t)row_ids_offset };init_pushconst_fastdiv(pc);ggml_vk_dispatch_pipeline(ctx, subctx, count_experts,- { vk_subbuffer{ d_ids, ids_buf_offset, ids_sz }, expert_count_buf }, pc,- { hoist_row_ids ? 1u : (uint32_t)n_as, 1, 1});+ { d_ids, expert_count_buf }, pc,+ { 1, 1, 1 });}if (x_non_contig) {@@ -7671,10 +7654,10 @@ static void ggml_vk_mul_mat_id_q_f16(ggml_backend_vk_context * ctx, vk_context&ggml_vk_matmul_id(ctx, subctx, pipeline,{ d_X, x_buf_offset, x_range }, { d_Y, y_buf_offset, y_range },- { d_D, d_buf_offset, d_sz }, { d_ids, ids_buf_offset, ids_sz }, expert_count_buf,+ { d_D, d_buf_offset, d_sz }, std::move(d_ids), expert_count_buf,ne01, ne21, ne10, ne10, stride_b_y, ne01,stride_batch_x, stride_batch_y, ne20*ne21,- n_as, nei0, nei1, nbi1 / ggml_type_size(ids->type), ne11, hoist_row_ids+ n_as, nei0, nei1, nbi1 / ggml_type_size(ids->type), ne11, row_ids_offset, max_tiles); // NOLINTif (x_non_contig || qx_needs_dequant) {@@ -7923,11 +7906,14 @@ static void ggml_vk_mul_mat_vec_id_q_f16(ggml_backend_vk_context * ctx, vk_conte}}+static bool ggml_vk_use_mul_mat_vec_id(const ggml_tensor * dst) {+ const ggml_tensor * src0 = dst->src[0];+ const ggml_tensor * ids = dst->src[2];+ return ids->ne[1] <= 8 && (src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16 || ggml_is_quantized(src0->type));+}+bool ggml_vk_use_mul_mat_vec_id(const struct ggml_cgraph * cgraph, int node_idx) {- ggml_tensor * dst = cgraph->nodes[node_idx];- ggml_tensor * src0 = dst->src[0];- ggml_tensor * src2 = dst->src[2];- return (src2->ne[1] <= 8) && (src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16 || ggml_is_quantized(src0->type));+ return ggml_vk_use_mul_mat_vec_id(cgraph->nodes[node_idx]);}void ggml_vk_mul_mat_id(ggml_backend_vk_context * ctx, vk_context& subctx, const struct ggml_cgraph * cgraph, int node_idx) {@@ -15431,6 +15417,13 @@ static bool ggml_backend_vk_device_supports_op(ggml_backend_dev_t dev, const ggm{ggml_type src0_type = op->src[0]->type;if (op->op == GGML_OP_MUL_MAT_ID) {+ if (!ggml_vk_use_mul_mat_vec_id(op)) {+ const ggml_tensor * ids = op->src[2];+ // Shaders store row IDs as uint16_t slot and token indices.+ if (ids->ne[0] > int64_t(UINT16_MAX) + 1 || ids->ne[1] > int64_t(UINT16_MAX) + 1) {+ return false;+ }+ }if (!device->mul_mat_id_s[src0_type] && !device->mul_mat_id_m[src0_type] && !device->mul_mat_id_l[src0_type]) {// If there's not enough shared memory for row_ids and the result tile, fallback to CPUreturn false;diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/count_experts.comp b/ggml/src/ggml-vulkan/vulkan-shaders/count_experts.compindex 06a50181c4..e7e295115d 100644--- a/ggml/src/ggml-vulkan/vulkan-shaders/count_experts.comp+++ b/ggml/src/ggml-vulkan/vulkan-shaders/count_experts.comp@@ -5,6 +5,7 @@#ifdef USE_SUBGROUPS#extension GL_KHR_shader_subgroup_basic : enable#extension GL_KHR_shader_subgroup_arithmetic : enable+#extension GL_KHR_shader_subgroup_vote : enable#endif#include "types.glsl"@@ -18,40 +19,47 @@ layout (push_constant) uniform parameteruint32_t nb01;uint32_t a_offset;uint32_t n_experts;- uint32_t hoist_row_ids;uint32_t ne00mp;uint32_t ne00L;+ uint32_t row_tile_size;+ uint32_t row_ids_offset;} p;#define BLOCK_SIZE 256+#define EXPERTS_PER_CHUNK 1024layout(local_size_x = BLOCK_SIZE, local_size_y = 1, local_size_z = 1) in;layout (binding = 0) readonly buffer A {uint data_a[];};layout (binding = 1) writeonly buffer D {uint data_d[];};-// Upper bound on n_experts for the hoisted row-id path. Must match the limit in-// ggml_vk_mul_mat_id_q_f16 (hoist_row_ids). The non-hoisted reduction below only-// needs BLOCK_SIZE entries.-#define MAX_EXPERTS 1024--shared uint vals[MAX_EXPERTS];-shared uint offsets[MAX_EXPERTS];-shared uint cursors[MAX_EXPERTS];+shared uint vals[EXPERTS_PER_CHUNK];+shared uint offsets[EXPERTS_PER_CHUNK];+shared uint cursors[EXPERTS_PER_CHUNK];++// data_d[0] is the tile count. Each tile has (expert, valid rows).+// Packed (token << 16) | slot row ids start at p.row_ids_offset, with each expert on a tile boundary.+void write_tiles(uint expert, uint count, uint tile_begin, uint n_tiles) {+ for (uint t = 0; t < n_tiles; ++t) {+ const uint descriptor = 1 + 2 * (tile_begin + t);+ data_d[descriptor] = expert;+ data_d[descriptor + 1] = min(p.row_tile_size, count - t * p.row_tile_size);+ }+}-// data_d layout when p.hoist_row_ids is set:-// [0, n_experts) per-expert row count-// [n_experts, 2*n_experts) per-expert start offset into the row id region-// [2*n_experts] total row count-// [2*n_experts + 1, ) row ids grouped by expert, packed as (i01 << 16) | (i00 & 0xffff)-// Otherwise only data_d[expert_id] is written, holding that expert's row count.void main() {- const uint expert_id = gl_WorkGroupID.x;const uint num_elements = p.ne00 * p.ne01;const uint tid = gl_LocalInvocationID.x;+ const uint tile_shift = findLSB(p.row_tile_size);+ uint total_tiles = 0;+#ifdef USE_SUBGROUPS+ // Use the subgroup that contains invocation 0.+ const bool prefix_subgroup = subgroupAny(tid == 0);+#endif- if (p.hoist_row_ids != 0) {- for (uint e = tid; e < p.n_experts; e += BLOCK_SIZE) {+ for (uint expert_base = 0; expert_base < p.n_experts;) {+ const uint n_experts = min(EXPERTS_PER_CHUNK, p.n_experts - expert_base);+ for (uint e = tid; e < n_experts; e += BLOCK_SIZE) {vals[e] = 0;}barrier();@@ -59,46 +67,43 @@ void main() {for (uint idx = tid; idx < num_elements; idx += BLOCK_SIZE) {const uint i01 = fastdiv(idx, p.ne00mp, p.ne00L);const uint i00 = idx - i01 * p.ne00;- const uint expert = data_a[p.a_offset + i01 * p.nb01 + i00 * p.nb00];- if (expert < p.n_experts) {+ const uint expert = data_a[p.a_offset + i01 * p.nb01 + i00 * p.nb00] - expert_base;+ if (expert < n_experts) {atomicAdd(vals[expert], 1);}}barrier();#ifdef USE_SUBGROUPS- if (gl_SubgroupID == 0) {- // pad the trip count so the subgroup ops stay in uniform control flow- const uint n_experts_padded = (p.n_experts + gl_SubgroupSize - 1) & ~(gl_SubgroupSize - 1);- uint base = 0;- for (uint expert = gl_SubgroupInvocationID; expert < n_experts_padded; expert += gl_SubgroupSize) {- const bool in_range = expert < p.n_experts;- const uint count = in_range ? vals[expert] : 0;- const uint offset = base + subgroupExclusiveAdd(count);+ if (prefix_subgroup) {+ const uint n_experts_padded = (n_experts + gl_SubgroupSize - 1) & ~(gl_SubgroupSize - 1);+ uint base = subgroupAdd(tid == 0 ? total_tiles : 0u);+ for (uint e = gl_SubgroupInvocationID; e < n_experts_padded; e += gl_SubgroupSize) {+ const bool in_range = e < n_experts;+ const uint count = in_range ? vals[e] : 0;+ const uint n_tiles = (count + p.row_tile_size - 1) >> tile_shift;+ const uint tile_begin = base + subgroupExclusiveAdd(n_tiles);if (in_range) {- data_d[expert] = count;- data_d[p.n_experts + expert] = offset;- offsets[expert] = offset;- cursors[expert] = 0;+ offsets[e] = tile_begin * p.row_tile_size;+ cursors[e] = 0;+ write_tiles(expert_base + e, count, tile_begin, n_tiles);}- base += subgroupAdd(count);+ base += subgroupAdd(n_tiles);}- if (subgroupElect()) {- data_d[2 * p.n_experts] = base;+ if (tid == 0) {+ total_tiles = base;}}#elseif (tid == 0) {- uint offset = 0;- for (uint expert = 0; expert < p.n_experts; ++expert) {- const uint count = vals[expert];- data_d[expert] = count;- data_d[p.n_experts + expert] = offset;- offsets[expert] = offset;- cursors[expert] = 0;- offset += count;+ for (uint e = 0; e < n_experts; ++e) {+ const uint count = vals[e];+ const uint n_tiles = (count + p.row_tile_size - 1) >> tile_shift;+ offsets[e] = total_tiles * p.row_tile_size;+ cursors[e] = 0;+ write_tiles(expert_base + e, count, total_tiles, n_tiles);+ total_tiles += n_tiles;}- data_d[2 * p.n_experts] = offset;}#endifbarrier();@@ -106,35 +111,16 @@ void main() {for (uint idx = tid; idx < num_elements; idx += BLOCK_SIZE) {const uint i01 = fastdiv(idx, p.ne00mp, p.ne00L);const uint i00 = idx - i01 * p.ne00;- const uint expert = data_a[p.a_offset + i01 * p.nb01 + i00 * p.nb00];- if (expert < p.n_experts) {+ const uint expert = data_a[p.a_offset + i01 * p.nb01 + i00 * p.nb00] - expert_base;+ if (expert < n_experts) {const uint row = atomicAdd(cursors[expert], 1);- const uint packed_row_id = (i01 << 16) | (i00 & 0xffffu);- data_d[2 * p.n_experts + 1 + offsets[expert] + row] = packed_row_id;+ data_d[p.row_ids_offset + offsets[expert] + row] = (i01 << 16) | i00;}}- return;+ // The next chunk clears vals. Scatter uses only offsets and cursors.+ expert_base += n_experts;}-- uint count = 0;- for (uint idx = tid; idx < num_elements; idx += BLOCK_SIZE) {- const uint i01 = fastdiv(idx, p.ne00mp, p.ne00L);- const uint i00 = idx - i01 * p.ne00;- const uint a = data_a[p.a_offset + i01 * p.nb01 + i00 * p.nb00];-- count += uint(a == expert_id);- }-- vals[tid] = count;- barrier();- [[unroll]] for (uint s = BLOCK_SIZE / 2; s > 0; s >>= 1) {- if (tid < s) {- vals[tid] += vals[tid + s];- }- barrier();- }-if (tid == 0) {- data_d[expert_id] = vals[0];+ data_d[0] = total_tiles;}}diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm.comp b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm.compindex 11098ee7b3..9f31ef126b 100644--- a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm.comp+++ b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm.comp@@ -124,7 +124,7 @@ layout (binding = 2) writeonly buffer D {D_TYPE data_d[];};#ifdef MUL_MAT_IDlayout (binding = 3) readonly buffer IDS {int data_ids[];};-layout (binding = 4) readonly buffer Counts {int data_expert_count[];};+layout (binding = 4) readonly buffer RowMap {uint data_row_map[];};#endiflayout (push_constant) uniform parameter@@ -146,7 +146,7 @@ layout (push_constant) uniform parameteruint nbi1;uint ne11;uint n_experts;- uint hoist_row_ids;+ uint row_ids_offset;#elseuint base_work_group_z;uint num_batches;@@ -208,13 +208,17 @@ shared ACC_TYPE coopmat_stage[TM * TN * NUM_WARPS];#endifvoid main() {- const uint ic = gl_WorkGroupID.y;-#ifdef MUL_MAT_ID- const uint expert_idx = gl_WorkGroupID.z;- if (ic * BN >= data_expert_count[expert_idx]) {+ const uint tile_idx = gl_WorkGroupID.y + gl_WorkGroupID.z * gl_NumWorkGroups.y;+ if (tile_idx >= data_row_map[0]) {return;}+ const uint expert_idx = data_row_map[1 + 2 * tile_idx];+ const uint row_begin = tile_idx * BN;+ _ne1 = data_row_map[2 + 2 * tile_idx];+ const uint ic = 0;+#else+ const uint ic = gl_WorkGroupID.y;#endif#if defined(NEEDS_INIT_IQ_SHMEM) || defined(MULMAT_QUANT)init_iq_shmem(gl_WorkGroupSize);@@ -285,37 +289,9 @@ void main() {const uint loadstride_b = gl_WorkGroupSize.x * LOAD_VEC_B_EFF * LOAD_VEC_BATCH_B / BK;#ifdef MUL_MAT_ID- if (p.hoist_row_ids != 0) {- load_row_ids_hoisted(expert_idx, ic);- } else {-#ifdef MUL_MAT_ID_USE_SUBGROUPS- if (bitCount(p.nei0) == 1) {- load_row_ids(expert_idx, true, ic);- } else {- load_row_ids(expert_idx, false, ic);- }-#else- _ne1 = 0;- for (uint ii1 = 0; ii1 < p.nei1 && _ne1 < (ic + 1) * BN; ii1++) {- for (uint ii0 = 0; ii0 < p.nei0 && _ne1 < (ic + 1) * BN; ii0++) {- if (data_ids[ii1*p.nbi1 + ii0] == expert_idx) {- if (_ne1 >= ic * BN) {- row_ids[_ne1 - ic * BN] = u16vec2(ii0, ii1);- }- _ne1++;- }- }- }-- barrier();-#endif- }-- // Workgroup has no work- if (ic * BN >= _ne1) return;-- uint required_work_items = (_ne1 - ic * BN) * BK / LOAD_VEC_B_EFF / LOAD_VEC_BATCH_B;- uint required_warp_c = (_ne1 - ic * BN + WN - 1) / WN;+ load_row_ids(row_begin);+ uint required_work_items = _ne1 * BK / LOAD_VEC_B_EFF / LOAD_VEC_BATCH_B;+ uint required_warp_c = (_ne1 + WN - 1) / WN;#endif#ifdef MUL_MAT_IDdiff --git a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_cm2.comp b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_cm2.compindex 9b59e8bca7..e3f1dd5abb 100644--- a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_cm2.comp+++ b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_cm2.comp@@ -82,7 +82,7 @@ layout (push_constant) uniform parameteruint nbi1;uint ne11;uint n_experts;- uint hoist_row_ids;+ uint row_ids_offset;#elseuint base_work_group_z;uint num_batches;@@ -182,7 +182,7 @@ f16vec4 mmDecodeA_v(const in decodeBufA bl_in, const in uint blockCoords[2], con#ifdef MUL_MAT_IDlayout (binding = 3) readonly buffer IDS {int data_ids[];};-layout (binding = 4) readonly buffer Counts {int data_expert_count[];};+layout (binding = 4) readonly buffer RowMap {uint data_row_map[];};shared u16vec4 row_ids[BN];@@ -191,7 +191,6 @@ layout(buffer_reference, std430, buffer_reference_align = 2) buffer decodeBufB {};uint _ne1;-shared uvec4 ballots_sh[BLOCK_SIZE / subgroup_size];B_TYPE decodeFuncB(const in decodeBufB bl, const in uint blockCoords[2], const in uint coordInBlock[2]){@@ -238,82 +237,9 @@ D_TYPE perElemOpD(const in uint32_t r, const in uint32_t c, const in D_TYPE elemreturn elem;}-void load_row_ids(uint expert_idx, bool nei0_is_pow2, uint ic) {- _ne1 = 0;- uint num_elements = p.nei1 * p.nei0;- uint nei0shift = findLSB(p.nei0);-- uint ids[16];- uint iter = 0;-- uint expert_count = data_expert_count[expert_idx];-- for (uint j = 0; j < num_elements; j += BLOCK_SIZE) {- // prefetch up to 16 elements- if (iter == 0) {- [[unroll]] for (uint k = 0; k < 16; ++k) {- uint i = j + gl_LocalInvocationIndex + k*BLOCK_SIZE;- bool in_range = i < num_elements;- uint ii1;- if (nei0_is_pow2) {- ii1 = i >> nei0shift;- } else {- ii1 = i / p.nei0;- }- uint ii0 = i - ii1 * p.nei0;- ids[k] = in_range ? data_ids[ii1*p.nbi1 + ii0] : 0;- }- }- uint i = j + gl_LocalInvocationIndex;- bool in_range = i < num_elements;- uint ii1;- if (nei0_is_pow2) {- ii1 = i >> nei0shift;- } else {- ii1 = i / p.nei0;- }- uint ii0 = i - ii1 * p.nei0;- uint id = ids[iter++];- uvec4 ballot = subgroupBallot(in_range && id == expert_idx);-- if (gl_SubgroupInvocationID == 0) {- ballots_sh[gl_SubgroupID] = ballot;- }- barrier();-- uint subgroup_base = 0;- uint total = 0;- for (uint k = 0; k < gl_NumSubgroups; ++k) {- if (k == gl_SubgroupID) {- subgroup_base = total;- }- total += subgroupBallotBitCount(ballots_sh[k]);- }- barrier();-- uint idx = subgroup_base + subgroupBallotExclusiveBitCount(ballot);- if (in_range && id == expert_idx && _ne1 + idx >= ic * BN && _ne1 + idx < (ic + 1) * BN) {- row_ids[_ne1 + idx - ic * BN] = u16vec4(fastmod(ii0, p.ne11), ii1, ii0, 0);- }- _ne1 += total;- iter &= 15;- if (_ne1 >= (ic + 1) * BN || _ne1 == expert_count) {- break;- }- }- barrier();-}--void load_row_ids_hoisted(uint expert_idx, uint ic) {- _ne1 = uint(data_expert_count[expert_idx]);-- const uint tile_begin = ic * BN;- const uint tile_count = tile_begin < _ne1 ? min(BN, _ne1 - tile_begin) : 0;- const uint expert_offset = uint(data_expert_count[p.n_experts + expert_idx]);- const uint row_ids_offset = 2 * p.n_experts + 1 + expert_offset + tile_begin;-- for (uint i = gl_LocalInvocationIndex; i < tile_count; i += BLOCK_SIZE) {- const uint packed_row_id = uint(data_expert_count[row_ids_offset + i]);+void load_row_ids(uint row_begin) {+ for (uint i = gl_LocalInvocationIndex; i < _ne1; i += BLOCK_SIZE) {+ const uint packed_row_id = data_row_map[p.row_ids_offset + row_begin + i];const uint ii0 = packed_row_id & 0xffffu;const uint ii1 = packed_row_id >> 16;row_ids[i] = u16vec4(fastmod(ii0, p.ne11), ii1, ii0, 0);@@ -328,13 +254,20 @@ void load_row_ids_hoisted(uint expert_idx, uint ic) {void main() {const uint tid = gl_LocalInvocationIndex;- const uint ic = gl_WorkGroupID.y;-#ifdef MUL_MAT_ID- const uint expert_idx = gl_WorkGroupID.z;- if (ic * BN >= data_expert_count[expert_idx]) {+ const uint tile_idx = gl_WorkGroupID.y + gl_WorkGroupID.z * gl_NumWorkGroups.y;+ if (tile_idx >= data_row_map[0]) {return;}+ const uint expert_idx = data_row_map[1 + 2 * tile_idx];+ const uint row_begin = tile_idx * BN;+ _ne1 = data_row_map[2 + 2 * tile_idx];+ const uint ic = 0;+#else+ const uint ic = gl_WorkGroupID.y;+#endif++#ifdef MUL_MAT_ID// initialize to row 0 so we don't need to bounds checkif (tid < BN) {row_ids[tid] = u16vec4(0);@@ -365,16 +298,7 @@ void main() {const uint ik = gl_WorkGroupID.x / blocks_m;#ifdef MUL_MAT_ID- if (p.hoist_row_ids != 0) {- load_row_ids_hoisted(expert_idx, ic);- } else if (bitCount(p.nei0) == 1) {- load_row_ids(expert_idx, true, ic);- } else {- load_row_ids(expert_idx, false, ic);- }-- // Workgroup has no work- if (ic * BN >= _ne1) return;+ load_row_ids(row_begin);#endif#ifdef MUL_MAT_IDdiff --git a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_id_funcs.glsl b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_id_funcs.glslindex 54ad60b2ef..99a644e5ab 100644--- a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_id_funcs.glsl+++ b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_id_funcs.glsl@@ -2,86 +2,9 @@shared u16vec2 row_ids[BN];uint _ne1;-#ifdef MUL_MAT_ID_USE_SUBGROUPS-shared uvec4 ballots_sh[NUM_WARPS];--void load_row_ids(uint expert_idx, bool nei0_is_pow2, uint ic) {- _ne1 = 0;- uint num_elements = p.nei1 * p.nei0;- uint nei0shift = findLSB(p.nei0);-- uint ids[16];- uint iter = 0;-- uint expert_count = data_expert_count[expert_idx];-- for (uint j = 0; j < num_elements; j += BLOCK_SIZE) {- // prefetch up to 16 elements- if (iter == 0) {- [[unroll]] for (uint k = 0; k < 16; ++k) {- uint i = j + gl_LocalInvocationIndex + k*BLOCK_SIZE;- bool in_range = i < num_elements;- uint ii1;- if (nei0_is_pow2) {- ii1 = i >> nei0shift;- } else {- ii1 = i / p.nei0;- }- uint ii0 = i - ii1 * p.nei0;- ids[k] = in_range ? data_ids[ii1*p.nbi1 + ii0] : 0;- }- }- uint i = j + gl_LocalInvocationIndex;- bool in_range = i < num_elements;- uint ii1;- if (nei0_is_pow2) {- ii1 = i >> nei0shift;- } else {- ii1 = i / p.nei0;- }- uint ii0 = i - ii1 * p.nei0;- uint id = ids[iter++];- uvec4 ballot = subgroupBallot(in_range && id == expert_idx);-- if (gl_SubgroupInvocationID == 0) {- ballots_sh[gl_SubgroupID] = ballot;- }- barrier();-- uint subgroup_base = 0;- uint total = 0;- for (uint k = 0; k < gl_NumSubgroups; ++k) {- if (k == gl_SubgroupID) {- subgroup_base = total;- }- total += subgroupBallotBitCount(ballots_sh[k]);- }- barrier();-- uint idx = subgroup_base + subgroupBallotExclusiveBitCount(ballot);- if (in_range && id == expert_idx && _ne1 + idx >= ic * BN && _ne1 + idx < (ic + 1) * BN) {- row_ids[_ne1 + idx - ic * BN] = u16vec2(ii0, ii1);- }- _ne1 += total;- iter &= 15;- if (_ne1 >= (ic + 1) * BN || _ne1 == expert_count) {- break;- }- }- barrier();-}-#endif // MUL_MAT_ID_USE_SUBGROUPS--void load_row_ids_hoisted(uint expert_idx, uint ic) {- _ne1 = uint(data_expert_count[expert_idx]);-- const uint tile_begin = ic * BN;- const uint tile_count = tile_begin < _ne1 ? min(BN, _ne1 - tile_begin) : 0;- const uint expert_offset = uint(data_expert_count[p.n_experts + expert_idx]);- const uint row_ids_offset = 2 * p.n_experts + 1 + expert_offset + tile_begin;-- for (uint i = gl_LocalInvocationIndex; i < tile_count; i += BLOCK_SIZE) {- const uint packed_row_id = uint(data_expert_count[row_ids_offset + i]);+void load_row_ids(uint row_begin) {+ for (uint i = gl_LocalInvocationIndex; i < _ne1; i += BLOCK_SIZE) {+ const uint packed_row_id = data_row_map[p.row_ids_offset + row_begin + i];row_ids[i] = u16vec2(packed_row_id & 0xffffu, packed_row_id >> 16);}barrier();diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mmq.comp b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mmq.compindex 67908e7048..2819eb46e9 100644--- a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mmq.comp+++ b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mmq.comp@@ -37,7 +37,7 @@ layout (binding = 2) writeonly buffer D {D_TYPE data_d[];};#ifdef MUL_MAT_IDlayout (binding = 3) readonly buffer IDS {int data_ids[];};-layout (binding = 4) readonly buffer Counts {int data_expert_count[];};+layout (binding = 4) readonly buffer RowMap {uint data_row_map[];};#endiflayout (push_constant) uniform parameter@@ -59,7 +59,7 @@ layout (push_constant) uniform parameteruint nbi1;uint ne11;uint n_experts;- uint hoist_row_ids;+ uint row_ids_offset;#elseuint base_work_group_z;uint num_batches;@@ -105,19 +105,22 @@ block_b_cache cache_b;#define LOAD_VEC_A (4 * QUANT_R_MMQ)#define LOAD_VEC_B 16-#define NUM_WARPS (BLOCK_SIZE / WARP)#include "mul_mm_id_funcs.glsl"#include "mul_mmq_funcs.glsl"void main() {- const uint ic = gl_WorkGroupID.y;-#ifdef MUL_MAT_ID- const uint expert_idx = gl_WorkGroupID.z;- if (ic * BN >= data_expert_count[expert_idx]) {+ const uint tile_idx = gl_WorkGroupID.y + gl_WorkGroupID.z * gl_NumWorkGroups.y;+ if (tile_idx >= data_row_map[0]) {return;}+ const uint expert_idx = data_row_map[1 + 2 * tile_idx];+ const uint row_begin = tile_idx * BN;+ _ne1 = data_row_map[2 + 2 * tile_idx];+ const uint ic = 0;+#else+ const uint ic = gl_WorkGroupID.y;#endif#ifdef NEEDS_INIT_IQ_SHMEMinit_iq_shmem(gl_WorkGroupSize);@@ -161,34 +164,7 @@ void main() {const uint loadstride_b = BLOCK_SIZE * LOAD_VEC_B / BK;#ifdef MUL_MAT_ID- if (p.hoist_row_ids != 0) {- load_row_ids_hoisted(expert_idx, ic);- } else {-#ifdef MUL_MAT_ID_USE_SUBGROUPS- if (bitCount(p.nei0) == 1) {- load_row_ids(expert_idx, true, ic);- } else {- load_row_ids(expert_idx, false, ic);- }-#else- _ne1 = 0;- for (uint ii1 = 0; ii1 < p.nei1 && _ne1 < (ic + 1) * BN; ii1++) {- for (uint ii0 = 0; ii0 < p.nei0 && _ne1 < (ic + 1) * BN; ii0++) {- if (data_ids[ii1*p.nbi1 + ii0] == expert_idx) {- if (_ne1 >= ic * BN) {- row_ids[_ne1 - ic * BN] = u16vec2(ii0, ii1);- }- _ne1++;- }- }- }-- barrier();-#endif- }-- // Workgroup has no work- if (ic * BN >= _ne1) return;+ load_row_ids(row_begin);#endif#ifdef MUL_MAT_IDdiff --git a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mmq_cm1.comp b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mmq_cm1.compindex 7cab9a1195..17280d17b4 100644--- a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mmq_cm1.comp+++ b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mmq_cm1.comp@@ -39,7 +39,7 @@ layout (binding = 2) writeonly buffer D {D_TYPE data_d[];};#ifdef MUL_MAT_IDlayout (binding = 3) readonly buffer IDS {int data_ids[];};-layout (binding = 4) readonly buffer Counts {int data_expert_count[];};+layout (binding = 4) readonly buffer RowMap {uint data_row_map[];};#endiflayout (push_constant) uniform parameter@@ -61,7 +61,7 @@ layout (push_constant) uniform parameteruint nbi1;uint ne11;uint n_experts;- uint hoist_row_ids;+ uint row_ids_offset;#elseuint base_work_group_z;uint num_batches;@@ -142,13 +142,22 @@ ACC_TYPE cm1_accumulate(ACC_TYPE prev, int acc_e, float scale_a, float nbias_a,}#ifdef MUL_MAT_ID-#define NUM_WARPS (BLOCK_SIZE / WARP)#include "mul_mm_id_funcs.glsl"#endif#include "mul_mmq_cm1_funcs.glsl"void main() {+#ifdef MUL_MAT_ID+ const uint tile_idx = gl_WorkGroupID.y + gl_WorkGroupID.z * gl_NumWorkGroups.y;+ if (tile_idx >= data_row_map[0]) {+ return;+ }+ const uint expert_idx = data_row_map[1 + 2 * tile_idx];+ const uint row_begin = tile_idx * BN;+ _ne1 = data_row_map[2 + 2 * tile_idx];+ const uint ic = 0;+#endif#if defined(DATA_A_IQ4_NL) || defined(DATA_A_IQ4_XS)if (gl_LocalInvocationIndex < 16u) {cm1_kvalues[gl_LocalInvocationIndex] = kvalues_iq4nl_const[gl_LocalInvocationIndex];@@ -175,12 +184,7 @@ void main() {const uint ik = gl_WorkGroupID.x / blocks_m;#ifdef MUL_MAT_ID- const uint ic = gl_WorkGroupID.y;const uint ir = gl_WorkGroupID.x % blocks_m;- const uint expert_idx = gl_WorkGroupID.z;- if (ic * BN >= data_expert_count[expert_idx]) {- return;- }#else// L2-friendly workgroup schedulingconst uint blocks_n = (p.N + BN - 1) / BN;@@ -231,33 +235,7 @@ void main() {const uint loadstride_b = BLOCK_SIZE * LOAD_VEC_B / BK;#ifdef MUL_MAT_ID- if (p.hoist_row_ids != 0) {- load_row_ids_hoisted(expert_idx, ic);- } else {-#ifdef MUL_MAT_ID_USE_SUBGROUPS- if (bitCount(p.nei0) == 1) {- load_row_ids(expert_idx, true, ic);- } else {- load_row_ids(expert_idx, false, ic);- }-#else- _ne1 = 0;- for (uint ii1 = 0; ii1 < p.nei1 && _ne1 < (ic + 1) * BN; ii1++) {- for (uint ii0 = 0; ii0 < p.nei0 && _ne1 < (ic + 1) * BN; ii0++) {- if (data_ids[ii1*p.nbi1 + ii0] == expert_idx) {- if (_ne1 >= ic * BN) {- row_ids[_ne1 - ic * BN] = u16vec2(ii0, ii1);- }- _ne1++;- }- }- }-- barrier();-#endif- }-- if (ic * BN >= _ne1) return;+ load_row_ids(row_begin);#endif#ifdef MUL_MAT_IDdiff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cppindex 4e3c6942b4..bd599e99e5 100644--- a/tests/test-backend-ops.cpp+++ b/tests/test-backend-ops.cpp@@ -5440,9 +5440,10 @@ struct test_mul_mat_id : public test_case {const int64_t k;const float amax; // magnitude of src1const int64_t m_v; // rows of as in memory, the experts of as are strided for m_v > m, no view for m_v == 0+ const bool ids_offset;std::string vars() override {- return VARS_TO_STR10(type_a, type_b, n_mats, n_used, b, m, n, k, amax, m_v);+ return VARS_TO_STR11(type_a, type_b, n_mats, n_used, b, m, n, k, amax, m_v, ids_offset);}double max_nmse_err() override {@@ -5467,9 +5468,9 @@ struct test_mul_mat_id : public test_case {test_mul_mat_id(ggml_type type_a = GGML_TYPE_F32, ggml_type type_b = GGML_TYPE_F32,int n_mats = 8, int n_used = 2, bool b = false,int64_t m = 32, int64_t n = 32, int64_t k = 32,- float amax = 1.0f, int64_t m_v = 0)+ float amax = 1.0f, int64_t m_v = 0, bool ids_offset = false): type_a(type_a), type_b(type_b), n_mats(n_mats), n_used(n_used), b(b),- m(m), n(n), k(k), amax(amax), m_v(m_v) {+ m(m), n(n), k(k), amax(amax), m_v(m_v), ids_offset(ids_offset) {GGML_ASSERT(n_used <= n_mats);GGML_ASSERT(m_v == 0 || m_v > m);}@@ -5482,10 +5483,10 @@ struct test_mul_mat_id : public test_case {}ggml_set_name(as, "as");- ggml_tensor * ids = ggml_new_tensor_2d(ctx, GGML_TYPE_I32, n_mats, n);+ ggml_tensor * ids = ggml_new_tensor_2d(ctx, GGML_TYPE_I32, n_mats + int(ids_offset), n);ggml_set_name(ids, "ids");- if (n_used != n_mats) {- ids = ggml_view_2d(ctx, ids, n_used, n, ids->nb[1], 0);+ if (n_used != n_mats || ids_offset) {+ ids = ggml_view_2d(ctx, ids, n_used, n, ids->nb[1], ids_offset ? sizeof(int32_t) : 0);ggml_set_name(ids, "view_of_ids");}@@ -5512,6 +5513,32 @@ struct test_mul_mat_id : public test_case {}};+// MUL_MAT_ID with expert ids repeated within a token's row+struct test_mul_mat_id_dup : public test_mul_mat_id {+ using test_mul_mat_id::test_mul_mat_id;+ std::string vars() override { return test_mul_mat_id::vars() + ",dup=1"; }+ void initialize_tensors(ggml_context * ctx) override {+ std::default_random_engine rng(1234);+ for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, t)) {+ if (t->type == GGML_TYPE_I32) {+ if (ggml_is_view_op(t->op)) { continue; }+ for (int64_t r = 0; r < ggml_nrows(t); r++) {+ std::vector<int32_t> data(t->ne[0]);+ for (int i = 0; i < t->ne[0]; i++) {+ data[i] = (rng() % 4 == 0) ? (int32_t) (rng() % n_mats) : 0; // repeated ids, mostly expert 0+ }+ if (r == ggml_nrows(t) - 1) {+ data[ids_offset ? 1 : 0] = n_mats - 1;+ }+ ggml_backend_tensor_set(t, data.data(), r * t->nb[1], t->ne[0] * sizeof(int32_t));+ }+ } else {+ init_tensor_uniform(t);+ }+ }+ }+};+// FP4 W4A8 path on the MoE path (GGML_PREC_Q8 on src1 disallows 4-bit activations)struct test_mul_mat_id_w4a8 : public test_mul_mat_id {test_mul_mat_id_w4a8(ggml_type type_a = GGML_TYPE_NVFP4, ggml_type type_b = GGML_TYPE_F32,@@ -10972,6 +10999,18 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_TQ1_0, GGML_TYPE_F32, 28, 10, false, 1024, 1, 4096));test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_TQ1_0, GGML_TYPE_F32, 128, 8, false, 1024, 1, 2048));+ // repeated expert ids within a row+ for (ggml_type ta : {GGML_TYPE_F16, GGML_TYPE_Q8_0, GGML_TYPE_Q4_0}) {+ for (int n : {9, 16, 33, 64}) {+ test_cases.emplace_back(new test_mul_mat_id_dup(ta, GGML_TYPE_F32, 28, 10, false, 1024, n, 256));+ }+ }++ for (ggml_type ta : {GGML_TYPE_F16, GGML_TYPE_Q4_0}) {+ test_cases.emplace_back(new test_mul_mat_id_dup(ta, GGML_TYPE_F32, 1025, 10, false, 64, 33, 256, 1.0f, 0, true));+ test_cases.emplace_back(new test_mul_mat_id_dup(ta, GGML_TYPE_F32, 2050, 10, true, 64, 33, 256));+ }+for (ggml_type type_a : all_types) {test_cases.emplace_back(new test_mul_mat_id(type_a, GGML_TYPE_F32, 4, 2, false, 64, 16, 3*ggml_blck_size(type_a)));}