Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

dslify-moe-scatter (#205)

Aleksandr Golokoz9a979ccdc6e0
diff --git a/crates/uzu/src/backends/metal/encodable_block/moe_block/moe_block.rs b/crates/uzu/src/backends/metal/encodable_block/moe_block/moe_block.rs
index add71978f..a5cbbca4a 100644
--- a/crates/uzu/src/backends/metal/encodable_block/moe_block/moe_block.rs
+++ b/crates/uzu/src/backends/metal/encodable_block/moe_block/moe_block.rs
@@ -10,16 +10,21 @@ use crate::{
Activation, DataType, LinearConfig, MixtureOfExpertsConfig, RoutingFunctionConfig,
array::Array,
backends::{
- common::kernel::{MoeCountsOffsetsFusedKernel, MoeFinalizeKernel, MoeRouterTopKKernel},
+ common::kernel::{
+ MoeBlockBasesFromPartialsKernel, MoeCountsOffsetsFusedKernel, MoeFinalizeKernel, MoeRouterTopKKernel,
+ MoeScatterBucketsMapKernel,
+ },
metal::{
Metal, MetalContext,
kernel::{
MoeGatherKernels,
- dsl::{MoeCountsOffsetsFusedMetalKernel, MoeFinalizeMetalKernel, MoeRouterTopKMetalKernel},
+ dsl::{
+ MoeBlockBasesFromPartialsMetalKernel, MoeCountsOffsetsFusedMetalKernel, MoeFinalizeMetalKernel,
+ MoeRouterTopKMetalKernel, MoeScatterBucketsMapMetalKernel,
+ },
moe::{
- MoeBlockBasesArguments, MoeExpertsTwoPassArguments, MoeExpertsTwoPassDecodeBlock,
- MoeExpertsTwoPassPrefillBlock, MoeGatherArguments, MoeScatterArguments, MoeScatterKernels,
- MoeScatterWithMapArguments,
+ MoeExpertsTwoPassArguments, MoeExpertsTwoPassDecodeBlock, MoeExpertsTwoPassPrefillBlock,
+ MoeGatherArguments,
},
},
},
@@ -41,7 +46,8 @@ pub struct MoeBlock {
router_renorm: bool,
router_topk_kernel: MoeRouterTopKMetalKernel,
counts_offsets_kernel: MoeCountsOffsetsFusedMetalKernel,
- scatter_kernels: MoeScatterKernels,
+ scatter_bases_kernel: MoeBlockBasesFromPartialsMetalKernel,
+ scatter_map_kernel: MoeScatterBucketsMapMetalKernel,
gather_kernels: MoeGatherKernels,
experts_two_pass_decode_kernel: MoeExpertsTwoPassDecodeBlock,
experts_two_pass_prefill_kernel: MoeExpertsTwoPassPrefillBlock,
@@ -109,9 +115,18 @@ impl MoeBlock {
let counts_offsets_kernel = MoeCountsOffsetsFusedKernel::new(context).map_err(|e| {
crate::backends::metal::MetalError::Generic(format!("Counts+offsets kernel error: {:?}", e))
})?;
- let scatter_kernels = MoeScatterKernels::new(context)
- .map_err(|e| crate::backends::metal::MetalError::Generic(format!("Scatter kernels error: {:?}", e)))?;
- let gather_kernel = MoeGatherKernels::new(context)
+
+ let scatter_bases_kernel = MoeBlockBasesFromPartialsMetalKernel::new(context).map_err(|e| {
+ crate::backends::metal::MetalError::Generic(format!(
+ "MoeBlockBasesFromPartialsMetalKernel kernel error: {:?}",
+ e
+ ))
+ })?;
+ let scatter_map_kernel = MoeScatterBucketsMapMetalKernel::new(context, data_type).map_err(|e| {
+ crate::backends::metal::MetalError::Generic(format!("MoeScatterBucketsMetalKernel kernel error: {:?}", e))
+ })?;
+
+ let gather_kernels = MoeGatherKernels::new(context)
.map_err(|e| crate::backends::metal::MetalError::Generic(format!("Gather kernel error: {:?}", e)))?;
let experts_two_pass_decode_kernel = MoeExpertsTwoPassDecodeBlock::new(context).map_err(|e| {
crate::backends::metal::MetalError::Generic(format!("Experts two-pass decode kernel error: {:?}", e))
@@ -166,8 +181,9 @@ impl MoeBlock {
router_renorm,
router_topk_kernel,
counts_offsets_kernel,
- scatter_kernels,
- gather_kernels: gather_kernel,
+ scatter_bases_kernel,
+ scatter_map_kernel,
+ gather_kernels,
experts_two_pass_decode_kernel,
experts_two_pass_prefill_kernel,
finalize_kernel,
@@ -349,44 +365,41 @@ impl EncodableBlock<Metal> for MoeBlock {
let num_blocks = ((suffix_length + 255) / 256).max(1);
let num_tiles = ((e + 512 - 1) / 512).max(1);
-
- self.scatter_kernels
- .encode_block_bases(
- root,
- MoeBlockBasesArguments {
- partials_buffer: &partials_buf,
- block_bases_buffer: &block_bases_buf,
- block_alloc_buffer: &block_alloc_buf,
- e,
- num_blocks,
- num_tiles,
- },
- )
- .expect("MoE block bases failed");
-
- self.scatter_kernels
- .encode_scatter_with_map(
- root,
- MoeScatterWithMapArguments {
- base: MoeScatterArguments {
- topk_ids_buffer: &topk_ids_buf,
- topk_probs_buffer: &topk_probs_buf,
- offsets_buffer: &offsets_buf,
- block_bases_buffer: &block_bases_buf,
- block_alloc_buffer: &block_alloc_buf,
- out_ids_buffer: &bucketed_ids_buf,
- out_probs_buffer: &bucketed_probs_buf,
- t: suffix_length,
- e,
- k,
- num_blocks,
- num_tiles,
- },
- tok2row_buffer: &tok2row_buf,
- },
- self.data_type,
- )
- .expect("MoE scatter failed");
+ let scatter_bases_encoder = command_buffer
+ .new_compute_command_encoder()
+ .expect("Failed to create compute command encoder for scatter bases kernel");
+ self.scatter_bases_kernel.encode(
+ &partials_buf,
+ &block_bases_buf,
+ &block_alloc_buf,
+ e as u32,
+ num_blocks as u32,
+ num_tiles as u32,
+ 0u32,
+ &scatter_bases_encoder,
+ );
+ scatter_bases_encoder.end_encoding();
+
+ let scatter_encoder = command_buffer
+ .new_compute_command_encoder()
+ .expect("Failed to create compute command encoder for scatter map kernel");
+ self.scatter_map_kernel.encode(
+ &topk_ids_buf,
+ &topk_probs_buf,
+ &offsets_buf,
+ &block_bases_buf,
+ &block_alloc_buf,
+ &bucketed_ids_buf,
+ &bucketed_probs_buf,
+ suffix_length as u32,
+ e as u32,
+ k as u32,
+ num_blocks as u32,
+ num_tiles as u32,
+ &tok2row_buf,
+ &scatter_encoder,
+ );
+ scatter_encoder.end_encoding();
self.gather_kernels.encode(
root,
diff --git a/crates/uzu/src/backends/metal/kernel/mod.rs b/crates/uzu/src/backends/metal/kernel/mod.rs
index 332685497..0323968b5 100644
--- a/crates/uzu/src/backends/metal/kernel/mod.rs
+++ b/crates/uzu/src/backends/metal/kernel/mod.rs
@@ -9,8 +9,8 @@ pub mod moe;
pub use matmul::{MatmulArguments, MatmulKernel};
pub use moe::{
- MoeBlockBasesArguments, MoeExpertsTwoPassArguments, MoeExpertsTwoPassDecodeBlock, MoeExpertsTwoPassPrefillBlock,
- MoeGatherArguments, MoeGatherKernels, MoeScatterArguments, MoeScatterKernels, MoeScatterWithMapArguments,
+ MoeExpertsTwoPassArguments, MoeExpertsTwoPassDecodeBlock, MoeExpertsTwoPassPrefillBlock, MoeGatherArguments,
+ MoeGatherKernels,
};
impl MatmulKernels for dsl::MetalKernels {
diff --git a/crates/uzu/src/backends/metal/kernel/moe/mod.rs b/crates/uzu/src/backends/metal/kernel/moe/mod.rs
index 15c18190f..beb45c942 100644
--- a/crates/uzu/src/backends/metal/kernel/moe/mod.rs
+++ b/crates/uzu/src/backends/metal/kernel/moe/mod.rs
@@ -3,7 +3,6 @@ mod experts_single;
mod experts_two_pass_decode;
mod experts_two_pass_prefill;
mod gather;
-mod scatter;
pub mod tiles_map;
pub mod tiles_pass_a;
@@ -12,7 +11,6 @@ pub use experts_single::{MoeExpertsSingleDecodeArguments, MoeExpertsSingleDecode
pub use experts_two_pass_decode::MoeExpertsTwoPassDecodeBlock;
pub use experts_two_pass_prefill::{MoeExpertsTwoPassArguments, MoeExpertsTwoPassPrefillBlock};
pub use gather::{MoeGatherArguments, MoeGatherKernels};
-pub use scatter::{MoeBlockBasesArguments, MoeScatterArguments, MoeScatterKernels, MoeScatterWithMapArguments};
pub use tiles_pass_a::{
MoePassARowMapArguments, MoePassATileBuildArguments, MoePassATileCountsArguments, MoePassATileDispatchArguments,
MoePassATileKernels, MoePassATileScanArguments,
diff --git a/crates/uzu/src/backends/metal/kernel/moe/scatter.rs b/crates/uzu/src/backends/metal/kernel/moe/scatter.rs
deleted file mode 100644
index 1159b0654..000000000
--- a/crates/uzu/src/backends/metal/kernel/moe/scatter.rs
+++ /dev/null
@@ -1,205 +0,0 @@
-use metal::{
- MTLBuffer, MTLCommandBuffer, MTLCommandEncoder, MTLComputeCommandEncoder, MTLComputePipelineState, MTLSize,
-};
-use objc2::{rc::Retained, runtime::ProtocolObject};
-
-use crate::{
- DataType,
- backends::metal::{MetalContext, metal_extensions::ComputeEncoderSetValue},
-};
-
-// ---- Scatter Buckets Kernels ----
-
-#[derive(Debug, thiserror::Error)]
-pub enum MoeScatterError {
- #[error("Metal error: {0}")]
- MetalError(#[from] crate::backends::metal::MetalError),
-}
-
-pub struct MoeScatterKernels {
- pipeline_bases: Retained<ProtocolObject<dyn MTLComputePipelineState>>,
- pipeline_scatter_f16: Retained<ProtocolObject<dyn MTLComputePipelineState>>,
- pipeline_scatter_f32: Retained<ProtocolObject<dyn MTLComputePipelineState>>,
- pipeline_scatter_bf16: Retained<ProtocolObject<dyn MTLComputePipelineState>>,
- // map variants
- pipeline_scatter_map_f16: Retained<ProtocolObject<dyn MTLComputePipelineState>>,
- pipeline_scatter_map_f32: Retained<ProtocolObject<dyn MTLComputePipelineState>>,
- pipeline_scatter_map_bf16: Retained<ProtocolObject<dyn MTLComputePipelineState>>,
-}
-
-#[derive(Debug)]
-pub struct MoeBlockBasesArguments<'a> {
- pub partials_buffer: &'a ProtocolObject<dyn MTLBuffer>, // [num_blocks * num_tiles * 512]
- pub block_bases_buffer: &'a ProtocolObject<dyn MTLBuffer>, // same shape as partials
- pub block_alloc_buffer: &'a ProtocolObject<dyn MTLBuffer>, // [num_blocks * num_tiles * 512]
- pub e: usize,
- pub num_blocks: usize,
- pub num_tiles: usize,
-}
-
-#[derive(Debug)]
-pub struct MoeScatterArguments<'a> {
- pub topk_ids_buffer: &'a ProtocolObject<dyn MTLBuffer>,
- pub topk_probs_buffer: &'a ProtocolObject<dyn MTLBuffer>,
- pub offsets_buffer: &'a ProtocolObject<dyn MTLBuffer>,
- pub block_bases_buffer: &'a ProtocolObject<dyn MTLBuffer>,
- pub block_alloc_buffer: &'a ProtocolObject<dyn MTLBuffer>,
- pub out_ids_buffer: &'a ProtocolObject<dyn MTLBuffer>,
- pub out_probs_buffer: &'a ProtocolObject<dyn MTLBuffer>,
- pub t: usize,
- pub e: usize,
- pub k: usize,
- pub num_blocks: usize,
- pub num_tiles: usize,
-}
-
-#[derive(Debug)]
-pub struct MoeScatterWithMapArguments<'a> {
- pub base: MoeScatterArguments<'a>,
- pub tok2row_buffer: &'a ProtocolObject<dyn MTLBuffer>, // [T*K] int32, initialized to -1
-}
-
-impl MoeScatterKernels {
- pub fn new(mtl_context: &MetalContext) -> Result<Self, MoeScatterError> {
- let pipeline_bases = mtl_context.compute_pipeline_state("moe_block_bases_from_partials", None)?;
- let pipeline_scatter_f16 = mtl_context.compute_pipeline_state("moe_scatter_buckets_f16", None)?;
- let pipeline_scatter_f32 = mtl_context.compute_pipeline_state("moe_scatter_buckets_f32", None)?;
- let pipeline_scatter_bf16 = mtl_context.compute_pipeline_state("moe_scatter_buckets_bf16", None)?;
- let pipeline_scatter_map_f16 = mtl_context.compute_pipeline_state("moe_scatter_buckets_map_f16", None)?;
- let pipeline_scatter_map_f32 = mtl_context.compute_pipeline_state("moe_scatter_buckets_map_f32", None)?;
- let pipeline_scatter_map_bf16 = mtl_context.compute_pipeline_state("moe_scatter_buckets_map_bf16", None)?;
-
- Ok(Self {
- pipeline_bases,
- pipeline_scatter_f16,
- pipeline_scatter_f32,
- pipeline_scatter_bf16,
- pipeline_scatter_map_f16,
- pipeline_scatter_map_f32,
- pipeline_scatter_map_bf16,
- })
- }
-
- pub fn encode_block_bases(
- &self,
- command_buffer: &Retained<ProtocolObject<dyn MTLCommandBuffer>>,
- args: MoeBlockBasesArguments,
- ) -> Result<(), MoeScatterError> {
- let compute_encoder =
- command_buffer.new_compute_command_encoder().expect("Failed to create compute command encoder");
- compute_encoder.set_compute_pipeline_state(&self.pipeline_bases);
- compute_encoder.set_buffer(Some(args.partials_buffer), 0, 0);
- compute_encoder.set_buffer(Some(args.block_bases_buffer), 0, 1);
- compute_encoder.set_buffer(Some(args.block_alloc_buffer), 0, 2);
-
- let e_u32 = args.e as u32;
- let nb_u32 = args.num_blocks as u32;
- let nt_u32 = args.num_tiles as u32;
- let cap_u32: u32 = 0;
- compute_encoder.set_value(&e_u32, 3);
- compute_encoder.set_value(&nb_u32, 4);
- compute_encoder.set_value(&nt_u32, 5);
- compute_encoder.set_value(&cap_u32, 6);
-
- let total_entries = args.num_tiles * 512usize;
- let threads_per_threadgroup = MTLSize::new(256, 1, 1);
- let tg = MTLSize::new((total_entries + 255) / 256, 1, 1);
- if total_entries > 0 {
- compute_encoder.dispatch_threadgroups(tg, threads_per_threadgroup);
- }
- compute_encoder.end_encoding();
- Ok(())
- }
-
- pub fn encode_scatter(
- &self,
- command_buffer: &Retained<ProtocolObject<dyn MTLCommandBuffer>>,
- args: MoeScatterArguments,
- dtype: DataType,
- ) -> Result<(), MoeScatterError> {
- let compute_encoder =
- command_buffer.new_compute_command_encoder().expect("Failed to create compute command encoder");
- // Select pipeline based on dtype
- match dtype {
- DataType::F16 => {
- compute_encoder.set_compute_pipeline_state(&self.pipeline_scatter_f16);
- },
- DataType::F32 => {
- compute_encoder.set_compute_pipeline_state(&self.pipeline_scatter_f32);
- },
- DataType::BF16 => {
- compute_encoder.set_compute_pipeline_state(&self.pipeline_scatter_bf16);
- },
- _ => panic!("Unsupported data type: {:?}", dtype),
- }
- compute_encoder.set_buffer(Some(args.topk_ids_buffer), 0, 0);
- compute_encoder.set_buffer(Some(args.topk_probs_buffer), 0, 1);
- compute_encoder.set_buffer(Some(args.offsets_buffer), 0, 2);
- compute_encoder.set_buffer(Some(args.block_bases_buffer), 0, 3);
- compute_encoder.set_buffer(Some(args.block_alloc_buffer), 0, 4);
- compute_encoder.set_buffer(Some(args.out_ids_buffer), 0, 5);
- compute_encoder.set_buffer(Some(args.out_probs_buffer), 0, 6);
- let t_u32 = args.t as u32;
- let e_u32 = args.e as u32;
- let k_u32 = args.k as u32;
- let nb_u32 = args.num_blocks as u32;
- let nt_u32 = args.num_tiles as u32;
- compute_encoder.set_value(&t_u32, 7);
- compute_encoder.set_value(&e_u32, 8);
- compute_encoder.set_value(&k_u32, 9);
- compute_encoder.set_value(&nb_u32, 10);
- compute_encoder.set_value(&nt_u32, 11);
-
- let threads_per_threadgroup = MTLSize::new(256, 1, 1);
- let tg = MTLSize::new(args.num_blocks, 1, 1);
- if args.num_blocks > 0 {
- compute_encoder.dispatch_threadgroups(tg, threads_per_threadgroup);
- }
- compute_encoder.end_encoding();
- Ok(())
- }
-
- pub fn encode_scatter_with_map(
- &self,
- command_buffer: &Retained<ProtocolObject<dyn MTLCommandBuffer>>,
- args: MoeScatterWithMapArguments,
- dtype: DataType,
- ) -> Result<(), MoeScatterError> {
- let compute_encoder =
- command_buffer.new_compute_command_encoder().expect("Failed to create compute command encoder");
- let pipeline = match dtype {
- DataType::F16 => &self.pipeline_scatter_map_f16,
- DataType::F32 => &self.pipeline_scatter_map_f32,
- DataType::BF16 => &self.pipeline_scatter_map_bf16,
- _ => panic!("Unsupported data type: {:?}", dtype),
- };
- compute_encoder.set_compute_pipeline_state(pipeline);
- let base = &args.base;
- compute_encoder.set_buffer(Some(base.topk_ids_buffer), 0, 0);
- compute_encoder.set_buffer(Some(base.topk_probs_buffer), 0, 1);
- compute_encoder.set_buffer(Some(base.offsets_buffer), 0, 2);
- compute_encoder.set_buffer(Some(base.block_bases_buffer), 0, 3);
- compute_encoder.set_buffer(Some(base.block_alloc_buffer), 0, 4);
- compute_encoder.set_buffer(Some(base.out_ids_buffer), 0, 5);
- compute_encoder.set_buffer(Some(base.out_probs_buffer), 0, 6);
- let t_u32 = base.t as u32;
- let e_u32 = base.e as u32;
- let k_u32 = base.k as u32;
- let nb_u32 = base.num_blocks as u32;
- let nt_u32 = base.num_tiles as u32;
- compute_encoder.set_value(&t_u32, 7);
- compute_encoder.set_value(&e_u32, 8);
- compute_encoder.set_value(&k_u32, 9);
- compute_encoder.set_value(&nb_u32, 10);
- compute_encoder.set_value(&nt_u32, 11);
- compute_encoder.set_buffer(Some(args.tok2row_buffer), 0, 12);
-
- let threads_per_threadgroup = MTLSize::new(256, 1, 1);
- let tg = MTLSize::new(base.num_blocks, 1, 1);
- if base.num_blocks > 0 {
- compute_encoder.dispatch_threadgroups(tg, threads_per_threadgroup);
- }
- compute_encoder.end_encoding();
- Ok(())
- }
-}
diff --git a/crates/uzu/src/backends/metal/kernel/moe/scatter_buckets.metal b/crates/uzu/src/backends/metal/kernel/moe/scatter_buckets.metal
index cdb91e750..3248e0a30 100644
--- a/crates/uzu/src/backends/metal/kernel/moe/scatter_buckets.metal
+++ b/crates/uzu/src/backends/metal/kernel/moe/scatter_buckets.metal
@@ -1,5 +1,6 @@
#include <metal_stdlib>
#include <metal_atomic>
+#include "../definitions.metal"
using namespace metal;
#define BLOCK_SIZE 256
@@ -7,52 +8,6 @@ using namespace metal;
#define SIMD_WIDTH 32
#define NUM_SG (BLOCK_SIZE / SIMD_WIDTH)
-// 4A: Compute per-block base offsets from partials
-// partials layout: index = (block_id * num_tiles + tile_id) * TILE_E + te
-[[max_total_threads_per_threadgroup(256)]]
-kernel void moe_block_bases_from_partials(
- device const uint* partials [[buffer(0)]],
- device uint* block_bases [[buffer(1)]],
- device uint* block_alloc [[buffer(2)]],
- constant uint& E [[buffer(3)]],
- constant uint& num_blocks [[buffer(4)]],
- constant uint& num_tiles [[buffer(5)]],
- constant uint& capacity_per_expert [[buffer(6)]], // 0 => no clamp
- uint gid [[thread_position_in_grid]]
-) {
- const uint entries = num_tiles * TILE_E;
- if (gid >= entries)
- return;
- const uint tile_id = gid / TILE_E;
- const uint te = gid % TILE_E;
- const uint e = tile_id * TILE_E + te;
- if (e >= E)
- return;
-
- uint running = 0u;
- const bool clamp = (capacity_per_expert > 0u);
- for (uint b = 0; b < num_blocks; ++b) {
- const uint idx = (b * num_tiles + tile_id) * TILE_E + te;
- const uint p = partials[idx];
- const uint alloc = clamp ? (running < capacity_per_expert
- ? min(p, capacity_per_expert - running)
- : 0u)
- : p;
- block_bases[idx] = running;
- block_alloc[idx] = alloc;
- running += alloc;
- if (clamp && running >= capacity_per_expert) {
- // Remainder allocate zero and base stays at capacity
- for (uint bb = b + 1; bb < num_blocks; ++bb) {
- const uint idx2 = (bb * num_tiles + tile_id) * TILE_E + te;
- block_bases[idx2] = running;
- block_alloc[idx2] = 0u;
- }
- break;
- }
- }
-}
-
// 4B: Scatter into expert-major buckets using atomic-free device writes
template <typename ProbT>
inline void moe_scatter_buckets_impl(
@@ -188,98 +143,136 @@ inline void moe_scatter_buckets_impl(
}
}
-#define DEFINE_MOE_SCATTER_BUCKETS_KERNEL(SUFFIX, ProbT) \
- [[max_total_threads_per_threadgroup(256)]] \
- kernel void moe_scatter_buckets_##SUFFIX( \
- device const int* topk_ids [[buffer(0)]], \
- device const ProbT* topk_probs [[buffer(1)]], \
- device const uint* offsets [[buffer(2)]], \
- device const uint* block_bases [[buffer(3)]], \
- device const uint* block_alloc [[buffer(4)]], \
- device int* out_ids [[buffer(5)]], \
- device ProbT* out_probs [[buffer(6)]], \
- constant uint& T [[buffer(7)]], \
- constant uint& E [[buffer(8)]], \
- constant uint& K [[buffer(9)]], \
- constant uint& num_blocks [[buffer(10)]], \
- constant uint& num_tiles [[buffer(11)]], \
- uint3 tid3 [[thread_position_in_grid]], \
- uint lid [[thread_index_in_threadgroup]], \
- uint3 tgpig [[threadgroup_position_in_grid]] \
- ) { \
- threadgroup _atomic<uint> sg_counts[NUM_SG * TILE_E]; \
- threadgroup uint sg_base[NUM_SG * TILE_E]; \
- moe_scatter_buckets_impl<ProbT>( \
- topk_ids, \
- topk_probs, \
- offsets, \
- block_bases, \
- block_alloc, \
- out_ids, \
- out_probs, \
- (device int*)nullptr, \
- T, \
- E, \
- K, \
- num_blocks, \
- num_tiles, \
- tid3, \
- lid, \
- tgpig, \
- sg_counts, \
- sg_base \
- ); \
+// 4A: Compute per-block base offsets from partials
+// partials layout: index = (block_id * num_tiles + tile_id) * TILE_E + te
+KERNEL(MoeBlockBasesFromPartials)(
+ device const uint* partials,
+ device uint* block_bases,
+ device uint* block_alloc,
+ constant uint& e_input,
+ constant uint& num_blocks,
+ constant uint& num_tiles,
+ constant uint& capacity_per_expert, // 0 => no clamp
+ const uint gid AXIS(num_tiles * 512, 256)
+) {
+ const uint tile_id = gid / TILE_E;
+ const uint te = gid % TILE_E;
+ const uint e = tile_id * TILE_E + te;
+ if (e >= e_input)
+ return;
+
+ uint running = 0u;
+ const bool clamp = (capacity_per_expert > 0u);
+ for (uint b = 0; b < num_blocks; ++b) {
+ const uint idx = (b * num_tiles + tile_id) * TILE_E + te;
+ const uint p = partials[idx];
+ const uint alloc = clamp ? (running < capacity_per_expert
+ ? min(p, capacity_per_expert - running)
+ : 0u)
+ : p;
+ block_bases[idx] = running;
+ block_alloc[idx] = alloc;
+ running += alloc;
+ if (clamp && running >= capacity_per_expert) {
+ // Remainder allocate zero and base stays at capacity
+ for (uint bb = b + 1; bb < num_blocks; ++bb) {
+ const uint idx2 = (bb * num_tiles + tile_id) * TILE_E + te;
+ block_bases[idx2] = running;
+ block_alloc[idx2] = 0u;
+ }
+ break;
+ }
}
+}
-DEFINE_MOE_SCATTER_BUCKETS_KERNEL(f16, half)
-DEFINE_MOE_SCATTER_BUCKETS_KERNEL(f32, float)
-DEFINE_MOE_SCATTER_BUCKETS_KERNEL(bf16, bfloat)
+template <typename T>
+VARIANTS(T, float, half, bfloat)
+KERNEL(MoeScatterBuckets)(
+ device const int* topk_ids,
+ device const T* topk_probs,
+ device const uint* offsets,
+ device const uint* block_bases,
+ device const uint* block_alloc,
+ device int* out_ids,
+ device T* out_probs,
+ constant uint& t,
+ constant uint& e,
+ constant uint& k,
+ constant uint& num_blocks,
+ constant uint& num_tiles,
+ threadgroup _atomic<uint> sg_counts[NUM_SG * TILE_E],
+ threadgroup uint sg_base[NUM_SG * TILE_E],
+ const uint tgpig_x GROUPS(num_blocks),
+ const uint lid THREADS(256)
+) {
+ const uint3 tgpig = {tgpig_x, 0, 0};
+ const uint tid_x = tgpig.x * BLOCK_SIZE + lid;
+ const uint3 tid3 = {tid_x, 0, 0};
+ moe_scatter_buckets_impl<T>(
+ topk_ids,
+ topk_probs,
+ offsets,
+ block_bases,
+ block_alloc,
+ out_ids,
+ out_probs,
+ (device int*)nullptr,
+ t,
+ e,
+ k,
+ num_blocks,
+ num_tiles,
+ tid3,
+ lid,
+ tgpig,
+ sg_counts,
+ sg_base
+ );
+}
// Map variants
-#define DEFINE_MOE_SCATTER_BUCKETS_MAP_KERNEL(SUFFIX, ProbT) \
- [[max_total_threads_per_threadgroup(256)]] \
- kernel void moe_scatter_buckets_map_##SUFFIX( \
- device const int* topk_ids [[buffer(0)]], \
- device const ProbT* topk_probs [[buffer(1)]], \
- device const uint* offsets [[buffer(2)]], \
- device const uint* block_bases [[buffer(3)]], \
- device const uint* block_alloc [[buffer(4)]], \
- device int* out_ids [[buffer(5)]], \
- device ProbT* out_probs [[buffer(6)]], \
- constant uint& T [[buffer(7)]], \
- constant uint& E [[buffer(8)]], \
- constant uint& K [[buffer(9)]], \
- constant uint& num_blocks [[buffer(10)]], \
- constant uint& num_tiles [[buffer(11)]], \
- device int* tok2row [[buffer(12)]], \
- uint3 tid3 [[thread_position_in_grid]], \
- uint lid [[thread_index_in_threadgroup]], \
- uint3 tgpig [[threadgroup_position_in_grid]] \
- ) { \
- threadgroup _atomic<uint> sg_counts[NUM_SG * TILE_E]; \
- threadgroup uint sg_base[NUM_SG * TILE_E]; \
- moe_scatter_buckets_impl<ProbT>( \
- topk_ids, \
- topk_probs, \
- offsets, \
- block_bases, \
- block_alloc, \
- out_ids, \
- out_probs, \
- tok2row, \
- T, \
- E, \
- K, \
- num_blocks, \
- num_tiles, \
- tid3, \
- lid, \
- tgpig, \
- sg_counts, \
- sg_base \
- ); \
- }
-
-DEFINE_MOE_SCATTER_BUCKETS_MAP_KERNEL(f16, half)
-DEFINE_MOE_SCATTER_BUCKETS_MAP_KERNEL(f32, float)
-DEFINE_MOE_SCATTER_BUCKETS_MAP_KERNEL(bf16, bfloat)
+template <typename T>
+VARIANTS(T, float, half, bfloat)
+KERNEL(MoeScatterBucketsMap)(
+ device const int* topk_ids,
+ device const T* topk_probs,
+ device const uint* offsets,
+ device const uint* block_bases,
+ device const uint* block_alloc,
+ device int* out_ids,
+ device T* out_probs,
+ constant uint& t,
+ constant uint& e,
+ constant uint& k,
+ constant uint& num_blocks,
+ constant uint& num_tiles,
+ device int* tok2row,
+ threadgroup _atomic<uint> sg_counts[NUM_SG * TILE_E],
+ threadgroup uint sg_base[NUM_SG * TILE_E],
+ const uint tgpig_x GROUPS(num_blocks),
+ const uint lid THREADS(256)
+) {
+ const uint3 tgpig = {tgpig_x, 0, 0};
+ const uint tid_x = tgpig.x * BLOCK_SIZE + lid;
+ const uint3 tid3 = {tid_x, 0, 0};
+ moe_scatter_buckets_impl<T>(
+ topk_ids,
+ topk_probs,
+ offsets,
+ block_bases,
+ block_alloc,
+ out_ids,
+ out_probs,
+ tok2row,
+ t,
+ e,
+ k,
+ num_blocks,
+ num_tiles,
+ tid3,
+ lid,
+ tgpig,
+ sg_counts,
+ sg_base
+ );
+}
\ No newline at end of file
diff --git a/crates/uzu/tests/moe/moe_block_e2e_test.rs b/crates/uzu/tests/moe/moe_block_e2e_test.rs
index 1e3f7090b..7815565d8 100644
--- a/crates/uzu/tests/moe/moe_block_e2e_test.rs
+++ b/crates/uzu/tests/moe/moe_block_e2e_test.rs
@@ -4,12 +4,17 @@ use rand::{RngExt, SeedableRng, rngs::StdRng};
use uzu::{
DataType,
backends::{
- common::kernel::{MoeCountsOffsetsFusedKernel, MoeFinalizeKernel, MoeRouterTopKKernel},
+ common::kernel::{
+ MoeBlockBasesFromPartialsKernel, MoeCountsOffsetsFusedKernel, MoeFinalizeKernel, MoeRouterTopKKernel,
+ MoeScatterBucketsMapKernel,
+ },
metal::{
MetalContext,
kernel::{
- MoeBlockBasesArguments, MoeScatterKernels, MoeScatterWithMapArguments,
- dsl::{MoeCountsOffsetsFusedMetalKernel, MoeFinalizeMetalKernel, MoeRouterTopKMetalKernel},
+ dsl::{
+ MoeBlockBasesFromPartialsMetalKernel, MoeCountsOffsetsFusedMetalKernel, MoeFinalizeMetalKernel,
+ MoeRouterTopKMetalKernel, MoeScatterBucketsMapMetalKernel,
+ },
moe::{
MoeExpertsTwoPassArguments, MoeExpertsTwoPassPrefillBlock, MoeGatherArguments, MoeGatherKernels,
},
@@ -371,44 +376,40 @@ fn run_moe_parity_test_internal(
fused_kernel.encode(&topk_ids_buf, &offsets_buf, &sumk_buf, &partials_buf, t as u32, e as u32, k as u32, &encoder);
encoder.end_encoding();
- let scatter = MoeScatterKernels::new(&ctx).expect("scatter");
- scatter
- .encode_block_bases(
- &cb,
- MoeBlockBasesArguments {
- partials_buffer: &partials_buf,
- block_bases_buffer: &block_bases_buf,
- block_alloc_buffer: &block_alloc_buf,
- e,
- num_blocks,
- num_tiles,
- },
- )
- .expect("block bases");
-
- scatter
- .encode_scatter_with_map(
- &cb,
- MoeScatterWithMapArguments {
- base: uzu::backends::metal::kernel::MoeScatterArguments {
- topk_ids_buffer: &topk_ids_buf,
- topk_probs_buffer: &topk_probs_buf,
- offsets_buffer: &offsets_buf,
- block_bases_buffer: &block_bases_buf,
- block_alloc_buffer: &block_alloc_buf,
- out_ids_buffer: &bucketed_ids_buf,
- out_probs_buffer: &bucketed_probs_buf,
- t,
- e,
- k,
- num_blocks,
- num_tiles,
- },
- tok2row_buffer: &tok2row_buf,
- },
- DataType::BF16,
- )
- .expect("scatter");
+ let scatter_bases_encoder = cb.new_compute_command_encoder().expect("scatter_bases_encoder");
+ let scatter_bases_kernel = MoeBlockBasesFromPartialsMetalKernel::new(&ctx).expect("scatter bases kernel");
+ scatter_bases_kernel.encode(
+ &partials_buf,
+ &block_bases_buf,
+ &block_alloc_buf,
+ e as u32,
+ num_blocks as u32,
+ num_tiles as u32,
+ 0u32,
+ &scatter_bases_encoder,
+ );
+ scatter_bases_encoder.end_encoding();
+
+ let scatter_encoder = cb.new_compute_command_encoder().expect("scatter encoder");
+ let scatter_map_kernel = MoeScatterBucketsMapMetalKernel::new(&ctx, DataType::BF16)
+ .expect("Failed to create MoeScatterBucketsMapMetalKernel");
+ scatter_map_kernel.encode(
+ &topk_ids_buf,
+ &topk_probs_buf,
+ &offsets_buf,
+ &block_bases_buf,
+ &block_alloc_buf,
+ &bucketed_ids_buf,
+ &bucketed_probs_buf,
+ t as u32,
+ e as u32,
+ k as u32,
+ num_blocks as u32,
+ num_tiles as u32,
+ &tok2row_buf,
+ &scatter_encoder,
+ );
+ scatter_encoder.end_encoding();
let gather = MoeGatherKernels::new(&ctx).expect("gather");
gather.encode(
diff --git a/crates/uzu/tests/moe/moe_perf_test.rs b/crates/uzu/tests/moe/moe_perf_test.rs
index dd4f4d8f9..624b5ddb3 100644
--- a/crates/uzu/tests/moe/moe_perf_test.rs
+++ b/crates/uzu/tests/moe/moe_perf_test.rs
@@ -6,13 +6,16 @@ use rand::{RngExt, SeedableRng, rngs::StdRng};
use uzu::{
DataType,
backends::{
- common::kernel::{MoeCountsOffsetsFusedKernel, MoeFinalizeKernel, MoeRouterTopKKernel},
+ common::kernel::{
+ MoeBlockBasesFromPartialsKernel, MoeCountsOffsetsFusedKernel, MoeFinalizeKernel, MoeRouterTopKKernel,
+ MoeScatterBucketsMapKernel,
+ },
metal::kernel::{
- dsl::{MoeCountsOffsetsFusedMetalKernel, MoeFinalizeMetalKernel, MoeRouterTopKMetalKernel},
- moe::{
- MoeExpertsTwoPassArguments, MoeExpertsTwoPassDecodeBlock, MoeGatherArguments, MoeGatherKernels,
- MoeScatterKernels, MoeScatterWithMapArguments,
+ dsl::{
+ MoeBlockBasesFromPartialsMetalKernel, MoeCountsOffsetsFusedMetalKernel, MoeFinalizeMetalKernel,
+ MoeRouterTopKMetalKernel, MoeScatterBucketsMapMetalKernel,
},
+ moe::{MoeExpertsTwoPassArguments, MoeExpertsTwoPassDecodeBlock, MoeGatherArguments, MoeGatherKernels},
},
},
};
@@ -292,7 +295,10 @@ fn test_moe_pipeline_breakdown_decode() {
// Create kernel structs (use production-validated encoding logic)
let counts_offsets_kernel = MoeCountsOffsetsFusedMetalKernel::new(&ctx).expect("counts+offsets fused");
- let scatter_kernel = MoeScatterKernels::new(&ctx).expect("scatter");
+ let scatter_bases_kernel =
+ MoeBlockBasesFromPartialsMetalKernel::new(&ctx).expect("MoeBlockBasesFromPartialsMetalKernel");
+ let scatter_map_kernel =
+ MoeScatterBucketsMapMetalKernel::new(&ctx, DataType::BF16).expect("MoeScatterBucketsMapMetalKernel");
let gather_kernel = MoeGatherKernels::new(&ctx).expect("gather");
let experts_kernel = MoeExpertsTwoPassDecodeBlock::new(&ctx).expect("experts two-pass decode");
let finalize_kernel = MoeFinalizeMetalKernel::new(&ctx, DataType::BF16).expect("finalize");
@@ -340,42 +346,38 @@ fn test_moe_pipeline_breakdown_decode() {
let scatter_perf = time_kernel("Scatter", 2, 5, || {
let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");
- scatter_kernel
- .encode_block_bases(
- &cb,
- uzu::backends::metal::kernel::moe::MoeBlockBasesArguments {
- partials_buffer: &partials_buf,
- block_bases_buffer: &block_bases_buf,
- block_alloc_buffer: &block_alloc_buf,
- e,
- num_blocks,
- num_tiles,
- },
- )
- .expect("block bases");
- scatter_kernel
- .encode_scatter_with_map(
- &cb,
- MoeScatterWithMapArguments {
- base: uzu::backends::metal::kernel::moe::MoeScatterArguments {
- topk_ids_buffer: &topk_ids_buf,
- topk_probs_buffer: &topk_probs_buf,
- offsets_buffer: &offsets_buf,
- block_bases_buffer: &block_bases_buf,
- block_alloc_buffer: &block_alloc_buf,
- out_ids_buffer: &bucketed_ids_buf,
- out_probs_buffer: &bucketed_probs_buf,
- t,
- e,
- k,
- num_blocks,
- num_tiles,
- },
- tok2row_buffer: &tok2row_buf,
- },
- DataType::BF16,
- )
- .expect("scatter");
+ let scatter_bases_encoder = cb.new_compute_command_encoder().expect("scatter_bases_encoder");
+ scatter_bases_kernel.encode(
+ &partials_buf,
+ &block_bases_buf,
+ &block_alloc_buf,
+ e as u32,
+ num_blocks as u32,
+ num_tiles as u32,
+ 0u32,
+ &scatter_bases_encoder,
+ );
+ scatter_bases_encoder.end_encoding();
+
+ let scatter_map_encoder = cb.new_compute_command_encoder().expect("scatter_map_encoder");
+ scatter_map_kernel.encode(
+ &topk_ids_buf,
+ &topk_probs_buf,
+ &offsets_buf,
+ &block_bases_buf,
+ &block_alloc_buf,
+ &bucketed_ids_buf,
+ &bucketed_probs_buf,
+ t as u32,
+ e as u32,
+ k as u32,
+ num_blocks as u32,
+ num_tiles as u32,
+ &tok2row_buf,
+ &scatter_map_encoder,
+ );
+ scatter_map_encoder.end_encoding();
+
cb.commit();
cb.wait_until_completed();
});
diff --git a/crates/uzu/tests/moe/moe_scatter_test.rs b/crates/uzu/tests/moe/moe_scatter_test.rs
index 568d0606e..2a2449b60 100644
--- a/crates/uzu/tests/moe/moe_scatter_test.rs
+++ b/crates/uzu/tests/moe/moe_scatter_test.rs
@@ -6,10 +6,12 @@ use rand::{RngExt, SeedableRng, rngs::StdRng};
use uzu::{
DataType,
backends::{
- common::kernel::{MoeCountsOffsetsFusedKernel, MoeRouterTopKKernel},
- metal::kernel::{
- MoeBlockBasesArguments, MoeScatterArguments, MoeScatterKernels,
- dsl::{MoeCountsOffsetsFusedMetalKernel, MoeRouterTopKMetalKernel},
+ common::kernel::{
+ MoeBlockBasesFromPartialsKernel, MoeCountsOffsetsFusedKernel, MoeRouterTopKKernel, MoeScatterBucketsKernel,
+ },
+ metal::kernel::dsl::{
+ MoeBlockBasesFromPartialsMetalKernel, MoeCountsOffsetsFusedMetalKernel, MoeRouterTopKMetalKernel,
+ MoeScatterBucketsMetalKernel,
},
},
};
@@ -133,22 +135,26 @@ fn test_scatter_buckets_parity() {
let entries = num_blocks * num_tiles * 512usize;
let block_bases_buf = alloc_buffer::<u32>(&ctx, entries);
- let scatter = MoeScatterKernels::new(&ctx).expect("scatter kernels");
+ let scatter_bases_kernel = MoeBlockBasesFromPartialsMetalKernel::new(&ctx)
+ .expect("Failed to create MoeBlockBasesFromPartialsMetalKernel");
+ let scatter_kernel = MoeScatterBucketsMetalKernel::new(&ctx, DataType::BF16)
+ .expect("Failed to create MoeScatterBucketsMetalKernel");
let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");
let block_alloc_buf = alloc_buffer::<u32>(&ctx, entries);
- scatter
- .encode_block_bases(
- &cb,
- MoeBlockBasesArguments {
- partials_buffer: &partials_buf,
- block_bases_buffer: &block_bases_buf,
- block_alloc_buffer: &block_alloc_buf,
- e,
- num_blocks,
- num_tiles,
- },
- )
- .expect("encode bases");
+
+ let scatter_bases_encoder = cb.new_compute_command_encoder().expect("Failed to create scatter_bases_encoder");
+ scatter_bases_kernel.encode(
+ &partials_buf,
+ &block_bases_buf,
+ &block_alloc_buf,
+ e as u32,
+ num_blocks as u32,
+ num_tiles as u32,
+ 0u32,
+ &scatter_bases_encoder,
+ );
+ scatter_bases_encoder.end_encoding();
+
cb.commit();
cb.wait_until_completed();
@@ -157,26 +163,24 @@ fn test_scatter_buckets_parity() {
let out_probs_buf = alloc_buffer::<bf16>(&ctx, sumk);
let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");
- scatter
- .encode_scatter(
- &cb,
- MoeScatterArguments {
- topk_ids_buffer: &topk_ids_buf,
- topk_probs_buffer: &topk_probs_buf,
- offsets_buffer: &offsets_buf,
- block_bases_buffer: &block_bases_buf,
- block_alloc_buffer: &block_alloc_buf,
- out_ids_buffer: &out_ids_buf,
- out_probs_buffer: &out_probs_buf,
- t,
- e,
- k,
- num_blocks,
- num_tiles,
- },
- DataType::BF16,
- )
- .expect("encode scatter");
+ let scatter_encoder = cb.new_compute_command_encoder().expect("Failed to create scatter_encoder");
+ scatter_kernel.encode(
+ &topk_ids_buf,
+ &topk_probs_buf,
+ &offsets_buf,
+ &block_bases_buf,
+ &block_alloc_buf,
+ &out_ids_buf,
+ &out_probs_buf,
+ t as u32,
+ e as u32,
+ k as u32,
+ num_blocks as u32,
+ num_tiles as u32,
+ &scatter_encoder,
+ );
+ scatter_encoder.end_encoding();
+
cb.commit();
cb.wait_until_completed();