A high-performance inference engine for AI models
dslify-moe-scatter (#205)
Aleksandr Golokoz9a979ccdc6e0
diff --git a/crates/uzu/src/backends/metal/encodable_block/moe_block/moe_block.rs b/crates/uzu/src/backends/metal/encodable_block/moe_block/moe_block.rsindex add71978f..a5cbbca4a 100644--- a/crates/uzu/src/backends/metal/encodable_block/moe_block/moe_block.rs+++ b/crates/uzu/src/backends/metal/encodable_block/moe_block/moe_block.rs@@ -10,16 +10,21 @@ use crate::{Activation, DataType, LinearConfig, MixtureOfExpertsConfig, RoutingFunctionConfig,array::Array,backends::{- common::kernel::{MoeCountsOffsetsFusedKernel, MoeFinalizeKernel, MoeRouterTopKKernel},+ common::kernel::{+ MoeBlockBasesFromPartialsKernel, MoeCountsOffsetsFusedKernel, MoeFinalizeKernel, MoeRouterTopKKernel,+ MoeScatterBucketsMapKernel,+ },metal::{Metal, MetalContext,kernel::{MoeGatherKernels,- dsl::{MoeCountsOffsetsFusedMetalKernel, MoeFinalizeMetalKernel, MoeRouterTopKMetalKernel},+ dsl::{+ MoeBlockBasesFromPartialsMetalKernel, MoeCountsOffsetsFusedMetalKernel, MoeFinalizeMetalKernel,+ MoeRouterTopKMetalKernel, MoeScatterBucketsMapMetalKernel,+ },moe::{- MoeBlockBasesArguments, MoeExpertsTwoPassArguments, MoeExpertsTwoPassDecodeBlock,- MoeExpertsTwoPassPrefillBlock, MoeGatherArguments, MoeScatterArguments, MoeScatterKernels,- MoeScatterWithMapArguments,+ MoeExpertsTwoPassArguments, MoeExpertsTwoPassDecodeBlock, MoeExpertsTwoPassPrefillBlock,+ MoeGatherArguments,},},},@@ -41,7 +46,8 @@ pub struct MoeBlock {router_renorm: bool,router_topk_kernel: MoeRouterTopKMetalKernel,counts_offsets_kernel: MoeCountsOffsetsFusedMetalKernel,- scatter_kernels: MoeScatterKernels,+ scatter_bases_kernel: MoeBlockBasesFromPartialsMetalKernel,+ scatter_map_kernel: MoeScatterBucketsMapMetalKernel,gather_kernels: MoeGatherKernels,experts_two_pass_decode_kernel: MoeExpertsTwoPassDecodeBlock,experts_two_pass_prefill_kernel: MoeExpertsTwoPassPrefillBlock,@@ -109,9 +115,18 @@ impl MoeBlock {let counts_offsets_kernel = MoeCountsOffsetsFusedKernel::new(context).map_err(|e| {crate::backends::metal::MetalError::Generic(format!("Counts+offsets kernel error: {:?}", e))})?;- let scatter_kernels = MoeScatterKernels::new(context)- .map_err(|e| crate::backends::metal::MetalError::Generic(format!("Scatter kernels error: {:?}", e)))?;- let gather_kernel = MoeGatherKernels::new(context)++ let scatter_bases_kernel = MoeBlockBasesFromPartialsMetalKernel::new(context).map_err(|e| {+ crate::backends::metal::MetalError::Generic(format!(+ "MoeBlockBasesFromPartialsMetalKernel kernel error: {:?}",+ e+ ))+ })?;+ let scatter_map_kernel = MoeScatterBucketsMapMetalKernel::new(context, data_type).map_err(|e| {+ crate::backends::metal::MetalError::Generic(format!("MoeScatterBucketsMetalKernel kernel error: {:?}", e))+ })?;++ let gather_kernels = MoeGatherKernels::new(context).map_err(|e| crate::backends::metal::MetalError::Generic(format!("Gather kernel error: {:?}", e)))?;let experts_two_pass_decode_kernel = MoeExpertsTwoPassDecodeBlock::new(context).map_err(|e| {crate::backends::metal::MetalError::Generic(format!("Experts two-pass decode kernel error: {:?}", e))@@ -166,8 +181,9 @@ impl MoeBlock {router_renorm,router_topk_kernel,counts_offsets_kernel,- scatter_kernels,- gather_kernels: gather_kernel,+ scatter_bases_kernel,+ scatter_map_kernel,+ gather_kernels,experts_two_pass_decode_kernel,experts_two_pass_prefill_kernel,finalize_kernel,@@ -349,44 +365,41 @@ impl EncodableBlock<Metal> for MoeBlock {let num_blocks = ((suffix_length + 255) / 256).max(1);let num_tiles = ((e + 512 - 1) / 512).max(1);-- self.scatter_kernels- .encode_block_bases(- root,- MoeBlockBasesArguments {- partials_buffer: &partials_buf,- block_bases_buffer: &block_bases_buf,- block_alloc_buffer: &block_alloc_buf,- e,- num_blocks,- num_tiles,- },- )- .expect("MoE block bases failed");-- self.scatter_kernels- .encode_scatter_with_map(- root,- MoeScatterWithMapArguments {- base: MoeScatterArguments {- topk_ids_buffer: &topk_ids_buf,- topk_probs_buffer: &topk_probs_buf,- offsets_buffer: &offsets_buf,- block_bases_buffer: &block_bases_buf,- block_alloc_buffer: &block_alloc_buf,- out_ids_buffer: &bucketed_ids_buf,- out_probs_buffer: &bucketed_probs_buf,- t: suffix_length,- e,- k,- num_blocks,- num_tiles,- },- tok2row_buffer: &tok2row_buf,- },- self.data_type,- )- .expect("MoE scatter failed");+ let scatter_bases_encoder = command_buffer+ .new_compute_command_encoder()+ .expect("Failed to create compute command encoder for scatter bases kernel");+ self.scatter_bases_kernel.encode(+ &partials_buf,+ &block_bases_buf,+ &block_alloc_buf,+ e as u32,+ num_blocks as u32,+ num_tiles as u32,+ 0u32,+ &scatter_bases_encoder,+ );+ scatter_bases_encoder.end_encoding();++ let scatter_encoder = command_buffer+ .new_compute_command_encoder()+ .expect("Failed to create compute command encoder for scatter map kernel");+ self.scatter_map_kernel.encode(+ &topk_ids_buf,+ &topk_probs_buf,+ &offsets_buf,+ &block_bases_buf,+ &block_alloc_buf,+ &bucketed_ids_buf,+ &bucketed_probs_buf,+ suffix_length as u32,+ e as u32,+ k as u32,+ num_blocks as u32,+ num_tiles as u32,+ &tok2row_buf,+ &scatter_encoder,+ );+ scatter_encoder.end_encoding();self.gather_kernels.encode(root,diff --git a/crates/uzu/src/backends/metal/kernel/mod.rs b/crates/uzu/src/backends/metal/kernel/mod.rsindex 332685497..0323968b5 100644--- a/crates/uzu/src/backends/metal/kernel/mod.rs+++ b/crates/uzu/src/backends/metal/kernel/mod.rs@@ -9,8 +9,8 @@ pub mod moe;pub use matmul::{MatmulArguments, MatmulKernel};pub use moe::{- MoeBlockBasesArguments, MoeExpertsTwoPassArguments, MoeExpertsTwoPassDecodeBlock, MoeExpertsTwoPassPrefillBlock,- MoeGatherArguments, MoeGatherKernels, MoeScatterArguments, MoeScatterKernels, MoeScatterWithMapArguments,+ MoeExpertsTwoPassArguments, MoeExpertsTwoPassDecodeBlock, MoeExpertsTwoPassPrefillBlock, MoeGatherArguments,+ MoeGatherKernels,};impl MatmulKernels for dsl::MetalKernels {diff --git a/crates/uzu/src/backends/metal/kernel/moe/mod.rs b/crates/uzu/src/backends/metal/kernel/moe/mod.rsindex 15c18190f..beb45c942 100644--- a/crates/uzu/src/backends/metal/kernel/moe/mod.rs+++ b/crates/uzu/src/backends/metal/kernel/moe/mod.rs@@ -3,7 +3,6 @@ mod experts_single;mod experts_two_pass_decode;mod experts_two_pass_prefill;mod gather;-mod scatter;pub mod tiles_map;pub mod tiles_pass_a;@@ -12,7 +11,6 @@ pub use experts_single::{MoeExpertsSingleDecodeArguments, MoeExpertsSingleDecodepub use experts_two_pass_decode::MoeExpertsTwoPassDecodeBlock;pub use experts_two_pass_prefill::{MoeExpertsTwoPassArguments, MoeExpertsTwoPassPrefillBlock};pub use gather::{MoeGatherArguments, MoeGatherKernels};-pub use scatter::{MoeBlockBasesArguments, MoeScatterArguments, MoeScatterKernels, MoeScatterWithMapArguments};pub use tiles_pass_a::{MoePassARowMapArguments, MoePassATileBuildArguments, MoePassATileCountsArguments, MoePassATileDispatchArguments,MoePassATileKernels, MoePassATileScanArguments,diff --git a/crates/uzu/src/backends/metal/kernel/moe/scatter.rs b/crates/uzu/src/backends/metal/kernel/moe/scatter.rsdeleted file mode 100644index 1159b0654..000000000--- a/crates/uzu/src/backends/metal/kernel/moe/scatter.rs+++ /dev/null@@ -1,205 +0,0 @@-use metal::{- MTLBuffer, MTLCommandBuffer, MTLCommandEncoder, MTLComputeCommandEncoder, MTLComputePipelineState, MTLSize,-};-use objc2::{rc::Retained, runtime::ProtocolObject};--use crate::{- DataType,- backends::metal::{MetalContext, metal_extensions::ComputeEncoderSetValue},-};--// ---- Scatter Buckets Kernels ------#[derive(Debug, thiserror::Error)]-pub enum MoeScatterError {- #[error("Metal error: {0}")]- MetalError(#[from] crate::backends::metal::MetalError),-}--pub struct MoeScatterKernels {- pipeline_bases: Retained<ProtocolObject<dyn MTLComputePipelineState>>,- pipeline_scatter_f16: Retained<ProtocolObject<dyn MTLComputePipelineState>>,- pipeline_scatter_f32: Retained<ProtocolObject<dyn MTLComputePipelineState>>,- pipeline_scatter_bf16: Retained<ProtocolObject<dyn MTLComputePipelineState>>,- // map variants- pipeline_scatter_map_f16: Retained<ProtocolObject<dyn MTLComputePipelineState>>,- pipeline_scatter_map_f32: Retained<ProtocolObject<dyn MTLComputePipelineState>>,- pipeline_scatter_map_bf16: Retained<ProtocolObject<dyn MTLComputePipelineState>>,-}--#[derive(Debug)]-pub struct MoeBlockBasesArguments<'a> {- pub partials_buffer: &'a ProtocolObject<dyn MTLBuffer>, // [num_blocks * num_tiles * 512]- pub block_bases_buffer: &'a ProtocolObject<dyn MTLBuffer>, // same shape as partials- pub block_alloc_buffer: &'a ProtocolObject<dyn MTLBuffer>, // [num_blocks * num_tiles * 512]- pub e: usize,- pub num_blocks: usize,- pub num_tiles: usize,-}--#[derive(Debug)]-pub struct MoeScatterArguments<'a> {- pub topk_ids_buffer: &'a ProtocolObject<dyn MTLBuffer>,- pub topk_probs_buffer: &'a ProtocolObject<dyn MTLBuffer>,- pub offsets_buffer: &'a ProtocolObject<dyn MTLBuffer>,- pub block_bases_buffer: &'a ProtocolObject<dyn MTLBuffer>,- pub block_alloc_buffer: &'a ProtocolObject<dyn MTLBuffer>,- pub out_ids_buffer: &'a ProtocolObject<dyn MTLBuffer>,- pub out_probs_buffer: &'a ProtocolObject<dyn MTLBuffer>,- pub t: usize,- pub e: usize,- pub k: usize,- pub num_blocks: usize,- pub num_tiles: usize,-}--#[derive(Debug)]-pub struct MoeScatterWithMapArguments<'a> {- pub base: MoeScatterArguments<'a>,- pub tok2row_buffer: &'a ProtocolObject<dyn MTLBuffer>, // [T*K] int32, initialized to -1-}--impl MoeScatterKernels {- pub fn new(mtl_context: &MetalContext) -> Result<Self, MoeScatterError> {- let pipeline_bases = mtl_context.compute_pipeline_state("moe_block_bases_from_partials", None)?;- let pipeline_scatter_f16 = mtl_context.compute_pipeline_state("moe_scatter_buckets_f16", None)?;- let pipeline_scatter_f32 = mtl_context.compute_pipeline_state("moe_scatter_buckets_f32", None)?;- let pipeline_scatter_bf16 = mtl_context.compute_pipeline_state("moe_scatter_buckets_bf16", None)?;- let pipeline_scatter_map_f16 = mtl_context.compute_pipeline_state("moe_scatter_buckets_map_f16", None)?;- let pipeline_scatter_map_f32 = mtl_context.compute_pipeline_state("moe_scatter_buckets_map_f32", None)?;- let pipeline_scatter_map_bf16 = mtl_context.compute_pipeline_state("moe_scatter_buckets_map_bf16", None)?;-- Ok(Self {- pipeline_bases,- pipeline_scatter_f16,- pipeline_scatter_f32,- pipeline_scatter_bf16,- pipeline_scatter_map_f16,- pipeline_scatter_map_f32,- pipeline_scatter_map_bf16,- })- }-- pub fn encode_block_bases(- &self,- command_buffer: &Retained<ProtocolObject<dyn MTLCommandBuffer>>,- args: MoeBlockBasesArguments,- ) -> Result<(), MoeScatterError> {- let compute_encoder =- command_buffer.new_compute_command_encoder().expect("Failed to create compute command encoder");- compute_encoder.set_compute_pipeline_state(&self.pipeline_bases);- compute_encoder.set_buffer(Some(args.partials_buffer), 0, 0);- compute_encoder.set_buffer(Some(args.block_bases_buffer), 0, 1);- compute_encoder.set_buffer(Some(args.block_alloc_buffer), 0, 2);-- let e_u32 = args.e as u32;- let nb_u32 = args.num_blocks as u32;- let nt_u32 = args.num_tiles as u32;- let cap_u32: u32 = 0;- compute_encoder.set_value(&e_u32, 3);- compute_encoder.set_value(&nb_u32, 4);- compute_encoder.set_value(&nt_u32, 5);- compute_encoder.set_value(&cap_u32, 6);-- let total_entries = args.num_tiles * 512usize;- let threads_per_threadgroup = MTLSize::new(256, 1, 1);- let tg = MTLSize::new((total_entries + 255) / 256, 1, 1);- if total_entries > 0 {- compute_encoder.dispatch_threadgroups(tg, threads_per_threadgroup);- }- compute_encoder.end_encoding();- Ok(())- }-- pub fn encode_scatter(- &self,- command_buffer: &Retained<ProtocolObject<dyn MTLCommandBuffer>>,- args: MoeScatterArguments,- dtype: DataType,- ) -> Result<(), MoeScatterError> {- let compute_encoder =- command_buffer.new_compute_command_encoder().expect("Failed to create compute command encoder");- // Select pipeline based on dtype- match dtype {- DataType::F16 => {- compute_encoder.set_compute_pipeline_state(&self.pipeline_scatter_f16);- },- DataType::F32 => {- compute_encoder.set_compute_pipeline_state(&self.pipeline_scatter_f32);- },- DataType::BF16 => {- compute_encoder.set_compute_pipeline_state(&self.pipeline_scatter_bf16);- },- _ => panic!("Unsupported data type: {:?}", dtype),- }- compute_encoder.set_buffer(Some(args.topk_ids_buffer), 0, 0);- compute_encoder.set_buffer(Some(args.topk_probs_buffer), 0, 1);- compute_encoder.set_buffer(Some(args.offsets_buffer), 0, 2);- compute_encoder.set_buffer(Some(args.block_bases_buffer), 0, 3);- compute_encoder.set_buffer(Some(args.block_alloc_buffer), 0, 4);- compute_encoder.set_buffer(Some(args.out_ids_buffer), 0, 5);- compute_encoder.set_buffer(Some(args.out_probs_buffer), 0, 6);- let t_u32 = args.t as u32;- let e_u32 = args.e as u32;- let k_u32 = args.k as u32;- let nb_u32 = args.num_blocks as u32;- let nt_u32 = args.num_tiles as u32;- compute_encoder.set_value(&t_u32, 7);- compute_encoder.set_value(&e_u32, 8);- compute_encoder.set_value(&k_u32, 9);- compute_encoder.set_value(&nb_u32, 10);- compute_encoder.set_value(&nt_u32, 11);-- let threads_per_threadgroup = MTLSize::new(256, 1, 1);- let tg = MTLSize::new(args.num_blocks, 1, 1);- if args.num_blocks > 0 {- compute_encoder.dispatch_threadgroups(tg, threads_per_threadgroup);- }- compute_encoder.end_encoding();- Ok(())- }-- pub fn encode_scatter_with_map(- &self,- command_buffer: &Retained<ProtocolObject<dyn MTLCommandBuffer>>,- args: MoeScatterWithMapArguments,- dtype: DataType,- ) -> Result<(), MoeScatterError> {- let compute_encoder =- command_buffer.new_compute_command_encoder().expect("Failed to create compute command encoder");- let pipeline = match dtype {- DataType::F16 => &self.pipeline_scatter_map_f16,- DataType::F32 => &self.pipeline_scatter_map_f32,- DataType::BF16 => &self.pipeline_scatter_map_bf16,- _ => panic!("Unsupported data type: {:?}", dtype),- };- compute_encoder.set_compute_pipeline_state(pipeline);- let base = &args.base;- compute_encoder.set_buffer(Some(base.topk_ids_buffer), 0, 0);- compute_encoder.set_buffer(Some(base.topk_probs_buffer), 0, 1);- compute_encoder.set_buffer(Some(base.offsets_buffer), 0, 2);- compute_encoder.set_buffer(Some(base.block_bases_buffer), 0, 3);- compute_encoder.set_buffer(Some(base.block_alloc_buffer), 0, 4);- compute_encoder.set_buffer(Some(base.out_ids_buffer), 0, 5);- compute_encoder.set_buffer(Some(base.out_probs_buffer), 0, 6);- let t_u32 = base.t as u32;- let e_u32 = base.e as u32;- let k_u32 = base.k as u32;- let nb_u32 = base.num_blocks as u32;- let nt_u32 = base.num_tiles as u32;- compute_encoder.set_value(&t_u32, 7);- compute_encoder.set_value(&e_u32, 8);- compute_encoder.set_value(&k_u32, 9);- compute_encoder.set_value(&nb_u32, 10);- compute_encoder.set_value(&nt_u32, 11);- compute_encoder.set_buffer(Some(args.tok2row_buffer), 0, 12);-- let threads_per_threadgroup = MTLSize::new(256, 1, 1);- let tg = MTLSize::new(base.num_blocks, 1, 1);- if base.num_blocks > 0 {- compute_encoder.dispatch_threadgroups(tg, threads_per_threadgroup);- }- compute_encoder.end_encoding();- Ok(())- }-}diff --git a/crates/uzu/src/backends/metal/kernel/moe/scatter_buckets.metal b/crates/uzu/src/backends/metal/kernel/moe/scatter_buckets.metalindex cdb91e750..3248e0a30 100644--- a/crates/uzu/src/backends/metal/kernel/moe/scatter_buckets.metal+++ b/crates/uzu/src/backends/metal/kernel/moe/scatter_buckets.metal@@ -1,5 +1,6 @@#include <metal_stdlib>#include <metal_atomic>+#include "../definitions.metal"using namespace metal;#define BLOCK_SIZE 256@@ -7,52 +8,6 @@ using namespace metal;#define SIMD_WIDTH 32#define NUM_SG (BLOCK_SIZE / SIMD_WIDTH)-// 4A: Compute per-block base offsets from partials-// partials layout: index = (block_id * num_tiles + tile_id) * TILE_E + te-[[max_total_threads_per_threadgroup(256)]]-kernel void moe_block_bases_from_partials(- device const uint* partials [[buffer(0)]],- device uint* block_bases [[buffer(1)]],- device uint* block_alloc [[buffer(2)]],- constant uint& E [[buffer(3)]],- constant uint& num_blocks [[buffer(4)]],- constant uint& num_tiles [[buffer(5)]],- constant uint& capacity_per_expert [[buffer(6)]], // 0 => no clamp- uint gid [[thread_position_in_grid]]-) {- const uint entries = num_tiles * TILE_E;- if (gid >= entries)- return;- const uint tile_id = gid / TILE_E;- const uint te = gid % TILE_E;- const uint e = tile_id * TILE_E + te;- if (e >= E)- return;-- uint running = 0u;- const bool clamp = (capacity_per_expert > 0u);- for (uint b = 0; b < num_blocks; ++b) {- const uint idx = (b * num_tiles + tile_id) * TILE_E + te;- const uint p = partials[idx];- const uint alloc = clamp ? (running < capacity_per_expert- ? min(p, capacity_per_expert - running)- : 0u)- : p;- block_bases[idx] = running;- block_alloc[idx] = alloc;- running += alloc;- if (clamp && running >= capacity_per_expert) {- // Remainder allocate zero and base stays at capacity- for (uint bb = b + 1; bb < num_blocks; ++bb) {- const uint idx2 = (bb * num_tiles + tile_id) * TILE_E + te;- block_bases[idx2] = running;- block_alloc[idx2] = 0u;- }- break;- }- }-}-// 4B: Scatter into expert-major buckets using atomic-free device writestemplate <typename ProbT>inline void moe_scatter_buckets_impl(@@ -188,98 +143,136 @@ inline void moe_scatter_buckets_impl(}}-#define DEFINE_MOE_SCATTER_BUCKETS_KERNEL(SUFFIX, ProbT) \- [[max_total_threads_per_threadgroup(256)]] \- kernel void moe_scatter_buckets_##SUFFIX( \- device const int* topk_ids [[buffer(0)]], \- device const ProbT* topk_probs [[buffer(1)]], \- device const uint* offsets [[buffer(2)]], \- device const uint* block_bases [[buffer(3)]], \- device const uint* block_alloc [[buffer(4)]], \- device int* out_ids [[buffer(5)]], \- device ProbT* out_probs [[buffer(6)]], \- constant uint& T [[buffer(7)]], \- constant uint& E [[buffer(8)]], \- constant uint& K [[buffer(9)]], \- constant uint& num_blocks [[buffer(10)]], \- constant uint& num_tiles [[buffer(11)]], \- uint3 tid3 [[thread_position_in_grid]], \- uint lid [[thread_index_in_threadgroup]], \- uint3 tgpig [[threadgroup_position_in_grid]] \- ) { \- threadgroup _atomic<uint> sg_counts[NUM_SG * TILE_E]; \- threadgroup uint sg_base[NUM_SG * TILE_E]; \- moe_scatter_buckets_impl<ProbT>( \- topk_ids, \- topk_probs, \- offsets, \- block_bases, \- block_alloc, \- out_ids, \- out_probs, \- (device int*)nullptr, \- T, \- E, \- K, \- num_blocks, \- num_tiles, \- tid3, \- lid, \- tgpig, \- sg_counts, \- sg_base \- ); \+// 4A: Compute per-block base offsets from partials+// partials layout: index = (block_id * num_tiles + tile_id) * TILE_E + te+KERNEL(MoeBlockBasesFromPartials)(+ device const uint* partials,+ device uint* block_bases,+ device uint* block_alloc,+ constant uint& e_input,+ constant uint& num_blocks,+ constant uint& num_tiles,+ constant uint& capacity_per_expert, // 0 => no clamp+ const uint gid AXIS(num_tiles * 512, 256)+) {+ const uint tile_id = gid / TILE_E;+ const uint te = gid % TILE_E;+ const uint e = tile_id * TILE_E + te;+ if (e >= e_input)+ return;++ uint running = 0u;+ const bool clamp = (capacity_per_expert > 0u);+ for (uint b = 0; b < num_blocks; ++b) {+ const uint idx = (b * num_tiles + tile_id) * TILE_E + te;+ const uint p = partials[idx];+ const uint alloc = clamp ? (running < capacity_per_expert+ ? min(p, capacity_per_expert - running)+ : 0u)+ : p;+ block_bases[idx] = running;+ block_alloc[idx] = alloc;+ running += alloc;+ if (clamp && running >= capacity_per_expert) {+ // Remainder allocate zero and base stays at capacity+ for (uint bb = b + 1; bb < num_blocks; ++bb) {+ const uint idx2 = (bb * num_tiles + tile_id) * TILE_E + te;+ block_bases[idx2] = running;+ block_alloc[idx2] = 0u;+ }+ break;+ }}+}-DEFINE_MOE_SCATTER_BUCKETS_KERNEL(f16, half)-DEFINE_MOE_SCATTER_BUCKETS_KERNEL(f32, float)-DEFINE_MOE_SCATTER_BUCKETS_KERNEL(bf16, bfloat)+template <typename T>+VARIANTS(T, float, half, bfloat)+KERNEL(MoeScatterBuckets)(+ device const int* topk_ids,+ device const T* topk_probs,+ device const uint* offsets,+ device const uint* block_bases,+ device const uint* block_alloc,+ device int* out_ids,+ device T* out_probs,+ constant uint& t,+ constant uint& e,+ constant uint& k,+ constant uint& num_blocks,+ constant uint& num_tiles,+ threadgroup _atomic<uint> sg_counts[NUM_SG * TILE_E],+ threadgroup uint sg_base[NUM_SG * TILE_E],+ const uint tgpig_x GROUPS(num_blocks),+ const uint lid THREADS(256)+) {+ const uint3 tgpig = {tgpig_x, 0, 0};+ const uint tid_x = tgpig.x * BLOCK_SIZE + lid;+ const uint3 tid3 = {tid_x, 0, 0};+ moe_scatter_buckets_impl<T>(+ topk_ids,+ topk_probs,+ offsets,+ block_bases,+ block_alloc,+ out_ids,+ out_probs,+ (device int*)nullptr,+ t,+ e,+ k,+ num_blocks,+ num_tiles,+ tid3,+ lid,+ tgpig,+ sg_counts,+ sg_base+ );+}// Map variants-#define DEFINE_MOE_SCATTER_BUCKETS_MAP_KERNEL(SUFFIX, ProbT) \- [[max_total_threads_per_threadgroup(256)]] \- kernel void moe_scatter_buckets_map_##SUFFIX( \- device const int* topk_ids [[buffer(0)]], \- device const ProbT* topk_probs [[buffer(1)]], \- device const uint* offsets [[buffer(2)]], \- device const uint* block_bases [[buffer(3)]], \- device const uint* block_alloc [[buffer(4)]], \- device int* out_ids [[buffer(5)]], \- device ProbT* out_probs [[buffer(6)]], \- constant uint& T [[buffer(7)]], \- constant uint& E [[buffer(8)]], \- constant uint& K [[buffer(9)]], \- constant uint& num_blocks [[buffer(10)]], \- constant uint& num_tiles [[buffer(11)]], \- device int* tok2row [[buffer(12)]], \- uint3 tid3 [[thread_position_in_grid]], \- uint lid [[thread_index_in_threadgroup]], \- uint3 tgpig [[threadgroup_position_in_grid]] \- ) { \- threadgroup _atomic<uint> sg_counts[NUM_SG * TILE_E]; \- threadgroup uint sg_base[NUM_SG * TILE_E]; \- moe_scatter_buckets_impl<ProbT>( \- topk_ids, \- topk_probs, \- offsets, \- block_bases, \- block_alloc, \- out_ids, \- out_probs, \- tok2row, \- T, \- E, \- K, \- num_blocks, \- num_tiles, \- tid3, \- lid, \- tgpig, \- sg_counts, \- sg_base \- ); \- }--DEFINE_MOE_SCATTER_BUCKETS_MAP_KERNEL(f16, half)-DEFINE_MOE_SCATTER_BUCKETS_MAP_KERNEL(f32, float)-DEFINE_MOE_SCATTER_BUCKETS_MAP_KERNEL(bf16, bfloat)+template <typename T>+VARIANTS(T, float, half, bfloat)+KERNEL(MoeScatterBucketsMap)(+ device const int* topk_ids,+ device const T* topk_probs,+ device const uint* offsets,+ device const uint* block_bases,+ device const uint* block_alloc,+ device int* out_ids,+ device T* out_probs,+ constant uint& t,+ constant uint& e,+ constant uint& k,+ constant uint& num_blocks,+ constant uint& num_tiles,+ device int* tok2row,+ threadgroup _atomic<uint> sg_counts[NUM_SG * TILE_E],+ threadgroup uint sg_base[NUM_SG * TILE_E],+ const uint tgpig_x GROUPS(num_blocks),+ const uint lid THREADS(256)+) {+ const uint3 tgpig = {tgpig_x, 0, 0};+ const uint tid_x = tgpig.x * BLOCK_SIZE + lid;+ const uint3 tid3 = {tid_x, 0, 0};+ moe_scatter_buckets_impl<T>(+ topk_ids,+ topk_probs,+ offsets,+ block_bases,+ block_alloc,+ out_ids,+ out_probs,+ tok2row,+ t,+ e,+ k,+ num_blocks,+ num_tiles,+ tid3,+ lid,+ tgpig,+ sg_counts,+ sg_base+ );+}\ No newline at end of filediff --git a/crates/uzu/tests/moe/moe_block_e2e_test.rs b/crates/uzu/tests/moe/moe_block_e2e_test.rsindex 1e3f7090b..7815565d8 100644--- a/crates/uzu/tests/moe/moe_block_e2e_test.rs+++ b/crates/uzu/tests/moe/moe_block_e2e_test.rs@@ -4,12 +4,17 @@ use rand::{RngExt, SeedableRng, rngs::StdRng};use uzu::{DataType,backends::{- common::kernel::{MoeCountsOffsetsFusedKernel, MoeFinalizeKernel, MoeRouterTopKKernel},+ common::kernel::{+ MoeBlockBasesFromPartialsKernel, MoeCountsOffsetsFusedKernel, MoeFinalizeKernel, MoeRouterTopKKernel,+ MoeScatterBucketsMapKernel,+ },metal::{MetalContext,kernel::{- MoeBlockBasesArguments, MoeScatterKernels, MoeScatterWithMapArguments,- dsl::{MoeCountsOffsetsFusedMetalKernel, MoeFinalizeMetalKernel, MoeRouterTopKMetalKernel},+ dsl::{+ MoeBlockBasesFromPartialsMetalKernel, MoeCountsOffsetsFusedMetalKernel, MoeFinalizeMetalKernel,+ MoeRouterTopKMetalKernel, MoeScatterBucketsMapMetalKernel,+ },moe::{MoeExpertsTwoPassArguments, MoeExpertsTwoPassPrefillBlock, MoeGatherArguments, MoeGatherKernels,},@@ -371,44 +376,40 @@ fn run_moe_parity_test_internal(fused_kernel.encode(&topk_ids_buf, &offsets_buf, &sumk_buf, &partials_buf, t as u32, e as u32, k as u32, &encoder);encoder.end_encoding();- let scatter = MoeScatterKernels::new(&ctx).expect("scatter");- scatter- .encode_block_bases(- &cb,- MoeBlockBasesArguments {- partials_buffer: &partials_buf,- block_bases_buffer: &block_bases_buf,- block_alloc_buffer: &block_alloc_buf,- e,- num_blocks,- num_tiles,- },- )- .expect("block bases");-- scatter- .encode_scatter_with_map(- &cb,- MoeScatterWithMapArguments {- base: uzu::backends::metal::kernel::MoeScatterArguments {- topk_ids_buffer: &topk_ids_buf,- topk_probs_buffer: &topk_probs_buf,- offsets_buffer: &offsets_buf,- block_bases_buffer: &block_bases_buf,- block_alloc_buffer: &block_alloc_buf,- out_ids_buffer: &bucketed_ids_buf,- out_probs_buffer: &bucketed_probs_buf,- t,- e,- k,- num_blocks,- num_tiles,- },- tok2row_buffer: &tok2row_buf,- },- DataType::BF16,- )- .expect("scatter");+ let scatter_bases_encoder = cb.new_compute_command_encoder().expect("scatter_bases_encoder");+ let scatter_bases_kernel = MoeBlockBasesFromPartialsMetalKernel::new(&ctx).expect("scatter bases kernel");+ scatter_bases_kernel.encode(+ &partials_buf,+ &block_bases_buf,+ &block_alloc_buf,+ e as u32,+ num_blocks as u32,+ num_tiles as u32,+ 0u32,+ &scatter_bases_encoder,+ );+ scatter_bases_encoder.end_encoding();++ let scatter_encoder = cb.new_compute_command_encoder().expect("scatter encoder");+ let scatter_map_kernel = MoeScatterBucketsMapMetalKernel::new(&ctx, DataType::BF16)+ .expect("Failed to create MoeScatterBucketsMapMetalKernel");+ scatter_map_kernel.encode(+ &topk_ids_buf,+ &topk_probs_buf,+ &offsets_buf,+ &block_bases_buf,+ &block_alloc_buf,+ &bucketed_ids_buf,+ &bucketed_probs_buf,+ t as u32,+ e as u32,+ k as u32,+ num_blocks as u32,+ num_tiles as u32,+ &tok2row_buf,+ &scatter_encoder,+ );+ scatter_encoder.end_encoding();let gather = MoeGatherKernels::new(&ctx).expect("gather");gather.encode(diff --git a/crates/uzu/tests/moe/moe_perf_test.rs b/crates/uzu/tests/moe/moe_perf_test.rsindex dd4f4d8f9..624b5ddb3 100644--- a/crates/uzu/tests/moe/moe_perf_test.rs+++ b/crates/uzu/tests/moe/moe_perf_test.rs@@ -6,13 +6,16 @@ use rand::{RngExt, SeedableRng, rngs::StdRng};use uzu::{DataType,backends::{- common::kernel::{MoeCountsOffsetsFusedKernel, MoeFinalizeKernel, MoeRouterTopKKernel},+ common::kernel::{+ MoeBlockBasesFromPartialsKernel, MoeCountsOffsetsFusedKernel, MoeFinalizeKernel, MoeRouterTopKKernel,+ MoeScatterBucketsMapKernel,+ },metal::kernel::{- dsl::{MoeCountsOffsetsFusedMetalKernel, MoeFinalizeMetalKernel, MoeRouterTopKMetalKernel},- moe::{- MoeExpertsTwoPassArguments, MoeExpertsTwoPassDecodeBlock, MoeGatherArguments, MoeGatherKernels,- MoeScatterKernels, MoeScatterWithMapArguments,+ dsl::{+ MoeBlockBasesFromPartialsMetalKernel, MoeCountsOffsetsFusedMetalKernel, MoeFinalizeMetalKernel,+ MoeRouterTopKMetalKernel, MoeScatterBucketsMapMetalKernel,},+ moe::{MoeExpertsTwoPassArguments, MoeExpertsTwoPassDecodeBlock, MoeGatherArguments, MoeGatherKernels},},},};@@ -292,7 +295,10 @@ fn test_moe_pipeline_breakdown_decode() {// Create kernel structs (use production-validated encoding logic)let counts_offsets_kernel = MoeCountsOffsetsFusedMetalKernel::new(&ctx).expect("counts+offsets fused");- let scatter_kernel = MoeScatterKernels::new(&ctx).expect("scatter");+ let scatter_bases_kernel =+ MoeBlockBasesFromPartialsMetalKernel::new(&ctx).expect("MoeBlockBasesFromPartialsMetalKernel");+ let scatter_map_kernel =+ MoeScatterBucketsMapMetalKernel::new(&ctx, DataType::BF16).expect("MoeScatterBucketsMapMetalKernel");let gather_kernel = MoeGatherKernels::new(&ctx).expect("gather");let experts_kernel = MoeExpertsTwoPassDecodeBlock::new(&ctx).expect("experts two-pass decode");let finalize_kernel = MoeFinalizeMetalKernel::new(&ctx, DataType::BF16).expect("finalize");@@ -340,42 +346,38 @@ fn test_moe_pipeline_breakdown_decode() {let scatter_perf = time_kernel("Scatter", 2, 5, || {let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");- scatter_kernel- .encode_block_bases(- &cb,- uzu::backends::metal::kernel::moe::MoeBlockBasesArguments {- partials_buffer: &partials_buf,- block_bases_buffer: &block_bases_buf,- block_alloc_buffer: &block_alloc_buf,- e,- num_blocks,- num_tiles,- },- )- .expect("block bases");- scatter_kernel- .encode_scatter_with_map(- &cb,- MoeScatterWithMapArguments {- base: uzu::backends::metal::kernel::moe::MoeScatterArguments {- topk_ids_buffer: &topk_ids_buf,- topk_probs_buffer: &topk_probs_buf,- offsets_buffer: &offsets_buf,- block_bases_buffer: &block_bases_buf,- block_alloc_buffer: &block_alloc_buf,- out_ids_buffer: &bucketed_ids_buf,- out_probs_buffer: &bucketed_probs_buf,- t,- e,- k,- num_blocks,- num_tiles,- },- tok2row_buffer: &tok2row_buf,- },- DataType::BF16,- )- .expect("scatter");+ let scatter_bases_encoder = cb.new_compute_command_encoder().expect("scatter_bases_encoder");+ scatter_bases_kernel.encode(+ &partials_buf,+ &block_bases_buf,+ &block_alloc_buf,+ e as u32,+ num_blocks as u32,+ num_tiles as u32,+ 0u32,+ &scatter_bases_encoder,+ );+ scatter_bases_encoder.end_encoding();++ let scatter_map_encoder = cb.new_compute_command_encoder().expect("scatter_map_encoder");+ scatter_map_kernel.encode(+ &topk_ids_buf,+ &topk_probs_buf,+ &offsets_buf,+ &block_bases_buf,+ &block_alloc_buf,+ &bucketed_ids_buf,+ &bucketed_probs_buf,+ t as u32,+ e as u32,+ k as u32,+ num_blocks as u32,+ num_tiles as u32,+ &tok2row_buf,+ &scatter_map_encoder,+ );+ scatter_map_encoder.end_encoding();+cb.commit();cb.wait_until_completed();});diff --git a/crates/uzu/tests/moe/moe_scatter_test.rs b/crates/uzu/tests/moe/moe_scatter_test.rsindex 568d0606e..2a2449b60 100644--- a/crates/uzu/tests/moe/moe_scatter_test.rs+++ b/crates/uzu/tests/moe/moe_scatter_test.rs@@ -6,10 +6,12 @@ use rand::{RngExt, SeedableRng, rngs::StdRng};use uzu::{DataType,backends::{- common::kernel::{MoeCountsOffsetsFusedKernel, MoeRouterTopKKernel},- metal::kernel::{- MoeBlockBasesArguments, MoeScatterArguments, MoeScatterKernels,- dsl::{MoeCountsOffsetsFusedMetalKernel, MoeRouterTopKMetalKernel},+ common::kernel::{+ MoeBlockBasesFromPartialsKernel, MoeCountsOffsetsFusedKernel, MoeRouterTopKKernel, MoeScatterBucketsKernel,+ },+ metal::kernel::dsl::{+ MoeBlockBasesFromPartialsMetalKernel, MoeCountsOffsetsFusedMetalKernel, MoeRouterTopKMetalKernel,+ MoeScatterBucketsMetalKernel,},},};@@ -133,22 +135,26 @@ fn test_scatter_buckets_parity() {let entries = num_blocks * num_tiles * 512usize;let block_bases_buf = alloc_buffer::<u32>(&ctx, entries);- let scatter = MoeScatterKernels::new(&ctx).expect("scatter kernels");+ let scatter_bases_kernel = MoeBlockBasesFromPartialsMetalKernel::new(&ctx)+ .expect("Failed to create MoeBlockBasesFromPartialsMetalKernel");+ let scatter_kernel = MoeScatterBucketsMetalKernel::new(&ctx, DataType::BF16)+ .expect("Failed to create MoeScatterBucketsMetalKernel");let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");let block_alloc_buf = alloc_buffer::<u32>(&ctx, entries);- scatter- .encode_block_bases(- &cb,- MoeBlockBasesArguments {- partials_buffer: &partials_buf,- block_bases_buffer: &block_bases_buf,- block_alloc_buffer: &block_alloc_buf,- e,- num_blocks,- num_tiles,- },- )- .expect("encode bases");++ let scatter_bases_encoder = cb.new_compute_command_encoder().expect("Failed to create scatter_bases_encoder");+ scatter_bases_kernel.encode(+ &partials_buf,+ &block_bases_buf,+ &block_alloc_buf,+ e as u32,+ num_blocks as u32,+ num_tiles as u32,+ 0u32,+ &scatter_bases_encoder,+ );+ scatter_bases_encoder.end_encoding();+cb.commit();cb.wait_until_completed();@@ -157,26 +163,24 @@ fn test_scatter_buckets_parity() {let out_probs_buf = alloc_buffer::<bf16>(&ctx, sumk);let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");- scatter- .encode_scatter(- &cb,- MoeScatterArguments {- topk_ids_buffer: &topk_ids_buf,- topk_probs_buffer: &topk_probs_buf,- offsets_buffer: &offsets_buf,- block_bases_buffer: &block_bases_buf,- block_alloc_buffer: &block_alloc_buf,- out_ids_buffer: &out_ids_buf,- out_probs_buffer: &out_probs_buf,- t,- e,- k,- num_blocks,- num_tiles,- },- DataType::BF16,- )- .expect("encode scatter");+ let scatter_encoder = cb.new_compute_command_encoder().expect("Failed to create scatter_encoder");+ scatter_kernel.encode(+ &topk_ids_buf,+ &topk_probs_buf,+ &offsets_buf,+ &block_bases_buf,+ &block_alloc_buf,+ &out_ids_buf,+ &out_probs_buf,+ t as u32,+ e as u32,+ k as u32,+ num_blocks as u32,+ num_tiles as u32,+ &scatter_encoder,+ );+ scatter_encoder.end_encoding();+cb.commit();cb.wait_until_completed();