A high-performance inference engine for AI models
Moe improvements
LuckyIYIeab58be42efd
diff --git a/crates/uzu/src/backends/metal/kernel/mlp/mod.rs b/crates/uzu/src/backends/metal/kernel/mlp/mod.rsindex 4f58d1d46..1d4e3e541 100644--- a/crates/uzu/src/backends/metal/kernel/mlp/mod.rs+++ b/crates/uzu/src/backends/metal/kernel/mlp/mod.rs@@ -1,5 +1,6 @@use metal::{- Buffer as MTLBuffer, ComputeCommandEncoderRef, ComputePipelineState, MTLSize,+ Buffer as MTLBuffer, ComputeCommandEncoderRef, ComputePipelineState,+ MTLSize,};use mpsgraph::CommandBuffer as MPSCommandBuffer;diff --git a/crates/uzu/src/backends/metal/kernel/moe/encodable.rs b/crates/uzu/src/backends/metal/kernel/moe/encodable.rsindex f58a406da..ff59b0e77 100644--- a/crates/uzu/src/backends/metal/kernel/moe/encodable.rs+++ b/crates/uzu/src/backends/metal/kernel/moe/encodable.rs@@ -1,13 +1,20 @@+//! MoE Block Encodable with path selection:+//! - suffix == 1: Single-token fused decode (no scatter/gather)+//! - suffix <= 32: Indirect decode path (GEMV-based)+//! - suffix > 32: Indirect prefill path (GEMM/MMA-based)+use std::{cell::RefCell, rc::Rc};+use metal::CommandBufferRef;use mpsgraph::CommandBuffer as MPSCommandBuffer;use super::{MoeCountsOffsetsFusedArguments, MoeCountsOffsetsFusedKernel,+ MoeExpertsSingleDecodeArguments, MoeExpertsSingleDecodeKernel,MoeExpertsTwoPassArguments, MoeExpertsTwoPassDecodeKernel,- MoeExpertsTwoPassPrefillKernel, MoeFinalizeKernel, MoeGatherArguments,- MoeGatherKernel, MoeRouterTopKArguments, MoeRouterTopKKernel,- MoeScatterKernels, MoeScatterWithMapArguments,+ MoeExpertsTwoPassPrefillKernel, MoeFinalizeArguments, MoeFinalizeKernel,+ MoeGatherArguments, MoeGatherKernel, MoeRouterTopKArguments,+ MoeRouterTopKKernel, MoeScatterKernels, MoeScatterWithMapArguments,};use crate::{DataType,@@ -25,6 +32,9 @@ use crate::{parameters::ParameterTree,};+/// Threshold for decode vs prefill path selection+const INDIRECT_DECODE_THRESHOLD: usize = 32;+enum RouterBlock {Metal {weights_buf: metal::Buffer,@@ -40,17 +50,51 @@ pub struct SharedMoeWeights {pub down_biases_buf: Rc<metal::Buffer>,}+/// Buffers used by MoE encoding paths+struct MoeBuffers {+ main: metal::Buffer,+ topk_ids: metal::Buffer,+ topk_probs: metal::Buffer,+ offsets: metal::Buffer,+ sumk: metal::Buffer,+ bucketed_ids: metal::Buffer,+ bucketed_probs: metal::Buffer,+ x_perm: metal::Buffer,+ tok2row: metal::Buffer,+ y_partial: metal::Buffer,+ tile_counts: metal::Buffer,+ tile_offsets: metal::Buffer,+ tile_map: metal::Buffer,+ total_tiles: metal::Buffer,+ dispatch_args: metal::Buffer,+ partials: metal::Buffer,+ block_bases: metal::Buffer,+ block_alloc: metal::Buffer,+ hidden: metal::Buffer,+ row_expert_map: metal::Buffer,+}+pub struct MoeBlockEncodable {+ // Routerrouter: RouterBlock,router_data_type: KernelDataType,router_renorm: bool,router_topk_kernel: MoeRouterTopKKernel,++ // Indirect path shared componentscounts_offsets_kernel: MoeCountsOffsetsFusedKernel,scatter_kernels: MoeScatterKernels,gather_kernel: MoeGatherKernel,+ finalize_kernel: MoeFinalizeKernel,++ // Indirect path variantsexperts_two_pass_decode_kernel: MoeExpertsTwoPassDecodeKernel,experts_two_pass_prefill_kernel: MoeExpertsTwoPassPrefillKernel,- finalize_kernel: MoeFinalizeKernel,++ // Single-token decode path+ single_decode_kernel: MoeExpertsSingleDecodeKernel,++ // Config and weightsmoe_config: MixtureOfExpertsConfig,model_dim: usize,hidden_dim: usize,@@ -133,6 +177,7 @@ impl MoeBlockEncodable {},};+ // Initialize kernelslet router_topk_kernel =MoeRouterTopKKernel::new(context).map_err(|e| {crate::backends::metal::MTLError::Generic(format!(@@ -173,6 +218,13 @@ impl MoeBlockEncodable {e))})?;+ let single_decode_kernel = MoeExpertsSingleDecodeKernel::new(context)+ .map_err(|e| {+ crate::backends::metal::MTLError::Generic(format!(+ "Simple decode fused kernel error: {:?}",+ e+ ))+ })?;let finalize_kernel = MoeFinalizeKernel::new(context).map_err(|e| {crate::backends::metal::MTLError::Generic(format!("Finalize kernel error: {:?}",@@ -180,6 +232,7 @@ impl MoeBlockEncodable {))})?;+ // Load expert weightslet experts_tree = parameter_tree.subtree("experts").map_err(|e| {crate::backends::metal::MTLError::Generic(format!("experts subtree error: {:?}",@@ -272,6 +325,7 @@ impl MoeBlockEncodable {gather_kernel,experts_two_pass_decode_kernel,experts_two_pass_prefill_kernel,+ single_decode_kernel,finalize_kernel,moe_config: moe_config.clone(),model_dim,@@ -283,167 +337,47 @@ impl MoeBlockEncodable {fn gating_code_from_activation(activation: &Activation) -> u32 {match activation {- Activation::Gelu => 3,+ Activation::Gelu => 3, // GEGLUActivation::SiLU {..- } => 2,+ } => 2, // SwiGLUActivation::Identity => {panic!("Identity activation is not supported for MoE kernels")},}}-}--impl EncodableWithState for MoeBlockEncodable {- fn encode(- &self,- state: &mut ForwardPassState,- command_buffer: &MPSCommandBuffer,- parameters: &EncodingParameters,- ) {- let suffix_length = state.active_suffix_length();- let arrays = state.arrays(&[- ArrayId::Main,- ArrayId::MoeTopkIds,- ArrayId::MoeTopkProbs,- ArrayId::MoeOffsets,- ArrayId::MoeSumK,- ArrayId::MoeBucketedTokenIds,- ArrayId::MoeBucketedProbs,- ArrayId::MoeXPerm,- ArrayId::MoeTok2Row,- ArrayId::MoeYPartial,- ArrayId::MoeTileCounts,- ArrayId::MoeTileOffsets,- ArrayId::MoeTileMap,- ArrayId::MoeTotalTiles,- ArrayId::MoeDispatchArgs,- ArrayId::MoeScatterPartials,- ArrayId::MoeScatterBlockBases,- ArrayId::MoeBlockAlloc,- ArrayId::MoeHidden,- ArrayId::MoeTwoPassRowExpertMap,- ]);-- let clone_buffer = |array: &RefCell<MetalArray>| -> metal::Buffer {- let mut borrow = array.borrow_mut();- let buffer = unsafe { borrow.mtl_buffer().clone() };- buffer- };- let mut array_iter = arrays.iter();- let main_buf = clone_buffer(array_iter.next().unwrap());- let topk_ids_buf = clone_buffer(array_iter.next().unwrap());- let topk_probs_buf = clone_buffer(array_iter.next().unwrap());- let offsets_buf = clone_buffer(array_iter.next().unwrap());- let sumk_buf = clone_buffer(array_iter.next().unwrap());- let bucketed_ids_buf = clone_buffer(array_iter.next().unwrap());- let bucketed_probs_buf = clone_buffer(array_iter.next().unwrap());- let x_perm_buf = clone_buffer(array_iter.next().unwrap());- let tok2row_buf = clone_buffer(array_iter.next().unwrap());- let y_partial_buf = clone_buffer(array_iter.next().unwrap());- let tile_counts_buf = clone_buffer(array_iter.next().unwrap());- let tile_offsets_buf = clone_buffer(array_iter.next().unwrap());- let tile_map_buf = clone_buffer(array_iter.next().unwrap());- let total_tiles_buf = clone_buffer(array_iter.next().unwrap());- let dispatch_args_buf = clone_buffer(array_iter.next().unwrap());- let partials_buf = clone_buffer(array_iter.next().unwrap());- let block_bases_buf = clone_buffer(array_iter.next().unwrap());- let block_alloc_buf = clone_buffer(array_iter.next().unwrap());- let hidden_buf = clone_buffer(array_iter.next().unwrap());- let row_expert_map_buf = clone_buffer(array_iter.next().unwrap());- debug_assert!(array_iter.next().is_none());-- let e = self.moe_config.mixture_size;- let k = self.moe_config.num_experts_per_token;-- let mtl_command_buffer =- command_buffer.root_command_buffer().to_owned();- let root = &*mtl_command_buffer;- let k_tile = 128;-- // Clear internal MoE buffers- let dtype_size = match self.data_type {+ fn dtype_size(&self) -> usize {+ match self.data_type {KernelDataType::BFloat16 | KernelDataType::Float16 => 2,KernelDataType::Float32 => 4,- };-- let blit_encoder = root.new_blit_command_encoder();-- if suffix_length > 0 && k > 0 {- let entries = suffix_length * k;- let topk_bytes = entries * std::mem::size_of::<u32>();- let tok2row_bytes = entries * std::mem::size_of::<i32>();-- // Clear topk_ids and tok2row buffers- if topk_bytes > 0 {- blit_encoder.fill_buffer(- &topk_ids_buf,- metal::NSRange::new(0, topk_bytes as u64),- 0xFF,- );- }- if tok2row_bytes > 0 {- blit_encoder.fill_buffer(- &tok2row_buf,- metal::NSRange::new(0, tok2row_bytes as u64),- 0xFF,- );- }-- // Clear hidden buffer- let hidden_bytes =- (suffix_length * k * self.hidden_dim * dtype_size) as u64;- if hidden_bytes > 0 {- blit_encoder.fill_buffer(- &hidden_buf,- metal::NSRange::new(0, hidden_bytes),- 0,- );- }-- // Clear y_partial buffer- let y_partial_bytes =- (suffix_length * k * self.model_dim * dtype_size) as u64;- if y_partial_bytes > 0 {- blit_encoder.fill_buffer(- &y_partial_buf,- metal::NSRange::new(0, y_partial_bytes),- 0,- );- }-- // Clear x_perm buffer- let x_perm_bytes =- (suffix_length * k * self.model_dim * dtype_size) as u64;- if x_perm_bytes > 0 {- blit_encoder.fill_buffer(- &x_perm_buf,- metal::NSRange::new(0, x_perm_bytes),- 0,- );- }}+ }- blit_encoder.end_encoding();+ fn encode_router(+ &self,+ root: &CommandBufferRef,+ buffers: &MoeBuffers,+ suffix_length: usize,+ ) {+ let e = self.moe_config.mixture_size;+ let k = self.moe_config.num_experts_per_token;- // Use fused Router+TopK kernel for non-quantized routersmatch &self.router {RouterBlock::Metal {weights_buf,biases_buf,} => {- // Use the fused router+topk kernelself.router_topk_kernel.encode(root,self.router_data_type,MoeRouterTopKArguments {- input_buffer: &main_buf,+ input_buffer: &buffers.main,weight_buffer: weights_buf,bias_buffer: biases_buf,- topk_ids_buffer: &topk_ids_buf,- topk_probs_buffer: &topk_probs_buf,+ topk_ids_buffer: &buffers.topk_ids,+ topk_probs_buffer: &buffers.topk_probs,t: suffix_length,d_model: self.model_dim,e,@@ -451,18 +385,29 @@ impl EncodableWithState for MoeBlockEncodable {renorm: self.router_renorm,},)- .expect("MoE fused router+topk failed");+ .expect("MoE router+topk failed");},}+ }+ fn encode_indirect_setup(+ &self,+ root: &CommandBufferRef,+ buffers: &MoeBuffers,+ suffix_length: usize,+ ) {+ let e = self.moe_config.mixture_size;+ let k = self.moe_config.num_experts_per_token;++ // Counts + offsetsself.counts_offsets_kernel.encode(root,MoeCountsOffsetsFusedArguments {- topk_ids_buffer: &topk_ids_buf,- offsets_buffer: &offsets_buf,- sum_k_buffer: &sumk_buf,- partials_buffer: &partials_buf,+ topk_ids_buffer: &buffers.topk_ids,+ offsets_buffer: &buffers.offsets,+ sum_k_buffer: &buffers.sumk,+ partials_buffer: &buffers.partials,t: suffix_length,e,k,@@ -473,13 +418,14 @@ impl EncodableWithState for MoeBlockEncodable {let num_blocks = ((suffix_length + 255) / 256).max(1);let num_tiles = ((e + 512 - 1) / 512).max(1);+ // Block basesself.scatter_kernels.encode_block_bases(root,super::MoeBlockBasesArguments {- partials_buffer: &partials_buf,- block_bases_buffer: &block_bases_buf,- block_alloc_buffer: &block_alloc_buf,+ partials_buffer: &buffers.partials,+ block_bases_buffer: &buffers.block_bases,+ block_alloc_buffer: &buffers.block_alloc,e,num_blocks,num_tiles,@@ -487,51 +433,171 @@ impl EncodableWithState for MoeBlockEncodable {).expect("MoE block bases failed");+ // Scatter with tok2row mapself.scatter_kernels.encode_scatter_with_map(root,MoeScatterWithMapArguments {base: super::MoeScatterArguments {- topk_ids_buffer: &topk_ids_buf,- topk_probs_buffer: &topk_probs_buf,- offsets_buffer: &offsets_buf,- block_bases_buffer: &block_bases_buf,- block_alloc_buffer: &block_alloc_buf,- out_ids_buffer: &bucketed_ids_buf,- out_probs_buffer: &bucketed_probs_buf,+ topk_ids_buffer: &buffers.topk_ids,+ topk_probs_buffer: &buffers.topk_probs,+ offsets_buffer: &buffers.offsets,+ block_bases_buffer: &buffers.block_bases,+ block_alloc_buffer: &buffers.block_alloc,+ out_ids_buffer: &buffers.bucketed_ids,+ out_probs_buffer: &buffers.bucketed_probs,t: suffix_length,e,k,num_blocks,num_tiles,},- tok2row_buffer: &tok2row_buf,+ tok2row_buffer: &buffers.tok2row,},self.data_type,).expect("MoE scatter failed");+ // Gatherself.gather_kernel.encode(root,self.data_type,MoeGatherArguments {- x_buffer: &main_buf,- bucketed_ids_buffer: &bucketed_ids_buf,- x_perm_buffer: &x_perm_buf,- sumk_buffer: &sumk_buf,+ x_buffer: &buffers.main,+ bucketed_ids_buffer: &buffers.bucketed_ids,+ x_perm_buffer: &buffers.x_perm,+ sumk_buffer: &buffers.sumk,t: suffix_length,k,d_model: self.model_dim,},).expect("MoE gather failed");+ }++ fn clear_indirect_buffers(+ &self,+ root: &CommandBufferRef,+ buffers: &MoeBuffers,+ suffix_length: usize,+ ) {+ let k = self.moe_config.num_experts_per_token;+ if k == 0 {+ return;+ }++ let blit_encoder = root.new_blit_command_encoder();+ let entries = suffix_length * k;+ let topk_bytes = entries * std::mem::size_of::<u32>();+ let tok2row_bytes = entries * std::mem::size_of::<i32>();+ let dtype_size = self.dtype_size();++ if topk_bytes > 0 {+ blit_encoder.fill_buffer(+ &buffers.topk_ids,+ metal::NSRange::new(0, topk_bytes as u64),+ 0xFF,+ );+ }+ if tok2row_bytes > 0 {+ blit_encoder.fill_buffer(+ &buffers.tok2row,+ metal::NSRange::new(0, tok2row_bytes as u64),+ 0xFF,+ );+ }++ let hidden_bytes =+ (suffix_length * k * self.hidden_dim * dtype_size) as u64;+ if hidden_bytes > 0 {+ blit_encoder.fill_buffer(+ &buffers.hidden,+ metal::NSRange::new(0, hidden_bytes),+ 0,+ );+ }++ let y_partial_bytes =+ (suffix_length * k * self.model_dim * dtype_size) as u64;+ if y_partial_bytes > 0 {+ blit_encoder.fill_buffer(+ &buffers.y_partial,+ metal::NSRange::new(0, y_partial_bytes),+ 0,+ );+ }++ let x_perm_bytes =+ (suffix_length * k * self.model_dim * dtype_size) as u64;+ if x_perm_bytes > 0 {+ blit_encoder.fill_buffer(+ &buffers.x_perm,+ metal::NSRange::new(0, x_perm_bytes),+ 0,+ );+ }++ blit_encoder.end_encoding();+ }++ fn encode_single_token_path(+ &self,+ root: &CommandBufferRef,+ buffers: &MoeBuffers,+ ) {+ let k = self.moe_config.num_experts_per_token;+ let gating_code = Self::gating_code_from_activation(+ &self.moe_config.expert_config.activation,+ );+ let silu_alpha = self.moe_config.expert_config.activation.alpha();+ let gate_clip_min = self.moe_config.expert_config.gate_clipping[0]+ .unwrap_or(f32::NEG_INFINITY);+ let gate_clip_max = self.moe_config.expert_config.gate_clipping[1]+ .unwrap_or(f32::INFINITY);+ let up_clip_min = self.moe_config.expert_config.up_clipping[0];+ let up_clip_max = self.moe_config.expert_config.up_clipping[1];++ self.single_decode_kernel+ .encode(+ root,+ MoeExpertsSingleDecodeArguments {+ x: &buffers.main,+ topk_ids: &buffers.topk_ids,+ topk_probs: &buffers.topk_probs,+ w13_all: &self.shared_weights.w13_buf,+ w2_all: &self.shared_weights.w2_buf,+ up_biases: &self.shared_weights.up_biases_buf,+ down_biases: &self.shared_weights.down_biases_buf,+ hidden: &buffers.hidden,+ y: &buffers.main,+ d_model: self.model_dim,+ d_ff: self.hidden_dim,+ k,+ gating_code,+ silu_alpha,+ gate_clip_min,+ gate_clip_max,+ up_clip_min,+ up_clip_max,+ data_type: self.data_type,+ },+ )+ .expect("MoE single-token fused decode failed");+ }+ fn encode_indirect_decode_path(+ &self,+ root: &CommandBufferRef,+ buffers: &MoeBuffers,+ suffix_length: usize,+ ) {+ let e = self.moe_config.mixture_size;+ let k = self.moe_config.num_experts_per_token;let gating_code = Self::gating_code_from_activation(&self.moe_config.expert_config.activation,);- // Compute clipping values and alpha for expert kernelslet gate_clip_min = self.moe_config.expert_config.gate_clipping[0].unwrap_or(f32::NEG_INFINITY);let gate_clip_max = self.moe_config.expert_config.gate_clipping[1]@@ -540,90 +606,124 @@ impl EncodableWithState for MoeBlockEncodable {let up_clip_max = self.moe_config.expert_config.up_clipping[1];let silu_alpha = self.moe_config.expert_config.activation.alpha();- if suffix_length == 1 {- let total_rows = suffix_length * k;- let num_tiles_k = ((self.hidden_dim + k_tile - 1) / k_tile) as u32;-- self.experts_two_pass_decode_kernel- .encode(- root,- MoeExpertsTwoPassArguments {- x_perm_buffer: &x_perm_buf,- expert_offsets: &offsets_buf,- row_expert_map: &row_expert_map_buf,- hidden_buffer: &hidden_buf,- output_buffer: &y_partial_buf,- w13_all: &self.shared_weights.w13_buf,- w2_all: &self.shared_weights.w2_buf,- up_biases: &self.shared_weights.up_biases_buf,- down_biases: &self.shared_weights.down_biases_buf,- tile_counts: &tile_counts_buf,- tile_offsets: &tile_offsets_buf,- tile_map: &tile_map_buf,- total_tiles: &total_tiles_buf,- dispatch_args: &dispatch_args_buf,- total_rows,- d_model: self.model_dim,- d_ff: self.hidden_dim,- e,- num_tiles_k,- gating_code,- gate_clip_min,- gate_clip_max,- up_clip_min,- up_clip_max,- silu_alpha,- data_type: self.data_type,- },- )- .expect("MoE experts two-pass failed");- } else {- let total_rows = suffix_length * k;- let num_tiles_k = ((self.hidden_dim + k_tile - 1) / k_tile) as u32;-- self.experts_two_pass_prefill_kernel- .encode(- root,- MoeExpertsTwoPassArguments {- x_perm_buffer: &x_perm_buf,- expert_offsets: &offsets_buf,- row_expert_map: &row_expert_map_buf,- hidden_buffer: &hidden_buf,- output_buffer: &y_partial_buf,- w13_all: &self.shared_weights.w13_buf,- w2_all: &self.shared_weights.w2_buf,- up_biases: &self.shared_weights.up_biases_buf,- down_biases: &self.shared_weights.down_biases_buf,- tile_counts: &tile_counts_buf,- tile_offsets: &tile_offsets_buf,- tile_map: &tile_map_buf,- total_tiles: &total_tiles_buf,- dispatch_args: &dispatch_args_buf,- total_rows,- d_model: self.model_dim,- d_ff: self.hidden_dim,- e,- num_tiles_k,- gating_code,- gate_clip_min,- gate_clip_max,- up_clip_min,- up_clip_max,- silu_alpha,- data_type: self.data_type,- },- )- .expect("MoE experts two-pass prefill failed");- }+ let total_rows = suffix_length * k;+ const K_TILE: usize = 64;+ let num_tiles_k = ((self.hidden_dim + K_TILE - 1) / K_TILE) as u32;++ self.experts_two_pass_decode_kernel+ .encode(+ root,+ MoeExpertsTwoPassArguments {+ x_perm_buffer: &buffers.x_perm,+ expert_offsets: &buffers.offsets,+ row_expert_map: &buffers.row_expert_map,+ hidden_buffer: &buffers.hidden,+ output_buffer: &buffers.y_partial,+ w13_all: &self.shared_weights.w13_buf,+ w2_all: &self.shared_weights.w2_buf,+ up_biases: &self.shared_weights.up_biases_buf,+ down_biases: &self.shared_weights.down_biases_buf,+ tile_counts: &buffers.tile_counts,+ tile_offsets: &buffers.tile_offsets,+ tile_map: &buffers.tile_map,+ total_tiles: &buffers.total_tiles,+ dispatch_args: &buffers.dispatch_args,+ total_rows,+ d_model: self.model_dim,+ d_ff: self.hidden_dim,+ e,+ num_tiles_k,+ gating_code,+ gate_clip_min,+ gate_clip_max,+ up_clip_min,+ up_clip_max,+ silu_alpha,+ data_type: self.data_type,+ },+ )+ .expect("MoE indirect decode failed");++ self.encode_finalize(root, buffers, suffix_length);+ }++ fn encode_indirect_prefill_path(+ &self,+ root: &CommandBufferRef,+ buffers: &MoeBuffers,+ suffix_length: usize,+ ) {+ let e = self.moe_config.mixture_size;+ let k = self.moe_config.num_experts_per_token;+ let gating_code = Self::gating_code_from_activation(+ &self.moe_config.expert_config.activation,+ );++ let gate_clip_min = self.moe_config.expert_config.gate_clipping[0]+ .unwrap_or(f32::NEG_INFINITY);+ let gate_clip_max = self.moe_config.expert_config.gate_clipping[1]+ .unwrap_or(f32::INFINITY);+ let up_clip_min = self.moe_config.expert_config.up_clipping[0];+ let up_clip_max = self.moe_config.expert_config.up_clipping[1];+ let silu_alpha = self.moe_config.expert_config.activation.alpha();++ let total_rows = suffix_length * k;+ const K_TILE: usize = 128;+ let num_tiles_k = ((self.hidden_dim + K_TILE - 1) / K_TILE) as u32;++ self.experts_two_pass_prefill_kernel+ .encode(+ root,+ MoeExpertsTwoPassArguments {+ x_perm_buffer: &buffers.x_perm,+ expert_offsets: &buffers.offsets,+ row_expert_map: &buffers.row_expert_map,+ hidden_buffer: &buffers.hidden,+ output_buffer: &buffers.y_partial,+ w13_all: &self.shared_weights.w13_buf,+ w2_all: &self.shared_weights.w2_buf,+ up_biases: &self.shared_weights.up_biases_buf,+ down_biases: &self.shared_weights.down_biases_buf,+ tile_counts: &buffers.tile_counts,+ tile_offsets: &buffers.tile_offsets,+ tile_map: &buffers.tile_map,+ total_tiles: &buffers.total_tiles,+ dispatch_args: &buffers.dispatch_args,+ total_rows,+ d_model: self.model_dim,+ d_ff: self.hidden_dim,+ e,+ num_tiles_k,+ gating_code,+ gate_clip_min,+ gate_clip_max,+ up_clip_min,+ up_clip_max,+ silu_alpha,+ data_type: self.data_type,+ },+ )+ .expect("MoE indirect prefill failed");++ self.encode_finalize(root, buffers, suffix_length);+ }++ fn encode_finalize(+ &self,+ root: &CommandBufferRef,+ buffers: &MoeBuffers,+ suffix_length: usize,+ ) {+ let k = self.moe_config.num_experts_per_token;self.finalize_kernel.encode(root,- super::MoeFinalizeArguments {- tok2row_buffer: &tok2row_buf,- probs_buffer: &topk_probs_buf,- y_partial_buffer: &y_partial_buf,- y_out_buffer: &main_buf,+ MoeFinalizeArguments {+ tok2row_buffer: &buffers.tok2row,+ probs_buffer: &buffers.topk_probs,+ y_partial_buffer: &buffers.y_partial,+ y_out_buffer: &buffers.main,t: suffix_length,d_model: self.model_dim,k,@@ -631,6 +731,95 @@ impl EncodableWithState for MoeBlockEncodable {self.data_type,).expect("MoE finalize failed");+ }+}++impl EncodableWithState for MoeBlockEncodable {+ fn encode(+ &self,+ state: &mut ForwardPassState,+ command_buffer: &MPSCommandBuffer,+ parameters: &EncodingParameters,+ ) {+ let suffix_length = state.active_suffix_length();+ let arrays = state.arrays(&[+ ArrayId::Main,+ ArrayId::MoeTopkIds,+ ArrayId::MoeTopkProbs,+ ArrayId::MoeOffsets,+ ArrayId::MoeSumK,+ ArrayId::MoeBucketedTokenIds,+ ArrayId::MoeBucketedProbs,+ ArrayId::MoeXPerm,+ ArrayId::MoeTok2Row,+ ArrayId::MoeYPartial,+ ArrayId::MoeTileCounts,+ ArrayId::MoeTileOffsets,+ ArrayId::MoeTileMap,+ ArrayId::MoeTotalTiles,+ ArrayId::MoeDispatchArgs,+ ArrayId::MoeScatterPartials,+ ArrayId::MoeScatterBlockBases,+ ArrayId::MoeBlockAlloc,+ ArrayId::MoeHidden,+ ArrayId::MoeTwoPassRowExpertMap,+ ]);++ let clone_buffer = |array: &RefCell<MetalArray>| -> metal::Buffer {+ let mut borrow = array.borrow_mut();+ unsafe { borrow.mtl_buffer().clone() }+ };++ let mut array_iter = arrays.iter();+ let buffers = MoeBuffers {+ main: clone_buffer(array_iter.next().unwrap()),+ topk_ids: clone_buffer(array_iter.next().unwrap()),+ topk_probs: clone_buffer(array_iter.next().unwrap()),+ offsets: clone_buffer(array_iter.next().unwrap()),+ sumk: clone_buffer(array_iter.next().unwrap()),+ bucketed_ids: clone_buffer(array_iter.next().unwrap()),+ bucketed_probs: clone_buffer(array_iter.next().unwrap()),+ x_perm: clone_buffer(array_iter.next().unwrap()),+ tok2row: clone_buffer(array_iter.next().unwrap()),+ y_partial: clone_buffer(array_iter.next().unwrap()),+ tile_counts: clone_buffer(array_iter.next().unwrap()),+ tile_offsets: clone_buffer(array_iter.next().unwrap()),+ tile_map: clone_buffer(array_iter.next().unwrap()),+ total_tiles: clone_buffer(array_iter.next().unwrap()),+ dispatch_args: clone_buffer(array_iter.next().unwrap()),+ partials: clone_buffer(array_iter.next().unwrap()),+ block_bases: clone_buffer(array_iter.next().unwrap()),+ block_alloc: clone_buffer(array_iter.next().unwrap()),+ hidden: clone_buffer(array_iter.next().unwrap()),+ row_expert_map: clone_buffer(array_iter.next().unwrap()),+ };+ debug_assert!(array_iter.next().is_none());++ let mtl_command_buffer =+ command_buffer.root_command_buffer().to_owned();+ let root = &*mtl_command_buffer;++ // Path selection:+ // - suffix == 1: Single-token fused decode (no scatter/gather)+ // - suffix <= 32: Indirect decode (GEMV)+ // - suffix > 32: Indirect prefill (GEMM)+ if suffix_length == 1 {+ // Single-token path: router -> fused decode+ self.encode_router(root, &buffers, suffix_length);+ self.encode_single_token_path(root, &buffers);+ } else if suffix_length <= INDIRECT_DECODE_THRESHOLD {+ // Indirect decode path: router -> setup -> decode experts -> finalize+ self.clear_indirect_buffers(root, &buffers, suffix_length);+ self.encode_router(root, &buffers, suffix_length);+ self.encode_indirect_setup(root, &buffers, suffix_length);+ self.encode_indirect_decode_path(root, &buffers, suffix_length);+ } else {+ // Indirect prefill path: router -> setup -> prefill experts -> finalize+ self.clear_indirect_buffers(root, &buffers, suffix_length);+ self.encode_router(root, &buffers, suffix_length);+ self.encode_indirect_setup(root, &buffers, suffix_length);+ self.encode_indirect_prefill_path(root, &buffers, suffix_length);+ }if parameters.wait_until_completed {command_buffer.commit_and_continue();diff --git a/crates/uzu/src/backends/metal/kernel/moe/experts.rs b/crates/uzu/src/backends/metal/kernel/moe/experts.rsindex 8d8ffc35d..d516c13a4 100644--- a/crates/uzu/src/backends/metal/kernel/moe/experts.rs+++ b/crates/uzu/src/backends/metal/kernel/moe/experts.rs@@ -538,3 +538,221 @@ impl MoeExpertsTwoPassPrefillKernel {Ok(())}}++/// Arguments for single-token MoE decode (T=1 optimized path)+#[derive(Debug)]+pub struct MoeExpertsSingleDecodeArguments<'a> {+ /// Input activation [d_model]+ pub x: &'a MTLBuffer,+ /// Top-K expert indices from router [K]+ pub topk_ids: &'a MTLBuffer,+ /// Top-K probabilities from router [K]+ pub topk_probs: &'a MTLBuffer,+ /// Up/gate projection weights [E, 2*d_ff, d_model]+ pub w13_all: &'a MTLBuffer,+ /// Down projection weights [E, d_model, d_ff]+ pub w2_all: &'a MTLBuffer,+ /// Up/gate biases [E, 2*d_ff]+ pub up_biases: &'a MTLBuffer,+ /// Down biases [E, d_model]+ pub down_biases: &'a MTLBuffer,+ /// Hidden buffer [K, d_ff] - intermediate storage (f32)+ pub hidden: &'a MTLBuffer,+ /// Final output [d_model]+ pub y: &'a MTLBuffer,+ /// Model dimension+ pub d_model: usize,+ /// FFN hidden dimension+ pub d_ff: usize,+ /// Number of active experts per token+ pub k: usize,+ /// Gating activation: 0=GELU, 1=SiLU, 2=SwiGLU, 3=GEGLU+ pub gating_code: u32,+ /// SiLU activation alpha parameter+ pub silu_alpha: f32,+ /// Gate clipping min+ pub gate_clip_min: f32,+ /// Gate clipping max+ pub gate_clip_max: f32,+ /// Up clipping min+ pub up_clip_min: f32,+ /// Up clipping max+ pub up_clip_max: f32,+ /// Data type+ pub data_type: KernelDataType,+}++pub struct MoeExpertsSingleDecodeKernel {+ pass_a: Vec<Vec<MTLComputePipelineState>>, // [gate][dtype]+ pass_b: Vec<MTLComputePipelineState>, // [dtype]+}++impl MoeExpertsSingleDecodeKernel {+ pub fn new(ctx: &MTLContext) -> Result<Self, MoeExpertsError> {+ let dtypes = [+ KernelDataType::Float16,+ KernelDataType::BFloat16,+ KernelDataType::Float32,+ ];++ // Pass A: needs gating function constant+ let mut pass_a = vec![Vec::with_capacity(dtypes.len()); 4];+ for gate in 0u32..4u32 {+ for dtype in &dtypes {+ let suffix = dtype_suffix(*dtype);+ let fcv = FunctionConstantValues::new();+ fcv.set_constant_value_at_index(+ &gate as *const u32 as *const std::ffi::c_void,+ MTLDataType::UInt,+ 30,+ );+ let kernel_name =+ format!("moe_experts_decode_single_pass_a_{}", suffix);+ let cache_key = format!("{}_gate_{}", kernel_name, gate);+ pass_a[gate as usize].push(ctx.compute_pipeline_state_cached(+ &cache_key,+ &kernel_name,+ Some(&fcv),+ )?);+ }+ }++ // Pass B (fused with finalize): no function constants needed+ let mut pass_b = Vec::with_capacity(dtypes.len());+ for dtype in &dtypes {+ let suffix = dtype_suffix(*dtype);+ let kernel_name =+ format!("moe_experts_decode_single_pass_b_{}", suffix);+ pass_b.push(ctx.compute_pipeline_state(&kernel_name, None)?);+ }++ Ok(Self {+ pass_a,+ pass_b,+ })+ }++ /// Encode the single-token decode pipeline+ ///+ /// Dispatches:+ /// 1. Pass A: grid = (h_blocks, K) - K parallel expert up+gate projections+ /// 2. Pass B: grid = (d_blocks, 1) - computes final y directly (fused finalize)+ pub fn encode(+ &self,+ command_buffer: &CommandBufferRef,+ args: MoeExpertsSingleDecodeArguments,+ ) -> Result<(), MoeExpertsError> {+ if args.k == 0 {+ return Ok(());+ }++ let gate_idx = (args.gating_code.min(3)) as usize;+ let dtype_idx = dtype_index(args.data_type);++ let d_model_u32 = args.d_model as u32;+ let d_ff_u32 = args.d_ff as u32;+ let k_u32 = args.k as u32;++ // Pass A: 4 simdgroups (128 threads), each threadgroup outputs 4 elements+ const PASSA_BLOCK_H: u32 = 4;+ const PASSA_THREADS: u32 = 128;+ let h_blocks = (d_ff_u32 + PASSA_BLOCK_H - 1) / PASSA_BLOCK_H;++ // Pass A: x @ W13[expert] -> hidden+ {+ let pipeline = &self.pass_a[gate_idx][dtype_idx];+ let encoder = command_buffer.new_compute_command_encoder();+ encoder.set_compute_pipeline_state(pipeline);+ encoder.set_buffer(0, Some(args.x), 0);+ encoder.set_buffer(1, Some(args.topk_ids), 0);+ encoder.set_buffer(2, Some(args.w13_all), 0);+ encoder.set_buffer(3, Some(args.up_biases), 0);+ encoder.set_buffer(4, Some(args.hidden), 0);+ encoder.set_bytes(+ 5,+ size_of::<u32>() as u64,+ &d_model_u32 as *const u32 as *const _,+ );+ encoder.set_bytes(+ 6,+ size_of::<u32>() as u64,+ &d_ff_u32 as *const u32 as *const _,+ );+ encoder.set_bytes(+ 7,+ size_of::<u32>() as u64,+ &k_u32 as *const u32 as *const _,+ );+ encoder.set_bytes(+ 8,+ size_of::<f32>() as u64,+ &args.silu_alpha as *const f32 as *const _,+ );+ encoder.set_bytes(+ 9,+ size_of::<f32>() as u64,+ &args.gate_clip_min as *const f32 as *const _,+ );+ encoder.set_bytes(+ 10,+ size_of::<f32>() as u64,+ &args.gate_clip_max as *const f32 as *const _,+ );+ encoder.set_bytes(+ 11,+ size_of::<f32>() as u64,+ &args.up_clip_min as *const f32 as *const _,+ );+ encoder.set_bytes(+ 12,+ size_of::<f32>() as u64,+ &args.up_clip_max as *const f32 as *const _,+ );+ encoder.dispatch_thread_groups(+ MTLSize::new(h_blocks as u64, args.k as u64, 1),+ MTLSize::new(PASSA_THREADS as u64, 1, 1),+ );+ encoder.end_encoding();+ }++ // Pass B: 8 simdgroups (256 threads), outputs final y directly+ const PASSB_BLOCK_D: u32 = 8;+ const PASSB_THREADS: u32 = 256;+ let d_blocks = (d_model_u32 + PASSB_BLOCK_D - 1) / PASSB_BLOCK_D;++ // Pass B: hidden @ W2[expert] -> y (with weighted sum)+ {+ let pipeline = &self.pass_b[dtype_idx];+ let encoder = command_buffer.new_compute_command_encoder();+ encoder.set_compute_pipeline_state(pipeline);+ encoder.set_buffer(0, Some(args.hidden), 0);+ encoder.set_buffer(1, Some(args.topk_ids), 0);+ encoder.set_buffer(2, Some(args.topk_probs), 0);+ encoder.set_buffer(3, Some(args.w2_all), 0);+ encoder.set_buffer(4, Some(args.down_biases), 0);+ encoder.set_buffer(5, Some(args.y), 0);+ encoder.set_bytes(+ 6,+ size_of::<u32>() as u64,+ &d_model_u32 as *const u32 as *const _,+ );+ encoder.set_bytes(+ 7,+ size_of::<u32>() as u64,+ &d_ff_u32 as *const u32 as *const _,+ );+ encoder.set_bytes(+ 8,+ size_of::<u32>() as u64,+ &k_u32 as *const u32 as *const _,+ );+ encoder.dispatch_thread_groups(+ MTLSize::new(d_blocks as u64, 1, 1),+ MTLSize::new(PASSB_THREADS as u64, 1, 1),+ );+ encoder.end_encoding();+ }++ Ok(())+ }+}diff --git a/crates/uzu/src/backends/metal/kernel/moe/experts_single.metal b/crates/uzu/src/backends/metal/kernel/moe/experts_single.metalnew file mode 100644index 000000000..23c3c1252--- /dev/null+++ b/crates/uzu/src/backends/metal/kernel/moe/experts_single.metal@@ -0,0 +1,239 @@+#include <metal_stdlib>+#include <metal_simdgroup>+using namespace metal;++constant uint GATING_SEL [[function_constant(30)]]; // 0=GELU, 1=SiLU, 2=SwiGLU, 3=GEGLU++static inline float gelu_approx(float x) {+ const float k0 = 0.7978845608f;+ const float k1 = 0.044715f;+ if (x > 10.0f) return x;+ if (x < -10.0f) return 0.0f;+ return 0.5f * x * (1.0f + tanh(clamp(k0 * (x + k1 * x * x * x), -10.0f, 10.0f)));+}++static inline float silu(float x, float alpha) {+ return x / (1.0f + exp(-alpha * x));+}++// ============================================================================+// Pass A: x @ W13[expert] → hidden[k]+// Each threadgroup: 4 simdgroups, outputs 4 elements (1 per simdgroup)+// Each simdgroup: 32 threads reduce d_model with float4 vectorized loads+// Grid: (ceil(d_ff/4), K)+// ============================================================================++template<typename T, typename T4>+inline void moe_experts_decode_single_pass_a_impl(+ device const T* x,+ device const int* topk_ids,+ device const T* W13_all,+ device const T* biases,+ device float* hidden_out,+ uint d_model,+ uint d_ff,+ uint K,+ float silu_alpha,+ float gate_clip_min,+ float gate_clip_max,+ float up_clip_min,+ float up_clip_max,+ uint k_slot,+ uint h_block_idx,+ uint simd_gid,+ uint simd_lid+) {+ const int expert_id = topk_ids[k_slot];+ if (expert_id < 0) return;+ const uint expert_u = uint(expert_id);++ const uint h_idx = h_block_idx * 4 + simd_gid;+ if (h_idx >= d_ff) return;++ const ulong w13_stride = (ulong)d_model * (ulong)(2 * d_ff);+ const ulong w13_base = (ulong)expert_u * w13_stride;+ const ulong bias_base = (ulong)expert_u * (ulong)(2 * d_ff);++ device const T* w_up_row = W13_all + w13_base + (ulong)h_idx * (ulong)d_model;+ device const T* w_gate_row = W13_all + w13_base + (ulong)(d_ff + h_idx) * (ulong)d_model;++ float acc_up = 0.0f;+ float acc_gate = 0.0f;++ const uint vec_iters = d_model / 128;++ for (uint i = 0; i < vec_iters; ++i) {+ uint base_idx = i * 128 + simd_lid * 4;++ T4 x_vec = *reinterpret_cast<device const T4*>(x + base_idx);+ T4 w_up_vec = *reinterpret_cast<device const T4*>(w_up_row + base_idx);++ acc_up += float(x_vec.x) * float(w_up_vec.x);+ acc_up += float(x_vec.y) * float(w_up_vec.y);+ acc_up += float(x_vec.z) * float(w_up_vec.z);+ acc_up += float(x_vec.w) * float(w_up_vec.w);++ if (GATING_SEL > 1) {+ T4 w_gate_vec = *reinterpret_cast<device const T4*>(w_gate_row + base_idx);+ acc_gate += float(x_vec.x) * float(w_gate_vec.x);+ acc_gate += float(x_vec.y) * float(w_gate_vec.y);+ acc_gate += float(x_vec.z) * float(w_gate_vec.z);+ acc_gate += float(x_vec.w) * float(w_gate_vec.w);+ }+ }++ uint leftover_start = vec_iters * 128 + simd_lid;+ for (uint idx = leftover_start; idx < d_model; idx += 32) {+ float xv = float(x[idx]);+ acc_up += xv * float(w_up_row[idx]);+ if (GATING_SEL > 1) {+ acc_gate += xv * float(w_gate_row[idx]);+ }+ }++ acc_up = simd_sum(acc_up);+ if (GATING_SEL > 1) {+ acc_gate = simd_sum(acc_gate);+ }++ if (simd_lid == 0) {+ float up_val = clamp(acc_up + float(biases[bias_base + h_idx]), up_clip_min, up_clip_max);++ float activated;+ if (GATING_SEL <= 1) {+ activated = (GATING_SEL == 0) ? gelu_approx(up_val) : silu(up_val, silu_alpha);+ } else {+ float gate_val = clamp(acc_gate + float(biases[bias_base + d_ff + h_idx]), gate_clip_min, gate_clip_max);+ float gate_act = (GATING_SEL == 2) ? silu(gate_val, silu_alpha) : gelu_approx(gate_val);+ activated = gate_act * up_val;+ }++ hidden_out[(ulong)k_slot * (ulong)d_ff + (ulong)h_idx] = activated;+ }+}++#define MOE_DECODE_SINGLE_PASS_A_KERNEL(DTYPE, DTYPE4, SUFFIX) \+kernel void moe_experts_decode_single_pass_a_##SUFFIX( \+ device const DTYPE* x [[buffer(0)]], \+ device const int* topk_ids [[buffer(1)]], \+ device const DTYPE* W13_all [[buffer(2)]], \+ device const DTYPE* biases [[buffer(3)]], \+ device float* hidden_out [[buffer(4)]], \+ constant uint& d_model [[buffer(5)]], \+ constant uint& d_ff [[buffer(6)]], \+ constant uint& K [[buffer(7)]], \+ constant float& silu_alpha [[buffer(8)]], \+ constant float& gate_clip_min [[buffer(9)]], \+ constant float& gate_clip_max [[buffer(10)]], \+ constant float& up_clip_min [[buffer(11)]], \+ constant float& up_clip_max [[buffer(12)]], \+ uint2 tgpig [[threadgroup_position_in_grid]], \+ uint simd_gid [[simdgroup_index_in_threadgroup]], \+ uint simd_lid [[thread_index_in_simdgroup]]) \+{ \+ moe_experts_decode_single_pass_a_impl<DTYPE, DTYPE4>( \+ x, topk_ids, W13_all, biases, hidden_out, \+ d_model, d_ff, K, silu_alpha, \+ gate_clip_min, gate_clip_max, up_clip_min, up_clip_max, \+ tgpig.y, tgpig.x, simd_gid, simd_lid); \+}++MOE_DECODE_SINGLE_PASS_A_KERNEL(half, half4, f16)+MOE_DECODE_SINGLE_PASS_A_KERNEL(bfloat, bfloat4, bf16)+MOE_DECODE_SINGLE_PASS_A_KERNEL(float, float4, f32)+++// ============================================================================+// Pass B (fused with finalize): hidden[k] @ W2[expert] → y (directly)+// Computes: y[d] = Σ_k prob[k] * (hidden[k] @ W2[expert_k, d] + bias)+// Each simdgroup computes one final output element+// Grid: (ceil(d_model/8), 1) - NOT per K!+// ============================================================================++template<typename T, typename T4>+inline void moe_experts_decode_single_pass_b_impl(+ device const float* hidden, // [K, d_ff]+ device const int* topk_ids, // [K]+ device const T* topk_probs, // [K]+ device const T* W2_all, // [E, d_model, d_ff]+ device const T* biases, // [E, d_model]+ device T* y, // [d_model]+ uint d_model,+ uint d_ff,+ uint K,+ uint d_block,+ uint simd_gid,+ uint simd_lid+) {+ const uint my_col = d_block * 8 + simd_gid;+ if (my_col >= d_model) return;++ const uint vec_iters = d_ff / 128;+ const ulong w2_expert_stride = (ulong)d_model * (ulong)d_ff;++ float final_acc = 0.0f;++ // Loop over K experts+ for (uint k = 0; k < K; ++k) {+ const uint expert_u = uint(topk_ids[k]);+ const float prob = float(topk_probs[k]);++ device const float* hidden_ptr = hidden + (ulong)k * (ulong)d_ff;+ device const T* w2_ptr = W2_all + (ulong)expert_u * w2_expert_stride + (ulong)my_col * (ulong)d_ff;++ float acc = 0.0f;++ // Vectorized reduction+ for (uint i = 0; i < vec_iters; ++i) {+ uint base_idx = i * 128 + simd_lid * 4;++ float4 h_vec = *reinterpret_cast<device const float4*>(hidden_ptr + base_idx);+ T4 w_vec = *reinterpret_cast<device const T4*>(w2_ptr + base_idx);++ acc += h_vec.x * float(w_vec.x);+ acc += h_vec.y * float(w_vec.y);+ acc += h_vec.z * float(w_vec.z);+ acc += h_vec.w * float(w_vec.w);+ }++ // Remainder+ for (uint idx = vec_iters * 128 + simd_lid; idx < d_ff; idx += 32) {+ acc += hidden_ptr[idx] * float(w2_ptr[idx]);+ }++ // Simdgroup reduction and accumulate+ float result = simd_sum(acc);+ if (simd_lid == 0) {+ result += float(biases[(ulong)expert_u * (ulong)d_model + (ulong)my_col]);+ final_acc += prob * result;+ }+ }++ if (simd_lid == 0) {+ y[my_col] = T(final_acc);+ }+}++#define MOE_DECODE_SINGLE_PASS_B_KERNEL(DTYPE, DTYPE4, SUFFIX) \+kernel void moe_experts_decode_single_pass_b_##SUFFIX( \+ device const float* hidden [[buffer(0)]], \+ device const int* topk_ids [[buffer(1)]], \+ device const DTYPE* topk_probs [[buffer(2)]], \+ device const DTYPE* W2_all [[buffer(3)]], \+ device const DTYPE* biases [[buffer(4)]], \+ device DTYPE* y [[buffer(5)]], \+ constant uint& d_model [[buffer(6)]], \+ constant uint& d_ff [[buffer(7)]], \+ constant uint& K [[buffer(8)]], \+ uint tgpig [[threadgroup_position_in_grid]], \+ uint simd_gid [[simdgroup_index_in_threadgroup]], \+ uint simd_lid [[thread_index_in_simdgroup]]) \+{ \+ moe_experts_decode_single_pass_b_impl<DTYPE, DTYPE4>( \+ hidden, topk_ids, topk_probs, W2_all, biases, y, \+ d_model, d_ff, K, tgpig, simd_gid, simd_lid); \+}++MOE_DECODE_SINGLE_PASS_B_KERNEL(half, half4, f16)+MOE_DECODE_SINGLE_PASS_B_KERNEL(bfloat, bfloat4, bf16)+MOE_DECODE_SINGLE_PASS_B_KERNEL(float, float4, f32)diff --git a/crates/uzu/src/backends/metal/kernel/moe/experts_two_pass_decode.metal b/crates/uzu/src/backends/metal/kernel/moe/experts_two_pass_decode.metalindex 67d95608e..d1631c395 100644--- a/crates/uzu/src/backends/metal/kernel/moe/experts_two_pass_decode.metal+++ b/crates/uzu/src/backends/metal/kernel/moe/experts_two_pass_decode.metal@@ -2,59 +2,31 @@#include <metal_simdgroup>using namespace metal;-constant uint GATING_SEL- [[function_constant(30)]]; // 0=GELU,1=SiLU,2=SwiGLU,3=GEGLU-constant uint TILE_H- [[function_constant(32)]]; // tile size for hidden dimension in Pass A+constant uint GATING_SEL [[function_constant(30)]]; // 0=GELU,1=SiLU,2=SwiGLU,3=GEGLUstatic inline float gelu_approx(float x) {- const float k0 = 0.7978845608f;- const float k1 = 0.044715f;- if (x > 10.0f)- return x;- if (x < -10.0f)- return 0.0f;- return 0.5f * x *- (1.0f + tanh(clamp(k0 * (x + k1 * x * x * x), -10.0f, 10.0f)));+ const float k0 = 0.7978845608f;+ const float k1 = 0.044715f;+ if (x > 10.0f) return x;+ if (x < -10.0f) return 0.0f;+ return 0.5f * x * (1.0f + tanh(clamp(k0 * (x + k1 * x * x * x), -10.0f, 10.0f)));}static inline float silu(float x, float alpha) {- return x / (1.0f + exp(-alpha * x));+ return x / (1.0f + exp(-alpha * x));}-#define MTL_CONST static constant constexpr const-#define MTL_PRAGMA_UNROLL _Pragma("clang loop unroll(full)")--// Tiling configuration (tunable)-MTL_CONST uint BM = 1; // Simdgroups in M dimension within threadgroup-MTL_CONST uint BN = 4; // Simdgroups in N dimension within threadgroup-MTL_CONST uint SM = 1; // Thread rows per simdgroup-MTL_CONST uint SN = 32; // Thread cols per simdgroup (full warp width)-MTL_CONST uint TM = 4; // Output elements per thread in M-MTL_CONST uint TN = 4; // Input elements per thread in N (per iteration)--MTL_CONST uint THREADS_M = BM * SM;-MTL_CONST uint THREADS_N = BN * SN;-MTL_CONST uint BLOCK_M = THREADS_M * TM;-MTL_CONST uint BLOCK_N = THREADS_N * TN;--static_assert(SM * SN == 32, "simdgroup must have 32 threads");-static_assert(- SN == 4 || SN == 8 || SN == 16 || SN == 32,- "SN must be 4, 8, 16, or 32"-);--// === Pass A: Optimized GEMV with hierarchical tiling ===-// Computes: hidden[row, h] = activation(x[row, d] @ W13[d, 2*h])-// AccumT is used for internal accumulations, output still stored as T-template <typename T, typename AccumT>+// === Pass A: Vectorized GEMV with float4 loads ===+// Structure: 4 simdgroups (128 threads), each outputs 1 hidden element+// Each simdgroup: 32 threads reduce d_model using float4 vectorized loads+// Grid: (h_blocks, rows, 1) where h_blocks = ceil(d_ff/4)+template<typename T, typename T4>void moe_experts_decode_pass_a_impl(- device const T* X_perm, // [total_rows, d_model]- device const uint* expert_offsets, // [E + 1]- device const T* W13_all, // weights in layout [E, 2*d_ff, d_model]- device const T* up_biases, // [E, 2*d_ff]- device float*- hidden_out, // [total_rows, d_ff] - f32 for activation precision+ device const T* X_perm, // [total_rows, d_model]+ device const uint* expert_offsets, // [E + 1]+ device const T* W13_all, // [E, 2*d_ff, d_model]+ device const T* up_biases, // [E, 2*d_ff]+ device float* hidden_out, // [total_rows, d_ff]uint d_model,uint d_ff,uint E,@@ -67,409 +39,260 @@ void moe_experts_decode_pass_a_impl(uint row_in_expert,uint h_block_idx,uint simd_gid,- uint simd_lid,- threadgroup float* tgp_memory+ uint simd_lid) {- // Thread position within simdgroup- const uint thrM = SN != 32 ? simd_lid / SN : 0;- const uint thrN = SN != 32 ? simd_lid % SN : uint(simd_lid);-- // Simdgroup position within threadgroup- const uint sgN = BN != 1 ? (simd_gid % BN) : 0;- const uint simdM = BN != 1 ? SM * (simd_gid / BN) : uint(SM * simd_gid);- const uint simdN = BN != 1 ? SN * (simd_gid % BN) : 0;-- // Thread's work block- int bm = (simdM + thrM) * TM;- int bn = (simdN + thrN) * TN;-- // Output row position- const uint seg_start = expert_offsets[expert_idx];- const uint seg_end = expert_offsets[expert_idx + 1];- uint global_row = seg_start + row_in_expert;-- if (global_row >= seg_end)- return;-- // Calculate output h index- int h_row = h_block_idx * BLOCK_M + bm;-- // Adjust tail block to ensure in-bounds reads- if (h_row + TM > d_ff) {- h_row = d_ff > TM ? int(d_ff - TM) : 0;- bm = h_row - int(h_block_idx * BLOCK_M);- }-- if (h_row >= int(d_ff))- return;-- // Accumulation arrays for up and gate projections- thread AccumT result_up[TM] = {0};- thread AccumT result_gate[TM] = {0};- thread T weights_up[TN];- thread T weights_gate[TN];- thread AccumT x_vals[TN];-- // Expert weight base addresses- const ulong w13_base = (ulong)expert_idx * (ulong)d_model * (ulong)(2 * d_ff);- const ulong bias_base = (ulong)expert_idx * (ulong)(2 * d_ff);- const ulong x_row_base = (ulong)global_row * (ulong)d_model;-- // Main accumulation loop over d_model in blocks of BLOCK_N- const uint n_iter = d_model / BLOCK_N;- const uint leftover = d_model - (n_iter * BLOCK_N);-- for (uint i = 0; i < n_iter; ++i) {- // Load x values for this block- MTL_PRAGMA_UNROLL- for (uint tn = 0; tn < TN; tn++) {- uint d_idx = bn + tn;- x_vals[tn] = AccumT(X_perm[x_row_base + d_idx]);- }+ // Validate row bounds+ const uint seg_start = expert_offsets[expert_idx];+ const uint seg_end = expert_offsets[expert_idx + 1];+ const uint global_row = seg_start + row_in_expert;+ if (global_row >= seg_end) return;- // Accumulate for each output row- MTL_PRAGMA_UNROLL- for (uint tm = 0; tm < TM; tm++) {- uint h_idx = h_row + tm;-- // Load weights for up projection- MTL_PRAGMA_UNROLL- for (uint tn = 0; tn < TN; tn++) {- uint d_idx = bn + tn;- ulong w_idx = w13_base + (ulong)h_idx * (ulong)d_model + (ulong)d_idx;- weights_up[tn] = W13_all[w_idx];- }-- // Accumulate up projection- MTL_PRAGMA_UNROLL- for (uint tn = 0; tn < TN; tn++) {- result_up[tm] += x_vals[tn] * AccumT(weights_up[tn]);- }-- // Load and accumulate gate projection if needed- if (GATING_SEL > 1) {- MTL_PRAGMA_UNROLL- for (uint tn = 0; tn < TN; tn++) {- uint d_idx = bn + tn;- ulong w_gate_idx =- w13_base + (ulong)(d_ff + h_idx) * (ulong)d_model + (ulong)d_idx;- weights_gate[tn] = W13_all[w_gate_idx];- }+ // Each simdgroup outputs one hidden element+ const uint h_idx = h_block_idx * 4 + simd_gid;+ if (h_idx >= d_ff) return;- MTL_PRAGMA_UNROLL- for (uint tn = 0; tn < TN; tn++) {- result_gate[tm] += x_vals[tn] * AccumT(weights_gate[tn]);- }- }- }+ // Base addresses+ const ulong w13_stride = (ulong)d_model * (ulong)(2 * d_ff);+ const ulong w13_base = (ulong)expert_idx * w13_stride;+ const ulong bias_base = (ulong)expert_idx * (ulong)(2 * d_ff);+ const ulong x_row_base = (ulong)global_row * (ulong)d_model;++ device const T* x_ptr = X_perm + x_row_base;+ device const T* w_up_row = W13_all + w13_base + (ulong)h_idx * (ulong)d_model;+ device const T* w_gate_row = W13_all + w13_base + (ulong)(d_ff + h_idx) * (ulong)d_model;- bn += BLOCK_N;- }+ float acc_up = 0.0f;+ float acc_gate = 0.0f;- // Handle leftover elements- if (leftover > 0) {- MTL_PRAGMA_UNROLL- for (uint tn = 0; tn < TN; tn++) {- uint d_idx = bn + tn;- x_vals[tn] =- (d_idx < d_model) ? AccumT(X_perm[x_row_base + d_idx]) : AccumT(0.0);+ // Vectorized reduction: 32 threads × 4 elements = 128 elements per iteration+ const uint vec_iters = d_model / 128;++ for (uint i = 0; i < vec_iters; ++i) {+ uint base_idx = i * 128 + simd_lid * 4;++ T4 x_vec = *reinterpret_cast<device const T4*>(x_ptr + base_idx);+ T4 w_up_vec = *reinterpret_cast<device const T4*>(w_up_row + base_idx);++ acc_up += float(x_vec.x) * float(w_up_vec.x);+ acc_up += float(x_vec.y) * float(w_up_vec.y);+ acc_up += float(x_vec.z) * float(w_up_vec.z);+ acc_up += float(x_vec.w) * float(w_up_vec.w);++ if (GATING_SEL > 1) {+ T4 w_gate_vec = *reinterpret_cast<device const T4*>(w_gate_row + base_idx);+ acc_gate += float(x_vec.x) * float(w_gate_vec.x);+ acc_gate += float(x_vec.y) * float(w_gate_vec.y);+ acc_gate += float(x_vec.z) * float(w_gate_vec.z);+ acc_gate += float(x_vec.w) * float(w_gate_vec.w);+ }}- MTL_PRAGMA_UNROLL- for (uint tm = 0; tm < TM; tm++) {- uint h_idx = h_row + tm;-- MTL_PRAGMA_UNROLL- for (uint tn = 0; tn < TN; tn++) {- uint d_idx = bn + tn;- if (d_idx < d_model) {- ulong w_idx = w13_base + (ulong)h_idx * (ulong)d_model + (ulong)d_idx;- weights_up[tn] = W13_all[w_idx];- result_up[tm] += x_vals[tn] * AccumT(weights_up[tn]);-- if (GATING_SEL > 1) {- ulong w_gate_idx = w13_base +- (ulong)(d_ff + h_idx) * (ulong)d_model +- (ulong)d_idx;- weights_gate[tn] = W13_all[w_gate_idx];- result_gate[tm] += x_vals[tn] * AccumT(weights_gate[tn]);- }+ // Handle leftover elements+ uint leftover_start = vec_iters * 128 + simd_lid;+ for (uint idx = leftover_start; idx < d_model; idx += 32) {+ float xv = float(x_ptr[idx]);+ acc_up += xv * float(w_up_row[idx]);+ if (GATING_SEL > 1) {+ acc_gate += xv * float(w_gate_row[idx]);}- }}- }- // Simdgroup reduction across thrN dimension- MTL_PRAGMA_UNROLL- for (uint tm = 0; tm < TM; tm++) {- result_up[tm] = simd_sum(result_up[tm]);+ // Simdgroup reduction+ acc_up = simd_sum(acc_up);if (GATING_SEL > 1) {- result_gate[tm] = simd_sum(result_gate[tm]);+ acc_gate = simd_sum(acc_gate);}- }- // Threadgroup reduction if BN > 1- if (BN > 1) {- threadgroup float* tgp_up = tgp_memory;- threadgroup float* tgp_gate = tgp_memory + BN * (BLOCK_M + TM);+ // Lane 0 applies activation and writes result+ if (simd_lid == 0) {+ float up_val = acc_up + float(up_biases[bias_base + h_idx]);+ up_val = clamp(up_val, up_clip_min, up_clip_max);- if (thrN == 0) {- MTL_PRAGMA_UNROLL- for (uint tm = 0; tm < TM; tm++) {- tgp_up[sgN * (BLOCK_M + TM) + bm + tm] = float(result_up[tm]);- if (GATING_SEL > 1) {- tgp_gate[sgN * (BLOCK_M + TM) + bm + tm] = float(result_gate[tm]);- }- }-- threadgroup_barrier(mem_flags::mem_threadgroup);-- if (sgN == 0) {- MTL_PRAGMA_UNROLL- for (uint sgn = 1; sgn < BN; sgn++) {- MTL_PRAGMA_UNROLL- for (uint tm = 0; tm < TM; tm++) {- result_up[tm] += AccumT(tgp_up[sgn * (BLOCK_M + TM) + bm + tm]);- if (GATING_SEL > 1) {- result_gate[tm] +=- AccumT(tgp_gate[sgn * (BLOCK_M + TM) + bm + tm]);- }- }- }- }- }- }-- // Apply activation and write output (only first thread in simdgroup width)- if (simdN == 0 && thrN == 0) {- MTL_PRAGMA_UNROLL- for (uint tm = 0; tm < TM; tm++) {- uint h_idx = h_row + tm;- if (h_idx < d_ff) {- // Add bias and clip- AccumT up_val = result_up[tm] + AccumT(up_biases[bias_base + h_idx]);- up_val = AccumT(clamp(float(up_val), up_clip_min, up_clip_max));-- AccumT activated_val;+ float activated;if (GATING_SEL <= 1) {- activated_val = (GATING_SEL == 0)- ? AccumT(gelu_approx(float(up_val)))- : AccumT(silu(float(up_val), silu_alpha));+ activated = (GATING_SEL == 0) ? gelu_approx(up_val) : silu(up_val, silu_alpha);} else {- AccumT gate_val =- result_gate[tm] + AccumT(up_biases[bias_base + d_ff + h_idx]);- gate_val =- AccumT(clamp(float(gate_val), gate_clip_min, gate_clip_max));- AccumT gate_activated =- (GATING_SEL == 2) ? AccumT(silu(float(gate_val), silu_alpha))- : AccumT(gelu_approx(float(gate_val)));- activated_val = gate_activated * up_val;+ float gate_val = acc_gate + float(up_biases[bias_base + d_ff + h_idx]);+ gate_val = clamp(gate_val, gate_clip_min, gate_clip_max);+ float gate_act = (GATING_SEL == 2) ? silu(gate_val, silu_alpha) : gelu_approx(gate_val);+ activated = gate_act * up_val;}- ulong hidden_idx = (ulong)global_row * (ulong)d_ff + (ulong)h_idx;- hidden_out[hidden_idx] = float(activated_val);- }+ hidden_out[(ulong)global_row * (ulong)d_ff + (ulong)h_idx] = activated;}- }}-#define MOE_PASS_A_KERNEL(DTYPE, SUFFIX) \- kernel void moe_experts_decode_pass_a_##SUFFIX( \- device const DTYPE* X_perm [[buffer(0)]], \- device const uint* expert_offsets [[buffer(1)]], \- device const DTYPE* W13_all [[buffer(2)]], \- device float* hidden_out [[buffer(3)]], \- device const DTYPE* up_biases [[buffer(4)]], \- constant uint& d_model [[buffer(5)]], \- constant uint& d_ff [[buffer(6)]], \- constant uint& E [[buffer(7)]], \- constant float& gate_clip_min [[buffer(8)]], \- constant float& gate_clip_max [[buffer(9)]], \- constant float& up_clip_min [[buffer(10)]], \- constant float& up_clip_max [[buffer(11)]], \- constant float& silu_alpha [[buffer(12)]], \- uint3 tgpig [[threadgroup_position_in_grid]], \- uint simd_gid [[simdgroup_index_in_threadgroup]], \- uint simd_lid [[thread_index_in_simdgroup]] \- ) { \- constexpr uint tgp_mem_size = (BN > 1) ? 2 * BN * (BLOCK_M + TM) : 1; \- threadgroup float tgp_memory[tgp_mem_size]; \- \- moe_experts_decode_pass_a_impl<DTYPE, float>( \- X_perm, \- expert_offsets, \- W13_all, \- up_biases, \- hidden_out, \- d_model, \- d_ff, \- E, \- gate_clip_min, \- gate_clip_max, \- up_clip_min, \- up_clip_max, \- silu_alpha, \- tgpig.y, \- tgpig.z, \- tgpig.x, \- simd_gid, \- simd_lid, \- (BN > 1) ? tgp_memory : nullptr \- ); \- }--MOE_PASS_A_KERNEL(bfloat, bf16)-MOE_PASS_A_KERNEL(half, f16)-MOE_PASS_A_KERNEL(float, f32)+#define MOE_PASS_A_KERNEL(DTYPE, DTYPE4, SUFFIX) \+kernel void moe_experts_decode_pass_a_##SUFFIX( \+ device const DTYPE* X_perm [[buffer(0)]], \+ device const uint* expert_offsets [[buffer(1)]], \+ device const DTYPE* W13_all [[buffer(2)]], \+ device float* hidden_out [[buffer(3)]], \+ device const DTYPE* up_biases [[buffer(4)]], \+ constant uint& d_model [[buffer(5)]], \+ constant uint& d_ff [[buffer(6)]], \+ constant uint& E [[buffer(7)]], \+ constant float& gate_clip_min [[buffer(8)]], \+ constant float& gate_clip_max [[buffer(9)]], \+ constant float& up_clip_min [[buffer(10)]], \+ constant float& up_clip_max [[buffer(11)]], \+ constant float& silu_alpha [[buffer(12)]], \+ uint3 tgpig [[threadgroup_position_in_grid]], \+ uint simd_gid [[simdgroup_index_in_threadgroup]], \+ uint simd_lid [[thread_index_in_simdgroup]]) \+{ \+ moe_experts_decode_pass_a_impl<DTYPE, DTYPE4>( \+ X_perm, expert_offsets, W13_all, up_biases, hidden_out, \+ d_model, d_ff, E, gate_clip_min, gate_clip_max, \+ up_clip_min, up_clip_max, silu_alpha, \+ tgpig.y, tgpig.z, tgpig.x, \+ simd_gid, simd_lid); \+}++MOE_PASS_A_KERNEL(bfloat, bfloat4, bf16)+MOE_PASS_A_KERNEL(half, half4, f16)+MOE_PASS_A_KERNEL(float, float4, f32)// === Helper kernels for indirect dispatch of Pass A ===// Count tiles per expert: tiles = (num_rows > 0) ? num_rows * h_blocks : 0kernel void moe_pass_a_tile_counts(- device const uint* expert_offsets [[buffer(0)]], // [E+1]- device uint* tile_counts [[buffer(1)]], // [E]+ device const uint* expert_offsets [[buffer(0)]], // [E+1]+ device uint* tile_counts [[buffer(1)]], // [E]constant uint& E [[buffer(2)]],constant uint& h_blocks [[buffer(3)]],uint tid [[thread_position_in_grid]]) {- if (tid >= E)- return;- const uint start = expert_offsets[tid];- const uint end = expert_offsets[tid + 1];- const uint num_rows = end - start;- tile_counts[tid] = (num_rows > 0) ? (num_rows * h_blocks) : 0;+ if (tid >= E) return;+ const uint start = expert_offsets[tid];+ const uint end = expert_offsets[tid + 1];+ const uint num_rows = end - start;+ tile_counts[tid] = (num_rows > 0) ? (num_rows * h_blocks) : 0;}// Exclusive scan of tile_counts to get tile_offsets and total_tileskernel void moe_pass_a_tile_scan(- device const uint* tile_counts [[buffer(0)]], // [E]- device uint* tile_offsets [[buffer(1)]], // [E+1]- device uint* total_tiles [[buffer(2)]], // [1]+ device const uint* tile_counts [[buffer(0)]], // [E]+ device uint* tile_offsets [[buffer(1)]], // [E+1]+ device uint* total_tiles [[buffer(2)]], // [1]constant uint& E [[buffer(3)]],uint lid [[thread_index_in_threadgroup]],threadgroup uint* scratch [[threadgroup(0)]]) {- // Simple single-threadgroup scan (works for E <= 1024)- const uint idx = lid;-- // Load into threadgroup memory- if (idx < E) {- scratch[idx] = tile_counts[idx];- } else {- scratch[idx] = 0;- }- threadgroup_barrier(mem_flags::mem_threadgroup);-- // Kogge-Stone scan- uint val = scratch[idx];- MTL_PRAGMA_UNROLL- for (uint offset = 1; offset < 1024; offset *= 2) {- uint temp = 0;- if (idx >= offset && idx < E) {- temp = scratch[idx - offset];+ // Simple single-threadgroup scan (works for E <= 1024)+ const uint idx = lid;++ // Load into threadgroup memory+ if (idx < E) {+ scratch[idx] = tile_counts[idx];+ } else {+ scratch[idx] = 0;}threadgroup_barrier(mem_flags::mem_threadgroup);- if (idx >= offset && idx < E) {- val += temp;- scratch[idx] = val;++ // Kogge-Stone scan+ uint val = scratch[idx];+ for (uint offset = 1; offset < 1024; offset *= 2) {+ uint temp = 0;+ if (idx >= offset && idx < E) {+ temp = scratch[idx - offset];+ }+ threadgroup_barrier(mem_flags::mem_threadgroup);+ if (idx >= offset && idx < E) {+ val += temp;+ scratch[idx] = val;+ }+ threadgroup_barrier(mem_flags::mem_threadgroup);}- threadgroup_barrier(mem_flags::mem_threadgroup);- }-- // Write exclusive scan (shift right by 1)- if (idx == 0) {- tile_offsets[0] = 0;- }- if (idx < E) {- tile_offsets[idx + 1] = scratch[idx];- if (idx == E - 1) {- total_tiles[0] = scratch[idx];++ // Write exclusive scan (shift right by 1)+ if (idx == 0) {+ tile_offsets[0] = 0;+ }+ if (idx < E) {+ tile_offsets[idx + 1] = scratch[idx];+ if (idx == E - 1) {+ total_tiles[0] = scratch[idx];+ }}- }}// Build row→expert map: one thread per routed rowkernel void moe_pass_a_build_row_map(- device const uint* expert_offsets [[buffer(0)]], // [E+1]- device uint* row_expert_map [[buffer(1)]], // [total_rows]+ device const uint* expert_offsets [[buffer(0)]] , // [E+1]+ device uint* row_expert_map [[buffer(1)]] , // [total_rows]constant uint& total_rows [[buffer(2)]],constant uint& E [[buffer(3)]],uint tid [[thread_position_in_grid]]) {- if (tid >= total_rows)- return;+ if (tid >= total_rows) return;- uint left = 0u;- uint right = E;- const uint row = tid;+ uint left = 0u;+ uint right = E;+ const uint row = tid;- while (left + 1u < right) {- const uint mid = (left + right) >> 1;- if (row < expert_offsets[mid]) {- right = mid;- } else {- left = mid;+ while (left + 1u < right) {+ const uint mid = (left + right) >> 1;+ if (row < expert_offsets[mid]) {+ right = mid;+ } else {+ left = mid;+ }}- }- row_expert_map[row] = left;+ row_expert_map[row] = left;}// Build tile map entries from row→expert mapkernel void moe_pass_a_build_tile_map(- device const uint* expert_offsets [[buffer(0)]], // [E+1]- device const uint* tile_offsets [[buffer(1)]], // [E+1]- device const uint* row_expert_map [[buffer(2)]], // [total_rows]- device uint* tile_map [[buffer(3)]], // [total_tiles * 3]+ device const uint* expert_offsets [[buffer(0)]], // [E+1]+ device const uint* tile_offsets [[buffer(1)]], // [E+1]+ device const uint* row_expert_map [[buffer(2)]], // [total_rows]+ device uint* tile_map [[buffer(3)]], // [total_tiles * 3]constant uint& total_rows [[buffer(4)]],constant uint& h_blocks [[buffer(5)]],uint tid [[thread_position_in_grid]]) {- const uint total_tiles = total_rows * h_blocks;- if (tid >= total_tiles)- return;+ const uint total_tiles = total_rows * h_blocks;+ if (tid >= total_tiles) return;- const uint row_idx = tid / h_blocks;- const uint h_block = tid % h_blocks;+ const uint row_idx = tid / h_blocks;+ const uint h_block = tid % h_blocks;- if (row_idx >= total_rows)- return;+ if (row_idx >= total_rows) return;- const uint expert_idx = row_expert_map[row_idx];- const uint row_start = expert_offsets[expert_idx];- const uint row_in_expert = row_idx - row_start;- const uint tile_base =- tile_offsets[expert_idx] + row_in_expert * h_blocks + h_block;+ const uint expert_idx = row_expert_map[row_idx];+ const uint row_start = expert_offsets[expert_idx];+ const uint row_in_expert = row_idx - row_start;+ const uint tile_base = tile_offsets[expert_idx] + row_in_expert * h_blocks + h_block;- tile_map[tile_base * 3 + 0] = h_block;- tile_map[tile_base * 3 + 1] = expert_idx;- tile_map[tile_base * 3 + 2] = row_in_expert;+ tile_map[tile_base * 3 + 0] = h_block;+ tile_map[tile_base * 3 + 1] = expert_idx;+ tile_map[tile_base * 3 + 2] = row_in_expert;}// Write dispatch args for indirect dispatch (reusable from tiled version)kernel void moe_pass_a_write_dispatch_args(- device const uint* total_tiles [[buffer(0)]], // [1]- device uint* dispatch_args- [[buffer(1)]], // [3] - MTLDispatchThreadgroupsIndirectArguments- constant uint& num_tiles_y [[buffer(2)]], // usually 1 for Pass A+ device const uint* total_tiles [[buffer(0)]], // [1]+ device uint* dispatch_args [[buffer(1)]], // [3] - MTLDispatchThreadgroupsIndirectArguments+ constant uint& num_tiles_y [[buffer(2)]], // usually 1 for Pass Auint tid [[thread_position_in_grid]]) {- if (tid > 0)- return;- dispatch_args[0] = total_tiles[0]; // x dimension = total tiles- dispatch_args[1] = num_tiles_y; // y dimension- dispatch_args[2] = 1; // z dimension+ if (tid > 0) return;+ dispatch_args[0] = total_tiles[0]; // x dimension = total tiles+ dispatch_args[1] = num_tiles_y; // y dimension+ dispatch_args[2] = 1; // z dimension}// Modified Pass A that reads from tile map for indirect dispatch-template <typename T>+template<typename T, typename T4>void moe_experts_decode_pass_a_indirect_impl(device const T* X_perm,device const uint* expert_offsets,device const T* W13_all,device const T* up_biases,device float* hidden_out,- device const uint* tile_map, // [total_tiles * 3]+ device const uint* tile_map, // [total_tiles * 3]uint d_model,uint d_ff,uint E,@@ -478,101 +301,68 @@ void moe_experts_decode_pass_a_indirect_impl(float up_clip_min,float up_clip_max,float silu_alpha,- uint tile_idx, // flat threadgroup index+ uint tile_idx, // flat threadgroup indexuint simd_gid,- uint simd_lid,- threadgroup float* tgp_memory+ uint simd_lid) {- // Read tile descriptor- const uint h_block_idx = tile_map[tile_idx * 3 + 0];- const uint expert_idx = tile_map[tile_idx * 3 + 1];- const uint row_in_expert = tile_map[tile_idx * 3 + 2];-- // Call original implementation- moe_experts_decode_pass_a_impl<T, float>(- X_perm,- expert_offsets,- W13_all,- up_biases,- hidden_out,- d_model,- d_ff,- E,- gate_clip_min,- gate_clip_max,- up_clip_min,- up_clip_max,- silu_alpha,- expert_idx,- row_in_expert,- h_block_idx,- simd_gid,- simd_lid,- tgp_memory- );+ // Read tile descriptor+ const uint h_block_idx = tile_map[tile_idx * 3 + 0];+ const uint expert_idx = tile_map[tile_idx * 3 + 1];+ const uint row_in_expert = tile_map[tile_idx * 3 + 2];++ // Call original implementation+ moe_experts_decode_pass_a_impl<T, T4>(+ X_perm, expert_offsets, W13_all, up_biases, hidden_out,+ d_model, d_ff, E,+ gate_clip_min, gate_clip_max,+ up_clip_min, up_clip_max, silu_alpha,+ expert_idx, row_in_expert, h_block_idx,+ simd_gid, simd_lid+ );}-#define MOE_PASS_A_INDIRECT_KERNEL(DTYPE, SUFFIX) \- kernel void moe_experts_decode_pass_a_indirect_##SUFFIX( \- device const DTYPE* X_perm [[buffer(0)]], \- device const uint* expert_offsets [[buffer(1)]], \- device const DTYPE* W13_all [[buffer(2)]], \- device float* hidden_out [[buffer(3)]], \- device const DTYPE* up_biases [[buffer(4)]], \- constant uint& d_model [[buffer(5)]], \- constant uint& d_ff [[buffer(6)]], \- constant uint& E [[buffer(7)]], \- constant float& gate_clip_min [[buffer(8)]], \- constant float& gate_clip_max [[buffer(9)]], \- constant float& up_clip_min [[buffer(10)]], \- constant float& up_clip_max [[buffer(11)]], \- constant float& silu_alpha [[buffer(12)]], \- device const uint* tile_map [[buffer(13)]], \- uint3 tgpig [[threadgroup_position_in_grid]], \- uint simd_gid [[simdgroup_index_in_threadgroup]], \- uint simd_lid [[thread_index_in_simdgroup]] \- ) { \- constexpr uint tgp_mem_size = (BN > 1) ? 2 * BN * (BLOCK_M + TM) : 1; \- threadgroup float tgp_memory[tgp_mem_size]; \- \- moe_experts_decode_pass_a_indirect_impl<DTYPE>( \- X_perm, \- expert_offsets, \- W13_all, \- up_biases, \- hidden_out, \- tile_map, \- d_model, \- d_ff, \- E, \- gate_clip_min, \- gate_clip_max, \- up_clip_min, \- up_clip_max, \- silu_alpha, \- tgpig.x, \- simd_gid, \- simd_lid, \- (BN > 1) ? tgp_memory : nullptr \- ); \- }--MOE_PASS_A_INDIRECT_KERNEL(bfloat, bf16)-MOE_PASS_A_INDIRECT_KERNEL(half, f16)-MOE_PASS_A_INDIRECT_KERNEL(float, f32)+#define MOE_PASS_A_INDIRECT_KERNEL(DTYPE, DTYPE4, SUFFIX) \+kernel void moe_experts_decode_pass_a_indirect_##SUFFIX( \+ device const DTYPE* X_perm [[buffer(0)]], \+ device const uint* expert_offsets [[buffer(1)]], \+ device const DTYPE* W13_all [[buffer(2)]], \+ device float* hidden_out [[buffer(3)]], \+ device const DTYPE* up_biases [[buffer(4)]], \+ constant uint& d_model [[buffer(5)]], \+ constant uint& d_ff [[buffer(6)]], \+ constant uint& E [[buffer(7)]], \+ constant float& gate_clip_min [[buffer(8)]], \+ constant float& gate_clip_max [[buffer(9)]], \+ constant float& up_clip_min [[buffer(10)]], \+ constant float& up_clip_max [[buffer(11)]], \+ constant float& silu_alpha [[buffer(12)]], \+ device const uint* tile_map [[buffer(13)]], \+ uint3 tgpig [[threadgroup_position_in_grid]], \+ uint simd_gid [[simdgroup_index_in_threadgroup]], \+ uint simd_lid [[thread_index_in_simdgroup]]) \+{ \+ moe_experts_decode_pass_a_indirect_impl<DTYPE, DTYPE4>( \+ X_perm, expert_offsets, W13_all, up_biases, hidden_out, tile_map, \+ d_model, d_ff, E, gate_clip_min, gate_clip_max, \+ up_clip_min, up_clip_max, silu_alpha, \+ tgpig.x, simd_gid, simd_lid); \+}++MOE_PASS_A_INDIRECT_KERNEL(bfloat, bfloat4, bf16)+MOE_PASS_A_INDIRECT_KERNEL(half, half4, f16)+MOE_PASS_A_INDIRECT_KERNEL(float, float4, f32)+// === Pass B: Simdgroup cooperation along K for coalescing ===-// W2 layout [E, d_model, d_ff] - 32 threads cooperate on one output, reading-// consecutive K elements+// W2 layout [E, d_model, d_ff] - 32 threads cooperate on one output, reading consecutive K elements-template <typename T, typename AccumT>+template<typename T, typename AccumT>void moe_experts_decode_down_fused_2d_impl(- device const float* hidden, // [total_rows, d_ff] - f32 from Pass A- device const uint*- row_expert_map, // [total_rows] - direct row->expert lookup- device const T* w2_all, // [E, d_model, d_ff] - layout- device const T* down_biases, // [E, d_model]- device T* y_out, // [total_rows, d_model]+ device const float* hidden, // [total_rows, d_ff] - f32 from Pass A+ device const uint* row_expert_map, // [total_rows] - direct row->expert lookup+ device const T* w2_all, // [E, d_model, d_ff] - layout+ device const T* down_biases, // [E, d_model]+ device T* y_out, // [total_rows, d_model]uint total_rows,uint d_model,uint d_ff,@@ -581,143 +371,114 @@ void moe_experts_decode_down_fused_2d_impl(uint simd_gid,uint simd_lid) {- constexpr uint THREADS_PER_SIMD = 32;- constexpr uint SIMDGROUPS_PER_TG = 8;-- const uint row_idx = tgpig.y;- if (row_idx >= total_rows)- return;-- // Each simdgroup computes one output column- const uint my_col = tgpig.x * SIMDGROUPS_PER_TG + simd_gid;- if (my_col >= d_model)- return;-- const uint expert_idx = row_expert_map[row_idx];-- // Base addresses for this output column- const ulong hidden_base = (ulong)row_idx * (ulong)d_ff;- const ulong w2_col_base = (ulong)expert_idx * (ulong)d_model * (ulong)d_ff +- (ulong)my_col * (ulong)d_ff;-- // Dual accumulators for ILP: breaks FMA dependency chains- AccumT acc0 = AccumT(0.0);- AccumT acc1 = AccumT(0.0);- AccumT acc = AccumT(0.0);-- // Main loop: stride-32 with vec8 loads for ILP- const uint k_iters = d_ff / THREADS_PER_SIMD;- const uint k_vec_iters = k_iters / 8;-- MTL_PRAGMA_UNROLL- for (uint iter = 0; iter < k_vec_iters; ++iter) {- const uint k_base = iter * (8 * THREADS_PER_SIMD) + simd_lid;-- // hidden: stride-32 per lane, already f32 from Pass A- const AccumT h0 = hidden[hidden_base + k_base + 0 * THREADS_PER_SIMD];- const AccumT h1 = hidden[hidden_base + k_base + 1 * THREADS_PER_SIMD];- const AccumT h2 = hidden[hidden_base + k_base + 2 * THREADS_PER_SIMD];- const AccumT h3 = hidden[hidden_base + k_base + 3 * THREADS_PER_SIMD];- const AccumT h4 = hidden[hidden_base + k_base + 4 * THREADS_PER_SIMD];- const AccumT h5 = hidden[hidden_base + k_base + 5 * THREADS_PER_SIMD];- const AccumT h6 = hidden[hidden_base + k_base + 6 * THREADS_PER_SIMD];- const AccumT h7 = hidden[hidden_base + k_base + 7 * THREADS_PER_SIMD];-- // W2: lane-coalesced- const AccumT w0 =- AccumT(w2_all[w2_col_base + k_base + 0 * THREADS_PER_SIMD]);- const AccumT w1 =- AccumT(w2_all[w2_col_base + k_base + 1 * THREADS_PER_SIMD]);- const AccumT w2 =- AccumT(w2_all[w2_col_base + k_base + 2 * THREADS_PER_SIMD]);- const AccumT w3 =- AccumT(w2_all[w2_col_base + k_base + 3 * THREADS_PER_SIMD]);- const AccumT w4 =- AccumT(w2_all[w2_col_base + k_base + 4 * THREADS_PER_SIMD]);- const AccumT w5 =- AccumT(w2_all[w2_col_base + k_base + 5 * THREADS_PER_SIMD]);- const AccumT w6 =- AccumT(w2_all[w2_col_base + k_base + 6 * THREADS_PER_SIMD]);- const AccumT w7 =- AccumT(w2_all[w2_col_base + k_base + 7 * THREADS_PER_SIMD]);-- // dual trees for ILP- acc0 = fma(h0, w0, acc0);- acc1 = fma(h1, w1, acc1);- acc0 = fma(h2, w2, acc0);- acc1 = fma(h3, w3, acc1);- acc0 = fma(h4, w4, acc0);- acc1 = fma(h5, w5, acc1);- acc0 = fma(h6, w6, acc0);- acc1 = fma(h7, w7, acc1);- }-- // fold the vectorized contribution once- acc += (acc0 + acc1);-- // Handle remaining full iterations- MTL_PRAGMA_UNROLL- for (uint iter = k_vec_iters * 8; iter < k_iters; ++iter) {- const uint k = iter * THREADS_PER_SIMD + simd_lid;- acc =- fma(AccumT(hidden[hidden_base + k]),- AccumT(w2_all[w2_col_base + k]),- acc);- }-- // Handle leftover elements (d_ff % 32)- const uint leftover_start = k_iters * THREADS_PER_SIMD;- if (leftover_start + simd_lid < d_ff) {- const uint k = leftover_start + simd_lid;- acc =- fma(AccumT(hidden[hidden_base + k]),- AccumT(w2_all[w2_col_base + k]),- acc);- }-- // Simdgroup reduction- AccumT result = simd_sum(acc);-- // Lane 0 writes result- if (simd_lid == 0) {- const ulong bias_idx = (ulong)expert_idx * (ulong)d_model + (ulong)my_col;- result += AccumT(down_biases[bias_idx]);-- const ulong out_idx = (ulong)row_idx * (ulong)d_model + (ulong)my_col;- y_out[out_idx] = T(result);- }+ constexpr uint THREADS_PER_SIMD = 32;+ constexpr uint SIMDGROUPS_PER_TG = 8;++ const uint row_idx = tgpig.y;+ if (row_idx >= total_rows) return;++ // Each simdgroup computes one output column+ const uint my_col = tgpig.x * SIMDGROUPS_PER_TG + simd_gid;+ if (my_col >= d_model) return;++ const uint expert_idx = row_expert_map[row_idx];++ // Base addresses for this output column+ const ulong hidden_base = (ulong)row_idx * (ulong)d_ff;+ const ulong w2_col_base = (ulong)expert_idx * (ulong)d_model * (ulong)d_ff + (ulong)my_col * (ulong)d_ff;++ // Dual accumulators for ILP: breaks FMA dependency chains+ AccumT acc0 = AccumT(0.0);+ AccumT acc1 = AccumT(0.0);+ AccumT acc = AccumT(0.0);++ // Main loop: stride-32 with vec8 loads for ILP+ const uint k_iters = d_ff / THREADS_PER_SIMD;+ const uint k_vec_iters = k_iters / 8;++ for (uint iter = 0; iter < k_vec_iters; ++iter) {+ const uint k_base = iter * (8 * THREADS_PER_SIMD) + simd_lid;++ // hidden: stride-32 per lane, already f32 from Pass A+ const AccumT h0 = hidden[hidden_base + k_base + 0 * THREADS_PER_SIMD];+ const AccumT h1 = hidden[hidden_base + k_base + 1 * THREADS_PER_SIMD];+ const AccumT h2 = hidden[hidden_base + k_base + 2 * THREADS_PER_SIMD];+ const AccumT h3 = hidden[hidden_base + k_base + 3 * THREADS_PER_SIMD];+ const AccumT h4 = hidden[hidden_base + k_base + 4 * THREADS_PER_SIMD];+ const AccumT h5 = hidden[hidden_base + k_base + 5 * THREADS_PER_SIMD];+ const AccumT h6 = hidden[hidden_base + k_base + 6 * THREADS_PER_SIMD];+ const AccumT h7 = hidden[hidden_base + k_base + 7 * THREADS_PER_SIMD];++ // W2: lane-coalesced+ const AccumT w0 = AccumT(w2_all[w2_col_base + k_base + 0 * THREADS_PER_SIMD]);+ const AccumT w1 = AccumT(w2_all[w2_col_base + k_base + 1 * THREADS_PER_SIMD]);+ const AccumT w2 = AccumT(w2_all[w2_col_base + k_base + 2 * THREADS_PER_SIMD]);+ const AccumT w3 = AccumT(w2_all[w2_col_base + k_base + 3 * THREADS_PER_SIMD]);+ const AccumT w4 = AccumT(w2_all[w2_col_base + k_base + 4 * THREADS_PER_SIMD]);+ const AccumT w5 = AccumT(w2_all[w2_col_base + k_base + 5 * THREADS_PER_SIMD]);+ const AccumT w6 = AccumT(w2_all[w2_col_base + k_base + 6 * THREADS_PER_SIMD]);+ const AccumT w7 = AccumT(w2_all[w2_col_base + k_base + 7 * THREADS_PER_SIMD]);++ // dual trees for ILP+ acc0 = fma(h0, w0, acc0);+ acc1 = fma(h1, w1, acc1);+ acc0 = fma(h2, w2, acc0);+ acc1 = fma(h3, w3, acc1);+ acc0 = fma(h4, w4, acc0);+ acc1 = fma(h5, w5, acc1);+ acc0 = fma(h6, w6, acc0);+ acc1 = fma(h7, w7, acc1);+ }++ // fold the vectorized contribution once+ acc += (acc0 + acc1);++ // Handle remaining full iterations+ for (uint iter = k_vec_iters * 8; iter < k_iters; ++iter) {+ const uint k = iter * THREADS_PER_SIMD + simd_lid;+ acc = fma(AccumT(hidden[hidden_base + k]), AccumT(w2_all[w2_col_base + k]), acc);+ }++ // Handle leftover elements (d_ff % 32)+ const uint leftover_start = k_iters * THREADS_PER_SIMD;+ if (leftover_start + simd_lid < d_ff) {+ const uint k = leftover_start + simd_lid;+ acc = fma(AccumT(hidden[hidden_base + k]), AccumT(w2_all[w2_col_base + k]), acc);+ }++ // Simdgroup reduction+ AccumT result = simd_sum(acc);++ // Lane 0 writes result+ if (simd_lid == 0) {+ const ulong bias_idx = (ulong)expert_idx * (ulong)d_model + (ulong)my_col;+ result += AccumT(down_biases[bias_idx]);++ const ulong out_idx = (ulong)row_idx * (ulong)d_model + (ulong)my_col;+ y_out[out_idx] = T(result);+ }}-#define MOE_PASS_B_FUSED_2D_KERNEL(DTYPE, SUFFIX) \- kernel void moe_experts_decode_down_fused_2d_##SUFFIX( \- device const float* hidden [[buffer(0)]], \- device const uint* row_expert_map [[buffer(1)]], \- device const DTYPE* w2_all [[buffer(2)]], \- device const DTYPE* down_biases [[buffer(3)]], \- device DTYPE* y_out [[buffer(4)]], \- constant uint& total_rows [[buffer(5)]], \- constant uint& d_model [[buffer(6)]], \- constant uint& d_ff [[buffer(7)]], \- constant uint& E [[buffer(8)]], \- uint2 tgpig [[threadgroup_position_in_grid]], \- uint simd_gid [[simdgroup_index_in_threadgroup]], \- uint simd_lid [[thread_index_in_simdgroup]] \- ) { \- moe_experts_decode_down_fused_2d_impl<DTYPE, float>( \- hidden, \- row_expert_map, \- w2_all, \- down_biases, \- y_out, \- total_rows, \- d_model, \- d_ff, \- E, \- tgpig, \- simd_gid, \- simd_lid \- ); \- }+#define MOE_PASS_B_FUSED_2D_KERNEL(DTYPE, SUFFIX) \+kernel void moe_experts_decode_down_fused_2d_##SUFFIX( \+ device const float* hidden [[buffer(0)]], \+ device const uint* row_expert_map [[buffer(1)]], \+ device const DTYPE* w2_all [[buffer(2)]], \+ device const DTYPE* down_biases [[buffer(3)]], \+ device DTYPE* y_out [[buffer(4)]], \+ constant uint& total_rows [[buffer(5)]], \+ constant uint& d_model [[buffer(6)]], \+ constant uint& d_ff [[buffer(7)]], \+ constant uint& E [[buffer(8)]], \+ uint2 tgpig [[threadgroup_position_in_grid]], \+ uint simd_gid [[simdgroup_index_in_threadgroup]], \+ uint simd_lid [[thread_index_in_simdgroup]]) \+{ \+ moe_experts_decode_down_fused_2d_impl<DTYPE, float>( \+ hidden, row_expert_map, w2_all, down_biases, y_out, \+ total_rows, d_model, d_ff, E, \+ tgpig, simd_gid, simd_lid); \+}MOE_PASS_B_FUSED_2D_KERNEL(bfloat, bf16)MOE_PASS_B_FUSED_2D_KERNEL(half, f16)diff --git a/crates/uzu/src/backends/metal/kernel/moe/experts_two_pass_prefill.metal b/crates/uzu/src/backends/metal/kernel/moe/experts_two_pass_prefill.metalindex 03aad6517..9f9a5810d 100644--- a/crates/uzu/src/backends/metal/kernel/moe/experts_two_pass_prefill.metal+++ b/crates/uzu/src/backends/metal/kernel/moe/experts_two_pass_prefill.metal@@ -1,25 +1,7 @@-// UZU MoE Two-Pass Prefill-// - Pass A: X @ W13 → hidden (f32) with gating-// - Pass B: hidden @ W2 → output (T)-//-// X_perm : [total_rows, D]-// W13_all : [E, 2*FF, D] // first FF = up, second FF = gate-// up_biases : [E, 2*FF] // [up | gate]-// hidden_out (f32): [total_rows, FF]-// W2_all : [E, D, FF] // FF contiguous (col-major-ish use below)-// down_biases : [E, D]-// output (T) : [total_rows, D]-//-// Notes:-// - 8x8 simdgroup MMA tiles; threadgroup staging is float.-// - Vectorized float4 cooperative loads.-// - Linearized thread id covers both 1-D and 2/3-D TGs safely.-// - We guard tails; K/N can be non-multiples.-#include <metal_stdlib>#include <metal_simdgroup>#include <metal_simdgroup_matrix>-#include "../quant_matmul/mma.h"+#include "../quant_matmul/mma.h"using namespace metal;@@ -27,52 +9,50 @@ using namespace metal;static inline uint ceil_div(uint a, uint b) { return (a + b - 1u) / b; }static inline uint linear_tid(uint3 tid, uint3 tpg) {- return tid.z * (tpg.x * tpg.y) + tid.y * tpg.x + tid.x;+ return tid.z * (tpg.x * tpg.y) + tid.y * tpg.x + tid.x;}static inline float gelu_approx(float x) {- const float k0 = 0.7978845608f; // sqrt(2/pi)- const float k1 = 0.044715f;- if (x > 10.0f)- return x;- if (x < -10.0f)- return 0.0f;- float x3 = x * x * x;- float t = clamp(k0 * (x + k1 * x3), -10.0f, 10.0f);- return 0.5f * x * (1.0f + tanh(t));+ const float k0 = 0.7978845608f; // sqrt(2/pi)+ const float k1 = 0.044715f;+ if (x > 10.0f) return x;+ if (x < -10.0f) return 0.0f;+ float x3 = x * x * x;+ float t = clamp(k0 * (x + k1 * x3), -10.0f, 10.0f);+ return 0.5f * x * (1.0f + tanh(t));}static inline float silu(float x, float alpha) {- // alpha=1 gives standard SiLU; keep alpha param for parity with your API- return x / (1.0f + exp(-alpha * x));+ // alpha=1 gives standard SiLU; keep alpha param for parity with your API+ return x / (1.0f + exp(-alpha * x));}-template <typename T>+template<typename T>inline void store_vec4(device T* dst, ulong base, float4 vals) {- dst[base + 0] = T(vals.x);- dst[base + 1] = T(vals.y);- dst[base + 2] = T(vals.z);- dst[base + 3] = T(vals.w);+ dst[base + 0] = T(vals.x);+ dst[base + 1] = T(vals.y);+ dst[base + 2] = T(vals.z);+ dst[base + 3] = T(vals.w);}-template <>+template<>inline void store_vec4<float>(device float* dst, ulong base, float4 vals) {- *reinterpret_cast<device float4*>(dst + base) = vals;+ *reinterpret_cast<device float4*>(dst + base) = vals;}-template <typename T>+template<typename T>struct StageStorage {- using type = float;+ using type = float;};-template <>+template<>struct StageStorage<half> {- using type = half;+ using type = half;};-template <>+template<>struct StageStorage<bfloat> {- using type = bfloat;+ using type = bfloat;};// 0=GELU(up), 1=SiLU(up), 2=SwiGLU(gate)*up, 3=GEGLU(gate)*up@@ -85,917 +65,680 @@ constant uint PASSA_BN = 32;constant uint PASSA_BK = 64;constant uint PASSA_SG_BM = 8;constant uint PASSA_SG_BN = 16;+constant uint PASSA_TG_PAD = 4; // Padding to avoid bank conflicts-template <typename T>+template<typename T>inline void pass_a_impl(- device const T* X_perm, // [total_rows, D]- device const uint* expert_offsets, // [E+1]- device const T* W13_all, // [E, 2*FF, D]- device const T* up_biases, // [E, 2*FF] (up | gate)- device float* hidden_out, // [total_rows, FF]- uint d_model,- uint d_ff,- uint E,- float gate_clip_min,- float gate_clip_max,- float up_clip_min,- float up_clip_max,+ device const T* X_perm,+ device const uint* expert_offsets,+ device const T* W13_all,+ device const T* up_biases,+ device float* hidden_out,+ uint d_model, uint d_ff, uint E,+ float gate_clip_min, float gate_clip_max,+ float up_clip_min, float up_clip_max,float silu_alpha,// work partition- uint tile_seg_start,- uint expert_idx,- uint tile_m,- uint tile_n,+ uint tile_seg_start, uint expert_idx, uint tile_m, uint tile_n,// per-thread info- uint sg_id,- uint3 threads_per_tg,- uint3 local_tid,+ uint sg_id, uint3 threads_per_tg, uint3 local_tid,// TG storage- threadgroup typename StageStorage<T>::type* Xs, // [BM,BK]- threadgroup typename StageStorage<T>::type* Wk_up, // [BN,BK]- threadgroup typename StageStorage<T>::type* Wk_gate, // [BN,BK]- threadgroup float* bias_up, // [BN]- threadgroup float* bias_gate // [BN]+ threadgroup typename StageStorage<T>::type* Xs, // [BM,BK]+ threadgroup typename StageStorage<T>::type* Wk_up, // [BN,BK]+ threadgroup typename StageStorage<T>::type* Wk_gate, // [BN,BK]+ threadgroup float* bias_up, // [BN]+ threadgroup float* bias_gate // [BN]) {- using StageT = typename StageStorage<T>::type;- using StageVec4 = metal::vec<StageT, 4>;-- const uint Bm = PASSA_BM, Bn = PASSA_BN, Bk = PASSA_BK;- const uint SgBm = PASSA_SG_BM, SgBn = PASSA_SG_BN;-- if (expert_idx >= E)- return;-- const uint seg_start = expert_offsets[expert_idx];- const uint seg_end = expert_offsets[expert_idx + 1];- const uint seg_len = (seg_end > seg_start) ? (seg_end - seg_start) : 0u;- if (seg_len == 0)- return;-- const uint row_tg_off = tile_m * Bm;- const uint col_tg_off = tile_n * Bn;- if (row_tg_off >= seg_len || col_tg_off >= d_ff)- return;-- const uint m_rows = min(Bm, seg_len - row_tg_off);- const uint n_cols = min(Bn, d_ff - col_tg_off);-- const ulong x_base =- ((ulong)tile_seg_start + (ulong)row_tg_off) * (ulong)d_model;- const ulong w13_expert_base =- (ulong)expert_idx * (ulong)(2 * d_ff) * (ulong)d_model;- const ulong bias_base = (ulong)expert_idx * (ulong)(2 * d_ff);-- // simdgroup tile mapping- const uint sg_col_count = Bn / SgBn;- const uint row_sg = sg_id / sg_col_count;- const uint col_sg = sg_id % sg_col_count;- const uint row_sg_off = row_sg * SgBm;- const uint col_sg_off = col_sg * SgBn;-- // Guard against misconfigured TG sizes- constexpr uint SG_TILE = 8; // simdgroup fragment dimension (8x8)- constexpr uint SG_EXPECTED = (PASSA_BM / SG_TILE) * (PASSA_BN / SG_TILE);- if (sg_id >= SG_EXPECTED)- return;-- // partial accumulators (8x8 tiles per simdgroup)- constexpr uint TEMP = (PASSA_SG_BM / SG_TILE) * (PASSA_SG_BN / SG_TILE);- metal::simdgroup_float8x8 OutUp[TEMP];- metal::simdgroup_float8x8 OutGate[TEMP];- for (uint i = 0; i < TEMP; ++i) {- OutUp[i] = metal::make_filled_simdgroup_matrix<float, 8, 8>(0.0f);- if (GATING_SEL > 1u) {- OutGate[i] = metal::make_filled_simdgroup_matrix<float, 8, 8>(0.0f);- }- }-- const uint threads_total =- threads_per_tg.x * threads_per_tg.y * threads_per_tg.z;- const uint lin = linear_tid(local_tid, threads_per_tg);-- // K-loop (tile along D)- for (uint k_off = 0; k_off < d_model; k_off += Bk) {- const uint valid_k = min(Bk, d_model - k_off);-- threadgroup_barrier(mem_flags::mem_threadgroup);-- // ---- Stage LHS: Xs [Bm,Bk] as float, vectorized 4 ----- {- constexpr uint row_stride = PASSA_BK;- constexpr uint vec_cols = PASSA_BK / 4;- const uint total_vecs = PASSA_BM * vec_cols;- const uint iters = ceil_div(total_vecs, threads_total);-- UZU_PRAGMA_UNROLL- for (uint t = 0; t < iters; ++t) {- const uint i = t * threads_total + lin;- if (i < total_vecs) {- const uint r = i / vec_cols;- const uint c4 = i % vec_cols;-- threadgroup StageVec4* dst4 =- reinterpret_cast<threadgroup StageVec4*>(- Xs + r * row_stride + (c4 << 2)- );-- StageVec4 packed = StageVec4(StageT(0.0f));- if (r < m_rows) {- const ulong base =- x_base + (ulong)r * (ulong)d_model + (ulong)(k_off + (c4 << 2));- const uint base_vec = (c4 << 2);- float v0 = 0.0f, v1 = 0.0f, v2 = 0.0f, v3 = 0.0f;- if (base_vec + 0 < valid_k)- v0 = float(X_perm[base + 0]);- if (base_vec + 1 < valid_k)- v1 = float(X_perm[base + 1]);- if (base_vec + 2 < valid_k)- v2 = float(X_perm[base + 2]);- if (base_vec + 3 < valid_k)- v3 = float(X_perm[base + 3]);- packed = StageVec4(StageT(v0), StageT(v1), StageT(v2), StageT(v3));- }- *dst4 = packed;+ using StageT = typename StageStorage<T>::type;+ using StageVec4 = metal::vec<StageT, 4>;++ constexpr uint Bm = PASSA_BM, Bn = PASSA_BN, Bk = PASSA_BK;+ constexpr uint SgBm = PASSA_SG_BM, SgBn = PASSA_SG_BN;+ constexpr uint TG_PAD = PASSA_TG_PAD;+ constexpr uint X_LD = Bk + TG_PAD; // Padded stride for X+ constexpr uint W_LD = Bk + TG_PAD; // Padded stride for weights++ if (expert_idx >= E) return;++ const uint seg_start = expert_offsets[expert_idx];+ const uint seg_end = expert_offsets[expert_idx + 1];+ const uint seg_len = (seg_end > seg_start) ? (seg_end - seg_start) : 0u;+ if (seg_len == 0) return;++ const uint row_tg_off = tile_m * Bm;+ const uint col_tg_off = tile_n * Bn;+ if (row_tg_off >= seg_len || col_tg_off >= d_ff) return;++ const uint m_rows = min(Bm, seg_len - row_tg_off);+ const uint n_cols = min(Bn, d_ff - col_tg_off);++ const ulong x_base = ((ulong)tile_seg_start + (ulong)row_tg_off) * (ulong)d_model;+ const ulong w13_expert_base = (ulong)expert_idx * (ulong)(2 * d_ff) * (ulong)d_model;+ const ulong bias_base = (ulong)expert_idx * (ulong)(2 * d_ff);++ // simdgroup tile mapping+ const uint sg_col_count = Bn / SgBn;+ const uint row_sg = sg_id / sg_col_count;+ const uint col_sg = sg_id % sg_col_count;+ const uint row_sg_off = row_sg * SgBm;+ const uint col_sg_off = col_sg * SgBn;++ // Guard against misconfigured TG sizes+ constexpr uint SG_TILE = 8; // simdgroup fragment dimension (8x8)+ constexpr uint SG_EXPECTED = (Bm / SG_TILE) * (Bn / SG_TILE);+ if (sg_id >= SG_EXPECTED) return;++ // partial accumulators (8x8 tiles per simdgroup)+ constexpr uint TEMP = (SgBm / SG_TILE) * (SgBn / SG_TILE);+ metal::simdgroup_float8x8 OutUp[TEMP];+ metal::simdgroup_float8x8 OutGate[TEMP];+ for (uint i = 0; i < TEMP; ++i) {+ OutUp[i] = metal::make_filled_simdgroup_matrix<float, 8, 8>(0.0f);+ if (GATING_SEL > 1u) {+ OutGate[i] = metal::make_filled_simdgroup_matrix<float, 8, 8>(0.0f);}- }}- // ---- Stage RHS: W_up [Bn,Bk], W_gate [Bn,Bk], vectorized 4 ----- {- constexpr uint row_stride = PASSA_BK;- constexpr uint vec_cols = PASSA_BK / 4;- const uint total_vecs = PASSA_BN * vec_cols;- const uint iters = ceil_div(total_vecs, threads_total);-- UZU_PRAGMA_UNROLL- for (uint t = 0; t < iters; ++t) {- const uint i = t * threads_total + lin;- if (i < total_vecs) {- const uint n = i / vec_cols; // column within the BN tile- const uint c4 = i % vec_cols;-- threadgroup StageVec4* up4 = reinterpret_cast<threadgroup StageVec4*>(- Wk_up + n * row_stride + (c4 << 2)- );- StageVec4 upv = StageVec4(StageT(0.0f));-- threadgroup StageVec4* gt4 = reinterpret_cast<threadgroup StageVec4*>(- Wk_gate + n * row_stride + (c4 << 2)- );- StageVec4 gtv = StageVec4(StageT(0.0f));-- if (n < n_cols) {- const uint gcol = col_tg_off + n;- const ulong up_base = w13_expert_base +- (ulong)gcol * (ulong)d_model +- (ulong)(k_off + (c4 << 2));- const ulong gate_base = w13_expert_base +- (ulong)(d_ff + gcol) * (ulong)d_model +- (ulong)(k_off + (c4 << 2));- const uint base_vec = (c4 << 2);- float up0 = 0.0f, up1 = 0.0f, up2 = 0.0f, up3 = 0.0f;- float gt0 = 0.0f, gt1 = 0.0f, gt2 = 0.0f, gt3 = 0.0f;- if (base_vec + 0 < valid_k) {- up0 = float(W13_all[up_base + 0]);- gt0 = float(W13_all[gate_base + 0]);+ const uint threads_total = threads_per_tg.x * threads_per_tg.y * threads_per_tg.z;+ const uint lin = linear_tid(local_tid, threads_per_tg);++ // K-loop (tile along D)+ for (uint k_off = 0; k_off < d_model; k_off += Bk) {+ const uint valid_k = min(Bk, d_model - k_off);++ threadgroup_barrier(mem_flags::mem_threadgroup);++ // ---- Stage LHS: Xs [Bm,Bk] with vec8 loads and padding ----+ {+ constexpr uint vec_size = 8;+ constexpr uint vec_cols = Bk / vec_size; // 64/8 = 8+ constexpr uint total_vecs = Bm * vec_cols; // 16*8 = 128+ const uint iters = ceil_div(total_vecs, threads_total);++UZU_PRAGMA_UNROLL+ for (uint t = 0; t < iters; ++t) {+ const uint i = t * threads_total + lin;+ if (i < total_vecs) {+ const uint r = i / vec_cols;+ const uint c8 = i % vec_cols;+ const uint col_base = c8 * vec_size;++ threadgroup StageT* dst = Xs + r * X_LD + col_base;++ if (r < m_rows && col_base < valid_k) {+ // Safe to read - at least first element is valid+ const ulong base = x_base + (ulong)r * (ulong)d_model + (ulong)(k_off + col_base);+UZU_PRAGMA_UNROLL+ for (uint j = 0; j < vec_size; j++) {+ dst[j] = (col_base + j < valid_k) ? StageT(float(X_perm[base + j])) : StageT(0.0f);+ }+ } else {+ // Row out of bounds or entire vec8 chunk is out of K bounds+UZU_PRAGMA_UNROLL+ for (uint j = 0; j < vec_size; j++) {+ dst[j] = StageT(0.0f);+ }+ }+ }}- if (base_vec + 1 < valid_k) {- up1 = float(W13_all[up_base + 1]);- gt1 = float(W13_all[gate_base + 1]);- }- if (base_vec + 2 < valid_k) {- up2 = float(W13_all[up_base + 2]);- gt2 = float(W13_all[gate_base + 2]);- }- if (base_vec + 3 < valid_k) {- up3 = float(W13_all[up_base + 3]);- gt3 = float(W13_all[gate_base + 3]);+ }++ // ---- Stage RHS: W_up [Bn,Bk], W_gate [Bn,Bk] with vec8 loads and padding ----+ {+ constexpr uint vec_size = 8;+ constexpr uint vec_cols = Bk / vec_size; // 64/8 = 8+ constexpr uint total_vecs = Bn * vec_cols; // 32*8 = 256+ const uint iters = ceil_div(total_vecs, threads_total);++UZU_PRAGMA_UNROLL+ for (uint t = 0; t < iters; ++t) {+ const uint i = t * threads_total + lin;+ if (i < total_vecs) {+ const uint n = i / vec_cols; // row within the BN tile+ const uint c8 = i % vec_cols;+ const uint col_base = c8 * vec_size;++ threadgroup StageT* up_dst = Wk_up + n * W_LD + col_base;+ threadgroup StageT* gt_dst = Wk_gate + n * W_LD + col_base;++ if (n < n_cols && col_base < valid_k) {+ // Safe to read - at least first element is valid+ const uint gcol = col_tg_off + n;+ const ulong up_base = w13_expert_base + (ulong)gcol * (ulong)d_model + (ulong)(k_off + col_base);+ const ulong gate_base = w13_expert_base + (ulong)(d_ff + gcol) * (ulong)d_model + (ulong)(k_off + col_base);+UZU_PRAGMA_UNROLL+ for (uint j = 0; j < vec_size; j++) {+ up_dst[j] = (col_base + j < valid_k) ? StageT(float(W13_all[up_base + j])) : StageT(0.0f);+ if (GATING_SEL > 1u) {+ gt_dst[j] = (col_base + j < valid_k) ? StageT(float(W13_all[gate_base + j])) : StageT(0.0f);+ }+ }+ } else {+ // Row out of bounds or entire vec8 chunk is out of K bounds+UZU_PRAGMA_UNROLL+ for (uint j = 0; j < vec_size; j++) {+ up_dst[j] = StageT(0.0f);+ if (GATING_SEL > 1u) gt_dst[j] = StageT(0.0f);+ }+ }+ }}- upv = StageVec4(StageT(up0), StageT(up1), StageT(up2), StageT(up3));- gtv = StageVec4(StageT(gt0), StageT(gt1), StageT(gt2), StageT(gt3));- }+ }- *up4 = upv;- if (GATING_SEL > 1u)- *gt4 = gtv;+ threadgroup_barrier(mem_flags::mem_threadgroup);++ // ---- MMA across the BK tile (in simdgroup-sized chunks) ----+UZU_PRAGMA_UNROLL+ for (uint kk = 0; kk < Bk; kk += SG_TILE) {+UZU_PRAGMA_UNROLL+ for (uint m_sub = 0; m_sub < SgBm; m_sub += SG_TILE) {+ const uint r_idx = m_sub / SG_TILE;+ metal::simdgroup_matrix<StageT, 8, 8> lhs_frag;+ simdgroup_load(lhs_frag, Xs, X_LD, ulong2(kk, row_sg_off + m_sub));++UZU_PRAGMA_UNROLL+ for (uint n_sub = 0; n_sub < SgBn; n_sub += SG_TILE) {+ const uint c_idx = n_sub / SG_TILE;+ const uint tile = r_idx * (SgBn / SG_TILE) + c_idx;+ metal::simdgroup_matrix<StageT, 8, 8> rhs_up;+ simdgroup_load(rhs_up, Wk_up, W_LD, ulong2(kk, col_sg_off + n_sub), true);+ simdgroup_multiply_accumulate(OutUp[tile], lhs_frag, rhs_up, OutUp[tile]);++ if (GATING_SEL > 1u) {+ metal::simdgroup_matrix<StageT, 8, 8> rhs_gate;+ simdgroup_load(rhs_gate, Wk_gate, W_LD, ulong2(kk, col_sg_off + n_sub), true);+ simdgroup_multiply_accumulate(OutGate[tile], lhs_frag, rhs_gate, OutGate[tile]);+ }+ }+ }+ }+ } // K loop++ // ---- Bias tile [Bn] ----+ // Note: bias layout is [up | gate] in contiguous FF chunks.+ for (uint c_local = lin; c_local < Bn; c_local += threads_total) {+ const uint c_global = col_tg_off + c_local;+ bias_up[c_local] = (c_global < d_ff) ? float(up_biases[bias_base + c_global]) : 0.0f;+ if (GATING_SEL > 1u) {+ bias_gate[c_local] = (c_global < d_ff) ? float(up_biases[bias_base + d_ff + c_global]) : 0.0f;}- }}threadgroup_barrier(mem_flags::mem_threadgroup);- // ---- MMA across the BK tile (in simdgroup-sized chunks) ----- UZU_PRAGMA_UNROLL- for (uint kk = 0; kk < PASSA_BK; kk += SG_TILE) {- UZU_PRAGMA_UNROLL- for (uint m_sub = 0; m_sub < PASSA_SG_BM; m_sub += SG_TILE) {- const uint r_idx = m_sub / SG_TILE;- metal::simdgroup_matrix<StageT, 8, 8> lhs_frag;- simdgroup_load(lhs_frag, Xs, PASSA_BK, ulong2(kk, row_sg_off + m_sub));-- UZU_PRAGMA_UNROLL- for (uint n_sub = 0; n_sub < PASSA_SG_BN; n_sub += SG_TILE) {- const uint c_idx = n_sub / SG_TILE;- const uint tile = r_idx * (PASSA_SG_BN / SG_TILE) + c_idx;- metal::simdgroup_matrix<StageT, 8, 8> rhs_up;- simdgroup_load(- rhs_up,- Wk_up,- PASSA_BK,- ulong2(kk, col_sg_off + n_sub),- true- );- simdgroup_multiply_accumulate(- OutUp[tile],- lhs_frag,- rhs_up,- OutUp[tile]- );-- if (GATING_SEL > 1u) {- metal::simdgroup_matrix<StageT, 8, 8> rhs_gate;- simdgroup_load(- rhs_gate,- Wk_gate,- PASSA_BK,- ulong2(kk, col_sg_off + n_sub),- true- );- simdgroup_multiply_accumulate(- OutGate[tile],- lhs_frag,- rhs_gate,- OutGate[tile]- );- }- }- }- }- } // K loop-- // ---- Bias tile [Bn] ----- // Note: bias layout is [up | gate] in contiguous FF chunks.- for (uint c_local = lin; c_local < Bn; c_local += threads_total) {- const uint c_global = col_tg_off + c_local;- bias_up[c_local] =- (c_global < d_ff) ? float(up_biases[bias_base + c_global]) : 0.0f;- if (GATING_SEL > 1u) {- bias_gate[c_local] = (c_global < d_ff)- ? float(up_biases[bias_base + d_ff + c_global])- : 0.0f;- }- }-- threadgroup_barrier(mem_flags::mem_threadgroup);-- // ---- Epilogue: activation/gating → hidden_out (f32) ----- const uint sg_lane_start = sg_id * 32;- const uint sg_lane_end = sg_lane_start + 32;- if (lin < sg_lane_start || lin >= sg_lane_end)- return;- const uint sg_lin = lin - sg_lane_start;- // Lane→(row,col) mapping for 8x8 simdgroup fragments, matches the fragment- // layout used above.- const uint lane_qid = sg_lin >> 2;- const uint lane_row = (lane_qid & 4u) + ((sg_lin >> 1) & 3u);- const uint lane_col_base = ((lane_qid & 2u) << 1) + ((sg_lin & 1u) << 1);-- UZU_PRAGMA_UNROLL- for (uint n_sub = 0; n_sub < PASSA_SG_BN; n_sub += SG_TILE) {- const uint c_idx = n_sub / SG_TILE;- UZU_PRAGMA_UNROLL- for (uint m_sub = 0; m_sub < PASSA_SG_BM; m_sub += SG_TILE) {- const uint r_idx = m_sub / SG_TILE;- const uint tile = r_idx * (PASSA_SG_BN / SG_TILE) + c_idx;-- const uint tile_row_base = row_sg_off + m_sub;- const uint tile_col_base = col_sg_off + n_sub;- const uint local_row = tile_row_base + lane_row;- if (local_row >= m_rows || tile_col_base >= n_cols) {- continue;- }-- const auto up_frag = OutUp[tile].thread_elements();- float gate_frag_0 = 0.0f;- float gate_frag_1 = 0.0f;- if (GATING_SEL > 1u) {- const auto gate_frag = OutGate[tile].thread_elements();- gate_frag_0 = gate_frag[0];- gate_frag_1 = gate_frag[1];- }-- const uint col0 = tile_col_base + lane_col_base;- const uint col1 = col0 + 1;- const ulong out_row = seg_start + row_tg_off + local_row;-- if (col0 < n_cols) {- float up_v =- clamp(up_frag[0] + bias_up[col0], up_clip_min, up_clip_max);- float out_val;- if (GATING_SEL <= 1u) {- out_val =- (GATING_SEL == 0u) ? gelu_approx(up_v) : silu(up_v, silu_alpha);- } else {- float gate_v = clamp(- gate_frag_0 + bias_gate[col0],- gate_clip_min,- gate_clip_max- );- const float gate_act = (GATING_SEL == 2u) ? silu(gate_v, silu_alpha)- : gelu_approx(gate_v);- out_val = gate_act * up_v;- }- const ulong out_col = col_tg_off + col0;- hidden_out[out_row * (ulong)d_ff + out_col] = out_val;- }-- if (col1 < n_cols) {- float up_v =- clamp(up_frag[1] + bias_up[col1], up_clip_min, up_clip_max);- float out_val;- if (GATING_SEL <= 1u) {- out_val =- (GATING_SEL == 0u) ? gelu_approx(up_v) : silu(up_v, silu_alpha);- } else {- float gate_v = clamp(- gate_frag_1 + bias_gate[col1],- gate_clip_min,- gate_clip_max- );- const float gate_act = (GATING_SEL == 2u) ? silu(gate_v, silu_alpha)- : gelu_approx(gate_v);- out_val = gate_act * up_v;+ // ---- Epilogue: activation/gating → hidden_out (f32) ----+ const uint sg_lane_start = sg_id * 32;+ const uint sg_lane_end = sg_lane_start + 32;+ if (lin < sg_lane_start || lin >= sg_lane_end) return;+ const uint sg_lin = lin - sg_lane_start;+ // Lane→(row,col) mapping for 8x8 simdgroup fragments, matches the fragment layout used above.+ const uint lane_qid = sg_lin >> 2;+ const uint lane_row = (lane_qid & 4u) + ((sg_lin >> 1) & 3u);+ const uint lane_col_base = ((lane_qid & 2u) << 1) + ((sg_lin & 1u) << 1);++UZU_PRAGMA_UNROLL+ for (uint n_sub = 0; n_sub < SgBn; n_sub += SG_TILE) {+ const uint c_idx = n_sub / SG_TILE;+UZU_PRAGMA_UNROLL+ for (uint m_sub = 0; m_sub < SgBm; m_sub += SG_TILE) {+ const uint r_idx = m_sub / SG_TILE;+ const uint tile = r_idx * (SgBn / SG_TILE) + c_idx;++ const uint tile_row_base = row_sg_off + m_sub;+ const uint tile_col_base = col_sg_off + n_sub;+ const uint local_row = tile_row_base + lane_row;+ if (local_row >= m_rows || tile_col_base >= n_cols) {+ continue;+ }++ const auto up_frag = OutUp[tile].thread_elements();+ float gate_frag_0 = 0.0f;+ float gate_frag_1 = 0.0f;+ if (GATING_SEL > 1u) {+ const auto gate_frag = OutGate[tile].thread_elements();+ gate_frag_0 = gate_frag[0];+ gate_frag_1 = gate_frag[1];+ }++ const uint col0 = tile_col_base + lane_col_base;+ const uint col1 = col0 + 1;+ const ulong out_row = seg_start + row_tg_off + local_row;++ if (col0 < n_cols) {+ float up_v = clamp(up_frag[0] + bias_up[col0], up_clip_min, up_clip_max);+ float out_val;+ if (GATING_SEL <= 1u) {+ out_val =+ (GATING_SEL == 0u) ? gelu_approx(up_v) : silu(up_v, silu_alpha);+ } else {+ float gate_v =+ clamp(gate_frag_0 + bias_gate[col0], gate_clip_min, gate_clip_max);+ const float gate_act =+ (GATING_SEL == 2u) ? silu(gate_v, silu_alpha) : gelu_approx(gate_v);+ out_val = gate_act * up_v;+ }+ const ulong out_col = col_tg_off + col0;+ hidden_out[out_row * (ulong)d_ff + out_col] = out_val;+ }++ if (col1 < n_cols) {+ float up_v = clamp(up_frag[1] + bias_up[col1], up_clip_min, up_clip_max);+ float out_val;+ if (GATING_SEL <= 1u) {+ out_val =+ (GATING_SEL == 0u) ? gelu_approx(up_v) : silu(up_v, silu_alpha);+ } else {+ float gate_v =+ clamp(gate_frag_1 + bias_gate[col1], gate_clip_min, gate_clip_max);+ const float gate_act =+ (GATING_SEL == 2u) ? silu(gate_v, silu_alpha) : gelu_approx(gate_v);+ out_val = gate_act * up_v;+ }+ const ulong out_col = col_tg_off + col1;+ hidden_out[out_row * (ulong)d_ff + out_col] = out_val;+ }}- const ulong out_col = col_tg_off + col1;- hidden_out[out_row * (ulong)d_ff + out_col] = out_val;- }}- }}// ----- Kernel entry points (Pass A) ------#define MOE_PASS_A_KERNEL(DTYPE, SUFFIX) \- kernel void moe_two_pass_prefill_pass_a_##SUFFIX( \- device const DTYPE* X_perm [[buffer(0)]], \- device const uint* expert_offsets [[buffer(1)]], \- device const DTYPE* W13_all [[buffer(2)]], \- device const DTYPE* up_biases [[buffer(3)]], \- device float* hidden_out [[buffer(4)]], \- constant uint& d_model [[buffer(5)]], \- constant uint& d_ff [[buffer(6)]], \- constant uint& E [[buffer(7)]], \- constant float& gate_clip_min [[buffer(8)]], \- constant float& gate_clip_max [[buffer(9)]], \- constant float& up_clip_min [[buffer(10)]], \- constant float& up_clip_max [[buffer(11)]], \- constant float& silu_alpha [[buffer(12)]], \- uint sg_id [[simdgroup_index_in_threadgroup]], \- uint3 threads_per_tg [[threads_per_threadgroup]], \- uint3 tg_pos [[threadgroup_position_in_grid]], \- uint3 local_tid [[thread_position_in_threadgroup]] \- ) { \- using StageT = typename StageStorage<DTYPE>::type; \- threadgroup StageT Xs[PASSA_BM * PASSA_BK]; \- threadgroup StageT Wk_up[PASSA_BN * PASSA_BK]; \- threadgroup StageT Wk_gate[PASSA_BN * PASSA_BK]; \- threadgroup float bias_up[PASSA_BN]; \- threadgroup float bias_gate[PASSA_BN]; \- pass_a_impl<DTYPE>( \- X_perm, \- expert_offsets, \- W13_all, \- up_biases, \- hidden_out, \- d_model, \- d_ff, \- E, \- gate_clip_min, \- gate_clip_max, \- up_clip_min, \- up_clip_max, \- silu_alpha, \- /*tile_seg_start*/ expert_offsets[tg_pos.z], \- /*expert*/ tg_pos.z, \- /*tile_m*/ tg_pos.y, \- /*tile_n*/ tg_pos.x, \- sg_id, \- threads_per_tg, \- local_tid, \- Xs, \- Wk_up, \- Wk_gate, \- bias_up, \- bias_gate \- ); \- }+#define MOE_PASS_A_KERNEL(DTYPE, SUFFIX) \+kernel void moe_two_pass_prefill_pass_a_##SUFFIX( \+ device const DTYPE* X_perm [[buffer(0)]], \+ device const uint* expert_offsets [[buffer(1)]], \+ device const DTYPE* W13_all [[buffer(2)]], \+ device const DTYPE* up_biases [[buffer(3)]], \+ device float* hidden_out [[buffer(4)]], \+ constant uint& d_model [[buffer(5)]], \+ constant uint& d_ff [[buffer(6)]], \+ constant uint& E [[buffer(7)]], \+ constant float& gate_clip_min [[buffer(8)]], \+ constant float& gate_clip_max [[buffer(9)]], \+ constant float& up_clip_min [[buffer(10)]], \+ constant float& up_clip_max [[buffer(11)]], \+ constant float& silu_alpha [[buffer(12)]], \+ uint sg_id [[simdgroup_index_in_threadgroup]], \+ uint3 threads_per_tg [[threads_per_threadgroup]], \+ uint3 tg_pos [[threadgroup_position_in_grid]], \+ uint3 local_tid [[thread_position_in_threadgroup]]) \+{ \+ using StageT = typename StageStorage<DTYPE>::type; \+ threadgroup StageT Xs[PASSA_BM * (PASSA_BK + PASSA_TG_PAD)]; \+ threadgroup StageT Wk_up[PASSA_BN * (PASSA_BK + PASSA_TG_PAD)]; \+ threadgroup StageT Wk_gate[PASSA_BN * (PASSA_BK + PASSA_TG_PAD)]; \+ threadgroup float bias_up[PASSA_BN]; \+ threadgroup float bias_gate[PASSA_BN]; \+ pass_a_impl<DTYPE>( \+ X_perm, expert_offsets, W13_all, up_biases, hidden_out, \+ d_model, d_ff, E, \+ gate_clip_min, gate_clip_max, up_clip_min, up_clip_max, silu_alpha, \+ expert_offsets[tg_pos.z], tg_pos.z, tg_pos.y, tg_pos.x, \+ sg_id, threads_per_tg, local_tid, \+ Xs, Wk_up, Wk_gate, bias_up, bias_gate); \+}MOE_PASS_A_KERNEL(bfloat, bf16)MOE_PASS_A_KERNEL(half, f16)MOE_PASS_A_KERNEL(float, f32)// Indirect variant (consumes [expert, seg_start, tile_row_offset] triples)-template <typename T>+template<typename T>inline void pass_a_indirect_impl(device const T* X_perm,device const uint* expert_offsets,device const T* W13_all,device const T* up_biases,device float* hidden_out,- device const uint*- tile_map, // [tiles * 3]: (expert, seg_start, row_offset_elems)- uint d_model,- uint d_ff,- uint E,- float gate_clip_min,- float gate_clip_max,- float up_clip_min,- float up_clip_max,+ device const uint* tile_map,+ uint d_model, uint d_ff, uint E,+ float gate_clip_min, float gate_clip_max,+ float up_clip_min, float up_clip_max,float silu_alpha,- uint row_tile_idx,- uint n_tile_idx,- uint sg_id,- uint3 threads_per_tg,- uint3 local_tid,+ uint row_tile_idx, uint n_tile_idx,+ uint sg_id, uint3 threads_per_tg, uint3 local_tid,threadgroup typename StageStorage<T>::type* Xs,threadgroup typename StageStorage<T>::type* Wk_up,threadgroup typename StageStorage<T>::type* Wk_gate,threadgroup float* bias_up,- threadgroup float* bias_gate-) {- constexpr uint ROW_TILE = PASSA_BM;- const uint base = row_tile_idx * 3u;- const uint expert_idx = tile_map[base + 0u];- if (expert_idx >= E)- return;- const uint tile_seg_start = tile_map[base + 1u];- const uint row_off_elems = tile_map[base + 2u];- const uint tile_m = row_off_elems / ROW_TILE;-- pass_a_impl<T>(- X_perm,- expert_offsets,- W13_all,- up_biases,- hidden_out,- d_model,- d_ff,- E,- gate_clip_min,- gate_clip_max,- up_clip_min,- up_clip_max,- silu_alpha,- tile_seg_start,- expert_idx,- tile_m,- n_tile_idx,- sg_id,- threads_per_tg,- local_tid,- Xs,- Wk_up,- Wk_gate,- bias_up,- bias_gate- );+ threadgroup float* bias_gate)+{+ constexpr uint ROW_TILE = PASSA_BM;+ const uint base = row_tile_idx * 3u;+ const uint expert_idx = tile_map[base + 0u];+ if (expert_idx >= E) return;+ const uint tile_seg_start = tile_map[base + 1u];+ const uint row_off_elems = tile_map[base + 2u];+ const uint tile_m = row_off_elems / ROW_TILE;++ pass_a_impl<T>(+ X_perm, expert_offsets, W13_all, up_biases, hidden_out,+ d_model, d_ff, E,+ gate_clip_min, gate_clip_max, up_clip_min, up_clip_max, silu_alpha,+ tile_seg_start, expert_idx, tile_m, n_tile_idx,+ sg_id, threads_per_tg, local_tid,+ Xs, Wk_up, Wk_gate, bias_up, bias_gate);}-#define MOE_PASS_A_INDIRECT_KERNEL(DTYPE, SUFFIX) \- kernel void moe_two_pass_prefill_pass_a_indirect_##SUFFIX( \- device const DTYPE* X_perm [[buffer(0)]], \- device const uint* expert_offsets [[buffer(1)]], \- device const DTYPE* W13_all [[buffer(2)]], \- device const DTYPE* up_biases [[buffer(3)]], \- device float* hidden_out [[buffer(4)]], \- constant uint& d_model [[buffer(5)]], \- constant uint& d_ff [[buffer(6)]], \- constant uint& E [[buffer(7)]], \- constant float& gate_clip_min [[buffer(8)]], \- constant float& gate_clip_max [[buffer(9)]], \- constant float& up_clip_min [[buffer(10)]], \- constant float& up_clip_max [[buffer(11)]], \- constant float& silu_alpha [[buffer(12)]], \- device const uint* tile_map [[buffer(13)]], \- uint sg_id [[simdgroup_index_in_threadgroup]], \- uint3 threads_per_tg [[threads_per_threadgroup]], \- uint3 tg_pos [[threadgroup_position_in_grid]], \- uint3 local_tid [[thread_position_in_threadgroup]] \- ) { \- using StageT = typename StageStorage<DTYPE>::type; \- threadgroup StageT Xs[PASSA_BM * PASSA_BK]; \- threadgroup StageT Wk_up[PASSA_BN * PASSA_BK]; \- threadgroup StageT Wk_gate[PASSA_BN * PASSA_BK]; \- threadgroup float bias_up[PASSA_BN]; \- threadgroup float bias_gate[PASSA_BN]; \- pass_a_indirect_impl<DTYPE>( \- X_perm, \- expert_offsets, \- W13_all, \- up_biases, \- hidden_out, \- tile_map, \- d_model, \- d_ff, \- E, \- gate_clip_min, \- gate_clip_max, \- up_clip_min, \- up_clip_max, \- silu_alpha, \- /*row_tile_idx*/ tg_pos.y, \- /*n_tile_idx*/ tg_pos.x, \- sg_id, \- threads_per_tg, \- local_tid, \- Xs, \- Wk_up, \- Wk_gate, \- bias_up, \- bias_gate \- ); \- }+#define MOE_PASS_A_INDIRECT_KERNEL(DTYPE, SUFFIX) \+kernel void moe_two_pass_prefill_pass_a_indirect_##SUFFIX( \+ device const DTYPE* X_perm [[buffer(0)]], \+ device const uint* expert_offsets [[buffer(1)]], \+ device const DTYPE* W13_all [[buffer(2)]], \+ device const DTYPE* up_biases [[buffer(3)]], \+ device float* hidden_out [[buffer(4)]], \+ constant uint& d_model [[buffer(5)]], \+ constant uint& d_ff [[buffer(6)]], \+ constant uint& E [[buffer(7)]], \+ constant float& gate_clip_min [[buffer(8)]], \+ constant float& gate_clip_max [[buffer(9)]], \+ constant float& up_clip_min [[buffer(10)]], \+ constant float& up_clip_max [[buffer(11)]], \+ constant float& silu_alpha [[buffer(12)]], \+ device const uint* tile_map [[buffer(13)]], \+ uint sg_id [[simdgroup_index_in_threadgroup]], \+ uint3 threads_per_tg [[threads_per_threadgroup]], \+ uint3 tg_pos [[threadgroup_position_in_grid]], \+ uint3 local_tid [[thread_position_in_threadgroup]]) \+{ \+ using StageT = typename StageStorage<DTYPE>::type; \+ threadgroup StageT Xs[PASSA_BM * (PASSA_BK + PASSA_TG_PAD)]; \+ threadgroup StageT Wk_up[PASSA_BN * (PASSA_BK + PASSA_TG_PAD)]; \+ threadgroup StageT Wk_gate[PASSA_BN * (PASSA_BK + PASSA_TG_PAD)]; \+ threadgroup float bias_up[PASSA_BN]; \+ threadgroup float bias_gate[PASSA_BN]; \+ pass_a_indirect_impl<DTYPE>( \+ X_perm, expert_offsets, W13_all, up_biases, hidden_out, tile_map, \+ d_model, d_ff, E, \+ gate_clip_min, gate_clip_max, up_clip_min, up_clip_max, silu_alpha, \+ tg_pos.y, tg_pos.x, \+ sg_id, threads_per_tg, local_tid, \+ Xs, Wk_up, Wk_gate, bias_up, bias_gate); \+}MOE_PASS_A_INDIRECT_KERNEL(bfloat, bf16)MOE_PASS_A_INDIRECT_KERNEL(half, f16)MOE_PASS_A_INDIRECT_KERNEL(float, f32)-// ------------------------ Pass B (hidden @ W2 → output)-// ------------------------+// ------------------------ Pass B (hidden @ W2 → output) ------------------------+// Optimized config: 16x64 output tile, 4 simdgroups (128 threads)constant uint PASSB_BM = 16;constant uint PASSB_BN = 64;constant uint PASSB_BK = 64;-constant uint PASSB_SG_BM = 8;+constant uint PASSB_SG_BM = 8; // Each simdgroup handles 8x32constant uint PASSB_SG_BN = 32;+constant uint PASSB_TG_PAD = 4; // Padding to avoid bank conflicts-template <typename T>+template<typename T>inline void pass_b_impl(- device const float* hidden, // [total_rows, FF]- device const uint* expert_offsets, // [E+1]- device const T* W2_all, // [E, D, FF] (FF contiguous)- device const T* down_biases, // [E, D]- device T* output, // [total_rows, D]- uint d_model,- uint d_ff,- uint E,- uint expert_idx,- uint tile_m,- uint tile_n,- uint sg_id,- uint3 threads_per_tg,- uint3 local_tid,- threadgroup float* Hs, // [BM,BK]- threadgroup typename StageStorage<T>::type* Wk, // [BN,BK]- threadgroup float* bias_tile // [BN]+ device const float* hidden,+ device const uint* expert_offsets,+ device const T* W2_all,+ device const T* down_biases,+ device T* output,+ uint d_model, uint d_ff, uint E,+ uint expert_idx, uint tile_m, uint tile_n,+ uint sg_id, uint simd_lid, uint lin,+ threadgroup float* Hs,+ threadgroup typename StageStorage<T>::type* Wk,+ threadgroup float* bias_tile) {- using StageT = typename StageStorage<T>::type;- using StageVec4 = metal::vec<StageT, 4>;-- const uint Bm = PASSB_BM, Bn = PASSB_BN, Bk = PASSB_BK;- const uint SgBm = PASSB_SG_BM, SgBn = PASSB_SG_BN;-- if (expert_idx >= E)- return;-- const uint seg_start = expert_offsets[expert_idx];- const uint seg_end = expert_offsets[expert_idx + 1];- const uint seg_len = (seg_end > seg_start) ? (seg_end - seg_start) : 0u;- if (seg_len == 0)- return;-- const uint row_tg_off = tile_m * Bm;- const uint col_tg_off = tile_n * Bn;- if (row_tg_off >= seg_len || col_tg_off >= d_model)- return;-- const uint m_rows = min(Bm, seg_len - row_tg_off);- const uint n_cols = min(Bn, d_model - col_tg_off);-- const ulong h_base = (ulong)(seg_start + row_tg_off) * (ulong)d_ff;- const ulong w2_expert_base = (ulong)expert_idx * (ulong)d_model * (ulong)d_ff;- const ulong bias_base = (ulong)expert_idx * (ulong)d_model;-- const uint sg_col_count = Bn / SgBn;- const uint row_sg = sg_id / sg_col_count;- const uint col_sg = sg_id % sg_col_count;- const uint row_sg_off = row_sg * SgBm;- const uint col_sg_off = col_sg * SgBn;-- constexpr uint SG_TILE = 8;- constexpr uint TEMP = (PASSB_SG_BM / SG_TILE) * (PASSB_SG_BN / SG_TILE);- metal::simdgroup_float8x8 Out[TEMP];- for (uint i = 0; i < TEMP; ++i) {- Out[i] = metal::make_filled_simdgroup_matrix<float, 8, 8>(0.0f);- }-- const uint threads_total =- threads_per_tg.x * threads_per_tg.y * threads_per_tg.z;- const uint lin = linear_tid(local_tid, threads_per_tg);-- // K-loop across FF- for (uint k_off = 0; k_off < d_ff; k_off += Bk) {- const uint valid_k = min(Bk, d_ff - k_off);+ using StageT = typename StageStorage<T>::type;++ constexpr uint Bm = PASSB_BM, Bn = PASSB_BN, Bk = PASSB_BK;+ constexpr uint SgBm = PASSB_SG_BM, SgBn = PASSB_SG_BN;+ constexpr uint TG_PAD = PASSB_TG_PAD;+ constexpr uint H_LD = Bk + TG_PAD; // Padded stride for hidden+ constexpr uint W_LD = Bk + TG_PAD; // Padded stride for weights+ constexpr uint SG_TILE = 8;+ constexpr uint TGP_SIZE = 128; // 4 simdgroups × 32 threads++ if (expert_idx >= E) return;++ const uint seg_start = expert_offsets[expert_idx];+ const uint seg_end = expert_offsets[expert_idx + 1];+ const uint seg_len = (seg_end > seg_start) ? (seg_end - seg_start) : 0u;+ if (seg_len == 0) return;++ const uint row_tg_off = tile_m * Bm;+ const uint col_tg_off = tile_n * Bn;+ if (row_tg_off >= seg_len || col_tg_off >= d_model) return;++ const uint m_rows = min(Bm, seg_len - row_tg_off);+ const uint n_cols = min(Bn, d_model - col_tg_off);++ const ulong h_base = (ulong)(seg_start + row_tg_off) * (ulong)d_ff;+ const ulong w2_expert_base = (ulong)expert_idx * (ulong)d_model * (ulong)d_ff;+ const ulong bias_base = (ulong)expert_idx * (ulong)d_model;++ // 2×2 simdgroup layout for 16×64 output (each sg handles 8×32)+ const uint row_sg = sg_id / 2; // 0-1 → rows+ const uint col_sg = sg_id % 2; // 0-1 → cols+ const uint row_sg_off = row_sg * SgBm;+ const uint col_sg_off = col_sg * SgBn;++ // Guard against excessive simdgroups+ constexpr uint SG_EXPECTED = (PASSB_BM / SgBm) * (PASSB_BN / SgBn);+ if (sg_id >= SG_EXPECTED) return;++ // 4 accumulators per simdgroup: 1×4 layout of 8×8 tiles+ constexpr uint TEMP = (SgBm / SG_TILE) * (SgBn / SG_TILE); // 1×4 = 4+ metal::simdgroup_float8x8 Out[TEMP];+UZU_PRAGMA_UNROLL+ for (uint i = 0; i < TEMP; ++i) {+ Out[i] = metal::make_filled_simdgroup_matrix<float, 8, 8>(0.0f);+ }- threadgroup_barrier(mem_flags::mem_threadgroup);+ // Cooperative load config - vec8 loads for better bandwidth+ constexpr uint H_VEC = 8;+ constexpr uint H_TCOLS = Bk / H_VEC; // 64/8 = 8+ constexpr uint H_TROWS = TGP_SIZE / H_TCOLS; // 128/8 = 16+ const uint h_bi = lin / H_TCOLS;+ const uint h_bj = (lin % H_TCOLS) * H_VEC;++ constexpr uint W_VEC = 8;+ constexpr uint W_TCOLS = Bk / W_VEC; // 64/8 = 8+ constexpr uint W_TROWS = TGP_SIZE / W_TCOLS; // 128/8 = 16+ const uint w_bi = lin / W_TCOLS;+ const uint w_bj = (lin % W_TCOLS) * W_VEC;++ // K-loop across FF+ for (uint k_off = 0; k_off < d_ff; k_off += Bk) {+ const uint valid_k = min(Bk, d_ff - k_off);+ threadgroup_barrier(mem_flags::mem_threadgroup);++ // ---- Stage hidden [Bm=16, Bk=64] with vec8 loads ----+ // 128 threads, 8 vec8 loads per row → 16 rows covered in one pass+ {+ threadgroup float* my_dst = Hs + h_bi * H_LD + h_bj;+ if (h_bi < m_rows && h_bj < valid_k) {+ // Safe to read - at least first element is valid+ device const float* my_src = hidden + h_base + (ulong)h_bi * (ulong)d_ff + (ulong)(k_off + h_bj);+UZU_PRAGMA_UNROLL+ for (uint j = 0; j < H_VEC; j++) {+ my_dst[j] = (h_bj + j < valid_k) ? my_src[j] : 0.0f;+ }+ } else {+UZU_PRAGMA_UNROLL+ for (uint j = 0; j < H_VEC; j++) {+ my_dst[j] = 0.0f;+ }+ }+ }- // ---- Stage hidden [Bm,Bk] as float4 ----- {- constexpr uint row_stride = PASSB_BK;- constexpr uint vec_cols = PASSB_BK / 4;- const uint total_vecs = PASSB_BM * vec_cols;- const uint iters = ceil_div(total_vecs, threads_total);-- UZU_PRAGMA_UNROLL- for (uint t = 0; t < iters; ++t) {- const uint i = t * threads_total + lin;- if (i < total_vecs) {- const uint r = i / vec_cols;- const uint c4 = i % vec_cols;-- threadgroup float4* dst4 = reinterpret_cast<threadgroup float4*>(- Hs + r * row_stride + (c4 << 2)- );-- float4 v = float4(0.0f);- if (r < m_rows) {- const uint base_vec = (c4 << 2);- const ulong base =- h_base + (ulong)r * (ulong)d_ff + (ulong)(k_off + base_vec);- if (base_vec + 0u < valid_k)- v.x = hidden[base + 0];- if (base_vec + 1u < valid_k)- v.y = hidden[base + 1];- if (base_vec + 2u < valid_k)- v.z = hidden[base + 2];- if (base_vec + 3u < valid_k)- v.w = hidden[base + 3];- }- *dst4 = v;+ // ---- Stage W2 [Bn=64, Bk=64] with vec8 loads ----+ // Need 4 passes: 128 threads cover 16 rows, Bn=64 needs 4×16+ {+ for (uint pass = 0; pass < 4; pass++) {+ const uint row = w_bi + pass * W_TROWS;+ threadgroup StageT* my_dst = Wk + row * W_LD + w_bj;+ if (row < n_cols && w_bj < valid_k) {+ // Safe to read - at least first element is valid+ const uint gcol = col_tg_off + row;+ device const T* my_src = W2_all + w2_expert_base + (ulong)gcol * (ulong)d_ff + (ulong)(k_off + w_bj);+UZU_PRAGMA_UNROLL+ for (uint j = 0; j < W_VEC; j++) {+ my_dst[j] = (w_bj + j < valid_k) ? StageT(float(my_src[j])) : StageT(0.0f);+ }+ } else {+UZU_PRAGMA_UNROLL+ for (uint j = 0; j < W_VEC; j++) {+ my_dst[j] = StageT(0.0f);+ }+ }+ }}- }- }- // ---- Stage W2 [Bn,Bk] as StageT (column-major for simd transpose) ----- {- constexpr uint row_stride = PASSB_BK;- constexpr uint vec_cols = PASSB_BK / 4;- const uint total_vecs = PASSB_BN * vec_cols;- const uint iters = ceil_div(total_vecs, threads_total);-- UZU_PRAGMA_UNROLL- for (uint t = 0; t < iters; ++t) {- const uint i = t * threads_total + lin;- if (i < total_vecs) {- const uint n = i / vec_cols;- const uint c4 = i % vec_cols;-- threadgroup StageVec4* dst4 =- reinterpret_cast<threadgroup StageVec4*>(- Wk + n * row_stride + (c4 << 2)- );-- StageVec4 packed = StageVec4(StageT(0.0f));- if (n < n_cols) {- const uint gcol = col_tg_off + n;- const ulong base = w2_expert_base + (ulong)gcol * (ulong)d_ff +- (ulong)(k_off + (c4 << 2));- const uint base_vec = (c4 << 2);- float v0 = 0.0f, v1 = 0.0f, v2 = 0.0f, v3 = 0.0f;- if (base_vec + 0u < valid_k)- v0 = float(W2_all[base + 0]);- if (base_vec + 1u < valid_k)- v1 = float(W2_all[base + 1]);- if (base_vec + 2u < valid_k)- v2 = float(W2_all[base + 2]);- if (base_vec + 3u < valid_k)- v3 = float(W2_all[base + 3]);- packed = StageVec4(StageT(v0), StageT(v1), StageT(v2), StageT(v3));- }- *dst4 = packed;+ threadgroup_barrier(mem_flags::mem_threadgroup);++ // ---- Serpentine MMA: 1×4 tiles per simdgroup ----+UZU_PRAGMA_UNROLL+ for (uint kk = 0; kk < Bk; kk += SG_TILE) {+ // Load 1 LHS fragment+ metal::simdgroup_float8x8 lhs;+ simdgroup_load(lhs, Hs, H_LD, ulong2(kk, row_sg_off));++ // Load 4 RHS fragments (cols)+ metal::simdgroup_matrix<StageT, 8, 8> rhs0, rhs1, rhs2, rhs3;+ simdgroup_load(rhs0, Wk, W_LD, ulong2(kk, col_sg_off), true);+ simdgroup_load(rhs1, Wk, W_LD, ulong2(kk, col_sg_off + 8), true);+ simdgroup_load(rhs2, Wk, W_LD, ulong2(kk, col_sg_off + 16), true);+ simdgroup_load(rhs3, Wk, W_LD, ulong2(kk, col_sg_off + 24), true);++ // Serpentine pattern: 0→1→2→3 (forward for first row)+ simdgroup_multiply_accumulate(Out[0], lhs, rhs0, Out[0]);+ simdgroup_multiply_accumulate(Out[1], lhs, rhs1, Out[1]);+ simdgroup_multiply_accumulate(Out[2], lhs, rhs2, Out[2]);+ simdgroup_multiply_accumulate(Out[3], lhs, rhs3, Out[3]);}- }+ }++ // ---- Bias tile (cooperative load) ----+ for (uint c_local = lin; c_local < Bn; c_local += TGP_SIZE) {+ const uint c_global = col_tg_off + c_local;+ bias_tile[c_local] = (c_global < d_model) ? float(down_biases[bias_base + c_global]) : 0.0f;}threadgroup_barrier(mem_flags::mem_threadgroup);- // ---- MMA ----- UZU_PRAGMA_UNROLL- for (uint kk = 0; kk < PASSB_BK; kk += SG_TILE) {- UZU_PRAGMA_UNROLL- for (uint m_sub = 0; m_sub < PASSB_SG_BM; m_sub += SG_TILE) {- const uint r_idx = m_sub / SG_TILE;- metal::simdgroup_float8x8 lhs;- simdgroup_load(lhs, Hs, PASSB_BK, ulong2(kk, row_sg_off + m_sub));- UZU_PRAGMA_UNROLL- for (uint n_sub = 0; n_sub < PASSB_SG_BN; n_sub += SG_TILE) {- const uint c_idx = n_sub / SG_TILE;- const uint tile = r_idx * (PASSB_SG_BN / SG_TILE) + c_idx;- metal::simdgroup_matrix<StageT, 8, 8> rhs;- simdgroup_load(- rhs,- Wk,- PASSB_BK,- ulong2(kk, col_sg_off + n_sub),- true- );- simdgroup_multiply_accumulate(Out[tile], lhs, rhs, Out[tile]);+ // ---- Writeout with fragment extraction ----+ const uint lane_qid = simd_lid >> 2;+ const uint lane_row = (lane_qid & 4u) + ((simd_lid >> 1) & 3u);+ const uint lane_col_base = ((lane_qid & 2u) << 1) + ((simd_lid & 1u) << 1);++ const uint local_row = row_sg_off + lane_row;+ if (local_row >= m_rows) return;+ const ulong out_row = seg_start + row_tg_off + local_row;++UZU_PRAGMA_UNROLL+ for (uint ni = 0; ni < 4; ni++) { // 4 col tiles per simdgroup+ const auto accum = Out[ni].thread_elements();+ const uint local_col = col_sg_off + ni * 8 + lane_col_base;++ if (local_col < n_cols) {+ const ulong out_col = col_tg_off + local_col;+ const float val = accum[0] + bias_tile[local_col];+ output[out_row * (ulong)d_model + out_col] = T(val);+ }+ if (local_col + 1 < n_cols) {+ const ulong out_col = col_tg_off + local_col + 1;+ const float val = accum[1] + bias_tile[local_col + 1];+ output[out_row * (ulong)d_model + out_col] = T(val);}- }- }- }-- // ---- Bias tile ----- for (uint c_local = lin; c_local < Bn; c_local += threads_total) {- const uint c_global = col_tg_off + c_local;- bias_tile[c_local] =- (c_global < d_model) ? float(down_biases[bias_base + c_global]) : 0.0f;- }-- threadgroup_barrier(mem_flags::mem_threadgroup);-- // ---- Writeout ----- const uint sg_lane_start = sg_id * 32;- const uint sg_lane_end = sg_lane_start + 32;- if (lin < sg_lane_start || lin >= sg_lane_end)- return;- const uint sg_lin = lin - sg_lane_start;-- // Lane→(row,col) mapping for 8x8 simdgroup fragments, matches PASSB fragment- // packing.- const uint lane_qid = sg_lin >> 2;- const uint lane_row = (lane_qid & 4u) + ((sg_lin >> 1) & 3u);- const uint lane_col_base = ((lane_qid & 2u) << 1) + ((sg_lin & 1u) << 1);-- UZU_PRAGMA_UNROLL- for (uint n_sub = 0; n_sub < PASSB_SG_BN; n_sub += SG_TILE) {- const uint c_idx = n_sub / SG_TILE;- UZU_PRAGMA_UNROLL- for (uint m_sub = 0; m_sub < PASSB_SG_BM; m_sub += SG_TILE) {- const uint r_idx = m_sub / SG_TILE;- const uint tile = r_idx * (PASSB_SG_BN / SG_TILE) + c_idx;-- const uint tile_row_base = row_sg_off + m_sub;- const uint tile_col_base = col_sg_off + n_sub;- const uint local_row = tile_row_base + lane_row;- if (local_row >= m_rows || tile_col_base >= n_cols) {- continue;- }-- const auto accum = Out[tile].thread_elements();-- const uint col0 = tile_col_base + lane_col_base;- const uint col1 = col0 + 1u;- const ulong out_row = seg_start + row_tg_off + local_row;-- if (col0 < n_cols) {- const ulong out_col = col_tg_off + col0;- const float val = accum[0] + bias_tile[col0];- output[out_row * (ulong)d_model + out_col] = T(val);- }-- if (col1 < n_cols) {- const ulong out_col = col_tg_off + col1;- const float val = accum[1] + bias_tile[col1];- output[out_row * (ulong)d_model + out_col] = T(val);- }}- }}// ----- Kernel entry points (Pass B) ------#define MOE_PASS_B_KERNEL(DTYPE, SUFFIX) \- kernel void moe_two_pass_prefill_pass_b_##SUFFIX( \- device const float* hidden [[buffer(0)]], \- device const uint* expert_offsets [[buffer(1)]], \- device const DTYPE* W2_all [[buffer(2)]], \- device const DTYPE* down_biases [[buffer(3)]], \- device DTYPE* output [[buffer(4)]], \- constant uint& d_model [[buffer(5)]], \- constant uint& d_ff [[buffer(6)]], \- constant uint& E [[buffer(7)]], \- uint sg_id [[simdgroup_index_in_threadgroup]], \- uint3 threads_per_tg [[threads_per_threadgroup]], \- uint3 tg_pos [[threadgroup_position_in_grid]], \- uint3 local_tid [[thread_position_in_threadgroup]] \- ) { \- using StageT = typename StageStorage<DTYPE>::type; \- threadgroup float Hs[PASSB_BM * PASSB_BK]; \- threadgroup StageT Wk[PASSB_BN * PASSB_BK]; \- threadgroup float bias[PASSB_BN]; \- pass_b_impl<DTYPE>( \- hidden, \- expert_offsets, \- W2_all, \- down_biases, \- output, \- d_model, \- d_ff, \- E, \- /*expert*/ tg_pos.z, \- /*tile_m*/ tg_pos.y, \- /*tile_n*/ tg_pos.x, \- sg_id, \- threads_per_tg, \- local_tid, \- Hs, \- Wk, \- bias \- ); \- }+#define MOE_PASS_B_KERNEL(DTYPE, SUFFIX) \+kernel void moe_two_pass_prefill_pass_b_##SUFFIX( \+ device const float* hidden [[buffer(0)]], \+ device const uint* expert_offsets [[buffer(1)]], \+ device const DTYPE* W2_all [[buffer(2)]], \+ device const DTYPE* down_biases [[buffer(3)]], \+ device DTYPE* output [[buffer(4)]], \+ constant uint& d_model [[buffer(5)]], \+ constant uint& d_ff [[buffer(6)]], \+ constant uint& E [[buffer(7)]], \+ uint sg_id [[simdgroup_index_in_threadgroup]], \+ uint simd_lid [[thread_index_in_simdgroup]], \+ uint3 tg_pos [[threadgroup_position_in_grid]], \+ uint3 local_tid [[thread_position_in_threadgroup]]) \+{ \+ using StageT = typename StageStorage<DTYPE>::type; \+ threadgroup float Hs[PASSB_BM * (PASSB_BK + PASSB_TG_PAD)]; \+ threadgroup StageT Wk[PASSB_BN * (PASSB_BK + PASSB_TG_PAD)]; \+ threadgroup float bias[PASSB_BN]; \+ const uint lin = local_tid.x; \+ pass_b_impl<DTYPE>( \+ hidden, expert_offsets, W2_all, down_biases, output, \+ d_model, d_ff, E, \+ tg_pos.z, tg_pos.y, tg_pos.x, \+ sg_id, simd_lid, lin, \+ Hs, Wk, bias); \+}MOE_PASS_B_KERNEL(bfloat, bf16)MOE_PASS_B_KERNEL(half, f16)MOE_PASS_B_KERNEL(float, f32)// Indirect variant-template <typename T>+template<typename T>inline void pass_b_indirect_impl(device const float* hidden,device const uint* expert_offsets,device const T* W2_all,device const T* down_biases,device T* output,- device const uint* tile_map, // [tiles * 3]- uint d_model,- uint d_ff,- uint E,- uint row_tile_idx,- uint n_tile_idx,- uint sg_id,- uint3 threads_per_tg,- uint3 local_tid,+ device const uint* tile_map,+ uint d_model, uint d_ff, uint E,+ uint row_tile_idx, uint n_tile_idx,+ uint sg_id, uint simd_lid, uint lin,threadgroup float* Hs,threadgroup typename StageStorage<T>::type* Wk,- threadgroup float* bias_tile-) {- constexpr uint ROW_TILE = PASSB_BM;- const uint base = row_tile_idx * 3u;- const uint expert_idx = tile_map[base + 0u];- if (expert_idx >= E)- return;- const uint row_off_elems = tile_map[base + 2u];- const uint tile_m = row_off_elems / ROW_TILE;-- pass_b_impl<T>(- hidden,- expert_offsets,- W2_all,- down_biases,- output,- d_model,- d_ff,- E,- expert_idx,- tile_m,- n_tile_idx,- sg_id,- threads_per_tg,- local_tid,- Hs,- Wk,- bias_tile- );+ threadgroup float* bias_tile)+{+ constexpr uint ROW_TILE = PASSB_BM;+ const uint base = row_tile_idx * 3u;+ const uint expert_idx = tile_map[base + 0u];+ if (expert_idx >= E) return;+ const uint row_off_elems = tile_map[base + 2u];+ const uint tile_m = row_off_elems / ROW_TILE;++ pass_b_impl<T>(+ hidden, expert_offsets, W2_all, down_biases, output,+ d_model, d_ff, E,+ expert_idx, tile_m, n_tile_idx,+ sg_id, simd_lid, lin,+ Hs, Wk, bias_tile);}-#define MOE_PASS_B_INDIRECT_KERNEL(DTYPE, SUFFIX) \- kernel void moe_two_pass_prefill_pass_b_indirect_##SUFFIX( \- device const float* hidden [[buffer(0)]], \- device const uint* expert_offsets [[buffer(1)]], \- device const DTYPE* W2_all [[buffer(2)]], \- device const DTYPE* down_biases [[buffer(3)]], \- device DTYPE* output [[buffer(4)]], \- constant uint& d_model [[buffer(5)]], \- constant uint& d_ff [[buffer(6)]], \- constant uint& E [[buffer(7)]], \- device const uint* tile_map [[buffer(8)]], \- uint sg_id [[simdgroup_index_in_threadgroup]], \- uint3 threads_per_tg [[threads_per_threadgroup]], \- uint3 tg_pos [[threadgroup_position_in_grid]], \- uint3 local_tid [[thread_position_in_threadgroup]] \- ) { \- using StageT = typename StageStorage<DTYPE>::type; \- threadgroup float Hs[PASSB_BM * PASSB_BK]; \- threadgroup StageT Wk[PASSB_BN * PASSB_BK]; \- threadgroup float bias[PASSB_BN]; \- pass_b_indirect_impl<DTYPE>( \- hidden, \- expert_offsets, \- W2_all, \- down_biases, \- output, \- tile_map, \- d_model, \- d_ff, \- E, \- /*row_tile_idx*/ tg_pos.y, \- /*n_tile_idx*/ tg_pos.x, \- sg_id, \- threads_per_tg, \- local_tid, \- Hs, \- Wk, \- bias \- ); \- }+#define MOE_PASS_B_INDIRECT_KERNEL(DTYPE, SUFFIX) \+kernel void moe_two_pass_prefill_pass_b_indirect_##SUFFIX( \+ device const float* hidden [[buffer(0)]], \+ device const uint* expert_offsets [[buffer(1)]], \+ device const DTYPE* W2_all [[buffer(2)]], \+ device const DTYPE* down_biases [[buffer(3)]], \+ device DTYPE* output [[buffer(4)]], \+ constant uint& d_model [[buffer(5)]], \+ constant uint& d_ff [[buffer(6)]], \+ constant uint& E [[buffer(7)]], \+ device const uint* tile_map [[buffer(8)]], \+ uint sg_id [[simdgroup_index_in_threadgroup]], \+ uint simd_lid [[thread_index_in_simdgroup]], \+ uint3 tg_pos [[threadgroup_position_in_grid]], \+ uint3 local_tid [[thread_position_in_threadgroup]]) \+{ \+ using StageT = typename StageStorage<DTYPE>::type; \+ threadgroup float Hs[PASSB_BM * (PASSB_BK + PASSB_TG_PAD)]; \+ threadgroup StageT Wk[PASSB_BN * (PASSB_BK + PASSB_TG_PAD)]; \+ threadgroup float bias[PASSB_BN]; \+ const uint lin = local_tid.x; \+ pass_b_indirect_impl<DTYPE>( \+ hidden, expert_offsets, W2_all, down_biases, output, tile_map, \+ d_model, d_ff, E, \+ tg_pos.y, tg_pos.x, \+ sg_id, simd_lid, lin, \+ Hs, Wk, bias); \+}MOE_PASS_B_INDIRECT_KERNEL(bfloat, bf16)MOE_PASS_B_INDIRECT_KERNEL(half, f16)diff --git a/crates/uzu/src/backends/metal/kernel/moe/mod.rs b/crates/uzu/src/backends/metal/kernel/moe/mod.rsindex 1fb3fd783..9dc1853ba 100644--- a/crates/uzu/src/backends/metal/kernel/moe/mod.rs+++ b/crates/uzu/src/backends/metal/kernel/moe/mod.rs@@ -18,7 +18,8 @@ pub use counts_offsets_fused::{MoeCountsOffsetsFusedKernel,};pub use experts::{- MoeExpertsArguments, MoeExpertsError, MoeExpertsTwoPassArguments,+ MoeExpertsArguments, MoeExpertsError, MoeExpertsSingleDecodeArguments,+ MoeExpertsSingleDecodeKernel, MoeExpertsTwoPassArguments,MoeExpertsTwoPassDecodeKernel, MoeExpertsTwoPassPrefillKernel,MoeScatterError,};diff --git a/crates/uzu/tests/moe/moe_experts_perf_test.rs b/crates/uzu/tests/moe/moe_experts_perf_test.rsindex 8f705b296..03a639388 100644--- a/crates/uzu/tests/moe/moe_experts_perf_test.rs+++ b/crates/uzu/tests/moe/moe_experts_perf_test.rs@@ -5,6 +5,7 @@ use rand::{Rng, SeedableRng, rngs::StdRng};use uzu::backends::metal::{KernelDataType, MTLContext,kernel::moe::{+ MoeExpertsSingleDecodeArguments, MoeExpertsSingleDecodeKernel,MoeExpertsTwoPassArguments, MoeExpertsTwoPassDecodeKernel,MoeExpertsTwoPassPrefillKernel,},@@ -346,6 +347,7 @@ fn test_two_pass_prefill_speed() {let cases = vec![("T32_E16_D1024_F4096", 32, 1024, 4096, 16, 4, 2, 10),("T64_E16_D2048_F6144", 64, 2048, 6144, 16, 4, 2, 10),+ ("T256_E16_D2048_F6144", 256, 2048, 6144, 16, 4, 2, 10),];for (name, t, d_model, d_ff, e, k, warmup, iters) in cases {@@ -354,3 +356,378 @@ fn test_two_pass_prefill_speed() {);}}++fn run_fused_single_token_case(+ ctx: &MTLContext,+ name: &str,+ d_model: usize,+ d_ff: usize,+ e: usize,+ k: usize,+ warmup: usize,+ iters: usize,+) {+ let mut rng = StdRng::seed_from_u64(0xF053ED);++ eprintln!(+ "\n[fused single-token] {} => D={}, FF={}, E={}, K={}",+ name, d_model, d_ff, e, k+ );++ let x: Vec<bf16> = (0..d_model)+ .map(|_| bf16::from_f32(rng.random_range(-1.0..1.0)))+ .collect();++ let topk_ids: Vec<i32> = (0..k).map(|i| (i % e) as i32).collect();++ let topk_probs: Vec<bf16> = {+ let raw: Vec<f32> =+ (0..k).map(|_| rng.random_range(0.1..1.0)).collect();+ let sum: f32 = raw.iter().sum();+ raw.iter().map(|p| bf16::from_f32(p / sum)).collect()+ };++ let w13_all: Vec<bf16> = (0..e * 2 * d_ff * d_model)+ .map(|_| bf16::from_f32(rng.random_range(-0.05..0.05)))+ .collect();+ let w2_all: Vec<bf16> = (0..e * d_ff * d_model)+ .map(|_| bf16::from_f32(rng.random_range(-0.05..0.05)))+ .collect();+ let up_biases: Vec<bf16> = (0..e * 2 * d_ff)+ .map(|_| bf16::from_f32(rng.random_range(-0.01..0.01)))+ .collect();+ let down_biases: Vec<bf16> = (0..e * d_model)+ .map(|_| bf16::from_f32(rng.random_range(-0.01..0.01)))+ .collect();++ let fused_kernel =+ MoeExpertsSingleDecodeKernel::new(ctx).expect("fused kernel");++ let x_buf = alloc_buffer_with_data(ctx, &x);+ let topk_ids_buf = alloc_buffer_with_data(ctx, &topk_ids);+ let topk_probs_buf = alloc_buffer_with_data(ctx, &topk_probs);+ let w13_buf = alloc_buffer_with_data(ctx, &w13_all);+ let w2_buf = alloc_buffer_with_data(ctx, &w2_all);+ let up_biases_buf = alloc_buffer_with_data(ctx, &up_biases);+ let down_biases_buf = alloc_buffer_with_data(ctx, &down_biases);+ let hidden_buf = alloc_buffer::<f32>(ctx, k * d_ff);+ let y_buf = alloc_buffer::<bf16>(ctx, d_model);++ let make_args = || MoeExpertsSingleDecodeArguments {+ x: &x_buf,+ topk_ids: &topk_ids_buf,+ topk_probs: &topk_probs_buf,+ w13_all: &w13_buf,+ w2_all: &w2_buf,+ up_biases: &up_biases_buf,+ down_biases: &down_biases_buf,+ hidden: &hidden_buf,+ y: &y_buf,+ d_model,+ d_ff,+ k,+ gating_code: 2, // SwiGLU+ silu_alpha: 1.0,+ gate_clip_min: f32::NEG_INFINITY,+ gate_clip_max: f32::INFINITY,+ up_clip_min: f32::NEG_INFINITY,+ up_clip_max: f32::INFINITY,+ data_type: KernelDataType::BFloat16,+ };++ for _ in 0..warmup {+ let cb = ctx.command_queue.new_command_buffer();+ fused_kernel.encode(&cb, make_args()).expect("encode");+ cb.commit();+ cb.wait_until_completed();+ }++ let mut times = Vec::with_capacity(iters);+ for _ in 0..iters {+ let start = Instant::now();+ let cb = ctx.command_queue.new_command_buffer();+ fused_kernel.encode(&cb, make_args()).expect("encode");+ cb.commit();+ cb.wait_until_completed();+ times.push(start.elapsed().as_secs_f64() * 1000.0);+ }++ times.sort_by(|a, b| a.partial_cmp(b).unwrap());+ let mean = times.iter().sum::<f64>() / times.len() as f64;+ let median = times[times.len() / 2];+ let min = times[0];+ let max = times[times.len() - 1];+ let var = times.iter().map(|t| (t - mean).powi(2)).sum::<f64>()+ / times.len() as f64;+ let std = var.sqrt();++ eprintln!(+ " mean={:.3}ms median={:.3}ms min={:.3}ms max={:.3}ms std={:.3}ms",+ mean, median, min, max, std+ );+ eprintln!(" → Latency: {:.1} µs/token", mean * 1000.0);+}++#[test]+#[ignore]+fn test_fused_single_token_speed() {+ let ctx = create_ctx();++ let cases = vec![+ ("E16_D1024_F4096_K4", 1024, 4096, 16, 4, 3, 20),+ ("E8_D2048_F8192_K4", 2048, 8192, 8, 4, 3, 20),+ ("E8_D512_F2048_K2", 512, 2048, 8, 2, 3, 20),+ ];++ for (name, d_model, d_ff, e, k, warmup, iters) in cases {+ run_fused_single_token_case(+ &ctx, name, d_model, d_ff, e, k, warmup, iters,+ );+ }+}++/// Compare indirect decode vs fused decode for single-token (T=1)+#[test]+#[ignore]+fn test_single_token_indirect_vs_fused() {+ let ctx = create_ctx();++ let cases = vec![+ ("E8_D512_F2048_K2", 512, 2048, 8, 2),+ ("E16_D1024_F4096_K4", 1024, 4096, 16, 4),+ ("E8_D2048_F8192_K4", 2048, 8192, 8, 4),+ ];++ eprintln!("\n=== Single-Token Decode: Indirect vs Fused ===\n");++ for (name, d_model, d_ff, e, k) in cases {+ let warmup = 3;+ let iters = 20;++ eprintln!("[{}] D={}, FF={}, E={}, K={}", name, d_model, d_ff, e, k);++ // Run indirect decode (T=1)+ let indirect_mean = run_indirect_decode_timed(+ &ctx, 1, d_model, d_ff, e, k, warmup, iters,+ );++ // Run fused decode+ let fused_mean =+ run_fused_decode_timed(&ctx, d_model, d_ff, e, k, warmup, iters);++ let speedup = indirect_mean / fused_mean;+ eprintln!(+ " Indirect: {:.3}ms, Fused: {:.3}ms, Speedup: {:.2}x\n",+ indirect_mean, fused_mean, speedup+ );+ }+}++fn run_indirect_decode_timed(+ ctx: &MTLContext,+ t: usize,+ d_model: usize,+ d_ff: usize,+ e: usize,+ k: usize,+ warmup: usize,+ iters: usize,+) -> f64 {+ const K_TILE: usize = 64;++ let mut rng = StdRng::seed_from_u64(0xDEC0DE1234567890);+ let sum_k = t * k;++ let x_perm: Vec<bf16> = (0..sum_k * d_model)+ .map(|_| bf16::from_f32(rng.random_range(-1.0..1.0)))+ .collect();+ let offsets = build_offsets(e, sum_k);++ let w13_original: Vec<bf16> = (0..e * d_model * 2 * d_ff)+ .map(|_| bf16::from_f32(rng.random_range(-0.05..0.05)))+ .collect();++ let mut w13 = vec![bf16::from_f32(0.0); e * d_model * 2 * d_ff];+ for expert in 0..e {+ let src_offset = expert * d_model * 2 * d_ff;+ let dst_offset = expert * 2 * d_ff * d_model;+ for dm in 0..d_model {+ for ff in 0..(2 * d_ff) {+ let src_idx = src_offset + dm * 2 * d_ff + ff;+ let dst_idx = dst_offset + ff * d_model + dm;+ w13[dst_idx] = w13_original[src_idx];+ }+ }+ }+ let w2: Vec<bf16> = (0..e * d_ff * d_model)+ .map(|_| bf16::from_f32(rng.random_range(-0.05..0.05)))+ .collect();+ let up_biases: Vec<bf16> = (0..e * 2 * d_ff)+ .map(|_| bf16::from_f32(rng.random_range(-0.01..0.01)))+ .collect();+ let down_biases: Vec<bf16> = (0..e * d_model)+ .map(|_| bf16::from_f32(rng.random_range(-0.01..0.01)))+ .collect();++ let experts_kernel =+ MoeExpertsTwoPassDecodeKernel::new(ctx).expect("decode kernel");++ let x_perm_buf = alloc_buffer_with_data(ctx, &x_perm);+ let offsets_buf = alloc_buffer_with_data(ctx, &offsets);+ let w13_buf = alloc_buffer_with_data(ctx, &w13);+ let w2_buf = alloc_buffer_with_data(ctx, &w2);+ let up_biases_buf = alloc_buffer_with_data(ctx, &up_biases);+ let down_biases_buf = alloc_buffer_with_data(ctx, &down_biases);++ let num_tiles_k = ((d_ff + K_TILE - 1) / K_TILE) as usize;+ let hidden_buf = alloc_buffer::<f32>(ctx, sum_k * d_ff);+ let output_buf = alloc_buffer::<bf16>(ctx, sum_k * d_model);++ let tile_counts_buf = alloc_buffer::<u32>(ctx, e);+ let tile_offsets_buf = alloc_buffer::<u32>(ctx, e + 1);+ let max_tiles = e * sum_k * 16384;+ let tile_map_buf = alloc_buffer::<u32>(ctx, max_tiles * 3);+ let total_tiles_buf = alloc_buffer::<u32>(ctx, 1);+ let dispatch_args_buf = alloc_buffer::<u32>(ctx, 3);+ let row_expert_map_buf = alloc_buffer::<u32>(ctx, sum_k);++ let make_args = || MoeExpertsTwoPassArguments {+ x_perm_buffer: &x_perm_buf,+ expert_offsets: &offsets_buf,+ row_expert_map: &row_expert_map_buf,+ hidden_buffer: &hidden_buf,+ output_buffer: &output_buf,+ w13_all: &w13_buf,+ w2_all: &w2_buf,+ up_biases: &up_biases_buf,+ down_biases: &down_biases_buf,+ tile_counts: &tile_counts_buf,+ tile_offsets: &tile_offsets_buf,+ tile_map: &tile_map_buf,+ total_tiles: &total_tiles_buf,+ dispatch_args: &dispatch_args_buf,+ total_rows: sum_k,+ d_model,+ d_ff,+ e,+ num_tiles_k: num_tiles_k as u32,+ gating_code: 2,+ gate_clip_min: f32::NEG_INFINITY,+ gate_clip_max: f32::INFINITY,+ up_clip_min: f32::NEG_INFINITY,+ up_clip_max: f32::INFINITY,+ silu_alpha: 1.0,+ data_type: KernelDataType::BFloat16,+ };++ for _ in 0..warmup {+ let cb = ctx.command_queue.new_command_buffer();+ experts_kernel.encode(&cb, make_args()).expect("encode");+ cb.commit();+ cb.wait_until_completed();+ }++ let mut times = Vec::with_capacity(iters);+ for _ in 0..iters {+ let start = Instant::now();+ let cb = ctx.command_queue.new_command_buffer();+ experts_kernel.encode(&cb, make_args()).expect("encode");+ cb.commit();+ cb.wait_until_completed();+ times.push(start.elapsed().as_secs_f64() * 1000.0);+ }++ times.iter().sum::<f64>() / times.len() as f64+}++fn run_fused_decode_timed(+ ctx: &MTLContext,+ d_model: usize,+ d_ff: usize,+ e: usize,+ k: usize,+ warmup: usize,+ iters: usize,+) -> f64 {+ let mut rng = StdRng::seed_from_u64(0xF053ED);++ let x: Vec<bf16> = (0..d_model)+ .map(|_| bf16::from_f32(rng.random_range(-1.0..1.0)))+ .collect();++ let topk_ids: Vec<i32> = (0..k).map(|i| (i % e) as i32).collect();++ let topk_probs: Vec<bf16> = {+ let raw: Vec<f32> =+ (0..k).map(|_| rng.random_range(0.1..1.0)).collect();+ let sum: f32 = raw.iter().sum();+ raw.iter().map(|p| bf16::from_f32(p / sum)).collect()+ };++ let w13_all: Vec<bf16> = (0..e * 2 * d_ff * d_model)+ .map(|_| bf16::from_f32(rng.random_range(-0.05..0.05)))+ .collect();+ let w2_all: Vec<bf16> = (0..e * d_ff * d_model)+ .map(|_| bf16::from_f32(rng.random_range(-0.05..0.05)))+ .collect();+ let up_biases: Vec<bf16> = (0..e * 2 * d_ff)+ .map(|_| bf16::from_f32(rng.random_range(-0.01..0.01)))+ .collect();+ let down_biases: Vec<bf16> = (0..e * d_model)+ .map(|_| bf16::from_f32(rng.random_range(-0.01..0.01)))+ .collect();++ let fused_kernel =+ MoeExpertsSingleDecodeKernel::new(ctx).expect("fused kernel");++ let x_buf = alloc_buffer_with_data(ctx, &x);+ let topk_ids_buf = alloc_buffer_with_data(ctx, &topk_ids);+ let topk_probs_buf = alloc_buffer_with_data(ctx, &topk_probs);+ let w13_buf = alloc_buffer_with_data(ctx, &w13_all);+ let w2_buf = alloc_buffer_with_data(ctx, &w2_all);+ let up_biases_buf = alloc_buffer_with_data(ctx, &up_biases);+ let down_biases_buf = alloc_buffer_with_data(ctx, &down_biases);+ let hidden_buf = alloc_buffer::<f32>(ctx, k * d_ff);+ let y_buf = alloc_buffer::<bf16>(ctx, d_model);++ let make_args = || MoeExpertsSingleDecodeArguments {+ x: &x_buf,+ topk_ids: &topk_ids_buf,+ topk_probs: &topk_probs_buf,+ w13_all: &w13_buf,+ w2_all: &w2_buf,+ up_biases: &up_biases_buf,+ down_biases: &down_biases_buf,+ hidden: &hidden_buf,+ y: &y_buf,+ d_model,+ d_ff,+ k,+ gating_code: 2,+ silu_alpha: 1.0,+ gate_clip_min: f32::NEG_INFINITY,+ gate_clip_max: f32::INFINITY,+ up_clip_min: f32::NEG_INFINITY,+ up_clip_max: f32::INFINITY,+ data_type: KernelDataType::BFloat16,+ };++ for _ in 0..warmup {+ let cb = ctx.command_queue.new_command_buffer();+ fused_kernel.encode(&cb, make_args()).expect("encode");+ cb.commit();+ cb.wait_until_completed();+ }++ let mut times = Vec::with_capacity(iters);+ for _ in 0..iters {+ let start = Instant::now();+ let cb = ctx.command_queue.new_command_buffer();+ fused_kernel.encode(&cb, make_args()).expect("encode");+ cb.commit();+ cb.wait_until_completed();+ times.push(start.elapsed().as_secs_f64() * 1000.0);+ }++ times.iter().sum::<f64>() / times.len() as f64+}diff --git a/crates/uzu/tests/moe/moe_experts_test.rs b/crates/uzu/tests/moe/moe_experts_test.rsindex 7aab2d2fe..ef1bb14af 100644--- a/crates/uzu/tests/moe/moe_experts_test.rs+++ b/crates/uzu/tests/moe/moe_experts_test.rs@@ -13,6 +13,7 @@ use rand::{Rng, SeedableRng, rngs::StdRng};use uzu::backends::metal::{KernelDataType,kernel::moe::{+ MoeExpertsSingleDecodeArguments, MoeExpertsSingleDecodeKernel,MoeExpertsTwoPassArguments, MoeExpertsTwoPassDecodeKernel,MoeExpertsTwoPassPrefillKernel,},@@ -25,62 +26,163 @@ use test_utils::{cpu_tile_scan, create_ctx,};-/// Build expert segment offsets - distributes sum_k rows across e experts-fn build_offsets(+/// Test data for MoE experts+struct MoeTestData {+ x: Vec<bf16>,+ topk_ids: Vec<i32>,+ topk_probs: Vec<bf16>,+ w13: Vec<bf16>,+ w2: Vec<bf16>,+ up_biases: Vec<bf16>,+ down_biases: Vec<bf16>,+}++impl MoeTestData {+ fn generate(rng: &mut StdRng, t: usize, k: usize, d_model: usize, d_ff: usize, e: usize) -> Self {+ let x: Vec<bf16> = (0..t * d_model)+ .map(|_| bf16::from_f32(rng.random_range(-1.0..1.0)))+ .collect();++ let topk_ids: Vec<i32> = (0..t * k)+ .map(|i| ((i / k) + (i % k) * 2) as i32 % e as i32)+ .collect();++ let topk_probs: Vec<bf16> = (0..t)+ .flat_map(|_| {+ let raw: Vec<f32> = (0..k).map(|_| rng.random_range(0.1..1.0)).collect();+ let sum: f32 = raw.iter().sum();+ raw.iter().map(|p| bf16::from_f32(p / sum)).collect::<Vec<_>>()+ })+ .collect();++ let w13: Vec<bf16> = (0..e * 2 * d_ff * d_model)+ .map(|_| bf16::from_f32(rng.random_range(-0.05..0.05)))+ .collect();+ let w2: Vec<bf16> = (0..e * d_ff * d_model)+ .map(|_| bf16::from_f32(rng.random_range(-0.05..0.05)))+ .collect();+ let up_biases: Vec<bf16> = (0..e * 2 * d_ff)+ .map(|_| bf16::from_f32(rng.random_range(-0.01..0.01)))+ .collect();+ let down_biases: Vec<bf16> = (0..e * d_model)+ .map(|_| bf16::from_f32(rng.random_range(-0.01..0.01)))+ .collect();++ Self { x, topk_ids, topk_probs, w13, w2, up_biases, down_biases }+ }+}++/// Result of scatter operation - buckets rows by expert+struct ScatterResult {+ x_perm: Vec<bf16>,+ offsets: Vec<u32>,+ row_expert_map: Vec<u32>,+ perm_idx: Vec<usize>, // original row -> bucketed row (for gather)+}++fn scatter_by_expert(+ x: &[bf16],+ topk_ids: &[i32],+ t: usize,+ k: usize,+ d_model: usize,e: usize,- sum_k: usize,-) -> Vec<u32> {+) -> ScatterResult {+ let sum_k = t * k;++ // Count rows per expert and build offsets+ let mut expert_counts = vec![0usize; e];+ for &eid in topk_ids {+ expert_counts[eid as usize] += 1;+ }let mut offsets = vec![0u32; e + 1];- let base_rows = sum_k / e;- let extra = sum_k % e;+ for i in 0..e {+ offsets[i + 1] = offsets[i] + expert_counts[i] as u32;+ }++ // Build permutation indices+ let mut expert_cursors = vec![0usize; e];+ let mut perm_idx = vec![0usize; sum_k];+ let mut inv_perm = vec![0usize; sum_k];+ for orig_row in 0..sum_k {+ let eid = topk_ids[orig_row] as usize;+ let bucket_pos = offsets[eid] as usize + expert_cursors[eid];+ perm_idx[orig_row] = bucket_pos;+ inv_perm[bucket_pos] = orig_row;+ expert_cursors[eid] += 1;+ }- for i in 0..=e {- offsets[i] = (base_rows * i + extra.min(i)) as u32;+ // Build x_perm and row_expert_map in bucketed order+ let mut x_perm = vec![bf16::from_f32(0.0); sum_k * d_model];+ let mut row_expert_map = vec![0u32; sum_k];+ for bucket_row in 0..sum_k {+ let orig_row = inv_perm[bucket_row];+ let tok = orig_row / k;+ for d in 0..d_model {+ x_perm[bucket_row * d_model + d] = x[tok * d_model + d];+ }+ row_expert_map[bucket_row] = topk_ids[orig_row] as u32;}- offsets++ ScatterResult { x_perm, offsets, row_expert_map, perm_idx }}-/// Build row-expert map: maps each row to its expert index-fn build_row_expert_map(- offsets: &[u32],- sum_k: usize,-) -> Vec<u32> {- let e = offsets.len() - 1;- let mut row_map = vec![0u32; sum_k];-- for expert in 0..e {- let start = offsets[expert] as usize;- let end = offsets[expert + 1] as usize;- for row in start..end {- row_map[row] = expert as u32;+/// Gather + finalize: reorder from bucketed order and apply weighted sum+fn gather_and_finalize(+ y_partial: &[bf16],+ topk_probs: &[bf16],+ perm_idx: &[usize],+ t: usize,+ k: usize,+ d_model: usize,+) -> Vec<bf16> {+ let mut y = vec![bf16::from_f32(0.0); t * d_model];+ for tok in 0..t {+ for d in 0..d_model {+ let mut acc = 0.0f32;+ for ki in 0..k {+ let orig_row = tok * k + ki;+ let bucket_row = perm_idx[orig_row];+ let prob = f32::from(topk_probs[orig_row]);+ let val = f32::from(y_partial[bucket_row * d_model + d]);+ acc += prob * val;+ }+ y[tok * d_model + d] = bf16::from_f32(acc);}}- row_map+ y}-/// CPU reference for full expert computation matching GPU implementation+/// Unified CPU reference for MoE computation+///+/// Computes: y[t] = Σ_k prob[t,k] * (activation(x[t] @ W13[expert_k]) @ W2[expert_k] + bias)////// Supports 4 gating modes:/// - 0: GELU(up)/// - 1: SiLU(up)/// - 2: SwiGLU = SiLU(gate) * up/// - 3: GEGLU = GELU(gate) * up-fn cpu_expert_ffn_full(- x_perm: &[bf16], // [sum_k, d_model]- offsets: &[u32], // [E+1]- w13: &[bf16], // [E, 2*d_ff, d_model] transposed- w2: &[bf16], // [E, d_model, d_ff] transposed+#[allow(clippy::too_many_arguments)]+fn cpu_moe_reference(+ x: &[bf16], // [T, d_model]+ topk_ids: &[i32], // [T * K]+ topk_probs: &[bf16], // [T * K]+ w13_all: &[bf16], // [E, 2*d_ff, d_model] transposed+ w2_all: &[bf16], // [E, d_model, d_ff] transposedup_biases: &[bf16], // [E, 2*d_ff]down_biases: &[bf16], // [E, d_model]+ t: usize,d_model: usize,d_ff: usize,+ k: usize,gating_code: u32,silu_alpha: f32,-) -> (Vec<f32>, Vec<bf16>) {- let e = offsets.len() - 1;- let sum_k = x_perm.len() / d_model;- let mut hidden_full = vec![0.0f32; sum_k * d_ff];- let mut output = vec![bf16::from_f32(0.0); sum_k * d_model];+ gate_clip_min: f32,+ gate_clip_max: f32,+ up_clip_min: f32,+ up_clip_max: f32,+) -> Vec<bf16> {+ let mut y = vec![bf16::from_f32(0.0); t * d_model];// Helper functions matching Metal kernelslet gelu_approx = |x: f32| -> f32 {@@ -100,78 +202,97 @@ fn cpu_expert_ffn_full(let silu = |x: f32, alpha: f32| -> f32 { x / (1.0 + (-alpha * x).exp()) };- for expert in 0..e {- let row_start = offsets[expert] as usize;- let row_end = offsets[expert + 1] as usize;- let expert_rows = row_end - row_start;+ // Process each token+ for tok in 0..t {+ let x_offset = tok * d_model;- if expert_rows == 0 {- continue;- }+ // Hidden buffer for this token's K experts [K, d_ff]+ let mut hidden = vec![0.0f32; k * d_ff];- let w1_offset = expert * 2 * d_ff * d_model;- let w2_offset = expert * d_model * d_ff;-- // FC1: [expert_rows, d_model] @ [d_model, 2*d_ff] -> [expert_rows, 2*d_ff]- let mut hidden = vec![bf16::from_f32(0.0); expert_rows * 2 * d_ff];- for row in 0..expert_rows {- for ff in 0..(2 * d_ff) {- let mut acc = f32::from(up_biases[expert * 2 * d_ff + ff]);- for dm in 0..d_model {- let x_val =- f32::from(x_perm[(row_start + row) * d_model + dm]);- // w13 transposed: [2*d_ff, d_model]- let w_val = f32::from(w13[w1_offset + ff * d_model + dm]);- acc += x_val * w_val;- }- hidden[row * 2 * d_ff + ff] = bf16::from_f32(acc);+ // Pass A: x @ W13[expert] -> hidden[k] with activation+ for k_idx in 0..k {+ let expert_id = topk_ids[tok * k + k_idx];+ if expert_id < 0 {+ continue;}- }-- // Apply activation based on gating_code- for row in 0..expert_rows {- for ff in 0..d_ff {- let up = f32::from(hidden[row * 2 * d_ff + ff]);+ let expert = expert_id as usize;++ let w13_base = expert * 2 * d_ff * d_model;+ let bias_base = expert * 2 * d_ff;++ for h in 0..d_ff {+ // Up projection+ let mut acc_up = f32::from(up_biases[bias_base + h]);+ for d in 0..d_model {+ let x_val = f32::from(x[x_offset + d]);+ let w_val = f32::from(w13_all[w13_base + h * d_model + d]);+ acc_up += x_val * w_val;+ }- let result = if gating_code <= 1 {- // GELU(up) or SiLU(up) - no gating+ // Apply activation based on gating_code+ let activated = if gating_code <= 1 {+ // GELU or SiLU on up (with clipping)+ let up_val = acc_up.clamp(up_clip_min, up_clip_max);if gating_code == 0 {- gelu_approx(up)+ gelu_approx(up_val)} else {- silu(up, silu_alpha)+ silu(up_val, silu_alpha)}} else {- // SwiGLU or GEGLU - gate * up- let gate = f32::from(hidden[row * 2 * d_ff + d_ff + ff]);- let activated_gate = if gating_code == 2 {- silu(gate, silu_alpha)+ // SwiGLU or GEGLU - need gate projection too+ let mut acc_gate = f32::from(up_biases[bias_base + d_ff + h]);+ for d in 0..d_model {+ let x_val = f32::from(x[x_offset + d]);+ let w_val =+ f32::from(w13_all[w13_base + (d_ff + h) * d_model + d]);+ acc_gate += x_val * w_val;+ }++ // Apply clipping+ let up_val = acc_up.clamp(up_clip_min, up_clip_max);+ let gate_val = acc_gate.clamp(gate_clip_min, gate_clip_max);++ let gate_act = if gating_code == 2 {+ silu(gate_val, silu_alpha)} else {- gelu_approx(gate)+ gelu_approx(gate_val)};- activated_gate * up+ gate_act * up_val};- hidden[row * 2 * d_ff + ff] = bf16::from_f32(result);- hidden_full[(row_start + row) * d_ff + ff] = result;+ hidden[k_idx * d_ff + h] = activated;}}- // FC2: [expert_rows, d_ff] @ [d_ff, d_model] -> [expert_rows, d_model]- for row in 0..expert_rows {- for dm in 0..d_model {- let mut acc = f32::from(down_biases[expert * d_model + dm]);- for ff in 0..d_ff {- let h_val = f32::from(hidden[row * 2 * d_ff + ff]);- // w2 transposed: [d_model, d_ff]- let w_val = f32::from(w2[w2_offset + dm * d_ff + ff]);+ // Pass B: hidden @ W2 -> y (with weighted sum)+ for d in 0..d_model {+ let mut final_acc = 0.0f32;++ for k_idx in 0..k {+ let expert_id = topk_ids[tok * k + k_idx];+ if expert_id < 0 {+ continue;+ }+ let expert = expert_id as usize;+ let prob = f32::from(topk_probs[tok * k + k_idx]);++ let w2_base = expert * d_model * d_ff;+ let bias_base = expert * d_model;++ let mut acc = f32::from(down_biases[bias_base + d]);+ for h in 0..d_ff {+ let h_val = hidden[k_idx * d_ff + h];+ let w_val = f32::from(w2_all[w2_base + d * d_ff + h]);acc += h_val * w_val;}- output[(row_start + row) * d_model + dm] = bf16::from_f32(acc);+ final_acc += prob * acc;}++ y[tok * d_model + d] = bf16::from_f32(final_acc);}}- (hidden_full, output)+ y}#[test]@@ -179,28 +300,71 @@ fn test_two_pass_decode_correctness() {let ctx = create_ctx();let mut rng = StdRng::seed_from_u64(0xDEC0DE42);- // Realistic decode test case: single token, K=2+ // End-to-end test: T=1 token, K=2 experts per tokenlet t = 1;let k = 2;let sum_k = t * k;let d_model = 512;let d_ff = 2048;let e = 8;+ let gating_code = 2u32; // SwiGLU+ let silu_alpha = 1.0f32;eprintln!("[2-pass decode] T={}, K={}, sum_k={}, d_model={}, d_ff={}, E={}",t, k, sum_k, d_model, d_ff, e);- // Generate test data- let x_perm: Vec<bf16> = (0..sum_k * d_model)+ // Generate input x [T, d_model]+ let x: Vec<bf16> = (0..t * d_model).map(|_| bf16::from_f32(rng.random_range(-1.0..1.0))).collect();- let offsets = build_offsets(e, sum_k);- let row_expert_map = build_row_expert_map(&offsets, sum_k);+ // Generate routing: topk_ids [T*K], topk_probs [T*K]+ let topk_ids: Vec<i32> = (0..t * k)+ .map(|i| ((i / k) + (i % k) * 2) as i32 % e as i32)+ .collect();+ let topk_probs: Vec<bf16> = (0..t)+ .flat_map(|_| {+ let raw: Vec<f32> =+ (0..k).map(|_| rng.random_range(0.1..1.0)).collect();+ let sum: f32 = raw.iter().sum();+ raw.iter().map(|p| bf16::from_f32(p / sum)).collect::<Vec<_>>()+ })+ .collect();++ // Scatter x based on topk_ids to get x_perm+ // Each token contributes K rows, one per selected expert+ let mut x_perm = vec![bf16::from_f32(0.0); sum_k * d_model];+ for tok in 0..t {+ for ki in 0..k {+ let row_idx = tok * k + ki;+ for d in 0..d_model {+ x_perm[row_idx * d_model + d] = x[tok * d_model + d];+ }+ }+ }++ // Build offsets and row_expert_map based on topk_ids+ // First count rows per expert+ let mut expert_counts = vec![0usize; e];+ for &eid in &topk_ids {+ expert_counts[eid as usize] += 1;+ }+ let mut offsets = vec![0u32; e + 1];+ for i in 0..e {+ offsets[i + 1] = offsets[i] + expert_counts[i] as u32;+ }- // Generate weights (W13 in GPU transposed layout [E, 2*d_ff, d_model])+ // Build row_expert_map (maps permuted row to expert)+ let mut row_expert_map = vec![0u32; sum_k];+ let mut expert_row_idx = vec![0usize; e];+ for (row, &eid) in topk_ids.iter().enumerate() {+ row_expert_map[row] = eid as u32;+ expert_row_idx[eid as usize] += 1;+ }++ // Generate weightslet w13: Vec<bf16> = (0..e * 2 * d_ff * d_model).map(|_| bf16::from_f32(rng.random_range(-0.05..0.05))).collect();@@ -214,18 +378,25 @@ fn test_two_pass_decode_correctness() {.map(|_| bf16::from_f32(rng.random_range(-0.01..0.01))).collect();- // CPU reference (gating_code=0 means GELU)- let (_hidden_expected, expected) = cpu_expert_ffn_full(- &x_perm,- &offsets,+ // CPU reference - end-to-end+ let y_expected = cpu_moe_reference(+ &x,+ &topk_ids,+ &topk_probs,&w13,&w2,&up_biases,&down_biases,+ t,d_model,d_ff,- 0,- 1.0,+ k,+ gating_code,+ silu_alpha,+ f32::NEG_INFINITY,+ f32::INFINITY,+ f32::NEG_INFINITY,+ f32::INFINITY,);// Prepare GPU buffers@@ -237,9 +408,8 @@ fn test_two_pass_decode_correctness() {let up_biases_buf = alloc_buffer_with_data(&ctx, &up_biases);let down_biases_buf = alloc_buffer_with_data(&ctx, &down_biases);- // Intermediate buffers - hidden is f32 for activation precisionlet hidden_buf = alloc_buffer::<f32>(&ctx, sum_k * d_ff);- let output_buf = alloc_buffer::<bf16>(&ctx, sum_k * d_model);+ let y_partial_buf = alloc_buffer::<bf16>(&ctx, sum_k * d_model);// Tile infrastructurelet h_blocks_decode = (d_ff + 3) / 4;@@ -267,7 +437,7 @@ fn test_two_pass_decode_correctness() {expert_offsets: &offsets_buf,row_expert_map: &row_expert_map_buf,hidden_buffer: &hidden_buf,- output_buffer: &output_buf,+ output_buffer: &y_partial_buf,w13_all: &w13_buf,w2_all: &w2_buf,up_biases: &up_biases_buf,@@ -282,12 +452,12 @@ fn test_two_pass_decode_correctness() {d_ff,e,num_tiles_k,- gating_code: 0, // GELU activation- gate_clip_min: -1e6,- gate_clip_max: 1e6,- up_clip_min: -1e6,- up_clip_max: 1e6,- silu_alpha: 1.0,+ gating_code,+ gate_clip_min: f32::NEG_INFINITY,+ gate_clip_max: f32::INFINITY,+ up_clip_min: f32::NEG_INFINITY,+ up_clip_max: f32::INFINITY,+ silu_alpha,data_type: KernelDataType::BFloat16,},)@@ -296,63 +466,126 @@ fn test_two_pass_decode_correctness() {cb.commit();cb.wait_until_completed();- // Read GPU output- let output_gpu = unsafe {+ // Read GPU partial output and do CPU finalize (weighted sum)+ let y_partial_gpu = unsafe {std::slice::from_raw_parts(- output_buf.contents() as *const bf16,+ y_partial_buf.contents() as *const bf16,sum_k * d_model,)};+ // Finalize: y[t] = Σ_k prob[t,k] * y_partial[t*k + k_idx]+ let mut y_gpu = vec![bf16::from_f32(0.0); t * d_model];+ for tok in 0..t {+ for d in 0..d_model {+ let mut acc = 0.0f32;+ for ki in 0..k {+ let row_idx = tok * k + ki;+ let prob = f32::from(topk_probs[row_idx]);+ let val = f32::from(y_partial_gpu[row_idx * d_model + d]);+ acc += prob * val;+ }+ y_gpu[tok * d_model + d] = bf16::from_f32(acc);+ }+ }+// Compute error metricslet mut max_abs_error = 0.0f32;- let mut max_rel_error = 0.0f32;- let mut max_abs_idx = 0;- let mut max_rel_idx = 0;+ let mut max_idx = 0;for (i, (&gpu_val, &cpu_val)) in- output_gpu.iter().zip(expected.iter()).enumerate()+ y_gpu.iter().zip(y_expected.iter()).enumerate(){- let gpu_f = f32::from(gpu_val);- let cpu_f = f32::from(cpu_val);- let abs_error = (gpu_f - cpu_f).abs();- let rel_error = if cpu_f.abs() > 1e-6 {- abs_error / cpu_f.abs()- } else {- 0.0- };-+ let abs_error = (f32::from(gpu_val) - f32::from(cpu_val)).abs();if abs_error > max_abs_error {max_abs_error = abs_error;- max_abs_idx = i;- }- if rel_error > max_rel_error {- max_rel_error = rel_error;- max_rel_idx = i;+ max_idx = i;}}eprintln!(- "[2-pass decode] Max absolute error: {:.6} at index {} (GPU={:.6}, CPU={:.6})",+ "[2-pass decode] Max error: {:.6} at idx {} (GPU={:.6}, CPU={:.6})",max_abs_error,- max_abs_idx,- f32::from(output_gpu[max_abs_idx]),- f32::from(expected[max_abs_idx])+ max_idx,+ f32::from(y_gpu[max_idx]),+ f32::from(y_expected[max_idx]));- eprintln!(- "[2-pass decode] Max relative error: {:.6} ({:.2}%) at index {} (GPU={:.6}, CPU={:.6})",- max_rel_error,- max_rel_error * 100.0,- max_rel_idx,- f32::from(output_gpu[max_rel_idx]),- f32::from(expected[max_rel_idx])++ let tolerance = 0.02;+ assert_bf16_close(&y_gpu, &y_expected, tolerance, "2-pass decode output");++ eprintln!("[2-pass decode] ✓ PASSED (tolerance={:.4})", tolerance);+}++#[test]+fn test_two_pass_decode_multi_token() {+ let ctx = create_ctx();+ let mut rng = StdRng::seed_from_u64(0xDEC0DE43);++ let t = 4;+ let k = 2;+ let sum_k = t * k;+ let d_model = 512;+ let d_ff = 2048;+ let e = 8;+ let gating_code = 2u32;+ let silu_alpha = 1.0f32;++ eprintln!("[2-pass decode multi-token] T={}, K={}, d_model={}, d_ff={}, E={}", t, k, d_model, d_ff, e);++ let data = MoeTestData::generate(&mut rng, t, k, d_model, d_ff, e);+ let scatter = scatter_by_expert(&data.x, &data.topk_ids, t, k, d_model, e);++ let y_expected = cpu_moe_reference(+ &data.x, &data.topk_ids, &data.topk_probs, &data.w13, &data.w2,+ &data.up_biases, &data.down_biases, t, d_model, d_ff, k, gating_code,+ silu_alpha, f32::NEG_INFINITY, f32::INFINITY, f32::NEG_INFINITY, f32::INFINITY,);- let tolerance = 0.01;+ // GPU buffers+ let x_perm_buf = alloc_buffer_with_data(&ctx, &scatter.x_perm);+ let offsets_buf = alloc_buffer_with_data(&ctx, &scatter.offsets);+ let row_expert_map_buf = alloc_buffer_with_data(&ctx, &scatter.row_expert_map);+ let w13_buf = alloc_buffer_with_data(&ctx, &data.w13);+ let w2_buf = alloc_buffer_with_data(&ctx, &data.w2);+ let up_biases_buf = alloc_buffer_with_data(&ctx, &data.up_biases);+ let down_biases_buf = alloc_buffer_with_data(&ctx, &data.down_biases);+ let hidden_buf = alloc_buffer::<f32>(&ctx, sum_k * d_ff);+ let y_partial_buf = alloc_buffer::<bf16>(&ctx, sum_k * d_model);- assert_bf16_close(output_gpu, &expected, tolerance, "2-pass decode output");+ // Tile infrastructure+ let max_total_tiles = sum_k * ((d_ff + 3) / 4);+ let tile_counts_buf = alloc_buffer::<u32>(&ctx, e);+ let tile_offsets_buf = alloc_buffer::<u32>(&ctx, e + 1);+ let total_tiles_buf = alloc_buffer::<u32>(&ctx, 8);+ let tile_map_buf = alloc_buffer::<u32>(&ctx, max_total_tiles * 3);+ let dispatch_args_buf = alloc_buffer::<u32>(&ctx, 3);- eprintln!("[2-pass decode] ✓ PASSED (tolerance={:.4})", tolerance);+ let experts_kernel = MoeExpertsTwoPassDecodeKernel::new(&ctx).expect("kernel");+ let cb = ctx.command_queue.new_command_buffer();+ experts_kernel.encode(&cb, MoeExpertsTwoPassArguments {+ x_perm_buffer: &x_perm_buf, expert_offsets: &offsets_buf,+ row_expert_map: &row_expert_map_buf, hidden_buffer: &hidden_buf,+ output_buffer: &y_partial_buf, w13_all: &w13_buf, w2_all: &w2_buf,+ up_biases: &up_biases_buf, down_biases: &down_biases_buf,+ tile_counts: &tile_counts_buf, tile_offsets: &tile_offsets_buf,+ tile_map: &tile_map_buf, total_tiles: &total_tiles_buf,+ dispatch_args: &dispatch_args_buf, total_rows: sum_k, d_model, d_ff, e,+ num_tiles_k: ((d_ff + 63) / 64) as u32, gating_code,+ gate_clip_min: f32::NEG_INFINITY, gate_clip_max: f32::INFINITY,+ up_clip_min: f32::NEG_INFINITY, up_clip_max: f32::INFINITY,+ silu_alpha, data_type: KernelDataType::BFloat16,+ }).expect("encode");+ cb.commit();+ cb.wait_until_completed();++ let y_partial_gpu = unsafe {+ std::slice::from_raw_parts(y_partial_buf.contents() as *const bf16, sum_k * d_model)+ };+ let y_gpu = gather_and_finalize(y_partial_gpu, &data.topk_probs, &scatter.perm_idx, t, k, d_model);++ assert_bf16_close(&y_gpu, &y_expected, 0.02, "2-pass decode multi-token");+ eprintln!("[2-pass decode multi-token] ✓ PASSED");}#[test]@@ -360,32 +593,128 @@ fn test_two_pass_prefill_correctness() {let ctx = create_ctx();let mut rng = StdRng::seed_from_u64(0xFE1111);- // Realistic prefill case: 8 tokens, K=2let t = 8;let k = 2;let sum_k = t * k;let d_model = 512;let d_ff = 2048;let e = 8;+ let gating_code = 2u32;+ let silu_alpha = 1.0f32;++ eprintln!("[2-pass prefill] T={}, K={}, d_model={}, d_ff={}, E={}", t, k, d_model, d_ff, e);++ let data = MoeTestData::generate(&mut rng, t, k, d_model, d_ff, e);+ let scatter = scatter_by_expert(&data.x, &data.topk_ids, t, k, d_model, e);++ let y_expected = cpu_moe_reference(+ &data.x, &data.topk_ids, &data.topk_probs, &data.w13, &data.w2,+ &data.up_biases, &data.down_biases, t, d_model, d_ff, k, gating_code,+ silu_alpha, f32::NEG_INFINITY, f32::INFINITY, f32::NEG_INFINITY, f32::INFINITY,+ );++ // GPU buffers+ let x_perm_buf = alloc_buffer_with_data(&ctx, &scatter.x_perm);+ let offsets_buf = alloc_buffer_with_data(&ctx, &scatter.offsets);+ let row_expert_map_buf = alloc_buffer_with_data(&ctx, &scatter.row_expert_map);+ let w13_buf = alloc_buffer_with_data(&ctx, &data.w13);+ let w2_buf = alloc_buffer_with_data(&ctx, &data.w2);+ let up_biases_buf = alloc_buffer_with_data(&ctx, &data.up_biases);+ let down_biases_buf = alloc_buffer_with_data(&ctx, &data.down_biases);+ let hidden_buf = alloc_buffer::<f32>(&ctx, sum_k * d_ff);+ let y_partial_buf = alloc_buffer::<bf16>(&ctx, sum_k * d_model);++ // Tile infrastructure+ let max_total_tiles = sum_k * ((d_ff + 3) / 4);+ let tile_counts_buf = alloc_buffer::<u32>(&ctx, e);+ let tile_offsets_buf = alloc_buffer::<u32>(&ctx, e + 1);+ let total_tiles_buf = alloc_buffer::<u32>(&ctx, 8);+ let tile_map_buf = alloc_buffer::<u32>(&ctx, max_total_tiles * 3);+ let dispatch_args_buf = alloc_buffer::<u32>(&ctx, 3);++ let experts_kernel = MoeExpertsTwoPassPrefillKernel::new(&ctx).expect("kernel");+ let cb = ctx.command_queue.new_command_buffer();+ experts_kernel.encode(&cb, MoeExpertsTwoPassArguments {+ x_perm_buffer: &x_perm_buf, expert_offsets: &offsets_buf,+ row_expert_map: &row_expert_map_buf, hidden_buffer: &hidden_buf,+ output_buffer: &y_partial_buf, w13_all: &w13_buf, w2_all: &w2_buf,+ up_biases: &up_biases_buf, down_biases: &down_biases_buf,+ tile_counts: &tile_counts_buf, tile_offsets: &tile_offsets_buf,+ tile_map: &tile_map_buf, total_tiles: &total_tiles_buf,+ dispatch_args: &dispatch_args_buf, total_rows: sum_k, d_model, d_ff, e,+ num_tiles_k: ((d_ff + 63) / 64) as u32, gating_code,+ gate_clip_min: f32::NEG_INFINITY, gate_clip_max: f32::INFINITY,+ up_clip_min: f32::NEG_INFINITY, up_clip_max: f32::INFINITY,+ silu_alpha, data_type: KernelDataType::BFloat16,+ }).expect("encode");+ cb.commit();+ cb.wait_until_completed();++ let y_partial_gpu = unsafe {+ std::slice::from_raw_parts(y_partial_buf.contents() as *const bf16, sum_k * d_model)+ };+ let y_gpu = gather_and_finalize(y_partial_gpu, &data.topk_probs, &scatter.perm_idx, t, k, d_model);++ assert_bf16_close(&y_gpu, &y_expected, 0.02, "2-pass prefill");+ eprintln!("[2-pass prefill] ✓ PASSED");+}++#[test]+fn test_tile_infrastructure() {+ // Test that tile counts/scan/map are computed correctly+ // This is already tested in moe_tiles_test.rs, but we verify here+ // that the tiles match our expectations for expert workload distribution++ let offsets = vec![0, 5, 20, 35, 40];+ const BM: usize = 16;++ let tile_counts = cpu_tile_counts(&offsets, BM);+ let (tile_offsets, total_tiles) = cpu_tile_scan(&tile_counts);++ // Expert 0: 5 rows -> 1 tile+ // Expert 1: 15 rows -> 1 tile+ // Expert 2: 15 rows -> 1 tile+ // Expert 3: 5 rows -> 1 tile+ assert_eq!(tile_counts, vec![1, 1, 1, 1]);+ assert_eq!(tile_offsets, vec![0, 1, 2, 3, 4]);+ assert_eq!(total_tiles, 4);++ eprintln!("[tile infrastructure] ✓ PASSED");+}++#[test]+fn test_fused_single_token_decode() {+ let ctx = create_ctx();+ let mut rng = StdRng::seed_from_u64(0xF053ED);++ let d_model = 512;+ let d_ff = 2048;+ let e = 8;+ let k = 2;eprintln!(- "[2-pass prefill] T={}, K={}, sum_k={}, d_model={}, d_ff={}, E={}",- t, k, sum_k, d_model, d_ff, e+ "[fused single-token] d_model={}, d_ff={}, E={}, K={}",+ d_model, d_ff, e, k);// Generate test data- let x_perm: Vec<bf16> = (0..sum_k * d_model)+ let x: Vec<bf16> = (0..d_model).map(|_| bf16::from_f32(rng.random_range(-1.0..1.0))).collect();- let offsets = build_offsets(e, sum_k);- let row_expert_map = build_row_expert_map(&offsets, sum_k);+ let topk_ids: Vec<i32> = (0..k).map(|i| (i % e) as i32).collect();- // Generate weights- let w13: Vec<bf16> = (0..e * 2 * d_ff * d_model)+ let topk_probs: Vec<bf16> = {+ let raw: Vec<f32> =+ (0..k).map(|_| rng.random_range(0.1..1.0)).collect();+ let sum: f32 = raw.iter().sum();+ raw.iter().map(|p| bf16::from_f32(p / sum)).collect()+ };++ let w13_all: Vec<bf16> = (0..e * 2 * d_ff * d_model).map(|_| bf16::from_f32(rng.random_range(-0.05..0.05))).collect();- let w2: Vec<bf16> = (0..e * d_ff * d_model)+ let w2_all: Vec<bf16> = (0..e * d_ff * d_model).map(|_| bf16::from_f32(rng.random_range(-0.05..0.05))).collect();let up_biases: Vec<bf16> = (0..e * 2 * d_ff)@@ -395,222 +724,259 @@ fn test_two_pass_prefill_correctness() {.map(|_| bf16::from_f32(rng.random_range(-0.01..0.01))).collect();- // CPU reference (gating_code=0 means GELU)- let (hidden_expected, expected) = cpu_expert_ffn_full(- &x_perm,- &offsets,- &w13,- &w2,+ // CPU reference (gating_code=2 for SwiGLU)+ let gating_code = 2u32;+ let silu_alpha = 1.0f32;+ let y_expected = cpu_moe_reference(+ &x,+ &topk_ids,+ &topk_probs,+ &w13_all,+ &w2_all,&up_biases,&down_biases,+ 1, // T=1 for single tokend_model,d_ff,- 0,- 1.0,+ k,+ gating_code,+ silu_alpha,+ f32::NEG_INFINITY,+ f32::INFINITY,+ f32::NEG_INFINITY,+ f32::INFINITY,);- // Prepare GPU buffers- let x_perm_buf = alloc_buffer_with_data(&ctx, &x_perm);- let offsets_buf = alloc_buffer_with_data(&ctx, &offsets);- let row_expert_map_buf = alloc_buffer_with_data(&ctx, &row_expert_map);- let w13_buf = alloc_buffer_with_data(&ctx, &w13);- let w2_buf = alloc_buffer_with_data(&ctx, &w2);+ // GPU buffers+ let x_buf = alloc_buffer_with_data(&ctx, &x);+ let topk_ids_buf = alloc_buffer_with_data(&ctx, &topk_ids);+ let topk_probs_buf = alloc_buffer_with_data(&ctx, &topk_probs);+ let w13_buf = alloc_buffer_with_data(&ctx, &w13_all);+ let w2_buf = alloc_buffer_with_data(&ctx, &w2_all);let up_biases_buf = alloc_buffer_with_data(&ctx, &up_biases);let down_biases_buf = alloc_buffer_with_data(&ctx, &down_biases);+ let hidden_buf = alloc_buffer::<f32>(&ctx, k * d_ff);+ let y_buf = alloc_buffer::<bf16>(&ctx, d_model);- // Intermediate buffers - hidden is f32 for activation precision- let hidden_buf = alloc_buffer::<f32>(&ctx, sum_k * d_ff);- let output_buf = alloc_buffer::<bf16>(&ctx, sum_k * d_model);-- // Tile infrastructure- let h_blocks_decode = (d_ff + 3) / 4;- let max_total_tiles = sum_k * h_blocks_decode;-- let tile_counts_buf = alloc_buffer::<u32>(&ctx, e);- let tile_offsets_buf = alloc_buffer::<u32>(&ctx, e + 1);- let total_tiles_buf = alloc_buffer::<u32>(&ctx, 8);- let tile_map_buf = alloc_buffer::<u32>(&ctx, max_total_tiles * 3);- let dispatch_args_buf = alloc_buffer::<u32>(&ctx, 3);-- // Execute 2-pass prefill kernel- let experts_kernel = MoeExpertsTwoPassPrefillKernel::new(&ctx)- .expect("MoeExpertsTwoPassPrefillKernel::new");+ // Run fused decode kernel+ let fused_kernel = MoeExpertsSingleDecodeKernel::new(&ctx)+ .expect("MoeExpertsSingleDecodeKernel::new");let cb = ctx.command_queue.new_command_buffer();- const K_TILE: usize = 64;- let num_tiles_k = ((d_ff + K_TILE - 1) / K_TILE) as u32;-- experts_kernel+ fused_kernel.encode(&cb,- MoeExpertsTwoPassArguments {- x_perm_buffer: &x_perm_buf,- expert_offsets: &offsets_buf,- row_expert_map: &row_expert_map_buf,- hidden_buffer: &hidden_buf,- output_buffer: &output_buf,+ MoeExpertsSingleDecodeArguments {+ x: &x_buf,+ topk_ids: &topk_ids_buf,+ topk_probs: &topk_probs_buf,w13_all: &w13_buf,w2_all: &w2_buf,up_biases: &up_biases_buf,down_biases: &down_biases_buf,- tile_counts: &tile_counts_buf,- tile_offsets: &tile_offsets_buf,- tile_map: &tile_map_buf,- total_tiles: &total_tiles_buf,- dispatch_args: &dispatch_args_buf,- total_rows: sum_k,+ hidden: &hidden_buf,+ y: &y_buf,d_model,d_ff,- e,- num_tiles_k,- gating_code: 0, // GELU activation- gate_clip_min: -1e6,- gate_clip_max: 1e6,- up_clip_min: -1e6,- up_clip_max: 1e6,- silu_alpha: 1.0,+ k,+ gating_code,+ silu_alpha: 1.0, // Standard SiLU for testing+ gate_clip_min: f32::NEG_INFINITY,+ gate_clip_max: f32::INFINITY,+ up_clip_min: f32::NEG_INFINITY,+ up_clip_max: f32::INFINITY,data_type: KernelDataType::BFloat16,},)- .expect("encode_two_pass_prefill");+ .expect("fused decode encode");cb.commit();cb.wait_until_completed();// Read GPU output- let output_gpu = unsafe {- std::slice::from_raw_parts(- output_buf.contents() as *const bf16,- sum_k * d_model,- )+ let y_gpu = unsafe {+ std::slice::from_raw_parts(y_buf.contents() as *const bf16, d_model)};// Compute error metricslet mut max_abs_error = 0.0f32;- let mut max_rel_error = 0.0f32;- let mut max_abs_idx = 0;- let mut max_rel_idx = 0;+ let mut sum_abs_error = 0.0f32;+ let mut max_idx = 0;for (i, (&gpu_val, &cpu_val)) in- output_gpu.iter().zip(expected.iter()).enumerate()+ y_gpu.iter().zip(y_expected.iter()).enumerate(){- let gpu_f = f32::from(gpu_val);- let cpu_f = f32::from(cpu_val);- let abs_error = (gpu_f - cpu_f).abs();- let rel_error = if cpu_f.abs() > 1e-6 {- abs_error / cpu_f.abs()- } else {- 0.0- };-+ let abs_error = (f32::from(gpu_val) - f32::from(cpu_val)).abs();+ sum_abs_error += abs_error;if abs_error > max_abs_error {max_abs_error = abs_error;- max_abs_idx = i;- }- if rel_error > max_rel_error {- max_rel_error = rel_error;- max_rel_idx = i;+ max_idx = i;}}+ let mean_abs_error = sum_abs_error / d_model as f32;eprintln!(- "[2-pass prefill] Max absolute error: {:.6} at index {} (GPU={:.6}, CPU={:.6})",+ "[fused single-token] Max error: {:.6} at idx {} (GPU={:.6}, CPU={:.6}), Mean error: {:.6}",max_abs_error,- max_abs_idx,- f32::from(output_gpu[max_abs_idx]),- f32::from(expected[max_abs_idx])+ max_idx,+ f32::from(y_gpu[max_idx]),+ f32::from(y_expected[max_idx]),+ mean_abs_error+ );++ let tolerance = 0.02;+ assert_bf16_close(+ y_gpu,+ &y_expected,+ tolerance,+ "fused single-token output",);++ eprintln!("[fused single-token] PASSED (tolerance={:.4})", tolerance);+}++#[test]+fn test_fused_single_token_k4() {+ let ctx = create_ctx();+ let mut rng = StdRng::seed_from_u64(0xE001A);++ let d_model = 512;+ let d_ff = 2048;+ let e = 8;+ let k = 4;+eprintln!(- "[2-pass prefill] Max relative error: {:.6} ({:.2}%) at index {} (GPU={:.6}, CPU={:.6})",- max_rel_error,- max_rel_error * 100.0,- max_rel_idx,- f32::from(output_gpu[max_rel_idx]),- f32::from(expected[max_rel_idx])+ "[fused single-token K=4] d_model={}, d_ff={}, E={}, K={}",+ d_model, d_ff, e, k);- // Verify correctness with tight tolerance- let tolerance = 0.01;+ let x: Vec<bf16> = (0..d_model)+ .map(|_| bf16::from_f32(rng.random_range(-1.0..1.0)))+ .collect();++ let topk_ids: Vec<i32> = (0..k).map(|i| ((i * 2) % e) as i32).collect();- let hidden_gpu = unsafe {- std::slice::from_raw_parts(- hidden_buf.contents() as *const f32,- sum_k * d_ff,+ let topk_probs: Vec<bf16> = {+ let raw: Vec<f32> =+ (0..k).map(|_| rng.random_range(0.1..1.0)).collect();+ let sum: f32 = raw.iter().sum();+ raw.iter().map(|p| bf16::from_f32(p / sum)).collect()+ };++ let w13_all: Vec<bf16> = (0..e * 2 * d_ff * d_model)+ .map(|_| bf16::from_f32(rng.random_range(-0.05..0.05)))+ .collect();+ let w2_all: Vec<bf16> = (0..e * d_ff * d_model)+ .map(|_| bf16::from_f32(rng.random_range(-0.05..0.05)))+ .collect();+ let up_biases: Vec<bf16> = (0..e * 2 * d_ff)+ .map(|_| bf16::from_f32(rng.random_range(-0.01..0.01)))+ .collect();+ let down_biases: Vec<bf16> = (0..e * d_model)+ .map(|_| bf16::from_f32(rng.random_range(-0.01..0.01)))+ .collect();++ let gating_code = 2u32;+ let silu_alpha = 1.0f32;+ let y_expected = cpu_moe_reference(+ &x,+ &topk_ids,+ &topk_probs,+ &w13_all,+ &w2_all,+ &up_biases,+ &down_biases,+ 1, // T=1 for single token+ d_model,+ d_ff,+ k,+ gating_code,+ silu_alpha,+ f32::NEG_INFINITY,+ f32::INFINITY,+ f32::NEG_INFINITY,+ f32::INFINITY,+ );++ let x_buf = alloc_buffer_with_data(&ctx, &x);+ let topk_ids_buf = alloc_buffer_with_data(&ctx, &topk_ids);+ let topk_probs_buf = alloc_buffer_with_data(&ctx, &topk_probs);+ let w13_buf = alloc_buffer_with_data(&ctx, &w13_all);+ let w2_buf = alloc_buffer_with_data(&ctx, &w2_all);+ let up_biases_buf = alloc_buffer_with_data(&ctx, &up_biases);+ let down_biases_buf = alloc_buffer_with_data(&ctx, &down_biases);+ let hidden_buf = alloc_buffer::<f32>(&ctx, k * d_ff);+ let y_buf = alloc_buffer::<bf16>(&ctx, d_model);++ let fused_kernel =+ MoeExpertsSingleDecodeKernel::new(&ctx).expect("fused kernel");+ let cb = ctx.command_queue.new_command_buffer();+ fused_kernel+ .encode(+ &cb,+ MoeExpertsSingleDecodeArguments {+ x: &x_buf,+ topk_ids: &topk_ids_buf,+ topk_probs: &topk_probs_buf,+ w13_all: &w13_buf,+ w2_all: &w2_buf,+ up_biases: &up_biases_buf,+ down_biases: &down_biases_buf,+ hidden: &hidden_buf,+ y: &y_buf,+ d_model,+ d_ff,+ k,+ gating_code,+ silu_alpha,+ gate_clip_min: f32::NEG_INFINITY,+ gate_clip_max: f32::INFINITY,+ up_clip_min: f32::NEG_INFINITY,+ up_clip_max: f32::INFINITY,+ data_type: KernelDataType::BFloat16,+ },)+ .expect("fused encode");+ cb.commit();+ cb.wait_until_completed();++ let y_gpu = unsafe {+ std::slice::from_raw_parts(y_buf.contents() as *const bf16, d_model)};- let mut hidden_max_diff = 0.0f32;- let mut hidden_max_idx = 0usize;- for (idx, (&gpu, &cpu)) in- hidden_gpu.iter().zip(hidden_expected.iter()).enumerate()+ // Compute error metrics+ let mut max_abs_error = 0.0f32;+ let mut sum_abs_error = 0.0f32;+ let mut max_idx = 0;++ for (i, (&gpu_val, &cpu_val)) in+ y_gpu.iter().zip(y_expected.iter()).enumerate(){- let diff = (gpu - cpu).abs();- if diff > hidden_max_diff {- hidden_max_diff = diff;- hidden_max_idx = idx;+ let abs_error = (f32::from(gpu_val) - f32::from(cpu_val)).abs();+ sum_abs_error += abs_error;+ if abs_error > max_abs_error {+ max_abs_error = abs_error;+ max_idx = i;}}+ let mean_abs_error = sum_abs_error / d_model as f32;- if hidden_max_diff > tolerance {- let row = hidden_max_idx / d_ff;- let col = hidden_max_idx % d_ff;- eprintln!(- "[2-pass prefill] hidden mismatch row={} col={} gpu={} cpu={} diff={}",- row,- col,- hidden_gpu[hidden_max_idx],- hidden_expected[hidden_max_idx],- hidden_max_diff- );-- for offset in 0..8 {- let idx = row * d_ff + (col + offset).min(d_ff - 1);- let g = hidden_gpu[idx];- let c = hidden_expected[idx];- eprintln!(- " col {} -> gpu {:.6} cpu {:.6} diff {:.6}",- (col + offset).min(d_ff - 1),- g,- c,- (g - c).abs()- );- }- } else {- eprintln!(- "[2-pass prefill] hidden max diff {:.6} within tolerance",- hidden_max_diff- );- }+ eprintln!(+ "[fused single-token K=4] Max error: {:.6} at idx {} (GPU={:.6}, CPU={:.6}), Mean error: {:.6}",+ max_abs_error,+ max_idx,+ f32::from(y_gpu[max_idx]),+ f32::from(y_expected[max_idx]),+ mean_abs_error+ );+ let tolerance = 0.02;assert_bf16_close(- output_gpu,- &expected,+ y_gpu,+ &y_expected,tolerance,- "2-pass prefill output",+ "fused single-token K=4 output",);- eprintln!("[2-pass prefill] ✓ PASSED (tolerance={:.4})", tolerance);-}--#[test]-fn test_tile_infrastructure() {- // Test that tile counts/scan/map are computed correctly- // This is already tested in moe_tiles_test.rs, but we verify here- // that the tiles match our expectations for expert workload distribution-- let offsets = vec![0, 5, 20, 35, 40];- const BM: usize = 16;-- let tile_counts = cpu_tile_counts(&offsets, BM);- let (tile_offsets, total_tiles) = cpu_tile_scan(&tile_counts);-- // Expert 0: 5 rows -> 1 tile- // Expert 1: 15 rows -> 1 tile- // Expert 2: 15 rows -> 1 tile- // Expert 3: 5 rows -> 1 tile- assert_eq!(tile_counts, vec![1, 1, 1, 1]);- assert_eq!(tile_offsets, vec![0, 1, 2, 3, 4]);- assert_eq!(total_tiles, 4);-- eprintln!("[tile infrastructure] ✓ PASSED");+ eprintln!("[fused single-token K=4] PASSED (tolerance={:.4})", tolerance);}