Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

Moe improvements

LuckyIYIeab58be42efd
diff --git a/crates/uzu/src/backends/metal/kernel/mlp/mod.rs b/crates/uzu/src/backends/metal/kernel/mlp/mod.rs
index 4f58d1d46..1d4e3e541 100644
--- a/crates/uzu/src/backends/metal/kernel/mlp/mod.rs
+++ b/crates/uzu/src/backends/metal/kernel/mlp/mod.rs
@@ -1,5 +1,6 @@
use metal::{
- Buffer as MTLBuffer, ComputeCommandEncoderRef, ComputePipelineState, MTLSize,
+ Buffer as MTLBuffer, ComputeCommandEncoderRef, ComputePipelineState,
+ MTLSize,
};
use mpsgraph::CommandBuffer as MPSCommandBuffer;
diff --git a/crates/uzu/src/backends/metal/kernel/moe/encodable.rs b/crates/uzu/src/backends/metal/kernel/moe/encodable.rs
index f58a406da..ff59b0e77 100644
--- a/crates/uzu/src/backends/metal/kernel/moe/encodable.rs
+++ b/crates/uzu/src/backends/metal/kernel/moe/encodable.rs
@@ -1,13 +1,20 @@
+//! MoE Block Encodable with path selection:
+//! - suffix == 1: Single-token fused decode (no scatter/gather)
+//! - suffix <= 32: Indirect decode path (GEMV-based)
+//! - suffix > 32: Indirect prefill path (GEMM/MMA-based)
+
use std::{cell::RefCell, rc::Rc};
+use metal::CommandBufferRef;
use mpsgraph::CommandBuffer as MPSCommandBuffer;
use super::{
MoeCountsOffsetsFusedArguments, MoeCountsOffsetsFusedKernel,
+ MoeExpertsSingleDecodeArguments, MoeExpertsSingleDecodeKernel,
MoeExpertsTwoPassArguments, MoeExpertsTwoPassDecodeKernel,
- MoeExpertsTwoPassPrefillKernel, MoeFinalizeKernel, MoeGatherArguments,
- MoeGatherKernel, MoeRouterTopKArguments, MoeRouterTopKKernel,
- MoeScatterKernels, MoeScatterWithMapArguments,
+ MoeExpertsTwoPassPrefillKernel, MoeFinalizeArguments, MoeFinalizeKernel,
+ MoeGatherArguments, MoeGatherKernel, MoeRouterTopKArguments,
+ MoeRouterTopKKernel, MoeScatterKernels, MoeScatterWithMapArguments,
};
use crate::{
DataType,
@@ -25,6 +32,9 @@ use crate::{
parameters::ParameterTree,
};
+/// Threshold for decode vs prefill path selection
+const INDIRECT_DECODE_THRESHOLD: usize = 32;
+
enum RouterBlock {
Metal {
weights_buf: metal::Buffer,
@@ -40,17 +50,51 @@ pub struct SharedMoeWeights {
pub down_biases_buf: Rc<metal::Buffer>,
}
+/// Buffers used by MoE encoding paths
+struct MoeBuffers {
+ main: metal::Buffer,
+ topk_ids: metal::Buffer,
+ topk_probs: metal::Buffer,
+ offsets: metal::Buffer,
+ sumk: metal::Buffer,
+ bucketed_ids: metal::Buffer,
+ bucketed_probs: metal::Buffer,
+ x_perm: metal::Buffer,
+ tok2row: metal::Buffer,
+ y_partial: metal::Buffer,
+ tile_counts: metal::Buffer,
+ tile_offsets: metal::Buffer,
+ tile_map: metal::Buffer,
+ total_tiles: metal::Buffer,
+ dispatch_args: metal::Buffer,
+ partials: metal::Buffer,
+ block_bases: metal::Buffer,
+ block_alloc: metal::Buffer,
+ hidden: metal::Buffer,
+ row_expert_map: metal::Buffer,
+}
+
pub struct MoeBlockEncodable {
+ // Router
router: RouterBlock,
router_data_type: KernelDataType,
router_renorm: bool,
router_topk_kernel: MoeRouterTopKKernel,
+
+ // Indirect path shared components
counts_offsets_kernel: MoeCountsOffsetsFusedKernel,
scatter_kernels: MoeScatterKernels,
gather_kernel: MoeGatherKernel,
+ finalize_kernel: MoeFinalizeKernel,
+
+ // Indirect path variants
experts_two_pass_decode_kernel: MoeExpertsTwoPassDecodeKernel,
experts_two_pass_prefill_kernel: MoeExpertsTwoPassPrefillKernel,
- finalize_kernel: MoeFinalizeKernel,
+
+ // Single-token decode path
+ single_decode_kernel: MoeExpertsSingleDecodeKernel,
+
+ // Config and weights
moe_config: MixtureOfExpertsConfig,
model_dim: usize,
hidden_dim: usize,
@@ -133,6 +177,7 @@ impl MoeBlockEncodable {
},
};
+ // Initialize kernels
let router_topk_kernel =
MoeRouterTopKKernel::new(context).map_err(|e| {
crate::backends::metal::MTLError::Generic(format!(
@@ -173,6 +218,13 @@ impl MoeBlockEncodable {
e
))
})?;
+ let single_decode_kernel = MoeExpertsSingleDecodeKernel::new(context)
+ .map_err(|e| {
+ crate::backends::metal::MTLError::Generic(format!(
+ "Simple decode fused kernel error: {:?}",
+ e
+ ))
+ })?;
let finalize_kernel = MoeFinalizeKernel::new(context).map_err(|e| {
crate::backends::metal::MTLError::Generic(format!(
"Finalize kernel error: {:?}",
@@ -180,6 +232,7 @@ impl MoeBlockEncodable {
))
})?;
+ // Load expert weights
let experts_tree = parameter_tree.subtree("experts").map_err(|e| {
crate::backends::metal::MTLError::Generic(format!(
"experts subtree error: {:?}",
@@ -272,6 +325,7 @@ impl MoeBlockEncodable {
gather_kernel,
experts_two_pass_decode_kernel,
experts_two_pass_prefill_kernel,
+ single_decode_kernel,
finalize_kernel,
moe_config: moe_config.clone(),
model_dim,
@@ -283,167 +337,47 @@ impl MoeBlockEncodable {
fn gating_code_from_activation(activation: &Activation) -> u32 {
match activation {
- Activation::Gelu => 3,
+ Activation::Gelu => 3, // GEGLU
Activation::SiLU {
..
- } => 2,
+ } => 2, // SwiGLU
Activation::Identity => {
panic!("Identity activation is not supported for MoE kernels")
},
}
}
-}
-
-impl EncodableWithState for MoeBlockEncodable {
- fn encode(
- &self,
- state: &mut ForwardPassState,
- command_buffer: &MPSCommandBuffer,
- parameters: &EncodingParameters,
- ) {
- let suffix_length = state.active_suffix_length();
- let arrays = state.arrays(&[
- ArrayId::Main,
- ArrayId::MoeTopkIds,
- ArrayId::MoeTopkProbs,
- ArrayId::MoeOffsets,
- ArrayId::MoeSumK,
- ArrayId::MoeBucketedTokenIds,
- ArrayId::MoeBucketedProbs,
- ArrayId::MoeXPerm,
- ArrayId::MoeTok2Row,
- ArrayId::MoeYPartial,
- ArrayId::MoeTileCounts,
- ArrayId::MoeTileOffsets,
- ArrayId::MoeTileMap,
- ArrayId::MoeTotalTiles,
- ArrayId::MoeDispatchArgs,
- ArrayId::MoeScatterPartials,
- ArrayId::MoeScatterBlockBases,
- ArrayId::MoeBlockAlloc,
- ArrayId::MoeHidden,
- ArrayId::MoeTwoPassRowExpertMap,
- ]);
-
- let clone_buffer = |array: &RefCell<MetalArray>| -> metal::Buffer {
- let mut borrow = array.borrow_mut();
- let buffer = unsafe { borrow.mtl_buffer().clone() };
- buffer
- };
- let mut array_iter = arrays.iter();
- let main_buf = clone_buffer(array_iter.next().unwrap());
- let topk_ids_buf = clone_buffer(array_iter.next().unwrap());
- let topk_probs_buf = clone_buffer(array_iter.next().unwrap());
- let offsets_buf = clone_buffer(array_iter.next().unwrap());
- let sumk_buf = clone_buffer(array_iter.next().unwrap());
- let bucketed_ids_buf = clone_buffer(array_iter.next().unwrap());
- let bucketed_probs_buf = clone_buffer(array_iter.next().unwrap());
- let x_perm_buf = clone_buffer(array_iter.next().unwrap());
- let tok2row_buf = clone_buffer(array_iter.next().unwrap());
- let y_partial_buf = clone_buffer(array_iter.next().unwrap());
- let tile_counts_buf = clone_buffer(array_iter.next().unwrap());
- let tile_offsets_buf = clone_buffer(array_iter.next().unwrap());
- let tile_map_buf = clone_buffer(array_iter.next().unwrap());
- let total_tiles_buf = clone_buffer(array_iter.next().unwrap());
- let dispatch_args_buf = clone_buffer(array_iter.next().unwrap());
- let partials_buf = clone_buffer(array_iter.next().unwrap());
- let block_bases_buf = clone_buffer(array_iter.next().unwrap());
- let block_alloc_buf = clone_buffer(array_iter.next().unwrap());
- let hidden_buf = clone_buffer(array_iter.next().unwrap());
- let row_expert_map_buf = clone_buffer(array_iter.next().unwrap());
- debug_assert!(array_iter.next().is_none());
-
- let e = self.moe_config.mixture_size;
- let k = self.moe_config.num_experts_per_token;
-
- let mtl_command_buffer =
- command_buffer.root_command_buffer().to_owned();
- let root = &*mtl_command_buffer;
- let k_tile = 128;
-
- // Clear internal MoE buffers
- let dtype_size = match self.data_type {
+ fn dtype_size(&self) -> usize {
+ match self.data_type {
KernelDataType::BFloat16 | KernelDataType::Float16 => 2,
KernelDataType::Float32 => 4,
- };
-
- let blit_encoder = root.new_blit_command_encoder();
-
- if suffix_length > 0 && k > 0 {
- let entries = suffix_length * k;
- let topk_bytes = entries * std::mem::size_of::<u32>();
- let tok2row_bytes = entries * std::mem::size_of::<i32>();
-
- // Clear topk_ids and tok2row buffers
- if topk_bytes > 0 {
- blit_encoder.fill_buffer(
- &topk_ids_buf,
- metal::NSRange::new(0, topk_bytes as u64),
- 0xFF,
- );
- }
- if tok2row_bytes > 0 {
- blit_encoder.fill_buffer(
- &tok2row_buf,
- metal::NSRange::new(0, tok2row_bytes as u64),
- 0xFF,
- );
- }
-
- // Clear hidden buffer
- let hidden_bytes =
- (suffix_length * k * self.hidden_dim * dtype_size) as u64;
- if hidden_bytes > 0 {
- blit_encoder.fill_buffer(
- &hidden_buf,
- metal::NSRange::new(0, hidden_bytes),
- 0,
- );
- }
-
- // Clear y_partial buffer
- let y_partial_bytes =
- (suffix_length * k * self.model_dim * dtype_size) as u64;
- if y_partial_bytes > 0 {
- blit_encoder.fill_buffer(
- &y_partial_buf,
- metal::NSRange::new(0, y_partial_bytes),
- 0,
- );
- }
-
- // Clear x_perm buffer
- let x_perm_bytes =
- (suffix_length * k * self.model_dim * dtype_size) as u64;
- if x_perm_bytes > 0 {
- blit_encoder.fill_buffer(
- &x_perm_buf,
- metal::NSRange::new(0, x_perm_bytes),
- 0,
- );
- }
}
+ }
- blit_encoder.end_encoding();
+ fn encode_router(
+ &self,
+ root: &CommandBufferRef,
+ buffers: &MoeBuffers,
+ suffix_length: usize,
+ ) {
+ let e = self.moe_config.mixture_size;
+ let k = self.moe_config.num_experts_per_token;
- // Use fused Router+TopK kernel for non-quantized routers
match &self.router {
RouterBlock::Metal {
weights_buf,
biases_buf,
} => {
- // Use the fused router+topk kernel
self.router_topk_kernel
.encode(
root,
self.router_data_type,
MoeRouterTopKArguments {
- input_buffer: &main_buf,
+ input_buffer: &buffers.main,
weight_buffer: weights_buf,
bias_buffer: biases_buf,
- topk_ids_buffer: &topk_ids_buf,
- topk_probs_buffer: &topk_probs_buf,
+ topk_ids_buffer: &buffers.topk_ids,
+ topk_probs_buffer: &buffers.topk_probs,
t: suffix_length,
d_model: self.model_dim,
e,
@@ -451,18 +385,29 @@ impl EncodableWithState for MoeBlockEncodable {
renorm: self.router_renorm,
},
)
- .expect("MoE fused router+topk failed");
+ .expect("MoE router+topk failed");
},
}
+ }
+ fn encode_indirect_setup(
+ &self,
+ root: &CommandBufferRef,
+ buffers: &MoeBuffers,
+ suffix_length: usize,
+ ) {
+ let e = self.moe_config.mixture_size;
+ let k = self.moe_config.num_experts_per_token;
+
+ // Counts + offsets
self.counts_offsets_kernel
.encode(
root,
MoeCountsOffsetsFusedArguments {
- topk_ids_buffer: &topk_ids_buf,
- offsets_buffer: &offsets_buf,
- sum_k_buffer: &sumk_buf,
- partials_buffer: &partials_buf,
+ topk_ids_buffer: &buffers.topk_ids,
+ offsets_buffer: &buffers.offsets,
+ sum_k_buffer: &buffers.sumk,
+ partials_buffer: &buffers.partials,
t: suffix_length,
e,
k,
@@ -473,13 +418,14 @@ impl EncodableWithState for MoeBlockEncodable {
let num_blocks = ((suffix_length + 255) / 256).max(1);
let num_tiles = ((e + 512 - 1) / 512).max(1);
+ // Block bases
self.scatter_kernels
.encode_block_bases(
root,
super::MoeBlockBasesArguments {
- partials_buffer: &partials_buf,
- block_bases_buffer: &block_bases_buf,
- block_alloc_buffer: &block_alloc_buf,
+ partials_buffer: &buffers.partials,
+ block_bases_buffer: &buffers.block_bases,
+ block_alloc_buffer: &buffers.block_alloc,
e,
num_blocks,
num_tiles,
@@ -487,51 +433,171 @@ impl EncodableWithState for MoeBlockEncodable {
)
.expect("MoE block bases failed");
+ // Scatter with tok2row map
self.scatter_kernels
.encode_scatter_with_map(
root,
MoeScatterWithMapArguments {
base: super::MoeScatterArguments {
- topk_ids_buffer: &topk_ids_buf,
- topk_probs_buffer: &topk_probs_buf,
- offsets_buffer: &offsets_buf,
- block_bases_buffer: &block_bases_buf,
- block_alloc_buffer: &block_alloc_buf,
- out_ids_buffer: &bucketed_ids_buf,
- out_probs_buffer: &bucketed_probs_buf,
+ topk_ids_buffer: &buffers.topk_ids,
+ topk_probs_buffer: &buffers.topk_probs,
+ offsets_buffer: &buffers.offsets,
+ block_bases_buffer: &buffers.block_bases,
+ block_alloc_buffer: &buffers.block_alloc,
+ out_ids_buffer: &buffers.bucketed_ids,
+ out_probs_buffer: &buffers.bucketed_probs,
t: suffix_length,
e,
k,
num_blocks,
num_tiles,
},
- tok2row_buffer: &tok2row_buf,
+ tok2row_buffer: &buffers.tok2row,
},
self.data_type,
)
.expect("MoE scatter failed");
+ // Gather
self.gather_kernel
.encode(
root,
self.data_type,
MoeGatherArguments {
- x_buffer: &main_buf,
- bucketed_ids_buffer: &bucketed_ids_buf,
- x_perm_buffer: &x_perm_buf,
- sumk_buffer: &sumk_buf,
+ x_buffer: &buffers.main,
+ bucketed_ids_buffer: &buffers.bucketed_ids,
+ x_perm_buffer: &buffers.x_perm,
+ sumk_buffer: &buffers.sumk,
t: suffix_length,
k,
d_model: self.model_dim,
},
)
.expect("MoE gather failed");
+ }
+
+ fn clear_indirect_buffers(
+ &self,
+ root: &CommandBufferRef,
+ buffers: &MoeBuffers,
+ suffix_length: usize,
+ ) {
+ let k = self.moe_config.num_experts_per_token;
+ if k == 0 {
+ return;
+ }
+
+ let blit_encoder = root.new_blit_command_encoder();
+ let entries = suffix_length * k;
+ let topk_bytes = entries * std::mem::size_of::<u32>();
+ let tok2row_bytes = entries * std::mem::size_of::<i32>();
+ let dtype_size = self.dtype_size();
+
+ if topk_bytes > 0 {
+ blit_encoder.fill_buffer(
+ &buffers.topk_ids,
+ metal::NSRange::new(0, topk_bytes as u64),
+ 0xFF,
+ );
+ }
+ if tok2row_bytes > 0 {
+ blit_encoder.fill_buffer(
+ &buffers.tok2row,
+ metal::NSRange::new(0, tok2row_bytes as u64),
+ 0xFF,
+ );
+ }
+
+ let hidden_bytes =
+ (suffix_length * k * self.hidden_dim * dtype_size) as u64;
+ if hidden_bytes > 0 {
+ blit_encoder.fill_buffer(
+ &buffers.hidden,
+ metal::NSRange::new(0, hidden_bytes),
+ 0,
+ );
+ }
+
+ let y_partial_bytes =
+ (suffix_length * k * self.model_dim * dtype_size) as u64;
+ if y_partial_bytes > 0 {
+ blit_encoder.fill_buffer(
+ &buffers.y_partial,
+ metal::NSRange::new(0, y_partial_bytes),
+ 0,
+ );
+ }
+
+ let x_perm_bytes =
+ (suffix_length * k * self.model_dim * dtype_size) as u64;
+ if x_perm_bytes > 0 {
+ blit_encoder.fill_buffer(
+ &buffers.x_perm,
+ metal::NSRange::new(0, x_perm_bytes),
+ 0,
+ );
+ }
+
+ blit_encoder.end_encoding();
+ }
+
+ fn encode_single_token_path(
+ &self,
+ root: &CommandBufferRef,
+ buffers: &MoeBuffers,
+ ) {
+ let k = self.moe_config.num_experts_per_token;
+ let gating_code = Self::gating_code_from_activation(
+ &self.moe_config.expert_config.activation,
+ );
+ let silu_alpha = self.moe_config.expert_config.activation.alpha();
+ let gate_clip_min = self.moe_config.expert_config.gate_clipping[0]
+ .unwrap_or(f32::NEG_INFINITY);
+ let gate_clip_max = self.moe_config.expert_config.gate_clipping[1]
+ .unwrap_or(f32::INFINITY);
+ let up_clip_min = self.moe_config.expert_config.up_clipping[0];
+ let up_clip_max = self.moe_config.expert_config.up_clipping[1];
+
+ self.single_decode_kernel
+ .encode(
+ root,
+ MoeExpertsSingleDecodeArguments {
+ x: &buffers.main,
+ topk_ids: &buffers.topk_ids,
+ topk_probs: &buffers.topk_probs,
+ w13_all: &self.shared_weights.w13_buf,
+ w2_all: &self.shared_weights.w2_buf,
+ up_biases: &self.shared_weights.up_biases_buf,
+ down_biases: &self.shared_weights.down_biases_buf,
+ hidden: &buffers.hidden,
+ y: &buffers.main,
+ d_model: self.model_dim,
+ d_ff: self.hidden_dim,
+ k,
+ gating_code,
+ silu_alpha,
+ gate_clip_min,
+ gate_clip_max,
+ up_clip_min,
+ up_clip_max,
+ data_type: self.data_type,
+ },
+ )
+ .expect("MoE single-token fused decode failed");
+ }
+ fn encode_indirect_decode_path(
+ &self,
+ root: &CommandBufferRef,
+ buffers: &MoeBuffers,
+ suffix_length: usize,
+ ) {
+ let e = self.moe_config.mixture_size;
+ let k = self.moe_config.num_experts_per_token;
let gating_code = Self::gating_code_from_activation(
&self.moe_config.expert_config.activation,
);
- // Compute clipping values and alpha for expert kernels
let gate_clip_min = self.moe_config.expert_config.gate_clipping[0]
.unwrap_or(f32::NEG_INFINITY);
let gate_clip_max = self.moe_config.expert_config.gate_clipping[1]
@@ -540,90 +606,124 @@ impl EncodableWithState for MoeBlockEncodable {
let up_clip_max = self.moe_config.expert_config.up_clipping[1];
let silu_alpha = self.moe_config.expert_config.activation.alpha();
- if suffix_length == 1 {
- let total_rows = suffix_length * k;
- let num_tiles_k = ((self.hidden_dim + k_tile - 1) / k_tile) as u32;
-
- self.experts_two_pass_decode_kernel
- .encode(
- root,
- MoeExpertsTwoPassArguments {
- x_perm_buffer: &x_perm_buf,
- expert_offsets: &offsets_buf,
- row_expert_map: &row_expert_map_buf,
- hidden_buffer: &hidden_buf,
- output_buffer: &y_partial_buf,
- w13_all: &self.shared_weights.w13_buf,
- w2_all: &self.shared_weights.w2_buf,
- up_biases: &self.shared_weights.up_biases_buf,
- down_biases: &self.shared_weights.down_biases_buf,
- tile_counts: &tile_counts_buf,
- tile_offsets: &tile_offsets_buf,
- tile_map: &tile_map_buf,
- total_tiles: &total_tiles_buf,
- dispatch_args: &dispatch_args_buf,
- total_rows,
- d_model: self.model_dim,
- d_ff: self.hidden_dim,
- e,
- num_tiles_k,
- gating_code,
- gate_clip_min,
- gate_clip_max,
- up_clip_min,
- up_clip_max,
- silu_alpha,
- data_type: self.data_type,
- },
- )
- .expect("MoE experts two-pass failed");
- } else {
- let total_rows = suffix_length * k;
- let num_tiles_k = ((self.hidden_dim + k_tile - 1) / k_tile) as u32;
-
- self.experts_two_pass_prefill_kernel
- .encode(
- root,
- MoeExpertsTwoPassArguments {
- x_perm_buffer: &x_perm_buf,
- expert_offsets: &offsets_buf,
- row_expert_map: &row_expert_map_buf,
- hidden_buffer: &hidden_buf,
- output_buffer: &y_partial_buf,
- w13_all: &self.shared_weights.w13_buf,
- w2_all: &self.shared_weights.w2_buf,
- up_biases: &self.shared_weights.up_biases_buf,
- down_biases: &self.shared_weights.down_biases_buf,
- tile_counts: &tile_counts_buf,
- tile_offsets: &tile_offsets_buf,
- tile_map: &tile_map_buf,
- total_tiles: &total_tiles_buf,
- dispatch_args: &dispatch_args_buf,
- total_rows,
- d_model: self.model_dim,
- d_ff: self.hidden_dim,
- e,
- num_tiles_k,
- gating_code,
- gate_clip_min,
- gate_clip_max,
- up_clip_min,
- up_clip_max,
- silu_alpha,
- data_type: self.data_type,
- },
- )
- .expect("MoE experts two-pass prefill failed");
- }
+ let total_rows = suffix_length * k;
+ const K_TILE: usize = 64;
+ let num_tiles_k = ((self.hidden_dim + K_TILE - 1) / K_TILE) as u32;
+
+ self.experts_two_pass_decode_kernel
+ .encode(
+ root,
+ MoeExpertsTwoPassArguments {
+ x_perm_buffer: &buffers.x_perm,
+ expert_offsets: &buffers.offsets,
+ row_expert_map: &buffers.row_expert_map,
+ hidden_buffer: &buffers.hidden,
+ output_buffer: &buffers.y_partial,
+ w13_all: &self.shared_weights.w13_buf,
+ w2_all: &self.shared_weights.w2_buf,
+ up_biases: &self.shared_weights.up_biases_buf,
+ down_biases: &self.shared_weights.down_biases_buf,
+ tile_counts: &buffers.tile_counts,
+ tile_offsets: &buffers.tile_offsets,
+ tile_map: &buffers.tile_map,
+ total_tiles: &buffers.total_tiles,
+ dispatch_args: &buffers.dispatch_args,
+ total_rows,
+ d_model: self.model_dim,
+ d_ff: self.hidden_dim,
+ e,
+ num_tiles_k,
+ gating_code,
+ gate_clip_min,
+ gate_clip_max,
+ up_clip_min,
+ up_clip_max,
+ silu_alpha,
+ data_type: self.data_type,
+ },
+ )
+ .expect("MoE indirect decode failed");
+
+ self.encode_finalize(root, buffers, suffix_length);
+ }
+
+ fn encode_indirect_prefill_path(
+ &self,
+ root: &CommandBufferRef,
+ buffers: &MoeBuffers,
+ suffix_length: usize,
+ ) {
+ let e = self.moe_config.mixture_size;
+ let k = self.moe_config.num_experts_per_token;
+ let gating_code = Self::gating_code_from_activation(
+ &self.moe_config.expert_config.activation,
+ );
+
+ let gate_clip_min = self.moe_config.expert_config.gate_clipping[0]
+ .unwrap_or(f32::NEG_INFINITY);
+ let gate_clip_max = self.moe_config.expert_config.gate_clipping[1]
+ .unwrap_or(f32::INFINITY);
+ let up_clip_min = self.moe_config.expert_config.up_clipping[0];
+ let up_clip_max = self.moe_config.expert_config.up_clipping[1];
+ let silu_alpha = self.moe_config.expert_config.activation.alpha();
+
+ let total_rows = suffix_length * k;
+ const K_TILE: usize = 128;
+ let num_tiles_k = ((self.hidden_dim + K_TILE - 1) / K_TILE) as u32;
+
+ self.experts_two_pass_prefill_kernel
+ .encode(
+ root,
+ MoeExpertsTwoPassArguments {
+ x_perm_buffer: &buffers.x_perm,
+ expert_offsets: &buffers.offsets,
+ row_expert_map: &buffers.row_expert_map,
+ hidden_buffer: &buffers.hidden,
+ output_buffer: &buffers.y_partial,
+ w13_all: &self.shared_weights.w13_buf,
+ w2_all: &self.shared_weights.w2_buf,
+ up_biases: &self.shared_weights.up_biases_buf,
+ down_biases: &self.shared_weights.down_biases_buf,
+ tile_counts: &buffers.tile_counts,
+ tile_offsets: &buffers.tile_offsets,
+ tile_map: &buffers.tile_map,
+ total_tiles: &buffers.total_tiles,
+ dispatch_args: &buffers.dispatch_args,
+ total_rows,
+ d_model: self.model_dim,
+ d_ff: self.hidden_dim,
+ e,
+ num_tiles_k,
+ gating_code,
+ gate_clip_min,
+ gate_clip_max,
+ up_clip_min,
+ up_clip_max,
+ silu_alpha,
+ data_type: self.data_type,
+ },
+ )
+ .expect("MoE indirect prefill failed");
+
+ self.encode_finalize(root, buffers, suffix_length);
+ }
+
+ fn encode_finalize(
+ &self,
+ root: &CommandBufferRef,
+ buffers: &MoeBuffers,
+ suffix_length: usize,
+ ) {
+ let k = self.moe_config.num_experts_per_token;
self.finalize_kernel
.encode(
root,
- super::MoeFinalizeArguments {
- tok2row_buffer: &tok2row_buf,
- probs_buffer: &topk_probs_buf,
- y_partial_buffer: &y_partial_buf,
- y_out_buffer: &main_buf,
+ MoeFinalizeArguments {
+ tok2row_buffer: &buffers.tok2row,
+ probs_buffer: &buffers.topk_probs,
+ y_partial_buffer: &buffers.y_partial,
+ y_out_buffer: &buffers.main,
t: suffix_length,
d_model: self.model_dim,
k,
@@ -631,6 +731,95 @@ impl EncodableWithState for MoeBlockEncodable {
self.data_type,
)
.expect("MoE finalize failed");
+ }
+}
+
+impl EncodableWithState for MoeBlockEncodable {
+ fn encode(
+ &self,
+ state: &mut ForwardPassState,
+ command_buffer: &MPSCommandBuffer,
+ parameters: &EncodingParameters,
+ ) {
+ let suffix_length = state.active_suffix_length();
+ let arrays = state.arrays(&[
+ ArrayId::Main,
+ ArrayId::MoeTopkIds,
+ ArrayId::MoeTopkProbs,
+ ArrayId::MoeOffsets,
+ ArrayId::MoeSumK,
+ ArrayId::MoeBucketedTokenIds,
+ ArrayId::MoeBucketedProbs,
+ ArrayId::MoeXPerm,
+ ArrayId::MoeTok2Row,
+ ArrayId::MoeYPartial,
+ ArrayId::MoeTileCounts,
+ ArrayId::MoeTileOffsets,
+ ArrayId::MoeTileMap,
+ ArrayId::MoeTotalTiles,
+ ArrayId::MoeDispatchArgs,
+ ArrayId::MoeScatterPartials,
+ ArrayId::MoeScatterBlockBases,
+ ArrayId::MoeBlockAlloc,
+ ArrayId::MoeHidden,
+ ArrayId::MoeTwoPassRowExpertMap,
+ ]);
+
+ let clone_buffer = |array: &RefCell<MetalArray>| -> metal::Buffer {
+ let mut borrow = array.borrow_mut();
+ unsafe { borrow.mtl_buffer().clone() }
+ };
+
+ let mut array_iter = arrays.iter();
+ let buffers = MoeBuffers {
+ main: clone_buffer(array_iter.next().unwrap()),
+ topk_ids: clone_buffer(array_iter.next().unwrap()),
+ topk_probs: clone_buffer(array_iter.next().unwrap()),
+ offsets: clone_buffer(array_iter.next().unwrap()),
+ sumk: clone_buffer(array_iter.next().unwrap()),
+ bucketed_ids: clone_buffer(array_iter.next().unwrap()),
+ bucketed_probs: clone_buffer(array_iter.next().unwrap()),
+ x_perm: clone_buffer(array_iter.next().unwrap()),
+ tok2row: clone_buffer(array_iter.next().unwrap()),
+ y_partial: clone_buffer(array_iter.next().unwrap()),
+ tile_counts: clone_buffer(array_iter.next().unwrap()),
+ tile_offsets: clone_buffer(array_iter.next().unwrap()),
+ tile_map: clone_buffer(array_iter.next().unwrap()),
+ total_tiles: clone_buffer(array_iter.next().unwrap()),
+ dispatch_args: clone_buffer(array_iter.next().unwrap()),
+ partials: clone_buffer(array_iter.next().unwrap()),
+ block_bases: clone_buffer(array_iter.next().unwrap()),
+ block_alloc: clone_buffer(array_iter.next().unwrap()),
+ hidden: clone_buffer(array_iter.next().unwrap()),
+ row_expert_map: clone_buffer(array_iter.next().unwrap()),
+ };
+ debug_assert!(array_iter.next().is_none());
+
+ let mtl_command_buffer =
+ command_buffer.root_command_buffer().to_owned();
+ let root = &*mtl_command_buffer;
+
+ // Path selection:
+ // - suffix == 1: Single-token fused decode (no scatter/gather)
+ // - suffix <= 32: Indirect decode (GEMV)
+ // - suffix > 32: Indirect prefill (GEMM)
+ if suffix_length == 1 {
+ // Single-token path: router -> fused decode
+ self.encode_router(root, &buffers, suffix_length);
+ self.encode_single_token_path(root, &buffers);
+ } else if suffix_length <= INDIRECT_DECODE_THRESHOLD {
+ // Indirect decode path: router -> setup -> decode experts -> finalize
+ self.clear_indirect_buffers(root, &buffers, suffix_length);
+ self.encode_router(root, &buffers, suffix_length);
+ self.encode_indirect_setup(root, &buffers, suffix_length);
+ self.encode_indirect_decode_path(root, &buffers, suffix_length);
+ } else {
+ // Indirect prefill path: router -> setup -> prefill experts -> finalize
+ self.clear_indirect_buffers(root, &buffers, suffix_length);
+ self.encode_router(root, &buffers, suffix_length);
+ self.encode_indirect_setup(root, &buffers, suffix_length);
+ self.encode_indirect_prefill_path(root, &buffers, suffix_length);
+ }
if parameters.wait_until_completed {
command_buffer.commit_and_continue();
diff --git a/crates/uzu/src/backends/metal/kernel/moe/experts.rs b/crates/uzu/src/backends/metal/kernel/moe/experts.rs
index 8d8ffc35d..d516c13a4 100644
--- a/crates/uzu/src/backends/metal/kernel/moe/experts.rs
+++ b/crates/uzu/src/backends/metal/kernel/moe/experts.rs
@@ -538,3 +538,221 @@ impl MoeExpertsTwoPassPrefillKernel {
Ok(())
}
}
+
+/// Arguments for single-token MoE decode (T=1 optimized path)
+#[derive(Debug)]
+pub struct MoeExpertsSingleDecodeArguments<'a> {
+ /// Input activation [d_model]
+ pub x: &'a MTLBuffer,
+ /// Top-K expert indices from router [K]
+ pub topk_ids: &'a MTLBuffer,
+ /// Top-K probabilities from router [K]
+ pub topk_probs: &'a MTLBuffer,
+ /// Up/gate projection weights [E, 2*d_ff, d_model]
+ pub w13_all: &'a MTLBuffer,
+ /// Down projection weights [E, d_model, d_ff]
+ pub w2_all: &'a MTLBuffer,
+ /// Up/gate biases [E, 2*d_ff]
+ pub up_biases: &'a MTLBuffer,
+ /// Down biases [E, d_model]
+ pub down_biases: &'a MTLBuffer,
+ /// Hidden buffer [K, d_ff] - intermediate storage (f32)
+ pub hidden: &'a MTLBuffer,
+ /// Final output [d_model]
+ pub y: &'a MTLBuffer,
+ /// Model dimension
+ pub d_model: usize,
+ /// FFN hidden dimension
+ pub d_ff: usize,
+ /// Number of active experts per token
+ pub k: usize,
+ /// Gating activation: 0=GELU, 1=SiLU, 2=SwiGLU, 3=GEGLU
+ pub gating_code: u32,
+ /// SiLU activation alpha parameter
+ pub silu_alpha: f32,
+ /// Gate clipping min
+ pub gate_clip_min: f32,
+ /// Gate clipping max
+ pub gate_clip_max: f32,
+ /// Up clipping min
+ pub up_clip_min: f32,
+ /// Up clipping max
+ pub up_clip_max: f32,
+ /// Data type
+ pub data_type: KernelDataType,
+}
+
+pub struct MoeExpertsSingleDecodeKernel {
+ pass_a: Vec<Vec<MTLComputePipelineState>>, // [gate][dtype]
+ pass_b: Vec<MTLComputePipelineState>, // [dtype]
+}
+
+impl MoeExpertsSingleDecodeKernel {
+ pub fn new(ctx: &MTLContext) -> Result<Self, MoeExpertsError> {
+ let dtypes = [
+ KernelDataType::Float16,
+ KernelDataType::BFloat16,
+ KernelDataType::Float32,
+ ];
+
+ // Pass A: needs gating function constant
+ let mut pass_a = vec![Vec::with_capacity(dtypes.len()); 4];
+ for gate in 0u32..4u32 {
+ for dtype in &dtypes {
+ let suffix = dtype_suffix(*dtype);
+ let fcv = FunctionConstantValues::new();
+ fcv.set_constant_value_at_index(
+ &gate as *const u32 as *const std::ffi::c_void,
+ MTLDataType::UInt,
+ 30,
+ );
+ let kernel_name =
+ format!("moe_experts_decode_single_pass_a_{}", suffix);
+ let cache_key = format!("{}_gate_{}", kernel_name, gate);
+ pass_a[gate as usize].push(ctx.compute_pipeline_state_cached(
+ &cache_key,
+ &kernel_name,
+ Some(&fcv),
+ )?);
+ }
+ }
+
+ // Pass B (fused with finalize): no function constants needed
+ let mut pass_b = Vec::with_capacity(dtypes.len());
+ for dtype in &dtypes {
+ let suffix = dtype_suffix(*dtype);
+ let kernel_name =
+ format!("moe_experts_decode_single_pass_b_{}", suffix);
+ pass_b.push(ctx.compute_pipeline_state(&kernel_name, None)?);
+ }
+
+ Ok(Self {
+ pass_a,
+ pass_b,
+ })
+ }
+
+ /// Encode the single-token decode pipeline
+ ///
+ /// Dispatches:
+ /// 1. Pass A: grid = (h_blocks, K) - K parallel expert up+gate projections
+ /// 2. Pass B: grid = (d_blocks, 1) - computes final y directly (fused finalize)
+ pub fn encode(
+ &self,
+ command_buffer: &CommandBufferRef,
+ args: MoeExpertsSingleDecodeArguments,
+ ) -> Result<(), MoeExpertsError> {
+ if args.k == 0 {
+ return Ok(());
+ }
+
+ let gate_idx = (args.gating_code.min(3)) as usize;
+ let dtype_idx = dtype_index(args.data_type);
+
+ let d_model_u32 = args.d_model as u32;
+ let d_ff_u32 = args.d_ff as u32;
+ let k_u32 = args.k as u32;
+
+ // Pass A: 4 simdgroups (128 threads), each threadgroup outputs 4 elements
+ const PASSA_BLOCK_H: u32 = 4;
+ const PASSA_THREADS: u32 = 128;
+ let h_blocks = (d_ff_u32 + PASSA_BLOCK_H - 1) / PASSA_BLOCK_H;
+
+ // Pass A: x @ W13[expert] -> hidden
+ {
+ let pipeline = &self.pass_a[gate_idx][dtype_idx];
+ let encoder = command_buffer.new_compute_command_encoder();
+ encoder.set_compute_pipeline_state(pipeline);
+ encoder.set_buffer(0, Some(args.x), 0);
+ encoder.set_buffer(1, Some(args.topk_ids), 0);
+ encoder.set_buffer(2, Some(args.w13_all), 0);
+ encoder.set_buffer(3, Some(args.up_biases), 0);
+ encoder.set_buffer(4, Some(args.hidden), 0);
+ encoder.set_bytes(
+ 5,
+ size_of::<u32>() as u64,
+ &d_model_u32 as *const u32 as *const _,
+ );
+ encoder.set_bytes(
+ 6,
+ size_of::<u32>() as u64,
+ &d_ff_u32 as *const u32 as *const _,
+ );
+ encoder.set_bytes(
+ 7,
+ size_of::<u32>() as u64,
+ &k_u32 as *const u32 as *const _,
+ );
+ encoder.set_bytes(
+ 8,
+ size_of::<f32>() as u64,
+ &args.silu_alpha as *const f32 as *const _,
+ );
+ encoder.set_bytes(
+ 9,
+ size_of::<f32>() as u64,
+ &args.gate_clip_min as *const f32 as *const _,
+ );
+ encoder.set_bytes(
+ 10,
+ size_of::<f32>() as u64,
+ &args.gate_clip_max as *const f32 as *const _,
+ );
+ encoder.set_bytes(
+ 11,
+ size_of::<f32>() as u64,
+ &args.up_clip_min as *const f32 as *const _,
+ );
+ encoder.set_bytes(
+ 12,
+ size_of::<f32>() as u64,
+ &args.up_clip_max as *const f32 as *const _,
+ );
+ encoder.dispatch_thread_groups(
+ MTLSize::new(h_blocks as u64, args.k as u64, 1),
+ MTLSize::new(PASSA_THREADS as u64, 1, 1),
+ );
+ encoder.end_encoding();
+ }
+
+ // Pass B: 8 simdgroups (256 threads), outputs final y directly
+ const PASSB_BLOCK_D: u32 = 8;
+ const PASSB_THREADS: u32 = 256;
+ let d_blocks = (d_model_u32 + PASSB_BLOCK_D - 1) / PASSB_BLOCK_D;
+
+ // Pass B: hidden @ W2[expert] -> y (with weighted sum)
+ {
+ let pipeline = &self.pass_b[dtype_idx];
+ let encoder = command_buffer.new_compute_command_encoder();
+ encoder.set_compute_pipeline_state(pipeline);
+ encoder.set_buffer(0, Some(args.hidden), 0);
+ encoder.set_buffer(1, Some(args.topk_ids), 0);
+ encoder.set_buffer(2, Some(args.topk_probs), 0);
+ encoder.set_buffer(3, Some(args.w2_all), 0);
+ encoder.set_buffer(4, Some(args.down_biases), 0);
+ encoder.set_buffer(5, Some(args.y), 0);
+ encoder.set_bytes(
+ 6,
+ size_of::<u32>() as u64,
+ &d_model_u32 as *const u32 as *const _,
+ );
+ encoder.set_bytes(
+ 7,
+ size_of::<u32>() as u64,
+ &d_ff_u32 as *const u32 as *const _,
+ );
+ encoder.set_bytes(
+ 8,
+ size_of::<u32>() as u64,
+ &k_u32 as *const u32 as *const _,
+ );
+ encoder.dispatch_thread_groups(
+ MTLSize::new(d_blocks as u64, 1, 1),
+ MTLSize::new(PASSB_THREADS as u64, 1, 1),
+ );
+ encoder.end_encoding();
+ }
+
+ Ok(())
+ }
+}
diff --git a/crates/uzu/src/backends/metal/kernel/moe/experts_single.metal b/crates/uzu/src/backends/metal/kernel/moe/experts_single.metal
new file mode 100644
index 000000000..23c3c1252
--- /dev/null
+++ b/crates/uzu/src/backends/metal/kernel/moe/experts_single.metal
@@ -0,0 +1,239 @@
+#include <metal_stdlib>
+#include <metal_simdgroup>
+using namespace metal;
+
+constant uint GATING_SEL [[function_constant(30)]]; // 0=GELU, 1=SiLU, 2=SwiGLU, 3=GEGLU
+
+static inline float gelu_approx(float x) {
+ const float k0 = 0.7978845608f;
+ const float k1 = 0.044715f;
+ if (x > 10.0f) return x;
+ if (x < -10.0f) return 0.0f;
+ return 0.5f * x * (1.0f + tanh(clamp(k0 * (x + k1 * x * x * x), -10.0f, 10.0f)));
+}
+
+static inline float silu(float x, float alpha) {
+ return x / (1.0f + exp(-alpha * x));
+}
+
+// ============================================================================
+// Pass A: x @ W13[expert] → hidden[k]
+// Each threadgroup: 4 simdgroups, outputs 4 elements (1 per simdgroup)
+// Each simdgroup: 32 threads reduce d_model with float4 vectorized loads
+// Grid: (ceil(d_ff/4), K)
+// ============================================================================
+
+template<typename T, typename T4>
+inline void moe_experts_decode_single_pass_a_impl(
+ device const T* x,
+ device const int* topk_ids,
+ device const T* W13_all,
+ device const T* biases,
+ device float* hidden_out,
+ uint d_model,
+ uint d_ff,
+ uint K,
+ float silu_alpha,
+ float gate_clip_min,
+ float gate_clip_max,
+ float up_clip_min,
+ float up_clip_max,
+ uint k_slot,
+ uint h_block_idx,
+ uint simd_gid,
+ uint simd_lid
+) {
+ const int expert_id = topk_ids[k_slot];
+ if (expert_id < 0) return;
+ const uint expert_u = uint(expert_id);
+
+ const uint h_idx = h_block_idx * 4 + simd_gid;
+ if (h_idx >= d_ff) return;
+
+ const ulong w13_stride = (ulong)d_model * (ulong)(2 * d_ff);
+ const ulong w13_base = (ulong)expert_u * w13_stride;
+ const ulong bias_base = (ulong)expert_u * (ulong)(2 * d_ff);
+
+ device const T* w_up_row = W13_all + w13_base + (ulong)h_idx * (ulong)d_model;
+ device const T* w_gate_row = W13_all + w13_base + (ulong)(d_ff + h_idx) * (ulong)d_model;
+
+ float acc_up = 0.0f;
+ float acc_gate = 0.0f;
+
+ const uint vec_iters = d_model / 128;
+
+ for (uint i = 0; i < vec_iters; ++i) {
+ uint base_idx = i * 128 + simd_lid * 4;
+
+ T4 x_vec = *reinterpret_cast<device const T4*>(x + base_idx);
+ T4 w_up_vec = *reinterpret_cast<device const T4*>(w_up_row + base_idx);
+
+ acc_up += float(x_vec.x) * float(w_up_vec.x);
+ acc_up += float(x_vec.y) * float(w_up_vec.y);
+ acc_up += float(x_vec.z) * float(w_up_vec.z);
+ acc_up += float(x_vec.w) * float(w_up_vec.w);
+
+ if (GATING_SEL > 1) {
+ T4 w_gate_vec = *reinterpret_cast<device const T4*>(w_gate_row + base_idx);
+ acc_gate += float(x_vec.x) * float(w_gate_vec.x);
+ acc_gate += float(x_vec.y) * float(w_gate_vec.y);
+ acc_gate += float(x_vec.z) * float(w_gate_vec.z);
+ acc_gate += float(x_vec.w) * float(w_gate_vec.w);
+ }
+ }
+
+ uint leftover_start = vec_iters * 128 + simd_lid;
+ for (uint idx = leftover_start; idx < d_model; idx += 32) {
+ float xv = float(x[idx]);
+ acc_up += xv * float(w_up_row[idx]);
+ if (GATING_SEL > 1) {
+ acc_gate += xv * float(w_gate_row[idx]);
+ }
+ }
+
+ acc_up = simd_sum(acc_up);
+ if (GATING_SEL > 1) {
+ acc_gate = simd_sum(acc_gate);
+ }
+
+ if (simd_lid == 0) {
+ float up_val = clamp(acc_up + float(biases[bias_base + h_idx]), up_clip_min, up_clip_max);
+
+ float activated;
+ if (GATING_SEL <= 1) {
+ activated = (GATING_SEL == 0) ? gelu_approx(up_val) : silu(up_val, silu_alpha);
+ } else {
+ float gate_val = clamp(acc_gate + float(biases[bias_base + d_ff + h_idx]), gate_clip_min, gate_clip_max);
+ float gate_act = (GATING_SEL == 2) ? silu(gate_val, silu_alpha) : gelu_approx(gate_val);
+ activated = gate_act * up_val;
+ }
+
+ hidden_out[(ulong)k_slot * (ulong)d_ff + (ulong)h_idx] = activated;
+ }
+}
+
+#define MOE_DECODE_SINGLE_PASS_A_KERNEL(DTYPE, DTYPE4, SUFFIX) \
+kernel void moe_experts_decode_single_pass_a_##SUFFIX( \
+ device const DTYPE* x [[buffer(0)]], \
+ device const int* topk_ids [[buffer(1)]], \
+ device const DTYPE* W13_all [[buffer(2)]], \
+ device const DTYPE* biases [[buffer(3)]], \
+ device float* hidden_out [[buffer(4)]], \
+ constant uint& d_model [[buffer(5)]], \
+ constant uint& d_ff [[buffer(6)]], \
+ constant uint& K [[buffer(7)]], \
+ constant float& silu_alpha [[buffer(8)]], \
+ constant float& gate_clip_min [[buffer(9)]], \
+ constant float& gate_clip_max [[buffer(10)]], \
+ constant float& up_clip_min [[buffer(11)]], \
+ constant float& up_clip_max [[buffer(12)]], \
+ uint2 tgpig [[threadgroup_position_in_grid]], \
+ uint simd_gid [[simdgroup_index_in_threadgroup]], \
+ uint simd_lid [[thread_index_in_simdgroup]]) \
+{ \
+ moe_experts_decode_single_pass_a_impl<DTYPE, DTYPE4>( \
+ x, topk_ids, W13_all, biases, hidden_out, \
+ d_model, d_ff, K, silu_alpha, \
+ gate_clip_min, gate_clip_max, up_clip_min, up_clip_max, \
+ tgpig.y, tgpig.x, simd_gid, simd_lid); \
+}
+
+MOE_DECODE_SINGLE_PASS_A_KERNEL(half, half4, f16)
+MOE_DECODE_SINGLE_PASS_A_KERNEL(bfloat, bfloat4, bf16)
+MOE_DECODE_SINGLE_PASS_A_KERNEL(float, float4, f32)
+
+
+// ============================================================================
+// Pass B (fused with finalize): hidden[k] @ W2[expert] → y (directly)
+// Computes: y[d] = Σ_k prob[k] * (hidden[k] @ W2[expert_k, d] + bias)
+// Each simdgroup computes one final output element
+// Grid: (ceil(d_model/8), 1) - NOT per K!
+// ============================================================================
+
+template<typename T, typename T4>
+inline void moe_experts_decode_single_pass_b_impl(
+ device const float* hidden, // [K, d_ff]
+ device const int* topk_ids, // [K]
+ device const T* topk_probs, // [K]
+ device const T* W2_all, // [E, d_model, d_ff]
+ device const T* biases, // [E, d_model]
+ device T* y, // [d_model]
+ uint d_model,
+ uint d_ff,
+ uint K,
+ uint d_block,
+ uint simd_gid,
+ uint simd_lid
+) {
+ const uint my_col = d_block * 8 + simd_gid;
+ if (my_col >= d_model) return;
+
+ const uint vec_iters = d_ff / 128;
+ const ulong w2_expert_stride = (ulong)d_model * (ulong)d_ff;
+
+ float final_acc = 0.0f;
+
+ // Loop over K experts
+ for (uint k = 0; k < K; ++k) {
+ const uint expert_u = uint(topk_ids[k]);
+ const float prob = float(topk_probs[k]);
+
+ device const float* hidden_ptr = hidden + (ulong)k * (ulong)d_ff;
+ device const T* w2_ptr = W2_all + (ulong)expert_u * w2_expert_stride + (ulong)my_col * (ulong)d_ff;
+
+ float acc = 0.0f;
+
+ // Vectorized reduction
+ for (uint i = 0; i < vec_iters; ++i) {
+ uint base_idx = i * 128 + simd_lid * 4;
+
+ float4 h_vec = *reinterpret_cast<device const float4*>(hidden_ptr + base_idx);
+ T4 w_vec = *reinterpret_cast<device const T4*>(w2_ptr + base_idx);
+
+ acc += h_vec.x * float(w_vec.x);
+ acc += h_vec.y * float(w_vec.y);
+ acc += h_vec.z * float(w_vec.z);
+ acc += h_vec.w * float(w_vec.w);
+ }
+
+ // Remainder
+ for (uint idx = vec_iters * 128 + simd_lid; idx < d_ff; idx += 32) {
+ acc += hidden_ptr[idx] * float(w2_ptr[idx]);
+ }
+
+ // Simdgroup reduction and accumulate
+ float result = simd_sum(acc);
+ if (simd_lid == 0) {
+ result += float(biases[(ulong)expert_u * (ulong)d_model + (ulong)my_col]);
+ final_acc += prob * result;
+ }
+ }
+
+ if (simd_lid == 0) {
+ y[my_col] = T(final_acc);
+ }
+}
+
+#define MOE_DECODE_SINGLE_PASS_B_KERNEL(DTYPE, DTYPE4, SUFFIX) \
+kernel void moe_experts_decode_single_pass_b_##SUFFIX( \
+ device const float* hidden [[buffer(0)]], \
+ device const int* topk_ids [[buffer(1)]], \
+ device const DTYPE* topk_probs [[buffer(2)]], \
+ device const DTYPE* W2_all [[buffer(3)]], \
+ device const DTYPE* biases [[buffer(4)]], \
+ device DTYPE* y [[buffer(5)]], \
+ constant uint& d_model [[buffer(6)]], \
+ constant uint& d_ff [[buffer(7)]], \
+ constant uint& K [[buffer(8)]], \
+ uint tgpig [[threadgroup_position_in_grid]], \
+ uint simd_gid [[simdgroup_index_in_threadgroup]], \
+ uint simd_lid [[thread_index_in_simdgroup]]) \
+{ \
+ moe_experts_decode_single_pass_b_impl<DTYPE, DTYPE4>( \
+ hidden, topk_ids, topk_probs, W2_all, biases, y, \
+ d_model, d_ff, K, tgpig, simd_gid, simd_lid); \
+}
+
+MOE_DECODE_SINGLE_PASS_B_KERNEL(half, half4, f16)
+MOE_DECODE_SINGLE_PASS_B_KERNEL(bfloat, bfloat4, bf16)
+MOE_DECODE_SINGLE_PASS_B_KERNEL(float, float4, f32)
diff --git a/crates/uzu/src/backends/metal/kernel/moe/experts_two_pass_decode.metal b/crates/uzu/src/backends/metal/kernel/moe/experts_two_pass_decode.metal
index 67d95608e..d1631c395 100644
--- a/crates/uzu/src/backends/metal/kernel/moe/experts_two_pass_decode.metal
+++ b/crates/uzu/src/backends/metal/kernel/moe/experts_two_pass_decode.metal
@@ -2,59 +2,31 @@
#include <metal_simdgroup>
using namespace metal;
-constant uint GATING_SEL
- [[function_constant(30)]]; // 0=GELU,1=SiLU,2=SwiGLU,3=GEGLU
-constant uint TILE_H
- [[function_constant(32)]]; // tile size for hidden dimension in Pass A
+constant uint GATING_SEL [[function_constant(30)]]; // 0=GELU,1=SiLU,2=SwiGLU,3=GEGLU
static inline float gelu_approx(float x) {
- const float k0 = 0.7978845608f;
- const float k1 = 0.044715f;
- if (x > 10.0f)
- return x;
- if (x < -10.0f)
- return 0.0f;
- return 0.5f * x *
- (1.0f + tanh(clamp(k0 * (x + k1 * x * x * x), -10.0f, 10.0f)));
+ const float k0 = 0.7978845608f;
+ const float k1 = 0.044715f;
+ if (x > 10.0f) return x;
+ if (x < -10.0f) return 0.0f;
+ return 0.5f * x * (1.0f + tanh(clamp(k0 * (x + k1 * x * x * x), -10.0f, 10.0f)));
}
static inline float silu(float x, float alpha) {
- return x / (1.0f + exp(-alpha * x));
+ return x / (1.0f + exp(-alpha * x));
}
-#define MTL_CONST static constant constexpr const
-#define MTL_PRAGMA_UNROLL _Pragma("clang loop unroll(full)")
-
-// Tiling configuration (tunable)
-MTL_CONST uint BM = 1; // Simdgroups in M dimension within threadgroup
-MTL_CONST uint BN = 4; // Simdgroups in N dimension within threadgroup
-MTL_CONST uint SM = 1; // Thread rows per simdgroup
-MTL_CONST uint SN = 32; // Thread cols per simdgroup (full warp width)
-MTL_CONST uint TM = 4; // Output elements per thread in M
-MTL_CONST uint TN = 4; // Input elements per thread in N (per iteration)
-
-MTL_CONST uint THREADS_M = BM * SM;
-MTL_CONST uint THREADS_N = BN * SN;
-MTL_CONST uint BLOCK_M = THREADS_M * TM;
-MTL_CONST uint BLOCK_N = THREADS_N * TN;
-
-static_assert(SM * SN == 32, "simdgroup must have 32 threads");
-static_assert(
- SN == 4 || SN == 8 || SN == 16 || SN == 32,
- "SN must be 4, 8, 16, or 32"
-);
-
-// === Pass A: Optimized GEMV with hierarchical tiling ===
-// Computes: hidden[row, h] = activation(x[row, d] @ W13[d, 2*h])
-// AccumT is used for internal accumulations, output still stored as T
-template <typename T, typename AccumT>
+// === Pass A: Vectorized GEMV with float4 loads ===
+// Structure: 4 simdgroups (128 threads), each outputs 1 hidden element
+// Each simdgroup: 32 threads reduce d_model using float4 vectorized loads
+// Grid: (h_blocks, rows, 1) where h_blocks = ceil(d_ff/4)
+template<typename T, typename T4>
void moe_experts_decode_pass_a_impl(
- device const T* X_perm, // [total_rows, d_model]
- device const uint* expert_offsets, // [E + 1]
- device const T* W13_all, // weights in layout [E, 2*d_ff, d_model]
- device const T* up_biases, // [E, 2*d_ff]
- device float*
- hidden_out, // [total_rows, d_ff] - f32 for activation precision
+ device const T* X_perm, // [total_rows, d_model]
+ device const uint* expert_offsets, // [E + 1]
+ device const T* W13_all, // [E, 2*d_ff, d_model]
+ device const T* up_biases, // [E, 2*d_ff]
+ device float* hidden_out, // [total_rows, d_ff]
uint d_model,
uint d_ff,
uint E,
@@ -67,409 +39,260 @@ void moe_experts_decode_pass_a_impl(
uint row_in_expert,
uint h_block_idx,
uint simd_gid,
- uint simd_lid,
- threadgroup float* tgp_memory
+ uint simd_lid
) {
- // Thread position within simdgroup
- const uint thrM = SN != 32 ? simd_lid / SN : 0;
- const uint thrN = SN != 32 ? simd_lid % SN : uint(simd_lid);
-
- // Simdgroup position within threadgroup
- const uint sgN = BN != 1 ? (simd_gid % BN) : 0;
- const uint simdM = BN != 1 ? SM * (simd_gid / BN) : uint(SM * simd_gid);
- const uint simdN = BN != 1 ? SN * (simd_gid % BN) : 0;
-
- // Thread's work block
- int bm = (simdM + thrM) * TM;
- int bn = (simdN + thrN) * TN;
-
- // Output row position
- const uint seg_start = expert_offsets[expert_idx];
- const uint seg_end = expert_offsets[expert_idx + 1];
- uint global_row = seg_start + row_in_expert;
-
- if (global_row >= seg_end)
- return;
-
- // Calculate output h index
- int h_row = h_block_idx * BLOCK_M + bm;
-
- // Adjust tail block to ensure in-bounds reads
- if (h_row + TM > d_ff) {
- h_row = d_ff > TM ? int(d_ff - TM) : 0;
- bm = h_row - int(h_block_idx * BLOCK_M);
- }
-
- if (h_row >= int(d_ff))
- return;
-
- // Accumulation arrays for up and gate projections
- thread AccumT result_up[TM] = {0};
- thread AccumT result_gate[TM] = {0};
- thread T weights_up[TN];
- thread T weights_gate[TN];
- thread AccumT x_vals[TN];
-
- // Expert weight base addresses
- const ulong w13_base = (ulong)expert_idx * (ulong)d_model * (ulong)(2 * d_ff);
- const ulong bias_base = (ulong)expert_idx * (ulong)(2 * d_ff);
- const ulong x_row_base = (ulong)global_row * (ulong)d_model;
-
- // Main accumulation loop over d_model in blocks of BLOCK_N
- const uint n_iter = d_model / BLOCK_N;
- const uint leftover = d_model - (n_iter * BLOCK_N);
-
- for (uint i = 0; i < n_iter; ++i) {
- // Load x values for this block
- MTL_PRAGMA_UNROLL
- for (uint tn = 0; tn < TN; tn++) {
- uint d_idx = bn + tn;
- x_vals[tn] = AccumT(X_perm[x_row_base + d_idx]);
- }
+ // Validate row bounds
+ const uint seg_start = expert_offsets[expert_idx];
+ const uint seg_end = expert_offsets[expert_idx + 1];
+ const uint global_row = seg_start + row_in_expert;
+ if (global_row >= seg_end) return;
- // Accumulate for each output row
- MTL_PRAGMA_UNROLL
- for (uint tm = 0; tm < TM; tm++) {
- uint h_idx = h_row + tm;
-
- // Load weights for up projection
- MTL_PRAGMA_UNROLL
- for (uint tn = 0; tn < TN; tn++) {
- uint d_idx = bn + tn;
- ulong w_idx = w13_base + (ulong)h_idx * (ulong)d_model + (ulong)d_idx;
- weights_up[tn] = W13_all[w_idx];
- }
-
- // Accumulate up projection
- MTL_PRAGMA_UNROLL
- for (uint tn = 0; tn < TN; tn++) {
- result_up[tm] += x_vals[tn] * AccumT(weights_up[tn]);
- }
-
- // Load and accumulate gate projection if needed
- if (GATING_SEL > 1) {
- MTL_PRAGMA_UNROLL
- for (uint tn = 0; tn < TN; tn++) {
- uint d_idx = bn + tn;
- ulong w_gate_idx =
- w13_base + (ulong)(d_ff + h_idx) * (ulong)d_model + (ulong)d_idx;
- weights_gate[tn] = W13_all[w_gate_idx];
- }
+ // Each simdgroup outputs one hidden element
+ const uint h_idx = h_block_idx * 4 + simd_gid;
+ if (h_idx >= d_ff) return;
- MTL_PRAGMA_UNROLL
- for (uint tn = 0; tn < TN; tn++) {
- result_gate[tm] += x_vals[tn] * AccumT(weights_gate[tn]);
- }
- }
- }
+ // Base addresses
+ const ulong w13_stride = (ulong)d_model * (ulong)(2 * d_ff);
+ const ulong w13_base = (ulong)expert_idx * w13_stride;
+ const ulong bias_base = (ulong)expert_idx * (ulong)(2 * d_ff);
+ const ulong x_row_base = (ulong)global_row * (ulong)d_model;
+
+ device const T* x_ptr = X_perm + x_row_base;
+ device const T* w_up_row = W13_all + w13_base + (ulong)h_idx * (ulong)d_model;
+ device const T* w_gate_row = W13_all + w13_base + (ulong)(d_ff + h_idx) * (ulong)d_model;
- bn += BLOCK_N;
- }
+ float acc_up = 0.0f;
+ float acc_gate = 0.0f;
- // Handle leftover elements
- if (leftover > 0) {
- MTL_PRAGMA_UNROLL
- for (uint tn = 0; tn < TN; tn++) {
- uint d_idx = bn + tn;
- x_vals[tn] =
- (d_idx < d_model) ? AccumT(X_perm[x_row_base + d_idx]) : AccumT(0.0);
+ // Vectorized reduction: 32 threads × 4 elements = 128 elements per iteration
+ const uint vec_iters = d_model / 128;
+
+ for (uint i = 0; i < vec_iters; ++i) {
+ uint base_idx = i * 128 + simd_lid * 4;
+
+ T4 x_vec = *reinterpret_cast<device const T4*>(x_ptr + base_idx);
+ T4 w_up_vec = *reinterpret_cast<device const T4*>(w_up_row + base_idx);
+
+ acc_up += float(x_vec.x) * float(w_up_vec.x);
+ acc_up += float(x_vec.y) * float(w_up_vec.y);
+ acc_up += float(x_vec.z) * float(w_up_vec.z);
+ acc_up += float(x_vec.w) * float(w_up_vec.w);
+
+ if (GATING_SEL > 1) {
+ T4 w_gate_vec = *reinterpret_cast<device const T4*>(w_gate_row + base_idx);
+ acc_gate += float(x_vec.x) * float(w_gate_vec.x);
+ acc_gate += float(x_vec.y) * float(w_gate_vec.y);
+ acc_gate += float(x_vec.z) * float(w_gate_vec.z);
+ acc_gate += float(x_vec.w) * float(w_gate_vec.w);
+ }
}
- MTL_PRAGMA_UNROLL
- for (uint tm = 0; tm < TM; tm++) {
- uint h_idx = h_row + tm;
-
- MTL_PRAGMA_UNROLL
- for (uint tn = 0; tn < TN; tn++) {
- uint d_idx = bn + tn;
- if (d_idx < d_model) {
- ulong w_idx = w13_base + (ulong)h_idx * (ulong)d_model + (ulong)d_idx;
- weights_up[tn] = W13_all[w_idx];
- result_up[tm] += x_vals[tn] * AccumT(weights_up[tn]);
-
- if (GATING_SEL > 1) {
- ulong w_gate_idx = w13_base +
- (ulong)(d_ff + h_idx) * (ulong)d_model +
- (ulong)d_idx;
- weights_gate[tn] = W13_all[w_gate_idx];
- result_gate[tm] += x_vals[tn] * AccumT(weights_gate[tn]);
- }
+ // Handle leftover elements
+ uint leftover_start = vec_iters * 128 + simd_lid;
+ for (uint idx = leftover_start; idx < d_model; idx += 32) {
+ float xv = float(x_ptr[idx]);
+ acc_up += xv * float(w_up_row[idx]);
+ if (GATING_SEL > 1) {
+ acc_gate += xv * float(w_gate_row[idx]);
}
- }
}
- }
- // Simdgroup reduction across thrN dimension
- MTL_PRAGMA_UNROLL
- for (uint tm = 0; tm < TM; tm++) {
- result_up[tm] = simd_sum(result_up[tm]);
+ // Simdgroup reduction
+ acc_up = simd_sum(acc_up);
if (GATING_SEL > 1) {
- result_gate[tm] = simd_sum(result_gate[tm]);
+ acc_gate = simd_sum(acc_gate);
}
- }
- // Threadgroup reduction if BN > 1
- if (BN > 1) {
- threadgroup float* tgp_up = tgp_memory;
- threadgroup float* tgp_gate = tgp_memory + BN * (BLOCK_M + TM);
+ // Lane 0 applies activation and writes result
+ if (simd_lid == 0) {
+ float up_val = acc_up + float(up_biases[bias_base + h_idx]);
+ up_val = clamp(up_val, up_clip_min, up_clip_max);
- if (thrN == 0) {
- MTL_PRAGMA_UNROLL
- for (uint tm = 0; tm < TM; tm++) {
- tgp_up[sgN * (BLOCK_M + TM) + bm + tm] = float(result_up[tm]);
- if (GATING_SEL > 1) {
- tgp_gate[sgN * (BLOCK_M + TM) + bm + tm] = float(result_gate[tm]);
- }
- }
-
- threadgroup_barrier(mem_flags::mem_threadgroup);
-
- if (sgN == 0) {
- MTL_PRAGMA_UNROLL
- for (uint sgn = 1; sgn < BN; sgn++) {
- MTL_PRAGMA_UNROLL
- for (uint tm = 0; tm < TM; tm++) {
- result_up[tm] += AccumT(tgp_up[sgn * (BLOCK_M + TM) + bm + tm]);
- if (GATING_SEL > 1) {
- result_gate[tm] +=
- AccumT(tgp_gate[sgn * (BLOCK_M + TM) + bm + tm]);
- }
- }
- }
- }
- }
- }
-
- // Apply activation and write output (only first thread in simdgroup width)
- if (simdN == 0 && thrN == 0) {
- MTL_PRAGMA_UNROLL
- for (uint tm = 0; tm < TM; tm++) {
- uint h_idx = h_row + tm;
- if (h_idx < d_ff) {
- // Add bias and clip
- AccumT up_val = result_up[tm] + AccumT(up_biases[bias_base + h_idx]);
- up_val = AccumT(clamp(float(up_val), up_clip_min, up_clip_max));
-
- AccumT activated_val;
+ float activated;
if (GATING_SEL <= 1) {
- activated_val = (GATING_SEL == 0)
- ? AccumT(gelu_approx(float(up_val)))
- : AccumT(silu(float(up_val), silu_alpha));
+ activated = (GATING_SEL == 0) ? gelu_approx(up_val) : silu(up_val, silu_alpha);
} else {
- AccumT gate_val =
- result_gate[tm] + AccumT(up_biases[bias_base + d_ff + h_idx]);
- gate_val =
- AccumT(clamp(float(gate_val), gate_clip_min, gate_clip_max));
- AccumT gate_activated =
- (GATING_SEL == 2) ? AccumT(silu(float(gate_val), silu_alpha))
- : AccumT(gelu_approx(float(gate_val)));
- activated_val = gate_activated * up_val;
+ float gate_val = acc_gate + float(up_biases[bias_base + d_ff + h_idx]);
+ gate_val = clamp(gate_val, gate_clip_min, gate_clip_max);
+ float gate_act = (GATING_SEL == 2) ? silu(gate_val, silu_alpha) : gelu_approx(gate_val);
+ activated = gate_act * up_val;
}
- ulong hidden_idx = (ulong)global_row * (ulong)d_ff + (ulong)h_idx;
- hidden_out[hidden_idx] = float(activated_val);
- }
+ hidden_out[(ulong)global_row * (ulong)d_ff + (ulong)h_idx] = activated;
}
- }
}
-#define MOE_PASS_A_KERNEL(DTYPE, SUFFIX) \
- kernel void moe_experts_decode_pass_a_##SUFFIX( \
- device const DTYPE* X_perm [[buffer(0)]], \
- device const uint* expert_offsets [[buffer(1)]], \
- device const DTYPE* W13_all [[buffer(2)]], \
- device float* hidden_out [[buffer(3)]], \
- device const DTYPE* up_biases [[buffer(4)]], \
- constant uint& d_model [[buffer(5)]], \
- constant uint& d_ff [[buffer(6)]], \
- constant uint& E [[buffer(7)]], \
- constant float& gate_clip_min [[buffer(8)]], \
- constant float& gate_clip_max [[buffer(9)]], \
- constant float& up_clip_min [[buffer(10)]], \
- constant float& up_clip_max [[buffer(11)]], \
- constant float& silu_alpha [[buffer(12)]], \
- uint3 tgpig [[threadgroup_position_in_grid]], \
- uint simd_gid [[simdgroup_index_in_threadgroup]], \
- uint simd_lid [[thread_index_in_simdgroup]] \
- ) { \
- constexpr uint tgp_mem_size = (BN > 1) ? 2 * BN * (BLOCK_M + TM) : 1; \
- threadgroup float tgp_memory[tgp_mem_size]; \
- \
- moe_experts_decode_pass_a_impl<DTYPE, float>( \
- X_perm, \
- expert_offsets, \
- W13_all, \
- up_biases, \
- hidden_out, \
- d_model, \
- d_ff, \
- E, \
- gate_clip_min, \
- gate_clip_max, \
- up_clip_min, \
- up_clip_max, \
- silu_alpha, \
- tgpig.y, \
- tgpig.z, \
- tgpig.x, \
- simd_gid, \
- simd_lid, \
- (BN > 1) ? tgp_memory : nullptr \
- ); \
- }
-
-MOE_PASS_A_KERNEL(bfloat, bf16)
-MOE_PASS_A_KERNEL(half, f16)
-MOE_PASS_A_KERNEL(float, f32)
+#define MOE_PASS_A_KERNEL(DTYPE, DTYPE4, SUFFIX) \
+kernel void moe_experts_decode_pass_a_##SUFFIX( \
+ device const DTYPE* X_perm [[buffer(0)]], \
+ device const uint* expert_offsets [[buffer(1)]], \
+ device const DTYPE* W13_all [[buffer(2)]], \
+ device float* hidden_out [[buffer(3)]], \
+ device const DTYPE* up_biases [[buffer(4)]], \
+ constant uint& d_model [[buffer(5)]], \
+ constant uint& d_ff [[buffer(6)]], \
+ constant uint& E [[buffer(7)]], \
+ constant float& gate_clip_min [[buffer(8)]], \
+ constant float& gate_clip_max [[buffer(9)]], \
+ constant float& up_clip_min [[buffer(10)]], \
+ constant float& up_clip_max [[buffer(11)]], \
+ constant float& silu_alpha [[buffer(12)]], \
+ uint3 tgpig [[threadgroup_position_in_grid]], \
+ uint simd_gid [[simdgroup_index_in_threadgroup]], \
+ uint simd_lid [[thread_index_in_simdgroup]]) \
+{ \
+ moe_experts_decode_pass_a_impl<DTYPE, DTYPE4>( \
+ X_perm, expert_offsets, W13_all, up_biases, hidden_out, \
+ d_model, d_ff, E, gate_clip_min, gate_clip_max, \
+ up_clip_min, up_clip_max, silu_alpha, \
+ tgpig.y, tgpig.z, tgpig.x, \
+ simd_gid, simd_lid); \
+}
+
+MOE_PASS_A_KERNEL(bfloat, bfloat4, bf16)
+MOE_PASS_A_KERNEL(half, half4, f16)
+MOE_PASS_A_KERNEL(float, float4, f32)
// === Helper kernels for indirect dispatch of Pass A ===
// Count tiles per expert: tiles = (num_rows > 0) ? num_rows * h_blocks : 0
kernel void moe_pass_a_tile_counts(
- device const uint* expert_offsets [[buffer(0)]], // [E+1]
- device uint* tile_counts [[buffer(1)]], // [E]
+ device const uint* expert_offsets [[buffer(0)]], // [E+1]
+ device uint* tile_counts [[buffer(1)]], // [E]
constant uint& E [[buffer(2)]],
constant uint& h_blocks [[buffer(3)]],
uint tid [[thread_position_in_grid]]
) {
- if (tid >= E)
- return;
- const uint start = expert_offsets[tid];
- const uint end = expert_offsets[tid + 1];
- const uint num_rows = end - start;
- tile_counts[tid] = (num_rows > 0) ? (num_rows * h_blocks) : 0;
+ if (tid >= E) return;
+ const uint start = expert_offsets[tid];
+ const uint end = expert_offsets[tid + 1];
+ const uint num_rows = end - start;
+ tile_counts[tid] = (num_rows > 0) ? (num_rows * h_blocks) : 0;
}
// Exclusive scan of tile_counts to get tile_offsets and total_tiles
kernel void moe_pass_a_tile_scan(
- device const uint* tile_counts [[buffer(0)]], // [E]
- device uint* tile_offsets [[buffer(1)]], // [E+1]
- device uint* total_tiles [[buffer(2)]], // [1]
+ device const uint* tile_counts [[buffer(0)]], // [E]
+ device uint* tile_offsets [[buffer(1)]], // [E+1]
+ device uint* total_tiles [[buffer(2)]], // [1]
constant uint& E [[buffer(3)]],
uint lid [[thread_index_in_threadgroup]],
threadgroup uint* scratch [[threadgroup(0)]]
) {
- // Simple single-threadgroup scan (works for E <= 1024)
- const uint idx = lid;
-
- // Load into threadgroup memory
- if (idx < E) {
- scratch[idx] = tile_counts[idx];
- } else {
- scratch[idx] = 0;
- }
- threadgroup_barrier(mem_flags::mem_threadgroup);
-
- // Kogge-Stone scan
- uint val = scratch[idx];
- MTL_PRAGMA_UNROLL
- for (uint offset = 1; offset < 1024; offset *= 2) {
- uint temp = 0;
- if (idx >= offset && idx < E) {
- temp = scratch[idx - offset];
+ // Simple single-threadgroup scan (works for E <= 1024)
+ const uint idx = lid;
+
+ // Load into threadgroup memory
+ if (idx < E) {
+ scratch[idx] = tile_counts[idx];
+ } else {
+ scratch[idx] = 0;
}
threadgroup_barrier(mem_flags::mem_threadgroup);
- if (idx >= offset && idx < E) {
- val += temp;
- scratch[idx] = val;
+
+ // Kogge-Stone scan
+ uint val = scratch[idx];
+ for (uint offset = 1; offset < 1024; offset *= 2) {
+ uint temp = 0;
+ if (idx >= offset && idx < E) {
+ temp = scratch[idx - offset];
+ }
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ if (idx >= offset && idx < E) {
+ val += temp;
+ scratch[idx] = val;
+ }
+ threadgroup_barrier(mem_flags::mem_threadgroup);
}
- threadgroup_barrier(mem_flags::mem_threadgroup);
- }
-
- // Write exclusive scan (shift right by 1)
- if (idx == 0) {
- tile_offsets[0] = 0;
- }
- if (idx < E) {
- tile_offsets[idx + 1] = scratch[idx];
- if (idx == E - 1) {
- total_tiles[0] = scratch[idx];
+
+ // Write exclusive scan (shift right by 1)
+ if (idx == 0) {
+ tile_offsets[0] = 0;
+ }
+ if (idx < E) {
+ tile_offsets[idx + 1] = scratch[idx];
+ if (idx == E - 1) {
+ total_tiles[0] = scratch[idx];
+ }
}
- }
}
// Build row→expert map: one thread per routed row
kernel void moe_pass_a_build_row_map(
- device const uint* expert_offsets [[buffer(0)]], // [E+1]
- device uint* row_expert_map [[buffer(1)]], // [total_rows]
+ device const uint* expert_offsets [[buffer(0)]] , // [E+1]
+ device uint* row_expert_map [[buffer(1)]] , // [total_rows]
constant uint& total_rows [[buffer(2)]],
constant uint& E [[buffer(3)]],
uint tid [[thread_position_in_grid]]
) {
- if (tid >= total_rows)
- return;
+ if (tid >= total_rows) return;
- uint left = 0u;
- uint right = E;
- const uint row = tid;
+ uint left = 0u;
+ uint right = E;
+ const uint row = tid;
- while (left + 1u < right) {
- const uint mid = (left + right) >> 1;
- if (row < expert_offsets[mid]) {
- right = mid;
- } else {
- left = mid;
+ while (left + 1u < right) {
+ const uint mid = (left + right) >> 1;
+ if (row < expert_offsets[mid]) {
+ right = mid;
+ } else {
+ left = mid;
+ }
}
- }
- row_expert_map[row] = left;
+ row_expert_map[row] = left;
}
// Build tile map entries from row→expert map
kernel void moe_pass_a_build_tile_map(
- device const uint* expert_offsets [[buffer(0)]], // [E+1]
- device const uint* tile_offsets [[buffer(1)]], // [E+1]
- device const uint* row_expert_map [[buffer(2)]], // [total_rows]
- device uint* tile_map [[buffer(3)]], // [total_tiles * 3]
+ device const uint* expert_offsets [[buffer(0)]], // [E+1]
+ device const uint* tile_offsets [[buffer(1)]], // [E+1]
+ device const uint* row_expert_map [[buffer(2)]], // [total_rows]
+ device uint* tile_map [[buffer(3)]], // [total_tiles * 3]
constant uint& total_rows [[buffer(4)]],
constant uint& h_blocks [[buffer(5)]],
uint tid [[thread_position_in_grid]]
) {
- const uint total_tiles = total_rows * h_blocks;
- if (tid >= total_tiles)
- return;
+ const uint total_tiles = total_rows * h_blocks;
+ if (tid >= total_tiles) return;
- const uint row_idx = tid / h_blocks;
- const uint h_block = tid % h_blocks;
+ const uint row_idx = tid / h_blocks;
+ const uint h_block = tid % h_blocks;
- if (row_idx >= total_rows)
- return;
+ if (row_idx >= total_rows) return;
- const uint expert_idx = row_expert_map[row_idx];
- const uint row_start = expert_offsets[expert_idx];
- const uint row_in_expert = row_idx - row_start;
- const uint tile_base =
- tile_offsets[expert_idx] + row_in_expert * h_blocks + h_block;
+ const uint expert_idx = row_expert_map[row_idx];
+ const uint row_start = expert_offsets[expert_idx];
+ const uint row_in_expert = row_idx - row_start;
+ const uint tile_base = tile_offsets[expert_idx] + row_in_expert * h_blocks + h_block;
- tile_map[tile_base * 3 + 0] = h_block;
- tile_map[tile_base * 3 + 1] = expert_idx;
- tile_map[tile_base * 3 + 2] = row_in_expert;
+ tile_map[tile_base * 3 + 0] = h_block;
+ tile_map[tile_base * 3 + 1] = expert_idx;
+ tile_map[tile_base * 3 + 2] = row_in_expert;
}
// Write dispatch args for indirect dispatch (reusable from tiled version)
kernel void moe_pass_a_write_dispatch_args(
- device const uint* total_tiles [[buffer(0)]], // [1]
- device uint* dispatch_args
- [[buffer(1)]], // [3] - MTLDispatchThreadgroupsIndirectArguments
- constant uint& num_tiles_y [[buffer(2)]], // usually 1 for Pass A
+ device const uint* total_tiles [[buffer(0)]], // [1]
+ device uint* dispatch_args [[buffer(1)]], // [3] - MTLDispatchThreadgroupsIndirectArguments
+ constant uint& num_tiles_y [[buffer(2)]], // usually 1 for Pass A
uint tid [[thread_position_in_grid]]
) {
- if (tid > 0)
- return;
- dispatch_args[0] = total_tiles[0]; // x dimension = total tiles
- dispatch_args[1] = num_tiles_y; // y dimension
- dispatch_args[2] = 1; // z dimension
+ if (tid > 0) return;
+ dispatch_args[0] = total_tiles[0]; // x dimension = total tiles
+ dispatch_args[1] = num_tiles_y; // y dimension
+ dispatch_args[2] = 1; // z dimension
}
// Modified Pass A that reads from tile map for indirect dispatch
-template <typename T>
+template<typename T, typename T4>
void moe_experts_decode_pass_a_indirect_impl(
device const T* X_perm,
device const uint* expert_offsets,
device const T* W13_all,
device const T* up_biases,
device float* hidden_out,
- device const uint* tile_map, // [total_tiles * 3]
+ device const uint* tile_map, // [total_tiles * 3]
uint d_model,
uint d_ff,
uint E,
@@ -478,101 +301,68 @@ void moe_experts_decode_pass_a_indirect_impl(
float up_clip_min,
float up_clip_max,
float silu_alpha,
- uint tile_idx, // flat threadgroup index
+ uint tile_idx, // flat threadgroup index
uint simd_gid,
- uint simd_lid,
- threadgroup float* tgp_memory
+ uint simd_lid
) {
- // Read tile descriptor
- const uint h_block_idx = tile_map[tile_idx * 3 + 0];
- const uint expert_idx = tile_map[tile_idx * 3 + 1];
- const uint row_in_expert = tile_map[tile_idx * 3 + 2];
-
- // Call original implementation
- moe_experts_decode_pass_a_impl<T, float>(
- X_perm,
- expert_offsets,
- W13_all,
- up_biases,
- hidden_out,
- d_model,
- d_ff,
- E,
- gate_clip_min,
- gate_clip_max,
- up_clip_min,
- up_clip_max,
- silu_alpha,
- expert_idx,
- row_in_expert,
- h_block_idx,
- simd_gid,
- simd_lid,
- tgp_memory
- );
+ // Read tile descriptor
+ const uint h_block_idx = tile_map[tile_idx * 3 + 0];
+ const uint expert_idx = tile_map[tile_idx * 3 + 1];
+ const uint row_in_expert = tile_map[tile_idx * 3 + 2];
+
+ // Call original implementation
+ moe_experts_decode_pass_a_impl<T, T4>(
+ X_perm, expert_offsets, W13_all, up_biases, hidden_out,
+ d_model, d_ff, E,
+ gate_clip_min, gate_clip_max,
+ up_clip_min, up_clip_max, silu_alpha,
+ expert_idx, row_in_expert, h_block_idx,
+ simd_gid, simd_lid
+ );
}
-#define MOE_PASS_A_INDIRECT_KERNEL(DTYPE, SUFFIX) \
- kernel void moe_experts_decode_pass_a_indirect_##SUFFIX( \
- device const DTYPE* X_perm [[buffer(0)]], \
- device const uint* expert_offsets [[buffer(1)]], \
- device const DTYPE* W13_all [[buffer(2)]], \
- device float* hidden_out [[buffer(3)]], \
- device const DTYPE* up_biases [[buffer(4)]], \
- constant uint& d_model [[buffer(5)]], \
- constant uint& d_ff [[buffer(6)]], \
- constant uint& E [[buffer(7)]], \
- constant float& gate_clip_min [[buffer(8)]], \
- constant float& gate_clip_max [[buffer(9)]], \
- constant float& up_clip_min [[buffer(10)]], \
- constant float& up_clip_max [[buffer(11)]], \
- constant float& silu_alpha [[buffer(12)]], \
- device const uint* tile_map [[buffer(13)]], \
- uint3 tgpig [[threadgroup_position_in_grid]], \
- uint simd_gid [[simdgroup_index_in_threadgroup]], \
- uint simd_lid [[thread_index_in_simdgroup]] \
- ) { \
- constexpr uint tgp_mem_size = (BN > 1) ? 2 * BN * (BLOCK_M + TM) : 1; \
- threadgroup float tgp_memory[tgp_mem_size]; \
- \
- moe_experts_decode_pass_a_indirect_impl<DTYPE>( \
- X_perm, \
- expert_offsets, \
- W13_all, \
- up_biases, \
- hidden_out, \
- tile_map, \
- d_model, \
- d_ff, \
- E, \
- gate_clip_min, \
- gate_clip_max, \
- up_clip_min, \
- up_clip_max, \
- silu_alpha, \
- tgpig.x, \
- simd_gid, \
- simd_lid, \
- (BN > 1) ? tgp_memory : nullptr \
- ); \
- }
-
-MOE_PASS_A_INDIRECT_KERNEL(bfloat, bf16)
-MOE_PASS_A_INDIRECT_KERNEL(half, f16)
-MOE_PASS_A_INDIRECT_KERNEL(float, f32)
+#define MOE_PASS_A_INDIRECT_KERNEL(DTYPE, DTYPE4, SUFFIX) \
+kernel void moe_experts_decode_pass_a_indirect_##SUFFIX( \
+ device const DTYPE* X_perm [[buffer(0)]], \
+ device const uint* expert_offsets [[buffer(1)]], \
+ device const DTYPE* W13_all [[buffer(2)]], \
+ device float* hidden_out [[buffer(3)]], \
+ device const DTYPE* up_biases [[buffer(4)]], \
+ constant uint& d_model [[buffer(5)]], \
+ constant uint& d_ff [[buffer(6)]], \
+ constant uint& E [[buffer(7)]], \
+ constant float& gate_clip_min [[buffer(8)]], \
+ constant float& gate_clip_max [[buffer(9)]], \
+ constant float& up_clip_min [[buffer(10)]], \
+ constant float& up_clip_max [[buffer(11)]], \
+ constant float& silu_alpha [[buffer(12)]], \
+ device const uint* tile_map [[buffer(13)]], \
+ uint3 tgpig [[threadgroup_position_in_grid]], \
+ uint simd_gid [[simdgroup_index_in_threadgroup]], \
+ uint simd_lid [[thread_index_in_simdgroup]]) \
+{ \
+ moe_experts_decode_pass_a_indirect_impl<DTYPE, DTYPE4>( \
+ X_perm, expert_offsets, W13_all, up_biases, hidden_out, tile_map, \
+ d_model, d_ff, E, gate_clip_min, gate_clip_max, \
+ up_clip_min, up_clip_max, silu_alpha, \
+ tgpig.x, simd_gid, simd_lid); \
+}
+
+MOE_PASS_A_INDIRECT_KERNEL(bfloat, bfloat4, bf16)
+MOE_PASS_A_INDIRECT_KERNEL(half, half4, f16)
+MOE_PASS_A_INDIRECT_KERNEL(float, float4, f32)
+
// === Pass B: Simdgroup cooperation along K for coalescing ===
-// W2 layout [E, d_model, d_ff] - 32 threads cooperate on one output, reading
-// consecutive K elements
+// W2 layout [E, d_model, d_ff] - 32 threads cooperate on one output, reading consecutive K elements
-template <typename T, typename AccumT>
+template<typename T, typename AccumT>
void moe_experts_decode_down_fused_2d_impl(
- device const float* hidden, // [total_rows, d_ff] - f32 from Pass A
- device const uint*
- row_expert_map, // [total_rows] - direct row->expert lookup
- device const T* w2_all, // [E, d_model, d_ff] - layout
- device const T* down_biases, // [E, d_model]
- device T* y_out, // [total_rows, d_model]
+ device const float* hidden, // [total_rows, d_ff] - f32 from Pass A
+ device const uint* row_expert_map, // [total_rows] - direct row->expert lookup
+ device const T* w2_all, // [E, d_model, d_ff] - layout
+ device const T* down_biases, // [E, d_model]
+ device T* y_out, // [total_rows, d_model]
uint total_rows,
uint d_model,
uint d_ff,
@@ -581,143 +371,114 @@ void moe_experts_decode_down_fused_2d_impl(
uint simd_gid,
uint simd_lid
) {
- constexpr uint THREADS_PER_SIMD = 32;
- constexpr uint SIMDGROUPS_PER_TG = 8;
-
- const uint row_idx = tgpig.y;
- if (row_idx >= total_rows)
- return;
-
- // Each simdgroup computes one output column
- const uint my_col = tgpig.x * SIMDGROUPS_PER_TG + simd_gid;
- if (my_col >= d_model)
- return;
-
- const uint expert_idx = row_expert_map[row_idx];
-
- // Base addresses for this output column
- const ulong hidden_base = (ulong)row_idx * (ulong)d_ff;
- const ulong w2_col_base = (ulong)expert_idx * (ulong)d_model * (ulong)d_ff +
- (ulong)my_col * (ulong)d_ff;
-
- // Dual accumulators for ILP: breaks FMA dependency chains
- AccumT acc0 = AccumT(0.0);
- AccumT acc1 = AccumT(0.0);
- AccumT acc = AccumT(0.0);
-
- // Main loop: stride-32 with vec8 loads for ILP
- const uint k_iters = d_ff / THREADS_PER_SIMD;
- const uint k_vec_iters = k_iters / 8;
-
- MTL_PRAGMA_UNROLL
- for (uint iter = 0; iter < k_vec_iters; ++iter) {
- const uint k_base = iter * (8 * THREADS_PER_SIMD) + simd_lid;
-
- // hidden: stride-32 per lane, already f32 from Pass A
- const AccumT h0 = hidden[hidden_base + k_base + 0 * THREADS_PER_SIMD];
- const AccumT h1 = hidden[hidden_base + k_base + 1 * THREADS_PER_SIMD];
- const AccumT h2 = hidden[hidden_base + k_base + 2 * THREADS_PER_SIMD];
- const AccumT h3 = hidden[hidden_base + k_base + 3 * THREADS_PER_SIMD];
- const AccumT h4 = hidden[hidden_base + k_base + 4 * THREADS_PER_SIMD];
- const AccumT h5 = hidden[hidden_base + k_base + 5 * THREADS_PER_SIMD];
- const AccumT h6 = hidden[hidden_base + k_base + 6 * THREADS_PER_SIMD];
- const AccumT h7 = hidden[hidden_base + k_base + 7 * THREADS_PER_SIMD];
-
- // W2: lane-coalesced
- const AccumT w0 =
- AccumT(w2_all[w2_col_base + k_base + 0 * THREADS_PER_SIMD]);
- const AccumT w1 =
- AccumT(w2_all[w2_col_base + k_base + 1 * THREADS_PER_SIMD]);
- const AccumT w2 =
- AccumT(w2_all[w2_col_base + k_base + 2 * THREADS_PER_SIMD]);
- const AccumT w3 =
- AccumT(w2_all[w2_col_base + k_base + 3 * THREADS_PER_SIMD]);
- const AccumT w4 =
- AccumT(w2_all[w2_col_base + k_base + 4 * THREADS_PER_SIMD]);
- const AccumT w5 =
- AccumT(w2_all[w2_col_base + k_base + 5 * THREADS_PER_SIMD]);
- const AccumT w6 =
- AccumT(w2_all[w2_col_base + k_base + 6 * THREADS_PER_SIMD]);
- const AccumT w7 =
- AccumT(w2_all[w2_col_base + k_base + 7 * THREADS_PER_SIMD]);
-
- // dual trees for ILP
- acc0 = fma(h0, w0, acc0);
- acc1 = fma(h1, w1, acc1);
- acc0 = fma(h2, w2, acc0);
- acc1 = fma(h3, w3, acc1);
- acc0 = fma(h4, w4, acc0);
- acc1 = fma(h5, w5, acc1);
- acc0 = fma(h6, w6, acc0);
- acc1 = fma(h7, w7, acc1);
- }
-
- // fold the vectorized contribution once
- acc += (acc0 + acc1);
-
- // Handle remaining full iterations
- MTL_PRAGMA_UNROLL
- for (uint iter = k_vec_iters * 8; iter < k_iters; ++iter) {
- const uint k = iter * THREADS_PER_SIMD + simd_lid;
- acc =
- fma(AccumT(hidden[hidden_base + k]),
- AccumT(w2_all[w2_col_base + k]),
- acc);
- }
-
- // Handle leftover elements (d_ff % 32)
- const uint leftover_start = k_iters * THREADS_PER_SIMD;
- if (leftover_start + simd_lid < d_ff) {
- const uint k = leftover_start + simd_lid;
- acc =
- fma(AccumT(hidden[hidden_base + k]),
- AccumT(w2_all[w2_col_base + k]),
- acc);
- }
-
- // Simdgroup reduction
- AccumT result = simd_sum(acc);
-
- // Lane 0 writes result
- if (simd_lid == 0) {
- const ulong bias_idx = (ulong)expert_idx * (ulong)d_model + (ulong)my_col;
- result += AccumT(down_biases[bias_idx]);
-
- const ulong out_idx = (ulong)row_idx * (ulong)d_model + (ulong)my_col;
- y_out[out_idx] = T(result);
- }
+ constexpr uint THREADS_PER_SIMD = 32;
+ constexpr uint SIMDGROUPS_PER_TG = 8;
+
+ const uint row_idx = tgpig.y;
+ if (row_idx >= total_rows) return;
+
+ // Each simdgroup computes one output column
+ const uint my_col = tgpig.x * SIMDGROUPS_PER_TG + simd_gid;
+ if (my_col >= d_model) return;
+
+ const uint expert_idx = row_expert_map[row_idx];
+
+ // Base addresses for this output column
+ const ulong hidden_base = (ulong)row_idx * (ulong)d_ff;
+ const ulong w2_col_base = (ulong)expert_idx * (ulong)d_model * (ulong)d_ff + (ulong)my_col * (ulong)d_ff;
+
+ // Dual accumulators for ILP: breaks FMA dependency chains
+ AccumT acc0 = AccumT(0.0);
+ AccumT acc1 = AccumT(0.0);
+ AccumT acc = AccumT(0.0);
+
+ // Main loop: stride-32 with vec8 loads for ILP
+ const uint k_iters = d_ff / THREADS_PER_SIMD;
+ const uint k_vec_iters = k_iters / 8;
+
+ for (uint iter = 0; iter < k_vec_iters; ++iter) {
+ const uint k_base = iter * (8 * THREADS_PER_SIMD) + simd_lid;
+
+ // hidden: stride-32 per lane, already f32 from Pass A
+ const AccumT h0 = hidden[hidden_base + k_base + 0 * THREADS_PER_SIMD];
+ const AccumT h1 = hidden[hidden_base + k_base + 1 * THREADS_PER_SIMD];
+ const AccumT h2 = hidden[hidden_base + k_base + 2 * THREADS_PER_SIMD];
+ const AccumT h3 = hidden[hidden_base + k_base + 3 * THREADS_PER_SIMD];
+ const AccumT h4 = hidden[hidden_base + k_base + 4 * THREADS_PER_SIMD];
+ const AccumT h5 = hidden[hidden_base + k_base + 5 * THREADS_PER_SIMD];
+ const AccumT h6 = hidden[hidden_base + k_base + 6 * THREADS_PER_SIMD];
+ const AccumT h7 = hidden[hidden_base + k_base + 7 * THREADS_PER_SIMD];
+
+ // W2: lane-coalesced
+ const AccumT w0 = AccumT(w2_all[w2_col_base + k_base + 0 * THREADS_PER_SIMD]);
+ const AccumT w1 = AccumT(w2_all[w2_col_base + k_base + 1 * THREADS_PER_SIMD]);
+ const AccumT w2 = AccumT(w2_all[w2_col_base + k_base + 2 * THREADS_PER_SIMD]);
+ const AccumT w3 = AccumT(w2_all[w2_col_base + k_base + 3 * THREADS_PER_SIMD]);
+ const AccumT w4 = AccumT(w2_all[w2_col_base + k_base + 4 * THREADS_PER_SIMD]);
+ const AccumT w5 = AccumT(w2_all[w2_col_base + k_base + 5 * THREADS_PER_SIMD]);
+ const AccumT w6 = AccumT(w2_all[w2_col_base + k_base + 6 * THREADS_PER_SIMD]);
+ const AccumT w7 = AccumT(w2_all[w2_col_base + k_base + 7 * THREADS_PER_SIMD]);
+
+ // dual trees for ILP
+ acc0 = fma(h0, w0, acc0);
+ acc1 = fma(h1, w1, acc1);
+ acc0 = fma(h2, w2, acc0);
+ acc1 = fma(h3, w3, acc1);
+ acc0 = fma(h4, w4, acc0);
+ acc1 = fma(h5, w5, acc1);
+ acc0 = fma(h6, w6, acc0);
+ acc1 = fma(h7, w7, acc1);
+ }
+
+ // fold the vectorized contribution once
+ acc += (acc0 + acc1);
+
+ // Handle remaining full iterations
+ for (uint iter = k_vec_iters * 8; iter < k_iters; ++iter) {
+ const uint k = iter * THREADS_PER_SIMD + simd_lid;
+ acc = fma(AccumT(hidden[hidden_base + k]), AccumT(w2_all[w2_col_base + k]), acc);
+ }
+
+ // Handle leftover elements (d_ff % 32)
+ const uint leftover_start = k_iters * THREADS_PER_SIMD;
+ if (leftover_start + simd_lid < d_ff) {
+ const uint k = leftover_start + simd_lid;
+ acc = fma(AccumT(hidden[hidden_base + k]), AccumT(w2_all[w2_col_base + k]), acc);
+ }
+
+ // Simdgroup reduction
+ AccumT result = simd_sum(acc);
+
+ // Lane 0 writes result
+ if (simd_lid == 0) {
+ const ulong bias_idx = (ulong)expert_idx * (ulong)d_model + (ulong)my_col;
+ result += AccumT(down_biases[bias_idx]);
+
+ const ulong out_idx = (ulong)row_idx * (ulong)d_model + (ulong)my_col;
+ y_out[out_idx] = T(result);
+ }
}
-#define MOE_PASS_B_FUSED_2D_KERNEL(DTYPE, SUFFIX) \
- kernel void moe_experts_decode_down_fused_2d_##SUFFIX( \
- device const float* hidden [[buffer(0)]], \
- device const uint* row_expert_map [[buffer(1)]], \
- device const DTYPE* w2_all [[buffer(2)]], \
- device const DTYPE* down_biases [[buffer(3)]], \
- device DTYPE* y_out [[buffer(4)]], \
- constant uint& total_rows [[buffer(5)]], \
- constant uint& d_model [[buffer(6)]], \
- constant uint& d_ff [[buffer(7)]], \
- constant uint& E [[buffer(8)]], \
- uint2 tgpig [[threadgroup_position_in_grid]], \
- uint simd_gid [[simdgroup_index_in_threadgroup]], \
- uint simd_lid [[thread_index_in_simdgroup]] \
- ) { \
- moe_experts_decode_down_fused_2d_impl<DTYPE, float>( \
- hidden, \
- row_expert_map, \
- w2_all, \
- down_biases, \
- y_out, \
- total_rows, \
- d_model, \
- d_ff, \
- E, \
- tgpig, \
- simd_gid, \
- simd_lid \
- ); \
- }
+#define MOE_PASS_B_FUSED_2D_KERNEL(DTYPE, SUFFIX) \
+kernel void moe_experts_decode_down_fused_2d_##SUFFIX( \
+ device const float* hidden [[buffer(0)]], \
+ device const uint* row_expert_map [[buffer(1)]], \
+ device const DTYPE* w2_all [[buffer(2)]], \
+ device const DTYPE* down_biases [[buffer(3)]], \
+ device DTYPE* y_out [[buffer(4)]], \
+ constant uint& total_rows [[buffer(5)]], \
+ constant uint& d_model [[buffer(6)]], \
+ constant uint& d_ff [[buffer(7)]], \
+ constant uint& E [[buffer(8)]], \
+ uint2 tgpig [[threadgroup_position_in_grid]], \
+ uint simd_gid [[simdgroup_index_in_threadgroup]], \
+ uint simd_lid [[thread_index_in_simdgroup]]) \
+{ \
+ moe_experts_decode_down_fused_2d_impl<DTYPE, float>( \
+ hidden, row_expert_map, w2_all, down_biases, y_out, \
+ total_rows, d_model, d_ff, E, \
+ tgpig, simd_gid, simd_lid); \
+}
MOE_PASS_B_FUSED_2D_KERNEL(bfloat, bf16)
MOE_PASS_B_FUSED_2D_KERNEL(half, f16)
diff --git a/crates/uzu/src/backends/metal/kernel/moe/experts_two_pass_prefill.metal b/crates/uzu/src/backends/metal/kernel/moe/experts_two_pass_prefill.metal
index 03aad6517..9f9a5810d 100644
--- a/crates/uzu/src/backends/metal/kernel/moe/experts_two_pass_prefill.metal
+++ b/crates/uzu/src/backends/metal/kernel/moe/experts_two_pass_prefill.metal
@@ -1,25 +1,7 @@
-// UZU MoE Two-Pass Prefill
-// - Pass A: X @ W13 → hidden (f32) with gating
-// - Pass B: hidden @ W2 → output (T)
-//
-// X_perm : [total_rows, D]
-// W13_all : [E, 2*FF, D] // first FF = up, second FF = gate
-// up_biases : [E, 2*FF] // [up | gate]
-// hidden_out (f32): [total_rows, FF]
-// W2_all : [E, D, FF] // FF contiguous (col-major-ish use below)
-// down_biases : [E, D]
-// output (T) : [total_rows, D]
-//
-// Notes:
-// - 8x8 simdgroup MMA tiles; threadgroup staging is float.
-// - Vectorized float4 cooperative loads.
-// - Linearized thread id covers both 1-D and 2/3-D TGs safely.
-// - We guard tails; K/N can be non-multiples.
-
#include <metal_stdlib>
#include <metal_simdgroup>
#include <metal_simdgroup_matrix>
-#include "../quant_matmul/mma.h"
+#include "../quant_matmul/mma.h"
using namespace metal;
@@ -27,52 +9,50 @@ using namespace metal;
static inline uint ceil_div(uint a, uint b) { return (a + b - 1u) / b; }
static inline uint linear_tid(uint3 tid, uint3 tpg) {
- return tid.z * (tpg.x * tpg.y) + tid.y * tpg.x + tid.x;
+ return tid.z * (tpg.x * tpg.y) + tid.y * tpg.x + tid.x;
}
static inline float gelu_approx(float x) {
- const float k0 = 0.7978845608f; // sqrt(2/pi)
- const float k1 = 0.044715f;
- if (x > 10.0f)
- return x;
- if (x < -10.0f)
- return 0.0f;
- float x3 = x * x * x;
- float t = clamp(k0 * (x + k1 * x3), -10.0f, 10.0f);
- return 0.5f * x * (1.0f + tanh(t));
+ const float k0 = 0.7978845608f; // sqrt(2/pi)
+ const float k1 = 0.044715f;
+ if (x > 10.0f) return x;
+ if (x < -10.0f) return 0.0f;
+ float x3 = x * x * x;
+ float t = clamp(k0 * (x + k1 * x3), -10.0f, 10.0f);
+ return 0.5f * x * (1.0f + tanh(t));
}
static inline float silu(float x, float alpha) {
- // alpha=1 gives standard SiLU; keep alpha param for parity with your API
- return x / (1.0f + exp(-alpha * x));
+ // alpha=1 gives standard SiLU; keep alpha param for parity with your API
+ return x / (1.0f + exp(-alpha * x));
}
-template <typename T>
+template<typename T>
inline void store_vec4(device T* dst, ulong base, float4 vals) {
- dst[base + 0] = T(vals.x);
- dst[base + 1] = T(vals.y);
- dst[base + 2] = T(vals.z);
- dst[base + 3] = T(vals.w);
+ dst[base + 0] = T(vals.x);
+ dst[base + 1] = T(vals.y);
+ dst[base + 2] = T(vals.z);
+ dst[base + 3] = T(vals.w);
}
-template <>
+template<>
inline void store_vec4<float>(device float* dst, ulong base, float4 vals) {
- *reinterpret_cast<device float4*>(dst + base) = vals;
+ *reinterpret_cast<device float4*>(dst + base) = vals;
}
-template <typename T>
+template<typename T>
struct StageStorage {
- using type = float;
+ using type = float;
};
-template <>
+template<>
struct StageStorage<half> {
- using type = half;
+ using type = half;
};
-template <>
+template<>
struct StageStorage<bfloat> {
- using type = bfloat;
+ using type = bfloat;
};
// 0=GELU(up), 1=SiLU(up), 2=SwiGLU(gate)*up, 3=GEGLU(gate)*up
@@ -85,917 +65,680 @@ constant uint PASSA_BN = 32;
constant uint PASSA_BK = 64;
constant uint PASSA_SG_BM = 8;
constant uint PASSA_SG_BN = 16;
+constant uint PASSA_TG_PAD = 4; // Padding to avoid bank conflicts
-template <typename T>
+template<typename T>
inline void pass_a_impl(
- device const T* X_perm, // [total_rows, D]
- device const uint* expert_offsets, // [E+1]
- device const T* W13_all, // [E, 2*FF, D]
- device const T* up_biases, // [E, 2*FF] (up | gate)
- device float* hidden_out, // [total_rows, FF]
- uint d_model,
- uint d_ff,
- uint E,
- float gate_clip_min,
- float gate_clip_max,
- float up_clip_min,
- float up_clip_max,
+ device const T* X_perm,
+ device const uint* expert_offsets,
+ device const T* W13_all,
+ device const T* up_biases,
+ device float* hidden_out,
+ uint d_model, uint d_ff, uint E,
+ float gate_clip_min, float gate_clip_max,
+ float up_clip_min, float up_clip_max,
float silu_alpha,
// work partition
- uint tile_seg_start,
- uint expert_idx,
- uint tile_m,
- uint tile_n,
+ uint tile_seg_start, uint expert_idx, uint tile_m, uint tile_n,
// per-thread info
- uint sg_id,
- uint3 threads_per_tg,
- uint3 local_tid,
+ uint sg_id, uint3 threads_per_tg, uint3 local_tid,
// TG storage
- threadgroup typename StageStorage<T>::type* Xs, // [BM,BK]
- threadgroup typename StageStorage<T>::type* Wk_up, // [BN,BK]
- threadgroup typename StageStorage<T>::type* Wk_gate, // [BN,BK]
- threadgroup float* bias_up, // [BN]
- threadgroup float* bias_gate // [BN]
+ threadgroup typename StageStorage<T>::type* Xs, // [BM,BK]
+ threadgroup typename StageStorage<T>::type* Wk_up, // [BN,BK]
+ threadgroup typename StageStorage<T>::type* Wk_gate, // [BN,BK]
+ threadgroup float* bias_up, // [BN]
+ threadgroup float* bias_gate // [BN]
) {
- using StageT = typename StageStorage<T>::type;
- using StageVec4 = metal::vec<StageT, 4>;
-
- const uint Bm = PASSA_BM, Bn = PASSA_BN, Bk = PASSA_BK;
- const uint SgBm = PASSA_SG_BM, SgBn = PASSA_SG_BN;
-
- if (expert_idx >= E)
- return;
-
- const uint seg_start = expert_offsets[expert_idx];
- const uint seg_end = expert_offsets[expert_idx + 1];
- const uint seg_len = (seg_end > seg_start) ? (seg_end - seg_start) : 0u;
- if (seg_len == 0)
- return;
-
- const uint row_tg_off = tile_m * Bm;
- const uint col_tg_off = tile_n * Bn;
- if (row_tg_off >= seg_len || col_tg_off >= d_ff)
- return;
-
- const uint m_rows = min(Bm, seg_len - row_tg_off);
- const uint n_cols = min(Bn, d_ff - col_tg_off);
-
- const ulong x_base =
- ((ulong)tile_seg_start + (ulong)row_tg_off) * (ulong)d_model;
- const ulong w13_expert_base =
- (ulong)expert_idx * (ulong)(2 * d_ff) * (ulong)d_model;
- const ulong bias_base = (ulong)expert_idx * (ulong)(2 * d_ff);
-
- // simdgroup tile mapping
- const uint sg_col_count = Bn / SgBn;
- const uint row_sg = sg_id / sg_col_count;
- const uint col_sg = sg_id % sg_col_count;
- const uint row_sg_off = row_sg * SgBm;
- const uint col_sg_off = col_sg * SgBn;
-
- // Guard against misconfigured TG sizes
- constexpr uint SG_TILE = 8; // simdgroup fragment dimension (8x8)
- constexpr uint SG_EXPECTED = (PASSA_BM / SG_TILE) * (PASSA_BN / SG_TILE);
- if (sg_id >= SG_EXPECTED)
- return;
-
- // partial accumulators (8x8 tiles per simdgroup)
- constexpr uint TEMP = (PASSA_SG_BM / SG_TILE) * (PASSA_SG_BN / SG_TILE);
- metal::simdgroup_float8x8 OutUp[TEMP];
- metal::simdgroup_float8x8 OutGate[TEMP];
- for (uint i = 0; i < TEMP; ++i) {
- OutUp[i] = metal::make_filled_simdgroup_matrix<float, 8, 8>(0.0f);
- if (GATING_SEL > 1u) {
- OutGate[i] = metal::make_filled_simdgroup_matrix<float, 8, 8>(0.0f);
- }
- }
-
- const uint threads_total =
- threads_per_tg.x * threads_per_tg.y * threads_per_tg.z;
- const uint lin = linear_tid(local_tid, threads_per_tg);
-
- // K-loop (tile along D)
- for (uint k_off = 0; k_off < d_model; k_off += Bk) {
- const uint valid_k = min(Bk, d_model - k_off);
-
- threadgroup_barrier(mem_flags::mem_threadgroup);
-
- // ---- Stage LHS: Xs [Bm,Bk] as float, vectorized 4 ----
- {
- constexpr uint row_stride = PASSA_BK;
- constexpr uint vec_cols = PASSA_BK / 4;
- const uint total_vecs = PASSA_BM * vec_cols;
- const uint iters = ceil_div(total_vecs, threads_total);
-
- UZU_PRAGMA_UNROLL
- for (uint t = 0; t < iters; ++t) {
- const uint i = t * threads_total + lin;
- if (i < total_vecs) {
- const uint r = i / vec_cols;
- const uint c4 = i % vec_cols;
-
- threadgroup StageVec4* dst4 =
- reinterpret_cast<threadgroup StageVec4*>(
- Xs + r * row_stride + (c4 << 2)
- );
-
- StageVec4 packed = StageVec4(StageT(0.0f));
- if (r < m_rows) {
- const ulong base =
- x_base + (ulong)r * (ulong)d_model + (ulong)(k_off + (c4 << 2));
- const uint base_vec = (c4 << 2);
- float v0 = 0.0f, v1 = 0.0f, v2 = 0.0f, v3 = 0.0f;
- if (base_vec + 0 < valid_k)
- v0 = float(X_perm[base + 0]);
- if (base_vec + 1 < valid_k)
- v1 = float(X_perm[base + 1]);
- if (base_vec + 2 < valid_k)
- v2 = float(X_perm[base + 2]);
- if (base_vec + 3 < valid_k)
- v3 = float(X_perm[base + 3]);
- packed = StageVec4(StageT(v0), StageT(v1), StageT(v2), StageT(v3));
- }
- *dst4 = packed;
+ using StageT = typename StageStorage<T>::type;
+ using StageVec4 = metal::vec<StageT, 4>;
+
+ constexpr uint Bm = PASSA_BM, Bn = PASSA_BN, Bk = PASSA_BK;
+ constexpr uint SgBm = PASSA_SG_BM, SgBn = PASSA_SG_BN;
+ constexpr uint TG_PAD = PASSA_TG_PAD;
+ constexpr uint X_LD = Bk + TG_PAD; // Padded stride for X
+ constexpr uint W_LD = Bk + TG_PAD; // Padded stride for weights
+
+ if (expert_idx >= E) return;
+
+ const uint seg_start = expert_offsets[expert_idx];
+ const uint seg_end = expert_offsets[expert_idx + 1];
+ const uint seg_len = (seg_end > seg_start) ? (seg_end - seg_start) : 0u;
+ if (seg_len == 0) return;
+
+ const uint row_tg_off = tile_m * Bm;
+ const uint col_tg_off = tile_n * Bn;
+ if (row_tg_off >= seg_len || col_tg_off >= d_ff) return;
+
+ const uint m_rows = min(Bm, seg_len - row_tg_off);
+ const uint n_cols = min(Bn, d_ff - col_tg_off);
+
+ const ulong x_base = ((ulong)tile_seg_start + (ulong)row_tg_off) * (ulong)d_model;
+ const ulong w13_expert_base = (ulong)expert_idx * (ulong)(2 * d_ff) * (ulong)d_model;
+ const ulong bias_base = (ulong)expert_idx * (ulong)(2 * d_ff);
+
+ // simdgroup tile mapping
+ const uint sg_col_count = Bn / SgBn;
+ const uint row_sg = sg_id / sg_col_count;
+ const uint col_sg = sg_id % sg_col_count;
+ const uint row_sg_off = row_sg * SgBm;
+ const uint col_sg_off = col_sg * SgBn;
+
+ // Guard against misconfigured TG sizes
+ constexpr uint SG_TILE = 8; // simdgroup fragment dimension (8x8)
+ constexpr uint SG_EXPECTED = (Bm / SG_TILE) * (Bn / SG_TILE);
+ if (sg_id >= SG_EXPECTED) return;
+
+ // partial accumulators (8x8 tiles per simdgroup)
+ constexpr uint TEMP = (SgBm / SG_TILE) * (SgBn / SG_TILE);
+ metal::simdgroup_float8x8 OutUp[TEMP];
+ metal::simdgroup_float8x8 OutGate[TEMP];
+ for (uint i = 0; i < TEMP; ++i) {
+ OutUp[i] = metal::make_filled_simdgroup_matrix<float, 8, 8>(0.0f);
+ if (GATING_SEL > 1u) {
+ OutGate[i] = metal::make_filled_simdgroup_matrix<float, 8, 8>(0.0f);
}
- }
}
- // ---- Stage RHS: W_up [Bn,Bk], W_gate [Bn,Bk], vectorized 4 ----
- {
- constexpr uint row_stride = PASSA_BK;
- constexpr uint vec_cols = PASSA_BK / 4;
- const uint total_vecs = PASSA_BN * vec_cols;
- const uint iters = ceil_div(total_vecs, threads_total);
-
- UZU_PRAGMA_UNROLL
- for (uint t = 0; t < iters; ++t) {
- const uint i = t * threads_total + lin;
- if (i < total_vecs) {
- const uint n = i / vec_cols; // column within the BN tile
- const uint c4 = i % vec_cols;
-
- threadgroup StageVec4* up4 = reinterpret_cast<threadgroup StageVec4*>(
- Wk_up + n * row_stride + (c4 << 2)
- );
- StageVec4 upv = StageVec4(StageT(0.0f));
-
- threadgroup StageVec4* gt4 = reinterpret_cast<threadgroup StageVec4*>(
- Wk_gate + n * row_stride + (c4 << 2)
- );
- StageVec4 gtv = StageVec4(StageT(0.0f));
-
- if (n < n_cols) {
- const uint gcol = col_tg_off + n;
- const ulong up_base = w13_expert_base +
- (ulong)gcol * (ulong)d_model +
- (ulong)(k_off + (c4 << 2));
- const ulong gate_base = w13_expert_base +
- (ulong)(d_ff + gcol) * (ulong)d_model +
- (ulong)(k_off + (c4 << 2));
- const uint base_vec = (c4 << 2);
- float up0 = 0.0f, up1 = 0.0f, up2 = 0.0f, up3 = 0.0f;
- float gt0 = 0.0f, gt1 = 0.0f, gt2 = 0.0f, gt3 = 0.0f;
- if (base_vec + 0 < valid_k) {
- up0 = float(W13_all[up_base + 0]);
- gt0 = float(W13_all[gate_base + 0]);
+ const uint threads_total = threads_per_tg.x * threads_per_tg.y * threads_per_tg.z;
+ const uint lin = linear_tid(local_tid, threads_per_tg);
+
+ // K-loop (tile along D)
+ for (uint k_off = 0; k_off < d_model; k_off += Bk) {
+ const uint valid_k = min(Bk, d_model - k_off);
+
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+
+ // ---- Stage LHS: Xs [Bm,Bk] with vec8 loads and padding ----
+ {
+ constexpr uint vec_size = 8;
+ constexpr uint vec_cols = Bk / vec_size; // 64/8 = 8
+ constexpr uint total_vecs = Bm * vec_cols; // 16*8 = 128
+ const uint iters = ceil_div(total_vecs, threads_total);
+
+UZU_PRAGMA_UNROLL
+ for (uint t = 0; t < iters; ++t) {
+ const uint i = t * threads_total + lin;
+ if (i < total_vecs) {
+ const uint r = i / vec_cols;
+ const uint c8 = i % vec_cols;
+ const uint col_base = c8 * vec_size;
+
+ threadgroup StageT* dst = Xs + r * X_LD + col_base;
+
+ if (r < m_rows && col_base < valid_k) {
+ // Safe to read - at least first element is valid
+ const ulong base = x_base + (ulong)r * (ulong)d_model + (ulong)(k_off + col_base);
+UZU_PRAGMA_UNROLL
+ for (uint j = 0; j < vec_size; j++) {
+ dst[j] = (col_base + j < valid_k) ? StageT(float(X_perm[base + j])) : StageT(0.0f);
+ }
+ } else {
+ // Row out of bounds or entire vec8 chunk is out of K bounds
+UZU_PRAGMA_UNROLL
+ for (uint j = 0; j < vec_size; j++) {
+ dst[j] = StageT(0.0f);
+ }
+ }
+ }
}
- if (base_vec + 1 < valid_k) {
- up1 = float(W13_all[up_base + 1]);
- gt1 = float(W13_all[gate_base + 1]);
- }
- if (base_vec + 2 < valid_k) {
- up2 = float(W13_all[up_base + 2]);
- gt2 = float(W13_all[gate_base + 2]);
- }
- if (base_vec + 3 < valid_k) {
- up3 = float(W13_all[up_base + 3]);
- gt3 = float(W13_all[gate_base + 3]);
+ }
+
+ // ---- Stage RHS: W_up [Bn,Bk], W_gate [Bn,Bk] with vec8 loads and padding ----
+ {
+ constexpr uint vec_size = 8;
+ constexpr uint vec_cols = Bk / vec_size; // 64/8 = 8
+ constexpr uint total_vecs = Bn * vec_cols; // 32*8 = 256
+ const uint iters = ceil_div(total_vecs, threads_total);
+
+UZU_PRAGMA_UNROLL
+ for (uint t = 0; t < iters; ++t) {
+ const uint i = t * threads_total + lin;
+ if (i < total_vecs) {
+ const uint n = i / vec_cols; // row within the BN tile
+ const uint c8 = i % vec_cols;
+ const uint col_base = c8 * vec_size;
+
+ threadgroup StageT* up_dst = Wk_up + n * W_LD + col_base;
+ threadgroup StageT* gt_dst = Wk_gate + n * W_LD + col_base;
+
+ if (n < n_cols && col_base < valid_k) {
+ // Safe to read - at least first element is valid
+ const uint gcol = col_tg_off + n;
+ const ulong up_base = w13_expert_base + (ulong)gcol * (ulong)d_model + (ulong)(k_off + col_base);
+ const ulong gate_base = w13_expert_base + (ulong)(d_ff + gcol) * (ulong)d_model + (ulong)(k_off + col_base);
+UZU_PRAGMA_UNROLL
+ for (uint j = 0; j < vec_size; j++) {
+ up_dst[j] = (col_base + j < valid_k) ? StageT(float(W13_all[up_base + j])) : StageT(0.0f);
+ if (GATING_SEL > 1u) {
+ gt_dst[j] = (col_base + j < valid_k) ? StageT(float(W13_all[gate_base + j])) : StageT(0.0f);
+ }
+ }
+ } else {
+ // Row out of bounds or entire vec8 chunk is out of K bounds
+UZU_PRAGMA_UNROLL
+ for (uint j = 0; j < vec_size; j++) {
+ up_dst[j] = StageT(0.0f);
+ if (GATING_SEL > 1u) gt_dst[j] = StageT(0.0f);
+ }
+ }
+ }
}
- upv = StageVec4(StageT(up0), StageT(up1), StageT(up2), StageT(up3));
- gtv = StageVec4(StageT(gt0), StageT(gt1), StageT(gt2), StageT(gt3));
- }
+ }
- *up4 = upv;
- if (GATING_SEL > 1u)
- *gt4 = gtv;
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+
+ // ---- MMA across the BK tile (in simdgroup-sized chunks) ----
+UZU_PRAGMA_UNROLL
+ for (uint kk = 0; kk < Bk; kk += SG_TILE) {
+UZU_PRAGMA_UNROLL
+ for (uint m_sub = 0; m_sub < SgBm; m_sub += SG_TILE) {
+ const uint r_idx = m_sub / SG_TILE;
+ metal::simdgroup_matrix<StageT, 8, 8> lhs_frag;
+ simdgroup_load(lhs_frag, Xs, X_LD, ulong2(kk, row_sg_off + m_sub));
+
+UZU_PRAGMA_UNROLL
+ for (uint n_sub = 0; n_sub < SgBn; n_sub += SG_TILE) {
+ const uint c_idx = n_sub / SG_TILE;
+ const uint tile = r_idx * (SgBn / SG_TILE) + c_idx;
+ metal::simdgroup_matrix<StageT, 8, 8> rhs_up;
+ simdgroup_load(rhs_up, Wk_up, W_LD, ulong2(kk, col_sg_off + n_sub), true);
+ simdgroup_multiply_accumulate(OutUp[tile], lhs_frag, rhs_up, OutUp[tile]);
+
+ if (GATING_SEL > 1u) {
+ metal::simdgroup_matrix<StageT, 8, 8> rhs_gate;
+ simdgroup_load(rhs_gate, Wk_gate, W_LD, ulong2(kk, col_sg_off + n_sub), true);
+ simdgroup_multiply_accumulate(OutGate[tile], lhs_frag, rhs_gate, OutGate[tile]);
+ }
+ }
+ }
+ }
+ } // K loop
+
+ // ---- Bias tile [Bn] ----
+ // Note: bias layout is [up | gate] in contiguous FF chunks.
+ for (uint c_local = lin; c_local < Bn; c_local += threads_total) {
+ const uint c_global = col_tg_off + c_local;
+ bias_up[c_local] = (c_global < d_ff) ? float(up_biases[bias_base + c_global]) : 0.0f;
+ if (GATING_SEL > 1u) {
+ bias_gate[c_local] = (c_global < d_ff) ? float(up_biases[bias_base + d_ff + c_global]) : 0.0f;
}
- }
}
threadgroup_barrier(mem_flags::mem_threadgroup);
- // ---- MMA across the BK tile (in simdgroup-sized chunks) ----
- UZU_PRAGMA_UNROLL
- for (uint kk = 0; kk < PASSA_BK; kk += SG_TILE) {
- UZU_PRAGMA_UNROLL
- for (uint m_sub = 0; m_sub < PASSA_SG_BM; m_sub += SG_TILE) {
- const uint r_idx = m_sub / SG_TILE;
- metal::simdgroup_matrix<StageT, 8, 8> lhs_frag;
- simdgroup_load(lhs_frag, Xs, PASSA_BK, ulong2(kk, row_sg_off + m_sub));
-
- UZU_PRAGMA_UNROLL
- for (uint n_sub = 0; n_sub < PASSA_SG_BN; n_sub += SG_TILE) {
- const uint c_idx = n_sub / SG_TILE;
- const uint tile = r_idx * (PASSA_SG_BN / SG_TILE) + c_idx;
- metal::simdgroup_matrix<StageT, 8, 8> rhs_up;
- simdgroup_load(
- rhs_up,
- Wk_up,
- PASSA_BK,
- ulong2(kk, col_sg_off + n_sub),
- true
- );
- simdgroup_multiply_accumulate(
- OutUp[tile],
- lhs_frag,
- rhs_up,
- OutUp[tile]
- );
-
- if (GATING_SEL > 1u) {
- metal::simdgroup_matrix<StageT, 8, 8> rhs_gate;
- simdgroup_load(
- rhs_gate,
- Wk_gate,
- PASSA_BK,
- ulong2(kk, col_sg_off + n_sub),
- true
- );
- simdgroup_multiply_accumulate(
- OutGate[tile],
- lhs_frag,
- rhs_gate,
- OutGate[tile]
- );
- }
- }
- }
- }
- } // K loop
-
- // ---- Bias tile [Bn] ----
- // Note: bias layout is [up | gate] in contiguous FF chunks.
- for (uint c_local = lin; c_local < Bn; c_local += threads_total) {
- const uint c_global = col_tg_off + c_local;
- bias_up[c_local] =
- (c_global < d_ff) ? float(up_biases[bias_base + c_global]) : 0.0f;
- if (GATING_SEL > 1u) {
- bias_gate[c_local] = (c_global < d_ff)
- ? float(up_biases[bias_base + d_ff + c_global])
- : 0.0f;
- }
- }
-
- threadgroup_barrier(mem_flags::mem_threadgroup);
-
- // ---- Epilogue: activation/gating → hidden_out (f32) ----
- const uint sg_lane_start = sg_id * 32;
- const uint sg_lane_end = sg_lane_start + 32;
- if (lin < sg_lane_start || lin >= sg_lane_end)
- return;
- const uint sg_lin = lin - sg_lane_start;
- // Lane→(row,col) mapping for 8x8 simdgroup fragments, matches the fragment
- // layout used above.
- const uint lane_qid = sg_lin >> 2;
- const uint lane_row = (lane_qid & 4u) + ((sg_lin >> 1) & 3u);
- const uint lane_col_base = ((lane_qid & 2u) << 1) + ((sg_lin & 1u) << 1);
-
- UZU_PRAGMA_UNROLL
- for (uint n_sub = 0; n_sub < PASSA_SG_BN; n_sub += SG_TILE) {
- const uint c_idx = n_sub / SG_TILE;
- UZU_PRAGMA_UNROLL
- for (uint m_sub = 0; m_sub < PASSA_SG_BM; m_sub += SG_TILE) {
- const uint r_idx = m_sub / SG_TILE;
- const uint tile = r_idx * (PASSA_SG_BN / SG_TILE) + c_idx;
-
- const uint tile_row_base = row_sg_off + m_sub;
- const uint tile_col_base = col_sg_off + n_sub;
- const uint local_row = tile_row_base + lane_row;
- if (local_row >= m_rows || tile_col_base >= n_cols) {
- continue;
- }
-
- const auto up_frag = OutUp[tile].thread_elements();
- float gate_frag_0 = 0.0f;
- float gate_frag_1 = 0.0f;
- if (GATING_SEL > 1u) {
- const auto gate_frag = OutGate[tile].thread_elements();
- gate_frag_0 = gate_frag[0];
- gate_frag_1 = gate_frag[1];
- }
-
- const uint col0 = tile_col_base + lane_col_base;
- const uint col1 = col0 + 1;
- const ulong out_row = seg_start + row_tg_off + local_row;
-
- if (col0 < n_cols) {
- float up_v =
- clamp(up_frag[0] + bias_up[col0], up_clip_min, up_clip_max);
- float out_val;
- if (GATING_SEL <= 1u) {
- out_val =
- (GATING_SEL == 0u) ? gelu_approx(up_v) : silu(up_v, silu_alpha);
- } else {
- float gate_v = clamp(
- gate_frag_0 + bias_gate[col0],
- gate_clip_min,
- gate_clip_max
- );
- const float gate_act = (GATING_SEL == 2u) ? silu(gate_v, silu_alpha)
- : gelu_approx(gate_v);
- out_val = gate_act * up_v;
- }
- const ulong out_col = col_tg_off + col0;
- hidden_out[out_row * (ulong)d_ff + out_col] = out_val;
- }
-
- if (col1 < n_cols) {
- float up_v =
- clamp(up_frag[1] + bias_up[col1], up_clip_min, up_clip_max);
- float out_val;
- if (GATING_SEL <= 1u) {
- out_val =
- (GATING_SEL == 0u) ? gelu_approx(up_v) : silu(up_v, silu_alpha);
- } else {
- float gate_v = clamp(
- gate_frag_1 + bias_gate[col1],
- gate_clip_min,
- gate_clip_max
- );
- const float gate_act = (GATING_SEL == 2u) ? silu(gate_v, silu_alpha)
- : gelu_approx(gate_v);
- out_val = gate_act * up_v;
+ // ---- Epilogue: activation/gating → hidden_out (f32) ----
+ const uint sg_lane_start = sg_id * 32;
+ const uint sg_lane_end = sg_lane_start + 32;
+ if (lin < sg_lane_start || lin >= sg_lane_end) return;
+ const uint sg_lin = lin - sg_lane_start;
+ // Lane→(row,col) mapping for 8x8 simdgroup fragments, matches the fragment layout used above.
+ const uint lane_qid = sg_lin >> 2;
+ const uint lane_row = (lane_qid & 4u) + ((sg_lin >> 1) & 3u);
+ const uint lane_col_base = ((lane_qid & 2u) << 1) + ((sg_lin & 1u) << 1);
+
+UZU_PRAGMA_UNROLL
+ for (uint n_sub = 0; n_sub < SgBn; n_sub += SG_TILE) {
+ const uint c_idx = n_sub / SG_TILE;
+UZU_PRAGMA_UNROLL
+ for (uint m_sub = 0; m_sub < SgBm; m_sub += SG_TILE) {
+ const uint r_idx = m_sub / SG_TILE;
+ const uint tile = r_idx * (SgBn / SG_TILE) + c_idx;
+
+ const uint tile_row_base = row_sg_off + m_sub;
+ const uint tile_col_base = col_sg_off + n_sub;
+ const uint local_row = tile_row_base + lane_row;
+ if (local_row >= m_rows || tile_col_base >= n_cols) {
+ continue;
+ }
+
+ const auto up_frag = OutUp[tile].thread_elements();
+ float gate_frag_0 = 0.0f;
+ float gate_frag_1 = 0.0f;
+ if (GATING_SEL > 1u) {
+ const auto gate_frag = OutGate[tile].thread_elements();
+ gate_frag_0 = gate_frag[0];
+ gate_frag_1 = gate_frag[1];
+ }
+
+ const uint col0 = tile_col_base + lane_col_base;
+ const uint col1 = col0 + 1;
+ const ulong out_row = seg_start + row_tg_off + local_row;
+
+ if (col0 < n_cols) {
+ float up_v = clamp(up_frag[0] + bias_up[col0], up_clip_min, up_clip_max);
+ float out_val;
+ if (GATING_SEL <= 1u) {
+ out_val =
+ (GATING_SEL == 0u) ? gelu_approx(up_v) : silu(up_v, silu_alpha);
+ } else {
+ float gate_v =
+ clamp(gate_frag_0 + bias_gate[col0], gate_clip_min, gate_clip_max);
+ const float gate_act =
+ (GATING_SEL == 2u) ? silu(gate_v, silu_alpha) : gelu_approx(gate_v);
+ out_val = gate_act * up_v;
+ }
+ const ulong out_col = col_tg_off + col0;
+ hidden_out[out_row * (ulong)d_ff + out_col] = out_val;
+ }
+
+ if (col1 < n_cols) {
+ float up_v = clamp(up_frag[1] + bias_up[col1], up_clip_min, up_clip_max);
+ float out_val;
+ if (GATING_SEL <= 1u) {
+ out_val =
+ (GATING_SEL == 0u) ? gelu_approx(up_v) : silu(up_v, silu_alpha);
+ } else {
+ float gate_v =
+ clamp(gate_frag_1 + bias_gate[col1], gate_clip_min, gate_clip_max);
+ const float gate_act =
+ (GATING_SEL == 2u) ? silu(gate_v, silu_alpha) : gelu_approx(gate_v);
+ out_val = gate_act * up_v;
+ }
+ const ulong out_col = col_tg_off + col1;
+ hidden_out[out_row * (ulong)d_ff + out_col] = out_val;
+ }
}
- const ulong out_col = col_tg_off + col1;
- hidden_out[out_row * (ulong)d_ff + out_col] = out_val;
- }
}
- }
}
// ----- Kernel entry points (Pass A) -----
-#define MOE_PASS_A_KERNEL(DTYPE, SUFFIX) \
- kernel void moe_two_pass_prefill_pass_a_##SUFFIX( \
- device const DTYPE* X_perm [[buffer(0)]], \
- device const uint* expert_offsets [[buffer(1)]], \
- device const DTYPE* W13_all [[buffer(2)]], \
- device const DTYPE* up_biases [[buffer(3)]], \
- device float* hidden_out [[buffer(4)]], \
- constant uint& d_model [[buffer(5)]], \
- constant uint& d_ff [[buffer(6)]], \
- constant uint& E [[buffer(7)]], \
- constant float& gate_clip_min [[buffer(8)]], \
- constant float& gate_clip_max [[buffer(9)]], \
- constant float& up_clip_min [[buffer(10)]], \
- constant float& up_clip_max [[buffer(11)]], \
- constant float& silu_alpha [[buffer(12)]], \
- uint sg_id [[simdgroup_index_in_threadgroup]], \
- uint3 threads_per_tg [[threads_per_threadgroup]], \
- uint3 tg_pos [[threadgroup_position_in_grid]], \
- uint3 local_tid [[thread_position_in_threadgroup]] \
- ) { \
- using StageT = typename StageStorage<DTYPE>::type; \
- threadgroup StageT Xs[PASSA_BM * PASSA_BK]; \
- threadgroup StageT Wk_up[PASSA_BN * PASSA_BK]; \
- threadgroup StageT Wk_gate[PASSA_BN * PASSA_BK]; \
- threadgroup float bias_up[PASSA_BN]; \
- threadgroup float bias_gate[PASSA_BN]; \
- pass_a_impl<DTYPE>( \
- X_perm, \
- expert_offsets, \
- W13_all, \
- up_biases, \
- hidden_out, \
- d_model, \
- d_ff, \
- E, \
- gate_clip_min, \
- gate_clip_max, \
- up_clip_min, \
- up_clip_max, \
- silu_alpha, \
- /*tile_seg_start*/ expert_offsets[tg_pos.z], \
- /*expert*/ tg_pos.z, \
- /*tile_m*/ tg_pos.y, \
- /*tile_n*/ tg_pos.x, \
- sg_id, \
- threads_per_tg, \
- local_tid, \
- Xs, \
- Wk_up, \
- Wk_gate, \
- bias_up, \
- bias_gate \
- ); \
- }
+#define MOE_PASS_A_KERNEL(DTYPE, SUFFIX) \
+kernel void moe_two_pass_prefill_pass_a_##SUFFIX( \
+ device const DTYPE* X_perm [[buffer(0)]], \
+ device const uint* expert_offsets [[buffer(1)]], \
+ device const DTYPE* W13_all [[buffer(2)]], \
+ device const DTYPE* up_biases [[buffer(3)]], \
+ device float* hidden_out [[buffer(4)]], \
+ constant uint& d_model [[buffer(5)]], \
+ constant uint& d_ff [[buffer(6)]], \
+ constant uint& E [[buffer(7)]], \
+ constant float& gate_clip_min [[buffer(8)]], \
+ constant float& gate_clip_max [[buffer(9)]], \
+ constant float& up_clip_min [[buffer(10)]], \
+ constant float& up_clip_max [[buffer(11)]], \
+ constant float& silu_alpha [[buffer(12)]], \
+ uint sg_id [[simdgroup_index_in_threadgroup]], \
+ uint3 threads_per_tg [[threads_per_threadgroup]], \
+ uint3 tg_pos [[threadgroup_position_in_grid]], \
+ uint3 local_tid [[thread_position_in_threadgroup]]) \
+{ \
+ using StageT = typename StageStorage<DTYPE>::type; \
+ threadgroup StageT Xs[PASSA_BM * (PASSA_BK + PASSA_TG_PAD)]; \
+ threadgroup StageT Wk_up[PASSA_BN * (PASSA_BK + PASSA_TG_PAD)]; \
+ threadgroup StageT Wk_gate[PASSA_BN * (PASSA_BK + PASSA_TG_PAD)]; \
+ threadgroup float bias_up[PASSA_BN]; \
+ threadgroup float bias_gate[PASSA_BN]; \
+ pass_a_impl<DTYPE>( \
+ X_perm, expert_offsets, W13_all, up_biases, hidden_out, \
+ d_model, d_ff, E, \
+ gate_clip_min, gate_clip_max, up_clip_min, up_clip_max, silu_alpha, \
+ expert_offsets[tg_pos.z], tg_pos.z, tg_pos.y, tg_pos.x, \
+ sg_id, threads_per_tg, local_tid, \
+ Xs, Wk_up, Wk_gate, bias_up, bias_gate); \
+}
MOE_PASS_A_KERNEL(bfloat, bf16)
MOE_PASS_A_KERNEL(half, f16)
MOE_PASS_A_KERNEL(float, f32)
// Indirect variant (consumes [expert, seg_start, tile_row_offset] triples)
-template <typename T>
+template<typename T>
inline void pass_a_indirect_impl(
device const T* X_perm,
device const uint* expert_offsets,
device const T* W13_all,
device const T* up_biases,
device float* hidden_out,
- device const uint*
- tile_map, // [tiles * 3]: (expert, seg_start, row_offset_elems)
- uint d_model,
- uint d_ff,
- uint E,
- float gate_clip_min,
- float gate_clip_max,
- float up_clip_min,
- float up_clip_max,
+ device const uint* tile_map,
+ uint d_model, uint d_ff, uint E,
+ float gate_clip_min, float gate_clip_max,
+ float up_clip_min, float up_clip_max,
float silu_alpha,
- uint row_tile_idx,
- uint n_tile_idx,
- uint sg_id,
- uint3 threads_per_tg,
- uint3 local_tid,
+ uint row_tile_idx, uint n_tile_idx,
+ uint sg_id, uint3 threads_per_tg, uint3 local_tid,
threadgroup typename StageStorage<T>::type* Xs,
threadgroup typename StageStorage<T>::type* Wk_up,
threadgroup typename StageStorage<T>::type* Wk_gate,
threadgroup float* bias_up,
- threadgroup float* bias_gate
-) {
- constexpr uint ROW_TILE = PASSA_BM;
- const uint base = row_tile_idx * 3u;
- const uint expert_idx = tile_map[base + 0u];
- if (expert_idx >= E)
- return;
- const uint tile_seg_start = tile_map[base + 1u];
- const uint row_off_elems = tile_map[base + 2u];
- const uint tile_m = row_off_elems / ROW_TILE;
-
- pass_a_impl<T>(
- X_perm,
- expert_offsets,
- W13_all,
- up_biases,
- hidden_out,
- d_model,
- d_ff,
- E,
- gate_clip_min,
- gate_clip_max,
- up_clip_min,
- up_clip_max,
- silu_alpha,
- tile_seg_start,
- expert_idx,
- tile_m,
- n_tile_idx,
- sg_id,
- threads_per_tg,
- local_tid,
- Xs,
- Wk_up,
- Wk_gate,
- bias_up,
- bias_gate
- );
+ threadgroup float* bias_gate)
+{
+ constexpr uint ROW_TILE = PASSA_BM;
+ const uint base = row_tile_idx * 3u;
+ const uint expert_idx = tile_map[base + 0u];
+ if (expert_idx >= E) return;
+ const uint tile_seg_start = tile_map[base + 1u];
+ const uint row_off_elems = tile_map[base + 2u];
+ const uint tile_m = row_off_elems / ROW_TILE;
+
+ pass_a_impl<T>(
+ X_perm, expert_offsets, W13_all, up_biases, hidden_out,
+ d_model, d_ff, E,
+ gate_clip_min, gate_clip_max, up_clip_min, up_clip_max, silu_alpha,
+ tile_seg_start, expert_idx, tile_m, n_tile_idx,
+ sg_id, threads_per_tg, local_tid,
+ Xs, Wk_up, Wk_gate, bias_up, bias_gate);
}
-#define MOE_PASS_A_INDIRECT_KERNEL(DTYPE, SUFFIX) \
- kernel void moe_two_pass_prefill_pass_a_indirect_##SUFFIX( \
- device const DTYPE* X_perm [[buffer(0)]], \
- device const uint* expert_offsets [[buffer(1)]], \
- device const DTYPE* W13_all [[buffer(2)]], \
- device const DTYPE* up_biases [[buffer(3)]], \
- device float* hidden_out [[buffer(4)]], \
- constant uint& d_model [[buffer(5)]], \
- constant uint& d_ff [[buffer(6)]], \
- constant uint& E [[buffer(7)]], \
- constant float& gate_clip_min [[buffer(8)]], \
- constant float& gate_clip_max [[buffer(9)]], \
- constant float& up_clip_min [[buffer(10)]], \
- constant float& up_clip_max [[buffer(11)]], \
- constant float& silu_alpha [[buffer(12)]], \
- device const uint* tile_map [[buffer(13)]], \
- uint sg_id [[simdgroup_index_in_threadgroup]], \
- uint3 threads_per_tg [[threads_per_threadgroup]], \
- uint3 tg_pos [[threadgroup_position_in_grid]], \
- uint3 local_tid [[thread_position_in_threadgroup]] \
- ) { \
- using StageT = typename StageStorage<DTYPE>::type; \
- threadgroup StageT Xs[PASSA_BM * PASSA_BK]; \
- threadgroup StageT Wk_up[PASSA_BN * PASSA_BK]; \
- threadgroup StageT Wk_gate[PASSA_BN * PASSA_BK]; \
- threadgroup float bias_up[PASSA_BN]; \
- threadgroup float bias_gate[PASSA_BN]; \
- pass_a_indirect_impl<DTYPE>( \
- X_perm, \
- expert_offsets, \
- W13_all, \
- up_biases, \
- hidden_out, \
- tile_map, \
- d_model, \
- d_ff, \
- E, \
- gate_clip_min, \
- gate_clip_max, \
- up_clip_min, \
- up_clip_max, \
- silu_alpha, \
- /*row_tile_idx*/ tg_pos.y, \
- /*n_tile_idx*/ tg_pos.x, \
- sg_id, \
- threads_per_tg, \
- local_tid, \
- Xs, \
- Wk_up, \
- Wk_gate, \
- bias_up, \
- bias_gate \
- ); \
- }
+#define MOE_PASS_A_INDIRECT_KERNEL(DTYPE, SUFFIX) \
+kernel void moe_two_pass_prefill_pass_a_indirect_##SUFFIX( \
+ device const DTYPE* X_perm [[buffer(0)]], \
+ device const uint* expert_offsets [[buffer(1)]], \
+ device const DTYPE* W13_all [[buffer(2)]], \
+ device const DTYPE* up_biases [[buffer(3)]], \
+ device float* hidden_out [[buffer(4)]], \
+ constant uint& d_model [[buffer(5)]], \
+ constant uint& d_ff [[buffer(6)]], \
+ constant uint& E [[buffer(7)]], \
+ constant float& gate_clip_min [[buffer(8)]], \
+ constant float& gate_clip_max [[buffer(9)]], \
+ constant float& up_clip_min [[buffer(10)]], \
+ constant float& up_clip_max [[buffer(11)]], \
+ constant float& silu_alpha [[buffer(12)]], \
+ device const uint* tile_map [[buffer(13)]], \
+ uint sg_id [[simdgroup_index_in_threadgroup]], \
+ uint3 threads_per_tg [[threads_per_threadgroup]], \
+ uint3 tg_pos [[threadgroup_position_in_grid]], \
+ uint3 local_tid [[thread_position_in_threadgroup]]) \
+{ \
+ using StageT = typename StageStorage<DTYPE>::type; \
+ threadgroup StageT Xs[PASSA_BM * (PASSA_BK + PASSA_TG_PAD)]; \
+ threadgroup StageT Wk_up[PASSA_BN * (PASSA_BK + PASSA_TG_PAD)]; \
+ threadgroup StageT Wk_gate[PASSA_BN * (PASSA_BK + PASSA_TG_PAD)]; \
+ threadgroup float bias_up[PASSA_BN]; \
+ threadgroup float bias_gate[PASSA_BN]; \
+ pass_a_indirect_impl<DTYPE>( \
+ X_perm, expert_offsets, W13_all, up_biases, hidden_out, tile_map, \
+ d_model, d_ff, E, \
+ gate_clip_min, gate_clip_max, up_clip_min, up_clip_max, silu_alpha, \
+ tg_pos.y, tg_pos.x, \
+ sg_id, threads_per_tg, local_tid, \
+ Xs, Wk_up, Wk_gate, bias_up, bias_gate); \
+}
MOE_PASS_A_INDIRECT_KERNEL(bfloat, bf16)
MOE_PASS_A_INDIRECT_KERNEL(half, f16)
MOE_PASS_A_INDIRECT_KERNEL(float, f32)
-// ------------------------ Pass B (hidden @ W2 → output)
-// ------------------------
+// ------------------------ Pass B (hidden @ W2 → output) ------------------------
+// Optimized config: 16x64 output tile, 4 simdgroups (128 threads)
constant uint PASSB_BM = 16;
constant uint PASSB_BN = 64;
constant uint PASSB_BK = 64;
-constant uint PASSB_SG_BM = 8;
+constant uint PASSB_SG_BM = 8; // Each simdgroup handles 8x32
constant uint PASSB_SG_BN = 32;
+constant uint PASSB_TG_PAD = 4; // Padding to avoid bank conflicts
-template <typename T>
+template<typename T>
inline void pass_b_impl(
- device const float* hidden, // [total_rows, FF]
- device const uint* expert_offsets, // [E+1]
- device const T* W2_all, // [E, D, FF] (FF contiguous)
- device const T* down_biases, // [E, D]
- device T* output, // [total_rows, D]
- uint d_model,
- uint d_ff,
- uint E,
- uint expert_idx,
- uint tile_m,
- uint tile_n,
- uint sg_id,
- uint3 threads_per_tg,
- uint3 local_tid,
- threadgroup float* Hs, // [BM,BK]
- threadgroup typename StageStorage<T>::type* Wk, // [BN,BK]
- threadgroup float* bias_tile // [BN]
+ device const float* hidden,
+ device const uint* expert_offsets,
+ device const T* W2_all,
+ device const T* down_biases,
+ device T* output,
+ uint d_model, uint d_ff, uint E,
+ uint expert_idx, uint tile_m, uint tile_n,
+ uint sg_id, uint simd_lid, uint lin,
+ threadgroup float* Hs,
+ threadgroup typename StageStorage<T>::type* Wk,
+ threadgroup float* bias_tile
) {
- using StageT = typename StageStorage<T>::type;
- using StageVec4 = metal::vec<StageT, 4>;
-
- const uint Bm = PASSB_BM, Bn = PASSB_BN, Bk = PASSB_BK;
- const uint SgBm = PASSB_SG_BM, SgBn = PASSB_SG_BN;
-
- if (expert_idx >= E)
- return;
-
- const uint seg_start = expert_offsets[expert_idx];
- const uint seg_end = expert_offsets[expert_idx + 1];
- const uint seg_len = (seg_end > seg_start) ? (seg_end - seg_start) : 0u;
- if (seg_len == 0)
- return;
-
- const uint row_tg_off = tile_m * Bm;
- const uint col_tg_off = tile_n * Bn;
- if (row_tg_off >= seg_len || col_tg_off >= d_model)
- return;
-
- const uint m_rows = min(Bm, seg_len - row_tg_off);
- const uint n_cols = min(Bn, d_model - col_tg_off);
-
- const ulong h_base = (ulong)(seg_start + row_tg_off) * (ulong)d_ff;
- const ulong w2_expert_base = (ulong)expert_idx * (ulong)d_model * (ulong)d_ff;
- const ulong bias_base = (ulong)expert_idx * (ulong)d_model;
-
- const uint sg_col_count = Bn / SgBn;
- const uint row_sg = sg_id / sg_col_count;
- const uint col_sg = sg_id % sg_col_count;
- const uint row_sg_off = row_sg * SgBm;
- const uint col_sg_off = col_sg * SgBn;
-
- constexpr uint SG_TILE = 8;
- constexpr uint TEMP = (PASSB_SG_BM / SG_TILE) * (PASSB_SG_BN / SG_TILE);
- metal::simdgroup_float8x8 Out[TEMP];
- for (uint i = 0; i < TEMP; ++i) {
- Out[i] = metal::make_filled_simdgroup_matrix<float, 8, 8>(0.0f);
- }
-
- const uint threads_total =
- threads_per_tg.x * threads_per_tg.y * threads_per_tg.z;
- const uint lin = linear_tid(local_tid, threads_per_tg);
-
- // K-loop across FF
- for (uint k_off = 0; k_off < d_ff; k_off += Bk) {
- const uint valid_k = min(Bk, d_ff - k_off);
+ using StageT = typename StageStorage<T>::type;
+
+ constexpr uint Bm = PASSB_BM, Bn = PASSB_BN, Bk = PASSB_BK;
+ constexpr uint SgBm = PASSB_SG_BM, SgBn = PASSB_SG_BN;
+ constexpr uint TG_PAD = PASSB_TG_PAD;
+ constexpr uint H_LD = Bk + TG_PAD; // Padded stride for hidden
+ constexpr uint W_LD = Bk + TG_PAD; // Padded stride for weights
+ constexpr uint SG_TILE = 8;
+ constexpr uint TGP_SIZE = 128; // 4 simdgroups × 32 threads
+
+ if (expert_idx >= E) return;
+
+ const uint seg_start = expert_offsets[expert_idx];
+ const uint seg_end = expert_offsets[expert_idx + 1];
+ const uint seg_len = (seg_end > seg_start) ? (seg_end - seg_start) : 0u;
+ if (seg_len == 0) return;
+
+ const uint row_tg_off = tile_m * Bm;
+ const uint col_tg_off = tile_n * Bn;
+ if (row_tg_off >= seg_len || col_tg_off >= d_model) return;
+
+ const uint m_rows = min(Bm, seg_len - row_tg_off);
+ const uint n_cols = min(Bn, d_model - col_tg_off);
+
+ const ulong h_base = (ulong)(seg_start + row_tg_off) * (ulong)d_ff;
+ const ulong w2_expert_base = (ulong)expert_idx * (ulong)d_model * (ulong)d_ff;
+ const ulong bias_base = (ulong)expert_idx * (ulong)d_model;
+
+ // 2×2 simdgroup layout for 16×64 output (each sg handles 8×32)
+ const uint row_sg = sg_id / 2; // 0-1 → rows
+ const uint col_sg = sg_id % 2; // 0-1 → cols
+ const uint row_sg_off = row_sg * SgBm;
+ const uint col_sg_off = col_sg * SgBn;
+
+ // Guard against excessive simdgroups
+ constexpr uint SG_EXPECTED = (PASSB_BM / SgBm) * (PASSB_BN / SgBn);
+ if (sg_id >= SG_EXPECTED) return;
+
+ // 4 accumulators per simdgroup: 1×4 layout of 8×8 tiles
+ constexpr uint TEMP = (SgBm / SG_TILE) * (SgBn / SG_TILE); // 1×4 = 4
+ metal::simdgroup_float8x8 Out[TEMP];
+UZU_PRAGMA_UNROLL
+ for (uint i = 0; i < TEMP; ++i) {
+ Out[i] = metal::make_filled_simdgroup_matrix<float, 8, 8>(0.0f);
+ }
- threadgroup_barrier(mem_flags::mem_threadgroup);
+ // Cooperative load config - vec8 loads for better bandwidth
+ constexpr uint H_VEC = 8;
+ constexpr uint H_TCOLS = Bk / H_VEC; // 64/8 = 8
+ constexpr uint H_TROWS = TGP_SIZE / H_TCOLS; // 128/8 = 16
+ const uint h_bi = lin / H_TCOLS;
+ const uint h_bj = (lin % H_TCOLS) * H_VEC;
+
+ constexpr uint W_VEC = 8;
+ constexpr uint W_TCOLS = Bk / W_VEC; // 64/8 = 8
+ constexpr uint W_TROWS = TGP_SIZE / W_TCOLS; // 128/8 = 16
+ const uint w_bi = lin / W_TCOLS;
+ const uint w_bj = (lin % W_TCOLS) * W_VEC;
+
+ // K-loop across FF
+ for (uint k_off = 0; k_off < d_ff; k_off += Bk) {
+ const uint valid_k = min(Bk, d_ff - k_off);
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+
+ // ---- Stage hidden [Bm=16, Bk=64] with vec8 loads ----
+ // 128 threads, 8 vec8 loads per row → 16 rows covered in one pass
+ {
+ threadgroup float* my_dst = Hs + h_bi * H_LD + h_bj;
+ if (h_bi < m_rows && h_bj < valid_k) {
+ // Safe to read - at least first element is valid
+ device const float* my_src = hidden + h_base + (ulong)h_bi * (ulong)d_ff + (ulong)(k_off + h_bj);
+UZU_PRAGMA_UNROLL
+ for (uint j = 0; j < H_VEC; j++) {
+ my_dst[j] = (h_bj + j < valid_k) ? my_src[j] : 0.0f;
+ }
+ } else {
+UZU_PRAGMA_UNROLL
+ for (uint j = 0; j < H_VEC; j++) {
+ my_dst[j] = 0.0f;
+ }
+ }
+ }
- // ---- Stage hidden [Bm,Bk] as float4 ----
- {
- constexpr uint row_stride = PASSB_BK;
- constexpr uint vec_cols = PASSB_BK / 4;
- const uint total_vecs = PASSB_BM * vec_cols;
- const uint iters = ceil_div(total_vecs, threads_total);
-
- UZU_PRAGMA_UNROLL
- for (uint t = 0; t < iters; ++t) {
- const uint i = t * threads_total + lin;
- if (i < total_vecs) {
- const uint r = i / vec_cols;
- const uint c4 = i % vec_cols;
-
- threadgroup float4* dst4 = reinterpret_cast<threadgroup float4*>(
- Hs + r * row_stride + (c4 << 2)
- );
-
- float4 v = float4(0.0f);
- if (r < m_rows) {
- const uint base_vec = (c4 << 2);
- const ulong base =
- h_base + (ulong)r * (ulong)d_ff + (ulong)(k_off + base_vec);
- if (base_vec + 0u < valid_k)
- v.x = hidden[base + 0];
- if (base_vec + 1u < valid_k)
- v.y = hidden[base + 1];
- if (base_vec + 2u < valid_k)
- v.z = hidden[base + 2];
- if (base_vec + 3u < valid_k)
- v.w = hidden[base + 3];
- }
- *dst4 = v;
+ // ---- Stage W2 [Bn=64, Bk=64] with vec8 loads ----
+ // Need 4 passes: 128 threads cover 16 rows, Bn=64 needs 4×16
+ {
+ for (uint pass = 0; pass < 4; pass++) {
+ const uint row = w_bi + pass * W_TROWS;
+ threadgroup StageT* my_dst = Wk + row * W_LD + w_bj;
+ if (row < n_cols && w_bj < valid_k) {
+ // Safe to read - at least first element is valid
+ const uint gcol = col_tg_off + row;
+ device const T* my_src = W2_all + w2_expert_base + (ulong)gcol * (ulong)d_ff + (ulong)(k_off + w_bj);
+UZU_PRAGMA_UNROLL
+ for (uint j = 0; j < W_VEC; j++) {
+ my_dst[j] = (w_bj + j < valid_k) ? StageT(float(my_src[j])) : StageT(0.0f);
+ }
+ } else {
+UZU_PRAGMA_UNROLL
+ for (uint j = 0; j < W_VEC; j++) {
+ my_dst[j] = StageT(0.0f);
+ }
+ }
+ }
}
- }
- }
- // ---- Stage W2 [Bn,Bk] as StageT (column-major for simd transpose) ----
- {
- constexpr uint row_stride = PASSB_BK;
- constexpr uint vec_cols = PASSB_BK / 4;
- const uint total_vecs = PASSB_BN * vec_cols;
- const uint iters = ceil_div(total_vecs, threads_total);
-
- UZU_PRAGMA_UNROLL
- for (uint t = 0; t < iters; ++t) {
- const uint i = t * threads_total + lin;
- if (i < total_vecs) {
- const uint n = i / vec_cols;
- const uint c4 = i % vec_cols;
-
- threadgroup StageVec4* dst4 =
- reinterpret_cast<threadgroup StageVec4*>(
- Wk + n * row_stride + (c4 << 2)
- );
-
- StageVec4 packed = StageVec4(StageT(0.0f));
- if (n < n_cols) {
- const uint gcol = col_tg_off + n;
- const ulong base = w2_expert_base + (ulong)gcol * (ulong)d_ff +
- (ulong)(k_off + (c4 << 2));
- const uint base_vec = (c4 << 2);
- float v0 = 0.0f, v1 = 0.0f, v2 = 0.0f, v3 = 0.0f;
- if (base_vec + 0u < valid_k)
- v0 = float(W2_all[base + 0]);
- if (base_vec + 1u < valid_k)
- v1 = float(W2_all[base + 1]);
- if (base_vec + 2u < valid_k)
- v2 = float(W2_all[base + 2]);
- if (base_vec + 3u < valid_k)
- v3 = float(W2_all[base + 3]);
- packed = StageVec4(StageT(v0), StageT(v1), StageT(v2), StageT(v3));
- }
- *dst4 = packed;
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+
+ // ---- Serpentine MMA: 1×4 tiles per simdgroup ----
+UZU_PRAGMA_UNROLL
+ for (uint kk = 0; kk < Bk; kk += SG_TILE) {
+ // Load 1 LHS fragment
+ metal::simdgroup_float8x8 lhs;
+ simdgroup_load(lhs, Hs, H_LD, ulong2(kk, row_sg_off));
+
+ // Load 4 RHS fragments (cols)
+ metal::simdgroup_matrix<StageT, 8, 8> rhs0, rhs1, rhs2, rhs3;
+ simdgroup_load(rhs0, Wk, W_LD, ulong2(kk, col_sg_off), true);
+ simdgroup_load(rhs1, Wk, W_LD, ulong2(kk, col_sg_off + 8), true);
+ simdgroup_load(rhs2, Wk, W_LD, ulong2(kk, col_sg_off + 16), true);
+ simdgroup_load(rhs3, Wk, W_LD, ulong2(kk, col_sg_off + 24), true);
+
+ // Serpentine pattern: 0→1→2→3 (forward for first row)
+ simdgroup_multiply_accumulate(Out[0], lhs, rhs0, Out[0]);
+ simdgroup_multiply_accumulate(Out[1], lhs, rhs1, Out[1]);
+ simdgroup_multiply_accumulate(Out[2], lhs, rhs2, Out[2]);
+ simdgroup_multiply_accumulate(Out[3], lhs, rhs3, Out[3]);
}
- }
+ }
+
+ // ---- Bias tile (cooperative load) ----
+ for (uint c_local = lin; c_local < Bn; c_local += TGP_SIZE) {
+ const uint c_global = col_tg_off + c_local;
+ bias_tile[c_local] = (c_global < d_model) ? float(down_biases[bias_base + c_global]) : 0.0f;
}
threadgroup_barrier(mem_flags::mem_threadgroup);
- // ---- MMA ----
- UZU_PRAGMA_UNROLL
- for (uint kk = 0; kk < PASSB_BK; kk += SG_TILE) {
- UZU_PRAGMA_UNROLL
- for (uint m_sub = 0; m_sub < PASSB_SG_BM; m_sub += SG_TILE) {
- const uint r_idx = m_sub / SG_TILE;
- metal::simdgroup_float8x8 lhs;
- simdgroup_load(lhs, Hs, PASSB_BK, ulong2(kk, row_sg_off + m_sub));
- UZU_PRAGMA_UNROLL
- for (uint n_sub = 0; n_sub < PASSB_SG_BN; n_sub += SG_TILE) {
- const uint c_idx = n_sub / SG_TILE;
- const uint tile = r_idx * (PASSB_SG_BN / SG_TILE) + c_idx;
- metal::simdgroup_matrix<StageT, 8, 8> rhs;
- simdgroup_load(
- rhs,
- Wk,
- PASSB_BK,
- ulong2(kk, col_sg_off + n_sub),
- true
- );
- simdgroup_multiply_accumulate(Out[tile], lhs, rhs, Out[tile]);
+ // ---- Writeout with fragment extraction ----
+ const uint lane_qid = simd_lid >> 2;
+ const uint lane_row = (lane_qid & 4u) + ((simd_lid >> 1) & 3u);
+ const uint lane_col_base = ((lane_qid & 2u) << 1) + ((simd_lid & 1u) << 1);
+
+ const uint local_row = row_sg_off + lane_row;
+ if (local_row >= m_rows) return;
+ const ulong out_row = seg_start + row_tg_off + local_row;
+
+UZU_PRAGMA_UNROLL
+ for (uint ni = 0; ni < 4; ni++) { // 4 col tiles per simdgroup
+ const auto accum = Out[ni].thread_elements();
+ const uint local_col = col_sg_off + ni * 8 + lane_col_base;
+
+ if (local_col < n_cols) {
+ const ulong out_col = col_tg_off + local_col;
+ const float val = accum[0] + bias_tile[local_col];
+ output[out_row * (ulong)d_model + out_col] = T(val);
+ }
+ if (local_col + 1 < n_cols) {
+ const ulong out_col = col_tg_off + local_col + 1;
+ const float val = accum[1] + bias_tile[local_col + 1];
+ output[out_row * (ulong)d_model + out_col] = T(val);
}
- }
- }
- }
-
- // ---- Bias tile ----
- for (uint c_local = lin; c_local < Bn; c_local += threads_total) {
- const uint c_global = col_tg_off + c_local;
- bias_tile[c_local] =
- (c_global < d_model) ? float(down_biases[bias_base + c_global]) : 0.0f;
- }
-
- threadgroup_barrier(mem_flags::mem_threadgroup);
-
- // ---- Writeout ----
- const uint sg_lane_start = sg_id * 32;
- const uint sg_lane_end = sg_lane_start + 32;
- if (lin < sg_lane_start || lin >= sg_lane_end)
- return;
- const uint sg_lin = lin - sg_lane_start;
-
- // Lane→(row,col) mapping for 8x8 simdgroup fragments, matches PASSB fragment
- // packing.
- const uint lane_qid = sg_lin >> 2;
- const uint lane_row = (lane_qid & 4u) + ((sg_lin >> 1) & 3u);
- const uint lane_col_base = ((lane_qid & 2u) << 1) + ((sg_lin & 1u) << 1);
-
- UZU_PRAGMA_UNROLL
- for (uint n_sub = 0; n_sub < PASSB_SG_BN; n_sub += SG_TILE) {
- const uint c_idx = n_sub / SG_TILE;
- UZU_PRAGMA_UNROLL
- for (uint m_sub = 0; m_sub < PASSB_SG_BM; m_sub += SG_TILE) {
- const uint r_idx = m_sub / SG_TILE;
- const uint tile = r_idx * (PASSB_SG_BN / SG_TILE) + c_idx;
-
- const uint tile_row_base = row_sg_off + m_sub;
- const uint tile_col_base = col_sg_off + n_sub;
- const uint local_row = tile_row_base + lane_row;
- if (local_row >= m_rows || tile_col_base >= n_cols) {
- continue;
- }
-
- const auto accum = Out[tile].thread_elements();
-
- const uint col0 = tile_col_base + lane_col_base;
- const uint col1 = col0 + 1u;
- const ulong out_row = seg_start + row_tg_off + local_row;
-
- if (col0 < n_cols) {
- const ulong out_col = col_tg_off + col0;
- const float val = accum[0] + bias_tile[col0];
- output[out_row * (ulong)d_model + out_col] = T(val);
- }
-
- if (col1 < n_cols) {
- const ulong out_col = col_tg_off + col1;
- const float val = accum[1] + bias_tile[col1];
- output[out_row * (ulong)d_model + out_col] = T(val);
- }
}
- }
}
// ----- Kernel entry points (Pass B) -----
-#define MOE_PASS_B_KERNEL(DTYPE, SUFFIX) \
- kernel void moe_two_pass_prefill_pass_b_##SUFFIX( \
- device const float* hidden [[buffer(0)]], \
- device const uint* expert_offsets [[buffer(1)]], \
- device const DTYPE* W2_all [[buffer(2)]], \
- device const DTYPE* down_biases [[buffer(3)]], \
- device DTYPE* output [[buffer(4)]], \
- constant uint& d_model [[buffer(5)]], \
- constant uint& d_ff [[buffer(6)]], \
- constant uint& E [[buffer(7)]], \
- uint sg_id [[simdgroup_index_in_threadgroup]], \
- uint3 threads_per_tg [[threads_per_threadgroup]], \
- uint3 tg_pos [[threadgroup_position_in_grid]], \
- uint3 local_tid [[thread_position_in_threadgroup]] \
- ) { \
- using StageT = typename StageStorage<DTYPE>::type; \
- threadgroup float Hs[PASSB_BM * PASSB_BK]; \
- threadgroup StageT Wk[PASSB_BN * PASSB_BK]; \
- threadgroup float bias[PASSB_BN]; \
- pass_b_impl<DTYPE>( \
- hidden, \
- expert_offsets, \
- W2_all, \
- down_biases, \
- output, \
- d_model, \
- d_ff, \
- E, \
- /*expert*/ tg_pos.z, \
- /*tile_m*/ tg_pos.y, \
- /*tile_n*/ tg_pos.x, \
- sg_id, \
- threads_per_tg, \
- local_tid, \
- Hs, \
- Wk, \
- bias \
- ); \
- }
+#define MOE_PASS_B_KERNEL(DTYPE, SUFFIX) \
+kernel void moe_two_pass_prefill_pass_b_##SUFFIX( \
+ device const float* hidden [[buffer(0)]], \
+ device const uint* expert_offsets [[buffer(1)]], \
+ device const DTYPE* W2_all [[buffer(2)]], \
+ device const DTYPE* down_biases [[buffer(3)]], \
+ device DTYPE* output [[buffer(4)]], \
+ constant uint& d_model [[buffer(5)]], \
+ constant uint& d_ff [[buffer(6)]], \
+ constant uint& E [[buffer(7)]], \
+ uint sg_id [[simdgroup_index_in_threadgroup]], \
+ uint simd_lid [[thread_index_in_simdgroup]], \
+ uint3 tg_pos [[threadgroup_position_in_grid]], \
+ uint3 local_tid [[thread_position_in_threadgroup]]) \
+{ \
+ using StageT = typename StageStorage<DTYPE>::type; \
+ threadgroup float Hs[PASSB_BM * (PASSB_BK + PASSB_TG_PAD)]; \
+ threadgroup StageT Wk[PASSB_BN * (PASSB_BK + PASSB_TG_PAD)]; \
+ threadgroup float bias[PASSB_BN]; \
+ const uint lin = local_tid.x; \
+ pass_b_impl<DTYPE>( \
+ hidden, expert_offsets, W2_all, down_biases, output, \
+ d_model, d_ff, E, \
+ tg_pos.z, tg_pos.y, tg_pos.x, \
+ sg_id, simd_lid, lin, \
+ Hs, Wk, bias); \
+}
MOE_PASS_B_KERNEL(bfloat, bf16)
MOE_PASS_B_KERNEL(half, f16)
MOE_PASS_B_KERNEL(float, f32)
// Indirect variant
-template <typename T>
+template<typename T>
inline void pass_b_indirect_impl(
device const float* hidden,
device const uint* expert_offsets,
device const T* W2_all,
device const T* down_biases,
device T* output,
- device const uint* tile_map, // [tiles * 3]
- uint d_model,
- uint d_ff,
- uint E,
- uint row_tile_idx,
- uint n_tile_idx,
- uint sg_id,
- uint3 threads_per_tg,
- uint3 local_tid,
+ device const uint* tile_map,
+ uint d_model, uint d_ff, uint E,
+ uint row_tile_idx, uint n_tile_idx,
+ uint sg_id, uint simd_lid, uint lin,
threadgroup float* Hs,
threadgroup typename StageStorage<T>::type* Wk,
- threadgroup float* bias_tile
-) {
- constexpr uint ROW_TILE = PASSB_BM;
- const uint base = row_tile_idx * 3u;
- const uint expert_idx = tile_map[base + 0u];
- if (expert_idx >= E)
- return;
- const uint row_off_elems = tile_map[base + 2u];
- const uint tile_m = row_off_elems / ROW_TILE;
-
- pass_b_impl<T>(
- hidden,
- expert_offsets,
- W2_all,
- down_biases,
- output,
- d_model,
- d_ff,
- E,
- expert_idx,
- tile_m,
- n_tile_idx,
- sg_id,
- threads_per_tg,
- local_tid,
- Hs,
- Wk,
- bias_tile
- );
+ threadgroup float* bias_tile)
+{
+ constexpr uint ROW_TILE = PASSB_BM;
+ const uint base = row_tile_idx * 3u;
+ const uint expert_idx = tile_map[base + 0u];
+ if (expert_idx >= E) return;
+ const uint row_off_elems = tile_map[base + 2u];
+ const uint tile_m = row_off_elems / ROW_TILE;
+
+ pass_b_impl<T>(
+ hidden, expert_offsets, W2_all, down_biases, output,
+ d_model, d_ff, E,
+ expert_idx, tile_m, n_tile_idx,
+ sg_id, simd_lid, lin,
+ Hs, Wk, bias_tile);
}
-#define MOE_PASS_B_INDIRECT_KERNEL(DTYPE, SUFFIX) \
- kernel void moe_two_pass_prefill_pass_b_indirect_##SUFFIX( \
- device const float* hidden [[buffer(0)]], \
- device const uint* expert_offsets [[buffer(1)]], \
- device const DTYPE* W2_all [[buffer(2)]], \
- device const DTYPE* down_biases [[buffer(3)]], \
- device DTYPE* output [[buffer(4)]], \
- constant uint& d_model [[buffer(5)]], \
- constant uint& d_ff [[buffer(6)]], \
- constant uint& E [[buffer(7)]], \
- device const uint* tile_map [[buffer(8)]], \
- uint sg_id [[simdgroup_index_in_threadgroup]], \
- uint3 threads_per_tg [[threads_per_threadgroup]], \
- uint3 tg_pos [[threadgroup_position_in_grid]], \
- uint3 local_tid [[thread_position_in_threadgroup]] \
- ) { \
- using StageT = typename StageStorage<DTYPE>::type; \
- threadgroup float Hs[PASSB_BM * PASSB_BK]; \
- threadgroup StageT Wk[PASSB_BN * PASSB_BK]; \
- threadgroup float bias[PASSB_BN]; \
- pass_b_indirect_impl<DTYPE>( \
- hidden, \
- expert_offsets, \
- W2_all, \
- down_biases, \
- output, \
- tile_map, \
- d_model, \
- d_ff, \
- E, \
- /*row_tile_idx*/ tg_pos.y, \
- /*n_tile_idx*/ tg_pos.x, \
- sg_id, \
- threads_per_tg, \
- local_tid, \
- Hs, \
- Wk, \
- bias \
- ); \
- }
+#define MOE_PASS_B_INDIRECT_KERNEL(DTYPE, SUFFIX) \
+kernel void moe_two_pass_prefill_pass_b_indirect_##SUFFIX( \
+ device const float* hidden [[buffer(0)]], \
+ device const uint* expert_offsets [[buffer(1)]], \
+ device const DTYPE* W2_all [[buffer(2)]], \
+ device const DTYPE* down_biases [[buffer(3)]], \
+ device DTYPE* output [[buffer(4)]], \
+ constant uint& d_model [[buffer(5)]], \
+ constant uint& d_ff [[buffer(6)]], \
+ constant uint& E [[buffer(7)]], \
+ device const uint* tile_map [[buffer(8)]], \
+ uint sg_id [[simdgroup_index_in_threadgroup]], \
+ uint simd_lid [[thread_index_in_simdgroup]], \
+ uint3 tg_pos [[threadgroup_position_in_grid]], \
+ uint3 local_tid [[thread_position_in_threadgroup]]) \
+{ \
+ using StageT = typename StageStorage<DTYPE>::type; \
+ threadgroup float Hs[PASSB_BM * (PASSB_BK + PASSB_TG_PAD)]; \
+ threadgroup StageT Wk[PASSB_BN * (PASSB_BK + PASSB_TG_PAD)]; \
+ threadgroup float bias[PASSB_BN]; \
+ const uint lin = local_tid.x; \
+ pass_b_indirect_impl<DTYPE>( \
+ hidden, expert_offsets, W2_all, down_biases, output, tile_map, \
+ d_model, d_ff, E, \
+ tg_pos.y, tg_pos.x, \
+ sg_id, simd_lid, lin, \
+ Hs, Wk, bias); \
+}
MOE_PASS_B_INDIRECT_KERNEL(bfloat, bf16)
MOE_PASS_B_INDIRECT_KERNEL(half, f16)
diff --git a/crates/uzu/src/backends/metal/kernel/moe/mod.rs b/crates/uzu/src/backends/metal/kernel/moe/mod.rs
index 1fb3fd783..9dc1853ba 100644
--- a/crates/uzu/src/backends/metal/kernel/moe/mod.rs
+++ b/crates/uzu/src/backends/metal/kernel/moe/mod.rs
@@ -18,7 +18,8 @@ pub use counts_offsets_fused::{
MoeCountsOffsetsFusedKernel,
};
pub use experts::{
- MoeExpertsArguments, MoeExpertsError, MoeExpertsTwoPassArguments,
+ MoeExpertsArguments, MoeExpertsError, MoeExpertsSingleDecodeArguments,
+ MoeExpertsSingleDecodeKernel, MoeExpertsTwoPassArguments,
MoeExpertsTwoPassDecodeKernel, MoeExpertsTwoPassPrefillKernel,
MoeScatterError,
};
diff --git a/crates/uzu/tests/moe/moe_experts_perf_test.rs b/crates/uzu/tests/moe/moe_experts_perf_test.rs
index 8f705b296..03a639388 100644
--- a/crates/uzu/tests/moe/moe_experts_perf_test.rs
+++ b/crates/uzu/tests/moe/moe_experts_perf_test.rs
@@ -5,6 +5,7 @@ use rand::{Rng, SeedableRng, rngs::StdRng};
use uzu::backends::metal::{
KernelDataType, MTLContext,
kernel::moe::{
+ MoeExpertsSingleDecodeArguments, MoeExpertsSingleDecodeKernel,
MoeExpertsTwoPassArguments, MoeExpertsTwoPassDecodeKernel,
MoeExpertsTwoPassPrefillKernel,
},
@@ -346,6 +347,7 @@ fn test_two_pass_prefill_speed() {
let cases = vec![
("T32_E16_D1024_F4096", 32, 1024, 4096, 16, 4, 2, 10),
("T64_E16_D2048_F6144", 64, 2048, 6144, 16, 4, 2, 10),
+ ("T256_E16_D2048_F6144", 256, 2048, 6144, 16, 4, 2, 10),
];
for (name, t, d_model, d_ff, e, k, warmup, iters) in cases {
@@ -354,3 +356,378 @@ fn test_two_pass_prefill_speed() {
);
}
}
+
+fn run_fused_single_token_case(
+ ctx: &MTLContext,
+ name: &str,
+ d_model: usize,
+ d_ff: usize,
+ e: usize,
+ k: usize,
+ warmup: usize,
+ iters: usize,
+) {
+ let mut rng = StdRng::seed_from_u64(0xF053ED);
+
+ eprintln!(
+ "\n[fused single-token] {} => D={}, FF={}, E={}, K={}",
+ name, d_model, d_ff, e, k
+ );
+
+ let x: Vec<bf16> = (0..d_model)
+ .map(|_| bf16::from_f32(rng.random_range(-1.0..1.0)))
+ .collect();
+
+ let topk_ids: Vec<i32> = (0..k).map(|i| (i % e) as i32).collect();
+
+ let topk_probs: Vec<bf16> = {
+ let raw: Vec<f32> =
+ (0..k).map(|_| rng.random_range(0.1..1.0)).collect();
+ let sum: f32 = raw.iter().sum();
+ raw.iter().map(|p| bf16::from_f32(p / sum)).collect()
+ };
+
+ let w13_all: Vec<bf16> = (0..e * 2 * d_ff * d_model)
+ .map(|_| bf16::from_f32(rng.random_range(-0.05..0.05)))
+ .collect();
+ let w2_all: Vec<bf16> = (0..e * d_ff * d_model)
+ .map(|_| bf16::from_f32(rng.random_range(-0.05..0.05)))
+ .collect();
+ let up_biases: Vec<bf16> = (0..e * 2 * d_ff)
+ .map(|_| bf16::from_f32(rng.random_range(-0.01..0.01)))
+ .collect();
+ let down_biases: Vec<bf16> = (0..e * d_model)
+ .map(|_| bf16::from_f32(rng.random_range(-0.01..0.01)))
+ .collect();
+
+ let fused_kernel =
+ MoeExpertsSingleDecodeKernel::new(ctx).expect("fused kernel");
+
+ let x_buf = alloc_buffer_with_data(ctx, &x);
+ let topk_ids_buf = alloc_buffer_with_data(ctx, &topk_ids);
+ let topk_probs_buf = alloc_buffer_with_data(ctx, &topk_probs);
+ let w13_buf = alloc_buffer_with_data(ctx, &w13_all);
+ let w2_buf = alloc_buffer_with_data(ctx, &w2_all);
+ let up_biases_buf = alloc_buffer_with_data(ctx, &up_biases);
+ let down_biases_buf = alloc_buffer_with_data(ctx, &down_biases);
+ let hidden_buf = alloc_buffer::<f32>(ctx, k * d_ff);
+ let y_buf = alloc_buffer::<bf16>(ctx, d_model);
+
+ let make_args = || MoeExpertsSingleDecodeArguments {
+ x: &x_buf,
+ topk_ids: &topk_ids_buf,
+ topk_probs: &topk_probs_buf,
+ w13_all: &w13_buf,
+ w2_all: &w2_buf,
+ up_biases: &up_biases_buf,
+ down_biases: &down_biases_buf,
+ hidden: &hidden_buf,
+ y: &y_buf,
+ d_model,
+ d_ff,
+ k,
+ gating_code: 2, // SwiGLU
+ silu_alpha: 1.0,
+ gate_clip_min: f32::NEG_INFINITY,
+ gate_clip_max: f32::INFINITY,
+ up_clip_min: f32::NEG_INFINITY,
+ up_clip_max: f32::INFINITY,
+ data_type: KernelDataType::BFloat16,
+ };
+
+ for _ in 0..warmup {
+ let cb = ctx.command_queue.new_command_buffer();
+ fused_kernel.encode(&cb, make_args()).expect("encode");
+ cb.commit();
+ cb.wait_until_completed();
+ }
+
+ let mut times = Vec::with_capacity(iters);
+ for _ in 0..iters {
+ let start = Instant::now();
+ let cb = ctx.command_queue.new_command_buffer();
+ fused_kernel.encode(&cb, make_args()).expect("encode");
+ cb.commit();
+ cb.wait_until_completed();
+ times.push(start.elapsed().as_secs_f64() * 1000.0);
+ }
+
+ times.sort_by(|a, b| a.partial_cmp(b).unwrap());
+ let mean = times.iter().sum::<f64>() / times.len() as f64;
+ let median = times[times.len() / 2];
+ let min = times[0];
+ let max = times[times.len() - 1];
+ let var = times.iter().map(|t| (t - mean).powi(2)).sum::<f64>()
+ / times.len() as f64;
+ let std = var.sqrt();
+
+ eprintln!(
+ " mean={:.3}ms median={:.3}ms min={:.3}ms max={:.3}ms std={:.3}ms",
+ mean, median, min, max, std
+ );
+ eprintln!(" → Latency: {:.1} µs/token", mean * 1000.0);
+}
+
+#[test]
+#[ignore]
+fn test_fused_single_token_speed() {
+ let ctx = create_ctx();
+
+ let cases = vec![
+ ("E16_D1024_F4096_K4", 1024, 4096, 16, 4, 3, 20),
+ ("E8_D2048_F8192_K4", 2048, 8192, 8, 4, 3, 20),
+ ("E8_D512_F2048_K2", 512, 2048, 8, 2, 3, 20),
+ ];
+
+ for (name, d_model, d_ff, e, k, warmup, iters) in cases {
+ run_fused_single_token_case(
+ &ctx, name, d_model, d_ff, e, k, warmup, iters,
+ );
+ }
+}
+
+/// Compare indirect decode vs fused decode for single-token (T=1)
+#[test]
+#[ignore]
+fn test_single_token_indirect_vs_fused() {
+ let ctx = create_ctx();
+
+ let cases = vec![
+ ("E8_D512_F2048_K2", 512, 2048, 8, 2),
+ ("E16_D1024_F4096_K4", 1024, 4096, 16, 4),
+ ("E8_D2048_F8192_K4", 2048, 8192, 8, 4),
+ ];
+
+ eprintln!("\n=== Single-Token Decode: Indirect vs Fused ===\n");
+
+ for (name, d_model, d_ff, e, k) in cases {
+ let warmup = 3;
+ let iters = 20;
+
+ eprintln!("[{}] D={}, FF={}, E={}, K={}", name, d_model, d_ff, e, k);
+
+ // Run indirect decode (T=1)
+ let indirect_mean = run_indirect_decode_timed(
+ &ctx, 1, d_model, d_ff, e, k, warmup, iters,
+ );
+
+ // Run fused decode
+ let fused_mean =
+ run_fused_decode_timed(&ctx, d_model, d_ff, e, k, warmup, iters);
+
+ let speedup = indirect_mean / fused_mean;
+ eprintln!(
+ " Indirect: {:.3}ms, Fused: {:.3}ms, Speedup: {:.2}x\n",
+ indirect_mean, fused_mean, speedup
+ );
+ }
+}
+
+fn run_indirect_decode_timed(
+ ctx: &MTLContext,
+ t: usize,
+ d_model: usize,
+ d_ff: usize,
+ e: usize,
+ k: usize,
+ warmup: usize,
+ iters: usize,
+) -> f64 {
+ const K_TILE: usize = 64;
+
+ let mut rng = StdRng::seed_from_u64(0xDEC0DE1234567890);
+ let sum_k = t * k;
+
+ let x_perm: Vec<bf16> = (0..sum_k * d_model)
+ .map(|_| bf16::from_f32(rng.random_range(-1.0..1.0)))
+ .collect();
+ let offsets = build_offsets(e, sum_k);
+
+ let w13_original: Vec<bf16> = (0..e * d_model * 2 * d_ff)
+ .map(|_| bf16::from_f32(rng.random_range(-0.05..0.05)))
+ .collect();
+
+ let mut w13 = vec![bf16::from_f32(0.0); e * d_model * 2 * d_ff];
+ for expert in 0..e {
+ let src_offset = expert * d_model * 2 * d_ff;
+ let dst_offset = expert * 2 * d_ff * d_model;
+ for dm in 0..d_model {
+ for ff in 0..(2 * d_ff) {
+ let src_idx = src_offset + dm * 2 * d_ff + ff;
+ let dst_idx = dst_offset + ff * d_model + dm;
+ w13[dst_idx] = w13_original[src_idx];
+ }
+ }
+ }
+ let w2: Vec<bf16> = (0..e * d_ff * d_model)
+ .map(|_| bf16::from_f32(rng.random_range(-0.05..0.05)))
+ .collect();
+ let up_biases: Vec<bf16> = (0..e * 2 * d_ff)
+ .map(|_| bf16::from_f32(rng.random_range(-0.01..0.01)))
+ .collect();
+ let down_biases: Vec<bf16> = (0..e * d_model)
+ .map(|_| bf16::from_f32(rng.random_range(-0.01..0.01)))
+ .collect();
+
+ let experts_kernel =
+ MoeExpertsTwoPassDecodeKernel::new(ctx).expect("decode kernel");
+
+ let x_perm_buf = alloc_buffer_with_data(ctx, &x_perm);
+ let offsets_buf = alloc_buffer_with_data(ctx, &offsets);
+ let w13_buf = alloc_buffer_with_data(ctx, &w13);
+ let w2_buf = alloc_buffer_with_data(ctx, &w2);
+ let up_biases_buf = alloc_buffer_with_data(ctx, &up_biases);
+ let down_biases_buf = alloc_buffer_with_data(ctx, &down_biases);
+
+ let num_tiles_k = ((d_ff + K_TILE - 1) / K_TILE) as usize;
+ let hidden_buf = alloc_buffer::<f32>(ctx, sum_k * d_ff);
+ let output_buf = alloc_buffer::<bf16>(ctx, sum_k * d_model);
+
+ let tile_counts_buf = alloc_buffer::<u32>(ctx, e);
+ let tile_offsets_buf = alloc_buffer::<u32>(ctx, e + 1);
+ let max_tiles = e * sum_k * 16384;
+ let tile_map_buf = alloc_buffer::<u32>(ctx, max_tiles * 3);
+ let total_tiles_buf = alloc_buffer::<u32>(ctx, 1);
+ let dispatch_args_buf = alloc_buffer::<u32>(ctx, 3);
+ let row_expert_map_buf = alloc_buffer::<u32>(ctx, sum_k);
+
+ let make_args = || MoeExpertsTwoPassArguments {
+ x_perm_buffer: &x_perm_buf,
+ expert_offsets: &offsets_buf,
+ row_expert_map: &row_expert_map_buf,
+ hidden_buffer: &hidden_buf,
+ output_buffer: &output_buf,
+ w13_all: &w13_buf,
+ w2_all: &w2_buf,
+ up_biases: &up_biases_buf,
+ down_biases: &down_biases_buf,
+ tile_counts: &tile_counts_buf,
+ tile_offsets: &tile_offsets_buf,
+ tile_map: &tile_map_buf,
+ total_tiles: &total_tiles_buf,
+ dispatch_args: &dispatch_args_buf,
+ total_rows: sum_k,
+ d_model,
+ d_ff,
+ e,
+ num_tiles_k: num_tiles_k as u32,
+ gating_code: 2,
+ gate_clip_min: f32::NEG_INFINITY,
+ gate_clip_max: f32::INFINITY,
+ up_clip_min: f32::NEG_INFINITY,
+ up_clip_max: f32::INFINITY,
+ silu_alpha: 1.0,
+ data_type: KernelDataType::BFloat16,
+ };
+
+ for _ in 0..warmup {
+ let cb = ctx.command_queue.new_command_buffer();
+ experts_kernel.encode(&cb, make_args()).expect("encode");
+ cb.commit();
+ cb.wait_until_completed();
+ }
+
+ let mut times = Vec::with_capacity(iters);
+ for _ in 0..iters {
+ let start = Instant::now();
+ let cb = ctx.command_queue.new_command_buffer();
+ experts_kernel.encode(&cb, make_args()).expect("encode");
+ cb.commit();
+ cb.wait_until_completed();
+ times.push(start.elapsed().as_secs_f64() * 1000.0);
+ }
+
+ times.iter().sum::<f64>() / times.len() as f64
+}
+
+fn run_fused_decode_timed(
+ ctx: &MTLContext,
+ d_model: usize,
+ d_ff: usize,
+ e: usize,
+ k: usize,
+ warmup: usize,
+ iters: usize,
+) -> f64 {
+ let mut rng = StdRng::seed_from_u64(0xF053ED);
+
+ let x: Vec<bf16> = (0..d_model)
+ .map(|_| bf16::from_f32(rng.random_range(-1.0..1.0)))
+ .collect();
+
+ let topk_ids: Vec<i32> = (0..k).map(|i| (i % e) as i32).collect();
+
+ let topk_probs: Vec<bf16> = {
+ let raw: Vec<f32> =
+ (0..k).map(|_| rng.random_range(0.1..1.0)).collect();
+ let sum: f32 = raw.iter().sum();
+ raw.iter().map(|p| bf16::from_f32(p / sum)).collect()
+ };
+
+ let w13_all: Vec<bf16> = (0..e * 2 * d_ff * d_model)
+ .map(|_| bf16::from_f32(rng.random_range(-0.05..0.05)))
+ .collect();
+ let w2_all: Vec<bf16> = (0..e * d_ff * d_model)
+ .map(|_| bf16::from_f32(rng.random_range(-0.05..0.05)))
+ .collect();
+ let up_biases: Vec<bf16> = (0..e * 2 * d_ff)
+ .map(|_| bf16::from_f32(rng.random_range(-0.01..0.01)))
+ .collect();
+ let down_biases: Vec<bf16> = (0..e * d_model)
+ .map(|_| bf16::from_f32(rng.random_range(-0.01..0.01)))
+ .collect();
+
+ let fused_kernel =
+ MoeExpertsSingleDecodeKernel::new(ctx).expect("fused kernel");
+
+ let x_buf = alloc_buffer_with_data(ctx, &x);
+ let topk_ids_buf = alloc_buffer_with_data(ctx, &topk_ids);
+ let topk_probs_buf = alloc_buffer_with_data(ctx, &topk_probs);
+ let w13_buf = alloc_buffer_with_data(ctx, &w13_all);
+ let w2_buf = alloc_buffer_with_data(ctx, &w2_all);
+ let up_biases_buf = alloc_buffer_with_data(ctx, &up_biases);
+ let down_biases_buf = alloc_buffer_with_data(ctx, &down_biases);
+ let hidden_buf = alloc_buffer::<f32>(ctx, k * d_ff);
+ let y_buf = alloc_buffer::<bf16>(ctx, d_model);
+
+ let make_args = || MoeExpertsSingleDecodeArguments {
+ x: &x_buf,
+ topk_ids: &topk_ids_buf,
+ topk_probs: &topk_probs_buf,
+ w13_all: &w13_buf,
+ w2_all: &w2_buf,
+ up_biases: &up_biases_buf,
+ down_biases: &down_biases_buf,
+ hidden: &hidden_buf,
+ y: &y_buf,
+ d_model,
+ d_ff,
+ k,
+ gating_code: 2,
+ silu_alpha: 1.0,
+ gate_clip_min: f32::NEG_INFINITY,
+ gate_clip_max: f32::INFINITY,
+ up_clip_min: f32::NEG_INFINITY,
+ up_clip_max: f32::INFINITY,
+ data_type: KernelDataType::BFloat16,
+ };
+
+ for _ in 0..warmup {
+ let cb = ctx.command_queue.new_command_buffer();
+ fused_kernel.encode(&cb, make_args()).expect("encode");
+ cb.commit();
+ cb.wait_until_completed();
+ }
+
+ let mut times = Vec::with_capacity(iters);
+ for _ in 0..iters {
+ let start = Instant::now();
+ let cb = ctx.command_queue.new_command_buffer();
+ fused_kernel.encode(&cb, make_args()).expect("encode");
+ cb.commit();
+ cb.wait_until_completed();
+ times.push(start.elapsed().as_secs_f64() * 1000.0);
+ }
+
+ times.iter().sum::<f64>() / times.len() as f64
+}
diff --git a/crates/uzu/tests/moe/moe_experts_test.rs b/crates/uzu/tests/moe/moe_experts_test.rs
index 7aab2d2fe..ef1bb14af 100644
--- a/crates/uzu/tests/moe/moe_experts_test.rs
+++ b/crates/uzu/tests/moe/moe_experts_test.rs
@@ -13,6 +13,7 @@ use rand::{Rng, SeedableRng, rngs::StdRng};
use uzu::backends::metal::{
KernelDataType,
kernel::moe::{
+ MoeExpertsSingleDecodeArguments, MoeExpertsSingleDecodeKernel,
MoeExpertsTwoPassArguments, MoeExpertsTwoPassDecodeKernel,
MoeExpertsTwoPassPrefillKernel,
},
@@ -25,62 +26,163 @@ use test_utils::{
cpu_tile_scan, create_ctx,
};
-/// Build expert segment offsets - distributes sum_k rows across e experts
-fn build_offsets(
+/// Test data for MoE experts
+struct MoeTestData {
+ x: Vec<bf16>,
+ topk_ids: Vec<i32>,
+ topk_probs: Vec<bf16>,
+ w13: Vec<bf16>,
+ w2: Vec<bf16>,
+ up_biases: Vec<bf16>,
+ down_biases: Vec<bf16>,
+}
+
+impl MoeTestData {
+ fn generate(rng: &mut StdRng, t: usize, k: usize, d_model: usize, d_ff: usize, e: usize) -> Self {
+ let x: Vec<bf16> = (0..t * d_model)
+ .map(|_| bf16::from_f32(rng.random_range(-1.0..1.0)))
+ .collect();
+
+ let topk_ids: Vec<i32> = (0..t * k)
+ .map(|i| ((i / k) + (i % k) * 2) as i32 % e as i32)
+ .collect();
+
+ let topk_probs: Vec<bf16> = (0..t)
+ .flat_map(|_| {
+ let raw: Vec<f32> = (0..k).map(|_| rng.random_range(0.1..1.0)).collect();
+ let sum: f32 = raw.iter().sum();
+ raw.iter().map(|p| bf16::from_f32(p / sum)).collect::<Vec<_>>()
+ })
+ .collect();
+
+ let w13: Vec<bf16> = (0..e * 2 * d_ff * d_model)
+ .map(|_| bf16::from_f32(rng.random_range(-0.05..0.05)))
+ .collect();
+ let w2: Vec<bf16> = (0..e * d_ff * d_model)
+ .map(|_| bf16::from_f32(rng.random_range(-0.05..0.05)))
+ .collect();
+ let up_biases: Vec<bf16> = (0..e * 2 * d_ff)
+ .map(|_| bf16::from_f32(rng.random_range(-0.01..0.01)))
+ .collect();
+ let down_biases: Vec<bf16> = (0..e * d_model)
+ .map(|_| bf16::from_f32(rng.random_range(-0.01..0.01)))
+ .collect();
+
+ Self { x, topk_ids, topk_probs, w13, w2, up_biases, down_biases }
+ }
+}
+
+/// Result of scatter operation - buckets rows by expert
+struct ScatterResult {
+ x_perm: Vec<bf16>,
+ offsets: Vec<u32>,
+ row_expert_map: Vec<u32>,
+ perm_idx: Vec<usize>, // original row -> bucketed row (for gather)
+}
+
+fn scatter_by_expert(
+ x: &[bf16],
+ topk_ids: &[i32],
+ t: usize,
+ k: usize,
+ d_model: usize,
e: usize,
- sum_k: usize,
-) -> Vec<u32> {
+) -> ScatterResult {
+ let sum_k = t * k;
+
+ // Count rows per expert and build offsets
+ let mut expert_counts = vec![0usize; e];
+ for &eid in topk_ids {
+ expert_counts[eid as usize] += 1;
+ }
let mut offsets = vec![0u32; e + 1];
- let base_rows = sum_k / e;
- let extra = sum_k % e;
+ for i in 0..e {
+ offsets[i + 1] = offsets[i] + expert_counts[i] as u32;
+ }
+
+ // Build permutation indices
+ let mut expert_cursors = vec![0usize; e];
+ let mut perm_idx = vec![0usize; sum_k];
+ let mut inv_perm = vec![0usize; sum_k];
+ for orig_row in 0..sum_k {
+ let eid = topk_ids[orig_row] as usize;
+ let bucket_pos = offsets[eid] as usize + expert_cursors[eid];
+ perm_idx[orig_row] = bucket_pos;
+ inv_perm[bucket_pos] = orig_row;
+ expert_cursors[eid] += 1;
+ }
- for i in 0..=e {
- offsets[i] = (base_rows * i + extra.min(i)) as u32;
+ // Build x_perm and row_expert_map in bucketed order
+ let mut x_perm = vec![bf16::from_f32(0.0); sum_k * d_model];
+ let mut row_expert_map = vec![0u32; sum_k];
+ for bucket_row in 0..sum_k {
+ let orig_row = inv_perm[bucket_row];
+ let tok = orig_row / k;
+ for d in 0..d_model {
+ x_perm[bucket_row * d_model + d] = x[tok * d_model + d];
+ }
+ row_expert_map[bucket_row] = topk_ids[orig_row] as u32;
}
- offsets
+
+ ScatterResult { x_perm, offsets, row_expert_map, perm_idx }
}
-/// Build row-expert map: maps each row to its expert index
-fn build_row_expert_map(
- offsets: &[u32],
- sum_k: usize,
-) -> Vec<u32> {
- let e = offsets.len() - 1;
- let mut row_map = vec![0u32; sum_k];
-
- for expert in 0..e {
- let start = offsets[expert] as usize;
- let end = offsets[expert + 1] as usize;
- for row in start..end {
- row_map[row] = expert as u32;
+/// Gather + finalize: reorder from bucketed order and apply weighted sum
+fn gather_and_finalize(
+ y_partial: &[bf16],
+ topk_probs: &[bf16],
+ perm_idx: &[usize],
+ t: usize,
+ k: usize,
+ d_model: usize,
+) -> Vec<bf16> {
+ let mut y = vec![bf16::from_f32(0.0); t * d_model];
+ for tok in 0..t {
+ for d in 0..d_model {
+ let mut acc = 0.0f32;
+ for ki in 0..k {
+ let orig_row = tok * k + ki;
+ let bucket_row = perm_idx[orig_row];
+ let prob = f32::from(topk_probs[orig_row]);
+ let val = f32::from(y_partial[bucket_row * d_model + d]);
+ acc += prob * val;
+ }
+ y[tok * d_model + d] = bf16::from_f32(acc);
}
}
- row_map
+ y
}
-/// CPU reference for full expert computation matching GPU implementation
+/// Unified CPU reference for MoE computation
+///
+/// Computes: y[t] = Σ_k prob[t,k] * (activation(x[t] @ W13[expert_k]) @ W2[expert_k] + bias)
///
/// Supports 4 gating modes:
/// - 0: GELU(up)
/// - 1: SiLU(up)
/// - 2: SwiGLU = SiLU(gate) * up
/// - 3: GEGLU = GELU(gate) * up
-fn cpu_expert_ffn_full(
- x_perm: &[bf16], // [sum_k, d_model]
- offsets: &[u32], // [E+1]
- w13: &[bf16], // [E, 2*d_ff, d_model] transposed
- w2: &[bf16], // [E, d_model, d_ff] transposed
+#[allow(clippy::too_many_arguments)]
+fn cpu_moe_reference(
+ x: &[bf16], // [T, d_model]
+ topk_ids: &[i32], // [T * K]
+ topk_probs: &[bf16], // [T * K]
+ w13_all: &[bf16], // [E, 2*d_ff, d_model] transposed
+ w2_all: &[bf16], // [E, d_model, d_ff] transposed
up_biases: &[bf16], // [E, 2*d_ff]
down_biases: &[bf16], // [E, d_model]
+ t: usize,
d_model: usize,
d_ff: usize,
+ k: usize,
gating_code: u32,
silu_alpha: f32,
-) -> (Vec<f32>, Vec<bf16>) {
- let e = offsets.len() - 1;
- let sum_k = x_perm.len() / d_model;
- let mut hidden_full = vec![0.0f32; sum_k * d_ff];
- let mut output = vec![bf16::from_f32(0.0); sum_k * d_model];
+ gate_clip_min: f32,
+ gate_clip_max: f32,
+ up_clip_min: f32,
+ up_clip_max: f32,
+) -> Vec<bf16> {
+ let mut y = vec![bf16::from_f32(0.0); t * d_model];
// Helper functions matching Metal kernels
let gelu_approx = |x: f32| -> f32 {
@@ -100,78 +202,97 @@ fn cpu_expert_ffn_full(
let silu = |x: f32, alpha: f32| -> f32 { x / (1.0 + (-alpha * x).exp()) };
- for expert in 0..e {
- let row_start = offsets[expert] as usize;
- let row_end = offsets[expert + 1] as usize;
- let expert_rows = row_end - row_start;
+ // Process each token
+ for tok in 0..t {
+ let x_offset = tok * d_model;
- if expert_rows == 0 {
- continue;
- }
+ // Hidden buffer for this token's K experts [K, d_ff]
+ let mut hidden = vec![0.0f32; k * d_ff];
- let w1_offset = expert * 2 * d_ff * d_model;
- let w2_offset = expert * d_model * d_ff;
-
- // FC1: [expert_rows, d_model] @ [d_model, 2*d_ff] -> [expert_rows, 2*d_ff]
- let mut hidden = vec![bf16::from_f32(0.0); expert_rows * 2 * d_ff];
- for row in 0..expert_rows {
- for ff in 0..(2 * d_ff) {
- let mut acc = f32::from(up_biases[expert * 2 * d_ff + ff]);
- for dm in 0..d_model {
- let x_val =
- f32::from(x_perm[(row_start + row) * d_model + dm]);
- // w13 transposed: [2*d_ff, d_model]
- let w_val = f32::from(w13[w1_offset + ff * d_model + dm]);
- acc += x_val * w_val;
- }
- hidden[row * 2 * d_ff + ff] = bf16::from_f32(acc);
+ // Pass A: x @ W13[expert] -> hidden[k] with activation
+ for k_idx in 0..k {
+ let expert_id = topk_ids[tok * k + k_idx];
+ if expert_id < 0 {
+ continue;
}
- }
-
- // Apply activation based on gating_code
- for row in 0..expert_rows {
- for ff in 0..d_ff {
- let up = f32::from(hidden[row * 2 * d_ff + ff]);
+ let expert = expert_id as usize;
+
+ let w13_base = expert * 2 * d_ff * d_model;
+ let bias_base = expert * 2 * d_ff;
+
+ for h in 0..d_ff {
+ // Up projection
+ let mut acc_up = f32::from(up_biases[bias_base + h]);
+ for d in 0..d_model {
+ let x_val = f32::from(x[x_offset + d]);
+ let w_val = f32::from(w13_all[w13_base + h * d_model + d]);
+ acc_up += x_val * w_val;
+ }
- let result = if gating_code <= 1 {
- // GELU(up) or SiLU(up) - no gating
+ // Apply activation based on gating_code
+ let activated = if gating_code <= 1 {
+ // GELU or SiLU on up (with clipping)
+ let up_val = acc_up.clamp(up_clip_min, up_clip_max);
if gating_code == 0 {
- gelu_approx(up)
+ gelu_approx(up_val)
} else {
- silu(up, silu_alpha)
+ silu(up_val, silu_alpha)
}
} else {
- // SwiGLU or GEGLU - gate * up
- let gate = f32::from(hidden[row * 2 * d_ff + d_ff + ff]);
- let activated_gate = if gating_code == 2 {
- silu(gate, silu_alpha)
+ // SwiGLU or GEGLU - need gate projection too
+ let mut acc_gate = f32::from(up_biases[bias_base + d_ff + h]);
+ for d in 0..d_model {
+ let x_val = f32::from(x[x_offset + d]);
+ let w_val =
+ f32::from(w13_all[w13_base + (d_ff + h) * d_model + d]);
+ acc_gate += x_val * w_val;
+ }
+
+ // Apply clipping
+ let up_val = acc_up.clamp(up_clip_min, up_clip_max);
+ let gate_val = acc_gate.clamp(gate_clip_min, gate_clip_max);
+
+ let gate_act = if gating_code == 2 {
+ silu(gate_val, silu_alpha)
} else {
- gelu_approx(gate)
+ gelu_approx(gate_val)
};
- activated_gate * up
+ gate_act * up_val
};
- hidden[row * 2 * d_ff + ff] = bf16::from_f32(result);
- hidden_full[(row_start + row) * d_ff + ff] = result;
+ hidden[k_idx * d_ff + h] = activated;
}
}
- // FC2: [expert_rows, d_ff] @ [d_ff, d_model] -> [expert_rows, d_model]
- for row in 0..expert_rows {
- for dm in 0..d_model {
- let mut acc = f32::from(down_biases[expert * d_model + dm]);
- for ff in 0..d_ff {
- let h_val = f32::from(hidden[row * 2 * d_ff + ff]);
- // w2 transposed: [d_model, d_ff]
- let w_val = f32::from(w2[w2_offset + dm * d_ff + ff]);
+ // Pass B: hidden @ W2 -> y (with weighted sum)
+ for d in 0..d_model {
+ let mut final_acc = 0.0f32;
+
+ for k_idx in 0..k {
+ let expert_id = topk_ids[tok * k + k_idx];
+ if expert_id < 0 {
+ continue;
+ }
+ let expert = expert_id as usize;
+ let prob = f32::from(topk_probs[tok * k + k_idx]);
+
+ let w2_base = expert * d_model * d_ff;
+ let bias_base = expert * d_model;
+
+ let mut acc = f32::from(down_biases[bias_base + d]);
+ for h in 0..d_ff {
+ let h_val = hidden[k_idx * d_ff + h];
+ let w_val = f32::from(w2_all[w2_base + d * d_ff + h]);
acc += h_val * w_val;
}
- output[(row_start + row) * d_model + dm] = bf16::from_f32(acc);
+ final_acc += prob * acc;
}
+
+ y[tok * d_model + d] = bf16::from_f32(final_acc);
}
}
- (hidden_full, output)
+ y
}
#[test]
@@ -179,28 +300,71 @@ fn test_two_pass_decode_correctness() {
let ctx = create_ctx();
let mut rng = StdRng::seed_from_u64(0xDEC0DE42);
- // Realistic decode test case: single token, K=2
+ // End-to-end test: T=1 token, K=2 experts per token
let t = 1;
let k = 2;
let sum_k = t * k;
let d_model = 512;
let d_ff = 2048;
let e = 8;
+ let gating_code = 2u32; // SwiGLU
+ let silu_alpha = 1.0f32;
eprintln!(
"[2-pass decode] T={}, K={}, sum_k={}, d_model={}, d_ff={}, E={}",
t, k, sum_k, d_model, d_ff, e
);
- // Generate test data
- let x_perm: Vec<bf16> = (0..sum_k * d_model)
+ // Generate input x [T, d_model]
+ let x: Vec<bf16> = (0..t * d_model)
.map(|_| bf16::from_f32(rng.random_range(-1.0..1.0)))
.collect();
- let offsets = build_offsets(e, sum_k);
- let row_expert_map = build_row_expert_map(&offsets, sum_k);
+ // Generate routing: topk_ids [T*K], topk_probs [T*K]
+ let topk_ids: Vec<i32> = (0..t * k)
+ .map(|i| ((i / k) + (i % k) * 2) as i32 % e as i32)
+ .collect();
+ let topk_probs: Vec<bf16> = (0..t)
+ .flat_map(|_| {
+ let raw: Vec<f32> =
+ (0..k).map(|_| rng.random_range(0.1..1.0)).collect();
+ let sum: f32 = raw.iter().sum();
+ raw.iter().map(|p| bf16::from_f32(p / sum)).collect::<Vec<_>>()
+ })
+ .collect();
+
+ // Scatter x based on topk_ids to get x_perm
+ // Each token contributes K rows, one per selected expert
+ let mut x_perm = vec![bf16::from_f32(0.0); sum_k * d_model];
+ for tok in 0..t {
+ for ki in 0..k {
+ let row_idx = tok * k + ki;
+ for d in 0..d_model {
+ x_perm[row_idx * d_model + d] = x[tok * d_model + d];
+ }
+ }
+ }
+
+ // Build offsets and row_expert_map based on topk_ids
+ // First count rows per expert
+ let mut expert_counts = vec![0usize; e];
+ for &eid in &topk_ids {
+ expert_counts[eid as usize] += 1;
+ }
+ let mut offsets = vec![0u32; e + 1];
+ for i in 0..e {
+ offsets[i + 1] = offsets[i] + expert_counts[i] as u32;
+ }
- // Generate weights (W13 in GPU transposed layout [E, 2*d_ff, d_model])
+ // Build row_expert_map (maps permuted row to expert)
+ let mut row_expert_map = vec![0u32; sum_k];
+ let mut expert_row_idx = vec![0usize; e];
+ for (row, &eid) in topk_ids.iter().enumerate() {
+ row_expert_map[row] = eid as u32;
+ expert_row_idx[eid as usize] += 1;
+ }
+
+ // Generate weights
let w13: Vec<bf16> = (0..e * 2 * d_ff * d_model)
.map(|_| bf16::from_f32(rng.random_range(-0.05..0.05)))
.collect();
@@ -214,18 +378,25 @@ fn test_two_pass_decode_correctness() {
.map(|_| bf16::from_f32(rng.random_range(-0.01..0.01)))
.collect();
- // CPU reference (gating_code=0 means GELU)
- let (_hidden_expected, expected) = cpu_expert_ffn_full(
- &x_perm,
- &offsets,
+ // CPU reference - end-to-end
+ let y_expected = cpu_moe_reference(
+ &x,
+ &topk_ids,
+ &topk_probs,
&w13,
&w2,
&up_biases,
&down_biases,
+ t,
d_model,
d_ff,
- 0,
- 1.0,
+ k,
+ gating_code,
+ silu_alpha,
+ f32::NEG_INFINITY,
+ f32::INFINITY,
+ f32::NEG_INFINITY,
+ f32::INFINITY,
);
// Prepare GPU buffers
@@ -237,9 +408,8 @@ fn test_two_pass_decode_correctness() {
let up_biases_buf = alloc_buffer_with_data(&ctx, &up_biases);
let down_biases_buf = alloc_buffer_with_data(&ctx, &down_biases);
- // Intermediate buffers - hidden is f32 for activation precision
let hidden_buf = alloc_buffer::<f32>(&ctx, sum_k * d_ff);
- let output_buf = alloc_buffer::<bf16>(&ctx, sum_k * d_model);
+ let y_partial_buf = alloc_buffer::<bf16>(&ctx, sum_k * d_model);
// Tile infrastructure
let h_blocks_decode = (d_ff + 3) / 4;
@@ -267,7 +437,7 @@ fn test_two_pass_decode_correctness() {
expert_offsets: &offsets_buf,
row_expert_map: &row_expert_map_buf,
hidden_buffer: &hidden_buf,
- output_buffer: &output_buf,
+ output_buffer: &y_partial_buf,
w13_all: &w13_buf,
w2_all: &w2_buf,
up_biases: &up_biases_buf,
@@ -282,12 +452,12 @@ fn test_two_pass_decode_correctness() {
d_ff,
e,
num_tiles_k,
- gating_code: 0, // GELU activation
- gate_clip_min: -1e6,
- gate_clip_max: 1e6,
- up_clip_min: -1e6,
- up_clip_max: 1e6,
- silu_alpha: 1.0,
+ gating_code,
+ gate_clip_min: f32::NEG_INFINITY,
+ gate_clip_max: f32::INFINITY,
+ up_clip_min: f32::NEG_INFINITY,
+ up_clip_max: f32::INFINITY,
+ silu_alpha,
data_type: KernelDataType::BFloat16,
},
)
@@ -296,63 +466,126 @@ fn test_two_pass_decode_correctness() {
cb.commit();
cb.wait_until_completed();
- // Read GPU output
- let output_gpu = unsafe {
+ // Read GPU partial output and do CPU finalize (weighted sum)
+ let y_partial_gpu = unsafe {
std::slice::from_raw_parts(
- output_buf.contents() as *const bf16,
+ y_partial_buf.contents() as *const bf16,
sum_k * d_model,
)
};
+ // Finalize: y[t] = Σ_k prob[t,k] * y_partial[t*k + k_idx]
+ let mut y_gpu = vec![bf16::from_f32(0.0); t * d_model];
+ for tok in 0..t {
+ for d in 0..d_model {
+ let mut acc = 0.0f32;
+ for ki in 0..k {
+ let row_idx = tok * k + ki;
+ let prob = f32::from(topk_probs[row_idx]);
+ let val = f32::from(y_partial_gpu[row_idx * d_model + d]);
+ acc += prob * val;
+ }
+ y_gpu[tok * d_model + d] = bf16::from_f32(acc);
+ }
+ }
+
// Compute error metrics
let mut max_abs_error = 0.0f32;
- let mut max_rel_error = 0.0f32;
- let mut max_abs_idx = 0;
- let mut max_rel_idx = 0;
+ let mut max_idx = 0;
for (i, (&gpu_val, &cpu_val)) in
- output_gpu.iter().zip(expected.iter()).enumerate()
+ y_gpu.iter().zip(y_expected.iter()).enumerate()
{
- let gpu_f = f32::from(gpu_val);
- let cpu_f = f32::from(cpu_val);
- let abs_error = (gpu_f - cpu_f).abs();
- let rel_error = if cpu_f.abs() > 1e-6 {
- abs_error / cpu_f.abs()
- } else {
- 0.0
- };
-
+ let abs_error = (f32::from(gpu_val) - f32::from(cpu_val)).abs();
if abs_error > max_abs_error {
max_abs_error = abs_error;
- max_abs_idx = i;
- }
- if rel_error > max_rel_error {
- max_rel_error = rel_error;
- max_rel_idx = i;
+ max_idx = i;
}
}
eprintln!(
- "[2-pass decode] Max absolute error: {:.6} at index {} (GPU={:.6}, CPU={:.6})",
+ "[2-pass decode] Max error: {:.6} at idx {} (GPU={:.6}, CPU={:.6})",
max_abs_error,
- max_abs_idx,
- f32::from(output_gpu[max_abs_idx]),
- f32::from(expected[max_abs_idx])
+ max_idx,
+ f32::from(y_gpu[max_idx]),
+ f32::from(y_expected[max_idx])
);
- eprintln!(
- "[2-pass decode] Max relative error: {:.6} ({:.2}%) at index {} (GPU={:.6}, CPU={:.6})",
- max_rel_error,
- max_rel_error * 100.0,
- max_rel_idx,
- f32::from(output_gpu[max_rel_idx]),
- f32::from(expected[max_rel_idx])
+
+ let tolerance = 0.02;
+ assert_bf16_close(&y_gpu, &y_expected, tolerance, "2-pass decode output");
+
+ eprintln!("[2-pass decode] ✓ PASSED (tolerance={:.4})", tolerance);
+}
+
+#[test]
+fn test_two_pass_decode_multi_token() {
+ let ctx = create_ctx();
+ let mut rng = StdRng::seed_from_u64(0xDEC0DE43);
+
+ let t = 4;
+ let k = 2;
+ let sum_k = t * k;
+ let d_model = 512;
+ let d_ff = 2048;
+ let e = 8;
+ let gating_code = 2u32;
+ let silu_alpha = 1.0f32;
+
+ eprintln!("[2-pass decode multi-token] T={}, K={}, d_model={}, d_ff={}, E={}", t, k, d_model, d_ff, e);
+
+ let data = MoeTestData::generate(&mut rng, t, k, d_model, d_ff, e);
+ let scatter = scatter_by_expert(&data.x, &data.topk_ids, t, k, d_model, e);
+
+ let y_expected = cpu_moe_reference(
+ &data.x, &data.topk_ids, &data.topk_probs, &data.w13, &data.w2,
+ &data.up_biases, &data.down_biases, t, d_model, d_ff, k, gating_code,
+ silu_alpha, f32::NEG_INFINITY, f32::INFINITY, f32::NEG_INFINITY, f32::INFINITY,
);
- let tolerance = 0.01;
+ // GPU buffers
+ let x_perm_buf = alloc_buffer_with_data(&ctx, &scatter.x_perm);
+ let offsets_buf = alloc_buffer_with_data(&ctx, &scatter.offsets);
+ let row_expert_map_buf = alloc_buffer_with_data(&ctx, &scatter.row_expert_map);
+ let w13_buf = alloc_buffer_with_data(&ctx, &data.w13);
+ let w2_buf = alloc_buffer_with_data(&ctx, &data.w2);
+ let up_biases_buf = alloc_buffer_with_data(&ctx, &data.up_biases);
+ let down_biases_buf = alloc_buffer_with_data(&ctx, &data.down_biases);
+ let hidden_buf = alloc_buffer::<f32>(&ctx, sum_k * d_ff);
+ let y_partial_buf = alloc_buffer::<bf16>(&ctx, sum_k * d_model);
- assert_bf16_close(output_gpu, &expected, tolerance, "2-pass decode output");
+ // Tile infrastructure
+ let max_total_tiles = sum_k * ((d_ff + 3) / 4);
+ let tile_counts_buf = alloc_buffer::<u32>(&ctx, e);
+ let tile_offsets_buf = alloc_buffer::<u32>(&ctx, e + 1);
+ let total_tiles_buf = alloc_buffer::<u32>(&ctx, 8);
+ let tile_map_buf = alloc_buffer::<u32>(&ctx, max_total_tiles * 3);
+ let dispatch_args_buf = alloc_buffer::<u32>(&ctx, 3);
- eprintln!("[2-pass decode] ✓ PASSED (tolerance={:.4})", tolerance);
+ let experts_kernel = MoeExpertsTwoPassDecodeKernel::new(&ctx).expect("kernel");
+ let cb = ctx.command_queue.new_command_buffer();
+ experts_kernel.encode(&cb, MoeExpertsTwoPassArguments {
+ x_perm_buffer: &x_perm_buf, expert_offsets: &offsets_buf,
+ row_expert_map: &row_expert_map_buf, hidden_buffer: &hidden_buf,
+ output_buffer: &y_partial_buf, w13_all: &w13_buf, w2_all: &w2_buf,
+ up_biases: &up_biases_buf, down_biases: &down_biases_buf,
+ tile_counts: &tile_counts_buf, tile_offsets: &tile_offsets_buf,
+ tile_map: &tile_map_buf, total_tiles: &total_tiles_buf,
+ dispatch_args: &dispatch_args_buf, total_rows: sum_k, d_model, d_ff, e,
+ num_tiles_k: ((d_ff + 63) / 64) as u32, gating_code,
+ gate_clip_min: f32::NEG_INFINITY, gate_clip_max: f32::INFINITY,
+ up_clip_min: f32::NEG_INFINITY, up_clip_max: f32::INFINITY,
+ silu_alpha, data_type: KernelDataType::BFloat16,
+ }).expect("encode");
+ cb.commit();
+ cb.wait_until_completed();
+
+ let y_partial_gpu = unsafe {
+ std::slice::from_raw_parts(y_partial_buf.contents() as *const bf16, sum_k * d_model)
+ };
+ let y_gpu = gather_and_finalize(y_partial_gpu, &data.topk_probs, &scatter.perm_idx, t, k, d_model);
+
+ assert_bf16_close(&y_gpu, &y_expected, 0.02, "2-pass decode multi-token");
+ eprintln!("[2-pass decode multi-token] ✓ PASSED");
}
#[test]
@@ -360,32 +593,128 @@ fn test_two_pass_prefill_correctness() {
let ctx = create_ctx();
let mut rng = StdRng::seed_from_u64(0xFE1111);
- // Realistic prefill case: 8 tokens, K=2
let t = 8;
let k = 2;
let sum_k = t * k;
let d_model = 512;
let d_ff = 2048;
let e = 8;
+ let gating_code = 2u32;
+ let silu_alpha = 1.0f32;
+
+ eprintln!("[2-pass prefill] T={}, K={}, d_model={}, d_ff={}, E={}", t, k, d_model, d_ff, e);
+
+ let data = MoeTestData::generate(&mut rng, t, k, d_model, d_ff, e);
+ let scatter = scatter_by_expert(&data.x, &data.topk_ids, t, k, d_model, e);
+
+ let y_expected = cpu_moe_reference(
+ &data.x, &data.topk_ids, &data.topk_probs, &data.w13, &data.w2,
+ &data.up_biases, &data.down_biases, t, d_model, d_ff, k, gating_code,
+ silu_alpha, f32::NEG_INFINITY, f32::INFINITY, f32::NEG_INFINITY, f32::INFINITY,
+ );
+
+ // GPU buffers
+ let x_perm_buf = alloc_buffer_with_data(&ctx, &scatter.x_perm);
+ let offsets_buf = alloc_buffer_with_data(&ctx, &scatter.offsets);
+ let row_expert_map_buf = alloc_buffer_with_data(&ctx, &scatter.row_expert_map);
+ let w13_buf = alloc_buffer_with_data(&ctx, &data.w13);
+ let w2_buf = alloc_buffer_with_data(&ctx, &data.w2);
+ let up_biases_buf = alloc_buffer_with_data(&ctx, &data.up_biases);
+ let down_biases_buf = alloc_buffer_with_data(&ctx, &data.down_biases);
+ let hidden_buf = alloc_buffer::<f32>(&ctx, sum_k * d_ff);
+ let y_partial_buf = alloc_buffer::<bf16>(&ctx, sum_k * d_model);
+
+ // Tile infrastructure
+ let max_total_tiles = sum_k * ((d_ff + 3) / 4);
+ let tile_counts_buf = alloc_buffer::<u32>(&ctx, e);
+ let tile_offsets_buf = alloc_buffer::<u32>(&ctx, e + 1);
+ let total_tiles_buf = alloc_buffer::<u32>(&ctx, 8);
+ let tile_map_buf = alloc_buffer::<u32>(&ctx, max_total_tiles * 3);
+ let dispatch_args_buf = alloc_buffer::<u32>(&ctx, 3);
+
+ let experts_kernel = MoeExpertsTwoPassPrefillKernel::new(&ctx).expect("kernel");
+ let cb = ctx.command_queue.new_command_buffer();
+ experts_kernel.encode(&cb, MoeExpertsTwoPassArguments {
+ x_perm_buffer: &x_perm_buf, expert_offsets: &offsets_buf,
+ row_expert_map: &row_expert_map_buf, hidden_buffer: &hidden_buf,
+ output_buffer: &y_partial_buf, w13_all: &w13_buf, w2_all: &w2_buf,
+ up_biases: &up_biases_buf, down_biases: &down_biases_buf,
+ tile_counts: &tile_counts_buf, tile_offsets: &tile_offsets_buf,
+ tile_map: &tile_map_buf, total_tiles: &total_tiles_buf,
+ dispatch_args: &dispatch_args_buf, total_rows: sum_k, d_model, d_ff, e,
+ num_tiles_k: ((d_ff + 63) / 64) as u32, gating_code,
+ gate_clip_min: f32::NEG_INFINITY, gate_clip_max: f32::INFINITY,
+ up_clip_min: f32::NEG_INFINITY, up_clip_max: f32::INFINITY,
+ silu_alpha, data_type: KernelDataType::BFloat16,
+ }).expect("encode");
+ cb.commit();
+ cb.wait_until_completed();
+
+ let y_partial_gpu = unsafe {
+ std::slice::from_raw_parts(y_partial_buf.contents() as *const bf16, sum_k * d_model)
+ };
+ let y_gpu = gather_and_finalize(y_partial_gpu, &data.topk_probs, &scatter.perm_idx, t, k, d_model);
+
+ assert_bf16_close(&y_gpu, &y_expected, 0.02, "2-pass prefill");
+ eprintln!("[2-pass prefill] ✓ PASSED");
+}
+
+#[test]
+fn test_tile_infrastructure() {
+ // Test that tile counts/scan/map are computed correctly
+ // This is already tested in moe_tiles_test.rs, but we verify here
+ // that the tiles match our expectations for expert workload distribution
+
+ let offsets = vec![0, 5, 20, 35, 40];
+ const BM: usize = 16;
+
+ let tile_counts = cpu_tile_counts(&offsets, BM);
+ let (tile_offsets, total_tiles) = cpu_tile_scan(&tile_counts);
+
+ // Expert 0: 5 rows -> 1 tile
+ // Expert 1: 15 rows -> 1 tile
+ // Expert 2: 15 rows -> 1 tile
+ // Expert 3: 5 rows -> 1 tile
+ assert_eq!(tile_counts, vec![1, 1, 1, 1]);
+ assert_eq!(tile_offsets, vec![0, 1, 2, 3, 4]);
+ assert_eq!(total_tiles, 4);
+
+ eprintln!("[tile infrastructure] ✓ PASSED");
+}
+
+#[test]
+fn test_fused_single_token_decode() {
+ let ctx = create_ctx();
+ let mut rng = StdRng::seed_from_u64(0xF053ED);
+
+ let d_model = 512;
+ let d_ff = 2048;
+ let e = 8;
+ let k = 2;
eprintln!(
- "[2-pass prefill] T={}, K={}, sum_k={}, d_model={}, d_ff={}, E={}",
- t, k, sum_k, d_model, d_ff, e
+ "[fused single-token] d_model={}, d_ff={}, E={}, K={}",
+ d_model, d_ff, e, k
);
// Generate test data
- let x_perm: Vec<bf16> = (0..sum_k * d_model)
+ let x: Vec<bf16> = (0..d_model)
.map(|_| bf16::from_f32(rng.random_range(-1.0..1.0)))
.collect();
- let offsets = build_offsets(e, sum_k);
- let row_expert_map = build_row_expert_map(&offsets, sum_k);
+ let topk_ids: Vec<i32> = (0..k).map(|i| (i % e) as i32).collect();
- // Generate weights
- let w13: Vec<bf16> = (0..e * 2 * d_ff * d_model)
+ let topk_probs: Vec<bf16> = {
+ let raw: Vec<f32> =
+ (0..k).map(|_| rng.random_range(0.1..1.0)).collect();
+ let sum: f32 = raw.iter().sum();
+ raw.iter().map(|p| bf16::from_f32(p / sum)).collect()
+ };
+
+ let w13_all: Vec<bf16> = (0..e * 2 * d_ff * d_model)
.map(|_| bf16::from_f32(rng.random_range(-0.05..0.05)))
.collect();
- let w2: Vec<bf16> = (0..e * d_ff * d_model)
+ let w2_all: Vec<bf16> = (0..e * d_ff * d_model)
.map(|_| bf16::from_f32(rng.random_range(-0.05..0.05)))
.collect();
let up_biases: Vec<bf16> = (0..e * 2 * d_ff)
@@ -395,222 +724,259 @@ fn test_two_pass_prefill_correctness() {
.map(|_| bf16::from_f32(rng.random_range(-0.01..0.01)))
.collect();
- // CPU reference (gating_code=0 means GELU)
- let (hidden_expected, expected) = cpu_expert_ffn_full(
- &x_perm,
- &offsets,
- &w13,
- &w2,
+ // CPU reference (gating_code=2 for SwiGLU)
+ let gating_code = 2u32;
+ let silu_alpha = 1.0f32;
+ let y_expected = cpu_moe_reference(
+ &x,
+ &topk_ids,
+ &topk_probs,
+ &w13_all,
+ &w2_all,
&up_biases,
&down_biases,
+ 1, // T=1 for single token
d_model,
d_ff,
- 0,
- 1.0,
+ k,
+ gating_code,
+ silu_alpha,
+ f32::NEG_INFINITY,
+ f32::INFINITY,
+ f32::NEG_INFINITY,
+ f32::INFINITY,
);
- // Prepare GPU buffers
- let x_perm_buf = alloc_buffer_with_data(&ctx, &x_perm);
- let offsets_buf = alloc_buffer_with_data(&ctx, &offsets);
- let row_expert_map_buf = alloc_buffer_with_data(&ctx, &row_expert_map);
- let w13_buf = alloc_buffer_with_data(&ctx, &w13);
- let w2_buf = alloc_buffer_with_data(&ctx, &w2);
+ // GPU buffers
+ let x_buf = alloc_buffer_with_data(&ctx, &x);
+ let topk_ids_buf = alloc_buffer_with_data(&ctx, &topk_ids);
+ let topk_probs_buf = alloc_buffer_with_data(&ctx, &topk_probs);
+ let w13_buf = alloc_buffer_with_data(&ctx, &w13_all);
+ let w2_buf = alloc_buffer_with_data(&ctx, &w2_all);
let up_biases_buf = alloc_buffer_with_data(&ctx, &up_biases);
let down_biases_buf = alloc_buffer_with_data(&ctx, &down_biases);
+ let hidden_buf = alloc_buffer::<f32>(&ctx, k * d_ff);
+ let y_buf = alloc_buffer::<bf16>(&ctx, d_model);
- // Intermediate buffers - hidden is f32 for activation precision
- let hidden_buf = alloc_buffer::<f32>(&ctx, sum_k * d_ff);
- let output_buf = alloc_buffer::<bf16>(&ctx, sum_k * d_model);
-
- // Tile infrastructure
- let h_blocks_decode = (d_ff + 3) / 4;
- let max_total_tiles = sum_k * h_blocks_decode;
-
- let tile_counts_buf = alloc_buffer::<u32>(&ctx, e);
- let tile_offsets_buf = alloc_buffer::<u32>(&ctx, e + 1);
- let total_tiles_buf = alloc_buffer::<u32>(&ctx, 8);
- let tile_map_buf = alloc_buffer::<u32>(&ctx, max_total_tiles * 3);
- let dispatch_args_buf = alloc_buffer::<u32>(&ctx, 3);
-
- // Execute 2-pass prefill kernel
- let experts_kernel = MoeExpertsTwoPassPrefillKernel::new(&ctx)
- .expect("MoeExpertsTwoPassPrefillKernel::new");
+ // Run fused decode kernel
+ let fused_kernel = MoeExpertsSingleDecodeKernel::new(&ctx)
+ .expect("MoeExpertsSingleDecodeKernel::new");
let cb = ctx.command_queue.new_command_buffer();
- const K_TILE: usize = 64;
- let num_tiles_k = ((d_ff + K_TILE - 1) / K_TILE) as u32;
-
- experts_kernel
+ fused_kernel
.encode(
&cb,
- MoeExpertsTwoPassArguments {
- x_perm_buffer: &x_perm_buf,
- expert_offsets: &offsets_buf,
- row_expert_map: &row_expert_map_buf,
- hidden_buffer: &hidden_buf,
- output_buffer: &output_buf,
+ MoeExpertsSingleDecodeArguments {
+ x: &x_buf,
+ topk_ids: &topk_ids_buf,
+ topk_probs: &topk_probs_buf,
w13_all: &w13_buf,
w2_all: &w2_buf,
up_biases: &up_biases_buf,
down_biases: &down_biases_buf,
- tile_counts: &tile_counts_buf,
- tile_offsets: &tile_offsets_buf,
- tile_map: &tile_map_buf,
- total_tiles: &total_tiles_buf,
- dispatch_args: &dispatch_args_buf,
- total_rows: sum_k,
+ hidden: &hidden_buf,
+ y: &y_buf,
d_model,
d_ff,
- e,
- num_tiles_k,
- gating_code: 0, // GELU activation
- gate_clip_min: -1e6,
- gate_clip_max: 1e6,
- up_clip_min: -1e6,
- up_clip_max: 1e6,
- silu_alpha: 1.0,
+ k,
+ gating_code,
+ silu_alpha: 1.0, // Standard SiLU for testing
+ gate_clip_min: f32::NEG_INFINITY,
+ gate_clip_max: f32::INFINITY,
+ up_clip_min: f32::NEG_INFINITY,
+ up_clip_max: f32::INFINITY,
data_type: KernelDataType::BFloat16,
},
)
- .expect("encode_two_pass_prefill");
+ .expect("fused decode encode");
cb.commit();
cb.wait_until_completed();
// Read GPU output
- let output_gpu = unsafe {
- std::slice::from_raw_parts(
- output_buf.contents() as *const bf16,
- sum_k * d_model,
- )
+ let y_gpu = unsafe {
+ std::slice::from_raw_parts(y_buf.contents() as *const bf16, d_model)
};
// Compute error metrics
let mut max_abs_error = 0.0f32;
- let mut max_rel_error = 0.0f32;
- let mut max_abs_idx = 0;
- let mut max_rel_idx = 0;
+ let mut sum_abs_error = 0.0f32;
+ let mut max_idx = 0;
for (i, (&gpu_val, &cpu_val)) in
- output_gpu.iter().zip(expected.iter()).enumerate()
+ y_gpu.iter().zip(y_expected.iter()).enumerate()
{
- let gpu_f = f32::from(gpu_val);
- let cpu_f = f32::from(cpu_val);
- let abs_error = (gpu_f - cpu_f).abs();
- let rel_error = if cpu_f.abs() > 1e-6 {
- abs_error / cpu_f.abs()
- } else {
- 0.0
- };
-
+ let abs_error = (f32::from(gpu_val) - f32::from(cpu_val)).abs();
+ sum_abs_error += abs_error;
if abs_error > max_abs_error {
max_abs_error = abs_error;
- max_abs_idx = i;
- }
- if rel_error > max_rel_error {
- max_rel_error = rel_error;
- max_rel_idx = i;
+ max_idx = i;
}
}
+ let mean_abs_error = sum_abs_error / d_model as f32;
eprintln!(
- "[2-pass prefill] Max absolute error: {:.6} at index {} (GPU={:.6}, CPU={:.6})",
+ "[fused single-token] Max error: {:.6} at idx {} (GPU={:.6}, CPU={:.6}), Mean error: {:.6}",
max_abs_error,
- max_abs_idx,
- f32::from(output_gpu[max_abs_idx]),
- f32::from(expected[max_abs_idx])
+ max_idx,
+ f32::from(y_gpu[max_idx]),
+ f32::from(y_expected[max_idx]),
+ mean_abs_error
+ );
+
+ let tolerance = 0.02;
+ assert_bf16_close(
+ y_gpu,
+ &y_expected,
+ tolerance,
+ "fused single-token output",
);
+
+ eprintln!("[fused single-token] PASSED (tolerance={:.4})", tolerance);
+}
+
+#[test]
+fn test_fused_single_token_k4() {
+ let ctx = create_ctx();
+ let mut rng = StdRng::seed_from_u64(0xE001A);
+
+ let d_model = 512;
+ let d_ff = 2048;
+ let e = 8;
+ let k = 4;
+
eprintln!(
- "[2-pass prefill] Max relative error: {:.6} ({:.2}%) at index {} (GPU={:.6}, CPU={:.6})",
- max_rel_error,
- max_rel_error * 100.0,
- max_rel_idx,
- f32::from(output_gpu[max_rel_idx]),
- f32::from(expected[max_rel_idx])
+ "[fused single-token K=4] d_model={}, d_ff={}, E={}, K={}",
+ d_model, d_ff, e, k
);
- // Verify correctness with tight tolerance
- let tolerance = 0.01;
+ let x: Vec<bf16> = (0..d_model)
+ .map(|_| bf16::from_f32(rng.random_range(-1.0..1.0)))
+ .collect();
+
+ let topk_ids: Vec<i32> = (0..k).map(|i| ((i * 2) % e) as i32).collect();
- let hidden_gpu = unsafe {
- std::slice::from_raw_parts(
- hidden_buf.contents() as *const f32,
- sum_k * d_ff,
+ let topk_probs: Vec<bf16> = {
+ let raw: Vec<f32> =
+ (0..k).map(|_| rng.random_range(0.1..1.0)).collect();
+ let sum: f32 = raw.iter().sum();
+ raw.iter().map(|p| bf16::from_f32(p / sum)).collect()
+ };
+
+ let w13_all: Vec<bf16> = (0..e * 2 * d_ff * d_model)
+ .map(|_| bf16::from_f32(rng.random_range(-0.05..0.05)))
+ .collect();
+ let w2_all: Vec<bf16> = (0..e * d_ff * d_model)
+ .map(|_| bf16::from_f32(rng.random_range(-0.05..0.05)))
+ .collect();
+ let up_biases: Vec<bf16> = (0..e * 2 * d_ff)
+ .map(|_| bf16::from_f32(rng.random_range(-0.01..0.01)))
+ .collect();
+ let down_biases: Vec<bf16> = (0..e * d_model)
+ .map(|_| bf16::from_f32(rng.random_range(-0.01..0.01)))
+ .collect();
+
+ let gating_code = 2u32;
+ let silu_alpha = 1.0f32;
+ let y_expected = cpu_moe_reference(
+ &x,
+ &topk_ids,
+ &topk_probs,
+ &w13_all,
+ &w2_all,
+ &up_biases,
+ &down_biases,
+ 1, // T=1 for single token
+ d_model,
+ d_ff,
+ k,
+ gating_code,
+ silu_alpha,
+ f32::NEG_INFINITY,
+ f32::INFINITY,
+ f32::NEG_INFINITY,
+ f32::INFINITY,
+ );
+
+ let x_buf = alloc_buffer_with_data(&ctx, &x);
+ let topk_ids_buf = alloc_buffer_with_data(&ctx, &topk_ids);
+ let topk_probs_buf = alloc_buffer_with_data(&ctx, &topk_probs);
+ let w13_buf = alloc_buffer_with_data(&ctx, &w13_all);
+ let w2_buf = alloc_buffer_with_data(&ctx, &w2_all);
+ let up_biases_buf = alloc_buffer_with_data(&ctx, &up_biases);
+ let down_biases_buf = alloc_buffer_with_data(&ctx, &down_biases);
+ let hidden_buf = alloc_buffer::<f32>(&ctx, k * d_ff);
+ let y_buf = alloc_buffer::<bf16>(&ctx, d_model);
+
+ let fused_kernel =
+ MoeExpertsSingleDecodeKernel::new(&ctx).expect("fused kernel");
+ let cb = ctx.command_queue.new_command_buffer();
+ fused_kernel
+ .encode(
+ &cb,
+ MoeExpertsSingleDecodeArguments {
+ x: &x_buf,
+ topk_ids: &topk_ids_buf,
+ topk_probs: &topk_probs_buf,
+ w13_all: &w13_buf,
+ w2_all: &w2_buf,
+ up_biases: &up_biases_buf,
+ down_biases: &down_biases_buf,
+ hidden: &hidden_buf,
+ y: &y_buf,
+ d_model,
+ d_ff,
+ k,
+ gating_code,
+ silu_alpha,
+ gate_clip_min: f32::NEG_INFINITY,
+ gate_clip_max: f32::INFINITY,
+ up_clip_min: f32::NEG_INFINITY,
+ up_clip_max: f32::INFINITY,
+ data_type: KernelDataType::BFloat16,
+ },
)
+ .expect("fused encode");
+ cb.commit();
+ cb.wait_until_completed();
+
+ let y_gpu = unsafe {
+ std::slice::from_raw_parts(y_buf.contents() as *const bf16, d_model)
};
- let mut hidden_max_diff = 0.0f32;
- let mut hidden_max_idx = 0usize;
- for (idx, (&gpu, &cpu)) in
- hidden_gpu.iter().zip(hidden_expected.iter()).enumerate()
+ // Compute error metrics
+ let mut max_abs_error = 0.0f32;
+ let mut sum_abs_error = 0.0f32;
+ let mut max_idx = 0;
+
+ for (i, (&gpu_val, &cpu_val)) in
+ y_gpu.iter().zip(y_expected.iter()).enumerate()
{
- let diff = (gpu - cpu).abs();
- if diff > hidden_max_diff {
- hidden_max_diff = diff;
- hidden_max_idx = idx;
+ let abs_error = (f32::from(gpu_val) - f32::from(cpu_val)).abs();
+ sum_abs_error += abs_error;
+ if abs_error > max_abs_error {
+ max_abs_error = abs_error;
+ max_idx = i;
}
}
+ let mean_abs_error = sum_abs_error / d_model as f32;
- if hidden_max_diff > tolerance {
- let row = hidden_max_idx / d_ff;
- let col = hidden_max_idx % d_ff;
- eprintln!(
- "[2-pass prefill] hidden mismatch row={} col={} gpu={} cpu={} diff={}",
- row,
- col,
- hidden_gpu[hidden_max_idx],
- hidden_expected[hidden_max_idx],
- hidden_max_diff
- );
-
- for offset in 0..8 {
- let idx = row * d_ff + (col + offset).min(d_ff - 1);
- let g = hidden_gpu[idx];
- let c = hidden_expected[idx];
- eprintln!(
- " col {} -> gpu {:.6} cpu {:.6} diff {:.6}",
- (col + offset).min(d_ff - 1),
- g,
- c,
- (g - c).abs()
- );
- }
- } else {
- eprintln!(
- "[2-pass prefill] hidden max diff {:.6} within tolerance",
- hidden_max_diff
- );
- }
+ eprintln!(
+ "[fused single-token K=4] Max error: {:.6} at idx {} (GPU={:.6}, CPU={:.6}), Mean error: {:.6}",
+ max_abs_error,
+ max_idx,
+ f32::from(y_gpu[max_idx]),
+ f32::from(y_expected[max_idx]),
+ mean_abs_error
+ );
+ let tolerance = 0.02;
assert_bf16_close(
- output_gpu,
- &expected,
+ y_gpu,
+ &y_expected,
tolerance,
- "2-pass prefill output",
+ "fused single-token K=4 output",
);
- eprintln!("[2-pass prefill] ✓ PASSED (tolerance={:.4})", tolerance);
-}
-
-#[test]
-fn test_tile_infrastructure() {
- // Test that tile counts/scan/map are computed correctly
- // This is already tested in moe_tiles_test.rs, but we verify here
- // that the tiles match our expectations for expert workload distribution
-
- let offsets = vec![0, 5, 20, 35, 40];
- const BM: usize = 16;
-
- let tile_counts = cpu_tile_counts(&offsets, BM);
- let (tile_offsets, total_tiles) = cpu_tile_scan(&tile_counts);
-
- // Expert 0: 5 rows -> 1 tile
- // Expert 1: 15 rows -> 1 tile
- // Expert 2: 15 rows -> 1 tile
- // Expert 3: 5 rows -> 1 tile
- assert_eq!(tile_counts, vec![1, 1, 1, 1]);
- assert_eq!(tile_offsets, vec![0, 1, 2, 3, 4]);
- assert_eq!(total_tiles, 4);
-
- eprintln!("[tile infrastructure] ✓ PASSED");
+ eprintln!("[fused single-token K=4] PASSED (tolerance={:.4})", tolerance);
}