Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

dslify-moe-2pass-prefill (#197)

Aleksandr Golokozf23a7582b154
diff --git a/crates/uzu/src/backends/metal/encodable_block/moe_block/moe_block.rs b/crates/uzu/src/backends/metal/encodable_block/moe_block/moe_block.rs
index b776abea3..e703e2b62 100644
--- a/crates/uzu/src/backends/metal/encodable_block/moe_block/moe_block.rs
+++ b/crates/uzu/src/backends/metal/encodable_block/moe_block/moe_block.rs
@@ -17,8 +17,8 @@ use crate::{
MoeGatherKernels,
dsl::{MoeCountsOffsetsFusedMetalKernel, MoeFinalizeMetalKernel},
moe::{
- MoeBlockBasesArguments, MoeExpertsTwoPassArguments, MoeExpertsTwoPassDecodeKernels,
- MoeExpertsTwoPassPrefillKernel, MoeGatherArguments, MoeRouterTopKArguments, MoeRouterTopKKernel,
+ MoeBlockBasesArguments, MoeExpertsTwoPassArguments, MoeExpertsTwoPassDecodeBlock,
+ MoeExpertsTwoPassPrefillBlock, MoeGatherArguments, MoeRouterTopKArguments, MoeRouterTopKKernel,
MoeScatterArguments, MoeScatterKernels, MoeScatterWithMapArguments,
},
},
@@ -44,8 +44,8 @@ pub struct MoeBlock {
counts_offsets_kernel: MoeCountsOffsetsFusedMetalKernel,
scatter_kernels: MoeScatterKernels,
gather_kernels: MoeGatherKernels,
- experts_two_pass_decode_kernel: MoeExpertsTwoPassDecodeKernels,
- experts_two_pass_prefill_kernel: MoeExpertsTwoPassPrefillKernel,
+ experts_two_pass_decode_kernel: MoeExpertsTwoPassDecodeBlock,
+ experts_two_pass_prefill_kernel: MoeExpertsTwoPassPrefillBlock,
finalize_kernel: MoeFinalizeMetalKernel,
moe_config: MixtureOfExpertsConfig,
model_dim: usize,
@@ -114,10 +114,10 @@ impl MoeBlock {
.map_err(|e| crate::backends::metal::MetalError::Generic(format!("Scatter kernels error: {:?}", e)))?;
let gather_kernel = MoeGatherKernels::new(context)
.map_err(|e| crate::backends::metal::MetalError::Generic(format!("Gather kernel error: {:?}", e)))?;
- let experts_two_pass_decode_kernel = MoeExpertsTwoPassDecodeKernels::new(context).map_err(|e| {
+ let experts_two_pass_decode_kernel = MoeExpertsTwoPassDecodeBlock::new(context).map_err(|e| {
crate::backends::metal::MetalError::Generic(format!("Experts two-pass decode kernel error: {:?}", e))
})?;
- let experts_two_pass_prefill_kernel = MoeExpertsTwoPassPrefillKernel::new(context).map_err(|e| {
+ let experts_two_pass_prefill_kernel = MoeExpertsTwoPassPrefillBlock::new(context).map_err(|e| {
crate::backends::metal::MetalError::Generic(format!("Experts two-pass prefill kernel error: {:?}", e))
})?;
let finalize_kernel = MoeFinalizeMetalKernel::new(context, data_type)
@@ -442,39 +442,35 @@ impl EncodableBlock<Metal> for MoeBlock {
let total_rows = suffix_length * k;
let num_tiles_k = ((self.hidden_dim + k_tile - 1) / k_tile) as u32;
- self.experts_two_pass_prefill_kernel
- .encode(
- root,
- MoeExpertsTwoPassArguments {
- x_perm_buffer: &x_perm_buf,
- expert_offsets: &offsets_buf,
- row_expert_map: &row_expert_map_buf,
- hidden_buffer: &hidden_buf,
- output_buffer: &y_partial_buf,
- w13_all: &self.shared_weights.w13_buf,
- w2_all: &self.shared_weights.w2_buf,
- up_biases: &self.shared_weights.up_biases_buf,
- down_biases: &self.shared_weights.down_biases_buf,
- tile_counts: &tile_counts_buf,
- tile_offsets: &tile_offsets_buf,
- tile_map: &tile_map_buf,
- total_tiles: &total_tiles_buf,
- dispatch_args: &dispatch_args_buf,
- total_rows,
- d_model: self.model_dim,
- d_ff: self.hidden_dim,
- e,
- num_tiles_k,
- gating_code,
- gate_clip_min,
- gate_clip_max,
- up_clip_min,
- up_clip_max,
- silu_alpha,
- data_type: self.data_type,
- },
- )
- .expect("MoE experts two-pass prefill failed");
+ let args = MoeExpertsTwoPassArguments {
+ x_perm_buffer: &x_perm_buf,
+ expert_offsets: &offsets_buf,
+ row_expert_map: &row_expert_map_buf,
+ hidden_buffer: &hidden_buf,
+ output_buffer: &y_partial_buf,
+ w13_all: &self.shared_weights.w13_buf,
+ w2_all: &self.shared_weights.w2_buf,
+ up_biases: &self.shared_weights.up_biases_buf,
+ down_biases: &self.shared_weights.down_biases_buf,
+ tile_counts: &tile_counts_buf,
+ tile_offsets: &tile_offsets_buf,
+ tile_map: &tile_map_buf,
+ total_tiles: &total_tiles_buf,
+ dispatch_args: &dispatch_args_buf,
+ total_rows,
+ d_model: self.model_dim,
+ d_ff: self.hidden_dim,
+ e,
+ num_tiles_k,
+ gating_code,
+ gate_clip_min,
+ gate_clip_max,
+ up_clip_min,
+ up_clip_max,
+ silu_alpha,
+ data_type: self.data_type,
+ };
+ self.experts_two_pass_prefill_kernel.encode(root, &args);
}
{
diff --git a/crates/uzu/src/backends/metal/kernel/mod.rs b/crates/uzu/src/backends/metal/kernel/mod.rs
index 6f02adf2c..daae0807a 100644
--- a/crates/uzu/src/backends/metal/kernel/mod.rs
+++ b/crates/uzu/src/backends/metal/kernel/mod.rs
@@ -10,10 +10,9 @@ pub mod quant_matmul;
pub use matmul::{MatmulArguments, MatmulKernel};
pub use moe::{
- MoeBlockBasesArguments, MoeExpertsArguments, MoeExpertsError, MoeExpertsTwoPassArguments,
- MoeExpertsTwoPassDecodeKernels, MoeExpertsTwoPassPrefillKernel, MoeGatherArguments, MoeGatherKernels,
- MoeRouterTopKArguments, MoeRouterTopKKernel, MoeScatterArguments, MoeScatterError, MoeScatterKernels,
- MoeScatterWithMapArguments,
+ MoeBlockBasesArguments, MoeExpertsTwoPassArguments, MoeExpertsTwoPassDecodeBlock, MoeExpertsTwoPassPrefillBlock,
+ MoeGatherArguments, MoeGatherKernels, MoeRouterTopKArguments, MoeRouterTopKKernel, MoeScatterArguments,
+ MoeScatterKernels, MoeScatterWithMapArguments,
};
impl MatmulKernels for dsl::MetalKernels {
diff --git a/crates/uzu/src/backends/metal/kernel/moe/experts.rs b/crates/uzu/src/backends/metal/kernel/moe/experts.rs
deleted file mode 100644
index 19b669744..000000000
--- a/crates/uzu/src/backends/metal/kernel/moe/experts.rs
+++ /dev/null
@@ -1,244 +0,0 @@
-use std::{mem::size_of, ptr::NonNull};
-
-use metal::{
- MTLBlitCommandEncoderExt, MTLBuffer, MTLCommandBuffer, MTLCommandEncoder, MTLComputeCommandEncoder,
- MTLComputePipelineState, MTLFunctionConstantValues, MTLSize,
-};
-use objc2::{rc::Retained, runtime::ProtocolObject};
-
-use super::{dtype_index, dtype_suffix};
-use crate::{
- DataType,
- backends::metal::{
- FunctionConstantValuesSetValue, MetalContext, MetalError,
- kernel::moe::tiles_map::{
- MoeTileCountsArguments, MoeTileDispatchArguments, MoeTileMapBuildArguments, MoeTileMapKernels,
- MoeTileScanArguments,
- },
- },
-};
-
-#[derive(Debug, thiserror::Error)]
-pub enum MoeScatterError {
- #[error("Metal error: {0}")]
- MetalError(#[from] MetalError),
-}
-#[derive(Debug, thiserror::Error)]
-pub enum MoeExpertsError {
- #[error("Metal error: {0}")]
- MetalError(#[from] MetalError),
- #[error("{0}")]
- Generic(String),
-}
-
-#[derive(Debug)]
-pub struct MoeExpertsArguments<'a> {
- pub x_perm_buffer: &'a ProtocolObject<dyn MTLBuffer>, // [sum_k, d_model] - permuted input
- pub expert_offsets: &'a ProtocolObject<dyn MTLBuffer>, // [E+1] - expert segment offsets
- pub w13_all: &'a ProtocolObject<dyn MTLBuffer>, // [E, 2*d_ff, d_model] - transposed up projection weights
- pub w2_all: &'a ProtocolObject<dyn MTLBuffer>, // [E, d_model, d_ff] - transposed down projection weights
- pub y_partial: &'a ProtocolObject<dyn MTLBuffer>, // [sum_k, d_model] - output buffer
- pub up_biases: &'a ProtocolObject<dyn MTLBuffer>, // [E, 2*d_ff] - up projection biases
- pub down_biases: &'a ProtocolObject<dyn MTLBuffer>, // [E, d_model] - down projection biases
- pub tile_counts: &'a ProtocolObject<dyn MTLBuffer>, // [E]
- pub tile_row_offsets: &'a ProtocolObject<dyn MTLBuffer>, // [E+1]
- pub tile_map: &'a ProtocolObject<dyn MTLBuffer>, // [max_tiles * 3]
- pub total_tiles: &'a ProtocolObject<dyn MTLBuffer>, // [2]
- pub dispatch_args: &'a ProtocolObject<dyn MTLBuffer>, // [3]
- pub num_tiles_n: usize,
- pub t: usize,
- pub d_model: usize,
- pub d_ff: usize,
- pub e: usize,
- pub k: usize,
- pub gating_code: u32,
- pub gate_clip_min: f32,
- pub gate_clip_max: f32,
- pub up_clip_min: f32,
- pub up_clip_max: f32,
- pub silu_alpha: f32,
- pub data_type: DataType,
-}
-
-pub struct MoeExpertsTwoPassPrefillKernel {
- tile_map: MoeTileMapKernels,
- pass_a_indirect: Vec<Vec<Retained<ProtocolObject<dyn MTLComputePipelineState>>>>, // [gate][dtype]
- pass_b_indirect: Vec<Retained<ProtocolObject<dyn MTLComputePipelineState>>>, // [dtype]
-}
-
-#[derive(Debug)]
-pub struct MoeExpertsTwoPassArguments<'a> {
- pub x_perm_buffer: &'a ProtocolObject<dyn MTLBuffer>,
- pub expert_offsets: &'a ProtocolObject<dyn MTLBuffer>,
- pub row_expert_map: &'a ProtocolObject<dyn MTLBuffer>,
- pub hidden_buffer: &'a ProtocolObject<dyn MTLBuffer>,
- pub output_buffer: &'a ProtocolObject<dyn MTLBuffer>,
- pub w13_all: &'a ProtocolObject<dyn MTLBuffer>,
- pub w2_all: &'a ProtocolObject<dyn MTLBuffer>,
- pub up_biases: &'a ProtocolObject<dyn MTLBuffer>,
- pub down_biases: &'a ProtocolObject<dyn MTLBuffer>,
- pub tile_counts: &'a ProtocolObject<dyn MTLBuffer>,
- pub tile_offsets: &'a ProtocolObject<dyn MTLBuffer>,
- pub tile_map: &'a ProtocolObject<dyn MTLBuffer>,
- pub total_tiles: &'a ProtocolObject<dyn MTLBuffer>,
- pub dispatch_args: &'a ProtocolObject<dyn MTLBuffer>,
- pub total_rows: usize,
- pub d_model: usize,
- pub d_ff: usize,
- pub e: usize,
- pub num_tiles_k: u32,
- pub gating_code: u32,
- pub gate_clip_min: f32,
- pub gate_clip_max: f32,
- pub up_clip_min: f32,
- pub up_clip_max: f32,
- pub silu_alpha: f32,
- pub data_type: DataType,
-}
-
-impl MoeExpertsTwoPassPrefillKernel {
- pub fn new(ctx: &MetalContext) -> Result<Self, MoeExpertsError> {
- let dtypes = [DataType::F16, DataType::BF16, DataType::F32];
- let mut pass_a_indirect = vec![Vec::with_capacity(dtypes.len()); 4];
- for gate in 0u32..4u32 {
- for dtype in &dtypes {
- let dtype_suffix = dtype_suffix(*dtype);
- let fcv = MTLFunctionConstantValues::new();
- fcv.set_value(&gate, 30);
- let kernel_name = format!("moe_two_pass_prefill_pass_a_indirect_{}", dtype_suffix);
- let cache_key = format!("{}_gate_{}", kernel_name, gate);
- pass_a_indirect[gate as usize].push(ctx.compute_pipeline_state_cached(
- &cache_key,
- &kernel_name,
- Some(&fcv),
- )?);
- }
- }
- let mut pass_b_indirect = Vec::with_capacity(dtypes.len());
- for dtype in &dtypes {
- let dtype_suffix = dtype_suffix(*dtype);
- let kernel_name = format!("moe_two_pass_prefill_pass_b_indirect_{}", dtype_suffix);
- pass_b_indirect.push(ctx.compute_pipeline_state(&kernel_name, None)?);
- }
- Ok(Self {
- tile_map: MoeTileMapKernels::new(ctx)?,
- pass_a_indirect,
- pass_b_indirect,
- })
- }
-
- pub fn encode(
- &self,
- command_buffer: &Retained<ProtocolObject<dyn MTLCommandBuffer>>,
- args: MoeExpertsTwoPassArguments,
- ) -> Result<(), MoeExpertsError> {
- if args.total_rows == 0 {
- return Ok(());
- }
- let gate_idx = (args.gating_code.min(3)) as usize;
- let dtype_idx = dtype_index(args.data_type);
- let hidden_bytes = args.total_rows * args.d_ff * args.data_type.size_in_bytes();
- let blit_encoder = command_buffer.new_blit_command_encoder().expect("Failed to create blit command encoder");
- blit_encoder.fill_buffer_range_value(args.hidden_buffer, 0..hidden_bytes, 0);
- blit_encoder.end_encoding();
- self.tile_map.encode_counts(
- command_buffer,
- &MoeTileCountsArguments {
- offsets_buffer: args.expert_offsets,
- tile_counts_buffer: args.tile_counts,
- e: args.e,
- },
- );
- self.tile_map.encode_scan(
- command_buffer,
- &MoeTileScanArguments {
- tile_counts_buffer: args.tile_counts,
- tile_offsets_buffer: args.tile_offsets,
- total_tiles_buffer: args.total_tiles,
- e: args.e,
- },
- );
- self.tile_map.encode_build_map(
- command_buffer,
- &MoeTileMapBuildArguments {
- expert_offsets: args.expert_offsets,
- tile_offsets: args.tile_offsets,
- tile_counts: args.tile_counts,
- tile_map: args.tile_map,
- e: args.e,
- },
- );
- let d_model_u32 = args.d_model as u32;
- let d_ff_u32 = args.d_ff as u32;
- let e_u32 = args.e as u32;
- const COL_TILE_FF: u32 = 32; // Must match PASSA_BN in Metal kernel
- const COL_TILE_MODEL: u32 = 64;
- let n_tiles_ff = if d_ff_u32 == 0 {
- 0
- } else {
- (d_ff_u32 + COL_TILE_FF - 1) / COL_TILE_FF
- };
- let n_tiles_model = if d_model_u32 == 0 {
- 0
- } else {
- (d_model_u32 + COL_TILE_MODEL - 1) / COL_TILE_MODEL
- };
- self.tile_map.encode_dispatch_args(
- command_buffer,
- &MoeTileDispatchArguments {
- total_tiles: args.total_tiles,
- dispatch_args: args.dispatch_args,
- num_tiles_x: n_tiles_ff,
- },
- );
- let pass_a_pipeline = &self.pass_a_indirect[gate_idx][dtype_idx];
- let encoder_a = command_buffer.new_compute_command_encoder().expect("Failed to create compute command encoder");
- encoder_a.set_compute_pipeline_state(pass_a_pipeline);
- encoder_a.set_buffer(Some(args.x_perm_buffer), 0, 0);
- encoder_a.set_buffer(Some(args.expert_offsets), 0, 1);
- encoder_a.set_buffer(Some(args.w13_all), 0, 2);
- encoder_a.set_buffer(Some(args.up_biases), 0, 3);
- encoder_a.set_buffer(Some(args.hidden_buffer), 0, 4);
- unsafe {
- encoder_a.set_bytes(NonNull::new_unchecked(&d_model_u32 as *const _ as *mut _), size_of::<u32>(), 5);
- encoder_a.set_bytes(NonNull::new_unchecked(&d_ff_u32 as *const _ as *mut _), size_of::<u32>(), 6);
- encoder_a.set_bytes(NonNull::new_unchecked(&e_u32 as *const _ as *mut _), size_of::<u32>(), 7);
- encoder_a.set_bytes(NonNull::new_unchecked(&args.gate_clip_min as *const _ as *mut _), size_of::<f32>(), 8);
- encoder_a.set_bytes(NonNull::new_unchecked(&args.gate_clip_max as *const _ as *mut _), size_of::<f32>(), 9);
- encoder_a.set_bytes(NonNull::new_unchecked(&args.up_clip_min as *const _ as *mut _), size_of::<f32>(), 10);
- encoder_a.set_bytes(NonNull::new_unchecked(&args.up_clip_max as *const _ as *mut _), size_of::<f32>(), 11);
- encoder_a.set_bytes(NonNull::new_unchecked(&args.silu_alpha as *const _ as *mut _), size_of::<f32>(), 12);
- }
- encoder_a.set_buffer(Some(args.tile_map), 0, 13);
- // Match kernel config: WM=2, WN=2 => 4 SIMDgroups => 128 threads
- const SIMDGROUPS_PER_TG: u32 = 4;
- const THREADS_PER_TG: u32 = SIMDGROUPS_PER_TG * 32;
- encoder_a.dispatch_threadgroups_indirect(args.dispatch_args, 0, MTLSize::new(THREADS_PER_TG as usize, 1, 1));
- encoder_a.end_encoding();
- self.tile_map.encode_dispatch_args(
- command_buffer,
- &MoeTileDispatchArguments {
- total_tiles: args.total_tiles,
- dispatch_args: args.dispatch_args,
- num_tiles_x: n_tiles_model,
- },
- );
- let pass_b_pipeline = &self.pass_b_indirect[dtype_idx];
- let encoder_b = command_buffer.new_compute_command_encoder().expect("Failed to create compute command encoder");
- encoder_b.set_compute_pipeline_state(pass_b_pipeline);
- encoder_b.set_buffer(Some(args.hidden_buffer), 0, 0);
- encoder_b.set_buffer(Some(args.expert_offsets), 0, 1);
- encoder_b.set_buffer(Some(args.w2_all), 0, 2);
- encoder_b.set_buffer(Some(args.down_biases), 0, 3);
- encoder_b.set_buffer(Some(args.output_buffer), 0, 4);
- unsafe {
- encoder_b.set_bytes(NonNull::new_unchecked(&d_model_u32 as *const _ as *mut _), size_of::<u32>(), 5);
- encoder_b.set_bytes(NonNull::new_unchecked(&d_ff_u32 as *const _ as *mut _), size_of::<u32>(), 6);
- encoder_b.set_bytes(NonNull::new_unchecked(&e_u32 as *const _ as *mut _), size_of::<u32>(), 7);
- }
- encoder_b.set_buffer(Some(args.tile_map), 0, 8);
- encoder_b.dispatch_threadgroups_indirect(args.dispatch_args, 0, MTLSize::new(THREADS_PER_TG as usize, 1, 1));
- encoder_b.end_encoding();
- Ok(())
- }
-}
diff --git a/crates/uzu/src/backends/metal/kernel/moe/experts_two_pass_decode.rs b/crates/uzu/src/backends/metal/kernel/moe/experts_two_pass_decode.rs
index 4f1c3725e..8a07282f8 100644
--- a/crates/uzu/src/backends/metal/kernel/moe/experts_two_pass_decode.rs
+++ b/crates/uzu/src/backends/metal/kernel/moe/experts_two_pass_decode.rs
@@ -15,7 +15,7 @@ use crate::{
dsl::{MoeExpertsDecodeDownFused2DMetalKernel, MoeExpertsDecodePassAMetalKernel},
moe::{
MoePassARowMapArguments, MoePassATileBuildArguments, MoePassATileCountsArguments,
- MoePassATileDispatchArguments, MoePassATileKernels, MoePassATileScanArguments, dtype_index,
+ MoePassATileDispatchArguments, MoePassATileKernels, MoePassATileScanArguments,
},
},
},
@@ -24,13 +24,13 @@ use crate::{
static DTYPES: [DataType; 3] = [DataType::F16, DataType::BF16, DataType::F32];
-pub struct MoeExpertsTwoPassDecodeKernels {
+pub struct MoeExpertsTwoPassDecodeBlock {
pass_a_tile: MoePassATileKernels,
pass_a_indirect: Vec<Vec<MoeExpertsDecodePassAMetalKernel>>,
fused_down: Vec<MoeExpertsDecodeDownFused2DMetalKernel>,
}
-impl MoeExpertsTwoPassDecodeKernels {
+impl MoeExpertsTwoPassDecodeBlock {
pub fn new(ctx: &MetalContext) -> Result<Self, MetalError> {
let mut pass_a_indirect = vec![];
for gate in 0u32..4u32 {
@@ -116,7 +116,7 @@ impl MoeExpertsTwoPassDecodeKernels {
// pass a
let gate_idx = args.gating_code.min(3) as usize;
- let dtype_idx = dtype_index(args.data_type);
+ let dtype_idx = DTYPES.iter().position(|dtype| *dtype == args.data_type).unwrap();
let pass_a_encoder =
command_buffer.new_compute_command_encoder().expect("Failed to create compute command encoder");
let pass_a_kernel = &self.pass_a_indirect[gate_idx][dtype_idx];
diff --git a/crates/uzu/src/backends/metal/kernel/moe/experts_two_pass_prefill.metal b/crates/uzu/src/backends/metal/kernel/moe/experts_two_pass_prefill.metal
index f1818b697..9f28ba119 100644
--- a/crates/uzu/src/backends/metal/kernel/moe/experts_two_pass_prefill.metal
+++ b/crates/uzu/src/backends/metal/kernel/moe/experts_two_pass_prefill.metal
@@ -1,11 +1,8 @@
#include <metal_stdlib>
-#include <metal_simdgroup>
-#include <metal_simdgroup_matrix>
+#include "../definitions.metal"
#include "../quant_matmul/mma.h"
#include "moe_commons.h"
-using namespace metal;
-
// ------------------------ helpers ------------------------
static inline uint ceil_div(uint a, uint b) { return (a + b - 1u) / b; }
@@ -26,24 +23,6 @@ inline void store_vec4<float>(device float* dst, ulong base, float4 vals) {
*reinterpret_cast<device float4*>(dst + base) = vals;
}
-template <typename T>
-struct StageStorage {
- using type = float;
-};
-
-template <>
-struct StageStorage<half> {
- using type = half;
-};
-
-template <>
-struct StageStorage<bfloat> {
- using type = bfloat;
-};
-
-// 0=GELU(up), 1=SiLU(up), 2=SwiGLU(gate)*up, 3=GEGLU(gate)*up
-constant uint GATING_SEL [[function_constant(30)]];
-
// ------------------------ Pass A (X @ W13 → hidden) ------------------------
// Tile config (tuned for A-series; matches your prior defaults)
constant uint PASSA_BM = 16;
@@ -54,38 +33,50 @@ constant uint PASSA_SG_BN = 16;
constant uint PASSA_TG_PAD = 4; // Padding to avoid bank conflicts
template <typename T>
-inline void pass_a_impl(
- device const T* X_perm,
+VARIANTS(T, float, half, bfloat)
+KERNEL(MoeExpertsPrefillPassA)(
+ device const T* x_perm,
device const uint* expert_offsets,
- device const T* W13_all,
+ device const T* w13_all,
device const T* up_biases,
device float* hidden_out,
- uint d_model,
- uint d_ff,
- uint E,
- float gate_clip_min,
- float gate_clip_max,
- float up_clip_min,
- float up_clip_max,
- float silu_alpha,
- // work partition
- uint tile_seg_start,
- uint expert_idx,
- uint tile_m,
- uint tile_n,
- // per-thread info
- uint sg_id,
- uint3 threads_per_tg,
- uint3 local_tid,
- // TG storage
- threadgroup typename StageStorage<T>::type* Xs, // [BM,BK]
- threadgroup typename StageStorage<T>::type* Wk_up, // [BN,BK]
- threadgroup typename StageStorage<T>::type* Wk_gate, // [BN,BK]
- threadgroup float* bias_up, // [BN]
- threadgroup float* bias_gate // [BN]
+ constant uint& d_model,
+ constant uint& d_ff,
+ constant uint& e,
+ constant float& gate_clip_min,
+ constant float& gate_clip_max,
+ constant float& up_clip_min,
+ constant float& up_clip_max,
+ constant float& silu_alpha,
+ device const uint* tile_map,
+ threadgroup T Xs[PASSA_BM * (PASSA_BK + PASSA_TG_PAD)],
+ threadgroup T Wk_up[PASSA_BN * (PASSA_BK + PASSA_TG_PAD)],
+ threadgroup T Wk_gate[PASSA_BN * (PASSA_BK + PASSA_TG_PAD)],
+ threadgroup float bias_up[PASSA_BN],
+ threadgroup float bias_gate[PASSA_BN],
+ // 0=GELU(up), 1=SiLU(up), 2=SwiGLU(gate)*up, 3=GEGLU(gate)*up
+ const uint gating_sel SPECIALIZE,
+ const uint n_tile_idx GROUPS(INDIRECT),
+ const uint row_tile_idx GROUPS(INDIRECT),
+ const uint local_tid_x THREADS(128),
+ const Simd simd
) {
- using StageT = typename StageStorage<T>::type;
- using StageVec4 = metal::vec<StageT, 4>;
+ const uint base = row_tile_idx * 3u;
+ const uint expert_idx = tile_map[base + 0u];
+ if (expert_idx >= e)
+ return;
+
+ const uint3 threads_per_tg = {128, 1, 1};
+ const uint3 local_tid = {local_tid_x, 0, 0};
+ const uint sg_id = simd.group_idx;
+
+ constexpr uint ROW_TILE = PASSA_BM;
+ const uint tile_seg_start = tile_map[base + 1u];
+ const uint row_off_elems = tile_map[base + 2u];
+ const uint tile_m = row_off_elems / ROW_TILE;
+ const uint tile_n = n_tile_idx;
+
+ using StageVec4 = metal::vec<T, 4>;
constexpr uint Bm = PASSA_BM, Bn = PASSA_BN, Bk = PASSA_BK;
constexpr uint SgBm = PASSA_SG_BM, SgBn = PASSA_SG_BN;
@@ -93,9 +84,6 @@ inline void pass_a_impl(
constexpr uint X_LD = Bk + TG_PAD; // Padded stride for X
constexpr uint W_LD = Bk + TG_PAD; // Padded stride for weights
- if (expert_idx >= E)
- return;
-
const uint seg_start = expert_offsets[expert_idx];
const uint seg_end = expert_offsets[expert_idx + 1];
const uint seg_len = (seg_end > seg_start) ? (seg_end - seg_start) : 0u;
@@ -135,7 +123,7 @@ inline void pass_a_impl(
metal::simdgroup_float8x8 OutGate[TEMP];
for (uint i = 0; i < TEMP; ++i) {
OutUp[i] = metal::make_filled_simdgroup_matrix<float, 8, 8>(0.0f);
- if (GATING_SEL > 1u) {
+ if (gating_sel > 1u) {
OutGate[i] = metal::make_filled_simdgroup_matrix<float, 8, 8>(0.0f);
}
}
@@ -165,7 +153,7 @@ inline void pass_a_impl(
const uint c8 = i % vec_cols;
const uint col_base = c8 * vec_size;
- threadgroup StageT* dst = Xs + r * X_LD + col_base;
+ threadgroup T* dst = Xs + r * X_LD + col_base;
if (r < m_rows && col_base < valid_k) {
// Safe to read - at least first element is valid
@@ -174,14 +162,14 @@ inline void pass_a_impl(
UZU_PRAGMA_UNROLL
for (uint j = 0; j < vec_size; j++) {
dst[j] = (col_base + j < valid_k)
- ? StageT(float(X_perm[base + j]))
- : StageT(0.0f);
+ ? static_cast<T>(float(x_perm[base + j]))
+ : static_cast<T>(0.0f);
}
} else {
// Row out of bounds or entire vec8 chunk is out of K bounds
UZU_PRAGMA_UNROLL
for (uint j = 0; j < vec_size; j++) {
- dst[j] = StageT(0.0f);
+ dst[j] = static_cast<T>(0.0f);
}
}
}
@@ -204,8 +192,8 @@ inline void pass_a_impl(
const uint c8 = i % vec_cols;
const uint col_base = c8 * vec_size;
- threadgroup StageT* up_dst = Wk_up + n * W_LD + col_base;
- threadgroup StageT* gt_dst = Wk_gate + n * W_LD + col_base;
+ threadgroup T* up_dst = Wk_up + n * W_LD + col_base;
+ threadgroup T* gt_dst = Wk_gate + n * W_LD + col_base;
if (n < n_cols && col_base < valid_k) {
// Safe to read - at least first element is valid
@@ -219,21 +207,21 @@ inline void pass_a_impl(
UZU_PRAGMA_UNROLL
for (uint j = 0; j < vec_size; j++) {
up_dst[j] = (col_base + j < valid_k)
- ? StageT(float(W13_all[up_base + j]))
- : StageT(0.0f);
- if (GATING_SEL > 1u) {
+ ? static_cast<T>(float(w13_all[up_base + j]))
+ : static_cast<T>(0.0f);
+ if (gating_sel > 1u) {
gt_dst[j] = (col_base + j < valid_k)
- ? StageT(float(W13_all[gate_base + j]))
- : StageT(0.0f);
+ ? static_cast<T>(float(w13_all[gate_base + j]))
+ : static_cast<T>(0.0f);
}
}
} else {
// Row out of bounds or entire vec8 chunk is out of K bounds
UZU_PRAGMA_UNROLL
for (uint j = 0; j < vec_size; j++) {
- up_dst[j] = StageT(0.0f);
- if (GATING_SEL > 1u)
- gt_dst[j] = StageT(0.0f);
+ up_dst[j] = static_cast<T>(0.0f);
+ if (gating_sel > 1u)
+ gt_dst[j] = static_cast<T>(0.0f);
}
}
}
@@ -248,14 +236,14 @@ inline void pass_a_impl(
UZU_PRAGMA_UNROLL
for (uint m_sub = 0; m_sub < SgBm; m_sub += SG_TILE) {
const uint r_idx = m_sub / SG_TILE;
- metal::simdgroup_matrix<StageT, 8, 8> lhs_frag;
+ metal::simdgroup_matrix<T, 8, 8> lhs_frag;
simdgroup_load(lhs_frag, Xs, X_LD, ulong2(kk, row_sg_off + m_sub));
UZU_PRAGMA_UNROLL
for (uint n_sub = 0; n_sub < SgBn; n_sub += SG_TILE) {
const uint c_idx = n_sub / SG_TILE;
const uint tile = r_idx * (SgBn / SG_TILE) + c_idx;
- metal::simdgroup_matrix<StageT, 8, 8> rhs_up;
+ metal::simdgroup_matrix<T, 8, 8> rhs_up;
simdgroup_load(
rhs_up,
Wk_up,
@@ -270,8 +258,8 @@ inline void pass_a_impl(
OutUp[tile]
);
- if (GATING_SEL > 1u) {
- metal::simdgroup_matrix<StageT, 8, 8> rhs_gate;
+ if (gating_sel > 1u) {
+ metal::simdgroup_matrix<T, 8, 8> rhs_gate;
simdgroup_load(
rhs_gate,
Wk_gate,
@@ -297,7 +285,7 @@ inline void pass_a_impl(
const uint c_global = col_tg_off + c_local;
bias_up[c_local] =
(c_global < d_ff) ? float(up_biases[bias_base + c_global]) : 0.0f;
- if (GATING_SEL > 1u) {
+ if (gating_sel > 1u) {
bias_gate[c_local] = (c_global < d_ff)
? float(up_biases[bias_base + d_ff + c_global])
: 0.0f;
@@ -336,7 +324,7 @@ inline void pass_a_impl(
const auto up_frag = OutUp[tile].thread_elements();
float gate_frag_0 = 0.0f;
float gate_frag_1 = 0.0f;
- if (GATING_SEL > 1u) {
+ if (gating_sel > 1u) {
const auto gate_frag = OutGate[tile].thread_elements();
gate_frag_0 = gate_frag[0];
gate_frag_1 = gate_frag[1];
@@ -350,16 +338,16 @@ inline void pass_a_impl(
float up_v =
clamp(up_frag[0] + bias_up[col0], up_clip_min, up_clip_max);
float out_val;
- if (GATING_SEL <= 1u) {
+ if (gating_sel <= 1u) {
out_val =
- (GATING_SEL == 0u) ? gelu_approx(up_v) : silu(up_v, silu_alpha);
+ (gating_sel == 0u) ? gelu_approx(up_v) : silu(up_v, silu_alpha);
} else {
float gate_v = clamp(
gate_frag_0 + bias_gate[col0],
gate_clip_min,
gate_clip_max
);
- const float gate_act = (GATING_SEL == 2u) ? silu(gate_v, silu_alpha)
+ const float gate_act = (gating_sel == 2u) ? silu(gate_v, silu_alpha)
: gelu_approx(gate_v);
out_val = gate_act * up_v;
}
@@ -371,16 +359,16 @@ inline void pass_a_impl(
float up_v =
clamp(up_frag[1] + bias_up[col1], up_clip_min, up_clip_max);
float out_val;
- if (GATING_SEL <= 1u) {
+ if (gating_sel <= 1u) {
out_val =
- (GATING_SEL == 0u) ? gelu_approx(up_v) : silu(up_v, silu_alpha);
+ (gating_sel == 0u) ? gelu_approx(up_v) : silu(up_v, silu_alpha);
} else {
float gate_v = clamp(
gate_frag_1 + bias_gate[col1],
gate_clip_min,
gate_clip_max
);
- const float gate_act = (GATING_SEL == 2u) ? silu(gate_v, silu_alpha)
+ const float gate_act = (gating_sel == 2u) ? silu(gate_v, silu_alpha)
: gelu_approx(gate_v);
out_val = gate_act * up_v;
}
@@ -391,132 +379,6 @@ inline void pass_a_impl(
}
}
-// Indirect variant (consumes [expert, seg_start, tile_row_offset] triples)
-template <typename T>
-inline void pass_a_indirect_impl(
- device const T* X_perm,
- device const uint* expert_offsets,
- device const T* W13_all,
- device const T* up_biases,
- device float* hidden_out,
- device const uint* tile_map,
- uint d_model,
- uint d_ff,
- uint E,
- float gate_clip_min,
- float gate_clip_max,
- float up_clip_min,
- float up_clip_max,
- float silu_alpha,
- uint row_tile_idx,
- uint n_tile_idx,
- uint sg_id,
- uint3 threads_per_tg,
- uint3 local_tid,
- threadgroup typename StageStorage<T>::type* Xs,
- threadgroup typename StageStorage<T>::type* Wk_up,
- threadgroup typename StageStorage<T>::type* Wk_gate,
- threadgroup float* bias_up,
- threadgroup float* bias_gate
-) {
- constexpr uint ROW_TILE = PASSA_BM;
- const uint base = row_tile_idx * 3u;
- const uint expert_idx = tile_map[base + 0u];
- if (expert_idx >= E)
- return;
- const uint tile_seg_start = tile_map[base + 1u];
- const uint row_off_elems = tile_map[base + 2u];
- const uint tile_m = row_off_elems / ROW_TILE;
-
- pass_a_impl<T>(
- X_perm,
- expert_offsets,
- W13_all,
- up_biases,
- hidden_out,
- d_model,
- d_ff,
- E,
- gate_clip_min,
- gate_clip_max,
- up_clip_min,
- up_clip_max,
- silu_alpha,
- tile_seg_start,
- expert_idx,
- tile_m,
- n_tile_idx,
- sg_id,
- threads_per_tg,
- local_tid,
- Xs,
- Wk_up,
- Wk_gate,
- bias_up,
- bias_gate
- );
-}
-
-#define MOE_PASS_A_INDIRECT_KERNEL(DTYPE, SUFFIX) \
- [[max_total_threads_per_threadgroup(128)]] \
- kernel void moe_two_pass_prefill_pass_a_indirect_##SUFFIX( \
- device const DTYPE* X_perm [[buffer(0)]], \
- device const uint* expert_offsets [[buffer(1)]], \
- device const DTYPE* W13_all [[buffer(2)]], \
- device const DTYPE* up_biases [[buffer(3)]], \
- device float* hidden_out [[buffer(4)]], \
- constant uint& d_model [[buffer(5)]], \
- constant uint& d_ff [[buffer(6)]], \
- constant uint& E [[buffer(7)]], \
- constant float& gate_clip_min [[buffer(8)]], \
- constant float& gate_clip_max [[buffer(9)]], \
- constant float& up_clip_min [[buffer(10)]], \
- constant float& up_clip_max [[buffer(11)]], \
- constant float& silu_alpha [[buffer(12)]], \
- device const uint* tile_map [[buffer(13)]], \
- uint sg_id [[simdgroup_index_in_threadgroup]], \
- uint3 threads_per_tg [[threads_per_threadgroup]], \
- uint3 tg_pos [[threadgroup_position_in_grid]], \
- uint3 local_tid [[thread_position_in_threadgroup]] \
- ) { \
- using StageT = typename StageStorage<DTYPE>::type; \
- threadgroup StageT Xs[PASSA_BM * (PASSA_BK + PASSA_TG_PAD)]; \
- threadgroup StageT Wk_up[PASSA_BN * (PASSA_BK + PASSA_TG_PAD)]; \
- threadgroup StageT Wk_gate[PASSA_BN * (PASSA_BK + PASSA_TG_PAD)]; \
- threadgroup float bias_up[PASSA_BN]; \
- threadgroup float bias_gate[PASSA_BN]; \
- pass_a_indirect_impl<DTYPE>( \
- X_perm, \
- expert_offsets, \
- W13_all, \
- up_biases, \
- hidden_out, \
- tile_map, \
- d_model, \
- d_ff, \
- E, \
- gate_clip_min, \
- gate_clip_max, \
- up_clip_min, \
- up_clip_max, \
- silu_alpha, \
- tg_pos.y, \
- tg_pos.x, \
- sg_id, \
- threads_per_tg, \
- local_tid, \
- Xs, \
- Wk_up, \
- Wk_gate, \
- bias_up, \
- bias_gate \
- ); \
- }
-
-MOE_PASS_A_INDIRECT_KERNEL(bfloat, bf16)
-MOE_PASS_A_INDIRECT_KERNEL(half, f16)
-MOE_PASS_A_INDIRECT_KERNEL(float, f32)
-
// ------------------------ Pass B (hidden @ W2 → output)
// ------------------------ Optimized config: 16x64 output tile, 4 simdgroups
// (128 threads)
@@ -528,26 +390,37 @@ constant uint PASSB_SG_BN = 32;
constant uint PASSB_TG_PAD = 4; // Padding to avoid bank conflicts
template <typename T>
-inline void pass_b_impl(
+VARIANTS(T, float, half, bfloat)
+KERNEL(MoeExpertsPrefillPassB)(
device const float* hidden,
device const uint* expert_offsets,
- device const T* W2_all,
+ device const T* w2_all,
device const T* down_biases,
device T* output,
- uint d_model,
- uint d_ff,
- uint E,
- uint expert_idx,
- uint tile_m,
- uint tile_n,
- uint sg_id,
- uint simd_lid,
- uint lin,
- threadgroup float* Hs,
- threadgroup typename StageStorage<T>::type* Wk,
- threadgroup float* bias_tile
+ constant uint& d_model,
+ constant uint& d_ff,
+ constant uint& e,
+ device const uint* tile_map,
+ threadgroup float Hs[PASSB_BM * (PASSB_BK + PASSB_TG_PAD)],
+ threadgroup T Wk[PASSB_BN * (PASSB_BK + PASSB_TG_PAD)],
+ threadgroup float bias_tile[PASSB_BN],
+ const Simd simd,
+ const uint n_tile_idx GROUPS(INDIRECT),
+ const uint row_tile_idx GROUPS(INDIRECT),
+ const uint lin THREADS(128)
) {
- using StageT = typename StageStorage<T>::type;
+ const uint base = row_tile_idx * 3u;
+ const uint expert_idx = tile_map[base + 0u];
+ if (expert_idx >= e)
+ return;
+
+ const uint simd_lid = simd.lane_idx;
+ const uint sg_id = simd.group_idx;
+
+ constexpr uint ROW_TILE = PASSB_BM;
+ const uint row_off_elems = tile_map[base + 2u];
+ const uint tile_m = row_off_elems / ROW_TILE;
+ const uint tile_n = n_tile_idx;
constexpr uint Bm = PASSB_BM, Bn = PASSB_BN, Bk = PASSB_BK;
constexpr uint SgBm = PASSB_SG_BM, SgBn = PASSB_SG_BN;
@@ -557,9 +430,6 @@ inline void pass_b_impl(
constexpr uint SG_TILE = 8;
constexpr uint TGP_SIZE = 128; // 4 simdgroups × 32 threads
- if (expert_idx >= E)
- return;
-
const uint seg_start = expert_offsets[expert_idx];
const uint seg_end = expert_offsets[expert_idx + 1];
const uint seg_len = (seg_end > seg_start) ? (seg_end - seg_start) : 0u;
@@ -640,22 +510,22 @@ inline void pass_b_impl(
{
for (uint pass = 0; pass < 4; pass++) {
const uint row = w_bi + pass * W_TROWS;
- threadgroup StageT* my_dst = Wk + row * W_LD + w_bj;
+ threadgroup T* my_dst = Wk + row * W_LD + w_bj;
if (row < n_cols && w_bj < valid_k) {
// Safe to read - at least first element is valid
const uint gcol = col_tg_off + row;
- device const T* my_src = W2_all + w2_expert_base +
+ device const T* my_src = w2_all + w2_expert_base +
(ulong)gcol * (ulong)d_ff +
(ulong)(k_off + w_bj);
UZU_PRAGMA_UNROLL
for (uint j = 0; j < W_VEC; j++) {
- my_dst[j] =
- (w_bj + j < valid_k) ? StageT(float(my_src[j])) : StageT(0.0f);
+ my_dst[j] = (w_bj + j < valid_k) ? static_cast<T>(float(my_src[j]))
+ : static_cast<T>(0.0f);
}
} else {
UZU_PRAGMA_UNROLL
for (uint j = 0; j < W_VEC; j++) {
- my_dst[j] = StageT(0.0f);
+ my_dst[j] = static_cast<T>(0.0f);
}
}
}
@@ -671,7 +541,7 @@ inline void pass_b_impl(
simdgroup_load(lhs, Hs, H_LD, ulong2(kk, row_sg_off));
// Load 4 RHS fragments (cols)
- metal::simdgroup_matrix<StageT, 8, 8> rhs0, rhs1, rhs2, rhs3;
+ metal::simdgroup_matrix<T, 8, 8> rhs0, rhs1, rhs2, rhs3;
simdgroup_load(rhs0, Wk, W_LD, ulong2(kk, col_sg_off), true);
simdgroup_load(rhs1, Wk, W_LD, ulong2(kk, col_sg_off + 8), true);
simdgroup_load(rhs2, Wk, W_LD, ulong2(kk, col_sg_off + 16), true);
@@ -720,101 +590,4 @@ inline void pass_b_impl(
output[out_row * (ulong)d_model + out_col] = T(val);
}
}
-}
-
-// Indirect variant
-template <typename T>
-inline void pass_b_indirect_impl(
- device const float* hidden,
- device const uint* expert_offsets,
- device const T* W2_all,
- device const T* down_biases,
- device T* output,
- device const uint* tile_map,
- uint d_model,
- uint d_ff,
- uint E,
- uint row_tile_idx,
- uint n_tile_idx,
- uint sg_id,
- uint simd_lid,
- uint lin,
- threadgroup float* Hs,
- threadgroup typename StageStorage<T>::type* Wk,
- threadgroup float* bias_tile
-) {
- constexpr uint ROW_TILE = PASSB_BM;
- const uint base = row_tile_idx * 3u;
- const uint expert_idx = tile_map[base + 0u];
- if (expert_idx >= E)
- return;
- const uint row_off_elems = tile_map[base + 2u];
- const uint tile_m = row_off_elems / ROW_TILE;
-
- pass_b_impl<T>(
- hidden,
- expert_offsets,
- W2_all,
- down_biases,
- output,
- d_model,
- d_ff,
- E,
- expert_idx,
- tile_m,
- n_tile_idx,
- sg_id,
- simd_lid,
- lin,
- Hs,
- Wk,
- bias_tile
- );
-}
-
-#define MOE_PASS_B_INDIRECT_KERNEL(DTYPE, SUFFIX) \
- [[max_total_threads_per_threadgroup(128)]] \
- kernel void moe_two_pass_prefill_pass_b_indirect_##SUFFIX( \
- device const float* hidden [[buffer(0)]], \
- device const uint* expert_offsets [[buffer(1)]], \
- device const DTYPE* W2_all [[buffer(2)]], \
- device const DTYPE* down_biases [[buffer(3)]], \
- device DTYPE* output [[buffer(4)]], \
- constant uint& d_model [[buffer(5)]], \
- constant uint& d_ff [[buffer(6)]], \
- constant uint& E [[buffer(7)]], \
- device const uint* tile_map [[buffer(8)]], \
- uint sg_id [[simdgroup_index_in_threadgroup]], \
- uint simd_lid [[thread_index_in_simdgroup]], \
- uint3 tg_pos [[threadgroup_position_in_grid]], \
- uint3 local_tid [[thread_position_in_threadgroup]] \
- ) { \
- using StageT = typename StageStorage<DTYPE>::type; \
- threadgroup float Hs[PASSB_BM * (PASSB_BK + PASSB_TG_PAD)]; \
- threadgroup StageT Wk[PASSB_BN * (PASSB_BK + PASSB_TG_PAD)]; \
- threadgroup float bias[PASSB_BN]; \
- const uint lin = local_tid.x; \
- pass_b_indirect_impl<DTYPE>( \
- hidden, \
- expert_offsets, \
- W2_all, \
- down_biases, \
- output, \
- tile_map, \
- d_model, \
- d_ff, \
- E, \
- tg_pos.y, \
- tg_pos.x, \
- sg_id, \
- simd_lid, \
- lin, \
- Hs, \
- Wk, \
- bias \
- ); \
- }
-
-MOE_PASS_B_INDIRECT_KERNEL(bfloat, bf16)
-MOE_PASS_B_INDIRECT_KERNEL(half, f16)
-MOE_PASS_B_INDIRECT_KERNEL(float, f32)
+}
\ No newline at end of file
diff --git a/crates/uzu/src/backends/metal/kernel/moe/experts_two_pass_prefill.rs b/crates/uzu/src/backends/metal/kernel/moe/experts_two_pass_prefill.rs
new file mode 100644
index 000000000..a1e28cf39
--- /dev/null
+++ b/crates/uzu/src/backends/metal/kernel/moe/experts_two_pass_prefill.rs
@@ -0,0 +1,203 @@
+use metal::{MTLBlitCommandEncoderExt, MTLBuffer, MTLCommandBuffer, MTLCommandEncoder};
+use objc2::{
+ __framework_prelude::{ProtocolObject, Retained},
+ Message,
+};
+
+use crate::{
+ DataType,
+ backends::{
+ common::kernel::{MoeExpertsPrefillPassAKernel, MoeExpertsPrefillPassBKernel},
+ metal::{
+ MetalContext, MetalError,
+ kernel::{
+ dsl::{MoeExpertsPrefillPassAMetalKernel, MoeExpertsPrefillPassBMetalKernel},
+ moe::tiles_map::{
+ MoeTileCountsArguments, MoeTileDispatchArguments, MoeTileMapBuildArguments, MoeTileMapKernels,
+ MoeTileScanArguments,
+ },
+ },
+ },
+ },
+};
+
+const DTYPES: [DataType; 3] = [DataType::F16, DataType::BF16, DataType::F32];
+
+pub struct MoeExpertsTwoPassPrefillBlock {
+ tile_map: MoeTileMapKernels,
+ pass_a_indirect: Vec<Vec<MoeExpertsPrefillPassAMetalKernel>>, // [gate][dtype]
+ pass_b_indirect: Vec<MoeExpertsPrefillPassBMetalKernel>, // [dtype]
+}
+
+impl MoeExpertsTwoPassPrefillBlock {
+ pub fn new(ctx: &MetalContext) -> Result<Self, MetalError> {
+ let dtypes = [DataType::F16, DataType::BF16, DataType::F32];
+
+ let mut pass_a_indirect = vec![];
+ for gate in 0u32..4u32 {
+ let mut kernels = vec![];
+ for dtype in dtypes {
+ let kernel = MoeExpertsPrefillPassAMetalKernel::new(ctx, dtype, gate)?;
+ kernels.push(kernel);
+ }
+ pass_a_indirect.push(kernels);
+ }
+
+ let mut pass_b_indirect = vec![];
+ for dtype in dtypes {
+ let kernel = MoeExpertsPrefillPassBMetalKernel::new(ctx, dtype)?;
+ pass_b_indirect.push(kernel);
+ }
+
+ Ok(Self {
+ tile_map: MoeTileMapKernels::new(ctx)?,
+ pass_a_indirect,
+ pass_b_indirect,
+ })
+ }
+
+ pub fn encode(
+ &self,
+ command_buffer: &Retained<ProtocolObject<dyn MTLCommandBuffer>>,
+ args: &MoeExpertsTwoPassArguments,
+ ) {
+ if args.total_rows == 0 {
+ return;
+ }
+
+ let hidden_bytes = args.total_rows * args.d_ff * args.data_type.size_in_bytes();
+ let blit_encoder = command_buffer.new_blit_command_encoder().expect("Failed to create blit command encoder");
+ blit_encoder.fill_buffer_range_value(args.hidden_buffer, 0..hidden_bytes, 0);
+ blit_encoder.end_encoding();
+
+ self.tile_map.encode_counts(
+ command_buffer,
+ &MoeTileCountsArguments {
+ offsets_buffer: args.expert_offsets,
+ tile_counts_buffer: args.tile_counts,
+ e: args.e,
+ },
+ );
+ self.tile_map.encode_scan(
+ command_buffer,
+ &MoeTileScanArguments {
+ tile_counts_buffer: args.tile_counts,
+ tile_offsets_buffer: args.tile_offsets,
+ total_tiles_buffer: args.total_tiles,
+ e: args.e,
+ },
+ );
+ self.tile_map.encode_build_map(
+ command_buffer,
+ &MoeTileMapBuildArguments {
+ expert_offsets: args.expert_offsets,
+ tile_offsets: args.tile_offsets,
+ tile_counts: args.tile_counts,
+ tile_map: args.tile_map,
+ e: args.e,
+ },
+ );
+ let d_model_u32 = args.d_model as u32;
+ let d_ff_u32 = args.d_ff as u32;
+ const COL_TILE_FF: u32 = 32; // Must match PASSA_BN in Metal kernel
+ const COL_TILE_MODEL: u32 = 64;
+ let n_tiles_ff = if d_ff_u32 == 0 {
+ 0
+ } else {
+ (d_ff_u32 + COL_TILE_FF - 1) / COL_TILE_FF
+ };
+ let n_tiles_model = if d_model_u32 == 0 {
+ 0
+ } else {
+ (d_model_u32 + COL_TILE_MODEL - 1) / COL_TILE_MODEL
+ };
+ self.tile_map.encode_dispatch_args(
+ command_buffer,
+ &MoeTileDispatchArguments {
+ total_tiles: args.total_tiles,
+ dispatch_args: args.dispatch_args,
+ num_tiles_x: n_tiles_ff,
+ },
+ );
+
+ let encoder_a =
+ command_buffer.new_compute_command_encoder().expect("Failed to create compute command encoder A");
+ let gate_idx = args.gating_code.min(3) as usize;
+ let dtype_idx = DTYPES.iter().position(|t| *t == args.data_type).unwrap();
+ let kernel_pass_a = &self.pass_a_indirect[gate_idx][dtype_idx];
+ kernel_pass_a.encode(
+ &args.x_perm_buffer.retain(),
+ &args.expert_offsets.retain(),
+ &args.w13_all.retain(),
+ &args.up_biases.retain(),
+ &args.hidden_buffer.retain(),
+ args.d_model as u32,
+ args.d_ff as u32,
+ args.e as u32,
+ args.gate_clip_min,
+ args.gate_clip_max,
+ args.up_clip_min,
+ args.up_clip_max,
+ args.silu_alpha,
+ &args.tile_map.retain(),
+ &args.dispatch_args.retain(),
+ &encoder_a,
+ );
+ encoder_a.end_encoding();
+
+ let dispatch_args = MoeTileDispatchArguments {
+ total_tiles: args.total_tiles,
+ dispatch_args: args.dispatch_args,
+ num_tiles_x: n_tiles_model,
+ };
+ self.tile_map.encode_dispatch_args(command_buffer, &dispatch_args);
+
+ let encoder_b =
+ command_buffer.new_compute_command_encoder().expect("Failed to create compute command encoder B");
+ let kernel_pass_b = &self.pass_b_indirect[dtype_idx];
+ kernel_pass_b.encode(
+ &args.hidden_buffer.retain(),
+ &args.expert_offsets.retain(),
+ &args.w2_all.retain(),
+ &args.down_biases.retain(),
+ &args.output_buffer.retain(),
+ args.d_model as u32,
+ args.d_ff as u32,
+ args.e as u32,
+ &args.tile_map.retain(),
+ &args.dispatch_args.retain(),
+ &encoder_b,
+ );
+ encoder_b.end_encoding();
+ }
+}
+
+#[derive(Debug)]
+pub struct MoeExpertsTwoPassArguments<'a> {
+ pub x_perm_buffer: &'a ProtocolObject<dyn MTLBuffer>,
+ pub expert_offsets: &'a ProtocolObject<dyn MTLBuffer>,
+ pub row_expert_map: &'a ProtocolObject<dyn MTLBuffer>,
+ pub hidden_buffer: &'a ProtocolObject<dyn MTLBuffer>,
+ pub output_buffer: &'a ProtocolObject<dyn MTLBuffer>,
+ pub w13_all: &'a ProtocolObject<dyn MTLBuffer>,
+ pub w2_all: &'a ProtocolObject<dyn MTLBuffer>,
+ pub up_biases: &'a ProtocolObject<dyn MTLBuffer>,
+ pub down_biases: &'a ProtocolObject<dyn MTLBuffer>,
+ pub tile_counts: &'a ProtocolObject<dyn MTLBuffer>,
+ pub tile_offsets: &'a ProtocolObject<dyn MTLBuffer>,
+ pub tile_map: &'a ProtocolObject<dyn MTLBuffer>,
+ pub total_tiles: &'a ProtocolObject<dyn MTLBuffer>,
+ pub dispatch_args: &'a ProtocolObject<dyn MTLBuffer>,
+ pub total_rows: usize,
+ pub d_model: usize,
+ pub d_ff: usize,
+ pub e: usize,
+ pub num_tiles_k: u32,
+ pub gating_code: u32,
+ pub gate_clip_min: f32,
+ pub gate_clip_max: f32,
+ pub up_clip_min: f32,
+ pub up_clip_max: f32,
+ pub silu_alpha: f32,
+ pub data_type: DataType,
+}
diff --git a/crates/uzu/src/backends/metal/kernel/moe/mod.rs b/crates/uzu/src/backends/metal/kernel/moe/mod.rs
index d868ace3a..227de2672 100644
--- a/crates/uzu/src/backends/metal/kernel/moe/mod.rs
+++ b/crates/uzu/src/backends/metal/kernel/moe/mod.rs
@@ -1,9 +1,7 @@
-use crate::DataType;
-
// Submodules
-mod experts;
mod experts_single;
mod experts_two_pass_decode;
+mod experts_two_pass_prefill;
mod gather;
mod router_topk;
mod scatter;
@@ -11,11 +9,9 @@ pub mod tiles_map;
pub mod tiles_pass_a;
// Re-export public items from submodules
-pub use experts::{
- MoeExpertsArguments, MoeExpertsError, MoeExpertsTwoPassArguments, MoeExpertsTwoPassPrefillKernel, MoeScatterError,
-};
pub use experts_single::{MoeExpertsSingleDecodeArguments, MoeExpertsSingleDecodeKernels};
-pub use experts_two_pass_decode::MoeExpertsTwoPassDecodeKernels;
+pub use experts_two_pass_decode::MoeExpertsTwoPassDecodeBlock;
+pub use experts_two_pass_prefill::{MoeExpertsTwoPassArguments, MoeExpertsTwoPassPrefillBlock};
pub use gather::{MoeGatherArguments, MoeGatherKernels};
pub use router_topk::{MoeRouterTopKArguments, MoeRouterTopKError, MoeRouterTopKKernel};
pub use scatter::{MoeBlockBasesArguments, MoeScatterArguments, MoeScatterKernels, MoeScatterWithMapArguments};
@@ -23,22 +19,3 @@ pub use tiles_pass_a::{
MoePassARowMapArguments, MoePassATileBuildArguments, MoePassATileCountsArguments, MoePassATileDispatchArguments,
MoePassATileKernels, MoePassATileScanArguments,
};
-
-// Common utility functions
-pub(crate) fn dtype_suffix(dtype: DataType) -> &'static str {
- match dtype {
- DataType::F16 => "f16",
- DataType::BF16 => "bf16",
- DataType::F32 => "f32",
- _ => panic!("Unsupported data type: {:?}", dtype),
- }
-}
-
-pub(crate) fn dtype_index(dtype: DataType) -> usize {
- match dtype {
- DataType::F16 => 0,
- DataType::BF16 => 1,
- DataType::F32 => 2,
- _ => panic!("Unsupported data type: {:?}", dtype),
- }
-}
diff --git a/crates/uzu/tests/moe/moe_block_e2e_test.rs b/crates/uzu/tests/moe/moe_block_e2e_test.rs
index c8dd522ee..f0a936e24 100644
--- a/crates/uzu/tests/moe/moe_block_e2e_test.rs
+++ b/crates/uzu/tests/moe/moe_block_e2e_test.rs
@@ -11,7 +11,7 @@ use uzu::{
MoeBlockBasesArguments, MoeScatterKernels, MoeScatterWithMapArguments,
dsl::{MoeCountsOffsetsFusedMetalKernel, MoeFinalizeMetalKernel},
moe::{
- MoeExpertsTwoPassArguments, MoeExpertsTwoPassPrefillKernel, MoeGatherArguments, MoeGatherKernels,
+ MoeExpertsTwoPassArguments, MoeExpertsTwoPassPrefillBlock, MoeGatherArguments, MoeGatherKernels,
MoeRouterTopKArguments, MoeRouterTopKKernel,
},
},
@@ -434,41 +434,37 @@ fn run_moe_parity_test_internal(
let hidden_buf = alloc_buffer::<f32>(&ctx, total_rows * d_ff);
let row_expert_map_buf = alloc_buffer::<u32>(&ctx, total_rows);
- let experts = MoeExpertsTwoPassPrefillKernel::new(&ctx).expect("experts");
+ let experts = MoeExpertsTwoPassPrefillBlock::new(&ctx).expect("experts");
let num_tiles_k = ((d_ff + 64 - 1) / 64) as u32;
- experts
- .encode(
- &cb,
- MoeExpertsTwoPassArguments {
- x_perm_buffer: &x_perm_buf,
- expert_offsets: &offsets_buf,
- row_expert_map: &row_expert_map_buf,
- hidden_buffer: &hidden_buf,
- output_buffer: &y_partial_buf,
- w13_all: &w13_buf,
- w2_all: &w2_buf,
- up_biases: &up_biases_buf,
- down_biases: &down_biases_buf,
- tile_counts: &tile_counts_buf,
- tile_offsets: &tile_offsets_buf,
- tile_map: &tile_map_buf,
- total_tiles: &total_tiles_buf,
- dispatch_args: &dispatch_args_buf,
- total_rows,
- d_model,
- d_ff,
- e,
- num_tiles_k,
- gating_code,
- gate_clip_min: gate_clip.0,
- gate_clip_max: gate_clip.1,
- up_clip_min: up_clip.0,
- up_clip_max: up_clip.1,
- silu_alpha,
- data_type: DataType::BF16,
- },
- )
- .expect("experts encode");
+ let args = MoeExpertsTwoPassArguments {
+ x_perm_buffer: &x_perm_buf,
+ expert_offsets: &offsets_buf,
+ row_expert_map: &row_expert_map_buf,
+ hidden_buffer: &hidden_buf,
+ output_buffer: &y_partial_buf,
+ w13_all: &w13_buf,
+ w2_all: &w2_buf,
+ up_biases: &up_biases_buf,
+ down_biases: &down_biases_buf,
+ tile_counts: &tile_counts_buf,
+ tile_offsets: &tile_offsets_buf,
+ tile_map: &tile_map_buf,
+ total_tiles: &total_tiles_buf,
+ dispatch_args: &dispatch_args_buf,
+ total_rows,
+ d_model,
+ d_ff,
+ e,
+ num_tiles_k,
+ gating_code,
+ gate_clip_min: gate_clip.0,
+ gate_clip_max: gate_clip.1,
+ up_clip_min: up_clip.0,
+ up_clip_max: up_clip.1,
+ silu_alpha,
+ data_type: DataType::BF16,
+ };
+ experts.encode(&cb, &args);
let finalize = MoeFinalizeMetalKernel::new(&ctx, DataType::BF16).expect("finalize");
let encoder = cb.new_compute_command_encoder().expect("encoder");
diff --git a/crates/uzu/tests/moe/moe_experts_perf_test.rs b/crates/uzu/tests/moe/moe_experts_perf_test.rs
index ba2d602c2..2bb58c14f 100644
--- a/crates/uzu/tests/moe/moe_experts_perf_test.rs
+++ b/crates/uzu/tests/moe/moe_experts_perf_test.rs
@@ -9,7 +9,7 @@ use uzu::{
MetalContext,
kernel::moe::{
MoeExpertsSingleDecodeArguments, MoeExpertsSingleDecodeKernels, MoeExpertsTwoPassArguments,
- MoeExpertsTwoPassDecodeKernels, MoeExpertsTwoPassPrefillKernel,
+ MoeExpertsTwoPassDecodeBlock, MoeExpertsTwoPassPrefillBlock,
},
},
};
@@ -76,7 +76,7 @@ fn run_decode_case(
let up_biases: Vec<bf16> = (0..e * 2 * d_ff).map(|_| bf16::from_f32(rng.random_range(-0.01..0.01))).collect();
let down_biases: Vec<bf16> = (0..e * d_model).map(|_| bf16::from_f32(rng.random_range(-0.01..0.01))).collect();
- let experts_kernel = MoeExpertsTwoPassDecodeKernels::new(ctx).expect("experts decode kernel");
+ let experts_kernel = MoeExpertsTwoPassDecodeBlock::new(ctx).expect("experts decode kernel");
let x_perm_buf = alloc_buffer_with_data(&ctx, &x_perm);
let offsets_buf = alloc_buffer_with_data(&ctx, &offsets);
@@ -213,7 +213,7 @@ fn run_two_pass_prefill_case(
let up_biases: Vec<bf16> = (0..e * 2 * d_ff).map(|_| bf16::from_f32(rng.random_range(-0.01..0.01))).collect();
let down_biases: Vec<bf16> = (0..e * d_model).map(|_| bf16::from_f32(rng.random_range(-0.01..0.01))).collect();
- let experts_kernel = MoeExpertsTwoPassPrefillKernel::new(ctx).expect("experts prefill kernel");
+ let experts_kernel = MoeExpertsTwoPassPrefillBlock::new(ctx).expect("experts prefill kernel");
let x_perm_buf = alloc_buffer_with_data(&ctx, &x_perm);
let offsets_buf = alloc_buffer_with_data(&ctx, &offsets);
@@ -266,7 +266,7 @@ fn run_two_pass_prefill_case(
for _ in 0..warmup {
let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");
- experts_kernel.encode(&cb, make_two_pass_args()).expect("two-pass prefill encode");
+ experts_kernel.encode(&cb, &make_two_pass_args());
cb.commit();
cb.wait_until_completed();
}
@@ -275,7 +275,7 @@ fn run_two_pass_prefill_case(
for _ in 0..iters {
let start = Instant::now();
let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");
- experts_kernel.encode(&cb, make_two_pass_args()).expect("two-pass prefill encode");
+ experts_kernel.encode(&cb, &make_two_pass_args());
cb.commit();
cb.wait_until_completed();
times.push(start.elapsed().as_secs_f64() * 1000.0);
@@ -486,7 +486,7 @@ fn run_indirect_decode_timed(
let up_biases: Vec<bf16> = (0..e * 2 * d_ff).map(|_| bf16::from_f32(rng.random_range(-0.01..0.01))).collect();
let down_biases: Vec<bf16> = (0..e * d_model).map(|_| bf16::from_f32(rng.random_range(-0.01..0.01))).collect();
- let experts_kernel = MoeExpertsTwoPassDecodeKernels::new(ctx).expect("decode kernel");
+ let experts_kernel = MoeExpertsTwoPassDecodeBlock::new(ctx).expect("decode kernel");
let x_perm_buf = alloc_buffer_with_data(ctx, &x_perm);
let offsets_buf = alloc_buffer_with_data(ctx, &offsets);
diff --git a/crates/uzu/tests/moe/moe_experts_test.rs b/crates/uzu/tests/moe/moe_experts_test.rs
index 663d0ed4b..6c566785e 100644
--- a/crates/uzu/tests/moe/moe_experts_test.rs
+++ b/crates/uzu/tests/moe/moe_experts_test.rs
@@ -14,8 +14,8 @@ use rand::{RngExt, SeedableRng, rngs::StdRng};
use uzu::{
DataType,
backends::metal::kernel::moe::{
- MoeExpertsSingleDecodeKernels, MoeExpertsTwoPassArguments, MoeExpertsTwoPassDecodeKernels,
- MoeExpertsTwoPassPrefillKernel,
+ MoeExpertsSingleDecodeKernels, MoeExpertsTwoPassArguments, MoeExpertsTwoPassDecodeBlock,
+ MoeExpertsTwoPassPrefillBlock,
},
};
@@ -412,7 +412,7 @@ fn test_two_pass_decode_correctness() {
let dispatch_args_buf = alloc_buffer::<u32>(&ctx, 3);
// Execute 2-pass decode kernel
- let experts_kernel = MoeExpertsTwoPassDecodeKernels::new(&ctx).expect("MoeExpertsTwoPassDecodeKernel::new");
+ let experts_kernel = MoeExpertsTwoPassDecodeBlock::new(&ctx).expect("MoeExpertsTwoPassDecodeKernel::new");
let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");
const K_TILE: usize = 64;
@@ -556,7 +556,7 @@ fn test_two_pass_decode_multi_token() {
let tile_map_buf = alloc_buffer::<u32>(&ctx, max_total_tiles * 3);
let dispatch_args_buf = alloc_buffer::<u32>(&ctx, 3);
- let experts_kernel = MoeExpertsTwoPassDecodeKernels::new(&ctx).expect("kernel");
+ let experts_kernel = MoeExpertsTwoPassDecodeBlock::new(&ctx).expect("kernel");
let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");
experts_kernel.encode(
&cb,
@@ -658,41 +658,37 @@ fn test_two_pass_prefill_correctness() {
let tile_map_buf = alloc_buffer::<u32>(&ctx, max_total_tiles * 3);
let dispatch_args_buf = alloc_buffer::<u32>(&ctx, 3);
- let experts_kernel = MoeExpertsTwoPassPrefillKernel::new(&ctx).expect("kernel");
+ let experts_kernel = MoeExpertsTwoPassPrefillBlock::new(&ctx).expect("kernel");
let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");
- experts_kernel
- .encode(
- &cb,
- MoeExpertsTwoPassArguments {
- x_perm_buffer: &x_perm_buf,
- expert_offsets: &offsets_buf,
- row_expert_map: &row_expert_map_buf,
- hidden_buffer: &hidden_buf,
- output_buffer: &y_partial_buf,
- w13_all: &w13_buf,
- w2_all: &w2_buf,
- up_biases: &up_biases_buf,
- down_biases: &down_biases_buf,
- tile_counts: &tile_counts_buf,
- tile_offsets: &tile_offsets_buf,
- tile_map: &tile_map_buf,
- total_tiles: &total_tiles_buf,
- dispatch_args: &dispatch_args_buf,
- total_rows: sum_k,
- d_model,
- d_ff,
- e,
- num_tiles_k: ((d_ff + 63) / 64) as u32,
- gating_code,
- gate_clip_min: f32::NEG_INFINITY,
- gate_clip_max: f32::INFINITY,
- up_clip_min: f32::NEG_INFINITY,
- up_clip_max: f32::INFINITY,
- silu_alpha,
- data_type: DataType::BF16,
- },
- )
- .expect("encode");
+ let args = MoeExpertsTwoPassArguments {
+ x_perm_buffer: &x_perm_buf,
+ expert_offsets: &offsets_buf,
+ row_expert_map: &row_expert_map_buf,
+ hidden_buffer: &hidden_buf,
+ output_buffer: &y_partial_buf,
+ w13_all: &w13_buf,
+ w2_all: &w2_buf,
+ up_biases: &up_biases_buf,
+ down_biases: &down_biases_buf,
+ tile_counts: &tile_counts_buf,
+ tile_offsets: &tile_offsets_buf,
+ tile_map: &tile_map_buf,
+ total_tiles: &total_tiles_buf,
+ dispatch_args: &dispatch_args_buf,
+ total_rows: sum_k,
+ d_model,
+ d_ff,
+ e,
+ num_tiles_k: ((d_ff + 63) / 64) as u32,
+ gating_code,
+ gate_clip_min: f32::NEG_INFINITY,
+ gate_clip_max: f32::INFINITY,
+ up_clip_min: f32::NEG_INFINITY,
+ up_clip_max: f32::INFINITY,
+ silu_alpha,
+ data_type: DataType::BF16,
+ };
+ experts_kernel.encode(&cb, &args);
cb.commit();
cb.wait_until_completed();
diff --git a/crates/uzu/tests/moe/moe_perf_test.rs b/crates/uzu/tests/moe/moe_perf_test.rs
index 5759b538e..ff54da09e 100644
--- a/crates/uzu/tests/moe/moe_perf_test.rs
+++ b/crates/uzu/tests/moe/moe_perf_test.rs
@@ -10,7 +10,7 @@ use uzu::{
metal::kernel::{
dsl::{MoeCountsOffsetsFusedMetalKernel, MoeFinalizeMetalKernel},
moe::{
- MoeExpertsTwoPassArguments, MoeExpertsTwoPassDecodeKernels, MoeGatherArguments, MoeGatherKernels,
+ MoeExpertsTwoPassArguments, MoeExpertsTwoPassDecodeBlock, MoeGatherArguments, MoeGatherKernels,
MoeRouterTopKArguments, MoeRouterTopKKernel, MoeScatterKernels, MoeScatterWithMapArguments,
},
},
@@ -300,7 +300,7 @@ fn test_moe_pipeline_breakdown_decode() {
let counts_offsets_kernel = MoeCountsOffsetsFusedMetalKernel::new(&ctx).expect("counts+offsets fused");
let scatter_kernel = MoeScatterKernels::new(&ctx).expect("scatter");
let gather_kernel = MoeGatherKernels::new(&ctx).expect("gather");
- let experts_kernel = MoeExpertsTwoPassDecodeKernels::new(&ctx).expect("experts two-pass decode");
+ let experts_kernel = MoeExpertsTwoPassDecodeBlock::new(&ctx).expect("experts two-pass decode");
let finalize_kernel = MoeFinalizeMetalKernel::new(&ctx, DataType::BF16).expect("finalize");
let router_topk_fused_kernel = MoeRouterTopKKernel::new(&ctx).expect("router+topk fused");