A high-performance inference engine for AI models
dslify-moe-2pass-prefill (#197)
Aleksandr Golokozf23a7582b154
diff --git a/crates/uzu/src/backends/metal/encodable_block/moe_block/moe_block.rs b/crates/uzu/src/backends/metal/encodable_block/moe_block/moe_block.rsindex b776abea3..e703e2b62 100644--- a/crates/uzu/src/backends/metal/encodable_block/moe_block/moe_block.rs+++ b/crates/uzu/src/backends/metal/encodable_block/moe_block/moe_block.rs@@ -17,8 +17,8 @@ use crate::{MoeGatherKernels,dsl::{MoeCountsOffsetsFusedMetalKernel, MoeFinalizeMetalKernel},moe::{- MoeBlockBasesArguments, MoeExpertsTwoPassArguments, MoeExpertsTwoPassDecodeKernels,- MoeExpertsTwoPassPrefillKernel, MoeGatherArguments, MoeRouterTopKArguments, MoeRouterTopKKernel,+ MoeBlockBasesArguments, MoeExpertsTwoPassArguments, MoeExpertsTwoPassDecodeBlock,+ MoeExpertsTwoPassPrefillBlock, MoeGatherArguments, MoeRouterTopKArguments, MoeRouterTopKKernel,MoeScatterArguments, MoeScatterKernels, MoeScatterWithMapArguments,},},@@ -44,8 +44,8 @@ pub struct MoeBlock {counts_offsets_kernel: MoeCountsOffsetsFusedMetalKernel,scatter_kernels: MoeScatterKernels,gather_kernels: MoeGatherKernels,- experts_two_pass_decode_kernel: MoeExpertsTwoPassDecodeKernels,- experts_two_pass_prefill_kernel: MoeExpertsTwoPassPrefillKernel,+ experts_two_pass_decode_kernel: MoeExpertsTwoPassDecodeBlock,+ experts_two_pass_prefill_kernel: MoeExpertsTwoPassPrefillBlock,finalize_kernel: MoeFinalizeMetalKernel,moe_config: MixtureOfExpertsConfig,model_dim: usize,@@ -114,10 +114,10 @@ impl MoeBlock {.map_err(|e| crate::backends::metal::MetalError::Generic(format!("Scatter kernels error: {:?}", e)))?;let gather_kernel = MoeGatherKernels::new(context).map_err(|e| crate::backends::metal::MetalError::Generic(format!("Gather kernel error: {:?}", e)))?;- let experts_two_pass_decode_kernel = MoeExpertsTwoPassDecodeKernels::new(context).map_err(|e| {+ let experts_two_pass_decode_kernel = MoeExpertsTwoPassDecodeBlock::new(context).map_err(|e| {crate::backends::metal::MetalError::Generic(format!("Experts two-pass decode kernel error: {:?}", e))})?;- let experts_two_pass_prefill_kernel = MoeExpertsTwoPassPrefillKernel::new(context).map_err(|e| {+ let experts_two_pass_prefill_kernel = MoeExpertsTwoPassPrefillBlock::new(context).map_err(|e| {crate::backends::metal::MetalError::Generic(format!("Experts two-pass prefill kernel error: {:?}", e))})?;let finalize_kernel = MoeFinalizeMetalKernel::new(context, data_type)@@ -442,39 +442,35 @@ impl EncodableBlock<Metal> for MoeBlock {let total_rows = suffix_length * k;let num_tiles_k = ((self.hidden_dim + k_tile - 1) / k_tile) as u32;- self.experts_two_pass_prefill_kernel- .encode(- root,- MoeExpertsTwoPassArguments {- x_perm_buffer: &x_perm_buf,- expert_offsets: &offsets_buf,- row_expert_map: &row_expert_map_buf,- hidden_buffer: &hidden_buf,- output_buffer: &y_partial_buf,- w13_all: &self.shared_weights.w13_buf,- w2_all: &self.shared_weights.w2_buf,- up_biases: &self.shared_weights.up_biases_buf,- down_biases: &self.shared_weights.down_biases_buf,- tile_counts: &tile_counts_buf,- tile_offsets: &tile_offsets_buf,- tile_map: &tile_map_buf,- total_tiles: &total_tiles_buf,- dispatch_args: &dispatch_args_buf,- total_rows,- d_model: self.model_dim,- d_ff: self.hidden_dim,- e,- num_tiles_k,- gating_code,- gate_clip_min,- gate_clip_max,- up_clip_min,- up_clip_max,- silu_alpha,- data_type: self.data_type,- },- )- .expect("MoE experts two-pass prefill failed");+ let args = MoeExpertsTwoPassArguments {+ x_perm_buffer: &x_perm_buf,+ expert_offsets: &offsets_buf,+ row_expert_map: &row_expert_map_buf,+ hidden_buffer: &hidden_buf,+ output_buffer: &y_partial_buf,+ w13_all: &self.shared_weights.w13_buf,+ w2_all: &self.shared_weights.w2_buf,+ up_biases: &self.shared_weights.up_biases_buf,+ down_biases: &self.shared_weights.down_biases_buf,+ tile_counts: &tile_counts_buf,+ tile_offsets: &tile_offsets_buf,+ tile_map: &tile_map_buf,+ total_tiles: &total_tiles_buf,+ dispatch_args: &dispatch_args_buf,+ total_rows,+ d_model: self.model_dim,+ d_ff: self.hidden_dim,+ e,+ num_tiles_k,+ gating_code,+ gate_clip_min,+ gate_clip_max,+ up_clip_min,+ up_clip_max,+ silu_alpha,+ data_type: self.data_type,+ };+ self.experts_two_pass_prefill_kernel.encode(root, &args);}{diff --git a/crates/uzu/src/backends/metal/kernel/mod.rs b/crates/uzu/src/backends/metal/kernel/mod.rsindex 6f02adf2c..daae0807a 100644--- a/crates/uzu/src/backends/metal/kernel/mod.rs+++ b/crates/uzu/src/backends/metal/kernel/mod.rs@@ -10,10 +10,9 @@ pub mod quant_matmul;pub use matmul::{MatmulArguments, MatmulKernel};pub use moe::{- MoeBlockBasesArguments, MoeExpertsArguments, MoeExpertsError, MoeExpertsTwoPassArguments,- MoeExpertsTwoPassDecodeKernels, MoeExpertsTwoPassPrefillKernel, MoeGatherArguments, MoeGatherKernels,- MoeRouterTopKArguments, MoeRouterTopKKernel, MoeScatterArguments, MoeScatterError, MoeScatterKernels,- MoeScatterWithMapArguments,+ MoeBlockBasesArguments, MoeExpertsTwoPassArguments, MoeExpertsTwoPassDecodeBlock, MoeExpertsTwoPassPrefillBlock,+ MoeGatherArguments, MoeGatherKernels, MoeRouterTopKArguments, MoeRouterTopKKernel, MoeScatterArguments,+ MoeScatterKernels, MoeScatterWithMapArguments,};impl MatmulKernels for dsl::MetalKernels {diff --git a/crates/uzu/src/backends/metal/kernel/moe/experts.rs b/crates/uzu/src/backends/metal/kernel/moe/experts.rsdeleted file mode 100644index 19b669744..000000000--- a/crates/uzu/src/backends/metal/kernel/moe/experts.rs+++ /dev/null@@ -1,244 +0,0 @@-use std::{mem::size_of, ptr::NonNull};--use metal::{- MTLBlitCommandEncoderExt, MTLBuffer, MTLCommandBuffer, MTLCommandEncoder, MTLComputeCommandEncoder,- MTLComputePipelineState, MTLFunctionConstantValues, MTLSize,-};-use objc2::{rc::Retained, runtime::ProtocolObject};--use super::{dtype_index, dtype_suffix};-use crate::{- DataType,- backends::metal::{- FunctionConstantValuesSetValue, MetalContext, MetalError,- kernel::moe::tiles_map::{- MoeTileCountsArguments, MoeTileDispatchArguments, MoeTileMapBuildArguments, MoeTileMapKernels,- MoeTileScanArguments,- },- },-};--#[derive(Debug, thiserror::Error)]-pub enum MoeScatterError {- #[error("Metal error: {0}")]- MetalError(#[from] MetalError),-}-#[derive(Debug, thiserror::Error)]-pub enum MoeExpertsError {- #[error("Metal error: {0}")]- MetalError(#[from] MetalError),- #[error("{0}")]- Generic(String),-}--#[derive(Debug)]-pub struct MoeExpertsArguments<'a> {- pub x_perm_buffer: &'a ProtocolObject<dyn MTLBuffer>, // [sum_k, d_model] - permuted input- pub expert_offsets: &'a ProtocolObject<dyn MTLBuffer>, // [E+1] - expert segment offsets- pub w13_all: &'a ProtocolObject<dyn MTLBuffer>, // [E, 2*d_ff, d_model] - transposed up projection weights- pub w2_all: &'a ProtocolObject<dyn MTLBuffer>, // [E, d_model, d_ff] - transposed down projection weights- pub y_partial: &'a ProtocolObject<dyn MTLBuffer>, // [sum_k, d_model] - output buffer- pub up_biases: &'a ProtocolObject<dyn MTLBuffer>, // [E, 2*d_ff] - up projection biases- pub down_biases: &'a ProtocolObject<dyn MTLBuffer>, // [E, d_model] - down projection biases- pub tile_counts: &'a ProtocolObject<dyn MTLBuffer>, // [E]- pub tile_row_offsets: &'a ProtocolObject<dyn MTLBuffer>, // [E+1]- pub tile_map: &'a ProtocolObject<dyn MTLBuffer>, // [max_tiles * 3]- pub total_tiles: &'a ProtocolObject<dyn MTLBuffer>, // [2]- pub dispatch_args: &'a ProtocolObject<dyn MTLBuffer>, // [3]- pub num_tiles_n: usize,- pub t: usize,- pub d_model: usize,- pub d_ff: usize,- pub e: usize,- pub k: usize,- pub gating_code: u32,- pub gate_clip_min: f32,- pub gate_clip_max: f32,- pub up_clip_min: f32,- pub up_clip_max: f32,- pub silu_alpha: f32,- pub data_type: DataType,-}--pub struct MoeExpertsTwoPassPrefillKernel {- tile_map: MoeTileMapKernels,- pass_a_indirect: Vec<Vec<Retained<ProtocolObject<dyn MTLComputePipelineState>>>>, // [gate][dtype]- pass_b_indirect: Vec<Retained<ProtocolObject<dyn MTLComputePipelineState>>>, // [dtype]-}--#[derive(Debug)]-pub struct MoeExpertsTwoPassArguments<'a> {- pub x_perm_buffer: &'a ProtocolObject<dyn MTLBuffer>,- pub expert_offsets: &'a ProtocolObject<dyn MTLBuffer>,- pub row_expert_map: &'a ProtocolObject<dyn MTLBuffer>,- pub hidden_buffer: &'a ProtocolObject<dyn MTLBuffer>,- pub output_buffer: &'a ProtocolObject<dyn MTLBuffer>,- pub w13_all: &'a ProtocolObject<dyn MTLBuffer>,- pub w2_all: &'a ProtocolObject<dyn MTLBuffer>,- pub up_biases: &'a ProtocolObject<dyn MTLBuffer>,- pub down_biases: &'a ProtocolObject<dyn MTLBuffer>,- pub tile_counts: &'a ProtocolObject<dyn MTLBuffer>,- pub tile_offsets: &'a ProtocolObject<dyn MTLBuffer>,- pub tile_map: &'a ProtocolObject<dyn MTLBuffer>,- pub total_tiles: &'a ProtocolObject<dyn MTLBuffer>,- pub dispatch_args: &'a ProtocolObject<dyn MTLBuffer>,- pub total_rows: usize,- pub d_model: usize,- pub d_ff: usize,- pub e: usize,- pub num_tiles_k: u32,- pub gating_code: u32,- pub gate_clip_min: f32,- pub gate_clip_max: f32,- pub up_clip_min: f32,- pub up_clip_max: f32,- pub silu_alpha: f32,- pub data_type: DataType,-}--impl MoeExpertsTwoPassPrefillKernel {- pub fn new(ctx: &MetalContext) -> Result<Self, MoeExpertsError> {- let dtypes = [DataType::F16, DataType::BF16, DataType::F32];- let mut pass_a_indirect = vec![Vec::with_capacity(dtypes.len()); 4];- for gate in 0u32..4u32 {- for dtype in &dtypes {- let dtype_suffix = dtype_suffix(*dtype);- let fcv = MTLFunctionConstantValues::new();- fcv.set_value(&gate, 30);- let kernel_name = format!("moe_two_pass_prefill_pass_a_indirect_{}", dtype_suffix);- let cache_key = format!("{}_gate_{}", kernel_name, gate);- pass_a_indirect[gate as usize].push(ctx.compute_pipeline_state_cached(- &cache_key,- &kernel_name,- Some(&fcv),- )?);- }- }- let mut pass_b_indirect = Vec::with_capacity(dtypes.len());- for dtype in &dtypes {- let dtype_suffix = dtype_suffix(*dtype);- let kernel_name = format!("moe_two_pass_prefill_pass_b_indirect_{}", dtype_suffix);- pass_b_indirect.push(ctx.compute_pipeline_state(&kernel_name, None)?);- }- Ok(Self {- tile_map: MoeTileMapKernels::new(ctx)?,- pass_a_indirect,- pass_b_indirect,- })- }-- pub fn encode(- &self,- command_buffer: &Retained<ProtocolObject<dyn MTLCommandBuffer>>,- args: MoeExpertsTwoPassArguments,- ) -> Result<(), MoeExpertsError> {- if args.total_rows == 0 {- return Ok(());- }- let gate_idx = (args.gating_code.min(3)) as usize;- let dtype_idx = dtype_index(args.data_type);- let hidden_bytes = args.total_rows * args.d_ff * args.data_type.size_in_bytes();- let blit_encoder = command_buffer.new_blit_command_encoder().expect("Failed to create blit command encoder");- blit_encoder.fill_buffer_range_value(args.hidden_buffer, 0..hidden_bytes, 0);- blit_encoder.end_encoding();- self.tile_map.encode_counts(- command_buffer,- &MoeTileCountsArguments {- offsets_buffer: args.expert_offsets,- tile_counts_buffer: args.tile_counts,- e: args.e,- },- );- self.tile_map.encode_scan(- command_buffer,- &MoeTileScanArguments {- tile_counts_buffer: args.tile_counts,- tile_offsets_buffer: args.tile_offsets,- total_tiles_buffer: args.total_tiles,- e: args.e,- },- );- self.tile_map.encode_build_map(- command_buffer,- &MoeTileMapBuildArguments {- expert_offsets: args.expert_offsets,- tile_offsets: args.tile_offsets,- tile_counts: args.tile_counts,- tile_map: args.tile_map,- e: args.e,- },- );- let d_model_u32 = args.d_model as u32;- let d_ff_u32 = args.d_ff as u32;- let e_u32 = args.e as u32;- const COL_TILE_FF: u32 = 32; // Must match PASSA_BN in Metal kernel- const COL_TILE_MODEL: u32 = 64;- let n_tiles_ff = if d_ff_u32 == 0 {- 0- } else {- (d_ff_u32 + COL_TILE_FF - 1) / COL_TILE_FF- };- let n_tiles_model = if d_model_u32 == 0 {- 0- } else {- (d_model_u32 + COL_TILE_MODEL - 1) / COL_TILE_MODEL- };- self.tile_map.encode_dispatch_args(- command_buffer,- &MoeTileDispatchArguments {- total_tiles: args.total_tiles,- dispatch_args: args.dispatch_args,- num_tiles_x: n_tiles_ff,- },- );- let pass_a_pipeline = &self.pass_a_indirect[gate_idx][dtype_idx];- let encoder_a = command_buffer.new_compute_command_encoder().expect("Failed to create compute command encoder");- encoder_a.set_compute_pipeline_state(pass_a_pipeline);- encoder_a.set_buffer(Some(args.x_perm_buffer), 0, 0);- encoder_a.set_buffer(Some(args.expert_offsets), 0, 1);- encoder_a.set_buffer(Some(args.w13_all), 0, 2);- encoder_a.set_buffer(Some(args.up_biases), 0, 3);- encoder_a.set_buffer(Some(args.hidden_buffer), 0, 4);- unsafe {- encoder_a.set_bytes(NonNull::new_unchecked(&d_model_u32 as *const _ as *mut _), size_of::<u32>(), 5);- encoder_a.set_bytes(NonNull::new_unchecked(&d_ff_u32 as *const _ as *mut _), size_of::<u32>(), 6);- encoder_a.set_bytes(NonNull::new_unchecked(&e_u32 as *const _ as *mut _), size_of::<u32>(), 7);- encoder_a.set_bytes(NonNull::new_unchecked(&args.gate_clip_min as *const _ as *mut _), size_of::<f32>(), 8);- encoder_a.set_bytes(NonNull::new_unchecked(&args.gate_clip_max as *const _ as *mut _), size_of::<f32>(), 9);- encoder_a.set_bytes(NonNull::new_unchecked(&args.up_clip_min as *const _ as *mut _), size_of::<f32>(), 10);- encoder_a.set_bytes(NonNull::new_unchecked(&args.up_clip_max as *const _ as *mut _), size_of::<f32>(), 11);- encoder_a.set_bytes(NonNull::new_unchecked(&args.silu_alpha as *const _ as *mut _), size_of::<f32>(), 12);- }- encoder_a.set_buffer(Some(args.tile_map), 0, 13);- // Match kernel config: WM=2, WN=2 => 4 SIMDgroups => 128 threads- const SIMDGROUPS_PER_TG: u32 = 4;- const THREADS_PER_TG: u32 = SIMDGROUPS_PER_TG * 32;- encoder_a.dispatch_threadgroups_indirect(args.dispatch_args, 0, MTLSize::new(THREADS_PER_TG as usize, 1, 1));- encoder_a.end_encoding();- self.tile_map.encode_dispatch_args(- command_buffer,- &MoeTileDispatchArguments {- total_tiles: args.total_tiles,- dispatch_args: args.dispatch_args,- num_tiles_x: n_tiles_model,- },- );- let pass_b_pipeline = &self.pass_b_indirect[dtype_idx];- let encoder_b = command_buffer.new_compute_command_encoder().expect("Failed to create compute command encoder");- encoder_b.set_compute_pipeline_state(pass_b_pipeline);- encoder_b.set_buffer(Some(args.hidden_buffer), 0, 0);- encoder_b.set_buffer(Some(args.expert_offsets), 0, 1);- encoder_b.set_buffer(Some(args.w2_all), 0, 2);- encoder_b.set_buffer(Some(args.down_biases), 0, 3);- encoder_b.set_buffer(Some(args.output_buffer), 0, 4);- unsafe {- encoder_b.set_bytes(NonNull::new_unchecked(&d_model_u32 as *const _ as *mut _), size_of::<u32>(), 5);- encoder_b.set_bytes(NonNull::new_unchecked(&d_ff_u32 as *const _ as *mut _), size_of::<u32>(), 6);- encoder_b.set_bytes(NonNull::new_unchecked(&e_u32 as *const _ as *mut _), size_of::<u32>(), 7);- }- encoder_b.set_buffer(Some(args.tile_map), 0, 8);- encoder_b.dispatch_threadgroups_indirect(args.dispatch_args, 0, MTLSize::new(THREADS_PER_TG as usize, 1, 1));- encoder_b.end_encoding();- Ok(())- }-}diff --git a/crates/uzu/src/backends/metal/kernel/moe/experts_two_pass_decode.rs b/crates/uzu/src/backends/metal/kernel/moe/experts_two_pass_decode.rsindex 4f1c3725e..8a07282f8 100644--- a/crates/uzu/src/backends/metal/kernel/moe/experts_two_pass_decode.rs+++ b/crates/uzu/src/backends/metal/kernel/moe/experts_two_pass_decode.rs@@ -15,7 +15,7 @@ use crate::{dsl::{MoeExpertsDecodeDownFused2DMetalKernel, MoeExpertsDecodePassAMetalKernel},moe::{MoePassARowMapArguments, MoePassATileBuildArguments, MoePassATileCountsArguments,- MoePassATileDispatchArguments, MoePassATileKernels, MoePassATileScanArguments, dtype_index,+ MoePassATileDispatchArguments, MoePassATileKernels, MoePassATileScanArguments,},},},@@ -24,13 +24,13 @@ use crate::{static DTYPES: [DataType; 3] = [DataType::F16, DataType::BF16, DataType::F32];-pub struct MoeExpertsTwoPassDecodeKernels {+pub struct MoeExpertsTwoPassDecodeBlock {pass_a_tile: MoePassATileKernels,pass_a_indirect: Vec<Vec<MoeExpertsDecodePassAMetalKernel>>,fused_down: Vec<MoeExpertsDecodeDownFused2DMetalKernel>,}-impl MoeExpertsTwoPassDecodeKernels {+impl MoeExpertsTwoPassDecodeBlock {pub fn new(ctx: &MetalContext) -> Result<Self, MetalError> {let mut pass_a_indirect = vec![];for gate in 0u32..4u32 {@@ -116,7 +116,7 @@ impl MoeExpertsTwoPassDecodeKernels {// pass alet gate_idx = args.gating_code.min(3) as usize;- let dtype_idx = dtype_index(args.data_type);+ let dtype_idx = DTYPES.iter().position(|dtype| *dtype == args.data_type).unwrap();let pass_a_encoder =command_buffer.new_compute_command_encoder().expect("Failed to create compute command encoder");let pass_a_kernel = &self.pass_a_indirect[gate_idx][dtype_idx];diff --git a/crates/uzu/src/backends/metal/kernel/moe/experts_two_pass_prefill.metal b/crates/uzu/src/backends/metal/kernel/moe/experts_two_pass_prefill.metalindex f1818b697..9f28ba119 100644--- a/crates/uzu/src/backends/metal/kernel/moe/experts_two_pass_prefill.metal+++ b/crates/uzu/src/backends/metal/kernel/moe/experts_two_pass_prefill.metal@@ -1,11 +1,8 @@#include <metal_stdlib>-#include <metal_simdgroup>-#include <metal_simdgroup_matrix>+#include "../definitions.metal"#include "../quant_matmul/mma.h"#include "moe_commons.h"-using namespace metal;-// ------------------------ helpers ------------------------static inline uint ceil_div(uint a, uint b) { return (a + b - 1u) / b; }@@ -26,24 +23,6 @@ inline void store_vec4<float>(device float* dst, ulong base, float4 vals) {*reinterpret_cast<device float4*>(dst + base) = vals;}-template <typename T>-struct StageStorage {- using type = float;-};--template <>-struct StageStorage<half> {- using type = half;-};--template <>-struct StageStorage<bfloat> {- using type = bfloat;-};--// 0=GELU(up), 1=SiLU(up), 2=SwiGLU(gate)*up, 3=GEGLU(gate)*up-constant uint GATING_SEL [[function_constant(30)]];-// ------------------------ Pass A (X @ W13 → hidden) ------------------------// Tile config (tuned for A-series; matches your prior defaults)constant uint PASSA_BM = 16;@@ -54,38 +33,50 @@ constant uint PASSA_SG_BN = 16;constant uint PASSA_TG_PAD = 4; // Padding to avoid bank conflictstemplate <typename T>-inline void pass_a_impl(- device const T* X_perm,+VARIANTS(T, float, half, bfloat)+KERNEL(MoeExpertsPrefillPassA)(+ device const T* x_perm,device const uint* expert_offsets,- device const T* W13_all,+ device const T* w13_all,device const T* up_biases,device float* hidden_out,- uint d_model,- uint d_ff,- uint E,- float gate_clip_min,- float gate_clip_max,- float up_clip_min,- float up_clip_max,- float silu_alpha,- // work partition- uint tile_seg_start,- uint expert_idx,- uint tile_m,- uint tile_n,- // per-thread info- uint sg_id,- uint3 threads_per_tg,- uint3 local_tid,- // TG storage- threadgroup typename StageStorage<T>::type* Xs, // [BM,BK]- threadgroup typename StageStorage<T>::type* Wk_up, // [BN,BK]- threadgroup typename StageStorage<T>::type* Wk_gate, // [BN,BK]- threadgroup float* bias_up, // [BN]- threadgroup float* bias_gate // [BN]+ constant uint& d_model,+ constant uint& d_ff,+ constant uint& e,+ constant float& gate_clip_min,+ constant float& gate_clip_max,+ constant float& up_clip_min,+ constant float& up_clip_max,+ constant float& silu_alpha,+ device const uint* tile_map,+ threadgroup T Xs[PASSA_BM * (PASSA_BK + PASSA_TG_PAD)],+ threadgroup T Wk_up[PASSA_BN * (PASSA_BK + PASSA_TG_PAD)],+ threadgroup T Wk_gate[PASSA_BN * (PASSA_BK + PASSA_TG_PAD)],+ threadgroup float bias_up[PASSA_BN],+ threadgroup float bias_gate[PASSA_BN],+ // 0=GELU(up), 1=SiLU(up), 2=SwiGLU(gate)*up, 3=GEGLU(gate)*up+ const uint gating_sel SPECIALIZE,+ const uint n_tile_idx GROUPS(INDIRECT),+ const uint row_tile_idx GROUPS(INDIRECT),+ const uint local_tid_x THREADS(128),+ const Simd simd) {- using StageT = typename StageStorage<T>::type;- using StageVec4 = metal::vec<StageT, 4>;+ const uint base = row_tile_idx * 3u;+ const uint expert_idx = tile_map[base + 0u];+ if (expert_idx >= e)+ return;++ const uint3 threads_per_tg = {128, 1, 1};+ const uint3 local_tid = {local_tid_x, 0, 0};+ const uint sg_id = simd.group_idx;++ constexpr uint ROW_TILE = PASSA_BM;+ const uint tile_seg_start = tile_map[base + 1u];+ const uint row_off_elems = tile_map[base + 2u];+ const uint tile_m = row_off_elems / ROW_TILE;+ const uint tile_n = n_tile_idx;++ using StageVec4 = metal::vec<T, 4>;constexpr uint Bm = PASSA_BM, Bn = PASSA_BN, Bk = PASSA_BK;constexpr uint SgBm = PASSA_SG_BM, SgBn = PASSA_SG_BN;@@ -93,9 +84,6 @@ inline void pass_a_impl(constexpr uint X_LD = Bk + TG_PAD; // Padded stride for Xconstexpr uint W_LD = Bk + TG_PAD; // Padded stride for weights- if (expert_idx >= E)- return;-const uint seg_start = expert_offsets[expert_idx];const uint seg_end = expert_offsets[expert_idx + 1];const uint seg_len = (seg_end > seg_start) ? (seg_end - seg_start) : 0u;@@ -135,7 +123,7 @@ inline void pass_a_impl(metal::simdgroup_float8x8 OutGate[TEMP];for (uint i = 0; i < TEMP; ++i) {OutUp[i] = metal::make_filled_simdgroup_matrix<float, 8, 8>(0.0f);- if (GATING_SEL > 1u) {+ if (gating_sel > 1u) {OutGate[i] = metal::make_filled_simdgroup_matrix<float, 8, 8>(0.0f);}}@@ -165,7 +153,7 @@ inline void pass_a_impl(const uint c8 = i % vec_cols;const uint col_base = c8 * vec_size;- threadgroup StageT* dst = Xs + r * X_LD + col_base;+ threadgroup T* dst = Xs + r * X_LD + col_base;if (r < m_rows && col_base < valid_k) {// Safe to read - at least first element is valid@@ -174,14 +162,14 @@ inline void pass_a_impl(UZU_PRAGMA_UNROLLfor (uint j = 0; j < vec_size; j++) {dst[j] = (col_base + j < valid_k)- ? StageT(float(X_perm[base + j]))- : StageT(0.0f);+ ? static_cast<T>(float(x_perm[base + j]))+ : static_cast<T>(0.0f);}} else {// Row out of bounds or entire vec8 chunk is out of K boundsUZU_PRAGMA_UNROLLfor (uint j = 0; j < vec_size; j++) {- dst[j] = StageT(0.0f);+ dst[j] = static_cast<T>(0.0f);}}}@@ -204,8 +192,8 @@ inline void pass_a_impl(const uint c8 = i % vec_cols;const uint col_base = c8 * vec_size;- threadgroup StageT* up_dst = Wk_up + n * W_LD + col_base;- threadgroup StageT* gt_dst = Wk_gate + n * W_LD + col_base;+ threadgroup T* up_dst = Wk_up + n * W_LD + col_base;+ threadgroup T* gt_dst = Wk_gate + n * W_LD + col_base;if (n < n_cols && col_base < valid_k) {// Safe to read - at least first element is valid@@ -219,21 +207,21 @@ inline void pass_a_impl(UZU_PRAGMA_UNROLLfor (uint j = 0; j < vec_size; j++) {up_dst[j] = (col_base + j < valid_k)- ? StageT(float(W13_all[up_base + j]))- : StageT(0.0f);- if (GATING_SEL > 1u) {+ ? static_cast<T>(float(w13_all[up_base + j]))+ : static_cast<T>(0.0f);+ if (gating_sel > 1u) {gt_dst[j] = (col_base + j < valid_k)- ? StageT(float(W13_all[gate_base + j]))- : StageT(0.0f);+ ? static_cast<T>(float(w13_all[gate_base + j]))+ : static_cast<T>(0.0f);}}} else {// Row out of bounds or entire vec8 chunk is out of K boundsUZU_PRAGMA_UNROLLfor (uint j = 0; j < vec_size; j++) {- up_dst[j] = StageT(0.0f);- if (GATING_SEL > 1u)- gt_dst[j] = StageT(0.0f);+ up_dst[j] = static_cast<T>(0.0f);+ if (gating_sel > 1u)+ gt_dst[j] = static_cast<T>(0.0f);}}}@@ -248,14 +236,14 @@ inline void pass_a_impl(UZU_PRAGMA_UNROLLfor (uint m_sub = 0; m_sub < SgBm; m_sub += SG_TILE) {const uint r_idx = m_sub / SG_TILE;- metal::simdgroup_matrix<StageT, 8, 8> lhs_frag;+ metal::simdgroup_matrix<T, 8, 8> lhs_frag;simdgroup_load(lhs_frag, Xs, X_LD, ulong2(kk, row_sg_off + m_sub));UZU_PRAGMA_UNROLLfor (uint n_sub = 0; n_sub < SgBn; n_sub += SG_TILE) {const uint c_idx = n_sub / SG_TILE;const uint tile = r_idx * (SgBn / SG_TILE) + c_idx;- metal::simdgroup_matrix<StageT, 8, 8> rhs_up;+ metal::simdgroup_matrix<T, 8, 8> rhs_up;simdgroup_load(rhs_up,Wk_up,@@ -270,8 +258,8 @@ inline void pass_a_impl(OutUp[tile]);- if (GATING_SEL > 1u) {- metal::simdgroup_matrix<StageT, 8, 8> rhs_gate;+ if (gating_sel > 1u) {+ metal::simdgroup_matrix<T, 8, 8> rhs_gate;simdgroup_load(rhs_gate,Wk_gate,@@ -297,7 +285,7 @@ inline void pass_a_impl(const uint c_global = col_tg_off + c_local;bias_up[c_local] =(c_global < d_ff) ? float(up_biases[bias_base + c_global]) : 0.0f;- if (GATING_SEL > 1u) {+ if (gating_sel > 1u) {bias_gate[c_local] = (c_global < d_ff)? float(up_biases[bias_base + d_ff + c_global]): 0.0f;@@ -336,7 +324,7 @@ inline void pass_a_impl(const auto up_frag = OutUp[tile].thread_elements();float gate_frag_0 = 0.0f;float gate_frag_1 = 0.0f;- if (GATING_SEL > 1u) {+ if (gating_sel > 1u) {const auto gate_frag = OutGate[tile].thread_elements();gate_frag_0 = gate_frag[0];gate_frag_1 = gate_frag[1];@@ -350,16 +338,16 @@ inline void pass_a_impl(float up_v =clamp(up_frag[0] + bias_up[col0], up_clip_min, up_clip_max);float out_val;- if (GATING_SEL <= 1u) {+ if (gating_sel <= 1u) {out_val =- (GATING_SEL == 0u) ? gelu_approx(up_v) : silu(up_v, silu_alpha);+ (gating_sel == 0u) ? gelu_approx(up_v) : silu(up_v, silu_alpha);} else {float gate_v = clamp(gate_frag_0 + bias_gate[col0],gate_clip_min,gate_clip_max);- const float gate_act = (GATING_SEL == 2u) ? silu(gate_v, silu_alpha)+ const float gate_act = (gating_sel == 2u) ? silu(gate_v, silu_alpha): gelu_approx(gate_v);out_val = gate_act * up_v;}@@ -371,16 +359,16 @@ inline void pass_a_impl(float up_v =clamp(up_frag[1] + bias_up[col1], up_clip_min, up_clip_max);float out_val;- if (GATING_SEL <= 1u) {+ if (gating_sel <= 1u) {out_val =- (GATING_SEL == 0u) ? gelu_approx(up_v) : silu(up_v, silu_alpha);+ (gating_sel == 0u) ? gelu_approx(up_v) : silu(up_v, silu_alpha);} else {float gate_v = clamp(gate_frag_1 + bias_gate[col1],gate_clip_min,gate_clip_max);- const float gate_act = (GATING_SEL == 2u) ? silu(gate_v, silu_alpha)+ const float gate_act = (gating_sel == 2u) ? silu(gate_v, silu_alpha): gelu_approx(gate_v);out_val = gate_act * up_v;}@@ -391,132 +379,6 @@ inline void pass_a_impl(}}-// Indirect variant (consumes [expert, seg_start, tile_row_offset] triples)-template <typename T>-inline void pass_a_indirect_impl(- device const T* X_perm,- device const uint* expert_offsets,- device const T* W13_all,- device const T* up_biases,- device float* hidden_out,- device const uint* tile_map,- uint d_model,- uint d_ff,- uint E,- float gate_clip_min,- float gate_clip_max,- float up_clip_min,- float up_clip_max,- float silu_alpha,- uint row_tile_idx,- uint n_tile_idx,- uint sg_id,- uint3 threads_per_tg,- uint3 local_tid,- threadgroup typename StageStorage<T>::type* Xs,- threadgroup typename StageStorage<T>::type* Wk_up,- threadgroup typename StageStorage<T>::type* Wk_gate,- threadgroup float* bias_up,- threadgroup float* bias_gate-) {- constexpr uint ROW_TILE = PASSA_BM;- const uint base = row_tile_idx * 3u;- const uint expert_idx = tile_map[base + 0u];- if (expert_idx >= E)- return;- const uint tile_seg_start = tile_map[base + 1u];- const uint row_off_elems = tile_map[base + 2u];- const uint tile_m = row_off_elems / ROW_TILE;-- pass_a_impl<T>(- X_perm,- expert_offsets,- W13_all,- up_biases,- hidden_out,- d_model,- d_ff,- E,- gate_clip_min,- gate_clip_max,- up_clip_min,- up_clip_max,- silu_alpha,- tile_seg_start,- expert_idx,- tile_m,- n_tile_idx,- sg_id,- threads_per_tg,- local_tid,- Xs,- Wk_up,- Wk_gate,- bias_up,- bias_gate- );-}--#define MOE_PASS_A_INDIRECT_KERNEL(DTYPE, SUFFIX) \- [[max_total_threads_per_threadgroup(128)]] \- kernel void moe_two_pass_prefill_pass_a_indirect_##SUFFIX( \- device const DTYPE* X_perm [[buffer(0)]], \- device const uint* expert_offsets [[buffer(1)]], \- device const DTYPE* W13_all [[buffer(2)]], \- device const DTYPE* up_biases [[buffer(3)]], \- device float* hidden_out [[buffer(4)]], \- constant uint& d_model [[buffer(5)]], \- constant uint& d_ff [[buffer(6)]], \- constant uint& E [[buffer(7)]], \- constant float& gate_clip_min [[buffer(8)]], \- constant float& gate_clip_max [[buffer(9)]], \- constant float& up_clip_min [[buffer(10)]], \- constant float& up_clip_max [[buffer(11)]], \- constant float& silu_alpha [[buffer(12)]], \- device const uint* tile_map [[buffer(13)]], \- uint sg_id [[simdgroup_index_in_threadgroup]], \- uint3 threads_per_tg [[threads_per_threadgroup]], \- uint3 tg_pos [[threadgroup_position_in_grid]], \- uint3 local_tid [[thread_position_in_threadgroup]] \- ) { \- using StageT = typename StageStorage<DTYPE>::type; \- threadgroup StageT Xs[PASSA_BM * (PASSA_BK + PASSA_TG_PAD)]; \- threadgroup StageT Wk_up[PASSA_BN * (PASSA_BK + PASSA_TG_PAD)]; \- threadgroup StageT Wk_gate[PASSA_BN * (PASSA_BK + PASSA_TG_PAD)]; \- threadgroup float bias_up[PASSA_BN]; \- threadgroup float bias_gate[PASSA_BN]; \- pass_a_indirect_impl<DTYPE>( \- X_perm, \- expert_offsets, \- W13_all, \- up_biases, \- hidden_out, \- tile_map, \- d_model, \- d_ff, \- E, \- gate_clip_min, \- gate_clip_max, \- up_clip_min, \- up_clip_max, \- silu_alpha, \- tg_pos.y, \- tg_pos.x, \- sg_id, \- threads_per_tg, \- local_tid, \- Xs, \- Wk_up, \- Wk_gate, \- bias_up, \- bias_gate \- ); \- }--MOE_PASS_A_INDIRECT_KERNEL(bfloat, bf16)-MOE_PASS_A_INDIRECT_KERNEL(half, f16)-MOE_PASS_A_INDIRECT_KERNEL(float, f32)-// ------------------------ Pass B (hidden @ W2 → output)// ------------------------ Optimized config: 16x64 output tile, 4 simdgroups// (128 threads)@@ -528,26 +390,37 @@ constant uint PASSB_SG_BN = 32;constant uint PASSB_TG_PAD = 4; // Padding to avoid bank conflictstemplate <typename T>-inline void pass_b_impl(+VARIANTS(T, float, half, bfloat)+KERNEL(MoeExpertsPrefillPassB)(device const float* hidden,device const uint* expert_offsets,- device const T* W2_all,+ device const T* w2_all,device const T* down_biases,device T* output,- uint d_model,- uint d_ff,- uint E,- uint expert_idx,- uint tile_m,- uint tile_n,- uint sg_id,- uint simd_lid,- uint lin,- threadgroup float* Hs,- threadgroup typename StageStorage<T>::type* Wk,- threadgroup float* bias_tile+ constant uint& d_model,+ constant uint& d_ff,+ constant uint& e,+ device const uint* tile_map,+ threadgroup float Hs[PASSB_BM * (PASSB_BK + PASSB_TG_PAD)],+ threadgroup T Wk[PASSB_BN * (PASSB_BK + PASSB_TG_PAD)],+ threadgroup float bias_tile[PASSB_BN],+ const Simd simd,+ const uint n_tile_idx GROUPS(INDIRECT),+ const uint row_tile_idx GROUPS(INDIRECT),+ const uint lin THREADS(128)) {- using StageT = typename StageStorage<T>::type;+ const uint base = row_tile_idx * 3u;+ const uint expert_idx = tile_map[base + 0u];+ if (expert_idx >= e)+ return;++ const uint simd_lid = simd.lane_idx;+ const uint sg_id = simd.group_idx;++ constexpr uint ROW_TILE = PASSB_BM;+ const uint row_off_elems = tile_map[base + 2u];+ const uint tile_m = row_off_elems / ROW_TILE;+ const uint tile_n = n_tile_idx;constexpr uint Bm = PASSB_BM, Bn = PASSB_BN, Bk = PASSB_BK;constexpr uint SgBm = PASSB_SG_BM, SgBn = PASSB_SG_BN;@@ -557,9 +430,6 @@ inline void pass_b_impl(constexpr uint SG_TILE = 8;constexpr uint TGP_SIZE = 128; // 4 simdgroups × 32 threads- if (expert_idx >= E)- return;-const uint seg_start = expert_offsets[expert_idx];const uint seg_end = expert_offsets[expert_idx + 1];const uint seg_len = (seg_end > seg_start) ? (seg_end - seg_start) : 0u;@@ -640,22 +510,22 @@ inline void pass_b_impl({for (uint pass = 0; pass < 4; pass++) {const uint row = w_bi + pass * W_TROWS;- threadgroup StageT* my_dst = Wk + row * W_LD + w_bj;+ threadgroup T* my_dst = Wk + row * W_LD + w_bj;if (row < n_cols && w_bj < valid_k) {// Safe to read - at least first element is validconst uint gcol = col_tg_off + row;- device const T* my_src = W2_all + w2_expert_base ++ device const T* my_src = w2_all + w2_expert_base +(ulong)gcol * (ulong)d_ff +(ulong)(k_off + w_bj);UZU_PRAGMA_UNROLLfor (uint j = 0; j < W_VEC; j++) {- my_dst[j] =- (w_bj + j < valid_k) ? StageT(float(my_src[j])) : StageT(0.0f);+ my_dst[j] = (w_bj + j < valid_k) ? static_cast<T>(float(my_src[j]))+ : static_cast<T>(0.0f);}} else {UZU_PRAGMA_UNROLLfor (uint j = 0; j < W_VEC; j++) {- my_dst[j] = StageT(0.0f);+ my_dst[j] = static_cast<T>(0.0f);}}}@@ -671,7 +541,7 @@ inline void pass_b_impl(simdgroup_load(lhs, Hs, H_LD, ulong2(kk, row_sg_off));// Load 4 RHS fragments (cols)- metal::simdgroup_matrix<StageT, 8, 8> rhs0, rhs1, rhs2, rhs3;+ metal::simdgroup_matrix<T, 8, 8> rhs0, rhs1, rhs2, rhs3;simdgroup_load(rhs0, Wk, W_LD, ulong2(kk, col_sg_off), true);simdgroup_load(rhs1, Wk, W_LD, ulong2(kk, col_sg_off + 8), true);simdgroup_load(rhs2, Wk, W_LD, ulong2(kk, col_sg_off + 16), true);@@ -720,101 +590,4 @@ inline void pass_b_impl(output[out_row * (ulong)d_model + out_col] = T(val);}}-}--// Indirect variant-template <typename T>-inline void pass_b_indirect_impl(- device const float* hidden,- device const uint* expert_offsets,- device const T* W2_all,- device const T* down_biases,- device T* output,- device const uint* tile_map,- uint d_model,- uint d_ff,- uint E,- uint row_tile_idx,- uint n_tile_idx,- uint sg_id,- uint simd_lid,- uint lin,- threadgroup float* Hs,- threadgroup typename StageStorage<T>::type* Wk,- threadgroup float* bias_tile-) {- constexpr uint ROW_TILE = PASSB_BM;- const uint base = row_tile_idx * 3u;- const uint expert_idx = tile_map[base + 0u];- if (expert_idx >= E)- return;- const uint row_off_elems = tile_map[base + 2u];- const uint tile_m = row_off_elems / ROW_TILE;-- pass_b_impl<T>(- hidden,- expert_offsets,- W2_all,- down_biases,- output,- d_model,- d_ff,- E,- expert_idx,- tile_m,- n_tile_idx,- sg_id,- simd_lid,- lin,- Hs,- Wk,- bias_tile- );-}--#define MOE_PASS_B_INDIRECT_KERNEL(DTYPE, SUFFIX) \- [[max_total_threads_per_threadgroup(128)]] \- kernel void moe_two_pass_prefill_pass_b_indirect_##SUFFIX( \- device const float* hidden [[buffer(0)]], \- device const uint* expert_offsets [[buffer(1)]], \- device const DTYPE* W2_all [[buffer(2)]], \- device const DTYPE* down_biases [[buffer(3)]], \- device DTYPE* output [[buffer(4)]], \- constant uint& d_model [[buffer(5)]], \- constant uint& d_ff [[buffer(6)]], \- constant uint& E [[buffer(7)]], \- device const uint* tile_map [[buffer(8)]], \- uint sg_id [[simdgroup_index_in_threadgroup]], \- uint simd_lid [[thread_index_in_simdgroup]], \- uint3 tg_pos [[threadgroup_position_in_grid]], \- uint3 local_tid [[thread_position_in_threadgroup]] \- ) { \- using StageT = typename StageStorage<DTYPE>::type; \- threadgroup float Hs[PASSB_BM * (PASSB_BK + PASSB_TG_PAD)]; \- threadgroup StageT Wk[PASSB_BN * (PASSB_BK + PASSB_TG_PAD)]; \- threadgroup float bias[PASSB_BN]; \- const uint lin = local_tid.x; \- pass_b_indirect_impl<DTYPE>( \- hidden, \- expert_offsets, \- W2_all, \- down_biases, \- output, \- tile_map, \- d_model, \- d_ff, \- E, \- tg_pos.y, \- tg_pos.x, \- sg_id, \- simd_lid, \- lin, \- Hs, \- Wk, \- bias \- ); \- }--MOE_PASS_B_INDIRECT_KERNEL(bfloat, bf16)-MOE_PASS_B_INDIRECT_KERNEL(half, f16)-MOE_PASS_B_INDIRECT_KERNEL(float, f32)+}\ No newline at end of filediff --git a/crates/uzu/src/backends/metal/kernel/moe/experts_two_pass_prefill.rs b/crates/uzu/src/backends/metal/kernel/moe/experts_two_pass_prefill.rsnew file mode 100644index 000000000..a1e28cf39--- /dev/null+++ b/crates/uzu/src/backends/metal/kernel/moe/experts_two_pass_prefill.rs@@ -0,0 +1,203 @@+use metal::{MTLBlitCommandEncoderExt, MTLBuffer, MTLCommandBuffer, MTLCommandEncoder};+use objc2::{+ __framework_prelude::{ProtocolObject, Retained},+ Message,+};++use crate::{+ DataType,+ backends::{+ common::kernel::{MoeExpertsPrefillPassAKernel, MoeExpertsPrefillPassBKernel},+ metal::{+ MetalContext, MetalError,+ kernel::{+ dsl::{MoeExpertsPrefillPassAMetalKernel, MoeExpertsPrefillPassBMetalKernel},+ moe::tiles_map::{+ MoeTileCountsArguments, MoeTileDispatchArguments, MoeTileMapBuildArguments, MoeTileMapKernels,+ MoeTileScanArguments,+ },+ },+ },+ },+};++const DTYPES: [DataType; 3] = [DataType::F16, DataType::BF16, DataType::F32];++pub struct MoeExpertsTwoPassPrefillBlock {+ tile_map: MoeTileMapKernels,+ pass_a_indirect: Vec<Vec<MoeExpertsPrefillPassAMetalKernel>>, // [gate][dtype]+ pass_b_indirect: Vec<MoeExpertsPrefillPassBMetalKernel>, // [dtype]+}++impl MoeExpertsTwoPassPrefillBlock {+ pub fn new(ctx: &MetalContext) -> Result<Self, MetalError> {+ let dtypes = [DataType::F16, DataType::BF16, DataType::F32];++ let mut pass_a_indirect = vec![];+ for gate in 0u32..4u32 {+ let mut kernels = vec![];+ for dtype in dtypes {+ let kernel = MoeExpertsPrefillPassAMetalKernel::new(ctx, dtype, gate)?;+ kernels.push(kernel);+ }+ pass_a_indirect.push(kernels);+ }++ let mut pass_b_indirect = vec![];+ for dtype in dtypes {+ let kernel = MoeExpertsPrefillPassBMetalKernel::new(ctx, dtype)?;+ pass_b_indirect.push(kernel);+ }++ Ok(Self {+ tile_map: MoeTileMapKernels::new(ctx)?,+ pass_a_indirect,+ pass_b_indirect,+ })+ }++ pub fn encode(+ &self,+ command_buffer: &Retained<ProtocolObject<dyn MTLCommandBuffer>>,+ args: &MoeExpertsTwoPassArguments,+ ) {+ if args.total_rows == 0 {+ return;+ }++ let hidden_bytes = args.total_rows * args.d_ff * args.data_type.size_in_bytes();+ let blit_encoder = command_buffer.new_blit_command_encoder().expect("Failed to create blit command encoder");+ blit_encoder.fill_buffer_range_value(args.hidden_buffer, 0..hidden_bytes, 0);+ blit_encoder.end_encoding();++ self.tile_map.encode_counts(+ command_buffer,+ &MoeTileCountsArguments {+ offsets_buffer: args.expert_offsets,+ tile_counts_buffer: args.tile_counts,+ e: args.e,+ },+ );+ self.tile_map.encode_scan(+ command_buffer,+ &MoeTileScanArguments {+ tile_counts_buffer: args.tile_counts,+ tile_offsets_buffer: args.tile_offsets,+ total_tiles_buffer: args.total_tiles,+ e: args.e,+ },+ );+ self.tile_map.encode_build_map(+ command_buffer,+ &MoeTileMapBuildArguments {+ expert_offsets: args.expert_offsets,+ tile_offsets: args.tile_offsets,+ tile_counts: args.tile_counts,+ tile_map: args.tile_map,+ e: args.e,+ },+ );+ let d_model_u32 = args.d_model as u32;+ let d_ff_u32 = args.d_ff as u32;+ const COL_TILE_FF: u32 = 32; // Must match PASSA_BN in Metal kernel+ const COL_TILE_MODEL: u32 = 64;+ let n_tiles_ff = if d_ff_u32 == 0 {+ 0+ } else {+ (d_ff_u32 + COL_TILE_FF - 1) / COL_TILE_FF+ };+ let n_tiles_model = if d_model_u32 == 0 {+ 0+ } else {+ (d_model_u32 + COL_TILE_MODEL - 1) / COL_TILE_MODEL+ };+ self.tile_map.encode_dispatch_args(+ command_buffer,+ &MoeTileDispatchArguments {+ total_tiles: args.total_tiles,+ dispatch_args: args.dispatch_args,+ num_tiles_x: n_tiles_ff,+ },+ );++ let encoder_a =+ command_buffer.new_compute_command_encoder().expect("Failed to create compute command encoder A");+ let gate_idx = args.gating_code.min(3) as usize;+ let dtype_idx = DTYPES.iter().position(|t| *t == args.data_type).unwrap();+ let kernel_pass_a = &self.pass_a_indirect[gate_idx][dtype_idx];+ kernel_pass_a.encode(+ &args.x_perm_buffer.retain(),+ &args.expert_offsets.retain(),+ &args.w13_all.retain(),+ &args.up_biases.retain(),+ &args.hidden_buffer.retain(),+ args.d_model as u32,+ args.d_ff as u32,+ args.e as u32,+ args.gate_clip_min,+ args.gate_clip_max,+ args.up_clip_min,+ args.up_clip_max,+ args.silu_alpha,+ &args.tile_map.retain(),+ &args.dispatch_args.retain(),+ &encoder_a,+ );+ encoder_a.end_encoding();++ let dispatch_args = MoeTileDispatchArguments {+ total_tiles: args.total_tiles,+ dispatch_args: args.dispatch_args,+ num_tiles_x: n_tiles_model,+ };+ self.tile_map.encode_dispatch_args(command_buffer, &dispatch_args);++ let encoder_b =+ command_buffer.new_compute_command_encoder().expect("Failed to create compute command encoder B");+ let kernel_pass_b = &self.pass_b_indirect[dtype_idx];+ kernel_pass_b.encode(+ &args.hidden_buffer.retain(),+ &args.expert_offsets.retain(),+ &args.w2_all.retain(),+ &args.down_biases.retain(),+ &args.output_buffer.retain(),+ args.d_model as u32,+ args.d_ff as u32,+ args.e as u32,+ &args.tile_map.retain(),+ &args.dispatch_args.retain(),+ &encoder_b,+ );+ encoder_b.end_encoding();+ }+}++#[derive(Debug)]+pub struct MoeExpertsTwoPassArguments<'a> {+ pub x_perm_buffer: &'a ProtocolObject<dyn MTLBuffer>,+ pub expert_offsets: &'a ProtocolObject<dyn MTLBuffer>,+ pub row_expert_map: &'a ProtocolObject<dyn MTLBuffer>,+ pub hidden_buffer: &'a ProtocolObject<dyn MTLBuffer>,+ pub output_buffer: &'a ProtocolObject<dyn MTLBuffer>,+ pub w13_all: &'a ProtocolObject<dyn MTLBuffer>,+ pub w2_all: &'a ProtocolObject<dyn MTLBuffer>,+ pub up_biases: &'a ProtocolObject<dyn MTLBuffer>,+ pub down_biases: &'a ProtocolObject<dyn MTLBuffer>,+ pub tile_counts: &'a ProtocolObject<dyn MTLBuffer>,+ pub tile_offsets: &'a ProtocolObject<dyn MTLBuffer>,+ pub tile_map: &'a ProtocolObject<dyn MTLBuffer>,+ pub total_tiles: &'a ProtocolObject<dyn MTLBuffer>,+ pub dispatch_args: &'a ProtocolObject<dyn MTLBuffer>,+ pub total_rows: usize,+ pub d_model: usize,+ pub d_ff: usize,+ pub e: usize,+ pub num_tiles_k: u32,+ pub gating_code: u32,+ pub gate_clip_min: f32,+ pub gate_clip_max: f32,+ pub up_clip_min: f32,+ pub up_clip_max: f32,+ pub silu_alpha: f32,+ pub data_type: DataType,+}diff --git a/crates/uzu/src/backends/metal/kernel/moe/mod.rs b/crates/uzu/src/backends/metal/kernel/moe/mod.rsindex d868ace3a..227de2672 100644--- a/crates/uzu/src/backends/metal/kernel/moe/mod.rs+++ b/crates/uzu/src/backends/metal/kernel/moe/mod.rs@@ -1,9 +1,7 @@-use crate::DataType;-// Submodules-mod experts;mod experts_single;mod experts_two_pass_decode;+mod experts_two_pass_prefill;mod gather;mod router_topk;mod scatter;@@ -11,11 +9,9 @@ pub mod tiles_map;pub mod tiles_pass_a;// Re-export public items from submodules-pub use experts::{- MoeExpertsArguments, MoeExpertsError, MoeExpertsTwoPassArguments, MoeExpertsTwoPassPrefillKernel, MoeScatterError,-};pub use experts_single::{MoeExpertsSingleDecodeArguments, MoeExpertsSingleDecodeKernels};-pub use experts_two_pass_decode::MoeExpertsTwoPassDecodeKernels;+pub use experts_two_pass_decode::MoeExpertsTwoPassDecodeBlock;+pub use experts_two_pass_prefill::{MoeExpertsTwoPassArguments, MoeExpertsTwoPassPrefillBlock};pub use gather::{MoeGatherArguments, MoeGatherKernels};pub use router_topk::{MoeRouterTopKArguments, MoeRouterTopKError, MoeRouterTopKKernel};pub use scatter::{MoeBlockBasesArguments, MoeScatterArguments, MoeScatterKernels, MoeScatterWithMapArguments};@@ -23,22 +19,3 @@ pub use tiles_pass_a::{MoePassARowMapArguments, MoePassATileBuildArguments, MoePassATileCountsArguments, MoePassATileDispatchArguments,MoePassATileKernels, MoePassATileScanArguments,};--// Common utility functions-pub(crate) fn dtype_suffix(dtype: DataType) -> &'static str {- match dtype {- DataType::F16 => "f16",- DataType::BF16 => "bf16",- DataType::F32 => "f32",- _ => panic!("Unsupported data type: {:?}", dtype),- }-}--pub(crate) fn dtype_index(dtype: DataType) -> usize {- match dtype {- DataType::F16 => 0,- DataType::BF16 => 1,- DataType::F32 => 2,- _ => panic!("Unsupported data type: {:?}", dtype),- }-}diff --git a/crates/uzu/tests/moe/moe_block_e2e_test.rs b/crates/uzu/tests/moe/moe_block_e2e_test.rsindex c8dd522ee..f0a936e24 100644--- a/crates/uzu/tests/moe/moe_block_e2e_test.rs+++ b/crates/uzu/tests/moe/moe_block_e2e_test.rs@@ -11,7 +11,7 @@ use uzu::{MoeBlockBasesArguments, MoeScatterKernels, MoeScatterWithMapArguments,dsl::{MoeCountsOffsetsFusedMetalKernel, MoeFinalizeMetalKernel},moe::{- MoeExpertsTwoPassArguments, MoeExpertsTwoPassPrefillKernel, MoeGatherArguments, MoeGatherKernels,+ MoeExpertsTwoPassArguments, MoeExpertsTwoPassPrefillBlock, MoeGatherArguments, MoeGatherKernels,MoeRouterTopKArguments, MoeRouterTopKKernel,},},@@ -434,41 +434,37 @@ fn run_moe_parity_test_internal(let hidden_buf = alloc_buffer::<f32>(&ctx, total_rows * d_ff);let row_expert_map_buf = alloc_buffer::<u32>(&ctx, total_rows);- let experts = MoeExpertsTwoPassPrefillKernel::new(&ctx).expect("experts");+ let experts = MoeExpertsTwoPassPrefillBlock::new(&ctx).expect("experts");let num_tiles_k = ((d_ff + 64 - 1) / 64) as u32;- experts- .encode(- &cb,- MoeExpertsTwoPassArguments {- x_perm_buffer: &x_perm_buf,- expert_offsets: &offsets_buf,- row_expert_map: &row_expert_map_buf,- hidden_buffer: &hidden_buf,- output_buffer: &y_partial_buf,- w13_all: &w13_buf,- w2_all: &w2_buf,- up_biases: &up_biases_buf,- down_biases: &down_biases_buf,- tile_counts: &tile_counts_buf,- tile_offsets: &tile_offsets_buf,- tile_map: &tile_map_buf,- total_tiles: &total_tiles_buf,- dispatch_args: &dispatch_args_buf,- total_rows,- d_model,- d_ff,- e,- num_tiles_k,- gating_code,- gate_clip_min: gate_clip.0,- gate_clip_max: gate_clip.1,- up_clip_min: up_clip.0,- up_clip_max: up_clip.1,- silu_alpha,- data_type: DataType::BF16,- },- )- .expect("experts encode");+ let args = MoeExpertsTwoPassArguments {+ x_perm_buffer: &x_perm_buf,+ expert_offsets: &offsets_buf,+ row_expert_map: &row_expert_map_buf,+ hidden_buffer: &hidden_buf,+ output_buffer: &y_partial_buf,+ w13_all: &w13_buf,+ w2_all: &w2_buf,+ up_biases: &up_biases_buf,+ down_biases: &down_biases_buf,+ tile_counts: &tile_counts_buf,+ tile_offsets: &tile_offsets_buf,+ tile_map: &tile_map_buf,+ total_tiles: &total_tiles_buf,+ dispatch_args: &dispatch_args_buf,+ total_rows,+ d_model,+ d_ff,+ e,+ num_tiles_k,+ gating_code,+ gate_clip_min: gate_clip.0,+ gate_clip_max: gate_clip.1,+ up_clip_min: up_clip.0,+ up_clip_max: up_clip.1,+ silu_alpha,+ data_type: DataType::BF16,+ };+ experts.encode(&cb, &args);let finalize = MoeFinalizeMetalKernel::new(&ctx, DataType::BF16).expect("finalize");let encoder = cb.new_compute_command_encoder().expect("encoder");diff --git a/crates/uzu/tests/moe/moe_experts_perf_test.rs b/crates/uzu/tests/moe/moe_experts_perf_test.rsindex ba2d602c2..2bb58c14f 100644--- a/crates/uzu/tests/moe/moe_experts_perf_test.rs+++ b/crates/uzu/tests/moe/moe_experts_perf_test.rs@@ -9,7 +9,7 @@ use uzu::{MetalContext,kernel::moe::{MoeExpertsSingleDecodeArguments, MoeExpertsSingleDecodeKernels, MoeExpertsTwoPassArguments,- MoeExpertsTwoPassDecodeKernels, MoeExpertsTwoPassPrefillKernel,+ MoeExpertsTwoPassDecodeBlock, MoeExpertsTwoPassPrefillBlock,},},};@@ -76,7 +76,7 @@ fn run_decode_case(let up_biases: Vec<bf16> = (0..e * 2 * d_ff).map(|_| bf16::from_f32(rng.random_range(-0.01..0.01))).collect();let down_biases: Vec<bf16> = (0..e * d_model).map(|_| bf16::from_f32(rng.random_range(-0.01..0.01))).collect();- let experts_kernel = MoeExpertsTwoPassDecodeKernels::new(ctx).expect("experts decode kernel");+ let experts_kernel = MoeExpertsTwoPassDecodeBlock::new(ctx).expect("experts decode kernel");let x_perm_buf = alloc_buffer_with_data(&ctx, &x_perm);let offsets_buf = alloc_buffer_with_data(&ctx, &offsets);@@ -213,7 +213,7 @@ fn run_two_pass_prefill_case(let up_biases: Vec<bf16> = (0..e * 2 * d_ff).map(|_| bf16::from_f32(rng.random_range(-0.01..0.01))).collect();let down_biases: Vec<bf16> = (0..e * d_model).map(|_| bf16::from_f32(rng.random_range(-0.01..0.01))).collect();- let experts_kernel = MoeExpertsTwoPassPrefillKernel::new(ctx).expect("experts prefill kernel");+ let experts_kernel = MoeExpertsTwoPassPrefillBlock::new(ctx).expect("experts prefill kernel");let x_perm_buf = alloc_buffer_with_data(&ctx, &x_perm);let offsets_buf = alloc_buffer_with_data(&ctx, &offsets);@@ -266,7 +266,7 @@ fn run_two_pass_prefill_case(for _ in 0..warmup {let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");- experts_kernel.encode(&cb, make_two_pass_args()).expect("two-pass prefill encode");+ experts_kernel.encode(&cb, &make_two_pass_args());cb.commit();cb.wait_until_completed();}@@ -275,7 +275,7 @@ fn run_two_pass_prefill_case(for _ in 0..iters {let start = Instant::now();let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");- experts_kernel.encode(&cb, make_two_pass_args()).expect("two-pass prefill encode");+ experts_kernel.encode(&cb, &make_two_pass_args());cb.commit();cb.wait_until_completed();times.push(start.elapsed().as_secs_f64() * 1000.0);@@ -486,7 +486,7 @@ fn run_indirect_decode_timed(let up_biases: Vec<bf16> = (0..e * 2 * d_ff).map(|_| bf16::from_f32(rng.random_range(-0.01..0.01))).collect();let down_biases: Vec<bf16> = (0..e * d_model).map(|_| bf16::from_f32(rng.random_range(-0.01..0.01))).collect();- let experts_kernel = MoeExpertsTwoPassDecodeKernels::new(ctx).expect("decode kernel");+ let experts_kernel = MoeExpertsTwoPassDecodeBlock::new(ctx).expect("decode kernel");let x_perm_buf = alloc_buffer_with_data(ctx, &x_perm);let offsets_buf = alloc_buffer_with_data(ctx, &offsets);diff --git a/crates/uzu/tests/moe/moe_experts_test.rs b/crates/uzu/tests/moe/moe_experts_test.rsindex 663d0ed4b..6c566785e 100644--- a/crates/uzu/tests/moe/moe_experts_test.rs+++ b/crates/uzu/tests/moe/moe_experts_test.rs@@ -14,8 +14,8 @@ use rand::{RngExt, SeedableRng, rngs::StdRng};use uzu::{DataType,backends::metal::kernel::moe::{- MoeExpertsSingleDecodeKernels, MoeExpertsTwoPassArguments, MoeExpertsTwoPassDecodeKernels,- MoeExpertsTwoPassPrefillKernel,+ MoeExpertsSingleDecodeKernels, MoeExpertsTwoPassArguments, MoeExpertsTwoPassDecodeBlock,+ MoeExpertsTwoPassPrefillBlock,},};@@ -412,7 +412,7 @@ fn test_two_pass_decode_correctness() {let dispatch_args_buf = alloc_buffer::<u32>(&ctx, 3);// Execute 2-pass decode kernel- let experts_kernel = MoeExpertsTwoPassDecodeKernels::new(&ctx).expect("MoeExpertsTwoPassDecodeKernel::new");+ let experts_kernel = MoeExpertsTwoPassDecodeBlock::new(&ctx).expect("MoeExpertsTwoPassDecodeKernel::new");let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");const K_TILE: usize = 64;@@ -556,7 +556,7 @@ fn test_two_pass_decode_multi_token() {let tile_map_buf = alloc_buffer::<u32>(&ctx, max_total_tiles * 3);let dispatch_args_buf = alloc_buffer::<u32>(&ctx, 3);- let experts_kernel = MoeExpertsTwoPassDecodeKernels::new(&ctx).expect("kernel");+ let experts_kernel = MoeExpertsTwoPassDecodeBlock::new(&ctx).expect("kernel");let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");experts_kernel.encode(&cb,@@ -658,41 +658,37 @@ fn test_two_pass_prefill_correctness() {let tile_map_buf = alloc_buffer::<u32>(&ctx, max_total_tiles * 3);let dispatch_args_buf = alloc_buffer::<u32>(&ctx, 3);- let experts_kernel = MoeExpertsTwoPassPrefillKernel::new(&ctx).expect("kernel");+ let experts_kernel = MoeExpertsTwoPassPrefillBlock::new(&ctx).expect("kernel");let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");- experts_kernel- .encode(- &cb,- MoeExpertsTwoPassArguments {- x_perm_buffer: &x_perm_buf,- expert_offsets: &offsets_buf,- row_expert_map: &row_expert_map_buf,- hidden_buffer: &hidden_buf,- output_buffer: &y_partial_buf,- w13_all: &w13_buf,- w2_all: &w2_buf,- up_biases: &up_biases_buf,- down_biases: &down_biases_buf,- tile_counts: &tile_counts_buf,- tile_offsets: &tile_offsets_buf,- tile_map: &tile_map_buf,- total_tiles: &total_tiles_buf,- dispatch_args: &dispatch_args_buf,- total_rows: sum_k,- d_model,- d_ff,- e,- num_tiles_k: ((d_ff + 63) / 64) as u32,- gating_code,- gate_clip_min: f32::NEG_INFINITY,- gate_clip_max: f32::INFINITY,- up_clip_min: f32::NEG_INFINITY,- up_clip_max: f32::INFINITY,- silu_alpha,- data_type: DataType::BF16,- },- )- .expect("encode");+ let args = MoeExpertsTwoPassArguments {+ x_perm_buffer: &x_perm_buf,+ expert_offsets: &offsets_buf,+ row_expert_map: &row_expert_map_buf,+ hidden_buffer: &hidden_buf,+ output_buffer: &y_partial_buf,+ w13_all: &w13_buf,+ w2_all: &w2_buf,+ up_biases: &up_biases_buf,+ down_biases: &down_biases_buf,+ tile_counts: &tile_counts_buf,+ tile_offsets: &tile_offsets_buf,+ tile_map: &tile_map_buf,+ total_tiles: &total_tiles_buf,+ dispatch_args: &dispatch_args_buf,+ total_rows: sum_k,+ d_model,+ d_ff,+ e,+ num_tiles_k: ((d_ff + 63) / 64) as u32,+ gating_code,+ gate_clip_min: f32::NEG_INFINITY,+ gate_clip_max: f32::INFINITY,+ up_clip_min: f32::NEG_INFINITY,+ up_clip_max: f32::INFINITY,+ silu_alpha,+ data_type: DataType::BF16,+ };+ experts_kernel.encode(&cb, &args);cb.commit();cb.wait_until_completed();diff --git a/crates/uzu/tests/moe/moe_perf_test.rs b/crates/uzu/tests/moe/moe_perf_test.rsindex 5759b538e..ff54da09e 100644--- a/crates/uzu/tests/moe/moe_perf_test.rs+++ b/crates/uzu/tests/moe/moe_perf_test.rs@@ -10,7 +10,7 @@ use uzu::{metal::kernel::{dsl::{MoeCountsOffsetsFusedMetalKernel, MoeFinalizeMetalKernel},moe::{- MoeExpertsTwoPassArguments, MoeExpertsTwoPassDecodeKernels, MoeGatherArguments, MoeGatherKernels,+ MoeExpertsTwoPassArguments, MoeExpertsTwoPassDecodeBlock, MoeGatherArguments, MoeGatherKernels,MoeRouterTopKArguments, MoeRouterTopKKernel, MoeScatterKernels, MoeScatterWithMapArguments,},},@@ -300,7 +300,7 @@ fn test_moe_pipeline_breakdown_decode() {let counts_offsets_kernel = MoeCountsOffsetsFusedMetalKernel::new(&ctx).expect("counts+offsets fused");let scatter_kernel = MoeScatterKernels::new(&ctx).expect("scatter");let gather_kernel = MoeGatherKernels::new(&ctx).expect("gather");- let experts_kernel = MoeExpertsTwoPassDecodeKernels::new(&ctx).expect("experts two-pass decode");+ let experts_kernel = MoeExpertsTwoPassDecodeBlock::new(&ctx).expect("experts two-pass decode");let finalize_kernel = MoeFinalizeMetalKernel::new(&ctx, DataType::BF16).expect("finalize");let router_topk_fused_kernel = MoeRouterTopKKernel::new(&ctx).expect("router+topk fused");