A high-performance inference engine for AI models
generalize moe kernel wrappers and encodable block (#207)
uuuvn72ded1d2e5e1
diff --git a/crates/uzu/src/backends/common/copy_encoder.rs b/crates/uzu/src/backends/common/copy_encoder.rsindex f68665a2f..9b39ac447 100644--- a/crates/uzu/src/backends/common/copy_encoder.rs+++ b/crates/uzu/src/backends/common/copy_encoder.rs@@ -1,3 +1,5 @@+use std::ops::Range;+use super::Backend;pub trait CopyEncoder {@@ -9,4 +11,11 @@ pub trait CopyEncoder {dst: &<Self::Backend as Backend>::NativeBuffer,size: usize,);++ fn encode_fill(+ &self,+ dst: &<Self::Backend as Backend>::NativeBuffer,+ range: Range<usize>,+ value: u8,+ );}diff --git a/crates/uzu/src/backends/common/kernel/mod.rs b/crates/uzu/src/backends/common/kernel/mod.rsindex e97540287..b5a284288 100644--- a/crates/uzu/src/backends/common/kernel/mod.rs+++ b/crates/uzu/src/backends/common/kernel/mod.rs@@ -6,6 +6,7 @@ pub mod attention;pub mod kv_cache_update;pub mod matmul;pub mod mlp_gate_act_mul;+pub mod moe;pub mod quant_matmul;pub mod sampling;pub mod ssd_prefill;diff --git a/crates/uzu/src/backends/metal/kernel/moe/experts_single.rs b/crates/uzu/src/backends/common/kernel/moe/experts_single.rssimilarity index 52%rename from crates/uzu/src/backends/metal/kernel/moe/experts_single.rsrename to crates/uzu/src/backends/common/kernel/moe/experts_single.rsindex 9f1797b36..cc408d358 100644--- a/crates/uzu/src/backends/metal/kernel/moe/experts_single.rs+++ b/crates/uzu/src/backends/common/kernel/moe/experts_single.rs@@ -1,14 +1,8 @@-use metal::{MTLBuffer, MTLCommandBuffer, MTLCommandEncoder};-use objc2::{__framework_prelude::ProtocolObject, Message, rc::Retained};-use crate::{DataType,- backends::{- common::kernel::{MoeExpertsDecodeSinglePassAKernel as _, MoeExpertsDecodeSinglePassBKernel as _},- metal::{- MetalContext, MetalError,- kernel::dsl::{MoeExpertsDecodeSinglePassAMetalKernel, MoeExpertsDecodeSinglePassBMetalKernel},- },+ backends::common::{+ Backend, CommandBuffer, Kernels,+ kernel::{MoeExpertsDecodeSinglePassAKernel, MoeExpertsDecodeSinglePassBKernel},},};@@ -16,25 +10,25 @@ static DTYPES: [DataType; 3] = [DataType::F16, DataType::BF16, DataType::F32];/// Arguments for single-token MoE decode (T=1 optimized path)#[derive(Debug)]-pub struct MoeExpertsSingleDecodeArguments<'a> {+pub struct MoeExpertsSingleDecodeArguments<'a, B: Backend> {/// Input activation [d_model]- pub x: &'a ProtocolObject<dyn MTLBuffer>,+ pub x: &'a B::NativeBuffer,/// Top-K expert indices from router [K]- pub topk_ids: &'a ProtocolObject<dyn MTLBuffer>,+ pub topk_ids: &'a B::NativeBuffer,/// Top-K probabilities from router [K]- pub topk_probs: &'a ProtocolObject<dyn MTLBuffer>,+ pub topk_probs: &'a B::NativeBuffer,/// Up/gate projection weights [E, 2*d_ff, d_model]- pub w13_all: &'a ProtocolObject<dyn MTLBuffer>,+ pub w13_all: &'a B::NativeBuffer,/// Down projection weights [E, d_model, d_ff]- pub w2_all: &'a ProtocolObject<dyn MTLBuffer>,+ pub w2_all: &'a B::NativeBuffer,/// Up/gate biases [E, 2*d_ff]- pub up_biases: &'a ProtocolObject<dyn MTLBuffer>,+ pub up_biases: &'a B::NativeBuffer,/// Down biases [E, d_model]- pub down_biases: &'a ProtocolObject<dyn MTLBuffer>,+ pub down_biases: &'a B::NativeBuffer,/// Hidden buffer [K, d_ff] - intermediate storage (f32)- pub hidden: &'a ProtocolObject<dyn MTLBuffer>,+ pub hidden: &'a B::NativeBuffer,/// Final output [d_model]- pub y: &'a ProtocolObject<dyn MTLBuffer>,+ pub y: &'a B::NativeBuffer,/// Model dimensionpub d_model: usize,/// FFN hidden dimension@@ -57,18 +51,19 @@ pub struct MoeExpertsSingleDecodeArguments<'a> {pub data_type: DataType,}-pub struct MoeExpertsSingleDecodeKernels {- pass_a: Vec<Vec<MoeExpertsDecodeSinglePassAMetalKernel>>,- pass_b: Vec<MoeExpertsDecodeSinglePassBMetalKernel>,+pub struct MoeExpertsSingleDecodeKernels<B: Backend> {+ pass_a: Vec<Vec<<B::Kernels as Kernels>::MoeExpertsDecodeSinglePassAKernel>>,+ pass_b: Vec<<B::Kernels as Kernels>::MoeExpertsDecodeSinglePassBKernel>,}-impl MoeExpertsSingleDecodeKernels {- pub fn new(ctx: &MetalContext) -> Result<Self, MetalError> {+impl<B: Backend> MoeExpertsSingleDecodeKernels<B> {+ pub fn new(ctx: &B::Context) -> Result<Self, B::Error> {let mut pass_a = vec![];for gate in 0..4 {let mut kernels = vec![];for dtype in &DTYPES {- let kernel = MoeExpertsDecodeSinglePassAMetalKernel::new(ctx, (*dtype).into(), gate)?;+ let kernel =+ <B::Kernels as Kernels>::MoeExpertsDecodeSinglePassAKernel::new(ctx, (*dtype).into(), gate)?;kernels.push(kernel)}pass_a.push(kernels);@@ -76,7 +71,7 @@ impl MoeExpertsSingleDecodeKernels {let mut pass_b = Vec::with_capacity(DTYPES.len());for dtype in &DTYPES {- pass_b.push(MoeExpertsDecodeSinglePassBMetalKernel::new(ctx, (*dtype).into())?);+ pass_b.push(<B::Kernels as Kernels>::MoeExpertsDecodeSinglePassBKernel::new(ctx, (*dtype).into())?);}Ok(Self {@@ -87,8 +82,8 @@ impl MoeExpertsSingleDecodeKernels {pub fn encode(&self,- command_buffer: &Retained<ProtocolObject<dyn MTLCommandBuffer>>,- args: MoeExpertsSingleDecodeArguments,+ command_buffer: &B::CommandBuffer,+ args: MoeExpertsSingleDecodeArguments<B>,) {if args.k == 0 {return;@@ -98,16 +93,14 @@ impl MoeExpertsSingleDecodeKernels {let dtype_idx = DTYPES.iter().position(|dtype| *dtype == args.data_type).unwrap();// Pass A: x @ W13[expert] -> hidden- {+ command_buffer.with_compute_encoder(|encoder| {let kernel = &self.pass_a[gate_idx][dtype_idx];- let encoder =- command_buffer.new_compute_command_encoder().expect("Failed to create compute command encoder");kernel.encode(- &args.x.retain(),- &args.topk_ids.retain(),- &args.w13_all.retain(),- &args.up_biases.retain(),- &args.hidden.retain(),+ args.x,+ args.topk_ids,+ args.w13_all,+ args.up_biases,+ args.hidden,args.d_model as u32,args.d_ff as u32,args.k as u32,@@ -118,27 +111,23 @@ impl MoeExpertsSingleDecodeKernels {args.up_clip_max,&encoder,);- encoder.end_encoding();- }+ });// Pass B: 8 simdgroups (256 threads), outputs final y directly- {+ command_buffer.with_compute_encoder(|encoder| {let kernel = &self.pass_b[dtype_idx];- let encoder =- command_buffer.new_compute_command_encoder().expect("Failed to create compute command encoder");kernel.encode(- &args.hidden.retain(),- &args.topk_ids.retain(),- &args.topk_probs.retain(),- &args.w2_all.retain(),- &args.down_biases.retain(),- &args.y.retain(),+ args.hidden,+ args.topk_ids,+ args.topk_probs,+ args.w2_all,+ args.down_biases,+ args.y,args.d_model as u32,args.d_ff as u32,args.k as u32,&encoder,);- encoder.end_encoding();- }+ });}}diff --git a/crates/uzu/src/backends/common/kernel/moe/experts_two_pass_decode.rs b/crates/uzu/src/backends/common/kernel/moe/experts_two_pass_decode.rsnew file mode 100644index 000000000..9650dbe5d--- /dev/null+++ b/crates/uzu/src/backends/common/kernel/moe/experts_two_pass_decode.rs@@ -0,0 +1,148 @@+use super::{+ MoeExpertsTwoPassArguments, MoePassARowMapArguments, MoePassATileBuildArguments, MoePassATileCountsArguments,+ MoePassATileDispatchArguments, MoePassATileKernels, MoePassATileScanArguments,+};+use crate::{+ DataType,+ backends::common::{+ CommandBuffer,+ kernel::{Backend, Kernels, MoeExpertsDecodeDownFused2DKernel, MoeExpertsDecodePassAKernel},+ },+};++static DTYPES: [DataType; 3] = [DataType::F16, DataType::BF16, DataType::F32];++pub struct MoeExpertsTwoPassDecodeBlock<B: Backend> {+ pass_a_tile: MoePassATileKernels<B>,+ pass_a_indirect: Vec<Vec<<B::Kernels as Kernels>::MoeExpertsDecodePassAKernel>>,+ fused_down: Vec<<B::Kernels as Kernels>::MoeExpertsDecodeDownFused2DKernel>,+}++impl<B: Backend> MoeExpertsTwoPassDecodeBlock<B> {+ pub fn new(ctx: &B::Context) -> Result<Self, B::Error> {+ let mut pass_a_indirect = vec![];+ for gate in 0u32..4u32 {+ let mut kernels = vec![];+ for dtype in &DTYPES {+ let kernel = <B::Kernels as Kernels>::MoeExpertsDecodePassAKernel::new(ctx, *dtype, gate)?;+ kernels.push(kernel);+ }+ pass_a_indirect.push(kernels);+ }++ let mut fused_down = Vec::with_capacity(DTYPES.len());+ for dtype in &DTYPES {+ let kernel = <B::Kernels as Kernels>::MoeExpertsDecodeDownFused2DKernel::new(ctx, *dtype, DataType::F32)?;+ fused_down.push(kernel);+ }++ Ok(Self {+ pass_a_tile: MoePassATileKernels::new(ctx)?,+ pass_a_indirect,+ fused_down,+ })+ }++ pub fn encode(+ &self,+ command_buffer: &B::CommandBuffer,+ args: &MoeExpertsTwoPassArguments<B>,+ ) {+ if args.total_rows == 0 {+ return;+ }++ // pass a tile+ const BLOCK_M: u32 = 4;+ let h_blocks = (args.d_ff as u32 + BLOCK_M - 1) / BLOCK_M;+ self.pass_a_tile.encode_counts(+ command_buffer,+ &MoePassATileCountsArguments {+ expert_offsets: args.expert_offsets,+ tile_counts: args.tile_counts,+ e: args.e,+ h_blocks,+ },+ );+ self.pass_a_tile.encode_scan(+ command_buffer,+ &MoePassATileScanArguments {+ tile_counts: args.tile_counts,+ tile_offsets: args.tile_offsets,+ total_tiles: args.total_tiles,+ e: args.e,+ },+ );+ self.pass_a_tile.encode_row_map(+ command_buffer,+ &MoePassARowMapArguments {+ expert_offsets: args.expert_offsets,+ row_expert_map: args.row_expert_map,+ total_rows: args.total_rows,+ e: args.e,+ },+ );+ self.pass_a_tile.encode_build_map(+ command_buffer,+ &MoePassATileBuildArguments {+ expert_offsets: args.expert_offsets,+ tile_offsets: args.tile_offsets,+ row_expert_map: args.row_expert_map,+ tile_map: args.tile_map,+ total_rows: args.total_rows,+ h_blocks,+ },+ );+ self.pass_a_tile.encode_dispatch_args(+ command_buffer,+ &MoePassATileDispatchArguments {+ total_tiles: args.total_tiles,+ dispatch_args: args.dispatch_args,+ num_tiles_y: 1,+ },+ );++ let gate_idx = args.gating_code.min(3) as usize;+ let dtype_idx = DTYPES.iter().position(|dtype| *dtype == args.data_type).unwrap();++ // pass a+ command_buffer.with_compute_encoder(|encoder| {+ let pass_a_kernel = &self.pass_a_indirect[gate_idx][dtype_idx];+ pass_a_kernel.encode(+ args.x_perm_buffer,+ args.expert_offsets,+ args.w13_all,+ args.hidden_buffer,+ args.up_biases,+ args.d_model as u32,+ args.d_ff as u32,+ args.e as u32,+ args.gate_clip_min,+ args.gate_clip_max,+ args.up_clip_min,+ args.up_clip_max,+ args.silu_alpha,+ args.tile_map,+ args.dispatch_args,+ encoder,+ );+ });++ // pass b+ command_buffer.with_compute_encoder(|encoder| {+ let pass_b_kernel = &self.fused_down[dtype_idx];+ pass_b_kernel.encode(+ args.hidden_buffer,+ args.row_expert_map,+ args.w2_all,+ args.down_biases,+ args.output_buffer,+ args.total_rows as u32,+ args.d_model as u32,+ args.d_ff as u32,+ args.e as u32,+ encoder,+ );+ });+ }+}diff --git a/crates/uzu/src/backends/common/kernel/moe/experts_two_pass_prefill.rs b/crates/uzu/src/backends/common/kernel/moe/experts_two_pass_prefill.rsnew file mode 100644index 000000000..7936a116d--- /dev/null+++ b/crates/uzu/src/backends/common/kernel/moe/experts_two_pass_prefill.rs@@ -0,0 +1,188 @@+use super::{+ MoeTileCountsArguments, MoeTileDispatchArguments, MoeTileMapBuildArguments, MoeTileMapKernels, MoeTileScanArguments,+};+use crate::{+ DataType,+ backends::common::{+ Backend, CommandBuffer, CopyEncoder, Kernels,+ kernel::{MoeExpertsPrefillPassAKernel, MoeExpertsPrefillPassBKernel},+ },+};++const DTYPES: [DataType; 3] = [DataType::F16, DataType::BF16, DataType::F32];++pub struct MoeExpertsTwoPassPrefillBlock<B: Backend> {+ tile_map: MoeTileMapKernels<B>,+ pass_a_indirect: Vec<Vec<<B::Kernels as Kernels>::MoeExpertsPrefillPassAKernel>>, // [gate][dtype]+ pass_b_indirect: Vec<<B::Kernels as Kernels>::MoeExpertsPrefillPassBKernel>, // [dtype]+}++impl<B: Backend> MoeExpertsTwoPassPrefillBlock<B> {+ pub fn new(ctx: &B::Context) -> Result<Self, B::Error> {+ let dtypes = [DataType::F16, DataType::BF16, DataType::F32];++ let mut pass_a_indirect = vec![];+ for gate in 0u32..4u32 {+ let mut kernels = vec![];+ for dtype in dtypes {+ let kernel = <B::Kernels as Kernels>::MoeExpertsPrefillPassAKernel::new(ctx, dtype, gate)?;+ kernels.push(kernel);+ }+ pass_a_indirect.push(kernels);+ }++ let mut pass_b_indirect = vec![];+ for dtype in dtypes {+ let kernel = <B::Kernels as Kernels>::MoeExpertsPrefillPassBKernel::new(ctx, dtype)?;+ pass_b_indirect.push(kernel);+ }++ Ok(Self {+ tile_map: MoeTileMapKernels::new(ctx)?,+ pass_a_indirect,+ pass_b_indirect,+ })+ }++ pub fn encode(+ &self,+ command_buffer: &B::CommandBuffer,+ args: &MoeExpertsTwoPassArguments<B>,+ ) {+ if args.total_rows == 0 {+ return;+ }++ let hidden_bytes = args.total_rows * args.d_ff * args.data_type.size_in_bytes();+ command_buffer.with_copy_encoder(|encoder| encoder.encode_fill(args.hidden_buffer, 0..hidden_bytes, 0));++ self.tile_map.encode_counts(+ command_buffer,+ &MoeTileCountsArguments {+ offsets_buffer: args.expert_offsets,+ tile_counts_buffer: args.tile_counts,+ e: args.e,+ },+ );+ self.tile_map.encode_scan(+ command_buffer,+ &MoeTileScanArguments {+ tile_counts_buffer: args.tile_counts,+ tile_offsets_buffer: args.tile_offsets,+ total_tiles_buffer: args.total_tiles,+ e: args.e,+ },+ );+ self.tile_map.encode_build_map(+ command_buffer,+ &MoeTileMapBuildArguments {+ expert_offsets: args.expert_offsets,+ tile_offsets: args.tile_offsets,+ tile_counts: args.tile_counts,+ tile_map: args.tile_map,+ e: args.e,+ },+ );+ let d_model_u32 = args.d_model as u32;+ let d_ff_u32 = args.d_ff as u32;+ const COL_TILE_FF: u32 = 32; // Must match PASSA_BN in Metal kernel+ const COL_TILE_MODEL: u32 = 64;+ let n_tiles_ff = if d_ff_u32 == 0 {+ 0+ } else {+ (d_ff_u32 + COL_TILE_FF - 1) / COL_TILE_FF+ };+ let n_tiles_model = if d_model_u32 == 0 {+ 0+ } else {+ (d_model_u32 + COL_TILE_MODEL - 1) / COL_TILE_MODEL+ };+ self.tile_map.encode_dispatch_args(+ command_buffer,+ &MoeTileDispatchArguments {+ total_tiles: args.total_tiles,+ dispatch_args: args.dispatch_args,+ num_tiles_x: n_tiles_ff,+ },+ );++ let gate_idx = args.gating_code.min(3) as usize;+ let dtype_idx = DTYPES.iter().position(|t| *t == args.data_type).unwrap();++ command_buffer.with_compute_encoder(|encoder| {+ let kernel_pass_a = &self.pass_a_indirect[gate_idx][dtype_idx];+ kernel_pass_a.encode(+ args.x_perm_buffer,+ args.expert_offsets,+ args.w13_all,+ args.up_biases,+ args.hidden_buffer,+ args.d_model as u32,+ args.d_ff as u32,+ args.e as u32,+ args.gate_clip_min,+ args.gate_clip_max,+ args.up_clip_min,+ args.up_clip_max,+ args.silu_alpha,+ args.tile_map,+ args.dispatch_args,+ encoder,+ );+ });++ let dispatch_args = MoeTileDispatchArguments {+ total_tiles: args.total_tiles,+ dispatch_args: args.dispatch_args,+ num_tiles_x: n_tiles_model,+ };+ self.tile_map.encode_dispatch_args(command_buffer, &dispatch_args);++ command_buffer.with_compute_encoder(|encoder| {+ let kernel_pass_b = &self.pass_b_indirect[dtype_idx];+ kernel_pass_b.encode(+ args.hidden_buffer,+ args.expert_offsets,+ args.w2_all,+ args.down_biases,+ args.output_buffer,+ args.d_model as u32,+ args.d_ff as u32,+ args.e as u32,+ args.tile_map,+ args.dispatch_args,+ encoder,+ );+ });+ }+}++#[derive(Debug)]+pub struct MoeExpertsTwoPassArguments<'a, B: Backend> {+ pub x_perm_buffer: &'a B::NativeBuffer,+ pub expert_offsets: &'a B::NativeBuffer,+ pub row_expert_map: &'a B::NativeBuffer,+ pub hidden_buffer: &'a B::NativeBuffer,+ pub output_buffer: &'a B::NativeBuffer,+ pub w13_all: &'a B::NativeBuffer,+ pub w2_all: &'a B::NativeBuffer,+ pub up_biases: &'a B::NativeBuffer,+ pub down_biases: &'a B::NativeBuffer,+ pub tile_counts: &'a B::NativeBuffer,+ pub tile_offsets: &'a B::NativeBuffer,+ pub tile_map: &'a B::NativeBuffer,+ pub total_tiles: &'a B::NativeBuffer,+ pub dispatch_args: &'a B::NativeBuffer,+ pub total_rows: usize,+ pub d_model: usize,+ pub d_ff: usize,+ pub e: usize,+ pub num_tiles_k: u32,+ pub gating_code: u32,+ pub gate_clip_min: f32,+ pub gate_clip_max: f32,+ pub up_clip_min: f32,+ pub up_clip_max: f32,+ pub silu_alpha: f32,+ pub data_type: DataType,+}diff --git a/crates/uzu/src/backends/common/kernel/moe/gather.rs b/crates/uzu/src/backends/common/kernel/moe/gather.rsnew file mode 100644index 000000000..bd4bc07bb--- /dev/null+++ b/crates/uzu/src/backends/common/kernel/moe/gather.rs@@ -0,0 +1,75 @@+use crate::{+ DataType,+ backends::common::{+ Backend, CommandBuffer, Kernels,+ kernel::{MoeGatherXPerm1DKernel, MoeGatherXPerm2DKernel},+ },+};++#[derive(Debug)]+pub struct MoeGatherArguments<'a, B: Backend> {+ pub x_buffer: &'a B::NativeBuffer,+ pub bucketed_ids_buffer: &'a B::NativeBuffer,+ pub x_perm_buffer: &'a B::NativeBuffer,+ pub sumk_buffer: &'a B::NativeBuffer,+ pub t: usize,+ pub k: usize,+ pub d_model: usize,+}++pub struct MoeGatherKernels<B: Backend> {+ bf16: <B::Kernels as Kernels>::MoeGatherXPerm2DKernel,+ f16: <B::Kernels as Kernels>::MoeGatherXPerm1DKernel,+ f32: <B::Kernels as Kernels>::MoeGatherXPerm1DKernel,+}++impl<B: Backend> MoeGatherKernels<B> {+ pub fn new(ctx: &B::Context) -> Result<Self, B::Error> {+ Ok(Self {+ bf16: <B::Kernels as Kernels>::MoeGatherXPerm2DKernel::new(ctx, DataType::BF16)?,+ f16: <B::Kernels as Kernels>::MoeGatherXPerm1DKernel::new(ctx, DataType::F16)?,+ f32: <B::Kernels as Kernels>::MoeGatherXPerm1DKernel::new(ctx, DataType::F32)?,+ })+ }++ pub fn encode(+ &self,+ command_buffer: &B::CommandBuffer,+ dtype: DataType,+ args: &MoeGatherArguments<B>,+ ) {+ command_buffer.with_compute_encoder(|encoder| match dtype {+ DataType::F32 => self.f32.encode(+ args.x_buffer,+ args.bucketed_ids_buffer,+ args.x_perm_buffer,+ args.sumk_buffer,+ args.d_model as u32,+ args.t as u32,+ args.k as u32,+ encoder,+ ),+ DataType::F16 => self.f16.encode(+ args.x_buffer,+ args.bucketed_ids_buffer,+ args.x_perm_buffer,+ args.sumk_buffer,+ args.d_model as u32,+ args.t as u32,+ args.k as u32,+ encoder,+ ),+ DataType::BF16 => self.bf16.encode(+ args.x_buffer,+ args.bucketed_ids_buffer,+ args.x_perm_buffer,+ args.sumk_buffer,+ args.d_model as u32,+ args.t as u32,+ args.k as u32,+ encoder,+ ),+ _ => panic!("Unsupported data type: {:?}", dtype),+ });+ }+}diff --git a/crates/uzu/src/backends/metal/kernel/moe/mod.rs b/crates/uzu/src/backends/common/kernel/moe/mod.rssimilarity index 76%rename from crates/uzu/src/backends/metal/kernel/moe/mod.rsrename to crates/uzu/src/backends/common/kernel/moe/mod.rsindex beb45c942..1ac0db46a 100644--- a/crates/uzu/src/backends/metal/kernel/moe/mod.rs+++ b/crates/uzu/src/backends/common/kernel/moe/mod.rs@@ -1,16 +1,17 @@-// Submodulesmod experts_single;mod experts_two_pass_decode;mod experts_two_pass_prefill;mod gather;-pub mod tiles_map;-pub mod tiles_pass_a;+mod tiles_map;+mod tiles_pass_a;-// Re-export public items from submodulespub use experts_single::{MoeExpertsSingleDecodeArguments, MoeExpertsSingleDecodeKernels};pub use experts_two_pass_decode::MoeExpertsTwoPassDecodeBlock;pub use experts_two_pass_prefill::{MoeExpertsTwoPassArguments, MoeExpertsTwoPassPrefillBlock};pub use gather::{MoeGatherArguments, MoeGatherKernels};+pub use tiles_map::{+ MoeTileCountsArguments, MoeTileDispatchArguments, MoeTileMapBuildArguments, MoeTileMapKernels, MoeTileScanArguments,+};pub use tiles_pass_a::{MoePassARowMapArguments, MoePassATileBuildArguments, MoePassATileCountsArguments, MoePassATileDispatchArguments,MoePassATileKernels, MoePassATileScanArguments,diff --git a/crates/uzu/src/backends/common/kernel/moe/tiles_map.rs b/crates/uzu/src/backends/common/kernel/moe/tiles_map.rsnew file mode 100644index 000000000..3590cb360--- /dev/null+++ b/crates/uzu/src/backends/common/kernel/moe/tiles_map.rs@@ -0,0 +1,104 @@+use crate::backends::common::{+ Backend, CommandBuffer, Kernels,+ kernel::{MoeBuildTileMapKernel, MoeTileCountsKernel, MoeTileScanKernel, MoeWriteDispatchArgsKernel},+};++pub struct MoeTileMapKernels<B: Backend> {+ counts: <B::Kernels as Kernels>::MoeTileCountsKernel,+ scan: <B::Kernels as Kernels>::MoeTileScanKernel,+ build: <B::Kernels as Kernels>::MoeBuildTileMapKernel,+ dispatch: <B::Kernels as Kernels>::MoeWriteDispatchArgsKernel,+}++impl<B: Backend> MoeTileMapKernels<B> {+ pub fn new(ctx: &B::Context) -> Result<Self, B::Error> {+ Ok(Self {+ counts: <B::Kernels as Kernels>::MoeTileCountsKernel::new(ctx)?,+ scan: <B::Kernels as Kernels>::MoeTileScanKernel::new(ctx)?,+ build: <B::Kernels as Kernels>::MoeBuildTileMapKernel::new(ctx)?,+ dispatch: <B::Kernels as Kernels>::MoeWriteDispatchArgsKernel::new(ctx)?,+ })+ }++ pub fn encode_counts(+ &self,+ command_buffer: &B::CommandBuffer,+ args: &MoeTileCountsArguments<B>,+ ) {+ command_buffer.with_compute_encoder(|encoder| {+ self.counts.encode(args.offsets_buffer, args.tile_counts_buffer, args.e as u32, encoder);+ });+ }++ pub fn encode_scan(+ &self,+ command_buffer: &B::CommandBuffer,+ args: &MoeTileScanArguments<B>,+ ) {+ command_buffer.with_compute_encoder(|encoder| {+ self.scan.encode(+ args.tile_counts_buffer,+ args.tile_offsets_buffer,+ args.total_tiles_buffer,+ args.e as u32,+ encoder,+ );+ });+ }++ pub fn encode_build_map(+ &self,+ command_buffer: &B::CommandBuffer,+ args: &MoeTileMapBuildArguments<B>,+ ) {+ command_buffer.with_compute_encoder(|encoder| {+ self.build.encode(+ args.expert_offsets,+ args.tile_offsets,+ args.tile_counts,+ args.tile_map,+ args.e as u32,+ encoder,+ );+ });+ }++ pub fn encode_dispatch_args(+ &self,+ command_buffer: &B::CommandBuffer,+ args: &MoeTileDispatchArguments<B>,+ ) {+ command_buffer.with_compute_encoder(|encoder| {+ self.dispatch.encode(args.total_tiles, args.dispatch_args, args.num_tiles_x, encoder);+ });+ }+}++pub struct MoeTileCountsArguments<'a, B: Backend> {+ pub offsets_buffer: &'a B::NativeBuffer, // [E+1]+ pub tile_counts_buffer: &'a B::NativeBuffer, // [E]+ pub e: usize,+}++pub struct MoeTileScanArguments<'a, B: Backend> {+ pub tile_counts_buffer: &'a B::NativeBuffer, // [E]+ pub tile_offsets_buffer: &'a B::NativeBuffer, // [E+1]+ pub total_tiles_buffer: &'a B::NativeBuffer, // [>=2]+ pub e: usize,+}++#[derive(Debug)]+pub struct MoeTileMapBuildArguments<'a, B: Backend> {+ pub expert_offsets: &'a B::NativeBuffer, // [E+1]+ pub tile_offsets: &'a B::NativeBuffer, // [E+1]+ pub tile_counts: &'a B::NativeBuffer, // [E]+ pub tile_map: &'a B::NativeBuffer, // [total_tiles * 3]+ pub e: usize,+}++#[derive(Debug)]+pub struct MoeTileDispatchArguments<'a, B: Backend> {+ pub total_tiles: &'a B::NativeBuffer, // [>=1]+ pub dispatch_args: &'a B::NativeBuffer, // [3]+ pub num_tiles_x: u32, // x dimension for indirect dispatch+}diff --git a/crates/uzu/src/backends/common/kernel/moe/tiles_pass_a.rs b/crates/uzu/src/backends/common/kernel/moe/tiles_pass_a.rsnew file mode 100644index 000000000..43548c713--- /dev/null+++ b/crates/uzu/src/backends/common/kernel/moe/tiles_pass_a.rs@@ -0,0 +1,132 @@+use crate::backends::common::{+ CommandBuffer,+ kernel::{+ Backend, Kernels, MoePassABuildRowMapKernel, MoePassABuildTileMapKernel, MoePassATileCountsKernel,+ MoePassATileScanKernel, MoePassAWriteDispatchArgsKernel,+ },+};++#[derive(Debug)]+pub struct MoePassATileCountsArguments<'a, B: Backend> {+ pub expert_offsets: &'a B::NativeBuffer, // [E+1]+ pub tile_counts: &'a B::NativeBuffer, // [E]+ pub e: usize,+ pub h_blocks: u32,+}++#[derive(Debug)]+pub struct MoePassATileScanArguments<'a, B: Backend> {+ pub tile_counts: &'a B::NativeBuffer, // [E]+ pub tile_offsets: &'a B::NativeBuffer, // [E+1]+ pub total_tiles: &'a B::NativeBuffer, // [>=1]+ pub e: usize,+}++#[derive(Debug)]+pub struct MoePassARowMapArguments<'a, B: Backend> {+ pub expert_offsets: &'a B::NativeBuffer, // [E+1]+ pub row_expert_map: &'a B::NativeBuffer, // [total_rows]+ pub total_rows: usize,+ pub e: usize,+}++#[derive(Debug)]+pub struct MoePassATileBuildArguments<'a, B: Backend> {+ pub expert_offsets: &'a B::NativeBuffer, // [E+1]+ pub tile_offsets: &'a B::NativeBuffer, // [E+1]+ pub row_expert_map: &'a B::NativeBuffer, // [total_rows]+ pub tile_map: &'a B::NativeBuffer, // [total_tiles * 3]+ pub total_rows: usize,+ pub h_blocks: u32,+}++#[derive(Debug)]+pub struct MoePassATileDispatchArguments<'a, B: Backend> {+ pub total_tiles: &'a B::NativeBuffer, // [>=1]+ pub dispatch_args: &'a B::NativeBuffer, // [3]+ pub num_tiles_y: u32,+}++pub struct MoePassATileKernels<B: Backend> {+ counts: <B::Kernels as Kernels>::MoePassATileCountsKernel,+ scan: <B::Kernels as Kernels>::MoePassATileScanKernel,+ row_map: <B::Kernels as Kernels>::MoePassABuildRowMapKernel,+ build_map: <B::Kernels as Kernels>::MoePassABuildTileMapKernel,+ dispatch: <B::Kernels as Kernels>::MoePassAWriteDispatchArgsKernel,+}++impl<B: Backend> MoePassATileKernels<B> {+ pub fn new(ctx: &B::Context) -> Result<Self, B::Error> {+ Ok(Self {+ counts: <B::Kernels as Kernels>::MoePassATileCountsKernel::new(ctx)?,+ scan: <B::Kernels as Kernels>::MoePassATileScanKernel::new(ctx)?,+ row_map: <B::Kernels as Kernels>::MoePassABuildRowMapKernel::new(ctx)?,+ build_map: <B::Kernels as Kernels>::MoePassABuildTileMapKernel::new(ctx)?,+ dispatch: <B::Kernels as Kernels>::MoePassAWriteDispatchArgsKernel::new(ctx)?,+ })+ }++ pub fn encode_counts(+ &self,+ command_buffer: &B::CommandBuffer,+ args: &MoePassATileCountsArguments<B>,+ ) {+ command_buffer.with_compute_encoder(|encoder| {+ self.counts.encode(args.expert_offsets, args.tile_counts, args.e as u32, args.h_blocks, encoder);+ });+ }++ pub fn encode_scan(+ &self,+ command_buffer: &B::CommandBuffer,+ args: &MoePassATileScanArguments<B>,+ ) {+ command_buffer.with_compute_encoder(|encoder| {+ self.scan.encode(args.tile_counts, args.tile_offsets, args.total_tiles, args.e as u32, encoder);+ });+ }++ pub fn encode_row_map(+ &self,+ command_buffer: &B::CommandBuffer,+ args: &MoePassARowMapArguments<B>,+ ) {+ command_buffer.with_compute_encoder(|encoder| {+ self.row_map.encode(+ args.expert_offsets,+ args.row_expert_map,+ args.total_rows as u32,+ args.e as u32,+ encoder,+ );+ });+ }++ pub fn encode_build_map(+ &self,+ command_buffer: &B::CommandBuffer,+ args: &MoePassATileBuildArguments<B>,+ ) {+ command_buffer.with_compute_encoder(|encoder| {+ self.build_map.encode(+ args.expert_offsets,+ args.tile_offsets,+ args.row_expert_map,+ args.tile_map,+ args.total_rows as u32,+ args.h_blocks,+ encoder,+ );+ });+ }++ pub fn encode_dispatch_args(+ &self,+ command_buffer: &B::CommandBuffer,+ args: &MoePassATileDispatchArguments<B>,+ ) {+ command_buffer.with_compute_encoder(|encoder| {+ self.dispatch.encode(args.total_tiles, args.dispatch_args, args.num_tiles_y, encoder);+ });+ }+}diff --git a/crates/uzu/src/backends/metal/copy_encoder.rs b/crates/uzu/src/backends/metal/copy_encoder.rsindex ab9bcaab2..12cf03bff 100644--- a/crates/uzu/src/backends/metal/copy_encoder.rs+++ b/crates/uzu/src/backends/metal/copy_encoder.rs@@ -1,20 +1,34 @@-use metal::{MTLBlitCommandEncoder, MTLBuffer};+use std::ops::Range;++use metal::{MTLBlitCommandEncoder, MTLBlitCommandEncoderExt, MTLBuffer};use objc2::runtime::ProtocolObject;use super::Metal;-use crate::backends::common::CopyEncoder;+use crate::backends::common::{Backend, CopyEncoder};impl CopyEncoder for ProtocolObject<dyn MTLBlitCommandEncoder> {type Backend = Metal;fn encode_copy(&self,- src: &<Self::Backend as crate::backends::common::Backend>::NativeBuffer,- dst: &<Self::Backend as crate::backends::common::Backend>::NativeBuffer,+ src: &<Self::Backend as Backend>::NativeBuffer,+ dst: &<Self::Backend as Backend>::NativeBuffer,size: usize,) {assert!(src.length() >= size && dst.length() >= size);self.copy_buffer_to_buffer(src, 0, dst, 0, size);}++ fn encode_fill(+ &self,+ dst: &<Self::Backend as Backend>::NativeBuffer,+ range: Range<usize>,+ value: u8,+ ) {+ assert!(range.end > range.start && range.end <= dst.length());+ assert!(range.start % 4 == 0 && range.end % 4 == 0);++ self.fill_buffer_range_value(dst, range, value);+ }}diff --git a/crates/uzu/src/backends/metal/encodable_block/mod.rs b/crates/uzu/src/backends/metal/encodable_block/mod.rsindex ebcf7ef7d..155c30798 100644--- a/crates/uzu/src/backends/metal/encodable_block/mod.rs+++ b/crates/uzu/src/backends/metal/encodable_block/mod.rs@@ -11,7 +11,6 @@ mod decoder;mod embedding;mod layer;mod linear;-mod moe_block;pub mod transformer_layer;pub use classifier_layer::ClassifierLayer;@@ -22,4 +21,3 @@ pub use embedding::{};pub use layer::LayerExecutables;pub use linear::{FullPrecisionLinear, QuantizedLinear};-pub use moe_block::{MoeBlock, SharedMoeWeights};diff --git a/crates/uzu/src/backends/metal/encodable_block/moe_block/mod.rs b/crates/uzu/src/backends/metal/encodable_block/moe_block/mod.rsdeleted file mode 100644index 7e91b8451..000000000--- a/crates/uzu/src/backends/metal/encodable_block/moe_block/mod.rs+++ /dev/null@@ -1,7 +0,0 @@-//! MoE (Mixture of Experts) encodables.--mod moe_block;-mod shared_weights;--pub use moe_block::MoeBlock;-pub use shared_weights::SharedMoeWeights;diff --git a/crates/uzu/src/backends/metal/encodable_block/moe_block/shared_weights.rs b/crates/uzu/src/backends/metal/encodable_block/moe_block/shared_weights.rsdeleted file mode 100644index bc51a10bd..000000000--- a/crates/uzu/src/backends/metal/encodable_block/moe_block/shared_weights.rs+++ /dev/null@@ -1,14 +0,0 @@-//! Shared MoE expert weights.--use std::rc::Rc;--use metal::MTLBuffer;-use objc2::{rc::Retained, runtime::ProtocolObject};--#[derive(Clone)]-pub struct SharedMoeWeights {- pub w13_buf: Rc<Retained<ProtocolObject<dyn MTLBuffer>>>,- pub w2_buf: Rc<Retained<ProtocolObject<dyn MTLBuffer>>>,- pub up_biases_buf: Rc<Retained<ProtocolObject<dyn MTLBuffer>>>,- pub down_biases_buf: Rc<Retained<ProtocolObject<dyn MTLBuffer>>>,-}diff --git a/crates/uzu/src/backends/metal/encodable_block/transformer_layer.rs b/crates/uzu/src/backends/metal/encodable_block/transformer_layer.rsindex dcc4987c2..35ebe71f5 100644--- a/crates/uzu/src/backends/metal/encodable_block/transformer_layer.rs+++ b/crates/uzu/src/backends/metal/encodable_block/transformer_layer.rs@@ -1,6 +1,6 @@use super::{- FullPrecisionEmbeddingLookup, FullPrecisionEmbeddingReadout, FullPrecisionLinear, MoeBlock,- QuantizedEmbeddingLookup, QuantizedEmbeddingReadout, QuantizedLinear,+ FullPrecisionEmbeddingLookup, FullPrecisionEmbeddingReadout, FullPrecisionLinear, QuantizedEmbeddingLookup,+ QuantizedEmbeddingReadout, QuantizedLinear,};use crate::{DataType,@@ -9,7 +9,7 @@ use crate::{metal::{Metal, MetalContext, MetalError},},config::{DecoderConfig, EmbeddingConfig, LinearConfig, MLPConfig},- encodable_block::{EncodableBlock, MlpBlock},+ encodable_block::{EncodableBlock, MlpBlock, MoeBlock},forward_pass::state::ArrayId,parameters::ParameterTree,};@@ -104,7 +104,8 @@ pub fn mlp_block(if let crate::config::MLPConfig::MixtureOfExperts(mixture_of_experts_config) = config {let mixture_of_experts_block =- MoeBlock::new(context, mixture_of_experts_config, model_dimension, hidden_dimension, parameter_tree)?;+ MoeBlock::new(context, mixture_of_experts_config, model_dimension, hidden_dimension, parameter_tree)+ .map_err(|error| MetalError::Generic(format!("{:?}", error)))?;return Ok(Box::new(mixture_of_experts_block));}diff --git a/crates/uzu/src/backends/metal/kernel/mod.rs b/crates/uzu/src/backends/metal/kernel/mod.rsindex 0323968b5..bd94bb20f 100644--- a/crates/uzu/src/backends/metal/kernel/mod.rs+++ b/crates/uzu/src/backends/metal/kernel/mod.rs@@ -5,13 +5,8 @@ pub mod dsl {pub(super) const MTLB: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/default.metallib"));pub mod matmul;-pub mod moe;pub use matmul::{MatmulArguments, MatmulKernel};-pub use moe::{- MoeExpertsTwoPassArguments, MoeExpertsTwoPassDecodeBlock, MoeExpertsTwoPassPrefillBlock, MoeGatherArguments,- MoeGatherKernels,-};impl MatmulKernels for dsl::MetalKernels {type FullPrecisionMatmulKernel = MatmulKernel;diff --git a/crates/uzu/src/backends/metal/kernel/moe/experts_two_pass_decode.rs b/crates/uzu/src/backends/metal/kernel/moe/experts_two_pass_decode.rsdeleted file mode 100644index 8a07282f8..000000000--- a/crates/uzu/src/backends/metal/kernel/moe/experts_two_pass_decode.rs+++ /dev/null@@ -1,161 +0,0 @@-use metal::{MTLCommandBuffer, MTLCommandEncoder};-use objc2::{- __framework_prelude::{ProtocolObject, Retained},- Message,-};--use crate::{- DataType,- backends::{- common::kernel::{MoeExpertsDecodeDownFused2DKernel, MoeExpertsDecodePassAKernel},- metal::{- MetalContext, MetalError,- kernel::{- MoeExpertsTwoPassArguments,- dsl::{MoeExpertsDecodeDownFused2DMetalKernel, MoeExpertsDecodePassAMetalKernel},- moe::{- MoePassARowMapArguments, MoePassATileBuildArguments, MoePassATileCountsArguments,- MoePassATileDispatchArguments, MoePassATileKernels, MoePassATileScanArguments,- },- },- },- },-};--static DTYPES: [DataType; 3] = [DataType::F16, DataType::BF16, DataType::F32];--pub struct MoeExpertsTwoPassDecodeBlock {- pass_a_tile: MoePassATileKernels,- pass_a_indirect: Vec<Vec<MoeExpertsDecodePassAMetalKernel>>,- fused_down: Vec<MoeExpertsDecodeDownFused2DMetalKernel>,-}--impl MoeExpertsTwoPassDecodeBlock {- pub fn new(ctx: &MetalContext) -> Result<Self, MetalError> {- let mut pass_a_indirect = vec![];- for gate in 0u32..4u32 {- let mut kernels = vec![];- for dtype in &DTYPES {- let kernel = MoeExpertsDecodePassAMetalKernel::new(ctx, *dtype, gate)?;- kernels.push(kernel);- }- pass_a_indirect.push(kernels);- }-- let mut fused_down = Vec::with_capacity(DTYPES.len());- for dtype in &DTYPES {- let kernel = MoeExpertsDecodeDownFused2DMetalKernel::new(ctx, *dtype, DataType::F32)?;- fused_down.push(kernel);- }-- Ok(Self {- pass_a_tile: MoePassATileKernels::new(ctx)?,- pass_a_indirect,- fused_down,- })- }-- pub fn encode(- &self,- command_buffer: &Retained<ProtocolObject<dyn MTLCommandBuffer>>,- args: &MoeExpertsTwoPassArguments,- ) {- if args.total_rows == 0 {- return;- }-- // pass a tile- const BLOCK_M: u32 = 4;- let h_blocks = (args.d_ff as u32 + BLOCK_M - 1) / BLOCK_M;- self.pass_a_tile.encode_counts(- command_buffer,- &MoePassATileCountsArguments {- expert_offsets: args.expert_offsets,- tile_counts: args.tile_counts,- e: args.e,- h_blocks,- },- );- self.pass_a_tile.encode_scan(- command_buffer,- &MoePassATileScanArguments {- tile_counts: args.tile_counts,- tile_offsets: args.tile_offsets,- total_tiles: args.total_tiles,- e: args.e,- },- );- self.pass_a_tile.encode_row_map(- command_buffer,- &MoePassARowMapArguments {- expert_offsets: args.expert_offsets,- row_expert_map: args.row_expert_map,- total_rows: args.total_rows,- e: args.e,- },- );- self.pass_a_tile.encode_build_map(- command_buffer,- &MoePassATileBuildArguments {- expert_offsets: args.expert_offsets,- tile_offsets: args.tile_offsets,- row_expert_map: args.row_expert_map,- tile_map: args.tile_map,- total_rows: args.total_rows,- h_blocks,- },- );- self.pass_a_tile.encode_dispatch_args(- command_buffer,- &MoePassATileDispatchArguments {- total_tiles: args.total_tiles,- dispatch_args: args.dispatch_args,- num_tiles_y: 1,- },- );-- // pass a- let gate_idx = args.gating_code.min(3) as usize;- let dtype_idx = DTYPES.iter().position(|dtype| *dtype == args.data_type).unwrap();- let pass_a_encoder =- command_buffer.new_compute_command_encoder().expect("Failed to create compute command encoder");- let pass_a_kernel = &self.pass_a_indirect[gate_idx][dtype_idx];- pass_a_kernel.encode(- &args.x_perm_buffer.retain(),- &args.expert_offsets.retain(),- &args.w13_all.retain(),- &args.hidden_buffer.retain(),- &args.up_biases.retain(),- args.d_model as u32,- args.d_ff as u32,- args.e as u32,- args.gate_clip_min,- args.gate_clip_max,- args.up_clip_min,- args.up_clip_max,- args.silu_alpha,- &args.tile_map.retain(),- &args.dispatch_args.retain(),- &pass_a_encoder,- );- pass_a_encoder.end_encoding();-- // pass b- let pass_b_kernel = &self.fused_down[dtype_idx];- let pass_b_encoder =- command_buffer.new_compute_command_encoder().expect("Failed to create compute command encoder");- pass_b_kernel.encode(- &args.hidden_buffer.retain(),- &args.row_expert_map.retain(),- &args.w2_all.retain(),- &args.down_biases.retain(),- &args.output_buffer.retain(),- args.total_rows as u32,- args.d_model as u32,- args.d_ff as u32,- args.e as u32,- &pass_b_encoder,- );- pass_b_encoder.end_encoding();- }-}diff --git a/crates/uzu/src/backends/metal/kernel/moe/experts_two_pass_prefill.rs b/crates/uzu/src/backends/metal/kernel/moe/experts_two_pass_prefill.rsdeleted file mode 100644index a1e28cf39..000000000--- a/crates/uzu/src/backends/metal/kernel/moe/experts_two_pass_prefill.rs+++ /dev/null@@ -1,203 +0,0 @@-use metal::{MTLBlitCommandEncoderExt, MTLBuffer, MTLCommandBuffer, MTLCommandEncoder};-use objc2::{- __framework_prelude::{ProtocolObject, Retained},- Message,-};--use crate::{- DataType,- backends::{- common::kernel::{MoeExpertsPrefillPassAKernel, MoeExpertsPrefillPassBKernel},- metal::{- MetalContext, MetalError,- kernel::{- dsl::{MoeExpertsPrefillPassAMetalKernel, MoeExpertsPrefillPassBMetalKernel},- moe::tiles_map::{- MoeTileCountsArguments, MoeTileDispatchArguments, MoeTileMapBuildArguments, MoeTileMapKernels,- MoeTileScanArguments,- },- },- },- },-};--const DTYPES: [DataType; 3] = [DataType::F16, DataType::BF16, DataType::F32];--pub struct MoeExpertsTwoPassPrefillBlock {- tile_map: MoeTileMapKernels,- pass_a_indirect: Vec<Vec<MoeExpertsPrefillPassAMetalKernel>>, // [gate][dtype]- pass_b_indirect: Vec<MoeExpertsPrefillPassBMetalKernel>, // [dtype]-}--impl MoeExpertsTwoPassPrefillBlock {- pub fn new(ctx: &MetalContext) -> Result<Self, MetalError> {- let dtypes = [DataType::F16, DataType::BF16, DataType::F32];-- let mut pass_a_indirect = vec![];- for gate in 0u32..4u32 {- let mut kernels = vec![];- for dtype in dtypes {- let kernel = MoeExpertsPrefillPassAMetalKernel::new(ctx, dtype, gate)?;- kernels.push(kernel);- }- pass_a_indirect.push(kernels);- }-- let mut pass_b_indirect = vec![];- for dtype in dtypes {- let kernel = MoeExpertsPrefillPassBMetalKernel::new(ctx, dtype)?;- pass_b_indirect.push(kernel);- }-- Ok(Self {- tile_map: MoeTileMapKernels::new(ctx)?,- pass_a_indirect,- pass_b_indirect,- })- }-- pub fn encode(- &self,- command_buffer: &Retained<ProtocolObject<dyn MTLCommandBuffer>>,- args: &MoeExpertsTwoPassArguments,- ) {- if args.total_rows == 0 {- return;- }-- let hidden_bytes = args.total_rows * args.d_ff * args.data_type.size_in_bytes();- let blit_encoder = command_buffer.new_blit_command_encoder().expect("Failed to create blit command encoder");- blit_encoder.fill_buffer_range_value(args.hidden_buffer, 0..hidden_bytes, 0);- blit_encoder.end_encoding();-- self.tile_map.encode_counts(- command_buffer,- &MoeTileCountsArguments {- offsets_buffer: args.expert_offsets,- tile_counts_buffer: args.tile_counts,- e: args.e,- },- );- self.tile_map.encode_scan(- command_buffer,- &MoeTileScanArguments {- tile_counts_buffer: args.tile_counts,- tile_offsets_buffer: args.tile_offsets,- total_tiles_buffer: args.total_tiles,- e: args.e,- },- );- self.tile_map.encode_build_map(- command_buffer,- &MoeTileMapBuildArguments {- expert_offsets: args.expert_offsets,- tile_offsets: args.tile_offsets,- tile_counts: args.tile_counts,- tile_map: args.tile_map,- e: args.e,- },- );- let d_model_u32 = args.d_model as u32;- let d_ff_u32 = args.d_ff as u32;- const COL_TILE_FF: u32 = 32; // Must match PASSA_BN in Metal kernel- const COL_TILE_MODEL: u32 = 64;- let n_tiles_ff = if d_ff_u32 == 0 {- 0- } else {- (d_ff_u32 + COL_TILE_FF - 1) / COL_TILE_FF- };- let n_tiles_model = if d_model_u32 == 0 {- 0- } else {- (d_model_u32 + COL_TILE_MODEL - 1) / COL_TILE_MODEL- };- self.tile_map.encode_dispatch_args(- command_buffer,- &MoeTileDispatchArguments {- total_tiles: args.total_tiles,- dispatch_args: args.dispatch_args,- num_tiles_x: n_tiles_ff,- },- );-- let encoder_a =- command_buffer.new_compute_command_encoder().expect("Failed to create compute command encoder A");- let gate_idx = args.gating_code.min(3) as usize;- let dtype_idx = DTYPES.iter().position(|t| *t == args.data_type).unwrap();- let kernel_pass_a = &self.pass_a_indirect[gate_idx][dtype_idx];- kernel_pass_a.encode(- &args.x_perm_buffer.retain(),- &args.expert_offsets.retain(),- &args.w13_all.retain(),- &args.up_biases.retain(),- &args.hidden_buffer.retain(),- args.d_model as u32,- args.d_ff as u32,- args.e as u32,- args.gate_clip_min,- args.gate_clip_max,- args.up_clip_min,- args.up_clip_max,- args.silu_alpha,- &args.tile_map.retain(),- &args.dispatch_args.retain(),- &encoder_a,- );- encoder_a.end_encoding();-- let dispatch_args = MoeTileDispatchArguments {- total_tiles: args.total_tiles,- dispatch_args: args.dispatch_args,- num_tiles_x: n_tiles_model,- };- self.tile_map.encode_dispatch_args(command_buffer, &dispatch_args);-- let encoder_b =- command_buffer.new_compute_command_encoder().expect("Failed to create compute command encoder B");- let kernel_pass_b = &self.pass_b_indirect[dtype_idx];- kernel_pass_b.encode(- &args.hidden_buffer.retain(),- &args.expert_offsets.retain(),- &args.w2_all.retain(),- &args.down_biases.retain(),- &args.output_buffer.retain(),- args.d_model as u32,- args.d_ff as u32,- args.e as u32,- &args.tile_map.retain(),- &args.dispatch_args.retain(),- &encoder_b,- );- encoder_b.end_encoding();- }-}--#[derive(Debug)]-pub struct MoeExpertsTwoPassArguments<'a> {- pub x_perm_buffer: &'a ProtocolObject<dyn MTLBuffer>,- pub expert_offsets: &'a ProtocolObject<dyn MTLBuffer>,- pub row_expert_map: &'a ProtocolObject<dyn MTLBuffer>,- pub hidden_buffer: &'a ProtocolObject<dyn MTLBuffer>,- pub output_buffer: &'a ProtocolObject<dyn MTLBuffer>,- pub w13_all: &'a ProtocolObject<dyn MTLBuffer>,- pub w2_all: &'a ProtocolObject<dyn MTLBuffer>,- pub up_biases: &'a ProtocolObject<dyn MTLBuffer>,- pub down_biases: &'a ProtocolObject<dyn MTLBuffer>,- pub tile_counts: &'a ProtocolObject<dyn MTLBuffer>,- pub tile_offsets: &'a ProtocolObject<dyn MTLBuffer>,- pub tile_map: &'a ProtocolObject<dyn MTLBuffer>,- pub total_tiles: &'a ProtocolObject<dyn MTLBuffer>,- pub dispatch_args: &'a ProtocolObject<dyn MTLBuffer>,- pub total_rows: usize,- pub d_model: usize,- pub d_ff: usize,- pub e: usize,- pub num_tiles_k: u32,- pub gating_code: u32,- pub gate_clip_min: f32,- pub gate_clip_max: f32,- pub up_clip_min: f32,- pub up_clip_max: f32,- pub silu_alpha: f32,- pub data_type: DataType,-}diff --git a/crates/uzu/src/backends/metal/kernel/moe/gather.rs b/crates/uzu/src/backends/metal/kernel/moe/gather.rsdeleted file mode 100644index 3229d21be..000000000--- a/crates/uzu/src/backends/metal/kernel/moe/gather.rs+++ /dev/null@@ -1,83 +0,0 @@-use metal::{MTLBuffer, MTLCommandBuffer, MTLCommandEncoder};-use objc2::{Message, rc::Retained, runtime::ProtocolObject};--use crate::{- DataType,- backends::{- common::kernel::{MoeGatherXPerm1DKernel, MoeGatherXPerm2DKernel},- metal::{- MetalContext, MetalError,- kernel::dsl::{MoeGatherXPerm1DMetalKernel, MoeGatherXPerm2DMetalKernel},- },- },-};--#[derive(Debug)]-pub struct MoeGatherArguments<'a> {- pub x_buffer: &'a ProtocolObject<dyn MTLBuffer>,- pub bucketed_ids_buffer: &'a ProtocolObject<dyn MTLBuffer>,- pub x_perm_buffer: &'a ProtocolObject<dyn MTLBuffer>,- pub sumk_buffer: &'a ProtocolObject<dyn MTLBuffer>,- pub t: usize,- pub k: usize,- pub d_model: usize,-}--pub struct MoeGatherKernels {- bf16: MoeGatherXPerm2DMetalKernel,- f16: MoeGatherXPerm1DMetalKernel,- f32: MoeGatherXPerm1DMetalKernel,-}--impl MoeGatherKernels {- pub fn new(ctx: &MetalContext) -> Result<Self, MetalError> {- Ok(Self {- bf16: MoeGatherXPerm2DMetalKernel::new(ctx, DataType::BF16)?,- f16: MoeGatherXPerm1DMetalKernel::new(ctx, DataType::F16)?,- f32: MoeGatherXPerm1DMetalKernel::new(ctx, DataType::F32)?,- })- }-- pub fn encode(- &self,- command_buffer: &Retained<ProtocolObject<dyn MTLCommandBuffer>>,- dtype: DataType,- args: &MoeGatherArguments,- ) {- let encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute command encoder");- match dtype {- DataType::F32 => self.f32.encode(- &args.x_buffer.retain(),- &args.bucketed_ids_buffer.retain(),- &args.x_perm_buffer.retain(),- &args.sumk_buffer.retain(),- args.d_model as u32,- args.t as u32,- args.k as u32,- &encoder,- ),- DataType::F16 => self.f16.encode(- &args.x_buffer.retain(),- &args.bucketed_ids_buffer.retain(),- &args.x_perm_buffer.retain(),- &args.sumk_buffer.retain(),- args.d_model as u32,- args.t as u32,- args.k as u32,- &encoder,- ),- DataType::BF16 => self.bf16.encode(- &args.x_buffer.retain(),- &args.bucketed_ids_buffer.retain(),- &args.x_perm_buffer.retain(),- &args.sumk_buffer.retain(),- args.d_model as u32,- args.t as u32,- args.k as u32,- &encoder,- ),- _ => panic!("Unsupported data type: {:?}", dtype),- }- encoder.end_encoding();- }-}diff --git a/crates/uzu/src/backends/metal/kernel/moe/tiles_map.rs b/crates/uzu/src/backends/metal/kernel/moe/tiles_map.rsdeleted file mode 100644index 1f9ef7de5..000000000--- a/crates/uzu/src/backends/metal/kernel/moe/tiles_map.rs+++ /dev/null@@ -1,116 +0,0 @@-use metal::{MTLBuffer, MTLCommandBuffer, MTLCommandEncoder};-use objc2::{- __framework_prelude::{ProtocolObject, Retained},- Message,-};--use crate::backends::{- common::kernel::{MoeBuildTileMapKernel, MoeTileCountsKernel, MoeTileScanKernel, MoeWriteDispatchArgsKernel},- metal::{- MetalContext, MetalError,- kernel::dsl::{- MoeBuildTileMapMetalKernel, MoeTileCountsMetalKernel, MoeTileScanMetalKernel,- MoeWriteDispatchArgsMetalKernel,- },- },-};--pub struct MoeTileMapKernels {- counts: MoeTileCountsMetalKernel,- scan: MoeTileScanMetalKernel,- build: MoeBuildTileMapMetalKernel,- dispatch: MoeWriteDispatchArgsMetalKernel,-}--impl MoeTileMapKernels {- pub fn new(ctx: &MetalContext) -> Result<Self, MetalError> {- Ok(Self {- counts: MoeTileCountsMetalKernel::new(ctx)?,- scan: MoeTileScanMetalKernel::new(ctx)?,- build: MoeBuildTileMapMetalKernel::new(ctx)?,- dispatch: MoeWriteDispatchArgsMetalKernel::new(ctx)?,- })- }-- pub fn encode_counts(- &self,- command_buffer: &Retained<ProtocolObject<dyn MTLCommandBuffer>>,- args: &MoeTileCountsArguments,- ) {- let encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute command encoder");- self.counts.encode(&args.offsets_buffer.retain(), &args.tile_counts_buffer.retain(), args.e as u32, &encoder);- encoder.end_encoding();- }-- pub fn encode_scan(- &self,- command_buffer: &Retained<ProtocolObject<dyn MTLCommandBuffer>>,- args: &MoeTileScanArguments,- ) {- let encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute command encoder");- self.scan.encode(- &args.tile_counts_buffer.retain(),- &args.tile_offsets_buffer.retain(),- &args.total_tiles_buffer.retain(),- args.e as u32,- &encoder,- );- encoder.end_encoding();- }-- pub fn encode_build_map(- &self,- command_buffer: &Retained<ProtocolObject<dyn MTLCommandBuffer>>,- args: &MoeTileMapBuildArguments,- ) {- let encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute command encoder");- self.build.encode(- &args.expert_offsets.retain(),- &args.tile_offsets.retain(),- &args.tile_counts.retain(),- &args.tile_map.retain(),- args.e as u32,- &encoder,- );- encoder.end_encoding();- }-- pub fn encode_dispatch_args(- &self,- command_buffer: &Retained<ProtocolObject<dyn MTLCommandBuffer>>,- args: &MoeTileDispatchArguments,- ) {- let encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute command encoder");- self.dispatch.encode(&args.total_tiles.retain(), &args.dispatch_args.retain(), args.num_tiles_x, &encoder);- encoder.end_encoding();- }-}--pub struct MoeTileCountsArguments<'a> {- pub offsets_buffer: &'a ProtocolObject<dyn MTLBuffer>, // [E+1]- pub tile_counts_buffer: &'a ProtocolObject<dyn MTLBuffer>, // [E]- pub e: usize,-}--pub struct MoeTileScanArguments<'a> {- pub tile_counts_buffer: &'a ProtocolObject<dyn MTLBuffer>, // [E]- pub tile_offsets_buffer: &'a ProtocolObject<dyn MTLBuffer>, // [E+1]- pub total_tiles_buffer: &'a ProtocolObject<dyn MTLBuffer>, // [>=2]- pub e: usize,-}--#[derive(Debug)]-pub struct MoeTileMapBuildArguments<'a> {- pub expert_offsets: &'a ProtocolObject<dyn MTLBuffer>, // [E+1]- pub tile_offsets: &'a ProtocolObject<dyn MTLBuffer>, // [E+1]- pub tile_counts: &'a ProtocolObject<dyn MTLBuffer>, // [E]- pub tile_map: &'a ProtocolObject<dyn MTLBuffer>, // [total_tiles * 3]- pub e: usize,-}--#[derive(Debug)]-pub struct MoeTileDispatchArguments<'a> {- pub total_tiles: &'a ProtocolObject<dyn MTLBuffer>, // [>=1]- pub dispatch_args: &'a ProtocolObject<dyn MTLBuffer>, // [3]- pub num_tiles_x: u32, // x dimension for indirect dispatch-}diff --git a/crates/uzu/src/backends/metal/kernel/moe/tiles_pass_a.rs b/crates/uzu/src/backends/metal/kernel/moe/tiles_pass_a.rsdeleted file mode 100644index c2deccd86..000000000--- a/crates/uzu/src/backends/metal/kernel/moe/tiles_pass_a.rs+++ /dev/null@@ -1,153 +0,0 @@-use metal::{MTLBuffer, MTLCommandBuffer, MTLCommandEncoder};-use objc2::{Message, runtime::ProtocolObject};--use crate::backends::{- common::kernel::{- MoePassABuildRowMapKernel, MoePassABuildTileMapKernel, MoePassATileCountsKernel, MoePassATileScanKernel,- MoePassAWriteDispatchArgsKernel,- },- metal::{- MetalContext, MetalError,- kernel::dsl::{- MoePassABuildRowMapMetalKernel, MoePassABuildTileMapMetalKernel, MoePassATileCountsMetalKernel,- MoePassATileScanMetalKernel, MoePassAWriteDispatchArgsMetalKernel,- },- },-};--#[derive(Debug)]-pub struct MoePassATileCountsArguments<'a> {- pub expert_offsets: &'a ProtocolObject<dyn MTLBuffer>, // [E+1]- pub tile_counts: &'a ProtocolObject<dyn MTLBuffer>, // [E]- pub e: usize,- pub h_blocks: u32,-}--#[derive(Debug)]-pub struct MoePassATileScanArguments<'a> {- pub tile_counts: &'a ProtocolObject<dyn MTLBuffer>, // [E]- pub tile_offsets: &'a ProtocolObject<dyn MTLBuffer>, // [E+1]- pub total_tiles: &'a ProtocolObject<dyn MTLBuffer>, // [>=1]- pub e: usize,-}--#[derive(Debug)]-pub struct MoePassARowMapArguments<'a> {- pub expert_offsets: &'a ProtocolObject<dyn MTLBuffer>, // [E+1]- pub row_expert_map: &'a ProtocolObject<dyn MTLBuffer>, // [total_rows]- pub total_rows: usize,- pub e: usize,-}--#[derive(Debug)]-pub struct MoePassATileBuildArguments<'a> {- pub expert_offsets: &'a ProtocolObject<dyn MTLBuffer>, // [E+1]- pub tile_offsets: &'a ProtocolObject<dyn MTLBuffer>, // [E+1]- pub row_expert_map: &'a ProtocolObject<dyn MTLBuffer>, // [total_rows]- pub tile_map: &'a ProtocolObject<dyn MTLBuffer>, // [total_tiles * 3]- pub total_rows: usize,- pub h_blocks: u32,-}--#[derive(Debug)]-pub struct MoePassATileDispatchArguments<'a> {- pub total_tiles: &'a ProtocolObject<dyn MTLBuffer>, // [>=1]- pub dispatch_args: &'a ProtocolObject<dyn MTLBuffer>, // [3]- pub num_tiles_y: u32,-}--pub struct MoePassATileKernels {- counts: MoePassATileCountsMetalKernel,- scan: MoePassATileScanMetalKernel,- row_map: MoePassABuildRowMapMetalKernel,- build_map: MoePassABuildTileMapMetalKernel,- dispatch: MoePassAWriteDispatchArgsMetalKernel,-}--impl MoePassATileKernels {- pub fn new(ctx: &MetalContext) -> Result<Self, MetalError> {- Ok(Self {- counts: MoePassATileCountsMetalKernel::new(ctx)?,- scan: MoePassATileScanMetalKernel::new(ctx)?,- row_map: MoePassABuildRowMapMetalKernel::new(ctx)?,- build_map: MoePassABuildTileMapMetalKernel::new(ctx)?,- dispatch: MoePassAWriteDispatchArgsMetalKernel::new(ctx)?,- })- }-- pub fn encode_counts(- &self,- command_buffer: &ProtocolObject<dyn MTLCommandBuffer>,- args: &MoePassATileCountsArguments,- ) {- let encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute command encoder");- self.counts.encode(- &args.expert_offsets.retain(),- &args.tile_counts.retain(),- args.e as u32,- args.h_blocks,- &encoder,- );- encoder.end_encoding();- }-- pub fn encode_scan(- &self,- command_buffer: &ProtocolObject<dyn MTLCommandBuffer>,- args: &MoePassATileScanArguments,- ) {- let encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute command encoder");- self.scan.encode(- &args.tile_counts.retain(),- &args.tile_offsets.retain(),- &args.total_tiles.retain(),- args.e as u32,- &encoder,- );- encoder.end_encoding();- }-- pub fn encode_row_map(- &self,- command_buffer: &ProtocolObject<dyn MTLCommandBuffer>,- args: &MoePassARowMapArguments,- ) {- let encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute command encoder");- self.row_map.encode(- &args.expert_offsets.retain(),- &args.row_expert_map.retain(),- args.total_rows as u32,- args.e as u32,- &encoder,- );- encoder.end_encoding();- }-- pub fn encode_build_map(- &self,- command_buffer: &ProtocolObject<dyn MTLCommandBuffer>,- args: &MoePassATileBuildArguments,- ) {- let encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute command encoder");- self.build_map.encode(- &args.expert_offsets.retain(),- &args.tile_offsets.retain(),- &args.row_expert_map.retain(),- &args.tile_map.retain(),- args.total_rows as u32,- args.h_blocks,- &encoder,- );- encoder.end_encoding();- }-- pub fn encode_dispatch_args(- &self,- command_buffer: &ProtocolObject<dyn MTLCommandBuffer>,- args: &MoePassATileDispatchArguments,- ) {- let encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute command encoder");- self.dispatch.encode(&args.total_tiles.retain(), &args.dispatch_args.retain(), args.num_tiles_y, &encoder);- encoder.end_encoding();- }-}diff --git a/crates/uzu/src/encodable_block/mod.rs b/crates/uzu/src/encodable_block/mod.rsindex 1aaf72743..9911ca737 100644--- a/crates/uzu/src/encodable_block/mod.rs+++ b/crates/uzu/src/encodable_block/mod.rs@@ -11,6 +11,7 @@ mod full_precision_linear;mod layer_norm;mod mamba_mixer;mod mlp;+mod moe_block;mod normalization;mod pooling;mod prediction_head;@@ -33,6 +34,7 @@ pub use full_precision_linear::{FullPrecisionLinear, FullPrecisionLinearError};pub use layer_norm::{LayerNorm, LayerNormError};pub(crate) use mamba_mixer::MambaMixer;pub use mlp::MlpBlock;+pub use moe_block::MoeBlock;pub use normalization::{Normalization, NormalizationError};pub use pooling::Pooling;pub use prediction_head::ClassifierPredictionHead;diff --git a/crates/uzu/src/backends/metal/encodable_block/moe_block/moe_block.rs b/crates/uzu/src/encodable_block/moe_block.rssimilarity index 52%rename from crates/uzu/src/backends/metal/encodable_block/moe_block/moe_block.rsrename to crates/uzu/src/encodable_block/moe_block.rsindex a5cbbca4a..321e28f32 100644--- a/crates/uzu/src/backends/metal/encodable_block/moe_block/moe_block.rs+++ b/crates/uzu/src/encodable_block/moe_block.rs@@ -2,80 +2,81 @@use std::{cell::RefCell, rc::Rc};-use metal::{MTLBlitCommandEncoderExt, MTLBuffer, MTLCommandBuffer, MTLCommandEncoder, MTLComputeCommandEncoder};-use objc2::{rc::Retained, runtime::ProtocolObject};+use thiserror::Error;-use super::SharedMoeWeights;use crate::{Activation, DataType, LinearConfig, MixtureOfExpertsConfig, RoutingFunctionConfig,array::Array,- backends::{- common::kernel::{+ backends::common::{+ Backend, CommandBuffer, CopyEncoder, Kernels,+ kernel::{MoeBlockBasesFromPartialsKernel, MoeCountsOffsetsFusedKernel, MoeFinalizeKernel, MoeRouterTopKKernel,MoeScatterBucketsMapKernel,- },- metal::{- Metal, MetalContext,- kernel::{- MoeGatherKernels,- dsl::{- MoeBlockBasesFromPartialsMetalKernel, MoeCountsOffsetsFusedMetalKernel, MoeFinalizeMetalKernel,- MoeRouterTopKMetalKernel, MoeScatterBucketsMapMetalKernel,- },- moe::{- MoeExpertsTwoPassArguments, MoeExpertsTwoPassDecodeBlock, MoeExpertsTwoPassPrefillBlock,- MoeGatherArguments,- },+ moe::{+ MoeExpertsTwoPassArguments, MoeExpertsTwoPassDecodeBlock, MoeExpertsTwoPassPrefillBlock,+ MoeGatherArguments, MoeGatherKernels,},},},encodable_block::{EncodableBlock, EncodingParameters},forward_pass::state::{ArrayId, ForwardPassState},- parameters::ParameterTree,+ parameters::{ParameterLoaderError, ParameterTree},};-enum RouterBlock {- Metal {- weights_buf: Retained<ProtocolObject<dyn MTLBuffer>>,- biases_buf: Retained<ProtocolObject<dyn MTLBuffer>>,- },+struct RouterBlock<B: Backend> {+ weights_buf: B::NativeBuffer,+ biases_buf: B::NativeBuffer,}-pub struct MoeBlock {- router: RouterBlock,+#[derive(Clone)]+struct SharedMoeWeights<B: Backend> {+ pub w13_buf: Rc<B::NativeBuffer>,+ pub w2_buf: Rc<B::NativeBuffer>,+ pub up_biases_buf: Rc<B::NativeBuffer>,+ pub down_biases_buf: Rc<B::NativeBuffer>,+}++pub struct MoeBlock<B: Backend> {+ router: RouterBlock<B>,router_renorm: bool,- router_topk_kernel: MoeRouterTopKMetalKernel,- counts_offsets_kernel: MoeCountsOffsetsFusedMetalKernel,- scatter_bases_kernel: MoeBlockBasesFromPartialsMetalKernel,- scatter_map_kernel: MoeScatterBucketsMapMetalKernel,- gather_kernels: MoeGatherKernels,- experts_two_pass_decode_kernel: MoeExpertsTwoPassDecodeBlock,- experts_two_pass_prefill_kernel: MoeExpertsTwoPassPrefillBlock,- finalize_kernel: MoeFinalizeMetalKernel,+ router_topk_kernel: <B::Kernels as Kernels>::MoeRouterTopKKernel,+ counts_offsets_kernel: <B::Kernels as Kernels>::MoeCountsOffsetsFusedKernel,+ scatter_bases_kernel: <B::Kernels as Kernels>::MoeBlockBasesFromPartialsKernel,+ scatter_map_kernel: <B::Kernels as Kernels>::MoeScatterBucketsMapKernel,+ gather_kernels: MoeGatherKernels<B>,+ experts_two_pass_decode_kernel: MoeExpertsTwoPassDecodeBlock<B>,+ experts_two_pass_prefill_kernel: MoeExpertsTwoPassPrefillBlock<B>,+ finalize_kernel: <B::Kernels as Kernels>::MoeFinalizeKernel,moe_config: MixtureOfExpertsConfig,model_dim: usize,hidden_dim: usize,data_type: DataType,- shared_weights: SharedMoeWeights,+ shared_weights: SharedMoeWeights<B>,}-impl MoeBlock {+#[derive(Debug, Error)]+pub enum MoeBlockError<B: Backend> {+ #[error("Backend error: {0}")]+ BackendError(#[source] B::Error),+ #[error("Parameter loader error: {0}")]+ ParameterLoaderError(#[source] ParameterLoaderError),+}++impl<B: Backend> MoeBlock<B> {pub fn new(- context: &MetalContext,+ context: &B::Context,moe_config: &MixtureOfExpertsConfig,model_dim: usize,hidden_dim: usize,- parameter_tree: &ParameterTree<MetalContext>,- ) -> Result<Self, crate::backends::metal::MetalError> {+ parameter_tree: &ParameterTree<B::Context>,+ ) -> Result<Self, MoeBlockError<B>> {let data_type: DataType = moe_config.expert_config.linear_config.activation_precision().into();let router_data_type: DataType = moe_config.router_config.activation_precision().into();let router_renorm = matches!(moe_config.routing_function, RoutingFunctionConfig::SoftmaxRouting);- let router_tree = parameter_tree- .subtree("router")- .map_err(|e| crate::backends::metal::MetalError::Generic(format!("Router subtree error: {:?}", e)))?;+ let router_tree = parameter_tree.subtree("router").map_err(MoeBlockError::ParameterLoaderError)?;let router = match &moe_config.router_config {LinearConfig::Quantized(_) => {@@ -87,19 +88,15 @@ impl MoeBlock {LinearConfig::FullPrecision {..} => {- let weights_arr = router_tree.leaf("weights").map_err(|e| {- crate::backends::metal::MetalError::Generic(format!("Router weights error: {:?}", e))- })?;+ let weights_arr = router_tree.leaf("weights").map_err(MoeBlockError::ParameterLoaderError)?;let weights_buf = weights_arr.buffer();- let biases_arr = router_tree.leaf("biases").map_err(|e| {- crate::backends::metal::MetalError::Generic(format!("Router biases error: {:?}", e))- })?;+ let biases_arr = router_tree.leaf("biases").map_err(MoeBlockError::ParameterLoaderError)?;let biases_buf = biases_arr.buffer();- RouterBlock::Metal {- weights_buf: weights_buf.into(),- biases_buf: biases_buf.into(),+ RouterBlock {+ weights_buf: weights_buf.clone(),+ biases_buf: biases_buf.clone(),}},LinearConfig::QLoRA {@@ -109,64 +106,51 @@ impl MoeBlock {},};- let router_topk_kernel = MoeRouterTopKMetalKernel::new(context, router_data_type).map_err(|e| {- crate::backends::metal::MetalError::Generic(format!("RouterTopK fused kernel error: {:?}", e))- })?;- let counts_offsets_kernel = MoeCountsOffsetsFusedKernel::new(context).map_err(|e| {- crate::backends::metal::MetalError::Generic(format!("Counts+offsets kernel error: {:?}", e))- })?;-- let scatter_bases_kernel = MoeBlockBasesFromPartialsMetalKernel::new(context).map_err(|e| {- crate::backends::metal::MetalError::Generic(format!(- "MoeBlockBasesFromPartialsMetalKernel kernel error: {:?}",- e- ))- })?;- let scatter_map_kernel = MoeScatterBucketsMapMetalKernel::new(context, data_type).map_err(|e| {- crate::backends::metal::MetalError::Generic(format!("MoeScatterBucketsMetalKernel kernel error: {:?}", e))- })?;-- let gather_kernels = MoeGatherKernels::new(context)- .map_err(|e| crate::backends::metal::MetalError::Generic(format!("Gather kernel error: {:?}", e)))?;- let experts_two_pass_decode_kernel = MoeExpertsTwoPassDecodeBlock::new(context).map_err(|e| {- crate::backends::metal::MetalError::Generic(format!("Experts two-pass decode kernel error: {:?}", e))- })?;- let experts_two_pass_prefill_kernel = MoeExpertsTwoPassPrefillBlock::new(context).map_err(|e| {- crate::backends::metal::MetalError::Generic(format!("Experts two-pass prefill kernel error: {:?}", e))- })?;- let finalize_kernel = MoeFinalizeMetalKernel::new(context, data_type)- .map_err(|e| crate::backends::metal::MetalError::Generic(format!("Finalize kernel error: {:?}", e)))?;-- let experts_tree = parameter_tree- .subtree("experts")- .map_err(|e| crate::backends::metal::MetalError::Generic(format!("experts subtree error: {:?}", e)))?;+ let router_topk_kernel = <B::Kernels as Kernels>::MoeRouterTopKKernel::new(context, router_data_type)+ .map_err(MoeBlockError::BackendError)?;+ let counts_offsets_kernel = MoeCountsOffsetsFusedKernel::new(context).map_err(MoeBlockError::BackendError)?;++ let scatter_bases_kernel = <B::Kernels as Kernels>::MoeBlockBasesFromPartialsKernel::new(context)+ .map_err(MoeBlockError::BackendError)?;+ let scatter_map_kernel = <B::Kernels as Kernels>::MoeScatterBucketsMapKernel::new(context, data_type)+ .map_err(MoeBlockError::BackendError)?;++ let gather_kernels = MoeGatherKernels::new(context).map_err(MoeBlockError::BackendError)?;+ let experts_two_pass_decode_kernel =+ MoeExpertsTwoPassDecodeBlock::new(context).map_err(MoeBlockError::BackendError)?;+ let experts_two_pass_prefill_kernel =+ MoeExpertsTwoPassPrefillBlock::new(context).map_err(MoeBlockError::BackendError)?;+ let finalize_kernel =+ <B::Kernels as Kernels>::MoeFinalizeKernel::new(context, data_type).map_err(MoeBlockError::BackendError)?;++ let experts_tree = parameter_tree.subtree("experts").map_err(MoeBlockError::ParameterLoaderError)?;let w13_arr = experts_tree.subtree("up_projection")- .map_err(|e| crate::backends::metal::MetalError::Generic(format!("up_projection error: {:?}", e)))?+ .map_err(MoeBlockError::ParameterLoaderError)?.leaf("weights")- .map_err(|e| crate::backends::metal::MetalError::Generic(format!("up weights error: {:?}", e)))?;+ .map_err(MoeBlockError::ParameterLoaderError)?;let w13_buf = w13_arr.buffer().clone();let w2_arr = experts_tree.subtree("down_projection")- .map_err(|e| crate::backends::metal::MetalError::Generic(format!("down_projection error: {:?}", e)))?+ .map_err(MoeBlockError::ParameterLoaderError)?.leaf("weights")- .map_err(|e| crate::backends::metal::MetalError::Generic(format!("down weights error: {:?}", e)))?;+ .map_err(MoeBlockError::ParameterLoaderError)?;let w2_buf = w2_arr.buffer().clone();let up_biases_arr = experts_tree.subtree("up_projection")- .map_err(|e| crate::backends::metal::MetalError::Generic(format!("up_projection biases error: {:?}", e)))?+ .map_err(MoeBlockError::ParameterLoaderError)?.leaf("biases")- .map_err(|e| crate::backends::metal::MetalError::Generic(format!("up biases error: {:?}", e)))?;+ .map_err(MoeBlockError::ParameterLoaderError)?;let up_biases_buf = up_biases_arr.buffer().clone();let down_biases_arr = experts_tree.subtree("down_projection")- .map_err(|e| crate::backends::metal::MetalError::Generic(format!("down_projection biases error: {:?}", e)))?+ .map_err(MoeBlockError::ParameterLoaderError)?.leaf("biases")- .map_err(|e| crate::backends::metal::MetalError::Generic(format!("down biases error: {:?}", e)))?;+ .map_err(MoeBlockError::ParameterLoaderError)?;let down_biases_buf = down_biases_arr.buffer().clone();let shared_weights = SharedMoeWeights {@@ -208,12 +192,12 @@ impl MoeBlock {}}-impl EncodableBlock<Metal> for MoeBlock {+impl<B: Backend> EncodableBlock<B> for MoeBlock<B> {fn encode(&self,- state: &mut ForwardPassState<Metal>,- parameters: &EncodingParameters<Metal>,- command_buffer: &Retained<ProtocolObject<dyn MTLCommandBuffer>>,+ state: &mut ForwardPassState<B>,+ parameters: &EncodingParameters<B>,+ command_buffer: &B::CommandBuffer,) {let suffix_length = state.active_suffix_length();let arrays = state.arrays(&[@@ -239,9 +223,7 @@ impl EncodableBlock<Metal> for MoeBlock {ArrayId::MoeTwoPassRowExpertMap,]);- let clone_buffer = |array: &RefCell<Array<Metal>>| -> Retained<ProtocolObject<dyn MTLBuffer>> {- array.borrow().buffer().to_owned().into()- };+ let clone_buffer = |array: &RefCell<Array<B>>| -> B::NativeBuffer { array.borrow().buffer().to_owned().into() };let mut array_iter = arrays.iter();let main_buf = clone_buffer(array_iter.next().unwrap());@@ -273,83 +255,71 @@ impl EncodableBlock<Metal> for MoeBlock {let k_tile = 128;// Clear internal MoE buffers- let blit_encoder = root.new_blit_command_encoder().expect("Failed to create blit command encoder");-- if suffix_length > 0 && k > 0 {- let entries = suffix_length * k;- let topk_bytes = entries * std::mem::size_of::<u32>();- let tok2row_bytes = entries * std::mem::size_of::<i32>();-- // Clear topk_ids and tok2row buffers- if topk_bytes > 0 {- blit_encoder.fill_buffer_range_value(&topk_ids_buf, 0..topk_bytes, 0xFF);- }- if tok2row_bytes > 0 {- blit_encoder.fill_buffer_range_value(&tok2row_buf, 0..tok2row_bytes, 0xFF);- }+ root.with_copy_encoder(|encoder| {+ if suffix_length > 0 && k > 0 {+ let entries = suffix_length * k;+ let topk_bytes = entries * std::mem::size_of::<u32>();+ let tok2row_bytes = entries * std::mem::size_of::<i32>();++ // Clear topk_ids and tok2row buffers+ if topk_bytes > 0 {+ encoder.encode_fill(&topk_ids_buf, 0..topk_bytes, 0xFF);+ }+ if tok2row_bytes > 0 {+ encoder.encode_fill(&tok2row_buf, 0..tok2row_bytes, 0xFF);+ }- // Clear hidden buffer- let hidden_bytes = suffix_length * k * self.hidden_dim * self.data_type.size_in_bytes();- if hidden_bytes > 0 {- blit_encoder.fill_buffer_range_value(&hidden_buf, 0..hidden_bytes, 0);- }+ // Clear hidden buffer+ let hidden_bytes = suffix_length * k * self.hidden_dim * self.data_type.size_in_bytes();+ if hidden_bytes > 0 {+ encoder.encode_fill(&hidden_buf, 0..hidden_bytes, 0);+ }- // Clear y_partial buffer- let y_partial_bytes = suffix_length * k * self.model_dim * self.data_type.size_in_bytes();- if y_partial_bytes > 0 {- blit_encoder.fill_buffer_range_value(&y_partial_buf, 0..y_partial_bytes, 0);- }+ // Clear y_partial buffer+ let y_partial_bytes = suffix_length * k * self.model_dim * self.data_type.size_in_bytes();+ if y_partial_bytes > 0 {+ encoder.encode_fill(&y_partial_buf, 0..y_partial_bytes, 0);+ }- // Clear x_perm buffer- let x_perm_bytes = suffix_length * k * self.model_dim * self.data_type.size_in_bytes();- if x_perm_bytes > 0 {- blit_encoder.fill_buffer_range_value(&x_perm_buf, 0..x_perm_bytes, 0);+ // Clear x_perm buffer+ let x_perm_bytes = suffix_length * k * self.model_dim * self.data_type.size_in_bytes();+ if x_perm_bytes > 0 {+ encoder.encode_fill(&x_perm_buf, 0..x_perm_bytes, 0);+ }}- }-- blit_encoder.end_encoding();+ });// Use fused Router+TopK kernel for non-quantized routers- match &self.router {- RouterBlock::Metal {- weights_buf,- biases_buf,- } => {- if self.model_dim % 4 != 0 {- panic!("MoE fused router+topk failed: {} % 4 != 0", self.model_dim);- }- if e > 512 {- panic!("MoE fused router+topk failed: {e} > 512");- }- if k > 128 {- panic!("MoE fused router+bottomk failed: {k} > 128");- }+ if self.model_dim % 4 != 0 {+ panic!("MoE fused router+topk failed: {} % 4 != 0", self.model_dim);+ }+ if e > 512 {+ panic!("MoE fused router+topk failed: {e} > 512");+ }+ if k > 128 {+ panic!("MoE fused router+bottomk failed: {k} > 128");+ }- if suffix_length > 0 && e > 0 && k > 0 {- let encoder =- command_buffer.new_compute_command_encoder().expect("Failed to create compute command encoder");- // Use the fused router+topk kernel- self.router_topk_kernel.encode(- &main_buf,- weights_buf,- biases_buf,- &topk_ids_buf,- &topk_probs_buf,- suffix_length as u32,- self.model_dim as u32,- e as u32,- k as u32,- self.router_renorm,- &encoder,- );- encoder.end_encoding();- }- },+ if suffix_length > 0 && e > 0 && k > 0 {+ command_buffer.with_compute_encoder(|encoder| {+ // Use the fused router+topk kernel+ self.router_topk_kernel.encode(+ &main_buf,+ &self.router.weights_buf,+ &self.router.biases_buf,+ &topk_ids_buf,+ &topk_probs_buf,+ suffix_length as u32,+ self.model_dim as u32,+ e as u32,+ k as u32,+ self.router_renorm,+ &encoder,+ );+ });}- {- let encoder =- command_buffer.new_compute_command_encoder().expect("Failed to create compute command encoder");+ command_buffer.with_compute_encoder(|encoder| {self.counts_offsets_kernel.encode(&topk_ids_buf,&offsets_buf,@@ -360,46 +330,40 @@ impl EncodableBlock<Metal> for MoeBlock {k as u32,&encoder,);- encoder.end_encoding();- }+ });let num_blocks = ((suffix_length + 255) / 256).max(1);let num_tiles = ((e + 512 - 1) / 512).max(1);- let scatter_bases_encoder = command_buffer- .new_compute_command_encoder()- .expect("Failed to create compute command encoder for scatter bases kernel");- self.scatter_bases_kernel.encode(- &partials_buf,- &block_bases_buf,- &block_alloc_buf,- e as u32,- num_blocks as u32,- num_tiles as u32,- 0u32,- &scatter_bases_encoder,- );- scatter_bases_encoder.end_encoding();-- let scatter_encoder = command_buffer- .new_compute_command_encoder()- .expect("Failed to create compute command encoder for scatter map kernel");- self.scatter_map_kernel.encode(- &topk_ids_buf,- &topk_probs_buf,- &offsets_buf,- &block_bases_buf,- &block_alloc_buf,- &bucketed_ids_buf,- &bucketed_probs_buf,- suffix_length as u32,- e as u32,- k as u32,- num_blocks as u32,- num_tiles as u32,- &tok2row_buf,- &scatter_encoder,- );- scatter_encoder.end_encoding();+ command_buffer.with_compute_encoder(|encoder| {+ self.scatter_bases_kernel.encode(+ &partials_buf,+ &block_bases_buf,+ &block_alloc_buf,+ e as u32,+ num_blocks as u32,+ num_tiles as u32,+ 0u32,+ &encoder,+ );+ });+ command_buffer.with_compute_encoder(|encoder| {+ self.scatter_map_kernel.encode(+ &topk_ids_buf,+ &topk_probs_buf,+ &offsets_buf,+ &block_bases_buf,+ &block_alloc_buf,+ &bucketed_ids_buf,+ &bucketed_probs_buf,+ suffix_length as u32,+ e as u32,+ k as u32,+ num_blocks as u32,+ num_tiles as u32,+ &tok2row_buf,+ &encoder,+ );+ });self.gather_kernels.encode(root,@@ -469,10 +433,10 @@ impl EncodableBlock<Metal> for MoeBlock {row_expert_map: &row_expert_map_buf,hidden_buffer: &hidden_buf,output_buffer: &y_partial_buf,- w13_all: &self.shared_weights.w13_buf,- w2_all: &self.shared_weights.w2_buf,- up_biases: &self.shared_weights.up_biases_buf,- down_biases: &self.shared_weights.down_biases_buf,+ w13_all: self.shared_weights.w13_buf.as_ref(),+ w2_all: self.shared_weights.w2_buf.as_ref(),+ up_biases: self.shared_weights.up_biases_buf.as_ref(),+ down_biases: self.shared_weights.down_biases_buf.as_ref(),tile_counts: &tile_counts_buf,tile_offsets: &tile_offsets_buf,tile_map: &tile_map_buf,@@ -494,9 +458,7 @@ impl EncodableBlock<Metal> for MoeBlock {self.experts_two_pass_prefill_kernel.encode(root, &args);}- {- let encoder =- command_buffer.new_compute_command_encoder().expect("Failed to create compute command encoder");+ command_buffer.with_compute_encoder(|encoder| {self.finalize_kernel.encode(&tok2row_buf,&topk_probs_buf,@@ -507,11 +469,10 @@ impl EncodableBlock<Metal> for MoeBlock {k as u32,&encoder,);- encoder.end_encoding();- }+ });if parameters.wait_until_completed {- command_buffer.commit();+ command_buffer.submit();command_buffer.wait_until_completed();}}@@ -522,9 +483,9 @@ impl EncodableBlock<Metal> for MoeBlock {fn encode_with_shared_encoder(&self,- _state: &mut ForwardPassState<Metal>,- _parameters: &EncodingParameters<Metal>,- _encoder: &ProtocolObject<dyn MTLComputeCommandEncoder>,+ _state: &mut ForwardPassState<B>,+ _parameters: &EncodingParameters<B>,+ _encoder: &B::ComputeEncoder,) {unreachable!("MoeBlock does not support shared compute encoder");}diff --git a/crates/uzu/tests/moe/moe_block_e2e_test.rs b/crates/uzu/tests/moe/moe_block_e2e_test.rsindex 7815565d8..4eaac6417 100644--- a/crates/uzu/tests/moe/moe_block_e2e_test.rs+++ b/crates/uzu/tests/moe/moe_block_e2e_test.rs@@ -7,17 +7,13 @@ use uzu::{common::kernel::{MoeBlockBasesFromPartialsKernel, MoeCountsOffsetsFusedKernel, MoeFinalizeKernel, MoeRouterTopKKernel,MoeScatterBucketsMapKernel,+ moe::{MoeExpertsTwoPassArguments, MoeExpertsTwoPassPrefillBlock, MoeGatherArguments, MoeGatherKernels},},metal::{- MetalContext,- kernel::{- dsl::{- MoeBlockBasesFromPartialsMetalKernel, MoeCountsOffsetsFusedMetalKernel, MoeFinalizeMetalKernel,- MoeRouterTopKMetalKernel, MoeScatterBucketsMapMetalKernel,- },- moe::{- MoeExpertsTwoPassArguments, MoeExpertsTwoPassPrefillBlock, MoeGatherArguments, MoeGatherKernels,- },+ Metal, MetalContext,+ kernel::dsl::{+ MoeBlockBasesFromPartialsMetalKernel, MoeCountsOffsetsFusedMetalKernel, MoeFinalizeMetalKernel,+ MoeRouterTopKMetalKernel, MoeScatterBucketsMapMetalKernel,},},},@@ -411,7 +407,7 @@ fn run_moe_parity_test_internal();scatter_encoder.end_encoding();- let gather = MoeGatherKernels::new(&ctx).expect("gather");+ let gather = MoeGatherKernels::<Metal>::new(&ctx).expect("gather");gather.encode(&cb,DataType::BF16,@@ -431,7 +427,7 @@ fn run_moe_parity_test_internal(let hidden_buf = alloc_buffer::<f32>(&ctx, total_rows * d_ff);let row_expert_map_buf = alloc_buffer::<u32>(&ctx, total_rows);- let experts = MoeExpertsTwoPassPrefillBlock::new(&ctx).expect("experts");+ let experts = MoeExpertsTwoPassPrefillBlock::<Metal>::new(&ctx).expect("experts");let num_tiles_k = ((d_ff + 64 - 1) / 64) as u32;let args = MoeExpertsTwoPassArguments {x_perm_buffer: &x_perm_buf,diff --git a/crates/uzu/tests/moe/moe_experts_perf_test.rs b/crates/uzu/tests/moe/moe_experts_perf_test.rsindex 2bb58c14f..091a5b67a 100644--- a/crates/uzu/tests/moe/moe_experts_perf_test.rs+++ b/crates/uzu/tests/moe/moe_experts_perf_test.rs@@ -5,12 +5,12 @@ use metal::{MTLCommandBuffer, MTLCommandQueue};use rand::{RngExt, SeedableRng, rngs::StdRng};use uzu::{DataType,- backends::metal::{- MetalContext,- kernel::moe::{+ backends::{+ common::kernel::moe::{MoeExpertsSingleDecodeArguments, MoeExpertsSingleDecodeKernels, MoeExpertsTwoPassArguments,MoeExpertsTwoPassDecodeBlock, MoeExpertsTwoPassPrefillBlock,},+ metal::{Metal, MetalContext},},};@@ -76,7 +76,7 @@ fn run_decode_case(let up_biases: Vec<bf16> = (0..e * 2 * d_ff).map(|_| bf16::from_f32(rng.random_range(-0.01..0.01))).collect();let down_biases: Vec<bf16> = (0..e * d_model).map(|_| bf16::from_f32(rng.random_range(-0.01..0.01))).collect();- let experts_kernel = MoeExpertsTwoPassDecodeBlock::new(ctx).expect("experts decode kernel");+ let experts_kernel = MoeExpertsTwoPassDecodeBlock::<Metal>::new(ctx).expect("experts decode kernel");let x_perm_buf = alloc_buffer_with_data(&ctx, &x_perm);let offsets_buf = alloc_buffer_with_data(&ctx, &offsets);@@ -213,7 +213,7 @@ fn run_two_pass_prefill_case(let up_biases: Vec<bf16> = (0..e * 2 * d_ff).map(|_| bf16::from_f32(rng.random_range(-0.01..0.01))).collect();let down_biases: Vec<bf16> = (0..e * d_model).map(|_| bf16::from_f32(rng.random_range(-0.01..0.01))).collect();- let experts_kernel = MoeExpertsTwoPassPrefillBlock::new(ctx).expect("experts prefill kernel");+ let experts_kernel = MoeExpertsTwoPassPrefillBlock::<Metal>::new(ctx).expect("experts prefill kernel");let x_perm_buf = alloc_buffer_with_data(&ctx, &x_perm);let offsets_buf = alloc_buffer_with_data(&ctx, &offsets);@@ -339,7 +339,7 @@ fn run_fused_single_token_case(let up_biases: Vec<bf16> = (0..e * 2 * d_ff).map(|_| bf16::from_f32(rng.random_range(-0.01..0.01))).collect();let down_biases: Vec<bf16> = (0..e * d_model).map(|_| bf16::from_f32(rng.random_range(-0.01..0.01))).collect();- let fused_kernel = MoeExpertsSingleDecodeKernels::new(ctx).expect("fused kernel");+ let fused_kernel = MoeExpertsSingleDecodeKernels::<Metal>::new(ctx).expect("fused kernel");let x_buf = alloc_buffer_with_data(ctx, &x);let topk_ids_buf = alloc_buffer_with_data(ctx, &topk_ids);@@ -486,7 +486,7 @@ fn run_indirect_decode_timed(let up_biases: Vec<bf16> = (0..e * 2 * d_ff).map(|_| bf16::from_f32(rng.random_range(-0.01..0.01))).collect();let down_biases: Vec<bf16> = (0..e * d_model).map(|_| bf16::from_f32(rng.random_range(-0.01..0.01))).collect();- let experts_kernel = MoeExpertsTwoPassDecodeBlock::new(ctx).expect("decode kernel");+ let experts_kernel = MoeExpertsTwoPassDecodeBlock::<Metal>::new(ctx).expect("decode kernel");let x_perm_buf = alloc_buffer_with_data(ctx, &x_perm);let offsets_buf = alloc_buffer_with_data(ctx, &offsets);@@ -583,7 +583,7 @@ fn run_fused_decode_timed(let up_biases: Vec<bf16> = (0..e * 2 * d_ff).map(|_| bf16::from_f32(rng.random_range(-0.01..0.01))).collect();let down_biases: Vec<bf16> = (0..e * d_model).map(|_| bf16::from_f32(rng.random_range(-0.01..0.01))).collect();- let fused_kernel = MoeExpertsSingleDecodeKernels::new(ctx).expect("fused kernel");+ let fused_kernel = MoeExpertsSingleDecodeKernels::<Metal>::new(ctx).expect("fused kernel");let x_buf = alloc_buffer_with_data(ctx, &x);let topk_ids_buf = alloc_buffer_with_data(ctx, &topk_ids);diff --git a/crates/uzu/tests/moe/moe_experts_test.rs b/crates/uzu/tests/moe/moe_experts_test.rsindex 6c566785e..8bcd8bc53 100644--- a/crates/uzu/tests/moe/moe_experts_test.rs+++ b/crates/uzu/tests/moe/moe_experts_test.rs@@ -13,16 +13,19 @@ use metal::{MTLBuffer, MTLCommandBuffer, MTLCommandQueue};use rand::{RngExt, SeedableRng, rngs::StdRng};use uzu::{DataType,- backends::metal::kernel::moe::{- MoeExpertsSingleDecodeKernels, MoeExpertsTwoPassArguments, MoeExpertsTwoPassDecodeBlock,- MoeExpertsTwoPassPrefillBlock,+ backends::{+ common::kernel::moe::{+ MoeExpertsSingleDecodeKernels, MoeExpertsTwoPassArguments, MoeExpertsTwoPassDecodeBlock,+ MoeExpertsTwoPassPrefillBlock,+ },+ metal::Metal,},};#[path = "moe_test_utils.rs"]mod test_utils;use test_utils::{alloc_buffer, alloc_buffer_with_data, assert_bf16_close, cpu_tile_counts, cpu_tile_scan, create_ctx};-use uzu::backends::metal::kernel::moe::MoeExpertsSingleDecodeArguments;+use uzu::backends::common::kernel::moe::MoeExpertsSingleDecodeArguments;/// Test data for MoE expertsstruct MoeTestData {@@ -412,7 +415,7 @@ fn test_two_pass_decode_correctness() {let dispatch_args_buf = alloc_buffer::<u32>(&ctx, 3);// Execute 2-pass decode kernel- let experts_kernel = MoeExpertsTwoPassDecodeBlock::new(&ctx).expect("MoeExpertsTwoPassDecodeKernel::new");+ let experts_kernel = MoeExpertsTwoPassDecodeBlock::<Metal>::new(&ctx).expect("MoeExpertsTwoPassDecodeKernel::new");let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");const K_TILE: usize = 64;@@ -556,7 +559,7 @@ fn test_two_pass_decode_multi_token() {let tile_map_buf = alloc_buffer::<u32>(&ctx, max_total_tiles * 3);let dispatch_args_buf = alloc_buffer::<u32>(&ctx, 3);- let experts_kernel = MoeExpertsTwoPassDecodeBlock::new(&ctx).expect("kernel");+ let experts_kernel = MoeExpertsTwoPassDecodeBlock::<Metal>::new(&ctx).expect("kernel");let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");experts_kernel.encode(&cb,@@ -658,7 +661,7 @@ fn test_two_pass_prefill_correctness() {let tile_map_buf = alloc_buffer::<u32>(&ctx, max_total_tiles * 3);let dispatch_args_buf = alloc_buffer::<u32>(&ctx, 3);- let experts_kernel = MoeExpertsTwoPassPrefillBlock::new(&ctx).expect("kernel");+ let experts_kernel = MoeExpertsTwoPassPrefillBlock::<Metal>::new(&ctx).expect("kernel");let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");let args = MoeExpertsTwoPassArguments {x_perm_buffer: &x_perm_buf,@@ -787,7 +790,7 @@ fn test_fused_single_token_decode() {let y_buf = alloc_buffer::<bf16>(&ctx, d_model);// Run fused decode kernel- let fused_kernel = MoeExpertsSingleDecodeKernels::new(&ctx).expect("MoeExpertsSingleDecodeKernel::new");+ let fused_kernel = MoeExpertsSingleDecodeKernels::<Metal>::new(&ctx).expect("MoeExpertsSingleDecodeKernel::new");let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");fused_kernel.encode(@@ -911,7 +914,7 @@ fn test_fused_single_token_k4() {let hidden_buf = alloc_buffer::<f32>(&ctx, k * d_ff);let y_buf = alloc_buffer::<bf16>(&ctx, d_model);- let fused_kernel = MoeExpertsSingleDecodeKernels::new(&ctx).expect("fused kernel");+ let fused_kernel = MoeExpertsSingleDecodeKernels::<Metal>::new(&ctx).expect("fused kernel");let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");fused_kernel.encode(&cb,diff --git a/crates/uzu/tests/moe/moe_gather_test.rs b/crates/uzu/tests/moe/moe_gather_test.rsindex e9e1bc743..928823bd9 100644--- a/crates/uzu/tests/moe/moe_gather_test.rs+++ b/crates/uzu/tests/moe/moe_gather_test.rs@@ -5,7 +5,10 @@ use metal::{MTLBuffer, MTLCommandBuffer, MTLCommandQueue};use rand::{RngExt, SeedableRng, rngs::StdRng};use uzu::{DataType,- backends::metal::kernel::moe::{MoeGatherArguments, MoeGatherKernels},+ backends::{+ common::kernel::moe::{MoeGatherArguments, MoeGatherKernels},+ metal::Metal,+ },};use super::test_utils::{alloc_buffer, alloc_buffer_with_data, assert_bf16_close, create_ctx};@@ -74,7 +77,7 @@ fn test_gather_correctness() {let sumk_buf = alloc_buffer_with_data(&ctx, &sum_k_u32);// Execute gather kernel using kernel struct- let gather = MoeGatherKernels::new(&ctx).expect("MoeGatherKernel::new");+ let gather = MoeGatherKernels::<Metal>::new(&ctx).expect("MoeGatherKernel::new");let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");gather.encode(&cb,diff --git a/crates/uzu/tests/moe/moe_perf_test.rs b/crates/uzu/tests/moe/moe_perf_test.rsindex 624b5ddb3..147494163 100644--- a/crates/uzu/tests/moe/moe_perf_test.rs+++ b/crates/uzu/tests/moe/moe_perf_test.rs@@ -9,13 +9,14 @@ use uzu::{common::kernel::{MoeBlockBasesFromPartialsKernel, MoeCountsOffsetsFusedKernel, MoeFinalizeKernel, MoeRouterTopKKernel,MoeScatterBucketsMapKernel,+ moe::{MoeExpertsTwoPassArguments, MoeExpertsTwoPassDecodeBlock, MoeGatherArguments, MoeGatherKernels},},- metal::kernel::{- dsl::{+ metal::{+ Metal,+ kernel::dsl::{MoeBlockBasesFromPartialsMetalKernel, MoeCountsOffsetsFusedMetalKernel, MoeFinalizeMetalKernel,MoeRouterTopKMetalKernel, MoeScatterBucketsMapMetalKernel,},- moe::{MoeExpertsTwoPassArguments, MoeExpertsTwoPassDecodeBlock, MoeGatherArguments, MoeGatherKernels},},},};@@ -299,8 +300,8 @@ fn test_moe_pipeline_breakdown_decode() {MoeBlockBasesFromPartialsMetalKernel::new(&ctx).expect("MoeBlockBasesFromPartialsMetalKernel");let scatter_map_kernel =MoeScatterBucketsMapMetalKernel::new(&ctx, DataType::BF16).expect("MoeScatterBucketsMapMetalKernel");- let gather_kernel = MoeGatherKernels::new(&ctx).expect("gather");- let experts_kernel = MoeExpertsTwoPassDecodeBlock::new(&ctx).expect("experts two-pass decode");+ let gather_kernel = MoeGatherKernels::<Metal>::new(&ctx).expect("gather");+ let experts_kernel = MoeExpertsTwoPassDecodeBlock::<Metal>::new(&ctx).expect("experts two-pass decode");let finalize_kernel = MoeFinalizeMetalKernel::new(&ctx, DataType::BF16).expect("finalize");let router_topk_fused_kernel = MoeRouterTopKMetalKernel::new(&ctx, DataType::BF16).expect("router+topk fused");diff --git a/crates/uzu/tests/moe/moe_tiles_test.rs b/crates/uzu/tests/moe/moe_tiles_test.rsindex 13afb70a1..874d41329 100644--- a/crates/uzu/tests/moe/moe_tiles_test.rs+++ b/crates/uzu/tests/moe/moe_tiles_test.rs@@ -2,7 +2,10 @@use metal::{MTLBuffer, MTLCommandBuffer, MTLCommandQueue};use rand::{RngExt, SeedableRng, rngs::StdRng};-use uzu::backends::metal::kernel::moe::tiles_map::{MoeTileCountsArguments, MoeTileMapKernels, MoeTileScanArguments};+use uzu::backends::{+ common::kernel::moe::{MoeTileCountsArguments, MoeTileMapKernels, MoeTileScanArguments},+ metal::Metal,+};use super::test_utils::{alloc_buffer, alloc_buffer_with_data, cpu_tile_counts, cpu_tile_scan, create_ctx};@@ -31,7 +34,7 @@ fn test_tile_counts_correctness() {let tile_counts_buf = alloc_buffer::<u32>(&ctx, e);// Execute kernel using kernel struct- let tile_kernel = MoeTileMapKernels::new(&ctx).expect("MoeTileMapKernel::new");+ let tile_kernel = MoeTileMapKernels::<Metal>::new(&ctx).expect("MoeTileMapKernel::new");let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");tile_kernel.encode_counts(&cb,@@ -76,7 +79,7 @@ fn test_tile_scan_correctness() {let total_tiles_buf = alloc_buffer::<u32>(&ctx, 1);// Execute kernel using kernel struct- let tile_kernel = MoeTileMapKernels::new(&ctx).expect("MoeTileMapKernel::new");+ let tile_kernel = MoeTileMapKernels::<Metal>::new(&ctx).expect("MoeTileMapKernel::new");let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");tile_kernel.encode_scan(&cb,@@ -116,7 +119,7 @@ fn test_tile_edge_cases() {let offsets_buf = alloc_buffer_with_data(&ctx, &offsets);let tile_counts_buf = alloc_buffer::<u32>(&ctx, e);- let tile_kernel = MoeTileMapKernels::new(&ctx).expect("MoeTileMapKernel::new");+ let tile_kernel = MoeTileMapKernels::<Metal>::new(&ctx).expect("MoeTileMapKernel::new");let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");tile_kernel.encode_counts(&cb,@@ -147,7 +150,7 @@ fn test_tile_edge_cases() {let offsets_buf = alloc_buffer_with_data(&ctx, &offsets);let tile_counts_buf = alloc_buffer::<u32>(&ctx, e);- let tile_kernel = MoeTileMapKernels::new(&ctx).expect("MoeTileMapKernel::new");+ let tile_kernel = MoeTileMapKernels::<Metal>::new(&ctx).expect("MoeTileMapKernel::new");let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");tile_kernel.encode_counts(&cb,