Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

generalize moe kernel wrappers and encodable block (#207)

uuuvn72ded1d2e5e1
diff --git a/crates/uzu/src/backends/common/copy_encoder.rs b/crates/uzu/src/backends/common/copy_encoder.rs
index f68665a2f..9b39ac447 100644
--- a/crates/uzu/src/backends/common/copy_encoder.rs
+++ b/crates/uzu/src/backends/common/copy_encoder.rs
@@ -1,3 +1,5 @@
+use std::ops::Range;
+
use super::Backend;
pub trait CopyEncoder {
@@ -9,4 +11,11 @@ pub trait CopyEncoder {
dst: &<Self::Backend as Backend>::NativeBuffer,
size: usize,
);
+
+ fn encode_fill(
+ &self,
+ dst: &<Self::Backend as Backend>::NativeBuffer,
+ range: Range<usize>,
+ value: u8,
+ );
}
diff --git a/crates/uzu/src/backends/common/kernel/mod.rs b/crates/uzu/src/backends/common/kernel/mod.rs
index e97540287..b5a284288 100644
--- a/crates/uzu/src/backends/common/kernel/mod.rs
+++ b/crates/uzu/src/backends/common/kernel/mod.rs
@@ -6,6 +6,7 @@ pub mod attention;
pub mod kv_cache_update;
pub mod matmul;
pub mod mlp_gate_act_mul;
+pub mod moe;
pub mod quant_matmul;
pub mod sampling;
pub mod ssd_prefill;
diff --git a/crates/uzu/src/backends/metal/kernel/moe/experts_single.rs b/crates/uzu/src/backends/common/kernel/moe/experts_single.rs
similarity index 52%
rename from crates/uzu/src/backends/metal/kernel/moe/experts_single.rs
rename to crates/uzu/src/backends/common/kernel/moe/experts_single.rs
index 9f1797b36..cc408d358 100644
--- a/crates/uzu/src/backends/metal/kernel/moe/experts_single.rs
+++ b/crates/uzu/src/backends/common/kernel/moe/experts_single.rs
@@ -1,14 +1,8 @@
-use metal::{MTLBuffer, MTLCommandBuffer, MTLCommandEncoder};
-use objc2::{__framework_prelude::ProtocolObject, Message, rc::Retained};
-
use crate::{
DataType,
- backends::{
- common::kernel::{MoeExpertsDecodeSinglePassAKernel as _, MoeExpertsDecodeSinglePassBKernel as _},
- metal::{
- MetalContext, MetalError,
- kernel::dsl::{MoeExpertsDecodeSinglePassAMetalKernel, MoeExpertsDecodeSinglePassBMetalKernel},
- },
+ backends::common::{
+ Backend, CommandBuffer, Kernels,
+ kernel::{MoeExpertsDecodeSinglePassAKernel, MoeExpertsDecodeSinglePassBKernel},
},
};
@@ -16,25 +10,25 @@ static DTYPES: [DataType; 3] = [DataType::F16, DataType::BF16, DataType::F32];
/// Arguments for single-token MoE decode (T=1 optimized path)
#[derive(Debug)]
-pub struct MoeExpertsSingleDecodeArguments<'a> {
+pub struct MoeExpertsSingleDecodeArguments<'a, B: Backend> {
/// Input activation [d_model]
- pub x: &'a ProtocolObject<dyn MTLBuffer>,
+ pub x: &'a B::NativeBuffer,
/// Top-K expert indices from router [K]
- pub topk_ids: &'a ProtocolObject<dyn MTLBuffer>,
+ pub topk_ids: &'a B::NativeBuffer,
/// Top-K probabilities from router [K]
- pub topk_probs: &'a ProtocolObject<dyn MTLBuffer>,
+ pub topk_probs: &'a B::NativeBuffer,
/// Up/gate projection weights [E, 2*d_ff, d_model]
- pub w13_all: &'a ProtocolObject<dyn MTLBuffer>,
+ pub w13_all: &'a B::NativeBuffer,
/// Down projection weights [E, d_model, d_ff]
- pub w2_all: &'a ProtocolObject<dyn MTLBuffer>,
+ pub w2_all: &'a B::NativeBuffer,
/// Up/gate biases [E, 2*d_ff]
- pub up_biases: &'a ProtocolObject<dyn MTLBuffer>,
+ pub up_biases: &'a B::NativeBuffer,
/// Down biases [E, d_model]
- pub down_biases: &'a ProtocolObject<dyn MTLBuffer>,
+ pub down_biases: &'a B::NativeBuffer,
/// Hidden buffer [K, d_ff] - intermediate storage (f32)
- pub hidden: &'a ProtocolObject<dyn MTLBuffer>,
+ pub hidden: &'a B::NativeBuffer,
/// Final output [d_model]
- pub y: &'a ProtocolObject<dyn MTLBuffer>,
+ pub y: &'a B::NativeBuffer,
/// Model dimension
pub d_model: usize,
/// FFN hidden dimension
@@ -57,18 +51,19 @@ pub struct MoeExpertsSingleDecodeArguments<'a> {
pub data_type: DataType,
}
-pub struct MoeExpertsSingleDecodeKernels {
- pass_a: Vec<Vec<MoeExpertsDecodeSinglePassAMetalKernel>>,
- pass_b: Vec<MoeExpertsDecodeSinglePassBMetalKernel>,
+pub struct MoeExpertsSingleDecodeKernels<B: Backend> {
+ pass_a: Vec<Vec<<B::Kernels as Kernels>::MoeExpertsDecodeSinglePassAKernel>>,
+ pass_b: Vec<<B::Kernels as Kernels>::MoeExpertsDecodeSinglePassBKernel>,
}
-impl MoeExpertsSingleDecodeKernels {
- pub fn new(ctx: &MetalContext) -> Result<Self, MetalError> {
+impl<B: Backend> MoeExpertsSingleDecodeKernels<B> {
+ pub fn new(ctx: &B::Context) -> Result<Self, B::Error> {
let mut pass_a = vec![];
for gate in 0..4 {
let mut kernels = vec![];
for dtype in &DTYPES {
- let kernel = MoeExpertsDecodeSinglePassAMetalKernel::new(ctx, (*dtype).into(), gate)?;
+ let kernel =
+ <B::Kernels as Kernels>::MoeExpertsDecodeSinglePassAKernel::new(ctx, (*dtype).into(), gate)?;
kernels.push(kernel)
}
pass_a.push(kernels);
@@ -76,7 +71,7 @@ impl MoeExpertsSingleDecodeKernels {
let mut pass_b = Vec::with_capacity(DTYPES.len());
for dtype in &DTYPES {
- pass_b.push(MoeExpertsDecodeSinglePassBMetalKernel::new(ctx, (*dtype).into())?);
+ pass_b.push(<B::Kernels as Kernels>::MoeExpertsDecodeSinglePassBKernel::new(ctx, (*dtype).into())?);
}
Ok(Self {
@@ -87,8 +82,8 @@ impl MoeExpertsSingleDecodeKernels {
pub fn encode(
&self,
- command_buffer: &Retained<ProtocolObject<dyn MTLCommandBuffer>>,
- args: MoeExpertsSingleDecodeArguments,
+ command_buffer: &B::CommandBuffer,
+ args: MoeExpertsSingleDecodeArguments<B>,
) {
if args.k == 0 {
return;
@@ -98,16 +93,14 @@ impl MoeExpertsSingleDecodeKernels {
let dtype_idx = DTYPES.iter().position(|dtype| *dtype == args.data_type).unwrap();
// Pass A: x @ W13[expert] -> hidden
- {
+ command_buffer.with_compute_encoder(|encoder| {
let kernel = &self.pass_a[gate_idx][dtype_idx];
- let encoder =
- command_buffer.new_compute_command_encoder().expect("Failed to create compute command encoder");
kernel.encode(
- &args.x.retain(),
- &args.topk_ids.retain(),
- &args.w13_all.retain(),
- &args.up_biases.retain(),
- &args.hidden.retain(),
+ args.x,
+ args.topk_ids,
+ args.w13_all,
+ args.up_biases,
+ args.hidden,
args.d_model as u32,
args.d_ff as u32,
args.k as u32,
@@ -118,27 +111,23 @@ impl MoeExpertsSingleDecodeKernels {
args.up_clip_max,
&encoder,
);
- encoder.end_encoding();
- }
+ });
// Pass B: 8 simdgroups (256 threads), outputs final y directly
- {
+ command_buffer.with_compute_encoder(|encoder| {
let kernel = &self.pass_b[dtype_idx];
- let encoder =
- command_buffer.new_compute_command_encoder().expect("Failed to create compute command encoder");
kernel.encode(
- &args.hidden.retain(),
- &args.topk_ids.retain(),
- &args.topk_probs.retain(),
- &args.w2_all.retain(),
- &args.down_biases.retain(),
- &args.y.retain(),
+ args.hidden,
+ args.topk_ids,
+ args.topk_probs,
+ args.w2_all,
+ args.down_biases,
+ args.y,
args.d_model as u32,
args.d_ff as u32,
args.k as u32,
&encoder,
);
- encoder.end_encoding();
- }
+ });
}
}
diff --git a/crates/uzu/src/backends/common/kernel/moe/experts_two_pass_decode.rs b/crates/uzu/src/backends/common/kernel/moe/experts_two_pass_decode.rs
new file mode 100644
index 000000000..9650dbe5d
--- /dev/null
+++ b/crates/uzu/src/backends/common/kernel/moe/experts_two_pass_decode.rs
@@ -0,0 +1,148 @@
+use super::{
+ MoeExpertsTwoPassArguments, MoePassARowMapArguments, MoePassATileBuildArguments, MoePassATileCountsArguments,
+ MoePassATileDispatchArguments, MoePassATileKernels, MoePassATileScanArguments,
+};
+use crate::{
+ DataType,
+ backends::common::{
+ CommandBuffer,
+ kernel::{Backend, Kernels, MoeExpertsDecodeDownFused2DKernel, MoeExpertsDecodePassAKernel},
+ },
+};
+
+static DTYPES: [DataType; 3] = [DataType::F16, DataType::BF16, DataType::F32];
+
+pub struct MoeExpertsTwoPassDecodeBlock<B: Backend> {
+ pass_a_tile: MoePassATileKernels<B>,
+ pass_a_indirect: Vec<Vec<<B::Kernels as Kernels>::MoeExpertsDecodePassAKernel>>,
+ fused_down: Vec<<B::Kernels as Kernels>::MoeExpertsDecodeDownFused2DKernel>,
+}
+
+impl<B: Backend> MoeExpertsTwoPassDecodeBlock<B> {
+ pub fn new(ctx: &B::Context) -> Result<Self, B::Error> {
+ let mut pass_a_indirect = vec![];
+ for gate in 0u32..4u32 {
+ let mut kernels = vec![];
+ for dtype in &DTYPES {
+ let kernel = <B::Kernels as Kernels>::MoeExpertsDecodePassAKernel::new(ctx, *dtype, gate)?;
+ kernels.push(kernel);
+ }
+ pass_a_indirect.push(kernels);
+ }
+
+ let mut fused_down = Vec::with_capacity(DTYPES.len());
+ for dtype in &DTYPES {
+ let kernel = <B::Kernels as Kernels>::MoeExpertsDecodeDownFused2DKernel::new(ctx, *dtype, DataType::F32)?;
+ fused_down.push(kernel);
+ }
+
+ Ok(Self {
+ pass_a_tile: MoePassATileKernels::new(ctx)?,
+ pass_a_indirect,
+ fused_down,
+ })
+ }
+
+ pub fn encode(
+ &self,
+ command_buffer: &B::CommandBuffer,
+ args: &MoeExpertsTwoPassArguments<B>,
+ ) {
+ if args.total_rows == 0 {
+ return;
+ }
+
+ // pass a tile
+ const BLOCK_M: u32 = 4;
+ let h_blocks = (args.d_ff as u32 + BLOCK_M - 1) / BLOCK_M;
+ self.pass_a_tile.encode_counts(
+ command_buffer,
+ &MoePassATileCountsArguments {
+ expert_offsets: args.expert_offsets,
+ tile_counts: args.tile_counts,
+ e: args.e,
+ h_blocks,
+ },
+ );
+ self.pass_a_tile.encode_scan(
+ command_buffer,
+ &MoePassATileScanArguments {
+ tile_counts: args.tile_counts,
+ tile_offsets: args.tile_offsets,
+ total_tiles: args.total_tiles,
+ e: args.e,
+ },
+ );
+ self.pass_a_tile.encode_row_map(
+ command_buffer,
+ &MoePassARowMapArguments {
+ expert_offsets: args.expert_offsets,
+ row_expert_map: args.row_expert_map,
+ total_rows: args.total_rows,
+ e: args.e,
+ },
+ );
+ self.pass_a_tile.encode_build_map(
+ command_buffer,
+ &MoePassATileBuildArguments {
+ expert_offsets: args.expert_offsets,
+ tile_offsets: args.tile_offsets,
+ row_expert_map: args.row_expert_map,
+ tile_map: args.tile_map,
+ total_rows: args.total_rows,
+ h_blocks,
+ },
+ );
+ self.pass_a_tile.encode_dispatch_args(
+ command_buffer,
+ &MoePassATileDispatchArguments {
+ total_tiles: args.total_tiles,
+ dispatch_args: args.dispatch_args,
+ num_tiles_y: 1,
+ },
+ );
+
+ let gate_idx = args.gating_code.min(3) as usize;
+ let dtype_idx = DTYPES.iter().position(|dtype| *dtype == args.data_type).unwrap();
+
+ // pass a
+ command_buffer.with_compute_encoder(|encoder| {
+ let pass_a_kernel = &self.pass_a_indirect[gate_idx][dtype_idx];
+ pass_a_kernel.encode(
+ args.x_perm_buffer,
+ args.expert_offsets,
+ args.w13_all,
+ args.hidden_buffer,
+ args.up_biases,
+ args.d_model as u32,
+ args.d_ff as u32,
+ args.e as u32,
+ args.gate_clip_min,
+ args.gate_clip_max,
+ args.up_clip_min,
+ args.up_clip_max,
+ args.silu_alpha,
+ args.tile_map,
+ args.dispatch_args,
+ encoder,
+ );
+ });
+
+ // pass b
+ command_buffer.with_compute_encoder(|encoder| {
+ let pass_b_kernel = &self.fused_down[dtype_idx];
+ pass_b_kernel.encode(
+ args.hidden_buffer,
+ args.row_expert_map,
+ args.w2_all,
+ args.down_biases,
+ args.output_buffer,
+ args.total_rows as u32,
+ args.d_model as u32,
+ args.d_ff as u32,
+ args.e as u32,
+ encoder,
+ );
+ });
+ }
+}
diff --git a/crates/uzu/src/backends/common/kernel/moe/experts_two_pass_prefill.rs b/crates/uzu/src/backends/common/kernel/moe/experts_two_pass_prefill.rs
new file mode 100644
index 000000000..7936a116d
--- /dev/null
+++ b/crates/uzu/src/backends/common/kernel/moe/experts_two_pass_prefill.rs
@@ -0,0 +1,188 @@
+use super::{
+ MoeTileCountsArguments, MoeTileDispatchArguments, MoeTileMapBuildArguments, MoeTileMapKernels, MoeTileScanArguments,
+};
+use crate::{
+ DataType,
+ backends::common::{
+ Backend, CommandBuffer, CopyEncoder, Kernels,
+ kernel::{MoeExpertsPrefillPassAKernel, MoeExpertsPrefillPassBKernel},
+ },
+};
+
+const DTYPES: [DataType; 3] = [DataType::F16, DataType::BF16, DataType::F32];
+
+pub struct MoeExpertsTwoPassPrefillBlock<B: Backend> {
+ tile_map: MoeTileMapKernels<B>,
+ pass_a_indirect: Vec<Vec<<B::Kernels as Kernels>::MoeExpertsPrefillPassAKernel>>, // [gate][dtype]
+ pass_b_indirect: Vec<<B::Kernels as Kernels>::MoeExpertsPrefillPassBKernel>, // [dtype]
+}
+
+impl<B: Backend> MoeExpertsTwoPassPrefillBlock<B> {
+ pub fn new(ctx: &B::Context) -> Result<Self, B::Error> {
+ let dtypes = [DataType::F16, DataType::BF16, DataType::F32];
+
+ let mut pass_a_indirect = vec![];
+ for gate in 0u32..4u32 {
+ let mut kernels = vec![];
+ for dtype in dtypes {
+ let kernel = <B::Kernels as Kernels>::MoeExpertsPrefillPassAKernel::new(ctx, dtype, gate)?;
+ kernels.push(kernel);
+ }
+ pass_a_indirect.push(kernels);
+ }
+
+ let mut pass_b_indirect = vec![];
+ for dtype in dtypes {
+ let kernel = <B::Kernels as Kernels>::MoeExpertsPrefillPassBKernel::new(ctx, dtype)?;
+ pass_b_indirect.push(kernel);
+ }
+
+ Ok(Self {
+ tile_map: MoeTileMapKernels::new(ctx)?,
+ pass_a_indirect,
+ pass_b_indirect,
+ })
+ }
+
+ pub fn encode(
+ &self,
+ command_buffer: &B::CommandBuffer,
+ args: &MoeExpertsTwoPassArguments<B>,
+ ) {
+ if args.total_rows == 0 {
+ return;
+ }
+
+ let hidden_bytes = args.total_rows * args.d_ff * args.data_type.size_in_bytes();
+ command_buffer.with_copy_encoder(|encoder| encoder.encode_fill(args.hidden_buffer, 0..hidden_bytes, 0));
+
+ self.tile_map.encode_counts(
+ command_buffer,
+ &MoeTileCountsArguments {
+ offsets_buffer: args.expert_offsets,
+ tile_counts_buffer: args.tile_counts,
+ e: args.e,
+ },
+ );
+ self.tile_map.encode_scan(
+ command_buffer,
+ &MoeTileScanArguments {
+ tile_counts_buffer: args.tile_counts,
+ tile_offsets_buffer: args.tile_offsets,
+ total_tiles_buffer: args.total_tiles,
+ e: args.e,
+ },
+ );
+ self.tile_map.encode_build_map(
+ command_buffer,
+ &MoeTileMapBuildArguments {
+ expert_offsets: args.expert_offsets,
+ tile_offsets: args.tile_offsets,
+ tile_counts: args.tile_counts,
+ tile_map: args.tile_map,
+ e: args.e,
+ },
+ );
+ let d_model_u32 = args.d_model as u32;
+ let d_ff_u32 = args.d_ff as u32;
+ const COL_TILE_FF: u32 = 32; // Must match PASSA_BN in Metal kernel
+ const COL_TILE_MODEL: u32 = 64;
+ let n_tiles_ff = if d_ff_u32 == 0 {
+ 0
+ } else {
+ (d_ff_u32 + COL_TILE_FF - 1) / COL_TILE_FF
+ };
+ let n_tiles_model = if d_model_u32 == 0 {
+ 0
+ } else {
+ (d_model_u32 + COL_TILE_MODEL - 1) / COL_TILE_MODEL
+ };
+ self.tile_map.encode_dispatch_args(
+ command_buffer,
+ &MoeTileDispatchArguments {
+ total_tiles: args.total_tiles,
+ dispatch_args: args.dispatch_args,
+ num_tiles_x: n_tiles_ff,
+ },
+ );
+
+ let gate_idx = args.gating_code.min(3) as usize;
+ let dtype_idx = DTYPES.iter().position(|t| *t == args.data_type).unwrap();
+
+ command_buffer.with_compute_encoder(|encoder| {
+ let kernel_pass_a = &self.pass_a_indirect[gate_idx][dtype_idx];
+ kernel_pass_a.encode(
+ args.x_perm_buffer,
+ args.expert_offsets,
+ args.w13_all,
+ args.up_biases,
+ args.hidden_buffer,
+ args.d_model as u32,
+ args.d_ff as u32,
+ args.e as u32,
+ args.gate_clip_min,
+ args.gate_clip_max,
+ args.up_clip_min,
+ args.up_clip_max,
+ args.silu_alpha,
+ args.tile_map,
+ args.dispatch_args,
+ encoder,
+ );
+ });
+
+ let dispatch_args = MoeTileDispatchArguments {
+ total_tiles: args.total_tiles,
+ dispatch_args: args.dispatch_args,
+ num_tiles_x: n_tiles_model,
+ };
+ self.tile_map.encode_dispatch_args(command_buffer, &dispatch_args);
+
+ command_buffer.with_compute_encoder(|encoder| {
+ let kernel_pass_b = &self.pass_b_indirect[dtype_idx];
+ kernel_pass_b.encode(
+ args.hidden_buffer,
+ args.expert_offsets,
+ args.w2_all,
+ args.down_biases,
+ args.output_buffer,
+ args.d_model as u32,
+ args.d_ff as u32,
+ args.e as u32,
+ args.tile_map,
+ args.dispatch_args,
+ encoder,
+ );
+ });
+ }
+}
+
+#[derive(Debug)]
+pub struct MoeExpertsTwoPassArguments<'a, B: Backend> {
+ pub x_perm_buffer: &'a B::NativeBuffer,
+ pub expert_offsets: &'a B::NativeBuffer,
+ pub row_expert_map: &'a B::NativeBuffer,
+ pub hidden_buffer: &'a B::NativeBuffer,
+ pub output_buffer: &'a B::NativeBuffer,
+ pub w13_all: &'a B::NativeBuffer,
+ pub w2_all: &'a B::NativeBuffer,
+ pub up_biases: &'a B::NativeBuffer,
+ pub down_biases: &'a B::NativeBuffer,
+ pub tile_counts: &'a B::NativeBuffer,
+ pub tile_offsets: &'a B::NativeBuffer,
+ pub tile_map: &'a B::NativeBuffer,
+ pub total_tiles: &'a B::NativeBuffer,
+ pub dispatch_args: &'a B::NativeBuffer,
+ pub total_rows: usize,
+ pub d_model: usize,
+ pub d_ff: usize,
+ pub e: usize,
+ pub num_tiles_k: u32,
+ pub gating_code: u32,
+ pub gate_clip_min: f32,
+ pub gate_clip_max: f32,
+ pub up_clip_min: f32,
+ pub up_clip_max: f32,
+ pub silu_alpha: f32,
+ pub data_type: DataType,
+}
diff --git a/crates/uzu/src/backends/common/kernel/moe/gather.rs b/crates/uzu/src/backends/common/kernel/moe/gather.rs
new file mode 100644
index 000000000..bd4bc07bb
--- /dev/null
+++ b/crates/uzu/src/backends/common/kernel/moe/gather.rs
@@ -0,0 +1,75 @@
+use crate::{
+ DataType,
+ backends::common::{
+ Backend, CommandBuffer, Kernels,
+ kernel::{MoeGatherXPerm1DKernel, MoeGatherXPerm2DKernel},
+ },
+};
+
+#[derive(Debug)]
+pub struct MoeGatherArguments<'a, B: Backend> {
+ pub x_buffer: &'a B::NativeBuffer,
+ pub bucketed_ids_buffer: &'a B::NativeBuffer,
+ pub x_perm_buffer: &'a B::NativeBuffer,
+ pub sumk_buffer: &'a B::NativeBuffer,
+ pub t: usize,
+ pub k: usize,
+ pub d_model: usize,
+}
+
+pub struct MoeGatherKernels<B: Backend> {
+ bf16: <B::Kernels as Kernels>::MoeGatherXPerm2DKernel,
+ f16: <B::Kernels as Kernels>::MoeGatherXPerm1DKernel,
+ f32: <B::Kernels as Kernels>::MoeGatherXPerm1DKernel,
+}
+
+impl<B: Backend> MoeGatherKernels<B> {
+ pub fn new(ctx: &B::Context) -> Result<Self, B::Error> {
+ Ok(Self {
+ bf16: <B::Kernels as Kernels>::MoeGatherXPerm2DKernel::new(ctx, DataType::BF16)?,
+ f16: <B::Kernels as Kernels>::MoeGatherXPerm1DKernel::new(ctx, DataType::F16)?,
+ f32: <B::Kernels as Kernels>::MoeGatherXPerm1DKernel::new(ctx, DataType::F32)?,
+ })
+ }
+
+ pub fn encode(
+ &self,
+ command_buffer: &B::CommandBuffer,
+ dtype: DataType,
+ args: &MoeGatherArguments<B>,
+ ) {
+ command_buffer.with_compute_encoder(|encoder| match dtype {
+ DataType::F32 => self.f32.encode(
+ args.x_buffer,
+ args.bucketed_ids_buffer,
+ args.x_perm_buffer,
+ args.sumk_buffer,
+ args.d_model as u32,
+ args.t as u32,
+ args.k as u32,
+ encoder,
+ ),
+ DataType::F16 => self.f16.encode(
+ args.x_buffer,
+ args.bucketed_ids_buffer,
+ args.x_perm_buffer,
+ args.sumk_buffer,
+ args.d_model as u32,
+ args.t as u32,
+ args.k as u32,
+ encoder,
+ ),
+ DataType::BF16 => self.bf16.encode(
+ args.x_buffer,
+ args.bucketed_ids_buffer,
+ args.x_perm_buffer,
+ args.sumk_buffer,
+ args.d_model as u32,
+ args.t as u32,
+ args.k as u32,
+ encoder,
+ ),
+ _ => panic!("Unsupported data type: {:?}", dtype),
+ });
+ }
+}
diff --git a/crates/uzu/src/backends/metal/kernel/moe/mod.rs b/crates/uzu/src/backends/common/kernel/moe/mod.rs
similarity index 76%
rename from crates/uzu/src/backends/metal/kernel/moe/mod.rs
rename to crates/uzu/src/backends/common/kernel/moe/mod.rs
index beb45c942..1ac0db46a 100644
--- a/crates/uzu/src/backends/metal/kernel/moe/mod.rs
+++ b/crates/uzu/src/backends/common/kernel/moe/mod.rs
@@ -1,16 +1,17 @@
-// Submodules
mod experts_single;
mod experts_two_pass_decode;
mod experts_two_pass_prefill;
mod gather;
-pub mod tiles_map;
-pub mod tiles_pass_a;
+mod tiles_map;
+mod tiles_pass_a;
-// Re-export public items from submodules
pub use experts_single::{MoeExpertsSingleDecodeArguments, MoeExpertsSingleDecodeKernels};
pub use experts_two_pass_decode::MoeExpertsTwoPassDecodeBlock;
pub use experts_two_pass_prefill::{MoeExpertsTwoPassArguments, MoeExpertsTwoPassPrefillBlock};
pub use gather::{MoeGatherArguments, MoeGatherKernels};
+pub use tiles_map::{
+ MoeTileCountsArguments, MoeTileDispatchArguments, MoeTileMapBuildArguments, MoeTileMapKernels, MoeTileScanArguments,
+};
pub use tiles_pass_a::{
MoePassARowMapArguments, MoePassATileBuildArguments, MoePassATileCountsArguments, MoePassATileDispatchArguments,
MoePassATileKernels, MoePassATileScanArguments,
diff --git a/crates/uzu/src/backends/common/kernel/moe/tiles_map.rs b/crates/uzu/src/backends/common/kernel/moe/tiles_map.rs
new file mode 100644
index 000000000..3590cb360
--- /dev/null
+++ b/crates/uzu/src/backends/common/kernel/moe/tiles_map.rs
@@ -0,0 +1,104 @@
+use crate::backends::common::{
+ Backend, CommandBuffer, Kernels,
+ kernel::{MoeBuildTileMapKernel, MoeTileCountsKernel, MoeTileScanKernel, MoeWriteDispatchArgsKernel},
+};
+
+pub struct MoeTileMapKernels<B: Backend> {
+ counts: <B::Kernels as Kernels>::MoeTileCountsKernel,
+ scan: <B::Kernels as Kernels>::MoeTileScanKernel,
+ build: <B::Kernels as Kernels>::MoeBuildTileMapKernel,
+ dispatch: <B::Kernels as Kernels>::MoeWriteDispatchArgsKernel,
+}
+
+impl<B: Backend> MoeTileMapKernels<B> {
+ pub fn new(ctx: &B::Context) -> Result<Self, B::Error> {
+ Ok(Self {
+ counts: <B::Kernels as Kernels>::MoeTileCountsKernel::new(ctx)?,
+ scan: <B::Kernels as Kernels>::MoeTileScanKernel::new(ctx)?,
+ build: <B::Kernels as Kernels>::MoeBuildTileMapKernel::new(ctx)?,
+ dispatch: <B::Kernels as Kernels>::MoeWriteDispatchArgsKernel::new(ctx)?,
+ })
+ }
+
+ pub fn encode_counts(
+ &self,
+ command_buffer: &B::CommandBuffer,
+ args: &MoeTileCountsArguments<B>,
+ ) {
+ command_buffer.with_compute_encoder(|encoder| {
+ self.counts.encode(args.offsets_buffer, args.tile_counts_buffer, args.e as u32, encoder);
+ });
+ }
+
+ pub fn encode_scan(
+ &self,
+ command_buffer: &B::CommandBuffer,
+ args: &MoeTileScanArguments<B>,
+ ) {
+ command_buffer.with_compute_encoder(|encoder| {
+ self.scan.encode(
+ args.tile_counts_buffer,
+ args.tile_offsets_buffer,
+ args.total_tiles_buffer,
+ args.e as u32,
+ encoder,
+ );
+ });
+ }
+
+ pub fn encode_build_map(
+ &self,
+ command_buffer: &B::CommandBuffer,
+ args: &MoeTileMapBuildArguments<B>,
+ ) {
+ command_buffer.with_compute_encoder(|encoder| {
+ self.build.encode(
+ args.expert_offsets,
+ args.tile_offsets,
+ args.tile_counts,
+ args.tile_map,
+ args.e as u32,
+ encoder,
+ );
+ });
+ }
+
+ pub fn encode_dispatch_args(
+ &self,
+ command_buffer: &B::CommandBuffer,
+ args: &MoeTileDispatchArguments<B>,
+ ) {
+ command_buffer.with_compute_encoder(|encoder| {
+ self.dispatch.encode(args.total_tiles, args.dispatch_args, args.num_tiles_x, encoder);
+ });
+ }
+}
+
+pub struct MoeTileCountsArguments<'a, B: Backend> {
+ pub offsets_buffer: &'a B::NativeBuffer, // [E+1]
+ pub tile_counts_buffer: &'a B::NativeBuffer, // [E]
+ pub e: usize,
+}
+
+pub struct MoeTileScanArguments<'a, B: Backend> {
+ pub tile_counts_buffer: &'a B::NativeBuffer, // [E]
+ pub tile_offsets_buffer: &'a B::NativeBuffer, // [E+1]
+ pub total_tiles_buffer: &'a B::NativeBuffer, // [>=2]
+ pub e: usize,
+}
+
+#[derive(Debug)]
+pub struct MoeTileMapBuildArguments<'a, B: Backend> {
+ pub expert_offsets: &'a B::NativeBuffer, // [E+1]
+ pub tile_offsets: &'a B::NativeBuffer, // [E+1]
+ pub tile_counts: &'a B::NativeBuffer, // [E]
+ pub tile_map: &'a B::NativeBuffer, // [total_tiles * 3]
+ pub e: usize,
+}
+
+#[derive(Debug)]
+pub struct MoeTileDispatchArguments<'a, B: Backend> {
+ pub total_tiles: &'a B::NativeBuffer, // [>=1]
+ pub dispatch_args: &'a B::NativeBuffer, // [3]
+ pub num_tiles_x: u32, // x dimension for indirect dispatch
+}
diff --git a/crates/uzu/src/backends/common/kernel/moe/tiles_pass_a.rs b/crates/uzu/src/backends/common/kernel/moe/tiles_pass_a.rs
new file mode 100644
index 000000000..43548c713
--- /dev/null
+++ b/crates/uzu/src/backends/common/kernel/moe/tiles_pass_a.rs
@@ -0,0 +1,132 @@
+use crate::backends::common::{
+ CommandBuffer,
+ kernel::{
+ Backend, Kernels, MoePassABuildRowMapKernel, MoePassABuildTileMapKernel, MoePassATileCountsKernel,
+ MoePassATileScanKernel, MoePassAWriteDispatchArgsKernel,
+ },
+};
+
+#[derive(Debug)]
+pub struct MoePassATileCountsArguments<'a, B: Backend> {
+ pub expert_offsets: &'a B::NativeBuffer, // [E+1]
+ pub tile_counts: &'a B::NativeBuffer, // [E]
+ pub e: usize,
+ pub h_blocks: u32,
+}
+
+#[derive(Debug)]
+pub struct MoePassATileScanArguments<'a, B: Backend> {
+ pub tile_counts: &'a B::NativeBuffer, // [E]
+ pub tile_offsets: &'a B::NativeBuffer, // [E+1]
+ pub total_tiles: &'a B::NativeBuffer, // [>=1]
+ pub e: usize,
+}
+
+#[derive(Debug)]
+pub struct MoePassARowMapArguments<'a, B: Backend> {
+ pub expert_offsets: &'a B::NativeBuffer, // [E+1]
+ pub row_expert_map: &'a B::NativeBuffer, // [total_rows]
+ pub total_rows: usize,
+ pub e: usize,
+}
+
+#[derive(Debug)]
+pub struct MoePassATileBuildArguments<'a, B: Backend> {
+ pub expert_offsets: &'a B::NativeBuffer, // [E+1]
+ pub tile_offsets: &'a B::NativeBuffer, // [E+1]
+ pub row_expert_map: &'a B::NativeBuffer, // [total_rows]
+ pub tile_map: &'a B::NativeBuffer, // [total_tiles * 3]
+ pub total_rows: usize,
+ pub h_blocks: u32,
+}
+
+#[derive(Debug)]
+pub struct MoePassATileDispatchArguments<'a, B: Backend> {
+ pub total_tiles: &'a B::NativeBuffer, // [>=1]
+ pub dispatch_args: &'a B::NativeBuffer, // [3]
+ pub num_tiles_y: u32,
+}
+
+pub struct MoePassATileKernels<B: Backend> {
+ counts: <B::Kernels as Kernels>::MoePassATileCountsKernel,
+ scan: <B::Kernels as Kernels>::MoePassATileScanKernel,
+ row_map: <B::Kernels as Kernels>::MoePassABuildRowMapKernel,
+ build_map: <B::Kernels as Kernels>::MoePassABuildTileMapKernel,
+ dispatch: <B::Kernels as Kernels>::MoePassAWriteDispatchArgsKernel,
+}
+
+impl<B: Backend> MoePassATileKernels<B> {
+ pub fn new(ctx: &B::Context) -> Result<Self, B::Error> {
+ Ok(Self {
+ counts: <B::Kernels as Kernels>::MoePassATileCountsKernel::new(ctx)?,
+ scan: <B::Kernels as Kernels>::MoePassATileScanKernel::new(ctx)?,
+ row_map: <B::Kernels as Kernels>::MoePassABuildRowMapKernel::new(ctx)?,
+ build_map: <B::Kernels as Kernels>::MoePassABuildTileMapKernel::new(ctx)?,
+ dispatch: <B::Kernels as Kernels>::MoePassAWriteDispatchArgsKernel::new(ctx)?,
+ })
+ }
+
+ pub fn encode_counts(
+ &self,
+ command_buffer: &B::CommandBuffer,
+ args: &MoePassATileCountsArguments<B>,
+ ) {
+ command_buffer.with_compute_encoder(|encoder| {
+ self.counts.encode(args.expert_offsets, args.tile_counts, args.e as u32, args.h_blocks, encoder);
+ });
+ }
+
+ pub fn encode_scan(
+ &self,
+ command_buffer: &B::CommandBuffer,
+ args: &MoePassATileScanArguments<B>,
+ ) {
+ command_buffer.with_compute_encoder(|encoder| {
+ self.scan.encode(args.tile_counts, args.tile_offsets, args.total_tiles, args.e as u32, encoder);
+ });
+ }
+
+ pub fn encode_row_map(
+ &self,
+ command_buffer: &B::CommandBuffer,
+ args: &MoePassARowMapArguments<B>,
+ ) {
+ command_buffer.with_compute_encoder(|encoder| {
+ self.row_map.encode(
+ args.expert_offsets,
+ args.row_expert_map,
+ args.total_rows as u32,
+ args.e as u32,
+ encoder,
+ );
+ });
+ }
+
+ pub fn encode_build_map(
+ &self,
+ command_buffer: &B::CommandBuffer,
+ args: &MoePassATileBuildArguments<B>,
+ ) {
+ command_buffer.with_compute_encoder(|encoder| {
+ self.build_map.encode(
+ args.expert_offsets,
+ args.tile_offsets,
+ args.row_expert_map,
+ args.tile_map,
+ args.total_rows as u32,
+ args.h_blocks,
+ encoder,
+ );
+ });
+ }
+
+ pub fn encode_dispatch_args(
+ &self,
+ command_buffer: &B::CommandBuffer,
+ args: &MoePassATileDispatchArguments<B>,
+ ) {
+ command_buffer.with_compute_encoder(|encoder| {
+ self.dispatch.encode(args.total_tiles, args.dispatch_args, args.num_tiles_y, encoder);
+ });
+ }
+}
diff --git a/crates/uzu/src/backends/metal/copy_encoder.rs b/crates/uzu/src/backends/metal/copy_encoder.rs
index ab9bcaab2..12cf03bff 100644
--- a/crates/uzu/src/backends/metal/copy_encoder.rs
+++ b/crates/uzu/src/backends/metal/copy_encoder.rs
@@ -1,20 +1,34 @@
-use metal::{MTLBlitCommandEncoder, MTLBuffer};
+use std::ops::Range;
+
+use metal::{MTLBlitCommandEncoder, MTLBlitCommandEncoderExt, MTLBuffer};
use objc2::runtime::ProtocolObject;
use super::Metal;
-use crate::backends::common::CopyEncoder;
+use crate::backends::common::{Backend, CopyEncoder};
impl CopyEncoder for ProtocolObject<dyn MTLBlitCommandEncoder> {
type Backend = Metal;
fn encode_copy(
&self,
- src: &<Self::Backend as crate::backends::common::Backend>::NativeBuffer,
- dst: &<Self::Backend as crate::backends::common::Backend>::NativeBuffer,
+ src: &<Self::Backend as Backend>::NativeBuffer,
+ dst: &<Self::Backend as Backend>::NativeBuffer,
size: usize,
) {
assert!(src.length() >= size && dst.length() >= size);
self.copy_buffer_to_buffer(src, 0, dst, 0, size);
}
+
+ fn encode_fill(
+ &self,
+ dst: &<Self::Backend as Backend>::NativeBuffer,
+ range: Range<usize>,
+ value: u8,
+ ) {
+ assert!(range.end > range.start && range.end <= dst.length());
+ assert!(range.start % 4 == 0 && range.end % 4 == 0);
+
+ self.fill_buffer_range_value(dst, range, value);
+ }
}
diff --git a/crates/uzu/src/backends/metal/encodable_block/mod.rs b/crates/uzu/src/backends/metal/encodable_block/mod.rs
index ebcf7ef7d..155c30798 100644
--- a/crates/uzu/src/backends/metal/encodable_block/mod.rs
+++ b/crates/uzu/src/backends/metal/encodable_block/mod.rs
@@ -11,7 +11,6 @@ mod decoder;
mod embedding;
mod layer;
mod linear;
-mod moe_block;
pub mod transformer_layer;
pub use classifier_layer::ClassifierLayer;
@@ -22,4 +21,3 @@ pub use embedding::{
};
pub use layer::LayerExecutables;
pub use linear::{FullPrecisionLinear, QuantizedLinear};
-pub use moe_block::{MoeBlock, SharedMoeWeights};
diff --git a/crates/uzu/src/backends/metal/encodable_block/moe_block/mod.rs b/crates/uzu/src/backends/metal/encodable_block/moe_block/mod.rs
deleted file mode 100644
index 7e91b8451..000000000
--- a/crates/uzu/src/backends/metal/encodable_block/moe_block/mod.rs
+++ /dev/null
@@ -1,7 +0,0 @@
-//! MoE (Mixture of Experts) encodables.
-
-mod moe_block;
-mod shared_weights;
-
-pub use moe_block::MoeBlock;
-pub use shared_weights::SharedMoeWeights;
diff --git a/crates/uzu/src/backends/metal/encodable_block/moe_block/shared_weights.rs b/crates/uzu/src/backends/metal/encodable_block/moe_block/shared_weights.rs
deleted file mode 100644
index bc51a10bd..000000000
--- a/crates/uzu/src/backends/metal/encodable_block/moe_block/shared_weights.rs
+++ /dev/null
@@ -1,14 +0,0 @@
-//! Shared MoE expert weights.
-
-use std::rc::Rc;
-
-use metal::MTLBuffer;
-use objc2::{rc::Retained, runtime::ProtocolObject};
-
-#[derive(Clone)]
-pub struct SharedMoeWeights {
- pub w13_buf: Rc<Retained<ProtocolObject<dyn MTLBuffer>>>,
- pub w2_buf: Rc<Retained<ProtocolObject<dyn MTLBuffer>>>,
- pub up_biases_buf: Rc<Retained<ProtocolObject<dyn MTLBuffer>>>,
- pub down_biases_buf: Rc<Retained<ProtocolObject<dyn MTLBuffer>>>,
-}
diff --git a/crates/uzu/src/backends/metal/encodable_block/transformer_layer.rs b/crates/uzu/src/backends/metal/encodable_block/transformer_layer.rs
index dcc4987c2..35ebe71f5 100644
--- a/crates/uzu/src/backends/metal/encodable_block/transformer_layer.rs
+++ b/crates/uzu/src/backends/metal/encodable_block/transformer_layer.rs
@@ -1,6 +1,6 @@
use super::{
- FullPrecisionEmbeddingLookup, FullPrecisionEmbeddingReadout, FullPrecisionLinear, MoeBlock,
- QuantizedEmbeddingLookup, QuantizedEmbeddingReadout, QuantizedLinear,
+ FullPrecisionEmbeddingLookup, FullPrecisionEmbeddingReadout, FullPrecisionLinear, QuantizedEmbeddingLookup,
+ QuantizedEmbeddingReadout, QuantizedLinear,
};
use crate::{
DataType,
@@ -9,7 +9,7 @@ use crate::{
metal::{Metal, MetalContext, MetalError},
},
config::{DecoderConfig, EmbeddingConfig, LinearConfig, MLPConfig},
- encodable_block::{EncodableBlock, MlpBlock},
+ encodable_block::{EncodableBlock, MlpBlock, MoeBlock},
forward_pass::state::ArrayId,
parameters::ParameterTree,
};
@@ -104,7 +104,8 @@ pub fn mlp_block(
if let crate::config::MLPConfig::MixtureOfExperts(mixture_of_experts_config) = config {
let mixture_of_experts_block =
- MoeBlock::new(context, mixture_of_experts_config, model_dimension, hidden_dimension, parameter_tree)?;
+ MoeBlock::new(context, mixture_of_experts_config, model_dimension, hidden_dimension, parameter_tree)
+ .map_err(|error| MetalError::Generic(format!("{:?}", error)))?;
return Ok(Box::new(mixture_of_experts_block));
}
diff --git a/crates/uzu/src/backends/metal/kernel/mod.rs b/crates/uzu/src/backends/metal/kernel/mod.rs
index 0323968b5..bd94bb20f 100644
--- a/crates/uzu/src/backends/metal/kernel/mod.rs
+++ b/crates/uzu/src/backends/metal/kernel/mod.rs
@@ -5,13 +5,8 @@ pub mod dsl {
pub(super) const MTLB: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/default.metallib"));
pub mod matmul;
-pub mod moe;
pub use matmul::{MatmulArguments, MatmulKernel};
-pub use moe::{
- MoeExpertsTwoPassArguments, MoeExpertsTwoPassDecodeBlock, MoeExpertsTwoPassPrefillBlock, MoeGatherArguments,
- MoeGatherKernels,
-};
impl MatmulKernels for dsl::MetalKernels {
type FullPrecisionMatmulKernel = MatmulKernel;
diff --git a/crates/uzu/src/backends/metal/kernel/moe/experts_two_pass_decode.rs b/crates/uzu/src/backends/metal/kernel/moe/experts_two_pass_decode.rs
deleted file mode 100644
index 8a07282f8..000000000
--- a/crates/uzu/src/backends/metal/kernel/moe/experts_two_pass_decode.rs
+++ /dev/null
@@ -1,161 +0,0 @@
-use metal::{MTLCommandBuffer, MTLCommandEncoder};
-use objc2::{
- __framework_prelude::{ProtocolObject, Retained},
- Message,
-};
-
-use crate::{
- DataType,
- backends::{
- common::kernel::{MoeExpertsDecodeDownFused2DKernel, MoeExpertsDecodePassAKernel},
- metal::{
- MetalContext, MetalError,
- kernel::{
- MoeExpertsTwoPassArguments,
- dsl::{MoeExpertsDecodeDownFused2DMetalKernel, MoeExpertsDecodePassAMetalKernel},
- moe::{
- MoePassARowMapArguments, MoePassATileBuildArguments, MoePassATileCountsArguments,
- MoePassATileDispatchArguments, MoePassATileKernels, MoePassATileScanArguments,
- },
- },
- },
- },
-};
-
-static DTYPES: [DataType; 3] = [DataType::F16, DataType::BF16, DataType::F32];
-
-pub struct MoeExpertsTwoPassDecodeBlock {
- pass_a_tile: MoePassATileKernels,
- pass_a_indirect: Vec<Vec<MoeExpertsDecodePassAMetalKernel>>,
- fused_down: Vec<MoeExpertsDecodeDownFused2DMetalKernel>,
-}
-
-impl MoeExpertsTwoPassDecodeBlock {
- pub fn new(ctx: &MetalContext) -> Result<Self, MetalError> {
- let mut pass_a_indirect = vec![];
- for gate in 0u32..4u32 {
- let mut kernels = vec![];
- for dtype in &DTYPES {
- let kernel = MoeExpertsDecodePassAMetalKernel::new(ctx, *dtype, gate)?;
- kernels.push(kernel);
- }
- pass_a_indirect.push(kernels);
- }
-
- let mut fused_down = Vec::with_capacity(DTYPES.len());
- for dtype in &DTYPES {
- let kernel = MoeExpertsDecodeDownFused2DMetalKernel::new(ctx, *dtype, DataType::F32)?;
- fused_down.push(kernel);
- }
-
- Ok(Self {
- pass_a_tile: MoePassATileKernels::new(ctx)?,
- pass_a_indirect,
- fused_down,
- })
- }
-
- pub fn encode(
- &self,
- command_buffer: &Retained<ProtocolObject<dyn MTLCommandBuffer>>,
- args: &MoeExpertsTwoPassArguments,
- ) {
- if args.total_rows == 0 {
- return;
- }
-
- // pass a tile
- const BLOCK_M: u32 = 4;
- let h_blocks = (args.d_ff as u32 + BLOCK_M - 1) / BLOCK_M;
- self.pass_a_tile.encode_counts(
- command_buffer,
- &MoePassATileCountsArguments {
- expert_offsets: args.expert_offsets,
- tile_counts: args.tile_counts,
- e: args.e,
- h_blocks,
- },
- );
- self.pass_a_tile.encode_scan(
- command_buffer,
- &MoePassATileScanArguments {
- tile_counts: args.tile_counts,
- tile_offsets: args.tile_offsets,
- total_tiles: args.total_tiles,
- e: args.e,
- },
- );
- self.pass_a_tile.encode_row_map(
- command_buffer,
- &MoePassARowMapArguments {
- expert_offsets: args.expert_offsets,
- row_expert_map: args.row_expert_map,
- total_rows: args.total_rows,
- e: args.e,
- },
- );
- self.pass_a_tile.encode_build_map(
- command_buffer,
- &MoePassATileBuildArguments {
- expert_offsets: args.expert_offsets,
- tile_offsets: args.tile_offsets,
- row_expert_map: args.row_expert_map,
- tile_map: args.tile_map,
- total_rows: args.total_rows,
- h_blocks,
- },
- );
- self.pass_a_tile.encode_dispatch_args(
- command_buffer,
- &MoePassATileDispatchArguments {
- total_tiles: args.total_tiles,
- dispatch_args: args.dispatch_args,
- num_tiles_y: 1,
- },
- );
-
- // pass a
- let gate_idx = args.gating_code.min(3) as usize;
- let dtype_idx = DTYPES.iter().position(|dtype| *dtype == args.data_type).unwrap();
- let pass_a_encoder =
- command_buffer.new_compute_command_encoder().expect("Failed to create compute command encoder");
- let pass_a_kernel = &self.pass_a_indirect[gate_idx][dtype_idx];
- pass_a_kernel.encode(
- &args.x_perm_buffer.retain(),
- &args.expert_offsets.retain(),
- &args.w13_all.retain(),
- &args.hidden_buffer.retain(),
- &args.up_biases.retain(),
- args.d_model as u32,
- args.d_ff as u32,
- args.e as u32,
- args.gate_clip_min,
- args.gate_clip_max,
- args.up_clip_min,
- args.up_clip_max,
- args.silu_alpha,
- &args.tile_map.retain(),
- &args.dispatch_args.retain(),
- &pass_a_encoder,
- );
- pass_a_encoder.end_encoding();
-
- // pass b
- let pass_b_kernel = &self.fused_down[dtype_idx];
- let pass_b_encoder =
- command_buffer.new_compute_command_encoder().expect("Failed to create compute command encoder");
- pass_b_kernel.encode(
- &args.hidden_buffer.retain(),
- &args.row_expert_map.retain(),
- &args.w2_all.retain(),
- &args.down_biases.retain(),
- &args.output_buffer.retain(),
- args.total_rows as u32,
- args.d_model as u32,
- args.d_ff as u32,
- args.e as u32,
- &pass_b_encoder,
- );
- pass_b_encoder.end_encoding();
- }
-}
diff --git a/crates/uzu/src/backends/metal/kernel/moe/experts_two_pass_prefill.rs b/crates/uzu/src/backends/metal/kernel/moe/experts_two_pass_prefill.rs
deleted file mode 100644
index a1e28cf39..000000000
--- a/crates/uzu/src/backends/metal/kernel/moe/experts_two_pass_prefill.rs
+++ /dev/null
@@ -1,203 +0,0 @@
-use metal::{MTLBlitCommandEncoderExt, MTLBuffer, MTLCommandBuffer, MTLCommandEncoder};
-use objc2::{
- __framework_prelude::{ProtocolObject, Retained},
- Message,
-};
-
-use crate::{
- DataType,
- backends::{
- common::kernel::{MoeExpertsPrefillPassAKernel, MoeExpertsPrefillPassBKernel},
- metal::{
- MetalContext, MetalError,
- kernel::{
- dsl::{MoeExpertsPrefillPassAMetalKernel, MoeExpertsPrefillPassBMetalKernel},
- moe::tiles_map::{
- MoeTileCountsArguments, MoeTileDispatchArguments, MoeTileMapBuildArguments, MoeTileMapKernels,
- MoeTileScanArguments,
- },
- },
- },
- },
-};
-
-const DTYPES: [DataType; 3] = [DataType::F16, DataType::BF16, DataType::F32];
-
-pub struct MoeExpertsTwoPassPrefillBlock {
- tile_map: MoeTileMapKernels,
- pass_a_indirect: Vec<Vec<MoeExpertsPrefillPassAMetalKernel>>, // [gate][dtype]
- pass_b_indirect: Vec<MoeExpertsPrefillPassBMetalKernel>, // [dtype]
-}
-
-impl MoeExpertsTwoPassPrefillBlock {
- pub fn new(ctx: &MetalContext) -> Result<Self, MetalError> {
- let dtypes = [DataType::F16, DataType::BF16, DataType::F32];
-
- let mut pass_a_indirect = vec![];
- for gate in 0u32..4u32 {
- let mut kernels = vec![];
- for dtype in dtypes {
- let kernel = MoeExpertsPrefillPassAMetalKernel::new(ctx, dtype, gate)?;
- kernels.push(kernel);
- }
- pass_a_indirect.push(kernels);
- }
-
- let mut pass_b_indirect = vec![];
- for dtype in dtypes {
- let kernel = MoeExpertsPrefillPassBMetalKernel::new(ctx, dtype)?;
- pass_b_indirect.push(kernel);
- }
-
- Ok(Self {
- tile_map: MoeTileMapKernels::new(ctx)?,
- pass_a_indirect,
- pass_b_indirect,
- })
- }
-
- pub fn encode(
- &self,
- command_buffer: &Retained<ProtocolObject<dyn MTLCommandBuffer>>,
- args: &MoeExpertsTwoPassArguments,
- ) {
- if args.total_rows == 0 {
- return;
- }
-
- let hidden_bytes = args.total_rows * args.d_ff * args.data_type.size_in_bytes();
- let blit_encoder = command_buffer.new_blit_command_encoder().expect("Failed to create blit command encoder");
- blit_encoder.fill_buffer_range_value(args.hidden_buffer, 0..hidden_bytes, 0);
- blit_encoder.end_encoding();
-
- self.tile_map.encode_counts(
- command_buffer,
- &MoeTileCountsArguments {
- offsets_buffer: args.expert_offsets,
- tile_counts_buffer: args.tile_counts,
- e: args.e,
- },
- );
- self.tile_map.encode_scan(
- command_buffer,
- &MoeTileScanArguments {
- tile_counts_buffer: args.tile_counts,
- tile_offsets_buffer: args.tile_offsets,
- total_tiles_buffer: args.total_tiles,
- e: args.e,
- },
- );
- self.tile_map.encode_build_map(
- command_buffer,
- &MoeTileMapBuildArguments {
- expert_offsets: args.expert_offsets,
- tile_offsets: args.tile_offsets,
- tile_counts: args.tile_counts,
- tile_map: args.tile_map,
- e: args.e,
- },
- );
- let d_model_u32 = args.d_model as u32;
- let d_ff_u32 = args.d_ff as u32;
- const COL_TILE_FF: u32 = 32; // Must match PASSA_BN in Metal kernel
- const COL_TILE_MODEL: u32 = 64;
- let n_tiles_ff = if d_ff_u32 == 0 {
- 0
- } else {
- (d_ff_u32 + COL_TILE_FF - 1) / COL_TILE_FF
- };
- let n_tiles_model = if d_model_u32 == 0 {
- 0
- } else {
- (d_model_u32 + COL_TILE_MODEL - 1) / COL_TILE_MODEL
- };
- self.tile_map.encode_dispatch_args(
- command_buffer,
- &MoeTileDispatchArguments {
- total_tiles: args.total_tiles,
- dispatch_args: args.dispatch_args,
- num_tiles_x: n_tiles_ff,
- },
- );
-
- let encoder_a =
- command_buffer.new_compute_command_encoder().expect("Failed to create compute command encoder A");
- let gate_idx = args.gating_code.min(3) as usize;
- let dtype_idx = DTYPES.iter().position(|t| *t == args.data_type).unwrap();
- let kernel_pass_a = &self.pass_a_indirect[gate_idx][dtype_idx];
- kernel_pass_a.encode(
- &args.x_perm_buffer.retain(),
- &args.expert_offsets.retain(),
- &args.w13_all.retain(),
- &args.up_biases.retain(),
- &args.hidden_buffer.retain(),
- args.d_model as u32,
- args.d_ff as u32,
- args.e as u32,
- args.gate_clip_min,
- args.gate_clip_max,
- args.up_clip_min,
- args.up_clip_max,
- args.silu_alpha,
- &args.tile_map.retain(),
- &args.dispatch_args.retain(),
- &encoder_a,
- );
- encoder_a.end_encoding();
-
- let dispatch_args = MoeTileDispatchArguments {
- total_tiles: args.total_tiles,
- dispatch_args: args.dispatch_args,
- num_tiles_x: n_tiles_model,
- };
- self.tile_map.encode_dispatch_args(command_buffer, &dispatch_args);
-
- let encoder_b =
- command_buffer.new_compute_command_encoder().expect("Failed to create compute command encoder B");
- let kernel_pass_b = &self.pass_b_indirect[dtype_idx];
- kernel_pass_b.encode(
- &args.hidden_buffer.retain(),
- &args.expert_offsets.retain(),
- &args.w2_all.retain(),
- &args.down_biases.retain(),
- &args.output_buffer.retain(),
- args.d_model as u32,
- args.d_ff as u32,
- args.e as u32,
- &args.tile_map.retain(),
- &args.dispatch_args.retain(),
- &encoder_b,
- );
- encoder_b.end_encoding();
- }
-}
-
-#[derive(Debug)]
-pub struct MoeExpertsTwoPassArguments<'a> {
- pub x_perm_buffer: &'a ProtocolObject<dyn MTLBuffer>,
- pub expert_offsets: &'a ProtocolObject<dyn MTLBuffer>,
- pub row_expert_map: &'a ProtocolObject<dyn MTLBuffer>,
- pub hidden_buffer: &'a ProtocolObject<dyn MTLBuffer>,
- pub output_buffer: &'a ProtocolObject<dyn MTLBuffer>,
- pub w13_all: &'a ProtocolObject<dyn MTLBuffer>,
- pub w2_all: &'a ProtocolObject<dyn MTLBuffer>,
- pub up_biases: &'a ProtocolObject<dyn MTLBuffer>,
- pub down_biases: &'a ProtocolObject<dyn MTLBuffer>,
- pub tile_counts: &'a ProtocolObject<dyn MTLBuffer>,
- pub tile_offsets: &'a ProtocolObject<dyn MTLBuffer>,
- pub tile_map: &'a ProtocolObject<dyn MTLBuffer>,
- pub total_tiles: &'a ProtocolObject<dyn MTLBuffer>,
- pub dispatch_args: &'a ProtocolObject<dyn MTLBuffer>,
- pub total_rows: usize,
- pub d_model: usize,
- pub d_ff: usize,
- pub e: usize,
- pub num_tiles_k: u32,
- pub gating_code: u32,
- pub gate_clip_min: f32,
- pub gate_clip_max: f32,
- pub up_clip_min: f32,
- pub up_clip_max: f32,
- pub silu_alpha: f32,
- pub data_type: DataType,
-}
diff --git a/crates/uzu/src/backends/metal/kernel/moe/gather.rs b/crates/uzu/src/backends/metal/kernel/moe/gather.rs
deleted file mode 100644
index 3229d21be..000000000
--- a/crates/uzu/src/backends/metal/kernel/moe/gather.rs
+++ /dev/null
@@ -1,83 +0,0 @@
-use metal::{MTLBuffer, MTLCommandBuffer, MTLCommandEncoder};
-use objc2::{Message, rc::Retained, runtime::ProtocolObject};
-
-use crate::{
- DataType,
- backends::{
- common::kernel::{MoeGatherXPerm1DKernel, MoeGatherXPerm2DKernel},
- metal::{
- MetalContext, MetalError,
- kernel::dsl::{MoeGatherXPerm1DMetalKernel, MoeGatherXPerm2DMetalKernel},
- },
- },
-};
-
-#[derive(Debug)]
-pub struct MoeGatherArguments<'a> {
- pub x_buffer: &'a ProtocolObject<dyn MTLBuffer>,
- pub bucketed_ids_buffer: &'a ProtocolObject<dyn MTLBuffer>,
- pub x_perm_buffer: &'a ProtocolObject<dyn MTLBuffer>,
- pub sumk_buffer: &'a ProtocolObject<dyn MTLBuffer>,
- pub t: usize,
- pub k: usize,
- pub d_model: usize,
-}
-
-pub struct MoeGatherKernels {
- bf16: MoeGatherXPerm2DMetalKernel,
- f16: MoeGatherXPerm1DMetalKernel,
- f32: MoeGatherXPerm1DMetalKernel,
-}
-
-impl MoeGatherKernels {
- pub fn new(ctx: &MetalContext) -> Result<Self, MetalError> {
- Ok(Self {
- bf16: MoeGatherXPerm2DMetalKernel::new(ctx, DataType::BF16)?,
- f16: MoeGatherXPerm1DMetalKernel::new(ctx, DataType::F16)?,
- f32: MoeGatherXPerm1DMetalKernel::new(ctx, DataType::F32)?,
- })
- }
-
- pub fn encode(
- &self,
- command_buffer: &Retained<ProtocolObject<dyn MTLCommandBuffer>>,
- dtype: DataType,
- args: &MoeGatherArguments,
- ) {
- let encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute command encoder");
- match dtype {
- DataType::F32 => self.f32.encode(
- &args.x_buffer.retain(),
- &args.bucketed_ids_buffer.retain(),
- &args.x_perm_buffer.retain(),
- &args.sumk_buffer.retain(),
- args.d_model as u32,
- args.t as u32,
- args.k as u32,
- &encoder,
- ),
- DataType::F16 => self.f16.encode(
- &args.x_buffer.retain(),
- &args.bucketed_ids_buffer.retain(),
- &args.x_perm_buffer.retain(),
- &args.sumk_buffer.retain(),
- args.d_model as u32,
- args.t as u32,
- args.k as u32,
- &encoder,
- ),
- DataType::BF16 => self.bf16.encode(
- &args.x_buffer.retain(),
- &args.bucketed_ids_buffer.retain(),
- &args.x_perm_buffer.retain(),
- &args.sumk_buffer.retain(),
- args.d_model as u32,
- args.t as u32,
- args.k as u32,
- &encoder,
- ),
- _ => panic!("Unsupported data type: {:?}", dtype),
- }
- encoder.end_encoding();
- }
-}
diff --git a/crates/uzu/src/backends/metal/kernel/moe/tiles_map.rs b/crates/uzu/src/backends/metal/kernel/moe/tiles_map.rs
deleted file mode 100644
index 1f9ef7de5..000000000
--- a/crates/uzu/src/backends/metal/kernel/moe/tiles_map.rs
+++ /dev/null
@@ -1,116 +0,0 @@
-use metal::{MTLBuffer, MTLCommandBuffer, MTLCommandEncoder};
-use objc2::{
- __framework_prelude::{ProtocolObject, Retained},
- Message,
-};
-
-use crate::backends::{
- common::kernel::{MoeBuildTileMapKernel, MoeTileCountsKernel, MoeTileScanKernel, MoeWriteDispatchArgsKernel},
- metal::{
- MetalContext, MetalError,
- kernel::dsl::{
- MoeBuildTileMapMetalKernel, MoeTileCountsMetalKernel, MoeTileScanMetalKernel,
- MoeWriteDispatchArgsMetalKernel,
- },
- },
-};
-
-pub struct MoeTileMapKernels {
- counts: MoeTileCountsMetalKernel,
- scan: MoeTileScanMetalKernel,
- build: MoeBuildTileMapMetalKernel,
- dispatch: MoeWriteDispatchArgsMetalKernel,
-}
-
-impl MoeTileMapKernels {
- pub fn new(ctx: &MetalContext) -> Result<Self, MetalError> {
- Ok(Self {
- counts: MoeTileCountsMetalKernel::new(ctx)?,
- scan: MoeTileScanMetalKernel::new(ctx)?,
- build: MoeBuildTileMapMetalKernel::new(ctx)?,
- dispatch: MoeWriteDispatchArgsMetalKernel::new(ctx)?,
- })
- }
-
- pub fn encode_counts(
- &self,
- command_buffer: &Retained<ProtocolObject<dyn MTLCommandBuffer>>,
- args: &MoeTileCountsArguments,
- ) {
- let encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute command encoder");
- self.counts.encode(&args.offsets_buffer.retain(), &args.tile_counts_buffer.retain(), args.e as u32, &encoder);
- encoder.end_encoding();
- }
-
- pub fn encode_scan(
- &self,
- command_buffer: &Retained<ProtocolObject<dyn MTLCommandBuffer>>,
- args: &MoeTileScanArguments,
- ) {
- let encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute command encoder");
- self.scan.encode(
- &args.tile_counts_buffer.retain(),
- &args.tile_offsets_buffer.retain(),
- &args.total_tiles_buffer.retain(),
- args.e as u32,
- &encoder,
- );
- encoder.end_encoding();
- }
-
- pub fn encode_build_map(
- &self,
- command_buffer: &Retained<ProtocolObject<dyn MTLCommandBuffer>>,
- args: &MoeTileMapBuildArguments,
- ) {
- let encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute command encoder");
- self.build.encode(
- &args.expert_offsets.retain(),
- &args.tile_offsets.retain(),
- &args.tile_counts.retain(),
- &args.tile_map.retain(),
- args.e as u32,
- &encoder,
- );
- encoder.end_encoding();
- }
-
- pub fn encode_dispatch_args(
- &self,
- command_buffer: &Retained<ProtocolObject<dyn MTLCommandBuffer>>,
- args: &MoeTileDispatchArguments,
- ) {
- let encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute command encoder");
- self.dispatch.encode(&args.total_tiles.retain(), &args.dispatch_args.retain(), args.num_tiles_x, &encoder);
- encoder.end_encoding();
- }
-}
-
-pub struct MoeTileCountsArguments<'a> {
- pub offsets_buffer: &'a ProtocolObject<dyn MTLBuffer>, // [E+1]
- pub tile_counts_buffer: &'a ProtocolObject<dyn MTLBuffer>, // [E]
- pub e: usize,
-}
-
-pub struct MoeTileScanArguments<'a> {
- pub tile_counts_buffer: &'a ProtocolObject<dyn MTLBuffer>, // [E]
- pub tile_offsets_buffer: &'a ProtocolObject<dyn MTLBuffer>, // [E+1]
- pub total_tiles_buffer: &'a ProtocolObject<dyn MTLBuffer>, // [>=2]
- pub e: usize,
-}
-
-#[derive(Debug)]
-pub struct MoeTileMapBuildArguments<'a> {
- pub expert_offsets: &'a ProtocolObject<dyn MTLBuffer>, // [E+1]
- pub tile_offsets: &'a ProtocolObject<dyn MTLBuffer>, // [E+1]
- pub tile_counts: &'a ProtocolObject<dyn MTLBuffer>, // [E]
- pub tile_map: &'a ProtocolObject<dyn MTLBuffer>, // [total_tiles * 3]
- pub e: usize,
-}
-
-#[derive(Debug)]
-pub struct MoeTileDispatchArguments<'a> {
- pub total_tiles: &'a ProtocolObject<dyn MTLBuffer>, // [>=1]
- pub dispatch_args: &'a ProtocolObject<dyn MTLBuffer>, // [3]
- pub num_tiles_x: u32, // x dimension for indirect dispatch
-}
diff --git a/crates/uzu/src/backends/metal/kernel/moe/tiles_pass_a.rs b/crates/uzu/src/backends/metal/kernel/moe/tiles_pass_a.rs
deleted file mode 100644
index c2deccd86..000000000
--- a/crates/uzu/src/backends/metal/kernel/moe/tiles_pass_a.rs
+++ /dev/null
@@ -1,153 +0,0 @@
-use metal::{MTLBuffer, MTLCommandBuffer, MTLCommandEncoder};
-use objc2::{Message, runtime::ProtocolObject};
-
-use crate::backends::{
- common::kernel::{
- MoePassABuildRowMapKernel, MoePassABuildTileMapKernel, MoePassATileCountsKernel, MoePassATileScanKernel,
- MoePassAWriteDispatchArgsKernel,
- },
- metal::{
- MetalContext, MetalError,
- kernel::dsl::{
- MoePassABuildRowMapMetalKernel, MoePassABuildTileMapMetalKernel, MoePassATileCountsMetalKernel,
- MoePassATileScanMetalKernel, MoePassAWriteDispatchArgsMetalKernel,
- },
- },
-};
-
-#[derive(Debug)]
-pub struct MoePassATileCountsArguments<'a> {
- pub expert_offsets: &'a ProtocolObject<dyn MTLBuffer>, // [E+1]
- pub tile_counts: &'a ProtocolObject<dyn MTLBuffer>, // [E]
- pub e: usize,
- pub h_blocks: u32,
-}
-
-#[derive(Debug)]
-pub struct MoePassATileScanArguments<'a> {
- pub tile_counts: &'a ProtocolObject<dyn MTLBuffer>, // [E]
- pub tile_offsets: &'a ProtocolObject<dyn MTLBuffer>, // [E+1]
- pub total_tiles: &'a ProtocolObject<dyn MTLBuffer>, // [>=1]
- pub e: usize,
-}
-
-#[derive(Debug)]
-pub struct MoePassARowMapArguments<'a> {
- pub expert_offsets: &'a ProtocolObject<dyn MTLBuffer>, // [E+1]
- pub row_expert_map: &'a ProtocolObject<dyn MTLBuffer>, // [total_rows]
- pub total_rows: usize,
- pub e: usize,
-}
-
-#[derive(Debug)]
-pub struct MoePassATileBuildArguments<'a> {
- pub expert_offsets: &'a ProtocolObject<dyn MTLBuffer>, // [E+1]
- pub tile_offsets: &'a ProtocolObject<dyn MTLBuffer>, // [E+1]
- pub row_expert_map: &'a ProtocolObject<dyn MTLBuffer>, // [total_rows]
- pub tile_map: &'a ProtocolObject<dyn MTLBuffer>, // [total_tiles * 3]
- pub total_rows: usize,
- pub h_blocks: u32,
-}
-
-#[derive(Debug)]
-pub struct MoePassATileDispatchArguments<'a> {
- pub total_tiles: &'a ProtocolObject<dyn MTLBuffer>, // [>=1]
- pub dispatch_args: &'a ProtocolObject<dyn MTLBuffer>, // [3]
- pub num_tiles_y: u32,
-}
-
-pub struct MoePassATileKernels {
- counts: MoePassATileCountsMetalKernel,
- scan: MoePassATileScanMetalKernel,
- row_map: MoePassABuildRowMapMetalKernel,
- build_map: MoePassABuildTileMapMetalKernel,
- dispatch: MoePassAWriteDispatchArgsMetalKernel,
-}
-
-impl MoePassATileKernels {
- pub fn new(ctx: &MetalContext) -> Result<Self, MetalError> {
- Ok(Self {
- counts: MoePassATileCountsMetalKernel::new(ctx)?,
- scan: MoePassATileScanMetalKernel::new(ctx)?,
- row_map: MoePassABuildRowMapMetalKernel::new(ctx)?,
- build_map: MoePassABuildTileMapMetalKernel::new(ctx)?,
- dispatch: MoePassAWriteDispatchArgsMetalKernel::new(ctx)?,
- })
- }
-
- pub fn encode_counts(
- &self,
- command_buffer: &ProtocolObject<dyn MTLCommandBuffer>,
- args: &MoePassATileCountsArguments,
- ) {
- let encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute command encoder");
- self.counts.encode(
- &args.expert_offsets.retain(),
- &args.tile_counts.retain(),
- args.e as u32,
- args.h_blocks,
- &encoder,
- );
- encoder.end_encoding();
- }
-
- pub fn encode_scan(
- &self,
- command_buffer: &ProtocolObject<dyn MTLCommandBuffer>,
- args: &MoePassATileScanArguments,
- ) {
- let encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute command encoder");
- self.scan.encode(
- &args.tile_counts.retain(),
- &args.tile_offsets.retain(),
- &args.total_tiles.retain(),
- args.e as u32,
- &encoder,
- );
- encoder.end_encoding();
- }
-
- pub fn encode_row_map(
- &self,
- command_buffer: &ProtocolObject<dyn MTLCommandBuffer>,
- args: &MoePassARowMapArguments,
- ) {
- let encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute command encoder");
- self.row_map.encode(
- &args.expert_offsets.retain(),
- &args.row_expert_map.retain(),
- args.total_rows as u32,
- args.e as u32,
- &encoder,
- );
- encoder.end_encoding();
- }
-
- pub fn encode_build_map(
- &self,
- command_buffer: &ProtocolObject<dyn MTLCommandBuffer>,
- args: &MoePassATileBuildArguments,
- ) {
- let encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute command encoder");
- self.build_map.encode(
- &args.expert_offsets.retain(),
- &args.tile_offsets.retain(),
- &args.row_expert_map.retain(),
- &args.tile_map.retain(),
- args.total_rows as u32,
- args.h_blocks,
- &encoder,
- );
- encoder.end_encoding();
- }
-
- pub fn encode_dispatch_args(
- &self,
- command_buffer: &ProtocolObject<dyn MTLCommandBuffer>,
- args: &MoePassATileDispatchArguments,
- ) {
- let encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute command encoder");
- self.dispatch.encode(&args.total_tiles.retain(), &args.dispatch_args.retain(), args.num_tiles_y, &encoder);
- encoder.end_encoding();
- }
-}
diff --git a/crates/uzu/src/encodable_block/mod.rs b/crates/uzu/src/encodable_block/mod.rs
index 1aaf72743..9911ca737 100644
--- a/crates/uzu/src/encodable_block/mod.rs
+++ b/crates/uzu/src/encodable_block/mod.rs
@@ -11,6 +11,7 @@ mod full_precision_linear;
mod layer_norm;
mod mamba_mixer;
mod mlp;
+mod moe_block;
mod normalization;
mod pooling;
mod prediction_head;
@@ -33,6 +34,7 @@ pub use full_precision_linear::{FullPrecisionLinear, FullPrecisionLinearError};
pub use layer_norm::{LayerNorm, LayerNormError};
pub(crate) use mamba_mixer::MambaMixer;
pub use mlp::MlpBlock;
+pub use moe_block::MoeBlock;
pub use normalization::{Normalization, NormalizationError};
pub use pooling::Pooling;
pub use prediction_head::ClassifierPredictionHead;
diff --git a/crates/uzu/src/backends/metal/encodable_block/moe_block/moe_block.rs b/crates/uzu/src/encodable_block/moe_block.rs
similarity index 52%
rename from crates/uzu/src/backends/metal/encodable_block/moe_block/moe_block.rs
rename to crates/uzu/src/encodable_block/moe_block.rs
index a5cbbca4a..321e28f32 100644
--- a/crates/uzu/src/backends/metal/encodable_block/moe_block/moe_block.rs
+++ b/crates/uzu/src/encodable_block/moe_block.rs
@@ -2,80 +2,81 @@
use std::{cell::RefCell, rc::Rc};
-use metal::{MTLBlitCommandEncoderExt, MTLBuffer, MTLCommandBuffer, MTLCommandEncoder, MTLComputeCommandEncoder};
-use objc2::{rc::Retained, runtime::ProtocolObject};
+use thiserror::Error;
-use super::SharedMoeWeights;
use crate::{
Activation, DataType, LinearConfig, MixtureOfExpertsConfig, RoutingFunctionConfig,
array::Array,
- backends::{
- common::kernel::{
+ backends::common::{
+ Backend, CommandBuffer, CopyEncoder, Kernels,
+ kernel::{
MoeBlockBasesFromPartialsKernel, MoeCountsOffsetsFusedKernel, MoeFinalizeKernel, MoeRouterTopKKernel,
MoeScatterBucketsMapKernel,
- },
- metal::{
- Metal, MetalContext,
- kernel::{
- MoeGatherKernels,
- dsl::{
- MoeBlockBasesFromPartialsMetalKernel, MoeCountsOffsetsFusedMetalKernel, MoeFinalizeMetalKernel,
- MoeRouterTopKMetalKernel, MoeScatterBucketsMapMetalKernel,
- },
- moe::{
- MoeExpertsTwoPassArguments, MoeExpertsTwoPassDecodeBlock, MoeExpertsTwoPassPrefillBlock,
- MoeGatherArguments,
- },
+ moe::{
+ MoeExpertsTwoPassArguments, MoeExpertsTwoPassDecodeBlock, MoeExpertsTwoPassPrefillBlock,
+ MoeGatherArguments, MoeGatherKernels,
},
},
},
encodable_block::{EncodableBlock, EncodingParameters},
forward_pass::state::{ArrayId, ForwardPassState},
- parameters::ParameterTree,
+ parameters::{ParameterLoaderError, ParameterTree},
};
-enum RouterBlock {
- Metal {
- weights_buf: Retained<ProtocolObject<dyn MTLBuffer>>,
- biases_buf: Retained<ProtocolObject<dyn MTLBuffer>>,
- },
+struct RouterBlock<B: Backend> {
+ weights_buf: B::NativeBuffer,
+ biases_buf: B::NativeBuffer,
}
-pub struct MoeBlock {
- router: RouterBlock,
+#[derive(Clone)]
+struct SharedMoeWeights<B: Backend> {
+ pub w13_buf: Rc<B::NativeBuffer>,
+ pub w2_buf: Rc<B::NativeBuffer>,
+ pub up_biases_buf: Rc<B::NativeBuffer>,
+ pub down_biases_buf: Rc<B::NativeBuffer>,
+}
+
+pub struct MoeBlock<B: Backend> {
+ router: RouterBlock<B>,
router_renorm: bool,
- router_topk_kernel: MoeRouterTopKMetalKernel,
- counts_offsets_kernel: MoeCountsOffsetsFusedMetalKernel,
- scatter_bases_kernel: MoeBlockBasesFromPartialsMetalKernel,
- scatter_map_kernel: MoeScatterBucketsMapMetalKernel,
- gather_kernels: MoeGatherKernels,
- experts_two_pass_decode_kernel: MoeExpertsTwoPassDecodeBlock,
- experts_two_pass_prefill_kernel: MoeExpertsTwoPassPrefillBlock,
- finalize_kernel: MoeFinalizeMetalKernel,
+ router_topk_kernel: <B::Kernels as Kernels>::MoeRouterTopKKernel,
+ counts_offsets_kernel: <B::Kernels as Kernels>::MoeCountsOffsetsFusedKernel,
+ scatter_bases_kernel: <B::Kernels as Kernels>::MoeBlockBasesFromPartialsKernel,
+ scatter_map_kernel: <B::Kernels as Kernels>::MoeScatterBucketsMapKernel,
+ gather_kernels: MoeGatherKernels<B>,
+ experts_two_pass_decode_kernel: MoeExpertsTwoPassDecodeBlock<B>,
+ experts_two_pass_prefill_kernel: MoeExpertsTwoPassPrefillBlock<B>,
+ finalize_kernel: <B::Kernels as Kernels>::MoeFinalizeKernel,
moe_config: MixtureOfExpertsConfig,
model_dim: usize,
hidden_dim: usize,
data_type: DataType,
- shared_weights: SharedMoeWeights,
+ shared_weights: SharedMoeWeights<B>,
}
-impl MoeBlock {
+#[derive(Debug, Error)]
+pub enum MoeBlockError<B: Backend> {
+ #[error("Backend error: {0}")]
+ BackendError(#[source] B::Error),
+ #[error("Parameter loader error: {0}")]
+ ParameterLoaderError(#[source] ParameterLoaderError),
+}
+
+impl<B: Backend> MoeBlock<B> {
pub fn new(
- context: &MetalContext,
+ context: &B::Context,
moe_config: &MixtureOfExpertsConfig,
model_dim: usize,
hidden_dim: usize,
- parameter_tree: &ParameterTree<MetalContext>,
- ) -> Result<Self, crate::backends::metal::MetalError> {
+ parameter_tree: &ParameterTree<B::Context>,
+ ) -> Result<Self, MoeBlockError<B>> {
let data_type: DataType = moe_config.expert_config.linear_config.activation_precision().into();
let router_data_type: DataType = moe_config.router_config.activation_precision().into();
let router_renorm = matches!(moe_config.routing_function, RoutingFunctionConfig::SoftmaxRouting);
- let router_tree = parameter_tree
- .subtree("router")
- .map_err(|e| crate::backends::metal::MetalError::Generic(format!("Router subtree error: {:?}", e)))?;
+ let router_tree = parameter_tree.subtree("router").map_err(MoeBlockError::ParameterLoaderError)?;
let router = match &moe_config.router_config {
LinearConfig::Quantized(_) => {
@@ -87,19 +88,15 @@ impl MoeBlock {
LinearConfig::FullPrecision {
..
} => {
- let weights_arr = router_tree.leaf("weights").map_err(|e| {
- crate::backends::metal::MetalError::Generic(format!("Router weights error: {:?}", e))
- })?;
+ let weights_arr = router_tree.leaf("weights").map_err(MoeBlockError::ParameterLoaderError)?;
let weights_buf = weights_arr.buffer();
- let biases_arr = router_tree.leaf("biases").map_err(|e| {
- crate::backends::metal::MetalError::Generic(format!("Router biases error: {:?}", e))
- })?;
+ let biases_arr = router_tree.leaf("biases").map_err(MoeBlockError::ParameterLoaderError)?;
let biases_buf = biases_arr.buffer();
- RouterBlock::Metal {
- weights_buf: weights_buf.into(),
- biases_buf: biases_buf.into(),
+ RouterBlock {
+ weights_buf: weights_buf.clone(),
+ biases_buf: biases_buf.clone(),
}
},
LinearConfig::QLoRA {
@@ -109,64 +106,51 @@ impl MoeBlock {
},
};
- let router_topk_kernel = MoeRouterTopKMetalKernel::new(context, router_data_type).map_err(|e| {
- crate::backends::metal::MetalError::Generic(format!("RouterTopK fused kernel error: {:?}", e))
- })?;
- let counts_offsets_kernel = MoeCountsOffsetsFusedKernel::new(context).map_err(|e| {
- crate::backends::metal::MetalError::Generic(format!("Counts+offsets kernel error: {:?}", e))
- })?;
-
- let scatter_bases_kernel = MoeBlockBasesFromPartialsMetalKernel::new(context).map_err(|e| {
- crate::backends::metal::MetalError::Generic(format!(
- "MoeBlockBasesFromPartialsMetalKernel kernel error: {:?}",
- e
- ))
- })?;
- let scatter_map_kernel = MoeScatterBucketsMapMetalKernel::new(context, data_type).map_err(|e| {
- crate::backends::metal::MetalError::Generic(format!("MoeScatterBucketsMetalKernel kernel error: {:?}", e))
- })?;
-
- let gather_kernels = MoeGatherKernels::new(context)
- .map_err(|e| crate::backends::metal::MetalError::Generic(format!("Gather kernel error: {:?}", e)))?;
- let experts_two_pass_decode_kernel = MoeExpertsTwoPassDecodeBlock::new(context).map_err(|e| {
- crate::backends::metal::MetalError::Generic(format!("Experts two-pass decode kernel error: {:?}", e))
- })?;
- let experts_two_pass_prefill_kernel = MoeExpertsTwoPassPrefillBlock::new(context).map_err(|e| {
- crate::backends::metal::MetalError::Generic(format!("Experts two-pass prefill kernel error: {:?}", e))
- })?;
- let finalize_kernel = MoeFinalizeMetalKernel::new(context, data_type)
- .map_err(|e| crate::backends::metal::MetalError::Generic(format!("Finalize kernel error: {:?}", e)))?;
-
- let experts_tree = parameter_tree
- .subtree("experts")
- .map_err(|e| crate::backends::metal::MetalError::Generic(format!("experts subtree error: {:?}", e)))?;
+ let router_topk_kernel = <B::Kernels as Kernels>::MoeRouterTopKKernel::new(context, router_data_type)
+ .map_err(MoeBlockError::BackendError)?;
+ let counts_offsets_kernel = MoeCountsOffsetsFusedKernel::new(context).map_err(MoeBlockError::BackendError)?;
+
+ let scatter_bases_kernel = <B::Kernels as Kernels>::MoeBlockBasesFromPartialsKernel::new(context)
+ .map_err(MoeBlockError::BackendError)?;
+ let scatter_map_kernel = <B::Kernels as Kernels>::MoeScatterBucketsMapKernel::new(context, data_type)
+ .map_err(MoeBlockError::BackendError)?;
+
+ let gather_kernels = MoeGatherKernels::new(context).map_err(MoeBlockError::BackendError)?;
+ let experts_two_pass_decode_kernel =
+ MoeExpertsTwoPassDecodeBlock::new(context).map_err(MoeBlockError::BackendError)?;
+ let experts_two_pass_prefill_kernel =
+ MoeExpertsTwoPassPrefillBlock::new(context).map_err(MoeBlockError::BackendError)?;
+ let finalize_kernel =
+ <B::Kernels as Kernels>::MoeFinalizeKernel::new(context, data_type).map_err(MoeBlockError::BackendError)?;
+
+ let experts_tree = parameter_tree.subtree("experts").map_err(MoeBlockError::ParameterLoaderError)?;
let w13_arr = experts_tree
.subtree("up_projection")
- .map_err(|e| crate::backends::metal::MetalError::Generic(format!("up_projection error: {:?}", e)))?
+ .map_err(MoeBlockError::ParameterLoaderError)?
.leaf("weights")
- .map_err(|e| crate::backends::metal::MetalError::Generic(format!("up weights error: {:?}", e)))?;
+ .map_err(MoeBlockError::ParameterLoaderError)?;
let w13_buf = w13_arr.buffer().clone();
let w2_arr = experts_tree
.subtree("down_projection")
- .map_err(|e| crate::backends::metal::MetalError::Generic(format!("down_projection error: {:?}", e)))?
+ .map_err(MoeBlockError::ParameterLoaderError)?
.leaf("weights")
- .map_err(|e| crate::backends::metal::MetalError::Generic(format!("down weights error: {:?}", e)))?;
+ .map_err(MoeBlockError::ParameterLoaderError)?;
let w2_buf = w2_arr.buffer().clone();
let up_biases_arr = experts_tree
.subtree("up_projection")
- .map_err(|e| crate::backends::metal::MetalError::Generic(format!("up_projection biases error: {:?}", e)))?
+ .map_err(MoeBlockError::ParameterLoaderError)?
.leaf("biases")
- .map_err(|e| crate::backends::metal::MetalError::Generic(format!("up biases error: {:?}", e)))?;
+ .map_err(MoeBlockError::ParameterLoaderError)?;
let up_biases_buf = up_biases_arr.buffer().clone();
let down_biases_arr = experts_tree
.subtree("down_projection")
- .map_err(|e| crate::backends::metal::MetalError::Generic(format!("down_projection biases error: {:?}", e)))?
+ .map_err(MoeBlockError::ParameterLoaderError)?
.leaf("biases")
- .map_err(|e| crate::backends::metal::MetalError::Generic(format!("down biases error: {:?}", e)))?;
+ .map_err(MoeBlockError::ParameterLoaderError)?;
let down_biases_buf = down_biases_arr.buffer().clone();
let shared_weights = SharedMoeWeights {
@@ -208,12 +192,12 @@ impl MoeBlock {
}
}
-impl EncodableBlock<Metal> for MoeBlock {
+impl<B: Backend> EncodableBlock<B> for MoeBlock<B> {
fn encode(
&self,
- state: &mut ForwardPassState<Metal>,
- parameters: &EncodingParameters<Metal>,
- command_buffer: &Retained<ProtocolObject<dyn MTLCommandBuffer>>,
+ state: &mut ForwardPassState<B>,
+ parameters: &EncodingParameters<B>,
+ command_buffer: &B::CommandBuffer,
) {
let suffix_length = state.active_suffix_length();
let arrays = state.arrays(&[
@@ -239,9 +223,7 @@ impl EncodableBlock<Metal> for MoeBlock {
ArrayId::MoeTwoPassRowExpertMap,
]);
- let clone_buffer = |array: &RefCell<Array<Metal>>| -> Retained<ProtocolObject<dyn MTLBuffer>> {
- array.borrow().buffer().to_owned().into()
- };
+ let clone_buffer = |array: &RefCell<Array<B>>| -> B::NativeBuffer { array.borrow().buffer().to_owned().into() };
let mut array_iter = arrays.iter();
let main_buf = clone_buffer(array_iter.next().unwrap());
@@ -273,83 +255,71 @@ impl EncodableBlock<Metal> for MoeBlock {
let k_tile = 128;
// Clear internal MoE buffers
- let blit_encoder = root.new_blit_command_encoder().expect("Failed to create blit command encoder");
-
- if suffix_length > 0 && k > 0 {
- let entries = suffix_length * k;
- let topk_bytes = entries * std::mem::size_of::<u32>();
- let tok2row_bytes = entries * std::mem::size_of::<i32>();
-
- // Clear topk_ids and tok2row buffers
- if topk_bytes > 0 {
- blit_encoder.fill_buffer_range_value(&topk_ids_buf, 0..topk_bytes, 0xFF);
- }
- if tok2row_bytes > 0 {
- blit_encoder.fill_buffer_range_value(&tok2row_buf, 0..tok2row_bytes, 0xFF);
- }
+ root.with_copy_encoder(|encoder| {
+ if suffix_length > 0 && k > 0 {
+ let entries = suffix_length * k;
+ let topk_bytes = entries * std::mem::size_of::<u32>();
+ let tok2row_bytes = entries * std::mem::size_of::<i32>();
+
+ // Clear topk_ids and tok2row buffers
+ if topk_bytes > 0 {
+ encoder.encode_fill(&topk_ids_buf, 0..topk_bytes, 0xFF);
+ }
+ if tok2row_bytes > 0 {
+ encoder.encode_fill(&tok2row_buf, 0..tok2row_bytes, 0xFF);
+ }
- // Clear hidden buffer
- let hidden_bytes = suffix_length * k * self.hidden_dim * self.data_type.size_in_bytes();
- if hidden_bytes > 0 {
- blit_encoder.fill_buffer_range_value(&hidden_buf, 0..hidden_bytes, 0);
- }
+ // Clear hidden buffer
+ let hidden_bytes = suffix_length * k * self.hidden_dim * self.data_type.size_in_bytes();
+ if hidden_bytes > 0 {
+ encoder.encode_fill(&hidden_buf, 0..hidden_bytes, 0);
+ }
- // Clear y_partial buffer
- let y_partial_bytes = suffix_length * k * self.model_dim * self.data_type.size_in_bytes();
- if y_partial_bytes > 0 {
- blit_encoder.fill_buffer_range_value(&y_partial_buf, 0..y_partial_bytes, 0);
- }
+ // Clear y_partial buffer
+ let y_partial_bytes = suffix_length * k * self.model_dim * self.data_type.size_in_bytes();
+ if y_partial_bytes > 0 {
+ encoder.encode_fill(&y_partial_buf, 0..y_partial_bytes, 0);
+ }
- // Clear x_perm buffer
- let x_perm_bytes = suffix_length * k * self.model_dim * self.data_type.size_in_bytes();
- if x_perm_bytes > 0 {
- blit_encoder.fill_buffer_range_value(&x_perm_buf, 0..x_perm_bytes, 0);
+ // Clear x_perm buffer
+ let x_perm_bytes = suffix_length * k * self.model_dim * self.data_type.size_in_bytes();
+ if x_perm_bytes > 0 {
+ encoder.encode_fill(&x_perm_buf, 0..x_perm_bytes, 0);
+ }
}
- }
-
- blit_encoder.end_encoding();
+ });
// Use fused Router+TopK kernel for non-quantized routers
- match &self.router {
- RouterBlock::Metal {
- weights_buf,
- biases_buf,
- } => {
- if self.model_dim % 4 != 0 {
- panic!("MoE fused router+topk failed: {} % 4 != 0", self.model_dim);
- }
- if e > 512 {
- panic!("MoE fused router+topk failed: {e} > 512");
- }
- if k > 128 {
- panic!("MoE fused router+bottomk failed: {k} > 128");
- }
+ if self.model_dim % 4 != 0 {
+ panic!("MoE fused router+topk failed: {} % 4 != 0", self.model_dim);
+ }
+ if e > 512 {
+ panic!("MoE fused router+topk failed: {e} > 512");
+ }
+ if k > 128 {
+ panic!("MoE fused router+bottomk failed: {k} > 128");
+ }
- if suffix_length > 0 && e > 0 && k > 0 {
- let encoder =
- command_buffer.new_compute_command_encoder().expect("Failed to create compute command encoder");
- // Use the fused router+topk kernel
- self.router_topk_kernel.encode(
- &main_buf,
- weights_buf,
- biases_buf,
- &topk_ids_buf,
- &topk_probs_buf,
- suffix_length as u32,
- self.model_dim as u32,
- e as u32,
- k as u32,
- self.router_renorm,
- &encoder,
- );
- encoder.end_encoding();
- }
- },
+ if suffix_length > 0 && e > 0 && k > 0 {
+ command_buffer.with_compute_encoder(|encoder| {
+ // Use the fused router+topk kernel
+ self.router_topk_kernel.encode(
+ &main_buf,
+ &self.router.weights_buf,
+ &self.router.biases_buf,
+ &topk_ids_buf,
+ &topk_probs_buf,
+ suffix_length as u32,
+ self.model_dim as u32,
+ e as u32,
+ k as u32,
+ self.router_renorm,
+ &encoder,
+ );
+ });
}
- {
- let encoder =
- command_buffer.new_compute_command_encoder().expect("Failed to create compute command encoder");
+ command_buffer.with_compute_encoder(|encoder| {
self.counts_offsets_kernel.encode(
&topk_ids_buf,
&offsets_buf,
@@ -360,46 +330,40 @@ impl EncodableBlock<Metal> for MoeBlock {
k as u32,
&encoder,
);
- encoder.end_encoding();
- }
+ });
let num_blocks = ((suffix_length + 255) / 256).max(1);
let num_tiles = ((e + 512 - 1) / 512).max(1);
- let scatter_bases_encoder = command_buffer
- .new_compute_command_encoder()
- .expect("Failed to create compute command encoder for scatter bases kernel");
- self.scatter_bases_kernel.encode(
- &partials_buf,
- &block_bases_buf,
- &block_alloc_buf,
- e as u32,
- num_blocks as u32,
- num_tiles as u32,
- 0u32,
- &scatter_bases_encoder,
- );
- scatter_bases_encoder.end_encoding();
-
- let scatter_encoder = command_buffer
- .new_compute_command_encoder()
- .expect("Failed to create compute command encoder for scatter map kernel");
- self.scatter_map_kernel.encode(
- &topk_ids_buf,
- &topk_probs_buf,
- &offsets_buf,
- &block_bases_buf,
- &block_alloc_buf,
- &bucketed_ids_buf,
- &bucketed_probs_buf,
- suffix_length as u32,
- e as u32,
- k as u32,
- num_blocks as u32,
- num_tiles as u32,
- &tok2row_buf,
- &scatter_encoder,
- );
- scatter_encoder.end_encoding();
+ command_buffer.with_compute_encoder(|encoder| {
+ self.scatter_bases_kernel.encode(
+ &partials_buf,
+ &block_bases_buf,
+ &block_alloc_buf,
+ e as u32,
+ num_blocks as u32,
+ num_tiles as u32,
+ 0u32,
+ &encoder,
+ );
+ });
+ command_buffer.with_compute_encoder(|encoder| {
+ self.scatter_map_kernel.encode(
+ &topk_ids_buf,
+ &topk_probs_buf,
+ &offsets_buf,
+ &block_bases_buf,
+ &block_alloc_buf,
+ &bucketed_ids_buf,
+ &bucketed_probs_buf,
+ suffix_length as u32,
+ e as u32,
+ k as u32,
+ num_blocks as u32,
+ num_tiles as u32,
+ &tok2row_buf,
+ &encoder,
+ );
+ });
self.gather_kernels.encode(
root,
@@ -469,10 +433,10 @@ impl EncodableBlock<Metal> for MoeBlock {
row_expert_map: &row_expert_map_buf,
hidden_buffer: &hidden_buf,
output_buffer: &y_partial_buf,
- w13_all: &self.shared_weights.w13_buf,
- w2_all: &self.shared_weights.w2_buf,
- up_biases: &self.shared_weights.up_biases_buf,
- down_biases: &self.shared_weights.down_biases_buf,
+ w13_all: self.shared_weights.w13_buf.as_ref(),
+ w2_all: self.shared_weights.w2_buf.as_ref(),
+ up_biases: self.shared_weights.up_biases_buf.as_ref(),
+ down_biases: self.shared_weights.down_biases_buf.as_ref(),
tile_counts: &tile_counts_buf,
tile_offsets: &tile_offsets_buf,
tile_map: &tile_map_buf,
@@ -494,9 +458,7 @@ impl EncodableBlock<Metal> for MoeBlock {
self.experts_two_pass_prefill_kernel.encode(root, &args);
}
- {
- let encoder =
- command_buffer.new_compute_command_encoder().expect("Failed to create compute command encoder");
+ command_buffer.with_compute_encoder(|encoder| {
self.finalize_kernel.encode(
&tok2row_buf,
&topk_probs_buf,
@@ -507,11 +469,10 @@ impl EncodableBlock<Metal> for MoeBlock {
k as u32,
&encoder,
);
- encoder.end_encoding();
- }
+ });
if parameters.wait_until_completed {
- command_buffer.commit();
+ command_buffer.submit();
command_buffer.wait_until_completed();
}
}
@@ -522,9 +483,9 @@ impl EncodableBlock<Metal> for MoeBlock {
fn encode_with_shared_encoder(
&self,
- _state: &mut ForwardPassState<Metal>,
- _parameters: &EncodingParameters<Metal>,
- _encoder: &ProtocolObject<dyn MTLComputeCommandEncoder>,
+ _state: &mut ForwardPassState<B>,
+ _parameters: &EncodingParameters<B>,
+ _encoder: &B::ComputeEncoder,
) {
unreachable!("MoeBlock does not support shared compute encoder");
}
diff --git a/crates/uzu/tests/moe/moe_block_e2e_test.rs b/crates/uzu/tests/moe/moe_block_e2e_test.rs
index 7815565d8..4eaac6417 100644
--- a/crates/uzu/tests/moe/moe_block_e2e_test.rs
+++ b/crates/uzu/tests/moe/moe_block_e2e_test.rs
@@ -7,17 +7,13 @@ use uzu::{
common::kernel::{
MoeBlockBasesFromPartialsKernel, MoeCountsOffsetsFusedKernel, MoeFinalizeKernel, MoeRouterTopKKernel,
MoeScatterBucketsMapKernel,
+ moe::{MoeExpertsTwoPassArguments, MoeExpertsTwoPassPrefillBlock, MoeGatherArguments, MoeGatherKernels},
},
metal::{
- MetalContext,
- kernel::{
- dsl::{
- MoeBlockBasesFromPartialsMetalKernel, MoeCountsOffsetsFusedMetalKernel, MoeFinalizeMetalKernel,
- MoeRouterTopKMetalKernel, MoeScatterBucketsMapMetalKernel,
- },
- moe::{
- MoeExpertsTwoPassArguments, MoeExpertsTwoPassPrefillBlock, MoeGatherArguments, MoeGatherKernels,
- },
+ Metal, MetalContext,
+ kernel::dsl::{
+ MoeBlockBasesFromPartialsMetalKernel, MoeCountsOffsetsFusedMetalKernel, MoeFinalizeMetalKernel,
+ MoeRouterTopKMetalKernel, MoeScatterBucketsMapMetalKernel,
},
},
},
@@ -411,7 +407,7 @@ fn run_moe_parity_test_internal(
);
scatter_encoder.end_encoding();
- let gather = MoeGatherKernels::new(&ctx).expect("gather");
+ let gather = MoeGatherKernels::<Metal>::new(&ctx).expect("gather");
gather.encode(
&cb,
DataType::BF16,
@@ -431,7 +427,7 @@ fn run_moe_parity_test_internal(
let hidden_buf = alloc_buffer::<f32>(&ctx, total_rows * d_ff);
let row_expert_map_buf = alloc_buffer::<u32>(&ctx, total_rows);
- let experts = MoeExpertsTwoPassPrefillBlock::new(&ctx).expect("experts");
+ let experts = MoeExpertsTwoPassPrefillBlock::<Metal>::new(&ctx).expect("experts");
let num_tiles_k = ((d_ff + 64 - 1) / 64) as u32;
let args = MoeExpertsTwoPassArguments {
x_perm_buffer: &x_perm_buf,
diff --git a/crates/uzu/tests/moe/moe_experts_perf_test.rs b/crates/uzu/tests/moe/moe_experts_perf_test.rs
index 2bb58c14f..091a5b67a 100644
--- a/crates/uzu/tests/moe/moe_experts_perf_test.rs
+++ b/crates/uzu/tests/moe/moe_experts_perf_test.rs
@@ -5,12 +5,12 @@ use metal::{MTLCommandBuffer, MTLCommandQueue};
use rand::{RngExt, SeedableRng, rngs::StdRng};
use uzu::{
DataType,
- backends::metal::{
- MetalContext,
- kernel::moe::{
+ backends::{
+ common::kernel::moe::{
MoeExpertsSingleDecodeArguments, MoeExpertsSingleDecodeKernels, MoeExpertsTwoPassArguments,
MoeExpertsTwoPassDecodeBlock, MoeExpertsTwoPassPrefillBlock,
},
+ metal::{Metal, MetalContext},
},
};
@@ -76,7 +76,7 @@ fn run_decode_case(
let up_biases: Vec<bf16> = (0..e * 2 * d_ff).map(|_| bf16::from_f32(rng.random_range(-0.01..0.01))).collect();
let down_biases: Vec<bf16> = (0..e * d_model).map(|_| bf16::from_f32(rng.random_range(-0.01..0.01))).collect();
- let experts_kernel = MoeExpertsTwoPassDecodeBlock::new(ctx).expect("experts decode kernel");
+ let experts_kernel = MoeExpertsTwoPassDecodeBlock::<Metal>::new(ctx).expect("experts decode kernel");
let x_perm_buf = alloc_buffer_with_data(&ctx, &x_perm);
let offsets_buf = alloc_buffer_with_data(&ctx, &offsets);
@@ -213,7 +213,7 @@ fn run_two_pass_prefill_case(
let up_biases: Vec<bf16> = (0..e * 2 * d_ff).map(|_| bf16::from_f32(rng.random_range(-0.01..0.01))).collect();
let down_biases: Vec<bf16> = (0..e * d_model).map(|_| bf16::from_f32(rng.random_range(-0.01..0.01))).collect();
- let experts_kernel = MoeExpertsTwoPassPrefillBlock::new(ctx).expect("experts prefill kernel");
+ let experts_kernel = MoeExpertsTwoPassPrefillBlock::<Metal>::new(ctx).expect("experts prefill kernel");
let x_perm_buf = alloc_buffer_with_data(&ctx, &x_perm);
let offsets_buf = alloc_buffer_with_data(&ctx, &offsets);
@@ -339,7 +339,7 @@ fn run_fused_single_token_case(
let up_biases: Vec<bf16> = (0..e * 2 * d_ff).map(|_| bf16::from_f32(rng.random_range(-0.01..0.01))).collect();
let down_biases: Vec<bf16> = (0..e * d_model).map(|_| bf16::from_f32(rng.random_range(-0.01..0.01))).collect();
- let fused_kernel = MoeExpertsSingleDecodeKernels::new(ctx).expect("fused kernel");
+ let fused_kernel = MoeExpertsSingleDecodeKernels::<Metal>::new(ctx).expect("fused kernel");
let x_buf = alloc_buffer_with_data(ctx, &x);
let topk_ids_buf = alloc_buffer_with_data(ctx, &topk_ids);
@@ -486,7 +486,7 @@ fn run_indirect_decode_timed(
let up_biases: Vec<bf16> = (0..e * 2 * d_ff).map(|_| bf16::from_f32(rng.random_range(-0.01..0.01))).collect();
let down_biases: Vec<bf16> = (0..e * d_model).map(|_| bf16::from_f32(rng.random_range(-0.01..0.01))).collect();
- let experts_kernel = MoeExpertsTwoPassDecodeBlock::new(ctx).expect("decode kernel");
+ let experts_kernel = MoeExpertsTwoPassDecodeBlock::<Metal>::new(ctx).expect("decode kernel");
let x_perm_buf = alloc_buffer_with_data(ctx, &x_perm);
let offsets_buf = alloc_buffer_with_data(ctx, &offsets);
@@ -583,7 +583,7 @@ fn run_fused_decode_timed(
let up_biases: Vec<bf16> = (0..e * 2 * d_ff).map(|_| bf16::from_f32(rng.random_range(-0.01..0.01))).collect();
let down_biases: Vec<bf16> = (0..e * d_model).map(|_| bf16::from_f32(rng.random_range(-0.01..0.01))).collect();
- let fused_kernel = MoeExpertsSingleDecodeKernels::new(ctx).expect("fused kernel");
+ let fused_kernel = MoeExpertsSingleDecodeKernels::<Metal>::new(ctx).expect("fused kernel");
let x_buf = alloc_buffer_with_data(ctx, &x);
let topk_ids_buf = alloc_buffer_with_data(ctx, &topk_ids);
diff --git a/crates/uzu/tests/moe/moe_experts_test.rs b/crates/uzu/tests/moe/moe_experts_test.rs
index 6c566785e..8bcd8bc53 100644
--- a/crates/uzu/tests/moe/moe_experts_test.rs
+++ b/crates/uzu/tests/moe/moe_experts_test.rs
@@ -13,16 +13,19 @@ use metal::{MTLBuffer, MTLCommandBuffer, MTLCommandQueue};
use rand::{RngExt, SeedableRng, rngs::StdRng};
use uzu::{
DataType,
- backends::metal::kernel::moe::{
- MoeExpertsSingleDecodeKernels, MoeExpertsTwoPassArguments, MoeExpertsTwoPassDecodeBlock,
- MoeExpertsTwoPassPrefillBlock,
+ backends::{
+ common::kernel::moe::{
+ MoeExpertsSingleDecodeKernels, MoeExpertsTwoPassArguments, MoeExpertsTwoPassDecodeBlock,
+ MoeExpertsTwoPassPrefillBlock,
+ },
+ metal::Metal,
},
};
#[path = "moe_test_utils.rs"]
mod test_utils;
use test_utils::{alloc_buffer, alloc_buffer_with_data, assert_bf16_close, cpu_tile_counts, cpu_tile_scan, create_ctx};
-use uzu::backends::metal::kernel::moe::MoeExpertsSingleDecodeArguments;
+use uzu::backends::common::kernel::moe::MoeExpertsSingleDecodeArguments;
/// Test data for MoE experts
struct MoeTestData {
@@ -412,7 +415,7 @@ fn test_two_pass_decode_correctness() {
let dispatch_args_buf = alloc_buffer::<u32>(&ctx, 3);
// Execute 2-pass decode kernel
- let experts_kernel = MoeExpertsTwoPassDecodeBlock::new(&ctx).expect("MoeExpertsTwoPassDecodeKernel::new");
+ let experts_kernel = MoeExpertsTwoPassDecodeBlock::<Metal>::new(&ctx).expect("MoeExpertsTwoPassDecodeKernel::new");
let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");
const K_TILE: usize = 64;
@@ -556,7 +559,7 @@ fn test_two_pass_decode_multi_token() {
let tile_map_buf = alloc_buffer::<u32>(&ctx, max_total_tiles * 3);
let dispatch_args_buf = alloc_buffer::<u32>(&ctx, 3);
- let experts_kernel = MoeExpertsTwoPassDecodeBlock::new(&ctx).expect("kernel");
+ let experts_kernel = MoeExpertsTwoPassDecodeBlock::<Metal>::new(&ctx).expect("kernel");
let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");
experts_kernel.encode(
&cb,
@@ -658,7 +661,7 @@ fn test_two_pass_prefill_correctness() {
let tile_map_buf = alloc_buffer::<u32>(&ctx, max_total_tiles * 3);
let dispatch_args_buf = alloc_buffer::<u32>(&ctx, 3);
- let experts_kernel = MoeExpertsTwoPassPrefillBlock::new(&ctx).expect("kernel");
+ let experts_kernel = MoeExpertsTwoPassPrefillBlock::<Metal>::new(&ctx).expect("kernel");
let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");
let args = MoeExpertsTwoPassArguments {
x_perm_buffer: &x_perm_buf,
@@ -787,7 +790,7 @@ fn test_fused_single_token_decode() {
let y_buf = alloc_buffer::<bf16>(&ctx, d_model);
// Run fused decode kernel
- let fused_kernel = MoeExpertsSingleDecodeKernels::new(&ctx).expect("MoeExpertsSingleDecodeKernel::new");
+ let fused_kernel = MoeExpertsSingleDecodeKernels::<Metal>::new(&ctx).expect("MoeExpertsSingleDecodeKernel::new");
let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");
fused_kernel.encode(
@@ -911,7 +914,7 @@ fn test_fused_single_token_k4() {
let hidden_buf = alloc_buffer::<f32>(&ctx, k * d_ff);
let y_buf = alloc_buffer::<bf16>(&ctx, d_model);
- let fused_kernel = MoeExpertsSingleDecodeKernels::new(&ctx).expect("fused kernel");
+ let fused_kernel = MoeExpertsSingleDecodeKernels::<Metal>::new(&ctx).expect("fused kernel");
let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");
fused_kernel.encode(
&cb,
diff --git a/crates/uzu/tests/moe/moe_gather_test.rs b/crates/uzu/tests/moe/moe_gather_test.rs
index e9e1bc743..928823bd9 100644
--- a/crates/uzu/tests/moe/moe_gather_test.rs
+++ b/crates/uzu/tests/moe/moe_gather_test.rs
@@ -5,7 +5,10 @@ use metal::{MTLBuffer, MTLCommandBuffer, MTLCommandQueue};
use rand::{RngExt, SeedableRng, rngs::StdRng};
use uzu::{
DataType,
- backends::metal::kernel::moe::{MoeGatherArguments, MoeGatherKernels},
+ backends::{
+ common::kernel::moe::{MoeGatherArguments, MoeGatherKernels},
+ metal::Metal,
+ },
};
use super::test_utils::{alloc_buffer, alloc_buffer_with_data, assert_bf16_close, create_ctx};
@@ -74,7 +77,7 @@ fn test_gather_correctness() {
let sumk_buf = alloc_buffer_with_data(&ctx, &sum_k_u32);
// Execute gather kernel using kernel struct
- let gather = MoeGatherKernels::new(&ctx).expect("MoeGatherKernel::new");
+ let gather = MoeGatherKernels::<Metal>::new(&ctx).expect("MoeGatherKernel::new");
let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");
gather.encode(
&cb,
diff --git a/crates/uzu/tests/moe/moe_perf_test.rs b/crates/uzu/tests/moe/moe_perf_test.rs
index 624b5ddb3..147494163 100644
--- a/crates/uzu/tests/moe/moe_perf_test.rs
+++ b/crates/uzu/tests/moe/moe_perf_test.rs
@@ -9,13 +9,14 @@ use uzu::{
common::kernel::{
MoeBlockBasesFromPartialsKernel, MoeCountsOffsetsFusedKernel, MoeFinalizeKernel, MoeRouterTopKKernel,
MoeScatterBucketsMapKernel,
+ moe::{MoeExpertsTwoPassArguments, MoeExpertsTwoPassDecodeBlock, MoeGatherArguments, MoeGatherKernels},
},
- metal::kernel::{
- dsl::{
+ metal::{
+ Metal,
+ kernel::dsl::{
MoeBlockBasesFromPartialsMetalKernel, MoeCountsOffsetsFusedMetalKernel, MoeFinalizeMetalKernel,
MoeRouterTopKMetalKernel, MoeScatterBucketsMapMetalKernel,
},
- moe::{MoeExpertsTwoPassArguments, MoeExpertsTwoPassDecodeBlock, MoeGatherArguments, MoeGatherKernels},
},
},
};
@@ -299,8 +300,8 @@ fn test_moe_pipeline_breakdown_decode() {
MoeBlockBasesFromPartialsMetalKernel::new(&ctx).expect("MoeBlockBasesFromPartialsMetalKernel");
let scatter_map_kernel =
MoeScatterBucketsMapMetalKernel::new(&ctx, DataType::BF16).expect("MoeScatterBucketsMapMetalKernel");
- let gather_kernel = MoeGatherKernels::new(&ctx).expect("gather");
- let experts_kernel = MoeExpertsTwoPassDecodeBlock::new(&ctx).expect("experts two-pass decode");
+ let gather_kernel = MoeGatherKernels::<Metal>::new(&ctx).expect("gather");
+ let experts_kernel = MoeExpertsTwoPassDecodeBlock::<Metal>::new(&ctx).expect("experts two-pass decode");
let finalize_kernel = MoeFinalizeMetalKernel::new(&ctx, DataType::BF16).expect("finalize");
let router_topk_fused_kernel = MoeRouterTopKMetalKernel::new(&ctx, DataType::BF16).expect("router+topk fused");
diff --git a/crates/uzu/tests/moe/moe_tiles_test.rs b/crates/uzu/tests/moe/moe_tiles_test.rs
index 13afb70a1..874d41329 100644
--- a/crates/uzu/tests/moe/moe_tiles_test.rs
+++ b/crates/uzu/tests/moe/moe_tiles_test.rs
@@ -2,7 +2,10 @@
use metal::{MTLBuffer, MTLCommandBuffer, MTLCommandQueue};
use rand::{RngExt, SeedableRng, rngs::StdRng};
-use uzu::backends::metal::kernel::moe::tiles_map::{MoeTileCountsArguments, MoeTileMapKernels, MoeTileScanArguments};
+use uzu::backends::{
+ common::kernel::moe::{MoeTileCountsArguments, MoeTileMapKernels, MoeTileScanArguments},
+ metal::Metal,
+};
use super::test_utils::{alloc_buffer, alloc_buffer_with_data, cpu_tile_counts, cpu_tile_scan, create_ctx};
@@ -31,7 +34,7 @@ fn test_tile_counts_correctness() {
let tile_counts_buf = alloc_buffer::<u32>(&ctx, e);
// Execute kernel using kernel struct
- let tile_kernel = MoeTileMapKernels::new(&ctx).expect("MoeTileMapKernel::new");
+ let tile_kernel = MoeTileMapKernels::<Metal>::new(&ctx).expect("MoeTileMapKernel::new");
let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");
tile_kernel.encode_counts(
&cb,
@@ -76,7 +79,7 @@ fn test_tile_scan_correctness() {
let total_tiles_buf = alloc_buffer::<u32>(&ctx, 1);
// Execute kernel using kernel struct
- let tile_kernel = MoeTileMapKernels::new(&ctx).expect("MoeTileMapKernel::new");
+ let tile_kernel = MoeTileMapKernels::<Metal>::new(&ctx).expect("MoeTileMapKernel::new");
let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");
tile_kernel.encode_scan(
&cb,
@@ -116,7 +119,7 @@ fn test_tile_edge_cases() {
let offsets_buf = alloc_buffer_with_data(&ctx, &offsets);
let tile_counts_buf = alloc_buffer::<u32>(&ctx, e);
- let tile_kernel = MoeTileMapKernels::new(&ctx).expect("MoeTileMapKernel::new");
+ let tile_kernel = MoeTileMapKernels::<Metal>::new(&ctx).expect("MoeTileMapKernel::new");
let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");
tile_kernel.encode_counts(
&cb,
@@ -147,7 +150,7 @@ fn test_tile_edge_cases() {
let offsets_buf = alloc_buffer_with_data(&ctx, &offsets);
let tile_counts_buf = alloc_buffer::<u32>(&ctx, e);
- let tile_kernel = MoeTileMapKernels::new(&ctx).expect("MoeTileMapKernel::new");
+ let tile_kernel = MoeTileMapKernels::<Metal>::new(&ctx).expect("MoeTileMapKernel::new");
let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");
tile_kernel.encode_counts(
&cb,