Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

Allocations clean up (#403)

LuckyIYI4b1eab1a7b63
diff --git a/crates/backend-uzu/src/backends/common/kernel/mlp_gate_act_mul.rs b/crates/backend-uzu/src/backends/common/kernel/mlp_gate_act_mul.rs
index 8676b3e1e..e22dfaf45 100644
--- a/crates/backend-uzu/src/backends/common/kernel/mlp_gate_act_mul.rs
+++ b/crates/backend-uzu/src/backends/common/kernel/mlp_gate_act_mul.rs
@@ -1,5 +1,6 @@
use crate::{
DataType,
+ array::size_for_shape,
backends::common::{
ActivationConfig, Allocation, Backend, Encoder, Kernels, gpu_types::ActivationType, kernel::MlpGateActMulKernel,
},
@@ -9,6 +10,7 @@ pub struct MlpGateActMulEncodable<B: Backend> {
kernel: <B::Kernels as Kernels>::MlpGateActMulKernel,
activation: ActivationConfig,
hidden_dim: usize,
+ data_type: DataType,
hadamard_factors: Option<Allocation<B>>,
}
@@ -25,6 +27,7 @@ impl<B: Backend> MlpGateActMulEncodable<B> {
kernel,
activation,
hidden_dim,
+ data_type,
hadamard_factors,
})
}
@@ -33,21 +36,21 @@ impl<B: Backend> MlpGateActMulEncodable<B> {
&self,
encoder: &mut Encoder<B>,
fused_up: &Allocation<B>,
- hidden: &mut Allocation<B>,
- m: i32,
- ) -> Result<(), B::Error> {
+ batch_dim: usize,
+ ) -> Result<Allocation<B>, B::Error> {
if self.activation.act_type() == ActivationType::IDENTITY {
panic!("Identity activation is not supported for kernel")
}
+ let mut hidden = encoder.allocate_scratch(size_for_shape(&[batch_dim, self.hidden_dim], self.data_type))?;
self.kernel.encode(
fused_up,
- hidden,
+ &mut hidden,
self.hadamard_factors.as_ref(),
self.hidden_dim as i32,
- m,
+ batch_dim as i32,
self.activation.act_type(),
encoder,
);
- Ok(())
+ Ok(hidden)
}
}
diff --git a/crates/backend-uzu/src/backends/common/kernel/moe/experts_two_pass_decode.rs b/crates/backend-uzu/src/backends/common/kernel/moe/experts_two_pass_decode.rs
index 8fc689c81..8ba310214 100644
--- a/crates/backend-uzu/src/backends/common/kernel/moe/experts_two_pass_decode.rs
+++ b/crates/backend-uzu/src/backends/common/kernel/moe/experts_two_pass_decode.rs
@@ -1,7 +1,4 @@
-use super::{
- MoeExpertsTwoPassArguments, MoePassARowMapArguments, MoePassATileBuildArguments, MoePassATileCountsArguments,
- MoePassATileDispatchArguments, MoePassATileKernels, MoePassATileScanArguments,
-};
+use super::{MoeExpertsTwoPassArguments, MoePassATileKernels};
use crate::{
DataType,
array::size_for_shape,
@@ -49,63 +46,29 @@ impl<B: Backend> MoeExpertsTwoPassDecodeBlock<B> {
encoder: &mut Encoder<B>,
args: MoeExpertsTwoPassArguments<B>,
) -> Result<Allocation<B>, B::Error> {
- let mut hidden = encoder.allocate_scratch(size_for_shape(&[args.total_rows, args.d_ff], DataType::F32))?;
- let mut output = encoder.allocate_scratch(size_for_shape(&[args.total_rows, args.d_model], args.data_type))?;
-
- // pass a tile
const BLOCK_M: u32 = 4;
let h_blocks = (args.d_ff as u32 + BLOCK_M - 1) / BLOCK_M;
- self.pass_a_tile.encode_counts(
- encoder,
- MoePassATileCountsArguments {
- expert_offsets: args.expert_offsets,
- tile_counts: &mut *args.tile_counts,
- e: args.e,
- h_blocks,
- },
- );
- self.pass_a_tile.encode_scan(
- encoder,
- MoePassATileScanArguments {
- tile_counts: &*args.tile_counts,
- tile_offsets: &mut *args.tile_offsets,
- total_tiles: &mut *args.total_tiles,
- e: args.e,
- },
- );
- self.pass_a_tile.encode_row_map(
- encoder,
- MoePassARowMapArguments {
- expert_offsets: args.expert_offsets,
- row_expert_map: &mut *args.row_expert_map,
- total_rows: args.total_rows,
- e: args.e,
- },
- );
- self.pass_a_tile.encode_build_map(
- encoder,
- MoePassATileBuildArguments {
- expert_offsets: args.expert_offsets,
- tile_offsets: &*args.tile_offsets,
- row_expert_map: &*args.row_expert_map,
- tile_map: &mut *args.tile_map,
- total_rows: args.total_rows,
- h_blocks,
- },
- );
- self.pass_a_tile.encode_dispatch_args(
+ let tile_counts = self.pass_a_tile.encode_counts(encoder, args.expert_offsets, args.e, h_blocks)?;
+
+ let tile_scan = self.pass_a_tile.encode_scan(encoder, &tile_counts, args.e)?;
+
+ let row_expert_map = self.pass_a_tile.encode_row_map(encoder, args.expert_offsets, args.total_rows, args.e)?;
+
+ let tile_map = self.pass_a_tile.encode_build_map(
encoder,
- MoePassATileDispatchArguments {
- total_tiles: &*args.total_tiles,
- dispatch_args: &mut *args.dispatch_args,
- num_tiles_y: 1,
- },
- );
+ args.expert_offsets,
+ &tile_scan.tile_offsets,
+ &row_expert_map,
+ args.total_rows,
+ h_blocks,
+ )?;
+
+ let dispatch_args = self.pass_a_tile.encode_dispatch_args(encoder, &tile_scan.total_tiles, 1)?;
let gate_idx = args.gating_code.min(3) as usize;
let dtype_idx = DTYPES.iter().position(|dtype| *dtype == args.data_type).unwrap();
+ let mut hidden = encoder.allocate_scratch(size_for_shape(&[args.total_rows, args.d_ff], DataType::F32))?;
- // pass a
let pass_a_kernel = &self.pass_a_indirect[gate_idx][dtype_idx];
pass_a_kernel.encode(
args.x_perm,
@@ -121,16 +84,16 @@ impl<B: Backend> MoeExpertsTwoPassDecodeBlock<B> {
args.up_clip_min,
args.up_clip_max,
args.silu_alpha,
- &*args.tile_map,
- &*args.dispatch_args,
+ &tile_map,
+ &dispatch_args,
encoder,
);
- // pass b
+ let mut output = encoder.allocate_scratch(size_for_shape(&[args.total_rows, args.d_model], args.data_type))?;
let pass_b_kernel = &self.fused_down[dtype_idx];
pass_b_kernel.encode(
&hidden,
- &*args.row_expert_map,
+ &row_expert_map,
args.w2_all,
args.down_biases,
&mut output,
diff --git a/crates/backend-uzu/src/backends/common/kernel/moe/experts_two_pass_prefill.rs b/crates/backend-uzu/src/backends/common/kernel/moe/experts_two_pass_prefill.rs
index c4f65ef6e..d4d4e745a 100644
--- a/crates/backend-uzu/src/backends/common/kernel/moe/experts_two_pass_prefill.rs
+++ b/crates/backend-uzu/src/backends/common/kernel/moe/experts_two_pass_prefill.rs
@@ -1,6 +1,4 @@
-use super::{
- MoeTileCountsArguments, MoeTileDispatchArguments, MoeTileMapBuildArguments, MoeTileMapKernels, MoeTileScanArguments,
-};
+use super::MoeTileMapKernels;
use crate::{
DataType,
array::size_for_shape,
@@ -50,37 +48,18 @@ impl<B: Backend> MoeExpertsTwoPassPrefillBlock<B> {
encoder: &mut Encoder<B>,
args: MoeExpertsTwoPassArguments<B>,
) -> Result<Allocation<B>, B::Error> {
- let mut hidden = encoder.allocate_scratch(size_for_shape(&[args.total_rows, args.d_ff], DataType::F32))?;
- let mut output = encoder.allocate_scratch(size_for_shape(&[args.total_rows, args.d_model], args.data_type))?;
- encoder.encode_fill(&mut hidden, 0);
+ let tile_counts = self.tile_map.encode_counts(encoder, args.expert_offsets, args.e)?;
- self.tile_map.encode_counts(
- encoder,
- MoeTileCountsArguments {
- offsets: args.expert_offsets,
- tile_counts: &mut *args.tile_counts,
- e: args.e,
- },
- );
- self.tile_map.encode_scan(
- encoder,
- MoeTileScanArguments {
- tile_counts: &*args.tile_counts,
- tile_offsets: &mut *args.tile_offsets,
- total_tiles: &mut *args.total_tiles,
- e: args.e,
- },
- );
- self.tile_map.encode_build_map(
+ let tile_scan = self.tile_map.encode_scan(encoder, &tile_counts, args.e)?;
+
+ let tile_map = self.tile_map.encode_build_map(
encoder,
- MoeTileMapBuildArguments {
- expert_offsets: args.expert_offsets,
- tile_offsets: &*args.tile_offsets,
- tile_counts: &*args.tile_counts,
- tile_map: &mut *args.tile_map,
- e: args.e,
- },
- );
+ args.expert_offsets,
+ &tile_scan.tile_offsets,
+ &tile_counts,
+ args.total_rows,
+ args.e,
+ )?;
let d_model_u32 = args.d_model as u32;
let d_ff_u32 = args.d_ff as u32;
const COL_TILE_FF: u32 = 32; // Must match PASSA_BN in kernel
@@ -95,17 +74,12 @@ impl<B: Backend> MoeExpertsTwoPassPrefillBlock<B> {
} else {
(d_model_u32 + COL_TILE_MODEL - 1) / COL_TILE_MODEL
};
- self.tile_map.encode_dispatch_args(
- encoder,
- MoeTileDispatchArguments {
- total_tiles: &*args.total_tiles,
- dispatch_args: &mut *args.dispatch_args,
- num_tiles_x: n_tiles_ff,
- },
- );
+ let pass_a_dispatch_args = self.tile_map.encode_dispatch_args(encoder, &tile_scan.total_tiles, n_tiles_ff)?;
let gate_idx = args.gating_code.min(3) as usize;
let dtype_idx = DTYPES.iter().position(|t| *t == args.data_type).unwrap();
+ let mut hidden = encoder.allocate_scratch(size_for_shape(&[args.total_rows, args.d_ff], DataType::F32))?;
+ encoder.encode_fill(&mut hidden, 0);
let kernel_pass_a = &self.pass_a_indirect[gate_idx][dtype_idx];
kernel_pass_a.encode(
@@ -122,18 +96,15 @@ impl<B: Backend> MoeExpertsTwoPassPrefillBlock<B> {
args.up_clip_min,
args.up_clip_max,
args.silu_alpha,
- &*args.tile_map,
- &*args.dispatch_args,
+ &tile_map,
+ &pass_a_dispatch_args,
encoder,
);
- let dispatch_args = MoeTileDispatchArguments {
- total_tiles: &*args.total_tiles,
- dispatch_args: &mut *args.dispatch_args,
- num_tiles_x: n_tiles_model,
- };
- self.tile_map.encode_dispatch_args(encoder, dispatch_args);
+ let pass_b_dispatch_args =
+ self.tile_map.encode_dispatch_args(encoder, &tile_scan.total_tiles, n_tiles_model)?;
+ let mut output = encoder.allocate_scratch(size_for_shape(&[args.total_rows, args.d_model], args.data_type))?;
let kernel_pass_b = &self.pass_b_indirect[dtype_idx];
kernel_pass_b.encode(
&hidden,
@@ -144,8 +115,8 @@ impl<B: Backend> MoeExpertsTwoPassPrefillBlock<B> {
args.d_model as u32,
args.d_ff as u32,
args.e as u32,
- &*args.tile_map,
- &*args.dispatch_args,
+ &tile_map,
+ &pass_b_dispatch_args,
encoder,
);
Ok(output)
@@ -155,21 +126,14 @@ impl<B: Backend> MoeExpertsTwoPassPrefillBlock<B> {
pub struct MoeExpertsTwoPassArguments<'a, B: Backend> {
pub x_perm: &'a Allocation<B>,
pub expert_offsets: &'a Allocation<B>,
- pub row_expert_map: &'a mut Allocation<B>,
pub w13_all: &'a Allocation<B>,
pub w2_all: &'a Allocation<B>,
pub up_biases: &'a Allocation<B>,
pub down_biases: &'a Allocation<B>,
- pub tile_counts: &'a mut Allocation<B>,
- pub tile_offsets: &'a mut Allocation<B>,
- pub tile_map: &'a mut Allocation<B>,
- pub total_tiles: &'a mut Allocation<B>,
- pub dispatch_args: &'a mut Allocation<B>,
pub total_rows: usize,
pub d_model: usize,
pub d_ff: usize,
pub e: usize,
- pub num_tiles_k: u32,
pub gating_code: u32,
pub gate_clip_min: f32,
pub gate_clip_max: f32,
diff --git a/crates/backend-uzu/src/backends/common/kernel/moe/gather.rs b/crates/backend-uzu/src/backends/common/kernel/moe/gather.rs
index 9c6040b37..ad740ad4d 100644
--- a/crates/backend-uzu/src/backends/common/kernel/moe/gather.rs
+++ b/crates/backend-uzu/src/backends/common/kernel/moe/gather.rs
@@ -1,5 +1,6 @@
use crate::{
DataType,
+ array::size_for_shape,
backends::common::{
Allocation, Backend, Encoder, Kernels,
kernel::{MoeGatherXPerm1DKernel, MoeGatherXPerm2DKernel},
@@ -9,7 +10,6 @@ use crate::{
pub struct MoeGatherArguments<'a, B: Backend> {
pub x: &'a Allocation<B>,
pub bucketed_ids: &'a Allocation<B>,
- pub x_perm: &'a mut Allocation<B>,
pub sumk: &'a Allocation<B>,
pub t: usize,
pub k: usize,
@@ -36,12 +36,15 @@ impl<B: Backend> MoeGatherKernels<B> {
encoder: &mut Encoder<B>,
dtype: DataType,
args: MoeGatherArguments<B>,
- ) {
+ ) -> Result<Allocation<B>, B::Error> {
+ let mut x_perm = encoder.allocate_scratch(size_for_shape(&[args.t * args.k, args.d_model], dtype))?;
+ encoder.encode_fill(&mut x_perm, 0);
+
match dtype {
DataType::F32 => self.f32.encode(
args.x,
args.bucketed_ids,
- args.x_perm,
+ &mut x_perm,
args.sumk,
args.d_model as u32,
args.t as u32,
@@ -51,7 +54,7 @@ impl<B: Backend> MoeGatherKernels<B> {
DataType::F16 => self.f16.encode(
args.x,
args.bucketed_ids,
- args.x_perm,
+ &mut x_perm,
args.sumk,
args.d_model as u32,
args.t as u32,
@@ -61,7 +64,7 @@ impl<B: Backend> MoeGatherKernels<B> {
DataType::BF16 => self.bf16.encode(
args.x,
args.bucketed_ids,
- args.x_perm,
+ &mut x_perm,
args.sumk,
args.d_model as u32,
args.t as u32,
@@ -70,5 +73,6 @@ impl<B: Backend> MoeGatherKernels<B> {
),
_ => panic!("Unsupported data type: {:?}", dtype),
};
+ Ok(x_perm)
}
}
diff --git a/crates/backend-uzu/src/backends/common/kernel/moe/mod.rs b/crates/backend-uzu/src/backends/common/kernel/moe/mod.rs
index 1ac0db46a..a7dc41735 100644
--- a/crates/backend-uzu/src/backends/common/kernel/moe/mod.rs
+++ b/crates/backend-uzu/src/backends/common/kernel/moe/mod.rs
@@ -9,10 +9,5 @@ pub use experts_single::{MoeExpertsSingleDecodeArguments, MoeExpertsSingleDecode
pub use experts_two_pass_decode::MoeExpertsTwoPassDecodeBlock;
pub use experts_two_pass_prefill::{MoeExpertsTwoPassArguments, MoeExpertsTwoPassPrefillBlock};
pub use gather::{MoeGatherArguments, MoeGatherKernels};
-pub use tiles_map::{
- MoeTileCountsArguments, MoeTileDispatchArguments, MoeTileMapBuildArguments, MoeTileMapKernels, MoeTileScanArguments,
-};
-pub use tiles_pass_a::{
- MoePassARowMapArguments, MoePassATileBuildArguments, MoePassATileCountsArguments, MoePassATileDispatchArguments,
- MoePassATileKernels, MoePassATileScanArguments,
-};
+pub use tiles_map::{MoeTileMapKernels, MoeTileScanOutput};
+pub use tiles_pass_a::{MoePassATileKernels, MoePassATileScanOutput};
diff --git a/crates/backend-uzu/src/backends/common/kernel/moe/tiles_map.rs b/crates/backend-uzu/src/backends/common/kernel/moe/tiles_map.rs
index 6bff799cc..b549a6801 100644
--- a/crates/backend-uzu/src/backends/common/kernel/moe/tiles_map.rs
+++ b/crates/backend-uzu/src/backends/common/kernel/moe/tiles_map.rs
@@ -1,6 +1,10 @@
-use crate::backends::common::{
- Allocation, Backend, Encoder, Kernels,
- kernel::{MoeBuildTileMapKernel, MoeTileCountsKernel, MoeTileScanKernel, MoeWriteDispatchArgsKernel},
+use crate::{
+ DataType,
+ array::size_for_shape,
+ backends::common::{
+ Allocation, Backend, Encoder, Kernels,
+ kernel::{MoeBuildTileMapKernel, MoeTileCountsKernel, MoeTileScanKernel, MoeWriteDispatchArgsKernel},
+ },
};
pub struct MoeTileMapKernels<B: Backend> {
@@ -23,66 +27,56 @@ impl<B: Backend> MoeTileMapKernels<B> {
pub fn encode_counts(
&self,
encoder: &mut Encoder<B>,
- args: MoeTileCountsArguments<B>,
- ) {
- self.counts.encode(args.offsets, args.tile_counts, args.e as u32, encoder);
+ offsets: &Allocation<B>,
+ e: usize,
+ ) -> Result<Allocation<B>, B::Error> {
+ let mut tile_counts = encoder.allocate_scratch(size_for_shape(&[e], DataType::U32))?;
+ self.counts.encode(offsets, &mut tile_counts, e as u32, encoder);
+ Ok(tile_counts)
}
pub fn encode_scan(
&self,
encoder: &mut Encoder<B>,
- args: MoeTileScanArguments<B>,
- ) {
- self.scan.encode(args.tile_counts, args.tile_offsets, args.total_tiles, args.e as u32, encoder);
+ tile_counts: &Allocation<B>,
+ e: usize,
+ ) -> Result<MoeTileScanOutput<B>, B::Error> {
+ let mut tile_offsets = encoder.allocate_scratch(size_for_shape(&[e + 1], DataType::U32))?;
+ let mut total_tiles = encoder.allocate_scratch(size_for_shape(&[8], DataType::U32))?;
+ self.scan.encode(tile_counts, &mut tile_offsets, &mut total_tiles, e as u32, encoder);
+ Ok(MoeTileScanOutput {
+ tile_offsets,
+ total_tiles,
+ })
}
pub fn encode_build_map(
&self,
encoder: &mut Encoder<B>,
- args: MoeTileMapBuildArguments<B>,
- ) {
- self.build.encode(
- args.expert_offsets,
- args.tile_offsets,
- args.tile_counts,
- args.tile_map,
- args.e as u32,
- encoder,
- );
+ expert_offsets: &Allocation<B>,
+ tile_offsets: &Allocation<B>,
+ tile_counts: &Allocation<B>,
+ total_rows: usize,
+ e: usize,
+ ) -> Result<Allocation<B>, B::Error> {
+ let mut tile_map = encoder.allocate_scratch(size_for_shape(&[total_rows * 3], DataType::U32))?;
+ self.build.encode(expert_offsets, tile_offsets, tile_counts, &mut tile_map, e as u32, encoder);
+ Ok(tile_map)
}
pub fn encode_dispatch_args(
&self,
encoder: &mut Encoder<B>,
- args: MoeTileDispatchArguments<B>,
- ) {
- self.dispatch.encode(args.total_tiles, args.dispatch_args, args.num_tiles_x, encoder);
+ total_tiles: &Allocation<B>,
+ num_tiles_x: u32,
+ ) -> Result<Allocation<B>, B::Error> {
+ let mut dispatch_args = encoder.allocate_scratch(size_for_shape(&[3], DataType::U32))?;
+ self.dispatch.encode(total_tiles, &mut dispatch_args, num_tiles_x, encoder);
+ Ok(dispatch_args)
}
}
-pub struct MoeTileCountsArguments<'a, B: Backend> {
- pub offsets: &'a Allocation<B>, // [E+1]
- pub tile_counts: &'a mut Allocation<B>, // [E]
- pub e: usize,
-}
-
-pub struct MoeTileScanArguments<'a, B: Backend> {
- pub tile_counts: &'a Allocation<B>, // [E]
- pub tile_offsets: &'a mut Allocation<B>, // [E+1]
- pub total_tiles: &'a mut Allocation<B>, // [>=2]
- pub e: usize,
-}
-
-pub struct MoeTileMapBuildArguments<'a, B: Backend> {
- pub expert_offsets: &'a Allocation<B>, // [E+1]
- pub tile_offsets: &'a Allocation<B>, // [E+1]
- pub tile_counts: &'a Allocation<B>, // [E]
- pub tile_map: &'a mut Allocation<B>, // [total_tiles * 3]
- pub e: usize,
-}
-
-pub struct MoeTileDispatchArguments<'a, B: Backend> {
- pub total_tiles: &'a Allocation<B>, // [>=1]
- pub dispatch_args: &'a mut Allocation<B>, // [3]
- pub num_tiles_x: u32, // x dimension for indirect dispatch
+pub struct MoeTileScanOutput<B: Backend> {
+ pub tile_offsets: Allocation<B>,
+ pub total_tiles: Allocation<B>,
}
diff --git a/crates/backend-uzu/src/backends/common/kernel/moe/tiles_pass_a.rs b/crates/backend-uzu/src/backends/common/kernel/moe/tiles_pass_a.rs
index cc3f8674e..ad8fc9b47 100644
--- a/crates/backend-uzu/src/backends/common/kernel/moe/tiles_pass_a.rs
+++ b/crates/backend-uzu/src/backends/common/kernel/moe/tiles_pass_a.rs
@@ -1,45 +1,18 @@
-use crate::backends::common::{
- Allocation, Backend, Encoder,
- kernel::{
- Kernels, MoePassABuildRowMapKernel, MoePassABuildTileMapKernel, MoePassATileCountsKernel,
- MoePassATileScanKernel, MoePassAWriteDispatchArgsKernel,
+use crate::{
+ DataType,
+ array::size_for_shape,
+ backends::common::{
+ Allocation, Backend, Encoder,
+ kernel::{
+ Kernels, MoePassABuildRowMapKernel, MoePassABuildTileMapKernel, MoePassATileCountsKernel,
+ MoePassATileScanKernel, MoePassAWriteDispatchArgsKernel,
+ },
},
};
-pub struct MoePassATileCountsArguments<'a, B: Backend> {
- pub expert_offsets: &'a Allocation<B>, // [E+1]
- pub tile_counts: &'a mut Allocation<B>, // [E]
- pub e: usize,
- pub h_blocks: u32,
-}
-
-pub struct MoePassATileScanArguments<'a, B: Backend> {
- pub tile_counts: &'a Allocation<B>, // [E]
- pub tile_offsets: &'a mut Allocation<B>, // [E+1]
- pub total_tiles: &'a mut Allocation<B>, // [>=1]
- pub e: usize,
-}
-
-pub struct MoePassARowMapArguments<'a, B: Backend> {
- pub expert_offsets: &'a Allocation<B>, // [E+1]
- pub row_expert_map: &'a mut Allocation<B>, // [total_rows]
- pub total_rows: usize,
- pub e: usize,
-}
-
-pub struct MoePassATileBuildArguments<'a, B: Backend> {
- pub expert_offsets: &'a Allocation<B>, // [E+1]
- pub tile_offsets: &'a Allocation<B>, // [E+1]
- pub row_expert_map: &'a Allocation<B>, // [total_rows]
- pub tile_map: &'a mut Allocation<B>, // [total_tiles * 3]
- pub total_rows: usize,
- pub h_blocks: u32,
-}
-
-pub struct MoePassATileDispatchArguments<'a, B: Backend> {
- pub total_tiles: &'a Allocation<B>, // [>=1]
- pub dispatch_args: &'a mut Allocation<B>, // [3]
- pub num_tiles_y: u32,
+pub struct MoePassATileScanOutput<B: Backend> {
+ pub tile_offsets: Allocation<B>,
+ pub total_tiles: Allocation<B>,
}
pub struct MoePassATileKernels<B: Backend> {
@@ -64,48 +37,73 @@ impl<B: Backend> MoePassATileKernels<B> {
pub fn encode_counts(
&self,
encoder: &mut Encoder<B>,
- args: MoePassATileCountsArguments<B>,
- ) {
- self.counts.encode(args.expert_offsets, args.tile_counts, args.e as u32, args.h_blocks, encoder);
+ expert_offsets: &Allocation<B>,
+ e: usize,
+ h_blocks: u32,
+ ) -> Result<Allocation<B>, B::Error> {
+ let mut tile_counts = encoder.allocate_scratch(size_for_shape(&[e], DataType::U32))?;
+ self.counts.encode(expert_offsets, &mut tile_counts, e as u32, h_blocks, encoder);
+ Ok(tile_counts)
}
pub fn encode_scan(
&self,
encoder: &mut Encoder<B>,
- args: MoePassATileScanArguments<B>,
- ) {
- self.scan.encode(args.tile_counts, args.tile_offsets, args.total_tiles, args.e as u32, encoder);
+ tile_counts: &Allocation<B>,
+ e: usize,
+ ) -> Result<MoePassATileScanOutput<B>, B::Error> {
+ let mut tile_offsets = encoder.allocate_scratch(size_for_shape(&[e + 1], DataType::U32))?;
+ let mut total_tiles = encoder.allocate_scratch(size_for_shape(&[1], DataType::U32))?;
+ self.scan.encode(tile_counts, &mut tile_offsets, &mut total_tiles, e as u32, encoder);
+ Ok(MoePassATileScanOutput {
+ tile_offsets,
+ total_tiles,
+ })
}
pub fn encode_row_map(
&self,
encoder: &mut Encoder<B>,
- args: MoePassARowMapArguments<B>,
- ) {
- self.row_map.encode(args.expert_offsets, args.row_expert_map, args.total_rows as u32, args.e as u32, encoder);
+ expert_offsets: &Allocation<B>,
+ total_rows: usize,
+ e: usize,
+ ) -> Result<Allocation<B>, B::Error> {
+ let mut row_expert_map = encoder.allocate_scratch(size_for_shape(&[total_rows], DataType::U32))?;
+ self.row_map.encode(expert_offsets, &mut row_expert_map, total_rows as u32, e as u32, encoder);
+ Ok(row_expert_map)
}
pub fn encode_build_map(
&self,
encoder: &mut Encoder<B>,
- args: MoePassATileBuildArguments<B>,
- ) {
+ expert_offsets: &Allocation<B>,
+ tile_offsets: &Allocation<B>,
+ row_expert_map: &Allocation<B>,
+ total_rows: usize,
+ h_blocks: u32,
+ ) -> Result<Allocation<B>, B::Error> {
+ let mut tile_map =
+ encoder.allocate_scratch(size_for_shape(&[total_rows * h_blocks as usize * 3], DataType::U32))?;
self.build_map.encode(
- args.expert_offsets,
- args.tile_offsets,
- args.row_expert_map,
- args.tile_map,
- args.total_rows as u32,
- args.h_blocks,
+ expert_offsets,
+ tile_offsets,
+ row_expert_map,
+ &mut tile_map,
+ total_rows as u32,
+ h_blocks,
encoder,
);
+ Ok(tile_map)
}
pub fn encode_dispatch_args(
&self,
encoder: &mut Encoder<B>,
- args: MoePassATileDispatchArguments<B>,
- ) {
- self.dispatch.encode(args.total_tiles, args.dispatch_args, args.num_tiles_y, encoder);
+ total_tiles: &Allocation<B>,
+ num_tiles_y: u32,
+ ) -> Result<Allocation<B>, B::Error> {
+ let mut dispatch_args = encoder.allocate_scratch(size_for_shape(&[3], DataType::U32))?;
+ self.dispatch.encode(total_tiles, &mut dispatch_args, num_tiles_y, encoder);
+ Ok(dispatch_args)
}
}
diff --git a/crates/backend-uzu/src/backends/common/kernel/ssd_prefill.rs b/crates/backend-uzu/src/backends/common/kernel/ssd_prefill.rs
index b3769ec4a..127e275a9 100644
--- a/crates/backend-uzu/src/backends/common/kernel/ssd_prefill.rs
+++ b/crates/backend-uzu/src/backends/common/kernel/ssd_prefill.rs
@@ -1,5 +1,6 @@
use crate::{
DataType,
+ array::size_for_shape,
backends::common::{
Allocation, Backend, Encoder, Kernels,
kernel::{SSDPrefill64Kernel, SSDPrefillKernel, SSDPrefillSequentialKernel},
@@ -20,7 +21,6 @@ pub struct SSDPrefillArguments<'a, B: Backend> {
pub d: &'a Allocation<B>,
pub z: &'a Allocation<B>,
pub state: &'a mut Allocation<B>,
- pub y: &'a mut Allocation<B>,
pub suffix_len: usize,
pub group_size: u32,
pub state_size: u32,
@@ -36,6 +36,7 @@ pub struct SSDPrefillKernels<B: Backend> {
single: <B::Kernels as Kernels>::SSDPrefillKernel,
single_64: <B::Kernels as Kernels>::SSDPrefill64Kernel,
sequential: <B::Kernels as Kernels>::SSDPrefillSequentialKernel,
+ data_type: DataType,
}
impl<B: Backend> SSDPrefillKernels<B> {
@@ -50,6 +51,7 @@ impl<B: Backend> SSDPrefillKernels<B> {
single,
single_64,
sequential,
+ data_type,
})
}
@@ -58,11 +60,13 @@ impl<B: Backend> SSDPrefillKernels<B> {
encoder: &mut Encoder<B>,
args: SSDPrefillArguments<B>,
mode: SSDPrefillMode,
- ) {
+ ) -> Result<Allocation<B>, B::Error> {
let x_strides: [u32; 3] = args.x_strides.map(|x| x as u32);
let dt_strides: [u32; 2] = args.dt_strides.map(|x| x as u32);
let cb_strides: [u32; 3] = args.cb_strides.map(|x| x as u32);
let state_strides: [u32; 3] = args.state_strides.map(|x| x as u32);
+ let mut output = encoder
+ .allocate_scratch(size_for_shape(&[args.suffix_len, args.channels * args.head_dim], self.data_type))?;
if mode == SSDPrefillMode::SinglePass {
if args.state_size == 64 {
@@ -74,7 +78,7 @@ impl<B: Backend> SSDPrefillKernels<B> {
args.d,
args.z,
args.state,
- args.y,
+ &mut output,
args.suffix_len as u32,
args.group_size,
args.state_size,
@@ -95,7 +99,7 @@ impl<B: Backend> SSDPrefillKernels<B> {
args.d,
args.z,
args.state,
- args.y,
+ &mut output,
args.suffix_len as u32,
args.group_size,
args.state_size,
@@ -117,7 +121,7 @@ impl<B: Backend> SSDPrefillKernels<B> {
args.d,
args.z,
args.state,
- args.y,
+ &mut output,
args.suffix_len as u32,
args.group_size,
args.state_size,
@@ -130,6 +134,7 @@ impl<B: Backend> SSDPrefillKernels<B> {
encoder,
)
}
+ Ok(output)
}
}
diff --git a/crates/backend-uzu/src/encodable_block/delta_net_mixer.rs b/crates/backend-uzu/src/encodable_block/delta_net_mixer.rs
index 08beb2ac9..004560627 100644
--- a/crates/backend-uzu/src/encodable_block/delta_net_mixer.rs
+++ b/crates/backend-uzu/src/encodable_block/delta_net_mixer.rs
@@ -264,8 +264,6 @@ impl<B: Backend> DeltaNetMixer<B> {
encoder.allocate_scratch(size_for_shape(&[suffix_length * self.config.num_heads], DataType::F32))?;
let mut prep_decay =
encoder.allocate_scratch(size_for_shape(&[suffix_length * self.config.num_heads], DataType::F32))?;
- let mut out =
- encoder.allocate_scratch(size_for_shape(&[suffix_length, self.config.value_dim()], self.data_type))?;
self.prefill_prep.encode(
in_proj,
@@ -283,6 +281,8 @@ impl<B: Backend> DeltaNetMixer<B> {
encoder,
);
+ let mut out =
+ encoder.allocate_scratch(size_for_shape(&[suffix_length, self.config.value_dim()], self.data_type))?;
self.delta_net_prefill.encode(
&prep_q_norm,
&prep_k_norm,
diff --git a/crates/backend-uzu/src/encodable_block/mamba_mixer.rs b/crates/backend-uzu/src/encodable_block/mamba_mixer.rs
index fb3d2791c..a6af9f11e 100644
--- a/crates/backend-uzu/src/encodable_block/mamba_mixer.rs
+++ b/crates/backend-uzu/src/encodable_block/mamba_mixer.rs
@@ -296,8 +296,6 @@ impl<B: Backend> MambaMixer<B> {
encoder: &mut Encoder<B>,
suffix_length: usize,
) -> Result<Allocation<B>, B::Error> {
- let mut out =
- encoder.allocate_scratch(size_for_shape(&[suffix_length, self.config.inner_dim()], self.data_type))?;
self.ssd_prefill.encode(
encoder,
SSDPrefillArguments {
@@ -308,7 +306,6 @@ impl<B: Backend> MambaMixer<B> {
d: &self.skip_connection_weight,
z: gate,
state: &mut layer.ssm_state,
- y: &mut out,
suffix_len: suffix_length,
group_size: (self.config.num_heads / self.config.num_groups) as u32,
state_size: self.config.state_dim as u32,
@@ -320,8 +317,7 @@ impl<B: Backend> MambaMixer<B> {
head_dim: self.config.head_dim,
},
self.prefill_mode,
- );
- Ok(out)
+ )
}
fn run_decode_ssm(
diff --git a/crates/backend-uzu/src/encodable_block/mlp/dense.rs b/crates/backend-uzu/src/encodable_block/mlp/dense.rs
index 00449dea4..dd34e49cf 100644
--- a/crates/backend-uzu/src/encodable_block/mlp/dense.rs
+++ b/crates/backend-uzu/src/encodable_block/mlp/dense.rs
@@ -1,18 +1,12 @@
//! MLP block encodable.
use super::{super::linear::Linear, Mlp};
-use crate::{
- DataType,
- array::size_for_shape,
- backends::common::{Allocation, Backend, Encoder, kernel::mlp_gate_act_mul::MlpGateActMulEncodable},
-};
+use crate::backends::common::{Allocation, Backend, Encoder, kernel::mlp_gate_act_mul::MlpGateActMulEncodable};
pub struct DenseMlp<B: Backend> {
up: Box<dyn Linear<B>>,
gate: MlpGateActMulEncodable<B>,
down: Box<dyn Linear<B>>,
- hidden_dim: usize,
- data_type: DataType,
}
impl<B: Backend> DenseMlp<B> {
@@ -20,15 +14,11 @@ impl<B: Backend> DenseMlp<B> {
up: Box<dyn Linear<B>>,
gate: MlpGateActMulEncodable<B>,
down: Box<dyn Linear<B>>,
- hidden_dim: usize,
- data_type: DataType,
) -> Self {
Self {
up,
gate,
down,
- hidden_dim,
- data_type,
}
}
}
@@ -41,8 +31,7 @@ impl<B: Backend> Mlp<B> for DenseMlp<B> {
encoder: &mut Encoder<B>,
) -> Result<Allocation<B>, B::Error> {
let fused_up = self.up.encode(input, batch_dim, encoder)?;
- let mut hidden = encoder.allocate_scratch(size_for_shape(&[batch_dim, self.hidden_dim], self.data_type))?;
- self.gate.encode(encoder, &fused_up, &mut hidden, batch_dim as i32)?;
+ let hidden = self.gate.encode(encoder, &fused_up, batch_dim)?;
self.down.encode(hidden, batch_dim, encoder)
}
}
diff --git a/crates/backend-uzu/src/encodable_block/mlp/mod.rs b/crates/backend-uzu/src/encodable_block/mlp/mod.rs
index cc86e0b21..464f58a20 100644
--- a/crates/backend-uzu/src/encodable_block/mlp/mod.rs
+++ b/crates/backend-uzu/src/encodable_block/mlp/mod.rs
@@ -71,10 +71,7 @@ impl<B: Backend> dyn Mlp<B> {
)
.map_err(MlpBlockError::BackendError)?;
- Ok((
- Box::new(DenseMlp::new(up_projection, gate, down_projection, hidden_dimension, data_type)),
- up_input_hadamard_factors,
- ))
+ Ok((Box::new(DenseMlp::new(up_projection, gate, down_projection)), up_input_hadamard_factors))
},
MLPConfig::MixtureOfExperts(mixture_of_experts_config) => Ok((
Box::new(MoeBlock::new(context, mixture_of_experts_config, model_dimension, parameter_tree)?),
diff --git a/crates/backend-uzu/src/encodable_block/mlp/moe.rs b/crates/backend-uzu/src/encodable_block/mlp/moe.rs
index f206c9910..a12621c8e 100644
--- a/crates/backend-uzu/src/encodable_block/mlp/moe.rs
+++ b/crates/backend-uzu/src/encodable_block/mlp/moe.rs
@@ -173,7 +173,6 @@ impl<B: Backend> Mlp<B> for MoeBlock<B> {
let e = self.moe_config.num_routed_experts;
let k = self.moe_config.num_active_routed_experts;
- let k_tile = 128;
let total_rows = suffix_length * k;
let num_blocks = suffix_length.div_ceil(256).max(1);
let num_tiles = e.div_ceil(512).max(1);
@@ -264,24 +263,18 @@ impl<B: Backend> Mlp<B> for MoeBlock<B> {
encoder,
);
- let mut x_perm = encoder.allocate_scratch(size_for_shape(&[total_rows, self.model_dim], self.data_type))?;
- if suffix_length > 0 && k > 0 {
- encoder.encode_fill(&mut x_perm, 0);
- }
-
- self.gather_kernels.encode(
+ let x_perm = self.gather_kernels.encode(
encoder,
self.data_type,
MoeGatherArguments {
x: &input,
bucketed_ids: &bucketed_ids,
- x_perm: &mut x_perm,
sumk: &sumk,
t: suffix_length,
k,
d_model: self.model_dim,
},
- );
+ )?;
let gating_code = Self::gating_code_from_activation(&self.moe_config.expert_config.activation.act_type());
@@ -293,38 +286,20 @@ impl<B: Backend> Mlp<B> for MoeBlock<B> {
let up_clip_max = up_hi.unwrap_or(f32::INFINITY);
let silu_alpha = self.moe_config.expert_config.activation.alpha();
- let mut row_expert_map = encoder.allocate_scratch(size_for_shape(&[total_rows], DataType::U32))?;
- let mut tile_counts = encoder.allocate_scratch(size_for_shape(&[e], DataType::U32))?;
- let mut tile_offsets = encoder.allocate_scratch(size_for_shape(&[e + 1], DataType::U32))?;
- let h_blocks = self.hidden_dim.div_ceil(4);
- let mut tile_map =
- encoder.allocate_scratch(size_for_shape(&[total_rows * h_blocks.max(1) * 3], DataType::U32))?;
- let mut total_tiles = encoder.allocate_scratch(size_for_shape(&[8], DataType::U32))?;
- let mut dispatch_args = encoder.allocate_scratch(size_for_shape(&[3], DataType::U32))?;
-
let y_partial = if suffix_length == 1 {
- let num_tiles_k = ((self.hidden_dim + k_tile - 1) / k_tile) as u32;
-
self.experts_two_pass_decode_kernel.encode(
encoder,
MoeExpertsTwoPassArguments {
x_perm: &x_perm,
expert_offsets: &offsets,
- row_expert_map: &mut row_expert_map,
w13_all: &self.shared_weights.w13,
w2_all: &self.shared_weights.w2,
up_biases: &self.shared_weights.up_biases,
down_biases: &self.shared_weights.down_biases,
- tile_counts: &mut tile_counts,
- tile_offsets: &mut tile_offsets,
- tile_map: &mut tile_map,
- total_tiles: &mut total_tiles,
- dispatch_args: &mut dispatch_args,
total_rows,
d_model: self.model_dim,
d_ff: self.hidden_dim,
e,
- num_tiles_k,
gating_code,
gate_clip_min,
gate_clip_max,
@@ -335,25 +310,17 @@ impl<B: Backend> Mlp<B> for MoeBlock<B> {
},
)?
} else {
- let num_tiles_k = ((self.hidden_dim + k_tile - 1) / k_tile) as u32;
let args = MoeExpertsTwoPassArguments {
x_perm: &x_perm,
expert_offsets: &offsets,
- row_expert_map: &mut row_expert_map,
w13_all: &self.shared_weights.w13,
w2_all: &self.shared_weights.w2,
up_biases: &self.shared_weights.up_biases,
down_biases: &self.shared_weights.down_biases,
- tile_counts: &mut tile_counts,
- tile_offsets: &mut tile_offsets,
- tile_map: &mut tile_map,
- total_tiles: &mut total_tiles,
- dispatch_args: &mut dispatch_args,
total_rows,
d_model: self.model_dim,
d_ff: self.hidden_dim,
e,
- num_tiles_k,
gating_code,
gate_clip_min,
gate_clip_max,
diff --git a/crates/backend-uzu/tests/performance/encodable_block/moe/moe_experts_perf_test.rs b/crates/backend-uzu/tests/performance/encodable_block/moe/moe_experts_perf_test.rs
index 47a50d96e..544285f2a 100644
--- a/crates/backend-uzu/tests/performance/encodable_block/moe/moe_experts_perf_test.rs
+++ b/crates/backend-uzu/tests/performance/encodable_block/moe/moe_experts_perf_test.rs
@@ -13,7 +13,7 @@ use backend_uzu::{
use half::bf16;
use rand::{RngExt, SeedableRng, rngs::StdRng};
-use crate::common::helpers::{alloc_allocation, alloc_allocation_with_data, create_context};
+use crate::common::helpers::{alloc_allocation_with_data, create_context};
fn build_offsets(
e: usize,
@@ -44,8 +44,6 @@ fn run_decode_case<B: Backend>(
warmup: usize,
iters: usize,
) {
- const K_TILE: usize = 64;
-
let mut rng = StdRng::seed_from_u64(0xDEC0DE1234567890);
let sum_k = t * k;
@@ -84,17 +82,6 @@ fn run_decode_case<B: Backend>(
let up_biases_buf = alloc_allocation_with_data::<B, bf16>(&ctx, &up_biases);
let down_biases_buf = alloc_allocation_with_data::<B, bf16>(&ctx, &down_biases);
- let num_tiles_k = d_ff.div_ceil(K_TILE);
-
- // Buffers for indirect dispatch
- let mut tile_counts_buf = alloc_allocation::<B, u32>(&ctx, e);
- let mut tile_offsets_buf = alloc_allocation::<B, u32>(&ctx, e + 1);
- let max_tiles = e * sum_k * 16384; // Conservative upper bound
- let mut tile_map_buf = alloc_allocation::<B, u32>(&ctx, max_tiles * 3);
- let mut total_tiles_buf = alloc_allocation::<B, u32>(&ctx, 1);
- let mut dispatch_args_buf = alloc_allocation::<B, u32>(&ctx, 3);
- let mut row_expert_map_buf = alloc_allocation::<B, u32>(&ctx, sum_k);
-
for _ in 0..warmup {
let mut encoder = Encoder::new(ctx).expect("Failed to create encoder");
let output = experts_kernel
@@ -103,21 +90,14 @@ fn run_decode_case<B: Backend>(
MoeExpertsTwoPassArguments {
x_perm: &x_perm_buf,
expert_offsets: &offsets_buf,
- row_expert_map: &mut row_expert_map_buf,
w13_all: &w13_buf,
w2_all: &w2_buf,
up_biases: &up_biases_buf,
down_biases: &down_biases_buf,
- tile_counts: &mut tile_counts_buf,
- tile_offsets: &mut tile_offsets_buf,
- tile_map: &mut tile_map_buf,
- total_tiles: &mut total_tiles_buf,
- dispatch_args: &mut dispatch_args_buf,
total_rows: sum_k,
d_model,
d_ff,
e,
- num_tiles_k: num_tiles_k as u32,
gating_code: 2,
gate_clip_min: f32::NEG_INFINITY,
gate_clip_max: f32::INFINITY,
@@ -128,8 +108,9 @@ fn run_decode_case<B: Backend>(
},
)
.expect("failed to encode MoE experts");
- encoder.end_encoding().submit().wait_until_completed().unwrap();
+ let completed = encoder.end_encoding().submit().wait_until_completed().unwrap();
drop(output);
+ drop(completed);
}
let mut times = Vec::with_capacity(iters);
@@ -142,21 +123,14 @@ fn run_decode_case<B: Backend>(
MoeExpertsTwoPassArguments {
x_perm: &x_perm_buf,
expert_offsets: &offsets_buf,
- row_expert_map: &mut row_expert_map_buf,
w13_all: &w13_buf,
w2_all: &w2_buf,
up_biases: &up_biases_buf,
down_biases: &down_biases_buf,
- tile_counts: &mut tile_counts_buf,
- tile_offsets: &mut tile_offsets_buf,
- tile_map: &mut tile_map_buf,
- total_tiles: &mut total_tiles_buf,
- dispatch_args: &mut dispatch_args_buf,
total_rows: sum_k,
d_model,
d_ff,
e,
- num_tiles_k: num_tiles_k as u32,
gating_code: 2,
gate_clip_min: f32::NEG_INFINITY,
gate_clip_max: f32::INFINITY,
@@ -167,8 +141,9 @@ fn run_decode_case<B: Backend>(
},
)
.expect("failed to encode MoE experts");
- encoder.end_encoding().submit().wait_until_completed().unwrap();
+ let completed = encoder.end_encoding().submit().wait_until_completed().unwrap();
drop(output);
+ drop(completed);
times.push(start.elapsed().as_secs_f64() * 1000.0);
}
@@ -195,8 +170,6 @@ fn run_two_pass_prefill_case<B: Backend>(
warmup: usize,
iters: usize,
) {
- const K_TILE: usize = 64;
-
let mut rng = StdRng::seed_from_u64(0xDEC0DE1234567890);
let sum_k = t * k;
@@ -233,36 +206,19 @@ fn run_two_pass_prefill_case<B: Backend>(
let up_biases_buf = alloc_allocation_with_data::<B, bf16>(&ctx, &up_biases);
let down_biases_buf = alloc_allocation_with_data::<B, bf16>(&ctx, &down_biases);
- let num_tiles_k = d_ff.div_ceil(K_TILE);
-
- let mut tile_counts_buf = alloc_allocation::<B, u32>(&ctx, e);
- let mut tile_offsets_buf = alloc_allocation::<B, u32>(&ctx, e + 1);
- let max_tiles = e * sum_k * 16384;
- let mut tile_map_buf = alloc_allocation::<B, u32>(&ctx, max_tiles * 3);
- let mut total_tiles_buf = alloc_allocation::<B, u32>(&ctx, 1);
- let mut dispatch_args_buf = alloc_allocation::<B, u32>(&ctx, 3);
- let mut row_expert_map_buf = alloc_allocation::<B, u32>(&ctx, sum_k);
-
for _ in 0..warmup {
let mut encoder = Encoder::new(ctx).expect("Failed to create encoder");
let args = MoeExpertsTwoPassArguments {
x_perm: &x_perm_buf,
expert_offsets: &offsets_buf,
- row_expert_map: &mut row_expert_map_buf,
w13_all: &w13_buf,
w2_all: &w2_buf,
up_biases: &up_biases_buf,
down_biases: &down_biases_buf,
- tile_counts: &mut tile_counts_buf,
- tile_offsets: &mut tile_offsets_buf,
- tile_map: &mut tile_map_buf,
- total_tiles: &mut total_tiles_buf,
- dispatch_args: &mut dispatch_args_buf,
total_rows: sum_k,
d_model,
d_ff,
e,
- num_tiles_k: num_tiles_k as u32,
gating_code: 2,
gate_clip_min: f32::NEG_INFINITY,
gate_clip_max: f32::INFINITY,
@@ -272,8 +228,9 @@ fn run_two_pass_prefill_case<B: Backend>(
data_type: DataType::BF16,
};
let output = experts_kernel.encode(&mut encoder, args).expect("failed to encode MoE experts");
- encoder.end_encoding().submit().wait_until_completed().unwrap();
+ let completed = encoder.end_encoding().submit().wait_until_completed().unwrap();
drop(output);
+ drop(completed);
}
let mut times = Vec::with_capacity(iters);
@@ -283,21 +240,14 @@ fn run_two_pass_prefill_case<B: Backend>(
let args = MoeExpertsTwoPassArguments {
x_perm: &x_perm_buf,
expert_offsets: &offsets_buf,
- row_expert_map: &mut row_expert_map_buf,
w13_all: &w13_buf,
w2_all: &w2_buf,
up_biases: &up_biases_buf,
down_biases: &down_biases_buf,
- tile_counts: &mut tile_counts_buf,
- tile_offsets: &mut tile_offsets_buf,
- tile_map: &mut tile_map_buf,
- total_tiles: &mut total_tiles_buf,
- dispatch_args: &mut dispatch_args_buf,
total_rows: sum_k,
d_model,
d_ff,
e,
- num_tiles_k: num_tiles_k as u32,
gating_code: 2,
gate_clip_min: f32::NEG_INFINITY,
gate_clip_max: f32::INFINITY,
@@ -307,8 +257,9 @@ fn run_two_pass_prefill_case<B: Backend>(
data_type: DataType::BF16,
};
let output = experts_kernel.encode(&mut encoder, args).expect("failed to encode MoE experts");
- encoder.end_encoding().submit().wait_until_completed().unwrap();
+ let completed = encoder.end_encoding().submit().wait_until_completed().unwrap();
drop(output);
+ drop(completed);
times.push(start.elapsed().as_secs_f64() * 1000.0);
}
@@ -389,8 +340,9 @@ fn run_fused_single_token_case<B: Backend>(
},
)
.expect("failed to encode MoE experts");
- encoder.end_encoding().submit().wait_until_completed().unwrap();
+ let completed = encoder.end_encoding().submit().wait_until_completed().unwrap();
drop(output);
+ drop(completed);
}
let mut times = Vec::with_capacity(iters);
@@ -421,8 +373,9 @@ fn run_fused_single_token_case<B: Backend>(
},
)
.expect("failed to encode MoE experts");
- encoder.end_encoding().submit().wait_until_completed().unwrap();
+ let completed = encoder.end_encoding().submit().wait_until_completed().unwrap();
drop(output);
+ drop(completed);
times.push(start.elapsed().as_secs_f64() * 1000.0);
}
@@ -448,8 +401,6 @@ fn run_indirect_decode_timed<B: Backend>(
warmup: usize,
iters: usize,
) -> f64 {
- const K_TILE: usize = 64;
-
let mut rng = StdRng::seed_from_u64(0xDEC0DE1234567890);
let sum_k = t * k;
@@ -484,15 +435,6 @@ fn run_indirect_decode_timed<B: Backend>(
let up_biases_buf = alloc_allocation_with_data::<B, bf16>(ctx, &up_biases);
let down_biases_buf = alloc_allocation_with_data::<B, bf16>(ctx, &down_biases);
- let num_tiles_k = d_ff.div_ceil(K_TILE);
- let mut tile_counts_buf = alloc_allocation::<B, u32>(ctx, e);
- let mut tile_offsets_buf = alloc_allocation::<B, u32>(ctx, e + 1);
- let max_tiles = e * sum_k * 16384;
- let mut tile_map_buf = alloc_allocation::<B, u32>(ctx, max_tiles * 3);
- let mut total_tiles_buf = alloc_allocation::<B, u32>(ctx, 1);
- let mut dispatch_args_buf = alloc_allocation::<B, u32>(ctx, 3);
- let mut row_expert_map_buf = alloc_allocation::<B, u32>(ctx, sum_k);
-
for _ in 0..warmup {
let mut encoder = Encoder::new(ctx).expect("Failed to create encoder");
let output = experts_kernel
@@ -501,21 +443,14 @@ fn run_indirect_decode_timed<B: Backend>(
MoeExpertsTwoPassArguments {
x_perm: &x_perm_buf,
expert_offsets: &offsets_buf,
- row_expert_map: &mut row_expert_map_buf,
w13_all: &w13_buf,
w2_all: &w2_buf,
up_biases: &up_biases_buf,
down_biases: &down_biases_buf,
- tile_counts: &mut tile_counts_buf,
- tile_offsets: &mut tile_offsets_buf,
- tile_map: &mut tile_map_buf,
- total_tiles: &mut total_tiles_buf,
- dispatch_args: &mut dispatch_args_buf,
total_rows: sum_k,
d_model,
d_ff,
e,
- num_tiles_k: num_tiles_k as u32,
gating_code: 2,
gate_clip_min: f32::NEG_INFINITY,
gate_clip_max: f32::INFINITY,
@@ -526,8 +461,9 @@ fn run_indirect_decode_timed<B: Backend>(
},
)
.expect("failed to encode MoE experts");
- encoder.end_encoding().submit().wait_until_completed().unwrap();
+ let completed = encoder.end_encoding().submit().wait_until_completed().unwrap();
drop(output);
+ drop(completed);
}
let mut times = Vec::with_capacity(iters);
@@ -540,21 +476,14 @@ fn run_indirect_decode_timed<B: Backend>(
MoeExpertsTwoPassArguments {
x_perm: &x_perm_buf,
expert_offsets: &offsets_buf,
- row_expert_map: &mut row_expert_map_buf,
w13_all: &w13_buf,
w2_all: &w2_buf,
up_biases: &up_biases_buf,
down_biases: &down_biases_buf,
- tile_counts: &mut tile_counts_buf,
- tile_offsets: &mut tile_offsets_buf,
- tile_map: &mut tile_map_buf,
- total_tiles: &mut total_tiles_buf,
- dispatch_args: &mut dispatch_args_buf,
total_rows: sum_k,
d_model,
d_ff,
e,
- num_tiles_k: num_tiles_k as u32,
gating_code: 2,
gate_clip_min: f32::NEG_INFINITY,
gate_clip_max: f32::INFINITY,
@@ -565,8 +494,9 @@ fn run_indirect_decode_timed<B: Backend>(
},
)
.expect("failed to encode MoE experts");
- encoder.end_encoding().submit().wait_until_completed().unwrap();
+ let completed = encoder.end_encoding().submit().wait_until_completed().unwrap();
drop(output);
+ drop(completed);
times.push(start.elapsed().as_secs_f64() * 1000.0);
}
@@ -635,8 +565,9 @@ fn run_fused_decode_timed<B: Backend>(
},
)
.expect("failed to encode MoE experts");
- encoder.end_encoding().submit().wait_until_completed().unwrap();
+ let completed = encoder.end_encoding().submit().wait_until_completed().unwrap();
drop(output);
+ drop(completed);
}
let mut times = Vec::with_capacity(iters);
@@ -667,8 +598,9 @@ fn run_fused_decode_timed<B: Backend>(
},
)
.expect("failed to encode MoE experts");
- encoder.end_encoding().submit().wait_until_completed().unwrap();
+ let completed = encoder.end_encoding().submit().wait_until_completed().unwrap();
drop(output);
+ drop(completed);
times.push(start.elapsed().as_secs_f64() * 1000.0);
}
diff --git a/crates/backend-uzu/tests/performance/encodable_block/moe/moe_perf_test.rs b/crates/backend-uzu/tests/performance/encodable_block/moe/moe_perf_test.rs
index 355dec81a..b6a2950c4 100644
--- a/crates/backend-uzu/tests/performance/encodable_block/moe/moe_perf_test.rs
+++ b/crates/backend-uzu/tests/performance/encodable_block/moe/moe_perf_test.rs
@@ -162,7 +162,6 @@ fn test_moe_pipeline_breakdown_decode() {
let mut offsets_buf = alloc_allocation::<B, u32>(&ctx, e + 1);
let mut sumk_buf = alloc_allocation::<B, u32>(&ctx, 1);
let mut bucketed_ids_buf = alloc_allocation::<B, i32>(&ctx, t * k);
- let mut x_perm_buf = alloc_allocation::<B, bf16>(&ctx, t * k * d_model);
let y_partial: Vec<bf16> = (0..t * k * d_model).map(|_| bf16::from_f32(rng.random_range(-1.0..1.0))).collect();
let y_partial_buf = alloc_allocation_with_data::<B, bf16>(&ctx, &y_partial);
let mut tok2row_buf = alloc_allocation::<B, i32>(&ctx, t * k);
@@ -196,18 +195,7 @@ fn test_moe_pipeline_breakdown_decode() {
let up_biases_buf = alloc_allocation_with_data::<B, bf16>(&ctx, &up_biases);
let down_biases_buf = alloc_allocation_with_data::<B, bf16>(&ctx, &down_biases);
- // Experts tiling buffers for two-pass
- const K_TILE: usize = 64;
let sum_k = t * k;
- let num_tiles_k = (d_ff + K_TILE - 1) / K_TILE;
- let max_tiles = t * k * e * num_tiles_k;
- let mut tile_counts_buf = alloc_allocation::<B, u32>(&ctx, e);
- let mut tile_offsets_buf = alloc_allocation::<B, u32>(&ctx, e + 1);
- let mut tile_map_buf = alloc_allocation::<B, u32>(&ctx, max_tiles * 3);
- let mut total_tiles_buf = alloc_allocation::<B, u32>(&ctx, 1);
- let mut dispatch_args_buf = alloc_allocation::<B, u32>(&ctx, 3);
-
- let mut row_expert_map_buf = alloc_allocation::<B, u32>(&ctx, sum_k);
// Scatter block bases buffers
let mut block_bases_buf = alloc_allocation::<B, u32>(&ctx, num_blocks * num_tiles);
@@ -297,22 +285,45 @@ fn test_moe_pipeline_breakdown_decode() {
let gather_perf = run_perf_with_warmup("Gather", 2, 5, || {
let mut encoder = Encoder::new(ctx.as_ref()).expect("Failed to create encoder");
- gather_kernel.encode(
- &mut encoder,
- DataType::BF16,
- MoeGatherArguments {
- x: &x_buf,
- bucketed_ids: &bucketed_ids_buf,
- x_perm: &mut x_perm_buf,
- sumk: &sumk_buf,
- t,
- k,
- d_model,
- },
- );
- encoder.end_encoding().submit().wait_until_completed().unwrap();
+ let x_perm = gather_kernel
+ .encode(
+ &mut encoder,
+ DataType::BF16,
+ MoeGatherArguments {
+ x: &x_buf,
+ bucketed_ids: &bucketed_ids_buf,
+ sumk: &sumk_buf,
+ t,
+ k,
+ d_model,
+ },
+ )
+ .expect("failed to encode MoE gather");
+ let completed = encoder.end_encoding().submit().wait_until_completed().unwrap();
+ drop(x_perm);
+ drop(completed);
});
+ let (x_perm_buf, x_perm_completed) = {
+ let mut encoder = Encoder::new(ctx.as_ref()).expect("Failed to create encoder");
+ let x_perm = gather_kernel
+ .encode(
+ &mut encoder,
+ DataType::BF16,
+ MoeGatherArguments {
+ x: &x_buf,
+ bucketed_ids: &bucketed_ids_buf,
+ sumk: &sumk_buf,
+ t,
+ k,
+ d_model,
+ },
+ )
+ .expect("failed to encode MoE gather");
+ let completed = encoder.end_encoding().submit().wait_until_completed().unwrap();
+ (x_perm, completed)
+ };
+
let experts_perf = run_perf_with_warmup("Experts (MAIN COMPUTE)", 2, 5, || {
let mut encoder = Encoder::new(ctx.as_ref()).expect("Failed to create encoder");
let output = experts_kernel
@@ -321,21 +332,14 @@ fn test_moe_pipeline_breakdown_decode() {
MoeExpertsTwoPassArguments {
x_perm: &x_perm_buf,
expert_offsets: &offsets_buf,
- row_expert_map: &mut row_expert_map_buf,
w13_all: &w13_buf,
w2_all: &w2_buf,
up_biases: &up_biases_buf,
down_biases: &down_biases_buf,
- tile_counts: &mut tile_counts_buf,
- tile_offsets: &mut tile_offsets_buf,
- tile_map: &mut tile_map_buf,
- total_tiles: &mut total_tiles_buf,
- dispatch_args: &mut dispatch_args_buf,
total_rows: sum_k,
d_model,
d_ff,
e,
- num_tiles_k: num_tiles_k as u32,
gating_code: 2, // SILU
gate_clip_min: f32::NEG_INFINITY,
gate_clip_max: 20.0,
@@ -346,9 +350,12 @@ fn test_moe_pipeline_breakdown_decode() {
},
)
.expect("failed to encode MoE experts");
- encoder.end_encoding().submit().wait_until_completed().unwrap();
+ let completed = encoder.end_encoding().submit().wait_until_completed().unwrap();
drop(output);
+ drop(completed);
});
+ drop(x_perm_buf);
+ drop(x_perm_completed);
let finalize_perf = run_perf_with_warmup("Finalize", 2, 5, || {
let mut encoder = Encoder::new(ctx.as_ref()).expect("Failed to create encoder");
diff --git a/crates/backend-uzu/tests/unit/backends/common/kernel/ssd_prefill_test.rs b/crates/backend-uzu/tests/unit/backends/common/kernel/ssd_prefill_test.rs
index 85ad712af..e1d9b6b9a 100644
--- a/crates/backend-uzu/tests/unit/backends/common/kernel/ssd_prefill_test.rs
+++ b/crates/backend-uzu/tests/unit/backends/common/kernel/ssd_prefill_test.rs
@@ -85,7 +85,6 @@ struct SSDPrefillFixture {
head_dim: usize,
state_dim: usize,
group_size: i32,
- total_x: usize,
x_strides: [usize; 3],
dt_strides: [usize; 2],
cb_strides: [usize; 3],
@@ -132,7 +131,6 @@ impl SSDPrefillFixture {
head_dim,
state_dim,
group_size,
- total_x,
x_strides,
dt_strides,
cb_strides,
@@ -161,7 +159,6 @@ fn run_prefill_kernel_mode<B: Backend>(
let d_buf = allocation_from_slice::<B, _>(ctx, &fixture.d_data);
let z_buf = allocation_from_slice::<B, _>(ctx, &fixture.z_data);
let mut state_buf = allocation_from_slice::<B, _>(ctx, &fixture.state_init);
- let mut y_buf = allocation_from_slice::<B, _>(ctx, &vec![0f32; fixture.total_x]);
let args = SSDPrefillArguments {
x: &x_buf,
@@ -171,7 +168,6 @@ fn run_prefill_kernel_mode<B: Backend>(
d: &d_buf,
z: &z_buf,
state: &mut state_buf,
- y: &mut y_buf,
suffix_len: fixture.suffix_len,
group_size: fixture.group_size as u32,
state_size: fixture.state_dim as u32,
@@ -184,11 +180,13 @@ fn run_prefill_kernel_mode<B: Backend>(
};
let mut encoder = Encoder::new(ctx).unwrap();
- kernel.encode(&mut encoder, args, mode);
- encoder.end_encoding().submit().wait_until_completed().unwrap();
+ let y_buf = kernel.encode(&mut encoder, args, mode).expect("Failed to encode SSD prefill");
+ let completed = encoder.end_encoding().submit().wait_until_completed().unwrap();
let y_vec = allocation_to_vec::<B, f32>(&y_buf);
let state_vec = allocation_to_vec::<B, f32>(&state_buf);
+ drop(y_buf);
+ drop(completed);
(y_vec, state_vec, None)
}
diff --git a/crates/backend-uzu/tests/unit/encodable_block/moe/moe_block_e2e_test.rs b/crates/backend-uzu/tests/unit/encodable_block/moe/moe_block_e2e_test.rs
index a5e4c8073..64cc0747e 100644
--- a/crates/backend-uzu/tests/unit/encodable_block/moe/moe_block_e2e_test.rs
+++ b/crates/backend-uzu/tests/unit/encodable_block/moe/moe_block_e2e_test.rs
@@ -307,16 +307,7 @@ fn run_moe_parity_test_internal<B: Backend>(
let mut bucketed_ids_buf = alloc_allocation::<B, i32>(&ctx, max_sumk);
let mut bucketed_probs_buf = alloc_allocation::<B, bf16>(&ctx, max_sumk);
let mut tok2row_buf = alloc_allocation::<B, i32>(&ctx, t * k);
- let mut x_perm_buf = alloc_allocation::<B, bf16>(&ctx, max_sumk * d_model);
let mut y_out_buf = alloc_allocation::<B, bf16>(&ctx, t * d_model);
- const BLOCK_M_DECODE: usize = 4; // matches two-pass decode kernel configuration
- let h_blocks_decode = (d_ff + BLOCK_M_DECODE - 1) / BLOCK_M_DECODE;
- let max_tiles = max_sumk * h_blocks_decode;
- let mut tile_counts_buf = alloc_allocation::<B, u32>(&ctx, e);
- let mut tile_offsets_buf = alloc_allocation::<B, u32>(&ctx, e + 1);
- let mut tile_map_buf = alloc_allocation::<B, u32>(&ctx, max_tiles * 3);
- let mut total_tiles_buf = alloc_allocation::<B, u32>(&ctx, 8);
- let mut dispatch_args_buf = alloc_allocation::<B, u32>(&ctx, 3);
// Encode ALL kernels in one command buffer
eprintln!("[E2E] Encoding entire MoE pipeline in single command buffer...");
@@ -383,43 +374,35 @@ fn run_moe_parity_test_internal<B: Backend>(
);
let gather = MoeGatherKernels::<B>::new(&ctx).expect("gather");
- gather.encode(
- &mut encoder,
- DataType::BF16,
- MoeGatherArguments {
- x: &x_buf,
- bucketed_ids: &bucketed_ids_buf,
- x_perm: &mut x_perm_buf,
- sumk: &sumk_buf,
- t,
- k,
- d_model,
- },
- );
+ let x_perm_buf = gather
+ .encode(
+ &mut encoder,
+ DataType::BF16,
+ MoeGatherArguments {
+ x: &x_buf,
+ bucketed_ids: &bucketed_ids_buf,
+ sumk: &sumk_buf,
+ t,
+ k,
+ d_model,
+ },
+ )
+ .expect("gather");
let total_rows = t * k;
- let mut row_expert_map_buf = alloc_allocation::<B, u32>(&ctx, total_rows);
let experts = MoeExpertsTwoPassPrefillBlock::<B>::new(&ctx).expect("experts");
- let num_tiles_k = ((d_ff + 64 - 1) / 64) as u32;
let args = MoeExpertsTwoPassArguments {
x_perm: &x_perm_buf,
expert_offsets: &offsets_buf,
- row_expert_map: &mut row_expert_map_buf,
w13_all: &w13_buf,
w2_all: &w2_buf,
up_biases: &up_biases_buf,
down_biases: &down_biases_buf,
- tile_counts: &mut tile_counts_buf,
- tile_offsets: &mut tile_offsets_buf,
- tile_map: &mut tile_map_buf,
- total_tiles: &mut total_tiles_buf,
- dispatch_args: &mut dispatch_args_buf,
total_rows,
d_model,
d_ff,
e,
- num_tiles_k,
gating_code,
gate_clip_min: gate_clip.0,
gate_clip_max: gate_clip.1,
@@ -471,18 +454,8 @@ fn run_moe_parity_test_internal<B: Backend>(
f32::from(w2_cpu[0])
);
- // Probe tile bookkeeping
- let total_tiles_cpu = allocation_prefix_to_vec::<B, u32>(&total_tiles_buf, 2);
- let dispatch_args_cpu = allocation_prefix_to_vec::<B, u32>(&dispatch_args_buf, 3);
- let tile_offsets_cpu = allocation_prefix_to_vec::<B, u32>(&tile_offsets_buf, (e + 1).min(8));
let sumk_val = allocation_to_vec::<B, u32>(&sumk_buf)[0] as usize;
- eprintln!("[E2E] Tile bookkeeping:");
- eprintln!(
- "[E2E] total_tiles={}, dispatch_args=({}, {}, {})",
- total_tiles_cpu[0], dispatch_args_cpu[0], dispatch_args_cpu[1], dispatch_args_cpu[2]
- );
- eprintln!("[E2E] tile_offsets[0..{}]={:?}", (e + 1).min(8), &tile_offsets_cpu);
- eprintln!("[E2E] sumk={}, num_tiles_k={}", sumk_val, num_tiles_k);
+ eprintln!("[E2E] sumk={}", sumk_val);
// For multi-token tests with large d_ff, verify gather output (x_perm)
if t > 1 && d_ff >= 256 {
@@ -498,10 +471,6 @@ fn run_moe_parity_test_internal<B: Backend>(
);
}
- // Check tile_map for first few tiles
- let tile_map_cpu = allocation_prefix_to_vec::<B, u32>(&tile_map_buf, 12.min(max_tiles * 3));
- eprintln!("[E2E] tile_map (first 4 tiles): {:?}", &tile_map_cpu);
-
// CRITICAL: Check tok2row mapping for multi-token tests
let tok2row_cpu = allocation_prefix_to_vec::<B, i32>(&tok2row_buf, t * k);
eprintln!("[E2E] tok2row[0..{}]: {:?}", t * k, &tok2row_cpu);
@@ -669,6 +638,7 @@ fn run_moe_parity_test_internal<B: Backend>(
}
eprintln!("[{}] ✓ PASSED (GPU matches CPU reference)", test_name);
+ drop(x_perm_buf);
drop(y_partial_buf);
drop(completed);
}
diff --git a/crates/backend-uzu/tests/unit/encodable_block/moe/moe_experts_test.rs b/crates/backend-uzu/tests/unit/encodable_block/moe/moe_experts_test.rs
index 41dd6cfec..296a005fc 100644
--- a/crates/backend-uzu/tests/unit/encodable_block/moe/moe_experts_test.rs
+++ b/crates/backend-uzu/tests/unit/encodable_block/moe/moe_experts_test.rs
@@ -23,7 +23,7 @@ use rand::{RngExt, SeedableRng, rngs::StdRng};
use crate::encodable_block::mlp::moe::tests::{
common::{
assert::assert_eq_float,
- helpers::{alloc_allocation, alloc_allocation_with_data, allocation_prefix_to_vec, create_context},
+ helpers::{alloc_allocation_with_data, allocation_prefix_to_vec, create_context},
},
cpu_tile_counts, cpu_tile_scan,
};
@@ -82,7 +82,6 @@ impl MoeTestData {
struct ScatterResult {
x_perm: Vec<bf16>,
offsets: Vec<u32>,
- row_expert_map: Vec<u32>,
perm_idx: Vec<usize>, // original row -> bucketed row (for gather)
}
@@ -118,22 +117,19 @@ fn scatter_by_expert(
expert_cursors[eid] += 1;
}
- // Build x_perm and row_expert_map in bucketed order
+ // Build x_perm in bucketed order
let mut x_perm = vec![bf16::from_f32(0.0); sum_k * d_model];
- let mut row_expert_map = vec![0u32; sum_k];
for bucket_row in 0..sum_k {
let orig_row = inv_perm[bucket_row];
let tok = orig_row / k;
for d in 0..d_model {
x_perm[bucket_row * d_model + d] = x[tok * d_model + d];
}
- row_expert_map[bucket_row] = topk_ids[orig_row] as u32;
}
ScatterResult {
x_perm,
offsets,
- row_expert_map,
perm_idx,
}
}
@@ -330,8 +326,6 @@ fn test_two_pass_decode_correctness() {
}
}
- // Build offsets and row_expert_map based on topk_ids
- // First count rows per expert
let mut expert_counts = vec![0usize; e];
for &eid in &topk_ids {
expert_counts[eid as usize] += 1;
@@ -341,14 +335,6 @@ fn test_two_pass_decode_correctness() {
offsets[i + 1] = offsets[i] + expert_counts[i] as u32;
}
- // Build row_expert_map (maps permuted row to expert)
- let mut row_expert_map = vec![0u32; sum_k];
- let mut expert_row_idx = vec![0usize; e];
- for (row, &eid) in topk_ids.iter().enumerate() {
- row_expert_map[row] = eid as u32;
- expert_row_idx[eid as usize] += 1;
- }
-
// Generate weights
let w13: Vec<bf16> =
(0..e * 2 * d_ff * d_model).map(|_| bf16::from_f32(rng.random_range(-0.05..0.05))).collect();
@@ -380,50 +366,28 @@ fn test_two_pass_decode_correctness() {
// Prepare GPU buffers
let x_perm_buf = alloc_allocation_with_data::<B, bf16>(&ctx, &x_perm);
let offsets_buf = alloc_allocation_with_data::<B, u32>(&ctx, &offsets);
- let mut row_expert_map_buf = alloc_allocation_with_data::<B, u32>(&ctx, &row_expert_map);
let w13_buf = alloc_allocation_with_data::<B, bf16>(&ctx, &w13);
let w2_buf = alloc_allocation_with_data::<B, bf16>(&ctx, &w2);
let up_biases_buf = alloc_allocation_with_data::<B, bf16>(&ctx, &up_biases);
let down_biases_buf = alloc_allocation_with_data::<B, bf16>(&ctx, &down_biases);
- // Tile infrastructure
- let h_blocks_decode = (d_ff + 3) / 4;
- let max_total_tiles = sum_k * h_blocks_decode;
-
- let mut tile_counts_buf = alloc_allocation::<B, u32>(&ctx, e);
- let mut tile_offsets_buf = alloc_allocation::<B, u32>(&ctx, e + 1);
- let mut total_tiles_buf = alloc_allocation::<B, u32>(&ctx, 8);
- let mut tile_map_buf = alloc_allocation::<B, u32>(&ctx, max_total_tiles * 3);
- let mut dispatch_args_buf = alloc_allocation::<B, u32>(&ctx, 3);
-
- // Execute 2-pass decode kernel
let experts_kernel = MoeExpertsTwoPassDecodeBlock::<B>::new(&ctx).expect("MoeExpertsTwoPassDecodeKernel::new");
let mut encoder = Encoder::new(ctx.as_ref()).expect("Failed to create encoder");
- const K_TILE: usize = 64;
- let num_tiles_k = ((d_ff + K_TILE - 1) / K_TILE) as u32;
-
let y_partial_buf = experts_kernel
.encode(
&mut encoder,
MoeExpertsTwoPassArguments {
x_perm: &x_perm_buf,
expert_offsets: &offsets_buf,
- row_expert_map: &mut row_expert_map_buf,
w13_all: &w13_buf,
w2_all: &w2_buf,
up_biases: &up_biases_buf,
down_biases: &down_biases_buf,
- tile_counts: &mut tile_counts_buf,
- tile_offsets: &mut tile_offsets_buf,
- tile_map: &mut tile_map_buf,
- total_tiles: &mut total_tiles_buf,
- dispatch_args: &mut dispatch_args_buf,
total_rows: sum_k,
d_model,
d_ff,
e,
- num_tiles_k,
gating_code,
gate_clip_min: f32::NEG_INFINITY,
gate_clip_max: f32::INFINITY,
@@ -527,18 +491,10 @@ fn test_two_pass_decode_multi_token() {
// GPU buffers
let x_perm_buf = alloc_allocation_with_data::<B, bf16>(&ctx, &scatter.x_perm);
let offsets_buf = alloc_allocation_with_data::<B, u32>(&ctx, &scatter.offsets);
- let mut row_expert_map_buf = alloc_allocation_with_data::<B, u32>(&ctx, &scatter.row_expert_map);
let w13_buf = alloc_allocation_with_data::<B, bf16>(&ctx, &data.w13);
let w2_buf = alloc_allocation_with_data::<B, bf16>(&ctx, &data.w2);
let up_biases_buf = alloc_allocation_with_data::<B, bf16>(&ctx, &data.up_biases);
let down_biases_buf = alloc_allocation_with_data::<B, bf16>(&ctx, &data.down_biases);
- // Tile infrastructure
- let max_total_tiles = sum_k * ((d_ff + 3) / 4);
- let mut tile_counts_buf = alloc_allocation::<B, u32>(&ctx, e);
- let mut tile_offsets_buf = alloc_allocation::<B, u32>(&ctx, e + 1);
- let mut total_tiles_buf = alloc_allocation::<B, u32>(&ctx, 8);
- let mut tile_map_buf = alloc_allocation::<B, u32>(&ctx, max_total_tiles * 3);
- let mut dispatch_args_buf = alloc_allocation::<B, u32>(&ctx, 3);
let experts_kernel = MoeExpertsTwoPassDecodeBlock::<B>::new(&ctx).expect("kernel");
let mut encoder = Encoder::new(ctx.as_ref()).expect("Failed to create encoder");
@@ -548,21 +504,14 @@ fn test_two_pass_decode_multi_token() {
MoeExpertsTwoPassArguments {
x_perm: &x_perm_buf,
expert_offsets: &offsets_buf,
- row_expert_map: &mut row_expert_map_buf,
w13_all: &w13_buf,
w2_all: &w2_buf,
up_biases: &up_biases_buf,
down_biases: &down_biases_buf,
- tile_counts: &mut tile_counts_buf,
- tile_offsets: &mut tile_offsets_buf,
- tile_map: &mut tile_map_buf,
- total_tiles: &mut total_tiles_buf,
- dispatch_args: &mut dispatch_args_buf,
total_rows: sum_k,
d_model,
d_ff,
e,
- num_tiles_k: ((d_ff + 63) / 64) as u32,
gating_code,
gate_clip_min: f32::NEG_INFINITY,
gate_clip_max: f32::INFINITY,
@@ -628,39 +577,24 @@ fn test_two_pass_prefill_correctness() {
// GPU buffers
let x_perm_buf = alloc_allocation_with_data::<B, bf16>(&ctx, &scatter.x_perm);
let offsets_buf = alloc_allocation_with_data::<B, u32>(&ctx, &scatter.offsets);
- let mut row_expert_map_buf = alloc_allocation_with_data::<B, u32>(&ctx, &scatter.row_expert_map);
let w13_buf = alloc_allocation_with_data::<B, bf16>(&ctx, &data.w13);
let w2_buf = alloc_allocation_with_data::<B, bf16>(&ctx, &data.w2);
let up_biases_buf = alloc_allocation_with_data::<B, bf16>(&ctx, &data.up_biases);
let down_biases_buf = alloc_allocation_with_data::<B, bf16>(&ctx, &data.down_biases);
- // Tile infrastructure
- let max_total_tiles = sum_k * ((d_ff + 3) / 4);
- let mut tile_counts_buf = alloc_allocation::<B, u32>(&ctx, e);
- let mut tile_offsets_buf = alloc_allocation::<B, u32>(&ctx, e + 1);
- let mut total_tiles_buf = alloc_allocation::<B, u32>(&ctx, 8);
- let mut tile_map_buf = alloc_allocation::<B, u32>(&ctx, max_total_tiles * 3);
- let mut dispatch_args_buf = alloc_allocation::<B, u32>(&ctx, 3);
let experts_kernel = MoeExpertsTwoPassPrefillBlock::<B>::new(&ctx).expect("kernel");
let mut encoder = Encoder::new(ctx.as_ref()).expect("Failed to create encoder");
let args = MoeExpertsTwoPassArguments {
x_perm: &x_perm_buf,
expert_offsets: &offsets_buf,
- row_expert_map: &mut row_expert_map_buf,
w13_all: &w13_buf,
w2_all: &w2_buf,
up_biases: &up_biases_buf,
down_biases: &down_biases_buf,
- tile_counts: &mut tile_counts_buf,
- tile_offsets: &mut tile_offsets_buf,
- tile_map: &mut tile_map_buf,
- total_tiles: &mut total_tiles_buf,
- dispatch_args: &mut dispatch_args_buf,
total_rows: sum_k,
d_model,
d_ff,
e,
- num_tiles_k: ((d_ff + 63) / 64) as u32,
gating_code,
gate_clip_min: f32::NEG_INFINITY,
gate_clip_max: f32::INFINITY,
diff --git a/crates/backend-uzu/tests/unit/encodable_block/moe/moe_tiles_test.rs b/crates/backend-uzu/tests/unit/encodable_block/moe/moe_tiles_test.rs
index 85b4de671..25af38012 100644
--- a/crates/backend-uzu/tests/unit/encodable_block/moe/moe_tiles_test.rs
+++ b/crates/backend-uzu/tests/unit/encodable_block/moe/moe_tiles_test.rs
@@ -1,13 +1,8 @@
-use backend_uzu::backends::common::{
- Encoder,
- kernel::moe::{MoeTileCountsArguments, MoeTileMapKernels, MoeTileScanArguments},
-};
+use backend_uzu::backends::common::{Encoder, kernel::moe::MoeTileMapKernels};
use rand::{RngExt, SeedableRng, rngs::StdRng};
use crate::encodable_block::mlp::moe::tests::{
- common::helpers::{
- alloc_allocation, alloc_allocation_with_data, allocation_prefix_to_vec, allocation_to_vec, create_context,
- },
+ common::helpers::{alloc_allocation_with_data, allocation_prefix_to_vec, allocation_to_vec, create_context},
cpu_tile_counts, cpu_tile_scan,
};
@@ -34,22 +29,17 @@ fn test_tile_counts_correctness() {
// GPU buffers
let offsets_buf = alloc_allocation_with_data::<B, u32>(&ctx, &offsets);
- let mut tile_counts_buf = alloc_allocation::<B, u32>(&ctx, e);
// Execute kernel using kernel struct
let tile_kernel = MoeTileMapKernels::<B>::new(&ctx).expect("MoeTileMapKernel::new");
let mut encoder = Encoder::new(ctx.as_ref()).expect("Failed to create encoder");
- tile_kernel.encode_counts(
- &mut encoder,
- MoeTileCountsArguments {
- offsets: &offsets_buf,
- tile_counts: &mut tile_counts_buf,
- e,
- },
- );
- encoder.end_encoding().submit().wait_until_completed().unwrap();
+ let tile_counts_buf =
+ tile_kernel.encode_counts(&mut encoder, &offsets_buf, e).expect("MoeTileMapKernels::encode_counts");
+ let completed = encoder.end_encoding().submit().wait_until_completed().unwrap();
let tile_counts_gpu = allocation_prefix_to_vec::<B, u32>(&tile_counts_buf, e);
+ drop(tile_counts_buf);
+ drop(completed);
assert_eq!(tile_counts_gpu, tile_counts_cpu, "Tile counts mismatch for E={}", e);
@@ -77,25 +67,18 @@ fn test_tile_scan_correctness() {
// GPU buffers
let tile_counts_buf = alloc_allocation_with_data::<B, u32>(&ctx, &tile_counts);
- let mut tile_offsets_buf = alloc_allocation::<B, u32>(&ctx, e + 1);
- let mut total_tiles_buf = alloc_allocation::<B, u32>(&ctx, 1);
// Execute kernel using kernel struct
let tile_kernel = MoeTileMapKernels::<B>::new(&ctx).expect("MoeTileMapKernel::new");
let mut encoder = Encoder::new(ctx.as_ref()).expect("Failed to create encoder");
- tile_kernel.encode_scan(
- &mut encoder,
- MoeTileScanArguments {
- tile_counts: &tile_counts_buf,
- tile_offsets: &mut tile_offsets_buf,
- total_tiles: &mut total_tiles_buf,
- e,
- },
- );
- encoder.end_encoding().submit().wait_until_completed().unwrap();
-
- let tile_offsets_gpu = allocation_prefix_to_vec::<B, u32>(&tile_offsets_buf, e + 1);
- let total_tiles_gpu = allocation_to_vec::<B, u32>(&total_tiles_buf)[0];
+ let tile_scan =
+ tile_kernel.encode_scan(&mut encoder, &tile_counts_buf, e).expect("MoeTileMapKernels::encode_scan");
+ let completed = encoder.end_encoding().submit().wait_until_completed().unwrap();
+
+ let tile_offsets_gpu = allocation_prefix_to_vec::<B, u32>(&tile_scan.tile_offsets, e + 1);
+ let total_tiles_gpu = allocation_to_vec::<B, u32>(&tile_scan.total_tiles)[0];
+ drop(tile_scan);
+ drop(completed);
assert_eq!(tile_offsets_gpu, tile_offsets_cpu, "Tile offsets mismatch for E={}", e);
assert_eq!(total_tiles_gpu, total_tiles_cpu, "Total tiles mismatch for E={}", e);
@@ -118,21 +101,16 @@ fn test_tile_edge_cases() {
let tile_counts_cpu = cpu_tile_counts(&offsets, 16);
let offsets_buf = alloc_allocation_with_data::<B, u32>(&ctx, &offsets);
- let mut tile_counts_buf = alloc_allocation::<B, u32>(&ctx, e);
let tile_kernel = MoeTileMapKernels::<B>::new(&ctx).expect("MoeTileMapKernel::new");
let mut encoder = Encoder::new(ctx.as_ref()).expect("Failed to create encoder");
- tile_kernel.encode_counts(
- &mut encoder,
- MoeTileCountsArguments {
- offsets: &offsets_buf,
- tile_counts: &mut tile_counts_buf,
- e,
- },
- );
- encoder.end_encoding().submit().wait_until_completed().unwrap();
+ let tile_counts_buf =
+ tile_kernel.encode_counts(&mut encoder, &offsets_buf, e).expect("MoeTileMapKernels::encode_counts");
+ let completed = encoder.end_encoding().submit().wait_until_completed().unwrap();
let tile_counts_gpu = allocation_prefix_to_vec::<B, u32>(&tile_counts_buf, e);
+ drop(tile_counts_buf);
+ drop(completed);
assert_eq!(tile_counts_gpu, tile_counts_cpu);
assert!(tile_counts_gpu.iter().all(|&c| c == 0));
@@ -147,21 +125,16 @@ fn test_tile_edge_cases() {
let tile_counts_cpu = cpu_tile_counts(&offsets, 16);
let offsets_buf = alloc_allocation_with_data::<B, u32>(&ctx, &offsets);
- let mut tile_counts_buf = alloc_allocation::<B, u32>(&ctx, e);
let tile_kernel = MoeTileMapKernels::<B>::new(&ctx).expect("MoeTileMapKernels::new");
let mut encoder = Encoder::new(ctx.as_ref()).expect("Failed to create encoder");
- tile_kernel.encode_counts(
- &mut encoder,
- MoeTileCountsArguments {
- offsets: &offsets_buf,
- tile_counts: &mut tile_counts_buf,
- e,
- },
- );
- encoder.end_encoding().submit().wait_until_completed().unwrap();
+ let tile_counts_buf =
+ tile_kernel.encode_counts(&mut encoder, &offsets_buf, e).expect("MoeTileMapKernels::encode_counts");
+ let completed = encoder.end_encoding().submit().wait_until_completed().unwrap();
let tile_counts_gpu = allocation_prefix_to_vec::<B, u32>(&tile_counts_buf, e);
+ drop(tile_counts_buf);
+ drop(completed);
assert_eq!(tile_counts_gpu, tile_counts_cpu);
// With BM=16: seg_lens=[10, 5, 15, 20] -> tile_counts=[1, 1, 1, 2]
diff --git a/crates/backend-uzu/tests/unit/kernel/moe/moe_gather_test.rs b/crates/backend-uzu/tests/unit/kernel/moe/moe_gather_test.rs
index 493348dc4..4d2e04c6b 100644
--- a/crates/backend-uzu/tests/unit/kernel/moe/moe_gather_test.rs
+++ b/crates/backend-uzu/tests/unit/kernel/moe/moe_gather_test.rs
@@ -15,7 +15,7 @@ use backend_uzu::{
use crate::{
common::{
assert::assert_eq_float,
- helpers::{alloc_allocation, alloc_allocation_with_data, allocation_to_vec, create_context},
+ helpers::{alloc_allocation_with_data, allocation_to_vec, create_context},
},
uzu_test,
};
@@ -35,26 +35,28 @@ fn get_output<B: Backend, T: ArrayElement + Float>(input: &Input<T>) -> Vec<T> {
let x_allocation = alloc_allocation_with_data::<B, T>(&context, &input.x);
let ids_allocation = alloc_allocation_with_data::<B, i32>(&context, &input.bucket_ids);
let sumk_allocation = alloc_allocation_with_data::<B, u32>(&context, &sumk_data);
- let mut x_perm_allocation = alloc_allocation::<B, T>(&context, input.sum_k * input.d_model);
-
let gather = MoeGatherKernels::<B>::new(&context).expect("MoeGatherKernel::new");
let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");
- gather.encode(
- &mut encoder,
- T::data_type(),
- MoeGatherArguments {
- x: &x_allocation,
- bucketed_ids: &ids_allocation,
- x_perm: &mut x_perm_allocation,
- sumk: &sumk_allocation,
- t: input.t,
- k: input.sum_k / input.t, // Decompose sum_k into k per token
- d_model: input.d_model,
- },
- );
- encoder.end_encoding().submit().wait_until_completed().unwrap();
+ let x_perm_allocation = gather
+ .encode(
+ &mut encoder,
+ T::data_type(),
+ MoeGatherArguments {
+ x: &x_allocation,
+ bucketed_ids: &ids_allocation,
+ sumk: &sumk_allocation,
+ t: input.t,
+ k: input.sum_k / input.t, // Decompose sum_k into k per token
+ d_model: input.d_model,
+ },
+ )
+ .expect("Failed to encode MoE gather");
+ let completed = encoder.end_encoding().submit().wait_until_completed().unwrap();
- allocation_to_vec::<B, T>(&x_perm_allocation)
+ let output = allocation_to_vec::<B, T>(&x_perm_allocation);
+ drop(x_perm_allocation);
+ drop(completed);
+ output
}
fn test_gather_internal(