A high-performance inference engine for AI models
Allocations clean up (#403)
LuckyIYI4b1eab1a7b63
diff --git a/crates/backend-uzu/src/backends/common/kernel/mlp_gate_act_mul.rs b/crates/backend-uzu/src/backends/common/kernel/mlp_gate_act_mul.rsindex 8676b3e1e..e22dfaf45 100644--- a/crates/backend-uzu/src/backends/common/kernel/mlp_gate_act_mul.rs+++ b/crates/backend-uzu/src/backends/common/kernel/mlp_gate_act_mul.rs@@ -1,5 +1,6 @@use crate::{DataType,+ array::size_for_shape,backends::common::{ActivationConfig, Allocation, Backend, Encoder, Kernels, gpu_types::ActivationType, kernel::MlpGateActMulKernel,},@@ -9,6 +10,7 @@ pub struct MlpGateActMulEncodable<B: Backend> {kernel: <B::Kernels as Kernels>::MlpGateActMulKernel,activation: ActivationConfig,hidden_dim: usize,+ data_type: DataType,hadamard_factors: Option<Allocation<B>>,}@@ -25,6 +27,7 @@ impl<B: Backend> MlpGateActMulEncodable<B> {kernel,activation,hidden_dim,+ data_type,hadamard_factors,})}@@ -33,21 +36,21 @@ impl<B: Backend> MlpGateActMulEncodable<B> {&self,encoder: &mut Encoder<B>,fused_up: &Allocation<B>,- hidden: &mut Allocation<B>,- m: i32,- ) -> Result<(), B::Error> {+ batch_dim: usize,+ ) -> Result<Allocation<B>, B::Error> {if self.activation.act_type() == ActivationType::IDENTITY {panic!("Identity activation is not supported for kernel")}+ let mut hidden = encoder.allocate_scratch(size_for_shape(&[batch_dim, self.hidden_dim], self.data_type))?;self.kernel.encode(fused_up,- hidden,+ &mut hidden,self.hadamard_factors.as_ref(),self.hidden_dim as i32,- m,+ batch_dim as i32,self.activation.act_type(),encoder,);- Ok(())+ Ok(hidden)}}diff --git a/crates/backend-uzu/src/backends/common/kernel/moe/experts_two_pass_decode.rs b/crates/backend-uzu/src/backends/common/kernel/moe/experts_two_pass_decode.rsindex 8fc689c81..8ba310214 100644--- a/crates/backend-uzu/src/backends/common/kernel/moe/experts_two_pass_decode.rs+++ b/crates/backend-uzu/src/backends/common/kernel/moe/experts_two_pass_decode.rs@@ -1,7 +1,4 @@-use super::{- MoeExpertsTwoPassArguments, MoePassARowMapArguments, MoePassATileBuildArguments, MoePassATileCountsArguments,- MoePassATileDispatchArguments, MoePassATileKernels, MoePassATileScanArguments,-};+use super::{MoeExpertsTwoPassArguments, MoePassATileKernels};use crate::{DataType,array::size_for_shape,@@ -49,63 +46,29 @@ impl<B: Backend> MoeExpertsTwoPassDecodeBlock<B> {encoder: &mut Encoder<B>,args: MoeExpertsTwoPassArguments<B>,) -> Result<Allocation<B>, B::Error> {- let mut hidden = encoder.allocate_scratch(size_for_shape(&[args.total_rows, args.d_ff], DataType::F32))?;- let mut output = encoder.allocate_scratch(size_for_shape(&[args.total_rows, args.d_model], args.data_type))?;-- // pass a tileconst BLOCK_M: u32 = 4;let h_blocks = (args.d_ff as u32 + BLOCK_M - 1) / BLOCK_M;- self.pass_a_tile.encode_counts(- encoder,- MoePassATileCountsArguments {- expert_offsets: args.expert_offsets,- tile_counts: &mut *args.tile_counts,- e: args.e,- h_blocks,- },- );- self.pass_a_tile.encode_scan(- encoder,- MoePassATileScanArguments {- tile_counts: &*args.tile_counts,- tile_offsets: &mut *args.tile_offsets,- total_tiles: &mut *args.total_tiles,- e: args.e,- },- );- self.pass_a_tile.encode_row_map(- encoder,- MoePassARowMapArguments {- expert_offsets: args.expert_offsets,- row_expert_map: &mut *args.row_expert_map,- total_rows: args.total_rows,- e: args.e,- },- );- self.pass_a_tile.encode_build_map(- encoder,- MoePassATileBuildArguments {- expert_offsets: args.expert_offsets,- tile_offsets: &*args.tile_offsets,- row_expert_map: &*args.row_expert_map,- tile_map: &mut *args.tile_map,- total_rows: args.total_rows,- h_blocks,- },- );- self.pass_a_tile.encode_dispatch_args(+ let tile_counts = self.pass_a_tile.encode_counts(encoder, args.expert_offsets, args.e, h_blocks)?;++ let tile_scan = self.pass_a_tile.encode_scan(encoder, &tile_counts, args.e)?;++ let row_expert_map = self.pass_a_tile.encode_row_map(encoder, args.expert_offsets, args.total_rows, args.e)?;++ let tile_map = self.pass_a_tile.encode_build_map(encoder,- MoePassATileDispatchArguments {- total_tiles: &*args.total_tiles,- dispatch_args: &mut *args.dispatch_args,- num_tiles_y: 1,- },- );+ args.expert_offsets,+ &tile_scan.tile_offsets,+ &row_expert_map,+ args.total_rows,+ h_blocks,+ )?;++ let dispatch_args = self.pass_a_tile.encode_dispatch_args(encoder, &tile_scan.total_tiles, 1)?;let gate_idx = args.gating_code.min(3) as usize;let dtype_idx = DTYPES.iter().position(|dtype| *dtype == args.data_type).unwrap();+ let mut hidden = encoder.allocate_scratch(size_for_shape(&[args.total_rows, args.d_ff], DataType::F32))?;- // pass alet pass_a_kernel = &self.pass_a_indirect[gate_idx][dtype_idx];pass_a_kernel.encode(args.x_perm,@@ -121,16 +84,16 @@ impl<B: Backend> MoeExpertsTwoPassDecodeBlock<B> {args.up_clip_min,args.up_clip_max,args.silu_alpha,- &*args.tile_map,- &*args.dispatch_args,+ &tile_map,+ &dispatch_args,encoder,);- // pass b+ let mut output = encoder.allocate_scratch(size_for_shape(&[args.total_rows, args.d_model], args.data_type))?;let pass_b_kernel = &self.fused_down[dtype_idx];pass_b_kernel.encode(&hidden,- &*args.row_expert_map,+ &row_expert_map,args.w2_all,args.down_biases,&mut output,diff --git a/crates/backend-uzu/src/backends/common/kernel/moe/experts_two_pass_prefill.rs b/crates/backend-uzu/src/backends/common/kernel/moe/experts_two_pass_prefill.rsindex c4f65ef6e..d4d4e745a 100644--- a/crates/backend-uzu/src/backends/common/kernel/moe/experts_two_pass_prefill.rs+++ b/crates/backend-uzu/src/backends/common/kernel/moe/experts_two_pass_prefill.rs@@ -1,6 +1,4 @@-use super::{- MoeTileCountsArguments, MoeTileDispatchArguments, MoeTileMapBuildArguments, MoeTileMapKernels, MoeTileScanArguments,-};+use super::MoeTileMapKernels;use crate::{DataType,array::size_for_shape,@@ -50,37 +48,18 @@ impl<B: Backend> MoeExpertsTwoPassPrefillBlock<B> {encoder: &mut Encoder<B>,args: MoeExpertsTwoPassArguments<B>,) -> Result<Allocation<B>, B::Error> {- let mut hidden = encoder.allocate_scratch(size_for_shape(&[args.total_rows, args.d_ff], DataType::F32))?;- let mut output = encoder.allocate_scratch(size_for_shape(&[args.total_rows, args.d_model], args.data_type))?;- encoder.encode_fill(&mut hidden, 0);+ let tile_counts = self.tile_map.encode_counts(encoder, args.expert_offsets, args.e)?;- self.tile_map.encode_counts(- encoder,- MoeTileCountsArguments {- offsets: args.expert_offsets,- tile_counts: &mut *args.tile_counts,- e: args.e,- },- );- self.tile_map.encode_scan(- encoder,- MoeTileScanArguments {- tile_counts: &*args.tile_counts,- tile_offsets: &mut *args.tile_offsets,- total_tiles: &mut *args.total_tiles,- e: args.e,- },- );- self.tile_map.encode_build_map(+ let tile_scan = self.tile_map.encode_scan(encoder, &tile_counts, args.e)?;++ let tile_map = self.tile_map.encode_build_map(encoder,- MoeTileMapBuildArguments {- expert_offsets: args.expert_offsets,- tile_offsets: &*args.tile_offsets,- tile_counts: &*args.tile_counts,- tile_map: &mut *args.tile_map,- e: args.e,- },- );+ args.expert_offsets,+ &tile_scan.tile_offsets,+ &tile_counts,+ args.total_rows,+ args.e,+ )?;let d_model_u32 = args.d_model as u32;let d_ff_u32 = args.d_ff as u32;const COL_TILE_FF: u32 = 32; // Must match PASSA_BN in kernel@@ -95,17 +74,12 @@ impl<B: Backend> MoeExpertsTwoPassPrefillBlock<B> {} else {(d_model_u32 + COL_TILE_MODEL - 1) / COL_TILE_MODEL};- self.tile_map.encode_dispatch_args(- encoder,- MoeTileDispatchArguments {- total_tiles: &*args.total_tiles,- dispatch_args: &mut *args.dispatch_args,- num_tiles_x: n_tiles_ff,- },- );+ let pass_a_dispatch_args = self.tile_map.encode_dispatch_args(encoder, &tile_scan.total_tiles, n_tiles_ff)?;let gate_idx = args.gating_code.min(3) as usize;let dtype_idx = DTYPES.iter().position(|t| *t == args.data_type).unwrap();+ let mut hidden = encoder.allocate_scratch(size_for_shape(&[args.total_rows, args.d_ff], DataType::F32))?;+ encoder.encode_fill(&mut hidden, 0);let kernel_pass_a = &self.pass_a_indirect[gate_idx][dtype_idx];kernel_pass_a.encode(@@ -122,18 +96,15 @@ impl<B: Backend> MoeExpertsTwoPassPrefillBlock<B> {args.up_clip_min,args.up_clip_max,args.silu_alpha,- &*args.tile_map,- &*args.dispatch_args,+ &tile_map,+ &pass_a_dispatch_args,encoder,);- let dispatch_args = MoeTileDispatchArguments {- total_tiles: &*args.total_tiles,- dispatch_args: &mut *args.dispatch_args,- num_tiles_x: n_tiles_model,- };- self.tile_map.encode_dispatch_args(encoder, dispatch_args);+ let pass_b_dispatch_args =+ self.tile_map.encode_dispatch_args(encoder, &tile_scan.total_tiles, n_tiles_model)?;+ let mut output = encoder.allocate_scratch(size_for_shape(&[args.total_rows, args.d_model], args.data_type))?;let kernel_pass_b = &self.pass_b_indirect[dtype_idx];kernel_pass_b.encode(&hidden,@@ -144,8 +115,8 @@ impl<B: Backend> MoeExpertsTwoPassPrefillBlock<B> {args.d_model as u32,args.d_ff as u32,args.e as u32,- &*args.tile_map,- &*args.dispatch_args,+ &tile_map,+ &pass_b_dispatch_args,encoder,);Ok(output)@@ -155,21 +126,14 @@ impl<B: Backend> MoeExpertsTwoPassPrefillBlock<B> {pub struct MoeExpertsTwoPassArguments<'a, B: Backend> {pub x_perm: &'a Allocation<B>,pub expert_offsets: &'a Allocation<B>,- pub row_expert_map: &'a mut Allocation<B>,pub w13_all: &'a Allocation<B>,pub w2_all: &'a Allocation<B>,pub up_biases: &'a Allocation<B>,pub down_biases: &'a Allocation<B>,- pub tile_counts: &'a mut Allocation<B>,- pub tile_offsets: &'a mut Allocation<B>,- pub tile_map: &'a mut Allocation<B>,- pub total_tiles: &'a mut Allocation<B>,- pub dispatch_args: &'a mut Allocation<B>,pub total_rows: usize,pub d_model: usize,pub d_ff: usize,pub e: usize,- pub num_tiles_k: u32,pub gating_code: u32,pub gate_clip_min: f32,pub gate_clip_max: f32,diff --git a/crates/backend-uzu/src/backends/common/kernel/moe/gather.rs b/crates/backend-uzu/src/backends/common/kernel/moe/gather.rsindex 9c6040b37..ad740ad4d 100644--- a/crates/backend-uzu/src/backends/common/kernel/moe/gather.rs+++ b/crates/backend-uzu/src/backends/common/kernel/moe/gather.rs@@ -1,5 +1,6 @@use crate::{DataType,+ array::size_for_shape,backends::common::{Allocation, Backend, Encoder, Kernels,kernel::{MoeGatherXPerm1DKernel, MoeGatherXPerm2DKernel},@@ -9,7 +10,6 @@ use crate::{pub struct MoeGatherArguments<'a, B: Backend> {pub x: &'a Allocation<B>,pub bucketed_ids: &'a Allocation<B>,- pub x_perm: &'a mut Allocation<B>,pub sumk: &'a Allocation<B>,pub t: usize,pub k: usize,@@ -36,12 +36,15 @@ impl<B: Backend> MoeGatherKernels<B> {encoder: &mut Encoder<B>,dtype: DataType,args: MoeGatherArguments<B>,- ) {+ ) -> Result<Allocation<B>, B::Error> {+ let mut x_perm = encoder.allocate_scratch(size_for_shape(&[args.t * args.k, args.d_model], dtype))?;+ encoder.encode_fill(&mut x_perm, 0);+match dtype {DataType::F32 => self.f32.encode(args.x,args.bucketed_ids,- args.x_perm,+ &mut x_perm,args.sumk,args.d_model as u32,args.t as u32,@@ -51,7 +54,7 @@ impl<B: Backend> MoeGatherKernels<B> {DataType::F16 => self.f16.encode(args.x,args.bucketed_ids,- args.x_perm,+ &mut x_perm,args.sumk,args.d_model as u32,args.t as u32,@@ -61,7 +64,7 @@ impl<B: Backend> MoeGatherKernels<B> {DataType::BF16 => self.bf16.encode(args.x,args.bucketed_ids,- args.x_perm,+ &mut x_perm,args.sumk,args.d_model as u32,args.t as u32,@@ -70,5 +73,6 @@ impl<B: Backend> MoeGatherKernels<B> {),_ => panic!("Unsupported data type: {:?}", dtype),};+ Ok(x_perm)}}diff --git a/crates/backend-uzu/src/backends/common/kernel/moe/mod.rs b/crates/backend-uzu/src/backends/common/kernel/moe/mod.rsindex 1ac0db46a..a7dc41735 100644--- a/crates/backend-uzu/src/backends/common/kernel/moe/mod.rs+++ b/crates/backend-uzu/src/backends/common/kernel/moe/mod.rs@@ -9,10 +9,5 @@ pub use experts_single::{MoeExpertsSingleDecodeArguments, MoeExpertsSingleDecodepub use experts_two_pass_decode::MoeExpertsTwoPassDecodeBlock;pub use experts_two_pass_prefill::{MoeExpertsTwoPassArguments, MoeExpertsTwoPassPrefillBlock};pub use gather::{MoeGatherArguments, MoeGatherKernels};-pub use tiles_map::{- MoeTileCountsArguments, MoeTileDispatchArguments, MoeTileMapBuildArguments, MoeTileMapKernels, MoeTileScanArguments,-};-pub use tiles_pass_a::{- MoePassARowMapArguments, MoePassATileBuildArguments, MoePassATileCountsArguments, MoePassATileDispatchArguments,- MoePassATileKernels, MoePassATileScanArguments,-};+pub use tiles_map::{MoeTileMapKernels, MoeTileScanOutput};+pub use tiles_pass_a::{MoePassATileKernels, MoePassATileScanOutput};diff --git a/crates/backend-uzu/src/backends/common/kernel/moe/tiles_map.rs b/crates/backend-uzu/src/backends/common/kernel/moe/tiles_map.rsindex 6bff799cc..b549a6801 100644--- a/crates/backend-uzu/src/backends/common/kernel/moe/tiles_map.rs+++ b/crates/backend-uzu/src/backends/common/kernel/moe/tiles_map.rs@@ -1,6 +1,10 @@-use crate::backends::common::{- Allocation, Backend, Encoder, Kernels,- kernel::{MoeBuildTileMapKernel, MoeTileCountsKernel, MoeTileScanKernel, MoeWriteDispatchArgsKernel},+use crate::{+ DataType,+ array::size_for_shape,+ backends::common::{+ Allocation, Backend, Encoder, Kernels,+ kernel::{MoeBuildTileMapKernel, MoeTileCountsKernel, MoeTileScanKernel, MoeWriteDispatchArgsKernel},+ },};pub struct MoeTileMapKernels<B: Backend> {@@ -23,66 +27,56 @@ impl<B: Backend> MoeTileMapKernels<B> {pub fn encode_counts(&self,encoder: &mut Encoder<B>,- args: MoeTileCountsArguments<B>,- ) {- self.counts.encode(args.offsets, args.tile_counts, args.e as u32, encoder);+ offsets: &Allocation<B>,+ e: usize,+ ) -> Result<Allocation<B>, B::Error> {+ let mut tile_counts = encoder.allocate_scratch(size_for_shape(&[e], DataType::U32))?;+ self.counts.encode(offsets, &mut tile_counts, e as u32, encoder);+ Ok(tile_counts)}pub fn encode_scan(&self,encoder: &mut Encoder<B>,- args: MoeTileScanArguments<B>,- ) {- self.scan.encode(args.tile_counts, args.tile_offsets, args.total_tiles, args.e as u32, encoder);+ tile_counts: &Allocation<B>,+ e: usize,+ ) -> Result<MoeTileScanOutput<B>, B::Error> {+ let mut tile_offsets = encoder.allocate_scratch(size_for_shape(&[e + 1], DataType::U32))?;+ let mut total_tiles = encoder.allocate_scratch(size_for_shape(&[8], DataType::U32))?;+ self.scan.encode(tile_counts, &mut tile_offsets, &mut total_tiles, e as u32, encoder);+ Ok(MoeTileScanOutput {+ tile_offsets,+ total_tiles,+ })}pub fn encode_build_map(&self,encoder: &mut Encoder<B>,- args: MoeTileMapBuildArguments<B>,- ) {- self.build.encode(- args.expert_offsets,- args.tile_offsets,- args.tile_counts,- args.tile_map,- args.e as u32,- encoder,- );+ expert_offsets: &Allocation<B>,+ tile_offsets: &Allocation<B>,+ tile_counts: &Allocation<B>,+ total_rows: usize,+ e: usize,+ ) -> Result<Allocation<B>, B::Error> {+ let mut tile_map = encoder.allocate_scratch(size_for_shape(&[total_rows * 3], DataType::U32))?;+ self.build.encode(expert_offsets, tile_offsets, tile_counts, &mut tile_map, e as u32, encoder);+ Ok(tile_map)}pub fn encode_dispatch_args(&self,encoder: &mut Encoder<B>,- args: MoeTileDispatchArguments<B>,- ) {- self.dispatch.encode(args.total_tiles, args.dispatch_args, args.num_tiles_x, encoder);+ total_tiles: &Allocation<B>,+ num_tiles_x: u32,+ ) -> Result<Allocation<B>, B::Error> {+ let mut dispatch_args = encoder.allocate_scratch(size_for_shape(&[3], DataType::U32))?;+ self.dispatch.encode(total_tiles, &mut dispatch_args, num_tiles_x, encoder);+ Ok(dispatch_args)}}-pub struct MoeTileCountsArguments<'a, B: Backend> {- pub offsets: &'a Allocation<B>, // [E+1]- pub tile_counts: &'a mut Allocation<B>, // [E]- pub e: usize,-}--pub struct MoeTileScanArguments<'a, B: Backend> {- pub tile_counts: &'a Allocation<B>, // [E]- pub tile_offsets: &'a mut Allocation<B>, // [E+1]- pub total_tiles: &'a mut Allocation<B>, // [>=2]- pub e: usize,-}--pub struct MoeTileMapBuildArguments<'a, B: Backend> {- pub expert_offsets: &'a Allocation<B>, // [E+1]- pub tile_offsets: &'a Allocation<B>, // [E+1]- pub tile_counts: &'a Allocation<B>, // [E]- pub tile_map: &'a mut Allocation<B>, // [total_tiles * 3]- pub e: usize,-}--pub struct MoeTileDispatchArguments<'a, B: Backend> {- pub total_tiles: &'a Allocation<B>, // [>=1]- pub dispatch_args: &'a mut Allocation<B>, // [3]- pub num_tiles_x: u32, // x dimension for indirect dispatch+pub struct MoeTileScanOutput<B: Backend> {+ pub tile_offsets: Allocation<B>,+ pub total_tiles: Allocation<B>,}diff --git a/crates/backend-uzu/src/backends/common/kernel/moe/tiles_pass_a.rs b/crates/backend-uzu/src/backends/common/kernel/moe/tiles_pass_a.rsindex cc3f8674e..ad8fc9b47 100644--- a/crates/backend-uzu/src/backends/common/kernel/moe/tiles_pass_a.rs+++ b/crates/backend-uzu/src/backends/common/kernel/moe/tiles_pass_a.rs@@ -1,45 +1,18 @@-use crate::backends::common::{- Allocation, Backend, Encoder,- kernel::{- Kernels, MoePassABuildRowMapKernel, MoePassABuildTileMapKernel, MoePassATileCountsKernel,- MoePassATileScanKernel, MoePassAWriteDispatchArgsKernel,+use crate::{+ DataType,+ array::size_for_shape,+ backends::common::{+ Allocation, Backend, Encoder,+ kernel::{+ Kernels, MoePassABuildRowMapKernel, MoePassABuildTileMapKernel, MoePassATileCountsKernel,+ MoePassATileScanKernel, MoePassAWriteDispatchArgsKernel,+ },},};-pub struct MoePassATileCountsArguments<'a, B: Backend> {- pub expert_offsets: &'a Allocation<B>, // [E+1]- pub tile_counts: &'a mut Allocation<B>, // [E]- pub e: usize,- pub h_blocks: u32,-}--pub struct MoePassATileScanArguments<'a, B: Backend> {- pub tile_counts: &'a Allocation<B>, // [E]- pub tile_offsets: &'a mut Allocation<B>, // [E+1]- pub total_tiles: &'a mut Allocation<B>, // [>=1]- pub e: usize,-}--pub struct MoePassARowMapArguments<'a, B: Backend> {- pub expert_offsets: &'a Allocation<B>, // [E+1]- pub row_expert_map: &'a mut Allocation<B>, // [total_rows]- pub total_rows: usize,- pub e: usize,-}--pub struct MoePassATileBuildArguments<'a, B: Backend> {- pub expert_offsets: &'a Allocation<B>, // [E+1]- pub tile_offsets: &'a Allocation<B>, // [E+1]- pub row_expert_map: &'a Allocation<B>, // [total_rows]- pub tile_map: &'a mut Allocation<B>, // [total_tiles * 3]- pub total_rows: usize,- pub h_blocks: u32,-}--pub struct MoePassATileDispatchArguments<'a, B: Backend> {- pub total_tiles: &'a Allocation<B>, // [>=1]- pub dispatch_args: &'a mut Allocation<B>, // [3]- pub num_tiles_y: u32,+pub struct MoePassATileScanOutput<B: Backend> {+ pub tile_offsets: Allocation<B>,+ pub total_tiles: Allocation<B>,}pub struct MoePassATileKernels<B: Backend> {@@ -64,48 +37,73 @@ impl<B: Backend> MoePassATileKernels<B> {pub fn encode_counts(&self,encoder: &mut Encoder<B>,- args: MoePassATileCountsArguments<B>,- ) {- self.counts.encode(args.expert_offsets, args.tile_counts, args.e as u32, args.h_blocks, encoder);+ expert_offsets: &Allocation<B>,+ e: usize,+ h_blocks: u32,+ ) -> Result<Allocation<B>, B::Error> {+ let mut tile_counts = encoder.allocate_scratch(size_for_shape(&[e], DataType::U32))?;+ self.counts.encode(expert_offsets, &mut tile_counts, e as u32, h_blocks, encoder);+ Ok(tile_counts)}pub fn encode_scan(&self,encoder: &mut Encoder<B>,- args: MoePassATileScanArguments<B>,- ) {- self.scan.encode(args.tile_counts, args.tile_offsets, args.total_tiles, args.e as u32, encoder);+ tile_counts: &Allocation<B>,+ e: usize,+ ) -> Result<MoePassATileScanOutput<B>, B::Error> {+ let mut tile_offsets = encoder.allocate_scratch(size_for_shape(&[e + 1], DataType::U32))?;+ let mut total_tiles = encoder.allocate_scratch(size_for_shape(&[1], DataType::U32))?;+ self.scan.encode(tile_counts, &mut tile_offsets, &mut total_tiles, e as u32, encoder);+ Ok(MoePassATileScanOutput {+ tile_offsets,+ total_tiles,+ })}pub fn encode_row_map(&self,encoder: &mut Encoder<B>,- args: MoePassARowMapArguments<B>,- ) {- self.row_map.encode(args.expert_offsets, args.row_expert_map, args.total_rows as u32, args.e as u32, encoder);+ expert_offsets: &Allocation<B>,+ total_rows: usize,+ e: usize,+ ) -> Result<Allocation<B>, B::Error> {+ let mut row_expert_map = encoder.allocate_scratch(size_for_shape(&[total_rows], DataType::U32))?;+ self.row_map.encode(expert_offsets, &mut row_expert_map, total_rows as u32, e as u32, encoder);+ Ok(row_expert_map)}pub fn encode_build_map(&self,encoder: &mut Encoder<B>,- args: MoePassATileBuildArguments<B>,- ) {+ expert_offsets: &Allocation<B>,+ tile_offsets: &Allocation<B>,+ row_expert_map: &Allocation<B>,+ total_rows: usize,+ h_blocks: u32,+ ) -> Result<Allocation<B>, B::Error> {+ let mut tile_map =+ encoder.allocate_scratch(size_for_shape(&[total_rows * h_blocks as usize * 3], DataType::U32))?;self.build_map.encode(- args.expert_offsets,- args.tile_offsets,- args.row_expert_map,- args.tile_map,- args.total_rows as u32,- args.h_blocks,+ expert_offsets,+ tile_offsets,+ row_expert_map,+ &mut tile_map,+ total_rows as u32,+ h_blocks,encoder,);+ Ok(tile_map)}pub fn encode_dispatch_args(&self,encoder: &mut Encoder<B>,- args: MoePassATileDispatchArguments<B>,- ) {- self.dispatch.encode(args.total_tiles, args.dispatch_args, args.num_tiles_y, encoder);+ total_tiles: &Allocation<B>,+ num_tiles_y: u32,+ ) -> Result<Allocation<B>, B::Error> {+ let mut dispatch_args = encoder.allocate_scratch(size_for_shape(&[3], DataType::U32))?;+ self.dispatch.encode(total_tiles, &mut dispatch_args, num_tiles_y, encoder);+ Ok(dispatch_args)}}diff --git a/crates/backend-uzu/src/backends/common/kernel/ssd_prefill.rs b/crates/backend-uzu/src/backends/common/kernel/ssd_prefill.rsindex b3769ec4a..127e275a9 100644--- a/crates/backend-uzu/src/backends/common/kernel/ssd_prefill.rs+++ b/crates/backend-uzu/src/backends/common/kernel/ssd_prefill.rs@@ -1,5 +1,6 @@use crate::{DataType,+ array::size_for_shape,backends::common::{Allocation, Backend, Encoder, Kernels,kernel::{SSDPrefill64Kernel, SSDPrefillKernel, SSDPrefillSequentialKernel},@@ -20,7 +21,6 @@ pub struct SSDPrefillArguments<'a, B: Backend> {pub d: &'a Allocation<B>,pub z: &'a Allocation<B>,pub state: &'a mut Allocation<B>,- pub y: &'a mut Allocation<B>,pub suffix_len: usize,pub group_size: u32,pub state_size: u32,@@ -36,6 +36,7 @@ pub struct SSDPrefillKernels<B: Backend> {single: <B::Kernels as Kernels>::SSDPrefillKernel,single_64: <B::Kernels as Kernels>::SSDPrefill64Kernel,sequential: <B::Kernels as Kernels>::SSDPrefillSequentialKernel,+ data_type: DataType,}impl<B: Backend> SSDPrefillKernels<B> {@@ -50,6 +51,7 @@ impl<B: Backend> SSDPrefillKernels<B> {single,single_64,sequential,+ data_type,})}@@ -58,11 +60,13 @@ impl<B: Backend> SSDPrefillKernels<B> {encoder: &mut Encoder<B>,args: SSDPrefillArguments<B>,mode: SSDPrefillMode,- ) {+ ) -> Result<Allocation<B>, B::Error> {let x_strides: [u32; 3] = args.x_strides.map(|x| x as u32);let dt_strides: [u32; 2] = args.dt_strides.map(|x| x as u32);let cb_strides: [u32; 3] = args.cb_strides.map(|x| x as u32);let state_strides: [u32; 3] = args.state_strides.map(|x| x as u32);+ let mut output = encoder+ .allocate_scratch(size_for_shape(&[args.suffix_len, args.channels * args.head_dim], self.data_type))?;if mode == SSDPrefillMode::SinglePass {if args.state_size == 64 {@@ -74,7 +78,7 @@ impl<B: Backend> SSDPrefillKernels<B> {args.d,args.z,args.state,- args.y,+ &mut output,args.suffix_len as u32,args.group_size,args.state_size,@@ -95,7 +99,7 @@ impl<B: Backend> SSDPrefillKernels<B> {args.d,args.z,args.state,- args.y,+ &mut output,args.suffix_len as u32,args.group_size,args.state_size,@@ -117,7 +121,7 @@ impl<B: Backend> SSDPrefillKernels<B> {args.d,args.z,args.state,- args.y,+ &mut output,args.suffix_len as u32,args.group_size,args.state_size,@@ -130,6 +134,7 @@ impl<B: Backend> SSDPrefillKernels<B> {encoder,)}+ Ok(output)}}diff --git a/crates/backend-uzu/src/encodable_block/delta_net_mixer.rs b/crates/backend-uzu/src/encodable_block/delta_net_mixer.rsindex 08beb2ac9..004560627 100644--- a/crates/backend-uzu/src/encodable_block/delta_net_mixer.rs+++ b/crates/backend-uzu/src/encodable_block/delta_net_mixer.rs@@ -264,8 +264,6 @@ impl<B: Backend> DeltaNetMixer<B> {encoder.allocate_scratch(size_for_shape(&[suffix_length * self.config.num_heads], DataType::F32))?;let mut prep_decay =encoder.allocate_scratch(size_for_shape(&[suffix_length * self.config.num_heads], DataType::F32))?;- let mut out =- encoder.allocate_scratch(size_for_shape(&[suffix_length, self.config.value_dim()], self.data_type))?;self.prefill_prep.encode(in_proj,@@ -283,6 +281,8 @@ impl<B: Backend> DeltaNetMixer<B> {encoder,);+ let mut out =+ encoder.allocate_scratch(size_for_shape(&[suffix_length, self.config.value_dim()], self.data_type))?;self.delta_net_prefill.encode(&prep_q_norm,&prep_k_norm,diff --git a/crates/backend-uzu/src/encodable_block/mamba_mixer.rs b/crates/backend-uzu/src/encodable_block/mamba_mixer.rsindex fb3d2791c..a6af9f11e 100644--- a/crates/backend-uzu/src/encodable_block/mamba_mixer.rs+++ b/crates/backend-uzu/src/encodable_block/mamba_mixer.rs@@ -296,8 +296,6 @@ impl<B: Backend> MambaMixer<B> {encoder: &mut Encoder<B>,suffix_length: usize,) -> Result<Allocation<B>, B::Error> {- let mut out =- encoder.allocate_scratch(size_for_shape(&[suffix_length, self.config.inner_dim()], self.data_type))?;self.ssd_prefill.encode(encoder,SSDPrefillArguments {@@ -308,7 +306,6 @@ impl<B: Backend> MambaMixer<B> {d: &self.skip_connection_weight,z: gate,state: &mut layer.ssm_state,- y: &mut out,suffix_len: suffix_length,group_size: (self.config.num_heads / self.config.num_groups) as u32,state_size: self.config.state_dim as u32,@@ -320,8 +317,7 @@ impl<B: Backend> MambaMixer<B> {head_dim: self.config.head_dim,},self.prefill_mode,- );- Ok(out)+ )}fn run_decode_ssm(diff --git a/crates/backend-uzu/src/encodable_block/mlp/dense.rs b/crates/backend-uzu/src/encodable_block/mlp/dense.rsindex 00449dea4..dd34e49cf 100644--- a/crates/backend-uzu/src/encodable_block/mlp/dense.rs+++ b/crates/backend-uzu/src/encodable_block/mlp/dense.rs@@ -1,18 +1,12 @@//! MLP block encodable.use super::{super::linear::Linear, Mlp};-use crate::{- DataType,- array::size_for_shape,- backends::common::{Allocation, Backend, Encoder, kernel::mlp_gate_act_mul::MlpGateActMulEncodable},-};+use crate::backends::common::{Allocation, Backend, Encoder, kernel::mlp_gate_act_mul::MlpGateActMulEncodable};pub struct DenseMlp<B: Backend> {up: Box<dyn Linear<B>>,gate: MlpGateActMulEncodable<B>,down: Box<dyn Linear<B>>,- hidden_dim: usize,- data_type: DataType,}impl<B: Backend> DenseMlp<B> {@@ -20,15 +14,11 @@ impl<B: Backend> DenseMlp<B> {up: Box<dyn Linear<B>>,gate: MlpGateActMulEncodable<B>,down: Box<dyn Linear<B>>,- hidden_dim: usize,- data_type: DataType,) -> Self {Self {up,gate,down,- hidden_dim,- data_type,}}}@@ -41,8 +31,7 @@ impl<B: Backend> Mlp<B> for DenseMlp<B> {encoder: &mut Encoder<B>,) -> Result<Allocation<B>, B::Error> {let fused_up = self.up.encode(input, batch_dim, encoder)?;- let mut hidden = encoder.allocate_scratch(size_for_shape(&[batch_dim, self.hidden_dim], self.data_type))?;- self.gate.encode(encoder, &fused_up, &mut hidden, batch_dim as i32)?;+ let hidden = self.gate.encode(encoder, &fused_up, batch_dim)?;self.down.encode(hidden, batch_dim, encoder)}}diff --git a/crates/backend-uzu/src/encodable_block/mlp/mod.rs b/crates/backend-uzu/src/encodable_block/mlp/mod.rsindex cc86e0b21..464f58a20 100644--- a/crates/backend-uzu/src/encodable_block/mlp/mod.rs+++ b/crates/backend-uzu/src/encodable_block/mlp/mod.rs@@ -71,10 +71,7 @@ impl<B: Backend> dyn Mlp<B> {).map_err(MlpBlockError::BackendError)?;- Ok((- Box::new(DenseMlp::new(up_projection, gate, down_projection, hidden_dimension, data_type)),- up_input_hadamard_factors,- ))+ Ok((Box::new(DenseMlp::new(up_projection, gate, down_projection)), up_input_hadamard_factors))},MLPConfig::MixtureOfExperts(mixture_of_experts_config) => Ok((Box::new(MoeBlock::new(context, mixture_of_experts_config, model_dimension, parameter_tree)?),diff --git a/crates/backend-uzu/src/encodable_block/mlp/moe.rs b/crates/backend-uzu/src/encodable_block/mlp/moe.rsindex f206c9910..a12621c8e 100644--- a/crates/backend-uzu/src/encodable_block/mlp/moe.rs+++ b/crates/backend-uzu/src/encodable_block/mlp/moe.rs@@ -173,7 +173,6 @@ impl<B: Backend> Mlp<B> for MoeBlock<B> {let e = self.moe_config.num_routed_experts;let k = self.moe_config.num_active_routed_experts;- let k_tile = 128;let total_rows = suffix_length * k;let num_blocks = suffix_length.div_ceil(256).max(1);let num_tiles = e.div_ceil(512).max(1);@@ -264,24 +263,18 @@ impl<B: Backend> Mlp<B> for MoeBlock<B> {encoder,);- let mut x_perm = encoder.allocate_scratch(size_for_shape(&[total_rows, self.model_dim], self.data_type))?;- if suffix_length > 0 && k > 0 {- encoder.encode_fill(&mut x_perm, 0);- }-- self.gather_kernels.encode(+ let x_perm = self.gather_kernels.encode(encoder,self.data_type,MoeGatherArguments {x: &input,bucketed_ids: &bucketed_ids,- x_perm: &mut x_perm,sumk: &sumk,t: suffix_length,k,d_model: self.model_dim,},- );+ )?;let gating_code = Self::gating_code_from_activation(&self.moe_config.expert_config.activation.act_type());@@ -293,38 +286,20 @@ impl<B: Backend> Mlp<B> for MoeBlock<B> {let up_clip_max = up_hi.unwrap_or(f32::INFINITY);let silu_alpha = self.moe_config.expert_config.activation.alpha();- let mut row_expert_map = encoder.allocate_scratch(size_for_shape(&[total_rows], DataType::U32))?;- let mut tile_counts = encoder.allocate_scratch(size_for_shape(&[e], DataType::U32))?;- let mut tile_offsets = encoder.allocate_scratch(size_for_shape(&[e + 1], DataType::U32))?;- let h_blocks = self.hidden_dim.div_ceil(4);- let mut tile_map =- encoder.allocate_scratch(size_for_shape(&[total_rows * h_blocks.max(1) * 3], DataType::U32))?;- let mut total_tiles = encoder.allocate_scratch(size_for_shape(&[8], DataType::U32))?;- let mut dispatch_args = encoder.allocate_scratch(size_for_shape(&[3], DataType::U32))?;-let y_partial = if suffix_length == 1 {- let num_tiles_k = ((self.hidden_dim + k_tile - 1) / k_tile) as u32;-self.experts_two_pass_decode_kernel.encode(encoder,MoeExpertsTwoPassArguments {x_perm: &x_perm,expert_offsets: &offsets,- row_expert_map: &mut row_expert_map,w13_all: &self.shared_weights.w13,w2_all: &self.shared_weights.w2,up_biases: &self.shared_weights.up_biases,down_biases: &self.shared_weights.down_biases,- tile_counts: &mut tile_counts,- tile_offsets: &mut tile_offsets,- tile_map: &mut tile_map,- total_tiles: &mut total_tiles,- dispatch_args: &mut dispatch_args,total_rows,d_model: self.model_dim,d_ff: self.hidden_dim,e,- num_tiles_k,gating_code,gate_clip_min,gate_clip_max,@@ -335,25 +310,17 @@ impl<B: Backend> Mlp<B> for MoeBlock<B> {},)?} else {- let num_tiles_k = ((self.hidden_dim + k_tile - 1) / k_tile) as u32;let args = MoeExpertsTwoPassArguments {x_perm: &x_perm,expert_offsets: &offsets,- row_expert_map: &mut row_expert_map,w13_all: &self.shared_weights.w13,w2_all: &self.shared_weights.w2,up_biases: &self.shared_weights.up_biases,down_biases: &self.shared_weights.down_biases,- tile_counts: &mut tile_counts,- tile_offsets: &mut tile_offsets,- tile_map: &mut tile_map,- total_tiles: &mut total_tiles,- dispatch_args: &mut dispatch_args,total_rows,d_model: self.model_dim,d_ff: self.hidden_dim,e,- num_tiles_k,gating_code,gate_clip_min,gate_clip_max,diff --git a/crates/backend-uzu/tests/performance/encodable_block/moe/moe_experts_perf_test.rs b/crates/backend-uzu/tests/performance/encodable_block/moe/moe_experts_perf_test.rsindex 47a50d96e..544285f2a 100644--- a/crates/backend-uzu/tests/performance/encodable_block/moe/moe_experts_perf_test.rs+++ b/crates/backend-uzu/tests/performance/encodable_block/moe/moe_experts_perf_test.rs@@ -13,7 +13,7 @@ use backend_uzu::{use half::bf16;use rand::{RngExt, SeedableRng, rngs::StdRng};-use crate::common::helpers::{alloc_allocation, alloc_allocation_with_data, create_context};+use crate::common::helpers::{alloc_allocation_with_data, create_context};fn build_offsets(e: usize,@@ -44,8 +44,6 @@ fn run_decode_case<B: Backend>(warmup: usize,iters: usize,) {- const K_TILE: usize = 64;-let mut rng = StdRng::seed_from_u64(0xDEC0DE1234567890);let sum_k = t * k;@@ -84,17 +82,6 @@ fn run_decode_case<B: Backend>(let up_biases_buf = alloc_allocation_with_data::<B, bf16>(&ctx, &up_biases);let down_biases_buf = alloc_allocation_with_data::<B, bf16>(&ctx, &down_biases);- let num_tiles_k = d_ff.div_ceil(K_TILE);-- // Buffers for indirect dispatch- let mut tile_counts_buf = alloc_allocation::<B, u32>(&ctx, e);- let mut tile_offsets_buf = alloc_allocation::<B, u32>(&ctx, e + 1);- let max_tiles = e * sum_k * 16384; // Conservative upper bound- let mut tile_map_buf = alloc_allocation::<B, u32>(&ctx, max_tiles * 3);- let mut total_tiles_buf = alloc_allocation::<B, u32>(&ctx, 1);- let mut dispatch_args_buf = alloc_allocation::<B, u32>(&ctx, 3);- let mut row_expert_map_buf = alloc_allocation::<B, u32>(&ctx, sum_k);-for _ in 0..warmup {let mut encoder = Encoder::new(ctx).expect("Failed to create encoder");let output = experts_kernel@@ -103,21 +90,14 @@ fn run_decode_case<B: Backend>(MoeExpertsTwoPassArguments {x_perm: &x_perm_buf,expert_offsets: &offsets_buf,- row_expert_map: &mut row_expert_map_buf,w13_all: &w13_buf,w2_all: &w2_buf,up_biases: &up_biases_buf,down_biases: &down_biases_buf,- tile_counts: &mut tile_counts_buf,- tile_offsets: &mut tile_offsets_buf,- tile_map: &mut tile_map_buf,- total_tiles: &mut total_tiles_buf,- dispatch_args: &mut dispatch_args_buf,total_rows: sum_k,d_model,d_ff,e,- num_tiles_k: num_tiles_k as u32,gating_code: 2,gate_clip_min: f32::NEG_INFINITY,gate_clip_max: f32::INFINITY,@@ -128,8 +108,9 @@ fn run_decode_case<B: Backend>(},).expect("failed to encode MoE experts");- encoder.end_encoding().submit().wait_until_completed().unwrap();+ let completed = encoder.end_encoding().submit().wait_until_completed().unwrap();drop(output);+ drop(completed);}let mut times = Vec::with_capacity(iters);@@ -142,21 +123,14 @@ fn run_decode_case<B: Backend>(MoeExpertsTwoPassArguments {x_perm: &x_perm_buf,expert_offsets: &offsets_buf,- row_expert_map: &mut row_expert_map_buf,w13_all: &w13_buf,w2_all: &w2_buf,up_biases: &up_biases_buf,down_biases: &down_biases_buf,- tile_counts: &mut tile_counts_buf,- tile_offsets: &mut tile_offsets_buf,- tile_map: &mut tile_map_buf,- total_tiles: &mut total_tiles_buf,- dispatch_args: &mut dispatch_args_buf,total_rows: sum_k,d_model,d_ff,e,- num_tiles_k: num_tiles_k as u32,gating_code: 2,gate_clip_min: f32::NEG_INFINITY,gate_clip_max: f32::INFINITY,@@ -167,8 +141,9 @@ fn run_decode_case<B: Backend>(},).expect("failed to encode MoE experts");- encoder.end_encoding().submit().wait_until_completed().unwrap();+ let completed = encoder.end_encoding().submit().wait_until_completed().unwrap();drop(output);+ drop(completed);times.push(start.elapsed().as_secs_f64() * 1000.0);}@@ -195,8 +170,6 @@ fn run_two_pass_prefill_case<B: Backend>(warmup: usize,iters: usize,) {- const K_TILE: usize = 64;-let mut rng = StdRng::seed_from_u64(0xDEC0DE1234567890);let sum_k = t * k;@@ -233,36 +206,19 @@ fn run_two_pass_prefill_case<B: Backend>(let up_biases_buf = alloc_allocation_with_data::<B, bf16>(&ctx, &up_biases);let down_biases_buf = alloc_allocation_with_data::<B, bf16>(&ctx, &down_biases);- let num_tiles_k = d_ff.div_ceil(K_TILE);-- let mut tile_counts_buf = alloc_allocation::<B, u32>(&ctx, e);- let mut tile_offsets_buf = alloc_allocation::<B, u32>(&ctx, e + 1);- let max_tiles = e * sum_k * 16384;- let mut tile_map_buf = alloc_allocation::<B, u32>(&ctx, max_tiles * 3);- let mut total_tiles_buf = alloc_allocation::<B, u32>(&ctx, 1);- let mut dispatch_args_buf = alloc_allocation::<B, u32>(&ctx, 3);- let mut row_expert_map_buf = alloc_allocation::<B, u32>(&ctx, sum_k);-for _ in 0..warmup {let mut encoder = Encoder::new(ctx).expect("Failed to create encoder");let args = MoeExpertsTwoPassArguments {x_perm: &x_perm_buf,expert_offsets: &offsets_buf,- row_expert_map: &mut row_expert_map_buf,w13_all: &w13_buf,w2_all: &w2_buf,up_biases: &up_biases_buf,down_biases: &down_biases_buf,- tile_counts: &mut tile_counts_buf,- tile_offsets: &mut tile_offsets_buf,- tile_map: &mut tile_map_buf,- total_tiles: &mut total_tiles_buf,- dispatch_args: &mut dispatch_args_buf,total_rows: sum_k,d_model,d_ff,e,- num_tiles_k: num_tiles_k as u32,gating_code: 2,gate_clip_min: f32::NEG_INFINITY,gate_clip_max: f32::INFINITY,@@ -272,8 +228,9 @@ fn run_two_pass_prefill_case<B: Backend>(data_type: DataType::BF16,};let output = experts_kernel.encode(&mut encoder, args).expect("failed to encode MoE experts");- encoder.end_encoding().submit().wait_until_completed().unwrap();+ let completed = encoder.end_encoding().submit().wait_until_completed().unwrap();drop(output);+ drop(completed);}let mut times = Vec::with_capacity(iters);@@ -283,21 +240,14 @@ fn run_two_pass_prefill_case<B: Backend>(let args = MoeExpertsTwoPassArguments {x_perm: &x_perm_buf,expert_offsets: &offsets_buf,- row_expert_map: &mut row_expert_map_buf,w13_all: &w13_buf,w2_all: &w2_buf,up_biases: &up_biases_buf,down_biases: &down_biases_buf,- tile_counts: &mut tile_counts_buf,- tile_offsets: &mut tile_offsets_buf,- tile_map: &mut tile_map_buf,- total_tiles: &mut total_tiles_buf,- dispatch_args: &mut dispatch_args_buf,total_rows: sum_k,d_model,d_ff,e,- num_tiles_k: num_tiles_k as u32,gating_code: 2,gate_clip_min: f32::NEG_INFINITY,gate_clip_max: f32::INFINITY,@@ -307,8 +257,9 @@ fn run_two_pass_prefill_case<B: Backend>(data_type: DataType::BF16,};let output = experts_kernel.encode(&mut encoder, args).expect("failed to encode MoE experts");- encoder.end_encoding().submit().wait_until_completed().unwrap();+ let completed = encoder.end_encoding().submit().wait_until_completed().unwrap();drop(output);+ drop(completed);times.push(start.elapsed().as_secs_f64() * 1000.0);}@@ -389,8 +340,9 @@ fn run_fused_single_token_case<B: Backend>(},).expect("failed to encode MoE experts");- encoder.end_encoding().submit().wait_until_completed().unwrap();+ let completed = encoder.end_encoding().submit().wait_until_completed().unwrap();drop(output);+ drop(completed);}let mut times = Vec::with_capacity(iters);@@ -421,8 +373,9 @@ fn run_fused_single_token_case<B: Backend>(},).expect("failed to encode MoE experts");- encoder.end_encoding().submit().wait_until_completed().unwrap();+ let completed = encoder.end_encoding().submit().wait_until_completed().unwrap();drop(output);+ drop(completed);times.push(start.elapsed().as_secs_f64() * 1000.0);}@@ -448,8 +401,6 @@ fn run_indirect_decode_timed<B: Backend>(warmup: usize,iters: usize,) -> f64 {- const K_TILE: usize = 64;-let mut rng = StdRng::seed_from_u64(0xDEC0DE1234567890);let sum_k = t * k;@@ -484,15 +435,6 @@ fn run_indirect_decode_timed<B: Backend>(let up_biases_buf = alloc_allocation_with_data::<B, bf16>(ctx, &up_biases);let down_biases_buf = alloc_allocation_with_data::<B, bf16>(ctx, &down_biases);- let num_tiles_k = d_ff.div_ceil(K_TILE);- let mut tile_counts_buf = alloc_allocation::<B, u32>(ctx, e);- let mut tile_offsets_buf = alloc_allocation::<B, u32>(ctx, e + 1);- let max_tiles = e * sum_k * 16384;- let mut tile_map_buf = alloc_allocation::<B, u32>(ctx, max_tiles * 3);- let mut total_tiles_buf = alloc_allocation::<B, u32>(ctx, 1);- let mut dispatch_args_buf = alloc_allocation::<B, u32>(ctx, 3);- let mut row_expert_map_buf = alloc_allocation::<B, u32>(ctx, sum_k);-for _ in 0..warmup {let mut encoder = Encoder::new(ctx).expect("Failed to create encoder");let output = experts_kernel@@ -501,21 +443,14 @@ fn run_indirect_decode_timed<B: Backend>(MoeExpertsTwoPassArguments {x_perm: &x_perm_buf,expert_offsets: &offsets_buf,- row_expert_map: &mut row_expert_map_buf,w13_all: &w13_buf,w2_all: &w2_buf,up_biases: &up_biases_buf,down_biases: &down_biases_buf,- tile_counts: &mut tile_counts_buf,- tile_offsets: &mut tile_offsets_buf,- tile_map: &mut tile_map_buf,- total_tiles: &mut total_tiles_buf,- dispatch_args: &mut dispatch_args_buf,total_rows: sum_k,d_model,d_ff,e,- num_tiles_k: num_tiles_k as u32,gating_code: 2,gate_clip_min: f32::NEG_INFINITY,gate_clip_max: f32::INFINITY,@@ -526,8 +461,9 @@ fn run_indirect_decode_timed<B: Backend>(},).expect("failed to encode MoE experts");- encoder.end_encoding().submit().wait_until_completed().unwrap();+ let completed = encoder.end_encoding().submit().wait_until_completed().unwrap();drop(output);+ drop(completed);}let mut times = Vec::with_capacity(iters);@@ -540,21 +476,14 @@ fn run_indirect_decode_timed<B: Backend>(MoeExpertsTwoPassArguments {x_perm: &x_perm_buf,expert_offsets: &offsets_buf,- row_expert_map: &mut row_expert_map_buf,w13_all: &w13_buf,w2_all: &w2_buf,up_biases: &up_biases_buf,down_biases: &down_biases_buf,- tile_counts: &mut tile_counts_buf,- tile_offsets: &mut tile_offsets_buf,- tile_map: &mut tile_map_buf,- total_tiles: &mut total_tiles_buf,- dispatch_args: &mut dispatch_args_buf,total_rows: sum_k,d_model,d_ff,e,- num_tiles_k: num_tiles_k as u32,gating_code: 2,gate_clip_min: f32::NEG_INFINITY,gate_clip_max: f32::INFINITY,@@ -565,8 +494,9 @@ fn run_indirect_decode_timed<B: Backend>(},).expect("failed to encode MoE experts");- encoder.end_encoding().submit().wait_until_completed().unwrap();+ let completed = encoder.end_encoding().submit().wait_until_completed().unwrap();drop(output);+ drop(completed);times.push(start.elapsed().as_secs_f64() * 1000.0);}@@ -635,8 +565,9 @@ fn run_fused_decode_timed<B: Backend>(},).expect("failed to encode MoE experts");- encoder.end_encoding().submit().wait_until_completed().unwrap();+ let completed = encoder.end_encoding().submit().wait_until_completed().unwrap();drop(output);+ drop(completed);}let mut times = Vec::with_capacity(iters);@@ -667,8 +598,9 @@ fn run_fused_decode_timed<B: Backend>(},).expect("failed to encode MoE experts");- encoder.end_encoding().submit().wait_until_completed().unwrap();+ let completed = encoder.end_encoding().submit().wait_until_completed().unwrap();drop(output);+ drop(completed);times.push(start.elapsed().as_secs_f64() * 1000.0);}diff --git a/crates/backend-uzu/tests/performance/encodable_block/moe/moe_perf_test.rs b/crates/backend-uzu/tests/performance/encodable_block/moe/moe_perf_test.rsindex 355dec81a..b6a2950c4 100644--- a/crates/backend-uzu/tests/performance/encodable_block/moe/moe_perf_test.rs+++ b/crates/backend-uzu/tests/performance/encodable_block/moe/moe_perf_test.rs@@ -162,7 +162,6 @@ fn test_moe_pipeline_breakdown_decode() {let mut offsets_buf = alloc_allocation::<B, u32>(&ctx, e + 1);let mut sumk_buf = alloc_allocation::<B, u32>(&ctx, 1);let mut bucketed_ids_buf = alloc_allocation::<B, i32>(&ctx, t * k);- let mut x_perm_buf = alloc_allocation::<B, bf16>(&ctx, t * k * d_model);let y_partial: Vec<bf16> = (0..t * k * d_model).map(|_| bf16::from_f32(rng.random_range(-1.0..1.0))).collect();let y_partial_buf = alloc_allocation_with_data::<B, bf16>(&ctx, &y_partial);let mut tok2row_buf = alloc_allocation::<B, i32>(&ctx, t * k);@@ -196,18 +195,7 @@ fn test_moe_pipeline_breakdown_decode() {let up_biases_buf = alloc_allocation_with_data::<B, bf16>(&ctx, &up_biases);let down_biases_buf = alloc_allocation_with_data::<B, bf16>(&ctx, &down_biases);- // Experts tiling buffers for two-pass- const K_TILE: usize = 64;let sum_k = t * k;- let num_tiles_k = (d_ff + K_TILE - 1) / K_TILE;- let max_tiles = t * k * e * num_tiles_k;- let mut tile_counts_buf = alloc_allocation::<B, u32>(&ctx, e);- let mut tile_offsets_buf = alloc_allocation::<B, u32>(&ctx, e + 1);- let mut tile_map_buf = alloc_allocation::<B, u32>(&ctx, max_tiles * 3);- let mut total_tiles_buf = alloc_allocation::<B, u32>(&ctx, 1);- let mut dispatch_args_buf = alloc_allocation::<B, u32>(&ctx, 3);-- let mut row_expert_map_buf = alloc_allocation::<B, u32>(&ctx, sum_k);// Scatter block bases bufferslet mut block_bases_buf = alloc_allocation::<B, u32>(&ctx, num_blocks * num_tiles);@@ -297,22 +285,45 @@ fn test_moe_pipeline_breakdown_decode() {let gather_perf = run_perf_with_warmup("Gather", 2, 5, || {let mut encoder = Encoder::new(ctx.as_ref()).expect("Failed to create encoder");- gather_kernel.encode(- &mut encoder,- DataType::BF16,- MoeGatherArguments {- x: &x_buf,- bucketed_ids: &bucketed_ids_buf,- x_perm: &mut x_perm_buf,- sumk: &sumk_buf,- t,- k,- d_model,- },- );- encoder.end_encoding().submit().wait_until_completed().unwrap();+ let x_perm = gather_kernel+ .encode(+ &mut encoder,+ DataType::BF16,+ MoeGatherArguments {+ x: &x_buf,+ bucketed_ids: &bucketed_ids_buf,+ sumk: &sumk_buf,+ t,+ k,+ d_model,+ },+ )+ .expect("failed to encode MoE gather");+ let completed = encoder.end_encoding().submit().wait_until_completed().unwrap();+ drop(x_perm);+ drop(completed);});+ let (x_perm_buf, x_perm_completed) = {+ let mut encoder = Encoder::new(ctx.as_ref()).expect("Failed to create encoder");+ let x_perm = gather_kernel+ .encode(+ &mut encoder,+ DataType::BF16,+ MoeGatherArguments {+ x: &x_buf,+ bucketed_ids: &bucketed_ids_buf,+ sumk: &sumk_buf,+ t,+ k,+ d_model,+ },+ )+ .expect("failed to encode MoE gather");+ let completed = encoder.end_encoding().submit().wait_until_completed().unwrap();+ (x_perm, completed)+ };+let experts_perf = run_perf_with_warmup("Experts (MAIN COMPUTE)", 2, 5, || {let mut encoder = Encoder::new(ctx.as_ref()).expect("Failed to create encoder");let output = experts_kernel@@ -321,21 +332,14 @@ fn test_moe_pipeline_breakdown_decode() {MoeExpertsTwoPassArguments {x_perm: &x_perm_buf,expert_offsets: &offsets_buf,- row_expert_map: &mut row_expert_map_buf,w13_all: &w13_buf,w2_all: &w2_buf,up_biases: &up_biases_buf,down_biases: &down_biases_buf,- tile_counts: &mut tile_counts_buf,- tile_offsets: &mut tile_offsets_buf,- tile_map: &mut tile_map_buf,- total_tiles: &mut total_tiles_buf,- dispatch_args: &mut dispatch_args_buf,total_rows: sum_k,d_model,d_ff,e,- num_tiles_k: num_tiles_k as u32,gating_code: 2, // SILUgate_clip_min: f32::NEG_INFINITY,gate_clip_max: 20.0,@@ -346,9 +350,12 @@ fn test_moe_pipeline_breakdown_decode() {},).expect("failed to encode MoE experts");- encoder.end_encoding().submit().wait_until_completed().unwrap();+ let completed = encoder.end_encoding().submit().wait_until_completed().unwrap();drop(output);+ drop(completed);});+ drop(x_perm_buf);+ drop(x_perm_completed);let finalize_perf = run_perf_with_warmup("Finalize", 2, 5, || {let mut encoder = Encoder::new(ctx.as_ref()).expect("Failed to create encoder");diff --git a/crates/backend-uzu/tests/unit/backends/common/kernel/ssd_prefill_test.rs b/crates/backend-uzu/tests/unit/backends/common/kernel/ssd_prefill_test.rsindex 85ad712af..e1d9b6b9a 100644--- a/crates/backend-uzu/tests/unit/backends/common/kernel/ssd_prefill_test.rs+++ b/crates/backend-uzu/tests/unit/backends/common/kernel/ssd_prefill_test.rs@@ -85,7 +85,6 @@ struct SSDPrefillFixture {head_dim: usize,state_dim: usize,group_size: i32,- total_x: usize,x_strides: [usize; 3],dt_strides: [usize; 2],cb_strides: [usize; 3],@@ -132,7 +131,6 @@ impl SSDPrefillFixture {head_dim,state_dim,group_size,- total_x,x_strides,dt_strides,cb_strides,@@ -161,7 +159,6 @@ fn run_prefill_kernel_mode<B: Backend>(let d_buf = allocation_from_slice::<B, _>(ctx, &fixture.d_data);let z_buf = allocation_from_slice::<B, _>(ctx, &fixture.z_data);let mut state_buf = allocation_from_slice::<B, _>(ctx, &fixture.state_init);- let mut y_buf = allocation_from_slice::<B, _>(ctx, &vec![0f32; fixture.total_x]);let args = SSDPrefillArguments {x: &x_buf,@@ -171,7 +168,6 @@ fn run_prefill_kernel_mode<B: Backend>(d: &d_buf,z: &z_buf,state: &mut state_buf,- y: &mut y_buf,suffix_len: fixture.suffix_len,group_size: fixture.group_size as u32,state_size: fixture.state_dim as u32,@@ -184,11 +180,13 @@ fn run_prefill_kernel_mode<B: Backend>(};let mut encoder = Encoder::new(ctx).unwrap();- kernel.encode(&mut encoder, args, mode);- encoder.end_encoding().submit().wait_until_completed().unwrap();+ let y_buf = kernel.encode(&mut encoder, args, mode).expect("Failed to encode SSD prefill");+ let completed = encoder.end_encoding().submit().wait_until_completed().unwrap();let y_vec = allocation_to_vec::<B, f32>(&y_buf);let state_vec = allocation_to_vec::<B, f32>(&state_buf);+ drop(y_buf);+ drop(completed);(y_vec, state_vec, None)}diff --git a/crates/backend-uzu/tests/unit/encodable_block/moe/moe_block_e2e_test.rs b/crates/backend-uzu/tests/unit/encodable_block/moe/moe_block_e2e_test.rsindex a5e4c8073..64cc0747e 100644--- a/crates/backend-uzu/tests/unit/encodable_block/moe/moe_block_e2e_test.rs+++ b/crates/backend-uzu/tests/unit/encodable_block/moe/moe_block_e2e_test.rs@@ -307,16 +307,7 @@ fn run_moe_parity_test_internal<B: Backend>(let mut bucketed_ids_buf = alloc_allocation::<B, i32>(&ctx, max_sumk);let mut bucketed_probs_buf = alloc_allocation::<B, bf16>(&ctx, max_sumk);let mut tok2row_buf = alloc_allocation::<B, i32>(&ctx, t * k);- let mut x_perm_buf = alloc_allocation::<B, bf16>(&ctx, max_sumk * d_model);let mut y_out_buf = alloc_allocation::<B, bf16>(&ctx, t * d_model);- const BLOCK_M_DECODE: usize = 4; // matches two-pass decode kernel configuration- let h_blocks_decode = (d_ff + BLOCK_M_DECODE - 1) / BLOCK_M_DECODE;- let max_tiles = max_sumk * h_blocks_decode;- let mut tile_counts_buf = alloc_allocation::<B, u32>(&ctx, e);- let mut tile_offsets_buf = alloc_allocation::<B, u32>(&ctx, e + 1);- let mut tile_map_buf = alloc_allocation::<B, u32>(&ctx, max_tiles * 3);- let mut total_tiles_buf = alloc_allocation::<B, u32>(&ctx, 8);- let mut dispatch_args_buf = alloc_allocation::<B, u32>(&ctx, 3);// Encode ALL kernels in one command buffereprintln!("[E2E] Encoding entire MoE pipeline in single command buffer...");@@ -383,43 +374,35 @@ fn run_moe_parity_test_internal<B: Backend>();let gather = MoeGatherKernels::<B>::new(&ctx).expect("gather");- gather.encode(- &mut encoder,- DataType::BF16,- MoeGatherArguments {- x: &x_buf,- bucketed_ids: &bucketed_ids_buf,- x_perm: &mut x_perm_buf,- sumk: &sumk_buf,- t,- k,- d_model,- },- );+ let x_perm_buf = gather+ .encode(+ &mut encoder,+ DataType::BF16,+ MoeGatherArguments {+ x: &x_buf,+ bucketed_ids: &bucketed_ids_buf,+ sumk: &sumk_buf,+ t,+ k,+ d_model,+ },+ )+ .expect("gather");let total_rows = t * k;- let mut row_expert_map_buf = alloc_allocation::<B, u32>(&ctx, total_rows);let experts = MoeExpertsTwoPassPrefillBlock::<B>::new(&ctx).expect("experts");- let num_tiles_k = ((d_ff + 64 - 1) / 64) as u32;let args = MoeExpertsTwoPassArguments {x_perm: &x_perm_buf,expert_offsets: &offsets_buf,- row_expert_map: &mut row_expert_map_buf,w13_all: &w13_buf,w2_all: &w2_buf,up_biases: &up_biases_buf,down_biases: &down_biases_buf,- tile_counts: &mut tile_counts_buf,- tile_offsets: &mut tile_offsets_buf,- tile_map: &mut tile_map_buf,- total_tiles: &mut total_tiles_buf,- dispatch_args: &mut dispatch_args_buf,total_rows,d_model,d_ff,e,- num_tiles_k,gating_code,gate_clip_min: gate_clip.0,gate_clip_max: gate_clip.1,@@ -471,18 +454,8 @@ fn run_moe_parity_test_internal<B: Backend>(f32::from(w2_cpu[0]));- // Probe tile bookkeeping- let total_tiles_cpu = allocation_prefix_to_vec::<B, u32>(&total_tiles_buf, 2);- let dispatch_args_cpu = allocation_prefix_to_vec::<B, u32>(&dispatch_args_buf, 3);- let tile_offsets_cpu = allocation_prefix_to_vec::<B, u32>(&tile_offsets_buf, (e + 1).min(8));let sumk_val = allocation_to_vec::<B, u32>(&sumk_buf)[0] as usize;- eprintln!("[E2E] Tile bookkeeping:");- eprintln!(- "[E2E] total_tiles={}, dispatch_args=({}, {}, {})",- total_tiles_cpu[0], dispatch_args_cpu[0], dispatch_args_cpu[1], dispatch_args_cpu[2]- );- eprintln!("[E2E] tile_offsets[0..{}]={:?}", (e + 1).min(8), &tile_offsets_cpu);- eprintln!("[E2E] sumk={}, num_tiles_k={}", sumk_val, num_tiles_k);+ eprintln!("[E2E] sumk={}", sumk_val);// For multi-token tests with large d_ff, verify gather output (x_perm)if t > 1 && d_ff >= 256 {@@ -498,10 +471,6 @@ fn run_moe_parity_test_internal<B: Backend>();}- // Check tile_map for first few tiles- let tile_map_cpu = allocation_prefix_to_vec::<B, u32>(&tile_map_buf, 12.min(max_tiles * 3));- eprintln!("[E2E] tile_map (first 4 tiles): {:?}", &tile_map_cpu);-// CRITICAL: Check tok2row mapping for multi-token testslet tok2row_cpu = allocation_prefix_to_vec::<B, i32>(&tok2row_buf, t * k);eprintln!("[E2E] tok2row[0..{}]: {:?}", t * k, &tok2row_cpu);@@ -669,6 +638,7 @@ fn run_moe_parity_test_internal<B: Backend>(}eprintln!("[{}] ✓ PASSED (GPU matches CPU reference)", test_name);+ drop(x_perm_buf);drop(y_partial_buf);drop(completed);}diff --git a/crates/backend-uzu/tests/unit/encodable_block/moe/moe_experts_test.rs b/crates/backend-uzu/tests/unit/encodable_block/moe/moe_experts_test.rsindex 41dd6cfec..296a005fc 100644--- a/crates/backend-uzu/tests/unit/encodable_block/moe/moe_experts_test.rs+++ b/crates/backend-uzu/tests/unit/encodable_block/moe/moe_experts_test.rs@@ -23,7 +23,7 @@ use rand::{RngExt, SeedableRng, rngs::StdRng};use crate::encodable_block::mlp::moe::tests::{common::{assert::assert_eq_float,- helpers::{alloc_allocation, alloc_allocation_with_data, allocation_prefix_to_vec, create_context},+ helpers::{alloc_allocation_with_data, allocation_prefix_to_vec, create_context},},cpu_tile_counts, cpu_tile_scan,};@@ -82,7 +82,6 @@ impl MoeTestData {struct ScatterResult {x_perm: Vec<bf16>,offsets: Vec<u32>,- row_expert_map: Vec<u32>,perm_idx: Vec<usize>, // original row -> bucketed row (for gather)}@@ -118,22 +117,19 @@ fn scatter_by_expert(expert_cursors[eid] += 1;}- // Build x_perm and row_expert_map in bucketed order+ // Build x_perm in bucketed orderlet mut x_perm = vec![bf16::from_f32(0.0); sum_k * d_model];- let mut row_expert_map = vec![0u32; sum_k];for bucket_row in 0..sum_k {let orig_row = inv_perm[bucket_row];let tok = orig_row / k;for d in 0..d_model {x_perm[bucket_row * d_model + d] = x[tok * d_model + d];}- row_expert_map[bucket_row] = topk_ids[orig_row] as u32;}ScatterResult {x_perm,offsets,- row_expert_map,perm_idx,}}@@ -330,8 +326,6 @@ fn test_two_pass_decode_correctness() {}}- // Build offsets and row_expert_map based on topk_ids- // First count rows per expertlet mut expert_counts = vec![0usize; e];for &eid in &topk_ids {expert_counts[eid as usize] += 1;@@ -341,14 +335,6 @@ fn test_two_pass_decode_correctness() {offsets[i + 1] = offsets[i] + expert_counts[i] as u32;}- // Build row_expert_map (maps permuted row to expert)- let mut row_expert_map = vec![0u32; sum_k];- let mut expert_row_idx = vec![0usize; e];- for (row, &eid) in topk_ids.iter().enumerate() {- row_expert_map[row] = eid as u32;- expert_row_idx[eid as usize] += 1;- }-// Generate weightslet w13: Vec<bf16> =(0..e * 2 * d_ff * d_model).map(|_| bf16::from_f32(rng.random_range(-0.05..0.05))).collect();@@ -380,50 +366,28 @@ fn test_two_pass_decode_correctness() {// Prepare GPU bufferslet x_perm_buf = alloc_allocation_with_data::<B, bf16>(&ctx, &x_perm);let offsets_buf = alloc_allocation_with_data::<B, u32>(&ctx, &offsets);- let mut row_expert_map_buf = alloc_allocation_with_data::<B, u32>(&ctx, &row_expert_map);let w13_buf = alloc_allocation_with_data::<B, bf16>(&ctx, &w13);let w2_buf = alloc_allocation_with_data::<B, bf16>(&ctx, &w2);let up_biases_buf = alloc_allocation_with_data::<B, bf16>(&ctx, &up_biases);let down_biases_buf = alloc_allocation_with_data::<B, bf16>(&ctx, &down_biases);- // Tile infrastructure- let h_blocks_decode = (d_ff + 3) / 4;- let max_total_tiles = sum_k * h_blocks_decode;-- let mut tile_counts_buf = alloc_allocation::<B, u32>(&ctx, e);- let mut tile_offsets_buf = alloc_allocation::<B, u32>(&ctx, e + 1);- let mut total_tiles_buf = alloc_allocation::<B, u32>(&ctx, 8);- let mut tile_map_buf = alloc_allocation::<B, u32>(&ctx, max_total_tiles * 3);- let mut dispatch_args_buf = alloc_allocation::<B, u32>(&ctx, 3);-- // Execute 2-pass decode kernellet experts_kernel = MoeExpertsTwoPassDecodeBlock::<B>::new(&ctx).expect("MoeExpertsTwoPassDecodeKernel::new");let mut encoder = Encoder::new(ctx.as_ref()).expect("Failed to create encoder");- const K_TILE: usize = 64;- let num_tiles_k = ((d_ff + K_TILE - 1) / K_TILE) as u32;-let y_partial_buf = experts_kernel.encode(&mut encoder,MoeExpertsTwoPassArguments {x_perm: &x_perm_buf,expert_offsets: &offsets_buf,- row_expert_map: &mut row_expert_map_buf,w13_all: &w13_buf,w2_all: &w2_buf,up_biases: &up_biases_buf,down_biases: &down_biases_buf,- tile_counts: &mut tile_counts_buf,- tile_offsets: &mut tile_offsets_buf,- tile_map: &mut tile_map_buf,- total_tiles: &mut total_tiles_buf,- dispatch_args: &mut dispatch_args_buf,total_rows: sum_k,d_model,d_ff,e,- num_tiles_k,gating_code,gate_clip_min: f32::NEG_INFINITY,gate_clip_max: f32::INFINITY,@@ -527,18 +491,10 @@ fn test_two_pass_decode_multi_token() {// GPU bufferslet x_perm_buf = alloc_allocation_with_data::<B, bf16>(&ctx, &scatter.x_perm);let offsets_buf = alloc_allocation_with_data::<B, u32>(&ctx, &scatter.offsets);- let mut row_expert_map_buf = alloc_allocation_with_data::<B, u32>(&ctx, &scatter.row_expert_map);let w13_buf = alloc_allocation_with_data::<B, bf16>(&ctx, &data.w13);let w2_buf = alloc_allocation_with_data::<B, bf16>(&ctx, &data.w2);let up_biases_buf = alloc_allocation_with_data::<B, bf16>(&ctx, &data.up_biases);let down_biases_buf = alloc_allocation_with_data::<B, bf16>(&ctx, &data.down_biases);- // Tile infrastructure- let max_total_tiles = sum_k * ((d_ff + 3) / 4);- let mut tile_counts_buf = alloc_allocation::<B, u32>(&ctx, e);- let mut tile_offsets_buf = alloc_allocation::<B, u32>(&ctx, e + 1);- let mut total_tiles_buf = alloc_allocation::<B, u32>(&ctx, 8);- let mut tile_map_buf = alloc_allocation::<B, u32>(&ctx, max_total_tiles * 3);- let mut dispatch_args_buf = alloc_allocation::<B, u32>(&ctx, 3);let experts_kernel = MoeExpertsTwoPassDecodeBlock::<B>::new(&ctx).expect("kernel");let mut encoder = Encoder::new(ctx.as_ref()).expect("Failed to create encoder");@@ -548,21 +504,14 @@ fn test_two_pass_decode_multi_token() {MoeExpertsTwoPassArguments {x_perm: &x_perm_buf,expert_offsets: &offsets_buf,- row_expert_map: &mut row_expert_map_buf,w13_all: &w13_buf,w2_all: &w2_buf,up_biases: &up_biases_buf,down_biases: &down_biases_buf,- tile_counts: &mut tile_counts_buf,- tile_offsets: &mut tile_offsets_buf,- tile_map: &mut tile_map_buf,- total_tiles: &mut total_tiles_buf,- dispatch_args: &mut dispatch_args_buf,total_rows: sum_k,d_model,d_ff,e,- num_tiles_k: ((d_ff + 63) / 64) as u32,gating_code,gate_clip_min: f32::NEG_INFINITY,gate_clip_max: f32::INFINITY,@@ -628,39 +577,24 @@ fn test_two_pass_prefill_correctness() {// GPU bufferslet x_perm_buf = alloc_allocation_with_data::<B, bf16>(&ctx, &scatter.x_perm);let offsets_buf = alloc_allocation_with_data::<B, u32>(&ctx, &scatter.offsets);- let mut row_expert_map_buf = alloc_allocation_with_data::<B, u32>(&ctx, &scatter.row_expert_map);let w13_buf = alloc_allocation_with_data::<B, bf16>(&ctx, &data.w13);let w2_buf = alloc_allocation_with_data::<B, bf16>(&ctx, &data.w2);let up_biases_buf = alloc_allocation_with_data::<B, bf16>(&ctx, &data.up_biases);let down_biases_buf = alloc_allocation_with_data::<B, bf16>(&ctx, &data.down_biases);- // Tile infrastructure- let max_total_tiles = sum_k * ((d_ff + 3) / 4);- let mut tile_counts_buf = alloc_allocation::<B, u32>(&ctx, e);- let mut tile_offsets_buf = alloc_allocation::<B, u32>(&ctx, e + 1);- let mut total_tiles_buf = alloc_allocation::<B, u32>(&ctx, 8);- let mut tile_map_buf = alloc_allocation::<B, u32>(&ctx, max_total_tiles * 3);- let mut dispatch_args_buf = alloc_allocation::<B, u32>(&ctx, 3);let experts_kernel = MoeExpertsTwoPassPrefillBlock::<B>::new(&ctx).expect("kernel");let mut encoder = Encoder::new(ctx.as_ref()).expect("Failed to create encoder");let args = MoeExpertsTwoPassArguments {x_perm: &x_perm_buf,expert_offsets: &offsets_buf,- row_expert_map: &mut row_expert_map_buf,w13_all: &w13_buf,w2_all: &w2_buf,up_biases: &up_biases_buf,down_biases: &down_biases_buf,- tile_counts: &mut tile_counts_buf,- tile_offsets: &mut tile_offsets_buf,- tile_map: &mut tile_map_buf,- total_tiles: &mut total_tiles_buf,- dispatch_args: &mut dispatch_args_buf,total_rows: sum_k,d_model,d_ff,e,- num_tiles_k: ((d_ff + 63) / 64) as u32,gating_code,gate_clip_min: f32::NEG_INFINITY,gate_clip_max: f32::INFINITY,diff --git a/crates/backend-uzu/tests/unit/encodable_block/moe/moe_tiles_test.rs b/crates/backend-uzu/tests/unit/encodable_block/moe/moe_tiles_test.rsindex 85b4de671..25af38012 100644--- a/crates/backend-uzu/tests/unit/encodable_block/moe/moe_tiles_test.rs+++ b/crates/backend-uzu/tests/unit/encodable_block/moe/moe_tiles_test.rs@@ -1,13 +1,8 @@-use backend_uzu::backends::common::{- Encoder,- kernel::moe::{MoeTileCountsArguments, MoeTileMapKernels, MoeTileScanArguments},-};+use backend_uzu::backends::common::{Encoder, kernel::moe::MoeTileMapKernels};use rand::{RngExt, SeedableRng, rngs::StdRng};use crate::encodable_block::mlp::moe::tests::{- common::helpers::{- alloc_allocation, alloc_allocation_with_data, allocation_prefix_to_vec, allocation_to_vec, create_context,- },+ common::helpers::{alloc_allocation_with_data, allocation_prefix_to_vec, allocation_to_vec, create_context},cpu_tile_counts, cpu_tile_scan,};@@ -34,22 +29,17 @@ fn test_tile_counts_correctness() {// GPU bufferslet offsets_buf = alloc_allocation_with_data::<B, u32>(&ctx, &offsets);- let mut tile_counts_buf = alloc_allocation::<B, u32>(&ctx, e);// Execute kernel using kernel structlet tile_kernel = MoeTileMapKernels::<B>::new(&ctx).expect("MoeTileMapKernel::new");let mut encoder = Encoder::new(ctx.as_ref()).expect("Failed to create encoder");- tile_kernel.encode_counts(- &mut encoder,- MoeTileCountsArguments {- offsets: &offsets_buf,- tile_counts: &mut tile_counts_buf,- e,- },- );- encoder.end_encoding().submit().wait_until_completed().unwrap();+ let tile_counts_buf =+ tile_kernel.encode_counts(&mut encoder, &offsets_buf, e).expect("MoeTileMapKernels::encode_counts");+ let completed = encoder.end_encoding().submit().wait_until_completed().unwrap();let tile_counts_gpu = allocation_prefix_to_vec::<B, u32>(&tile_counts_buf, e);+ drop(tile_counts_buf);+ drop(completed);assert_eq!(tile_counts_gpu, tile_counts_cpu, "Tile counts mismatch for E={}", e);@@ -77,25 +67,18 @@ fn test_tile_scan_correctness() {// GPU bufferslet tile_counts_buf = alloc_allocation_with_data::<B, u32>(&ctx, &tile_counts);- let mut tile_offsets_buf = alloc_allocation::<B, u32>(&ctx, e + 1);- let mut total_tiles_buf = alloc_allocation::<B, u32>(&ctx, 1);// Execute kernel using kernel structlet tile_kernel = MoeTileMapKernels::<B>::new(&ctx).expect("MoeTileMapKernel::new");let mut encoder = Encoder::new(ctx.as_ref()).expect("Failed to create encoder");- tile_kernel.encode_scan(- &mut encoder,- MoeTileScanArguments {- tile_counts: &tile_counts_buf,- tile_offsets: &mut tile_offsets_buf,- total_tiles: &mut total_tiles_buf,- e,- },- );- encoder.end_encoding().submit().wait_until_completed().unwrap();-- let tile_offsets_gpu = allocation_prefix_to_vec::<B, u32>(&tile_offsets_buf, e + 1);- let total_tiles_gpu = allocation_to_vec::<B, u32>(&total_tiles_buf)[0];+ let tile_scan =+ tile_kernel.encode_scan(&mut encoder, &tile_counts_buf, e).expect("MoeTileMapKernels::encode_scan");+ let completed = encoder.end_encoding().submit().wait_until_completed().unwrap();++ let tile_offsets_gpu = allocation_prefix_to_vec::<B, u32>(&tile_scan.tile_offsets, e + 1);+ let total_tiles_gpu = allocation_to_vec::<B, u32>(&tile_scan.total_tiles)[0];+ drop(tile_scan);+ drop(completed);assert_eq!(tile_offsets_gpu, tile_offsets_cpu, "Tile offsets mismatch for E={}", e);assert_eq!(total_tiles_gpu, total_tiles_cpu, "Total tiles mismatch for E={}", e);@@ -118,21 +101,16 @@ fn test_tile_edge_cases() {let tile_counts_cpu = cpu_tile_counts(&offsets, 16);let offsets_buf = alloc_allocation_with_data::<B, u32>(&ctx, &offsets);- let mut tile_counts_buf = alloc_allocation::<B, u32>(&ctx, e);let tile_kernel = MoeTileMapKernels::<B>::new(&ctx).expect("MoeTileMapKernel::new");let mut encoder = Encoder::new(ctx.as_ref()).expect("Failed to create encoder");- tile_kernel.encode_counts(- &mut encoder,- MoeTileCountsArguments {- offsets: &offsets_buf,- tile_counts: &mut tile_counts_buf,- e,- },- );- encoder.end_encoding().submit().wait_until_completed().unwrap();+ let tile_counts_buf =+ tile_kernel.encode_counts(&mut encoder, &offsets_buf, e).expect("MoeTileMapKernels::encode_counts");+ let completed = encoder.end_encoding().submit().wait_until_completed().unwrap();let tile_counts_gpu = allocation_prefix_to_vec::<B, u32>(&tile_counts_buf, e);+ drop(tile_counts_buf);+ drop(completed);assert_eq!(tile_counts_gpu, tile_counts_cpu);assert!(tile_counts_gpu.iter().all(|&c| c == 0));@@ -147,21 +125,16 @@ fn test_tile_edge_cases() {let tile_counts_cpu = cpu_tile_counts(&offsets, 16);let offsets_buf = alloc_allocation_with_data::<B, u32>(&ctx, &offsets);- let mut tile_counts_buf = alloc_allocation::<B, u32>(&ctx, e);let tile_kernel = MoeTileMapKernels::<B>::new(&ctx).expect("MoeTileMapKernels::new");let mut encoder = Encoder::new(ctx.as_ref()).expect("Failed to create encoder");- tile_kernel.encode_counts(- &mut encoder,- MoeTileCountsArguments {- offsets: &offsets_buf,- tile_counts: &mut tile_counts_buf,- e,- },- );- encoder.end_encoding().submit().wait_until_completed().unwrap();+ let tile_counts_buf =+ tile_kernel.encode_counts(&mut encoder, &offsets_buf, e).expect("MoeTileMapKernels::encode_counts");+ let completed = encoder.end_encoding().submit().wait_until_completed().unwrap();let tile_counts_gpu = allocation_prefix_to_vec::<B, u32>(&tile_counts_buf, e);+ drop(tile_counts_buf);+ drop(completed);assert_eq!(tile_counts_gpu, tile_counts_cpu);// With BM=16: seg_lens=[10, 5, 15, 20] -> tile_counts=[1, 1, 1, 2]diff --git a/crates/backend-uzu/tests/unit/kernel/moe/moe_gather_test.rs b/crates/backend-uzu/tests/unit/kernel/moe/moe_gather_test.rsindex 493348dc4..4d2e04c6b 100644--- a/crates/backend-uzu/tests/unit/kernel/moe/moe_gather_test.rs+++ b/crates/backend-uzu/tests/unit/kernel/moe/moe_gather_test.rs@@ -15,7 +15,7 @@ use backend_uzu::{use crate::{common::{assert::assert_eq_float,- helpers::{alloc_allocation, alloc_allocation_with_data, allocation_to_vec, create_context},+ helpers::{alloc_allocation_with_data, allocation_to_vec, create_context},},uzu_test,};@@ -35,26 +35,28 @@ fn get_output<B: Backend, T: ArrayElement + Float>(input: &Input<T>) -> Vec<T> {let x_allocation = alloc_allocation_with_data::<B, T>(&context, &input.x);let ids_allocation = alloc_allocation_with_data::<B, i32>(&context, &input.bucket_ids);let sumk_allocation = alloc_allocation_with_data::<B, u32>(&context, &sumk_data);- let mut x_perm_allocation = alloc_allocation::<B, T>(&context, input.sum_k * input.d_model);-let gather = MoeGatherKernels::<B>::new(&context).expect("MoeGatherKernel::new");let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");- gather.encode(- &mut encoder,- T::data_type(),- MoeGatherArguments {- x: &x_allocation,- bucketed_ids: &ids_allocation,- x_perm: &mut x_perm_allocation,- sumk: &sumk_allocation,- t: input.t,- k: input.sum_k / input.t, // Decompose sum_k into k per token- d_model: input.d_model,- },- );- encoder.end_encoding().submit().wait_until_completed().unwrap();+ let x_perm_allocation = gather+ .encode(+ &mut encoder,+ T::data_type(),+ MoeGatherArguments {+ x: &x_allocation,+ bucketed_ids: &ids_allocation,+ sumk: &sumk_allocation,+ t: input.t,+ k: input.sum_k / input.t, // Decompose sum_k into k per token+ d_model: input.d_model,+ },+ )+ .expect("Failed to encode MoE gather");+ let completed = encoder.end_encoding().submit().wait_until_completed().unwrap();- allocation_to_vec::<B, T>(&x_perm_allocation)+ let output = allocation_to_vec::<B, T>(&x_perm_allocation);+ drop(x_perm_allocation);+ drop(completed);+ output}fn test_gather_internal(