A high-performance inference engine for AI models
Burn MoE (#886)
uuuvn3a11b5d10f15
diff --git a/crates/uzu-engine/src/backends/common/gpu_types/mod.rs b/crates/uzu-engine/src/backends/common/gpu_types/mod.rsindex c4aa68f87..5c1089585 100644--- a/crates/uzu-engine/src/backends/common/gpu_types/mod.rs+++ b/crates/uzu-engine/src/backends/common/gpu_types/mod.rs@@ -14,7 +14,6 @@ pub mod matmul;pub mod quantization;pub mod quantization_method;pub mod ring;-pub mod router_topk;pub mod trie;pub mod weaver;diff --git a/crates/uzu-engine/src/backends/common/gpu_types/router_topk.rs b/crates/uzu-engine/src/backends/common/gpu_types/router_topk.rsdeleted file mode 100644index 6d033bc2c..000000000--- a/crates/uzu-engine/src/backends/common/gpu_types/router_topk.rs+++ /dev/null@@ -1,5 +0,0 @@-pub const ROUTER_TOPK_MAX_EXPERTS: u32 = 512;--pub const ROUTER_TOPK_MAX_SELECTED_EXPERTS: u32 = 128;--pub const ROUTER_TOPK_MAX_MODEL_DIM: u32 = 4096;diff --git a/crates/uzu-engine/src/backends/cpu/kernel/mod.rs b/crates/uzu-engine/src/backends/cpu/kernel/mod.rsindex 8a079aeba..e96c27e1c 100644--- a/crates/uzu-engine/src/backends/cpu/kernel/mod.rs+++ b/crates/uzu-engine/src/backends/cpu/kernel/mod.rs@@ -12,7 +12,6 @@ mod gated_act_mul;mod gdn;mod logit_transform;mod matmul;-mod moe;mod normalization;mod pooling;mod radix_top_k_small;diff --git a/crates/uzu-engine/src/backends/cpu/kernel/moe/counts_offsets_fused.rs b/crates/uzu-engine/src/backends/cpu/kernel/moe/counts_offsets_fused.rsdeleted file mode 100644index fb718c3dd..000000000--- a/crates/uzu-engine/src/backends/cpu/kernel/moe/counts_offsets_fused.rs+++ /dev/null@@ -1,55 +0,0 @@-use uzu_engine_macros::kernel;--#[kernel(MoeCountsOffsetsFused)]-pub fn moe_counts_offsets_fused(- topk_ids: *const i32,- offsets: *mut u32,- sum_k_out: *mut u32,- partials: *mut u32,- t_input: u32,- e_input: u32,- k_input: u32,-) {- let e = e_input as usize;- let t = t_input as usize;- let k = k_input as usize;-- if e == 0 {- unsafe {- *offsets = 0;- *sum_k_out = 0;- }- return;- }-- // Phase 1: Count tokens per expert (histogram)- let mut counts = vec![0u32; e];- for ti in 0..t {- let base = ti * k;- for kk in 0..k {- let eid = unsafe { *topk_ids.add(base + kk) };- if eid >= 0 {- let ue = eid as usize;- if ue < e {- counts[ue] += 1;- }- }- }- }-- // Write partials (same as counts for single-threadgroup case)- for i in 0..e {- unsafe { *partials.add(i) = counts[i] };- }-- // Phase 2: Exclusive prefix scan to produce offsets- let mut sum = 0u32;- for i in 0..e {- unsafe { *offsets.add(i) = sum };- sum += counts[i];- }- unsafe {- *offsets.add(e) = sum;- *sum_k_out = sum;- }-}diff --git a/crates/uzu-engine/src/backends/cpu/kernel/moe/experts_two_pass_decode.rs b/crates/uzu-engine/src/backends/cpu/kernel/moe/experts_two_pass_decode.rsdeleted file mode 100644index 96f6b1503..000000000--- a/crates/uzu-engine/src/backends/cpu/kernel/moe/experts_two_pass_decode.rs+++ /dev/null@@ -1,77 +0,0 @@-use half::{bf16, f16};-use num_traits::Float;-use uzu_engine_macros::kernel;--use crate::array::ArrayElement;--#[kernel(MoeExpertsDecodePassA)]-#[variants(T, f32, f16, bf16)]-pub fn moe_experts_decode_pass_a<T: ArrayElement + Float>(- #[allow(unused)] x_perm: *const T,- #[allow(unused)] expert_offsets: *const u32,- #[allow(unused)] w13_all: *const T,- #[allow(unused)] hidden_out: *mut f32,- #[allow(unused)] up_biases: *const T,- #[allow(unused)] d_model: u32,- #[allow(unused)] d_ff: u32,- #[allow(unused)] e: u32,- #[allow(unused)] gate_clip_min: f32,- #[allow(unused)] gate_clip_max: f32,- #[allow(unused)] up_clip_min: f32,- #[allow(unused)] up_clip_max: f32,- #[allow(unused)] silu_alpha: f32,- #[allow(unused)] tile_map: *const u32,- #[allow(unused)]- #[specialize]- gating_sel: u32,- #[allow(unused)] __dsl_indirect_dispatch_buffer: *const u32,-) {- todo!()-}--#[kernel(MoeExpertsDecodeDownFused2D)]-#[variants(T, f32, f16, bf16)]-#[variants(AccumT, f32)]-#[allow(clippy::extra_unused_type_parameters)]-pub fn moe_experts_decode_down_fused2_d<T: ArrayElement + Float, AccumT: ArrayElement + Float>(- hidden: *const f32,- row_expert_map: *const u32,- w2_all: *const T,- down_biases: *const T,- y_out: *mut T,- total_rows: u32,- d_model: u32,- d_ff: u32,- e: u32,-) {- let dm = d_model as usize;- let df = d_ff as usize;- let e = e as usize;-- for row_idx in 0..total_rows as usize {- let expert_idx = unsafe { *row_expert_map.add(row_idx) } as usize;- assert!(expert_idx < e);- let hidden_base = row_idx * df;- let w2_expert_base = expert_idx * dm * df;-- for my_col in 0..dm {- let w2_col_base = w2_expert_base + my_col * df;-- let mut acc = 0.0f32;- for h in 0..df {- let h_val = unsafe { *hidden.add(hidden_base + h) };- let w_val = unsafe { *w2_all.add(w2_col_base + h) }.to_f32().unwrap();- acc = h_val.mul_add(w_val, acc);- }-- // Add bias- let bias_idx = expert_idx * dm + my_col;- acc += unsafe { *down_biases.add(bias_idx) }.to_f32().unwrap();-- let out_idx = row_idx * dm + my_col;- unsafe {- *y_out.add(out_idx) = T::from(acc).unwrap();- }- }- }-}diff --git a/crates/uzu-engine/src/backends/cpu/kernel/moe/experts_two_pass_prefill.rs b/crates/uzu-engine/src/backends/cpu/kernel/moe/experts_two_pass_prefill.rsdeleted file mode 100644index 629e7bfaa..000000000--- a/crates/uzu-engine/src/backends/cpu/kernel/moe/experts_two_pass_prefill.rs+++ /dev/null@@ -1,47 +0,0 @@-use half::{bf16, f16};-use num_traits::Float;-use uzu_engine_macros::kernel;--use crate::array::ArrayElement;--#[kernel(MoeExpertsPrefillPassA)]-#[variants(T, f32, f16, bf16)]-pub fn moe_experts_prefill_pass_a<T: ArrayElement + Float>(- #[allow(unused)] x_perm: *const T,- #[allow(unused)] expert_offsets: *const u32,- #[allow(unused)] w13_all: *const T,- #[allow(unused)] up_biases: *const T,- #[allow(unused)] hidden_out: *mut f32,- #[allow(unused)] d_model: u32,- #[allow(unused)] d_ff: u32,- #[allow(unused)] e: u32,- #[allow(unused)] gate_clip_min: f32,- #[allow(unused)] gate_clip_max: f32,- #[allow(unused)] up_clip_min: f32,- #[allow(unused)] up_clip_max: f32,- #[allow(unused)] silu_alpha: f32,- #[allow(unused)] tile_map: *const u32,- #[allow(unused)]- #[specialize]- gating_sel: u32,- #[allow(unused)] __dsl_indirect_dispatch_buffer: *const u32,-) {- todo!()-}--#[kernel(MoeExpertsPrefillPassB)]-#[variants(T, f32, f16, bf16)]-pub fn moe_experts_prefill_pass_b<T: ArrayElement + Float>(- #[allow(unused)] hidden: *const f32,- #[allow(unused)] expert_offsets: *const u32,- #[allow(unused)] w2_all: *const T,- #[allow(unused)] down_biases: *const T,- #[allow(unused)] output: *mut T,- #[allow(unused)] d_model: u32,- #[allow(unused)] d_ff: u32,- #[allow(unused)] e: u32,- #[allow(unused)] tile_map: *const u32,- #[allow(unused)] __dsl_indirect_dispatch_buffer: *const u32,-) {- todo!()-}diff --git a/crates/uzu-engine/src/backends/cpu/kernel/moe/finalize.rs b/crates/uzu-engine/src/backends/cpu/kernel/moe/finalize.rsdeleted file mode 100644index aa239d6ce..000000000--- a/crates/uzu-engine/src/backends/cpu/kernel/moe/finalize.rs+++ /dev/null@@ -1,50 +0,0 @@-use half::{bf16, f16};-use num_traits::Float;-use uzu_engine_macros::kernel;--use crate::array::ArrayElement;--#[kernel(MoeFinalize)]-#[variants(T, f32, f16, bf16)]-pub fn moe_finalize<T: ArrayElement + Float>(- tok2row: *const i32,- probs: *const T,- y_partial: *const T,- y: *mut T,- t_count: u32,- d_model: u32,- k_input: u32,-) {- let t_count = t_count as usize;- let d_model = d_model as usize;- let k_input = k_input as usize;-- unsafe {- for ti in 0..t_count {- for f in 0..d_model {- let mut acc = 0f32;- for kk in 0..k_input {- let idx = ti * k_input + kk;- let row = *tok2row.add(idx);- if row >= 0 {- let mut prob = (*probs.add(idx)).to_f32().unwrap();- if !prob.is_finite() {- prob = 0.0;- }-- let row_u = row as usize;- let mut val = (*y_partial.add(row_u * d_model + f)).to_f32().unwrap();- if !val.is_finite() {- val = 0.0;- }- acc += prob * val;- }- }- if !acc.is_finite() {- acc = 0.0;- }- *y.add(ti * d_model + f) = T::from(acc).unwrap();- }- }- }-}diff --git a/crates/uzu-engine/src/backends/cpu/kernel/moe/gather.rs b/crates/uzu-engine/src/backends/cpu/kernel/moe/gather.rsdeleted file mode 100644index 8e5045a0f..000000000--- a/crates/uzu-engine/src/backends/cpu/kernel/moe/gather.rs+++ /dev/null@@ -1,69 +0,0 @@-use half::{bf16, f16};-use num_traits::Float;-use uzu_engine_macros::kernel;--use crate::array::ArrayElement;--fn moe_gather<T: ArrayElement + Float>(- x: *const T,- bucketed_ids: *const i32,- x_perm: *mut T,- sumk_buf: *const u32,- d_model: u32,- t: u32,- k: u32,-) {- let total_rows = unsafe { *sumk_buf } as usize;- let d_model = d_model as usize;- let t = t as usize;- let k = k as usize;- let routed_capacity = t.checked_mul(k).expect("routed token count overflow");- assert!(total_rows <= routed_capacity);-- if total_rows == 0 || d_model == 0 {- return;- }-- unsafe {- for row in 0..total_rows {- let token = *bucketed_ids.add(row);- if token < 0 {- continue;- }- assert!((token as usize) < t);- let src = x.add(token as usize * d_model);- let dst = x_perm.add(row * d_model);- for col in 0..d_model {- *dst.add(col) = *src.add(col);- }- }- }-}--#[kernel(MoeGatherXPerm2D)]-#[variants(T, f32, f16, bf16)]-pub fn moe_gather_x_perm2_d<T: ArrayElement + Float>(- x: *const T,- bucketed_ids: *const i32,- x_perm: *mut T,- sumk_buf: *const u32,- d_model: u32,- t: u32,- k: u32,-) {- moe_gather(x, bucketed_ids, x_perm, sumk_buf, d_model, t, k);-}--#[kernel(MoeGatherXPerm1D)]-#[variants(T, f32, f16, bf16)]-pub fn moe_gather_x_perm1_d<T: ArrayElement + Float>(- x: *const T,- bucketed_ids: *const i32,- x_perm: *mut T,- sumk_buf: *const u32,- d_model: u32,- t: u32,- k: u32,-) {- moe_gather(x, bucketed_ids, x_perm, sumk_buf, d_model, t, k);-}diff --git a/crates/uzu-engine/src/backends/cpu/kernel/moe/mod.rs b/crates/uzu-engine/src/backends/cpu/kernel/moe/mod.rsdeleted file mode 100644index fae7fc37e..000000000--- a/crates/uzu-engine/src/backends/cpu/kernel/moe/mod.rs+++ /dev/null@@ -1,9 +0,0 @@-pub mod counts_offsets_fused;-pub mod experts_two_pass_decode;-pub mod experts_two_pass_prefill;-pub mod finalize;-pub mod gather;-pub mod router_topk;-pub mod scatter_buckets;-pub mod tiles_map;-pub mod tiles_pass_a;diff --git a/crates/uzu-engine/src/backends/cpu/kernel/moe/router_topk.rs b/crates/uzu-engine/src/backends/cpu/kernel/moe/router_topk.rsdeleted file mode 100644index 448095fbe..000000000--- a/crates/uzu-engine/src/backends/cpu/kernel/moe/router_topk.rs+++ /dev/null@@ -1,142 +0,0 @@-use half::{bf16, f16};-use num_traits::Float;-use uzu_engine_macros::kernel;--use crate::array::ArrayElement;--#[kernel(MoeRouterTopK)]-#[variants(ScalarT, f16, bf16, f32)]-pub fn moe_router_top_k<ScalarT: ArrayElement + Float>(- input: *const ScalarT,- weight: *const ScalarT,- #[optional(has_biases)] bias: Option<*const ScalarT>,- #[optional(has_router_scales)] router_scale: Option<*const ScalarT>,- #[optional(has_per_expert_scales)] per_expert_scale: Option<*const ScalarT>,- topk_ids: *mut i32,- topk_probs: *mut ScalarT,- t: u32,- d_model: u32,- e: u32,- k: u32,- renorm: bool,- #[optional(normalize_router_input)] router_norm_epsilon: Option<f32>,- #[optional(has_router_input_scale)] router_input_scale: Option<f32>,- #[specialize] has_biases: bool,- #[specialize] has_router_scales: bool,- #[specialize] has_per_expert_scales: bool,- #[specialize] has_router_input_scale: bool,- #[specialize] normalize_router_input: bool,-) {- assert_eq!(d_model % 4, 0, "d_model must be multiple of 4");- assert!(k >= 1 && e >= k);-- let t = t as usize;- let d_model = d_model as usize;- let e = e as usize;- let k = k as usize;- let router_input_scale = if has_router_input_scale {- router_input_scale.unwrap()- } else {- 1.0- };- let router_norm_epsilon = router_norm_epsilon.unwrap_or(0.0);-- let mut logits = vec![0.0f32; t * e];- unsafe {- for token in 0..t {- let x_row = input.add(token * d_model);- let mut inv_rms = 1.0f32;- if normalize_router_input {- let mut sum_sq = 0.0f32;- for idx in 0..d_model {- let x = (*x_row.add(idx)).to_f32().unwrap();- sum_sq += x * x;- }- inv_rms = (sum_sq / d_model as f32 + router_norm_epsilon).sqrt().recip();- }- for expert in 0..e {- let w_row = weight.add(expert * d_model);- let mut accum = [0.0f32; 4];-- // Simulate GPU vec4 processing: accumulate in 4-element chunks- for chunk in (0..d_model).step_by(4) {- for i in 0..4 {- let idx = chunk + i;- let scale = if has_router_scales {- (*router_scale.unwrap().add(idx)).to_f32().unwrap()- } else {- 1.0- };- let x = (*x_row.add(idx)).to_f32().unwrap() * inv_rms * router_input_scale * scale;- accum[i] += (*w_row.add(idx)).to_f32().unwrap() * x;- }- }-- // Sum the 4-vector: (a.x + a.y) + (a.z + a.w) - matches Metal shader line 60- let sum = (accum[0] + accum[1]) + (accum[2] + accum[3]);- logits[token * e + expert] = sum- + if has_biases {- (*bias.unwrap().add(expert)).to_f32().unwrap()- } else {- 0.0- };- }-- let mut best_vals = vec![f32::NEG_INFINITY; k];- let mut best_ids = vec![-1i32; k];- let row = &logits[token * e..(token + 1) * e];- for expert in 0..e {- let v = row[expert];- let mut insert_pos = None;- for j in (0..k).rev() {- if v > best_vals[j] || (v == best_vals[j] && (best_ids[j] < 0 || (expert as i32) < best_ids[j])) {- insert_pos = Some(j);- }- }- if let Some(pos) = insert_pos {- for s in (pos + 1..k).rev() {- best_vals[s] = best_vals[s - 1];- best_ids[s] = best_ids[s - 1];- }- best_vals[pos] = v;- best_ids[pos] = expert as i32;- }- }- let base = token * k;- for kk in 0..k {- *topk_ids.add(base + kk) = best_ids[kk];- }- let expert_scale = |id: i32| {- if has_per_expert_scales {- (*per_expert_scale.unwrap().add(id as usize)).to_f32().unwrap()- } else {- 1.0- }- };- if renorm {- let max_v = best_vals.iter().copied().fold(f32::NEG_INFINITY, f32::max);- let mut exps = vec![0.0f32; k];- let mut sum = 0.0f32;- for kk in 0..k {- exps[kk] = (best_vals[kk] - max_v).exp();- sum += exps[kk];- }- if sum > 0.0 {- for kk in 0..k {- *topk_probs.add(base + kk) =- ScalarT::from(exps[kk] / sum * expert_scale(best_ids[kk])).unwrap();- }- } else {- let uniform = 1.0f32 / k as f32;- for kk in 0..k {- *topk_probs.add(base + kk) = ScalarT::from(uniform * expert_scale(best_ids[kk])).unwrap();- }- }- } else {- for kk in 0..k {- *topk_probs.add(base + kk) = ScalarT::from(best_vals[kk] * expert_scale(best_ids[kk])).unwrap();- }- }- }- }-}diff --git a/crates/uzu-engine/src/backends/cpu/kernel/moe/scatter_buckets.rs b/crates/uzu-engine/src/backends/cpu/kernel/moe/scatter_buckets.rsdeleted file mode 100644index d9d15d6aa..000000000--- a/crates/uzu-engine/src/backends/cpu/kernel/moe/scatter_buckets.rs+++ /dev/null@@ -1,38 +0,0 @@-use half::{bf16, f16};-use num_traits::Float;-use uzu_engine_macros::kernel;--use crate::array::ArrayElement;--#[kernel(MoeBlockBasesFromPartials)]-pub fn moe_block_bases_from_partials(- #[allow(unused)] partials: *const u32,- #[allow(unused)] block_bases: *mut u32,- #[allow(unused)] block_alloc: *mut u32,- #[allow(unused)] e_input: u32,- #[allow(unused)] num_blocks: u32,- #[allow(unused)] num_tiles: u32,- #[allow(unused)] capacity_per_expert: u32,-) {- todo!()-}--#[kernel(MoeScatterBucketsMap)]-#[variants(T, f32, f16, bf16)]-pub fn moe_scatter_buckets_map<T: ArrayElement + Float>(- #[allow(unused)] topk_ids: *const i32,- #[allow(unused)] topk_probs: *const T,- #[allow(unused)] offsets: *const u32,- #[allow(unused)] block_bases: *const u32,- #[allow(unused)] block_alloc: *const u32,- #[allow(unused)] out_ids: *mut i32,- #[allow(unused)] out_probs: *mut T,- #[allow(unused)] t: u32,- #[allow(unused)] e: u32,- #[allow(unused)] k: u32,- #[allow(unused)] num_blocks: u32,- #[allow(unused)] num_tiles: u32,- #[allow(unused)] tok2row: *mut i32,-) {- todo!()-}diff --git a/crates/uzu-engine/src/backends/cpu/kernel/moe/tiles_map.rs b/crates/uzu-engine/src/backends/cpu/kernel/moe/tiles_map.rsdeleted file mode 100644index 0d7348ea2..000000000--- a/crates/uzu-engine/src/backends/cpu/kernel/moe/tiles_map.rs+++ /dev/null@@ -1,40 +0,0 @@-use uzu_engine_macros::kernel;--#[kernel(MoeTileCounts)]-pub fn moe_tile_counts(- #[allow(unused)] offsets: *const u32,- #[allow(unused)] tile_counts: *mut u32,- #[allow(unused)] e: u32,-) {- todo!()-}--#[kernel(MoeTileScan)]-pub fn moe_tile_scan(- #[allow(unused)] tile_counts: *const u32,- #[allow(unused)] tile_row_offsets: *mut u32,- #[allow(unused)] total_tiles_buf: *mut u32,- #[allow(unused)] e: u32,-) {- todo!()-}--#[kernel(MoeBuildTileMap)]-pub fn moe_build_tile_map(- #[allow(unused)] offsets: *const u32,- #[allow(unused)] tile_row_offsets: *const u32,- #[allow(unused)] tile_counts: *const u32,- #[allow(unused)] tile_map: *mut u32,- #[allow(unused)] e: u32,-) {- todo!()-}--#[kernel(MoeWriteDispatchArgs)]-pub fn moe_write_dispatch_args(- #[allow(unused)] total_tiles_buf: *const u32,- #[allow(unused)] dispatch_args: *mut u32,- #[allow(unused)] num_tiles_n: u32,-) {- todo!()-}diff --git a/crates/uzu-engine/src/backends/cpu/kernel/moe/tiles_pass_a.rs b/crates/uzu-engine/src/backends/cpu/kernel/moe/tiles_pass_a.rsdeleted file mode 100644index 81cb00205..000000000--- a/crates/uzu-engine/src/backends/cpu/kernel/moe/tiles_pass_a.rs+++ /dev/null@@ -1,52 +0,0 @@-use uzu_engine_macros::kernel;--#[kernel(MoePassATileCounts)]-pub fn moe_pass_a_tile_counts(- #[allow(unused)] expert_offsets: *const u32,- #[allow(unused)] tile_counts: *mut u32,- #[allow(unused)] e: u32,- #[allow(unused)] h_blocks: u32,-) {- todo!()-}--#[kernel(MoePassATileScan)]-pub fn moe_pass_a_tile_scan(- #[allow(unused)] tile_counts: *const u32,- #[allow(unused)] tile_offsets: *mut u32,- #[allow(unused)] total_tiles: *mut u32,- #[allow(unused)] e: u32,-) {- todo!()-}--#[kernel(MoePassABuildRowMap)]-pub fn moe_pass_a_build_row_map(- #[allow(unused)] expert_offsets: *const u32,- #[allow(unused)] row_expert_map: *mut u32,- #[allow(unused)] total_rows: u32,- #[allow(unused)] e: u32,-) {- todo!()-}--#[kernel(MoePassABuildTileMap)]-pub fn moe_pass_a_build_tile_map(- #[allow(unused)] expert_offsets: *const u32,- #[allow(unused)] tile_offsets: *const u32,- #[allow(unused)] row_expert_map: *const u32,- #[allow(unused)] tile_map: *mut u32,- #[allow(unused)] total_rows: u32,- #[allow(unused)] h_blocks: u32,-) {- todo!()-}--#[kernel(MoePassAWriteDispatchArgs)]-pub fn moe_pass_a_write_dispatch_args(- #[allow(unused)] total_tiles: *const u32,- #[allow(unused)] dispatch_args: *mut u32,- #[allow(unused)] num_tiles_y: u32,-) {- todo!()-}diff --git a/crates/uzu-engine/src/backends/metal/kernel/common/threadgroup_reduce.h b/crates/uzu-engine/src/backends/metal/kernel/common/threadgroup_reduce.hindex 85756d118..bf58ca6aa 100644--- a/crates/uzu-engine/src/backends/metal/kernel/common/threadgroup_reduce.h+++ b/crates/uzu-engine/src/backends/metal/kernel/common/threadgroup_reduce.h@@ -20,14 +20,6 @@ struct SimdReduceMax {static T simd_reduce(T x) { return simd_max(x); }};-template <typename T>-struct SimdReduceMin {- using value_type = T;- static constant constexpr T identity =- numeric_limits<T>::has_infinity ? numeric_limits<T>::infinity() : numeric_limits<T>::max();- static T simd_reduce(T x) { return simd_min(x); }-};-template <typename Op, ushort BLOCK_SIZE>static typename Op::value_type threadgroup_cooperative_reduce(typename Op::value_type value,diff --git a/crates/uzu-engine/src/backends/metal/kernel/generated/router_topk.h b/crates/uzu-engine/src/backends/metal/kernel/generated/router_topk.hdeleted file mode 100644index 40f37de5a..000000000--- a/crates/uzu-engine/src/backends/metal/kernel/generated/router_topk.h+++ /dev/null@@ -1,13 +0,0 @@-// Auto-generated from gpu_types/router_topk - do not edit manually-#pragma once--#include <metal_stdlib>-using namespace metal;--namespace uzu::router_topk {-static constant constexpr uint32_t ROUTER_TOPK_MAX_EXPERTS = 512;--static constant constexpr uint32_t ROUTER_TOPK_MAX_SELECTED_EXPERTS = 128;--static constant constexpr uint32_t ROUTER_TOPK_MAX_MODEL_DIM = 4096;-} // namespace uzu::router_topkdiff --git a/crates/uzu-engine/src/backends/metal/kernel/moe/counts_offsets_fused.metal b/crates/uzu-engine/src/backends/metal/kernel/moe/counts_offsets_fused.metaldeleted file mode 100644index b82f05de8..000000000--- a/crates/uzu-engine/src/backends/metal/kernel/moe/counts_offsets_fused.metal+++ /dev/null@@ -1,138 +0,0 @@-#include <metal_stdlib>-#include <metal_atomic>--#include "../common/dsl.h"-#include "../common/thread_context.h"-#include "../common/threadgroup_reduce.h"--template <ushort BLOCK_SIZE, typename T>-static T threadgroup_raking_prefix_exclusive_sum(T value, threadgroup T* shared, const ushort lid) {- shared[lid] = value;- threadgroup_barrier(mem_flags::mem_threadgroup);-- if (lid < 32) {- const short values_per_thread = BLOCK_SIZE / 32;- const short first_index = lid * values_per_thread;- for (short i = first_index + 1; i < first_index + values_per_thread; i++) {- shared[i] += shared[i - 1];- }- T partial_sum = shared[first_index + values_per_thread - 1];- for (short i = first_index + values_per_thread - 1; i > first_index; i--) {- shared[i] = shared[i - 1];- }- shared[first_index] = 0;-- T prefix = simd_prefix_exclusive_sum(partial_sum);-- for (short i = first_index; i < first_index + values_per_thread; i++) {- shared[i] += prefix;- }- }- threadgroup_barrier(mem_flags::mem_threadgroup);-- const T result = shared[lid];- threadgroup_barrier(mem_flags::mem_threadgroup);- return result;-}--#define BLOCK_SIZE 128-#define TILE_E 512--// Single-kernel fused: count all experts + scan to offsets-// This kernel is launched with SINGLE threadgroup-PUBLIC KERNEL(MoeCountsOffsetsFused)(- device const int* topk_ids,- device uint* offsets, // output: exclusive scan [E+1]- device uint* sum_k_out, // output: total count [1]- device uint* partials, // output: partials [num_tiles * TILE_E] (for block_bases)- constant uint& t_input,- constant uint& e_input,- constant uint& k_input,- threadgroup _atomic<uint> tg_hist[TILE_E],- threadgroup uint scan_shared[BLOCK_SIZE],- threadgroup uint reduce_shared[BLOCK_SIZE],- threadgroup uint counts_shared[BLOCK_SIZE], // Cache counts in threadgroup memory- threadgroup uint& carry,- const ThreadContext thread_context,- const uint lid THREADS(128)-) {- if (e_input == 0) {- if (lid == 0) {- offsets[0] = 0u;- sum_k_out[0] = 0u;- }- return;- }-- // ═══════════════════════════════════════════════════════════- // PHASE 1: Count tokens per expert using tiled histogram- // ═══════════════════════════════════════════════════════════- // Tile over E dimension- for (uint e0 = 0; e0 < e_input; e0 += TILE_E) {- const uint tile_e = (e0 + TILE_E <= e_input) ? TILE_E : (e_input - e0);-- // Zero threadgroup histogram- for (uint e = lid; e < tile_e; e += BLOCK_SIZE) {- atomic_store_explicit(&tg_hist[e], 0u, memory_order_relaxed);- }- threadgroup_barrier(mem_flags::mem_threadgroup);-- // Process all tokens in strided fashion- // Since we have only 1 threadgroup, we need to cover all T tokens- for (uint t = lid; t < t_input; t += BLOCK_SIZE) {- const uint base = t * k_input;- for (uint k = 0; k < k_input; ++k) {- int eid = topk_ids[base + k];- if (eid >= 0) {- uint ue = uint(eid);- if (ue >= e0 && ue < e0 + tile_e) {- uint te = ue - e0;- atomic_fetch_add_explicit(&tg_hist[te], 1u, memory_order_relaxed);- }- }- }- }- threadgroup_barrier(mem_flags::mem_threadgroup);-- // Write final counts and partials for this tile- for (uint e = lid; e < tile_e; e += BLOCK_SIZE) {- const uint count_val = atomic_load_explicit(&tg_hist[e], memory_order_relaxed);- counts_shared[e0 + e] = count_val;- partials[e0 + e] = count_val;- }- threadgroup_barrier(mem_flags::mem_threadgroup);- }-- // ═══════════════════════════════════════════════════════════- // PHASE 2: Compute exclusive prefix scan on counts- // ═══════════════════════════════════════════════════════════- if (lid == 0) {- carry = 0u;- }- threadgroup_barrier(mem_flags::mem_threadgroup);-- for (uint base = 0; base < e_input; base += BLOCK_SIZE) {- uint remaining = e_input - base;- uint chunk_n = remaining < BLOCK_SIZE ? remaining : BLOCK_SIZE;-- uint v = (lid < chunk_n) ? counts_shared[lid] : 0u;-- uint prefix_local = threadgroup_raking_prefix_exclusive_sum<BLOCK_SIZE>(v, scan_shared, (ushort)lid);- uint prefix_global = prefix_local + carry;- if (lid < chunk_n) {- offsets[base + lid] = prefix_global;- }-- uint block_sum = threadgroup_cooperative_reduce<SimdReduceSum<uint>, BLOCK_SIZE>(v, reduce_shared, thread_context);- if (lid == 0) {- carry += block_sum;- }- threadgroup_barrier(mem_flags::mem_threadgroup);- }-- // Write final offset and total- if (lid == 0) {- offsets[e_input] = carry;- sum_k_out[0] = carry;- }-}diff --git a/crates/uzu-engine/src/backends/metal/kernel/moe/experts_two_pass_decode.metal b/crates/uzu-engine/src/backends/metal/kernel/moe/experts_two_pass_decode.metaldeleted file mode 100644index dabc1a252..000000000--- a/crates/uzu-engine/src/backends/metal/kernel/moe/experts_two_pass_decode.metal+++ /dev/null@@ -1,228 +0,0 @@-#include <metal_stdlib>-#include <metal_simdgroup>-#include "../activation/activations.h"-#include "../common/dsl.h"-#include "../common/thread_context.h"-using namespace metal;--// === Pass A: Vectorized GEMV with float4 loads ===-// Structure: 4 simdgroups (128 threads), each outputs 1 hidden element-// Each simdgroup: 32 threads reduce d_model using float4 vectorized loads-// Grid: (h_blocks, rows, 1) where h_blocks = ceil(d_ff/4)-template <typename T>-VARIANTS(T, float, half, bfloat)-PUBLIC KERNEL(MoeExpertsDecodePassA)(- device const T* x_perm, // [total_rows, d_model]- device const uint* expert_offsets, // [E + 1]- device const T* w13_all, // [E, 2*d_ff, d_model]- device float* hidden_out, // [E, 2*d_ff]- device const T* up_biases, // [total_rows, d_ff]- constant uint& d_model,- constant uint& d_ff,- constant uint& e,- constant float& gate_clip_min,- constant float& gate_clip_max,- constant float& up_clip_min,- constant float& up_clip_max,- constant float& silu_alpha,- device const uint* tile_map,- const uint gating_sel SPECIALIZE,- const ThreadContext thread_context,- const uint tile_idx GROUPS(INDIRECT),- const uint tid THREADS(128)-) {- // Read tile descriptor- const uint h_block_idx = tile_map[tile_idx * 3 + 0];- const uint expert_idx = tile_map[tile_idx * 3 + 1];- const uint row_in_expert = tile_map[tile_idx * 3 + 2];-- // Validate row bounds- const uint seg_start = expert_offsets[expert_idx];- const uint seg_end = expert_offsets[expert_idx + 1];- const uint global_row = seg_start + row_in_expert;- if (global_row >= seg_end)- return;-- // Each simdgroup outputs one hidden element- const uint h_idx = h_block_idx * 4 + thread_context.simdgroup_index;- if (h_idx >= d_ff)- return;-- // Base addresses- const ulong w13_stride = (ulong)d_model * (ulong)(2 * d_ff);- const ulong w13_base = (ulong)expert_idx * w13_stride;- const ulong bias_base = (ulong)expert_idx * (ulong)(2 * d_ff);- const ulong x_row_base = (ulong)global_row * (ulong)d_model;-- device const T* x_ptr = x_perm + x_row_base;- device const T* w_up_row = w13_all + w13_base + (ulong)h_idx * (ulong)d_model;- device const T* w_gate_row = w13_all + w13_base + (ulong)(d_ff + h_idx) * (ulong)d_model;-- float acc_up = 0.0f;- float acc_gate = 0.0f;-- // Vectorized reduction: 32 threads × 4 elements = 128 elements per iteration- const uint vec_iters = d_model / 128;-- for (uint i = 0; i < vec_iters; ++i) {- uint base_idx = i * 128 + thread_context.simd_lane_id * 4;-- device const T* x_vec = reinterpret_cast<device const T*>(x_ptr + base_idx);- device const T* w_up_vec = reinterpret_cast<device const T*>(w_up_row + base_idx);- acc_up += float(x_vec[0]) * float(w_up_vec[0]);- acc_up += float(x_vec[1]) * float(w_up_vec[1]);- acc_up += float(x_vec[2]) * float(w_up_vec[2]);- acc_up += float(x_vec[3]) * float(w_up_vec[3]);-- if (gating_sel > 1) {- device const T* w_gate_vec = reinterpret_cast<device const T*>(w_gate_row + base_idx);- acc_gate += float(x_vec[0]) * float(w_gate_vec[0]);- acc_gate += float(x_vec[1]) * float(w_gate_vec[1]);- acc_gate += float(x_vec[2]) * float(w_gate_vec[2]);- acc_gate += float(x_vec[3]) * float(w_gate_vec[3]);- }- }-- // Handle leftover elements- uint leftover_start = vec_iters * 128 + thread_context.simd_lane_id;- for (uint idx = leftover_start; idx < d_model; idx += 32) {- float xv = float(x_ptr[idx]);- acc_up += xv * float(w_up_row[idx]);- if (gating_sel > 1) {- acc_gate += xv * float(w_gate_row[idx]);- }- }-- // Simdgroup reduction- acc_up = simd_sum(acc_up);- if (gating_sel > 1) {- acc_gate = simd_sum(acc_gate);- }-- // Lane 0 applies activation and writes result- if (thread_context.simd_lane_id == 0) {- float up_val = acc_up + float(up_biases[bias_base + h_idx]);- up_val = clamp(up_val, up_clip_min, up_clip_max);-- float activated;- if (gating_sel <= 1) {- activated = (gating_sel == 0) ? activate_gelu(up_val) : activate_silu_alpha(up_val, silu_alpha);- } else {- float gate_val = acc_gate + float(up_biases[bias_base + d_ff + h_idx]);- gate_val = clamp(gate_val, gate_clip_min, gate_clip_max);- float gate_act = (gating_sel == 2) ? activate_silu_alpha(gate_val, silu_alpha) : activate_gelu(gate_val);- activated = gate_act * up_val;- }-- hidden_out[(ulong)global_row * (ulong)d_ff + (ulong)h_idx] = activated;- }-}--// === Pass B: Simdgroup cooperation along K for coalescing ===-// W2 layout [E, d_model, d_ff] - 32 threads cooperate on one output, reading-// consecutive K elements-#define THREADS_PER_SIMD 32-#define SIMDGROUPS_PER_TG 8--template <typename T, typename AccumT>-VARIANTS(T, float, half, bfloat)-VARIANTS(AccumT, float)-PUBLIC KERNEL(MoeExpertsDecodeDownFused2D)(- device const float* hidden, // [total_rows, d_ff] - f32 from Pass A- device const uint* row_expert_map, // [total_rows] - direct row->expert lookup- device const T* w2_all, // [E, d_model, d_ff] - layout- device const T* down_biases, // [E, d_model]- device T* y_out, // [total_rows, d_model]- constant uint& total_rows,- constant uint& d_model,- constant uint& d_ff,- constant uint& e,- const ThreadContext thread_context,- const uint tgpig_x GROUPS(d_model.div_ceil(SIMDGROUPS_PER_TG)),- const uint tgpig_y GROUPS(total_rows),- const uint tid THREADS(256)-) {- const uint row_idx = tgpig_y;-- // Each simdgroup computes one output column- const uint my_col = tgpig_x * SIMDGROUPS_PER_TG + thread_context.simdgroup_index;- if (my_col >= d_model)- return;-- const uint expert_idx = row_expert_map[row_idx];-- // Base addresses for this output column- const ulong hidden_base = (ulong)row_idx * (ulong)d_ff;- const ulong w2_col_base = (ulong)expert_idx * (ulong)d_model * (ulong)d_ff + (ulong)my_col * (ulong)d_ff;-- // Dual accumulators for ILP: breaks FMA dependency chains- AccumT acc0 = AccumT(0.0);- AccumT acc1 = AccumT(0.0);- AccumT acc = AccumT(0.0);-- // Main loop: stride-32 with vec8 loads for ILP- const uint k_iters = d_ff / THREADS_PER_SIMD;- const uint k_vec_iters = k_iters / 8;-- for (uint iter = 0; iter < k_vec_iters; ++iter) {- const uint k_base = iter * (8 * THREADS_PER_SIMD) + thread_context.simd_lane_id;-- // hidden: stride-32 per lane, already f32 from Pass A- const AccumT h0 = hidden[hidden_base + k_base + 0 * THREADS_PER_SIMD];- const AccumT h1 = hidden[hidden_base + k_base + 1 * THREADS_PER_SIMD];- const AccumT h2 = hidden[hidden_base + k_base + 2 * THREADS_PER_SIMD];- const AccumT h3 = hidden[hidden_base + k_base + 3 * THREADS_PER_SIMD];- const AccumT h4 = hidden[hidden_base + k_base + 4 * THREADS_PER_SIMD];- const AccumT h5 = hidden[hidden_base + k_base + 5 * THREADS_PER_SIMD];- const AccumT h6 = hidden[hidden_base + k_base + 6 * THREADS_PER_SIMD];- const AccumT h7 = hidden[hidden_base + k_base + 7 * THREADS_PER_SIMD];-- // W2: lane-coalesced- const AccumT w0 = AccumT(w2_all[w2_col_base + k_base + 0 * THREADS_PER_SIMD]);- const AccumT w1 = AccumT(w2_all[w2_col_base + k_base + 1 * THREADS_PER_SIMD]);- const AccumT w2 = AccumT(w2_all[w2_col_base + k_base + 2 * THREADS_PER_SIMD]);- const AccumT w3 = AccumT(w2_all[w2_col_base + k_base + 3 * THREADS_PER_SIMD]);- const AccumT w4 = AccumT(w2_all[w2_col_base + k_base + 4 * THREADS_PER_SIMD]);- const AccumT w5 = AccumT(w2_all[w2_col_base + k_base + 5 * THREADS_PER_SIMD]);- const AccumT w6 = AccumT(w2_all[w2_col_base + k_base + 6 * THREADS_PER_SIMD]);- const AccumT w7 = AccumT(w2_all[w2_col_base + k_base + 7 * THREADS_PER_SIMD]);-- // dual trees for ILP- acc0 = fma(h0, w0, acc0);- acc1 = fma(h1, w1, acc1);- acc0 = fma(h2, w2, acc0);- acc1 = fma(h3, w3, acc1);- acc0 = fma(h4, w4, acc0);- acc1 = fma(h5, w5, acc1);- acc0 = fma(h6, w6, acc0);- acc1 = fma(h7, w7, acc1);- }-- // fold the vectorized contribution once- acc += (acc0 + acc1);-- // Handle remaining full iterations- for (uint iter = k_vec_iters * 8; iter < k_iters; ++iter) {- const uint k = iter * THREADS_PER_SIMD + thread_context.simd_lane_id;- acc = fma(AccumT(hidden[hidden_base + k]), AccumT(w2_all[w2_col_base + k]), acc);- }-- // Handle leftover elements (d_ff % 32)- const uint leftover_start = k_iters * THREADS_PER_SIMD;- if (leftover_start + thread_context.simd_lane_id < d_ff) {- const uint k = leftover_start + thread_context.simd_lane_id;- acc = fma(AccumT(hidden[hidden_base + k]), AccumT(w2_all[w2_col_base + k]), acc);- }-- // Simdgroup reduction- AccumT result = simd_sum(acc);-- // Lane 0 writes result- if (thread_context.simd_lane_id == 0) {- const ulong bias_idx = (ulong)expert_idx * (ulong)d_model + (ulong)my_col;- result += AccumT(down_biases[bias_idx]);-- const ulong out_idx = (ulong)row_idx * (ulong)d_model + (ulong)my_col;- y_out[out_idx] = T(result);- }-}\ No newline at end of filediff --git a/crates/uzu-engine/src/backends/metal/kernel/moe/experts_two_pass_prefill.metal b/crates/uzu-engine/src/backends/metal/kernel/moe/experts_two_pass_prefill.metaldeleted file mode 100644index 35576d775..000000000--- a/crates/uzu-engine/src/backends/metal/kernel/moe/experts_two_pass_prefill.metal+++ /dev/null@@ -1,517 +0,0 @@-#include <metal_stdlib>-#include "../activation/activations.h"-#include "../common/defines.h"-#include "../common/dsl.h"-#include "../common/thread_context.h"--// ------------------------ helpers -------------------------static inline uint ceil_div(uint a, uint b) { return (a + b - 1u) / b; }--static inline uint linear_tid(uint3 tid, uint3 tpg) { return tid.z * (tpg.x * tpg.y) + tid.y * tpg.x + tid.x; }--// ------------------------ Pass A (X @ W13 → hidden) -------------------------// Tile config (tuned for A-series; matches your prior defaults)-constant uint PASSA_BM = 16;-constant uint PASSA_BN = 32;-constant uint PASSA_BK = 64;-constant uint PASSA_SG_BM = 8;-constant uint PASSA_SG_BN = 16;-constant uint PASSA_TG_PAD = 4; // Padding to avoid bank conflicts--template <typename T>-VARIANTS(T, float, half, bfloat)-PUBLIC KERNEL(MoeExpertsPrefillPassA)(- device const T* x_perm,- device const uint* expert_offsets,- device const T* w13_all,- device const T* up_biases,- device float* hidden_out,- constant uint& d_model,- constant uint& d_ff,- constant uint& e,- constant float& gate_clip_min,- constant float& gate_clip_max,- constant float& up_clip_min,- constant float& up_clip_max,- constant float& silu_alpha,- device const uint* tile_map,- threadgroup T Xs[PASSA_BM * (PASSA_BK + PASSA_TG_PAD)],- threadgroup T Wk_up[PASSA_BN * (PASSA_BK + PASSA_TG_PAD)],- threadgroup T Wk_gate[PASSA_BN * (PASSA_BK + PASSA_TG_PAD)],- threadgroup float bias_up[PASSA_BN],- threadgroup float bias_gate[PASSA_BN],- // 0=GELU(up), 1=SiLU(up), 2=SwiGLU(gate)*up, 3=GEGLU(gate)*up- const uint gating_sel SPECIALIZE,- const uint n_tile_idx GROUPS(INDIRECT),- const uint row_tile_idx GROUPS(INDIRECT),- const uint local_tid_x THREADS(128),- const ThreadContext thread_context-) {- const uint base = row_tile_idx * 3u;- const uint expert_idx = tile_map[base + 0u];- if (expert_idx >= e)- return;-- const uint3 threads_per_tg = {128, 1, 1};- const uint3 local_tid = {local_tid_x, 0, 0};-- constexpr uint ROW_TILE = PASSA_BM;- const uint tile_seg_start = tile_map[base + 1u];- const uint row_off_elems = tile_map[base + 2u];- const uint tile_m = row_off_elems / ROW_TILE;- const uint tile_n = n_tile_idx;-- using StageVec4 = metal::vec<T, 4>;-- constexpr uint Bm = PASSA_BM, Bn = PASSA_BN, Bk = PASSA_BK;- constexpr uint SgBm = PASSA_SG_BM, SgBn = PASSA_SG_BN;- constexpr uint TG_PAD = PASSA_TG_PAD;- constexpr uint X_LD = Bk + TG_PAD; // Padded stride for X- constexpr uint W_LD = Bk + TG_PAD; // Padded stride for weights-- const uint seg_start = expert_offsets[expert_idx];- const uint seg_end = expert_offsets[expert_idx + 1];- const uint seg_len = (seg_end > seg_start) ? (seg_end - seg_start) : 0u;- if (seg_len == 0)- return;-- const uint row_tg_off = tile_m * Bm;- const uint col_tg_off = tile_n * Bn;- if (row_tg_off >= seg_len || col_tg_off >= d_ff)- return;-- const uint m_rows = min(Bm, seg_len - row_tg_off);- const uint n_cols = min(Bn, d_ff - col_tg_off);-- const ulong x_base = ((ulong)tile_seg_start + (ulong)row_tg_off) * (ulong)d_model;- const ulong w13_expert_base = (ulong)expert_idx * (ulong)(2 * d_ff) * (ulong)d_model;- const ulong bias_base = (ulong)expert_idx * (ulong)(2 * d_ff);-- // simdgroup tile mapping- const uint sg_col_count = Bn / SgBn;- const uint row_sg = thread_context.simdgroup_index / sg_col_count;- const uint col_sg = thread_context.simdgroup_index % sg_col_count;- const uint row_sg_off = row_sg * SgBm;- const uint col_sg_off = col_sg * SgBn;-- // Guard against misconfigured TG sizes- constexpr uint SG_TILE = 8; // simdgroup fragment dimension (8x8)- constexpr uint SG_EXPECTED = (Bm / SG_TILE) * (Bn / SG_TILE);- if (thread_context.simdgroup_index >= SG_EXPECTED)- return;-- // partial accumulators (8x8 tiles per simdgroup)- constexpr uint TEMP = (SgBm / SG_TILE) * (SgBn / SG_TILE);- metal::simdgroup_float8x8 OutUp[TEMP];- metal::simdgroup_float8x8 OutGate[TEMP];- for (uint i = 0; i < TEMP; ++i) {- OutUp[i] = metal::make_filled_simdgroup_matrix<float, 8, 8>(0.0f);- if (gating_sel > 1u) {- OutGate[i] = metal::make_filled_simdgroup_matrix<float, 8, 8>(0.0f);- }- }-- const uint threads_total = threads_per_tg.x * threads_per_tg.y * threads_per_tg.z;- const uint lin = linear_tid(local_tid, threads_per_tg);-- // K-loop (tile along D)- for (uint k_off = 0; k_off < d_model; k_off += Bk) {- const uint valid_k = min(Bk, d_model - k_off);-- threadgroup_barrier(mem_flags::mem_threadgroup);-- // ---- Stage LHS: Xs [Bm,Bk] with vec8 loads and padding ----- {- constexpr uint vec_size = 8;- constexpr uint vec_cols = Bk / vec_size; // 64/8 = 8- constexpr uint total_vecs = Bm * vec_cols; // 16*8 = 128- const uint iters = ceil_div(total_vecs, threads_total);-- METAL_PRAGMA_UNROLL- for (uint t = 0; t < iters; ++t) {- const uint i = t * threads_total + lin;- if (i < total_vecs) {- const uint r = i / vec_cols;- const uint c8 = i % vec_cols;- const uint col_base = c8 * vec_size;-- threadgroup T* dst = Xs + r * X_LD + col_base;-- if (r < m_rows && col_base < valid_k) {- // Safe to read - at least first element is valid- const ulong base = x_base + (ulong)r * (ulong)d_model + (ulong)(k_off + col_base);- METAL_PRAGMA_UNROLL- for (uint j = 0; j < vec_size; j++) {- dst[j] = (col_base + j < valid_k) ? static_cast<T>(float(x_perm[base + j])) : static_cast<T>(0.0f);- }- } else {- // Row out of bounds or entire vec8 chunk is out of K bounds- METAL_PRAGMA_UNROLL- for (uint j = 0; j < vec_size; j++) {- dst[j] = static_cast<T>(0.0f);- }- }- }- }- }-- // ---- Stage RHS: W_up [Bn,Bk], W_gate [Bn,Bk] with vec8 loads and padding- // ----- {- constexpr uint vec_size = 8;- constexpr uint vec_cols = Bk / vec_size; // 64/8 = 8- constexpr uint total_vecs = Bn * vec_cols; // 32*8 = 256- const uint iters = ceil_div(total_vecs, threads_total);-- METAL_PRAGMA_UNROLL- for (uint t = 0; t < iters; ++t) {- const uint i = t * threads_total + lin;- if (i < total_vecs) {- const uint n = i / vec_cols; // row within the BN tile- const uint c8 = i % vec_cols;- const uint col_base = c8 * vec_size;-- threadgroup T* up_dst = Wk_up + n * W_LD + col_base;- threadgroup T* gt_dst = Wk_gate + n * W_LD + col_base;-- if (n < n_cols && col_base < valid_k) {- // Safe to read - at least first element is valid- const uint gcol = col_tg_off + n;- const ulong up_base = w13_expert_base + (ulong)gcol * (ulong)d_model + (ulong)(k_off + col_base);- const ulong gate_base = w13_expert_base + (ulong)(d_ff + gcol) * (ulong)d_model + (ulong)(k_off + col_base);- METAL_PRAGMA_UNROLL- for (uint j = 0; j < vec_size; j++) {- up_dst[j] = (col_base + j < valid_k) ? static_cast<T>(float(w13_all[up_base + j])) : static_cast<T>(0.0f);- if (gating_sel > 1u) {- gt_dst[j] =- (col_base + j < valid_k) ? static_cast<T>(float(w13_all[gate_base + j])) : static_cast<T>(0.0f);- }- }- } else {- // Row out of bounds or entire vec8 chunk is out of K bounds- METAL_PRAGMA_UNROLL- for (uint j = 0; j < vec_size; j++) {- up_dst[j] = static_cast<T>(0.0f);- if (gating_sel > 1u)- gt_dst[j] = static_cast<T>(0.0f);- }- }- }- }- }-- threadgroup_barrier(mem_flags::mem_threadgroup);-- // ---- MMA across the BK tile (in simdgroup-sized chunks) ----- METAL_PRAGMA_UNROLL- for (uint kk = 0; kk < Bk; kk += SG_TILE) {- METAL_PRAGMA_UNROLL- for (uint m_sub = 0; m_sub < SgBm; m_sub += SG_TILE) {- const uint r_idx = m_sub / SG_TILE;- metal::simdgroup_matrix<T, 8, 8> lhs_frag;- simdgroup_load(lhs_frag, Xs, X_LD, ulong2(kk, row_sg_off + m_sub));-- METAL_PRAGMA_UNROLL- for (uint n_sub = 0; n_sub < SgBn; n_sub += SG_TILE) {- const uint c_idx = n_sub / SG_TILE;- const uint tile = r_idx * (SgBn / SG_TILE) + c_idx;- metal::simdgroup_matrix<T, 8, 8> rhs_up;- simdgroup_load(rhs_up, Wk_up, W_LD, ulong2(kk, col_sg_off + n_sub), true);- simdgroup_multiply_accumulate(OutUp[tile], lhs_frag, rhs_up, OutUp[tile]);-- if (gating_sel > 1u) {- metal::simdgroup_matrix<T, 8, 8> rhs_gate;- simdgroup_load(rhs_gate, Wk_gate, W_LD, ulong2(kk, col_sg_off + n_sub), true);- simdgroup_multiply_accumulate(OutGate[tile], lhs_frag, rhs_gate, OutGate[tile]);- }- }- }- }- } // K loop-- // ---- Bias tile [Bn] ----- // Note: bias layout is [up | gate] in contiguous FF chunks.- for (uint c_local = lin; c_local < Bn; c_local += threads_total) {- const uint c_global = col_tg_off + c_local;- bias_up[c_local] = (c_global < d_ff) ? float(up_biases[bias_base + c_global]) : 0.0f;- if (gating_sel > 1u) {- bias_gate[c_local] = (c_global < d_ff) ? float(up_biases[bias_base + d_ff + c_global]) : 0.0f;- }- }-- threadgroup_barrier(mem_flags::mem_threadgroup);-- // ---- Epilogue: activation/gating → hidden_out (f32) ----- const uint sg_lane_start = thread_context.simdgroup_index * 32;- const uint sg_lane_end = sg_lane_start + 32;- if (lin < sg_lane_start || lin >= sg_lane_end)- return;- const uint sg_lin = lin - sg_lane_start;- // Lane→(row,col) mapping for 8x8 simdgroup fragments, matches the fragment- // layout used above.- const uint lane_qid = sg_lin >> 2;- const uint lane_row = (lane_qid & 4u) + ((sg_lin >> 1) & 3u);- const uint lane_col_base = ((lane_qid & 2u) * 2u) + ((sg_lin & 1u) * 2u);-- METAL_PRAGMA_UNROLL- for (uint n_sub = 0; n_sub < SgBn; n_sub += SG_TILE) {- const uint c_idx = n_sub / SG_TILE;- METAL_PRAGMA_UNROLL- for (uint m_sub = 0; m_sub < SgBm; m_sub += SG_TILE) {- const uint r_idx = m_sub / SG_TILE;- const uint tile = r_idx * (SgBn / SG_TILE) + c_idx;-- const uint tile_row_base = row_sg_off + m_sub;- const uint tile_col_base = col_sg_off + n_sub;- const uint local_row = tile_row_base + lane_row;- if (local_row >= m_rows || tile_col_base >= n_cols) {- continue;- }-- const auto up_frag = OutUp[tile].thread_elements();- float gate_frag_0 = 0.0f;- float gate_frag_1 = 0.0f;- if (gating_sel > 1u) {- const auto gate_frag = OutGate[tile].thread_elements();- gate_frag_0 = gate_frag[0];- gate_frag_1 = gate_frag[1];- }-- const uint col0 = tile_col_base + lane_col_base;- const uint col1 = col0 + 1;- const ulong out_row = seg_start + row_tg_off + local_row;-- if (col0 < n_cols) {- float up_v = clamp(up_frag[0] + bias_up[col0], up_clip_min, up_clip_max);- float out_val;- if (gating_sel <= 1u) {- out_val = (gating_sel == 0u) ? activate_gelu(up_v) : activate_silu_alpha(up_v, silu_alpha);- } else {- float gate_v = clamp(gate_frag_0 + bias_gate[col0], gate_clip_min, gate_clip_max);- const float gate_act = (gating_sel == 2u) ? activate_silu_alpha(gate_v, silu_alpha) : activate_gelu(gate_v);- out_val = gate_act * up_v;- }- const ulong out_col = col_tg_off + col0;- hidden_out[out_row * (ulong)d_ff + out_col] = out_val;- }-- if (col1 < n_cols) {- float up_v = clamp(up_frag[1] + bias_up[col1], up_clip_min, up_clip_max);- float out_val;- if (gating_sel <= 1u) {- out_val = (gating_sel == 0u) ? activate_gelu(up_v) : activate_silu_alpha(up_v, silu_alpha);- } else {- float gate_v = clamp(gate_frag_1 + bias_gate[col1], gate_clip_min, gate_clip_max);- const float gate_act = (gating_sel == 2u) ? activate_silu_alpha(gate_v, silu_alpha) : activate_gelu(gate_v);- out_val = gate_act * up_v;- }- const ulong out_col = col_tg_off + col1;- hidden_out[out_row * (ulong)d_ff + out_col] = out_val;- }- }- }-}--// ------------------------ Pass B (hidden @ W2 → output)-// ------------------------ Optimized config: 16x64 output tile, 4 simdgroups-// (128 threads)-constant uint PASSB_BM = 16;-constant uint PASSB_BN = 64;-constant uint PASSB_BK = 64;-constant uint PASSB_SG_BM = 8; // Each simdgroup handles 8x32-constant uint PASSB_SG_BN = 32;-constant uint PASSB_TG_PAD = 4; // Padding to avoid bank conflicts--template <typename T>-VARIANTS(T, float, half, bfloat)-PUBLIC KERNEL(MoeExpertsPrefillPassB)(- device const float* hidden,- device const uint* expert_offsets,- device const T* w2_all,- device const T* down_biases,- device T* output,- constant uint& d_model,- constant uint& d_ff,- constant uint& e,- device const uint* tile_map,- threadgroup float Hs[PASSB_BM * (PASSB_BK + PASSB_TG_PAD)],- threadgroup T Wk[PASSB_BN * (PASSB_BK + PASSB_TG_PAD)],- threadgroup float bias_tile[PASSB_BN],- const ThreadContext thread_context,- const uint n_tile_idx GROUPS(INDIRECT),- const uint row_tile_idx GROUPS(INDIRECT),- const uint lin THREADS(128)-) {- const uint base = row_tile_idx * 3u;- const uint expert_idx = tile_map[base + 0u];- if (expert_idx >= e)- return;-- constexpr uint ROW_TILE = PASSB_BM;- const uint row_off_elems = tile_map[base + 2u];- const uint tile_m = row_off_elems / ROW_TILE;- const uint tile_n = n_tile_idx;-- constexpr uint Bm = PASSB_BM, Bn = PASSB_BN, Bk = PASSB_BK;- constexpr uint SgBm = PASSB_SG_BM, SgBn = PASSB_SG_BN;- constexpr uint TG_PAD = PASSB_TG_PAD;- constexpr uint H_LD = Bk + TG_PAD; // Padded stride for hidden- constexpr uint W_LD = Bk + TG_PAD; // Padded stride for weights- constexpr uint SG_TILE = 8;- constexpr uint TGP_SIZE = 128; // 4 simdgroups × 32 threads-- const uint seg_start = expert_offsets[expert_idx];- const uint seg_end = expert_offsets[expert_idx + 1];- const uint seg_len = (seg_end > seg_start) ? (seg_end - seg_start) : 0u;- if (seg_len == 0)- return;-- const uint row_tg_off = tile_m * Bm;- const uint col_tg_off = tile_n * Bn;- if (row_tg_off >= seg_len || col_tg_off >= d_model)- return;-- const uint m_rows = min(Bm, seg_len - row_tg_off);- const uint n_cols = min(Bn, d_model - col_tg_off);-- const ulong h_base = (ulong)(seg_start + row_tg_off) * (ulong)d_ff;- const ulong w2_expert_base = (ulong)expert_idx * (ulong)d_model * (ulong)d_ff;- const ulong bias_base = (ulong)expert_idx * (ulong)d_model;-- // 2×2 simdgroup layout for 16×64 output (each sg handles 8×32)- const uint row_sg = thread_context.simdgroup_index / 2; // 0-1 → rows- const uint col_sg = thread_context.simdgroup_index % 2; // 0-1 → cols- const uint row_sg_off = row_sg * SgBm;- const uint col_sg_off = col_sg * SgBn;-- // Guard against excessive simdgroups- constexpr uint SG_EXPECTED = (PASSB_BM / SgBm) * (PASSB_BN / SgBn);- if (thread_context.simdgroup_index >= SG_EXPECTED)- return;-- // 4 accumulators per simdgroup: 1×4 layout of 8×8 tiles- constexpr uint TEMP = (SgBm / SG_TILE) * (SgBn / SG_TILE); // 1×4 = 4- metal::simdgroup_float8x8 Out[TEMP];- METAL_PRAGMA_UNROLL- for (uint i = 0; i < TEMP; ++i) {- Out[i] = metal::make_filled_simdgroup_matrix<float, 8, 8>(0.0f);- }-- // Cooperative load config - vec8 loads for better bandwidth- constexpr uint H_VEC = 8;- constexpr uint H_TCOLS = Bk / H_VEC; // 64/8 = 8- const uint h_bi = lin / H_TCOLS;- const uint h_bj = (lin % H_TCOLS) * H_VEC;-- constexpr uint W_VEC = 8;- constexpr uint W_TCOLS = Bk / W_VEC; // 64/8 = 8- constexpr uint W_TROWS = TGP_SIZE / W_TCOLS; // 128/8 = 16- const uint w_bi = lin / W_TCOLS;- const uint w_bj = (lin % W_TCOLS) * W_VEC;-- // K-loop across FF- for (uint k_off = 0; k_off < d_ff; k_off += Bk) {- const uint valid_k = min(Bk, d_ff - k_off);- threadgroup_barrier(mem_flags::mem_threadgroup);-- // ---- Stage hidden [Bm=16, Bk=64] with vec8 loads ----- // 128 threads, 8 vec8 loads per row → 16 rows covered in one pass- {- threadgroup float* my_dst = Hs + h_bi * H_LD + h_bj;- if (h_bi < m_rows && h_bj < valid_k) {- // Safe to read - at least first element is valid- device const float* my_src = hidden + h_base + (ulong)h_bi * (ulong)d_ff + (ulong)(k_off + h_bj);- METAL_PRAGMA_UNROLL- for (uint j = 0; j < H_VEC; j++) {- my_dst[j] = (h_bj + j < valid_k) ? my_src[j] : 0.0f;- }- } else {- METAL_PRAGMA_UNROLL- for (uint j = 0; j < H_VEC; j++) {- my_dst[j] = 0.0f;- }- }- }-- // ---- Stage W2 [Bn=64, Bk=64] with vec8 loads ----- // Need 4 passes: 128 threads cover 16 rows, Bn=64 needs 4×16- {- for (uint pass = 0; pass < 4; pass++) {- const uint row = w_bi + pass * W_TROWS;- threadgroup T* my_dst = Wk + row * W_LD + w_bj;- if (row < n_cols && w_bj < valid_k) {- // Safe to read - at least first element is valid- const uint gcol = col_tg_off + row;- device const T* my_src = w2_all + w2_expert_base + (ulong)gcol * (ulong)d_ff + (ulong)(k_off + w_bj);- METAL_PRAGMA_UNROLL- for (uint j = 0; j < W_VEC; j++) {- my_dst[j] = (w_bj + j < valid_k) ? static_cast<T>(float(my_src[j])) : static_cast<T>(0.0f);- }- } else {- METAL_PRAGMA_UNROLL- for (uint j = 0; j < W_VEC; j++) {- my_dst[j] = static_cast<T>(0.0f);- }- }- }- }-- threadgroup_barrier(mem_flags::mem_threadgroup);-- // ---- Serpentine MMA: 1×4 tiles per simdgroup ----- METAL_PRAGMA_UNROLL- for (uint kk = 0; kk < Bk; kk += SG_TILE) {- // Load 1 LHS fragment- metal::simdgroup_float8x8 lhs;- simdgroup_load(lhs, Hs, H_LD, ulong2(kk, row_sg_off));-- // Load 4 RHS fragments (cols)- metal::simdgroup_matrix<T, 8, 8> rhs0, rhs1, rhs2, rhs3;- simdgroup_load(rhs0, Wk, W_LD, ulong2(kk, col_sg_off), true);- simdgroup_load(rhs1, Wk, W_LD, ulong2(kk, col_sg_off + 8), true);- simdgroup_load(rhs2, Wk, W_LD, ulong2(kk, col_sg_off + 16), true);- simdgroup_load(rhs3, Wk, W_LD, ulong2(kk, col_sg_off + 24), true);-- // Serpentine pattern: 0→1→2→3 (forward for first row)- simdgroup_multiply_accumulate(Out[0], lhs, rhs0, Out[0]);- simdgroup_multiply_accumulate(Out[1], lhs, rhs1, Out[1]);- simdgroup_multiply_accumulate(Out[2], lhs, rhs2, Out[2]);- simdgroup_multiply_accumulate(Out[3], lhs, rhs3, Out[3]);- }- }-- // ---- Bias tile (cooperative load) ----- for (uint c_local = lin; c_local < Bn; c_local += TGP_SIZE) {- const uint c_global = col_tg_off + c_local;- bias_tile[c_local] = (c_global < d_model) ? float(down_biases[bias_base + c_global]) : 0.0f;- }-- threadgroup_barrier(mem_flags::mem_threadgroup);-- // ---- Writeout with fragment extraction ----- const uint lane_qid = thread_context.simd_lane_id >> 2;- const uint lane_row = (lane_qid & 4u) + ((thread_context.simd_lane_id >> 1) & 3u);- const uint lane_col_base = ((lane_qid & 2u) * 2u) + ((thread_context.simd_lane_id & 1u) * 2u);-- const uint local_row = row_sg_off + lane_row;- if (local_row >= m_rows)- return;- const ulong out_row = seg_start + row_tg_off + local_row;-- METAL_PRAGMA_UNROLL- for (uint ni = 0; ni < 4; ni++) { // 4 col tiles per simdgroup- const auto accum = Out[ni].thread_elements();- const uint local_col = col_sg_off + ni * 8 + lane_col_base;-- if (local_col < n_cols) {- const ulong out_col = col_tg_off + local_col;- const float val = accum[0] + bias_tile[local_col];- output[out_row * (ulong)d_model + out_col] = T(val);- }- if (local_col + 1 < n_cols) {- const ulong out_col = col_tg_off + local_col + 1;- const float val = accum[1] + bias_tile[local_col + 1];- output[out_row * (ulong)d_model + out_col] = T(val);- }- }-}diff --git a/crates/uzu-engine/src/backends/metal/kernel/moe/finalize.metal b/crates/uzu-engine/src/backends/metal/kernel/moe/finalize.metaldeleted file mode 100644index f94928258..000000000--- a/crates/uzu-engine/src/backends/metal/kernel/moe/finalize.metal+++ /dev/null@@ -1,54 +0,0 @@-#include <metal_stdlib>-#include "../common/dsl.h"--#define BM 32-#define BN 64--template <typename T>-VARIANTS(T, float, half, bfloat)-PUBLIC KERNEL(MoeFinalize)(- device const int* tok2row, // [T*K], -1 if dropped- device const T* probs, // [T*K]- device const T* y_partial, // [sum_k, d_model]- device T* y, // [T, d_model]- constant uint& t_count,- constant uint& d_model,- constant uint& k_input,- const uint lid THREADS(128),- const uint tg_n GROUPS(d_model.div_ceil(BN)),- const uint tg_m GROUPS(t_count.div_ceil(BM))-) {- const uint tile_n0 = tg_n * BN;- const uint tile_m0 = tg_m * BM;- const uint m_rows = min((uint)BM, t_count - tile_m0);- const uint n_cols = min((uint)BN, d_model - tile_n0);-- // 128 threads per TG expected- for (uint idx = lid; idx < m_rows * n_cols; idx += 128u) {- const uint mi = idx / n_cols;- const uint nj = idx % n_cols;- const uint t = tile_m0 + mi;- const uint f = tile_n0 + nj;- float acc = 0.0f;- const uint base = t * k_input;- for (uint k = 0; k < k_input; ++k) {- const int row = tok2row[base + k];- if (row >= 0) {- const ulong yidx = (ulong)(uint)row * (ulong)d_model + (ulong)f;- float prob = (float)probs[base + k];- if (!isfinite(prob)) {- prob = 0.0f;- }- float val = (float)y_partial[yidx];- if (!isfinite(val)) {- val = 0.0f;- }- acc = fma(prob, val, acc);- }- }- if (!isfinite(acc)) {- acc = 0.0f;- }- y[t * d_model + f] = T(acc);- }-}diff --git a/crates/uzu-engine/src/backends/metal/kernel/moe/gather.metal b/crates/uzu-engine/src/backends/metal/kernel/moe/gather.metaldeleted file mode 100644index 98365a997..000000000--- a/crates/uzu-engine/src/backends/metal/kernel/moe/gather.metal+++ /dev/null@@ -1,112 +0,0 @@-#include <metal_stdlib>-#include "../common/dsl.h"--// Tiling config for 2D gather-#define ROWS_PER_TG 8-#define COL_VECS 32u // 256 threads = 8×32-#define ELEMS_PER_THREAD 8u // each thread copies 8 elements (2× vec4)-#define THREADS_PER_THREADGROUP 256--template <typename T>-VARIANTS(T, float, half, bfloat)-PUBLIC KERNEL(MoeGatherXPerm2D)(- device const T* x,- device const int* bucketed_ids,- device T* x_perm,- device const uint* sumk_buf,- constant uint& d_model,- constant uint& t,- constant uint& k,- const uint tgid_x GROUPS((t * k).div_ceil(ROWS_PER_TG)),- const uint lid THREADS(THREADS_PER_THREADGROUP)-) {- const uint total_rows = sumk_buf[0];- if (total_rows == 0u || d_model == 0u) {- return;- }-- // Thread organization: lid = row_in_tg * COL_VECS + col_vec- const uint row_in_tg = lid / COL_VECS;- const uint col_vec = lid % COL_VECS;-- // Global row assignment- const uint base_row = tgid_x * ROWS_PER_TG;- const uint row = base_row + row_in_tg;- if (row >= total_rows) {- return;- }-- int token = bucketed_ids[row];- if (token < 0) {- return;- }-- const device T* src = x + (ulong)(uint)token * (ulong)d_model;- device T* dst = x_perm + (ulong)row * (ulong)d_model;-- // Each thread copies ELEMS_PER_THREAD elements- // Thread i processes: col_vec*8, col_vec*8+256, col_vec*8+512, ...- uint col = col_vec * ELEMS_PER_THREAD;- const uint stride = COL_VECS * ELEMS_PER_THREAD;-- // Vectorized copy: 8 elements = 2× vec4- using Vec4 = typename metal::vec<T, 4>;- while (col + 7u < d_model) {- Vec4 v0 = *(reinterpret_cast<const device Vec4*>(src + col));- Vec4 v1 = *(reinterpret_cast<const device Vec4*>(src + col + 4u));- *(reinterpret_cast<device Vec4*>(dst + col)) = v0;- *(reinterpret_cast<device Vec4*>(dst + col + 4u)) = v1;- col += stride;- }-- // Scalar remainder- if (col < d_model) {- const uint limit = min(d_model, col + ELEMS_PER_THREAD);- for (uint tail = col; tail < limit; ++tail) {- dst[tail] = src[tail];- }- }-}--template <typename T>-VARIANTS(T, float, half, bfloat)-PUBLIC KERNEL(MoeGatherXPerm1D)(- device const T* x,- device const int* bucketed_ids,- device T* x_perm,- device const uint* sumk_buf,- constant uint& d_model,- constant uint& t,- constant uint& k,- const uint tgid_x GROUPS((t * k).div_ceil(THREADS_PER_THREADGROUP)),- const uint lid THREADS(THREADS_PER_THREADGROUP)-) {- const uint total_rows = sumk_buf[0];- if (total_rows == 0u || d_model == 0u)- return;-- const uint num_vec4 = d_model / 4u;-- // Each thread processes multiple rows sequentially- for (uint row = lid; row < total_rows; row += THREADS_PER_THREADGROUP) {- int token = bucketed_ids[row];- if (token < 0)- continue;-- const device T* src = x + (ulong)(uint)token * (ulong)d_model;- device T* dst = x_perm + (ulong)row * (ulong)d_model;-- // Vectorized copy: 4 elements at a time- using Vec4 = typename metal::vec<T, 4>;- for (uint vec_idx = 0u; vec_idx < num_vec4; vec_idx++) {- uint col = vec_idx * 4u;- Vec4 v = *(reinterpret_cast<const device Vec4*>(src + col));- *(reinterpret_cast<device Vec4*>(dst + col)) = v;- }-- // Scalar remainder- for (uint col = num_vec4 * 4u; col < d_model; col++) {- dst[col] = src[col];- }- }-}\ No newline at end of filediff --git a/crates/uzu-engine/src/backends/metal/kernel/moe/router_topk.metal b/crates/uzu-engine/src/backends/metal/kernel/moe/router_topk.metaldeleted file mode 100644index b69a5ac9a..000000000--- a/crates/uzu-engine/src/backends/metal/kernel/moe/router_topk.metal+++ /dev/null@@ -1,166 +0,0 @@-#include <metal_stdlib>-#include <metal_simdgroup>-#include "../common/dsl.h"-#include "../common/thread_context.h"-#include "../common/threadgroup_reduce.h"-#include "../generated/router_topk.h"-using namespace metal;-using namespace uzu::router_topk;--#define THREADS_PER_TG 256--constant float NEG_INF = -INFINITY;--template <typename ScalarT>-VARIANTS(ScalarT, half, bfloat, float)-PUBLIC KERNEL(MoeRouterTopK)(- const device ScalarT* input,- const device ScalarT* weight,- const device ScalarT* bias OPTIONAL(has_biases),- const device ScalarT* router_scale OPTIONAL(has_router_scales),- const device ScalarT* per_expert_scale OPTIONAL(has_per_expert_scales),- device int* topk_ids,- device ScalarT* topk_probs,- constant uint& t,- constant uint& d_model,- constant uint& e,- constant uint& k,- constant bool& renorm,- constant float& router_norm_epsilon OPTIONAL(normalize_router_input),- constant float& router_input_scale OPTIONAL(has_router_input_scale),- const bool has_biases SPECIALIZE,- const bool has_router_scales SPECIALIZE,- const bool has_per_expert_scales SPECIALIZE,- const bool has_router_input_scale SPECIALIZE,- const bool normalize_router_input SPECIALIZE,- threadgroup float4 x_cache[ROUTER_TOPK_MAX_MODEL_DIM / 4],- threadgroup float logits_shared[ROUTER_TOPK_MAX_EXPERTS],- threadgroup float reduce_tmp[THREADS_PER_TG],- threadgroup uint reduce_tmp_u[THREADS_PER_TG],- threadgroup uint shared_best_idx[1],- threadgroup float shared_best_val[1],- const ThreadContext thread_context,- const uint tgpig_x GROUPS(1),- const uint token_idx GROUPS(t),- const uint lid THREADS(THREADS_PER_TG)-) {- if (d_model == 0 || e == 0 || k == 0) {- return;- }-- const uint vecs = d_model / 4u;- const device ScalarT* x_vec = input + (ulong)token_idx * (ulong)vecs * 4;-- float local_sum_sq = 0.0f;- for (uint c = lid; c < vecs; c += THREADS_PER_TG) {- const float4 xv = float4(x_vec[c * 4 + 0], x_vec[c * 4 + 1], x_vec[c * 4 + 2], x_vec[c * 4 + 3]);- x_cache[c] = xv;- if (normalize_router_input) {- local_sum_sq += dot(xv, xv);- }- }-- float inv_rms = 1.0f;- if (normalize_router_input) {- const float sum_sq =- threadgroup_cooperative_reduce<SimdReduceSum<float>, THREADS_PER_TG>(local_sum_sq, reduce_tmp, thread_context);- inv_rms = rsqrt(sum_sq / float(d_model) + router_norm_epsilon);- }-- if (normalize_router_input || has_router_input_scale || has_router_scales) {- float4 base_scale4 = float4(inv_rms);- if (has_router_input_scale) {- base_scale4 *= float4(router_input_scale);- }- for (uint c = lid; c < vecs; c += THREADS_PER_TG) {- float4 scale4 = base_scale4;- if (has_router_scales) {- scale4 *=- float4(router_scale[c * 4 + 0], router_scale[c * 4 + 1], router_scale[c * 4 + 2], router_scale[c * 4 + 3]);- }- x_cache[c] *= scale4;- }- }- threadgroup_barrier(mem_flags::mem_threadgroup);-- for (uint row = thread_context.simdgroup_index; row < e; row += thread_context.simdgroups_per_threadgroup) {- const device ScalarT* w_vec = weight + (ulong)row * (ulong)vecs * 4;-- float4 accum4 = float4(0.0f);- for (uint c = thread_context.simd_lane_id; c < vecs; c += 32u) {- const float4 wv = float4(w_vec[c * 4 + 0], w_vec[c * 4 + 1], w_vec[c * 4 + 2], w_vec[c * 4 + 3]);- const float4 xv = x_cache[c];- accum4 = fma(wv, xv, accum4);- }- float sum = (accum4.x + accum4.y) + (accum4.z + accum4.w);- sum = simd_sum(sum);- if (simd_is_first()) {- if (has_biases) {- sum += float(bias[row]);- }- logits_shared[row] = sum;- }- }- threadgroup_barrier(mem_flags::mem_threadgroup);-- const uint effective_k = min(k, ROUTER_TOPK_MAX_SELECTED_EXPERTS);- for (uint sel = 0; sel < effective_k; ++sel) {- float local_best = NEG_INF;- uint local_idx = 0xFFFFFFFFu;- for (uint row = lid; row < e; row += THREADS_PER_TG) {- float candidate = logits_shared[row];- if (candidate > local_best || (candidate == local_best && row < local_idx)) {- local_best = candidate;- local_idx = row;- }- }-- float max_val =- threadgroup_cooperative_reduce<SimdReduceMax<float>, THREADS_PER_TG>(local_best, reduce_tmp, thread_context);-- uint candidate_id = (local_best == max_val) ? local_idx : 0xFFFFFFFFu;- uint best_idx =- threadgroup_cooperative_reduce<SimdReduceMin<uint>, THREADS_PER_TG>(candidate_id, reduce_tmp_u, thread_context);-- if (lid == 0) {- shared_best_idx[0] = best_idx;- shared_best_val[0] = max_val;- }- threadgroup_barrier(mem_flags::mem_threadgroup);-- uint winner_idx = shared_best_idx[0];- float winner_val = shared_best_val[0];- if (lid == 0 && winner_idx < ROUTER_TOPK_MAX_EXPERTS) {- if (!renorm && has_per_expert_scales) {- winner_val *= float(per_expert_scale[winner_idx]);- }- logits_shared[winner_idx] = NEG_INF;- topk_ids[token_idx * k + sel] = int(winner_idx);- topk_probs[token_idx * k + sel] = static_cast<ScalarT>(winner_val);- }- threadgroup_barrier(mem_flags::mem_threadgroup);- }-- if (lid == 0 && renorm && effective_k > 0) {- device ScalarT* out_probs = topk_probs + token_idx * k;- float max_logit = -INFINITY;- for (uint i = 0; i < effective_k; ++i) {- max_logit = fmax(max_logit, float(out_probs[i]));- }- float sum_exp = 0.0f;- for (uint i = 0; i < effective_k; ++i) {- sum_exp += exp(float(out_probs[i]) - max_logit);- }- float default_prob = 1.0f / float(effective_k);- float inv_sum = (sum_exp > 0.0f) ? (1.0f / sum_exp) : default_prob;- for (uint i = 0; i < effective_k; ++i) {- float prob = (sum_exp > 0.0f) ? exp(float(out_probs[i]) - max_logit) * inv_sum : default_prob;- const int expert_id = topk_ids[token_idx * k + i];- const float expert_scale = (has_per_expert_scales && expert_id >= 0) ? float(per_expert_scale[expert_id]) : 1.0f;- out_probs[i] = static_cast<ScalarT>(prob * expert_scale);- }- for (uint i = effective_k; i < k; ++i) {- out_probs[i] = static_cast<ScalarT>(0.0f);- }- }-}diff --git a/crates/uzu-engine/src/backends/metal/kernel/moe/scatter_buckets.metal b/crates/uzu-engine/src/backends/metal/kernel/moe/scatter_buckets.metaldeleted file mode 100644index 85adc349f..000000000--- a/crates/uzu-engine/src/backends/metal/kernel/moe/scatter_buckets.metal+++ /dev/null@@ -1,210 +0,0 @@-#include <metal_stdlib>-#include <metal_atomic>-#include "../common/dsl.h"-#include "../common/thread_context.h"-using namespace metal;--#define BLOCK_SIZE 256-#define TILE_E 512-#define SIMD_WIDTH 32-#define NUM_SG (BLOCK_SIZE / SIMD_WIDTH)--// 4B: Scatter into expert-major buckets using atomic-free device writes-template <typename ProbT>-inline void moe_scatter_buckets_impl(- device const int* topk_ids,- device const ProbT* topk_probs,- device const uint* offsets,- device const uint* block_bases,- device const uint* block_alloc,- device int* out_ids,- device ProbT* out_probs,- device int* tok2row_opt,- uint T,- uint E,- uint K,- uint num_tiles,- uint lid,- uint3 tgpig,- threadgroup _atomic<uint>* sg_counts,- threadgroup uint* sg_base,- const thread ThreadContext& thread_context-) {- if (E == 0 || T == 0 || K == 0)- return;- const uint block_id = tgpig.x;- const uint t_start = block_id * BLOCK_SIZE;- const uint t_end = min(t_start + BLOCK_SIZE, T);-- // Per-tile processing- for (uint e0 = 0; e0 < E; e0 += TILE_E) {- const uint tile_e = (e0 + TILE_E <= E) ? TILE_E : (E - e0);-- // Zero counts and bases for active tile- for (uint te = lid; te < tile_e; te += BLOCK_SIZE) {- for (uint sg = 0; sg < NUM_SG; ++sg) {- atomic_store_explicit(&sg_counts[sg * TILE_E + te], 0u, memory_order_relaxed);- sg_base[sg * TILE_E + te] = 0u;- }- }- threadgroup_barrier(mem_flags::mem_threadgroup);-- // Phase 1: per-sg counting using threadgroup atomics (order-independent)- for (uint t = t_start + lid; t < t_end; t += BLOCK_SIZE) {- const uint base = t * K;- for (uint k = 0; k < K; ++k) {- int eid = topk_ids[base + k];- if (eid >= 0) {- const uint ue = (uint)eid;- if (ue >= e0 && ue < e0 + tile_e) {- const uint te = ue - e0;- atomic_fetch_add_explicit(- &sg_counts[thread_context.simdgroup_index * TILE_E + te],- 1u,- memory_order_relaxed- );- }- }- }- }- threadgroup_barrier(mem_flags::mem_threadgroup);-- // Phase 2: compute per-sg bases via prefix across sgs with capacity clamp- for (uint te = lid; te < tile_e; te += BLOCK_SIZE) {- const uint tile_id = e0 / TILE_E;- const uint base_idx_block = (block_id * num_tiles + tile_id) * TILE_E + te;- const uint alloc_total = block_alloc[base_idx_block];- uint prefix = 0u;- for (uint sg = 0; sg < NUM_SG; ++sg) {- const uint idx = sg * TILE_E + te;- const uint c = atomic_load_explicit(&sg_counts[idx], memory_order_relaxed);- const uint room = (prefix < alloc_total) ? (alloc_total - prefix) : 0u;- const uint take = min(c, room);- sg_base[idx] = prefix;- prefix += take;- }- // Reuse sg_counts as write indices: reset to zero- for (uint sg = 0; sg < NUM_SG; ++sg) {- atomic_store_explicit(&sg_counts[sg * TILE_E + te], 0u, memory_order_relaxed);- }- }- threadgroup_barrier(mem_flags::mem_threadgroup);-- // Phase 3: deterministic write with 32-lane sequencing, no device atomics- for (uint step = 0; step < thread_context.simdgroup_size; ++step) {- if (thread_context.simd_lane_id == step) {- const uint t = t_start + lid;- if (t < t_end) {- const uint base = t * K;- for (uint k = 0; k < K; ++k) {- int eid = topk_ids[base + k];- if (eid >= 0) {- const uint ue = (uint)eid;- if (ue >= e0 && ue < e0 + tile_e) {- const uint te = ue - e0;- const uint tile_id = e0 / TILE_E;- const uint base_idx_block = (block_id * num_tiles + tile_id) * TILE_E + te;- const uint idx_sg_te = thread_context.simdgroup_index * TILE_E + te;- const uint local = atomic_fetch_add_explicit(&sg_counts[idx_sg_te], 1u, memory_order_relaxed);- const uint base_s = sg_base[idx_sg_te];-- const uint slot = offsets[ue] + block_bases[base_idx_block] + base_s + local;- out_ids[slot] = (int)t;- out_probs[slot] = topk_probs[base + k];- if (tok2row_opt != nullptr) {- tok2row_opt[base + k] = (int)slot;- }- }- }- }- }- }- threadgroup_barrier(mem_flags::mem_threadgroup);- }- threadgroup_barrier(mem_flags::mem_threadgroup);- }-}--// 4A: Compute per-block base offsets from partials-// partials layout: index = (block_id * num_tiles + tile_id) * TILE_E + te-PUBLIC KERNEL(MoeBlockBasesFromPartials)(- device const uint* partials,- device uint* block_bases,- device uint* block_alloc,- constant uint& e_input,- constant uint& num_blocks,- constant uint& num_tiles,- constant uint& capacity_per_expert, // 0 => no clamp- const uint gid AXIS(num_tiles * 512, 256)-) {- const uint tile_id = gid / TILE_E;- const uint te = gid % TILE_E;- const uint e = tile_id * TILE_E + te;- if (e >= e_input)- return;-- uint running = 0u;- const bool clamp = (capacity_per_expert > 0u);- for (uint b = 0; b < num_blocks; ++b) {- const uint idx = (b * num_tiles + tile_id) * TILE_E + te;- const uint p = partials[idx];- const uint alloc = clamp ? (running < capacity_per_expert ? min(p, capacity_per_expert - running) : 0u) : p;- block_bases[idx] = running;- block_alloc[idx] = alloc;- running += alloc;- if (clamp && running >= capacity_per_expert) {- // Remainder allocate zero and base stays at capacity- for (uint bb = b + 1; bb < num_blocks; ++bb) {- const uint idx2 = (bb * num_tiles + tile_id) * TILE_E + te;- block_bases[idx2] = running;- block_alloc[idx2] = 0u;- }- break;- }- }-}--// Map variants-template <typename T>-VARIANTS(T, float, half, bfloat)-PUBLIC KERNEL(MoeScatterBucketsMap)(- device const int* topk_ids,- device const T* topk_probs,- device const uint* offsets,- device const uint* block_bases,- device const uint* block_alloc,- device int* out_ids,- device T* out_probs,- constant uint& t,- constant uint& e,- constant uint& k,- constant uint& num_blocks,- constant uint& num_tiles,- device int* tok2row,- threadgroup _atomic<uint> sg_counts[NUM_SG * TILE_E],- threadgroup uint sg_base[NUM_SG * TILE_E],- const ThreadContext thread_context,- const uint tgpig_x GROUPS(num_blocks),- const uint lid THREADS(256)-) {- const uint3 tgpig = {tgpig_x, 0, 0};- moe_scatter_buckets_impl<T>(- topk_ids,- topk_probs,- offsets,- block_bases,- block_alloc,- out_ids,- out_probs,- tok2row,- t,- e,- k,- num_tiles,- lid,- tgpig,- sg_counts,- sg_base,- thread_context- );-}diff --git a/crates/uzu-engine/src/backends/metal/kernel/moe/tiles_map.metal b/crates/uzu-engine/src/backends/metal/kernel/moe/tiles_map.metaldeleted file mode 100644index d49745802..000000000--- a/crates/uzu-engine/src/backends/metal/kernel/moe/tiles_map.metal+++ /dev/null@@ -1,107 +0,0 @@-#include <metal_stdlib>-#include <metal_atomic>-#include "../common/dsl.h"--#define BM 16--// Compute per-expert tile counts: tiles_e = ceil((seg_len)/BM)-PUBLIC KERNEL(MoeTileCounts)(- device const uint* offsets, // [e+1]- device uint* tile_counts, // [e]- constant uint& e,- const uint gid AXIS(e, 256)-) {- const uint start = offsets[gid];- const uint end = offsets[gid + 1];- const uint seg_len = (end > start) ? (end - start) : 0u;- const uint tiles = (seg_len == 0u) ? 0u : ((seg_len + BM - 1u) / BM);- tile_counts[gid] = tiles;-}--// Single-threadgroup exclusive scan over tile_counts -> tile_row_offsets, also-// writes total_tiles-PUBLIC KERNEL(MoeTileScan)(- device const uint* tile_counts, // [e]- device uint* tile_row_offsets, // [e+1]- device uint* total_tiles_buf, // [>=2]- constant uint& e,- threadgroup uint scratch[1024], // supports up to E<=1023- uint tid THREADS(256)-) {- if (tid < e) {- scratch[tid] = tile_counts[tid];- }- threadgroup_barrier(mem_flags::mem_threadgroup);-- // Inclusive scan in-place over scratch- // Simple O(log E) scan; E is small- for (uint offset = 1u; offset < e; offset *= 2u) {- uint val = 0u;- if (tid < e) {- val = scratch[tid];- if (tid >= offset) {- val += scratch[tid - offset];- }- }- threadgroup_barrier(mem_flags::mem_threadgroup);- if (tid < e) {- scratch[tid] = val;- }- threadgroup_barrier(mem_flags::mem_threadgroup);- }-- if (tid == 0u) {- tile_row_offsets[0] = 0u;- }- if (tid < e) {- // exclusive: shift right by one- const uint prefix = (tid == 0u) ? 0u : scratch[tid - 1u];- tile_row_offsets[tid] = prefix;- }- if (tid == 0u) {- tile_row_offsets[e] = (e == 0u) ? 0u : scratch[e - 1u];- total_tiles_buf[0] = tile_row_offsets[e];- total_tiles_buf[1] = 0u;- total_tiles_buf[2] = 0u;- total_tiles_buf[3] = 0u;- total_tiles_buf[4] = 0u;- total_tiles_buf[5] = 0u;- total_tiles_buf[6] = 0u;- total_tiles_buf[7] = 0u;- }-}--// Build flattened tile_map of length total_tiles; each entry is 3 uints:-// [3*i+0]=expert_idx, [3*i+1]=seg_start, [3*i+2]=tile_m0-PUBLIC KERNEL(MoeBuildTileMap)(- device const uint* offsets, // [e+1]- device const uint* tile_row_offsets, // [e+1]- device const uint* tile_counts, // [e]- device uint* tile_map, // [3*total_tiles]- constant uint& e,- uint gid AXIS(e, 256)-) {- const uint start = offsets[gid];- const uint tiles = tile_counts[gid];- const uint base = tile_row_offsets[gid];- for (uint t = 0u; t < tiles; ++t) {- const uint idx = base + t;- tile_map[3u * idx + 0u] = gid;- tile_map[3u * idx + 1u] = start;- tile_map[3u * idx + 2u] = t * BM;- }-}--// Write MTLDispatchThreadgroupsIndirectArguments {x, y, z} where:-// x = num_tiles_n (computed on CPU and passed in), y = total_tiles, z = 1-PUBLIC KERNEL(MoeWriteDispatchArgs)(- device const uint* total_tiles_buf, // [>=1], total_tiles_buf[0] = total_rows- device uint* dispatch_args, // [3] u32- constant uint& num_tiles_n, // x dimension- uint gid AXIS(1, 1)-) {- const uint total_rows = total_tiles_buf[0];- dispatch_args[0] = num_tiles_n; // x- dispatch_args[1] = total_rows; // y- dispatch_args[2] = 1u; // z-}diff --git a/crates/uzu-engine/src/backends/metal/kernel/moe/tiles_pass_a.metal b/crates/uzu-engine/src/backends/metal/kernel/moe/tiles_pass_a.metaldeleted file mode 100644index 1e566a1ea..000000000--- a/crates/uzu-engine/src/backends/metal/kernel/moe/tiles_pass_a.metal+++ /dev/null@@ -1,128 +0,0 @@-#include <metal_stdlib>-#include "../common/dsl.h"--// === Helper kernels for indirect dispatch of Pass A ===--// Count tiles per expert: tiles = (num_rows > 0) ? num_rows * h_blocks : 0-PUBLIC KERNEL(MoePassATileCounts)(- device const uint* expert_offsets, // [e+1]- device uint* tile_counts, // [e]- constant uint& e,- constant uint& h_blocks,- const uint tid AXIS(e, 256)-) {- const uint start = expert_offsets[tid];- const uint end = expert_offsets[tid + 1];- const uint num_rows = end - start;- tile_counts[tid] = (num_rows > 0) ? (num_rows * h_blocks) : 0;-}--// Exclusive scan of tile_counts to get tile_offsets and total_tiles-PUBLIC KERNEL(MoePassATileScan)(- device const uint* tile_counts, // [e]- device uint* tile_offsets, // [e+1]- device uint* total_tiles, // [1]- constant uint& e,- threadgroup uint scratch[1024],- uint gid GROUPS(1),- uint lid THREADS(1024)-) {- // Simple single-threadgroup scan (works for E <= 1024)- const uint idx = lid;-- // Load into threadgroup memory- if (idx < e) {- scratch[idx] = tile_counts[idx];- } else {- scratch[idx] = 0;- }- threadgroup_barrier(mem_flags::mem_threadgroup);-- // Kogge-Stone scan- uint val = scratch[idx];- for (uint offset = 1; offset < 1024; offset *= 2) {- uint temp = 0;- if (idx >= offset && idx < e) {- temp = scratch[idx - offset];- }- threadgroup_barrier(mem_flags::mem_threadgroup);- if (idx >= offset && idx < e) {- val += temp;- scratch[idx] = val;- }- threadgroup_barrier(mem_flags::mem_threadgroup);- }-- // Write exclusive scan (shift right by 1)- if (idx == 0) {- tile_offsets[0] = 0;- }- if (idx < e) {- tile_offsets[idx + 1] = scratch[idx];- if (idx == e - 1) {- total_tiles[0] = scratch[idx];- }- }-}--// Build row→expert map: one thread per routed row-PUBLIC KERNEL(MoePassABuildRowMap)(- device const uint* expert_offsets, // [e+1]- device uint* row_expert_map, // [total_rows]- constant uint& total_rows,- constant uint& e,- uint tid AXIS(total_rows, 256)-) {- uint left = 0u;- uint right = e;- const uint row = tid;-- while (left + 1u < right) {- const uint mid = (left + right) >> 1;- if (row < expert_offsets[mid]) {- right = mid;- } else {- left = mid;- }- }-- row_expert_map[row] = left;-}--// Build tile map entries from row→expert map-PUBLIC KERNEL(MoePassABuildTileMap)(- device const uint* expert_offsets, // [E+1]- device const uint* tile_offsets, // [E+1]- device const uint* row_expert_map, // [total_rows]- device uint* tile_map, // [total_tiles * 3]- constant uint& total_rows,- constant uint& h_blocks,- uint tid AXIS(total_rows.saturating_mul(h_blocks), 256)-) {- const uint row_idx = tid / h_blocks;- const uint h_block = tid % h_blocks;- if (row_idx >= total_rows) {- return;- }-- const uint expert_idx = row_expert_map[row_idx];- const uint row_start = expert_offsets[expert_idx];- const uint row_in_expert = row_idx - row_start;- const uint tile_base = tile_offsets[expert_idx] + row_in_expert * h_blocks + h_block;-- tile_map[tile_base * 3 + 0] = h_block;- tile_map[tile_base * 3 + 1] = expert_idx;- tile_map[tile_base * 3 + 2] = row_in_expert;-}--// Write dispatch args for indirect dispatch (reusable from tiled version)-PUBLIC KERNEL(MoePassAWriteDispatchArgs)(- device const uint* total_tiles, // [1]- device uint* dispatch_args, // [3] - MTLDispatchThreadgroupsIndirectArguments- constant uint& num_tiles_y, // usually 1 for Pass A- uint tid AXIS(1, 1)-) {- dispatch_args[0] = total_tiles[0]; // x dimension = total tiles- dispatch_args[1] = num_tiles_y; // y dimension- dispatch_args[2] = 1; // z dimension-}diff --git a/crates/uzu-engine/src/encodable_block/mlp/mod.rs b/crates/uzu-engine/src/encodable_block/mlp/mod.rsindex 53e533370..5ae64a74e 100644--- a/crates/uzu-engine/src/encodable_block/mlp/mod.rs+++ b/crates/uzu-engine/src/encodable_block/mlp/mod.rs@@ -1,11 +1,9 @@mod dense;mod gate_act_mul;-mod moe;pub use dense::DenseMlp;use derive_more::Debug;use gate_act_mul::MlpGateActMulEncodable;-pub use moe::{MoeBlock, MoeBlockError};use thiserror::Error;use crate::{@@ -31,8 +29,8 @@ pub enum MlpBlockError<B: Backend> {BackendError(#[source] B::Error),#[error("Linear block error: {0}")]LinearBlockError(#[from] LinearBlockError<B>),- #[error("MoeBlock error: {0}")]- MoeBlockError(#[from] MoeBlockError<B>),+ #[error("Mixture of experts is not supported")]+ UnsupportedMixtureOfExperts,}impl<B: Backend> dyn Mlp<B> {@@ -77,16 +75,7 @@ impl<B: Backend> dyn Mlp<B> {Ok((Box::new(DenseMlp::new(up_projection, gate, down_projection)), up_input_hadamard_factors))},- AnyMLPConfig::MixtureOfExpertsConfig(mixture_of_experts_config) => Ok((- Box::new(MoeBlock::new(- context,- mixture_of_experts_config,- model_dimension,- data_type,- parameter_tree,- )?),- None,- )),+ AnyMLPConfig::MixtureOfExpertsConfig(_) => Err(MlpBlockError::UnsupportedMixtureOfExperts),}}}diff --git a/crates/uzu-engine/src/encodable_block/mlp/moe/experts_two_pass_decode.rs b/crates/uzu-engine/src/encodable_block/mlp/moe/experts_two_pass_decode.rsdeleted file mode 100644index 9fec0879b..000000000--- a/crates/uzu-engine/src/encodable_block/mlp/moe/experts_two_pass_decode.rs+++ /dev/null@@ -1,123 +0,0 @@-use crate::{- backends::common::{- Backend, BufferRef, CommandBuffer, CommandBufferEncoding,- kernel::{- Kernels, MoeExpertsDecodeDownFused2DKernel, MoeExpertsDecodePassAKernel, MoePassABuildRowMapKernel,- MoePassABuildTileMapKernel, MoePassATileCountsKernel, MoePassATileScanKernel,- MoePassAWriteDispatchArgsKernel,- },- },- data_type::DataType,- encodable_block::mlp::moe::experts_two_pass_prefill::MoeExpertsTwoPassArguments,-};--pub struct MoeExpertsTwoPassDecodeBlock<B: Backend> {- counts: <B::Kernels as Kernels>::MoePassATileCountsKernel,- scan: <B::Kernels as Kernels>::MoePassATileScanKernel,- row_map: <B::Kernels as Kernels>::MoePassABuildRowMapKernel,- build_map: <B::Kernels as Kernels>::MoePassABuildTileMapKernel,- dispatch: <B::Kernels as Kernels>::MoePassAWriteDispatchArgsKernel,- pass_a_indirect: <B::Kernels as Kernels>::MoeExpertsDecodePassAKernel,- fused_down: <B::Kernels as Kernels>::MoeExpertsDecodeDownFused2DKernel,- data_type: DataType,-}--impl<B: Backend> MoeExpertsTwoPassDecodeBlock<B> {- pub fn new(- ctx: &B::Context,- data_type: DataType,- gating_code: u32,- ) -> Result<Self, B::Error> {- Ok(Self {- counts: <B::Kernels as Kernels>::MoePassATileCountsKernel::new(ctx)?,- scan: <B::Kernels as Kernels>::MoePassATileScanKernel::new(ctx)?,- row_map: <B::Kernels as Kernels>::MoePassABuildRowMapKernel::new(ctx)?,- build_map: <B::Kernels as Kernels>::MoePassABuildTileMapKernel::new(ctx)?,- dispatch: <B::Kernels as Kernels>::MoePassAWriteDispatchArgsKernel::new(ctx)?,- pass_a_indirect: <B::Kernels as Kernels>::MoeExpertsDecodePassAKernel::new(ctx, data_type, gating_code)?,- fused_down: <B::Kernels as Kernels>::MoeExpertsDecodeDownFused2DKernel::new(ctx, data_type, DataType::F32)?,- data_type,- })- }-- pub fn encode(- &self,- args: MoeExpertsTwoPassArguments<- impl BufferRef<Backend = B>,- impl BufferRef<Backend = B>,- impl BufferRef<Backend = B>,- >,- command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,- ) -> Result<B::ScratchBuffer, B::Error> {- const BLOCK_M: u32 = 4;- let h_blocks = args.d_ff.div_ceil(BLOCK_M);-- let mut tile_counts = command_buffer.allocate_scratch_for_shape(&[args.num_routed_experts], DataType::U32)?;- self.counts.encode(args.expert_offsets, &mut tile_counts, args.num_routed_experts, h_blocks, command_buffer);-- let mut tile_offsets =- command_buffer.allocate_scratch_for_shape(&[args.num_routed_experts + 1], DataType::U32)?;- let mut total_tiles = command_buffer.allocate_scratch_for_shape(&[1], DataType::U32)?;- self.scan.encode(&tile_counts, &mut tile_offsets, &mut total_tiles, args.num_routed_experts, command_buffer);-- let mut row_expert_map = command_buffer.allocate_scratch_for_shape(&[args.total_rows], DataType::U32)?;- self.row_map.encode(- args.expert_offsets,- &mut row_expert_map,- args.total_rows,- args.num_routed_experts,- command_buffer,- );-- let mut tile_map = command_buffer.allocate_scratch_for_shape(&[args.total_rows, h_blocks, 3], DataType::U32)?;- self.build_map.encode(- args.expert_offsets,- &tile_offsets,- &row_expert_map,- &mut tile_map,- args.total_rows,- h_blocks,- command_buffer,- );-- let mut dispatch_args = command_buffer.allocate_scratch_for_shape(&[3], DataType::U32)?;- self.dispatch.encode(&total_tiles, &mut dispatch_args, 1, command_buffer);-- let mut hidden = command_buffer.allocate_scratch_for_shape(&[args.total_rows, args.d_ff], DataType::F32)?;-- self.pass_a_indirect.encode(- args.x_perm,- args.expert_offsets,- args.w13_all,- &mut hidden,- args.up_biases,- args.d_model,- args.d_ff,- args.num_routed_experts,- args.gate_clip_min,- args.gate_clip_max,- args.up_clip_min,- args.up_clip_max,- args.silu_alpha,- &tile_map,- &dispatch_args,- command_buffer,- );-- let mut output = command_buffer.allocate_scratch_for_shape(&[args.total_rows, args.d_model], self.data_type)?;- self.fused_down.encode(- &hidden,- &row_expert_map,- args.w2_all,- args.down_biases,- &mut output,- args.total_rows,- args.d_model,- args.d_ff,- args.num_routed_experts,- command_buffer,- );-- Ok(output)- }-}diff --git a/crates/uzu-engine/src/encodable_block/mlp/moe/experts_two_pass_prefill.rs b/crates/uzu-engine/src/encodable_block/mlp/moe/experts_two_pass_prefill.rsdeleted file mode 100644index 37a05cedb..000000000--- a/crates/uzu-engine/src/encodable_block/mlp/moe/experts_two_pass_prefill.rs+++ /dev/null@@ -1,138 +0,0 @@-use crate::{- backends::common::{- Backend, BufferRef, CommandBuffer, CommandBufferEncoding, Kernels,- kernel::{- MoeBuildTileMapKernel, MoeExpertsPrefillPassAKernel, MoeExpertsPrefillPassBKernel, MoeTileCountsKernel,- MoeTileScanKernel, MoeWriteDispatchArgsKernel,- },- },- data_type::DataType,-};--pub struct MoeExpertsTwoPassPrefillBlock<B: Backend> {- counts: <B::Kernels as Kernels>::MoeTileCountsKernel,- scan: <B::Kernels as Kernels>::MoeTileScanKernel,- build: <B::Kernels as Kernels>::MoeBuildTileMapKernel,- dispatch: <B::Kernels as Kernels>::MoeWriteDispatchArgsKernel,- pass_a_indirect: <B::Kernels as Kernels>::MoeExpertsPrefillPassAKernel,- pass_b_indirect: <B::Kernels as Kernels>::MoeExpertsPrefillPassBKernel,- data_type: DataType,-}--impl<B: Backend> MoeExpertsTwoPassPrefillBlock<B> {- pub fn new(- ctx: &B::Context,- data_type: DataType,- gating_code: u32,- ) -> Result<Self, B::Error> {- Ok(Self {- counts: <B::Kernels as Kernels>::MoeTileCountsKernel::new(ctx)?,- scan: <B::Kernels as Kernels>::MoeTileScanKernel::new(ctx)?,- build: <B::Kernels as Kernels>::MoeBuildTileMapKernel::new(ctx)?,- dispatch: <B::Kernels as Kernels>::MoeWriteDispatchArgsKernel::new(ctx)?,- pass_a_indirect: <B::Kernels as Kernels>::MoeExpertsPrefillPassAKernel::new(ctx, data_type, gating_code)?,- pass_b_indirect: <B::Kernels as Kernels>::MoeExpertsPrefillPassBKernel::new(ctx, data_type)?,- data_type,- })- }-- pub fn encode(- &self,- args: MoeExpertsTwoPassArguments<- impl BufferRef<Backend = B>,- impl BufferRef<Backend = B>,- impl BufferRef<Backend = B>,- >,- command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,- ) -> Result<B::ScratchBuffer, B::Error> {- let mut tile_counts = command_buffer.allocate_scratch_for_shape(&[args.num_routed_experts], DataType::U32)?;- self.counts.encode(args.expert_offsets, &mut tile_counts, args.num_routed_experts, command_buffer);-- let mut tile_offsets =- command_buffer.allocate_scratch_for_shape(&[args.num_routed_experts + 1], DataType::U32)?;- let mut total_tiles = command_buffer.allocate_scratch_for_shape(&[8], DataType::U32)?;- self.scan.encode(&tile_counts, &mut tile_offsets, &mut total_tiles, args.num_routed_experts, command_buffer);-- let mut tile_map = command_buffer.allocate_scratch_for_shape(&[args.total_rows, 3], DataType::U32)?;- self.build.encode(- args.expert_offsets,- &tile_offsets,- &tile_counts,- &mut tile_map,- args.num_routed_experts,- command_buffer,- );-- const COL_TILE_FF: u32 = 32; // Must match PASSA_BN in kernel- let n_tiles_ff = args.d_ff.div_ceil(COL_TILE_FF);-- let mut pass_a_dispatch_args = command_buffer.allocate_scratch_for_shape(&[3], DataType::U32)?;- self.dispatch.encode(&total_tiles, &mut pass_a_dispatch_args, n_tiles_ff, command_buffer);-- let mut hidden = command_buffer.allocate_scratch_for_shape(&[args.total_rows, args.d_ff], DataType::F32)?;- command_buffer.encode_fill(&mut hidden, 0);-- self.pass_a_indirect.encode(- args.x_perm,- args.expert_offsets,- args.w13_all,- args.up_biases,- &mut hidden,- args.d_model,- args.d_ff,- args.num_routed_experts,- args.gate_clip_min,- args.gate_clip_max,- args.up_clip_min,- args.up_clip_max,- args.silu_alpha,- &tile_map,- &pass_a_dispatch_args,- command_buffer,- );-- const COL_TILE_MODEL: u32 = 64;- let n_tiles_model = args.d_model.div_ceil(COL_TILE_MODEL);-- let mut pass_b_dispatch_args = command_buffer.allocate_scratch_for_shape(&[3], DataType::U32)?;- self.dispatch.encode(&total_tiles, &mut pass_b_dispatch_args, n_tiles_model, command_buffer);-- let mut output = command_buffer.allocate_scratch_for_shape(&[args.total_rows, args.d_model], self.data_type)?;- self.pass_b_indirect.encode(- &hidden,- args.expert_offsets,- args.w2_all,- args.down_biases,- &mut output,- args.d_model,- args.d_ff,- args.num_routed_experts,- &tile_map,- &pass_b_dispatch_args,- command_buffer,- );- Ok(output)- }-}--pub struct MoeExpertsTwoPassArguments<- X: BufferRef,- O: BufferRef<Backend = X::Backend>,- W: BufferRef<Backend = X::Backend>,-> {- pub x_perm: X,- pub expert_offsets: O,- pub w13_all: W,- pub w2_all: W,- pub up_biases: W,- pub down_biases: W,- pub total_rows: u32,- pub d_model: u32,- pub d_ff: u32,- pub num_routed_experts: u32,- pub gate_clip_min: f32,- pub gate_clip_max: f32,- pub up_clip_min: f32,- pub up_clip_max: f32,- pub silu_alpha: f32,-}diff --git a/crates/uzu-engine/src/encodable_block/mlp/moe/gather.rs b/crates/uzu-engine/src/encodable_block/mlp/moe/gather.rsdeleted file mode 100644index 7fadae4d6..000000000--- a/crates/uzu-engine/src/encodable_block/mlp/moe/gather.rs+++ /dev/null@@ -1,77 +0,0 @@-use crate::{- backends::common::{- Backend, BufferRef, CommandBuffer, CommandBufferEncoding, Kernels,- kernel::{MoeGatherXPerm1DKernel, MoeGatherXPerm2DKernel},- },- data_type::DataType,-};--enum MoeGatherVariant<B: Backend> {- OneD(<B::Kernels as Kernels>::MoeGatherXPerm1DKernel),- TwoD(<B::Kernels as Kernels>::MoeGatherXPerm2DKernel),-}--pub struct MoeGather<B: Backend> {- variant: MoeGatherVariant<B>,- data_type: DataType,-}--impl<B: Backend> MoeGather<B> {- pub fn new(- ctx: &B::Context,- data_type: DataType,- ) -> Result<Self, B::Error> {- Ok(Self {- variant: if data_type == DataType::BF16 {- MoeGatherVariant::TwoD(<B::Kernels as Kernels>::MoeGatherXPerm2DKernel::new(ctx, data_type)?)- } else {- MoeGatherVariant::OneD(<B::Kernels as Kernels>::MoeGatherXPerm1DKernel::new(ctx, data_type)?)- },- data_type,- })- }-- pub fn encode(- &self,- input: impl BufferRef<Backend = B>,- bucketed_ids: impl BufferRef<Backend = B>,- sumk: impl BufferRef<Backend = B>,- batch_dim: u32,- num_active_experts: u32,- d_model: u32,- command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,- ) -> Result<B::ScratchBuffer, B::Error> {- let mut x_perm =- command_buffer.allocate_scratch_for_shape(&[batch_dim, num_active_experts, d_model], self.data_type)?;- command_buffer.encode_fill(&mut x_perm, 0);-- match &self.variant {- MoeGatherVariant::OneD(kernel) => kernel.encode(- input,- bucketed_ids,- &mut x_perm,- sumk,- d_model,- batch_dim,- num_active_experts,- command_buffer,- ),- MoeGatherVariant::TwoD(kernel) => kernel.encode(- input,- bucketed_ids,- &mut x_perm,- sumk,- d_model,- batch_dim,- num_active_experts,- command_buffer,- ),- };-- Ok(x_perm)- }-}--#[cfg(test)]-#[path = "../../../../unit/encodable_block/moe/moe_gather_test.rs"]-mod tests;diff --git a/crates/uzu-engine/src/encodable_block/mlp/moe/mod.rs b/crates/uzu-engine/src/encodable_block/mlp/moe/mod.rsdeleted file mode 100644index 62b292f04..000000000--- a/crates/uzu-engine/src/encodable_block/mlp/moe/mod.rs+++ /dev/null@@ -1,361 +0,0 @@-//! MoE (Mixture of Experts) block encodable.--mod experts_two_pass_decode;-mod experts_two_pass_prefill;-mod gather;--use derive_more::Debug;-use experts_two_pass_decode::MoeExpertsTwoPassDecodeBlock;-use experts_two_pass_prefill::{MoeExpertsTwoPassArguments, MoeExpertsTwoPassPrefillBlock};-use gather::MoeGather;-use thiserror::Error;--use crate::{- backends::common::{- Backend, CommandBuffer, CommandBufferEncoding, Kernels,- gpu_types::{- ActivationType,- router_topk::{ROUTER_TOPK_MAX_EXPERTS, ROUTER_TOPK_MAX_MODEL_DIM, ROUTER_TOPK_MAX_SELECTED_EXPERTS},- },- kernel::{- MoeBlockBasesFromPartialsKernel, MoeCountsOffsetsFusedKernel, MoeFinalizeKernel, MoeRouterTopKKernel,- MoeScatterBucketsMapKernel,- },- },- config::{- mlp::{mixture_of_experts::MixtureOfExpertsConfig, routing_function::AnyRoutingFunction},- weight_matrix::{AnyWeightMatrixSpec, Layout, full_precision_spec::FullPrecisionSpec},- },- data_type::DataType,- encodable_block::mlp::Mlp,- parameters::{ParameterLoaderError, ParameterTree},-};--pub struct MoeBlock<B: Backend> {- router_topk_kernel: <B::Kernels as Kernels>::MoeRouterTopKKernel,- counts_offsets_kernel: <B::Kernels as Kernels>::MoeCountsOffsetsFusedKernel,- scatter_bases_kernel: <B::Kernels as Kernels>::MoeBlockBasesFromPartialsKernel,- scatter_map_kernel: <B::Kernels as Kernels>::MoeScatterBucketsMapKernel,- gather: MoeGather<B>,- experts_two_pass_decode_block: MoeExpertsTwoPassDecodeBlock<B>,- experts_two_pass_prefill_block: MoeExpertsTwoPassPrefillBlock<B>,- finalize_kernel: <B::Kernels as Kernels>::MoeFinalizeKernel,- router_weights: B::GlobalBuffer,- router_biases: B::GlobalBuffer,- router_renorm: bool,- w13: B::GlobalBuffer,- w2: B::GlobalBuffer,- up_biases: B::GlobalBuffer,- down_biases: B::GlobalBuffer,- model_dim: u32,- hidden_dim: u32,- num_routed_experts: u32,- num_active_experts: u32,- gate_clip_min: f32,- gate_clip_max: f32,- up_clip_min: f32,- up_clip_max: f32,- silu_alpha: f32,- data_type: DataType,-}--#[derive(Debug, Error)]-pub enum MoeBlockError<B: Backend> {- #[error("Backend error: {0}")]- BackendError(#[source] B::Error),- #[error("Parameter loader error: {0}")]- ParameterLoaderError(#[from] ParameterLoaderError<B>),- #[error("MoE model_dim must be nonzero, divisible by 4, and within the fused router capacity")]- InvalidModelDim,- #[error("MoE num_routed_experts must be nonzero and within the fused router capacity")]- InvalidRoutedExpertCount,- #[error("MoE num_active_routed_experts must be nonzero, within TopK capacity, and <= num_routed_experts")]- InvalidActiveExpertCount,- #[error("MoE shared experts are not supported")]- UnsupportedSharedExperts,- #[error("MoE expert gate is not supported")]- UnsupportedExpertGate,- #[error("MoE without router, up, and down biases is not supported")]- UnsupportedNoBiases,- #[error("Unsupported MoE router configuration: {0}")]- UnsupportedRouterConfiguration(String),- #[error("Unsupported MoE expert activation: {0:?}")]- UnsupportedExpertActivation(ActivationType),-}--impl<B: Backend> MoeBlock<B> {- pub fn new(- context: &B::Context,- moe_config: &MixtureOfExpertsConfig,- model_dim: u32,- data_type: DataType,- parameter_tree: &ParameterTree<B>,- ) -> Result<Self, MoeBlockError<B>> {- if model_dim == 0 || model_dim > ROUTER_TOPK_MAX_MODEL_DIM || !model_dim.is_multiple_of(4) {- return Err(MoeBlockError::InvalidModelDim);- }- if !(1..=ROUTER_TOPK_MAX_EXPERTS).contains(&moe_config.num_routed_experts) {- return Err(MoeBlockError::InvalidRoutedExpertCount);- }- if !(1..=ROUTER_TOPK_MAX_SELECTED_EXPERTS.min(moe_config.num_routed_experts))- .contains(&moe_config.num_active_routed_experts)- {- return Err(MoeBlockError::InvalidActiveExpertCount);- }- if moe_config.num_shared_experts != 0 {- return Err(MoeBlockError::UnsupportedSharedExperts);- }- if moe_config.gate_config.is_some() {- return Err(MoeBlockError::UnsupportedExpertGate);- }- if !moe_config.router_has_biases- || !moe_config.expert_config.has_up_biases- || !moe_config.expert_config.has_down_biases- {- return Err(MoeBlockError::UnsupportedNoBiases);- }-- let gating_code = match moe_config.expert_config.activation.act_type() {- ActivationType::GELUApprox => 3,- ActivationType::SILU => 2,- activation_type => return Err(MoeBlockError::UnsupportedExpertActivation(activation_type)),- };-- let router_renorm = matches!(moe_config.routing_function, AnyRoutingFunction::SoftmaxRouting(_));-- let router_tree = parameter_tree.subtree("router");- let router_weights_tree = router_tree.subtree("weights");- let router_spec = router_weights_tree.metadata::<AnyWeightMatrixSpec>("spec")?;- let AnyWeightMatrixSpec::FullPrecisionSpec(FullPrecisionSpec {- layout: Layout::OutputInput,- ..- }) = &router_spec- else {- return Err(MoeBlockError::UnsupportedRouterConfiguration(format!("{router_spec:?}")));- };- let router_weights = router_weights_tree- .leaf("weights")?- .validate(&[moe_config.num_routed_experts, model_dim], data_type)?- .read_buffer()?;- let router_biases =- router_tree.leaf("biases")?.validate(&[moe_config.num_routed_experts], data_type)?.read_buffer()?;-- let experts_tree = parameter_tree.subtree("experts");- let up_tree = experts_tree.subtree("up_projection");- let down_tree = experts_tree.subtree("down_projection");- let up_weights_tree = up_tree.subtree("weights");- let down_weights_tree = down_tree.subtree("weights");-- let w13 = up_weights_tree- .leaf("weights")?- .validate(&[moe_config.num_routed_experts, moe_config.expert_hidden_dim * 2, model_dim], data_type)?- .read_buffer()?;- let w2 = down_weights_tree- .leaf("weights")?- .validate(&[moe_config.num_routed_experts, model_dim, moe_config.expert_hidden_dim], data_type)?- .read_buffer()?;- let up_biases = up_tree- .leaf("biases")?- .validate(&[moe_config.num_routed_experts, moe_config.expert_hidden_dim * 2], data_type)?- .read_buffer()?;- let down_biases = down_tree- .leaf("biases")?- .validate(&[moe_config.num_routed_experts, model_dim], data_type)?- .read_buffer()?;-- let router_topk_kernel =- <B::Kernels as Kernels>::MoeRouterTopKKernel::new(context, data_type, true, false, false, false, false)- .map_err(MoeBlockError::BackendError)?;- let counts_offsets_kernel = MoeCountsOffsetsFusedKernel::new(context).map_err(MoeBlockError::BackendError)?;-- let scatter_bases_kernel = <B::Kernels as Kernels>::MoeBlockBasesFromPartialsKernel::new(context)- .map_err(MoeBlockError::BackendError)?;- let scatter_map_kernel = <B::Kernels as Kernels>::MoeScatterBucketsMapKernel::new(context, data_type)- .map_err(MoeBlockError::BackendError)?;-- let gather = MoeGather::new(context, data_type).map_err(MoeBlockError::BackendError)?;- let experts_two_pass_decode_block =- MoeExpertsTwoPassDecodeBlock::new(context, data_type, gating_code).map_err(MoeBlockError::BackendError)?;- let experts_two_pass_prefill_block =- MoeExpertsTwoPassPrefillBlock::new(context, data_type, gating_code).map_err(MoeBlockError::BackendError)?;- let finalize_kernel =- <B::Kernels as Kernels>::MoeFinalizeKernel::new(context, data_type).map_err(MoeBlockError::BackendError)?;-- let gate_clipping = moe_config.expert_config.gate_clipping;- let gate_clipping = gate_clipping.into_pair().unwrap_or((f32::MIN, f32::MAX));- let up_clipping = moe_config.expert_config.up_clipping;- let up_clipping = up_clipping.into_pair().unwrap_or((f32::MIN, f32::MAX));-- Ok(Self {- router_topk_kernel,- counts_offsets_kernel,- scatter_bases_kernel,- scatter_map_kernel,- gather,- experts_two_pass_decode_block,- experts_two_pass_prefill_block,- finalize_kernel,- router_weights,- router_biases,- router_renorm,- w13,- w2,- up_biases,- down_biases,- model_dim,- hidden_dim: moe_config.expert_hidden_dim,- num_routed_experts: moe_config.num_routed_experts,- num_active_experts: moe_config.num_active_routed_experts,- gate_clip_min: gate_clipping.0,- gate_clip_max: gate_clipping.1,- up_clip_min: up_clipping.0,- up_clip_max: up_clipping.1,- silu_alpha: moe_config.expert_config.activation.alpha(),- data_type,- })- }-}--impl<B: Backend> Mlp<B> for MoeBlock<B> {- fn encode(- &self,- input: B::ScratchBuffer,- batch_dim: u32,- command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,- ) -> Result<B::ScratchBuffer, B::Error> {- command_buffer.push_debug_group("mlp (moe)");-- let total_rows = batch_dim * self.num_active_experts;- let num_blocks = batch_dim.div_ceil(256);- let num_tiles = self.num_routed_experts.div_ceil(512);-- let mut topk_ids =- command_buffer.allocate_scratch_for_shape(&[batch_dim, self.num_active_experts], DataType::I32)?;- let mut topk_probs =- command_buffer.allocate_scratch_for_shape(&[batch_dim, self.num_active_experts], self.data_type)?;-- command_buffer.encode_fill(&mut topk_ids, 0xFF);-- self.router_topk_kernel.encode(- &input,- &self.router_weights,- Some(&self.router_biases),- None::<&B::GlobalBuffer>,- None::<&B::GlobalBuffer>,- &mut topk_ids,- &mut topk_probs,- batch_dim,- self.model_dim,- self.num_routed_experts,- self.num_active_experts,- self.router_renorm,- None::<f32>,- None::<f32>,- command_buffer,- );-- let mut offsets = command_buffer.allocate_scratch_for_shape(&[self.num_routed_experts + 1], DataType::U32)?;- let mut sumk = command_buffer.allocate_scratch_for_shape(&[1], DataType::U32)?;- let scatter_entries = num_blocks * num_tiles * 512;- let mut partials = command_buffer.allocate_scratch_for_shape(&[scatter_entries], DataType::U32)?;- self.counts_offsets_kernel.encode(- &topk_ids,- &mut offsets,- &mut sumk,- &mut partials,- batch_dim,- self.num_routed_experts,- self.num_active_experts,- command_buffer,- );-- let mut block_bases = command_buffer.allocate_scratch_for_shape(&[scatter_entries], DataType::U32)?;- let mut block_alloc = command_buffer.allocate_scratch_for_shape(&[scatter_entries], DataType::U32)?;- let mut bucketed_ids = command_buffer.allocate_scratch_for_shape(&[total_rows], DataType::I32)?;- let mut bucketed_probs = command_buffer.allocate_scratch_for_shape(&[total_rows], self.data_type)?;- let mut tok2row = command_buffer.allocate_scratch_for_shape(&[total_rows], DataType::I32)?;-- command_buffer.encode_fill(&mut tok2row, 0xFF);-- self.scatter_bases_kernel.encode(- &partials,- &mut block_bases,- &mut block_alloc,- self.num_routed_experts,- num_blocks,- num_tiles,- 0u32,- command_buffer,- );- self.scatter_map_kernel.encode(- &topk_ids,- &topk_probs,- &offsets,- &block_bases,- &block_alloc,- &mut bucketed_ids,- &mut bucketed_probs,- batch_dim,- self.num_routed_experts,- self.num_active_experts,- num_blocks,- num_tiles,- &mut tok2row,- command_buffer,- );-- let x_perm = self.gather.encode(- &input,- &bucketed_ids,- &sumk,- batch_dim,- self.num_active_experts,- self.model_dim,- command_buffer,- )?;-- let args = MoeExpertsTwoPassArguments {- x_perm: &x_perm,- expert_offsets: &offsets,- w13_all: &self.w13,- w2_all: &self.w2,- up_biases: &self.up_biases,- down_biases: &self.down_biases,- total_rows,- d_model: self.model_dim,- d_ff: self.hidden_dim,- num_routed_experts: self.num_routed_experts,- gate_clip_min: self.gate_clip_min,- gate_clip_max: self.gate_clip_max,- up_clip_min: self.up_clip_min,- up_clip_max: self.up_clip_max,- silu_alpha: self.silu_alpha,- };-- let y_partial = if batch_dim == 1 {- self.experts_two_pass_decode_block.encode(args, command_buffer)?- } else {- self.experts_two_pass_prefill_block.encode(args, command_buffer)?- };-- let mut output = command_buffer.allocate_scratch_for_shape(&[batch_dim, self.model_dim], self.data_type)?;- self.finalize_kernel.encode(- &tok2row,- &topk_probs,- &y_partial,- &mut output,- batch_dim,- self.model_dim,- self.num_active_experts,- command_buffer,- );-- command_buffer.pop_debug_group();-- Ok(output)- }-}--#[cfg(test)]-#[path = "../../../../unit/encodable_block/moe/mod.rs"]-mod tests;diff --git a/crates/uzu-engine/unit/backends/common/kernel/mod.rs b/crates/uzu-engine/unit/backends/common/kernel/mod.rsindex ad865edc5..337413e9e 100644--- a/crates/uzu-engine/unit/backends/common/kernel/mod.rs+++ b/crates/uzu-engine/unit/backends/common/kernel/mod.rs@@ -6,7 +6,6 @@ mod gated_act_mul_test;mod gdn;mod logit_transform_test;mod matmul;-mod moe;mod normalization_test;mod pooling;mod radix_top_k_small_test;diff --git a/crates/uzu-engine/unit/backends/common/kernel/moe/mod.rs b/crates/uzu-engine/unit/backends/common/kernel/moe/mod.rsdeleted file mode 100644index 82c2e9a28..000000000--- a/crates/uzu-engine/unit/backends/common/kernel/moe/mod.rs+++ /dev/null@@ -1,3 +0,0 @@-mod moe_counts_offsets_fused_test;-mod moe_finalize_test;-mod moe_router_topk_test;diff --git a/crates/uzu-engine/unit/backends/common/kernel/moe/moe_counts_offsets_fused_test.rs b/crates/uzu-engine/unit/backends/common/kernel/moe/moe_counts_offsets_fused_test.rsdeleted file mode 100644index a459958aa..000000000--- a/crates/uzu-engine/unit/backends/common/kernel/moe/moe_counts_offsets_fused_test.rs+++ /dev/null@@ -1,237 +0,0 @@-use rand::{RngExt, SeedableRng, rngs::StdRng};-use uzu_engine_macros::uzu_test;--use crate::{- backends::{- common::{- Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferPending, Context, Kernels,- kernel::MoeCountsOffsetsFusedKernel,- },- cpu::Cpu,- },- tests::helpers::{- buffer_prefix_to_vec, buffer_to_vec, create_buffer, create_buffer_with_data, for_each_backend,- for_each_non_cpu_backend,- },-};--fn gen_random_topk_ids(- t: usize,- e: usize,- k: usize,- seed: u64,-) -> Vec<i32> {- let mut rng = StdRng::seed_from_u64(seed);- let mut topk_ids = vec![0i32; t * k];- for ti in 0..t {- for kk in 0..k {- topk_ids[ti * k + kk] = rng.random_range(0..e as i32);- }- }- topk_ids-}--fn get_output<B: Backend>(- topk_ids: &[i32],- t: usize,- e: usize,- k: usize,-) -> (Vec<u32>, u32, Vec<u32>) {- let context = B::Context::new().expect("Failed to create Context");-- let kernel = <<B as Backend>::Kernels as Kernels>::MoeCountsOffsetsFusedKernel::new(&context)- .expect("Failed to create MoeCountsOffsetsFusedKernel");-- let topk_ids_len = (t * k).max(1);- let topk_ids_buffer = if topk_ids.is_empty() {- create_buffer::<B, i32>(&context, topk_ids_len)- } else {- create_buffer_with_data::<B, i32>(&context, topk_ids)- };- let mut offsets = create_buffer::<B, u32>(&context, e + 1);- let mut sum_k = create_buffer::<B, u32>(&context, 1);- let num_tiles = e.div_ceil(512).max(1);- let mut partials = create_buffer::<B, u32>(&context, num_tiles * 512);-- let mut command_buffer = context.create_command_buffer(None, None).expect("Failed to create command buffer");- kernel.encode(- &topk_ids_buffer,- &mut offsets,- &mut sum_k,- &mut partials,- t as u32,- e as u32,- k as u32,- &mut command_buffer,- );- command_buffer.end_encoding().submit().wait_until_completed().unwrap();-- let offsets = buffer_to_vec::<B, u32>(&offsets);- let sum_k = buffer_to_vec::<B, u32>(&sum_k)[0];- let partials = buffer_prefix_to_vec::<B, u32>(&partials, e);-- (offsets, sum_k, partials)-}--#[uzu_test]-fn test_counts_offsets_fused_parity_random() {- let shapes = vec![(1usize, 4usize), (7, 16), (64, 64), (1024, 128)];- let ks = vec![1usize, 2usize, 4usize];-- for &(t, e) in &shapes {- for &k in &ks {- if k > e {- continue;- }- let topk_ids = gen_random_topk_ids(t, e, k, 1234);-- let (offsets_ref, sum_ref, counts_ref) = get_output::<Cpu>(&topk_ids, t, e, k);- for_each_non_cpu_backend!(|B| {- let (offsets, sum_k, partials) = get_output::<B>(&topk_ids, t, e, k);- let backend_name = std::any::type_name::<B>();- assert_eq!(offsets, offsets_ref, "offsets mismatch T={} E={} K={} backend={}", t, e, k, backend_name);- assert_eq!(sum_k, sum_ref, "sum mismatch T={} E={} K={} backend={}", t, e, k, backend_name);- assert_eq!(partials, counts_ref, "partials mismatch T={} E={} K={} backend={}", t, e, k, backend_name);- });- }- }-}--#[uzu_test]-fn test_counts_offsets_fused_all_tokens_one_expert() {- let (t, e, k) = (16usize, 8usize, 2usize);- let topk_ids: Vec<i32> = vec![3i32; t * k];-- let mut expected_offsets = vec![0u32; e + 1];- expected_offsets[4..].fill((t * k) as u32);-- for_each_backend!(|B| {- let (offsets, sum_k, _) = get_output::<B>(&topk_ids, t, e, k);- let backend_name = std::any::type_name::<B>();- assert_eq!(offsets, expected_offsets, "offsets mismatch backend={}", backend_name);- assert_eq!(sum_k, (t * k) as u32, "sum mismatch backend={}", backend_name);- });-}--#[uzu_test]-fn test_counts_offsets_fused_zero_tokens() {- let (t, e, k) = (0usize, 8usize, 2usize);- let topk_ids: Vec<i32> = vec![];-- for_each_backend!(|B| {- let (offsets, sum_k, _) = get_output::<B>(&topk_ids, t, e, k);- let backend_name = std::any::type_name::<B>();- assert!(offsets.iter().all(|&v| v == 0), "expected all-zero offsets for T=0, backend={}", backend_name,);- assert_eq!(sum_k, 0, "sum should be 0 for T=0, backend={}", backend_name);- });-}--#[uzu_test]-fn test_counts_offsets_fused_negative_ids_ignored() {- let (t, e, k) = (8usize, 4usize, 2usize);- // Mix of valid IDs and -1 (invalid/padding)- let mut topk_ids = vec![-1i32; t * k];- // Only set the first slot of each token, leave second as -1- for ti in 0..t {- topk_ids[ti * k] = (ti % e) as i32;- }-- let (offsets_ref, sum_ref, counts_ref) = get_output::<Cpu>(&topk_ids, t, e, k);- // sum should be t (only one valid id per token)- assert_eq!(sum_ref, t as u32);-- for_each_non_cpu_backend!(|B| {- let (offsets, sum_k, partials) = get_output::<B>(&topk_ids, t, e, k);- let backend_name = std::any::type_name::<B>();- assert_eq!(offsets, offsets_ref, "offsets mismatch backend={}", backend_name);- assert_eq!(sum_k, sum_ref, "sum mismatch backend={}", backend_name);- assert_eq!(partials, counts_ref, "partials mismatch backend={}", backend_name);- });-}--#[uzu_test]-fn test_counts_offsets_fused_single_token() {- let (t, e, k) = (1usize, 16usize, 1usize);- let topk_ids = vec![7i32];-- let mut expected_offsets = vec![0u32; e + 1];- // Expert 7 gets 1 token, so offsets[8..] = 1- expected_offsets[8..].fill(1);-- for_each_backend!(|B| {- let (offsets, sum_k, _) = get_output::<B>(&topk_ids, t, e, k);- let backend_name = std::any::type_name::<B>();- assert_eq!(offsets, expected_offsets, "offsets mismatch backend={}", backend_name);- assert_eq!(sum_k, 1, "sum mismatch backend={}", backend_name);- });-}--#[uzu_test]-fn test_counts_offsets_fused_uniform_distribution() {- // Each expert gets exactly 1 token- let e = 8usize;- let (t, k) = (e, 1usize);- let topk_ids: Vec<i32> = (0..e as i32).collect();- let expected_offsets: Vec<u32> = (0..=e as u32).collect();-- for_each_backend!(|B| {- let (offsets, sum_k, partials) = get_output::<B>(&topk_ids, t, e, k);- let backend_name = std::any::type_name::<B>();- assert_eq!(offsets, expected_offsets, "offsets mismatch backend={}", backend_name);- assert_eq!(sum_k, e as u32, "sum mismatch backend={}", backend_name);- assert!(partials.iter().all(|&v| v == 1), "each expert should have count=1 backend={}", backend_name,);- });-}--#[uzu_test]-fn test_counts_offsets_fused_large_t() {- // Stress test with many tokens- let (t, e, k) = (2048usize, 128usize, 4usize);- let topk_ids = gen_random_topk_ids(t, e, k, 42);-- let (offsets_ref, sum_ref, counts_ref) = get_output::<Cpu>(&topk_ids, t, e, k);- for_each_non_cpu_backend!(|B| {- let (offsets, sum_k, partials) = get_output::<B>(&topk_ids, t, e, k);- let backend_name = std::any::type_name::<B>();- assert_eq!(offsets, offsets_ref, "offsets mismatch backend={}", backend_name);- assert_eq!(sum_k, sum_ref, "sum mismatch backend={}", backend_name);- assert_eq!(partials, counts_ref, "partials mismatch backend={}", backend_name);- });-}--#[uzu_test]-fn test_counts_offsets_fused_offsets_monotonic() {- // Verify offsets are monotonically non-decreasing for various shapes- let shapes = vec![(10, 8, 2), (100, 32, 4), (5, 128, 1)];-- for (t, e, k) in shapes {- let topk_ids = gen_random_topk_ids(t, e, k, 99);-- for_each_backend!(|B| {- let (offsets, sum_k, _) = get_output::<B>(&topk_ids, t, e, k);- let backend_name = std::any::type_name::<B>();- for i in 1..offsets.len() {- assert!(- offsets[i] >= offsets[i - 1],- "offsets not monotonic at i={} ({} < {}) T={} E={} K={} backend={}",- i,- offsets[i],- offsets[i - 1],- t,- e,- k,- backend_name,- );- }- assert_eq!(- *offsets.last().unwrap(),- sum_k,- "last offset should equal sum_k T={} E={} K={} backend={}",- t,- e,- k,- backend_name,- );- });- }-}diff --git a/crates/uzu-engine/unit/backends/common/kernel/moe/moe_finalize_test.rs b/crates/uzu-engine/unit/backends/common/kernel/moe/moe_finalize_test.rsdeleted file mode 100644index 4ecb80859..000000000--- a/crates/uzu-engine/unit/backends/common/kernel/moe/moe_finalize_test.rs+++ /dev/null@@ -1,121 +0,0 @@-use half::bf16;-use num_traits::Float;-use rand::{RngExt, SeedableRng, rngs::StdRng};-use uzu_engine_macros::uzu_test;--use crate::{- array::ArrayElement,- backends::{- common::{- Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferPending, Context, Kernels,- kernel::MoeFinalizeKernel,- },- cpu::Cpu,- },- data_type::DataType,- tests::{- assert::assert_eq_float,- helpers::{buffer_to_vec, create_buffer, create_buffer_with_data, create_context, for_each_non_cpu_backend},- },-};--struct Input<T: ArrayElement + Float> {- tok2row: Box<[i32]>,- probs: Box<[T]>,- y_partial: Box<[T]>,- t: usize,- d_model: usize,- k: usize,-}--fn get_output<B: Backend, T: ArrayElement + Float>(input: &Input<T>) -> Vec<T> {- let context = create_context::<B>();- let tok2row = create_buffer_with_data::<B, i32>(&context, &input.tok2row);- let probs = create_buffer_with_data::<B, T>(&context, &input.probs);- let y_partial = create_buffer_with_data::<B, T>(&context, &input.y_partial);- let mut y_out = create_buffer::<B, T>(&context, input.t * input.d_model);-- let finalize = <B::Kernels as Kernels>::MoeFinalizeKernel::new(&context, DataType::BF16).expect("finalize kernel");- let mut command_buffer = context.create_command_buffer(None, None).expect("Failed to create command buffer");- finalize.encode(- &tok2row,- &probs,- &y_partial,- &mut y_out,- input.t as u32,- input.d_model as u32,- input.k as u32,- &mut command_buffer,- );- command_buffer.end_encoding().submit().wait_until_completed().unwrap();-- buffer_to_vec(&y_out)-}--fn test_finalize_internal(- t: usize,- k: usize,- d_model: usize,- sum_k: usize,-) {- let mut rng = StdRng::seed_from_u64(2026);-- // Generate random tok2row mapping: maps (token, k_idx) → row in y_partial- // Some entries can be -1 (no expert selected)- let mut tok2row: Vec<i32> = (0..t * k)- .map(|_| {- if rng.random_bool(0.9) {- rng.random_range(0..sum_k as i32)- } else {- -1 // No expert selected- }- })- .collect();-- // Ensure we use all rows in sum_k (avoid unused rows)- for row in 0..sum_k.min(t * k) {- tok2row[row] = row as i32;- }-- // Generate random probabilities (should sum to 1 per token, but not critical for unit test)- let probs: Vec<bf16> = (0..t * k).map(|_| bf16::from_f32(rng.random_range(0.0..1.0))).collect();-- // Generate random y_partial (expert outputs)- let y_partial: Vec<bf16> = (0..sum_k * d_model).map(|_| bf16::from_f32(rng.random_range(-2.0..2.0))).collect();-- // CPU reference- let input = Input {- tok2row: tok2row.into_boxed_slice(),- probs: probs.into_boxed_slice(),- y_partial: y_partial.into_boxed_slice(),- t,- d_model,- k,- };- let y_cpu = get_output::<Cpu, bf16>(&input);-- for_each_non_cpu_backend!(|B| {- let y_gpu = get_output::<B, bf16>(&input);- assert_eq_float(&y_cpu, &y_gpu, 1e-2, "finalize output");- });-}--#[uzu_test]-fn test_finalize_single_token() {- test_finalize_internal(1, 2, 64, 2)-}--#[uzu_test]-fn test_finalize_small_batch() {- test_finalize_internal(4, 2, 128, 8)-}--#[uzu_test]-fn test_finalize_medium() {- test_finalize_internal(8, 4, 256, 32)-}--#[uzu_test]-fn test_finalize_large() {- test_finalize_internal(16, 2, 512, 32)-}diff --git a/crates/uzu-engine/unit/backends/common/kernel/moe/moe_router_topk_test.rs b/crates/uzu-engine/unit/backends/common/kernel/moe/moe_router_topk_test.rsdeleted file mode 100644index bd0ba06cc..000000000--- a/crates/uzu-engine/unit/backends/common/kernel/moe/moe_router_topk_test.rs+++ /dev/null@@ -1,201 +0,0 @@-use std::fmt::Debug;--use half::bf16;-use num_traits::Float;-use rand::{RngExt, SeedableRng, rngs::StdRng};-use uzu_engine_macros::uzu_test;--use crate::{- array::ArrayElement,- backends::{- common::{- Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferPending, Context, Kernels,- kernel::MoeRouterTopKKernel,- },- cpu::Cpu,- },- tests::helpers::{buffer_to_vec, create_buffer, create_buffer_with_data, create_context, for_each_non_cpu_backend},-};--#[derive(Copy, Clone, Debug)]-enum RouterInput {- Plain,- Biased,- Gemma4,-}--fn get_output<B: Backend, T: ArrayElement + Float>(- input: &[T],- weights: &[T],- bias: Option<&[T]>,- router_scale: Option<&[T]>,- per_expert_scale: Option<&[T]>,- t: usize,- d_model: usize,- e: usize,- k: usize,- renorm: bool,- router_norm_epsilon: Option<f32>,- router_input_scale: Option<f32>,-) -> (Vec<i32>, Vec<T>) {- let ctx = create_context::<B>();-- let input_buffer = create_buffer_with_data::<B, T>(&ctx, input);- let weights_buffer = create_buffer_with_data::<B, T>(&ctx, weights);- let bias_buffer = bias.map(|bias| create_buffer_with_data::<B, T>(&ctx, bias));- let router_scale_buffer = router_scale.map(|router_scale| create_buffer_with_data::<B, T>(&ctx, router_scale));- let per_expert_scale_buffer =- per_expert_scale.map(|per_expert_scale| create_buffer_with_data::<B, T>(&ctx, per_expert_scale));- let mut ids = create_buffer::<B, i32>(&ctx, t * k);- let mut probs = create_buffer::<B, T>(&ctx, t * k);-- let kernel = <<B as Backend>::Kernels as Kernels>::MoeRouterTopKKernel::new(- &ctx,- T::data_type(),- bias_buffer.is_some(),- router_scale_buffer.is_some(),- per_expert_scale_buffer.is_some(),- router_input_scale.is_some(),- router_norm_epsilon.is_some(),- )- .expect("kernel");- let mut command_buffer = ctx.create_command_buffer(None, None).expect("Failed to create command buffer");- kernel.encode(- &input_buffer,- &weights_buffer,- bias_buffer.as_ref(),- router_scale_buffer.as_ref(),- per_expert_scale_buffer.as_ref(),- &mut ids,- &mut probs,- t as u32,- d_model as u32,- e as u32,- k as u32,- renorm,- router_norm_epsilon,- router_input_scale,- &mut command_buffer,- );- command_buffer.end_encoding().submit().wait_until_completed().unwrap();-- (buffer_to_vec(&ids), buffer_to_vec(&probs))-}--fn run_router_topk_once<B: Backend, T: ArrayElement + Debug + Float>(- t: usize,- d_model: usize,- e: usize,- k: usize,- renorm: bool,- router_input: RouterInput,-) {- assert_eq!(d_model % 4, 0, "router_topk kernel processes d_model in 4-wide chunks");-- let mut rng = StdRng::seed_from_u64(1234);- let gemma4 = matches!(router_input, RouterInput::Gemma4);- let input: Vec<T> = (0..t * d_model).map(|_| T::from(rng.random_range(-1.0..1.0)).unwrap()).collect();- let weight_limit = if gemma4 {- 0.4- } else {- 1.0- };- let weight: Vec<T> =- (0..e * d_model).map(|_| T::from(rng.random_range(-weight_limit..weight_limit)).unwrap()).collect();- let bias: Option<Vec<T>> = matches!(router_input, RouterInput::Biased)- .then(|| (0..e).map(|_| T::from(rng.random_range(-0.5..0.5)).unwrap()).collect());- let router_scale: Option<Vec<T>> =- gemma4.then(|| (0..d_model).map(|i| T::from(0.75 + (i % 7) as f32 * 0.05).unwrap()).collect());- let per_expert_scale: Option<Vec<T>> =- gemma4.then(|| (0..e).map(|i| T::from(0.5 + (i % 11) as f32 * 0.03).unwrap()).collect());- let router_norm_epsilon = gemma4.then_some(1e-6);- let router_input_scale = gemma4.then_some((d_model as f32).sqrt().recip());-- let (ids_ref, probs_ref) = get_output::<Cpu, T>(- input.as_slice(),- weight.as_slice(),- bias.as_deref(),- router_scale.as_deref(),- per_expert_scale.as_deref(),- t,- d_model,- e,- k,- renorm,- router_norm_epsilon,- router_input_scale,- );- let (ids_gpu, probs_gpu) = get_output::<B, T>(- input.as_slice(),- weight.as_slice(),- bias.as_deref(),- router_scale.as_deref(),- per_expert_scale.as_deref(),- t,- d_model,- e,- k,- renorm,- router_norm_epsilon,- router_input_scale,- );- assert_eq!(- ids_gpu, ids_ref,- "Top-k ids mismatch for T={}, d_model={}, E={}, K={}, renorm={}, router_input={:?}",- t, d_model, e, k, renorm, router_input- );-- for i in 0..(t * k) {- let diff = (probs_gpu[i] - probs_ref[i]).abs().to_f32().unwrap();-- // Tolerance accounts for numerical differences in:- // 1. Router matmul: GPU uses vec4 FMA + SIMD reductions, CPU uses scalar loops- // With bf16 inputs and f32 accumulation, minor differences expected- // 2. bf16 quantization on probs output buffer- // 3. TopK selection (minor)- // 4. Softmax computation when renorm=true (exp/div operations)- let atol = if renorm {- 2e-2 // Normalized probabilities with bf16: softmax + bf16 quantization- } else {- 5e-2 // Raw logits with bf16: vec4 accumulation + bf16 output quantization- };- assert!(- diff <= atol,- "Top-k prob mismatch at {}: gpu={} ref={} (diff={}, atol={}) with T={}, d_model={}, E={}, K={}, renorm={}, router_input={:?}",- i,- probs_gpu[i].to_f32().unwrap(),- probs_ref[i].to_f32().unwrap(),- diff,- atol,- t,- d_model,- e,- k,- renorm,- router_input- );- }-}--#[uzu_test]-fn test_router_topk_fused_matches_reference() {- let configs: &[(usize, usize, usize, usize, RouterInput, &[bool])] = &[- (1, 64, 32, 4, RouterInput::Plain, &[false, true]),- (4, 256, 128, 16, RouterInput::Plain, &[false, true]),- (1, 64, 32, 4, RouterInput::Biased, &[false, true]),- (2, 128, 64, 8, RouterInput::Biased, &[false, true]),- (4, 256, 128, 16, RouterInput::Biased, &[false, true]),- (8, 256, 256, 32, RouterInput::Biased, &[false, true]),- (1, 512, 512, 64, RouterInput::Biased, &[false, true]),- (3, 256, 128, 8, RouterInput::Gemma4, &[true]),- (1, 2816, 128, 8, RouterInput::Gemma4, &[true]),- ];-- for_each_non_cpu_backend!(|B| {- for &(t, d_model, e, k, router_input, renorm_options) in configs {- for &renorm in renorm_options {- run_router_topk_once::<B, bf16>(t, d_model, e, k, renorm, router_input);- }- }- });-}diff --git a/crates/uzu-engine/unit/encodable_block/moe/mod.rs b/crates/uzu-engine/unit/encodable_block/moe/mod.rsdeleted file mode 100644index 771f44831..000000000--- a/crates/uzu-engine/unit/encodable_block/moe/mod.rs+++ /dev/null@@ -1,55 +0,0 @@-use super::{- experts_two_pass_decode::MoeExpertsTwoPassDecodeBlock,- experts_two_pass_prefill::{MoeExpertsTwoPassArguments, MoeExpertsTwoPassPrefillBlock},- gather::MoeGather,-};--mod moe_block_e2e_test;-mod moe_experts_perf_test;-mod moe_experts_test;-mod moe_perf_test;-mod moe_tiles_test;--/// CPU reference for tile counts: count number of BM-sized tiles per expert-///-/// # Arguments-/// * `offsets` - Expert segment offsets [E+1]-/// * `bm` - Tile size (must match kernel BM constant)-///-/// # Returns-/// Tile counts per expert [E]-pub fn cpu_tile_counts(- offsets: &[u32],- bm: usize,-) -> Vec<u32> {- let e = offsets.len() - 1;- let mut tile_counts = vec![0u32; e];- for expert in 0..e {- let seg_len = offsets[expert + 1] - offsets[expert];- tile_counts[expert] = if seg_len == 0 {- 0- } else {- (seg_len as usize).div_ceil(bm) as u32- };- }- tile_counts-}--/// CPU reference for tile scan: exclusive prefix sum of tile counts-///-/// # Arguments-/// * `tile_counts` - Tile counts per expert [E]-///-/// # Returns-/// * Tile offsets [E+1] (exclusive prefix sum)-/// * Total number of tiles-pub fn cpu_tile_scan(tile_counts: &[u32]) -> (Vec<u32>, u32) {- let e = tile_counts.len();- let mut tile_offsets = Vec::with_capacity(e + 1);- tile_offsets.push(0);- for i in 0..e {- tile_offsets.push(tile_offsets[i] + tile_counts[i]);- }- let total_tiles = *tile_offsets.last().unwrap();- (tile_offsets, total_tiles)-}diff --git a/crates/uzu-engine/unit/encodable_block/moe/moe_block_e2e_test.rs b/crates/uzu-engine/unit/encodable_block/moe/moe_block_e2e_test.rsdeleted file mode 100644index 96137bf72..000000000--- a/crates/uzu-engine/unit/encodable_block/moe/moe_block_e2e_test.rs+++ /dev/null@@ -1,1148 +0,0 @@-use half::bf16;-use rand::{RngExt, SeedableRng, rngs::StdRng};-use uzu_engine_macros::uzu_test;--use super::{MoeExpertsTwoPassArguments, MoeExpertsTwoPassPrefillBlock, MoeGather};-use crate::{- backends::common::{- Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferPending, Context, Kernels,- gpu_types::{ActivationType, activation_silu_alpha},- kernel::{- MoeBlockBasesFromPartialsKernel, MoeCountsOffsetsFusedKernel, MoeFinalizeKernel, MoeRouterTopKKernel,- MoeScatterBucketsMapKernel,- },- },- data_type::DataType,- tests::helpers::{- buffer_prefix_to_vec, buffer_readback, buffer_to_vec, create_buffer, create_buffer_with_data, create_context,- for_each_non_cpu_backend,- },-};--fn moe_cpu_reference(- x: &[bf16],- router_weight: &[f32], // [E, d_model] - kept as F32 for router (computed before BF16 conversion)- router_bias: &[f32], // [E]- w13: &[bf16], // source layout [E, d_model, 2*d_ff] (GPU transposes to [E, 2*d_ff, d_model])- w2: &[bf16], // [E, d_ff, d_model]- up_biases: &[bf16], // [E, 2*d_ff]- down_biases: &[bf16], // [E, d_model]- t: usize,- e: usize,- k: usize,- d_model: usize,- d_ff: usize,- gating_code: u32, // 0=GELU, 1=SiLU, 2=SwiGLU, 3=GEGLU- silu_alpha: f32,- gate_clip: (f32, f32),- up_clip: (f32, f32),-) -> Vec<f32> {- let mut output = vec![0.0f32; t * d_model];-- for token_idx in 0..t {- let x_start = token_idx * d_model;-- // Router: compute logits = x @ W_r^T + b_r- // Router uses F32 precision (router_weight/router_bias are F32)- let mut token_logits = vec![0.0f32; e];- for expert_idx in 0..e {- let mut logit = router_bias[expert_idx];- for d in 0..d_model {- let x_val = f32::from(x[x_start + d]);- logit += x_val * router_weight[expert_idx * d_model + d];- }- token_logits[expert_idx] = logit;- }-- // TopK selection- let mut indices_and_logits: Vec<(usize, f32)> = token_logits.iter().enumerate().map(|(i, &v)| (i, v)).collect();- indices_and_logits.sort_by(|a, b| b.1.partial_cmp(&a.1).unwrap());- let top_k_indices: Vec<usize> = indices_and_logits[..k].iter().map(|&(i, _)| i).collect();- let top_k_logits: Vec<f32> = indices_and_logits[..k].iter().map(|&(_, v)| v).collect();-- // Softmax normalization (matching Python: jax.nn.softmax)- let max_logit = top_k_logits.iter().copied().fold(f32::NEG_INFINITY, f32::max);- let exp_sum: f32 = top_k_logits.iter().map(|&v| (v - max_logit).exp()).sum();- let weights: Vec<f32> = top_k_logits.iter().map(|&v| (v - max_logit).exp() / exp_sum).collect();-- // Process each selected expert (matching Python: vmap(run_expert))- for (expert_idx, &weight) in top_k_indices.iter().zip(weights.iter()) {- let x_start = token_idx * d_model;-- // Expert MLP weights layout: CPU uses W13[E, d_model, 2*d_ff], GPU stores as [E, 2*d_ff, d_model]; W2[E, d_ff, d_model]- let w13_offset = expert_idx * d_model * 2 * d_ff;- let w2_offset = expert_idx * d_ff * d_model;- let bias_up_offset = expert_idx * 2 * d_ff;- let bias_down_offset = expert_idx * d_model;-- // Up projection: W13[expert, input, output] where output=[up, gate]- // Convert BF16→F32 on each access to match GPU precision- let mut up_out = vec![0.0f32; d_ff];- let mut gate_out = vec![0.0f32; d_ff];-- for ff_idx in 0..d_ff {- let mut up_sum = f32::from(up_biases[bias_up_offset + ff_idx]);- let mut gate_sum = f32::from(up_biases[bias_up_offset + d_ff + ff_idx]);-- for input_idx in 0..d_model {- let w_base = w13_offset + input_idx * 2 * d_ff;- let x_val = f32::from(x[x_start + input_idx]);- let w_up = f32::from(w13[w_base + ff_idx]);- let w_gate = f32::from(w13[w_base + d_ff + ff_idx]);- up_sum += x_val * w_up;- gate_sum += x_val * w_gate;- }-- up_out[ff_idx] = up_sum;- gate_out[ff_idx] = gate_sum;- }-- // Apply clipping- for i in 0..d_ff {- gate_out[i] = gate_out[i].clamp(gate_clip.0, gate_clip.1);- up_out[i] = up_out[i].clamp(up_clip.0, up_clip.1);- }-- // Activation: depends on gating_code- // 0=GELU(up), 1=SiLU(up), 2=SwiGLU(gate)*up, 3=GEGLU(gate)*up- let mut hidden = vec![0.0f32; d_ff];- for i in 0..d_ff {- hidden[i] = match gating_code {- 0 => ActivationType::GELUApprox.activate(up_out[i]),- 1 => activation_silu_alpha(up_out[i], silu_alpha),- 2 => activation_silu_alpha(gate_out[i], silu_alpha) * up_out[i],- 3 => ActivationType::GELUApprox.activate(gate_out[i]) * up_out[i],- _ => activation_silu_alpha(gate_out[i], silu_alpha) * up_out[i], // fallback to SwiGLU- };- }-- // Down projection: W2[expert, ff, output]- // Convert BF16→F32 on each access to match GPU precision- // CRITICAL: Two-pass kernel accumulates in F32, only quantizes at the very end- for out_idx in 0..d_model {- let mut fc2_sum = 0.0f32;- for ff_idx in 0..d_ff {- let w2_val = f32::from(w2[w2_offset + ff_idx * d_model + out_idx]);- fc2_sum += hidden[ff_idx] * w2_val;- }- // Two-pass: keep in F32 (no intermediate quantization)-- // Add down bias in F32- let down_bias_val = f32::from(down_biases[bias_down_offset + out_idx]);- let with_bias = fc2_sum + down_bias_val;-- // Only quantize to BF16 once at the end (matches two-pass final write)- let with_bias_bf16 = bf16::from_f32(with_bias);- let final_val = f32::from(with_bias_bf16);-- output[x_start + out_idx] += weight * final_val;- }- }- }-- output-}--// Main entry point - automatically tests both modes for T>1-fn run_moe_parity_test<B: Backend>(- ctx: &B::Context,- t: usize,- e: usize,- k: usize,- d_model: usize,- d_ff: usize,- gating_code: u32,- silu_alpha: f32,- gate_clip: (f32, f32),- up_clip: (f32, f32),- seed: u64,- test_name: &str,-) {- if t > 1 {- // Test 1-pass prefill (default, stable)- run_moe_parity_test_internal::<B>(- ctx,- t,- e,- k,- d_model,- d_ff,- gating_code,- silu_alpha,- gate_clip,- up_clip,- seed,- &format!("{}_decode", test_name),- );- // Test 2-pass prefill (experimental)- run_moe_parity_test_internal::<B>(- ctx,- t,- e,- k,- d_model,- d_ff,- gating_code,- silu_alpha,- gate_clip,- up_clip,- seed,- &format!("{}_prefill", test_name),- );- } else {- // Decode mode (T=1) - no prefill variants- run_moe_parity_test_internal::<B>(- ctx,- t,- e,- k,- d_model,- d_ff,- gating_code,- silu_alpha,- gate_clip,- up_clip,- seed,- test_name,- );- }-}--fn run_moe_parity_test_internal<B: Backend>(- ctx: &B::Context,- t: usize,- e: usize,- k: usize,- d_model: usize,- d_ff: usize,- gating_code: u32, // 0=GELU, 1=SiLU, 2=SwiGLU, 3=GEGLU- silu_alpha: f32,- gate_clip: (f32, f32),- up_clip: (f32, f32),- seed: u64,- test_name: &str,-) {- let mut rng = StdRng::seed_from_u64(seed);-- let prefill_mode = if t > 1 {- "2-pass"- } else {- "decode"- };-- eprintln!(- "\n[{}] T={}, E={}, K={}, d_model={}, d_ff={}, alpha={}, gate_clip={:?}, up_clip={:?}, mode={}",- test_name, t, e, k, d_model, d_ff, silu_alpha, gate_clip, up_clip, prefill_mode- );-- // Random BF16 inputs- let x: Vec<bf16> = (0..t * d_model).map(|_| bf16::from_f32(rng.random_range(-1.0..1.0))).collect();-- // Generate random router weights and biases for CPU reference- let router_weight_f32: Vec<f32> = (0..e * d_model).map(|_| rng.random_range(-0.5..0.5)).collect();- let router_bias_f32: Vec<f32> = (0..e).map(|_| rng.random_range(-0.1..0.1)).collect();-- // Convert router weights/bias to BF16 for GPU- let router_weight_bf16: Vec<bf16> = router_weight_f32.iter().map(|&v| bf16::from_f32(v)).collect();- let router_bias_bf16: Vec<bf16> = router_bias_f32.iter().map(|&v| bf16::from_f32(v)).collect();-- // Generate random expert weights and biases- let w13_len = e * d_model * 2 * d_ff;- let w2_len = e * d_ff * d_model;-- // Generate W13 in original layout [E, d_model, 2*d_ff] for CPU reference- let w13_cpu: Vec<bf16> = (0..w13_len).map(|_| bf16::from_f32(rng.random_range(-0.5..0.5))).collect();-- // Transpose to GPU layout [E, 2*d_ff, d_model]- let mut w13_gpu = vec![bf16::from_f32(0.0); w13_len];- for expert in 0..e {- let src_offset = expert * d_model * 2 * d_ff;- let dst_offset = expert * 2 * d_ff * d_model;- for dm in 0..d_model {- for ff in 0..(2 * d_ff) {- let src_idx = src_offset + dm * 2 * d_ff + ff;- let dst_idx = dst_offset + ff * d_model + dm;- w13_gpu[dst_idx] = w13_cpu[src_idx];- }- }- }-- // Generate W2 in original layout [E, d_ff, d_model] for CPU reference- let w2_cpu: Vec<bf16> = (0..w2_len).map(|_| bf16::from_f32(rng.random_range(-0.5..0.5))).collect();-- // Transpose W2 to GPU layout [E, d_model, d_ff]- let mut w2_gpu = vec![bf16::from_f32(0.0); w2_len];- for expert in 0..e {- let expert_offset = expert * d_ff * d_model;- for ff in 0..d_ff {- for dm in 0..d_model {- // src: [E, d_ff, d_model] -> index: expert_offset + ff * d_model + dm- // dst: [E, d_model, d_ff] -> index: expert_offset + dm * d_ff + ff- w2_gpu[expert_offset + dm * d_ff + ff] = w2_cpu[expert_offset + ff * d_model + dm];- }- }- }- let up_biases: Vec<bf16> = (0..e * 2 * d_ff).map(|_| bf16::from_f32(rng.random_range(-0.1..0.1))).collect();- let down_biases: Vec<bf16> = (0..e * d_model).map(|_| bf16::from_f32(rng.random_range(-0.1..0.1))).collect();-- // Create Metal buffers- let x_buf = create_buffer_with_data::<B, bf16>(ctx, &x);- let router_w_buf = create_buffer_with_data::<B, bf16>(ctx, &router_weight_bf16);- let router_b_buf = create_buffer_with_data::<B, bf16>(ctx, &router_bias_bf16);- let w13_buf = create_buffer_with_data::<B, bf16>(ctx, &w13_gpu);- let w2_buf = create_buffer_with_data::<B, bf16>(ctx, &w2_gpu);- let up_biases_buf = create_buffer_with_data::<B, bf16>(ctx, &up_biases);- let down_biases_buf = create_buffer_with_data::<B, bf16>(ctx, &down_biases);-- // Allocate intermediate buffers (max capacity)- let max_sumk = t * k;- let mut topk_ids_buf = create_buffer::<B, i32>(ctx, t * k);- let mut topk_probs_buf = create_buffer::<B, bf16>(ctx, t * k);- let mut offsets_buf = create_buffer::<B, u32>(ctx, e + 1);- let mut sumk_buf = create_buffer::<B, u32>(ctx, 1);- let num_blocks = t.div_ceil(256).max(1);- let num_tiles = e.div_ceil(512).max(1);- let entries = num_blocks * num_tiles * 512usize;- let mut partials_buf = create_buffer::<B, u32>(ctx, entries);- let mut block_bases_buf = create_buffer::<B, u32>(ctx, entries);- let mut block_alloc_buf = create_buffer::<B, u32>(ctx, entries);- let mut bucketed_ids_buf = create_buffer::<B, i32>(ctx, max_sumk);- let mut bucketed_probs_buf = create_buffer::<B, bf16>(ctx, max_sumk);- let mut tok2row_buf = create_buffer::<B, i32>(ctx, t * k);- let mut y_out_buf = create_buffer::<B, bf16>(ctx, t * d_model);-- // Encode ALL kernels in one command buffer- eprintln!("[E2E] Encoding entire MoE pipeline in single command buffer...");- let mut command_buffer = ctx.create_command_buffer(None, None).expect("Failed to create command buffer");-- // Router + TopK (fused kernel)- let router_topk =- <B::Kernels as Kernels>::MoeRouterTopKKernel::new(ctx, DataType::BF16, true, false, false, false, false)- .expect("router+topk");- router_topk.encode(- &x_buf,- &router_w_buf,- Some(&router_b_buf),- None::<&B::GlobalBuffer>,- None::<&B::GlobalBuffer>,- &mut topk_ids_buf,- &mut topk_probs_buf,- t as u32,- d_model as u32,- e as u32,- k as u32,- true,- None::<f32>,- None::<f32>,- &mut command_buffer,- );-- let fused_kernel = <B::Kernels as Kernels>::MoeCountsOffsetsFusedKernel::new(ctx).expect("fused kernel");- fused_kernel.encode(- &topk_ids_buf,- &mut offsets_buf,- &mut sumk_buf,- &mut partials_buf,- t as u32,- e as u32,- k as u32,- &mut command_buffer,- );-- let scatter_bases_kernel =- <B::Kernels as Kernels>::MoeBlockBasesFromPartialsKernel::new(ctx).expect("scatter bases kernel");- scatter_bases_kernel.encode(- &partials_buf,- &mut block_bases_buf,- &mut block_alloc_buf,- e as u32,- num_blocks as u32,- num_tiles as u32,- 0u32,- &mut command_buffer,- );-- let scatter_map_kernel = <B::Kernels as Kernels>::MoeScatterBucketsMapKernel::new(ctx, DataType::BF16)- .expect("Failed to create <<Metal as Backend>::Kernels as Kernels>::MoeScatterBucketsMapKernel");- scatter_map_kernel.encode(- &topk_ids_buf,- &topk_probs_buf,- &offsets_buf,- &block_bases_buf,- &block_alloc_buf,- &mut bucketed_ids_buf,- &mut bucketed_probs_buf,- t as u32,- e as u32,- k as u32,- num_blocks as u32,- num_tiles as u32,- &mut tok2row_buf,- &mut command_buffer,- );-- let gather = MoeGather::<B>::new(ctx, DataType::BF16).expect("gather");- let x_perm_buf = gather- .encode(&x_buf, &bucketed_ids_buf, &sumk_buf, t as u32, k as u32, d_model as u32, &mut command_buffer)- .expect("gather");-- let total_rows = t * k;-- let experts = MoeExpertsTwoPassPrefillBlock::<B>::new(ctx, DataType::BF16, gating_code).expect("experts");- let args = MoeExpertsTwoPassArguments {- x_perm: &x_perm_buf,- expert_offsets: &offsets_buf,- w13_all: &w13_buf,- w2_all: &w2_buf,- up_biases: &up_biases_buf,- down_biases: &down_biases_buf,- total_rows: total_rows as u32,- d_model: d_model as u32,- d_ff: d_ff as u32,- num_routed_experts: e as u32,- gate_clip_min: gate_clip.0,- gate_clip_max: gate_clip.1,- up_clip_min: up_clip.0,- up_clip_max: up_clip.1,- silu_alpha,- };- let y_partial_buf = experts.encode(args, &mut command_buffer).expect("failed to encode MoE experts");-- let finalize = <B::Kernels as Kernels>::MoeFinalizeKernel::new(ctx, DataType::BF16).expect("finalize");- finalize.encode(- &tok2row_buf,- &topk_probs_buf,- &y_partial_buf,- &mut y_out_buf,- t as u32,- d_model as u32,- k as u32,- &mut command_buffer,- );-- eprintln!("[E2E] All kernels encoded. Committing ONCE and waiting...");- let completed = command_buffer.end_encoding().submit().wait_until_completed().unwrap();- eprintln!("[E2E] GPU execution completed");-- // Read GPU output- let y_out_bf16 = buffer_prefix_to_vec::<B, bf16>(&y_out_buf, t * d_model);- let y_out_gpu: Vec<f32> = y_out_bf16.iter().map(|&v| f32::from(v)).collect();-- // Validate GPU output is finite- let nan_count = y_out_gpu.iter().filter(|v| v.is_nan()).count();- let inf_count = y_out_gpu.iter().filter(|v| v.is_infinite()).count();-- eprintln!("[E2E] GPU output stats: nan={}, inf={}", nan_count, inf_count);- assert_eq!(nan_count, 0, "GPU output contains {} NaN values", nan_count);- assert_eq!(inf_count, 0, "GPU output contains {} Inf values", inf_count);-- // Run CPU reference (router logits already computed above)- // NOTE: Keep weights as BF16 and convert per-access to match GPU precision- eprintln!("[{}] Running CPU reference implementation...", test_name);-- // Debug: Verify data consistency and tile bookkeeping for large-scale tests- if d_model >= 512 {- eprintln!(- "[E2E] Large-scale debug: x[0]={:.6}, w13[0]={:.6}, w2[0]={:.6}",- f32::from(x[0]),- f32::from(w13_cpu[0]),- f32::from(w2_cpu[0])- );-- let sumk_val = buffer_to_vec::<B, u32>(&sumk_buf)[0] as usize;- eprintln!("[E2E] sumk={}", sumk_val);-- // For multi-token tests with large d_ff, verify gather output (x_perm)- if t > 1 && d_ff >= 256 {- let x_perm_cpu =- buffer_prefix_to_vec::<B, bf16>(&buffer_readback::<B>(ctx, &x_perm_buf), sumk_val * d_model);- eprintln!("[E2E] x_perm diagnostics (sumk={}):", sumk_val);- eprintln!("[E2E] Row 0 [0:8]: {:?}", x_perm_cpu[0..8].iter().map(|&v| f32::from(v)).collect::<Vec<_>>());- if sumk_val > 1 {- eprintln!(- "[E2E] Row 1 [{}:{}]: {:?}",- d_model,- d_model + 8,- x_perm_cpu[d_model..d_model + 8].iter().map(|&v| f32::from(v)).collect::<Vec<_>>()- );- }-- // CRITICAL: Check tok2row mapping for multi-token tests- let tok2row_cpu = buffer_prefix_to_vec::<B, i32>(&tok2row_buf, t * k);- eprintln!("[E2E] tok2row[0..{}]: {:?}", t * k, tok2row_cpu);- eprintln!(- "[E2E] Expected: token 0→row {}, token 1→row {}",- 0,- if sumk_val > 1 {- 1- } else {- -1- }- );-- // Check y_partial at specific indices where finalize reads for token 1- let y_partial_full =- buffer_prefix_to_vec::<B, bf16>(&buffer_readback::<B>(ctx, &y_partial_buf), sumk_val * d_model);- eprintln!("[E2E] y_partial spot check:");- eprintln!("[E2E] y_partial[48] (row 0, col 48) = {:.6}", f32::from(y_partial_full[48]));- if sumk_val > 1 {- // Check positions where mismatches occur (odd indices near tile boundaries)- eprintln!("[E2E] Row 1 positions (even=working, odd=corrupted?):");- for &pos in &[48, 56, 57, 58, 59, 60, 61, 62, 63, 64, 120, 121, 122, 123, 124, 125, 126, 127, 128] {- let idx = d_model + pos;- if idx < y_partial_full.len() {- eprintln!("[E2E] y_partial[row1, col {}] = {:.3}", pos, f32::from(y_partial_full[idx]));- }- }- }- }- }-- let y_cpu = moe_cpu_reference(- &x,- &router_weight_f32,- &router_bias_f32,- &w13_cpu,- &w2_cpu,- &up_biases,- &down_biases,- t,- e,- k,- d_model,- d_ff,- gating_code,- silu_alpha,- gate_clip,- up_clip,- );-- // Compare GPU vs CPU- eprintln!("[E2E] Comparing GPU vs CPU outputs...");- let mut max_abs_diff = 0.0f32;- let mut max_rel_diff = 0.0f32;- let mut total_abs_error = 0.0f32;- let mut mismatches = 0;-- for i in 0..(t * d_model) {- let gpu_val = y_out_gpu[i];- let cpu_val = y_cpu[i];- let abs_diff = (gpu_val - cpu_val).abs();- let rel_diff = if cpu_val.abs() > 1e-6 {- abs_diff / cpu_val.abs()- } else {- abs_diff- };-- max_abs_diff = max_abs_diff.max(abs_diff);- max_rel_diff = max_rel_diff.max(rel_diff);- total_abs_error += abs_diff;-- let threshold_rel = 0.1;- let threshold_abs = 1e-3;-- if rel_diff > threshold_rel && abs_diff > threshold_abs {- let print_limit = if test_name.contains("BoundarySweep_D1024_FF256_T2") {- usize::MAX // Print ALL for debugging- } else {- 10- };- if mismatches < print_limit {- eprintln!(- "[{}] Mismatch idx {}: GPU={:.6}, CPU={:.6}, abs={:.6}, rel={:.6}",- test_name, i, gpu_val, cpu_val, abs_diff, rel_diff- );- }- mismatches += 1;- }- }-- let mean_abs_error = total_abs_error / (t * d_model) as f32;-- eprintln!(- "[E2E] Error metrics: max_abs={:.6}, max_rel={:.6}, mean_abs={:.6}, mismatches={}",- max_abs_diff, max_rel_diff, mean_abs_error, mismatches- );-- // Debug: print some sample values from intermediate buffers- eprintln!("[E2E] === DEBUG: Intermediate values ===");- let topk_ids_gpu = buffer_prefix_to_vec::<B, i32>(&topk_ids_buf, t * k);- let topk_probs_gpu = buffer_prefix_to_vec::<B, bf16>(&topk_probs_buf, t * k);- eprintln!("[E2E] TopK IDs: {:?}", topk_ids_gpu);- eprintln!("[E2E] TopK Probs: {:?}", topk_probs_gpu.iter().map(|&v| f32::from(v)).collect::<Vec<_>>());-- let y_partial_gpu = buffer_prefix_to_vec::<B, bf16>(&buffer_readback::<B>(ctx, &y_partial_buf), max_sumk * d_model);- let sumk_actual = buffer_to_vec::<B, u32>(&sumk_buf)[0] as usize;- eprintln!("[E2E] sumk={}", sumk_actual);- let sample_size = 16.min(sumk_actual * d_model);- eprintln!(- "[E2E] y_partial sample (first {}): {:?}",- sample_size,- y_partial_gpu[..sample_size].iter().map(|&v| f32::from(v)).collect::<Vec<_>>()- );-- // For multi-row debugging: print both rows- if sumk_actual > 1 && d_ff >= 256 {- eprintln!(- "[E2E] y_partial row 1 [{}-{}]: {:?}",- d_model,- d_model + 16,- y_partial_gpu[d_model..d_model + 16].iter().map(|&v| f32::from(v)).collect::<Vec<_>>()- );- }- let out_sample_size = 16.min(t * d_model);- eprintln!("[E2E] y_out sample (token 0): {:?}", &y_out_gpu[..out_sample_size]);- if t > 1 && d_model >= 512 {- eprintln!("[E2E] y_out sample (token 1, first 32): {:?}", y_out_gpu[d_model..d_model + 32].to_vec());- }- eprintln!("[E2E] CPU ref sample (token 0): {:?}", &y_cpu[..out_sample_size]);- if t > 1 && d_model >= 512 {- eprintln!("[E2E] CPU ref sample (token 1): {:?}", &y_cpu[d_model..d_model + 16]);-- // Compare token 1 outputs element-wise (more positions to find pattern)- let gpu_t1 = &y_out_gpu[d_model..];- let cpu_t1 = &y_cpu[d_model..];- eprintln!("[E2E] Token 1 detailed comparison:");- for i in [0, 1, 2, 3, 4, 5, 6, 7, 15, 16, 17, 18, 48, 64, 121, 128].iter() {- if *i < d_model {- let diff = (gpu_t1[*i] - cpu_t1[*i]).abs();- eprintln!("[E2E] [{}]: GPU={:.6}, CPU={:.6}, diff={:.6}", i, gpu_t1[*i], cpu_t1[*i], diff);- }- }- }-- let k0_iters = d_model.div_ceil(32);- let ff0_iters = d_ff.div_ceil(32);- let product = k0_iters * ff0_iters;- let sqrt_product = (product as f32).sqrt();-- let multi_expert_drift = 0.03 * sqrt_product * (k as f32 / 2.0);- let mean_abs_threshold = 0.1f32.max(multi_expert_drift);-- if mean_abs_error >= mean_abs_threshold {- panic!("[{}] Mean absolute error {:.6} exceeds threshold {:.6}", test_name, mean_abs_error, mean_abs_threshold);- }-- // Warn on high max_rel but don't fail (can be inflated by near-zero values)- if max_rel_diff > 1.0 && mismatches > (t * d_model / 4) {- eprintln!(- "[{}] WARNING: High max_rel={:.2} with {} mismatches, but mean_abs={:.6} is acceptable",- test_name, max_rel_diff, mismatches, mean_abs_error- );- }-- eprintln!("[{}] ✓ PASSED (GPU matches CPU reference)", test_name);- drop(x_perm_buf);- drop(y_partial_buf);- drop(completed);-}--// Test 1: Minimal (K=1, small dims, no clipping, alpha=1.0)-#[uzu_test]-fn test_moe_minimal() {- for_each_non_cpu_backend!(|B| {- let ctx = create_context::<B>();- run_moe_parity_test::<B>(- &ctx,- 1, // t- 2, // e- 1, // k- 4, // d_model- 4, // d_ff- 2, // gating_code (SwiGLU)- 1.0, // silu_alpha- (f32::NEG_INFINITY, f32::INFINITY), // gate_clip- (f32::NEG_INFINITY, f32::INFINITY), // up_clip- 0xE2E_0001,- "Minimal_K1_SwiGLU",- );- })-}--// Test 2: Multi-expert (K=2, tests finalize weighted sum)-#[uzu_test]-fn test_moe_multi_expert() {- for_each_non_cpu_backend!(|B| {- let ctx = create_context::<B>();- run_moe_parity_test::<B>(- &ctx,- 2, // t- 4, // e- 2, // k- 8, // d_model- 8, // d_ff- 2, // gating_code (SwiGLU)- 1.702, // silu_alpha (GPT-OSS value)- (f32::NEG_INFINITY, 7.0), // gate_clip (GPT-OSS config)- (-6.0, 8.0), // up_clip (GPT-OSS config)- 0xE2E_0002,- "Multi_K2_SwiGLU",- );- })-}--// Test 3: Large d_model only (K=1 to isolate from finalize complexity)-#[uzu_test]-fn test_moe_large_d_model_only() {- for_each_non_cpu_backend!(|B| {- let ctx = create_context::<B>();- run_moe_parity_test::<B>(- &ctx,- 1, // t- 2, // e- 1, // k- 64, // d_model (tests k0 chunking)- 8, // d_ff (single ff0 chunk)- 2, // gating_code (SwiGLU)- 1.0,- (f32::NEG_INFINITY, f32::INFINITY),- (f32::NEG_INFINITY, f32::INFINITY),- 0xE2E_0003,- "LargeDModel64",- );- })-}--// Test 4a: d_ff=32 (exactly 1 full chunk - baseline for multi-chunk)-#[uzu_test]-fn test_1_full_chunk() {- for_each_non_cpu_backend!(|B| {- let ctx = create_context::<B>();- run_moe_parity_test::<B>(- &ctx,- 1, // t- 2, // e- 1, // k- 8, // d_model- 32, // d_ff (exactly 1 full chunk)- 2, // gating_code (SwiGLU)- 1.0,- (f32::NEG_INFINITY, f32::INFINITY),- (f32::NEG_INFINITY, f32::INFINITY),- 0xE2E_0004,- "LargeFF32_SingleChunk",- );- })-}--// Test 4b: d_ff=48 (1.5 chunks - tests partial second chunk)-#[uzu_test]-fn test_1_5_chunks() {- for_each_non_cpu_backend!(|B| {- let ctx = create_context::<B>();- run_moe_parity_test::<B>(- &ctx,- 1, // t- 2, // e- 1, // k- 8, // d_model- 48, // d_ff (1 full + 1 partial chunk)- 2, // gating_code (SwiGLU)- 1.0,- (f32::NEG_INFINITY, f32::INFINITY),- (f32::NEG_INFINITY, f32::INFINITY),- 0xE2E_0005,- "LargeFF48_Partial",- );- })-}--// Test 4c: d_ff=64 (exactly 2 full chunks)-#[uzu_test]-fn test_2_full_chunks() {- for_each_non_cpu_backend!(|B| {- let ctx = create_context::<B>();- run_moe_parity_test::<B>(- &ctx,- 1, // t- 2, // e- 1, // k- 8, // d_model- 64, // d_ff (exactly 2 full chunks)- 2, // gating_code (SwiGLU)- 1.0,- (f32::NEG_INFINITY, f32::INFINITY),- (f32::NEG_INFINITY, f32::INFINITY),- 0xE2E_0006,- "LargeFF64_TwoChunks",- );- })-}--// Test 5: GELU activation (gating_code=0)-#[uzu_test]-fn test_gelu_activation() {- for_each_non_cpu_backend!(|B| {- let ctx = create_context::<B>();- run_moe_parity_test::<B>(- &ctx,- 1, // t- 2, // e- 1, // k- 8, // d_model- 16, // d_ff- 0, // gating_code (GELU)- 1.0, // silu_alpha (unused for GELU)- (f32::NEG_INFINITY, f32::INFINITY),- (f32::NEG_INFINITY, f32::INFINITY),- 0xE2E_0007,- "GELU_Activation",- );- })-}--// Test 6: SiLU activation (gating_code=1)-#[uzu_test]-fn test_silu_activation() {- for_each_non_cpu_backend!(|B| {- let ctx = create_context::<B>();- run_moe_parity_test::<B>(- &ctx,- 1, // t- 2, // e- 1, // k- 8, // d_model- 16, // d_ff- 1, // gating_code (SiLU)- 1.702,- (f32::NEG_INFINITY, f32::INFINITY),- (f32::NEG_INFINITY, f32::INFINITY),- 0xE2E_0008,- "SiLU_Activation",- );- })-}--// Test 7: GEGLU activation (gating_code=3)-#[uzu_test]-fn test_geglu_activation() {- for_each_non_cpu_backend!(|B| {- let ctx = create_context::<B>();- run_moe_parity_test::<B>(- &ctx,- 1, // t- 2, // e- 1, // k- 8, // d_model- 16, // d_ff- 3, // gating_code (GEGLU)- 1.0, // silu_alpha (unused for GEGLU)- (f32::NEG_INFINITY, f32::INFINITY),- (f32::NEG_INFINITY, f32::INFINITY),- 0xE2E_0009,- "GEGLU_Activation",- );- })-}--// Test 8: Bucket stress - multiple tokens, larger E, stress scatter/gather-#[uzu_test]-fn test_bucket_stress() {- for_each_non_cpu_backend!(|B| {- let ctx = create_context::<B>();- run_moe_parity_test::<B>(- &ctx,- 8, // t (8 tokens)- 8, // e (8 experts)- 2, // k (each token picks 2 experts)- 16, // d_model- 32, // d_ff- 2, // gating_code (SwiGLU)- 1.0,- (f32::NEG_INFINITY, f32::INFINITY),- (f32::NEG_INFINITY, f32::INFINITY),- 0xE2E_0010,- "BucketStress_T8_E8_K2",- );- })-}--// Test 9a: K=4 small-scale to verify multi-expert accumulation-#[uzu_test]-fn test_small_scale() {- for_each_non_cpu_backend!(|B| {- let ctx = create_context::<B>();- run_moe_parity_test::<B>(- &ctx,- 2, // t- 4, // e- 4, // k (all experts)- 8, // d_model- 16, // d_ff- 2, // gating_code (SwiGLU)- 1.0,- (f32::NEG_INFINITY, f32::INFINITY),- (f32::NEG_INFINITY, f32::INFINITY),- 0xE2E_0011,- "Multi_K4_Small",- );- })-}--// Test 9b: d_model=128 (4 k0 chunks, 2 n-tiles)-#[uzu_test]-fn test_model_128() {- for_each_non_cpu_backend!(|B| {- let ctx = create_context::<B>();- run_moe_parity_test::<B>(- &ctx,- 2, // t- 4, // e- 1, // k- 128, // d_model (4 k0 chunks, 2 n-tiles)- 32, // d_ff- 2, // gating_code (SwiGLU)- 1.0,- (f32::NEG_INFINITY, f32::INFINITY),- (f32::NEG_INFINITY, f32::INFINITY),- 0xE2E_0012,- "MultiNTile_D128",- );- })-}--// Test 9c: d_model=192 (6 k0 chunks, 3 n-tiles - test threshold)-#[uzu_test]-fn test_model_192() {- for_each_non_cpu_backend!(|B| {- let ctx = create_context::<B>();- run_moe_parity_test::<B>(- &ctx,- 2, // t- 4, // e- 1, // k- 192, // d_model (6 k0 chunks, 3 n-tiles)- 32, // d_ff- 2, // gating_code (SwiGLU)- 1.0,- (f32::NEG_INFINITY, f32::INFINITY),- (f32::NEG_INFINITY, f32::INFINITY),- 0xE2E_0013,- "MultiNTile_D192_3tiles",- );- })-}--// Test 9d: d_model=256 (8 k0 chunks, 4 n-tiles)-#[uzu_test]-fn test_model_256() {- for_each_non_cpu_backend!(|B| {- let ctx = create_context::<B>();- run_moe_parity_test::<B>(- &ctx,- 2, // t- 4, // e- 1, // k- 256, // d_model (8 k0 chunks, 4 n-tiles)- 32, // d_ff- 2, // gating_code (SwiGLU)- 1.0,- (f32::NEG_INFINITY, f32::INFINITY),- (f32::NEG_INFINITY, f32::INFINITY),- 0xE2E_0014,- "MultiNTile_D256_4tiles",- );- })-}--// Test 10a: Isolate large d_ff with small d_model (FF accumulation test)-#[uzu_test]-fn test_ff_accumulation() {- for_each_non_cpu_backend!(|B| {- let ctx = create_context::<B>();- run_moe_parity_test::<B>(- &ctx,- 2, // t- 4, // e- 1, // k- 64, // d_model (small, 1 n-tile, 2 k0 chunks)- 14336, // d_ff (448 ff0 chunks!) - production scale- 2, // gating_code (SwiGLU)- 1.0,- (f32::NEG_INFINITY, f32::INFINITY),- (f32::NEG_INFINITY, f32::INFINITY),- 0xE2E_0016,- "IsolateFF_D64_FF14336_K1",- );- })-}--// Test 10b: d_model=1024 (16 n-tiles)-#[uzu_test]-fn test_model_1024() {- for_each_non_cpu_backend!(|B| {- let ctx = create_context::<B>();- run_moe_parity_test::<B>(- &ctx,- 2, // t- 4, // e- 1, // k- 1024, // d_model (32 k0 chunks, 16 n-tiles)- 64, // d_ff (small)- 2, // gating_code (SwiGLU)- 1.0,- (f32::NEG_INFINITY, f32::INFINITY),- (f32::NEG_INFINITY, f32::INFINITY),- 0xE2E_0017,- "IsolateD_D1024_FF64_K1",- );- })-}--// Test 10c: Verify layout with d_model=96, d_ff=96 (3 chunks each)-// This tests non-power-of-2 to catch stride bugs-#[uzu_test]-fn test_stride() {- for_each_non_cpu_backend!(|B| {- let ctx = create_context::<B>();- run_moe_parity_test::<B>(- &ctx,- 1, // t (single token)- 2, // e- 1, // k- 96, // d_model (3 k0 chunks - odd number)- 96, // d_ff (3 ff0 chunks - odd number)- 2, // gating_code (SwiGLU)- 1.0,- (f32::NEG_INFINITY, f32::INFINITY),- (f32::NEG_INFINITY, f32::INFINITY),- 0xE2E_0018,- "VerifyLayout_D96_FF96",- );- })-}--// Test 10d: d_model=1536-#[uzu_test]-fn test_model_1536() {- for_each_non_cpu_backend!(|B| {- let ctx = create_context::<B>();- run_moe_parity_test::<B>(- &ctx,- 2, // t- 4, // e- 1, // k- 1536, // d_model (48 k0 chunks, 24 n-tiles)- 64, // d_ff (small)- 2, // gating_code (SwiGLU)- 1.0,- (f32::NEG_INFINITY, f32::INFINITY),- (f32::NEG_INFINITY, f32::INFINITY),- 0xE2E_0019,- "IsolateD_D1536_FF64_K1",- );- })-}--// Test 10e: 2 n-tiles-#[uzu_test]-fn test_2n_tiles() {- for_each_non_cpu_backend!(|B| {- let ctx = create_context::<B>();- run_moe_parity_test::<B>(- &ctx,- 1, // t- 2, // e- 1, // k- 128, // d_model (2 n-tiles)- 256, // d_ff- 2, // gating_code (SwiGLU)- 1.0,- (f32::NEG_INFINITY, f32::INFINITY),- (f32::NEG_INFINITY, f32::INFINITY),- 0xE2E_0020,- "NTileTest_D128_2tiles",- );- })-}--// Test 10g: 8 n-tiles - find threshold-#[uzu_test]-fn test_8n_tiles() {- for_each_non_cpu_backend!(|B| {- let ctx = create_context::<B>();- run_moe_parity_test::<B>(- &ctx,- 1, // t- 2, // e- 1, // k- 512, // d_model (8 n-tiles)- 256, // d_ff- 2, // gating_code (SwiGLU)- 1.0,- (f32::NEG_INFINITY, f32::INFINITY),- (f32::NEG_INFINITY, f32::INFINITY),- 0xE2E_0021,- "NTileTest_D512_8tiles",- );- })-}--// Test 11b: Production scale with clipping but K=1-#[uzu_test]-#[ignore]-fn test_prod_scale_clipping() {- for_each_non_cpu_backend!(|B| {- let ctx = create_context::<B>();- run_moe_parity_test::<B>(- &ctx,- 2, // t- 8, // e- 1, // k- 4096, // d_model- 14336, // d_ff- 2, // gating_code (SwiGLU)- 1.702, // silu_alpha (GPT-OSS value)- (f32::NEG_INFINITY, 20.0), // gate_clip_max- (-19.0, 21.0), // up_clip- 0xE2E_0029,- "ProductionScale_D4096_H14336_E8_K1",- );- })-}--// Test 11c: PRODUCTION SCALE T=1 decode (triggers GEMV v2)-#[uzu_test]-#[ignore]-fn test_prod_scale_decode() {- for_each_non_cpu_backend!(|B| {- let ctx = create_context::<B>();- run_moe_parity_test::<B>(- &ctx,- 1, // t (single token decode)- 16, // e (16 experts)- 2, // k (2 experts per token)- 4096, // d_model- 14336, // d_ff- 2, // gating_code (SwiGLU)- 1.702, // silu_alpha (GPT-OSS value)- (f32::NEG_INFINITY, 20.0), // gate_clip_max (GPT-OSS config)- (-19.0, 21.0), // up_clip (GPT-OSS config)- 0xE2E_0030,- "ProductionScale_T1_D4096_H14336_E16_K2",- );- })-}--// Test 11d: FULL PRODUCTION SCALE with K=2, T=4 (uses tiled MMA)-#[uzu_test]-#[ignore]-fn test_prod_scale_tile_mma() {- for_each_non_cpu_backend!(|B| {- let ctx = create_context::<B>();- run_moe_parity_test::<B>(- &ctx,- 4, // t (4 tokens)- 16, // e (16 experts)- 2, // k (2 experts per token)- 4096, // d_model (128 k0 chunks, 64 n-tiles!)- 14336, // d_ff (448 ff0 chunks!)- 2, // gating_code (SwiGLU)- 1.702, // silu_alpha (GPT-OSS value)- (f32::NEG_INFINITY, 20.0), // gate_clip_max (GPT-OSS config)- (-19.0, 21.0), // up_clip (GPT-OSS config)- 0xE2E_0031,- "ProductionScale_D4096_H14336_E16_K2",- );- })-}diff --git a/crates/uzu-engine/unit/encodable_block/moe/moe_experts_perf_test.rs b/crates/uzu-engine/unit/encodable_block/moe/moe_experts_perf_test.rsdeleted file mode 100644index d0b2acb22..000000000--- a/crates/uzu-engine/unit/encodable_block/moe/moe_experts_perf_test.rs+++ /dev/null@@ -1,300 +0,0 @@-use std::time::Instant;--use half::bf16;-use rand::{RngExt, SeedableRng, rngs::StdRng};-use uzu_engine_macros::uzu_test;--use super::{MoeExpertsTwoPassArguments, MoeExpertsTwoPassDecodeBlock, MoeExpertsTwoPassPrefillBlock};-use crate::{- backends::common::{Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferPending, Context},- data_type::DataType,- tests::helpers::{create_buffer_with_data, create_context, for_each_non_cpu_backend},-};--fn build_offsets(- e: usize,- sum_k: usize,-) -> Vec<u32> {- let mut offsets = vec![0u32; e + 1];- let mut running = 0u32;- let mut remaining = sum_k;- for expert in 0..e {- offsets[expert] = running;- if remaining > 0 {- running += 1;- remaining -= 1;- }- }- offsets[e] = running;- offsets-}--fn run_decode_case<B: Backend>(- ctx: &B::Context,- name: &str,- t: usize,- d_model: usize,- d_ff: usize,- e: usize,- k: usize,- warmup: usize,- iters: usize,-) {- let mut rng = StdRng::seed_from_u64(0xDEC0DE1234567890);- let sum_k = t * k;-- eprintln!("\n[decode/two-pass] {} => T={}, D={}, FF={}, E={}, K={}, sum_k={}", name, t, d_model, d_ff, e, k, sum_k);-- let x_perm: Vec<bf16> = (0..sum_k * d_model).map(|_| bf16::from_f32(rng.random_range(-1.0..1.0))).collect();- let offsets = build_offsets(e, sum_k);-- // Generate W13 in original layout [E, d_model, 2*d_ff]- let w13_original: Vec<bf16> =- (0..e * d_model * 2 * d_ff).map(|_| bf16::from_f32(rng.random_range(-0.05..0.05))).collect();-- // Transpose to GPU layout [E, 2*d_ff, d_model]- let mut w13 = vec![bf16::from_f32(0.0); e * d_model * 2 * d_ff];- for expert in 0..e {- let src_offset = expert * d_model * 2 * d_ff;- let dst_offset = expert * 2 * d_ff * d_model;- for dm in 0..d_model {- for ff in 0..(2 * d_ff) {- let src_idx = src_offset + dm * 2 * d_ff + ff;- let dst_idx = dst_offset + ff * d_model + dm;- w13[dst_idx] = w13_original[src_idx];- }- }- }- let w2: Vec<bf16> = (0..e * d_ff * d_model).map(|_| bf16::from_f32(rng.random_range(-0.05..0.05))).collect();- let up_biases: Vec<bf16> = (0..e * 2 * d_ff).map(|_| bf16::from_f32(rng.random_range(-0.01..0.01))).collect();- let down_biases: Vec<bf16> = (0..e * d_model).map(|_| bf16::from_f32(rng.random_range(-0.01..0.01))).collect();-- let experts_kernel = MoeExpertsTwoPassDecodeBlock::<B>::new(ctx, DataType::BF16, 2).expect("experts decode kernel");-- let x_perm_buf = create_buffer_with_data::<B, bf16>(ctx, &x_perm);- let offsets_buf = create_buffer_with_data::<B, u32>(ctx, &offsets);- let w13_buf = create_buffer_with_data::<B, bf16>(ctx, &w13);- let w2_buf = create_buffer_with_data::<B, bf16>(ctx, &w2);- let up_biases_buf = create_buffer_with_data::<B, bf16>(ctx, &up_biases);- let down_biases_buf = create_buffer_with_data::<B, bf16>(ctx, &down_biases);-- for _ in 0..warmup {- let mut command_buffer = ctx.create_command_buffer(None, None).expect("Failed to create command buffer");- let output = experts_kernel- .encode(- MoeExpertsTwoPassArguments {- x_perm: &x_perm_buf,- expert_offsets: &offsets_buf,- w13_all: &w13_buf,- w2_all: &w2_buf,- up_biases: &up_biases_buf,- down_biases: &down_biases_buf,- total_rows: sum_k as u32,- d_model: d_model as u32,- d_ff: d_ff as u32,- num_routed_experts: e as u32,- gate_clip_min: f32::NEG_INFINITY,- gate_clip_max: f32::INFINITY,- up_clip_min: f32::NEG_INFINITY,- up_clip_max: f32::INFINITY,- silu_alpha: 1.702,- },- &mut command_buffer,- )- .expect("failed to encode MoE experts");- let completed = command_buffer.end_encoding().submit().wait_until_completed().unwrap();- drop(output);- drop(completed);- }-- let mut times = Vec::with_capacity(iters);- for _ in 0..iters {- let start = Instant::now();- let mut command_buffer = ctx.create_command_buffer(None, None).expect("Failed to create command buffer");- let output = experts_kernel- .encode(- MoeExpertsTwoPassArguments {- x_perm: &x_perm_buf,- expert_offsets: &offsets_buf,- w13_all: &w13_buf,- w2_all: &w2_buf,- up_biases: &up_biases_buf,- down_biases: &down_biases_buf,- total_rows: sum_k as u32,- d_model: d_model as u32,- d_ff: d_ff as u32,- num_routed_experts: e as u32,- gate_clip_min: f32::NEG_INFINITY,- gate_clip_max: f32::INFINITY,- up_clip_min: f32::NEG_INFINITY,- up_clip_max: f32::INFINITY,- silu_alpha: 1.702,- },- &mut command_buffer,- )- .expect("failed to encode MoE experts");- let completed = command_buffer.end_encoding().submit().wait_until_completed().unwrap();- drop(output);- drop(completed);- times.push(start.elapsed().as_secs_f64() * 1000.0);- }-- times.sort_by(|a, b| a.partial_cmp(b).unwrap());- let mean = times.iter().sum::<f64>() / times.len() as f64;- let median = times[times.len() / 2];- let min = times[0];- let max = times[times.len() - 1];- let var = times.iter().map(|t| (t - mean).powi(2)).sum::<f64>() / times.len() as f64;- let std = var.sqrt();-- eprintln!(" mean={:.3}ms median={:.3}ms min={:.3}ms max={:.3}ms std={:.3}ms", mean, median, min, max, std);- eprintln!(" → Latency: {:.1} µs/token", (mean / t as f64) * 1000.0);-}--fn run_two_pass_prefill_case<B: Backend>(- ctx: &B::Context,- name: &str,- t: usize,- d_model: usize,- d_ff: usize,- e: usize,- k: usize,- warmup: usize,- iters: usize,-) {- let mut rng = StdRng::seed_from_u64(0xDEC0DE1234567890);- let sum_k = t * k;-- eprintln!("\n[2-pass prefill] {} => T={}, D={}, FF={}, E={}, K={}, sum_k={}", name, t, d_model, d_ff, e, k, sum_k);-- let x_perm: Vec<bf16> = (0..sum_k * d_model).map(|_| bf16::from_f32(rng.random_range(-1.0..1.0))).collect();- let offsets = build_offsets(e, sum_k);-- let w13_original: Vec<bf16> =- (0..e * d_model * 2 * d_ff).map(|_| bf16::from_f32(rng.random_range(-0.05..0.05))).collect();-- let mut w13 = vec![bf16::from_f32(0.0); e * d_model * 2 * d_ff];- for expert in 0..e {- let src_offset = expert * d_model * 2 * d_ff;- let dst_offset = expert * 2 * d_ff * d_model;- for dm in 0..d_model {- for ff in 0..(2 * d_ff) {- let src_idx = src_offset + dm * 2 * d_ff + ff;- let dst_idx = dst_offset + ff * d_model + dm;- w13[dst_idx] = w13_original[src_idx];- }- }- }- let w2: Vec<bf16> = (0..e * d_ff * d_model).map(|_| bf16::from_f32(rng.random_range(-0.05..0.05))).collect();- let up_biases: Vec<bf16> = (0..e * 2 * d_ff).map(|_| bf16::from_f32(rng.random_range(-0.01..0.01))).collect();- let down_biases: Vec<bf16> = (0..e * d_model).map(|_| bf16::from_f32(rng.random_range(-0.01..0.01))).collect();-- let experts_kernel =- MoeExpertsTwoPassPrefillBlock::<B>::new(ctx, DataType::BF16, 2).expect("experts prefill kernel");-- let x_perm_buf = create_buffer_with_data::<B, bf16>(ctx, &x_perm);- let offsets_buf = create_buffer_with_data::<B, u32>(ctx, &offsets);- let w13_buf = create_buffer_with_data::<B, bf16>(ctx, &w13);- let w2_buf = create_buffer_with_data::<B, bf16>(ctx, &w2);- let up_biases_buf = create_buffer_with_data::<B, bf16>(ctx, &up_biases);- let down_biases_buf = create_buffer_with_data::<B, bf16>(ctx, &down_biases);-- for _ in 0..warmup {- let mut command_buffer = ctx.create_command_buffer(None, None).expect("Failed to create command buffer");- let args = MoeExpertsTwoPassArguments {- x_perm: &x_perm_buf,- expert_offsets: &offsets_buf,- w13_all: &w13_buf,- w2_all: &w2_buf,- up_biases: &up_biases_buf,- down_biases: &down_biases_buf,- total_rows: sum_k as u32,- d_model: d_model as u32,- d_ff: d_ff as u32,- num_routed_experts: e as u32,- gate_clip_min: f32::NEG_INFINITY,- gate_clip_max: f32::INFINITY,- up_clip_min: f32::NEG_INFINITY,- up_clip_max: f32::INFINITY,- silu_alpha: 1.702,- };- let output = experts_kernel.encode(args, &mut command_buffer).expect("failed to encode MoE experts");- let completed = command_buffer.end_encoding().submit().wait_until_completed().unwrap();- drop(output);- drop(completed);- }-- let mut times = Vec::with_capacity(iters);- for _ in 0..iters {- let start = Instant::now();- let mut command_buffer = ctx.create_command_buffer(None, None).expect("Failed to create command buffer");- let args = MoeExpertsTwoPassArguments {- x_perm: &x_perm_buf,- expert_offsets: &offsets_buf,- w13_all: &w13_buf,- w2_all: &w2_buf,- up_biases: &up_biases_buf,- down_biases: &down_biases_buf,- total_rows: sum_k as u32,- d_model: d_model as u32,- d_ff: d_ff as u32,- num_routed_experts: e as u32,- gate_clip_min: f32::NEG_INFINITY,- gate_clip_max: f32::INFINITY,- up_clip_min: f32::NEG_INFINITY,- up_clip_max: f32::INFINITY,- silu_alpha: 1.702,- };- let output = experts_kernel.encode(args, &mut command_buffer).expect("failed to encode MoE experts");- let completed = command_buffer.end_encoding().submit().wait_until_completed().unwrap();- drop(output);- drop(completed);- times.push(start.elapsed().as_secs_f64() * 1000.0);- }-- times.sort_by(|a, b| a.partial_cmp(b).unwrap());- let mean = times.iter().sum::<f64>() / times.len() as f64;- let median = times[times.len() / 2];- let min = times[0];- let max = times[times.len() - 1];- let var = times.iter().map(|t| (t - mean).powi(2)).sum::<f64>() / times.len() as f64;- let std = var.sqrt();-- eprintln!(" mean={:.3}ms median={:.3}ms min={:.3}ms max={:.3}ms std={:.3}ms", mean, median, min, max, std);- eprintln!(" → Throughput: {:.1} µs/token (mean / sum_k)", (mean / sum_k as f64) * 1000.0);-}--#[uzu_test]-#[ignore]-fn test_two_pass_decode_speed() {- for_each_non_cpu_backend!(|B| {- let ctx = create_context::<B>();-- let cases = vec![- ("T1_E16_D1024_F4096", 1, 1024, 4096, 16, 4, 3, 20),- ("T1_E8_D2048_F8192", 1, 2048, 8192, 8, 4, 3, 20),- ];-- for (name, t, d_model, d_ff, e, k, warmup, iters) in &cases {- run_decode_case::<B>(&ctx, name, *t, *d_model, *d_ff, *e, *k, *warmup, *iters);- }- });-}--#[uzu_test]-#[ignore]-fn test_two_pass_prefill_speed() {- for_each_non_cpu_backend!(|B| {- let ctx = create_context::<B>();-- let cases = vec![- ("T32_E16_D1024_F4096", 32, 1024, 4096, 16, 4, 2, 10),- ("T64_E16_D2048_F6144", 64, 2048, 6144, 16, 4, 2, 10),- ("T256_E16_D2048_F6144", 256, 2048, 6144, 16, 4, 2, 10),- ];-- for (name, t, d_model, d_ff, e, k, warmup, iters) in cases {- run_two_pass_prefill_case::<B>(&ctx, name, t, d_model, d_ff, e, k, warmup, iters);- }- })-}diff --git a/crates/uzu-engine/unit/encodable_block/moe/moe_experts_test.rs b/crates/uzu-engine/unit/encodable_block/moe/moe_experts_test.rsdeleted file mode 100644index 08e5c1f4e..000000000--- a/crates/uzu-engine/unit/encodable_block/moe/moe_experts_test.rs+++ /dev/null@@ -1,638 +0,0 @@-//! Unit correctness tests for MoE Expert kernels (both 1-pass and 2-pass variants)-//!-//! Tests verify:-//! - Decode (suffix_length=1) with 2-pass tiled implementation-//! - Prefill (suffix_length>1) with 2-pass tiled implementation-//! - Intermediate buffer correctness (row maps, tiles, dispatch args)-//! - Numerical correctness against CPU reference--use half::bf16;-use rand::{RngExt, SeedableRng, rngs::StdRng};-use uzu_engine_macros::uzu_test;--use super::{- MoeExpertsTwoPassArguments, MoeExpertsTwoPassDecodeBlock, MoeExpertsTwoPassPrefillBlock, cpu_tile_counts,- cpu_tile_scan,-};-use crate::{- backends::common::{- CommandBufferEncoding, CommandBufferExecutable, CommandBufferPending, Context,- gpu_types::{ActivationType, activation_silu_alpha},- },- data_type::DataType,- tests::{- assert::assert_eq_float,- helpers::{- buffer_prefix_to_vec, buffer_readback, create_buffer_with_data, create_context, for_each_non_cpu_backend,- },- },-};--/// Test data for MoE experts-struct MoeTestData {- x: Vec<bf16>,- topk_ids: Vec<i32>,- topk_probs: Vec<bf16>,- w13: Vec<bf16>,- w2: Vec<bf16>,- up_biases: Vec<bf16>,- down_biases: Vec<bf16>,-}--impl MoeTestData {- fn generate(- rng: &mut StdRng,- t: usize,- k: usize,- d_model: usize,- d_ff: usize,- e: usize,- ) -> Self {- let x: Vec<bf16> = (0..t * d_model).map(|_| bf16::from_f32(rng.random_range(-1.0..1.0))).collect();-- let topk_ids: Vec<i32> = (0..t * k).map(|i| ((i / k) + (i % k) * 2) as i32 % e as i32).collect();-- let topk_probs: Vec<bf16> = (0..t)- .flat_map(|_| {- let raw: Vec<f32> = (0..k).map(|_| rng.random_range(0.1..1.0)).collect();- let sum: f32 = raw.iter().sum();- raw.iter().map(|p| bf16::from_f32(p / sum)).collect::<Vec<_>>()- })- .collect();-- let w13: Vec<bf16> =- (0..e * 2 * d_ff * d_model).map(|_| bf16::from_f32(rng.random_range(-0.05..0.05))).collect();- let w2: Vec<bf16> = (0..e * d_ff * d_model).map(|_| bf16::from_f32(rng.random_range(-0.05..0.05))).collect();- let up_biases: Vec<bf16> = (0..e * 2 * d_ff).map(|_| bf16::from_f32(rng.random_range(-0.01..0.01))).collect();- let down_biases: Vec<bf16> = (0..e * d_model).map(|_| bf16::from_f32(rng.random_range(-0.01..0.01))).collect();-- Self {- x,- topk_ids,- topk_probs,- w13,- w2,- up_biases,- down_biases,- }- }-}--/// Result of scatter operation - buckets rows by expert-struct ScatterResult {- x_perm: Vec<bf16>,- offsets: Vec<u32>,- perm_idx: Vec<usize>, // original row -> bucketed row (for gather)-}--fn scatter_by_expert(- x: &[bf16],- topk_ids: &[i32],- t: usize,- k: usize,- d_model: usize,- e: usize,-) -> ScatterResult {- let sum_k = t * k;-- // Count rows per expert and build offsets- let mut expert_counts = vec![0usize; e];- for &eid in topk_ids {- expert_counts[eid as usize] += 1;- }- let mut offsets = vec![0u32; e + 1];- for i in 0..e {- offsets[i + 1] = offsets[i] + expert_counts[i] as u32;- }-- // Build permutation indices- let mut expert_cursors = vec![0usize; e];- let mut perm_idx = vec![0usize; sum_k];- let mut inv_perm = vec![0usize; sum_k];- for orig_row in 0..sum_k {- let eid = topk_ids[orig_row] as usize;- let bucket_pos = offsets[eid] as usize + expert_cursors[eid];- perm_idx[orig_row] = bucket_pos;- inv_perm[bucket_pos] = orig_row;- expert_cursors[eid] += 1;- }-- // Build x_perm in bucketed order- let mut x_perm = vec![bf16::from_f32(0.0); sum_k * d_model];- for bucket_row in 0..sum_k {- let orig_row = inv_perm[bucket_row];- let tok = orig_row / k;- for d in 0..d_model {- x_perm[bucket_row * d_model + d] = x[tok * d_model + d];- }- }-- ScatterResult {- x_perm,- offsets,- perm_idx,- }-}--/// Gather + finalize: reorder from bucketed order and apply weighted sum-fn gather_and_finalize(- y_partial: &[bf16],- topk_probs: &[bf16],- perm_idx: &[usize],- t: usize,- k: usize,- d_model: usize,-) -> Vec<bf16> {- let mut y = vec![bf16::from_f32(0.0); t * d_model];- for tok in 0..t {- for d in 0..d_model {- let mut acc = 0.0f32;- for ki in 0..k {- let orig_row = tok * k + ki;- let bucket_row = perm_idx[orig_row];- let prob = f32::from(topk_probs[orig_row]);- let val = f32::from(y_partial[bucket_row * d_model + d]);- acc += prob * val;- }- y[tok * d_model + d] = bf16::from_f32(acc);- }- }- y-}--/// Unified CPU reference for MoE computation-///-/// Computes: y[t] = Σ_k prob[t,k] * (activation(x[t] @ W13[expert_k]) @ W2[expert_k] + bias)-///-/// Supports 4 gating modes:-/// - 0: GELU(up)-/// - 1: SiLU(up)-/// - 2: SwiGLU = SiLU(gate) * up-/// - 3: GEGLU = GELU(gate) * up-fn cpu_moe_reference(- x: &[bf16], // [T, d_model]- topk_ids: &[i32], // [T * K]- topk_probs: &[bf16], // [T * K]- w13_all: &[bf16], // [E, 2*d_ff, d_model] transposed- w2_all: &[bf16], // [E, d_model, d_ff] transposed- up_biases: &[bf16], // [E, 2*d_ff]- down_biases: &[bf16], // [E, d_model]- t: usize,- d_model: usize,- d_ff: usize,- k: usize,- gating_code: u32,- silu_alpha: f32,- gate_clip_min: f32,- gate_clip_max: f32,- up_clip_min: f32,- up_clip_max: f32,-) -> Vec<bf16> {- let mut y = vec![bf16::from_f32(0.0); t * d_model];-- // Process each token- for tok in 0..t {- let x_offset = tok * d_model;-- // Hidden buffer for this token's K experts [K, d_ff]- let mut hidden = vec![0.0f32; k * d_ff];-- // Pass A: x @ W13[expert] -> hidden[k] with activation- for k_idx in 0..k {- let expert_id = topk_ids[tok * k + k_idx];- if expert_id < 0 {- continue;- }- let expert = expert_id as usize;-- let w13_base = expert * 2 * d_ff * d_model;- let bias_base = expert * 2 * d_ff;-- for h in 0..d_ff {- // Up projection- let mut acc_up = f32::from(up_biases[bias_base + h]);- for d in 0..d_model {- let x_val = f32::from(x[x_offset + d]);- let w_val = f32::from(w13_all[w13_base + h * d_model + d]);- acc_up += x_val * w_val;- }-- // Apply activation based on gating_code- let activated = if gating_code <= 1 {- // GELU or SiLU on up (with clipping)- let up_val = acc_up.clamp(up_clip_min, up_clip_max);- if gating_code == 0 {- ActivationType::GELUApprox.activate(up_val)- } else {- activation_silu_alpha(up_val, silu_alpha)- }- } else {- // SwiGLU or GEGLU - need gate projection too- let mut acc_gate = f32::from(up_biases[bias_base + d_ff + h]);- for d in 0..d_model {- let x_val = f32::from(x[x_offset + d]);- let w_val = f32::from(w13_all[w13_base + (d_ff + h) * d_model + d]);- acc_gate += x_val * w_val;- }-- // Apply clipping- let up_val = acc_up.clamp(up_clip_min, up_clip_max);- let gate_val = acc_gate.clamp(gate_clip_min, gate_clip_max);-- let gate_act = if gating_code == 2 {- activation_silu_alpha(gate_val, silu_alpha)- } else {- ActivationType::GELUApprox.activate(gate_val)- };- gate_act * up_val- };-- hidden[k_idx * d_ff + h] = activated;- }- }-- // Pass B: hidden @ W2 -> y (with weighted sum)- for d in 0..d_model {- let mut final_acc = 0.0f32;-- for k_idx in 0..k {- let expert_id = topk_ids[tok * k + k_idx];- if expert_id < 0 {- continue;- }- let expert = expert_id as usize;- let prob = f32::from(topk_probs[tok * k + k_idx]);-- let w2_base = expert * d_model * d_ff;- let bias_base = expert * d_model;-- let mut acc = f32::from(down_biases[bias_base + d]);- for h in 0..d_ff {- let h_val = hidden[k_idx * d_ff + h];- let w_val = f32::from(w2_all[w2_base + d * d_ff + h]);- acc += h_val * w_val;- }- final_acc += prob * acc;- }-- y[tok * d_model + d] = bf16::from_f32(final_acc);- }- }-- y-}--#[uzu_test]-fn test_two_pass_decode_correctness() {- for_each_non_cpu_backend!(|B| {- let ctx = create_context::<B>();- let mut rng = StdRng::seed_from_u64(0xDEC0DE42);-- // End-to-end test: T=1 token, K=2 experts per token- let t = 1;- let k = 2;- let sum_k = t * k;- let d_model = 512;- let d_ff = 2048;- let e = 8;- let gating_code = 2u32; // SwiGLU- let silu_alpha = 1.0f32;-- eprintln!("[2-pass decode] T={}, K={}, sum_k={}, d_model={}, d_ff={}, E={}", t, k, sum_k, d_model, d_ff, e);-- // Generate input x [T, d_model]- let x: Vec<bf16> = (0..t * d_model).map(|_| bf16::from_f32(rng.random_range(-1.0..1.0))).collect();-- // Generate routing: topk_ids [T*K], topk_probs [T*K]- let topk_ids: Vec<i32> = (0..t * k).map(|i| ((i / k) + (i % k) * 2) as i32 % e as i32).collect();- let topk_probs: Vec<bf16> = (0..t)- .flat_map(|_| {- let raw: Vec<f32> = (0..k).map(|_| rng.random_range(0.1..1.0)).collect();- let sum: f32 = raw.iter().sum();- raw.iter().map(|p| bf16::from_f32(p / sum)).collect::<Vec<_>>()- })- .collect();-- // Scatter x based on topk_ids to get x_perm- // Each token contributes K rows, one per selected expert- let mut x_perm = vec![bf16::from_f32(0.0); sum_k * d_model];- for tok in 0..t {- for ki in 0..k {- let row_idx = tok * k + ki;- for d in 0..d_model {- x_perm[row_idx * d_model + d] = x[tok * d_model + d];- }- }- }-- let mut expert_counts = vec![0usize; e];- for &eid in &topk_ids {- expert_counts[eid as usize] += 1;- }- let mut offsets = vec![0u32; e + 1];- for i in 0..e {- offsets[i + 1] = offsets[i] + expert_counts[i] as u32;- }-- // Generate weights- let w13: Vec<bf16> =- (0..e * 2 * d_ff * d_model).map(|_| bf16::from_f32(rng.random_range(-0.05..0.05))).collect();- let w2: Vec<bf16> = (0..e * d_ff * d_model).map(|_| bf16::from_f32(rng.random_range(-0.05..0.05))).collect();- let up_biases: Vec<bf16> = (0..e * 2 * d_ff).map(|_| bf16::from_f32(rng.random_range(-0.01..0.01))).collect();- let down_biases: Vec<bf16> = (0..e * d_model).map(|_| bf16::from_f32(rng.random_range(-0.01..0.01))).collect();-- // CPU reference - end-to-end- let y_expected = cpu_moe_reference(- &x,- &topk_ids,- &topk_probs,- &w13,- &w2,- &up_biases,- &down_biases,- t,- d_model,- d_ff,- k,- gating_code,- silu_alpha,- f32::NEG_INFINITY,- f32::INFINITY,- f32::NEG_INFINITY,- f32::INFINITY,- );-- // Prepare GPU buffers- let x_perm_buf = create_buffer_with_data::<B, bf16>(&ctx, &x_perm);- let offsets_buf = create_buffer_with_data::<B, u32>(&ctx, &offsets);- let w13_buf = create_buffer_with_data::<B, bf16>(&ctx, &w13);- let w2_buf = create_buffer_with_data::<B, bf16>(&ctx, &w2);- let up_biases_buf = create_buffer_with_data::<B, bf16>(&ctx, &up_biases);- let down_biases_buf = create_buffer_with_data::<B, bf16>(&ctx, &down_biases);-- let experts_kernel = MoeExpertsTwoPassDecodeBlock::<B>::new(&ctx, DataType::BF16, gating_code)- .expect("MoeExpertsTwoPassDecodeKernel::new");- let mut command_buffer = ctx.create_command_buffer(None, None).expect("Failed to create command buffer");-- let y_partial_buf = experts_kernel- .encode(- MoeExpertsTwoPassArguments {- x_perm: &x_perm_buf,- expert_offsets: &offsets_buf,- w13_all: &w13_buf,- w2_all: &w2_buf,- up_biases: &up_biases_buf,- down_biases: &down_biases_buf,- total_rows: sum_k as u32,- d_model: d_model as u32,- d_ff: d_ff as u32,- num_routed_experts: e as u32,- gate_clip_min: f32::NEG_INFINITY,- gate_clip_max: f32::INFINITY,- up_clip_min: f32::NEG_INFINITY,- up_clip_max: f32::INFINITY,- silu_alpha,- },- &mut command_buffer,- )- .expect("failed to encode MoE experts");-- let completed = command_buffer.end_encoding().submit().wait_until_completed().unwrap();-- // Read GPU partial output and do CPU finalize (weighted sum)- let y_partial_gpu =- buffer_prefix_to_vec::<B, bf16>(&buffer_readback::<B>(&ctx, &y_partial_buf), sum_k * d_model);-- // Finalize: y[t] = Σ_k prob[t,k] * y_partial[t*k + k_idx]- let mut y_gpu = vec![bf16::from_f32(0.0); t * d_model];- for tok in 0..t {- for d in 0..d_model {- let mut acc = 0.0f32;- for ki in 0..k {- let row_idx = tok * k + ki;- let prob = f32::from(topk_probs[row_idx]);- let val = f32::from(y_partial_gpu[row_idx * d_model + d]);- acc += prob * val;- }- y_gpu[tok * d_model + d] = bf16::from_f32(acc);- }- }-- // Compute error metrics- let mut max_abs_error = 0.0f32;- let mut max_idx = 0;-- for (i, (&gpu_val, &cpu_val)) in y_gpu.iter().zip(y_expected.iter()).enumerate() {- let abs_error = (f32::from(gpu_val) - f32::from(cpu_val)).abs();- if abs_error > max_abs_error {- max_abs_error = abs_error;- max_idx = i;- }- }-- eprintln!(- "[2-pass decode] Max error: {:.6} at idx {} (GPU={:.6}, CPU={:.6})",- max_abs_error,- max_idx,- f32::from(y_gpu[max_idx]),- f32::from(y_expected[max_idx])- );-- let tolerance = 0.02;- assert_eq_float(&y_expected, &y_gpu, tolerance, "2-pass decode output");-- eprintln!("[2-pass decode] ✓ PASSED (tolerance={:.4})", tolerance);- drop(y_partial_buf);- drop(completed);- });-}--#[uzu_test]-fn test_two_pass_decode_multi_token() {- for_each_non_cpu_backend!(|B| {- let ctx = create_context::<B>();- let mut rng = StdRng::seed_from_u64(0xDEC0DE43);-- let t = 4;- let k = 2;- let sum_k = t * k;- let d_model = 512;- let d_ff = 2048;- let e = 8;- let gating_code = 2u32;- let silu_alpha = 1.0f32;-- eprintln!("[2-pass decode multi-token] T={}, K={}, d_model={}, d_ff={}, E={}", t, k, d_model, d_ff, e);-- let data = MoeTestData::generate(&mut rng, t, k, d_model, d_ff, e);- let scatter = scatter_by_expert(&data.x, &data.topk_ids, t, k, d_model, e);-- let y_expected = cpu_moe_reference(- &data.x,- &data.topk_ids,- &data.topk_probs,- &data.w13,- &data.w2,- &data.up_biases,- &data.down_biases,- t,- d_model,- d_ff,- k,- gating_code,- silu_alpha,- f32::NEG_INFINITY,- f32::INFINITY,- f32::NEG_INFINITY,- f32::INFINITY,- );-- // GPU buffers- let x_perm_buf = create_buffer_with_data::<B, bf16>(&ctx, &scatter.x_perm);- let offsets_buf = create_buffer_with_data::<B, u32>(&ctx, &scatter.offsets);- let w13_buf = create_buffer_with_data::<B, bf16>(&ctx, &data.w13);- let w2_buf = create_buffer_with_data::<B, bf16>(&ctx, &data.w2);- let up_biases_buf = create_buffer_with_data::<B, bf16>(&ctx, &data.up_biases);- let down_biases_buf = create_buffer_with_data::<B, bf16>(&ctx, &data.down_biases);-- let experts_kernel = MoeExpertsTwoPassDecodeBlock::<B>::new(&ctx, DataType::BF16, gating_code).expect("kernel");- let mut command_buffer = ctx.create_command_buffer(None, None).expect("Failed to create command buffer");- let y_partial_buf = experts_kernel- .encode(- MoeExpertsTwoPassArguments {- x_perm: &x_perm_buf,- expert_offsets: &offsets_buf,- w13_all: &w13_buf,- w2_all: &w2_buf,- up_biases: &up_biases_buf,- down_biases: &down_biases_buf,- total_rows: sum_k as u32,- d_model: d_model as u32,- d_ff: d_ff as u32,- num_routed_experts: e as u32,- gate_clip_min: f32::NEG_INFINITY,- gate_clip_max: f32::INFINITY,- up_clip_min: f32::NEG_INFINITY,- up_clip_max: f32::INFINITY,- silu_alpha,- },- &mut command_buffer,- )- .expect("failed to encode MoE experts");- let completed = command_buffer.end_encoding().submit().wait_until_completed().unwrap();-- let y_partial_gpu =- buffer_prefix_to_vec::<B, bf16>(&buffer_readback::<B>(&ctx, &y_partial_buf), sum_k * d_model);- let y_gpu = gather_and_finalize(&y_partial_gpu, &data.topk_probs, &scatter.perm_idx, t, k, d_model);-- assert_eq_float(&y_expected, &y_gpu, 0.02, "2-pass decode multi-token");- eprintln!("[2-pass decode multi-token] ✓ PASSED");- drop(y_partial_buf);- drop(completed);- });-}--#[uzu_test]-fn test_two_pass_prefill_correctness() {- for_each_non_cpu_backend!(|B| {- let ctx = create_context::<B>();- let mut rng = StdRng::seed_from_u64(0xFE1111);-- let t = 8;- let k = 2;- let sum_k = t * k;- let d_model = 512;- let d_ff = 2048;- let e = 8;- let gating_code = 2u32;- let silu_alpha = 1.0f32;-- eprintln!("[2-pass prefill] T={}, K={}, d_model={}, d_ff={}, E={}", t, k, d_model, d_ff, e);-- let data = MoeTestData::generate(&mut rng, t, k, d_model, d_ff, e);- let scatter = scatter_by_expert(&data.x, &data.topk_ids, t, k, d_model, e);-- let y_expected = cpu_moe_reference(- &data.x,- &data.topk_ids,- &data.topk_probs,- &data.w13,- &data.w2,- &data.up_biases,- &data.down_biases,- t,- d_model,- d_ff,- k,- gating_code,- silu_alpha,- f32::NEG_INFINITY,- f32::INFINITY,- f32::NEG_INFINITY,- f32::INFINITY,- );-- // GPU buffers- let x_perm_buf = create_buffer_with_data::<B, bf16>(&ctx, &scatter.x_perm);- let offsets_buf = create_buffer_with_data::<B, u32>(&ctx, &scatter.offsets);- let w13_buf = create_buffer_with_data::<B, bf16>(&ctx, &data.w13);- let w2_buf = create_buffer_with_data::<B, bf16>(&ctx, &data.w2);- let up_biases_buf = create_buffer_with_data::<B, bf16>(&ctx, &data.up_biases);- let down_biases_buf = create_buffer_with_data::<B, bf16>(&ctx, &data.down_biases);-- let experts_kernel =- MoeExpertsTwoPassPrefillBlock::<B>::new(&ctx, DataType::BF16, gating_code).expect("kernel");- let mut command_buffer = ctx.create_command_buffer(None, None).expect("Failed to create command buffer");- let args = MoeExpertsTwoPassArguments {- x_perm: &x_perm_buf,- expert_offsets: &offsets_buf,- w13_all: &w13_buf,- w2_all: &w2_buf,- up_biases: &up_biases_buf,- down_biases: &down_biases_buf,- total_rows: sum_k as u32,- d_model: d_model as u32,- d_ff: d_ff as u32,- num_routed_experts: e as u32,- gate_clip_min: f32::NEG_INFINITY,- gate_clip_max: f32::INFINITY,- up_clip_min: f32::NEG_INFINITY,- up_clip_max: f32::INFINITY,- silu_alpha,- };- let y_partial_buf = experts_kernel.encode(args, &mut command_buffer).expect("failed to encode MoE experts");- let completed = command_buffer.end_encoding().submit().wait_until_completed().unwrap();-- let y_partial_gpu =- buffer_prefix_to_vec::<B, bf16>(&buffer_readback::<B>(&ctx, &y_partial_buf), sum_k * d_model);- let y_gpu = gather_and_finalize(&y_partial_gpu, &data.topk_probs, &scatter.perm_idx, t, k, d_model);-- assert_eq_float(&y_expected, &y_gpu, 0.02, "2-pass prefill");- eprintln!("[2-pass prefill] ✓ PASSED");- drop(y_partial_buf);- drop(completed);- });-}--#[uzu_test]-fn test_tile_infrastructure() {- // Test that tile counts/scan/map are computed correctly- // This is already tested in moe_tiles_test.rs, but we verify here- // that the tiles match our expectations for expert workload distribution-- let offsets = vec![0, 5, 20, 35, 40];- const BM: usize = 16;-- let tile_counts = cpu_tile_counts(&offsets, BM);- let (tile_offsets, total_tiles) = cpu_tile_scan(&tile_counts);-- // Expert 0: 5 rows -> 1 tile- // Expert 1: 15 rows -> 1 tile- // Expert 2: 15 rows -> 1 tile- // Expert 3: 5 rows -> 1 tile- assert_eq!(tile_counts, vec![1, 1, 1, 1]);- assert_eq!(tile_offsets, vec![0, 1, 2, 3, 4]);- assert_eq!(total_tiles, 4);-- eprintln!("[tile infrastructure] ✓ PASSED");-}diff --git a/crates/uzu-engine/unit/encodable_block/moe/moe_gather_test.rs b/crates/uzu-engine/unit/encodable_block/moe/moe_gather_test.rsdeleted file mode 100644index 0e94bc8c5..000000000--- a/crates/uzu-engine/unit/encodable_block/moe/moe_gather_test.rs+++ /dev/null@@ -1,97 +0,0 @@-use half::bf16;-use num_traits::Float;-use rand::{RngExt, SeedableRng, rngs::StdRng};-use uzu_engine_macros::uzu_test;--use super::MoeGather;-use crate::{- array::ArrayElement,- backends::{- common::{Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferPending, Context},- cpu::Cpu,- },- tests::{- assert::assert_eq_float,- helpers::{buffer_readback, buffer_to_vec, create_buffer_with_data, create_context, for_each_non_cpu_backend},- },-};--struct Input<T: ArrayElement + Float> {- x: Box<[T]>,- bucket_ids: Box<[i32]>,- t: usize,- sum_k: usize,- d_model: usize,-}--fn get_output<B: Backend, T: ArrayElement + Float>(input: &Input<T>) -> Vec<T> {- let context = create_context::<B>();-- let sumk_data: [u32; 1] = [input.sum_k as u32];- let x_buffer = create_buffer_with_data::<B, T>(&context, &input.x);- let ids_buffer = create_buffer_with_data::<B, i32>(&context, &input.bucket_ids);- let sumk_buffer = create_buffer_with_data::<B, u32>(&context, &sumk_data);- let gather = MoeGather::<B>::new(&context, T::data_type()).expect("MoeGather::new");- let mut command_buffer = context.create_command_buffer(None, None).expect("Failed to create command buffer");- let x_perm_buffer = gather- .encode(- &x_buffer,- &ids_buffer,- &sumk_buffer,- input.t as u32,- (input.sum_k / input.t) as u32,- input.d_model as u32,- &mut command_buffer,- )- .expect("Failed to encode MoE gather");- let completed = command_buffer.end_encoding().submit().wait_until_completed().unwrap();-- let output = buffer_to_vec::<B, T>(&buffer_readback::<B>(&context, &x_perm_buffer));- drop(x_perm_buffer);- drop(completed);- output-}--fn test_gather_internal(- t: usize,- sum_k: usize,- d_model: usize,-) {- let mut rng = StdRng::seed_from_u64(2027);- let x: Vec<bf16> = (0..t * d_model).map(|_| bf16::from_f32(rng.random_range(-2.0..2.0))).collect();- let bucketed_ids: Vec<i32> = (0..sum_k).map(|_| rng.random_range(0..t as i32)).collect();-- let input = Input::<bf16> {- x: x.clone().into_boxed_slice(),- bucket_ids: bucketed_ids.clone().into_boxed_slice(),- t,- sum_k,- d_model,- };-- let cpu_ref_output = get_output::<Cpu, bf16>(&input);- for_each_non_cpu_backend!(|B| {- let output = get_output::<B, bf16>(&input);- assert_eq_float(&cpu_ref_output, &output, 1e-6, "Moe gather");- })-}--#[uzu_test]-fn test_gather_single_token() {- test_gather_internal(1, 2, 64);-}--#[uzu_test]-fn test_gather_small_batch() {- test_gather_internal(4, 8, 128);-}--#[uzu_test]-fn test_gather_medium_batch() {- test_gather_internal(16, 32, 256)-}--#[uzu_test]-fn test_gather_large_batch() {- test_gather_internal(64, 128, 512)-}diff --git a/crates/uzu-engine/unit/encodable_block/moe/moe_perf_test.rs b/crates/uzu-engine/unit/encodable_block/moe/moe_perf_test.rsdeleted file mode 100644index 2ada4cd89..000000000--- a/crates/uzu-engine/unit/encodable_block/moe/moe_perf_test.rs+++ /dev/null@@ -1,443 +0,0 @@-use half::bf16;-use rand::{RngExt, SeedableRng, rngs::StdRng};-use uzu_engine_macros::uzu_test;--use super::{MoeExpertsTwoPassArguments, MoeExpertsTwoPassDecodeBlock, MoeGather};-use crate::{- backends::common::{- Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferPending, Context, Kernels,- kernel::{- MoeBlockBasesFromPartialsKernel, MoeCountsOffsetsFusedKernel, MoeFinalizeKernel, MoeRouterTopKKernel,- MoeScatterBucketsMapKernel,- },- },- data_type::DataType,- tests::{- helpers::{create_buffer, create_buffer_with_data, create_context, for_each_non_cpu_backend},- perf::run_perf_with_warmup,- },-};--// Test E2E MoE performance with timing breakdown (decode mode, T=1)-#[uzu_test]-#[ignore]-fn test_moe_e2e_decode_perf() {- eprintln!("\n=== End-to-End MoE Performance (DECODE, T=1) ===");- for_each_non_cpu_backend!(|B| {- let configs = vec![- ("Small", 1usize, 1024usize, 256usize, 8usize, 2usize),- ("Medium", 1usize, 2048usize, 1024usize, 16usize, 2usize),- ("Production", 1usize, 4096usize, 14336usize, 16usize, 2usize),- ];- for (config_name, t, d_model, d_ff, e, k) in configs {- eprintln!("\n{} T={}, D={}, H={}, E={}, K={}", config_name, t, d_model, d_ff, e, k);- let ctx = create_context::<B>();- let mut rng = StdRng::seed_from_u64(0xA0075);-- // Generate data- let x: Vec<bf16> = (0..t * d_model).map(|_| bf16::from_f32(rng.random_range(-1.0..1.0))).collect();- let router_w: Vec<bf16> = (0..e * d_model).map(|_| bf16::from_f32(rng.random_range(-0.5..0.5))).collect();- let router_b: Vec<bf16> = (0..e).map(|_| bf16::from_f32(rng.random_range(-0.1..0.1))).collect();-- // Buffers (simplified - just key buffers for timing)- let x_buf = create_buffer_with_data::<B, bf16>(&ctx, &x);- let router_w_buf = create_buffer_with_data::<B, bf16>(&ctx, &router_w);- let router_b_buf = create_buffer_with_data::<B, bf16>(&ctx, &router_b);- let mut topk_ids_buf = create_buffer::<B, i32>(&ctx, t * k);- let mut topk_probs_buf = create_buffer::<B, bf16>(&ctx, t * k);-- let router_topk = <<B as Backend>::Kernels as Kernels>::MoeRouterTopKKernel::new(- &ctx,- DataType::BF16,- true,- false,- false,- false,- false,- )- .expect("router+topk fused kernel");-- // Time fused Router+TopK- let fused_perf = run_perf_with_warmup("Router+TopK (FUSED)", 5, 20, || {- let mut command_buffer =- ctx.create_command_buffer(None, None).expect("Failed to create command buffer");- router_topk.encode(- &x_buf,- &router_w_buf,- Some(&router_b_buf),- None::<&<B as Backend>::GlobalBuffer>,- None::<&<B as Backend>::GlobalBuffer>,- &mut topk_ids_buf,- &mut topk_probs_buf,- t as u32,- d_model as u32,- e as u32,- k as u32,- true,- None::<f32>,- None::<f32>,- &mut command_buffer,- );- command_buffer.end_encoding().submit().wait_until_completed().unwrap();- });- fused_perf.print();- eprintln!(" Total: {:8.1} µs/token", fused_perf.mean_ms * 1000.0);- }- });-}--// Test E2E MoE performance with timing breakdown (prefill mode, T>1)-#[uzu_test]-#[ignore]-fn test_moe_e2e_prefill_perf() {- eprintln!("\n=== End-to-End MoE Performance (PREFILL, T>1) ===");- for_each_non_cpu_backend!(|B| {- let configs = vec![- ("Batch4", 4, 4096, 14336, 16, 2),- ("Batch16", 16, 4096, 14336, 16, 2),- ("Batch32", 32, 4096, 14336, 16, 2),- ("Batch64", 64, 4096, 14336, 16, 2),- ];-- for (config_name, t, d_model, d_ff, e, k) in configs {- eprintln!("\n{} T={}, D={}, H={}, E={}, K={}", config_name, t, d_model, d_ff, e, k);- let ctx = create_context::<B>();- let mut rng = StdRng::seed_from_u64(0xA0076);-- // Generate data- let x: Vec<bf16> = (0..t * d_model).map(|_| bf16::from_f32(rng.random_range(-1.0..1.0))).collect();- let router_w: Vec<bf16> = (0..e * d_model).map(|_| bf16::from_f32(rng.random_range(-0.5..0.5))).collect();- let router_b: Vec<bf16> = (0..e).map(|_| bf16::from_f32(rng.random_range(-0.1..0.1))).collect();-- // Buffers (simplified - just key buffers for timing)- let x_buf = create_buffer_with_data::<B, bf16>(&ctx, &x);- let router_w_buf = create_buffer_with_data::<B, bf16>(&ctx, &router_w);- let router_b_buf = create_buffer_with_data::<B, bf16>(&ctx, &router_b);- let mut topk_ids_buf = create_buffer::<B, i32>(&ctx, t * k);- let mut topk_probs_buf = create_buffer::<B, bf16>(&ctx, t * k);-- let router_topk = <<B as Backend>::Kernels as Kernels>::MoeRouterTopKKernel::new(- &ctx,- DataType::BF16,- true,- false,- false,- false,- false,- )- .expect("router+topk fused kernel");-- // Time fused Router+TopK- let fused_perf = run_perf_with_warmup("Router+TopK (FUSED)", 5, 20, || {- let mut command_buffer =- ctx.create_command_buffer(None, None).expect("Failed to create command buffer");- router_topk.encode(- &x_buf,- &router_w_buf,- Some(&router_b_buf),- None::<&<B as Backend>::GlobalBuffer>,- None::<&<B as Backend>::GlobalBuffer>,- &mut topk_ids_buf,- &mut topk_probs_buf,- t as u32,- d_model as u32,- e as u32,- k as u32,- true,- None::<f32>,- None::<f32>,- &mut command_buffer,- );- command_buffer.end_encoding().submit().wait_until_completed().unwrap();- });- eprintln!(" Total: {:8.3} ms", fused_perf.mean_ms);- eprintln!(- " Throughput: {:.1} tokens/sec, {:.3} ms/token",- (t as f64 / fused_perf.mean_ms) * 1000.0,- fused_perf.mean_ms / t as f64- );- }- });-}--// Test complete MoE pipeline timing breakdown (decode mode, T=1)-#[uzu_test]-#[ignore]-fn test_moe_pipeline_breakdown_decode() {- for_each_non_cpu_backend!(|B| {- let ctx = create_context::<B>();- let mut rng = StdRng::seed_from_u64(0xDECADE);-- eprintln!("\n=== MoE Pipeline Breakdown (DECODE, T=1) ===");- eprintln!("Measures ALL MoE kernels: Router→TopK→Counts→Offsets→Scatter→Gather→Experts→Finalize\n");-- let (t, d_model, d_ff, e, k) = (1, 4096, 14336, 16, 2);-- // Allocate buffers- let x: Vec<bf16> = (0..t * d_model).map(|_| bf16::from_f32(rng.random_range(-1.0..1.0))).collect();- let router_w: Vec<bf16> = (0..e * d_model).map(|_| bf16::from_f32(rng.random_range(-0.5..0.5))).collect();- let router_b: Vec<bf16> = (0..e).map(|_| bf16::from_f32(rng.random_range(-0.1..0.1))).collect();-- let x_buf = create_buffer_with_data::<B, bf16>(&ctx, &x);- let router_w_buf = create_buffer_with_data::<B, bf16>(&ctx, &router_w);- let router_b_buf = create_buffer_with_data::<B, bf16>(&ctx, &router_b);- let mut topk_ids_buf = create_buffer::<B, i32>(&ctx, t * k);- let mut topk_probs_buf = create_buffer::<B, bf16>(&ctx, t * k);- let num_blocks = t.div_ceil(256).max(1);- let num_tiles = e.div_ceil(512).max(1);- let mut partials_buf = create_buffer::<B, i32>(&ctx, num_blocks * num_tiles * e);- let mut offsets_buf = create_buffer::<B, u32>(&ctx, e + 1);- let mut sumk_buf = create_buffer::<B, u32>(&ctx, 1);- let mut bucketed_ids_buf = create_buffer::<B, i32>(&ctx, t * k);- let y_partial: Vec<bf16> = (0..t * k * d_model).map(|_| bf16::from_f32(rng.random_range(-1.0..1.0))).collect();- let y_partial_buf = create_buffer_with_data::<B, bf16>(&ctx, &y_partial);- let mut tok2row_buf = create_buffer::<B, i32>(&ctx, t * k);- let mut y_out_buf = create_buffer::<B, bf16>(&ctx, t * d_model);-- // Expert weights buffers- // Generate W13 in original layout [E, d_model, 2*d_ff]- let w13_original: Vec<bf16> =- (0..e * d_model * 2 * d_ff).map(|_| bf16::from_f32(rng.random_range(-0.5..0.5))).collect();-- // Transpose to GPU layout [E, 2*d_ff, d_model]- let mut w13 = vec![bf16::from_f32(0.0); e * d_model * 2 * d_ff];- for expert in 0..e {- let src_offset = expert * d_model * 2 * d_ff;- let dst_offset = expert * 2 * d_ff * d_model;- for dm in 0..d_model {- for ff in 0..(2 * d_ff) {- let src_idx = src_offset + dm * 2 * d_ff + ff;- let dst_idx = dst_offset + ff * d_model + dm;- w13[dst_idx] = w13_original[src_idx];- }- }- }-- let w2: Vec<bf16> = (0..e * d_ff * d_model).map(|_| bf16::from_f32(rng.random_range(-0.5..0.5))).collect();- let up_biases: Vec<bf16> = (0..e * 2 * d_ff).map(|_| bf16::from_f32(rng.random_range(-0.1..0.1))).collect();- let down_biases: Vec<bf16> = (0..e * d_model).map(|_| bf16::from_f32(rng.random_range(-0.1..0.1))).collect();-- let w13_buf = create_buffer_with_data::<B, bf16>(&ctx, &w13);- let w2_buf = create_buffer_with_data::<B, bf16>(&ctx, &w2);- let up_biases_buf = create_buffer_with_data::<B, bf16>(&ctx, &up_biases);- let down_biases_buf = create_buffer_with_data::<B, bf16>(&ctx, &down_biases);-- let sum_k = t * k;-- // Scatter block bases buffers- let mut block_bases_buf = create_buffer::<B, u32>(&ctx, num_blocks * num_tiles);- let mut block_alloc_buf = create_buffer::<B, u32>(&ctx, num_blocks * num_tiles);- let mut bucketed_probs_buf = create_buffer::<B, bf16>(&ctx, t * k);-- // Create kernel structs (use production-validated encoding logic)- let counts_offsets_kernel =- <<B as Backend>::Kernels as Kernels>::MoeCountsOffsetsFusedKernel::new(&ctx).expect("counts+offsets fused");- let scatter_bases_kernel = <<B as Backend>::Kernels as Kernels>::MoeBlockBasesFromPartialsKernel::new(&ctx)- .expect("<<Metal as Backend>::Kernels as Kernels>::MoeBlockBasesFromPartialsKernel");- let scatter_map_kernel =- <<B as Backend>::Kernels as Kernels>::MoeScatterBucketsMapKernel::new(&ctx, DataType::BF16)- .expect("<<Metal as Backend>::Kernels as Kernels>::MoeScatterBucketsMapKernel");- let gather = MoeGather::<B>::new(&ctx, DataType::BF16).expect("gather");- let experts_kernel =- MoeExpertsTwoPassDecodeBlock::<B>::new(&ctx, DataType::BF16, 2).expect("experts two-pass decode");- let finalize_kernel =- <<B as Backend>::Kernels as Kernels>::MoeFinalizeKernel::new(&ctx, DataType::BF16).expect("finalize");- let router_topk_fused_kernel = <<B as Backend>::Kernels as Kernels>::MoeRouterTopKKernel::new(- &ctx,- DataType::BF16,- true,- false,- false,- false,- false,- )- .expect("router+topk fused");-- // Testing: Router + TopK + Counts+Offsets (FUSED)- let router_topk_fused_perf = run_perf_with_warmup("Router+TopK (FUSED)", 2, 5, || {- let mut command_buffer = ctx.create_command_buffer(None, None).expect("Failed to create command buffer");- router_topk_fused_kernel.encode(- &x_buf,- &router_w_buf,- Some(&router_b_buf),- None::<&<B as Backend>::GlobalBuffer>,- None::<&<B as Backend>::GlobalBuffer>,- &mut topk_ids_buf,- &mut topk_probs_buf,- t as u32,- d_model as u32,- e as u32,- k as u32,- true,- None::<f32>,- None::<f32>,- &mut command_buffer,- );- command_buffer.end_encoding().submit().wait_until_completed().unwrap();- });-- let counts_offsets_perf = run_perf_with_warmup("Counts+Offsets (FUSED)", 2, 5, || {- let mut command_buffer = ctx.create_command_buffer(None, None).expect("Failed to create command buffer");- counts_offsets_kernel.encode(- &topk_ids_buf,- &mut offsets_buf,- &mut sumk_buf,- &mut partials_buf,- t as u32,- e as u32,- k as u32,- &mut command_buffer,- );- command_buffer.end_encoding().submit().wait_until_completed().unwrap();- });-- let scatter_perf = run_perf_with_warmup("Scatter", 2, 5, || {- let mut command_buffer = ctx.create_command_buffer(None, None).expect("Failed to create command buffer");- scatter_bases_kernel.encode(- &partials_buf,- &mut block_bases_buf,- &mut block_alloc_buf,- e as u32,- num_blocks as u32,- num_tiles as u32,- 0u32,- &mut command_buffer,- );- scatter_map_kernel.encode(- &topk_ids_buf,- &topk_probs_buf,- &offsets_buf,- &block_bases_buf,- &block_alloc_buf,- &mut bucketed_ids_buf,- &mut bucketed_probs_buf,- t as u32,- e as u32,- k as u32,- num_blocks as u32,- num_tiles as u32,- &mut tok2row_buf,- &mut command_buffer,- );- command_buffer.end_encoding().submit().wait_until_completed().unwrap();- });-- let gather_perf = run_perf_with_warmup("Gather", 2, 5, || {- let mut command_buffer = ctx.create_command_buffer(None, None).expect("Failed to create command buffer");- let x_perm = gather- .encode(&x_buf, &bucketed_ids_buf, &sumk_buf, t as u32, k as u32, d_model as u32, &mut command_buffer)- .expect("failed to encode MoE gather");- let completed = command_buffer.end_encoding().submit().wait_until_completed().unwrap();- drop(x_perm);- drop(completed);- });-- let (x_perm_buf, x_perm_completed) = {- let mut command_buffer = ctx.create_command_buffer(None, None).expect("Failed to create command buffer");- let x_perm = gather- .encode(&x_buf, &bucketed_ids_buf, &sumk_buf, t as u32, k as u32, d_model as u32, &mut command_buffer)- .expect("failed to encode MoE gather");- let completed = command_buffer.end_encoding().submit().wait_until_completed().unwrap();- (x_perm, completed)- };-- let experts_perf = run_perf_with_warmup("Experts (MAIN COMPUTE)", 2, 5, || {- let mut command_buffer = ctx.create_command_buffer(None, None).expect("Failed to create command buffer");- let output = experts_kernel- .encode(- MoeExpertsTwoPassArguments {- x_perm: &x_perm_buf,- expert_offsets: &offsets_buf,- w13_all: &w13_buf,- w2_all: &w2_buf,- up_biases: &up_biases_buf,- down_biases: &down_biases_buf,- total_rows: sum_k as u32,- d_model: d_model as u32,- d_ff: d_ff as u32,- num_routed_experts: e as u32,- gate_clip_min: f32::NEG_INFINITY,- gate_clip_max: 20.0,- up_clip_min: -19.0,- up_clip_max: 21.0,- silu_alpha: 1.702,- },- &mut command_buffer,- )- .expect("failed to encode MoE experts");- let completed = command_buffer.end_encoding().submit().wait_until_completed().unwrap();- drop(output);- drop(completed);- });- drop(x_perm_buf);- drop(x_perm_completed);-- let finalize_perf = run_perf_with_warmup("Finalize", 2, 5, || {- let mut command_buffer = ctx.create_command_buffer(None, None).expect("Failed to create command buffer");- finalize_kernel.encode(- &tok2row_buf,- &topk_probs_buf,- &y_partial_buf,- &mut y_out_buf,- t as u32,- d_model as u32,- k as u32,- &mut command_buffer,- );- command_buffer.end_encoding().submit().wait_until_completed().unwrap();- });-- // Print results- router_topk_fused_perf.print();- counts_offsets_perf.print();- scatter_perf.print();- gather_perf.print();- experts_perf.print();- finalize_perf.print();-- // Calculate breakdown- let total_us = (router_topk_fused_perf.mean_ms- + counts_offsets_perf.mean_ms- + scatter_perf.mean_ms- + gather_perf.mean_ms- + experts_perf.mean_ms- + finalize_perf.mean_ms)- * 1000.0;-- eprintln!("\n ═══ Per-Kernel Latency (Production D=4096, H=14336, E=16, K=2, T=1) ═══");- eprintln!(- " Router+TopK (FUSED): {:8.1} us ({:5.1}%)",- router_topk_fused_perf.mean_ms * 1000.0,- (router_topk_fused_perf.mean_ms / (total_us / 1000.0)) * 100.0- );- eprintln!(- " Counts+Offsets: {:8.1} us ({:5.1}%)",- counts_offsets_perf.mean_ms * 1000.0,- (counts_offsets_perf.mean_ms / (total_us / 1000.0)) * 100.0- );- eprintln!(- " Scatter: {:8.1} us ({:5.1}%)",- scatter_perf.mean_ms * 1000.0,- (scatter_perf.mean_ms / (total_us / 1000.0)) * 100.0- );- eprintln!(- " Gather: {:8.1} us ({:5.1}%)",- gather_perf.mean_ms * 1000.0,- (gather_perf.mean_ms / (total_us / 1000.0)) * 100.0- );- eprintln!(- " Experts: {:8.1} us ({:5.1}%) ← MAIN COMPUTE",- experts_perf.mean_ms * 1000.0,- (experts_perf.mean_ms / (total_us / 1000.0)) * 100.0- );- eprintln!(- " Finalize: {:8.1} us ({:5.1}%)",- finalize_perf.mean_ms * 1000.0,- (finalize_perf.mean_ms / (total_us / 1000.0)) * 100.0- );- eprintln!(" ═══════════════════════════════════════════");- eprintln!(" TOTAL: {:8.1} us (100.0%)", total_us);- eprintln!("\n Note: Times include Metal CB overhead (~10-50ms per kernel).");- eprintln!(" Real GPU compute is much faster, but relative % shows bottleneck.");- })-}diff --git a/crates/uzu-engine/unit/encodable_block/moe/moe_tiles_test.rs b/crates/uzu-engine/unit/encodable_block/moe/moe_tiles_test.rsdeleted file mode 100644index 99c8018e4..000000000--- a/crates/uzu-engine/unit/encodable_block/moe/moe_tiles_test.rs+++ /dev/null@@ -1,164 +0,0 @@-use rand::{RngExt, SeedableRng, rngs::StdRng};-use uzu_engine_macros::uzu_test;--use crate::{- backends::common::{- Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferPending, Context, Kernels,- kernel::{MoeTileCountsKernel, MoeTileScanKernel},- },- encodable_block::mlp::moe::tests::{cpu_tile_counts, cpu_tile_scan},- tests::helpers::{- buffer_prefix_to_vec, buffer_to_vec, create_buffer, create_buffer_with_data, create_context,- for_each_non_cpu_backend,- },-};--#[uzu_test]-fn test_tile_counts_correctness() {- for_each_non_cpu_backend!(|B| {- let ctx = create_context::<B>();- let mut rng = StdRng::seed_from_u64(2028);-- // Test multiple expert counts- let expert_counts = vec![4, 8, 16, 64];-- for e in expert_counts {- eprintln!("[TileCountsTest] E={}", e);-- // Generate random offsets (simulating expert segment sizes)- let mut offsets = vec![0u32];- for _ in 0..e {- let seg_len = rng.random_range(0..200);- offsets.push(offsets.last().unwrap() + seg_len);- }-- let tile_counts_cpu = cpu_tile_counts(&offsets, 16);-- // GPU buffers- let offsets_buf = create_buffer_with_data::<B, u32>(&ctx, &offsets);-- // Execute kernel using kernel struct- let counts_kernel =- <<B as Backend>::Kernels as Kernels>::MoeTileCountsKernel::new(&ctx).expect("MoeTileCountsKernel::new");- let mut command_buffer = ctx.create_command_buffer(None, None).expect("Failed to create command buffer");- let mut tile_counts_buf = create_buffer::<B, u32>(&ctx, e);- counts_kernel.encode(&offsets_buf, &mut tile_counts_buf, e as u32, &mut command_buffer);- let completed = command_buffer.end_encoding().submit().wait_until_completed().unwrap();-- let tile_counts_gpu = buffer_prefix_to_vec::<B, u32>(&tile_counts_buf, e);- drop(tile_counts_buf);- drop(completed);-- assert_eq!(tile_counts_gpu, tile_counts_cpu, "Tile counts mismatch for E={}", e);-- eprintln!("[TileCountsTest] ✓ PASSED");- }- });-}--#[uzu_test]-fn test_tile_scan_correctness() {- for_each_non_cpu_backend!(|B| {- let ctx = create_context::<B>();- let mut rng = StdRng::seed_from_u64(2029);-- // Test multiple expert counts- let expert_counts = vec![4, 8, 16, 64];-- for e in expert_counts {- eprintln!("[TileScanTest] E={}", e);-- // Generate random tile counts- let tile_counts: Vec<u32> = (0..e).map(|_| rng.random_range(0..20)).collect();-- let (tile_offsets_cpu, total_tiles_cpu) = cpu_tile_scan(&tile_counts);-- // GPU buffers- let tile_counts_buf = create_buffer_with_data::<B, u32>(&ctx, &tile_counts);-- // Execute kernel using kernel struct- let scan_kernel =- <<B as Backend>::Kernels as Kernels>::MoeTileScanKernel::new(&ctx).expect("MoeTileScanKernel::new");- let mut command_buffer = ctx.create_command_buffer(None, None).expect("Failed to create command buffer");- let mut tile_offsets_buf = create_buffer::<B, u32>(&ctx, e + 1);- let mut total_tiles_buf = create_buffer::<B, u32>(&ctx, 8);- scan_kernel.encode(- &tile_counts_buf,- &mut tile_offsets_buf,- &mut total_tiles_buf,- e as u32,- &mut command_buffer,- );- let completed = command_buffer.end_encoding().submit().wait_until_completed().unwrap();-- let tile_offsets_gpu = buffer_prefix_to_vec::<B, u32>(&tile_offsets_buf, e + 1);- let total_tiles_gpu = buffer_to_vec::<B, u32>(&total_tiles_buf)[0];- drop(tile_offsets_buf);- drop(total_tiles_buf);- drop(completed);-- assert_eq!(tile_offsets_gpu, tile_offsets_cpu, "Tile offsets mismatch for E={}", e);- assert_eq!(total_tiles_gpu, total_tiles_cpu, "Total tiles mismatch for E={}", e);-- eprintln!("[TileScanTest] ✓ PASSED");- }- });-}--#[uzu_test]-fn test_tile_edge_cases() {- for_each_non_cpu_backend!(|B| {- let ctx = create_context::<B>();-- // Test with all empty experts (seg_len=0)- {- let e = 8;- let offsets = vec![0u32; e + 1]; // All zeros-- let tile_counts_cpu = cpu_tile_counts(&offsets, 16);-- let offsets_buf = create_buffer_with_data::<B, u32>(&ctx, &offsets);-- let counts_kernel =- <<B as Backend>::Kernels as Kernels>::MoeTileCountsKernel::new(&ctx).expect("MoeTileCountsKernel::new");- let mut command_buffer = ctx.create_command_buffer(None, None).expect("Failed to create command buffer");- let mut tile_counts_buf = create_buffer::<B, u32>(&ctx, e);- counts_kernel.encode(&offsets_buf, &mut tile_counts_buf, e as u32, &mut command_buffer);- let completed = command_buffer.end_encoding().submit().wait_until_completed().unwrap();-- let tile_counts_gpu = buffer_prefix_to_vec::<B, u32>(&tile_counts_buf, e);- drop(tile_counts_buf);- drop(completed);-- assert_eq!(tile_counts_gpu, tile_counts_cpu);- assert!(tile_counts_gpu.iter().all(|&c| c == 0));- eprintln!("[TileEdgeCases] ✓ Empty experts PASSED");- }-- // Test with single tile per expert (seg_len < BM=16)- {- let e = 4;- let offsets = vec![0, 10, 15, 30, 50]; // Small segments-- let tile_counts_cpu = cpu_tile_counts(&offsets, 16);-- let offsets_buf = create_buffer_with_data::<B, u32>(&ctx, &offsets);-- let counts_kernel =- <<B as Backend>::Kernels as Kernels>::MoeTileCountsKernel::new(&ctx).expect("MoeTileCountsKernel::new");- let mut command_buffer = ctx.create_command_buffer(None, None).expect("Failed to create command buffer");- let mut tile_counts_buf = create_buffer::<B, u32>(&ctx, e);- counts_kernel.encode(&offsets_buf, &mut tile_counts_buf, e as u32, &mut command_buffer);- let completed = command_buffer.end_encoding().submit().wait_until_completed().unwrap();-- let tile_counts_gpu = buffer_prefix_to_vec::<B, u32>(&tile_counts_buf, e);- drop(tile_counts_buf);- drop(completed);-- assert_eq!(tile_counts_gpu, tile_counts_cpu);- // With BM=16: seg_lens=[10, 5, 15, 20] -> tile_counts=[1, 1, 1, 2]- assert_eq!(tile_counts_gpu, &[1, 1, 1, 2]);- eprintln!("[TileEdgeCases] ✓ Small segments PASSED");- }- });-}