Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

Burn MoE (#886)

uuuvn3a11b5d10f15
diff --git a/crates/uzu-engine/src/backends/common/gpu_types/mod.rs b/crates/uzu-engine/src/backends/common/gpu_types/mod.rs
index c4aa68f87..5c1089585 100644
--- a/crates/uzu-engine/src/backends/common/gpu_types/mod.rs
+++ b/crates/uzu-engine/src/backends/common/gpu_types/mod.rs
@@ -14,7 +14,6 @@ pub mod matmul;
pub mod quantization;
pub mod quantization_method;
pub mod ring;
-pub mod router_topk;
pub mod trie;
pub mod weaver;
diff --git a/crates/uzu-engine/src/backends/common/gpu_types/router_topk.rs b/crates/uzu-engine/src/backends/common/gpu_types/router_topk.rs
deleted file mode 100644
index 6d033bc2c..000000000
--- a/crates/uzu-engine/src/backends/common/gpu_types/router_topk.rs
+++ /dev/null
@@ -1,5 +0,0 @@
-pub const ROUTER_TOPK_MAX_EXPERTS: u32 = 512;
-
-pub const ROUTER_TOPK_MAX_SELECTED_EXPERTS: u32 = 128;
-
-pub const ROUTER_TOPK_MAX_MODEL_DIM: u32 = 4096;
diff --git a/crates/uzu-engine/src/backends/cpu/kernel/mod.rs b/crates/uzu-engine/src/backends/cpu/kernel/mod.rs
index 8a079aeba..e96c27e1c 100644
--- a/crates/uzu-engine/src/backends/cpu/kernel/mod.rs
+++ b/crates/uzu-engine/src/backends/cpu/kernel/mod.rs
@@ -12,7 +12,6 @@ mod gated_act_mul;
mod gdn;
mod logit_transform;
mod matmul;
-mod moe;
mod normalization;
mod pooling;
mod radix_top_k_small;
diff --git a/crates/uzu-engine/src/backends/cpu/kernel/moe/counts_offsets_fused.rs b/crates/uzu-engine/src/backends/cpu/kernel/moe/counts_offsets_fused.rs
deleted file mode 100644
index fb718c3dd..000000000
--- a/crates/uzu-engine/src/backends/cpu/kernel/moe/counts_offsets_fused.rs
+++ /dev/null
@@ -1,55 +0,0 @@
-use uzu_engine_macros::kernel;
-
-#[kernel(MoeCountsOffsetsFused)]
-pub fn moe_counts_offsets_fused(
- topk_ids: *const i32,
- offsets: *mut u32,
- sum_k_out: *mut u32,
- partials: *mut u32,
- t_input: u32,
- e_input: u32,
- k_input: u32,
-) {
- let e = e_input as usize;
- let t = t_input as usize;
- let k = k_input as usize;
-
- if e == 0 {
- unsafe {
- *offsets = 0;
- *sum_k_out = 0;
- }
- return;
- }
-
- // Phase 1: Count tokens per expert (histogram)
- let mut counts = vec![0u32; e];
- for ti in 0..t {
- let base = ti * k;
- for kk in 0..k {
- let eid = unsafe { *topk_ids.add(base + kk) };
- if eid >= 0 {
- let ue = eid as usize;
- if ue < e {
- counts[ue] += 1;
- }
- }
- }
- }
-
- // Write partials (same as counts for single-threadgroup case)
- for i in 0..e {
- unsafe { *partials.add(i) = counts[i] };
- }
-
- // Phase 2: Exclusive prefix scan to produce offsets
- let mut sum = 0u32;
- for i in 0..e {
- unsafe { *offsets.add(i) = sum };
- sum += counts[i];
- }
- unsafe {
- *offsets.add(e) = sum;
- *sum_k_out = sum;
- }
-}
diff --git a/crates/uzu-engine/src/backends/cpu/kernel/moe/experts_two_pass_decode.rs b/crates/uzu-engine/src/backends/cpu/kernel/moe/experts_two_pass_decode.rs
deleted file mode 100644
index 96f6b1503..000000000
--- a/crates/uzu-engine/src/backends/cpu/kernel/moe/experts_two_pass_decode.rs
+++ /dev/null
@@ -1,77 +0,0 @@
-use half::{bf16, f16};
-use num_traits::Float;
-use uzu_engine_macros::kernel;
-
-use crate::array::ArrayElement;
-
-#[kernel(MoeExpertsDecodePassA)]
-#[variants(T, f32, f16, bf16)]
-pub fn moe_experts_decode_pass_a<T: ArrayElement + Float>(
- #[allow(unused)] x_perm: *const T,
- #[allow(unused)] expert_offsets: *const u32,
- #[allow(unused)] w13_all: *const T,
- #[allow(unused)] hidden_out: *mut f32,
- #[allow(unused)] up_biases: *const T,
- #[allow(unused)] d_model: u32,
- #[allow(unused)] d_ff: u32,
- #[allow(unused)] e: u32,
- #[allow(unused)] gate_clip_min: f32,
- #[allow(unused)] gate_clip_max: f32,
- #[allow(unused)] up_clip_min: f32,
- #[allow(unused)] up_clip_max: f32,
- #[allow(unused)] silu_alpha: f32,
- #[allow(unused)] tile_map: *const u32,
- #[allow(unused)]
- #[specialize]
- gating_sel: u32,
- #[allow(unused)] __dsl_indirect_dispatch_buffer: *const u32,
-) {
- todo!()
-}
-
-#[kernel(MoeExpertsDecodeDownFused2D)]
-#[variants(T, f32, f16, bf16)]
-#[variants(AccumT, f32)]
-#[allow(clippy::extra_unused_type_parameters)]
-pub fn moe_experts_decode_down_fused2_d<T: ArrayElement + Float, AccumT: ArrayElement + Float>(
- hidden: *const f32,
- row_expert_map: *const u32,
- w2_all: *const T,
- down_biases: *const T,
- y_out: *mut T,
- total_rows: u32,
- d_model: u32,
- d_ff: u32,
- e: u32,
-) {
- let dm = d_model as usize;
- let df = d_ff as usize;
- let e = e as usize;
-
- for row_idx in 0..total_rows as usize {
- let expert_idx = unsafe { *row_expert_map.add(row_idx) } as usize;
- assert!(expert_idx < e);
- let hidden_base = row_idx * df;
- let w2_expert_base = expert_idx * dm * df;
-
- for my_col in 0..dm {
- let w2_col_base = w2_expert_base + my_col * df;
-
- let mut acc = 0.0f32;
- for h in 0..df {
- let h_val = unsafe { *hidden.add(hidden_base + h) };
- let w_val = unsafe { *w2_all.add(w2_col_base + h) }.to_f32().unwrap();
- acc = h_val.mul_add(w_val, acc);
- }
-
- // Add bias
- let bias_idx = expert_idx * dm + my_col;
- acc += unsafe { *down_biases.add(bias_idx) }.to_f32().unwrap();
-
- let out_idx = row_idx * dm + my_col;
- unsafe {
- *y_out.add(out_idx) = T::from(acc).unwrap();
- }
- }
- }
-}
diff --git a/crates/uzu-engine/src/backends/cpu/kernel/moe/experts_two_pass_prefill.rs b/crates/uzu-engine/src/backends/cpu/kernel/moe/experts_two_pass_prefill.rs
deleted file mode 100644
index 629e7bfaa..000000000
--- a/crates/uzu-engine/src/backends/cpu/kernel/moe/experts_two_pass_prefill.rs
+++ /dev/null
@@ -1,47 +0,0 @@
-use half::{bf16, f16};
-use num_traits::Float;
-use uzu_engine_macros::kernel;
-
-use crate::array::ArrayElement;
-
-#[kernel(MoeExpertsPrefillPassA)]
-#[variants(T, f32, f16, bf16)]
-pub fn moe_experts_prefill_pass_a<T: ArrayElement + Float>(
- #[allow(unused)] x_perm: *const T,
- #[allow(unused)] expert_offsets: *const u32,
- #[allow(unused)] w13_all: *const T,
- #[allow(unused)] up_biases: *const T,
- #[allow(unused)] hidden_out: *mut f32,
- #[allow(unused)] d_model: u32,
- #[allow(unused)] d_ff: u32,
- #[allow(unused)] e: u32,
- #[allow(unused)] gate_clip_min: f32,
- #[allow(unused)] gate_clip_max: f32,
- #[allow(unused)] up_clip_min: f32,
- #[allow(unused)] up_clip_max: f32,
- #[allow(unused)] silu_alpha: f32,
- #[allow(unused)] tile_map: *const u32,
- #[allow(unused)]
- #[specialize]
- gating_sel: u32,
- #[allow(unused)] __dsl_indirect_dispatch_buffer: *const u32,
-) {
- todo!()
-}
-
-#[kernel(MoeExpertsPrefillPassB)]
-#[variants(T, f32, f16, bf16)]
-pub fn moe_experts_prefill_pass_b<T: ArrayElement + Float>(
- #[allow(unused)] hidden: *const f32,
- #[allow(unused)] expert_offsets: *const u32,
- #[allow(unused)] w2_all: *const T,
- #[allow(unused)] down_biases: *const T,
- #[allow(unused)] output: *mut T,
- #[allow(unused)] d_model: u32,
- #[allow(unused)] d_ff: u32,
- #[allow(unused)] e: u32,
- #[allow(unused)] tile_map: *const u32,
- #[allow(unused)] __dsl_indirect_dispatch_buffer: *const u32,
-) {
- todo!()
-}
diff --git a/crates/uzu-engine/src/backends/cpu/kernel/moe/finalize.rs b/crates/uzu-engine/src/backends/cpu/kernel/moe/finalize.rs
deleted file mode 100644
index aa239d6ce..000000000
--- a/crates/uzu-engine/src/backends/cpu/kernel/moe/finalize.rs
+++ /dev/null
@@ -1,50 +0,0 @@
-use half::{bf16, f16};
-use num_traits::Float;
-use uzu_engine_macros::kernel;
-
-use crate::array::ArrayElement;
-
-#[kernel(MoeFinalize)]
-#[variants(T, f32, f16, bf16)]
-pub fn moe_finalize<T: ArrayElement + Float>(
- tok2row: *const i32,
- probs: *const T,
- y_partial: *const T,
- y: *mut T,
- t_count: u32,
- d_model: u32,
- k_input: u32,
-) {
- let t_count = t_count as usize;
- let d_model = d_model as usize;
- let k_input = k_input as usize;
-
- unsafe {
- for ti in 0..t_count {
- for f in 0..d_model {
- let mut acc = 0f32;
- for kk in 0..k_input {
- let idx = ti * k_input + kk;
- let row = *tok2row.add(idx);
- if row >= 0 {
- let mut prob = (*probs.add(idx)).to_f32().unwrap();
- if !prob.is_finite() {
- prob = 0.0;
- }
-
- let row_u = row as usize;
- let mut val = (*y_partial.add(row_u * d_model + f)).to_f32().unwrap();
- if !val.is_finite() {
- val = 0.0;
- }
- acc += prob * val;
- }
- }
- if !acc.is_finite() {
- acc = 0.0;
- }
- *y.add(ti * d_model + f) = T::from(acc).unwrap();
- }
- }
- }
-}
diff --git a/crates/uzu-engine/src/backends/cpu/kernel/moe/gather.rs b/crates/uzu-engine/src/backends/cpu/kernel/moe/gather.rs
deleted file mode 100644
index 8e5045a0f..000000000
--- a/crates/uzu-engine/src/backends/cpu/kernel/moe/gather.rs
+++ /dev/null
@@ -1,69 +0,0 @@
-use half::{bf16, f16};
-use num_traits::Float;
-use uzu_engine_macros::kernel;
-
-use crate::array::ArrayElement;
-
-fn moe_gather<T: ArrayElement + Float>(
- x: *const T,
- bucketed_ids: *const i32,
- x_perm: *mut T,
- sumk_buf: *const u32,
- d_model: u32,
- t: u32,
- k: u32,
-) {
- let total_rows = unsafe { *sumk_buf } as usize;
- let d_model = d_model as usize;
- let t = t as usize;
- let k = k as usize;
- let routed_capacity = t.checked_mul(k).expect("routed token count overflow");
- assert!(total_rows <= routed_capacity);
-
- if total_rows == 0 || d_model == 0 {
- return;
- }
-
- unsafe {
- for row in 0..total_rows {
- let token = *bucketed_ids.add(row);
- if token < 0 {
- continue;
- }
- assert!((token as usize) < t);
- let src = x.add(token as usize * d_model);
- let dst = x_perm.add(row * d_model);
- for col in 0..d_model {
- *dst.add(col) = *src.add(col);
- }
- }
- }
-}
-
-#[kernel(MoeGatherXPerm2D)]
-#[variants(T, f32, f16, bf16)]
-pub fn moe_gather_x_perm2_d<T: ArrayElement + Float>(
- x: *const T,
- bucketed_ids: *const i32,
- x_perm: *mut T,
- sumk_buf: *const u32,
- d_model: u32,
- t: u32,
- k: u32,
-) {
- moe_gather(x, bucketed_ids, x_perm, sumk_buf, d_model, t, k);
-}
-
-#[kernel(MoeGatherXPerm1D)]
-#[variants(T, f32, f16, bf16)]
-pub fn moe_gather_x_perm1_d<T: ArrayElement + Float>(
- x: *const T,
- bucketed_ids: *const i32,
- x_perm: *mut T,
- sumk_buf: *const u32,
- d_model: u32,
- t: u32,
- k: u32,
-) {
- moe_gather(x, bucketed_ids, x_perm, sumk_buf, d_model, t, k);
-}
diff --git a/crates/uzu-engine/src/backends/cpu/kernel/moe/mod.rs b/crates/uzu-engine/src/backends/cpu/kernel/moe/mod.rs
deleted file mode 100644
index fae7fc37e..000000000
--- a/crates/uzu-engine/src/backends/cpu/kernel/moe/mod.rs
+++ /dev/null
@@ -1,9 +0,0 @@
-pub mod counts_offsets_fused;
-pub mod experts_two_pass_decode;
-pub mod experts_two_pass_prefill;
-pub mod finalize;
-pub mod gather;
-pub mod router_topk;
-pub mod scatter_buckets;
-pub mod tiles_map;
-pub mod tiles_pass_a;
diff --git a/crates/uzu-engine/src/backends/cpu/kernel/moe/router_topk.rs b/crates/uzu-engine/src/backends/cpu/kernel/moe/router_topk.rs
deleted file mode 100644
index 448095fbe..000000000
--- a/crates/uzu-engine/src/backends/cpu/kernel/moe/router_topk.rs
+++ /dev/null
@@ -1,142 +0,0 @@
-use half::{bf16, f16};
-use num_traits::Float;
-use uzu_engine_macros::kernel;
-
-use crate::array::ArrayElement;
-
-#[kernel(MoeRouterTopK)]
-#[variants(ScalarT, f16, bf16, f32)]
-pub fn moe_router_top_k<ScalarT: ArrayElement + Float>(
- input: *const ScalarT,
- weight: *const ScalarT,
- #[optional(has_biases)] bias: Option<*const ScalarT>,
- #[optional(has_router_scales)] router_scale: Option<*const ScalarT>,
- #[optional(has_per_expert_scales)] per_expert_scale: Option<*const ScalarT>,
- topk_ids: *mut i32,
- topk_probs: *mut ScalarT,
- t: u32,
- d_model: u32,
- e: u32,
- k: u32,
- renorm: bool,
- #[optional(normalize_router_input)] router_norm_epsilon: Option<f32>,
- #[optional(has_router_input_scale)] router_input_scale: Option<f32>,
- #[specialize] has_biases: bool,
- #[specialize] has_router_scales: bool,
- #[specialize] has_per_expert_scales: bool,
- #[specialize] has_router_input_scale: bool,
- #[specialize] normalize_router_input: bool,
-) {
- assert_eq!(d_model % 4, 0, "d_model must be multiple of 4");
- assert!(k >= 1 && e >= k);
-
- let t = t as usize;
- let d_model = d_model as usize;
- let e = e as usize;
- let k = k as usize;
- let router_input_scale = if has_router_input_scale {
- router_input_scale.unwrap()
- } else {
- 1.0
- };
- let router_norm_epsilon = router_norm_epsilon.unwrap_or(0.0);
-
- let mut logits = vec![0.0f32; t * e];
- unsafe {
- for token in 0..t {
- let x_row = input.add(token * d_model);
- let mut inv_rms = 1.0f32;
- if normalize_router_input {
- let mut sum_sq = 0.0f32;
- for idx in 0..d_model {
- let x = (*x_row.add(idx)).to_f32().unwrap();
- sum_sq += x * x;
- }
- inv_rms = (sum_sq / d_model as f32 + router_norm_epsilon).sqrt().recip();
- }
- for expert in 0..e {
- let w_row = weight.add(expert * d_model);
- let mut accum = [0.0f32; 4];
-
- // Simulate GPU vec4 processing: accumulate in 4-element chunks
- for chunk in (0..d_model).step_by(4) {
- for i in 0..4 {
- let idx = chunk + i;
- let scale = if has_router_scales {
- (*router_scale.unwrap().add(idx)).to_f32().unwrap()
- } else {
- 1.0
- };
- let x = (*x_row.add(idx)).to_f32().unwrap() * inv_rms * router_input_scale * scale;
- accum[i] += (*w_row.add(idx)).to_f32().unwrap() * x;
- }
- }
-
- // Sum the 4-vector: (a.x + a.y) + (a.z + a.w) - matches Metal shader line 60
- let sum = (accum[0] + accum[1]) + (accum[2] + accum[3]);
- logits[token * e + expert] = sum
- + if has_biases {
- (*bias.unwrap().add(expert)).to_f32().unwrap()
- } else {
- 0.0
- };
- }
-
- let mut best_vals = vec![f32::NEG_INFINITY; k];
- let mut best_ids = vec![-1i32; k];
- let row = &logits[token * e..(token + 1) * e];
- for expert in 0..e {
- let v = row[expert];
- let mut insert_pos = None;
- for j in (0..k).rev() {
- if v > best_vals[j] || (v == best_vals[j] && (best_ids[j] < 0 || (expert as i32) < best_ids[j])) {
- insert_pos = Some(j);
- }
- }
- if let Some(pos) = insert_pos {
- for s in (pos + 1..k).rev() {
- best_vals[s] = best_vals[s - 1];
- best_ids[s] = best_ids[s - 1];
- }
- best_vals[pos] = v;
- best_ids[pos] = expert as i32;
- }
- }
- let base = token * k;
- for kk in 0..k {
- *topk_ids.add(base + kk) = best_ids[kk];
- }
- let expert_scale = |id: i32| {
- if has_per_expert_scales {
- (*per_expert_scale.unwrap().add(id as usize)).to_f32().unwrap()
- } else {
- 1.0
- }
- };
- if renorm {
- let max_v = best_vals.iter().copied().fold(f32::NEG_INFINITY, f32::max);
- let mut exps = vec![0.0f32; k];
- let mut sum = 0.0f32;
- for kk in 0..k {
- exps[kk] = (best_vals[kk] - max_v).exp();
- sum += exps[kk];
- }
- if sum > 0.0 {
- for kk in 0..k {
- *topk_probs.add(base + kk) =
- ScalarT::from(exps[kk] / sum * expert_scale(best_ids[kk])).unwrap();
- }
- } else {
- let uniform = 1.0f32 / k as f32;
- for kk in 0..k {
- *topk_probs.add(base + kk) = ScalarT::from(uniform * expert_scale(best_ids[kk])).unwrap();
- }
- }
- } else {
- for kk in 0..k {
- *topk_probs.add(base + kk) = ScalarT::from(best_vals[kk] * expert_scale(best_ids[kk])).unwrap();
- }
- }
- }
- }
-}
diff --git a/crates/uzu-engine/src/backends/cpu/kernel/moe/scatter_buckets.rs b/crates/uzu-engine/src/backends/cpu/kernel/moe/scatter_buckets.rs
deleted file mode 100644
index d9d15d6aa..000000000
--- a/crates/uzu-engine/src/backends/cpu/kernel/moe/scatter_buckets.rs
+++ /dev/null
@@ -1,38 +0,0 @@
-use half::{bf16, f16};
-use num_traits::Float;
-use uzu_engine_macros::kernel;
-
-use crate::array::ArrayElement;
-
-#[kernel(MoeBlockBasesFromPartials)]
-pub fn moe_block_bases_from_partials(
- #[allow(unused)] partials: *const u32,
- #[allow(unused)] block_bases: *mut u32,
- #[allow(unused)] block_alloc: *mut u32,
- #[allow(unused)] e_input: u32,
- #[allow(unused)] num_blocks: u32,
- #[allow(unused)] num_tiles: u32,
- #[allow(unused)] capacity_per_expert: u32,
-) {
- todo!()
-}
-
-#[kernel(MoeScatterBucketsMap)]
-#[variants(T, f32, f16, bf16)]
-pub fn moe_scatter_buckets_map<T: ArrayElement + Float>(
- #[allow(unused)] topk_ids: *const i32,
- #[allow(unused)] topk_probs: *const T,
- #[allow(unused)] offsets: *const u32,
- #[allow(unused)] block_bases: *const u32,
- #[allow(unused)] block_alloc: *const u32,
- #[allow(unused)] out_ids: *mut i32,
- #[allow(unused)] out_probs: *mut T,
- #[allow(unused)] t: u32,
- #[allow(unused)] e: u32,
- #[allow(unused)] k: u32,
- #[allow(unused)] num_blocks: u32,
- #[allow(unused)] num_tiles: u32,
- #[allow(unused)] tok2row: *mut i32,
-) {
- todo!()
-}
diff --git a/crates/uzu-engine/src/backends/cpu/kernel/moe/tiles_map.rs b/crates/uzu-engine/src/backends/cpu/kernel/moe/tiles_map.rs
deleted file mode 100644
index 0d7348ea2..000000000
--- a/crates/uzu-engine/src/backends/cpu/kernel/moe/tiles_map.rs
+++ /dev/null
@@ -1,40 +0,0 @@
-use uzu_engine_macros::kernel;
-
-#[kernel(MoeTileCounts)]
-pub fn moe_tile_counts(
- #[allow(unused)] offsets: *const u32,
- #[allow(unused)] tile_counts: *mut u32,
- #[allow(unused)] e: u32,
-) {
- todo!()
-}
-
-#[kernel(MoeTileScan)]
-pub fn moe_tile_scan(
- #[allow(unused)] tile_counts: *const u32,
- #[allow(unused)] tile_row_offsets: *mut u32,
- #[allow(unused)] total_tiles_buf: *mut u32,
- #[allow(unused)] e: u32,
-) {
- todo!()
-}
-
-#[kernel(MoeBuildTileMap)]
-pub fn moe_build_tile_map(
- #[allow(unused)] offsets: *const u32,
- #[allow(unused)] tile_row_offsets: *const u32,
- #[allow(unused)] tile_counts: *const u32,
- #[allow(unused)] tile_map: *mut u32,
- #[allow(unused)] e: u32,
-) {
- todo!()
-}
-
-#[kernel(MoeWriteDispatchArgs)]
-pub fn moe_write_dispatch_args(
- #[allow(unused)] total_tiles_buf: *const u32,
- #[allow(unused)] dispatch_args: *mut u32,
- #[allow(unused)] num_tiles_n: u32,
-) {
- todo!()
-}
diff --git a/crates/uzu-engine/src/backends/cpu/kernel/moe/tiles_pass_a.rs b/crates/uzu-engine/src/backends/cpu/kernel/moe/tiles_pass_a.rs
deleted file mode 100644
index 81cb00205..000000000
--- a/crates/uzu-engine/src/backends/cpu/kernel/moe/tiles_pass_a.rs
+++ /dev/null
@@ -1,52 +0,0 @@
-use uzu_engine_macros::kernel;
-
-#[kernel(MoePassATileCounts)]
-pub fn moe_pass_a_tile_counts(
- #[allow(unused)] expert_offsets: *const u32,
- #[allow(unused)] tile_counts: *mut u32,
- #[allow(unused)] e: u32,
- #[allow(unused)] h_blocks: u32,
-) {
- todo!()
-}
-
-#[kernel(MoePassATileScan)]
-pub fn moe_pass_a_tile_scan(
- #[allow(unused)] tile_counts: *const u32,
- #[allow(unused)] tile_offsets: *mut u32,
- #[allow(unused)] total_tiles: *mut u32,
- #[allow(unused)] e: u32,
-) {
- todo!()
-}
-
-#[kernel(MoePassABuildRowMap)]
-pub fn moe_pass_a_build_row_map(
- #[allow(unused)] expert_offsets: *const u32,
- #[allow(unused)] row_expert_map: *mut u32,
- #[allow(unused)] total_rows: u32,
- #[allow(unused)] e: u32,
-) {
- todo!()
-}
-
-#[kernel(MoePassABuildTileMap)]
-pub fn moe_pass_a_build_tile_map(
- #[allow(unused)] expert_offsets: *const u32,
- #[allow(unused)] tile_offsets: *const u32,
- #[allow(unused)] row_expert_map: *const u32,
- #[allow(unused)] tile_map: *mut u32,
- #[allow(unused)] total_rows: u32,
- #[allow(unused)] h_blocks: u32,
-) {
- todo!()
-}
-
-#[kernel(MoePassAWriteDispatchArgs)]
-pub fn moe_pass_a_write_dispatch_args(
- #[allow(unused)] total_tiles: *const u32,
- #[allow(unused)] dispatch_args: *mut u32,
- #[allow(unused)] num_tiles_y: u32,
-) {
- todo!()
-}
diff --git a/crates/uzu-engine/src/backends/metal/kernel/common/threadgroup_reduce.h b/crates/uzu-engine/src/backends/metal/kernel/common/threadgroup_reduce.h
index 85756d118..bf58ca6aa 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/common/threadgroup_reduce.h
+++ b/crates/uzu-engine/src/backends/metal/kernel/common/threadgroup_reduce.h
@@ -20,14 +20,6 @@ struct SimdReduceMax {
static T simd_reduce(T x) { return simd_max(x); }
};
-template <typename T>
-struct SimdReduceMin {
- using value_type = T;
- static constant constexpr T identity =
- numeric_limits<T>::has_infinity ? numeric_limits<T>::infinity() : numeric_limits<T>::max();
- static T simd_reduce(T x) { return simd_min(x); }
-};
-
template <typename Op, ushort BLOCK_SIZE>
static typename Op::value_type threadgroup_cooperative_reduce(
typename Op::value_type value,
diff --git a/crates/uzu-engine/src/backends/metal/kernel/generated/router_topk.h b/crates/uzu-engine/src/backends/metal/kernel/generated/router_topk.h
deleted file mode 100644
index 40f37de5a..000000000
--- a/crates/uzu-engine/src/backends/metal/kernel/generated/router_topk.h
+++ /dev/null
@@ -1,13 +0,0 @@
-// Auto-generated from gpu_types/router_topk - do not edit manually
-#pragma once
-
-#include <metal_stdlib>
-using namespace metal;
-
-namespace uzu::router_topk {
-static constant constexpr uint32_t ROUTER_TOPK_MAX_EXPERTS = 512;
-
-static constant constexpr uint32_t ROUTER_TOPK_MAX_SELECTED_EXPERTS = 128;
-
-static constant constexpr uint32_t ROUTER_TOPK_MAX_MODEL_DIM = 4096;
-} // namespace uzu::router_topk
diff --git a/crates/uzu-engine/src/backends/metal/kernel/moe/counts_offsets_fused.metal b/crates/uzu-engine/src/backends/metal/kernel/moe/counts_offsets_fused.metal
deleted file mode 100644
index b82f05de8..000000000
--- a/crates/uzu-engine/src/backends/metal/kernel/moe/counts_offsets_fused.metal
+++ /dev/null
@@ -1,138 +0,0 @@
-#include <metal_stdlib>
-#include <metal_atomic>
-
-#include "../common/dsl.h"
-#include "../common/thread_context.h"
-#include "../common/threadgroup_reduce.h"
-
-template <ushort BLOCK_SIZE, typename T>
-static T threadgroup_raking_prefix_exclusive_sum(T value, threadgroup T* shared, const ushort lid) {
- shared[lid] = value;
- threadgroup_barrier(mem_flags::mem_threadgroup);
-
- if (lid < 32) {
- const short values_per_thread = BLOCK_SIZE / 32;
- const short first_index = lid * values_per_thread;
- for (short i = first_index + 1; i < first_index + values_per_thread; i++) {
- shared[i] += shared[i - 1];
- }
- T partial_sum = shared[first_index + values_per_thread - 1];
- for (short i = first_index + values_per_thread - 1; i > first_index; i--) {
- shared[i] = shared[i - 1];
- }
- shared[first_index] = 0;
-
- T prefix = simd_prefix_exclusive_sum(partial_sum);
-
- for (short i = first_index; i < first_index + values_per_thread; i++) {
- shared[i] += prefix;
- }
- }
- threadgroup_barrier(mem_flags::mem_threadgroup);
-
- const T result = shared[lid];
- threadgroup_barrier(mem_flags::mem_threadgroup);
- return result;
-}
-
-#define BLOCK_SIZE 128
-#define TILE_E 512
-
-// Single-kernel fused: count all experts + scan to offsets
-// This kernel is launched with SINGLE threadgroup
-PUBLIC KERNEL(MoeCountsOffsetsFused)(
- device const int* topk_ids,
- device uint* offsets, // output: exclusive scan [E+1]
- device uint* sum_k_out, // output: total count [1]
- device uint* partials, // output: partials [num_tiles * TILE_E] (for block_bases)
- constant uint& t_input,
- constant uint& e_input,
- constant uint& k_input,
- threadgroup _atomic<uint> tg_hist[TILE_E],
- threadgroup uint scan_shared[BLOCK_SIZE],
- threadgroup uint reduce_shared[BLOCK_SIZE],
- threadgroup uint counts_shared[BLOCK_SIZE], // Cache counts in threadgroup memory
- threadgroup uint& carry,
- const ThreadContext thread_context,
- const uint lid THREADS(128)
-) {
- if (e_input == 0) {
- if (lid == 0) {
- offsets[0] = 0u;
- sum_k_out[0] = 0u;
- }
- return;
- }
-
- // ═══════════════════════════════════════════════════════════
- // PHASE 1: Count tokens per expert using tiled histogram
- // ═══════════════════════════════════════════════════════════
- // Tile over E dimension
- for (uint e0 = 0; e0 < e_input; e0 += TILE_E) {
- const uint tile_e = (e0 + TILE_E <= e_input) ? TILE_E : (e_input - e0);
-
- // Zero threadgroup histogram
- for (uint e = lid; e < tile_e; e += BLOCK_SIZE) {
- atomic_store_explicit(&tg_hist[e], 0u, memory_order_relaxed);
- }
- threadgroup_barrier(mem_flags::mem_threadgroup);
-
- // Process all tokens in strided fashion
- // Since we have only 1 threadgroup, we need to cover all T tokens
- for (uint t = lid; t < t_input; t += BLOCK_SIZE) {
- const uint base = t * k_input;
- for (uint k = 0; k < k_input; ++k) {
- int eid = topk_ids[base + k];
- if (eid >= 0) {
- uint ue = uint(eid);
- if (ue >= e0 && ue < e0 + tile_e) {
- uint te = ue - e0;
- atomic_fetch_add_explicit(&tg_hist[te], 1u, memory_order_relaxed);
- }
- }
- }
- }
- threadgroup_barrier(mem_flags::mem_threadgroup);
-
- // Write final counts and partials for this tile
- for (uint e = lid; e < tile_e; e += BLOCK_SIZE) {
- const uint count_val = atomic_load_explicit(&tg_hist[e], memory_order_relaxed);
- counts_shared[e0 + e] = count_val;
- partials[e0 + e] = count_val;
- }
- threadgroup_barrier(mem_flags::mem_threadgroup);
- }
-
- // ═══════════════════════════════════════════════════════════
- // PHASE 2: Compute exclusive prefix scan on counts
- // ═══════════════════════════════════════════════════════════
- if (lid == 0) {
- carry = 0u;
- }
- threadgroup_barrier(mem_flags::mem_threadgroup);
-
- for (uint base = 0; base < e_input; base += BLOCK_SIZE) {
- uint remaining = e_input - base;
- uint chunk_n = remaining < BLOCK_SIZE ? remaining : BLOCK_SIZE;
-
- uint v = (lid < chunk_n) ? counts_shared[lid] : 0u;
-
- uint prefix_local = threadgroup_raking_prefix_exclusive_sum<BLOCK_SIZE>(v, scan_shared, (ushort)lid);
- uint prefix_global = prefix_local + carry;
- if (lid < chunk_n) {
- offsets[base + lid] = prefix_global;
- }
-
- uint block_sum = threadgroup_cooperative_reduce<SimdReduceSum<uint>, BLOCK_SIZE>(v, reduce_shared, thread_context);
- if (lid == 0) {
- carry += block_sum;
- }
- threadgroup_barrier(mem_flags::mem_threadgroup);
- }
-
- // Write final offset and total
- if (lid == 0) {
- offsets[e_input] = carry;
- sum_k_out[0] = carry;
- }
-}
diff --git a/crates/uzu-engine/src/backends/metal/kernel/moe/experts_two_pass_decode.metal b/crates/uzu-engine/src/backends/metal/kernel/moe/experts_two_pass_decode.metal
deleted file mode 100644
index dabc1a252..000000000
--- a/crates/uzu-engine/src/backends/metal/kernel/moe/experts_two_pass_decode.metal
+++ /dev/null
@@ -1,228 +0,0 @@
-#include <metal_stdlib>
-#include <metal_simdgroup>
-#include "../activation/activations.h"
-#include "../common/dsl.h"
-#include "../common/thread_context.h"
-using namespace metal;
-
-// === Pass A: Vectorized GEMV with float4 loads ===
-// Structure: 4 simdgroups (128 threads), each outputs 1 hidden element
-// Each simdgroup: 32 threads reduce d_model using float4 vectorized loads
-// Grid: (h_blocks, rows, 1) where h_blocks = ceil(d_ff/4)
-template <typename T>
-VARIANTS(T, float, half, bfloat)
-PUBLIC KERNEL(MoeExpertsDecodePassA)(
- device const T* x_perm, // [total_rows, d_model]
- device const uint* expert_offsets, // [E + 1]
- device const T* w13_all, // [E, 2*d_ff, d_model]
- device float* hidden_out, // [E, 2*d_ff]
- device const T* up_biases, // [total_rows, d_ff]
- constant uint& d_model,
- constant uint& d_ff,
- constant uint& e,
- constant float& gate_clip_min,
- constant float& gate_clip_max,
- constant float& up_clip_min,
- constant float& up_clip_max,
- constant float& silu_alpha,
- device const uint* tile_map,
- const uint gating_sel SPECIALIZE,
- const ThreadContext thread_context,
- const uint tile_idx GROUPS(INDIRECT),
- const uint tid THREADS(128)
-) {
- // Read tile descriptor
- const uint h_block_idx = tile_map[tile_idx * 3 + 0];
- const uint expert_idx = tile_map[tile_idx * 3 + 1];
- const uint row_in_expert = tile_map[tile_idx * 3 + 2];
-
- // Validate row bounds
- const uint seg_start = expert_offsets[expert_idx];
- const uint seg_end = expert_offsets[expert_idx + 1];
- const uint global_row = seg_start + row_in_expert;
- if (global_row >= seg_end)
- return;
-
- // Each simdgroup outputs one hidden element
- const uint h_idx = h_block_idx * 4 + thread_context.simdgroup_index;
- if (h_idx >= d_ff)
- return;
-
- // Base addresses
- const ulong w13_stride = (ulong)d_model * (ulong)(2 * d_ff);
- const ulong w13_base = (ulong)expert_idx * w13_stride;
- const ulong bias_base = (ulong)expert_idx * (ulong)(2 * d_ff);
- const ulong x_row_base = (ulong)global_row * (ulong)d_model;
-
- device const T* x_ptr = x_perm + x_row_base;
- device const T* w_up_row = w13_all + w13_base + (ulong)h_idx * (ulong)d_model;
- device const T* w_gate_row = w13_all + w13_base + (ulong)(d_ff + h_idx) * (ulong)d_model;
-
- float acc_up = 0.0f;
- float acc_gate = 0.0f;
-
- // Vectorized reduction: 32 threads × 4 elements = 128 elements per iteration
- const uint vec_iters = d_model / 128;
-
- for (uint i = 0; i < vec_iters; ++i) {
- uint base_idx = i * 128 + thread_context.simd_lane_id * 4;
-
- device const T* x_vec = reinterpret_cast<device const T*>(x_ptr + base_idx);
- device const T* w_up_vec = reinterpret_cast<device const T*>(w_up_row + base_idx);
- acc_up += float(x_vec[0]) * float(w_up_vec[0]);
- acc_up += float(x_vec[1]) * float(w_up_vec[1]);
- acc_up += float(x_vec[2]) * float(w_up_vec[2]);
- acc_up += float(x_vec[3]) * float(w_up_vec[3]);
-
- if (gating_sel > 1) {
- device const T* w_gate_vec = reinterpret_cast<device const T*>(w_gate_row + base_idx);
- acc_gate += float(x_vec[0]) * float(w_gate_vec[0]);
- acc_gate += float(x_vec[1]) * float(w_gate_vec[1]);
- acc_gate += float(x_vec[2]) * float(w_gate_vec[2]);
- acc_gate += float(x_vec[3]) * float(w_gate_vec[3]);
- }
- }
-
- // Handle leftover elements
- uint leftover_start = vec_iters * 128 + thread_context.simd_lane_id;
- for (uint idx = leftover_start; idx < d_model; idx += 32) {
- float xv = float(x_ptr[idx]);
- acc_up += xv * float(w_up_row[idx]);
- if (gating_sel > 1) {
- acc_gate += xv * float(w_gate_row[idx]);
- }
- }
-
- // Simdgroup reduction
- acc_up = simd_sum(acc_up);
- if (gating_sel > 1) {
- acc_gate = simd_sum(acc_gate);
- }
-
- // Lane 0 applies activation and writes result
- if (thread_context.simd_lane_id == 0) {
- float up_val = acc_up + float(up_biases[bias_base + h_idx]);
- up_val = clamp(up_val, up_clip_min, up_clip_max);
-
- float activated;
- if (gating_sel <= 1) {
- activated = (gating_sel == 0) ? activate_gelu(up_val) : activate_silu_alpha(up_val, silu_alpha);
- } else {
- float gate_val = acc_gate + float(up_biases[bias_base + d_ff + h_idx]);
- gate_val = clamp(gate_val, gate_clip_min, gate_clip_max);
- float gate_act = (gating_sel == 2) ? activate_silu_alpha(gate_val, silu_alpha) : activate_gelu(gate_val);
- activated = gate_act * up_val;
- }
-
- hidden_out[(ulong)global_row * (ulong)d_ff + (ulong)h_idx] = activated;
- }
-}
-
-// === Pass B: Simdgroup cooperation along K for coalescing ===
-// W2 layout [E, d_model, d_ff] - 32 threads cooperate on one output, reading
-// consecutive K elements
-#define THREADS_PER_SIMD 32
-#define SIMDGROUPS_PER_TG 8
-
-template <typename T, typename AccumT>
-VARIANTS(T, float, half, bfloat)
-VARIANTS(AccumT, float)
-PUBLIC KERNEL(MoeExpertsDecodeDownFused2D)(
- device const float* hidden, // [total_rows, d_ff] - f32 from Pass A
- device const uint* row_expert_map, // [total_rows] - direct row->expert lookup
- device const T* w2_all, // [E, d_model, d_ff] - layout
- device const T* down_biases, // [E, d_model]
- device T* y_out, // [total_rows, d_model]
- constant uint& total_rows,
- constant uint& d_model,
- constant uint& d_ff,
- constant uint& e,
- const ThreadContext thread_context,
- const uint tgpig_x GROUPS(d_model.div_ceil(SIMDGROUPS_PER_TG)),
- const uint tgpig_y GROUPS(total_rows),
- const uint tid THREADS(256)
-) {
- const uint row_idx = tgpig_y;
-
- // Each simdgroup computes one output column
- const uint my_col = tgpig_x * SIMDGROUPS_PER_TG + thread_context.simdgroup_index;
- if (my_col >= d_model)
- return;
-
- const uint expert_idx = row_expert_map[row_idx];
-
- // Base addresses for this output column
- const ulong hidden_base = (ulong)row_idx * (ulong)d_ff;
- const ulong w2_col_base = (ulong)expert_idx * (ulong)d_model * (ulong)d_ff + (ulong)my_col * (ulong)d_ff;
-
- // Dual accumulators for ILP: breaks FMA dependency chains
- AccumT acc0 = AccumT(0.0);
- AccumT acc1 = AccumT(0.0);
- AccumT acc = AccumT(0.0);
-
- // Main loop: stride-32 with vec8 loads for ILP
- const uint k_iters = d_ff / THREADS_PER_SIMD;
- const uint k_vec_iters = k_iters / 8;
-
- for (uint iter = 0; iter < k_vec_iters; ++iter) {
- const uint k_base = iter * (8 * THREADS_PER_SIMD) + thread_context.simd_lane_id;
-
- // hidden: stride-32 per lane, already f32 from Pass A
- const AccumT h0 = hidden[hidden_base + k_base + 0 * THREADS_PER_SIMD];
- const AccumT h1 = hidden[hidden_base + k_base + 1 * THREADS_PER_SIMD];
- const AccumT h2 = hidden[hidden_base + k_base + 2 * THREADS_PER_SIMD];
- const AccumT h3 = hidden[hidden_base + k_base + 3 * THREADS_PER_SIMD];
- const AccumT h4 = hidden[hidden_base + k_base + 4 * THREADS_PER_SIMD];
- const AccumT h5 = hidden[hidden_base + k_base + 5 * THREADS_PER_SIMD];
- const AccumT h6 = hidden[hidden_base + k_base + 6 * THREADS_PER_SIMD];
- const AccumT h7 = hidden[hidden_base + k_base + 7 * THREADS_PER_SIMD];
-
- // W2: lane-coalesced
- const AccumT w0 = AccumT(w2_all[w2_col_base + k_base + 0 * THREADS_PER_SIMD]);
- const AccumT w1 = AccumT(w2_all[w2_col_base + k_base + 1 * THREADS_PER_SIMD]);
- const AccumT w2 = AccumT(w2_all[w2_col_base + k_base + 2 * THREADS_PER_SIMD]);
- const AccumT w3 = AccumT(w2_all[w2_col_base + k_base + 3 * THREADS_PER_SIMD]);
- const AccumT w4 = AccumT(w2_all[w2_col_base + k_base + 4 * THREADS_PER_SIMD]);
- const AccumT w5 = AccumT(w2_all[w2_col_base + k_base + 5 * THREADS_PER_SIMD]);
- const AccumT w6 = AccumT(w2_all[w2_col_base + k_base + 6 * THREADS_PER_SIMD]);
- const AccumT w7 = AccumT(w2_all[w2_col_base + k_base + 7 * THREADS_PER_SIMD]);
-
- // dual trees for ILP
- acc0 = fma(h0, w0, acc0);
- acc1 = fma(h1, w1, acc1);
- acc0 = fma(h2, w2, acc0);
- acc1 = fma(h3, w3, acc1);
- acc0 = fma(h4, w4, acc0);
- acc1 = fma(h5, w5, acc1);
- acc0 = fma(h6, w6, acc0);
- acc1 = fma(h7, w7, acc1);
- }
-
- // fold the vectorized contribution once
- acc += (acc0 + acc1);
-
- // Handle remaining full iterations
- for (uint iter = k_vec_iters * 8; iter < k_iters; ++iter) {
- const uint k = iter * THREADS_PER_SIMD + thread_context.simd_lane_id;
- acc = fma(AccumT(hidden[hidden_base + k]), AccumT(w2_all[w2_col_base + k]), acc);
- }
-
- // Handle leftover elements (d_ff % 32)
- const uint leftover_start = k_iters * THREADS_PER_SIMD;
- if (leftover_start + thread_context.simd_lane_id < d_ff) {
- const uint k = leftover_start + thread_context.simd_lane_id;
- acc = fma(AccumT(hidden[hidden_base + k]), AccumT(w2_all[w2_col_base + k]), acc);
- }
-
- // Simdgroup reduction
- AccumT result = simd_sum(acc);
-
- // Lane 0 writes result
- if (thread_context.simd_lane_id == 0) {
- const ulong bias_idx = (ulong)expert_idx * (ulong)d_model + (ulong)my_col;
- result += AccumT(down_biases[bias_idx]);
-
- const ulong out_idx = (ulong)row_idx * (ulong)d_model + (ulong)my_col;
- y_out[out_idx] = T(result);
- }
-}
\ No newline at end of file
diff --git a/crates/uzu-engine/src/backends/metal/kernel/moe/experts_two_pass_prefill.metal b/crates/uzu-engine/src/backends/metal/kernel/moe/experts_two_pass_prefill.metal
deleted file mode 100644
index 35576d775..000000000
--- a/crates/uzu-engine/src/backends/metal/kernel/moe/experts_two_pass_prefill.metal
+++ /dev/null
@@ -1,517 +0,0 @@
-#include <metal_stdlib>
-#include "../activation/activations.h"
-#include "../common/defines.h"
-#include "../common/dsl.h"
-#include "../common/thread_context.h"
-
-// ------------------------ helpers ------------------------
-static inline uint ceil_div(uint a, uint b) { return (a + b - 1u) / b; }
-
-static inline uint linear_tid(uint3 tid, uint3 tpg) { return tid.z * (tpg.x * tpg.y) + tid.y * tpg.x + tid.x; }
-
-// ------------------------ Pass A (X @ W13 → hidden) ------------------------
-// Tile config (tuned for A-series; matches your prior defaults)
-constant uint PASSA_BM = 16;
-constant uint PASSA_BN = 32;
-constant uint PASSA_BK = 64;
-constant uint PASSA_SG_BM = 8;
-constant uint PASSA_SG_BN = 16;
-constant uint PASSA_TG_PAD = 4; // Padding to avoid bank conflicts
-
-template <typename T>
-VARIANTS(T, float, half, bfloat)
-PUBLIC KERNEL(MoeExpertsPrefillPassA)(
- device const T* x_perm,
- device const uint* expert_offsets,
- device const T* w13_all,
- device const T* up_biases,
- device float* hidden_out,
- constant uint& d_model,
- constant uint& d_ff,
- constant uint& e,
- constant float& gate_clip_min,
- constant float& gate_clip_max,
- constant float& up_clip_min,
- constant float& up_clip_max,
- constant float& silu_alpha,
- device const uint* tile_map,
- threadgroup T Xs[PASSA_BM * (PASSA_BK + PASSA_TG_PAD)],
- threadgroup T Wk_up[PASSA_BN * (PASSA_BK + PASSA_TG_PAD)],
- threadgroup T Wk_gate[PASSA_BN * (PASSA_BK + PASSA_TG_PAD)],
- threadgroup float bias_up[PASSA_BN],
- threadgroup float bias_gate[PASSA_BN],
- // 0=GELU(up), 1=SiLU(up), 2=SwiGLU(gate)*up, 3=GEGLU(gate)*up
- const uint gating_sel SPECIALIZE,
- const uint n_tile_idx GROUPS(INDIRECT),
- const uint row_tile_idx GROUPS(INDIRECT),
- const uint local_tid_x THREADS(128),
- const ThreadContext thread_context
-) {
- const uint base = row_tile_idx * 3u;
- const uint expert_idx = tile_map[base + 0u];
- if (expert_idx >= e)
- return;
-
- const uint3 threads_per_tg = {128, 1, 1};
- const uint3 local_tid = {local_tid_x, 0, 0};
-
- constexpr uint ROW_TILE = PASSA_BM;
- const uint tile_seg_start = tile_map[base + 1u];
- const uint row_off_elems = tile_map[base + 2u];
- const uint tile_m = row_off_elems / ROW_TILE;
- const uint tile_n = n_tile_idx;
-
- using StageVec4 = metal::vec<T, 4>;
-
- constexpr uint Bm = PASSA_BM, Bn = PASSA_BN, Bk = PASSA_BK;
- constexpr uint SgBm = PASSA_SG_BM, SgBn = PASSA_SG_BN;
- constexpr uint TG_PAD = PASSA_TG_PAD;
- constexpr uint X_LD = Bk + TG_PAD; // Padded stride for X
- constexpr uint W_LD = Bk + TG_PAD; // Padded stride for weights
-
- const uint seg_start = expert_offsets[expert_idx];
- const uint seg_end = expert_offsets[expert_idx + 1];
- const uint seg_len = (seg_end > seg_start) ? (seg_end - seg_start) : 0u;
- if (seg_len == 0)
- return;
-
- const uint row_tg_off = tile_m * Bm;
- const uint col_tg_off = tile_n * Bn;
- if (row_tg_off >= seg_len || col_tg_off >= d_ff)
- return;
-
- const uint m_rows = min(Bm, seg_len - row_tg_off);
- const uint n_cols = min(Bn, d_ff - col_tg_off);
-
- const ulong x_base = ((ulong)tile_seg_start + (ulong)row_tg_off) * (ulong)d_model;
- const ulong w13_expert_base = (ulong)expert_idx * (ulong)(2 * d_ff) * (ulong)d_model;
- const ulong bias_base = (ulong)expert_idx * (ulong)(2 * d_ff);
-
- // simdgroup tile mapping
- const uint sg_col_count = Bn / SgBn;
- const uint row_sg = thread_context.simdgroup_index / sg_col_count;
- const uint col_sg = thread_context.simdgroup_index % sg_col_count;
- const uint row_sg_off = row_sg * SgBm;
- const uint col_sg_off = col_sg * SgBn;
-
- // Guard against misconfigured TG sizes
- constexpr uint SG_TILE = 8; // simdgroup fragment dimension (8x8)
- constexpr uint SG_EXPECTED = (Bm / SG_TILE) * (Bn / SG_TILE);
- if (thread_context.simdgroup_index >= SG_EXPECTED)
- return;
-
- // partial accumulators (8x8 tiles per simdgroup)
- constexpr uint TEMP = (SgBm / SG_TILE) * (SgBn / SG_TILE);
- metal::simdgroup_float8x8 OutUp[TEMP];
- metal::simdgroup_float8x8 OutGate[TEMP];
- for (uint i = 0; i < TEMP; ++i) {
- OutUp[i] = metal::make_filled_simdgroup_matrix<float, 8, 8>(0.0f);
- if (gating_sel > 1u) {
- OutGate[i] = metal::make_filled_simdgroup_matrix<float, 8, 8>(0.0f);
- }
- }
-
- const uint threads_total = threads_per_tg.x * threads_per_tg.y * threads_per_tg.z;
- const uint lin = linear_tid(local_tid, threads_per_tg);
-
- // K-loop (tile along D)
- for (uint k_off = 0; k_off < d_model; k_off += Bk) {
- const uint valid_k = min(Bk, d_model - k_off);
-
- threadgroup_barrier(mem_flags::mem_threadgroup);
-
- // ---- Stage LHS: Xs [Bm,Bk] with vec8 loads and padding ----
- {
- constexpr uint vec_size = 8;
- constexpr uint vec_cols = Bk / vec_size; // 64/8 = 8
- constexpr uint total_vecs = Bm * vec_cols; // 16*8 = 128
- const uint iters = ceil_div(total_vecs, threads_total);
-
- METAL_PRAGMA_UNROLL
- for (uint t = 0; t < iters; ++t) {
- const uint i = t * threads_total + lin;
- if (i < total_vecs) {
- const uint r = i / vec_cols;
- const uint c8 = i % vec_cols;
- const uint col_base = c8 * vec_size;
-
- threadgroup T* dst = Xs + r * X_LD + col_base;
-
- if (r < m_rows && col_base < valid_k) {
- // Safe to read - at least first element is valid
- const ulong base = x_base + (ulong)r * (ulong)d_model + (ulong)(k_off + col_base);
- METAL_PRAGMA_UNROLL
- for (uint j = 0; j < vec_size; j++) {
- dst[j] = (col_base + j < valid_k) ? static_cast<T>(float(x_perm[base + j])) : static_cast<T>(0.0f);
- }
- } else {
- // Row out of bounds or entire vec8 chunk is out of K bounds
- METAL_PRAGMA_UNROLL
- for (uint j = 0; j < vec_size; j++) {
- dst[j] = static_cast<T>(0.0f);
- }
- }
- }
- }
- }
-
- // ---- Stage RHS: W_up [Bn,Bk], W_gate [Bn,Bk] with vec8 loads and padding
- // ----
- {
- constexpr uint vec_size = 8;
- constexpr uint vec_cols = Bk / vec_size; // 64/8 = 8
- constexpr uint total_vecs = Bn * vec_cols; // 32*8 = 256
- const uint iters = ceil_div(total_vecs, threads_total);
-
- METAL_PRAGMA_UNROLL
- for (uint t = 0; t < iters; ++t) {
- const uint i = t * threads_total + lin;
- if (i < total_vecs) {
- const uint n = i / vec_cols; // row within the BN tile
- const uint c8 = i % vec_cols;
- const uint col_base = c8 * vec_size;
-
- threadgroup T* up_dst = Wk_up + n * W_LD + col_base;
- threadgroup T* gt_dst = Wk_gate + n * W_LD + col_base;
-
- if (n < n_cols && col_base < valid_k) {
- // Safe to read - at least first element is valid
- const uint gcol = col_tg_off + n;
- const ulong up_base = w13_expert_base + (ulong)gcol * (ulong)d_model + (ulong)(k_off + col_base);
- const ulong gate_base = w13_expert_base + (ulong)(d_ff + gcol) * (ulong)d_model + (ulong)(k_off + col_base);
- METAL_PRAGMA_UNROLL
- for (uint j = 0; j < vec_size; j++) {
- up_dst[j] = (col_base + j < valid_k) ? static_cast<T>(float(w13_all[up_base + j])) : static_cast<T>(0.0f);
- if (gating_sel > 1u) {
- gt_dst[j] =
- (col_base + j < valid_k) ? static_cast<T>(float(w13_all[gate_base + j])) : static_cast<T>(0.0f);
- }
- }
- } else {
- // Row out of bounds or entire vec8 chunk is out of K bounds
- METAL_PRAGMA_UNROLL
- for (uint j = 0; j < vec_size; j++) {
- up_dst[j] = static_cast<T>(0.0f);
- if (gating_sel > 1u)
- gt_dst[j] = static_cast<T>(0.0f);
- }
- }
- }
- }
- }
-
- threadgroup_barrier(mem_flags::mem_threadgroup);
-
- // ---- MMA across the BK tile (in simdgroup-sized chunks) ----
- METAL_PRAGMA_UNROLL
- for (uint kk = 0; kk < Bk; kk += SG_TILE) {
- METAL_PRAGMA_UNROLL
- for (uint m_sub = 0; m_sub < SgBm; m_sub += SG_TILE) {
- const uint r_idx = m_sub / SG_TILE;
- metal::simdgroup_matrix<T, 8, 8> lhs_frag;
- simdgroup_load(lhs_frag, Xs, X_LD, ulong2(kk, row_sg_off + m_sub));
-
- METAL_PRAGMA_UNROLL
- for (uint n_sub = 0; n_sub < SgBn; n_sub += SG_TILE) {
- const uint c_idx = n_sub / SG_TILE;
- const uint tile = r_idx * (SgBn / SG_TILE) + c_idx;
- metal::simdgroup_matrix<T, 8, 8> rhs_up;
- simdgroup_load(rhs_up, Wk_up, W_LD, ulong2(kk, col_sg_off + n_sub), true);
- simdgroup_multiply_accumulate(OutUp[tile], lhs_frag, rhs_up, OutUp[tile]);
-
- if (gating_sel > 1u) {
- metal::simdgroup_matrix<T, 8, 8> rhs_gate;
- simdgroup_load(rhs_gate, Wk_gate, W_LD, ulong2(kk, col_sg_off + n_sub), true);
- simdgroup_multiply_accumulate(OutGate[tile], lhs_frag, rhs_gate, OutGate[tile]);
- }
- }
- }
- }
- } // K loop
-
- // ---- Bias tile [Bn] ----
- // Note: bias layout is [up | gate] in contiguous FF chunks.
- for (uint c_local = lin; c_local < Bn; c_local += threads_total) {
- const uint c_global = col_tg_off + c_local;
- bias_up[c_local] = (c_global < d_ff) ? float(up_biases[bias_base + c_global]) : 0.0f;
- if (gating_sel > 1u) {
- bias_gate[c_local] = (c_global < d_ff) ? float(up_biases[bias_base + d_ff + c_global]) : 0.0f;
- }
- }
-
- threadgroup_barrier(mem_flags::mem_threadgroup);
-
- // ---- Epilogue: activation/gating → hidden_out (f32) ----
- const uint sg_lane_start = thread_context.simdgroup_index * 32;
- const uint sg_lane_end = sg_lane_start + 32;
- if (lin < sg_lane_start || lin >= sg_lane_end)
- return;
- const uint sg_lin = lin - sg_lane_start;
- // Lane→(row,col) mapping for 8x8 simdgroup fragments, matches the fragment
- // layout used above.
- const uint lane_qid = sg_lin >> 2;
- const uint lane_row = (lane_qid & 4u) + ((sg_lin >> 1) & 3u);
- const uint lane_col_base = ((lane_qid & 2u) * 2u) + ((sg_lin & 1u) * 2u);
-
- METAL_PRAGMA_UNROLL
- for (uint n_sub = 0; n_sub < SgBn; n_sub += SG_TILE) {
- const uint c_idx = n_sub / SG_TILE;
- METAL_PRAGMA_UNROLL
- for (uint m_sub = 0; m_sub < SgBm; m_sub += SG_TILE) {
- const uint r_idx = m_sub / SG_TILE;
- const uint tile = r_idx * (SgBn / SG_TILE) + c_idx;
-
- const uint tile_row_base = row_sg_off + m_sub;
- const uint tile_col_base = col_sg_off + n_sub;
- const uint local_row = tile_row_base + lane_row;
- if (local_row >= m_rows || tile_col_base >= n_cols) {
- continue;
- }
-
- const auto up_frag = OutUp[tile].thread_elements();
- float gate_frag_0 = 0.0f;
- float gate_frag_1 = 0.0f;
- if (gating_sel > 1u) {
- const auto gate_frag = OutGate[tile].thread_elements();
- gate_frag_0 = gate_frag[0];
- gate_frag_1 = gate_frag[1];
- }
-
- const uint col0 = tile_col_base + lane_col_base;
- const uint col1 = col0 + 1;
- const ulong out_row = seg_start + row_tg_off + local_row;
-
- if (col0 < n_cols) {
- float up_v = clamp(up_frag[0] + bias_up[col0], up_clip_min, up_clip_max);
- float out_val;
- if (gating_sel <= 1u) {
- out_val = (gating_sel == 0u) ? activate_gelu(up_v) : activate_silu_alpha(up_v, silu_alpha);
- } else {
- float gate_v = clamp(gate_frag_0 + bias_gate[col0], gate_clip_min, gate_clip_max);
- const float gate_act = (gating_sel == 2u) ? activate_silu_alpha(gate_v, silu_alpha) : activate_gelu(gate_v);
- out_val = gate_act * up_v;
- }
- const ulong out_col = col_tg_off + col0;
- hidden_out[out_row * (ulong)d_ff + out_col] = out_val;
- }
-
- if (col1 < n_cols) {
- float up_v = clamp(up_frag[1] + bias_up[col1], up_clip_min, up_clip_max);
- float out_val;
- if (gating_sel <= 1u) {
- out_val = (gating_sel == 0u) ? activate_gelu(up_v) : activate_silu_alpha(up_v, silu_alpha);
- } else {
- float gate_v = clamp(gate_frag_1 + bias_gate[col1], gate_clip_min, gate_clip_max);
- const float gate_act = (gating_sel == 2u) ? activate_silu_alpha(gate_v, silu_alpha) : activate_gelu(gate_v);
- out_val = gate_act * up_v;
- }
- const ulong out_col = col_tg_off + col1;
- hidden_out[out_row * (ulong)d_ff + out_col] = out_val;
- }
- }
- }
-}
-
-// ------------------------ Pass B (hidden @ W2 → output)
-// ------------------------ Optimized config: 16x64 output tile, 4 simdgroups
-// (128 threads)
-constant uint PASSB_BM = 16;
-constant uint PASSB_BN = 64;
-constant uint PASSB_BK = 64;
-constant uint PASSB_SG_BM = 8; // Each simdgroup handles 8x32
-constant uint PASSB_SG_BN = 32;
-constant uint PASSB_TG_PAD = 4; // Padding to avoid bank conflicts
-
-template <typename T>
-VARIANTS(T, float, half, bfloat)
-PUBLIC KERNEL(MoeExpertsPrefillPassB)(
- device const float* hidden,
- device const uint* expert_offsets,
- device const T* w2_all,
- device const T* down_biases,
- device T* output,
- constant uint& d_model,
- constant uint& d_ff,
- constant uint& e,
- device const uint* tile_map,
- threadgroup float Hs[PASSB_BM * (PASSB_BK + PASSB_TG_PAD)],
- threadgroup T Wk[PASSB_BN * (PASSB_BK + PASSB_TG_PAD)],
- threadgroup float bias_tile[PASSB_BN],
- const ThreadContext thread_context,
- const uint n_tile_idx GROUPS(INDIRECT),
- const uint row_tile_idx GROUPS(INDIRECT),
- const uint lin THREADS(128)
-) {
- const uint base = row_tile_idx * 3u;
- const uint expert_idx = tile_map[base + 0u];
- if (expert_idx >= e)
- return;
-
- constexpr uint ROW_TILE = PASSB_BM;
- const uint row_off_elems = tile_map[base + 2u];
- const uint tile_m = row_off_elems / ROW_TILE;
- const uint tile_n = n_tile_idx;
-
- constexpr uint Bm = PASSB_BM, Bn = PASSB_BN, Bk = PASSB_BK;
- constexpr uint SgBm = PASSB_SG_BM, SgBn = PASSB_SG_BN;
- constexpr uint TG_PAD = PASSB_TG_PAD;
- constexpr uint H_LD = Bk + TG_PAD; // Padded stride for hidden
- constexpr uint W_LD = Bk + TG_PAD; // Padded stride for weights
- constexpr uint SG_TILE = 8;
- constexpr uint TGP_SIZE = 128; // 4 simdgroups × 32 threads
-
- const uint seg_start = expert_offsets[expert_idx];
- const uint seg_end = expert_offsets[expert_idx + 1];
- const uint seg_len = (seg_end > seg_start) ? (seg_end - seg_start) : 0u;
- if (seg_len == 0)
- return;
-
- const uint row_tg_off = tile_m * Bm;
- const uint col_tg_off = tile_n * Bn;
- if (row_tg_off >= seg_len || col_tg_off >= d_model)
- return;
-
- const uint m_rows = min(Bm, seg_len - row_tg_off);
- const uint n_cols = min(Bn, d_model - col_tg_off);
-
- const ulong h_base = (ulong)(seg_start + row_tg_off) * (ulong)d_ff;
- const ulong w2_expert_base = (ulong)expert_idx * (ulong)d_model * (ulong)d_ff;
- const ulong bias_base = (ulong)expert_idx * (ulong)d_model;
-
- // 2×2 simdgroup layout for 16×64 output (each sg handles 8×32)
- const uint row_sg = thread_context.simdgroup_index / 2; // 0-1 → rows
- const uint col_sg = thread_context.simdgroup_index % 2; // 0-1 → cols
- const uint row_sg_off = row_sg * SgBm;
- const uint col_sg_off = col_sg * SgBn;
-
- // Guard against excessive simdgroups
- constexpr uint SG_EXPECTED = (PASSB_BM / SgBm) * (PASSB_BN / SgBn);
- if (thread_context.simdgroup_index >= SG_EXPECTED)
- return;
-
- // 4 accumulators per simdgroup: 1×4 layout of 8×8 tiles
- constexpr uint TEMP = (SgBm / SG_TILE) * (SgBn / SG_TILE); // 1×4 = 4
- metal::simdgroup_float8x8 Out[TEMP];
- METAL_PRAGMA_UNROLL
- for (uint i = 0; i < TEMP; ++i) {
- Out[i] = metal::make_filled_simdgroup_matrix<float, 8, 8>(0.0f);
- }
-
- // Cooperative load config - vec8 loads for better bandwidth
- constexpr uint H_VEC = 8;
- constexpr uint H_TCOLS = Bk / H_VEC; // 64/8 = 8
- const uint h_bi = lin / H_TCOLS;
- const uint h_bj = (lin % H_TCOLS) * H_VEC;
-
- constexpr uint W_VEC = 8;
- constexpr uint W_TCOLS = Bk / W_VEC; // 64/8 = 8
- constexpr uint W_TROWS = TGP_SIZE / W_TCOLS; // 128/8 = 16
- const uint w_bi = lin / W_TCOLS;
- const uint w_bj = (lin % W_TCOLS) * W_VEC;
-
- // K-loop across FF
- for (uint k_off = 0; k_off < d_ff; k_off += Bk) {
- const uint valid_k = min(Bk, d_ff - k_off);
- threadgroup_barrier(mem_flags::mem_threadgroup);
-
- // ---- Stage hidden [Bm=16, Bk=64] with vec8 loads ----
- // 128 threads, 8 vec8 loads per row → 16 rows covered in one pass
- {
- threadgroup float* my_dst = Hs + h_bi * H_LD + h_bj;
- if (h_bi < m_rows && h_bj < valid_k) {
- // Safe to read - at least first element is valid
- device const float* my_src = hidden + h_base + (ulong)h_bi * (ulong)d_ff + (ulong)(k_off + h_bj);
- METAL_PRAGMA_UNROLL
- for (uint j = 0; j < H_VEC; j++) {
- my_dst[j] = (h_bj + j < valid_k) ? my_src[j] : 0.0f;
- }
- } else {
- METAL_PRAGMA_UNROLL
- for (uint j = 0; j < H_VEC; j++) {
- my_dst[j] = 0.0f;
- }
- }
- }
-
- // ---- Stage W2 [Bn=64, Bk=64] with vec8 loads ----
- // Need 4 passes: 128 threads cover 16 rows, Bn=64 needs 4×16
- {
- for (uint pass = 0; pass < 4; pass++) {
- const uint row = w_bi + pass * W_TROWS;
- threadgroup T* my_dst = Wk + row * W_LD + w_bj;
- if (row < n_cols && w_bj < valid_k) {
- // Safe to read - at least first element is valid
- const uint gcol = col_tg_off + row;
- device const T* my_src = w2_all + w2_expert_base + (ulong)gcol * (ulong)d_ff + (ulong)(k_off + w_bj);
- METAL_PRAGMA_UNROLL
- for (uint j = 0; j < W_VEC; j++) {
- my_dst[j] = (w_bj + j < valid_k) ? static_cast<T>(float(my_src[j])) : static_cast<T>(0.0f);
- }
- } else {
- METAL_PRAGMA_UNROLL
- for (uint j = 0; j < W_VEC; j++) {
- my_dst[j] = static_cast<T>(0.0f);
- }
- }
- }
- }
-
- threadgroup_barrier(mem_flags::mem_threadgroup);
-
- // ---- Serpentine MMA: 1×4 tiles per simdgroup ----
- METAL_PRAGMA_UNROLL
- for (uint kk = 0; kk < Bk; kk += SG_TILE) {
- // Load 1 LHS fragment
- metal::simdgroup_float8x8 lhs;
- simdgroup_load(lhs, Hs, H_LD, ulong2(kk, row_sg_off));
-
- // Load 4 RHS fragments (cols)
- metal::simdgroup_matrix<T, 8, 8> rhs0, rhs1, rhs2, rhs3;
- simdgroup_load(rhs0, Wk, W_LD, ulong2(kk, col_sg_off), true);
- simdgroup_load(rhs1, Wk, W_LD, ulong2(kk, col_sg_off + 8), true);
- simdgroup_load(rhs2, Wk, W_LD, ulong2(kk, col_sg_off + 16), true);
- simdgroup_load(rhs3, Wk, W_LD, ulong2(kk, col_sg_off + 24), true);
-
- // Serpentine pattern: 0→1→2→3 (forward for first row)
- simdgroup_multiply_accumulate(Out[0], lhs, rhs0, Out[0]);
- simdgroup_multiply_accumulate(Out[1], lhs, rhs1, Out[1]);
- simdgroup_multiply_accumulate(Out[2], lhs, rhs2, Out[2]);
- simdgroup_multiply_accumulate(Out[3], lhs, rhs3, Out[3]);
- }
- }
-
- // ---- Bias tile (cooperative load) ----
- for (uint c_local = lin; c_local < Bn; c_local += TGP_SIZE) {
- const uint c_global = col_tg_off + c_local;
- bias_tile[c_local] = (c_global < d_model) ? float(down_biases[bias_base + c_global]) : 0.0f;
- }
-
- threadgroup_barrier(mem_flags::mem_threadgroup);
-
- // ---- Writeout with fragment extraction ----
- const uint lane_qid = thread_context.simd_lane_id >> 2;
- const uint lane_row = (lane_qid & 4u) + ((thread_context.simd_lane_id >> 1) & 3u);
- const uint lane_col_base = ((lane_qid & 2u) * 2u) + ((thread_context.simd_lane_id & 1u) * 2u);
-
- const uint local_row = row_sg_off + lane_row;
- if (local_row >= m_rows)
- return;
- const ulong out_row = seg_start + row_tg_off + local_row;
-
- METAL_PRAGMA_UNROLL
- for (uint ni = 0; ni < 4; ni++) { // 4 col tiles per simdgroup
- const auto accum = Out[ni].thread_elements();
- const uint local_col = col_sg_off + ni * 8 + lane_col_base;
-
- if (local_col < n_cols) {
- const ulong out_col = col_tg_off + local_col;
- const float val = accum[0] + bias_tile[local_col];
- output[out_row * (ulong)d_model + out_col] = T(val);
- }
- if (local_col + 1 < n_cols) {
- const ulong out_col = col_tg_off + local_col + 1;
- const float val = accum[1] + bias_tile[local_col + 1];
- output[out_row * (ulong)d_model + out_col] = T(val);
- }
- }
-}
diff --git a/crates/uzu-engine/src/backends/metal/kernel/moe/finalize.metal b/crates/uzu-engine/src/backends/metal/kernel/moe/finalize.metal
deleted file mode 100644
index f94928258..000000000
--- a/crates/uzu-engine/src/backends/metal/kernel/moe/finalize.metal
+++ /dev/null
@@ -1,54 +0,0 @@
-#include <metal_stdlib>
-#include "../common/dsl.h"
-
-#define BM 32
-#define BN 64
-
-template <typename T>
-VARIANTS(T, float, half, bfloat)
-PUBLIC KERNEL(MoeFinalize)(
- device const int* tok2row, // [T*K], -1 if dropped
- device const T* probs, // [T*K]
- device const T* y_partial, // [sum_k, d_model]
- device T* y, // [T, d_model]
- constant uint& t_count,
- constant uint& d_model,
- constant uint& k_input,
- const uint lid THREADS(128),
- const uint tg_n GROUPS(d_model.div_ceil(BN)),
- const uint tg_m GROUPS(t_count.div_ceil(BM))
-) {
- const uint tile_n0 = tg_n * BN;
- const uint tile_m0 = tg_m * BM;
- const uint m_rows = min((uint)BM, t_count - tile_m0);
- const uint n_cols = min((uint)BN, d_model - tile_n0);
-
- // 128 threads per TG expected
- for (uint idx = lid; idx < m_rows * n_cols; idx += 128u) {
- const uint mi = idx / n_cols;
- const uint nj = idx % n_cols;
- const uint t = tile_m0 + mi;
- const uint f = tile_n0 + nj;
- float acc = 0.0f;
- const uint base = t * k_input;
- for (uint k = 0; k < k_input; ++k) {
- const int row = tok2row[base + k];
- if (row >= 0) {
- const ulong yidx = (ulong)(uint)row * (ulong)d_model + (ulong)f;
- float prob = (float)probs[base + k];
- if (!isfinite(prob)) {
- prob = 0.0f;
- }
- float val = (float)y_partial[yidx];
- if (!isfinite(val)) {
- val = 0.0f;
- }
- acc = fma(prob, val, acc);
- }
- }
- if (!isfinite(acc)) {
- acc = 0.0f;
- }
- y[t * d_model + f] = T(acc);
- }
-}
diff --git a/crates/uzu-engine/src/backends/metal/kernel/moe/gather.metal b/crates/uzu-engine/src/backends/metal/kernel/moe/gather.metal
deleted file mode 100644
index 98365a997..000000000
--- a/crates/uzu-engine/src/backends/metal/kernel/moe/gather.metal
+++ /dev/null
@@ -1,112 +0,0 @@
-#include <metal_stdlib>
-#include "../common/dsl.h"
-
-// Tiling config for 2D gather
-#define ROWS_PER_TG 8
-#define COL_VECS 32u // 256 threads = 8×32
-#define ELEMS_PER_THREAD 8u // each thread copies 8 elements (2× vec4)
-#define THREADS_PER_THREADGROUP 256
-
-template <typename T>
-VARIANTS(T, float, half, bfloat)
-PUBLIC KERNEL(MoeGatherXPerm2D)(
- device const T* x,
- device const int* bucketed_ids,
- device T* x_perm,
- device const uint* sumk_buf,
- constant uint& d_model,
- constant uint& t,
- constant uint& k,
- const uint tgid_x GROUPS((t * k).div_ceil(ROWS_PER_TG)),
- const uint lid THREADS(THREADS_PER_THREADGROUP)
-) {
- const uint total_rows = sumk_buf[0];
- if (total_rows == 0u || d_model == 0u) {
- return;
- }
-
- // Thread organization: lid = row_in_tg * COL_VECS + col_vec
- const uint row_in_tg = lid / COL_VECS;
- const uint col_vec = lid % COL_VECS;
-
- // Global row assignment
- const uint base_row = tgid_x * ROWS_PER_TG;
- const uint row = base_row + row_in_tg;
- if (row >= total_rows) {
- return;
- }
-
- int token = bucketed_ids[row];
- if (token < 0) {
- return;
- }
-
- const device T* src = x + (ulong)(uint)token * (ulong)d_model;
- device T* dst = x_perm + (ulong)row * (ulong)d_model;
-
- // Each thread copies ELEMS_PER_THREAD elements
- // Thread i processes: col_vec*8, col_vec*8+256, col_vec*8+512, ...
- uint col = col_vec * ELEMS_PER_THREAD;
- const uint stride = COL_VECS * ELEMS_PER_THREAD;
-
- // Vectorized copy: 8 elements = 2× vec4
- using Vec4 = typename metal::vec<T, 4>;
- while (col + 7u < d_model) {
- Vec4 v0 = *(reinterpret_cast<const device Vec4*>(src + col));
- Vec4 v1 = *(reinterpret_cast<const device Vec4*>(src + col + 4u));
- *(reinterpret_cast<device Vec4*>(dst + col)) = v0;
- *(reinterpret_cast<device Vec4*>(dst + col + 4u)) = v1;
- col += stride;
- }
-
- // Scalar remainder
- if (col < d_model) {
- const uint limit = min(d_model, col + ELEMS_PER_THREAD);
- for (uint tail = col; tail < limit; ++tail) {
- dst[tail] = src[tail];
- }
- }
-}
-
-template <typename T>
-VARIANTS(T, float, half, bfloat)
-PUBLIC KERNEL(MoeGatherXPerm1D)(
- device const T* x,
- device const int* bucketed_ids,
- device T* x_perm,
- device const uint* sumk_buf,
- constant uint& d_model,
- constant uint& t,
- constant uint& k,
- const uint tgid_x GROUPS((t * k).div_ceil(THREADS_PER_THREADGROUP)),
- const uint lid THREADS(THREADS_PER_THREADGROUP)
-) {
- const uint total_rows = sumk_buf[0];
- if (total_rows == 0u || d_model == 0u)
- return;
-
- const uint num_vec4 = d_model / 4u;
-
- // Each thread processes multiple rows sequentially
- for (uint row = lid; row < total_rows; row += THREADS_PER_THREADGROUP) {
- int token = bucketed_ids[row];
- if (token < 0)
- continue;
-
- const device T* src = x + (ulong)(uint)token * (ulong)d_model;
- device T* dst = x_perm + (ulong)row * (ulong)d_model;
-
- // Vectorized copy: 4 elements at a time
- using Vec4 = typename metal::vec<T, 4>;
- for (uint vec_idx = 0u; vec_idx < num_vec4; vec_idx++) {
- uint col = vec_idx * 4u;
- Vec4 v = *(reinterpret_cast<const device Vec4*>(src + col));
- *(reinterpret_cast<device Vec4*>(dst + col)) = v;
- }
-
- // Scalar remainder
- for (uint col = num_vec4 * 4u; col < d_model; col++) {
- dst[col] = src[col];
- }
- }
-}
\ No newline at end of file
diff --git a/crates/uzu-engine/src/backends/metal/kernel/moe/router_topk.metal b/crates/uzu-engine/src/backends/metal/kernel/moe/router_topk.metal
deleted file mode 100644
index b69a5ac9a..000000000
--- a/crates/uzu-engine/src/backends/metal/kernel/moe/router_topk.metal
+++ /dev/null
@@ -1,166 +0,0 @@
-#include <metal_stdlib>
-#include <metal_simdgroup>
-#include "../common/dsl.h"
-#include "../common/thread_context.h"
-#include "../common/threadgroup_reduce.h"
-#include "../generated/router_topk.h"
-using namespace metal;
-using namespace uzu::router_topk;
-
-#define THREADS_PER_TG 256
-
-constant float NEG_INF = -INFINITY;
-
-template <typename ScalarT>
-VARIANTS(ScalarT, half, bfloat, float)
-PUBLIC KERNEL(MoeRouterTopK)(
- const device ScalarT* input,
- const device ScalarT* weight,
- const device ScalarT* bias OPTIONAL(has_biases),
- const device ScalarT* router_scale OPTIONAL(has_router_scales),
- const device ScalarT* per_expert_scale OPTIONAL(has_per_expert_scales),
- device int* topk_ids,
- device ScalarT* topk_probs,
- constant uint& t,
- constant uint& d_model,
- constant uint& e,
- constant uint& k,
- constant bool& renorm,
- constant float& router_norm_epsilon OPTIONAL(normalize_router_input),
- constant float& router_input_scale OPTIONAL(has_router_input_scale),
- const bool has_biases SPECIALIZE,
- const bool has_router_scales SPECIALIZE,
- const bool has_per_expert_scales SPECIALIZE,
- const bool has_router_input_scale SPECIALIZE,
- const bool normalize_router_input SPECIALIZE,
- threadgroup float4 x_cache[ROUTER_TOPK_MAX_MODEL_DIM / 4],
- threadgroup float logits_shared[ROUTER_TOPK_MAX_EXPERTS],
- threadgroup float reduce_tmp[THREADS_PER_TG],
- threadgroup uint reduce_tmp_u[THREADS_PER_TG],
- threadgroup uint shared_best_idx[1],
- threadgroup float shared_best_val[1],
- const ThreadContext thread_context,
- const uint tgpig_x GROUPS(1),
- const uint token_idx GROUPS(t),
- const uint lid THREADS(THREADS_PER_TG)
-) {
- if (d_model == 0 || e == 0 || k == 0) {
- return;
- }
-
- const uint vecs = d_model / 4u;
- const device ScalarT* x_vec = input + (ulong)token_idx * (ulong)vecs * 4;
-
- float local_sum_sq = 0.0f;
- for (uint c = lid; c < vecs; c += THREADS_PER_TG) {
- const float4 xv = float4(x_vec[c * 4 + 0], x_vec[c * 4 + 1], x_vec[c * 4 + 2], x_vec[c * 4 + 3]);
- x_cache[c] = xv;
- if (normalize_router_input) {
- local_sum_sq += dot(xv, xv);
- }
- }
-
- float inv_rms = 1.0f;
- if (normalize_router_input) {
- const float sum_sq =
- threadgroup_cooperative_reduce<SimdReduceSum<float>, THREADS_PER_TG>(local_sum_sq, reduce_tmp, thread_context);
- inv_rms = rsqrt(sum_sq / float(d_model) + router_norm_epsilon);
- }
-
- if (normalize_router_input || has_router_input_scale || has_router_scales) {
- float4 base_scale4 = float4(inv_rms);
- if (has_router_input_scale) {
- base_scale4 *= float4(router_input_scale);
- }
- for (uint c = lid; c < vecs; c += THREADS_PER_TG) {
- float4 scale4 = base_scale4;
- if (has_router_scales) {
- scale4 *=
- float4(router_scale[c * 4 + 0], router_scale[c * 4 + 1], router_scale[c * 4 + 2], router_scale[c * 4 + 3]);
- }
- x_cache[c] *= scale4;
- }
- }
- threadgroup_barrier(mem_flags::mem_threadgroup);
-
- for (uint row = thread_context.simdgroup_index; row < e; row += thread_context.simdgroups_per_threadgroup) {
- const device ScalarT* w_vec = weight + (ulong)row * (ulong)vecs * 4;
-
- float4 accum4 = float4(0.0f);
- for (uint c = thread_context.simd_lane_id; c < vecs; c += 32u) {
- const float4 wv = float4(w_vec[c * 4 + 0], w_vec[c * 4 + 1], w_vec[c * 4 + 2], w_vec[c * 4 + 3]);
- const float4 xv = x_cache[c];
- accum4 = fma(wv, xv, accum4);
- }
- float sum = (accum4.x + accum4.y) + (accum4.z + accum4.w);
- sum = simd_sum(sum);
- if (simd_is_first()) {
- if (has_biases) {
- sum += float(bias[row]);
- }
- logits_shared[row] = sum;
- }
- }
- threadgroup_barrier(mem_flags::mem_threadgroup);
-
- const uint effective_k = min(k, ROUTER_TOPK_MAX_SELECTED_EXPERTS);
- for (uint sel = 0; sel < effective_k; ++sel) {
- float local_best = NEG_INF;
- uint local_idx = 0xFFFFFFFFu;
- for (uint row = lid; row < e; row += THREADS_PER_TG) {
- float candidate = logits_shared[row];
- if (candidate > local_best || (candidate == local_best && row < local_idx)) {
- local_best = candidate;
- local_idx = row;
- }
- }
-
- float max_val =
- threadgroup_cooperative_reduce<SimdReduceMax<float>, THREADS_PER_TG>(local_best, reduce_tmp, thread_context);
-
- uint candidate_id = (local_best == max_val) ? local_idx : 0xFFFFFFFFu;
- uint best_idx =
- threadgroup_cooperative_reduce<SimdReduceMin<uint>, THREADS_PER_TG>(candidate_id, reduce_tmp_u, thread_context);
-
- if (lid == 0) {
- shared_best_idx[0] = best_idx;
- shared_best_val[0] = max_val;
- }
- threadgroup_barrier(mem_flags::mem_threadgroup);
-
- uint winner_idx = shared_best_idx[0];
- float winner_val = shared_best_val[0];
- if (lid == 0 && winner_idx < ROUTER_TOPK_MAX_EXPERTS) {
- if (!renorm && has_per_expert_scales) {
- winner_val *= float(per_expert_scale[winner_idx]);
- }
- logits_shared[winner_idx] = NEG_INF;
- topk_ids[token_idx * k + sel] = int(winner_idx);
- topk_probs[token_idx * k + sel] = static_cast<ScalarT>(winner_val);
- }
- threadgroup_barrier(mem_flags::mem_threadgroup);
- }
-
- if (lid == 0 && renorm && effective_k > 0) {
- device ScalarT* out_probs = topk_probs + token_idx * k;
- float max_logit = -INFINITY;
- for (uint i = 0; i < effective_k; ++i) {
- max_logit = fmax(max_logit, float(out_probs[i]));
- }
- float sum_exp = 0.0f;
- for (uint i = 0; i < effective_k; ++i) {
- sum_exp += exp(float(out_probs[i]) - max_logit);
- }
- float default_prob = 1.0f / float(effective_k);
- float inv_sum = (sum_exp > 0.0f) ? (1.0f / sum_exp) : default_prob;
- for (uint i = 0; i < effective_k; ++i) {
- float prob = (sum_exp > 0.0f) ? exp(float(out_probs[i]) - max_logit) * inv_sum : default_prob;
- const int expert_id = topk_ids[token_idx * k + i];
- const float expert_scale = (has_per_expert_scales && expert_id >= 0) ? float(per_expert_scale[expert_id]) : 1.0f;
- out_probs[i] = static_cast<ScalarT>(prob * expert_scale);
- }
- for (uint i = effective_k; i < k; ++i) {
- out_probs[i] = static_cast<ScalarT>(0.0f);
- }
- }
-}
diff --git a/crates/uzu-engine/src/backends/metal/kernel/moe/scatter_buckets.metal b/crates/uzu-engine/src/backends/metal/kernel/moe/scatter_buckets.metal
deleted file mode 100644
index 85adc349f..000000000
--- a/crates/uzu-engine/src/backends/metal/kernel/moe/scatter_buckets.metal
+++ /dev/null
@@ -1,210 +0,0 @@
-#include <metal_stdlib>
-#include <metal_atomic>
-#include "../common/dsl.h"
-#include "../common/thread_context.h"
-using namespace metal;
-
-#define BLOCK_SIZE 256
-#define TILE_E 512
-#define SIMD_WIDTH 32
-#define NUM_SG (BLOCK_SIZE / SIMD_WIDTH)
-
-// 4B: Scatter into expert-major buckets using atomic-free device writes
-template <typename ProbT>
-inline void moe_scatter_buckets_impl(
- device const int* topk_ids,
- device const ProbT* topk_probs,
- device const uint* offsets,
- device const uint* block_bases,
- device const uint* block_alloc,
- device int* out_ids,
- device ProbT* out_probs,
- device int* tok2row_opt,
- uint T,
- uint E,
- uint K,
- uint num_tiles,
- uint lid,
- uint3 tgpig,
- threadgroup _atomic<uint>* sg_counts,
- threadgroup uint* sg_base,
- const thread ThreadContext& thread_context
-) {
- if (E == 0 || T == 0 || K == 0)
- return;
- const uint block_id = tgpig.x;
- const uint t_start = block_id * BLOCK_SIZE;
- const uint t_end = min(t_start + BLOCK_SIZE, T);
-
- // Per-tile processing
- for (uint e0 = 0; e0 < E; e0 += TILE_E) {
- const uint tile_e = (e0 + TILE_E <= E) ? TILE_E : (E - e0);
-
- // Zero counts and bases for active tile
- for (uint te = lid; te < tile_e; te += BLOCK_SIZE) {
- for (uint sg = 0; sg < NUM_SG; ++sg) {
- atomic_store_explicit(&sg_counts[sg * TILE_E + te], 0u, memory_order_relaxed);
- sg_base[sg * TILE_E + te] = 0u;
- }
- }
- threadgroup_barrier(mem_flags::mem_threadgroup);
-
- // Phase 1: per-sg counting using threadgroup atomics (order-independent)
- for (uint t = t_start + lid; t < t_end; t += BLOCK_SIZE) {
- const uint base = t * K;
- for (uint k = 0; k < K; ++k) {
- int eid = topk_ids[base + k];
- if (eid >= 0) {
- const uint ue = (uint)eid;
- if (ue >= e0 && ue < e0 + tile_e) {
- const uint te = ue - e0;
- atomic_fetch_add_explicit(
- &sg_counts[thread_context.simdgroup_index * TILE_E + te],
- 1u,
- memory_order_relaxed
- );
- }
- }
- }
- }
- threadgroup_barrier(mem_flags::mem_threadgroup);
-
- // Phase 2: compute per-sg bases via prefix across sgs with capacity clamp
- for (uint te = lid; te < tile_e; te += BLOCK_SIZE) {
- const uint tile_id = e0 / TILE_E;
- const uint base_idx_block = (block_id * num_tiles + tile_id) * TILE_E + te;
- const uint alloc_total = block_alloc[base_idx_block];
- uint prefix = 0u;
- for (uint sg = 0; sg < NUM_SG; ++sg) {
- const uint idx = sg * TILE_E + te;
- const uint c = atomic_load_explicit(&sg_counts[idx], memory_order_relaxed);
- const uint room = (prefix < alloc_total) ? (alloc_total - prefix) : 0u;
- const uint take = min(c, room);
- sg_base[idx] = prefix;
- prefix += take;
- }
- // Reuse sg_counts as write indices: reset to zero
- for (uint sg = 0; sg < NUM_SG; ++sg) {
- atomic_store_explicit(&sg_counts[sg * TILE_E + te], 0u, memory_order_relaxed);
- }
- }
- threadgroup_barrier(mem_flags::mem_threadgroup);
-
- // Phase 3: deterministic write with 32-lane sequencing, no device atomics
- for (uint step = 0; step < thread_context.simdgroup_size; ++step) {
- if (thread_context.simd_lane_id == step) {
- const uint t = t_start + lid;
- if (t < t_end) {
- const uint base = t * K;
- for (uint k = 0; k < K; ++k) {
- int eid = topk_ids[base + k];
- if (eid >= 0) {
- const uint ue = (uint)eid;
- if (ue >= e0 && ue < e0 + tile_e) {
- const uint te = ue - e0;
- const uint tile_id = e0 / TILE_E;
- const uint base_idx_block = (block_id * num_tiles + tile_id) * TILE_E + te;
- const uint idx_sg_te = thread_context.simdgroup_index * TILE_E + te;
- const uint local = atomic_fetch_add_explicit(&sg_counts[idx_sg_te], 1u, memory_order_relaxed);
- const uint base_s = sg_base[idx_sg_te];
-
- const uint slot = offsets[ue] + block_bases[base_idx_block] + base_s + local;
- out_ids[slot] = (int)t;
- out_probs[slot] = topk_probs[base + k];
- if (tok2row_opt != nullptr) {
- tok2row_opt[base + k] = (int)slot;
- }
- }
- }
- }
- }
- }
- threadgroup_barrier(mem_flags::mem_threadgroup);
- }
- threadgroup_barrier(mem_flags::mem_threadgroup);
- }
-}
-
-// 4A: Compute per-block base offsets from partials
-// partials layout: index = (block_id * num_tiles + tile_id) * TILE_E + te
-PUBLIC KERNEL(MoeBlockBasesFromPartials)(
- device const uint* partials,
- device uint* block_bases,
- device uint* block_alloc,
- constant uint& e_input,
- constant uint& num_blocks,
- constant uint& num_tiles,
- constant uint& capacity_per_expert, // 0 => no clamp
- const uint gid AXIS(num_tiles * 512, 256)
-) {
- const uint tile_id = gid / TILE_E;
- const uint te = gid % TILE_E;
- const uint e = tile_id * TILE_E + te;
- if (e >= e_input)
- return;
-
- uint running = 0u;
- const bool clamp = (capacity_per_expert > 0u);
- for (uint b = 0; b < num_blocks; ++b) {
- const uint idx = (b * num_tiles + tile_id) * TILE_E + te;
- const uint p = partials[idx];
- const uint alloc = clamp ? (running < capacity_per_expert ? min(p, capacity_per_expert - running) : 0u) : p;
- block_bases[idx] = running;
- block_alloc[idx] = alloc;
- running += alloc;
- if (clamp && running >= capacity_per_expert) {
- // Remainder allocate zero and base stays at capacity
- for (uint bb = b + 1; bb < num_blocks; ++bb) {
- const uint idx2 = (bb * num_tiles + tile_id) * TILE_E + te;
- block_bases[idx2] = running;
- block_alloc[idx2] = 0u;
- }
- break;
- }
- }
-}
-
-// Map variants
-template <typename T>
-VARIANTS(T, float, half, bfloat)
-PUBLIC KERNEL(MoeScatterBucketsMap)(
- device const int* topk_ids,
- device const T* topk_probs,
- device const uint* offsets,
- device const uint* block_bases,
- device const uint* block_alloc,
- device int* out_ids,
- device T* out_probs,
- constant uint& t,
- constant uint& e,
- constant uint& k,
- constant uint& num_blocks,
- constant uint& num_tiles,
- device int* tok2row,
- threadgroup _atomic<uint> sg_counts[NUM_SG * TILE_E],
- threadgroup uint sg_base[NUM_SG * TILE_E],
- const ThreadContext thread_context,
- const uint tgpig_x GROUPS(num_blocks),
- const uint lid THREADS(256)
-) {
- const uint3 tgpig = {tgpig_x, 0, 0};
- moe_scatter_buckets_impl<T>(
- topk_ids,
- topk_probs,
- offsets,
- block_bases,
- block_alloc,
- out_ids,
- out_probs,
- tok2row,
- t,
- e,
- k,
- num_tiles,
- lid,
- tgpig,
- sg_counts,
- sg_base,
- thread_context
- );
-}
diff --git a/crates/uzu-engine/src/backends/metal/kernel/moe/tiles_map.metal b/crates/uzu-engine/src/backends/metal/kernel/moe/tiles_map.metal
deleted file mode 100644
index d49745802..000000000
--- a/crates/uzu-engine/src/backends/metal/kernel/moe/tiles_map.metal
+++ /dev/null
@@ -1,107 +0,0 @@
-#include <metal_stdlib>
-#include <metal_atomic>
-#include "../common/dsl.h"
-
-#define BM 16
-
-// Compute per-expert tile counts: tiles_e = ceil((seg_len)/BM)
-PUBLIC KERNEL(MoeTileCounts)(
- device const uint* offsets, // [e+1]
- device uint* tile_counts, // [e]
- constant uint& e,
- const uint gid AXIS(e, 256)
-) {
- const uint start = offsets[gid];
- const uint end = offsets[gid + 1];
- const uint seg_len = (end > start) ? (end - start) : 0u;
- const uint tiles = (seg_len == 0u) ? 0u : ((seg_len + BM - 1u) / BM);
- tile_counts[gid] = tiles;
-}
-
-// Single-threadgroup exclusive scan over tile_counts -> tile_row_offsets, also
-// writes total_tiles
-PUBLIC KERNEL(MoeTileScan)(
- device const uint* tile_counts, // [e]
- device uint* tile_row_offsets, // [e+1]
- device uint* total_tiles_buf, // [>=2]
- constant uint& e,
- threadgroup uint scratch[1024], // supports up to E<=1023
- uint tid THREADS(256)
-) {
- if (tid < e) {
- scratch[tid] = tile_counts[tid];
- }
- threadgroup_barrier(mem_flags::mem_threadgroup);
-
- // Inclusive scan in-place over scratch
- // Simple O(log E) scan; E is small
- for (uint offset = 1u; offset < e; offset *= 2u) {
- uint val = 0u;
- if (tid < e) {
- val = scratch[tid];
- if (tid >= offset) {
- val += scratch[tid - offset];
- }
- }
- threadgroup_barrier(mem_flags::mem_threadgroup);
- if (tid < e) {
- scratch[tid] = val;
- }
- threadgroup_barrier(mem_flags::mem_threadgroup);
- }
-
- if (tid == 0u) {
- tile_row_offsets[0] = 0u;
- }
- if (tid < e) {
- // exclusive: shift right by one
- const uint prefix = (tid == 0u) ? 0u : scratch[tid - 1u];
- tile_row_offsets[tid] = prefix;
- }
- if (tid == 0u) {
- tile_row_offsets[e] = (e == 0u) ? 0u : scratch[e - 1u];
- total_tiles_buf[0] = tile_row_offsets[e];
- total_tiles_buf[1] = 0u;
- total_tiles_buf[2] = 0u;
- total_tiles_buf[3] = 0u;
- total_tiles_buf[4] = 0u;
- total_tiles_buf[5] = 0u;
- total_tiles_buf[6] = 0u;
- total_tiles_buf[7] = 0u;
- }
-}
-
-// Build flattened tile_map of length total_tiles; each entry is 3 uints:
-// [3*i+0]=expert_idx, [3*i+1]=seg_start, [3*i+2]=tile_m0
-PUBLIC KERNEL(MoeBuildTileMap)(
- device const uint* offsets, // [e+1]
- device const uint* tile_row_offsets, // [e+1]
- device const uint* tile_counts, // [e]
- device uint* tile_map, // [3*total_tiles]
- constant uint& e,
- uint gid AXIS(e, 256)
-) {
- const uint start = offsets[gid];
- const uint tiles = tile_counts[gid];
- const uint base = tile_row_offsets[gid];
- for (uint t = 0u; t < tiles; ++t) {
- const uint idx = base + t;
- tile_map[3u * idx + 0u] = gid;
- tile_map[3u * idx + 1u] = start;
- tile_map[3u * idx + 2u] = t * BM;
- }
-}
-
-// Write MTLDispatchThreadgroupsIndirectArguments {x, y, z} where:
-// x = num_tiles_n (computed on CPU and passed in), y = total_tiles, z = 1
-PUBLIC KERNEL(MoeWriteDispatchArgs)(
- device const uint* total_tiles_buf, // [>=1], total_tiles_buf[0] = total_rows
- device uint* dispatch_args, // [3] u32
- constant uint& num_tiles_n, // x dimension
- uint gid AXIS(1, 1)
-) {
- const uint total_rows = total_tiles_buf[0];
- dispatch_args[0] = num_tiles_n; // x
- dispatch_args[1] = total_rows; // y
- dispatch_args[2] = 1u; // z
-}
diff --git a/crates/uzu-engine/src/backends/metal/kernel/moe/tiles_pass_a.metal b/crates/uzu-engine/src/backends/metal/kernel/moe/tiles_pass_a.metal
deleted file mode 100644
index 1e566a1ea..000000000
--- a/crates/uzu-engine/src/backends/metal/kernel/moe/tiles_pass_a.metal
+++ /dev/null
@@ -1,128 +0,0 @@
-#include <metal_stdlib>
-#include "../common/dsl.h"
-
-// === Helper kernels for indirect dispatch of Pass A ===
-
-// Count tiles per expert: tiles = (num_rows > 0) ? num_rows * h_blocks : 0
-PUBLIC KERNEL(MoePassATileCounts)(
- device const uint* expert_offsets, // [e+1]
- device uint* tile_counts, // [e]
- constant uint& e,
- constant uint& h_blocks,
- const uint tid AXIS(e, 256)
-) {
- const uint start = expert_offsets[tid];
- const uint end = expert_offsets[tid + 1];
- const uint num_rows = end - start;
- tile_counts[tid] = (num_rows > 0) ? (num_rows * h_blocks) : 0;
-}
-
-// Exclusive scan of tile_counts to get tile_offsets and total_tiles
-PUBLIC KERNEL(MoePassATileScan)(
- device const uint* tile_counts, // [e]
- device uint* tile_offsets, // [e+1]
- device uint* total_tiles, // [1]
- constant uint& e,
- threadgroup uint scratch[1024],
- uint gid GROUPS(1),
- uint lid THREADS(1024)
-) {
- // Simple single-threadgroup scan (works for E <= 1024)
- const uint idx = lid;
-
- // Load into threadgroup memory
- if (idx < e) {
- scratch[idx] = tile_counts[idx];
- } else {
- scratch[idx] = 0;
- }
- threadgroup_barrier(mem_flags::mem_threadgroup);
-
- // Kogge-Stone scan
- uint val = scratch[idx];
- for (uint offset = 1; offset < 1024; offset *= 2) {
- uint temp = 0;
- if (idx >= offset && idx < e) {
- temp = scratch[idx - offset];
- }
- threadgroup_barrier(mem_flags::mem_threadgroup);
- if (idx >= offset && idx < e) {
- val += temp;
- scratch[idx] = val;
- }
- threadgroup_barrier(mem_flags::mem_threadgroup);
- }
-
- // Write exclusive scan (shift right by 1)
- if (idx == 0) {
- tile_offsets[0] = 0;
- }
- if (idx < e) {
- tile_offsets[idx + 1] = scratch[idx];
- if (idx == e - 1) {
- total_tiles[0] = scratch[idx];
- }
- }
-}
-
-// Build row→expert map: one thread per routed row
-PUBLIC KERNEL(MoePassABuildRowMap)(
- device const uint* expert_offsets, // [e+1]
- device uint* row_expert_map, // [total_rows]
- constant uint& total_rows,
- constant uint& e,
- uint tid AXIS(total_rows, 256)
-) {
- uint left = 0u;
- uint right = e;
- const uint row = tid;
-
- while (left + 1u < right) {
- const uint mid = (left + right) >> 1;
- if (row < expert_offsets[mid]) {
- right = mid;
- } else {
- left = mid;
- }
- }
-
- row_expert_map[row] = left;
-}
-
-// Build tile map entries from row→expert map
-PUBLIC KERNEL(MoePassABuildTileMap)(
- device const uint* expert_offsets, // [E+1]
- device const uint* tile_offsets, // [E+1]
- device const uint* row_expert_map, // [total_rows]
- device uint* tile_map, // [total_tiles * 3]
- constant uint& total_rows,
- constant uint& h_blocks,
- uint tid AXIS(total_rows.saturating_mul(h_blocks), 256)
-) {
- const uint row_idx = tid / h_blocks;
- const uint h_block = tid % h_blocks;
- if (row_idx >= total_rows) {
- return;
- }
-
- const uint expert_idx = row_expert_map[row_idx];
- const uint row_start = expert_offsets[expert_idx];
- const uint row_in_expert = row_idx - row_start;
- const uint tile_base = tile_offsets[expert_idx] + row_in_expert * h_blocks + h_block;
-
- tile_map[tile_base * 3 + 0] = h_block;
- tile_map[tile_base * 3 + 1] = expert_idx;
- tile_map[tile_base * 3 + 2] = row_in_expert;
-}
-
-// Write dispatch args for indirect dispatch (reusable from tiled version)
-PUBLIC KERNEL(MoePassAWriteDispatchArgs)(
- device const uint* total_tiles, // [1]
- device uint* dispatch_args, // [3] - MTLDispatchThreadgroupsIndirectArguments
- constant uint& num_tiles_y, // usually 1 for Pass A
- uint tid AXIS(1, 1)
-) {
- dispatch_args[0] = total_tiles[0]; // x dimension = total tiles
- dispatch_args[1] = num_tiles_y; // y dimension
- dispatch_args[2] = 1; // z dimension
-}
diff --git a/crates/uzu-engine/src/encodable_block/mlp/mod.rs b/crates/uzu-engine/src/encodable_block/mlp/mod.rs
index 53e533370..5ae64a74e 100644
--- a/crates/uzu-engine/src/encodable_block/mlp/mod.rs
+++ b/crates/uzu-engine/src/encodable_block/mlp/mod.rs
@@ -1,11 +1,9 @@
mod dense;
mod gate_act_mul;
-mod moe;
pub use dense::DenseMlp;
use derive_more::Debug;
use gate_act_mul::MlpGateActMulEncodable;
-pub use moe::{MoeBlock, MoeBlockError};
use thiserror::Error;
use crate::{
@@ -31,8 +29,8 @@ pub enum MlpBlockError<B: Backend> {
BackendError(#[source] B::Error),
#[error("Linear block error: {0}")]
LinearBlockError(#[from] LinearBlockError<B>),
- #[error("MoeBlock error: {0}")]
- MoeBlockError(#[from] MoeBlockError<B>),
+ #[error("Mixture of experts is not supported")]
+ UnsupportedMixtureOfExperts,
}
impl<B: Backend> dyn Mlp<B> {
@@ -77,16 +75,7 @@ impl<B: Backend> dyn Mlp<B> {
Ok((Box::new(DenseMlp::new(up_projection, gate, down_projection)), up_input_hadamard_factors))
},
- AnyMLPConfig::MixtureOfExpertsConfig(mixture_of_experts_config) => Ok((
- Box::new(MoeBlock::new(
- context,
- mixture_of_experts_config,
- model_dimension,
- data_type,
- parameter_tree,
- )?),
- None,
- )),
+ AnyMLPConfig::MixtureOfExpertsConfig(_) => Err(MlpBlockError::UnsupportedMixtureOfExperts),
}
}
}
diff --git a/crates/uzu-engine/src/encodable_block/mlp/moe/experts_two_pass_decode.rs b/crates/uzu-engine/src/encodable_block/mlp/moe/experts_two_pass_decode.rs
deleted file mode 100644
index 9fec0879b..000000000
--- a/crates/uzu-engine/src/encodable_block/mlp/moe/experts_two_pass_decode.rs
+++ /dev/null
@@ -1,123 +0,0 @@
-use crate::{
- backends::common::{
- Backend, BufferRef, CommandBuffer, CommandBufferEncoding,
- kernel::{
- Kernels, MoeExpertsDecodeDownFused2DKernel, MoeExpertsDecodePassAKernel, MoePassABuildRowMapKernel,
- MoePassABuildTileMapKernel, MoePassATileCountsKernel, MoePassATileScanKernel,
- MoePassAWriteDispatchArgsKernel,
- },
- },
- data_type::DataType,
- encodable_block::mlp::moe::experts_two_pass_prefill::MoeExpertsTwoPassArguments,
-};
-
-pub struct MoeExpertsTwoPassDecodeBlock<B: Backend> {
- counts: <B::Kernels as Kernels>::MoePassATileCountsKernel,
- scan: <B::Kernels as Kernels>::MoePassATileScanKernel,
- row_map: <B::Kernels as Kernels>::MoePassABuildRowMapKernel,
- build_map: <B::Kernels as Kernels>::MoePassABuildTileMapKernel,
- dispatch: <B::Kernels as Kernels>::MoePassAWriteDispatchArgsKernel,
- pass_a_indirect: <B::Kernels as Kernels>::MoeExpertsDecodePassAKernel,
- fused_down: <B::Kernels as Kernels>::MoeExpertsDecodeDownFused2DKernel,
- data_type: DataType,
-}
-
-impl<B: Backend> MoeExpertsTwoPassDecodeBlock<B> {
- pub fn new(
- ctx: &B::Context,
- data_type: DataType,
- gating_code: u32,
- ) -> Result<Self, B::Error> {
- Ok(Self {
- counts: <B::Kernels as Kernels>::MoePassATileCountsKernel::new(ctx)?,
- scan: <B::Kernels as Kernels>::MoePassATileScanKernel::new(ctx)?,
- row_map: <B::Kernels as Kernels>::MoePassABuildRowMapKernel::new(ctx)?,
- build_map: <B::Kernels as Kernels>::MoePassABuildTileMapKernel::new(ctx)?,
- dispatch: <B::Kernels as Kernels>::MoePassAWriteDispatchArgsKernel::new(ctx)?,
- pass_a_indirect: <B::Kernels as Kernels>::MoeExpertsDecodePassAKernel::new(ctx, data_type, gating_code)?,
- fused_down: <B::Kernels as Kernels>::MoeExpertsDecodeDownFused2DKernel::new(ctx, data_type, DataType::F32)?,
- data_type,
- })
- }
-
- pub fn encode(
- &self,
- args: MoeExpertsTwoPassArguments<
- impl BufferRef<Backend = B>,
- impl BufferRef<Backend = B>,
- impl BufferRef<Backend = B>,
- >,
- command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
- ) -> Result<B::ScratchBuffer, B::Error> {
- const BLOCK_M: u32 = 4;
- let h_blocks = args.d_ff.div_ceil(BLOCK_M);
-
- let mut tile_counts = command_buffer.allocate_scratch_for_shape(&[args.num_routed_experts], DataType::U32)?;
- self.counts.encode(args.expert_offsets, &mut tile_counts, args.num_routed_experts, h_blocks, command_buffer);
-
- let mut tile_offsets =
- command_buffer.allocate_scratch_for_shape(&[args.num_routed_experts + 1], DataType::U32)?;
- let mut total_tiles = command_buffer.allocate_scratch_for_shape(&[1], DataType::U32)?;
- self.scan.encode(&tile_counts, &mut tile_offsets, &mut total_tiles, args.num_routed_experts, command_buffer);
-
- let mut row_expert_map = command_buffer.allocate_scratch_for_shape(&[args.total_rows], DataType::U32)?;
- self.row_map.encode(
- args.expert_offsets,
- &mut row_expert_map,
- args.total_rows,
- args.num_routed_experts,
- command_buffer,
- );
-
- let mut tile_map = command_buffer.allocate_scratch_for_shape(&[args.total_rows, h_blocks, 3], DataType::U32)?;
- self.build_map.encode(
- args.expert_offsets,
- &tile_offsets,
- &row_expert_map,
- &mut tile_map,
- args.total_rows,
- h_blocks,
- command_buffer,
- );
-
- let mut dispatch_args = command_buffer.allocate_scratch_for_shape(&[3], DataType::U32)?;
- self.dispatch.encode(&total_tiles, &mut dispatch_args, 1, command_buffer);
-
- let mut hidden = command_buffer.allocate_scratch_for_shape(&[args.total_rows, args.d_ff], DataType::F32)?;
-
- self.pass_a_indirect.encode(
- args.x_perm,
- args.expert_offsets,
- args.w13_all,
- &mut hidden,
- args.up_biases,
- args.d_model,
- args.d_ff,
- args.num_routed_experts,
- args.gate_clip_min,
- args.gate_clip_max,
- args.up_clip_min,
- args.up_clip_max,
- args.silu_alpha,
- &tile_map,
- &dispatch_args,
- command_buffer,
- );
-
- let mut output = command_buffer.allocate_scratch_for_shape(&[args.total_rows, args.d_model], self.data_type)?;
- self.fused_down.encode(
- &hidden,
- &row_expert_map,
- args.w2_all,
- args.down_biases,
- &mut output,
- args.total_rows,
- args.d_model,
- args.d_ff,
- args.num_routed_experts,
- command_buffer,
- );
-
- Ok(output)
- }
-}
diff --git a/crates/uzu-engine/src/encodable_block/mlp/moe/experts_two_pass_prefill.rs b/crates/uzu-engine/src/encodable_block/mlp/moe/experts_two_pass_prefill.rs
deleted file mode 100644
index 37a05cedb..000000000
--- a/crates/uzu-engine/src/encodable_block/mlp/moe/experts_two_pass_prefill.rs
+++ /dev/null
@@ -1,138 +0,0 @@
-use crate::{
- backends::common::{
- Backend, BufferRef, CommandBuffer, CommandBufferEncoding, Kernels,
- kernel::{
- MoeBuildTileMapKernel, MoeExpertsPrefillPassAKernel, MoeExpertsPrefillPassBKernel, MoeTileCountsKernel,
- MoeTileScanKernel, MoeWriteDispatchArgsKernel,
- },
- },
- data_type::DataType,
-};
-
-pub struct MoeExpertsTwoPassPrefillBlock<B: Backend> {
- counts: <B::Kernels as Kernels>::MoeTileCountsKernel,
- scan: <B::Kernels as Kernels>::MoeTileScanKernel,
- build: <B::Kernels as Kernels>::MoeBuildTileMapKernel,
- dispatch: <B::Kernels as Kernels>::MoeWriteDispatchArgsKernel,
- pass_a_indirect: <B::Kernels as Kernels>::MoeExpertsPrefillPassAKernel,
- pass_b_indirect: <B::Kernels as Kernels>::MoeExpertsPrefillPassBKernel,
- data_type: DataType,
-}
-
-impl<B: Backend> MoeExpertsTwoPassPrefillBlock<B> {
- pub fn new(
- ctx: &B::Context,
- data_type: DataType,
- gating_code: u32,
- ) -> Result<Self, B::Error> {
- Ok(Self {
- counts: <B::Kernels as Kernels>::MoeTileCountsKernel::new(ctx)?,
- scan: <B::Kernels as Kernels>::MoeTileScanKernel::new(ctx)?,
- build: <B::Kernels as Kernels>::MoeBuildTileMapKernel::new(ctx)?,
- dispatch: <B::Kernels as Kernels>::MoeWriteDispatchArgsKernel::new(ctx)?,
- pass_a_indirect: <B::Kernels as Kernels>::MoeExpertsPrefillPassAKernel::new(ctx, data_type, gating_code)?,
- pass_b_indirect: <B::Kernels as Kernels>::MoeExpertsPrefillPassBKernel::new(ctx, data_type)?,
- data_type,
- })
- }
-
- pub fn encode(
- &self,
- args: MoeExpertsTwoPassArguments<
- impl BufferRef<Backend = B>,
- impl BufferRef<Backend = B>,
- impl BufferRef<Backend = B>,
- >,
- command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
- ) -> Result<B::ScratchBuffer, B::Error> {
- let mut tile_counts = command_buffer.allocate_scratch_for_shape(&[args.num_routed_experts], DataType::U32)?;
- self.counts.encode(args.expert_offsets, &mut tile_counts, args.num_routed_experts, command_buffer);
-
- let mut tile_offsets =
- command_buffer.allocate_scratch_for_shape(&[args.num_routed_experts + 1], DataType::U32)?;
- let mut total_tiles = command_buffer.allocate_scratch_for_shape(&[8], DataType::U32)?;
- self.scan.encode(&tile_counts, &mut tile_offsets, &mut total_tiles, args.num_routed_experts, command_buffer);
-
- let mut tile_map = command_buffer.allocate_scratch_for_shape(&[args.total_rows, 3], DataType::U32)?;
- self.build.encode(
- args.expert_offsets,
- &tile_offsets,
- &tile_counts,
- &mut tile_map,
- args.num_routed_experts,
- command_buffer,
- );
-
- const COL_TILE_FF: u32 = 32; // Must match PASSA_BN in kernel
- let n_tiles_ff = args.d_ff.div_ceil(COL_TILE_FF);
-
- let mut pass_a_dispatch_args = command_buffer.allocate_scratch_for_shape(&[3], DataType::U32)?;
- self.dispatch.encode(&total_tiles, &mut pass_a_dispatch_args, n_tiles_ff, command_buffer);
-
- let mut hidden = command_buffer.allocate_scratch_for_shape(&[args.total_rows, args.d_ff], DataType::F32)?;
- command_buffer.encode_fill(&mut hidden, 0);
-
- self.pass_a_indirect.encode(
- args.x_perm,
- args.expert_offsets,
- args.w13_all,
- args.up_biases,
- &mut hidden,
- args.d_model,
- args.d_ff,
- args.num_routed_experts,
- args.gate_clip_min,
- args.gate_clip_max,
- args.up_clip_min,
- args.up_clip_max,
- args.silu_alpha,
- &tile_map,
- &pass_a_dispatch_args,
- command_buffer,
- );
-
- const COL_TILE_MODEL: u32 = 64;
- let n_tiles_model = args.d_model.div_ceil(COL_TILE_MODEL);
-
- let mut pass_b_dispatch_args = command_buffer.allocate_scratch_for_shape(&[3], DataType::U32)?;
- self.dispatch.encode(&total_tiles, &mut pass_b_dispatch_args, n_tiles_model, command_buffer);
-
- let mut output = command_buffer.allocate_scratch_for_shape(&[args.total_rows, args.d_model], self.data_type)?;
- self.pass_b_indirect.encode(
- &hidden,
- args.expert_offsets,
- args.w2_all,
- args.down_biases,
- &mut output,
- args.d_model,
- args.d_ff,
- args.num_routed_experts,
- &tile_map,
- &pass_b_dispatch_args,
- command_buffer,
- );
- Ok(output)
- }
-}
-
-pub struct MoeExpertsTwoPassArguments<
- X: BufferRef,
- O: BufferRef<Backend = X::Backend>,
- W: BufferRef<Backend = X::Backend>,
-> {
- pub x_perm: X,
- pub expert_offsets: O,
- pub w13_all: W,
- pub w2_all: W,
- pub up_biases: W,
- pub down_biases: W,
- pub total_rows: u32,
- pub d_model: u32,
- pub d_ff: u32,
- pub num_routed_experts: u32,
- pub gate_clip_min: f32,
- pub gate_clip_max: f32,
- pub up_clip_min: f32,
- pub up_clip_max: f32,
- pub silu_alpha: f32,
-}
diff --git a/crates/uzu-engine/src/encodable_block/mlp/moe/gather.rs b/crates/uzu-engine/src/encodable_block/mlp/moe/gather.rs
deleted file mode 100644
index 7fadae4d6..000000000
--- a/crates/uzu-engine/src/encodable_block/mlp/moe/gather.rs
+++ /dev/null
@@ -1,77 +0,0 @@
-use crate::{
- backends::common::{
- Backend, BufferRef, CommandBuffer, CommandBufferEncoding, Kernels,
- kernel::{MoeGatherXPerm1DKernel, MoeGatherXPerm2DKernel},
- },
- data_type::DataType,
-};
-
-enum MoeGatherVariant<B: Backend> {
- OneD(<B::Kernels as Kernels>::MoeGatherXPerm1DKernel),
- TwoD(<B::Kernels as Kernels>::MoeGatherXPerm2DKernel),
-}
-
-pub struct MoeGather<B: Backend> {
- variant: MoeGatherVariant<B>,
- data_type: DataType,
-}
-
-impl<B: Backend> MoeGather<B> {
- pub fn new(
- ctx: &B::Context,
- data_type: DataType,
- ) -> Result<Self, B::Error> {
- Ok(Self {
- variant: if data_type == DataType::BF16 {
- MoeGatherVariant::TwoD(<B::Kernels as Kernels>::MoeGatherXPerm2DKernel::new(ctx, data_type)?)
- } else {
- MoeGatherVariant::OneD(<B::Kernels as Kernels>::MoeGatherXPerm1DKernel::new(ctx, data_type)?)
- },
- data_type,
- })
- }
-
- pub fn encode(
- &self,
- input: impl BufferRef<Backend = B>,
- bucketed_ids: impl BufferRef<Backend = B>,
- sumk: impl BufferRef<Backend = B>,
- batch_dim: u32,
- num_active_experts: u32,
- d_model: u32,
- command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
- ) -> Result<B::ScratchBuffer, B::Error> {
- let mut x_perm =
- command_buffer.allocate_scratch_for_shape(&[batch_dim, num_active_experts, d_model], self.data_type)?;
- command_buffer.encode_fill(&mut x_perm, 0);
-
- match &self.variant {
- MoeGatherVariant::OneD(kernel) => kernel.encode(
- input,
- bucketed_ids,
- &mut x_perm,
- sumk,
- d_model,
- batch_dim,
- num_active_experts,
- command_buffer,
- ),
- MoeGatherVariant::TwoD(kernel) => kernel.encode(
- input,
- bucketed_ids,
- &mut x_perm,
- sumk,
- d_model,
- batch_dim,
- num_active_experts,
- command_buffer,
- ),
- };
-
- Ok(x_perm)
- }
-}
-
-#[cfg(test)]
-#[path = "../../../../unit/encodable_block/moe/moe_gather_test.rs"]
-mod tests;
diff --git a/crates/uzu-engine/src/encodable_block/mlp/moe/mod.rs b/crates/uzu-engine/src/encodable_block/mlp/moe/mod.rs
deleted file mode 100644
index 62b292f04..000000000
--- a/crates/uzu-engine/src/encodable_block/mlp/moe/mod.rs
+++ /dev/null
@@ -1,361 +0,0 @@
-//! MoE (Mixture of Experts) block encodable.
-
-mod experts_two_pass_decode;
-mod experts_two_pass_prefill;
-mod gather;
-
-use derive_more::Debug;
-use experts_two_pass_decode::MoeExpertsTwoPassDecodeBlock;
-use experts_two_pass_prefill::{MoeExpertsTwoPassArguments, MoeExpertsTwoPassPrefillBlock};
-use gather::MoeGather;
-use thiserror::Error;
-
-use crate::{
- backends::common::{
- Backend, CommandBuffer, CommandBufferEncoding, Kernels,
- gpu_types::{
- ActivationType,
- router_topk::{ROUTER_TOPK_MAX_EXPERTS, ROUTER_TOPK_MAX_MODEL_DIM, ROUTER_TOPK_MAX_SELECTED_EXPERTS},
- },
- kernel::{
- MoeBlockBasesFromPartialsKernel, MoeCountsOffsetsFusedKernel, MoeFinalizeKernel, MoeRouterTopKKernel,
- MoeScatterBucketsMapKernel,
- },
- },
- config::{
- mlp::{mixture_of_experts::MixtureOfExpertsConfig, routing_function::AnyRoutingFunction},
- weight_matrix::{AnyWeightMatrixSpec, Layout, full_precision_spec::FullPrecisionSpec},
- },
- data_type::DataType,
- encodable_block::mlp::Mlp,
- parameters::{ParameterLoaderError, ParameterTree},
-};
-
-pub struct MoeBlock<B: Backend> {
- router_topk_kernel: <B::Kernels as Kernels>::MoeRouterTopKKernel,
- counts_offsets_kernel: <B::Kernels as Kernels>::MoeCountsOffsetsFusedKernel,
- scatter_bases_kernel: <B::Kernels as Kernels>::MoeBlockBasesFromPartialsKernel,
- scatter_map_kernel: <B::Kernels as Kernels>::MoeScatterBucketsMapKernel,
- gather: MoeGather<B>,
- experts_two_pass_decode_block: MoeExpertsTwoPassDecodeBlock<B>,
- experts_two_pass_prefill_block: MoeExpertsTwoPassPrefillBlock<B>,
- finalize_kernel: <B::Kernels as Kernels>::MoeFinalizeKernel,
- router_weights: B::GlobalBuffer,
- router_biases: B::GlobalBuffer,
- router_renorm: bool,
- w13: B::GlobalBuffer,
- w2: B::GlobalBuffer,
- up_biases: B::GlobalBuffer,
- down_biases: B::GlobalBuffer,
- model_dim: u32,
- hidden_dim: u32,
- num_routed_experts: u32,
- num_active_experts: u32,
- gate_clip_min: f32,
- gate_clip_max: f32,
- up_clip_min: f32,
- up_clip_max: f32,
- silu_alpha: f32,
- data_type: DataType,
-}
-
-#[derive(Debug, Error)]
-pub enum MoeBlockError<B: Backend> {
- #[error("Backend error: {0}")]
- BackendError(#[source] B::Error),
- #[error("Parameter loader error: {0}")]
- ParameterLoaderError(#[from] ParameterLoaderError<B>),
- #[error("MoE model_dim must be nonzero, divisible by 4, and within the fused router capacity")]
- InvalidModelDim,
- #[error("MoE num_routed_experts must be nonzero and within the fused router capacity")]
- InvalidRoutedExpertCount,
- #[error("MoE num_active_routed_experts must be nonzero, within TopK capacity, and <= num_routed_experts")]
- InvalidActiveExpertCount,
- #[error("MoE shared experts are not supported")]
- UnsupportedSharedExperts,
- #[error("MoE expert gate is not supported")]
- UnsupportedExpertGate,
- #[error("MoE without router, up, and down biases is not supported")]
- UnsupportedNoBiases,
- #[error("Unsupported MoE router configuration: {0}")]
- UnsupportedRouterConfiguration(String),
- #[error("Unsupported MoE expert activation: {0:?}")]
- UnsupportedExpertActivation(ActivationType),
-}
-
-impl<B: Backend> MoeBlock<B> {
- pub fn new(
- context: &B::Context,
- moe_config: &MixtureOfExpertsConfig,
- model_dim: u32,
- data_type: DataType,
- parameter_tree: &ParameterTree<B>,
- ) -> Result<Self, MoeBlockError<B>> {
- if model_dim == 0 || model_dim > ROUTER_TOPK_MAX_MODEL_DIM || !model_dim.is_multiple_of(4) {
- return Err(MoeBlockError::InvalidModelDim);
- }
- if !(1..=ROUTER_TOPK_MAX_EXPERTS).contains(&moe_config.num_routed_experts) {
- return Err(MoeBlockError::InvalidRoutedExpertCount);
- }
- if !(1..=ROUTER_TOPK_MAX_SELECTED_EXPERTS.min(moe_config.num_routed_experts))
- .contains(&moe_config.num_active_routed_experts)
- {
- return Err(MoeBlockError::InvalidActiveExpertCount);
- }
- if moe_config.num_shared_experts != 0 {
- return Err(MoeBlockError::UnsupportedSharedExperts);
- }
- if moe_config.gate_config.is_some() {
- return Err(MoeBlockError::UnsupportedExpertGate);
- }
- if !moe_config.router_has_biases
- || !moe_config.expert_config.has_up_biases
- || !moe_config.expert_config.has_down_biases
- {
- return Err(MoeBlockError::UnsupportedNoBiases);
- }
-
- let gating_code = match moe_config.expert_config.activation.act_type() {
- ActivationType::GELUApprox => 3,
- ActivationType::SILU => 2,
- activation_type => return Err(MoeBlockError::UnsupportedExpertActivation(activation_type)),
- };
-
- let router_renorm = matches!(moe_config.routing_function, AnyRoutingFunction::SoftmaxRouting(_));
-
- let router_tree = parameter_tree.subtree("router");
- let router_weights_tree = router_tree.subtree("weights");
- let router_spec = router_weights_tree.metadata::<AnyWeightMatrixSpec>("spec")?;
- let AnyWeightMatrixSpec::FullPrecisionSpec(FullPrecisionSpec {
- layout: Layout::OutputInput,
- ..
- }) = &router_spec
- else {
- return Err(MoeBlockError::UnsupportedRouterConfiguration(format!("{router_spec:?}")));
- };
- let router_weights = router_weights_tree
- .leaf("weights")?
- .validate(&[moe_config.num_routed_experts, model_dim], data_type)?
- .read_buffer()?;
- let router_biases =
- router_tree.leaf("biases")?.validate(&[moe_config.num_routed_experts], data_type)?.read_buffer()?;
-
- let experts_tree = parameter_tree.subtree("experts");
- let up_tree = experts_tree.subtree("up_projection");
- let down_tree = experts_tree.subtree("down_projection");
- let up_weights_tree = up_tree.subtree("weights");
- let down_weights_tree = down_tree.subtree("weights");
-
- let w13 = up_weights_tree
- .leaf("weights")?
- .validate(&[moe_config.num_routed_experts, moe_config.expert_hidden_dim * 2, model_dim], data_type)?
- .read_buffer()?;
- let w2 = down_weights_tree
- .leaf("weights")?
- .validate(&[moe_config.num_routed_experts, model_dim, moe_config.expert_hidden_dim], data_type)?
- .read_buffer()?;
- let up_biases = up_tree
- .leaf("biases")?
- .validate(&[moe_config.num_routed_experts, moe_config.expert_hidden_dim * 2], data_type)?
- .read_buffer()?;
- let down_biases = down_tree
- .leaf("biases")?
- .validate(&[moe_config.num_routed_experts, model_dim], data_type)?
- .read_buffer()?;
-
- let router_topk_kernel =
- <B::Kernels as Kernels>::MoeRouterTopKKernel::new(context, data_type, true, false, false, false, false)
- .map_err(MoeBlockError::BackendError)?;
- let counts_offsets_kernel = MoeCountsOffsetsFusedKernel::new(context).map_err(MoeBlockError::BackendError)?;
-
- let scatter_bases_kernel = <B::Kernels as Kernels>::MoeBlockBasesFromPartialsKernel::new(context)
- .map_err(MoeBlockError::BackendError)?;
- let scatter_map_kernel = <B::Kernels as Kernels>::MoeScatterBucketsMapKernel::new(context, data_type)
- .map_err(MoeBlockError::BackendError)?;
-
- let gather = MoeGather::new(context, data_type).map_err(MoeBlockError::BackendError)?;
- let experts_two_pass_decode_block =
- MoeExpertsTwoPassDecodeBlock::new(context, data_type, gating_code).map_err(MoeBlockError::BackendError)?;
- let experts_two_pass_prefill_block =
- MoeExpertsTwoPassPrefillBlock::new(context, data_type, gating_code).map_err(MoeBlockError::BackendError)?;
- let finalize_kernel =
- <B::Kernels as Kernels>::MoeFinalizeKernel::new(context, data_type).map_err(MoeBlockError::BackendError)?;
-
- let gate_clipping = moe_config.expert_config.gate_clipping;
- let gate_clipping = gate_clipping.into_pair().unwrap_or((f32::MIN, f32::MAX));
- let up_clipping = moe_config.expert_config.up_clipping;
- let up_clipping = up_clipping.into_pair().unwrap_or((f32::MIN, f32::MAX));
-
- Ok(Self {
- router_topk_kernel,
- counts_offsets_kernel,
- scatter_bases_kernel,
- scatter_map_kernel,
- gather,
- experts_two_pass_decode_block,
- experts_two_pass_prefill_block,
- finalize_kernel,
- router_weights,
- router_biases,
- router_renorm,
- w13,
- w2,
- up_biases,
- down_biases,
- model_dim,
- hidden_dim: moe_config.expert_hidden_dim,
- num_routed_experts: moe_config.num_routed_experts,
- num_active_experts: moe_config.num_active_routed_experts,
- gate_clip_min: gate_clipping.0,
- gate_clip_max: gate_clipping.1,
- up_clip_min: up_clipping.0,
- up_clip_max: up_clipping.1,
- silu_alpha: moe_config.expert_config.activation.alpha(),
- data_type,
- })
- }
-}
-
-impl<B: Backend> Mlp<B> for MoeBlock<B> {
- fn encode(
- &self,
- input: B::ScratchBuffer,
- batch_dim: u32,
- command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
- ) -> Result<B::ScratchBuffer, B::Error> {
- command_buffer.push_debug_group("mlp (moe)");
-
- let total_rows = batch_dim * self.num_active_experts;
- let num_blocks = batch_dim.div_ceil(256);
- let num_tiles = self.num_routed_experts.div_ceil(512);
-
- let mut topk_ids =
- command_buffer.allocate_scratch_for_shape(&[batch_dim, self.num_active_experts], DataType::I32)?;
- let mut topk_probs =
- command_buffer.allocate_scratch_for_shape(&[batch_dim, self.num_active_experts], self.data_type)?;
-
- command_buffer.encode_fill(&mut topk_ids, 0xFF);
-
- self.router_topk_kernel.encode(
- &input,
- &self.router_weights,
- Some(&self.router_biases),
- None::<&B::GlobalBuffer>,
- None::<&B::GlobalBuffer>,
- &mut topk_ids,
- &mut topk_probs,
- batch_dim,
- self.model_dim,
- self.num_routed_experts,
- self.num_active_experts,
- self.router_renorm,
- None::<f32>,
- None::<f32>,
- command_buffer,
- );
-
- let mut offsets = command_buffer.allocate_scratch_for_shape(&[self.num_routed_experts + 1], DataType::U32)?;
- let mut sumk = command_buffer.allocate_scratch_for_shape(&[1], DataType::U32)?;
- let scatter_entries = num_blocks * num_tiles * 512;
- let mut partials = command_buffer.allocate_scratch_for_shape(&[scatter_entries], DataType::U32)?;
- self.counts_offsets_kernel.encode(
- &topk_ids,
- &mut offsets,
- &mut sumk,
- &mut partials,
- batch_dim,
- self.num_routed_experts,
- self.num_active_experts,
- command_buffer,
- );
-
- let mut block_bases = command_buffer.allocate_scratch_for_shape(&[scatter_entries], DataType::U32)?;
- let mut block_alloc = command_buffer.allocate_scratch_for_shape(&[scatter_entries], DataType::U32)?;
- let mut bucketed_ids = command_buffer.allocate_scratch_for_shape(&[total_rows], DataType::I32)?;
- let mut bucketed_probs = command_buffer.allocate_scratch_for_shape(&[total_rows], self.data_type)?;
- let mut tok2row = command_buffer.allocate_scratch_for_shape(&[total_rows], DataType::I32)?;
-
- command_buffer.encode_fill(&mut tok2row, 0xFF);
-
- self.scatter_bases_kernel.encode(
- &partials,
- &mut block_bases,
- &mut block_alloc,
- self.num_routed_experts,
- num_blocks,
- num_tiles,
- 0u32,
- command_buffer,
- );
- self.scatter_map_kernel.encode(
- &topk_ids,
- &topk_probs,
- &offsets,
- &block_bases,
- &block_alloc,
- &mut bucketed_ids,
- &mut bucketed_probs,
- batch_dim,
- self.num_routed_experts,
- self.num_active_experts,
- num_blocks,
- num_tiles,
- &mut tok2row,
- command_buffer,
- );
-
- let x_perm = self.gather.encode(
- &input,
- &bucketed_ids,
- &sumk,
- batch_dim,
- self.num_active_experts,
- self.model_dim,
- command_buffer,
- )?;
-
- let args = MoeExpertsTwoPassArguments {
- x_perm: &x_perm,
- expert_offsets: &offsets,
- w13_all: &self.w13,
- w2_all: &self.w2,
- up_biases: &self.up_biases,
- down_biases: &self.down_biases,
- total_rows,
- d_model: self.model_dim,
- d_ff: self.hidden_dim,
- num_routed_experts: self.num_routed_experts,
- gate_clip_min: self.gate_clip_min,
- gate_clip_max: self.gate_clip_max,
- up_clip_min: self.up_clip_min,
- up_clip_max: self.up_clip_max,
- silu_alpha: self.silu_alpha,
- };
-
- let y_partial = if batch_dim == 1 {
- self.experts_two_pass_decode_block.encode(args, command_buffer)?
- } else {
- self.experts_two_pass_prefill_block.encode(args, command_buffer)?
- };
-
- let mut output = command_buffer.allocate_scratch_for_shape(&[batch_dim, self.model_dim], self.data_type)?;
- self.finalize_kernel.encode(
- &tok2row,
- &topk_probs,
- &y_partial,
- &mut output,
- batch_dim,
- self.model_dim,
- self.num_active_experts,
- command_buffer,
- );
-
- command_buffer.pop_debug_group();
-
- Ok(output)
- }
-}
-
-#[cfg(test)]
-#[path = "../../../../unit/encodable_block/moe/mod.rs"]
-mod tests;
diff --git a/crates/uzu-engine/unit/backends/common/kernel/mod.rs b/crates/uzu-engine/unit/backends/common/kernel/mod.rs
index ad865edc5..337413e9e 100644
--- a/crates/uzu-engine/unit/backends/common/kernel/mod.rs
+++ b/crates/uzu-engine/unit/backends/common/kernel/mod.rs
@@ -6,7 +6,6 @@ mod gated_act_mul_test;
mod gdn;
mod logit_transform_test;
mod matmul;
-mod moe;
mod normalization_test;
mod pooling;
mod radix_top_k_small_test;
diff --git a/crates/uzu-engine/unit/backends/common/kernel/moe/mod.rs b/crates/uzu-engine/unit/backends/common/kernel/moe/mod.rs
deleted file mode 100644
index 82c2e9a28..000000000
--- a/crates/uzu-engine/unit/backends/common/kernel/moe/mod.rs
+++ /dev/null
@@ -1,3 +0,0 @@
-mod moe_counts_offsets_fused_test;
-mod moe_finalize_test;
-mod moe_router_topk_test;
diff --git a/crates/uzu-engine/unit/backends/common/kernel/moe/moe_counts_offsets_fused_test.rs b/crates/uzu-engine/unit/backends/common/kernel/moe/moe_counts_offsets_fused_test.rs
deleted file mode 100644
index a459958aa..000000000
--- a/crates/uzu-engine/unit/backends/common/kernel/moe/moe_counts_offsets_fused_test.rs
+++ /dev/null
@@ -1,237 +0,0 @@
-use rand::{RngExt, SeedableRng, rngs::StdRng};
-use uzu_engine_macros::uzu_test;
-
-use crate::{
- backends::{
- common::{
- Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferPending, Context, Kernels,
- kernel::MoeCountsOffsetsFusedKernel,
- },
- cpu::Cpu,
- },
- tests::helpers::{
- buffer_prefix_to_vec, buffer_to_vec, create_buffer, create_buffer_with_data, for_each_backend,
- for_each_non_cpu_backend,
- },
-};
-
-fn gen_random_topk_ids(
- t: usize,
- e: usize,
- k: usize,
- seed: u64,
-) -> Vec<i32> {
- let mut rng = StdRng::seed_from_u64(seed);
- let mut topk_ids = vec![0i32; t * k];
- for ti in 0..t {
- for kk in 0..k {
- topk_ids[ti * k + kk] = rng.random_range(0..e as i32);
- }
- }
- topk_ids
-}
-
-fn get_output<B: Backend>(
- topk_ids: &[i32],
- t: usize,
- e: usize,
- k: usize,
-) -> (Vec<u32>, u32, Vec<u32>) {
- let context = B::Context::new().expect("Failed to create Context");
-
- let kernel = <<B as Backend>::Kernels as Kernels>::MoeCountsOffsetsFusedKernel::new(&context)
- .expect("Failed to create MoeCountsOffsetsFusedKernel");
-
- let topk_ids_len = (t * k).max(1);
- let topk_ids_buffer = if topk_ids.is_empty() {
- create_buffer::<B, i32>(&context, topk_ids_len)
- } else {
- create_buffer_with_data::<B, i32>(&context, topk_ids)
- };
- let mut offsets = create_buffer::<B, u32>(&context, e + 1);
- let mut sum_k = create_buffer::<B, u32>(&context, 1);
- let num_tiles = e.div_ceil(512).max(1);
- let mut partials = create_buffer::<B, u32>(&context, num_tiles * 512);
-
- let mut command_buffer = context.create_command_buffer(None, None).expect("Failed to create command buffer");
- kernel.encode(
- &topk_ids_buffer,
- &mut offsets,
- &mut sum_k,
- &mut partials,
- t as u32,
- e as u32,
- k as u32,
- &mut command_buffer,
- );
- command_buffer.end_encoding().submit().wait_until_completed().unwrap();
-
- let offsets = buffer_to_vec::<B, u32>(&offsets);
- let sum_k = buffer_to_vec::<B, u32>(&sum_k)[0];
- let partials = buffer_prefix_to_vec::<B, u32>(&partials, e);
-
- (offsets, sum_k, partials)
-}
-
-#[uzu_test]
-fn test_counts_offsets_fused_parity_random() {
- let shapes = vec![(1usize, 4usize), (7, 16), (64, 64), (1024, 128)];
- let ks = vec![1usize, 2usize, 4usize];
-
- for &(t, e) in &shapes {
- for &k in &ks {
- if k > e {
- continue;
- }
- let topk_ids = gen_random_topk_ids(t, e, k, 1234);
-
- let (offsets_ref, sum_ref, counts_ref) = get_output::<Cpu>(&topk_ids, t, e, k);
- for_each_non_cpu_backend!(|B| {
- let (offsets, sum_k, partials) = get_output::<B>(&topk_ids, t, e, k);
- let backend_name = std::any::type_name::<B>();
- assert_eq!(offsets, offsets_ref, "offsets mismatch T={} E={} K={} backend={}", t, e, k, backend_name);
- assert_eq!(sum_k, sum_ref, "sum mismatch T={} E={} K={} backend={}", t, e, k, backend_name);
- assert_eq!(partials, counts_ref, "partials mismatch T={} E={} K={} backend={}", t, e, k, backend_name);
- });
- }
- }
-}
-
-#[uzu_test]
-fn test_counts_offsets_fused_all_tokens_one_expert() {
- let (t, e, k) = (16usize, 8usize, 2usize);
- let topk_ids: Vec<i32> = vec![3i32; t * k];
-
- let mut expected_offsets = vec![0u32; e + 1];
- expected_offsets[4..].fill((t * k) as u32);
-
- for_each_backend!(|B| {
- let (offsets, sum_k, _) = get_output::<B>(&topk_ids, t, e, k);
- let backend_name = std::any::type_name::<B>();
- assert_eq!(offsets, expected_offsets, "offsets mismatch backend={}", backend_name);
- assert_eq!(sum_k, (t * k) as u32, "sum mismatch backend={}", backend_name);
- });
-}
-
-#[uzu_test]
-fn test_counts_offsets_fused_zero_tokens() {
- let (t, e, k) = (0usize, 8usize, 2usize);
- let topk_ids: Vec<i32> = vec![];
-
- for_each_backend!(|B| {
- let (offsets, sum_k, _) = get_output::<B>(&topk_ids, t, e, k);
- let backend_name = std::any::type_name::<B>();
- assert!(offsets.iter().all(|&v| v == 0), "expected all-zero offsets for T=0, backend={}", backend_name,);
- assert_eq!(sum_k, 0, "sum should be 0 for T=0, backend={}", backend_name);
- });
-}
-
-#[uzu_test]
-fn test_counts_offsets_fused_negative_ids_ignored() {
- let (t, e, k) = (8usize, 4usize, 2usize);
- // Mix of valid IDs and -1 (invalid/padding)
- let mut topk_ids = vec![-1i32; t * k];
- // Only set the first slot of each token, leave second as -1
- for ti in 0..t {
- topk_ids[ti * k] = (ti % e) as i32;
- }
-
- let (offsets_ref, sum_ref, counts_ref) = get_output::<Cpu>(&topk_ids, t, e, k);
- // sum should be t (only one valid id per token)
- assert_eq!(sum_ref, t as u32);
-
- for_each_non_cpu_backend!(|B| {
- let (offsets, sum_k, partials) = get_output::<B>(&topk_ids, t, e, k);
- let backend_name = std::any::type_name::<B>();
- assert_eq!(offsets, offsets_ref, "offsets mismatch backend={}", backend_name);
- assert_eq!(sum_k, sum_ref, "sum mismatch backend={}", backend_name);
- assert_eq!(partials, counts_ref, "partials mismatch backend={}", backend_name);
- });
-}
-
-#[uzu_test]
-fn test_counts_offsets_fused_single_token() {
- let (t, e, k) = (1usize, 16usize, 1usize);
- let topk_ids = vec![7i32];
-
- let mut expected_offsets = vec![0u32; e + 1];
- // Expert 7 gets 1 token, so offsets[8..] = 1
- expected_offsets[8..].fill(1);
-
- for_each_backend!(|B| {
- let (offsets, sum_k, _) = get_output::<B>(&topk_ids, t, e, k);
- let backend_name = std::any::type_name::<B>();
- assert_eq!(offsets, expected_offsets, "offsets mismatch backend={}", backend_name);
- assert_eq!(sum_k, 1, "sum mismatch backend={}", backend_name);
- });
-}
-
-#[uzu_test]
-fn test_counts_offsets_fused_uniform_distribution() {
- // Each expert gets exactly 1 token
- let e = 8usize;
- let (t, k) = (e, 1usize);
- let topk_ids: Vec<i32> = (0..e as i32).collect();
- let expected_offsets: Vec<u32> = (0..=e as u32).collect();
-
- for_each_backend!(|B| {
- let (offsets, sum_k, partials) = get_output::<B>(&topk_ids, t, e, k);
- let backend_name = std::any::type_name::<B>();
- assert_eq!(offsets, expected_offsets, "offsets mismatch backend={}", backend_name);
- assert_eq!(sum_k, e as u32, "sum mismatch backend={}", backend_name);
- assert!(partials.iter().all(|&v| v == 1), "each expert should have count=1 backend={}", backend_name,);
- });
-}
-
-#[uzu_test]
-fn test_counts_offsets_fused_large_t() {
- // Stress test with many tokens
- let (t, e, k) = (2048usize, 128usize, 4usize);
- let topk_ids = gen_random_topk_ids(t, e, k, 42);
-
- let (offsets_ref, sum_ref, counts_ref) = get_output::<Cpu>(&topk_ids, t, e, k);
- for_each_non_cpu_backend!(|B| {
- let (offsets, sum_k, partials) = get_output::<B>(&topk_ids, t, e, k);
- let backend_name = std::any::type_name::<B>();
- assert_eq!(offsets, offsets_ref, "offsets mismatch backend={}", backend_name);
- assert_eq!(sum_k, sum_ref, "sum mismatch backend={}", backend_name);
- assert_eq!(partials, counts_ref, "partials mismatch backend={}", backend_name);
- });
-}
-
-#[uzu_test]
-fn test_counts_offsets_fused_offsets_monotonic() {
- // Verify offsets are monotonically non-decreasing for various shapes
- let shapes = vec![(10, 8, 2), (100, 32, 4), (5, 128, 1)];
-
- for (t, e, k) in shapes {
- let topk_ids = gen_random_topk_ids(t, e, k, 99);
-
- for_each_backend!(|B| {
- let (offsets, sum_k, _) = get_output::<B>(&topk_ids, t, e, k);
- let backend_name = std::any::type_name::<B>();
- for i in 1..offsets.len() {
- assert!(
- offsets[i] >= offsets[i - 1],
- "offsets not monotonic at i={} ({} < {}) T={} E={} K={} backend={}",
- i,
- offsets[i],
- offsets[i - 1],
- t,
- e,
- k,
- backend_name,
- );
- }
- assert_eq!(
- *offsets.last().unwrap(),
- sum_k,
- "last offset should equal sum_k T={} E={} K={} backend={}",
- t,
- e,
- k,
- backend_name,
- );
- });
- }
-}
diff --git a/crates/uzu-engine/unit/backends/common/kernel/moe/moe_finalize_test.rs b/crates/uzu-engine/unit/backends/common/kernel/moe/moe_finalize_test.rs
deleted file mode 100644
index 4ecb80859..000000000
--- a/crates/uzu-engine/unit/backends/common/kernel/moe/moe_finalize_test.rs
+++ /dev/null
@@ -1,121 +0,0 @@
-use half::bf16;
-use num_traits::Float;
-use rand::{RngExt, SeedableRng, rngs::StdRng};
-use uzu_engine_macros::uzu_test;
-
-use crate::{
- array::ArrayElement,
- backends::{
- common::{
- Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferPending, Context, Kernels,
- kernel::MoeFinalizeKernel,
- },
- cpu::Cpu,
- },
- data_type::DataType,
- tests::{
- assert::assert_eq_float,
- helpers::{buffer_to_vec, create_buffer, create_buffer_with_data, create_context, for_each_non_cpu_backend},
- },
-};
-
-struct Input<T: ArrayElement + Float> {
- tok2row: Box<[i32]>,
- probs: Box<[T]>,
- y_partial: Box<[T]>,
- t: usize,
- d_model: usize,
- k: usize,
-}
-
-fn get_output<B: Backend, T: ArrayElement + Float>(input: &Input<T>) -> Vec<T> {
- let context = create_context::<B>();
- let tok2row = create_buffer_with_data::<B, i32>(&context, &input.tok2row);
- let probs = create_buffer_with_data::<B, T>(&context, &input.probs);
- let y_partial = create_buffer_with_data::<B, T>(&context, &input.y_partial);
- let mut y_out = create_buffer::<B, T>(&context, input.t * input.d_model);
-
- let finalize = <B::Kernels as Kernels>::MoeFinalizeKernel::new(&context, DataType::BF16).expect("finalize kernel");
- let mut command_buffer = context.create_command_buffer(None, None).expect("Failed to create command buffer");
- finalize.encode(
- &tok2row,
- &probs,
- &y_partial,
- &mut y_out,
- input.t as u32,
- input.d_model as u32,
- input.k as u32,
- &mut command_buffer,
- );
- command_buffer.end_encoding().submit().wait_until_completed().unwrap();
-
- buffer_to_vec(&y_out)
-}
-
-fn test_finalize_internal(
- t: usize,
- k: usize,
- d_model: usize,
- sum_k: usize,
-) {
- let mut rng = StdRng::seed_from_u64(2026);
-
- // Generate random tok2row mapping: maps (token, k_idx) → row in y_partial
- // Some entries can be -1 (no expert selected)
- let mut tok2row: Vec<i32> = (0..t * k)
- .map(|_| {
- if rng.random_bool(0.9) {
- rng.random_range(0..sum_k as i32)
- } else {
- -1 // No expert selected
- }
- })
- .collect();
-
- // Ensure we use all rows in sum_k (avoid unused rows)
- for row in 0..sum_k.min(t * k) {
- tok2row[row] = row as i32;
- }
-
- // Generate random probabilities (should sum to 1 per token, but not critical for unit test)
- let probs: Vec<bf16> = (0..t * k).map(|_| bf16::from_f32(rng.random_range(0.0..1.0))).collect();
-
- // Generate random y_partial (expert outputs)
- let y_partial: Vec<bf16> = (0..sum_k * d_model).map(|_| bf16::from_f32(rng.random_range(-2.0..2.0))).collect();
-
- // CPU reference
- let input = Input {
- tok2row: tok2row.into_boxed_slice(),
- probs: probs.into_boxed_slice(),
- y_partial: y_partial.into_boxed_slice(),
- t,
- d_model,
- k,
- };
- let y_cpu = get_output::<Cpu, bf16>(&input);
-
- for_each_non_cpu_backend!(|B| {
- let y_gpu = get_output::<B, bf16>(&input);
- assert_eq_float(&y_cpu, &y_gpu, 1e-2, "finalize output");
- });
-}
-
-#[uzu_test]
-fn test_finalize_single_token() {
- test_finalize_internal(1, 2, 64, 2)
-}
-
-#[uzu_test]
-fn test_finalize_small_batch() {
- test_finalize_internal(4, 2, 128, 8)
-}
-
-#[uzu_test]
-fn test_finalize_medium() {
- test_finalize_internal(8, 4, 256, 32)
-}
-
-#[uzu_test]
-fn test_finalize_large() {
- test_finalize_internal(16, 2, 512, 32)
-}
diff --git a/crates/uzu-engine/unit/backends/common/kernel/moe/moe_router_topk_test.rs b/crates/uzu-engine/unit/backends/common/kernel/moe/moe_router_topk_test.rs
deleted file mode 100644
index bd0ba06cc..000000000
--- a/crates/uzu-engine/unit/backends/common/kernel/moe/moe_router_topk_test.rs
+++ /dev/null
@@ -1,201 +0,0 @@
-use std::fmt::Debug;
-
-use half::bf16;
-use num_traits::Float;
-use rand::{RngExt, SeedableRng, rngs::StdRng};
-use uzu_engine_macros::uzu_test;
-
-use crate::{
- array::ArrayElement,
- backends::{
- common::{
- Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferPending, Context, Kernels,
- kernel::MoeRouterTopKKernel,
- },
- cpu::Cpu,
- },
- tests::helpers::{buffer_to_vec, create_buffer, create_buffer_with_data, create_context, for_each_non_cpu_backend},
-};
-
-#[derive(Copy, Clone, Debug)]
-enum RouterInput {
- Plain,
- Biased,
- Gemma4,
-}
-
-fn get_output<B: Backend, T: ArrayElement + Float>(
- input: &[T],
- weights: &[T],
- bias: Option<&[T]>,
- router_scale: Option<&[T]>,
- per_expert_scale: Option<&[T]>,
- t: usize,
- d_model: usize,
- e: usize,
- k: usize,
- renorm: bool,
- router_norm_epsilon: Option<f32>,
- router_input_scale: Option<f32>,
-) -> (Vec<i32>, Vec<T>) {
- let ctx = create_context::<B>();
-
- let input_buffer = create_buffer_with_data::<B, T>(&ctx, input);
- let weights_buffer = create_buffer_with_data::<B, T>(&ctx, weights);
- let bias_buffer = bias.map(|bias| create_buffer_with_data::<B, T>(&ctx, bias));
- let router_scale_buffer = router_scale.map(|router_scale| create_buffer_with_data::<B, T>(&ctx, router_scale));
- let per_expert_scale_buffer =
- per_expert_scale.map(|per_expert_scale| create_buffer_with_data::<B, T>(&ctx, per_expert_scale));
- let mut ids = create_buffer::<B, i32>(&ctx, t * k);
- let mut probs = create_buffer::<B, T>(&ctx, t * k);
-
- let kernel = <<B as Backend>::Kernels as Kernels>::MoeRouterTopKKernel::new(
- &ctx,
- T::data_type(),
- bias_buffer.is_some(),
- router_scale_buffer.is_some(),
- per_expert_scale_buffer.is_some(),
- router_input_scale.is_some(),
- router_norm_epsilon.is_some(),
- )
- .expect("kernel");
- let mut command_buffer = ctx.create_command_buffer(None, None).expect("Failed to create command buffer");
- kernel.encode(
- &input_buffer,
- &weights_buffer,
- bias_buffer.as_ref(),
- router_scale_buffer.as_ref(),
- per_expert_scale_buffer.as_ref(),
- &mut ids,
- &mut probs,
- t as u32,
- d_model as u32,
- e as u32,
- k as u32,
- renorm,
- router_norm_epsilon,
- router_input_scale,
- &mut command_buffer,
- );
- command_buffer.end_encoding().submit().wait_until_completed().unwrap();
-
- (buffer_to_vec(&ids), buffer_to_vec(&probs))
-}
-
-fn run_router_topk_once<B: Backend, T: ArrayElement + Debug + Float>(
- t: usize,
- d_model: usize,
- e: usize,
- k: usize,
- renorm: bool,
- router_input: RouterInput,
-) {
- assert_eq!(d_model % 4, 0, "router_topk kernel processes d_model in 4-wide chunks");
-
- let mut rng = StdRng::seed_from_u64(1234);
- let gemma4 = matches!(router_input, RouterInput::Gemma4);
- let input: Vec<T> = (0..t * d_model).map(|_| T::from(rng.random_range(-1.0..1.0)).unwrap()).collect();
- let weight_limit = if gemma4 {
- 0.4
- } else {
- 1.0
- };
- let weight: Vec<T> =
- (0..e * d_model).map(|_| T::from(rng.random_range(-weight_limit..weight_limit)).unwrap()).collect();
- let bias: Option<Vec<T>> = matches!(router_input, RouterInput::Biased)
- .then(|| (0..e).map(|_| T::from(rng.random_range(-0.5..0.5)).unwrap()).collect());
- let router_scale: Option<Vec<T>> =
- gemma4.then(|| (0..d_model).map(|i| T::from(0.75 + (i % 7) as f32 * 0.05).unwrap()).collect());
- let per_expert_scale: Option<Vec<T>> =
- gemma4.then(|| (0..e).map(|i| T::from(0.5 + (i % 11) as f32 * 0.03).unwrap()).collect());
- let router_norm_epsilon = gemma4.then_some(1e-6);
- let router_input_scale = gemma4.then_some((d_model as f32).sqrt().recip());
-
- let (ids_ref, probs_ref) = get_output::<Cpu, T>(
- input.as_slice(),
- weight.as_slice(),
- bias.as_deref(),
- router_scale.as_deref(),
- per_expert_scale.as_deref(),
- t,
- d_model,
- e,
- k,
- renorm,
- router_norm_epsilon,
- router_input_scale,
- );
- let (ids_gpu, probs_gpu) = get_output::<B, T>(
- input.as_slice(),
- weight.as_slice(),
- bias.as_deref(),
- router_scale.as_deref(),
- per_expert_scale.as_deref(),
- t,
- d_model,
- e,
- k,
- renorm,
- router_norm_epsilon,
- router_input_scale,
- );
- assert_eq!(
- ids_gpu, ids_ref,
- "Top-k ids mismatch for T={}, d_model={}, E={}, K={}, renorm={}, router_input={:?}",
- t, d_model, e, k, renorm, router_input
- );
-
- for i in 0..(t * k) {
- let diff = (probs_gpu[i] - probs_ref[i]).abs().to_f32().unwrap();
-
- // Tolerance accounts for numerical differences in:
- // 1. Router matmul: GPU uses vec4 FMA + SIMD reductions, CPU uses scalar loops
- // With bf16 inputs and f32 accumulation, minor differences expected
- // 2. bf16 quantization on probs output buffer
- // 3. TopK selection (minor)
- // 4. Softmax computation when renorm=true (exp/div operations)
- let atol = if renorm {
- 2e-2 // Normalized probabilities with bf16: softmax + bf16 quantization
- } else {
- 5e-2 // Raw logits with bf16: vec4 accumulation + bf16 output quantization
- };
- assert!(
- diff <= atol,
- "Top-k prob mismatch at {}: gpu={} ref={} (diff={}, atol={}) with T={}, d_model={}, E={}, K={}, renorm={}, router_input={:?}",
- i,
- probs_gpu[i].to_f32().unwrap(),
- probs_ref[i].to_f32().unwrap(),
- diff,
- atol,
- t,
- d_model,
- e,
- k,
- renorm,
- router_input
- );
- }
-}
-
-#[uzu_test]
-fn test_router_topk_fused_matches_reference() {
- let configs: &[(usize, usize, usize, usize, RouterInput, &[bool])] = &[
- (1, 64, 32, 4, RouterInput::Plain, &[false, true]),
- (4, 256, 128, 16, RouterInput::Plain, &[false, true]),
- (1, 64, 32, 4, RouterInput::Biased, &[false, true]),
- (2, 128, 64, 8, RouterInput::Biased, &[false, true]),
- (4, 256, 128, 16, RouterInput::Biased, &[false, true]),
- (8, 256, 256, 32, RouterInput::Biased, &[false, true]),
- (1, 512, 512, 64, RouterInput::Biased, &[false, true]),
- (3, 256, 128, 8, RouterInput::Gemma4, &[true]),
- (1, 2816, 128, 8, RouterInput::Gemma4, &[true]),
- ];
-
- for_each_non_cpu_backend!(|B| {
- for &(t, d_model, e, k, router_input, renorm_options) in configs {
- for &renorm in renorm_options {
- run_router_topk_once::<B, bf16>(t, d_model, e, k, renorm, router_input);
- }
- }
- });
-}
diff --git a/crates/uzu-engine/unit/encodable_block/moe/mod.rs b/crates/uzu-engine/unit/encodable_block/moe/mod.rs
deleted file mode 100644
index 771f44831..000000000
--- a/crates/uzu-engine/unit/encodable_block/moe/mod.rs
+++ /dev/null
@@ -1,55 +0,0 @@
-use super::{
- experts_two_pass_decode::MoeExpertsTwoPassDecodeBlock,
- experts_two_pass_prefill::{MoeExpertsTwoPassArguments, MoeExpertsTwoPassPrefillBlock},
- gather::MoeGather,
-};
-
-mod moe_block_e2e_test;
-mod moe_experts_perf_test;
-mod moe_experts_test;
-mod moe_perf_test;
-mod moe_tiles_test;
-
-/// CPU reference for tile counts: count number of BM-sized tiles per expert
-///
-/// # Arguments
-/// * `offsets` - Expert segment offsets [E+1]
-/// * `bm` - Tile size (must match kernel BM constant)
-///
-/// # Returns
-/// Tile counts per expert [E]
-pub fn cpu_tile_counts(
- offsets: &[u32],
- bm: usize,
-) -> Vec<u32> {
- let e = offsets.len() - 1;
- let mut tile_counts = vec![0u32; e];
- for expert in 0..e {
- let seg_len = offsets[expert + 1] - offsets[expert];
- tile_counts[expert] = if seg_len == 0 {
- 0
- } else {
- (seg_len as usize).div_ceil(bm) as u32
- };
- }
- tile_counts
-}
-
-/// CPU reference for tile scan: exclusive prefix sum of tile counts
-///
-/// # Arguments
-/// * `tile_counts` - Tile counts per expert [E]
-///
-/// # Returns
-/// * Tile offsets [E+1] (exclusive prefix sum)
-/// * Total number of tiles
-pub fn cpu_tile_scan(tile_counts: &[u32]) -> (Vec<u32>, u32) {
- let e = tile_counts.len();
- let mut tile_offsets = Vec::with_capacity(e + 1);
- tile_offsets.push(0);
- for i in 0..e {
- tile_offsets.push(tile_offsets[i] + tile_counts[i]);
- }
- let total_tiles = *tile_offsets.last().unwrap();
- (tile_offsets, total_tiles)
-}
diff --git a/crates/uzu-engine/unit/encodable_block/moe/moe_block_e2e_test.rs b/crates/uzu-engine/unit/encodable_block/moe/moe_block_e2e_test.rs
deleted file mode 100644
index 96137bf72..000000000
--- a/crates/uzu-engine/unit/encodable_block/moe/moe_block_e2e_test.rs
+++ /dev/null
@@ -1,1148 +0,0 @@
-use half::bf16;
-use rand::{RngExt, SeedableRng, rngs::StdRng};
-use uzu_engine_macros::uzu_test;
-
-use super::{MoeExpertsTwoPassArguments, MoeExpertsTwoPassPrefillBlock, MoeGather};
-use crate::{
- backends::common::{
- Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferPending, Context, Kernels,
- gpu_types::{ActivationType, activation_silu_alpha},
- kernel::{
- MoeBlockBasesFromPartialsKernel, MoeCountsOffsetsFusedKernel, MoeFinalizeKernel, MoeRouterTopKKernel,
- MoeScatterBucketsMapKernel,
- },
- },
- data_type::DataType,
- tests::helpers::{
- buffer_prefix_to_vec, buffer_readback, buffer_to_vec, create_buffer, create_buffer_with_data, create_context,
- for_each_non_cpu_backend,
- },
-};
-
-fn moe_cpu_reference(
- x: &[bf16],
- router_weight: &[f32], // [E, d_model] - kept as F32 for router (computed before BF16 conversion)
- router_bias: &[f32], // [E]
- w13: &[bf16], // source layout [E, d_model, 2*d_ff] (GPU transposes to [E, 2*d_ff, d_model])
- w2: &[bf16], // [E, d_ff, d_model]
- up_biases: &[bf16], // [E, 2*d_ff]
- down_biases: &[bf16], // [E, d_model]
- t: usize,
- e: usize,
- k: usize,
- d_model: usize,
- d_ff: usize,
- gating_code: u32, // 0=GELU, 1=SiLU, 2=SwiGLU, 3=GEGLU
- silu_alpha: f32,
- gate_clip: (f32, f32),
- up_clip: (f32, f32),
-) -> Vec<f32> {
- let mut output = vec![0.0f32; t * d_model];
-
- for token_idx in 0..t {
- let x_start = token_idx * d_model;
-
- // Router: compute logits = x @ W_r^T + b_r
- // Router uses F32 precision (router_weight/router_bias are F32)
- let mut token_logits = vec![0.0f32; e];
- for expert_idx in 0..e {
- let mut logit = router_bias[expert_idx];
- for d in 0..d_model {
- let x_val = f32::from(x[x_start + d]);
- logit += x_val * router_weight[expert_idx * d_model + d];
- }
- token_logits[expert_idx] = logit;
- }
-
- // TopK selection
- let mut indices_and_logits: Vec<(usize, f32)> = token_logits.iter().enumerate().map(|(i, &v)| (i, v)).collect();
- indices_and_logits.sort_by(|a, b| b.1.partial_cmp(&a.1).unwrap());
- let top_k_indices: Vec<usize> = indices_and_logits[..k].iter().map(|&(i, _)| i).collect();
- let top_k_logits: Vec<f32> = indices_and_logits[..k].iter().map(|&(_, v)| v).collect();
-
- // Softmax normalization (matching Python: jax.nn.softmax)
- let max_logit = top_k_logits.iter().copied().fold(f32::NEG_INFINITY, f32::max);
- let exp_sum: f32 = top_k_logits.iter().map(|&v| (v - max_logit).exp()).sum();
- let weights: Vec<f32> = top_k_logits.iter().map(|&v| (v - max_logit).exp() / exp_sum).collect();
-
- // Process each selected expert (matching Python: vmap(run_expert))
- for (expert_idx, &weight) in top_k_indices.iter().zip(weights.iter()) {
- let x_start = token_idx * d_model;
-
- // Expert MLP weights layout: CPU uses W13[E, d_model, 2*d_ff], GPU stores as [E, 2*d_ff, d_model]; W2[E, d_ff, d_model]
- let w13_offset = expert_idx * d_model * 2 * d_ff;
- let w2_offset = expert_idx * d_ff * d_model;
- let bias_up_offset = expert_idx * 2 * d_ff;
- let bias_down_offset = expert_idx * d_model;
-
- // Up projection: W13[expert, input, output] where output=[up, gate]
- // Convert BF16→F32 on each access to match GPU precision
- let mut up_out = vec![0.0f32; d_ff];
- let mut gate_out = vec![0.0f32; d_ff];
-
- for ff_idx in 0..d_ff {
- let mut up_sum = f32::from(up_biases[bias_up_offset + ff_idx]);
- let mut gate_sum = f32::from(up_biases[bias_up_offset + d_ff + ff_idx]);
-
- for input_idx in 0..d_model {
- let w_base = w13_offset + input_idx * 2 * d_ff;
- let x_val = f32::from(x[x_start + input_idx]);
- let w_up = f32::from(w13[w_base + ff_idx]);
- let w_gate = f32::from(w13[w_base + d_ff + ff_idx]);
- up_sum += x_val * w_up;
- gate_sum += x_val * w_gate;
- }
-
- up_out[ff_idx] = up_sum;
- gate_out[ff_idx] = gate_sum;
- }
-
- // Apply clipping
- for i in 0..d_ff {
- gate_out[i] = gate_out[i].clamp(gate_clip.0, gate_clip.1);
- up_out[i] = up_out[i].clamp(up_clip.0, up_clip.1);
- }
-
- // Activation: depends on gating_code
- // 0=GELU(up), 1=SiLU(up), 2=SwiGLU(gate)*up, 3=GEGLU(gate)*up
- let mut hidden = vec![0.0f32; d_ff];
- for i in 0..d_ff {
- hidden[i] = match gating_code {
- 0 => ActivationType::GELUApprox.activate(up_out[i]),
- 1 => activation_silu_alpha(up_out[i], silu_alpha),
- 2 => activation_silu_alpha(gate_out[i], silu_alpha) * up_out[i],
- 3 => ActivationType::GELUApprox.activate(gate_out[i]) * up_out[i],
- _ => activation_silu_alpha(gate_out[i], silu_alpha) * up_out[i], // fallback to SwiGLU
- };
- }
-
- // Down projection: W2[expert, ff, output]
- // Convert BF16→F32 on each access to match GPU precision
- // CRITICAL: Two-pass kernel accumulates in F32, only quantizes at the very end
- for out_idx in 0..d_model {
- let mut fc2_sum = 0.0f32;
- for ff_idx in 0..d_ff {
- let w2_val = f32::from(w2[w2_offset + ff_idx * d_model + out_idx]);
- fc2_sum += hidden[ff_idx] * w2_val;
- }
- // Two-pass: keep in F32 (no intermediate quantization)
-
- // Add down bias in F32
- let down_bias_val = f32::from(down_biases[bias_down_offset + out_idx]);
- let with_bias = fc2_sum + down_bias_val;
-
- // Only quantize to BF16 once at the end (matches two-pass final write)
- let with_bias_bf16 = bf16::from_f32(with_bias);
- let final_val = f32::from(with_bias_bf16);
-
- output[x_start + out_idx] += weight * final_val;
- }
- }
- }
-
- output
-}
-
-// Main entry point - automatically tests both modes for T>1
-fn run_moe_parity_test<B: Backend>(
- ctx: &B::Context,
- t: usize,
- e: usize,
- k: usize,
- d_model: usize,
- d_ff: usize,
- gating_code: u32,
- silu_alpha: f32,
- gate_clip: (f32, f32),
- up_clip: (f32, f32),
- seed: u64,
- test_name: &str,
-) {
- if t > 1 {
- // Test 1-pass prefill (default, stable)
- run_moe_parity_test_internal::<B>(
- ctx,
- t,
- e,
- k,
- d_model,
- d_ff,
- gating_code,
- silu_alpha,
- gate_clip,
- up_clip,
- seed,
- &format!("{}_decode", test_name),
- );
- // Test 2-pass prefill (experimental)
- run_moe_parity_test_internal::<B>(
- ctx,
- t,
- e,
- k,
- d_model,
- d_ff,
- gating_code,
- silu_alpha,
- gate_clip,
- up_clip,
- seed,
- &format!("{}_prefill", test_name),
- );
- } else {
- // Decode mode (T=1) - no prefill variants
- run_moe_parity_test_internal::<B>(
- ctx,
- t,
- e,
- k,
- d_model,
- d_ff,
- gating_code,
- silu_alpha,
- gate_clip,
- up_clip,
- seed,
- test_name,
- );
- }
-}
-
-fn run_moe_parity_test_internal<B: Backend>(
- ctx: &B::Context,
- t: usize,
- e: usize,
- k: usize,
- d_model: usize,
- d_ff: usize,
- gating_code: u32, // 0=GELU, 1=SiLU, 2=SwiGLU, 3=GEGLU
- silu_alpha: f32,
- gate_clip: (f32, f32),
- up_clip: (f32, f32),
- seed: u64,
- test_name: &str,
-) {
- let mut rng = StdRng::seed_from_u64(seed);
-
- let prefill_mode = if t > 1 {
- "2-pass"
- } else {
- "decode"
- };
-
- eprintln!(
- "\n[{}] T={}, E={}, K={}, d_model={}, d_ff={}, alpha={}, gate_clip={:?}, up_clip={:?}, mode={}",
- test_name, t, e, k, d_model, d_ff, silu_alpha, gate_clip, up_clip, prefill_mode
- );
-
- // Random BF16 inputs
- let x: Vec<bf16> = (0..t * d_model).map(|_| bf16::from_f32(rng.random_range(-1.0..1.0))).collect();
-
- // Generate random router weights and biases for CPU reference
- let router_weight_f32: Vec<f32> = (0..e * d_model).map(|_| rng.random_range(-0.5..0.5)).collect();
- let router_bias_f32: Vec<f32> = (0..e).map(|_| rng.random_range(-0.1..0.1)).collect();
-
- // Convert router weights/bias to BF16 for GPU
- let router_weight_bf16: Vec<bf16> = router_weight_f32.iter().map(|&v| bf16::from_f32(v)).collect();
- let router_bias_bf16: Vec<bf16> = router_bias_f32.iter().map(|&v| bf16::from_f32(v)).collect();
-
- // Generate random expert weights and biases
- let w13_len = e * d_model * 2 * d_ff;
- let w2_len = e * d_ff * d_model;
-
- // Generate W13 in original layout [E, d_model, 2*d_ff] for CPU reference
- let w13_cpu: Vec<bf16> = (0..w13_len).map(|_| bf16::from_f32(rng.random_range(-0.5..0.5))).collect();
-
- // Transpose to GPU layout [E, 2*d_ff, d_model]
- let mut w13_gpu = vec![bf16::from_f32(0.0); w13_len];
- for expert in 0..e {
- let src_offset = expert * d_model * 2 * d_ff;
- let dst_offset = expert * 2 * d_ff * d_model;
- for dm in 0..d_model {
- for ff in 0..(2 * d_ff) {
- let src_idx = src_offset + dm * 2 * d_ff + ff;
- let dst_idx = dst_offset + ff * d_model + dm;
- w13_gpu[dst_idx] = w13_cpu[src_idx];
- }
- }
- }
-
- // Generate W2 in original layout [E, d_ff, d_model] for CPU reference
- let w2_cpu: Vec<bf16> = (0..w2_len).map(|_| bf16::from_f32(rng.random_range(-0.5..0.5))).collect();
-
- // Transpose W2 to GPU layout [E, d_model, d_ff]
- let mut w2_gpu = vec![bf16::from_f32(0.0); w2_len];
- for expert in 0..e {
- let expert_offset = expert * d_ff * d_model;
- for ff in 0..d_ff {
- for dm in 0..d_model {
- // src: [E, d_ff, d_model] -> index: expert_offset + ff * d_model + dm
- // dst: [E, d_model, d_ff] -> index: expert_offset + dm * d_ff + ff
- w2_gpu[expert_offset + dm * d_ff + ff] = w2_cpu[expert_offset + ff * d_model + dm];
- }
- }
- }
- let up_biases: Vec<bf16> = (0..e * 2 * d_ff).map(|_| bf16::from_f32(rng.random_range(-0.1..0.1))).collect();
- let down_biases: Vec<bf16> = (0..e * d_model).map(|_| bf16::from_f32(rng.random_range(-0.1..0.1))).collect();
-
- // Create Metal buffers
- let x_buf = create_buffer_with_data::<B, bf16>(ctx, &x);
- let router_w_buf = create_buffer_with_data::<B, bf16>(ctx, &router_weight_bf16);
- let router_b_buf = create_buffer_with_data::<B, bf16>(ctx, &router_bias_bf16);
- let w13_buf = create_buffer_with_data::<B, bf16>(ctx, &w13_gpu);
- let w2_buf = create_buffer_with_data::<B, bf16>(ctx, &w2_gpu);
- let up_biases_buf = create_buffer_with_data::<B, bf16>(ctx, &up_biases);
- let down_biases_buf = create_buffer_with_data::<B, bf16>(ctx, &down_biases);
-
- // Allocate intermediate buffers (max capacity)
- let max_sumk = t * k;
- let mut topk_ids_buf = create_buffer::<B, i32>(ctx, t * k);
- let mut topk_probs_buf = create_buffer::<B, bf16>(ctx, t * k);
- let mut offsets_buf = create_buffer::<B, u32>(ctx, e + 1);
- let mut sumk_buf = create_buffer::<B, u32>(ctx, 1);
- let num_blocks = t.div_ceil(256).max(1);
- let num_tiles = e.div_ceil(512).max(1);
- let entries = num_blocks * num_tiles * 512usize;
- let mut partials_buf = create_buffer::<B, u32>(ctx, entries);
- let mut block_bases_buf = create_buffer::<B, u32>(ctx, entries);
- let mut block_alloc_buf = create_buffer::<B, u32>(ctx, entries);
- let mut bucketed_ids_buf = create_buffer::<B, i32>(ctx, max_sumk);
- let mut bucketed_probs_buf = create_buffer::<B, bf16>(ctx, max_sumk);
- let mut tok2row_buf = create_buffer::<B, i32>(ctx, t * k);
- let mut y_out_buf = create_buffer::<B, bf16>(ctx, t * d_model);
-
- // Encode ALL kernels in one command buffer
- eprintln!("[E2E] Encoding entire MoE pipeline in single command buffer...");
- let mut command_buffer = ctx.create_command_buffer(None, None).expect("Failed to create command buffer");
-
- // Router + TopK (fused kernel)
- let router_topk =
- <B::Kernels as Kernels>::MoeRouterTopKKernel::new(ctx, DataType::BF16, true, false, false, false, false)
- .expect("router+topk");
- router_topk.encode(
- &x_buf,
- &router_w_buf,
- Some(&router_b_buf),
- None::<&B::GlobalBuffer>,
- None::<&B::GlobalBuffer>,
- &mut topk_ids_buf,
- &mut topk_probs_buf,
- t as u32,
- d_model as u32,
- e as u32,
- k as u32,
- true,
- None::<f32>,
- None::<f32>,
- &mut command_buffer,
- );
-
- let fused_kernel = <B::Kernels as Kernels>::MoeCountsOffsetsFusedKernel::new(ctx).expect("fused kernel");
- fused_kernel.encode(
- &topk_ids_buf,
- &mut offsets_buf,
- &mut sumk_buf,
- &mut partials_buf,
- t as u32,
- e as u32,
- k as u32,
- &mut command_buffer,
- );
-
- let scatter_bases_kernel =
- <B::Kernels as Kernels>::MoeBlockBasesFromPartialsKernel::new(ctx).expect("scatter bases kernel");
- scatter_bases_kernel.encode(
- &partials_buf,
- &mut block_bases_buf,
- &mut block_alloc_buf,
- e as u32,
- num_blocks as u32,
- num_tiles as u32,
- 0u32,
- &mut command_buffer,
- );
-
- let scatter_map_kernel = <B::Kernels as Kernels>::MoeScatterBucketsMapKernel::new(ctx, DataType::BF16)
- .expect("Failed to create <<Metal as Backend>::Kernels as Kernels>::MoeScatterBucketsMapKernel");
- scatter_map_kernel.encode(
- &topk_ids_buf,
- &topk_probs_buf,
- &offsets_buf,
- &block_bases_buf,
- &block_alloc_buf,
- &mut bucketed_ids_buf,
- &mut bucketed_probs_buf,
- t as u32,
- e as u32,
- k as u32,
- num_blocks as u32,
- num_tiles as u32,
- &mut tok2row_buf,
- &mut command_buffer,
- );
-
- let gather = MoeGather::<B>::new(ctx, DataType::BF16).expect("gather");
- let x_perm_buf = gather
- .encode(&x_buf, &bucketed_ids_buf, &sumk_buf, t as u32, k as u32, d_model as u32, &mut command_buffer)
- .expect("gather");
-
- let total_rows = t * k;
-
- let experts = MoeExpertsTwoPassPrefillBlock::<B>::new(ctx, DataType::BF16, gating_code).expect("experts");
- let args = MoeExpertsTwoPassArguments {
- x_perm: &x_perm_buf,
- expert_offsets: &offsets_buf,
- w13_all: &w13_buf,
- w2_all: &w2_buf,
- up_biases: &up_biases_buf,
- down_biases: &down_biases_buf,
- total_rows: total_rows as u32,
- d_model: d_model as u32,
- d_ff: d_ff as u32,
- num_routed_experts: e as u32,
- gate_clip_min: gate_clip.0,
- gate_clip_max: gate_clip.1,
- up_clip_min: up_clip.0,
- up_clip_max: up_clip.1,
- silu_alpha,
- };
- let y_partial_buf = experts.encode(args, &mut command_buffer).expect("failed to encode MoE experts");
-
- let finalize = <B::Kernels as Kernels>::MoeFinalizeKernel::new(ctx, DataType::BF16).expect("finalize");
- finalize.encode(
- &tok2row_buf,
- &topk_probs_buf,
- &y_partial_buf,
- &mut y_out_buf,
- t as u32,
- d_model as u32,
- k as u32,
- &mut command_buffer,
- );
-
- eprintln!("[E2E] All kernels encoded. Committing ONCE and waiting...");
- let completed = command_buffer.end_encoding().submit().wait_until_completed().unwrap();
- eprintln!("[E2E] GPU execution completed");
-
- // Read GPU output
- let y_out_bf16 = buffer_prefix_to_vec::<B, bf16>(&y_out_buf, t * d_model);
- let y_out_gpu: Vec<f32> = y_out_bf16.iter().map(|&v| f32::from(v)).collect();
-
- // Validate GPU output is finite
- let nan_count = y_out_gpu.iter().filter(|v| v.is_nan()).count();
- let inf_count = y_out_gpu.iter().filter(|v| v.is_infinite()).count();
-
- eprintln!("[E2E] GPU output stats: nan={}, inf={}", nan_count, inf_count);
- assert_eq!(nan_count, 0, "GPU output contains {} NaN values", nan_count);
- assert_eq!(inf_count, 0, "GPU output contains {} Inf values", inf_count);
-
- // Run CPU reference (router logits already computed above)
- // NOTE: Keep weights as BF16 and convert per-access to match GPU precision
- eprintln!("[{}] Running CPU reference implementation...", test_name);
-
- // Debug: Verify data consistency and tile bookkeeping for large-scale tests
- if d_model >= 512 {
- eprintln!(
- "[E2E] Large-scale debug: x[0]={:.6}, w13[0]={:.6}, w2[0]={:.6}",
- f32::from(x[0]),
- f32::from(w13_cpu[0]),
- f32::from(w2_cpu[0])
- );
-
- let sumk_val = buffer_to_vec::<B, u32>(&sumk_buf)[0] as usize;
- eprintln!("[E2E] sumk={}", sumk_val);
-
- // For multi-token tests with large d_ff, verify gather output (x_perm)
- if t > 1 && d_ff >= 256 {
- let x_perm_cpu =
- buffer_prefix_to_vec::<B, bf16>(&buffer_readback::<B>(ctx, &x_perm_buf), sumk_val * d_model);
- eprintln!("[E2E] x_perm diagnostics (sumk={}):", sumk_val);
- eprintln!("[E2E] Row 0 [0:8]: {:?}", x_perm_cpu[0..8].iter().map(|&v| f32::from(v)).collect::<Vec<_>>());
- if sumk_val > 1 {
- eprintln!(
- "[E2E] Row 1 [{}:{}]: {:?}",
- d_model,
- d_model + 8,
- x_perm_cpu[d_model..d_model + 8].iter().map(|&v| f32::from(v)).collect::<Vec<_>>()
- );
- }
-
- // CRITICAL: Check tok2row mapping for multi-token tests
- let tok2row_cpu = buffer_prefix_to_vec::<B, i32>(&tok2row_buf, t * k);
- eprintln!("[E2E] tok2row[0..{}]: {:?}", t * k, tok2row_cpu);
- eprintln!(
- "[E2E] Expected: token 0→row {}, token 1→row {}",
- 0,
- if sumk_val > 1 {
- 1
- } else {
- -1
- }
- );
-
- // Check y_partial at specific indices where finalize reads for token 1
- let y_partial_full =
- buffer_prefix_to_vec::<B, bf16>(&buffer_readback::<B>(ctx, &y_partial_buf), sumk_val * d_model);
- eprintln!("[E2E] y_partial spot check:");
- eprintln!("[E2E] y_partial[48] (row 0, col 48) = {:.6}", f32::from(y_partial_full[48]));
- if sumk_val > 1 {
- // Check positions where mismatches occur (odd indices near tile boundaries)
- eprintln!("[E2E] Row 1 positions (even=working, odd=corrupted?):");
- for &pos in &[48, 56, 57, 58, 59, 60, 61, 62, 63, 64, 120, 121, 122, 123, 124, 125, 126, 127, 128] {
- let idx = d_model + pos;
- if idx < y_partial_full.len() {
- eprintln!("[E2E] y_partial[row1, col {}] = {:.3}", pos, f32::from(y_partial_full[idx]));
- }
- }
- }
- }
- }
-
- let y_cpu = moe_cpu_reference(
- &x,
- &router_weight_f32,
- &router_bias_f32,
- &w13_cpu,
- &w2_cpu,
- &up_biases,
- &down_biases,
- t,
- e,
- k,
- d_model,
- d_ff,
- gating_code,
- silu_alpha,
- gate_clip,
- up_clip,
- );
-
- // Compare GPU vs CPU
- eprintln!("[E2E] Comparing GPU vs CPU outputs...");
- let mut max_abs_diff = 0.0f32;
- let mut max_rel_diff = 0.0f32;
- let mut total_abs_error = 0.0f32;
- let mut mismatches = 0;
-
- for i in 0..(t * d_model) {
- let gpu_val = y_out_gpu[i];
- let cpu_val = y_cpu[i];
- let abs_diff = (gpu_val - cpu_val).abs();
- let rel_diff = if cpu_val.abs() > 1e-6 {
- abs_diff / cpu_val.abs()
- } else {
- abs_diff
- };
-
- max_abs_diff = max_abs_diff.max(abs_diff);
- max_rel_diff = max_rel_diff.max(rel_diff);
- total_abs_error += abs_diff;
-
- let threshold_rel = 0.1;
- let threshold_abs = 1e-3;
-
- if rel_diff > threshold_rel && abs_diff > threshold_abs {
- let print_limit = if test_name.contains("BoundarySweep_D1024_FF256_T2") {
- usize::MAX // Print ALL for debugging
- } else {
- 10
- };
- if mismatches < print_limit {
- eprintln!(
- "[{}] Mismatch idx {}: GPU={:.6}, CPU={:.6}, abs={:.6}, rel={:.6}",
- test_name, i, gpu_val, cpu_val, abs_diff, rel_diff
- );
- }
- mismatches += 1;
- }
- }
-
- let mean_abs_error = total_abs_error / (t * d_model) as f32;
-
- eprintln!(
- "[E2E] Error metrics: max_abs={:.6}, max_rel={:.6}, mean_abs={:.6}, mismatches={}",
- max_abs_diff, max_rel_diff, mean_abs_error, mismatches
- );
-
- // Debug: print some sample values from intermediate buffers
- eprintln!("[E2E] === DEBUG: Intermediate values ===");
- let topk_ids_gpu = buffer_prefix_to_vec::<B, i32>(&topk_ids_buf, t * k);
- let topk_probs_gpu = buffer_prefix_to_vec::<B, bf16>(&topk_probs_buf, t * k);
- eprintln!("[E2E] TopK IDs: {:?}", topk_ids_gpu);
- eprintln!("[E2E] TopK Probs: {:?}", topk_probs_gpu.iter().map(|&v| f32::from(v)).collect::<Vec<_>>());
-
- let y_partial_gpu = buffer_prefix_to_vec::<B, bf16>(&buffer_readback::<B>(ctx, &y_partial_buf), max_sumk * d_model);
- let sumk_actual = buffer_to_vec::<B, u32>(&sumk_buf)[0] as usize;
- eprintln!("[E2E] sumk={}", sumk_actual);
- let sample_size = 16.min(sumk_actual * d_model);
- eprintln!(
- "[E2E] y_partial sample (first {}): {:?}",
- sample_size,
- y_partial_gpu[..sample_size].iter().map(|&v| f32::from(v)).collect::<Vec<_>>()
- );
-
- // For multi-row debugging: print both rows
- if sumk_actual > 1 && d_ff >= 256 {
- eprintln!(
- "[E2E] y_partial row 1 [{}-{}]: {:?}",
- d_model,
- d_model + 16,
- y_partial_gpu[d_model..d_model + 16].iter().map(|&v| f32::from(v)).collect::<Vec<_>>()
- );
- }
- let out_sample_size = 16.min(t * d_model);
- eprintln!("[E2E] y_out sample (token 0): {:?}", &y_out_gpu[..out_sample_size]);
- if t > 1 && d_model >= 512 {
- eprintln!("[E2E] y_out sample (token 1, first 32): {:?}", y_out_gpu[d_model..d_model + 32].to_vec());
- }
- eprintln!("[E2E] CPU ref sample (token 0): {:?}", &y_cpu[..out_sample_size]);
- if t > 1 && d_model >= 512 {
- eprintln!("[E2E] CPU ref sample (token 1): {:?}", &y_cpu[d_model..d_model + 16]);
-
- // Compare token 1 outputs element-wise (more positions to find pattern)
- let gpu_t1 = &y_out_gpu[d_model..];
- let cpu_t1 = &y_cpu[d_model..];
- eprintln!("[E2E] Token 1 detailed comparison:");
- for i in [0, 1, 2, 3, 4, 5, 6, 7, 15, 16, 17, 18, 48, 64, 121, 128].iter() {
- if *i < d_model {
- let diff = (gpu_t1[*i] - cpu_t1[*i]).abs();
- eprintln!("[E2E] [{}]: GPU={:.6}, CPU={:.6}, diff={:.6}", i, gpu_t1[*i], cpu_t1[*i], diff);
- }
- }
- }
-
- let k0_iters = d_model.div_ceil(32);
- let ff0_iters = d_ff.div_ceil(32);
- let product = k0_iters * ff0_iters;
- let sqrt_product = (product as f32).sqrt();
-
- let multi_expert_drift = 0.03 * sqrt_product * (k as f32 / 2.0);
- let mean_abs_threshold = 0.1f32.max(multi_expert_drift);
-
- if mean_abs_error >= mean_abs_threshold {
- panic!("[{}] Mean absolute error {:.6} exceeds threshold {:.6}", test_name, mean_abs_error, mean_abs_threshold);
- }
-
- // Warn on high max_rel but don't fail (can be inflated by near-zero values)
- if max_rel_diff > 1.0 && mismatches > (t * d_model / 4) {
- eprintln!(
- "[{}] WARNING: High max_rel={:.2} with {} mismatches, but mean_abs={:.6} is acceptable",
- test_name, max_rel_diff, mismatches, mean_abs_error
- );
- }
-
- eprintln!("[{}] ✓ PASSED (GPU matches CPU reference)", test_name);
- drop(x_perm_buf);
- drop(y_partial_buf);
- drop(completed);
-}
-
-// Test 1: Minimal (K=1, small dims, no clipping, alpha=1.0)
-#[uzu_test]
-fn test_moe_minimal() {
- for_each_non_cpu_backend!(|B| {
- let ctx = create_context::<B>();
- run_moe_parity_test::<B>(
- &ctx,
- 1, // t
- 2, // e
- 1, // k
- 4, // d_model
- 4, // d_ff
- 2, // gating_code (SwiGLU)
- 1.0, // silu_alpha
- (f32::NEG_INFINITY, f32::INFINITY), // gate_clip
- (f32::NEG_INFINITY, f32::INFINITY), // up_clip
- 0xE2E_0001,
- "Minimal_K1_SwiGLU",
- );
- })
-}
-
-// Test 2: Multi-expert (K=2, tests finalize weighted sum)
-#[uzu_test]
-fn test_moe_multi_expert() {
- for_each_non_cpu_backend!(|B| {
- let ctx = create_context::<B>();
- run_moe_parity_test::<B>(
- &ctx,
- 2, // t
- 4, // e
- 2, // k
- 8, // d_model
- 8, // d_ff
- 2, // gating_code (SwiGLU)
- 1.702, // silu_alpha (GPT-OSS value)
- (f32::NEG_INFINITY, 7.0), // gate_clip (GPT-OSS config)
- (-6.0, 8.0), // up_clip (GPT-OSS config)
- 0xE2E_0002,
- "Multi_K2_SwiGLU",
- );
- })
-}
-
-// Test 3: Large d_model only (K=1 to isolate from finalize complexity)
-#[uzu_test]
-fn test_moe_large_d_model_only() {
- for_each_non_cpu_backend!(|B| {
- let ctx = create_context::<B>();
- run_moe_parity_test::<B>(
- &ctx,
- 1, // t
- 2, // e
- 1, // k
- 64, // d_model (tests k0 chunking)
- 8, // d_ff (single ff0 chunk)
- 2, // gating_code (SwiGLU)
- 1.0,
- (f32::NEG_INFINITY, f32::INFINITY),
- (f32::NEG_INFINITY, f32::INFINITY),
- 0xE2E_0003,
- "LargeDModel64",
- );
- })
-}
-
-// Test 4a: d_ff=32 (exactly 1 full chunk - baseline for multi-chunk)
-#[uzu_test]
-fn test_1_full_chunk() {
- for_each_non_cpu_backend!(|B| {
- let ctx = create_context::<B>();
- run_moe_parity_test::<B>(
- &ctx,
- 1, // t
- 2, // e
- 1, // k
- 8, // d_model
- 32, // d_ff (exactly 1 full chunk)
- 2, // gating_code (SwiGLU)
- 1.0,
- (f32::NEG_INFINITY, f32::INFINITY),
- (f32::NEG_INFINITY, f32::INFINITY),
- 0xE2E_0004,
- "LargeFF32_SingleChunk",
- );
- })
-}
-
-// Test 4b: d_ff=48 (1.5 chunks - tests partial second chunk)
-#[uzu_test]
-fn test_1_5_chunks() {
- for_each_non_cpu_backend!(|B| {
- let ctx = create_context::<B>();
- run_moe_parity_test::<B>(
- &ctx,
- 1, // t
- 2, // e
- 1, // k
- 8, // d_model
- 48, // d_ff (1 full + 1 partial chunk)
- 2, // gating_code (SwiGLU)
- 1.0,
- (f32::NEG_INFINITY, f32::INFINITY),
- (f32::NEG_INFINITY, f32::INFINITY),
- 0xE2E_0005,
- "LargeFF48_Partial",
- );
- })
-}
-
-// Test 4c: d_ff=64 (exactly 2 full chunks)
-#[uzu_test]
-fn test_2_full_chunks() {
- for_each_non_cpu_backend!(|B| {
- let ctx = create_context::<B>();
- run_moe_parity_test::<B>(
- &ctx,
- 1, // t
- 2, // e
- 1, // k
- 8, // d_model
- 64, // d_ff (exactly 2 full chunks)
- 2, // gating_code (SwiGLU)
- 1.0,
- (f32::NEG_INFINITY, f32::INFINITY),
- (f32::NEG_INFINITY, f32::INFINITY),
- 0xE2E_0006,
- "LargeFF64_TwoChunks",
- );
- })
-}
-
-// Test 5: GELU activation (gating_code=0)
-#[uzu_test]
-fn test_gelu_activation() {
- for_each_non_cpu_backend!(|B| {
- let ctx = create_context::<B>();
- run_moe_parity_test::<B>(
- &ctx,
- 1, // t
- 2, // e
- 1, // k
- 8, // d_model
- 16, // d_ff
- 0, // gating_code (GELU)
- 1.0, // silu_alpha (unused for GELU)
- (f32::NEG_INFINITY, f32::INFINITY),
- (f32::NEG_INFINITY, f32::INFINITY),
- 0xE2E_0007,
- "GELU_Activation",
- );
- })
-}
-
-// Test 6: SiLU activation (gating_code=1)
-#[uzu_test]
-fn test_silu_activation() {
- for_each_non_cpu_backend!(|B| {
- let ctx = create_context::<B>();
- run_moe_parity_test::<B>(
- &ctx,
- 1, // t
- 2, // e
- 1, // k
- 8, // d_model
- 16, // d_ff
- 1, // gating_code (SiLU)
- 1.702,
- (f32::NEG_INFINITY, f32::INFINITY),
- (f32::NEG_INFINITY, f32::INFINITY),
- 0xE2E_0008,
- "SiLU_Activation",
- );
- })
-}
-
-// Test 7: GEGLU activation (gating_code=3)
-#[uzu_test]
-fn test_geglu_activation() {
- for_each_non_cpu_backend!(|B| {
- let ctx = create_context::<B>();
- run_moe_parity_test::<B>(
- &ctx,
- 1, // t
- 2, // e
- 1, // k
- 8, // d_model
- 16, // d_ff
- 3, // gating_code (GEGLU)
- 1.0, // silu_alpha (unused for GEGLU)
- (f32::NEG_INFINITY, f32::INFINITY),
- (f32::NEG_INFINITY, f32::INFINITY),
- 0xE2E_0009,
- "GEGLU_Activation",
- );
- })
-}
-
-// Test 8: Bucket stress - multiple tokens, larger E, stress scatter/gather
-#[uzu_test]
-fn test_bucket_stress() {
- for_each_non_cpu_backend!(|B| {
- let ctx = create_context::<B>();
- run_moe_parity_test::<B>(
- &ctx,
- 8, // t (8 tokens)
- 8, // e (8 experts)
- 2, // k (each token picks 2 experts)
- 16, // d_model
- 32, // d_ff
- 2, // gating_code (SwiGLU)
- 1.0,
- (f32::NEG_INFINITY, f32::INFINITY),
- (f32::NEG_INFINITY, f32::INFINITY),
- 0xE2E_0010,
- "BucketStress_T8_E8_K2",
- );
- })
-}
-
-// Test 9a: K=4 small-scale to verify multi-expert accumulation
-#[uzu_test]
-fn test_small_scale() {
- for_each_non_cpu_backend!(|B| {
- let ctx = create_context::<B>();
- run_moe_parity_test::<B>(
- &ctx,
- 2, // t
- 4, // e
- 4, // k (all experts)
- 8, // d_model
- 16, // d_ff
- 2, // gating_code (SwiGLU)
- 1.0,
- (f32::NEG_INFINITY, f32::INFINITY),
- (f32::NEG_INFINITY, f32::INFINITY),
- 0xE2E_0011,
- "Multi_K4_Small",
- );
- })
-}
-
-// Test 9b: d_model=128 (4 k0 chunks, 2 n-tiles)
-#[uzu_test]
-fn test_model_128() {
- for_each_non_cpu_backend!(|B| {
- let ctx = create_context::<B>();
- run_moe_parity_test::<B>(
- &ctx,
- 2, // t
- 4, // e
- 1, // k
- 128, // d_model (4 k0 chunks, 2 n-tiles)
- 32, // d_ff
- 2, // gating_code (SwiGLU)
- 1.0,
- (f32::NEG_INFINITY, f32::INFINITY),
- (f32::NEG_INFINITY, f32::INFINITY),
- 0xE2E_0012,
- "MultiNTile_D128",
- );
- })
-}
-
-// Test 9c: d_model=192 (6 k0 chunks, 3 n-tiles - test threshold)
-#[uzu_test]
-fn test_model_192() {
- for_each_non_cpu_backend!(|B| {
- let ctx = create_context::<B>();
- run_moe_parity_test::<B>(
- &ctx,
- 2, // t
- 4, // e
- 1, // k
- 192, // d_model (6 k0 chunks, 3 n-tiles)
- 32, // d_ff
- 2, // gating_code (SwiGLU)
- 1.0,
- (f32::NEG_INFINITY, f32::INFINITY),
- (f32::NEG_INFINITY, f32::INFINITY),
- 0xE2E_0013,
- "MultiNTile_D192_3tiles",
- );
- })
-}
-
-// Test 9d: d_model=256 (8 k0 chunks, 4 n-tiles)
-#[uzu_test]
-fn test_model_256() {
- for_each_non_cpu_backend!(|B| {
- let ctx = create_context::<B>();
- run_moe_parity_test::<B>(
- &ctx,
- 2, // t
- 4, // e
- 1, // k
- 256, // d_model (8 k0 chunks, 4 n-tiles)
- 32, // d_ff
- 2, // gating_code (SwiGLU)
- 1.0,
- (f32::NEG_INFINITY, f32::INFINITY),
- (f32::NEG_INFINITY, f32::INFINITY),
- 0xE2E_0014,
- "MultiNTile_D256_4tiles",
- );
- })
-}
-
-// Test 10a: Isolate large d_ff with small d_model (FF accumulation test)
-#[uzu_test]
-fn test_ff_accumulation() {
- for_each_non_cpu_backend!(|B| {
- let ctx = create_context::<B>();
- run_moe_parity_test::<B>(
- &ctx,
- 2, // t
- 4, // e
- 1, // k
- 64, // d_model (small, 1 n-tile, 2 k0 chunks)
- 14336, // d_ff (448 ff0 chunks!) - production scale
- 2, // gating_code (SwiGLU)
- 1.0,
- (f32::NEG_INFINITY, f32::INFINITY),
- (f32::NEG_INFINITY, f32::INFINITY),
- 0xE2E_0016,
- "IsolateFF_D64_FF14336_K1",
- );
- })
-}
-
-// Test 10b: d_model=1024 (16 n-tiles)
-#[uzu_test]
-fn test_model_1024() {
- for_each_non_cpu_backend!(|B| {
- let ctx = create_context::<B>();
- run_moe_parity_test::<B>(
- &ctx,
- 2, // t
- 4, // e
- 1, // k
- 1024, // d_model (32 k0 chunks, 16 n-tiles)
- 64, // d_ff (small)
- 2, // gating_code (SwiGLU)
- 1.0,
- (f32::NEG_INFINITY, f32::INFINITY),
- (f32::NEG_INFINITY, f32::INFINITY),
- 0xE2E_0017,
- "IsolateD_D1024_FF64_K1",
- );
- })
-}
-
-// Test 10c: Verify layout with d_model=96, d_ff=96 (3 chunks each)
-// This tests non-power-of-2 to catch stride bugs
-#[uzu_test]
-fn test_stride() {
- for_each_non_cpu_backend!(|B| {
- let ctx = create_context::<B>();
- run_moe_parity_test::<B>(
- &ctx,
- 1, // t (single token)
- 2, // e
- 1, // k
- 96, // d_model (3 k0 chunks - odd number)
- 96, // d_ff (3 ff0 chunks - odd number)
- 2, // gating_code (SwiGLU)
- 1.0,
- (f32::NEG_INFINITY, f32::INFINITY),
- (f32::NEG_INFINITY, f32::INFINITY),
- 0xE2E_0018,
- "VerifyLayout_D96_FF96",
- );
- })
-}
-
-// Test 10d: d_model=1536
-#[uzu_test]
-fn test_model_1536() {
- for_each_non_cpu_backend!(|B| {
- let ctx = create_context::<B>();
- run_moe_parity_test::<B>(
- &ctx,
- 2, // t
- 4, // e
- 1, // k
- 1536, // d_model (48 k0 chunks, 24 n-tiles)
- 64, // d_ff (small)
- 2, // gating_code (SwiGLU)
- 1.0,
- (f32::NEG_INFINITY, f32::INFINITY),
- (f32::NEG_INFINITY, f32::INFINITY),
- 0xE2E_0019,
- "IsolateD_D1536_FF64_K1",
- );
- })
-}
-
-// Test 10e: 2 n-tiles
-#[uzu_test]
-fn test_2n_tiles() {
- for_each_non_cpu_backend!(|B| {
- let ctx = create_context::<B>();
- run_moe_parity_test::<B>(
- &ctx,
- 1, // t
- 2, // e
- 1, // k
- 128, // d_model (2 n-tiles)
- 256, // d_ff
- 2, // gating_code (SwiGLU)
- 1.0,
- (f32::NEG_INFINITY, f32::INFINITY),
- (f32::NEG_INFINITY, f32::INFINITY),
- 0xE2E_0020,
- "NTileTest_D128_2tiles",
- );
- })
-}
-
-// Test 10g: 8 n-tiles - find threshold
-#[uzu_test]
-fn test_8n_tiles() {
- for_each_non_cpu_backend!(|B| {
- let ctx = create_context::<B>();
- run_moe_parity_test::<B>(
- &ctx,
- 1, // t
- 2, // e
- 1, // k
- 512, // d_model (8 n-tiles)
- 256, // d_ff
- 2, // gating_code (SwiGLU)
- 1.0,
- (f32::NEG_INFINITY, f32::INFINITY),
- (f32::NEG_INFINITY, f32::INFINITY),
- 0xE2E_0021,
- "NTileTest_D512_8tiles",
- );
- })
-}
-
-// Test 11b: Production scale with clipping but K=1
-#[uzu_test]
-#[ignore]
-fn test_prod_scale_clipping() {
- for_each_non_cpu_backend!(|B| {
- let ctx = create_context::<B>();
- run_moe_parity_test::<B>(
- &ctx,
- 2, // t
- 8, // e
- 1, // k
- 4096, // d_model
- 14336, // d_ff
- 2, // gating_code (SwiGLU)
- 1.702, // silu_alpha (GPT-OSS value)
- (f32::NEG_INFINITY, 20.0), // gate_clip_max
- (-19.0, 21.0), // up_clip
- 0xE2E_0029,
- "ProductionScale_D4096_H14336_E8_K1",
- );
- })
-}
-
-// Test 11c: PRODUCTION SCALE T=1 decode (triggers GEMV v2)
-#[uzu_test]
-#[ignore]
-fn test_prod_scale_decode() {
- for_each_non_cpu_backend!(|B| {
- let ctx = create_context::<B>();
- run_moe_parity_test::<B>(
- &ctx,
- 1, // t (single token decode)
- 16, // e (16 experts)
- 2, // k (2 experts per token)
- 4096, // d_model
- 14336, // d_ff
- 2, // gating_code (SwiGLU)
- 1.702, // silu_alpha (GPT-OSS value)
- (f32::NEG_INFINITY, 20.0), // gate_clip_max (GPT-OSS config)
- (-19.0, 21.0), // up_clip (GPT-OSS config)
- 0xE2E_0030,
- "ProductionScale_T1_D4096_H14336_E16_K2",
- );
- })
-}
-
-// Test 11d: FULL PRODUCTION SCALE with K=2, T=4 (uses tiled MMA)
-#[uzu_test]
-#[ignore]
-fn test_prod_scale_tile_mma() {
- for_each_non_cpu_backend!(|B| {
- let ctx = create_context::<B>();
- run_moe_parity_test::<B>(
- &ctx,
- 4, // t (4 tokens)
- 16, // e (16 experts)
- 2, // k (2 experts per token)
- 4096, // d_model (128 k0 chunks, 64 n-tiles!)
- 14336, // d_ff (448 ff0 chunks!)
- 2, // gating_code (SwiGLU)
- 1.702, // silu_alpha (GPT-OSS value)
- (f32::NEG_INFINITY, 20.0), // gate_clip_max (GPT-OSS config)
- (-19.0, 21.0), // up_clip (GPT-OSS config)
- 0xE2E_0031,
- "ProductionScale_D4096_H14336_E16_K2",
- );
- })
-}
diff --git a/crates/uzu-engine/unit/encodable_block/moe/moe_experts_perf_test.rs b/crates/uzu-engine/unit/encodable_block/moe/moe_experts_perf_test.rs
deleted file mode 100644
index d0b2acb22..000000000
--- a/crates/uzu-engine/unit/encodable_block/moe/moe_experts_perf_test.rs
+++ /dev/null
@@ -1,300 +0,0 @@
-use std::time::Instant;
-
-use half::bf16;
-use rand::{RngExt, SeedableRng, rngs::StdRng};
-use uzu_engine_macros::uzu_test;
-
-use super::{MoeExpertsTwoPassArguments, MoeExpertsTwoPassDecodeBlock, MoeExpertsTwoPassPrefillBlock};
-use crate::{
- backends::common::{Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferPending, Context},
- data_type::DataType,
- tests::helpers::{create_buffer_with_data, create_context, for_each_non_cpu_backend},
-};
-
-fn build_offsets(
- e: usize,
- sum_k: usize,
-) -> Vec<u32> {
- let mut offsets = vec![0u32; e + 1];
- let mut running = 0u32;
- let mut remaining = sum_k;
- for expert in 0..e {
- offsets[expert] = running;
- if remaining > 0 {
- running += 1;
- remaining -= 1;
- }
- }
- offsets[e] = running;
- offsets
-}
-
-fn run_decode_case<B: Backend>(
- ctx: &B::Context,
- name: &str,
- t: usize,
- d_model: usize,
- d_ff: usize,
- e: usize,
- k: usize,
- warmup: usize,
- iters: usize,
-) {
- let mut rng = StdRng::seed_from_u64(0xDEC0DE1234567890);
- let sum_k = t * k;
-
- eprintln!("\n[decode/two-pass] {} => T={}, D={}, FF={}, E={}, K={}, sum_k={}", name, t, d_model, d_ff, e, k, sum_k);
-
- let x_perm: Vec<bf16> = (0..sum_k * d_model).map(|_| bf16::from_f32(rng.random_range(-1.0..1.0))).collect();
- let offsets = build_offsets(e, sum_k);
-
- // Generate W13 in original layout [E, d_model, 2*d_ff]
- let w13_original: Vec<bf16> =
- (0..e * d_model * 2 * d_ff).map(|_| bf16::from_f32(rng.random_range(-0.05..0.05))).collect();
-
- // Transpose to GPU layout [E, 2*d_ff, d_model]
- let mut w13 = vec![bf16::from_f32(0.0); e * d_model * 2 * d_ff];
- for expert in 0..e {
- let src_offset = expert * d_model * 2 * d_ff;
- let dst_offset = expert * 2 * d_ff * d_model;
- for dm in 0..d_model {
- for ff in 0..(2 * d_ff) {
- let src_idx = src_offset + dm * 2 * d_ff + ff;
- let dst_idx = dst_offset + ff * d_model + dm;
- w13[dst_idx] = w13_original[src_idx];
- }
- }
- }
- let w2: Vec<bf16> = (0..e * d_ff * d_model).map(|_| bf16::from_f32(rng.random_range(-0.05..0.05))).collect();
- let up_biases: Vec<bf16> = (0..e * 2 * d_ff).map(|_| bf16::from_f32(rng.random_range(-0.01..0.01))).collect();
- let down_biases: Vec<bf16> = (0..e * d_model).map(|_| bf16::from_f32(rng.random_range(-0.01..0.01))).collect();
-
- let experts_kernel = MoeExpertsTwoPassDecodeBlock::<B>::new(ctx, DataType::BF16, 2).expect("experts decode kernel");
-
- let x_perm_buf = create_buffer_with_data::<B, bf16>(ctx, &x_perm);
- let offsets_buf = create_buffer_with_data::<B, u32>(ctx, &offsets);
- let w13_buf = create_buffer_with_data::<B, bf16>(ctx, &w13);
- let w2_buf = create_buffer_with_data::<B, bf16>(ctx, &w2);
- let up_biases_buf = create_buffer_with_data::<B, bf16>(ctx, &up_biases);
- let down_biases_buf = create_buffer_with_data::<B, bf16>(ctx, &down_biases);
-
- for _ in 0..warmup {
- let mut command_buffer = ctx.create_command_buffer(None, None).expect("Failed to create command buffer");
- let output = experts_kernel
- .encode(
- MoeExpertsTwoPassArguments {
- x_perm: &x_perm_buf,
- expert_offsets: &offsets_buf,
- w13_all: &w13_buf,
- w2_all: &w2_buf,
- up_biases: &up_biases_buf,
- down_biases: &down_biases_buf,
- total_rows: sum_k as u32,
- d_model: d_model as u32,
- d_ff: d_ff as u32,
- num_routed_experts: e as u32,
- gate_clip_min: f32::NEG_INFINITY,
- gate_clip_max: f32::INFINITY,
- up_clip_min: f32::NEG_INFINITY,
- up_clip_max: f32::INFINITY,
- silu_alpha: 1.702,
- },
- &mut command_buffer,
- )
- .expect("failed to encode MoE experts");
- let completed = command_buffer.end_encoding().submit().wait_until_completed().unwrap();
- drop(output);
- drop(completed);
- }
-
- let mut times = Vec::with_capacity(iters);
- for _ in 0..iters {
- let start = Instant::now();
- let mut command_buffer = ctx.create_command_buffer(None, None).expect("Failed to create command buffer");
- let output = experts_kernel
- .encode(
- MoeExpertsTwoPassArguments {
- x_perm: &x_perm_buf,
- expert_offsets: &offsets_buf,
- w13_all: &w13_buf,
- w2_all: &w2_buf,
- up_biases: &up_biases_buf,
- down_biases: &down_biases_buf,
- total_rows: sum_k as u32,
- d_model: d_model as u32,
- d_ff: d_ff as u32,
- num_routed_experts: e as u32,
- gate_clip_min: f32::NEG_INFINITY,
- gate_clip_max: f32::INFINITY,
- up_clip_min: f32::NEG_INFINITY,
- up_clip_max: f32::INFINITY,
- silu_alpha: 1.702,
- },
- &mut command_buffer,
- )
- .expect("failed to encode MoE experts");
- let completed = command_buffer.end_encoding().submit().wait_until_completed().unwrap();
- drop(output);
- drop(completed);
- times.push(start.elapsed().as_secs_f64() * 1000.0);
- }
-
- times.sort_by(|a, b| a.partial_cmp(b).unwrap());
- let mean = times.iter().sum::<f64>() / times.len() as f64;
- let median = times[times.len() / 2];
- let min = times[0];
- let max = times[times.len() - 1];
- let var = times.iter().map(|t| (t - mean).powi(2)).sum::<f64>() / times.len() as f64;
- let std = var.sqrt();
-
- eprintln!(" mean={:.3}ms median={:.3}ms min={:.3}ms max={:.3}ms std={:.3}ms", mean, median, min, max, std);
- eprintln!(" → Latency: {:.1} µs/token", (mean / t as f64) * 1000.0);
-}
-
-fn run_two_pass_prefill_case<B: Backend>(
- ctx: &B::Context,
- name: &str,
- t: usize,
- d_model: usize,
- d_ff: usize,
- e: usize,
- k: usize,
- warmup: usize,
- iters: usize,
-) {
- let mut rng = StdRng::seed_from_u64(0xDEC0DE1234567890);
- let sum_k = t * k;
-
- eprintln!("\n[2-pass prefill] {} => T={}, D={}, FF={}, E={}, K={}, sum_k={}", name, t, d_model, d_ff, e, k, sum_k);
-
- let x_perm: Vec<bf16> = (0..sum_k * d_model).map(|_| bf16::from_f32(rng.random_range(-1.0..1.0))).collect();
- let offsets = build_offsets(e, sum_k);
-
- let w13_original: Vec<bf16> =
- (0..e * d_model * 2 * d_ff).map(|_| bf16::from_f32(rng.random_range(-0.05..0.05))).collect();
-
- let mut w13 = vec![bf16::from_f32(0.0); e * d_model * 2 * d_ff];
- for expert in 0..e {
- let src_offset = expert * d_model * 2 * d_ff;
- let dst_offset = expert * 2 * d_ff * d_model;
- for dm in 0..d_model {
- for ff in 0..(2 * d_ff) {
- let src_idx = src_offset + dm * 2 * d_ff + ff;
- let dst_idx = dst_offset + ff * d_model + dm;
- w13[dst_idx] = w13_original[src_idx];
- }
- }
- }
- let w2: Vec<bf16> = (0..e * d_ff * d_model).map(|_| bf16::from_f32(rng.random_range(-0.05..0.05))).collect();
- let up_biases: Vec<bf16> = (0..e * 2 * d_ff).map(|_| bf16::from_f32(rng.random_range(-0.01..0.01))).collect();
- let down_biases: Vec<bf16> = (0..e * d_model).map(|_| bf16::from_f32(rng.random_range(-0.01..0.01))).collect();
-
- let experts_kernel =
- MoeExpertsTwoPassPrefillBlock::<B>::new(ctx, DataType::BF16, 2).expect("experts prefill kernel");
-
- let x_perm_buf = create_buffer_with_data::<B, bf16>(ctx, &x_perm);
- let offsets_buf = create_buffer_with_data::<B, u32>(ctx, &offsets);
- let w13_buf = create_buffer_with_data::<B, bf16>(ctx, &w13);
- let w2_buf = create_buffer_with_data::<B, bf16>(ctx, &w2);
- let up_biases_buf = create_buffer_with_data::<B, bf16>(ctx, &up_biases);
- let down_biases_buf = create_buffer_with_data::<B, bf16>(ctx, &down_biases);
-
- for _ in 0..warmup {
- let mut command_buffer = ctx.create_command_buffer(None, None).expect("Failed to create command buffer");
- let args = MoeExpertsTwoPassArguments {
- x_perm: &x_perm_buf,
- expert_offsets: &offsets_buf,
- w13_all: &w13_buf,
- w2_all: &w2_buf,
- up_biases: &up_biases_buf,
- down_biases: &down_biases_buf,
- total_rows: sum_k as u32,
- d_model: d_model as u32,
- d_ff: d_ff as u32,
- num_routed_experts: e as u32,
- gate_clip_min: f32::NEG_INFINITY,
- gate_clip_max: f32::INFINITY,
- up_clip_min: f32::NEG_INFINITY,
- up_clip_max: f32::INFINITY,
- silu_alpha: 1.702,
- };
- let output = experts_kernel.encode(args, &mut command_buffer).expect("failed to encode MoE experts");
- let completed = command_buffer.end_encoding().submit().wait_until_completed().unwrap();
- drop(output);
- drop(completed);
- }
-
- let mut times = Vec::with_capacity(iters);
- for _ in 0..iters {
- let start = Instant::now();
- let mut command_buffer = ctx.create_command_buffer(None, None).expect("Failed to create command buffer");
- let args = MoeExpertsTwoPassArguments {
- x_perm: &x_perm_buf,
- expert_offsets: &offsets_buf,
- w13_all: &w13_buf,
- w2_all: &w2_buf,
- up_biases: &up_biases_buf,
- down_biases: &down_biases_buf,
- total_rows: sum_k as u32,
- d_model: d_model as u32,
- d_ff: d_ff as u32,
- num_routed_experts: e as u32,
- gate_clip_min: f32::NEG_INFINITY,
- gate_clip_max: f32::INFINITY,
- up_clip_min: f32::NEG_INFINITY,
- up_clip_max: f32::INFINITY,
- silu_alpha: 1.702,
- };
- let output = experts_kernel.encode(args, &mut command_buffer).expect("failed to encode MoE experts");
- let completed = command_buffer.end_encoding().submit().wait_until_completed().unwrap();
- drop(output);
- drop(completed);
- times.push(start.elapsed().as_secs_f64() * 1000.0);
- }
-
- times.sort_by(|a, b| a.partial_cmp(b).unwrap());
- let mean = times.iter().sum::<f64>() / times.len() as f64;
- let median = times[times.len() / 2];
- let min = times[0];
- let max = times[times.len() - 1];
- let var = times.iter().map(|t| (t - mean).powi(2)).sum::<f64>() / times.len() as f64;
- let std = var.sqrt();
-
- eprintln!(" mean={:.3}ms median={:.3}ms min={:.3}ms max={:.3}ms std={:.3}ms", mean, median, min, max, std);
- eprintln!(" → Throughput: {:.1} µs/token (mean / sum_k)", (mean / sum_k as f64) * 1000.0);
-}
-
-#[uzu_test]
-#[ignore]
-fn test_two_pass_decode_speed() {
- for_each_non_cpu_backend!(|B| {
- let ctx = create_context::<B>();
-
- let cases = vec![
- ("T1_E16_D1024_F4096", 1, 1024, 4096, 16, 4, 3, 20),
- ("T1_E8_D2048_F8192", 1, 2048, 8192, 8, 4, 3, 20),
- ];
-
- for (name, t, d_model, d_ff, e, k, warmup, iters) in &cases {
- run_decode_case::<B>(&ctx, name, *t, *d_model, *d_ff, *e, *k, *warmup, *iters);
- }
- });
-}
-
-#[uzu_test]
-#[ignore]
-fn test_two_pass_prefill_speed() {
- for_each_non_cpu_backend!(|B| {
- let ctx = create_context::<B>();
-
- let cases = vec![
- ("T32_E16_D1024_F4096", 32, 1024, 4096, 16, 4, 2, 10),
- ("T64_E16_D2048_F6144", 64, 2048, 6144, 16, 4, 2, 10),
- ("T256_E16_D2048_F6144", 256, 2048, 6144, 16, 4, 2, 10),
- ];
-
- for (name, t, d_model, d_ff, e, k, warmup, iters) in cases {
- run_two_pass_prefill_case::<B>(&ctx, name, t, d_model, d_ff, e, k, warmup, iters);
- }
- })
-}
diff --git a/crates/uzu-engine/unit/encodable_block/moe/moe_experts_test.rs b/crates/uzu-engine/unit/encodable_block/moe/moe_experts_test.rs
deleted file mode 100644
index 08e5c1f4e..000000000
--- a/crates/uzu-engine/unit/encodable_block/moe/moe_experts_test.rs
+++ /dev/null
@@ -1,638 +0,0 @@
-//! Unit correctness tests for MoE Expert kernels (both 1-pass and 2-pass variants)
-//!
-//! Tests verify:
-//! - Decode (suffix_length=1) with 2-pass tiled implementation
-//! - Prefill (suffix_length>1) with 2-pass tiled implementation
-//! - Intermediate buffer correctness (row maps, tiles, dispatch args)
-//! - Numerical correctness against CPU reference
-
-use half::bf16;
-use rand::{RngExt, SeedableRng, rngs::StdRng};
-use uzu_engine_macros::uzu_test;
-
-use super::{
- MoeExpertsTwoPassArguments, MoeExpertsTwoPassDecodeBlock, MoeExpertsTwoPassPrefillBlock, cpu_tile_counts,
- cpu_tile_scan,
-};
-use crate::{
- backends::common::{
- CommandBufferEncoding, CommandBufferExecutable, CommandBufferPending, Context,
- gpu_types::{ActivationType, activation_silu_alpha},
- },
- data_type::DataType,
- tests::{
- assert::assert_eq_float,
- helpers::{
- buffer_prefix_to_vec, buffer_readback, create_buffer_with_data, create_context, for_each_non_cpu_backend,
- },
- },
-};
-
-/// Test data for MoE experts
-struct MoeTestData {
- x: Vec<bf16>,
- topk_ids: Vec<i32>,
- topk_probs: Vec<bf16>,
- w13: Vec<bf16>,
- w2: Vec<bf16>,
- up_biases: Vec<bf16>,
- down_biases: Vec<bf16>,
-}
-
-impl MoeTestData {
- fn generate(
- rng: &mut StdRng,
- t: usize,
- k: usize,
- d_model: usize,
- d_ff: usize,
- e: usize,
- ) -> Self {
- let x: Vec<bf16> = (0..t * d_model).map(|_| bf16::from_f32(rng.random_range(-1.0..1.0))).collect();
-
- let topk_ids: Vec<i32> = (0..t * k).map(|i| ((i / k) + (i % k) * 2) as i32 % e as i32).collect();
-
- let topk_probs: Vec<bf16> = (0..t)
- .flat_map(|_| {
- let raw: Vec<f32> = (0..k).map(|_| rng.random_range(0.1..1.0)).collect();
- let sum: f32 = raw.iter().sum();
- raw.iter().map(|p| bf16::from_f32(p / sum)).collect::<Vec<_>>()
- })
- .collect();
-
- let w13: Vec<bf16> =
- (0..e * 2 * d_ff * d_model).map(|_| bf16::from_f32(rng.random_range(-0.05..0.05))).collect();
- let w2: Vec<bf16> = (0..e * d_ff * d_model).map(|_| bf16::from_f32(rng.random_range(-0.05..0.05))).collect();
- let up_biases: Vec<bf16> = (0..e * 2 * d_ff).map(|_| bf16::from_f32(rng.random_range(-0.01..0.01))).collect();
- let down_biases: Vec<bf16> = (0..e * d_model).map(|_| bf16::from_f32(rng.random_range(-0.01..0.01))).collect();
-
- Self {
- x,
- topk_ids,
- topk_probs,
- w13,
- w2,
- up_biases,
- down_biases,
- }
- }
-}
-
-/// Result of scatter operation - buckets rows by expert
-struct ScatterResult {
- x_perm: Vec<bf16>,
- offsets: Vec<u32>,
- perm_idx: Vec<usize>, // original row -> bucketed row (for gather)
-}
-
-fn scatter_by_expert(
- x: &[bf16],
- topk_ids: &[i32],
- t: usize,
- k: usize,
- d_model: usize,
- e: usize,
-) -> ScatterResult {
- let sum_k = t * k;
-
- // Count rows per expert and build offsets
- let mut expert_counts = vec![0usize; e];
- for &eid in topk_ids {
- expert_counts[eid as usize] += 1;
- }
- let mut offsets = vec![0u32; e + 1];
- for i in 0..e {
- offsets[i + 1] = offsets[i] + expert_counts[i] as u32;
- }
-
- // Build permutation indices
- let mut expert_cursors = vec![0usize; e];
- let mut perm_idx = vec![0usize; sum_k];
- let mut inv_perm = vec![0usize; sum_k];
- for orig_row in 0..sum_k {
- let eid = topk_ids[orig_row] as usize;
- let bucket_pos = offsets[eid] as usize + expert_cursors[eid];
- perm_idx[orig_row] = bucket_pos;
- inv_perm[bucket_pos] = orig_row;
- expert_cursors[eid] += 1;
- }
-
- // Build x_perm in bucketed order
- let mut x_perm = vec![bf16::from_f32(0.0); sum_k * d_model];
- for bucket_row in 0..sum_k {
- let orig_row = inv_perm[bucket_row];
- let tok = orig_row / k;
- for d in 0..d_model {
- x_perm[bucket_row * d_model + d] = x[tok * d_model + d];
- }
- }
-
- ScatterResult {
- x_perm,
- offsets,
- perm_idx,
- }
-}
-
-/// Gather + finalize: reorder from bucketed order and apply weighted sum
-fn gather_and_finalize(
- y_partial: &[bf16],
- topk_probs: &[bf16],
- perm_idx: &[usize],
- t: usize,
- k: usize,
- d_model: usize,
-) -> Vec<bf16> {
- let mut y = vec![bf16::from_f32(0.0); t * d_model];
- for tok in 0..t {
- for d in 0..d_model {
- let mut acc = 0.0f32;
- for ki in 0..k {
- let orig_row = tok * k + ki;
- let bucket_row = perm_idx[orig_row];
- let prob = f32::from(topk_probs[orig_row]);
- let val = f32::from(y_partial[bucket_row * d_model + d]);
- acc += prob * val;
- }
- y[tok * d_model + d] = bf16::from_f32(acc);
- }
- }
- y
-}
-
-/// Unified CPU reference for MoE computation
-///
-/// Computes: y[t] = Σ_k prob[t,k] * (activation(x[t] @ W13[expert_k]) @ W2[expert_k] + bias)
-///
-/// Supports 4 gating modes:
-/// - 0: GELU(up)
-/// - 1: SiLU(up)
-/// - 2: SwiGLU = SiLU(gate) * up
-/// - 3: GEGLU = GELU(gate) * up
-fn cpu_moe_reference(
- x: &[bf16], // [T, d_model]
- topk_ids: &[i32], // [T * K]
- topk_probs: &[bf16], // [T * K]
- w13_all: &[bf16], // [E, 2*d_ff, d_model] transposed
- w2_all: &[bf16], // [E, d_model, d_ff] transposed
- up_biases: &[bf16], // [E, 2*d_ff]
- down_biases: &[bf16], // [E, d_model]
- t: usize,
- d_model: usize,
- d_ff: usize,
- k: usize,
- gating_code: u32,
- silu_alpha: f32,
- gate_clip_min: f32,
- gate_clip_max: f32,
- up_clip_min: f32,
- up_clip_max: f32,
-) -> Vec<bf16> {
- let mut y = vec![bf16::from_f32(0.0); t * d_model];
-
- // Process each token
- for tok in 0..t {
- let x_offset = tok * d_model;
-
- // Hidden buffer for this token's K experts [K, d_ff]
- let mut hidden = vec![0.0f32; k * d_ff];
-
- // Pass A: x @ W13[expert] -> hidden[k] with activation
- for k_idx in 0..k {
- let expert_id = topk_ids[tok * k + k_idx];
- if expert_id < 0 {
- continue;
- }
- let expert = expert_id as usize;
-
- let w13_base = expert * 2 * d_ff * d_model;
- let bias_base = expert * 2 * d_ff;
-
- for h in 0..d_ff {
- // Up projection
- let mut acc_up = f32::from(up_biases[bias_base + h]);
- for d in 0..d_model {
- let x_val = f32::from(x[x_offset + d]);
- let w_val = f32::from(w13_all[w13_base + h * d_model + d]);
- acc_up += x_val * w_val;
- }
-
- // Apply activation based on gating_code
- let activated = if gating_code <= 1 {
- // GELU or SiLU on up (with clipping)
- let up_val = acc_up.clamp(up_clip_min, up_clip_max);
- if gating_code == 0 {
- ActivationType::GELUApprox.activate(up_val)
- } else {
- activation_silu_alpha(up_val, silu_alpha)
- }
- } else {
- // SwiGLU or GEGLU - need gate projection too
- let mut acc_gate = f32::from(up_biases[bias_base + d_ff + h]);
- for d in 0..d_model {
- let x_val = f32::from(x[x_offset + d]);
- let w_val = f32::from(w13_all[w13_base + (d_ff + h) * d_model + d]);
- acc_gate += x_val * w_val;
- }
-
- // Apply clipping
- let up_val = acc_up.clamp(up_clip_min, up_clip_max);
- let gate_val = acc_gate.clamp(gate_clip_min, gate_clip_max);
-
- let gate_act = if gating_code == 2 {
- activation_silu_alpha(gate_val, silu_alpha)
- } else {
- ActivationType::GELUApprox.activate(gate_val)
- };
- gate_act * up_val
- };
-
- hidden[k_idx * d_ff + h] = activated;
- }
- }
-
- // Pass B: hidden @ W2 -> y (with weighted sum)
- for d in 0..d_model {
- let mut final_acc = 0.0f32;
-
- for k_idx in 0..k {
- let expert_id = topk_ids[tok * k + k_idx];
- if expert_id < 0 {
- continue;
- }
- let expert = expert_id as usize;
- let prob = f32::from(topk_probs[tok * k + k_idx]);
-
- let w2_base = expert * d_model * d_ff;
- let bias_base = expert * d_model;
-
- let mut acc = f32::from(down_biases[bias_base + d]);
- for h in 0..d_ff {
- let h_val = hidden[k_idx * d_ff + h];
- let w_val = f32::from(w2_all[w2_base + d * d_ff + h]);
- acc += h_val * w_val;
- }
- final_acc += prob * acc;
- }
-
- y[tok * d_model + d] = bf16::from_f32(final_acc);
- }
- }
-
- y
-}
-
-#[uzu_test]
-fn test_two_pass_decode_correctness() {
- for_each_non_cpu_backend!(|B| {
- let ctx = create_context::<B>();
- let mut rng = StdRng::seed_from_u64(0xDEC0DE42);
-
- // End-to-end test: T=1 token, K=2 experts per token
- let t = 1;
- let k = 2;
- let sum_k = t * k;
- let d_model = 512;
- let d_ff = 2048;
- let e = 8;
- let gating_code = 2u32; // SwiGLU
- let silu_alpha = 1.0f32;
-
- eprintln!("[2-pass decode] T={}, K={}, sum_k={}, d_model={}, d_ff={}, E={}", t, k, sum_k, d_model, d_ff, e);
-
- // Generate input x [T, d_model]
- let x: Vec<bf16> = (0..t * d_model).map(|_| bf16::from_f32(rng.random_range(-1.0..1.0))).collect();
-
- // Generate routing: topk_ids [T*K], topk_probs [T*K]
- let topk_ids: Vec<i32> = (0..t * k).map(|i| ((i / k) + (i % k) * 2) as i32 % e as i32).collect();
- let topk_probs: Vec<bf16> = (0..t)
- .flat_map(|_| {
- let raw: Vec<f32> = (0..k).map(|_| rng.random_range(0.1..1.0)).collect();
- let sum: f32 = raw.iter().sum();
- raw.iter().map(|p| bf16::from_f32(p / sum)).collect::<Vec<_>>()
- })
- .collect();
-
- // Scatter x based on topk_ids to get x_perm
- // Each token contributes K rows, one per selected expert
- let mut x_perm = vec![bf16::from_f32(0.0); sum_k * d_model];
- for tok in 0..t {
- for ki in 0..k {
- let row_idx = tok * k + ki;
- for d in 0..d_model {
- x_perm[row_idx * d_model + d] = x[tok * d_model + d];
- }
- }
- }
-
- let mut expert_counts = vec![0usize; e];
- for &eid in &topk_ids {
- expert_counts[eid as usize] += 1;
- }
- let mut offsets = vec![0u32; e + 1];
- for i in 0..e {
- offsets[i + 1] = offsets[i] + expert_counts[i] as u32;
- }
-
- // Generate weights
- let w13: Vec<bf16> =
- (0..e * 2 * d_ff * d_model).map(|_| bf16::from_f32(rng.random_range(-0.05..0.05))).collect();
- let w2: Vec<bf16> = (0..e * d_ff * d_model).map(|_| bf16::from_f32(rng.random_range(-0.05..0.05))).collect();
- let up_biases: Vec<bf16> = (0..e * 2 * d_ff).map(|_| bf16::from_f32(rng.random_range(-0.01..0.01))).collect();
- let down_biases: Vec<bf16> = (0..e * d_model).map(|_| bf16::from_f32(rng.random_range(-0.01..0.01))).collect();
-
- // CPU reference - end-to-end
- let y_expected = cpu_moe_reference(
- &x,
- &topk_ids,
- &topk_probs,
- &w13,
- &w2,
- &up_biases,
- &down_biases,
- t,
- d_model,
- d_ff,
- k,
- gating_code,
- silu_alpha,
- f32::NEG_INFINITY,
- f32::INFINITY,
- f32::NEG_INFINITY,
- f32::INFINITY,
- );
-
- // Prepare GPU buffers
- let x_perm_buf = create_buffer_with_data::<B, bf16>(&ctx, &x_perm);
- let offsets_buf = create_buffer_with_data::<B, u32>(&ctx, &offsets);
- let w13_buf = create_buffer_with_data::<B, bf16>(&ctx, &w13);
- let w2_buf = create_buffer_with_data::<B, bf16>(&ctx, &w2);
- let up_biases_buf = create_buffer_with_data::<B, bf16>(&ctx, &up_biases);
- let down_biases_buf = create_buffer_with_data::<B, bf16>(&ctx, &down_biases);
-
- let experts_kernel = MoeExpertsTwoPassDecodeBlock::<B>::new(&ctx, DataType::BF16, gating_code)
- .expect("MoeExpertsTwoPassDecodeKernel::new");
- let mut command_buffer = ctx.create_command_buffer(None, None).expect("Failed to create command buffer");
-
- let y_partial_buf = experts_kernel
- .encode(
- MoeExpertsTwoPassArguments {
- x_perm: &x_perm_buf,
- expert_offsets: &offsets_buf,
- w13_all: &w13_buf,
- w2_all: &w2_buf,
- up_biases: &up_biases_buf,
- down_biases: &down_biases_buf,
- total_rows: sum_k as u32,
- d_model: d_model as u32,
- d_ff: d_ff as u32,
- num_routed_experts: e as u32,
- gate_clip_min: f32::NEG_INFINITY,
- gate_clip_max: f32::INFINITY,
- up_clip_min: f32::NEG_INFINITY,
- up_clip_max: f32::INFINITY,
- silu_alpha,
- },
- &mut command_buffer,
- )
- .expect("failed to encode MoE experts");
-
- let completed = command_buffer.end_encoding().submit().wait_until_completed().unwrap();
-
- // Read GPU partial output and do CPU finalize (weighted sum)
- let y_partial_gpu =
- buffer_prefix_to_vec::<B, bf16>(&buffer_readback::<B>(&ctx, &y_partial_buf), sum_k * d_model);
-
- // Finalize: y[t] = Σ_k prob[t,k] * y_partial[t*k + k_idx]
- let mut y_gpu = vec![bf16::from_f32(0.0); t * d_model];
- for tok in 0..t {
- for d in 0..d_model {
- let mut acc = 0.0f32;
- for ki in 0..k {
- let row_idx = tok * k + ki;
- let prob = f32::from(topk_probs[row_idx]);
- let val = f32::from(y_partial_gpu[row_idx * d_model + d]);
- acc += prob * val;
- }
- y_gpu[tok * d_model + d] = bf16::from_f32(acc);
- }
- }
-
- // Compute error metrics
- let mut max_abs_error = 0.0f32;
- let mut max_idx = 0;
-
- for (i, (&gpu_val, &cpu_val)) in y_gpu.iter().zip(y_expected.iter()).enumerate() {
- let abs_error = (f32::from(gpu_val) - f32::from(cpu_val)).abs();
- if abs_error > max_abs_error {
- max_abs_error = abs_error;
- max_idx = i;
- }
- }
-
- eprintln!(
- "[2-pass decode] Max error: {:.6} at idx {} (GPU={:.6}, CPU={:.6})",
- max_abs_error,
- max_idx,
- f32::from(y_gpu[max_idx]),
- f32::from(y_expected[max_idx])
- );
-
- let tolerance = 0.02;
- assert_eq_float(&y_expected, &y_gpu, tolerance, "2-pass decode output");
-
- eprintln!("[2-pass decode] ✓ PASSED (tolerance={:.4})", tolerance);
- drop(y_partial_buf);
- drop(completed);
- });
-}
-
-#[uzu_test]
-fn test_two_pass_decode_multi_token() {
- for_each_non_cpu_backend!(|B| {
- let ctx = create_context::<B>();
- let mut rng = StdRng::seed_from_u64(0xDEC0DE43);
-
- let t = 4;
- let k = 2;
- let sum_k = t * k;
- let d_model = 512;
- let d_ff = 2048;
- let e = 8;
- let gating_code = 2u32;
- let silu_alpha = 1.0f32;
-
- eprintln!("[2-pass decode multi-token] T={}, K={}, d_model={}, d_ff={}, E={}", t, k, d_model, d_ff, e);
-
- let data = MoeTestData::generate(&mut rng, t, k, d_model, d_ff, e);
- let scatter = scatter_by_expert(&data.x, &data.topk_ids, t, k, d_model, e);
-
- let y_expected = cpu_moe_reference(
- &data.x,
- &data.topk_ids,
- &data.topk_probs,
- &data.w13,
- &data.w2,
- &data.up_biases,
- &data.down_biases,
- t,
- d_model,
- d_ff,
- k,
- gating_code,
- silu_alpha,
- f32::NEG_INFINITY,
- f32::INFINITY,
- f32::NEG_INFINITY,
- f32::INFINITY,
- );
-
- // GPU buffers
- let x_perm_buf = create_buffer_with_data::<B, bf16>(&ctx, &scatter.x_perm);
- let offsets_buf = create_buffer_with_data::<B, u32>(&ctx, &scatter.offsets);
- let w13_buf = create_buffer_with_data::<B, bf16>(&ctx, &data.w13);
- let w2_buf = create_buffer_with_data::<B, bf16>(&ctx, &data.w2);
- let up_biases_buf = create_buffer_with_data::<B, bf16>(&ctx, &data.up_biases);
- let down_biases_buf = create_buffer_with_data::<B, bf16>(&ctx, &data.down_biases);
-
- let experts_kernel = MoeExpertsTwoPassDecodeBlock::<B>::new(&ctx, DataType::BF16, gating_code).expect("kernel");
- let mut command_buffer = ctx.create_command_buffer(None, None).expect("Failed to create command buffer");
- let y_partial_buf = experts_kernel
- .encode(
- MoeExpertsTwoPassArguments {
- x_perm: &x_perm_buf,
- expert_offsets: &offsets_buf,
- w13_all: &w13_buf,
- w2_all: &w2_buf,
- up_biases: &up_biases_buf,
- down_biases: &down_biases_buf,
- total_rows: sum_k as u32,
- d_model: d_model as u32,
- d_ff: d_ff as u32,
- num_routed_experts: e as u32,
- gate_clip_min: f32::NEG_INFINITY,
- gate_clip_max: f32::INFINITY,
- up_clip_min: f32::NEG_INFINITY,
- up_clip_max: f32::INFINITY,
- silu_alpha,
- },
- &mut command_buffer,
- )
- .expect("failed to encode MoE experts");
- let completed = command_buffer.end_encoding().submit().wait_until_completed().unwrap();
-
- let y_partial_gpu =
- buffer_prefix_to_vec::<B, bf16>(&buffer_readback::<B>(&ctx, &y_partial_buf), sum_k * d_model);
- let y_gpu = gather_and_finalize(&y_partial_gpu, &data.topk_probs, &scatter.perm_idx, t, k, d_model);
-
- assert_eq_float(&y_expected, &y_gpu, 0.02, "2-pass decode multi-token");
- eprintln!("[2-pass decode multi-token] ✓ PASSED");
- drop(y_partial_buf);
- drop(completed);
- });
-}
-
-#[uzu_test]
-fn test_two_pass_prefill_correctness() {
- for_each_non_cpu_backend!(|B| {
- let ctx = create_context::<B>();
- let mut rng = StdRng::seed_from_u64(0xFE1111);
-
- let t = 8;
- let k = 2;
- let sum_k = t * k;
- let d_model = 512;
- let d_ff = 2048;
- let e = 8;
- let gating_code = 2u32;
- let silu_alpha = 1.0f32;
-
- eprintln!("[2-pass prefill] T={}, K={}, d_model={}, d_ff={}, E={}", t, k, d_model, d_ff, e);
-
- let data = MoeTestData::generate(&mut rng, t, k, d_model, d_ff, e);
- let scatter = scatter_by_expert(&data.x, &data.topk_ids, t, k, d_model, e);
-
- let y_expected = cpu_moe_reference(
- &data.x,
- &data.topk_ids,
- &data.topk_probs,
- &data.w13,
- &data.w2,
- &data.up_biases,
- &data.down_biases,
- t,
- d_model,
- d_ff,
- k,
- gating_code,
- silu_alpha,
- f32::NEG_INFINITY,
- f32::INFINITY,
- f32::NEG_INFINITY,
- f32::INFINITY,
- );
-
- // GPU buffers
- let x_perm_buf = create_buffer_with_data::<B, bf16>(&ctx, &scatter.x_perm);
- let offsets_buf = create_buffer_with_data::<B, u32>(&ctx, &scatter.offsets);
- let w13_buf = create_buffer_with_data::<B, bf16>(&ctx, &data.w13);
- let w2_buf = create_buffer_with_data::<B, bf16>(&ctx, &data.w2);
- let up_biases_buf = create_buffer_with_data::<B, bf16>(&ctx, &data.up_biases);
- let down_biases_buf = create_buffer_with_data::<B, bf16>(&ctx, &data.down_biases);
-
- let experts_kernel =
- MoeExpertsTwoPassPrefillBlock::<B>::new(&ctx, DataType::BF16, gating_code).expect("kernel");
- let mut command_buffer = ctx.create_command_buffer(None, None).expect("Failed to create command buffer");
- let args = MoeExpertsTwoPassArguments {
- x_perm: &x_perm_buf,
- expert_offsets: &offsets_buf,
- w13_all: &w13_buf,
- w2_all: &w2_buf,
- up_biases: &up_biases_buf,
- down_biases: &down_biases_buf,
- total_rows: sum_k as u32,
- d_model: d_model as u32,
- d_ff: d_ff as u32,
- num_routed_experts: e as u32,
- gate_clip_min: f32::NEG_INFINITY,
- gate_clip_max: f32::INFINITY,
- up_clip_min: f32::NEG_INFINITY,
- up_clip_max: f32::INFINITY,
- silu_alpha,
- };
- let y_partial_buf = experts_kernel.encode(args, &mut command_buffer).expect("failed to encode MoE experts");
- let completed = command_buffer.end_encoding().submit().wait_until_completed().unwrap();
-
- let y_partial_gpu =
- buffer_prefix_to_vec::<B, bf16>(&buffer_readback::<B>(&ctx, &y_partial_buf), sum_k * d_model);
- let y_gpu = gather_and_finalize(&y_partial_gpu, &data.topk_probs, &scatter.perm_idx, t, k, d_model);
-
- assert_eq_float(&y_expected, &y_gpu, 0.02, "2-pass prefill");
- eprintln!("[2-pass prefill] ✓ PASSED");
- drop(y_partial_buf);
- drop(completed);
- });
-}
-
-#[uzu_test]
-fn test_tile_infrastructure() {
- // Test that tile counts/scan/map are computed correctly
- // This is already tested in moe_tiles_test.rs, but we verify here
- // that the tiles match our expectations for expert workload distribution
-
- let offsets = vec![0, 5, 20, 35, 40];
- const BM: usize = 16;
-
- let tile_counts = cpu_tile_counts(&offsets, BM);
- let (tile_offsets, total_tiles) = cpu_tile_scan(&tile_counts);
-
- // Expert 0: 5 rows -> 1 tile
- // Expert 1: 15 rows -> 1 tile
- // Expert 2: 15 rows -> 1 tile
- // Expert 3: 5 rows -> 1 tile
- assert_eq!(tile_counts, vec![1, 1, 1, 1]);
- assert_eq!(tile_offsets, vec![0, 1, 2, 3, 4]);
- assert_eq!(total_tiles, 4);
-
- eprintln!("[tile infrastructure] ✓ PASSED");
-}
diff --git a/crates/uzu-engine/unit/encodable_block/moe/moe_gather_test.rs b/crates/uzu-engine/unit/encodable_block/moe/moe_gather_test.rs
deleted file mode 100644
index 0e94bc8c5..000000000
--- a/crates/uzu-engine/unit/encodable_block/moe/moe_gather_test.rs
+++ /dev/null
@@ -1,97 +0,0 @@
-use half::bf16;
-use num_traits::Float;
-use rand::{RngExt, SeedableRng, rngs::StdRng};
-use uzu_engine_macros::uzu_test;
-
-use super::MoeGather;
-use crate::{
- array::ArrayElement,
- backends::{
- common::{Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferPending, Context},
- cpu::Cpu,
- },
- tests::{
- assert::assert_eq_float,
- helpers::{buffer_readback, buffer_to_vec, create_buffer_with_data, create_context, for_each_non_cpu_backend},
- },
-};
-
-struct Input<T: ArrayElement + Float> {
- x: Box<[T]>,
- bucket_ids: Box<[i32]>,
- t: usize,
- sum_k: usize,
- d_model: usize,
-}
-
-fn get_output<B: Backend, T: ArrayElement + Float>(input: &Input<T>) -> Vec<T> {
- let context = create_context::<B>();
-
- let sumk_data: [u32; 1] = [input.sum_k as u32];
- let x_buffer = create_buffer_with_data::<B, T>(&context, &input.x);
- let ids_buffer = create_buffer_with_data::<B, i32>(&context, &input.bucket_ids);
- let sumk_buffer = create_buffer_with_data::<B, u32>(&context, &sumk_data);
- let gather = MoeGather::<B>::new(&context, T::data_type()).expect("MoeGather::new");
- let mut command_buffer = context.create_command_buffer(None, None).expect("Failed to create command buffer");
- let x_perm_buffer = gather
- .encode(
- &x_buffer,
- &ids_buffer,
- &sumk_buffer,
- input.t as u32,
- (input.sum_k / input.t) as u32,
- input.d_model as u32,
- &mut command_buffer,
- )
- .expect("Failed to encode MoE gather");
- let completed = command_buffer.end_encoding().submit().wait_until_completed().unwrap();
-
- let output = buffer_to_vec::<B, T>(&buffer_readback::<B>(&context, &x_perm_buffer));
- drop(x_perm_buffer);
- drop(completed);
- output
-}
-
-fn test_gather_internal(
- t: usize,
- sum_k: usize,
- d_model: usize,
-) {
- let mut rng = StdRng::seed_from_u64(2027);
- let x: Vec<bf16> = (0..t * d_model).map(|_| bf16::from_f32(rng.random_range(-2.0..2.0))).collect();
- let bucketed_ids: Vec<i32> = (0..sum_k).map(|_| rng.random_range(0..t as i32)).collect();
-
- let input = Input::<bf16> {
- x: x.clone().into_boxed_slice(),
- bucket_ids: bucketed_ids.clone().into_boxed_slice(),
- t,
- sum_k,
- d_model,
- };
-
- let cpu_ref_output = get_output::<Cpu, bf16>(&input);
- for_each_non_cpu_backend!(|B| {
- let output = get_output::<B, bf16>(&input);
- assert_eq_float(&cpu_ref_output, &output, 1e-6, "Moe gather");
- })
-}
-
-#[uzu_test]
-fn test_gather_single_token() {
- test_gather_internal(1, 2, 64);
-}
-
-#[uzu_test]
-fn test_gather_small_batch() {
- test_gather_internal(4, 8, 128);
-}
-
-#[uzu_test]
-fn test_gather_medium_batch() {
- test_gather_internal(16, 32, 256)
-}
-
-#[uzu_test]
-fn test_gather_large_batch() {
- test_gather_internal(64, 128, 512)
-}
diff --git a/crates/uzu-engine/unit/encodable_block/moe/moe_perf_test.rs b/crates/uzu-engine/unit/encodable_block/moe/moe_perf_test.rs
deleted file mode 100644
index 2ada4cd89..000000000
--- a/crates/uzu-engine/unit/encodable_block/moe/moe_perf_test.rs
+++ /dev/null
@@ -1,443 +0,0 @@
-use half::bf16;
-use rand::{RngExt, SeedableRng, rngs::StdRng};
-use uzu_engine_macros::uzu_test;
-
-use super::{MoeExpertsTwoPassArguments, MoeExpertsTwoPassDecodeBlock, MoeGather};
-use crate::{
- backends::common::{
- Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferPending, Context, Kernels,
- kernel::{
- MoeBlockBasesFromPartialsKernel, MoeCountsOffsetsFusedKernel, MoeFinalizeKernel, MoeRouterTopKKernel,
- MoeScatterBucketsMapKernel,
- },
- },
- data_type::DataType,
- tests::{
- helpers::{create_buffer, create_buffer_with_data, create_context, for_each_non_cpu_backend},
- perf::run_perf_with_warmup,
- },
-};
-
-// Test E2E MoE performance with timing breakdown (decode mode, T=1)
-#[uzu_test]
-#[ignore]
-fn test_moe_e2e_decode_perf() {
- eprintln!("\n=== End-to-End MoE Performance (DECODE, T=1) ===");
- for_each_non_cpu_backend!(|B| {
- let configs = vec![
- ("Small", 1usize, 1024usize, 256usize, 8usize, 2usize),
- ("Medium", 1usize, 2048usize, 1024usize, 16usize, 2usize),
- ("Production", 1usize, 4096usize, 14336usize, 16usize, 2usize),
- ];
- for (config_name, t, d_model, d_ff, e, k) in configs {
- eprintln!("\n{} T={}, D={}, H={}, E={}, K={}", config_name, t, d_model, d_ff, e, k);
- let ctx = create_context::<B>();
- let mut rng = StdRng::seed_from_u64(0xA0075);
-
- // Generate data
- let x: Vec<bf16> = (0..t * d_model).map(|_| bf16::from_f32(rng.random_range(-1.0..1.0))).collect();
- let router_w: Vec<bf16> = (0..e * d_model).map(|_| bf16::from_f32(rng.random_range(-0.5..0.5))).collect();
- let router_b: Vec<bf16> = (0..e).map(|_| bf16::from_f32(rng.random_range(-0.1..0.1))).collect();
-
- // Buffers (simplified - just key buffers for timing)
- let x_buf = create_buffer_with_data::<B, bf16>(&ctx, &x);
- let router_w_buf = create_buffer_with_data::<B, bf16>(&ctx, &router_w);
- let router_b_buf = create_buffer_with_data::<B, bf16>(&ctx, &router_b);
- let mut topk_ids_buf = create_buffer::<B, i32>(&ctx, t * k);
- let mut topk_probs_buf = create_buffer::<B, bf16>(&ctx, t * k);
-
- let router_topk = <<B as Backend>::Kernels as Kernels>::MoeRouterTopKKernel::new(
- &ctx,
- DataType::BF16,
- true,
- false,
- false,
- false,
- false,
- )
- .expect("router+topk fused kernel");
-
- // Time fused Router+TopK
- let fused_perf = run_perf_with_warmup("Router+TopK (FUSED)", 5, 20, || {
- let mut command_buffer =
- ctx.create_command_buffer(None, None).expect("Failed to create command buffer");
- router_topk.encode(
- &x_buf,
- &router_w_buf,
- Some(&router_b_buf),
- None::<&<B as Backend>::GlobalBuffer>,
- None::<&<B as Backend>::GlobalBuffer>,
- &mut topk_ids_buf,
- &mut topk_probs_buf,
- t as u32,
- d_model as u32,
- e as u32,
- k as u32,
- true,
- None::<f32>,
- None::<f32>,
- &mut command_buffer,
- );
- command_buffer.end_encoding().submit().wait_until_completed().unwrap();
- });
- fused_perf.print();
- eprintln!(" Total: {:8.1} µs/token", fused_perf.mean_ms * 1000.0);
- }
- });
-}
-
-// Test E2E MoE performance with timing breakdown (prefill mode, T>1)
-#[uzu_test]
-#[ignore]
-fn test_moe_e2e_prefill_perf() {
- eprintln!("\n=== End-to-End MoE Performance (PREFILL, T>1) ===");
- for_each_non_cpu_backend!(|B| {
- let configs = vec![
- ("Batch4", 4, 4096, 14336, 16, 2),
- ("Batch16", 16, 4096, 14336, 16, 2),
- ("Batch32", 32, 4096, 14336, 16, 2),
- ("Batch64", 64, 4096, 14336, 16, 2),
- ];
-
- for (config_name, t, d_model, d_ff, e, k) in configs {
- eprintln!("\n{} T={}, D={}, H={}, E={}, K={}", config_name, t, d_model, d_ff, e, k);
- let ctx = create_context::<B>();
- let mut rng = StdRng::seed_from_u64(0xA0076);
-
- // Generate data
- let x: Vec<bf16> = (0..t * d_model).map(|_| bf16::from_f32(rng.random_range(-1.0..1.0))).collect();
- let router_w: Vec<bf16> = (0..e * d_model).map(|_| bf16::from_f32(rng.random_range(-0.5..0.5))).collect();
- let router_b: Vec<bf16> = (0..e).map(|_| bf16::from_f32(rng.random_range(-0.1..0.1))).collect();
-
- // Buffers (simplified - just key buffers for timing)
- let x_buf = create_buffer_with_data::<B, bf16>(&ctx, &x);
- let router_w_buf = create_buffer_with_data::<B, bf16>(&ctx, &router_w);
- let router_b_buf = create_buffer_with_data::<B, bf16>(&ctx, &router_b);
- let mut topk_ids_buf = create_buffer::<B, i32>(&ctx, t * k);
- let mut topk_probs_buf = create_buffer::<B, bf16>(&ctx, t * k);
-
- let router_topk = <<B as Backend>::Kernels as Kernels>::MoeRouterTopKKernel::new(
- &ctx,
- DataType::BF16,
- true,
- false,
- false,
- false,
- false,
- )
- .expect("router+topk fused kernel");
-
- // Time fused Router+TopK
- let fused_perf = run_perf_with_warmup("Router+TopK (FUSED)", 5, 20, || {
- let mut command_buffer =
- ctx.create_command_buffer(None, None).expect("Failed to create command buffer");
- router_topk.encode(
- &x_buf,
- &router_w_buf,
- Some(&router_b_buf),
- None::<&<B as Backend>::GlobalBuffer>,
- None::<&<B as Backend>::GlobalBuffer>,
- &mut topk_ids_buf,
- &mut topk_probs_buf,
- t as u32,
- d_model as u32,
- e as u32,
- k as u32,
- true,
- None::<f32>,
- None::<f32>,
- &mut command_buffer,
- );
- command_buffer.end_encoding().submit().wait_until_completed().unwrap();
- });
- eprintln!(" Total: {:8.3} ms", fused_perf.mean_ms);
- eprintln!(
- " Throughput: {:.1} tokens/sec, {:.3} ms/token",
- (t as f64 / fused_perf.mean_ms) * 1000.0,
- fused_perf.mean_ms / t as f64
- );
- }
- });
-}
-
-// Test complete MoE pipeline timing breakdown (decode mode, T=1)
-#[uzu_test]
-#[ignore]
-fn test_moe_pipeline_breakdown_decode() {
- for_each_non_cpu_backend!(|B| {
- let ctx = create_context::<B>();
- let mut rng = StdRng::seed_from_u64(0xDECADE);
-
- eprintln!("\n=== MoE Pipeline Breakdown (DECODE, T=1) ===");
- eprintln!("Measures ALL MoE kernels: Router→TopK→Counts→Offsets→Scatter→Gather→Experts→Finalize\n");
-
- let (t, d_model, d_ff, e, k) = (1, 4096, 14336, 16, 2);
-
- // Allocate buffers
- let x: Vec<bf16> = (0..t * d_model).map(|_| bf16::from_f32(rng.random_range(-1.0..1.0))).collect();
- let router_w: Vec<bf16> = (0..e * d_model).map(|_| bf16::from_f32(rng.random_range(-0.5..0.5))).collect();
- let router_b: Vec<bf16> = (0..e).map(|_| bf16::from_f32(rng.random_range(-0.1..0.1))).collect();
-
- let x_buf = create_buffer_with_data::<B, bf16>(&ctx, &x);
- let router_w_buf = create_buffer_with_data::<B, bf16>(&ctx, &router_w);
- let router_b_buf = create_buffer_with_data::<B, bf16>(&ctx, &router_b);
- let mut topk_ids_buf = create_buffer::<B, i32>(&ctx, t * k);
- let mut topk_probs_buf = create_buffer::<B, bf16>(&ctx, t * k);
- let num_blocks = t.div_ceil(256).max(1);
- let num_tiles = e.div_ceil(512).max(1);
- let mut partials_buf = create_buffer::<B, i32>(&ctx, num_blocks * num_tiles * e);
- let mut offsets_buf = create_buffer::<B, u32>(&ctx, e + 1);
- let mut sumk_buf = create_buffer::<B, u32>(&ctx, 1);
- let mut bucketed_ids_buf = create_buffer::<B, i32>(&ctx, t * k);
- let y_partial: Vec<bf16> = (0..t * k * d_model).map(|_| bf16::from_f32(rng.random_range(-1.0..1.0))).collect();
- let y_partial_buf = create_buffer_with_data::<B, bf16>(&ctx, &y_partial);
- let mut tok2row_buf = create_buffer::<B, i32>(&ctx, t * k);
- let mut y_out_buf = create_buffer::<B, bf16>(&ctx, t * d_model);
-
- // Expert weights buffers
- // Generate W13 in original layout [E, d_model, 2*d_ff]
- let w13_original: Vec<bf16> =
- (0..e * d_model * 2 * d_ff).map(|_| bf16::from_f32(rng.random_range(-0.5..0.5))).collect();
-
- // Transpose to GPU layout [E, 2*d_ff, d_model]
- let mut w13 = vec![bf16::from_f32(0.0); e * d_model * 2 * d_ff];
- for expert in 0..e {
- let src_offset = expert * d_model * 2 * d_ff;
- let dst_offset = expert * 2 * d_ff * d_model;
- for dm in 0..d_model {
- for ff in 0..(2 * d_ff) {
- let src_idx = src_offset + dm * 2 * d_ff + ff;
- let dst_idx = dst_offset + ff * d_model + dm;
- w13[dst_idx] = w13_original[src_idx];
- }
- }
- }
-
- let w2: Vec<bf16> = (0..e * d_ff * d_model).map(|_| bf16::from_f32(rng.random_range(-0.5..0.5))).collect();
- let up_biases: Vec<bf16> = (0..e * 2 * d_ff).map(|_| bf16::from_f32(rng.random_range(-0.1..0.1))).collect();
- let down_biases: Vec<bf16> = (0..e * d_model).map(|_| bf16::from_f32(rng.random_range(-0.1..0.1))).collect();
-
- let w13_buf = create_buffer_with_data::<B, bf16>(&ctx, &w13);
- let w2_buf = create_buffer_with_data::<B, bf16>(&ctx, &w2);
- let up_biases_buf = create_buffer_with_data::<B, bf16>(&ctx, &up_biases);
- let down_biases_buf = create_buffer_with_data::<B, bf16>(&ctx, &down_biases);
-
- let sum_k = t * k;
-
- // Scatter block bases buffers
- let mut block_bases_buf = create_buffer::<B, u32>(&ctx, num_blocks * num_tiles);
- let mut block_alloc_buf = create_buffer::<B, u32>(&ctx, num_blocks * num_tiles);
- let mut bucketed_probs_buf = create_buffer::<B, bf16>(&ctx, t * k);
-
- // Create kernel structs (use production-validated encoding logic)
- let counts_offsets_kernel =
- <<B as Backend>::Kernels as Kernels>::MoeCountsOffsetsFusedKernel::new(&ctx).expect("counts+offsets fused");
- let scatter_bases_kernel = <<B as Backend>::Kernels as Kernels>::MoeBlockBasesFromPartialsKernel::new(&ctx)
- .expect("<<Metal as Backend>::Kernels as Kernels>::MoeBlockBasesFromPartialsKernel");
- let scatter_map_kernel =
- <<B as Backend>::Kernels as Kernels>::MoeScatterBucketsMapKernel::new(&ctx, DataType::BF16)
- .expect("<<Metal as Backend>::Kernels as Kernels>::MoeScatterBucketsMapKernel");
- let gather = MoeGather::<B>::new(&ctx, DataType::BF16).expect("gather");
- let experts_kernel =
- MoeExpertsTwoPassDecodeBlock::<B>::new(&ctx, DataType::BF16, 2).expect("experts two-pass decode");
- let finalize_kernel =
- <<B as Backend>::Kernels as Kernels>::MoeFinalizeKernel::new(&ctx, DataType::BF16).expect("finalize");
- let router_topk_fused_kernel = <<B as Backend>::Kernels as Kernels>::MoeRouterTopKKernel::new(
- &ctx,
- DataType::BF16,
- true,
- false,
- false,
- false,
- false,
- )
- .expect("router+topk fused");
-
- // Testing: Router + TopK + Counts+Offsets (FUSED)
- let router_topk_fused_perf = run_perf_with_warmup("Router+TopK (FUSED)", 2, 5, || {
- let mut command_buffer = ctx.create_command_buffer(None, None).expect("Failed to create command buffer");
- router_topk_fused_kernel.encode(
- &x_buf,
- &router_w_buf,
- Some(&router_b_buf),
- None::<&<B as Backend>::GlobalBuffer>,
- None::<&<B as Backend>::GlobalBuffer>,
- &mut topk_ids_buf,
- &mut topk_probs_buf,
- t as u32,
- d_model as u32,
- e as u32,
- k as u32,
- true,
- None::<f32>,
- None::<f32>,
- &mut command_buffer,
- );
- command_buffer.end_encoding().submit().wait_until_completed().unwrap();
- });
-
- let counts_offsets_perf = run_perf_with_warmup("Counts+Offsets (FUSED)", 2, 5, || {
- let mut command_buffer = ctx.create_command_buffer(None, None).expect("Failed to create command buffer");
- counts_offsets_kernel.encode(
- &topk_ids_buf,
- &mut offsets_buf,
- &mut sumk_buf,
- &mut partials_buf,
- t as u32,
- e as u32,
- k as u32,
- &mut command_buffer,
- );
- command_buffer.end_encoding().submit().wait_until_completed().unwrap();
- });
-
- let scatter_perf = run_perf_with_warmup("Scatter", 2, 5, || {
- let mut command_buffer = ctx.create_command_buffer(None, None).expect("Failed to create command buffer");
- scatter_bases_kernel.encode(
- &partials_buf,
- &mut block_bases_buf,
- &mut block_alloc_buf,
- e as u32,
- num_blocks as u32,
- num_tiles as u32,
- 0u32,
- &mut command_buffer,
- );
- scatter_map_kernel.encode(
- &topk_ids_buf,
- &topk_probs_buf,
- &offsets_buf,
- &block_bases_buf,
- &block_alloc_buf,
- &mut bucketed_ids_buf,
- &mut bucketed_probs_buf,
- t as u32,
- e as u32,
- k as u32,
- num_blocks as u32,
- num_tiles as u32,
- &mut tok2row_buf,
- &mut command_buffer,
- );
- command_buffer.end_encoding().submit().wait_until_completed().unwrap();
- });
-
- let gather_perf = run_perf_with_warmup("Gather", 2, 5, || {
- let mut command_buffer = ctx.create_command_buffer(None, None).expect("Failed to create command buffer");
- let x_perm = gather
- .encode(&x_buf, &bucketed_ids_buf, &sumk_buf, t as u32, k as u32, d_model as u32, &mut command_buffer)
- .expect("failed to encode MoE gather");
- let completed = command_buffer.end_encoding().submit().wait_until_completed().unwrap();
- drop(x_perm);
- drop(completed);
- });
-
- let (x_perm_buf, x_perm_completed) = {
- let mut command_buffer = ctx.create_command_buffer(None, None).expect("Failed to create command buffer");
- let x_perm = gather
- .encode(&x_buf, &bucketed_ids_buf, &sumk_buf, t as u32, k as u32, d_model as u32, &mut command_buffer)
- .expect("failed to encode MoE gather");
- let completed = command_buffer.end_encoding().submit().wait_until_completed().unwrap();
- (x_perm, completed)
- };
-
- let experts_perf = run_perf_with_warmup("Experts (MAIN COMPUTE)", 2, 5, || {
- let mut command_buffer = ctx.create_command_buffer(None, None).expect("Failed to create command buffer");
- let output = experts_kernel
- .encode(
- MoeExpertsTwoPassArguments {
- x_perm: &x_perm_buf,
- expert_offsets: &offsets_buf,
- w13_all: &w13_buf,
- w2_all: &w2_buf,
- up_biases: &up_biases_buf,
- down_biases: &down_biases_buf,
- total_rows: sum_k as u32,
- d_model: d_model as u32,
- d_ff: d_ff as u32,
- num_routed_experts: e as u32,
- gate_clip_min: f32::NEG_INFINITY,
- gate_clip_max: 20.0,
- up_clip_min: -19.0,
- up_clip_max: 21.0,
- silu_alpha: 1.702,
- },
- &mut command_buffer,
- )
- .expect("failed to encode MoE experts");
- let completed = command_buffer.end_encoding().submit().wait_until_completed().unwrap();
- drop(output);
- drop(completed);
- });
- drop(x_perm_buf);
- drop(x_perm_completed);
-
- let finalize_perf = run_perf_with_warmup("Finalize", 2, 5, || {
- let mut command_buffer = ctx.create_command_buffer(None, None).expect("Failed to create command buffer");
- finalize_kernel.encode(
- &tok2row_buf,
- &topk_probs_buf,
- &y_partial_buf,
- &mut y_out_buf,
- t as u32,
- d_model as u32,
- k as u32,
- &mut command_buffer,
- );
- command_buffer.end_encoding().submit().wait_until_completed().unwrap();
- });
-
- // Print results
- router_topk_fused_perf.print();
- counts_offsets_perf.print();
- scatter_perf.print();
- gather_perf.print();
- experts_perf.print();
- finalize_perf.print();
-
- // Calculate breakdown
- let total_us = (router_topk_fused_perf.mean_ms
- + counts_offsets_perf.mean_ms
- + scatter_perf.mean_ms
- + gather_perf.mean_ms
- + experts_perf.mean_ms
- + finalize_perf.mean_ms)
- * 1000.0;
-
- eprintln!("\n ═══ Per-Kernel Latency (Production D=4096, H=14336, E=16, K=2, T=1) ═══");
- eprintln!(
- " Router+TopK (FUSED): {:8.1} us ({:5.1}%)",
- router_topk_fused_perf.mean_ms * 1000.0,
- (router_topk_fused_perf.mean_ms / (total_us / 1000.0)) * 100.0
- );
- eprintln!(
- " Counts+Offsets: {:8.1} us ({:5.1}%)",
- counts_offsets_perf.mean_ms * 1000.0,
- (counts_offsets_perf.mean_ms / (total_us / 1000.0)) * 100.0
- );
- eprintln!(
- " Scatter: {:8.1} us ({:5.1}%)",
- scatter_perf.mean_ms * 1000.0,
- (scatter_perf.mean_ms / (total_us / 1000.0)) * 100.0
- );
- eprintln!(
- " Gather: {:8.1} us ({:5.1}%)",
- gather_perf.mean_ms * 1000.0,
- (gather_perf.mean_ms / (total_us / 1000.0)) * 100.0
- );
- eprintln!(
- " Experts: {:8.1} us ({:5.1}%) ← MAIN COMPUTE",
- experts_perf.mean_ms * 1000.0,
- (experts_perf.mean_ms / (total_us / 1000.0)) * 100.0
- );
- eprintln!(
- " Finalize: {:8.1} us ({:5.1}%)",
- finalize_perf.mean_ms * 1000.0,
- (finalize_perf.mean_ms / (total_us / 1000.0)) * 100.0
- );
- eprintln!(" ═══════════════════════════════════════════");
- eprintln!(" TOTAL: {:8.1} us (100.0%)", total_us);
- eprintln!("\n Note: Times include Metal CB overhead (~10-50ms per kernel).");
- eprintln!(" Real GPU compute is much faster, but relative % shows bottleneck.");
- })
-}
diff --git a/crates/uzu-engine/unit/encodable_block/moe/moe_tiles_test.rs b/crates/uzu-engine/unit/encodable_block/moe/moe_tiles_test.rs
deleted file mode 100644
index 99c8018e4..000000000
--- a/crates/uzu-engine/unit/encodable_block/moe/moe_tiles_test.rs
+++ /dev/null
@@ -1,164 +0,0 @@
-use rand::{RngExt, SeedableRng, rngs::StdRng};
-use uzu_engine_macros::uzu_test;
-
-use crate::{
- backends::common::{
- Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferPending, Context, Kernels,
- kernel::{MoeTileCountsKernel, MoeTileScanKernel},
- },
- encodable_block::mlp::moe::tests::{cpu_tile_counts, cpu_tile_scan},
- tests::helpers::{
- buffer_prefix_to_vec, buffer_to_vec, create_buffer, create_buffer_with_data, create_context,
- for_each_non_cpu_backend,
- },
-};
-
-#[uzu_test]
-fn test_tile_counts_correctness() {
- for_each_non_cpu_backend!(|B| {
- let ctx = create_context::<B>();
- let mut rng = StdRng::seed_from_u64(2028);
-
- // Test multiple expert counts
- let expert_counts = vec![4, 8, 16, 64];
-
- for e in expert_counts {
- eprintln!("[TileCountsTest] E={}", e);
-
- // Generate random offsets (simulating expert segment sizes)
- let mut offsets = vec![0u32];
- for _ in 0..e {
- let seg_len = rng.random_range(0..200);
- offsets.push(offsets.last().unwrap() + seg_len);
- }
-
- let tile_counts_cpu = cpu_tile_counts(&offsets, 16);
-
- // GPU buffers
- let offsets_buf = create_buffer_with_data::<B, u32>(&ctx, &offsets);
-
- // Execute kernel using kernel struct
- let counts_kernel =
- <<B as Backend>::Kernels as Kernels>::MoeTileCountsKernel::new(&ctx).expect("MoeTileCountsKernel::new");
- let mut command_buffer = ctx.create_command_buffer(None, None).expect("Failed to create command buffer");
- let mut tile_counts_buf = create_buffer::<B, u32>(&ctx, e);
- counts_kernel.encode(&offsets_buf, &mut tile_counts_buf, e as u32, &mut command_buffer);
- let completed = command_buffer.end_encoding().submit().wait_until_completed().unwrap();
-
- let tile_counts_gpu = buffer_prefix_to_vec::<B, u32>(&tile_counts_buf, e);
- drop(tile_counts_buf);
- drop(completed);
-
- assert_eq!(tile_counts_gpu, tile_counts_cpu, "Tile counts mismatch for E={}", e);
-
- eprintln!("[TileCountsTest] ✓ PASSED");
- }
- });
-}
-
-#[uzu_test]
-fn test_tile_scan_correctness() {
- for_each_non_cpu_backend!(|B| {
- let ctx = create_context::<B>();
- let mut rng = StdRng::seed_from_u64(2029);
-
- // Test multiple expert counts
- let expert_counts = vec![4, 8, 16, 64];
-
- for e in expert_counts {
- eprintln!("[TileScanTest] E={}", e);
-
- // Generate random tile counts
- let tile_counts: Vec<u32> = (0..e).map(|_| rng.random_range(0..20)).collect();
-
- let (tile_offsets_cpu, total_tiles_cpu) = cpu_tile_scan(&tile_counts);
-
- // GPU buffers
- let tile_counts_buf = create_buffer_with_data::<B, u32>(&ctx, &tile_counts);
-
- // Execute kernel using kernel struct
- let scan_kernel =
- <<B as Backend>::Kernels as Kernels>::MoeTileScanKernel::new(&ctx).expect("MoeTileScanKernel::new");
- let mut command_buffer = ctx.create_command_buffer(None, None).expect("Failed to create command buffer");
- let mut tile_offsets_buf = create_buffer::<B, u32>(&ctx, e + 1);
- let mut total_tiles_buf = create_buffer::<B, u32>(&ctx, 8);
- scan_kernel.encode(
- &tile_counts_buf,
- &mut tile_offsets_buf,
- &mut total_tiles_buf,
- e as u32,
- &mut command_buffer,
- );
- let completed = command_buffer.end_encoding().submit().wait_until_completed().unwrap();
-
- let tile_offsets_gpu = buffer_prefix_to_vec::<B, u32>(&tile_offsets_buf, e + 1);
- let total_tiles_gpu = buffer_to_vec::<B, u32>(&total_tiles_buf)[0];
- drop(tile_offsets_buf);
- drop(total_tiles_buf);
- drop(completed);
-
- assert_eq!(tile_offsets_gpu, tile_offsets_cpu, "Tile offsets mismatch for E={}", e);
- assert_eq!(total_tiles_gpu, total_tiles_cpu, "Total tiles mismatch for E={}", e);
-
- eprintln!("[TileScanTest] ✓ PASSED");
- }
- });
-}
-
-#[uzu_test]
-fn test_tile_edge_cases() {
- for_each_non_cpu_backend!(|B| {
- let ctx = create_context::<B>();
-
- // Test with all empty experts (seg_len=0)
- {
- let e = 8;
- let offsets = vec![0u32; e + 1]; // All zeros
-
- let tile_counts_cpu = cpu_tile_counts(&offsets, 16);
-
- let offsets_buf = create_buffer_with_data::<B, u32>(&ctx, &offsets);
-
- let counts_kernel =
- <<B as Backend>::Kernels as Kernels>::MoeTileCountsKernel::new(&ctx).expect("MoeTileCountsKernel::new");
- let mut command_buffer = ctx.create_command_buffer(None, None).expect("Failed to create command buffer");
- let mut tile_counts_buf = create_buffer::<B, u32>(&ctx, e);
- counts_kernel.encode(&offsets_buf, &mut tile_counts_buf, e as u32, &mut command_buffer);
- let completed = command_buffer.end_encoding().submit().wait_until_completed().unwrap();
-
- let tile_counts_gpu = buffer_prefix_to_vec::<B, u32>(&tile_counts_buf, e);
- drop(tile_counts_buf);
- drop(completed);
-
- assert_eq!(tile_counts_gpu, tile_counts_cpu);
- assert!(tile_counts_gpu.iter().all(|&c| c == 0));
- eprintln!("[TileEdgeCases] ✓ Empty experts PASSED");
- }
-
- // Test with single tile per expert (seg_len < BM=16)
- {
- let e = 4;
- let offsets = vec![0, 10, 15, 30, 50]; // Small segments
-
- let tile_counts_cpu = cpu_tile_counts(&offsets, 16);
-
- let offsets_buf = create_buffer_with_data::<B, u32>(&ctx, &offsets);
-
- let counts_kernel =
- <<B as Backend>::Kernels as Kernels>::MoeTileCountsKernel::new(&ctx).expect("MoeTileCountsKernel::new");
- let mut command_buffer = ctx.create_command_buffer(None, None).expect("Failed to create command buffer");
- let mut tile_counts_buf = create_buffer::<B, u32>(&ctx, e);
- counts_kernel.encode(&offsets_buf, &mut tile_counts_buf, e as u32, &mut command_buffer);
- let completed = command_buffer.end_encoding().submit().wait_until_completed().unwrap();
-
- let tile_counts_gpu = buffer_prefix_to_vec::<B, u32>(&tile_counts_buf, e);
- drop(tile_counts_buf);
- drop(completed);
-
- assert_eq!(tile_counts_gpu, tile_counts_cpu);
- // With BM=16: seg_lens=[10, 5, 15, 20] -> tile_counts=[1, 1, 1, 2]
- assert_eq!(tile_counts_gpu, &[1, 1, 1, 2]);
- eprintln!("[TileEdgeCases] ✓ Small segments PASSED");
- }
- });
-}