A high-performance inference engine for AI models
GDN speculation (#609)
CCYeh8377b7657bbd
diff --git a/crates/backend-uzu/src/backends/common/kernel/delta_net_tree_verify.rs b/crates/backend-uzu/src/backends/common/kernel/delta_net_tree_verify.rsnew file mode 100644index 000000000..e9d14bae9--- /dev/null+++ b/crates/backend-uzu/src/backends/common/kernel/delta_net_tree_verify.rs@@ -0,0 +1,42 @@+use std::convert::Infallible;++use crate::{+ backends::common::{Allocation, Backend, Encoder, Kernels},+ encodable_block::mixer::delta_net::tree_verify::{TreeVerifyEncodeArguments, TreeVerifyNewArguments},+};++pub trait DeltaNetTreeVerify<B: Backend<Kernels: Kernels<DeltaNetTreeVerify = Self>>>: Sized {+ fn is_supported(context: &B::Context) -> bool;++ fn new(+ context: &B::Context,+ arguments: &TreeVerifyNewArguments,+ ) -> Result<Self, B::Error>;++ fn encode(+ &self,+ arguments: TreeVerifyEncodeArguments<'_, B>,+ encoder: &mut Encoder<B>,+ ) -> Result<Allocation<B>, B::Error>;+}++impl<B: Backend<Kernels: Kernels<DeltaNetTreeVerify = Infallible>>> DeltaNetTreeVerify<B> for Infallible {+ fn is_supported(_context: &B::Context) -> bool {+ false+ }++ fn new(+ _context: &B::Context,+ _arguments: &TreeVerifyNewArguments,+ ) -> Result<Self, B::Error> {+ unreachable!("unsupported DeltaNet tree verifier should not be constructed")+ }++ fn encode(+ &self,+ _arguments: TreeVerifyEncodeArguments<'_, B>,+ _encoder: &mut Encoder<B>,+ ) -> Result<Allocation<B>, B::Error> {+ unreachable!("unsupported DeltaNet tree verifier cannot encode")+ }+}diff --git a/crates/backend-uzu/src/backends/common/kernel/mod.rs b/crates/backend-uzu/src/backends/common/kernel/mod.rsindex 59cfe359f..9d0d9c6d3 100644--- a/crates/backend-uzu/src/backends/common/kernel/mod.rs+++ b/crates/backend-uzu/src/backends/common/kernel/mod.rs@@ -2,6 +2,7 @@ use crate::backends::common::Backend;pub mod attention_gemm;pub mod delta_net_chunked_prefill;+pub mod delta_net_tree_verify;pub mod matmul;include!(concat!(env!("OUT_DIR"), "/traits.rs"));@@ -12,6 +13,7 @@ pub trait Kernels: Sized {autogen_kernels!();type AttentionGemmCore: attention_gemm::AttentionGemmCore<Self::Backend>;type DeltaNetChunkedPrefill: delta_net_chunked_prefill::DeltaNetChunkedPrefill<Self::Backend>;+ type DeltaNetTreeVerify: delta_net_tree_verify::DeltaNetTreeVerify<Self::Backend>;type MatmulKernel: matmul::MatmulKernel<Backend = Self::Backend>;}diff --git a/crates/backend-uzu/src/backends/cpu/kernel/gdn/conv_scan.rs b/crates/backend-uzu/src/backends/cpu/kernel/gdn/conv_scan.rsindex 36bc40ea5..984f53d0d 100644--- a/crates/backend-uzu/src/backends/cpu/kernel/gdn/conv_scan.rs+++ b/crates/backend-uzu/src/backends/cpu/kernel/gdn/conv_scan.rs@@ -1,4 +1,4 @@-use half::{bf16, f16};+use half::bf16;use num_traits::Float;use proc_macros::kernel;@@ -14,7 +14,7 @@ use crate::{array::ArrayElement, backends::common::gpu_types::ActivationType};// in_proj: [suffix_len, out_stride] — first conv_dim channels overwritten// state_out: [conv_dim, state_stride]#[kernel(DeltaNetConvScan)]-#[variants(T, f32, f16, bf16)]+#[variants(T, f32, bf16)]pub fn delta_net_conv_scan<T: ArrayElement + Float>(conv_padded: *const f32,conv_weight: *const f32,diff --git a/crates/backend-uzu/src/backends/cpu/kernel/gdn/conv_update.rs b/crates/backend-uzu/src/backends/cpu/kernel/gdn/conv_update.rsindex 3da868b84..203b54f31 100644--- a/crates/backend-uzu/src/backends/cpu/kernel/gdn/conv_update.rs+++ b/crates/backend-uzu/src/backends/cpu/kernel/gdn/conv_update.rs@@ -1,4 +1,4 @@-use half::{bf16, f16};+use half::bf16;use num_traits::Float;use proc_macros::kernel;@@ -6,7 +6,7 @@ use crate::{array::ArrayElement, backends::common::gpu_types::ActivationType};// Single-token causal conv1d with SiLU, in-place.#[kernel(DeltaNetConvUpdate)]-#[variants(T, f32, f16, bf16)]+#[variants(T, f32, bf16)]pub fn delta_net_conv_update<T: ArrayElement + Float>(conv_weight: *const f32,#[optional(has_bias)] bias: Option<*const f32>,diff --git a/crates/backend-uzu/src/backends/cpu/kernel/gdn/norm_gate.rs b/crates/backend-uzu/src/backends/cpu/kernel/gdn/norm_gate.rsindex eca1bf120..e305e5c00 100644--- a/crates/backend-uzu/src/backends/cpu/kernel/gdn/norm_gate.rs+++ b/crates/backend-uzu/src/backends/cpu/kernel/gdn/norm_gate.rs@@ -1,4 +1,4 @@-use half::{bf16, f16};+use half::bf16;use num_traits::Float;use proc_macros::kernel;@@ -14,7 +14,7 @@ use crate::{array::ArrayElement, backends::common::gpu_types::ActivationType};// where inv_rms = rsqrt(mean(in_out^2) + epsilon)// and z is at in_proj[token * total_proj_dim + conv_dim + hv * head_v_dim + i]#[kernel(DeltaNetNormGate)]-#[variants(T, f32, f16, bf16)]+#[variants(T, f32, bf16)]pub fn delta_net_norm_gate<T: ArrayElement + Float>(in_out: *mut T,in_proj: *const T,diff --git a/crates/backend-uzu/src/backends/cpu/kernel/gdn/prefill.rs b/crates/backend-uzu/src/backends/cpu/kernel/gdn/prefill.rsindex 12a1a8c9f..7d7fab7e9 100644--- a/crates/backend-uzu/src/backends/cpu/kernel/gdn/prefill.rs+++ b/crates/backend-uzu/src/backends/cpu/kernel/gdn/prefill.rs@@ -1,11 +1,11 @@-use half::{bf16, f16};+use half::bf16;use num_traits::Float;use proc_macros::kernel;use crate::array::ArrayElement;#[kernel(DeltaNetPrefill)]-#[variants(T, f32, f16, bf16)]+#[variants(T, f32, bf16)]#[variants(HEAD_K_DIM, 128)]pub fn delta_net_prefill<T: ArrayElement + Float, const HEAD_K_DIM: u32>(q_norm: *const f32,diff --git a/crates/backend-uzu/src/backends/cpu/kernel/gdn/prefill_prep.rs b/crates/backend-uzu/src/backends/cpu/kernel/gdn/prefill_prep.rsindex 00a2f601b..a87bbbc2d 100644--- a/crates/backend-uzu/src/backends/cpu/kernel/gdn/prefill_prep.rs+++ b/crates/backend-uzu/src/backends/cpu/kernel/gdn/prefill_prep.rs@@ -1,4 +1,4 @@-use half::{bf16, f16};+use half::bf16;use num_traits::Float;use proc_macros::kernel;@@ -6,14 +6,16 @@ use crate::array::ArrayElement;// Pre-compute L2-normalized q/k, beta, and decay for all tokens.#[kernel(DeltaNetPrefillPrep)]-#[variants(T, f32, f16, bf16)]+#[variants(T, f32, bf16)]+#[variants(QKT, f32, bf16)]#[variants(HEAD_K_DIM, 128)]-pub fn delta_net_prefill_prep<T: ArrayElement + Float, const HEAD_K_DIM: u32>(+pub fn delta_net_prefill_prep<T: ArrayElement + Float, QKT: ArrayElement + Float, const HEAD_K_DIM: u32>(in_proj: *const T,- a_log: *const T,- dt_bias: *const T,- q_norm_out: *mut f32,- k_norm_out: *mut f32,+ a_log: *const f32,+ dt_bias: *const f32,+ q_norm_out: *mut QKT,+ k_norm_out: *mut QKT,+ #[optional(write_compact_v)] compact_v_out: Option<*mut T>,beta_out: *mut f32,decay_out: *mut f32,num_v_heads: u32,@@ -22,7 +24,10 @@ pub fn delta_net_prefill_prep<T: ArrayElement + Float, const HEAD_K_DIM: u32>(value_dim: u32,suffix_len: u32,#[specialize] write_log_decay: bool,+ #[specialize] write_compact_v: bool,) {+ assert_eq!(compact_v_out.is_some(), write_compact_v, "compact V output presence mismatch");+let num_v_heads = num_v_heads as usize;let num_k_heads = num_k_heads as usize;let head_k_dim = HEAD_K_DIM as usize;@@ -36,6 +41,14 @@ pub fn delta_net_prefill_prep<T: ArrayElement + Float, const HEAD_K_DIM: u32>(for token in 0..suffix_len {let tok_offset = token * total_proj_dim;+ if let Some(compact_v_out) = compact_v_out {+ unsafe {+ in_proj+ .add(tok_offset + 2 * key_dim)+ .copy_to_nonoverlapping(compact_v_out.add(token * value_dim), value_dim)+ };+ }+for hk in 0..num_k_heads {// Load and L2-normalize qlet q_off = tok_offset + hk * head_k_dim;@@ -48,7 +61,9 @@ pub fn delta_net_prefill_prep<T: ArrayElement + Float, const HEAD_K_DIM: u32>(let q_scale = 1.0 / (head_k_dim as f32).sqrt();for j in 0..head_k_dim {let v = unsafe { (*in_proj.add(q_off + j)).to_f32().unwrap() };- unsafe { *q_norm_out.add(token * key_dim + hk * head_k_dim + j) = v * q_inv * q_scale };+ unsafe {+ *q_norm_out.add(token * key_dim + hk * head_k_dim + j) = QKT::from(v * q_inv * q_scale).unwrap()+ };}// Load and L2-normalize k@@ -61,7 +76,7 @@ pub fn delta_net_prefill_prep<T: ArrayElement + Float, const HEAD_K_DIM: u32>(let k_inv = 1.0 / (k_sq + 1e-6).sqrt();for j in 0..head_k_dim {let v = unsafe { (*in_proj.add(k_off + j)).to_f32().unwrap() };- unsafe { *k_norm_out.add(token * key_dim + hk * head_k_dim + j) = v * k_inv };+ unsafe { *k_norm_out.add(token * key_dim + hk * head_k_dim + j) = QKT::from(v * k_inv).unwrap() };}// Beta and decay for each v-head of this k-head@@ -71,8 +86,8 @@ pub fn delta_net_prefill_prep<T: ArrayElement + Float, const HEAD_K_DIM: u32>(let beta_raw = unsafe { (*in_proj.add(tok_offset + conv_dim + value_dim + hv)).to_f32().unwrap() };let beta = 1.0 / (1.0 + (-beta_raw).exp());- let a_log_val = unsafe { (*a_log.add(hv)).to_f32().unwrap() };- let dt_bias_val = unsafe { (*dt_bias.add(hv)).to_f32().unwrap() };+ let a_log_val = unsafe { *a_log.add(hv) };+ let dt_bias_val = unsafe { *dt_bias.add(hv) };let a_raw =unsafe { (*in_proj.add(tok_offset + conv_dim + value_dim + num_v_heads + hv)).to_f32().unwrap() };let sp_in = a_raw + dt_bias_val;diff --git a/crates/backend-uzu/src/backends/cpu/kernel/gdn/tree_verify/mod.rs b/crates/backend-uzu/src/backends/cpu/kernel/gdn/tree_verify/mod.rsindex fd25fad36..e4b9b6cf5 100644--- a/crates/backend-uzu/src/backends/cpu/kernel/gdn/tree_verify/mod.rs+++ b/crates/backend-uzu/src/backends/cpu/kernel/gdn/tree_verify/mod.rs@@ -1,6 +1,6 @@pub mod conv_scan;pub mod out;-pub mod prefix_beta;+pub mod prefix;pub mod state_advance;pub mod tree_gram;pub mod tree_update_solve;diff --git a/crates/backend-uzu/src/backends/cpu/kernel/gdn/tree_verify/out.rs b/crates/backend-uzu/src/backends/cpu/kernel/gdn/tree_verify/out.rsindex d3ac41635..c18492cc3 100644--- a/crates/backend-uzu/src/backends/cpu/kernel/gdn/tree_verify/out.rs+++ b/crates/backend-uzu/src/backends/cpu/kernel/gdn/tree_verify/out.rs@@ -5,15 +5,16 @@ use proc_macros::kernel;use crate::array::ArrayElement;#[kernel(BuildTreeOut)]-#[variants(T, f32, bf16)]-pub fn build_tree_out<T: ArrayElement + Float>(- q: *const T,+#[variants(QKT, f32, bf16)]+#[variants(OutputT, f32, bf16)]+pub fn build_tree_out<QKT: ArrayElement + Float, OutputT: ArrayElement + Float>(+ q: *const QKT,prefix: *const f32,qkd: *const f32,- u: *const T,+ u: *const f32,#[optional(use_h0)] h0: Option<*const f32>,#[optional(use_h0)] h0_indices: Option<*const i32>,- o: *mut T,+ o: *mut OutputT,scale: f32,batch_size: u32,tree_size: u32,@@ -75,12 +76,12 @@ pub fn build_tree_out<T: ArrayElement + Float>(for col in 0..tree_size {let qkd_value = unsafe { *qkd.add(qkd_base + row * tree_size + col) };- let u_value = unsafe { (*u.add(u_base + col * head_v_dim + value_col)).to_f32().unwrap() };+ let u_value = unsafe { *u.add(u_base + col * head_v_dim + value_col) };acc += qkd_value * u_value;}unsafe {- *o.add(out_base + row * value_heads * head_v_dim + value_col) = T::from(acc).unwrap();+ *o.add(out_base + row * value_heads * head_v_dim + value_col) = OutputT::from(acc).unwrap();}}}diff --git a/crates/backend-uzu/src/backends/cpu/kernel/gdn/tree_verify/prefix.rs b/crates/backend-uzu/src/backends/cpu/kernel/gdn/tree_verify/prefix.rsnew file mode 100644index 000000000..d225a61a2--- /dev/null+++ b/crates/backend-uzu/src/backends/cpu/kernel/gdn/tree_verify/prefix.rs@@ -0,0 +1,39 @@+use proc_macros::kernel;++use crate::backends::common::gpu_types::trie::TrieNode;++#[kernel(BuildTreePrefix)]+pub fn build_tree_prefix(+ trie: *const TrieNode,+ log_decay: *const f32,+ prefix: *mut f32,+ batch_size: u32,+ tree_size: u32,+ value_heads: u32,+) {+ let batch_size = batch_size as usize;+ let tree_size = tree_size as usize;+ let value_heads = value_heads as usize;++ for batch in 0..batch_size {+ let trie_batch = batch * tree_size;+ let batch_offset = batch * tree_size * value_heads;++ for row in 0..tree_size {+ let row_idx = row as u32;+ for head in 0..value_heads {+ let mut sum = 0.0f32;+ for col in 0..tree_size {+ // col is an ancestor-or-self of row iff row falls in col's+ // DFS subtree interval [trie_start, trie_end].+ let node = unsafe { *trie.add(trie_batch + col) };+ if row_idx < node.trie_start || row_idx > node.trie_end {+ continue;+ }+ sum += unsafe { *log_decay.add(batch_offset + col * value_heads + head) };+ }+ unsafe { *prefix.add(batch_offset + row * value_heads + head) = sum };+ }+ }+ }+}diff --git a/crates/backend-uzu/src/backends/cpu/kernel/gdn/tree_verify/prefix_beta.rs b/crates/backend-uzu/src/backends/cpu/kernel/gdn/tree_verify/prefix_beta.rsdeleted file mode 100644index 086119bd2..000000000--- a/crates/backend-uzu/src/backends/cpu/kernel/gdn/tree_verify/prefix_beta.rs+++ /dev/null@@ -1,58 +0,0 @@-use half::{bf16, f16};-use num_traits::Float;-use proc_macros::kernel;--use crate::{- array::ArrayElement,- backends::common::gpu_types::{ActivationType, trie::TrieNode},-};--#[kernel(BuildPrefixBeta)]-#[variants(T, f32, f16, bf16)]-pub fn build_prefix_beta<T: ArrayElement + Float>(- trie: *const TrieNode,- a_transposed: *const T,- b: *const T,- a_log: *const f32,- dt_bias: *const f32,- prefix: *mut f32,- beta: *mut f32,- batch_size: u32,- tree_size: u32,- value_heads: u32,-) {- let batch_size = batch_size as usize;- let tree_size = tree_size as usize;- let value_heads = value_heads as usize;-- for batch in 0..batch_size {- let trie_batch = batch * tree_size;- let batch_offset = batch * tree_size * value_heads;-- for row in 0..tree_size {- let row_idx = row as u32;- for head in 0..value_heads {- let out_idx = batch_offset + row * value_heads + head;- let b_val = unsafe { *b.add(out_idx) }.to_f32().unwrap();- unsafe { *beta.add(out_idx) = 1.0 / (1.0 + (-b_val).exp()) };-- let dt = unsafe { *dt_bias.add(head) };- let scale = unsafe { *a_log.add(head) }.exp();-- let mut sum = 0.0f32;- for col in 0..tree_size {- // col is an ancestor-or-self of row iff row falls in col's- // DFS subtree interval [trie_start, trie_end].- let node = unsafe { *trie.add(trie_batch + col) };- if row_idx < node.trie_start || row_idx > node.trie_end {- continue;- }- let a_val = unsafe { *a_transposed.add(batch_offset + head * tree_size + col) }.to_f32().unwrap();- let sp = ActivationType::SOFTPLUS.activate(a_val + dt);- sum -= scale * sp;- }- unsafe { *prefix.add(out_idx) = sum };- }- }- }-}diff --git a/crates/backend-uzu/src/backends/cpu/kernel/gdn/tree_verify/state_advance.rs b/crates/backend-uzu/src/backends/cpu/kernel/gdn/tree_verify/state_advance.rsindex 81ba1b006..095970463 100644--- a/crates/backend-uzu/src/backends/cpu/kernel/gdn/tree_verify/state_advance.rs+++ b/crates/backend-uzu/src/backends/cpu/kernel/gdn/tree_verify/state_advance.rs@@ -8,15 +8,15 @@ use crate::array::ArrayElement;#[variants(T, f32, bf16)]#[variants(HEAD_K_DIM, 128)]pub fn state_advance<T: ArrayElement + Float, const HEAD_K_DIM: u32>(- k_norm: *const f32,+ k_norm: *const T,v: *const T,log_decay_buf: *const f32,beta_buf: *const f32,accepted_indices: *const u32,state: *mut f32,accepted_len: u32,- num_v_heads: u32,- num_k_heads: u32,+ #[specialize] num_v_heads: u32,+ #[specialize] num_k_heads: u32,) {let accepted_len = accepted_len as usize;let num_v_heads = num_v_heads as usize;@@ -42,13 +42,16 @@ pub fn state_advance<T: ArrayElement + Float, const HEAD_K_DIM: u32>(for dk_idx in 0..head_k_dim {let state_element_ptr = unsafe { state.add(state_row_offset + dk_idx) };unsafe { *state_element_ptr *= decay };- kv_mem += unsafe { *state_element_ptr * *k_norm.add(k_offset + dk_idx) };+ kv_mem += unsafe { *state_element_ptr * (*k_norm.add(k_offset + dk_idx)).to_f32().unwrap() };}let v_value =unsafe { (*v.add(tree_idx * value_dim + hv_idx * head_v_dim + dv_idx)).to_f32().unwrap() };let delta = beta * (v_value - kv_mem);for dk_idx in 0..head_k_dim {- unsafe { *state.add(state_row_offset + dk_idx) += *k_norm.add(k_offset + dk_idx) * delta };+ unsafe {+ *state.add(state_row_offset + dk_idx) +=+ (*k_norm.add(k_offset + dk_idx)).to_f32().unwrap() * delta+ };}}}diff --git a/crates/backend-uzu/src/backends/cpu/kernel/gdn/tree_verify/tree_gram.rs b/crates/backend-uzu/src/backends/cpu/kernel/gdn/tree_verify/tree_gram.rsindex 7cbf882ea..ae1dce607 100644--- a/crates/backend-uzu/src/backends/cpu/kernel/gdn/tree_verify/tree_gram.rs+++ b/crates/backend-uzu/src/backends/cpu/kernel/gdn/tree_verify/tree_gram.rs@@ -15,7 +15,7 @@ pub fn build_tree_gram<T: ArrayElement + Float, const USE_MXU: bool>(trie: *const TrieNode,prefix: *const f32,beta: *const f32,- #[optional(use_h0)] h0: Option<*const T>,+ #[optional(use_h0)] h0: Option<*const f32>,#[optional(use_h0)] h0_idx: Option<*const i32>,a_packed: *mut f32,qkd: *mut f32,@@ -152,7 +152,7 @@ pub fn build_tree_gram<T: ArrayElement + Float, const USE_MXU: bool>(let mut sum = 0.0f32;for d in 0..head_k_dim {sum += unsafe { (*k.add(k_off + d)).to_f32().unwrap() }- * unsafe { (*h0.add(h0_head + dv * head_k_dim + d)).to_f32().unwrap() };+ * unsafe { *h0.add(h0_head + dv * head_k_dim + d) };}unsafe { *kh0.add(kh0_off + dv) = sum };}diff --git a/crates/backend-uzu/src/backends/cpu/kernel/gdn/tree_verify/tree_update_solve.rs b/crates/backend-uzu/src/backends/cpu/kernel/gdn/tree_verify/tree_update_solve.rsindex 0d25d0a30..ef46d2c38 100644--- a/crates/backend-uzu/src/backends/cpu/kernel/gdn/tree_verify/tree_update_solve.rs+++ b/crates/backend-uzu/src/backends/cpu/kernel/gdn/tree_verify/tree_update_solve.rs@@ -9,8 +9,7 @@ use crate::array::ArrayElement;#[kernel(TreeUpdateSolve)]#[variants(T, f32, bf16)]#[variants(BV, 16, 32)]-#[variants(USE_MXU, false)]-pub fn tree_update_solve<T: ArrayElement + Float, const BV: u32, const USE_MXU: bool>(+pub fn tree_update_solve<T: ArrayElement + Float, const BV: u32>(#[optional(use_h0)] kh0: Option<*const f32>,v: *const T,prefix: *const f32,diff --git a/crates/backend-uzu/src/backends/cpu/kernel/gdn/update.rs b/crates/backend-uzu/src/backends/cpu/kernel/gdn/update.rsindex 9ea29360f..26b323d5b 100644--- a/crates/backend-uzu/src/backends/cpu/kernel/gdn/update.rs+++ b/crates/backend-uzu/src/backends/cpu/kernel/gdn/update.rs@@ -1,4 +1,4 @@-use half::{bf16, f16};+use half::bf16;use num_traits::Float;use proc_macros::kernel;@@ -11,7 +11,7 @@ use crate::{array::ArrayElement, backends::common::gpu_types::ActivationType};// 3. For each v-dim: read state, compute output, update state with delta rule// 4. RMSNorm output, gate with SiLU(z), write result#[kernel(DeltaNetUpdate)]-#[variants(T, f32, f16, bf16)]+#[variants(T, f32, bf16)]#[variants(HEAD_K_DIM, 128)]pub fn delta_net_update<T: ArrayElement + Float, const HEAD_K_DIM: u32>(in_proj: *const T,diff --git a/crates/backend-uzu/src/backends/cpu/kernel/mod.rs b/crates/backend-uzu/src/backends/cpu/kernel/mod.rsindex 3d903e94d..60227949b 100644--- a/crates/backend-uzu/src/backends/cpu/kernel/mod.rs+++ b/crates/backend-uzu/src/backends/cpu/kernel/mod.rs@@ -33,5 +33,6 @@ impl Kernels for CpuKernels {autogen_kernels!();type AttentionGemmCore = Infallible;type DeltaNetChunkedPrefill = Infallible;+ type DeltaNetTreeVerify = Infallible;type MatmulKernel = matmul::MatmulCpuKernel;}diff --git a/crates/backend-uzu/src/backends/metal/kernel/gdn/chunked/mod.rs b/crates/backend-uzu/src/backends/metal/kernel/gdn/chunked/mod.rsindex 0ec83e804..937f0c00a 100644--- a/crates/backend-uzu/src/backends/metal/kernel/gdn/chunked/mod.rs+++ b/crates/backend-uzu/src/backends/metal/kernel/gdn/chunked/mod.rs@@ -54,7 +54,7 @@ impl DeltaNetChunkedPrefill<Metal> for MetalDeltaNetChunkedPrefill {Ok(Some(Self {min_t: MXU_MIN_T,- prep: DeltaNetPrefillPrepMetalKernel::new(context, outer_data_type, head_dim, true)?,+ prep: DeltaNetPrefillPrepMetalKernel::new(context, outer_data_type, DataType::F32, head_dim, true, false)?,cumsum: DeltaNetChunkedCumsumMetalKernel::new(context, CHUNK_SIZE as u32)?,gram: DeltaNetChunkedGramMetalKernel::new(context, head_dim, CHUNK_SIZE as u32)?,a_diag_inv: DeltaNetChunkedADiagInvMetalKernel::new(context, CHUNK_SIZE as u32)?,@@ -124,6 +124,7 @@ impl DeltaNetChunkedPrefill<Metal> for MetalDeltaNetChunkedPrefill {args.dt_bias,&mut q_norm,&mut k_norm,+ None::<&mut crate::backends::common::Allocation<Metal>>,&mut beta,&mut log_decay,args.num_heads,diff --git a/crates/backend-uzu/src/backends/metal/kernel/gdn/conv_scan.metal b/crates/backend-uzu/src/backends/metal/kernel/gdn/conv_scan.metalindex 8aa1f1d0b..44aebdbe0 100644--- a/crates/backend-uzu/src/backends/metal/kernel/gdn/conv_scan.metal+++ b/crates/backend-uzu/src/backends/metal/kernel/gdn/conv_scan.metal@@ -15,7 +15,7 @@ using namespace metal;// state_out: [conv_dim, state_stride]template <typename T>-VARIANTS(T, float, half, bfloat)+VARIANTS(T, float, bfloat)PUBLIC KERNEL(DeltaNetConvScan)(device const float* conv_padded,device const float* conv_weight,diff --git a/crates/backend-uzu/src/backends/metal/kernel/gdn/conv_update.metal b/crates/backend-uzu/src/backends/metal/kernel/gdn/conv_update.metalindex d9450a0f6..3547af513 100644--- a/crates/backend-uzu/src/backends/metal/kernel/gdn/conv_update.metal+++ b/crates/backend-uzu/src/backends/metal/kernel/gdn/conv_update.metal@@ -6,7 +6,7 @@ using namespace metal;// Single-token causal conv1d with SiLU, in-place.template <typename T>-VARIANTS(T, float, half, bfloat)+VARIANTS(T, float, bfloat)PUBLIC KERNEL(DeltaNetConvUpdate)(device const float* conv_weight,device const float* bias OPTIONAL(has_bias),diff --git a/crates/backend-uzu/src/backends/metal/kernel/gdn/mod.rs b/crates/backend-uzu/src/backends/metal/kernel/gdn/mod.rsindex 60c62629e..2082e1623 100644--- a/crates/backend-uzu/src/backends/metal/kernel/gdn/mod.rs+++ b/crates/backend-uzu/src/backends/metal/kernel/gdn/mod.rs@@ -1 +1,2 @@pub mod chunked;+pub mod tree_verify;diff --git a/crates/backend-uzu/src/backends/metal/kernel/gdn/norm_gate.metal b/crates/backend-uzu/src/backends/metal/kernel/gdn/norm_gate.metalindex 7933c475f..281c153ff 100644--- a/crates/backend-uzu/src/backends/metal/kernel/gdn/norm_gate.metal+++ b/crates/backend-uzu/src/backends/metal/kernel/gdn/norm_gate.metal@@ -15,7 +15,7 @@ using namespace metal;#define HEAD_V_DIM 128template <typename T>-VARIANTS(T, float, half, bfloat)+VARIANTS(T, float, bfloat)PUBLIC KERNEL(DeltaNetNormGate)(device T* in_out,device const T* in_proj,diff --git a/crates/backend-uzu/src/backends/metal/kernel/gdn/prefill.metal b/crates/backend-uzu/src/backends/metal/kernel/gdn/prefill.metalindex 8f24c8c02..82db56b15 100644--- a/crates/backend-uzu/src/backends/metal/kernel/gdn/prefill.metal+++ b/crates/backend-uzu/src/backends/metal/kernel/gdn/prefill.metal@@ -13,7 +13,7 @@ using namespace metal;static_assert(PREFILL_THREADS % METAL_SIMD_SIZE == 0, "PREFILL_THREADS must be a multiple of METAL_SIMD_SIZE");template <typename T, uint HEAD_K_DIM>-VARIANTS(T, float, half, bfloat)+VARIANTS(T, float, bfloat)VARIANTS(HEAD_K_DIM, 128)PUBLIC KERNEL(DeltaNetPrefill)(device const float* q_norm,diff --git a/crates/backend-uzu/src/backends/metal/kernel/gdn/prefill_prep.metal b/crates/backend-uzu/src/backends/metal/kernel/gdn/prefill_prep.metalindex ee6f9cd92..50ed4b4d0 100644--- a/crates/backend-uzu/src/backends/metal/kernel/gdn/prefill_prep.metal+++ b/crates/backend-uzu/src/backends/metal/kernel/gdn/prefill_prep.metal@@ -5,15 +5,17 @@using namespace metal;-template <typename T, uint HEAD_K_DIM>-VARIANTS(T, float, half, bfloat)+template <typename T, typename QKT, uint HEAD_K_DIM>+VARIANTS(T, float, bfloat)+VARIANTS(QKT, float, bfloat)VARIANTS(HEAD_K_DIM, 128)PUBLIC KERNEL(DeltaNetPrefillPrep)(device const T* in_proj,device const float* a_log,device const float* dt_bias,- device float* q_norm_out,- device float* k_norm_out,+ device QKT* q_norm_out,+ device QKT* k_norm_out,+ device T* compact_v_out OPTIONAL(write_compact_v),device float* beta_out,device float* decay_out,constant const uint& num_v_heads,@@ -24,6 +26,7 @@ PUBLIC KERNEL(DeltaNetPrefillPrep)(const uint token_idx GROUPS(suffix_len),const uint hk_idx GROUPS(num_k_heads),const bool write_log_decay SPECIALIZE,+ const bool write_compact_v SPECIALIZE,const uint lane THREADS(METAL_SIMD_SIZE)) {static_assert(HEAD_K_DIM % METAL_SIMD_SIZE == 0, "HEAD_K_DIM must be a multiple of METAL_SIMD_SIZE");@@ -57,8 +60,8 @@ PUBLIC KERNEL(DeltaNetPrefillPrep)(const uint out_base = token_idx * key_dim + hk_idx * HEAD_K_DIM;for (uint i = 0; i < elems_per_thread; ++i) {const uint idx = lane + i * METAL_SIMD_SIZE;- q_norm_out[out_base + idx] = q_vals[i] * q_inv_norm * q_scale;- k_norm_out[out_base + idx] = k_vals[i] * k_inv_norm;+ q_norm_out[out_base + idx] = static_cast<QKT>(q_vals[i] * q_inv_norm * q_scale);+ k_norm_out[out_base + idx] = static_cast<QKT>(k_vals[i] * k_inv_norm);}for (uint group = lane; group < groups_per_head; group += METAL_SIMD_SIZE) {@@ -74,4 +77,15 @@ PUBLIC KERNEL(DeltaNetPrefillPrep)(decay_out[token_idx * num_v_heads + hv] = fast::exp(log_decay);}}++ if (write_compact_v) {+ const uint head_v_dim = value_dim / num_v_heads;+ for (uint group = 0; group < groups_per_head; ++group) {+ const uint hv = hk_idx * groups_per_head + group;+ for (uint dv = lane; dv < head_v_dim; dv += METAL_SIMD_SIZE) {+ const uint v_idx = hv * head_v_dim + dv;+ compact_v_out[token_idx * value_dim + v_idx] = in_proj[tok_offset + 2 * key_dim + v_idx];+ }+ }+ }}diff --git a/crates/backend-uzu/src/backends/metal/kernel/gdn/tree_verify.rs b/crates/backend-uzu/src/backends/metal/kernel/gdn/tree_verify.rsnew file mode 100644index 000000000..c786faad6--- /dev/null+++ b/crates/backend-uzu/src/backends/metal/kernel/gdn/tree_verify.rs@@ -0,0 +1,166 @@+use crate::{+ array::size_for_shape,+ backends::{+ common::{+ Allocation, Encoder, Kernels,+ kernel::{+ BuildTreeGramKernel, BuildTreeOutKernel, BuildTreePrefixKernel, TreeUpdateSolveKernel,+ delta_net_tree_verify::DeltaNetTreeVerify,+ },+ },+ metal::{Metal, MetalContext, device_tier::DeviceTier, error::MetalError, kernel::MetalKernels},+ },+ data_type::DataType,+ encodable_block::mixer::delta_net::tree_verify::{TreeVerifyEncodeArguments, TreeVerifyNewArguments},+};++const TOKEN_BLOCK: usize = 16;+const BLOCK_PAIR_WIDTH: usize = 2 * TOKEN_BLOCK;++struct Layout {+ tree_size: usize,+ num_blocks: usize,+ num_block_pairs: usize,+ num_v_heads: usize,+ head_v_dim: usize,+}++impl Layout {+ const fn new(+ tree_size: usize,+ arguments: &TreeVerifyNewArguments,+ ) -> Self {+ let num_blocks = tree_size.div_ceil(TOKEN_BLOCK);+ Self {+ tree_size,+ num_blocks,+ num_block_pairs: num_blocks.div_ceil(2),+ num_v_heads: arguments.num_v_heads,+ head_v_dim: arguments.head_v_dim,+ }+ }++ const fn a_packed_shape(&self) -> [usize; 5] {+ [self.num_v_heads, self.num_blocks, self.num_block_pairs, TOKEN_BLOCK, BLOCK_PAIR_WIDTH]+ }++ const fn a_inverse_shape(&self) -> [usize; 4] {+ [self.num_v_heads, self.num_blocks, TOKEN_BLOCK, TOKEN_BLOCK]+ }+}++pub struct MetalDeltaNetTreeVerify {+ arguments: TreeVerifyNewArguments,+ prefix: <MetalKernels as Kernels>::BuildTreePrefixKernel,+ gram: <MetalKernels as Kernels>::BuildTreeGramKernel,+ solve: <MetalKernels as Kernels>::TreeUpdateSolveKernel,+ out: <MetalKernels as Kernels>::BuildTreeOutKernel,+}++impl DeltaNetTreeVerify<Metal> for MetalDeltaNetTreeVerify {+ fn is_supported(_context: &MetalContext) -> bool {+ true+ }++ fn new(+ context: &MetalContext,+ arguments: &TreeVerifyNewArguments,+ ) -> Result<Self, MetalError> {+ let use_mxu = arguments.data_type == DataType::BF16 && context.supports_mxu();+ let transposed_h0 =+ !use_mxu && matches!(context.device_tier(), DeviceTier::SmallLegacy | DeviceTier::SmallApple8);+ Ok(Self {+ arguments: *arguments,+ prefix: <MetalKernels as Kernels>::BuildTreePrefixKernel::new(context)?,+ gram: <MetalKernels as Kernels>::BuildTreeGramKernel::new(context, arguments.data_type, use_mxu, true)?,+ solve: <MetalKernels as Kernels>::TreeUpdateSolveKernel::new(context, arguments.data_type, 32, true)?,+ out: <MetalKernels as Kernels>::BuildTreeOutKernel::new(+ context,+ arguments.data_type,+ arguments.data_type,+ use_mxu,+ transposed_h0,+ true,+ )?,+ })+ }++ fn encode(+ &self,+ arguments: TreeVerifyEncodeArguments<'_, Metal>,+ encoder: &mut Encoder<Metal>,+ ) -> Result<Allocation<Metal>, MetalError> {+ let layout = Layout::new(arguments.tree_size, &self.arguments);+ let mut h0_indices = encoder.allocate_constant(DataType::I32.size_in_bytes())?;+ h0_indices.copyin(&[0i32]);+ let mut scratch = |shape: &[usize], data_type| encoder.allocate_scratch(size_for_shape(shape, data_type));+ let mut prefix = scratch(&[layout.tree_size, layout.num_v_heads], DataType::F32)?;+ let mut a_packed = scratch(&layout.a_packed_shape(), DataType::F32)?;+ let mut qkd = scratch(&[layout.num_v_heads, layout.tree_size, layout.tree_size], DataType::F32)?;+ let mut a_inverse = scratch(&layout.a_inverse_shape(), DataType::F32)?;+ let mut kh0 = scratch(&[layout.tree_size, layout.num_v_heads, layout.head_v_dim], DataType::F32)?;+ let mut u = scratch(&[layout.num_v_heads, layout.tree_size, layout.head_v_dim], DataType::F32)?;+ let mut output = scratch(&[layout.tree_size, layout.num_v_heads, layout.head_v_dim], self.arguments.data_type)?;+ let tree_size = arguments.tree_size.try_into().expect("tree size exceeds u32");+ let num_k_heads = self.arguments.num_k_heads.try_into().expect("K head count exceeds u32");+ let num_v_heads = self.arguments.num_v_heads.try_into().expect("V head count exceeds u32");+ let head_k_dim = self.arguments.head_k_dim.try_into().expect("K head dimension exceeds u32");+ let head_v_dim = self.arguments.head_v_dim.try_into().expect("V head dimension exceeds u32");++ self.prefix.encode(arguments.trie, arguments.log_decay, &mut prefix, 1, tree_size, num_v_heads, encoder);+ self.gram.encode(+ arguments.q,+ arguments.k,+ arguments.trie,+ &prefix,+ arguments.beta,+ Some(arguments.h0),+ Some(&h0_indices),+ &mut a_packed,+ &mut qkd,+ &mut a_inverse,+ Some(&mut kh0),+ 1.0,+ 1,+ tree_size,+ num_k_heads,+ num_v_heads,+ head_k_dim,+ head_v_dim,+ encoder,+ );+ self.solve.encode(+ Some(&kh0),+ arguments.v,+ &prefix,+ arguments.beta,+ &a_packed,+ &a_inverse,+ Some(&h0_indices),+ &mut u,+ 1,+ tree_size,+ num_v_heads,+ head_v_dim,+ encoder,+ );+ self.out.encode(+ arguments.q,+ &prefix,+ &qkd,+ &u,+ Some(arguments.h0),+ Some(&h0_indices),+ &mut output,+ 1.0,+ 1,+ tree_size,+ num_k_heads,+ num_v_heads,+ head_k_dim,+ head_v_dim,+ encoder,+ );+ Ok(output)+ }+}diff --git a/crates/backend-uzu/src/backends/metal/kernel/gdn/tree_verify/build_tree_out_dispatch_helper.rs b/crates/backend-uzu/src/backends/metal/kernel/gdn/tree_verify/build_tree_out_dispatch_helper.rsdeleted file mode 100644index a57e2f54e..000000000--- a/crates/backend-uzu/src/backends/metal/kernel/gdn/tree_verify/build_tree_out_dispatch_helper.rs+++ /dev/null@@ -1,11 +0,0 @@-#![allow(dead_code)]--use crate::backends::metal::device_tier::DeviceTier;--pub(crate) fn build_tree_out_transposed_h0(- use_mxu: bool,- device_tier: DeviceTier,- use_h0: bool,-) -> bool {- use_h0 && !use_mxu && matches!(device_tier, DeviceTier::SmallLegacy | DeviceTier::SmallApple8)-}diff --git a/crates/backend-uzu/src/backends/metal/kernel/gdn/tree_verify/out.metal b/crates/backend-uzu/src/backends/metal/kernel/gdn/tree_verify/out.metalindex 1a6446c7e..310aa8050 100644--- a/crates/backend-uzu/src/backends/metal/kernel/gdn/tree_verify/out.metal+++ b/crates/backend-uzu/src/backends/metal/kernel/gdn/tree_verify/out.metal@@ -17,29 +17,27 @@ using namespace uzu::matmul;// Build tree output for one (batch, value-head, row, value) tile.//// q: [B, T, Hg, K], with query head hq = hv / (HV / Hg)-// prefix: [B, T, HV] path log-decay prefix from BuildPrefixBeta+// prefix: [B, T, HV] path log-decay prefix from BuildTreePrefix// qkd: [B, HV, T, T] from BuildTreeGram// u: [B, HV, T, V]// h0: [H0, HV, V, K]//// o[row, value] = exp(prefix[row]) * scale * dot(q[row], h0[value])// + sum_j qkd[row, j] * u[j, value]-template <- typename T,- bool use_mxu,- bool transposed_h0>-VARIANTS(T, float, bfloat)+template <typename QKT, typename OutputT, bool use_mxu, bool transposed_h0>+VARIANTS(QKT, float, bfloat)+VARIANTS(OutputT, float, bfloat)VARIANTS(use_mxu, false, true)VARIANTS(transposed_h0, false, true)CONSTRAINT(!(use_mxu && transposed_h0))PUBLIC KERNEL(BuildTreeOut)(- const device T* q,+ const device QKT* q,const device float* prefix,const device float* qkd,- const device T* u,+ const device float* u,const device float* h0 OPTIONAL(use_h0),const device int* h0_indices OPTIONAL(use_h0),- device T* o,+ device OutputT* o,constant const float& scale,constant const uint& batch_size,constant const uint& tree_size,@@ -61,17 +59,17 @@ PUBLIC KERNEL(BuildTreeOut)(const uint batch_value_head_idx GROUPS(batch_size * value_heads),const uint tid THREADS(SIMDGROUPS_PER_TG * METAL_SIMD_SIZE)) {- using Ops = metal::conditional_t<use_mxu, MxuFragmentOps<>, SimdgroupFragmentOps>;- using InputType = metal::conditional_t<use_mxu, T, float>;+ // Relaxed MPP diverges from FP32 across layers.+ using Ops = metal::conditional_t<use_mxu, MxuStrictFragmentOps, SimdgroupFragmentOps>;using H0Read = metal::conditional_t<transposed_h0, ReadTranspose, ReadDirect>;constexpr ushort ROWS = Ops::FRAGMENT_ROWS;constexpr ushort COL_FRAGMENTS = MATMUL_COLS / Ops::FRAGMENT_COLS;using AccFragment = Fragment<float, 1, COL_FRAGMENTS, Ops>;- using QFragment = OperandFragment<InputType, 1, 1, Ops>;+ using QFragment = OperandFragment<float, 1, 1, Ops>;using H0Fragment = OperandFragment<float, 1, COL_FRAGMENTS, Ops, H0Read>;using QkdFragment = OperandFragment<float, 1, 1, Ops>;- using UFragment = OperandFragment<InputType, 1, COL_FRAGMENTS, Ops>;- threadgroup T* u_tile = reinterpret_cast<threadgroup T*>(u_tile_scratch);+ using UFragment = OperandFragment<float, 1, COL_FRAGMENTS, Ops>;+ threadgroup float* u_tile = reinterpret_cast<threadgroup float*>(u_tile_scratch);const uint batch_idx = batch_value_head_idx / value_heads;const uint value_head_idx = batch_value_head_idx - batch_idx * value_heads;@@ -136,7 +134,7 @@ PUBLIC KERNEL(BuildTreeOut)(const uint local_j = index / MATMUL_COLS;const uint local_col = index - local_j * MATMUL_COLS;const bool in_bounds = local_j < uint(valid_j) && local_col < uint(valid_cols);- u_tile[index] = in_bounds ? u[u_base + (j0 + local_j) * head_v_dim + local_col] : T(0);+ u_tile[index] = in_bounds ? u[u_base + (j0 + local_j) * head_v_dim + local_col] : 0.0f;}threadgroup_barrier(mem_flags::mem_threadgroup);u_frag.load_from(@@ -154,7 +152,7 @@ PUBLIC KERNEL(BuildTreeOut)(}}- device T* out_tile =+ device OutputT* out_tile =o + ((batch_idx * tree_size + row_base) * value_heads + value_head_idx) * head_v_dim + value_base;acc.store_safe(thread_context.simd_lane_id, out_tile, int(value_heads * head_v_dim), short2(valid_cols, valid_rows));}diff --git a/crates/backend-uzu/src/backends/metal/kernel/gdn/tree_verify/prefix_beta.metal b/crates/backend-uzu/src/backends/metal/kernel/gdn/tree_verify/prefix.metalsimilarity index 62%rename from crates/backend-uzu/src/backends/metal/kernel/gdn/tree_verify/prefix_beta.metalrename to crates/backend-uzu/src/backends/metal/kernel/gdn/tree_verify/prefix.metalindex e498d5aeb..6fe092adb 100644--- a/crates/backend-uzu/src/backends/metal/kernel/gdn/tree_verify/prefix_beta.metal+++ b/crates/backend-uzu/src/backends/metal/kernel/gdn/tree_verify/prefix.metal@@ -1,5 +1,4 @@#include <metal_stdlib>-#include "../../activation/activations.h"#include "../../common/defines.h"#include "../../common/dsl.h"#include "../../common/thread_context.h"@@ -12,19 +11,12 @@ using namespace uzu::trie;// trie: [B, T] TrieNode — col is an ancestor-or-self of row iff// trie[col].trie_start <= row <= trie[col].trie_end.-// a_transposed: [B, HV, T]-// b: [B, T, HV]-// prefix,beta: [B, T, HV] fp32-template <typename T>-VARIANTS(T, float, half, bfloat)-PUBLIC KERNEL(BuildPrefixBeta)(+// log_decay: [B, T, HV]+// prefix: [B, T, HV] fp32+PUBLIC KERNEL(BuildTreePrefix)(const device TrieNode* trie,- const device T* a_transposed,- const device T* b,- const device float* a_log,- const device float* dt_bias,+ const device float* log_decay,device float* prefix,- device float* beta,constant const uint& batch_size,constant const uint& tree_size,constant const uint& value_heads,@@ -32,7 +24,7 @@ PUBLIC KERNEL(BuildPrefixBeta)(const uint batch_idx GROUPS(batch_size),const uint row_idx GROUPS(tree_size),const uint head_group_idx GROUPS(value_heads.div_ceil(ROWS_PER_THREADGROUP)),- const uint tid THREADS(ROWS_PER_THREADGROUP * METAL_SIMD_SIZE)+ const uint tid THREADS(ROWS_PER_THREADGROUP* METAL_SIMD_SIZE)) {const uint simd_group = thread_context.simdgroup_index;const uint lane_id = thread_context.simd_lane_id;@@ -43,23 +35,16 @@ PUBLIC KERNEL(BuildPrefixBeta)(const uint batch_offset = batch_idx * tree_size * value_heads;const uint trie_offset = batch_idx * tree_size;- const uint out_idx = batch_offset + row_idx * value_heads + head_idx;-- const float scale = exp(float(a_log[head_idx]));- const float dt = float(dt_bias[head_idx]);float partial = 0.0f;for (uint col_idx = lane_id; col_idx < tree_size; col_idx += METAL_SIMD_SIZE) {const TrieNode node = trie[trie_offset + col_idx];if (row_idx >= node.trie_start && row_idx <= node.trie_end) {- const uint a_idx = batch_offset + head_idx * tree_size + col_idx;- partial -= scale * activate_softplus(float(a_transposed[a_idx]) + dt);+ partial += log_decay[batch_offset + col_idx * value_heads + head_idx];}}const float sum = simd_sum(partial);if (lane_id == 0) {- const float b_val = float(b[out_idx]);- beta[out_idx] = 1.0f / (1.0f + exp(-b_val));- prefix[out_idx] = sum;+ prefix[batch_offset + row_idx * value_heads + head_idx] = sum;}}diff --git a/crates/backend-uzu/src/backends/metal/kernel/gdn/tree_verify/state_advance.metal b/crates/backend-uzu/src/backends/metal/kernel/gdn/tree_verify/state_advance.metalindex 46d8f63dd..9b6c3afe6 100644--- a/crates/backend-uzu/src/backends/metal/kernel/gdn/tree_verify/state_advance.metal+++ b/crates/backend-uzu/src/backends/metal/kernel/gdn/tree_verify/state_advance.metal@@ -15,7 +15,7 @@ VARIANTS(T, float, bfloat)VARIANTS(HEAD_K_DIM, 128)PUBLIC KERNEL(StateAdvance)(// [tree_size, num_k_heads, HEAD_K_DIM]- device const float* k_norm,+ device const T* k_norm,// [tree_size, num_v_heads, HEAD_K_DIM]device const T* v,// [tree_size, num_v_heads]@@ -27,8 +27,8 @@ PUBLIC KERNEL(StateAdvance)(// [num_v_heads, HEAD_K_DIM, HEAD_K_DIM], input/outputdevice float* state,constant const uint& accepted_len,- constant const uint& num_v_heads,- constant const uint& num_k_heads,+ const uint num_v_heads SPECIALIZE,+ const uint num_k_heads SPECIALIZE,const ThreadContext thread_context,const uint hv_idx GROUPS(num_v_heads),const uint dv_group_idx GROUPS(STATE_ADVANCE_DV_GROUPS),@@ -60,7 +60,7 @@ PUBLIC KERNEL(StateAdvance)(const float decay = fast::exp(log_decay_buf[tree_head_offset]);const float beta = beta_buf[tree_head_offset];const uint k_offset = tree_idx * key_dim + hk_idx * HEAD_K_DIM + dk_base;- const float4 k = *reinterpret_cast<device const float4*>(k_norm + k_offset);+ const float4 k = float4(*reinterpret_cast<const device vec<T, 4>*>(k_norm + k_offset));state_rows[0] *= decay;state_rows[1] *= decay;diff --git a/crates/backend-uzu/src/backends/metal/kernel/gdn/tree_verify/tree_gram.metal b/crates/backend-uzu/src/backends/metal/kernel/gdn/tree_verify/tree_gram.metalindex dc2f1821c..75a681e98 100644--- a/crates/backend-uzu/src/backends/metal/kernel/gdn/tree_verify/tree_gram.metal+++ b/crates/backend-uzu/src/backends/metal/kernel/gdn/tree_verify/tree_gram.metal@@ -23,7 +23,7 @@ using namespace uzu::trie;template <typename T, typename Ops, typename AccFragment, typename LeftFragment, typename RightFragment>METAL_FUNC void tree_kh0(const device T* k_rows,- const device T* h0,+ const device float* h0,const device int* h0_idx,device float* kh0,const uint batch_idx,@@ -44,7 +44,7 @@ METAL_FUNC void tree_kh0(if (h0_slot < 0) {return;}- const device T* h0_head = h0 + (uint(h0_slot) * value_heads + value_head_idx) * head_v_dim * head_k_dim;+ const device float* h0_head = h0 + (uint(h0_slot) * value_heads + value_head_idx) * head_v_dim * head_k_dim;device float* kh0_rows =kh0 + (batch_idx * tree_size + row_base) * value_heads * head_v_dim + value_head_idx * head_v_dim;const uint kh0_row_stride = value_heads * head_v_dim;@@ -54,7 +54,7 @@ METAL_FUNC void tree_kh0(AccFragment kh0_acc;kh0_acc.clear();- const device T* h0_tile = h0_head + dv_base * head_k_dim;+ const device float* h0_tile = h0_head + dv_base * head_k_dim;const bool full_dvs = tile_dvs == COL_TILE;for (uint kb = 0; kb < head_k_dim; kb += Ops::FRAGMENT_ROWS) {@@ -96,7 +96,7 @@ PUBLIC KERNEL(BuildTreeGram)(const device TrieNode* trie,const device float* prefix,const device float* beta,- const device T* h0 OPTIONAL(use_h0),+ const device float* h0 OPTIONAL(use_h0),const device int* h0_idx OPTIONAL(use_h0),device float* a_packed,device float* qkd,@@ -128,10 +128,9 @@ PUBLIC KERNEL(BuildTreeGram)(using Ops = metal::conditional_t<USE_MXU, MxuFragmentOps<>, SimdgroupFragmentOps>;constexpr ushort ROW_FRAGMENTS = ROW_TILE / Ops::FRAGMENT_ROWS;constexpr ushort COL_FRAGMENTS = COL_TILE / Ops::FRAGMENT_COLS;- using InputType = metal::conditional_t<USE_MXU, T, float>;using AccFragment = Fragment<float, ROW_FRAGMENTS, COL_FRAGMENTS, Ops>;- using LeftFragment = OperandFragment<InputType, ROW_FRAGMENTS, 1, Ops>;- using RightFragment = OperandFragment<InputType, 1, COL_FRAGMENTS, Ops, ReadTranspose>;+ using LeftFragment = OperandFragment<float, ROW_FRAGMENTS, 1, Ops>;+ using RightFragment = OperandFragment<float, 1, COL_FRAGMENTS, Ops, ReadTranspose>;static_assert(COL_TILE == METAL_SIMD_SIZE, "COL_TILE must match the SIMD width");const uint key_head_idx = value_head_idx / (value_heads / k_heads);diff --git a/crates/backend-uzu/src/backends/metal/kernel/gdn/tree_verify/tree_update_solve.metal b/crates/backend-uzu/src/backends/metal/kernel/gdn/tree_verify/tree_update_solve.metalindex 8e8627c2e..a3c98aa93 100644--- a/crates/backend-uzu/src/backends/metal/kernel/gdn/tree_verify/tree_update_solve.metal+++ b/crates/backend-uzu/src/backends/metal/kernel/gdn/tree_verify/tree_update_solve.metal@@ -2,7 +2,6 @@#include "../../common/defines.h"#include "../../common/dsl.h"#include "../../matmul/common/fragment.h"-#include "../../matmul/common/mxu_fragment_ops.h"#include "../../matmul/common/simdgroup_fragment_ops.h"using namespace metal;@@ -27,12 +26,9 @@ using namespace uzu::matmul;// single-block tail when i is odd.// A and Ainv are kept in f32: the (I + A)^-1 forward-substitution cascade is// precision-sensitive.-template <typename T, uint BV, bool USE_MXU>+template <typename T, uint BV>VARIANTS(T, float, bfloat)VARIANTS(BV, 16, 32)-VARIANTS(USE_MXU, false, true)-CONSTRAINT(!USE_MXU || T != "float")-CONSTRAINT(!USE_MXU || BV >= 32)PUBLIC KERNEL(TreeUpdateSolve)(const device float* kh0 OPTIONAL(use_h0),const device T* v,@@ -53,7 +49,8 @@ PUBLIC KERNEL(TreeUpdateSolve)(const uint lane_idx THREADS(METAL_SIMD_SIZE)) {constexpr uint BT = 16;- using FragmentOps = metal::conditional_t<USE_MXU, MxuFragmentOps<>, SimdgroupFragmentOps>;+ // MXU was within 5% of SIMD-group at T=32..512.+ using FragmentOps = SimdgroupFragmentOps;constexpr ushort TOKEN_FRAGMENTS = BT / FragmentOps::FRAGMENT_ROWS;constexpr ushort VALUE_FRAGMENTS = BV / FragmentOps::FRAGMENT_COLS;using TileFragment = Fragment<float, TOKEN_FRAGMENTS, VALUE_FRAGMENTS, FragmentOps>;diff --git a/crates/backend-uzu/src/backends/metal/kernel/gdn/tree_verify/tree_update_solve_dispatch_helper.rs b/crates/backend-uzu/src/backends/metal/kernel/gdn/tree_verify/tree_update_solve_dispatch_helper.rsdeleted file mode 100644index d3e5c0905..000000000--- a/crates/backend-uzu/src/backends/metal/kernel/gdn/tree_verify/tree_update_solve_dispatch_helper.rs+++ /dev/null@@ -1,55 +0,0 @@-#![allow(dead_code)]--#[derive(Debug, Clone, Copy, PartialEq, Eq)]-pub(crate) struct TreeUpdateSolveVariant {- pub use_mxu: bool,- pub bv: u32,-}--/// Fastest TreeUpdateSolve variant per device + shape, from the 9-chip fleet sweep-/// (raw table in .claude/plans/solve_u_kernel_optimization.md). The kernel is a-/// latency-bound serial recurrence at B=1 small-T, where the narrower BV16 wins on-/// occupancy; the tree size at which the wider BV32 takes over grows with GPU-/// width. MXU chips switch to the matrix unit past that same B=1 small-T corner.-pub(crate) fn tree_update_solve_variant(- supports_mxu: bool,- gpu_core_count: u32,- batch_size: u32,- tree_size: u32,-) -> TreeUpdateSolveVariant {- if supports_mxu {- let small_corner = batch_size == 1 && tree_size <= 64;- return TreeUpdateSolveVariant {- use_mxu: !small_corner,- bv: if small_corner {- 16- } else {- 32- },- };- }- let bv16 = if gpu_core_count >= 30 {- batch_size == 1 || (batch_size <= 8 && tree_size <= 64)- } else {- let bv16_max_tree = if gpu_core_count >= 10 {- 128- } else if gpu_core_count >= 8 {- 33- } else {- 0- };- batch_size == 1 && tree_size <= bv16_max_tree- };- TreeUpdateSolveVariant {- use_mxu: false,- bv: if bv16 {- 16- } else {- 32- },- }-}--#[cfg(test)]-#[path = "../../../../../../tests/unit/backends/metal/tree_update_solve_dispatch_test.rs"]-mod tests;diff --git a/crates/backend-uzu/src/backends/metal/kernel/gdn/update.metal b/crates/backend-uzu/src/backends/metal/kernel/gdn/update.metalindex c99b2add8..b4ef42ce9 100644--- a/crates/backend-uzu/src/backends/metal/kernel/gdn/update.metal+++ b/crates/backend-uzu/src/backends/metal/kernel/gdn/update.metal@@ -14,7 +14,7 @@ using namespace metal;// One simd group per output dim dv; 32 lanes cover Dk (coalesced state IO,// reduction via simd_sum). Activations are model dtype T; state stays float.template <typename T, uint HEAD_K_DIM>-VARIANTS(T, float, half, bfloat)+VARIANTS(T, float, bfloat)VARIANTS(HEAD_K_DIM, 128)PUBLIC KERNEL(DeltaNetUpdate)(device const T* in_proj,diff --git a/crates/backend-uzu/src/backends/metal/kernel/mod.rs b/crates/backend-uzu/src/backends/metal/kernel/mod.rsindex f4a414d40..e3d1126b3 100644--- a/crates/backend-uzu/src/backends/metal/kernel/mod.rs+++ b/crates/backend-uzu/src/backends/metal/kernel/mod.rs@@ -7,12 +7,8 @@ use crate::backends::{};pub mod attention;-#[path = "gdn/tree_verify/build_tree_out_dispatch_helper.rs"]-mod build_tree_out_dispatch_helper;pub mod gdn;pub mod matmul;-#[path = "gdn/tree_verify/tree_update_solve_dispatch_helper.rs"]-mod tree_update_solve_dispatch_helper;pub const MTLB: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/default.metallib"));@@ -26,5 +22,6 @@ impl Kernels for MetalKernels {autogen_kernels!();type AttentionGemmCore = attention::AttentionGemmMetalCore;type DeltaNetChunkedPrefill = gdn::chunked::MetalDeltaNetChunkedPrefill;+ type DeltaNetTreeVerify = gdn::tree_verify::MetalDeltaNetTreeVerify;type MatmulKernel = matmul::MatmulMetalKernel;}diff --git a/crates/backend-uzu/src/encodable_block/mixer/delta_net.rs b/crates/backend-uzu/src/encodable_block/mixer/delta_net.rsindex f511818a5..0a15d89c9 100644--- a/crates/backend-uzu/src/encodable_block/mixer/delta_net.rs+++ b/crates/backend-uzu/src/encodable_block/mixer/delta_net.rs@@ -1,13 +1,18 @@+use std::mem::size_of;+use thiserror::Error;use crate::{array::size_for_shape,backends::common::{Allocation, AllocationType, Backend, Context, Encoder, Kernels,+ gpu_types::trie::TrieNode,kernel::{- Conv1dPackKernel, DeltaNetConvScanKernel, DeltaNetConvUpdateKernel, DeltaNetNormGateKernel,- DeltaNetPrefillKernel, DeltaNetPrefillPrepKernel, DeltaNetUpdateKernel,+ Conv1dPackKernel, ConvTreeScanKernel, DeltaNetConvScanKernel, DeltaNetConvUpdateKernel,+ DeltaNetNormGateKernel, DeltaNetPrefillKernel, DeltaNetPrefillPrepKernel, DeltaNetUpdateKernel,+ StateAdvanceKernel,delta_net_chunked_prefill::{DeltaNetChunkedPrefill, DeltaNetChunkedPrefillArgs},+ delta_net_tree_verify::DeltaNetTreeVerify as DeltaNetTreeVerifyTrait,},},config::token_mixer::delta_net::DeltaNetConfig,@@ -15,16 +20,37 @@ use crate::{encodable_block::{batch_topology::BatchTopology,linear::{Linear, LinearBlockError},- mixer::{Mixer, MixerState, attention::rope::PrecalculatedRoPE},+ mixer::{+ Mixer, MixerState,+ attention::rope::PrecalculatedRoPE,+ delta_net::tree_verify::{TreeVerifyEncodeArguments, TreeVerifyNewArguments},+ },},parameters::{ParameterLoaderError, ParameterTree},utils::maybe_mut::MaybeMut,};+pub(crate) mod tree_verify;++enum DeltaNetSuffixStatus<B: Backend> {+ Flat {+ suffix_length: usize,+ },+ Tree {+ conv_states: Allocation<B>,+ k: Allocation<B>,+ v: Allocation<B>,+ log_decay: Allocation<B>,+ beta: Allocation<B>,+ parents: Box<[i32]>,+ },+}+pub struct DeltaNetState<B: Backend> {conv_state: Allocation<B>,ssm_state: Allocation<B>,- suffix_length: Option<usize>,+ suffix_status: Option<DeltaNetSuffixStatus<B>>,+ state_advance: <B::Kernels as Kernels>::StateAdvanceKernel,}impl<B: Backend> MixerState<B> for DeltaNetState<B> {@@ -40,9 +66,53 @@ impl<B: Backend> MixerState<B> for DeltaNetState<B> {fn encode_accept(&mut self,accepted_indices: &[usize],- _encoder: &mut Encoder<B>,+ encoder: &mut Encoder<B>,) -> Result<(), B::Error> {- assert!(self.suffix_length.take() == Some(*accepted_indices.last().unwrap() + 1));+ let suffix_status = self.suffix_status.take().expect("delta net state has no suffix to accept");+ let accepted_index = *accepted_indices.last().expect("delta net state attempted to accept zero indices");++ match suffix_status {+ DeltaNetSuffixStatus::Flat {+ suffix_length,+ } => {+ assert!(accepted_index == suffix_length - 1, "attempted a partial flat delta net accept");+ },+ DeltaNetSuffixStatus::Tree {+ conv_states,+ k,+ v,+ log_decay,+ beta,+ parents,+ } => {+ assert!(accepted_indices.iter().all(|&index| index < parents.len()));+ assert_eq!(parents[accepted_indices[0]], -1);+ assert!(accepted_indices.windows(2).all(|edge| parents[edge[1]] == edge[0] as i32));++ let conv_state_size = self.conv_state.size();+ encoder.encode_copy(+ &conv_states,+ accepted_index * conv_state_size..(accepted_index + 1) * conv_state_size,+ &mut self.conv_state,+ ..,+ );++ let accepted_indices = accepted_indices.iter().map(|&index| index as u32).collect::<Box<[_]>>();+ let mut accepted_indices_buffer =+ encoder.allocate_constant(accepted_indices.len() * DataType::U32.size_in_bytes())?;+ accepted_indices_buffer.copyin(&accepted_indices);+ self.state_advance.encode(+ &k,+ &v,+ &log_decay,+ &beta,+ &accepted_indices_buffer,+ &mut self.ssm_state,+ accepted_indices.len() as u32,+ encoder,+ );+ },+ }Ok(())}}@@ -64,14 +134,17 @@ pub struct DeltaNet<B: Backend> {conv_update: <B::Kernels as Kernels>::DeltaNetConvUpdateKernel,conv_pack: <B::Kernels as Kernels>::Conv1dPackKernel,conv_scan: <B::Kernels as Kernels>::DeltaNetConvScanKernel,+ conv_tree_scan: <B::Kernels as Kernels>::ConvTreeScanKernel,a_log: Allocation<B>,dt_bias: Allocation<B>,norm_weight: Allocation<B>,norm_epsilon: f32,delta_net_update: <B::Kernels as Kernels>::DeltaNetUpdateKernel,delta_net_prefill_prep: <B::Kernels as Kernels>::DeltaNetPrefillPrepKernel,+ delta_net_tree_prep: <B::Kernels as Kernels>::DeltaNetPrefillPrepKernel,delta_net_prefill: <B::Kernels as Kernels>::DeltaNetPrefillKernel,delta_net_norm_gate: <B::Kernels as Kernels>::DeltaNetNormGateKernel,+ tree_verify: Option<<B::Kernels as Kernels>::DeltaNetTreeVerify>,chunked: Option<<B::Kernels as Kernels>::DeltaNetChunkedPrefill>,out_projection: Box<dyn Linear<B>>,}@@ -152,6 +225,13 @@ impl<B: Backend> DeltaNet<B> {let conv_scan =<B::Kernels as Kernels>::DeltaNetConvScanKernel::new(context, outer_data_type, conv_config.has_biases).map_err(DeltaNetNewError::Backend)?;+ let conv_tree_scan = <B::Kernels as Kernels>::ConvTreeScanKernel::new(+ context,+ outer_data_type,+ config.kernel_size as u32,+ conv_config.has_biases,+ )+ .map_err(DeltaNetNewError::Backend)?;let a_log = parameter_tree.leaf("a_log")?.validate(&[config.num_heads], inner_data_type)?.read_allocation()?;let dt_bias =@@ -167,8 +247,19 @@ impl<B: Backend> DeltaNet<B> {let delta_net_prefill_prep = <B::Kernels as Kernels>::DeltaNetPrefillPrepKernel::new(context,outer_data_type,+ DataType::F32,config.head_dim as u32,false,+ false,+ )+ .map_err(DeltaNetNewError::Backend)?;+ let delta_net_tree_prep = <B::Kernels as Kernels>::DeltaNetPrefillPrepKernel::new(+ context,+ outer_data_type,+ outer_data_type,+ config.head_dim as u32,+ true,+ true,).map_err(DeltaNetNewError::Backend)?;let delta_net_prefill =@@ -176,6 +267,24 @@ impl<B: Backend> DeltaNet<B> {.map_err(DeltaNetNewError::Backend)?;let delta_net_norm_gate = <B::Kernels as Kernels>::DeltaNetNormGateKernel::new(context, outer_data_type).map_err(DeltaNetNewError::Backend)?;+ let tree_verify =+ if <<B::Kernels as Kernels>::DeltaNetTreeVerify as DeltaNetTreeVerifyTrait<B>>::is_supported(context) {+ Some(+ <<B::Kernels as Kernels>::DeltaNetTreeVerify as DeltaNetTreeVerifyTrait<B>>::new(+ context,+ &TreeVerifyNewArguments {+ data_type: outer_data_type,+ num_k_heads: config.num_groups,+ num_v_heads: config.num_heads,+ head_k_dim: config.head_dim,+ head_v_dim: config.value_head_dim,+ },+ )+ .map_err(DeltaNetNewError::Backend)?,+ )+ } else {+ None+ };let chunked =<B::Kernels as Kernels>::DeltaNetChunkedPrefill::new(context, outer_data_type, config.head_dim as u32)@@ -210,25 +319,123 @@ impl<B: Backend> DeltaNet<B> {conv_update,conv_pack,conv_scan,+ conv_tree_scan,a_log,dt_bias,norm_weight,norm_epsilon,delta_net_update,delta_net_prefill_prep,+ delta_net_tree_prep,delta_net_prefill,delta_net_norm_gate,+ tree_verify,chunked,out_projection,},in_projection_input_hadamard_factors,))}++ fn encode_tree_verify(+ &self,+ in_projected: Allocation<B>,+ batch_dim: &BatchTopology,+ state: &mut DeltaNetState<B>,+ encoder: &mut Encoder<B>,+ ) -> Result<Allocation<B>, B::Error> {+ let tree_verify = self.tree_verify.as_ref().expect("DeltaNet tree verification is unsupported");+ let tree_size = batch_dim.size();+ let mut parents = encoder.allocate_constant(tree_size * DataType::I32.size_in_bytes())?;+ parents.copyin(batch_dim.parents());+ let mut trie = encoder.allocate_constant(tree_size * size_of::<TrieNode>())?;+ trie.copyin(batch_dim.nodes());++ let mut conv_states = encoder+ .allocate_scratch(size_for_shape(&[tree_size, self.conv_dim, self.kernel_size - 1], DataType::F32))?;+ let mut k = encoder.allocate_scratch(size_for_shape(&[tree_size, self.key_dim], self.outer_data_type))?;+ let mut v = encoder.allocate_scratch(size_for_shape(&[tree_size, self.value_dim], self.outer_data_type))?;+ let mut beta = encoder.allocate_scratch(size_for_shape(&[tree_size, self.num_heads], DataType::F32))?;+ let mut log_decay = encoder.allocate_scratch(size_for_shape(&[tree_size, self.num_heads], DataType::F32))?;++ let mut scratch = |shape: &[usize], data_type| encoder.allocate_scratch(size_for_shape(shape, data_type));+ let mut tree_projected = scratch(&[tree_size, self.total_proj_dim], self.outer_data_type)?;+ let mut q = scratch(&[tree_size, self.key_dim], self.outer_data_type)?;++ self.conv_tree_scan.encode(+ &in_projected,+ &self.conv_weight,+ self.conv_bias.as_ref(),+ &state.conv_state,+ &parents,+ &mut tree_projected,+ &mut conv_states,+ tree_size as u32,+ self.total_proj_dim as u32,+ self.conv_dim as u32,+ encoder,+ );++ self.delta_net_tree_prep.encode(+ &tree_projected,+ &self.a_log,+ &self.dt_bias,+ &mut q,+ &mut k,+ Some(&mut v),+ &mut beta,+ &mut log_decay,+ self.num_heads as u32,+ self.num_groups as u32,+ self.key_dim as u32,+ self.value_dim as u32,+ tree_size as u32,+ encoder,+ );++ let mut delta_output = tree_verify.encode(+ TreeVerifyEncodeArguments {+ q: &q,+ k: &k,+ v: &v,+ trie: &trie,+ log_decay: &log_decay,+ beta: &beta,+ h0: &state.ssm_state,+ tree_size,+ },+ encoder,+ )?;+ self.delta_net_norm_gate.encode(+ &mut delta_output,+ &tree_projected,+ &self.norm_weight,+ self.num_heads as u32,+ self.value_head_dim as u32,+ self.value_dim as u32,+ self.conv_dim as u32,+ self.total_proj_dim as u32,+ self.norm_epsilon,+ tree_size as u32,+ encoder,+ );++ let output = self.out_projection.encode(delta_output, tree_size, encoder)?;+ state.suffix_status = Some(DeltaNetSuffixStatus::Tree {+ conv_states,+ k,+ v,+ log_decay,+ beta,+ parents: batch_dim.parents().into(),+ });+ Ok(output)+ }}impl<B: Backend> Mixer<B> for DeltaNet<B> {fn speculation_supported(&self) -> bool {- false+ self.tree_verify.is_some()}fn max_context_length(&self) -> Option<usize> {@@ -255,10 +462,19 @@ impl<B: Backend> Mixer<B> for DeltaNet<B> {zero_encoder.encode_fill(&mut ssm_state, 0);zero_encoder.end_encoding().submit().wait_until_completed()?;+ let state_advance = <B::Kernels as Kernels>::StateAdvanceKernel::new(+ context,+ self.outer_data_type,+ self.head_dim as u32,+ self.num_heads as u32,+ self.num_groups as u32,+ )?;+Ok(Box::new(DeltaNetState {conv_state,ssm_state,- suffix_length: None,+ suffix_status: None,+ state_advance,}))}@@ -272,20 +488,20 @@ impl<B: Backend> Mixer<B> for DeltaNet<B> {) -> Result<Allocation<B>, B::Error> {assert!(precalculated_rope.is_none(), "unexpected rope for delta net mixer");- if !batch_dim.full_accept() {- panic!("delta net doesn't support speculation");- }-let state = state.expect("delta net requires state");let state = state.downcast::<DeltaNetState<B>>().expect("incorrect type of delta net state");let MaybeMut::Mut(state) = state else {panic!("delta net doesn't support immutable state");};- assert!(state.suffix_length.is_none(), "delta net called with state with unaccepted tokens");+ assert!(state.suffix_status.is_none(), "delta net called with state with an unaccepted suffix");let mut in_projected = self.in_projection.encode(hidden, batch_dim.size(), encoder)?;+ if !batch_dim.full_accept() {+ return self.encode_tree_verify(in_projected, batch_dim, state, encoder);+ }+let mut delta_output =encoder.allocate_scratch(size_for_shape(&[batch_dim.size(), self.value_dim], self.outer_data_type))?;if batch_dim.size() == 1 {@@ -376,6 +592,7 @@ impl<B: Backend> Mixer<B> for DeltaNet<B> {&self.dt_bias,&mut prep_q_norm,&mut prep_k_norm,+ None::<&mut Allocation<B>>,&mut prep_beta,&mut prep_decay,self.num_heads as u32,@@ -418,7 +635,9 @@ impl<B: Backend> Mixer<B> for DeltaNet<B> {);}- state.suffix_length = Some(batch_dim.size());+ state.suffix_status = Some(DeltaNetSuffixStatus::Flat {+ suffix_length: batch_dim.size(),+ });self.out_projection.encode(delta_output, batch_dim.size(), encoder)}diff --git a/crates/backend-uzu/src/encodable_block/mixer/delta_net/tree_verify.rs b/crates/backend-uzu/src/encodable_block/mixer/delta_net/tree_verify.rsnew file mode 100644index 000000000..4f8b6f7a7--- /dev/null+++ b/crates/backend-uzu/src/encodable_block/mixer/delta_net/tree_verify.rs@@ -0,0 +1,29 @@+use crate::{+ backends::common::{Allocation, Backend},+ data_type::DataType,+};++#[derive(Clone, Copy)]+pub struct TreeVerifyNewArguments {+ pub data_type: DataType,+ pub num_k_heads: usize,+ pub num_v_heads: usize,+ pub head_k_dim: usize,+ pub head_v_dim: usize,+}++#[derive(Clone, Copy)]+pub struct TreeVerifyEncodeArguments<'a, B: Backend> {+ pub q: &'a Allocation<B>,+ pub k: &'a Allocation<B>,+ pub v: &'a Allocation<B>,+ pub trie: &'a Allocation<B>,+ pub log_decay: &'a Allocation<B>,+ pub beta: &'a Allocation<B>,+ pub h0: &'a Allocation<B>,+ pub tree_size: usize,+}++#[cfg(test)]+#[path = "../../../../unit/encodable_block/delta_net_tree_verify_bench.rs"]+mod tests;diff --git a/crates/backend-uzu/tests/unit/backends/metal/tree_update_solve_dispatch_test.rs b/crates/backend-uzu/tests/unit/backends/metal/tree_update_solve_dispatch_test.rsdeleted file mode 100644index db98fffd7..000000000--- a/crates/backend-uzu/tests/unit/backends/metal/tree_update_solve_dispatch_test.rs+++ /dev/null@@ -1,35 +0,0 @@-use proc_macros::uzu_test;--use super::tree_update_solve_variant;--fn variant(- supports_mxu: bool,- cores: u32,- batch: u32,- tree: u32,-) -> (bool, u32) {- let v = tree_update_solve_variant(supports_mxu, cores, batch, tree);- (v.use_mxu, v.bv)-}--#[uzu_test]-fn tree_update_solve_dispatch_matches_fleet() {- // MXU chips (M5+): simdgroup BV16 only at the B=1 small-T corner, MXU BV32 else.- assert_eq!(variant(true, 40, 1, 64), (false, 16));- assert_eq!(variant(true, 40, 1, 128), (true, 32));- assert_eq!(variant(true, 40, 8, 256), (true, 32));-- // Max/Ultra (M3/M4 Max, 40): BV16 across B=1 and B<=8 tiny trees, BV32 beyond.- assert_eq!(variant(false, 40, 1, 256), (false, 16));- assert_eq!(variant(false, 40, 8, 64), (false, 16));- assert_eq!(variant(false, 40, 8, 256), (false, 32));-- // Mid (M2/M4/M4 Pro, 10-20): BV16 through B=1 T<=128.- assert_eq!(variant(false, 20, 1, 128), (false, 16));- assert_eq!(variant(false, 10, 1, 256), (false, 32));-- // Narrow (M1, 8): BV16 only at the tiniest tree. Phone-class (A18, 5): never.- assert_eq!(variant(false, 8, 1, 33), (false, 16));- assert_eq!(variant(false, 8, 1, 64), (false, 32));- assert_eq!(variant(false, 5, 1, 33), (false, 32));-}diff --git a/crates/backend-uzu/unit/backends/common/kernel/gdn/delta_net_test.rs b/crates/backend-uzu/unit/backends/common/kernel/gdn/delta_net_test.rsindex afd333ab2..78971cc1e 100644--- a/crates/backend-uzu/unit/backends/common/kernel/gdn/delta_net_test.rs+++ b/crates/backend-uzu/unit/backends/common/kernel/gdn/delta_net_test.rs@@ -329,8 +329,10 @@ fn run_prefill_with_norm_gate_typed<T: ArrayElement>(let prep_k = <<Metal as Backend>::Kernels as Kernels>::DeltaNetPrefillPrepKernel::new(&context,T::data_type(),+ DataType::F32,head_k_dim as u32,false,+ false,).unwrap();let prefill_k = <<Metal as Backend>::Kernels as Kernels>::DeltaNetPrefillKernel::new(@@ -349,6 +351,7 @@ fn run_prefill_with_norm_gate_typed<T: ArrayElement>(&dt_bias_array,&mut q_norm_array,&mut k_norm_array,+ None::<&mut crate::backends::common::Allocation<Metal>>,&mut beta_array,&mut decay_array,num_v_heads as u32,@@ -491,25 +494,29 @@ fn test_delta_net_prefill_prep() {let conv_dim = 2 * key_dim + value_dim;let total_proj_dim = conv_dim + value_dim + num_v_heads + num_v_heads;- let in_proj: Vec<f32> = (0..suffix_len * total_proj_dim).map(|i| ((i % 37) as f32) * 0.02 - 0.3).collect();+ let in_proj: Vec<bf16> =+ (0..suffix_len * total_proj_dim).map(|i| bf16::from_f32(((i % 37) as f32) * 0.02 - 0.3)).collect();let a_log: Vec<f32> = (0..num_v_heads).map(|i| -1.5 + (i as f32) * 0.05).collect();let dt_bias: Vec<f32> = (0..num_v_heads).map(|i| 0.3 + (i as f32) * 0.02).collect();// CPU reference via Kernels traitlet cpu_ctx = <Cpu as Backend>::Context::new().expect("cpu context");- let cpu_in_proj = alloc_allocation_with_data::<Cpu, f32>(&cpu_ctx, &in_proj);+ let cpu_in_proj = alloc_allocation_with_data::<Cpu, bf16>(&cpu_ctx, &in_proj);let cpu_a_log = alloc_allocation_with_data::<Cpu, f32>(&cpu_ctx, &a_log);let cpu_dt_bias = alloc_allocation_with_data::<Cpu, f32>(&cpu_ctx, &dt_bias);- let mut cpu_q = alloc_allocation::<Cpu, f32>(&cpu_ctx, suffix_len * key_dim);- let mut cpu_k = alloc_allocation::<Cpu, f32>(&cpu_ctx, suffix_len * key_dim);+ let mut cpu_q = alloc_allocation::<Cpu, bf16>(&cpu_ctx, suffix_len * key_dim);+ let mut cpu_k = alloc_allocation::<Cpu, bf16>(&cpu_ctx, suffix_len * key_dim);+ let mut cpu_v = alloc_allocation::<Cpu, bf16>(&cpu_ctx, suffix_len * value_dim);let mut cpu_beta = alloc_allocation::<Cpu, f32>(&cpu_ctx, suffix_len * num_v_heads);let mut cpu_decay = alloc_allocation::<Cpu, f32>(&cpu_ctx, suffix_len * num_v_heads);let cpu_prep = <<Cpu as Backend>::Kernels as Kernels>::DeltaNetPrefillPrepKernel::new(&cpu_ctx,- DataType::F32,+ DataType::BF16,+ DataType::BF16,head_k_dim as u32,false,+ true,).unwrap();let mut cpu_enc = Encoder::new(cpu_ctx.as_ref()).expect("encoder");@@ -519,6 +526,7 @@ fn test_delta_net_prefill_prep() {&cpu_dt_bias,&mut cpu_q,&mut cpu_k,+ Some(&mut cpu_v),&mut cpu_beta,&mut cpu_decay,num_v_heads as u32,@@ -530,27 +538,36 @@ fn test_delta_net_prefill_prep() {);cpu_enc.end_encoding().submit().wait_until_completed().unwrap();- let ref_q: Vec<f32> = allocation_to_vec(&cpu_q);- let ref_k: Vec<f32> = allocation_to_vec(&cpu_k);+ let ref_q = allocation_to_vec::<Cpu, bf16>(&cpu_q).into_iter().map(f32::from).collect::<Vec<_>>();+ let ref_k = allocation_to_vec::<Cpu, bf16>(&cpu_k).into_iter().map(f32::from).collect::<Vec<_>>();+ let ref_v: Vec<bf16> = allocation_to_vec(&cpu_v);let ref_beta: Vec<f32> = allocation_to_vec(&cpu_beta);let ref_decay: Vec<f32> = allocation_to_vec(&cpu_decay);+ let expected_v = in_proj+ .chunks_exact(total_proj_dim)+ .flat_map(|row| row[2 * key_dim..2 * key_dim + value_dim].iter().copied())+ .collect::<Vec<_>>();+ assert_eq!(ref_v, expected_v);// Metallet context = <Metal as Backend>::Context::new().expect("context");- let in_proj_array = alloc_allocation_with_data::<Metal, f32>(&context, &in_proj);+ let in_proj_array = alloc_allocation_with_data::<Metal, bf16>(&context, &in_proj);let a_log_array = alloc_allocation_with_data::<Metal, f32>(&context, &a_log);let dt_bias_array = alloc_allocation_with_data::<Metal, f32>(&context, &dt_bias);- let mut q_norm_array = alloc_allocation::<Metal, f32>(&context, suffix_len * key_dim);- let mut k_norm_array = alloc_allocation::<Metal, f32>(&context, suffix_len * key_dim);+ let mut q_norm_array = alloc_allocation::<Metal, bf16>(&context, suffix_len * key_dim);+ let mut k_norm_array = alloc_allocation::<Metal, bf16>(&context, suffix_len * key_dim);+ let mut compact_v_array = alloc_allocation::<Metal, bf16>(&context, suffix_len * value_dim);let mut beta_array = alloc_allocation::<Metal, f32>(&context, suffix_len * num_v_heads);let mut decay_array = alloc_allocation::<Metal, f32>(&context, suffix_len * num_v_heads);let prep_k = <<Metal as Backend>::Kernels as Kernels>::DeltaNetPrefillPrepKernel::new(&context,- DataType::F32,+ DataType::BF16,+ DataType::BF16,head_k_dim as u32,false,+ true,).unwrap();@@ -561,6 +578,7 @@ fn test_delta_net_prefill_prep() {&dt_bias_array,&mut q_norm_array,&mut k_norm_array,+ Some(&mut compact_v_array),&mut beta_array,&mut decay_array,num_v_heads as u32,@@ -572,13 +590,15 @@ fn test_delta_net_prefill_prep() {);encoder.end_encoding().submit().wait_until_completed().unwrap();- let gpu_q: Vec<f32> = allocation_to_vec(&q_norm_array);- let gpu_k: Vec<f32> = allocation_to_vec(&k_norm_array);+ let gpu_q = allocation_to_vec::<Metal, bf16>(&q_norm_array).into_iter().map(f32::from).collect::<Vec<_>>();+ let gpu_k = allocation_to_vec::<Metal, bf16>(&k_norm_array).into_iter().map(f32::from).collect::<Vec<_>>();+ let gpu_v: Vec<bf16> = allocation_to_vec(&compact_v_array);let gpu_beta: Vec<f32> = allocation_to_vec(&beta_array);let gpu_decay: Vec<f32> = allocation_to_vec(&decay_array);assert_close(&gpu_q, &ref_q, 1e-4, 1e-3, "prep q_norm");assert_close(&gpu_k, &ref_k, 1e-4, 1e-3, "prep k_norm");+ assert_eq!(gpu_v, ref_v);assert_close(&gpu_beta, &ref_beta, 1e-4, 1e-3, "prep beta");assert_close(&gpu_decay, &ref_decay, 1e-4, 1e-3, "prep decay");}@@ -622,8 +642,10 @@ fn bench_delta_net_prefill() {let prep_k = <<Metal as Backend>::Kernels as Kernels>::DeltaNetPrefillPrepKernel::new(&context,DataType::F32,+ DataType::F32,head_k_dim as u32,false,+ false,).unwrap();let prefill_k = <<Metal as Backend>::Kernels as Kernels>::DeltaNetPrefillKernel::new(@@ -649,6 +671,7 @@ fn bench_delta_net_prefill() {&dt_bias_array,&mut q_norm_array,&mut k_norm_array,+ None::<&mut crate::backends::common::Allocation<Metal>>,&mut beta_array,&mut decay_array,num_v_heads as u32,@@ -674,6 +697,7 @@ fn bench_delta_net_prefill() {&dt_bias_array,&mut q_norm_array,&mut k_norm_array,+ None::<&mut crate::backends::common::Allocation<Metal>>,&mut beta_array,&mut decay_array,num_v_heads as u32,diff --git a/crates/backend-uzu/unit/backends/common/kernel/gdn/tree_verify/mod.rs b/crates/backend-uzu/unit/backends/common/kernel/gdn/tree_verify/mod.rsindex 0416236e8..027eae27b 100644--- a/crates/backend-uzu/unit/backends/common/kernel/gdn/tree_verify/mod.rs+++ b/crates/backend-uzu/unit/backends/common/kernel/gdn/tree_verify/mod.rs@@ -1,8 +1,8 @@mod conv_scan_test;mod out_bench;mod out_test;-mod prefix_beta_bench;-mod prefix_beta_test;+mod prefix_bench;+mod prefix_test;mod state_advance_test;mod tree_gram_bench;mod tree_gram_test;diff --git a/crates/backend-uzu/unit/backends/common/kernel/gdn/tree_verify/out_bench.rs b/crates/backend-uzu/unit/backends/common/kernel/gdn/tree_verify/out_bench.rsindex 1c7c0b260..2942a30d4 100644--- a/crates/backend-uzu/unit/backends/common/kernel/gdn/tree_verify/out_bench.rs+++ b/crates/backend-uzu/unit/backends/common/kernel/gdn/tree_verify/out_bench.rs@@ -25,7 +25,7 @@ const HEAD_K_DIM: usize = 128;const HEAD_V_DIM: usize = 128;const TREE_OUT_MATMUL_MXU_ROWS: usize = 16;const TREE_OUT_MATMUL_SIMDGROUP_ROWS: usize = 8;-const TREE_SIZES: &[usize] = &[33, 49, 64, 128, 256, 512];+const TREE_SIZES: &[usize] = &[32, 33, 49, 64, 128, 256, 512];const BATCH_SIZES: &[usize] = &[1, 2, 4, 8];const TREE_OUT_VALUE_COLS: usize = 32;const TREE_OUT_SIMDGROUPS_PER_TG: usize = 4;@@ -60,7 +60,7 @@ fn make_buffers<T: ArrayElement + Float>(.map(|i| -((i % tree_size) as f32) * 0.01 - ((i % VALUE_HEADS) as f32) * 0.003).collect::<Vec<_>>();let qkd = (0..qkd_len).map(|i| ((i as f32 * 0.013).cos() * 0.03) - 0.01).collect::<Vec<_>>();- let u = (0..uv_len).map(|i| T::from(((i as f32 * 0.011).sin() * 0.3) + 0.04).unwrap()).collect::<Vec<_>>();+ let u = (0..uv_len).map(|i| (i as f32 * 0.011).sin() * 0.3 + 0.04).collect::<Vec<_>>();let h0 = (0..h0_len).map(|i| ((i as f32 * 0.019).cos() * 0.2) - 0.01).collect::<Vec<_>>();let h0_indices = (0..batch_size as i32).collect::<Vec<_>>();@@ -68,7 +68,7 @@ fn make_buffers<T: ArrayElement + Float>(q: alloc_allocation_with_data::<Metal, T>(context, &q),prefix: alloc_allocation_with_data::<Metal, f32>(context, &prefix),qkd: alloc_allocation_with_data::<Metal, f32>(context, &qkd),- u: alloc_allocation_with_data::<Metal, T>(context, &u),+ u: alloc_allocation_with_data::<Metal, f32>(context, &u),h0: alloc_allocation_with_data::<Metal, f32>(context, &h0),h0_indices: alloc_allocation_with_data::<Metal, i32>(context, &h0_indices),o: alloc_allocation::<Metal, T>(context, uv_len),@@ -109,6 +109,7 @@ fn bench_build_tree_out_type<T: ArrayElement + Float>(let kernel = <<Metal as Backend>::Kernels as Kernels>::BuildTreeOutKernel::new(context,T::data_type(),+ T::data_type(),use_mxu,transposed_h0,true,diff --git a/crates/backend-uzu/unit/backends/common/kernel/gdn/tree_verify/out_test.rs b/crates/backend-uzu/unit/backends/common/kernel/gdn/tree_verify/out_test.rsindex 4a13e9b48..da362a9c4 100644--- a/crates/backend-uzu/unit/backends/common/kernel/gdn/tree_verify/out_test.rs+++ b/crates/backend-uzu/unit/backends/common/kernel/gdn/tree_verify/out_test.rs@@ -32,7 +32,7 @@ struct Inputs<T> {q: Vec<T>,prefix: Vec<f32>,qkd: Vec<f32>,- u: Vec<T>,+ u: Vec<f32>,h0: Vec<f32>,h0_indices: Vec<i32>,}@@ -57,7 +57,7 @@ fn make_inputs<T: ArrayElement + Float>(shape: Shape) -> Inputs<T> {.map(|i| -((i % shape.tree_size) as f32) * 0.01 - ((i % shape.value_heads) as f32) * 0.003).collect(),qkd: (0..qkd_len).map(|i| ((i as f32 * 0.013).cos() * 0.1) - 0.02).collect(),- u: (0..u_len).map(|i| T::from(((i as f32 * 0.011).sin() * 0.3) + 0.04).unwrap()).collect(),+ u: (0..u_len).map(|i| ((i as f32 * 0.011).sin() * 0.3) + 0.04).collect(),h0: (0..h0_len).map(|i| ((i as f32 * 0.019).cos() * 0.2) - 0.01).collect(),h0_indices: (0..shape.batch_size).map(|i| {@@ -82,6 +82,7 @@ fn run_build_tree_out<B: Backend, T: ArrayElement + Float>(let kernel = <<B as Backend>::Kernels as Kernels>::BuildTreeOutKernel::new(&context,T::data_type(),+ T::data_type(),use_mxu,transposed_h0,use_h0,@@ -90,7 +91,7 @@ fn run_build_tree_out<B: Backend, T: ArrayElement + Float>(let q = alloc_allocation_with_data::<B, T>(&context, &inputs.q);let prefix = alloc_allocation_with_data::<B, f32>(&context, &inputs.prefix);let qkd = alloc_allocation_with_data::<B, f32>(&context, &inputs.qkd);- let u = alloc_allocation_with_data::<B, T>(&context, &inputs.u);+ let u = alloc_allocation_with_data::<B, f32>(&context, &inputs.u);let h0 = use_h0.then(|| alloc_allocation_with_data::<B, f32>(&context, &inputs.h0));let h0_indices = use_h0.then(|| alloc_allocation_with_data::<B, i32>(&context, &inputs.h0_indices));let mut o =@@ -187,7 +188,7 @@ fn test_build_tree_out_paths() {head_v_dim: 128,};for shape in [small, gdn] {- check_shape::<bf16>(shape, 0.08);- check_shape::<f32>(shape, 5e-3);+ check_shape::<bf16>(shape, 5e-3);+ check_shape::<f32>(shape, 5e-4);}}diff --git a/crates/backend-uzu/unit/backends/common/kernel/gdn/tree_verify/prefix_bench.rs b/crates/backend-uzu/unit/backends/common/kernel/gdn/tree_verify/prefix_bench.rsnew file mode 100644index 000000000..6aa242546--- /dev/null+++ b/crates/backend-uzu/unit/backends/common/kernel/gdn/tree_verify/prefix_bench.rs@@ -0,0 +1,67 @@+#![cfg(metal_backend)]++use std::mem::size_of;++use criterion::{BenchmarkId, Criterion, Throughput};+use proc_macros::uzu_bench;++use crate::{+ backends::{+ common::{Allocation, Backend, Kernels, kernel::BuildTreePrefixKernel},+ metal::Metal,+ },+ tests::{+ cold_pool::ColdPool,+ helpers::{alloc_allocation, alloc_allocation_with_data},+ matmul::iter_encode_loop_named,+ util::type_short_name,+ },+};++struct TreePrefixBuffers {+ trie: Allocation<Metal>,+ log_decay: Allocation<Metal>,+ prefix: Allocation<Metal>,+}++#[uzu_bench]+fn bench_build_tree_prefix(c: &mut Criterion) {+ let context = crate::tests::util::shared_metal_context();+ let kernel =+ <<Metal as Backend>::Kernels as Kernels>::BuildTreePrefixKernel::new(&context).expect("BuildTreePrefixKernel");+ let mut group = c.benchmark_group(format!("{}/Kernel/GDNTreeVerify/BuildTreePrefix", type_short_name::<Metal>()));++ for tree_size in [49usize, 64, 128] {+ let batch_size = 1usize;+ let value_heads = 48usize;+ let len = batch_size * tree_size * value_heads;+ let trie = (0..tree_size as u32).flat_map(|row| [row, tree_size as u32 - 1, row]).collect::<Vec<_>>();+ let log_decay = (0..len).map(|i| -0.001 - (i as f32 * 0.017).sin().abs() * 0.1).collect::<Vec<_>>();+ let bytes_per_copy = trie.len() * size_of::<u32>() + len * size_of::<f32>() * 2;+ let mut buffers = ColdPool::new(bytes_per_copy, || TreePrefixBuffers {+ trie: alloc_allocation_with_data::<Metal, u32>(&context, &trie),+ log_decay: alloc_allocation_with_data::<Metal, f32>(&context, &log_decay),+ prefix: alloc_allocation::<Metal, f32>(&context, len),+ });++ group.throughput(Throughput::Elements((len * tree_size) as u64));+ group.bench_function(BenchmarkId::from_parameter(format!("B1_T{tree_size}_HV{value_heads}")), |bencher| {+ let benchmark_path = format!(+ "{}/Kernel/GDNTreeVerify/BuildTreePrefix/B1_T{tree_size}_HV{value_heads}",+ type_short_name::<Metal>()+ );+ iter_encode_loop_named::<Metal, _>(&context, bencher, &benchmark_path, |encoder| {+ let buffers = buffers.next_mut();+ kernel.encode(+ &buffers.trie,+ &buffers.log_decay,+ &mut buffers.prefix,+ batch_size as u32,+ tree_size as u32,+ value_heads as u32,+ encoder,+ );+ });+ });+ }+}diff --git a/crates/backend-uzu/unit/backends/common/kernel/gdn/tree_verify/prefix_beta_bench.rs b/crates/backend-uzu/unit/backends/common/kernel/gdn/tree_verify/prefix_beta_bench.rsdeleted file mode 100644index dbde90093..000000000--- a/crates/backend-uzu/unit/backends/common/kernel/gdn/tree_verify/prefix_beta_bench.rs+++ /dev/null@@ -1,92 +0,0 @@-#![cfg(metal_backend)]--use std::mem::size_of;--use criterion::{BenchmarkId, Criterion, Throughput};-use proc_macros::uzu_bench;--use crate::{- backends::{- common::{Allocation, Backend, Kernels, kernel::BuildPrefixBetaKernel},- metal::Metal,- },- data_type::DataType,- tests::{- cold_pool::ColdPool,- helpers::{alloc_allocation, alloc_allocation_with_data},- matmul::iter_encode_loop_named,- util::type_short_name,- },-};--// chain trie: node `row` covers [row, T-1].-fn chain_trie(tree_size: usize) -> Vec<u32> {- let mut trie = Vec::with_capacity(tree_size * 3);- for row in 0..tree_size as u32 {- trie.extend_from_slice(&[row, tree_size as u32 - 1, row]);- }- trie-}--struct PrefixBetaBuffers {- trie: Allocation<Metal>,- a_transposed: Allocation<Metal>,- b: Allocation<Metal>,- prefix: Allocation<Metal>,- beta: Allocation<Metal>,-}--#[uzu_bench]-fn bench_build_prefix_beta(c: &mut Criterion) {- let context = crate::tests::util::shared_metal_context();- let kernel = <<Metal as Backend>::Kernels as Kernels>::BuildPrefixBetaKernel::new(&context, DataType::F32)- .expect("BuildPrefixBetaKernel");-- let mut group = c.benchmark_group(format!("{}/Kernel/GDNTreeVerify/BuildPrefixBeta", type_short_name::<Metal>()));-- for tree_size in [49usize, 64, 129] {- let batch_size = 1usize;- let value_heads = 48usize;- let len = batch_size * tree_size * value_heads;- let trie = chain_trie(tree_size);- let a_transposed = (0..len).map(|i| (i as f32 * 0.17).sin() - 0.4).collect::<Vec<_>>();- let b_data = (0..len).map(|i| (i as f32 * 0.11).cos() * 0.8).collect::<Vec<_>>();- let a_log = (0..value_heads).map(|i| -0.2 + i as f32 * 0.03).collect::<Vec<_>>();- let dt_bias = (0..value_heads).map(|i| 0.1 - i as f32 * 0.02).collect::<Vec<_>>();-- let a_log = alloc_allocation_with_data::<Metal, f32>(&context, &a_log);- let dt_bias = alloc_allocation_with_data::<Metal, f32>(&context, &dt_bias);- let bytes_per_copy = trie.len() * size_of::<u32>() + len * size_of::<f32>() * 4;- let mut buffers = ColdPool::new(bytes_per_copy, || PrefixBetaBuffers {- trie: alloc_allocation_with_data::<Metal, u32>(&context, &trie),- a_transposed: alloc_allocation_with_data::<Metal, f32>(&context, &a_transposed),- b: alloc_allocation_with_data::<Metal, f32>(&context, &b_data),- prefix: alloc_allocation::<Metal, f32>(&context, len),- beta: alloc_allocation::<Metal, f32>(&context, len),- });-- group.throughput(Throughput::Elements((len * tree_size) as u64));- group.bench_function(BenchmarkId::from_parameter(format!("B1_T{tree_size}_HV{value_heads}")), |bencher| {- let benchmark_path = format!(- "{}/Kernel/GDNTreeVerify/BuildPrefixBeta/B1_T{tree_size}_HV{value_heads}",- type_short_name::<Metal>()- );- iter_encode_loop_named::<Metal, _>(&context, bencher, &benchmark_path, |encoder| {- let buffers = buffers.next_mut();- kernel.encode(- &buffers.trie,- &buffers.a_transposed,- &buffers.b,- &a_log,- &dt_bias,- &mut buffers.prefix,- &mut buffers.beta,- batch_size as u32,- tree_size as u32,- value_heads as u32,- encoder,- );- });- });- }-}diff --git a/crates/backend-uzu/unit/backends/common/kernel/gdn/tree_verify/prefix_beta_test.rs b/crates/backend-uzu/unit/backends/common/kernel/gdn/tree_verify/prefix_beta_test.rsdeleted file mode 100644index 980ef8936..000000000--- a/crates/backend-uzu/unit/backends/common/kernel/gdn/tree_verify/prefix_beta_test.rs+++ /dev/null@@ -1,112 +0,0 @@-use proc_macros::uzu_test;-use test_runner::for_each_non_cpu_backend;--use crate::{- backends::{- common::{Backend, Context, Encoder, Kernels, kernel::BuildPrefixBetaKernel},- cpu::Cpu,- },- data_type::DataType,- tests::{- assert::assert_eq_float,- helpers::{alloc_allocation, alloc_allocation_with_data, allocation_to_vec},- },-};--fn get_output<B: Backend>(- trie: &[u32],- a_transposed: &[f32],- b: &[f32],- a_log: &[f32],- dt_bias: &[f32],- batch_size: u32,- tree_size: u32,- value_heads: u32,-) -> (Vec<f32>, Vec<f32>) {- let context = B::Context::new().expect("Failed to create Context");- let kernel = <<B as Backend>::Kernels as Kernels>::BuildPrefixBetaKernel::new(&context, DataType::F32)- .expect("Failed to create BuildPrefixBetaKernel");-- let output_len = a_transposed.len();- let trie = alloc_allocation_with_data::<B, u32>(&context, trie);- let a_transposed = alloc_allocation_with_data::<B, f32>(&context, a_transposed);- let b = alloc_allocation_with_data::<B, f32>(&context, b);- let a_log = alloc_allocation_with_data::<B, f32>(&context, a_log);- let dt_bias = alloc_allocation_with_data::<B, f32>(&context, dt_bias);- let mut prefix = alloc_allocation::<B, f32>(&context, output_len);- let mut beta = alloc_allocation::<B, f32>(&context, output_len);-- let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");- kernel.encode(- &trie,- &a_transposed,- &b,- &a_log,- &dt_bias,- &mut prefix,- &mut beta,- batch_size,- tree_size,- value_heads,- &mut encoder,- );- encoder.end_encoding().submit().wait_until_completed().unwrap();-- (allocation_to_vec(&prefix), allocation_to_vec(&beta))-}--// chain (batch 0) + broom (batch 1); the broom's varied [start, end] exercises-// trie_end, which the chain (end always T-1) can't.-fn build_trie(tree_size: usize) -> Vec<u32> {- let last = tree_size as u32 - 1;- let mut trie = Vec::with_capacity(2 * tree_size * 3);- for i in 0..tree_size as u32 {- trie.extend_from_slice(&[i, last, i]);- }- trie.extend_from_slice(&[0, last, 0]);- for i in 1..tree_size as u32 {- trie.extend_from_slice(&[i, i, 1]);- }- trie-}--#[uzu_test]-fn test_build_prefix_beta_matches_cpu() {- let batch_size = 2;- let value_heads = 5;-- for tree_size in [5, 33, 129] {- let len = batch_size * tree_size * value_heads;- let trie = build_trie(tree_size);- let a_transposed = (0..len).map(|i| (i as f32 * 0.13).sin() - 0.4).collect::<Vec<_>>();- let b = (0..len).map(|i| (i as f32 * 0.11).cos() * 0.8).collect::<Vec<_>>();- let a_log = (0..value_heads).map(|i| -0.2 + i as f32 * 0.03).collect::<Vec<_>>();- let dt_bias = (0..value_heads).map(|i| 0.1 - i as f32 * 0.02).collect::<Vec<_>>();- let expected = get_output::<Cpu>(- &trie,- &a_transposed,- &b,- &a_log,- &dt_bias,- batch_size as u32,- tree_size as u32,- value_heads as u32,- );-- for_each_non_cpu_backend!(|B| {- let output = get_output::<B>(- &trie,- &a_transposed,- &b,- &a_log,- &dt_bias,- batch_size as u32,- tree_size as u32,- value_heads as u32,- );- let msg = format!("backend {} tree_size {tree_size}", std::any::type_name::<B>());- assert_eq_float::<f32>(&expected.0, &output.0, 1e-4, &format!("prefix {msg}"));- assert_eq_float::<f32>(&expected.1, &output.1, 1e-5, &format!("beta {msg}"));- });- }-}diff --git a/crates/backend-uzu/unit/backends/common/kernel/gdn/tree_verify/prefix_test.rs b/crates/backend-uzu/unit/backends/common/kernel/gdn/tree_verify/prefix_test.rsnew file mode 100644index 000000000..6392d7508--- /dev/null+++ b/crates/backend-uzu/unit/backends/common/kernel/gdn/tree_verify/prefix_test.rs@@ -0,0 +1,46 @@+#![cfg(metal_backend)]++use proc_macros::uzu_test;+use test_runner::for_each_non_cpu_backend;++use crate::{+ backends::{+ common::{Backend, Context, Encoder, Kernels, kernel::BuildTreePrefixKernel},+ cpu::Cpu,+ },+ tests::{+ assert::assert_eq_float,+ helpers::{alloc_allocation, alloc_allocation_with_data, allocation_to_vec},+ },+};++fn run<B: Backend>(tree_size: usize) -> Vec<f32> {+ const HEADS: usize = 5;+ let context = B::Context::new().unwrap();+ let trie = (0..tree_size as u32).flat_map(|node| [node, tree_size as u32 - 1, node]).collect::<Vec<_>>();+ let log_decay = (0..tree_size * HEADS).map(|i| -0.001 - i as f32 * 0.0001).collect::<Vec<_>>();+ let trie = alloc_allocation_with_data::<B, u32>(&context, &trie);+ let log_decay = alloc_allocation_with_data::<B, f32>(&context, &log_decay);+ let mut prefix = alloc_allocation::<B, f32>(&context, tree_size * HEADS);+ let kernel = <B::Kernels as Kernels>::BuildTreePrefixKernel::new(&context).unwrap();+ let mut encoder = Encoder::new(context.as_ref()).unwrap();+ kernel.encode(&trie, &log_decay, &mut prefix, 1, tree_size as u32, HEADS as u32, &mut encoder);+ encoder.end_encoding().submit().wait_until_completed().unwrap();+ allocation_to_vec(&prefix)+}++#[uzu_test]+fn test_build_tree_prefix() {+ const HEADS: usize = 5;+ for tree_size in [49, 64, 128] {+ let expected = (0..tree_size)+ .flat_map(|row| {+ (0..HEADS).map(move |head| {+ (0..=row).map(|token| -0.001 - (token * HEADS + head) as f32 * 0.0001).sum::<f32>()+ })+ })+ .collect::<Vec<_>>();+ assert_eq_float(&expected, &run::<Cpu>(tree_size), 1e-6, "CPU");+ for_each_non_cpu_backend!(|B| assert_eq_float(&expected, &run::<B>(tree_size), 1e-6, "backend"));+ }+}diff --git a/crates/backend-uzu/unit/backends/common/kernel/gdn/tree_verify/state_advance_test.rs b/crates/backend-uzu/unit/backends/common/kernel/gdn/tree_verify/state_advance_test.rsindex f40a39e1d..1b141d5b2 100644--- a/crates/backend-uzu/unit/backends/common/kernel/gdn/tree_verify/state_advance_test.rs+++ b/crates/backend-uzu/unit/backends/common/kernel/gdn/tree_verify/state_advance_test.rs@@ -36,20 +36,25 @@ const TEST_NUM_K_HEADS: usize = 2;fn run<B: Backend, T: ArrayElement + Float>(accepted_indices: &[u32]) -> Vec<f32> {let context = B::Context::new().expect("context");- let kernel =- <<B as Backend>::Kernels as Kernels>::StateAdvanceKernel::new(&context, T::data_type(), HEAD_DIM as u32)- .expect("kernel");+ let kernel = <<B as Backend>::Kernels as Kernels>::StateAdvanceKernel::new(+ &context,+ T::data_type(),+ HEAD_DIM as u32,+ TEST_NUM_V_HEADS as u32,+ TEST_NUM_K_HEADS as u32,+ )+ .expect("kernel");let key_len = TREE_SIZE * TEST_NUM_K_HEADS * HEAD_DIM;let value_len = TREE_SIZE * TEST_NUM_V_HEADS * HEAD_DIM;let scalar_len = TREE_SIZE * TEST_NUM_V_HEADS;let state_len = TEST_NUM_V_HEADS * HEAD_DIM * HEAD_DIM;- let k_norm = (0..key_len).map(|i| (i % 17) as f32 * 0.002 - 0.016).collect::<Vec<_>>();+ let k_norm = (0..key_len).map(|i| T::from((i % 17) as f32 * 0.002 - 0.016).unwrap()).collect::<Vec<_>>();let v = (0..value_len).map(|i| T::from((i % 19) as f32 * 0.003 - 0.027).unwrap()).collect::<Vec<_>>();let log_decay = (0..scalar_len).map(|i| -0.01 - (i % 7) as f32 * 0.002).collect::<Vec<_>>();let beta = (0..scalar_len).map(|i| 0.2 + (i % 5) as f32 * 0.03).collect::<Vec<_>>();let initial_state = (0..state_len).map(|i| (i % 23) as f32 * 0.001 - 0.011).collect::<Vec<_>>();- let k_norm = alloc_allocation_with_data::<B, f32>(&context, &k_norm);+ let k_norm = alloc_allocation_with_data::<B, T>(&context, &k_norm);let v = alloc_allocation_with_data::<B, T>(&context, &v);let log_decay = alloc_allocation_with_data::<B, f32>(&context, &log_decay);let beta = alloc_allocation_with_data::<B, f32>(&context, &beta);@@ -66,8 +71,6 @@ fn run<B: Backend, T: ArrayElement + Float>(accepted_indices: &[u32]) -> Vec<f32&accepted_indices,&mut committed_state,accepted_len as u32,- TEST_NUM_V_HEADS as u32,- TEST_NUM_K_HEADS as u32,&mut encoder,);encoder.end_encoding().submit().wait_until_completed().unwrap();@@ -92,10 +95,18 @@ fn bench_state_advance(c: &mut Criterion) {const BENCHMARK: &str = "Metal/Kernel/GDNTreeVerify/StateAdvance";let context = crate::tests::util::shared_metal_context();- let kernel =- <<Metal as Backend>::Kernels as Kernels>::StateAdvanceKernel::new(&context, DataType::BF16, HEAD_DIM as u32)- .expect("kernel");- let k_norm = alloc_allocation_with_data::<Metal, f32>(&context, &vec![0.001; TREE_SIZE * NUM_K_HEADS * HEAD_DIM]);+ let kernel = <<Metal as Backend>::Kernels as Kernels>::StateAdvanceKernel::new(+ &context,+ DataType::BF16,+ HEAD_DIM as u32,+ NUM_V_HEADS as u32,+ NUM_K_HEADS as u32,+ )+ .expect("kernel");+ let k_norm = alloc_allocation_with_data::<Metal, bf16>(+ &context,+ &vec![bf16::from_f32(0.001); TREE_SIZE * NUM_K_HEADS * HEAD_DIM],+ );let v = alloc_allocation_with_data::<Metal, bf16>(&context,&vec![bf16::from_f32(0.01); TREE_SIZE * NUM_V_HEADS * HEAD_DIM],@@ -121,8 +132,6 @@ fn bench_state_advance(c: &mut Criterion) {&accepted_indices,committed_states.next_mut(),accepted_len as u32,- NUM_V_HEADS as u32,- NUM_K_HEADS as u32,encoder,);});diff --git a/crates/backend-uzu/unit/backends/common/kernel/gdn/tree_verify/tree_gram_bench.rs b/crates/backend-uzu/unit/backends/common/kernel/gdn/tree_verify/tree_gram_bench.rsindex ff5199ee9..0e01ed629 100644--- a/crates/backend-uzu/unit/backends/common/kernel/gdn/tree_verify/tree_gram_bench.rs+++ b/crates/backend-uzu/unit/backends/common/kernel/gdn/tree_verify/tree_gram_bench.rs@@ -23,7 +23,7 @@ const K_HEADS: usize = 16;const VALUE_HEADS: usize = 48;const HEAD_K_DIM: usize = 128;const HEAD_V_DIM: usize = 128;-const TREE_SIZES: &[usize] = &[33, 49, 64, 128, 256, 512];+const TREE_SIZES: &[usize] = &[32, 33, 49, 64, 128, 256, 512];const BATCH_SIZES: &[usize] = &[1, 2, 4, 8];struct TreeGramBuffers {@@ -104,7 +104,7 @@ fn make_buffers(.map(|i| -((i % tree_size) as f32) * 0.01 - ((i % VALUE_HEADS) as f32) * 0.003).collect::<Vec<_>>();let beta = (0..scalar_len).map(|i| 0.25 + ((i as f32 * 0.013).sin() + 1.0) * 0.2).collect::<Vec<_>>();- let h0 = (0..h0_len).map(|i| bf16::from_f32(((i as f32 * 0.007).sin() * 0.05) - 0.01)).collect::<Vec<_>>();+ let h0 = (0..h0_len).map(|i| ((i as f32 * 0.007).sin() * 0.05) - 0.01).collect::<Vec<_>>();let h0_idx = (0..batch_size).map(|i| i as i32).collect::<Vec<_>>();let scale = (HEAD_K_DIM as f32).sqrt().recip();(@@ -114,7 +114,7 @@ fn make_buffers(trie: alloc_allocation_with_data::<Metal, u32>(context, &trie),prefix: alloc_allocation_with_data::<Metal, f32>(context, &prefix),beta: alloc_allocation_with_data::<Metal, f32>(context, &beta),- h0: alloc_allocation_with_data::<Metal, bf16>(context, &h0),+ h0: alloc_allocation_with_data::<Metal, f32>(context, &h0),h0_idx: alloc_allocation_with_data::<Metal, i32>(context, &h0_idx),a_packed: alloc_allocation::<Metal, f32>(context, a_len),qkd: alloc_allocation::<Metal, f32>(context, out_len),@@ -137,7 +137,8 @@ fn buffers_bytes(let a_inv_len = batch_size * VALUE_HEADS * num_blocks * 16 * 16;let h0_len = batch_size * VALUE_HEADS * HEAD_V_DIM * HEAD_K_DIM;let kh0_len = batch_size * tree_size * VALUE_HEADS * HEAD_V_DIM;- (qk_len * 2 + h0_len) * size_of::<bf16>()+ qk_len * 2 * size_of::<bf16>()+ + h0_len * size_of::<f32>()+ batch_size * tree_size * 3 * size_of::<u32>()+ scalar_len * size_of::<f32>() * 2+ (out_len + a_len + a_inv_len + kh0_len) * size_of::<f32>()diff --git a/crates/backend-uzu/unit/backends/common/kernel/gdn/tree_verify/tree_gram_test.rs b/crates/backend-uzu/unit/backends/common/kernel/gdn/tree_verify/tree_gram_test.rsindex 627066065..ae7744b2b 100644--- a/crates/backend-uzu/unit/backends/common/kernel/gdn/tree_verify/tree_gram_test.rs+++ b/crates/backend-uzu/unit/backends/common/kernel/gdn/tree_verify/tree_gram_test.rs@@ -1,9 +1,12 @@#![cfg(metal_backend)]+use half::bf16;+use num_traits::Float;use proc_macros::uzu_test;use test_runner::for_each_non_cpu_backend;use crate::{+ array::ArrayElement,backends::{common::{Backend, Context, Encoder, Kernels, kernel::BuildTreeGramKernel},cpu::Cpu,@@ -51,9 +54,9 @@ fn make_inputs(tree_size: usize) -> Inputs {}}-fn get_output<B: Backend>(- q: &[f32],- k: &[f32],+fn get_output<B: Backend, T: ArrayElement + Float>(+ q: &[T],+ k: &[T],trie: &[u32],prefix: &[f32],beta: &[f32],@@ -64,16 +67,17 @@ fn get_output<B: Backend>(use_mxu: bool,) -> (Vec<f32>, Vec<f32>, Vec<f32>, Vec<f32>) {let context = B::Context::new().expect("Failed to create Context");- let kernel = <<B as Backend>::Kernels as Kernels>::BuildTreeGramKernel::new(&context, DataType::F32, use_mxu, true)- .expect("Failed to create BuildTreeGramKernel");+ let kernel =+ <<B as Backend>::Kernels as Kernels>::BuildTreeGramKernel::new(&context, T::data_type(), use_mxu, true)+ .expect("Failed to create BuildTreeGramKernel");let output_len = BATCH_SIZE * VALUE_HEADS * tree_size * tree_size;let num_blocks = tree_size.div_ceil(16);let a_len = BATCH_SIZE * VALUE_HEADS * num_blocks * num_blocks.div_ceil(2) * 16 * 32;let a_inv_len = BATCH_SIZE * VALUE_HEADS * num_blocks * 16 * 16;let kh0_len = BATCH_SIZE * tree_size * VALUE_HEADS * HEAD_V_DIM;- let q = alloc_allocation_with_data::<B, f32>(&context, q);- let k = alloc_allocation_with_data::<B, f32>(&context, k);+ let q = alloc_allocation_with_data::<B, T>(&context, q);+ let k = alloc_allocation_with_data::<B, T>(&context, k);let trie = alloc_allocation_with_data::<B, u32>(&context, trie);let prefix = alloc_allocation_with_data::<B, f32>(&context, prefix);let beta = alloc_allocation_with_data::<B, f32>(&context, beta);@@ -115,6 +119,72 @@ fn get_output<B: Backend>((allocation_to_vec(&a_packed), allocation_to_vec(&qkd), allocation_to_vec(&a_inv), allocation_to_vec(&kh0))}+fn check_type<T: ArrayElement + Float>(+ inputs: &Inputs,+ tree_size: usize,+ scale: f32,+) {+ let q = inputs.q.iter().map(|&value| T::from(value).unwrap()).collect::<Vec<_>>();+ let k = inputs.k.iter().map(|&value| T::from(value).unwrap()).collect::<Vec<_>>();+ let expected = get_output::<Cpu, T>(+ &q,+ &k,+ &inputs.trie,+ &inputs.prefix,+ &inputs.beta,+ &inputs.h0,+ &inputs.h0_idx,+ tree_size,+ scale,+ false,+ );++ for_each_non_cpu_backend!(|B| {+ let actual = get_output::<B, T>(+ &q,+ &k,+ &inputs.trie,+ &inputs.prefix,+ &inputs.beta,+ &inputs.h0,+ &inputs.h0_idx,+ tree_size,+ scale,+ false,+ );+ let msg = format!("backend {} simdgroup tree_size {tree_size}", std::any::type_name::<B>());+ assert_eq_float::<f32>(&expected.0, &actual.0, 2e-4, &format!("a_packed {msg}"));+ assert_eq_float::<f32>(&expected.1, &actual.1, 2e-4, &format!("qkd {msg}"));+ assert_eq_float::<f32>(&expected.2, &actual.2, 5e-4, &format!("a_inv {msg}"));+ assert_eq_float::<f32>(&expected.3, &actual.3, 2e-4, &format!("kh0 {msg}"));+ });++ if <Metal as Backend>::Context::new().expect("Failed to create Context").supports_mxu() {+ let actual = get_output::<Metal, T>(+ &q,+ &k,+ &inputs.trie,+ &inputs.prefix,+ &inputs.beta,+ &inputs.h0,+ &inputs.h0_idx,+ tree_size,+ scale,+ true,+ );+ let msg = format!("backend {} path MXU tree_size {tree_size}", std::any::type_name::<Metal>());+ let (a_eps, qkd_eps, inv_eps, kh0_eps) = if T::data_type() == DataType::BF16 {+ (1e-5, 1e-6, 1e-5, 5e-4)+ } else {+ (5e-3, 1e-3, 8e-3, 1e-3)+ };+ assert_eq_float::<f32>(&expected.0, &actual.0, a_eps, &format!("a_packed {msg}"));+ assert_eq_float::<f32>(&expected.1, &actual.1, qkd_eps, &format!("qkd {msg}"));+ assert_eq_float::<f32>(&expected.2, &actual.2, inv_eps, &format!("a_inv {msg}"));+ assert_eq_float::<f32>(&expected.3, &actual.3, kh0_eps, &format!("kh0 {msg}"));+ }+}+fn build_trie(tree_size: usize) -> Vec<u32> {let last = tree_size as u32 - 1;let mut trie = Vec::with_capacity(BATCH_SIZE * tree_size * 3);@@ -134,27 +204,7 @@ fn test_build_tree_gram_matches_cpu() {for tree_size in [17, 64, 128] {let i = make_inputs(tree_size);- let expected =- get_output::<Cpu>(&i.q, &i.k, &i.trie, &i.prefix, &i.beta, &i.h0, &i.h0_idx, tree_size, scale, false);-- for_each_non_cpu_backend!(|B| {- let actual =- get_output::<B>(&i.q, &i.k, &i.trie, &i.prefix, &i.beta, &i.h0, &i.h0_idx, tree_size, scale, false);- let msg = format!("backend {} simdgroup tree_size {tree_size}", std::any::type_name::<B>());- assert_eq_float::<f32>(&expected.0, &actual.0, 5e-3, &format!("a_packed {msg}"));- assert_eq_float::<f32>(&expected.1, &actual.1, 5e-3, &format!("qkd {msg}"));- assert_eq_float::<f32>(&expected.2, &actual.2, 1e-2, &format!("a_inv {msg}"));- assert_eq_float::<f32>(&expected.3, &actual.3, 5e-3, &format!("kh0 {msg}"));- });-- if <Metal as Backend>::Context::new().expect("Failed to create Context").supports_mxu() {- let actual =- get_output::<Metal>(&i.q, &i.k, &i.trie, &i.prefix, &i.beta, &i.h0, &i.h0_idx, tree_size, scale, true);- let msg = format!("backend {} path MXU tree_size {tree_size}", std::any::type_name::<Metal>());- assert_eq_float::<f32>(&expected.0, &actual.0, 5e-3, &format!("a_packed {msg}"));- assert_eq_float::<f32>(&expected.1, &actual.1, 5e-3, &format!("qkd {msg}"));- assert_eq_float::<f32>(&expected.2, &actual.2, 1e-2, &format!("a_inv {msg}"));- assert_eq_float::<f32>(&expected.3, &actual.3, 5e-3, &format!("kh0 {msg}"));- }+ check_type::<f32>(&i, tree_size, scale);+ check_type::<bf16>(&i, tree_size, scale);}}diff --git a/crates/backend-uzu/unit/backends/common/kernel/gdn/tree_verify/tree_update_solve_bench.rs b/crates/backend-uzu/unit/backends/common/kernel/gdn/tree_verify/tree_update_solve_bench.rsindex 897d564fb..01cf09285 100644--- a/crates/backend-uzu/unit/backends/common/kernel/gdn/tree_verify/tree_update_solve_bench.rs+++ b/crates/backend-uzu/unit/backends/common/kernel/gdn/tree_verify/tree_update_solve_bench.rs@@ -23,9 +23,9 @@ const NUM_V_HEADS: usize = 48;const HEAD_V_DIM: usize = 128;const BT: usize = 16;const BVS: &[usize] = &[16, 32];-const MXU_BVS: &[usize] = &[32];const BENCH_SHAPES: &[(usize, usize)] = &[+ (1, 32),(1, 33),(1, 49),(1, 64),@@ -172,51 +172,38 @@ fn buffers_bytes(#[uzu_bench]fn bench_tree_update_solve(c: &mut Criterion) {let context = <Metal as Backend>::Context::new().expect("metal context");- let kernel_paths = if context.supports_mxu() {- &[("Simdgroup", false, BVS), ("MXU", true, MXU_BVS)][..]- } else {- &[("Simdgroup", false, BVS)][..]- };-- for &(kernel_path, use_mxu, bvs) in kernel_paths {- for &bv in bvs {- let kernel = <<Metal as Backend>::Kernels as Kernels>::TreeUpdateSolveKernel::new(- &context,- DataType::BF16,- bv as u32,- use_mxu,- true,- )- .expect("TreeUpdateSolveKernel");-- let mut group =- c.benchmark_group(format!("Metal/Kernel/GDNTreeVerify/TreeUpdateSolve/{kernel_path}/BV{bv}"));- // 500ms warmup ramps GPU clocks before measuring (µs-scale benches never- // ramp otherwise and medians become run-composition artifacts).- group.sample_size(20).warm_up_time(Duration::from_millis(500)).measurement_time(Duration::from_secs(2));-- for &(batch_size, tree_size) in BENCH_SHAPES {- let benchmark_path = format!(- "Metal/Kernel/GDNTreeVerify/TreeUpdateSolve/{kernel_path}/BV{bv}/B{batch_size}_T{tree_size}"- );- let benchmark_id = BenchmarkId::from_parameter(format!(- "B{batch_size}_T{tree_size}_HV{NUM_V_HEADS}_V{HEAD_V_DIM}_BV{bv}"- ));- let mut buffers = ColdPool::new(buffers_bytes(batch_size, tree_size), || {- make_buffers(&context, batch_size, tree_size)- });-- group.throughput(Throughput::Elements((batch_size * tree_size * NUM_V_HEADS * HEAD_V_DIM) as u64));- group.bench_function(benchmark_id, |bencher| {- iter_encode_loop_named::<Metal, _>(context.as_ref(), bencher, &benchmark_path, |encoder| {- let buffers = buffers.next_mut();- encode(&kernel, buffers, batch_size, tree_size, encoder);- });+ for &bv in BVS {+ let kernel = <<Metal as Backend>::Kernels as Kernels>::TreeUpdateSolveKernel::new(+ &context,+ DataType::BF16,+ bv as u32,+ true,+ )+ .expect("TreeUpdateSolveKernel");++ let mut group = c.benchmark_group(format!("Metal/Kernel/GDNTreeVerify/TreeUpdateSolve/Simdgroup/BV{bv}"));+ // 500ms warmup ramps GPU clocks before measuring (µs-scale benches never+ // ramp otherwise and medians become run-composition artifacts).+ group.sample_size(20).warm_up_time(Duration::from_millis(500)).measurement_time(Duration::from_secs(2));++ for &(batch_size, tree_size) in BENCH_SHAPES {+ let benchmark_path =+ format!("Metal/Kernel/GDNTreeVerify/TreeUpdateSolve/Simdgroup/BV{bv}/B{batch_size}_T{tree_size}");+ let benchmark_id =+ BenchmarkId::from_parameter(format!("B{batch_size}_T{tree_size}_HV{NUM_V_HEADS}_V{HEAD_V_DIM}_BV{bv}"));+ let mut buffers =+ ColdPool::new(buffers_bytes(batch_size, tree_size), || make_buffers(&context, batch_size, tree_size));++ group.throughput(Throughput::Elements((batch_size * tree_size * NUM_V_HEADS * HEAD_V_DIM) as u64));+ group.bench_function(benchmark_id, |bencher| {+ iter_encode_loop_named::<Metal, _>(context.as_ref(), bencher, &benchmark_path, |encoder| {+ let buffers = buffers.next_mut();+ encode(&kernel, buffers, batch_size, tree_size, encoder);});- }- group.finish();- test_runner::metrics::wait_gpu_cooldown();+ });}+ group.finish();+ test_runner::metrics::wait_gpu_cooldown();}}diff --git a/crates/backend-uzu/unit/backends/common/kernel/gdn/tree_verify/tree_update_solve_test.rs b/crates/backend-uzu/unit/backends/common/kernel/gdn/tree_verify/tree_update_solve_test.rsindex 45e94396b..66b6db9b9 100644--- a/crates/backend-uzu/unit/backends/common/kernel/gdn/tree_verify/tree_update_solve_test.rs+++ b/crates/backend-uzu/unit/backends/common/kernel/gdn/tree_verify/tree_update_solve_test.rs@@ -9,7 +9,6 @@ use crate::{backends::{common::{Backend, Context, Encoder, Kernels, kernel::TreeUpdateSolveKernel},cpu::Cpu,- metal::Metal,},data_type::DataType,tests::{@@ -88,15 +87,13 @@ const CASES: &[SolveCase] = &[fn run_case<B: Backend, T: ArrayElement + Copy>(case: SolveCase,bv: u32,- use_mxu: bool,use_h0: bool,data_type: DataType,cast: impl Fn(f32) -> T + Copy,) -> Vec<f32> {let context = B::Context::new().expect("context");- let kernel =- <<B as Backend>::Kernels as Kernels>::TreeUpdateSolveKernel::new(&context, data_type, bv, use_mxu, use_h0)- .expect("kernel");+ let kernel = <<B as Backend>::Kernels as Kernels>::TreeUpdateSolveKernel::new(&context, data_type, bv, use_h0)+ .expect("kernel");let batch_size = case.batch_size as usize;let tree_size = case.tree_size as usize;@@ -191,34 +188,23 @@ fn test_tree_update_solve_cases() {for &bv in BVS {for &use_h0 in &[true, false] {for &case in CASES {- let expected = run_case::<Cpu, f32>(case, bv, false, use_h0, DataType::F32, |x| x);+ let expected = run_case::<Cpu, f32>(case, bv, use_h0, DataType::F32, |x| x);for_each_non_cpu_backend!(|B| {- let output = run_case::<B, f32>(case, bv, false, use_h0, DataType::F32, |x| x);+ let output = run_case::<B, f32>(case, bv, use_h0, DataType::F32, |x| x);assert_eq_float(&expected, &output, 1e-4, &format!("{} BV{bv} h0={use_h0}", case.name));});}}}- {- let supports_mxu = <Metal as Backend>::Context::new().expect("context").supports_mxu();- let paths: &[(&str, bool)] = if supports_mxu {- &[("Simdgroup", false), ("MXU", true)]- } else {- &[("Simdgroup", false)]- };-- // MXU needs an even number of column fragments or MPP row pairing; BV16- // (a single 16x16 fragment) is not supported.- for &bv in &[32] {- for &use_h0 in &[true, false] {- for &case in CASES {- let expected = run_case::<Cpu, bf16>(case, bv, false, use_h0, DataType::BF16, bf16::from_f32);- for &(path, use_mxu) in paths {- let output = run_case::<Metal, bf16>(case, bv, use_mxu, use_h0, DataType::BF16, bf16::from_f32);- assert_eq_float(&expected, &output, 1e-2, &format!("{} {path} BV{bv} h0={use_h0}", case.name));- }- }+ for &bv in BVS {+ for &use_h0 in &[true, false] {+ for &case in CASES {+ let expected = run_case::<Cpu, bf16>(case, bv, use_h0, DataType::BF16, bf16::from_f32);+ for_each_non_cpu_backend!(|B| {+ let output = run_case::<B, bf16>(case, bv, use_h0, DataType::BF16, bf16::from_f32);+ assert_eq_float(&expected, &output, 5e-4, &format!("{} BV{bv} h0={use_h0}", case.name));+ });}}}diff --git a/crates/backend-uzu/unit/backends/metal/kernel/gdn/chunked_test.rs b/crates/backend-uzu/unit/backends/metal/kernel/gdn/chunked_test.rsindex 0948a6fb5..220c352b2 100644--- a/crates/backend-uzu/unit/backends/metal/kernel/gdn/chunked_test.rs+++ b/crates/backend-uzu/unit/backends/metal/kernel/gdn/chunked_test.rs@@ -82,8 +82,10 @@ fn run_prefill<T: ArrayElement>(<BackendKernels<Metal> as Kernels>::DeltaNetPrefillPrepKernel::new(context,T::data_type(),+ DataType::F32,HEAD_K_DIM as u32,false,+ false,).expect("prep").encode(@@ -92,6 +94,7 @@ fn run_prefill<T: ArrayElement>(&dt_bias,&mut q,&mut k,+ None::<&mut crate::backends::common::Allocation<Metal>>,&mut beta,&mut decay,NUM_V_HEADS as u32,diff --git a/crates/backend-uzu/unit/encodable_block/delta_net_tree_verify_bench.rs b/crates/backend-uzu/unit/encodable_block/delta_net_tree_verify_bench.rsnew file mode 100644index 000000000..0910f2e17--- /dev/null+++ b/crates/backend-uzu/unit/encodable_block/delta_net_tree_verify_bench.rs@@ -0,0 +1,80 @@+#![cfg(metal_backend)]++use std::time::{Duration, Instant};++use criterion::Criterion;+use half::bf16;+use proc_macros::uzu_bench;++use crate::{+ backends::{+ common::{Backend, Context, Kernels, kernel::delta_net_tree_verify::DeltaNetTreeVerify},+ metal::Metal,+ },+ data_type::DataType,+ encodable_block::mixer::delta_net::tree_verify::{TreeVerifyEncodeArguments, TreeVerifyNewArguments},+ tests::{helpers::alloc_allocation_with_data, matmul::iter_encode_loop_named},+};++#[uzu_bench]+fn bench_delta_net_tree_verify(c: &mut Criterion) {+ const K_HEADS: usize = 16;+ const V_HEADS: usize = 48;+ const HEAD_DIM: usize = 128;+ let arguments = TreeVerifyNewArguments {+ data_type: DataType::BF16,+ num_k_heads: K_HEADS,+ num_v_heads: V_HEADS,+ head_k_dim: HEAD_DIM,+ head_v_dim: HEAD_DIM,+ };+ let context = <Metal as Backend>::Context::new().unwrap();+ let started = Instant::now();+ let tree_verify =+ <<Metal as Backend>::Kernels as Kernels>::DeltaNetTreeVerify::new(context.as_ref(), &arguments).unwrap();+ eprintln!("GDN tree verification PSO construction: {:.2} ms", started.elapsed().as_secs_f64() * 1e3);+ let h0 = alloc_allocation_with_data::<Metal, f32>(&context, &vec![0.001; V_HEADS * HEAD_DIM * HEAD_DIM]);+ let mut group = c.benchmark_group("Metal/Kernel/GDNTreeVerify");+ group.sample_size(20).warm_up_time(Duration::from_millis(300)).measurement_time(Duration::from_secs(1));++ for tree_size in [32usize, 49, 64, 128] {+ let q = alloc_allocation_with_data::<Metal, bf16>(+ &context,+ &vec![bf16::from_f32(0.01); tree_size * K_HEADS * HEAD_DIM],+ );+ let k = alloc_allocation_with_data::<Metal, bf16>(+ &context,+ &vec![bf16::from_f32(0.02); tree_size * K_HEADS * HEAD_DIM],+ );+ let v = alloc_allocation_with_data::<Metal, bf16>(+ &context,+ &vec![bf16::from_f32(0.03); tree_size * V_HEADS * HEAD_DIM],+ );+ let trie = alloc_allocation_with_data::<Metal, u32>(+ &context,+ &(0..tree_size as u32).flat_map(|node| [node, tree_size as u32 - 1, node]).collect::<Vec<_>>(),+ );+ let log_decay = alloc_allocation_with_data::<Metal, f32>(&context, &vec![-0.01; tree_size * V_HEADS]);+ let beta = alloc_allocation_with_data::<Metal, f32>(&context, &vec![0.2; tree_size * V_HEADS]);+ let benchmark_path = format!("Metal/Kernel/GDNTreeVerify/T{tree_size}");+ group.bench_function(format!("T{tree_size}"), |bencher| {+ iter_encode_loop_named(context.as_ref(), bencher, &benchmark_path, |encoder| {+ tree_verify+ .encode(+ TreeVerifyEncodeArguments {+ q: &q,+ k: &k,+ v: &v,+ trie: &trie,+ log_decay: &log_decay,+ beta: &beta,+ h0: &h0,+ tree_size,+ },+ encoder,+ )+ .unwrap();+ })+ });+ }+}