A high-performance inference engine for AI models
Fuse gdn tree verify (#996)
CCYeh43e9fe3dd88d
diff --git a/crates/uzu-engine/src/backends/cpu/kernel/gdn/tree_verify/mod.rs b/crates/uzu-engine/src/backends/cpu/kernel/gdn/tree_verify/mod.rsindex e4b9b6cf5..f7359462b 100644--- a/crates/uzu-engine/src/backends/cpu/kernel/gdn/tree_verify/mod.rs+++ b/crates/uzu-engine/src/backends/cpu/kernel/gdn/tree_verify/mod.rs@@ -1,6 +1,3 @@pub mod conv_scan;-pub mod out;-pub mod prefix;pub mod state_advance;-pub mod tree_gram;-pub mod tree_update_solve;+pub mod tree_verify;diff --git a/crates/uzu-engine/src/backends/cpu/kernel/gdn/tree_verify/out.rs b/crates/uzu-engine/src/backends/cpu/kernel/gdn/tree_verify/out.rsdeleted file mode 100644index fde31f84b..000000000--- a/crates/uzu-engine/src/backends/cpu/kernel/gdn/tree_verify/out.rs+++ /dev/null@@ -1,80 +0,0 @@-use half::bf16;-use num_traits::Float;-use uzu_engine_macros::kernel;--use crate::data_type::DataTypeRepr;--#[kernel(BuildTreeOut)]-#[variants(QKT, f32, bf16)]-#[variants(OutputT, f32, bf16)]-pub fn build_tree_out<QKT: DataTypeRepr + Float, OutputT: DataTypeRepr + Float>(- q: *const QKT,- prefix: *const f32,- qkd: *const f32,- u: *const f32,- #[optional(use_h0)] h0: Option<*const f32>,- o: *mut OutputT,- scale: f32,- batch_size: u32,- tree_size: u32,- qk_heads: u32,- value_heads: u32,- head_k_dim: u32,- head_v_dim: u32,- #[allow(unused)]- #[specialize]- use_mxu: bool,- #[allow(unused)]- #[specialize]- transposed_h0: bool,- #[specialize] use_h0: bool,-) {- let batch_size = batch_size as usize;- let tree_size = tree_size as usize;- let qk_heads = qk_heads as usize;- let value_heads = value_heads as usize;- let head_k_dim = head_k_dim as usize;- let head_v_dim = head_v_dim as usize;- let value_heads_per_qk_head = value_heads / qk_heads;- let h0 = h0.unwrap_or(std::ptr::null());-- for batch in 0..batch_size {- for value_head in 0..value_heads {- let qk_head = value_head / value_heads_per_qk_head;- let q_head_base = (batch * tree_size * qk_heads + qk_head) * head_k_dim;- let prefix_base = batch * tree_size * value_heads + value_head;- let qkd_base = (batch * value_heads + value_head) * tree_size * tree_size;- let u_base = (batch * value_heads + value_head) * tree_size * head_v_dim;- let out_base = ((batch * tree_size) * value_heads + value_head) * head_v_dim;-- for row in 0..tree_size {- let q_row = q_head_base + row * qk_heads * head_k_dim;- for value_col in 0..head_v_dim {- let mut acc = 0.0f32;-- if use_h0 {- let h0_base = ((batch * value_heads + value_head) * head_v_dim + value_col) * head_k_dim;- let mut dot = 0.0f32;- for dim in 0..head_k_dim {- let qv = unsafe { (*q.add(q_row + dim)).to_f32().unwrap() };- let hv = unsafe { *h0.add(h0_base + dim) };- dot += qv * hv;- }- let prefix_row = unsafe { *prefix.add(prefix_base + row * value_heads) };- acc += prefix_row.exp() * scale * dot;- }-- for col in 0..tree_size {- let qkd_value = unsafe { *qkd.add(qkd_base + row * tree_size + col) };- let u_value = unsafe { *u.add(u_base + col * head_v_dim + value_col) };- acc += qkd_value * u_value;- }-- unsafe {- *o.add(out_base + row * value_heads * head_v_dim + value_col) = OutputT::from(acc).unwrap();- }- }- }- }- }-}diff --git a/crates/uzu-engine/src/backends/cpu/kernel/gdn/tree_verify/prefix.rs b/crates/uzu-engine/src/backends/cpu/kernel/gdn/tree_verify/prefix.rsdeleted file mode 100644index 37c2b0332..000000000--- a/crates/uzu-engine/src/backends/cpu/kernel/gdn/tree_verify/prefix.rs+++ /dev/null@@ -1,39 +0,0 @@-use uzu_engine_macros::kernel;--use crate::backends::common::gpu_types::trie::TrieNode;--#[kernel(BuildTreePrefix)]-pub fn build_tree_prefix(- trie: *const TrieNode,- log_decay: *const f32,- prefix: *mut f32,- batch_size: u32,- tree_size: u32,- value_heads: u32,-) {- let batch_size = batch_size as usize;- let tree_size = tree_size as usize;- let value_heads = value_heads as usize;-- for batch in 0..batch_size {- let trie_batch = batch * tree_size;- let batch_offset = batch * tree_size * value_heads;-- for row in 0..tree_size {- let row_idx = row as u32;- for head in 0..value_heads {- let mut sum = 0.0f32;- for col in 0..tree_size {- // col is an ancestor-or-self of row iff row falls in col's- // DFS subtree interval [trie_start, trie_end].- let node = unsafe { *trie.add(trie_batch + col) };- if row_idx < node.trie_start || row_idx > node.trie_end {- continue;- }- sum += unsafe { *log_decay.add(batch_offset + col * value_heads + head) };- }- unsafe { *prefix.add(batch_offset + row * value_heads + head) = sum };- }- }- }-}diff --git a/crates/uzu-engine/src/backends/cpu/kernel/gdn/tree_verify/tree_gram.rs b/crates/uzu-engine/src/backends/cpu/kernel/gdn/tree_verify/tree_gram.rsdeleted file mode 100644index 8488e2ed7..000000000--- a/crates/uzu-engine/src/backends/cpu/kernel/gdn/tree_verify/tree_gram.rs+++ /dev/null@@ -1,158 +0,0 @@-use half::bf16;-use num_traits::Float;-use uzu_engine_macros::kernel;--use crate::{backends::common::gpu_types::trie::TrieNode, data_type::DataTypeRepr};--const BLOCK: usize = 16;--#[kernel(BuildTreeGram)]-#[variants(T, f32, bf16)]-#[variants(USE_MXU, false)]-pub fn build_tree_gram<T: DataTypeRepr + Float, const USE_MXU: bool>(- q: *const T,- k: *const T,- trie: *const TrieNode,- prefix: *const f32,- beta: *const f32,- #[optional(use_h0)] h0: Option<*const f32>,- a_packed: *mut f32,- qkd: *mut f32,- a_inv: *mut f32,- #[optional(use_h0)] kh0: Option<*mut f32>,- scale: f32,- batch_size: u32,- tree_size: u32,- k_heads: u32,- value_heads: u32,- head_k_dim: u32,- head_v_dim: u32,- #[specialize] use_h0: bool,-) {- let batch_size = batch_size as usize;- let tree_size = tree_size as usize;- let k_heads = k_heads as usize;- let value_heads = value_heads as usize;- let head_k_dim = head_k_dim as usize;- let head_v_dim = head_v_dim as usize;-- let value_heads_per_key_head = value_heads / k_heads;- let num_blocks = tree_size.div_ceil(BLOCK);- let num_col_pairs = num_blocks.div_ceil(2);-- for batch in 0..batch_size {- for hv in 0..value_heads {- let hk = hv / value_heads_per_key_head;-- // qkd[row, col] = scale * exp(prefix[row] - prefix[col]) * dot(q[row], k[col])- // for ancestor-or-self, else 0. Dense [T, T].- let mat_base = (batch * value_heads + hv) * tree_size * tree_size;- for row in 0..tree_size {- let prefix_row = unsafe { *prefix.add((batch * tree_size + row) * value_heads + hv) };- let row_off = ((batch * tree_size + row) * k_heads + hk) * head_k_dim;- for col in 0..tree_size {- let out = mat_base + row * tree_size + col;- let node = unsafe { *trie.add(batch * tree_size + col) };- if (row as u32) < node.trie_start || (row as u32) > node.trie_end {- unsafe { *qkd.add(out) = 0.0 };- continue;- }- let col_off = ((batch * tree_size + col) * k_heads + hk) * head_k_dim;- let mut qk = 0.0f32;- for d in 0..head_k_dim {- qk += unsafe { (*q.add(row_off + d)).to_f32().unwrap() }- * unsafe { (*k.add(col_off + d)).to_f32().unwrap() };- }- let prefix_col = unsafe { *prefix.add((batch * tree_size + col) * value_heads + hv) };- unsafe { *qkd.add(out) = (prefix_row - prefix_col).exp() * scale * qk };- }- }-- // A[row, col] = beta[row] * exp(prefix[row] - prefix[col]) * dot(k[row], k[col])- // for proper ancestors, else 0; packed [NB, ceil(NB/2), 16, 32] tiles,- // only pairs touching the block lower triangle (same tiles the GPU writes).- let a_base = (batch * value_heads + hv) * num_blocks * num_col_pairs * BLOCK * 2 * BLOCK;- for block in 0..num_blocks {- for pair in 0..=block / 2 {- let tile_base = a_base + (block * num_col_pairs + pair) * BLOCK * 2 * BLOCK;- for local_row in 0..BLOCK {- for local_col in 0..2 * BLOCK {- let row = block * BLOCK + local_row;- let col = pair * 2 * BLOCK + local_col;- let mut value = 0.0f32;- if row != col && row < tree_size && col < tree_size {- let node = unsafe { *trie.add(batch * tree_size + col) };- if (row as u32) >= node.trie_start && (row as u32) <= node.trie_end {- let row_off = ((batch * tree_size + row) * k_heads + hk) * head_k_dim;- let col_off = ((batch * tree_size + col) * k_heads + hk) * head_k_dim;- let mut kk = 0.0f32;- for d in 0..head_k_dim {- kk += unsafe { (*k.add(row_off + d)).to_f32().unwrap() }- * unsafe { (*k.add(col_off + d)).to_f32().unwrap() };- }- let prefix_row =- unsafe { *prefix.add((batch * tree_size + row) * value_heads + hv) };- let prefix_col =- unsafe { *prefix.add((batch * tree_size + col) * value_heads + hv) };- let beta_row = unsafe { *beta.add((batch * tree_size + row) * value_heads + hv) };- value = beta_row * (prefix_row - prefix_col).exp() * kk;- }- }- unsafe { *a_packed.add(tile_base + local_row * 2 * BLOCK + local_col) = value };- }- }- }- }-- // a_inv = (I + A_diag)^-1 per block, compact [NB, 16, 16], identity-padded;- // A is f32 so the diagonal values are read back from a_packed unrounded.- for block in 0..num_blocks {- let block_size = BLOCK.min(tree_size - block * BLOCK);- let block_base = ((batch * value_heads + hv) * num_blocks + block) * BLOCK * BLOCK;- let diag_tile = a_base + (block * num_col_pairs + block / 2) * BLOCK * 2 * BLOCK;- let diag_col = (block % 2) * BLOCK;- for row in 0..BLOCK {- for col in 0..BLOCK {- unsafe {- *a_inv.add(block_base + row * BLOCK + col) = if row == col {- 1.0- } else {- 0.0- }- };- }- }- for row in 0..block_size {- for col in 0..row {- let mut sum = 0.0f32;- for prev_row in col..row {- sum += unsafe { *a_packed.add(diag_tile + row * 2 * BLOCK + diag_col + prev_row) }- * unsafe { *a_inv.add(block_base + prev_row * BLOCK + col) };- }- unsafe { *a_inv.add(block_base + row * BLOCK + col) = -sum };- }- }- }-- // kh0[b, token, hv, dv] = dot(k[b, token, hk], h0[b, hv, dv])- if !use_h0 {- continue;- }- let h0 = h0.unwrap();- let kh0 = kh0.unwrap();- let h0_head = (batch * value_heads + hv) * head_v_dim * head_k_dim;- for token in 0..tree_size {- let k_off = ((batch * tree_size + token) * k_heads + hk) * head_k_dim;- let kh0_off = ((batch * tree_size + token) * value_heads + hv) * head_v_dim;- for dv in 0..head_v_dim {- let mut sum = 0.0f32;- for d in 0..head_k_dim {- sum += unsafe { (*k.add(k_off + d)).to_f32().unwrap() }- * unsafe { *h0.add(h0_head + dv * head_k_dim + d) };- }- unsafe { *kh0.add(kh0_off + dv) = sum };- }- }- }- }-}diff --git a/crates/uzu-engine/src/backends/cpu/kernel/gdn/tree_verify/tree_update_solve.rs b/crates/uzu-engine/src/backends/cpu/kernel/gdn/tree_verify/tree_update_solve.rsdeleted file mode 100644index fcc41335f..000000000--- a/crates/uzu-engine/src/backends/cpu/kernel/gdn/tree_verify/tree_update_solve.rs+++ /dev/null@@ -1,122 +0,0 @@-use half::bf16;-use num_traits::Float;-use uzu_engine_macros::kernel;--use crate::data_type::DataTypeRepr;--/// Solves chunked GDN update coefficients: `(I + A) U = beta * (v - exp(prefix) * kh0)`.-/// `kh0 = k @ h0^T` is precomputed by BuildTreeGram.-#[kernel(TreeUpdateSolve)]-#[variants(T, f32, bf16)]-#[variants(BV, 16, 32)]-pub fn tree_update_solve<T: DataTypeRepr + Float, const BV: u32>(- #[optional(use_h0)] kh0: Option<*const f32>,- v: *const T,- prefix: *const f32,- beta: *const f32,- a_packed: *const f32,- a_inv: *const f32,- u: *mut f32,- batch_size: u32,- tree_size: u32,- value_heads: u32,- head_v_dim: u32,- #[specialize] use_h0: bool,-) {- let batch_size = batch_size as usize;- let tree_size = tree_size as usize;- let value_heads = value_heads as usize;- let head_v_dim = head_v_dim as usize;- let block_size = 16usize;-- if batch_size == 0 || tree_size == 0 || value_heads == 0 || head_v_dim == 0 {- return;- }-- let num_blocks = tree_size.div_ceil(block_size);- let num_col_pairs = num_blocks.div_ceil(2);-- let kh0 = kh0.unwrap_or(std::ptr::null());- for batch in 0..batch_size {- for hv in 0..value_heads {- let batch_value_head_idx = batch * value_heads + hv;-- for block in 0..num_blocks {- let token_base = block * block_size;- let mut acc = vec![0.0f32; block_size * head_v_dim];-- for local_token in 0..block_size {- let token = token_base + local_token;- if token >= tree_size {- continue;- }-- let prefix_idx = (batch * tree_size + token) * value_heads + hv;- let beta_val = unsafe { *beta.add(prefix_idx) };- let decay_from_h0 = unsafe { *prefix.add(prefix_idx) }.exp();-- for dv in 0..head_v_dim {- let v_idx = (((batch * tree_size + token) * value_heads + hv) * head_v_dim) + dv;- let v_val = unsafe { (*v.add(v_idx)).to_f32().unwrap() };-- let kh0_val = if use_h0 {- unsafe { *kh0.add(v_idx) }- } else {- 0.0- };-- acc[local_token * head_v_dim + dv] = beta_val * (v_val - decay_from_h0 * kh0_val);- }- }-- for prev_token in 0..token_base {- let prev_block = prev_token / block_size;- let prev_local = prev_token % block_size;- for local_token in 0..block_size {- let token = token_base + local_token;- if token >= tree_size {- continue;- }-- // A is packed [B*HV, NB, ceil(NB/2), 16, 32] block-pair tiles.- let a_idx = ((batch_value_head_idx * num_blocks + block) * num_col_pairs + prev_block / 2)- * (block_size * 2 * block_size)- + local_token * (2 * block_size)- + (prev_block % 2) * block_size- + prev_local;- let a_val = unsafe { *a_packed.add(a_idx) };- for dv in 0..head_v_dim {- let u_prev_idx = (batch_value_head_idx * tree_size + prev_token) * head_v_dim + dv;- acc[local_token * head_v_dim + dv] -= a_val * unsafe { *u.add(u_prev_idx) };- }- }- }-- for local_token in 0..block_size {- let token = token_base + local_token;- if token >= tree_size {- continue;- }-- for dv in 0..head_v_dim {- let mut sum = 0.0f32;- for local_prev_token in 0..block_size {- let source_token = token_base + local_prev_token;- if source_token >= tree_size {- continue;- }- // Ainv is compact [B * HV, ceil(T/BT), BT, BT] diagonal blocks.- let inv_idx = ((batch_value_head_idx * num_blocks + block) * block_size + local_token)- * block_size- + local_prev_token;- sum += unsafe { *a_inv.add(inv_idx) } * acc[local_prev_token * head_v_dim + dv];- }-- let u_idx = (batch_value_head_idx * tree_size + token) * head_v_dim + dv;- unsafe { *u.add(u_idx) = sum };- }- }- }- }- }-}diff --git a/crates/uzu-engine/src/backends/cpu/kernel/gdn/tree_verify/tree_verify.rs b/crates/uzu-engine/src/backends/cpu/kernel/gdn/tree_verify/tree_verify.rsnew file mode 100644index 000000000..8c59896af--- /dev/null+++ b/crates/uzu-engine/src/backends/cpu/kernel/gdn/tree_verify/tree_verify.rs@@ -0,0 +1,105 @@+use half::bf16;+use num_traits::Float;+use uzu_engine_macros::kernel;++use crate::{backends::common::gpu_types::trie::TrieNode, data_type::DataTypeRepr};++#[kernel(TreeVerify)]+#[variants(T, f32, bf16)]+#[variants(NB, 1, 2)]+pub fn tree_verify<T: DataTypeRepr + Float, const NB: u32>(+ q: *const T,+ k: *const T,+ v: *const T,+ trie: *const TrieNode,+ log_decay: *const f32,+ beta: *const f32,+ h0: *const f32,+ in_proj: *const T,+ norm_weight: *const f32,+ o: *mut T,+ scale: f32,+ norm_epsilon: f32,+ batch_size: u32,+ tree_size: u32,+ num_k_heads: u32,+ num_v_heads: u32,+ head_k_dim: u32,+ head_v_dim: u32,+ conv_dim: u32,+ total_proj_dim: u32,+) {+ let batch_size = batch_size as usize;+ let tree_size = tree_size as usize;+ let num_k_heads = num_k_heads as usize;+ let num_v_heads = num_v_heads as usize;+ let head_k_dim = head_k_dim as usize;+ let head_v_dim = head_v_dim as usize;+ assert!(tree_size <= 16 * NB as usize);++ let load = |ptr: *const T, index: usize| unsafe { (*ptr.add(index)).to_f32().unwrap() };+ for batch in 0..batch_size {+ for hv_idx in 0..num_v_heads {+ let hk_idx = hv_idx / (num_v_heads / num_k_heads);+ let qk_off = |token: usize| ((batch * tree_size + token) * num_k_heads + hk_idx) * head_k_dim;+ let scalar = |token: usize| (batch * tree_size + token) * num_v_heads + hv_idx;+ let nodes: Vec<TrieNode> = (0..tree_size).map(|c| unsafe { *trie.add(batch * tree_size + c) }).collect();+ let in_subtree =+ |row: usize, col: usize| (row as u32) >= nodes[col].trie_start && (row as u32) <= nodes[col].trie_end;++ let prefix: Vec<f32> = (0..tree_size)+ .map(|row| {+ (0..tree_size).filter(|&c| in_subtree(row, c)).map(|c| unsafe { *log_decay.add(scalar(c)) }).sum()+ })+ .collect();+ let dot =+ |a: &dyn Fn(usize) -> f32, b: &dyn Fn(usize) -> f32| (0..head_k_dim).map(|d| a(d) * b(d)).sum::<f32>();+ let mut a = vec![0.0f32; tree_size * tree_size];+ let mut qkd = vec![0.0f32; tree_size * tree_size];+ for row in 0..tree_size {+ for col in 0..tree_size {+ if !in_subtree(row, col) {+ continue;+ }+ let decay = (prefix[row] - prefix[col]).exp();+ qkd[row * tree_size + col] =+ scale * decay * dot(&|d| load(q, qk_off(row) + d), &|d| load(k, qk_off(col) + d));+ if row != col {+ let kk = dot(&|d| load(k, qk_off(row) + d), &|d| load(k, qk_off(col) + d));+ a[row * tree_size + col] = unsafe { *beta.add(scalar(row)) } * decay * kk;+ }+ }+ }++ let h0_base = (batch * num_v_heads + hv_idx) * head_v_dim * head_k_dim;+ for dv in 0..head_v_dim {+ let h0_row = |d: usize| unsafe { *h0.add(h0_base + dv * head_k_dim + d) };+ let mut u = vec![0.0f32; tree_size];+ for row in 0..tree_size {+ let kh0 = dot(&|d| load(k, qk_off(row) + d), &h0_row);+ let v_value = load(v, scalar(row) * head_v_dim + dv);+ let rhs = unsafe { *beta.add(scalar(row)) } * (v_value - prefix[row].exp() * kh0);+ u[row] = rhs - (0..row).map(|c| a[row * tree_size + c] * u[c]).sum::<f32>();+ }+ for row in 0..tree_size {+ let qh0 = dot(&|d| load(q, qk_off(row) + d), &h0_row);+ let value = prefix[row].exp() * scale * qh0+ + (0..tree_size).map(|c| qkd[row * tree_size + c] * u[c]).sum::<f32>();+ unsafe { *o.add(scalar(row) * head_v_dim + dv) = T::from(value).unwrap() };+ }+ }+ }+ }+ super::super::norm_gate::delta_net_norm_gate::<T>(+ o,+ in_proj,+ norm_weight,+ num_v_heads as u32,+ head_v_dim as u32,+ (num_v_heads * head_v_dim) as u32,+ conv_dim,+ total_proj_dim,+ norm_epsilon,+ (batch_size * tree_size) as u32,+ );+}diff --git a/crates/uzu-engine/src/backends/metal/kernel/gdn/chunked/precompute.metal b/crates/uzu-engine/src/backends/metal/kernel/gdn/chunked/precompute.metalindex df8e83249..c60a91ca2 100644--- a/crates/uzu-engine/src/backends/metal/kernel/gdn/chunked/precompute.metal+++ b/crates/uzu-engine/src/backends/metal/kernel/gdn/chunked/precompute.metal@@ -60,8 +60,6 @@ KERNEL(DeltaNetChunkedPrecompute)(using GramRightFragment = OperandFragment<float, 1, GRAM_COL_FRAGMENTS, Ops, ReadTranspose>;using TileFragment = Fragment<float, ROW_FRAGMENTS, ROW_FRAGMENTS, Ops>;- using MatrixFragment = OperandFragment<float, ROW_FRAGMENTS, ROW_FRAGMENTS, Ops>;- using ValueFragment = OperandFragment<bfloat, ROW_FRAGMENTS, ROW_FRAGMENTS, Ops>;static_assert(HEAD_K_DIM % Ops::FRAGMENT_ROWS == 0, "HEAD_K_DIM must align to fragment K");constexpr uint num_blocks = CHUNK_SIZE / BLOCK_SIZE;@@ -211,50 +209,20 @@ KERNEL(DeltaNetChunkedPrecompute)(threadgroup_barrier(mem_flags::mem_threadgroup);// Solve this simdgroup's 16-column slice of T; blocks above the slice are zero, solved blocks stay in registers.- const uint slice_base = block_idx * BLOCK_SIZE;- device bfloat* t_head = t_out + head_base * CHUNK_SIZE * CHUNK_SIZE + slice_base;- ValueFragment t_blocks[num_blocks];- METAL_PRAGMA_UNROLL- for (uint b = 0; b < num_blocks; ++b) {- TileFragment t_block;- t_block.clear();- if (b >= block_idx) {- const threadgroup float* b_tiles = a_tiles + tile_base(b) * (BLOCK_SIZE * COL_TILE);- TileFragment acc_t;- acc_t.map_coords(lane, [&](short local_row, short local_col, float) UZU_ALWAYS_INLINE {- return b * BLOCK_SIZE + uint(local_row) == slice_base + uint(local_col) ? 1.0f : 0.0f;- });- METAL_PRAGMA_UNROLL- for (uint prev_block_idx = 0; prev_block_idx < b; ++prev_block_idx) {- if (prev_block_idx >= block_idx) {- MatrixFragment a_frag;- a_frag.load_from(- lane,- fragment_source(- b_tiles + (prev_block_idx / 2) * (BLOCK_SIZE * COL_TILE) + (prev_block_idx % 2) * BLOCK_SIZE,- COL_TILE- )- );- a_frag.map([](float value) UZU_ALWAYS_INLINE { return -value; });- fragment_mma(acc_t, a_frag, t_blocks[prev_block_idx]);- }- }- MatrixFragment inv_frag;- inv_frag.load_from(- lane,- fragment_source(- static_cast<const threadgroup float*>(diag_inverses + b * (BLOCK_SIZE * BLOCK_SIZE)),- BLOCK_SIZE- )- );- fragment_mma(t_block, inv_frag, acc_t);- }- t_block.store(lane, t_head + b * BLOCK_SIZE * CHUNK_SIZE, CHUNK_SIZE);- METAL_PRAGMA_UNROLL- for (ushort e = 0; e < ValueFragment::ELEMENTS_PER_FRAGMENT; ++e) {- t_blocks[b].elements()[e] = bfloat(t_block.elements()[e]);- }- }+ device bfloat* t_head = t_out + head_base * CHUNK_SIZE * CHUNK_SIZE + block_idx * BLOCK_SIZE;+ solve_lower_triangular_slice<BLOCK_SIZE, num_blocks, bfloat>(+ [&](uint b) UZU_ALWAYS_INLINE {+ return static_cast<const threadgroup float*>(a_tiles + tile_base(b) * (BLOCK_SIZE * COL_TILE));+ },+ [](const threadgroup float* row, uint m)+ UZU_ALWAYS_INLINE { return row + (m / 2) * (BLOCK_SIZE * COL_TILE) + (m % 2) * BLOCK_SIZE; },+ COL_TILE,+ diag_inverses,+ [&](uint b, thread TileFragment& t_block)+ UZU_ALWAYS_INLINE { t_block.store(lane, t_head + b * BLOCK_SIZE * CHUNK_SIZE, CHUNK_SIZE); },+ block_idx,+ lane+ );threadgroup_barrier(mem_flags::mem_threadgroup);}diff --git a/crates/uzu-engine/src/backends/metal/kernel/gdn/common/tri_inv.h b/crates/uzu-engine/src/backends/metal/kernel/gdn/common/tri_inv.hindex 947fc5922..9960d5bdc 100644--- a/crates/uzu-engine/src/backends/metal/kernel/gdn/common/tri_inv.h+++ b/crates/uzu-engine/src/backends/metal/kernel/gdn/common/tri_inv.h@@ -1,9 +1,12 @@#pragma once#include "../../common/defines.h"+#include "../../matmul/common/fragment.h"+#include "../../matmul/common/simdgroup_fragment_ops.h"#include <metal_stdlib>using namespace metal;+using namespace uzu::matmul;template <uint BLOCK, uint A_STRIDE = BLOCK, typename OutPtr>METAL_FUNC void invert_lower_triangular_block(@@ -39,3 +42,51 @@ METAL_FUNC void invert_lower_triangular_block(a_inv_block[row * BLOCK + col] = inverse_col[row];}}++template <uint BLOCK, uint NB, typename CarryT, typename ARow, typename ABlock, typename StoreBlock>+METAL_FUNC void solve_lower_triangular_slice(+ ARow a_row,+ ABlock a_block,+ const uint a_stride,+ const threadgroup float* diag_inverses,+ StoreBlock store_block,+ const uint slice,+ const uint lane+) {+ using Ops = SimdgroupFragmentOps;+ constexpr ushort ROW_FRAGMENTS = BLOCK / Ops::FRAGMENT_ROWS;+ using TileFragment = Fragment<float, ROW_FRAGMENTS, ROW_FRAGMENTS, Ops>;+ using MatrixFragment = OperandFragment<float, ROW_FRAGMENTS, ROW_FRAGMENTS, Ops>;+ using CarryFragment = OperandFragment<CarryT, ROW_FRAGMENTS, ROW_FRAGMENTS, Ops>;++ CarryFragment t_blocks[NB];+ METAL_PRAGMA_UNROLL+ for (uint b = 0; b < NB; ++b) {+ TileFragment t_block;+ t_block.clear();+ if (b >= slice) {+ const threadgroup float* b_tiles = a_row(b);+ TileFragment acc_t;+ acc_t.map_coords(lane, [&](short local_row, short local_col, float) UZU_ALWAYS_INLINE {+ return b * BLOCK + uint(local_row) == slice * BLOCK + uint(local_col) ? 1.0f : 0.0f;+ });+ METAL_PRAGMA_UNROLL+ for (uint m = 0; m < b; ++m) {+ if (m >= slice) {+ MatrixFragment a_frag;+ a_frag.load_from(lane, fragment_source(a_block(b_tiles, m), a_stride));+ a_frag.map([](float value) UZU_ALWAYS_INLINE { return -value; });+ fragment_mma(acc_t, a_frag, t_blocks[m]);+ }+ }+ MatrixFragment inv_frag;+ inv_frag.load_from(lane, fragment_source(diag_inverses + b * (BLOCK * BLOCK), BLOCK));+ fragment_mma(t_block, inv_frag, acc_t);+ }+ store_block(b, t_block);+ METAL_PRAGMA_UNROLL+ for (ushort e = 0; e < CarryFragment::ELEMENTS_PER_FRAGMENT; ++e) {+ t_blocks[b].elements()[e] = CarryT(t_block.elements()[e]);+ }+ }+}diff --git a/crates/uzu-engine/src/backends/metal/kernel/gdn/tree_verify.rs b/crates/uzu-engine/src/backends/metal/kernel/gdn/tree_verify.rsindex aa830c87d..20f3d6f64 100644--- a/crates/uzu-engine/src/backends/metal/kernel/gdn/tree_verify.rs+++ b/crates/uzu-engine/src/backends/metal/kernel/gdn/tree_verify.rs@@ -1,68 +1,23 @@-use metal::MTLGPUFamily;-use crate::{backends::{common::{Backend, CommandBufferEncoding, Kernels,- kernel::{- BuildTreeGramKernel, BuildTreeOutKernel, BuildTreePrefixKernel, TreeUpdateSolveKernel,- delta_net_tree_verify::DeltaNetTreeVerify,- },+ kernel::{TreeVerifyKernel, delta_net_tree_verify::DeltaNetTreeVerify},},metal::{- Metal,- command_buffer::MetalCommandBufferEncoding,- context::{LARGE_MIN_GPU_CORES, MetalContext},- error::MetalError,+ Metal, command_buffer::MetalCommandBufferEncoding, context::MetalContext, error::MetalError,kernel::MetalKernels,},},- data_type::DataType,encodable_block::mixer::delta_net::tree_verify::{TreeVerifyEncodeArguments, TreeVerifyNewArguments},};const TOKEN_BLOCK: u32 = 16;-const BLOCK_PAIR_WIDTH: u32 = 2 * TOKEN_BLOCK;-const INNER_DATA_TYPE: DataType = DataType::F32;--struct Layout {- tree_size: u32,- num_blocks: u32,- num_block_pairs: u32,- num_v_heads: u32,- head_v_dim: u32,-}--impl Layout {- const fn new(- tree_size: u32,- arguments: &TreeVerifyNewArguments,- ) -> Self {- let num_blocks = tree_size.div_ceil(TOKEN_BLOCK);- Self {- tree_size,- num_blocks,- num_block_pairs: num_blocks.div_ceil(2),- num_v_heads: arguments.num_v_heads,- head_v_dim: arguments.head_v_dim,- }- }-- const fn a_packed_shape(&self) -> [u32; 5] {- [self.num_v_heads, self.num_blocks, self.num_block_pairs, TOKEN_BLOCK, BLOCK_PAIR_WIDTH]- }-- const fn a_inverse_shape(&self) -> [u32; 4] {- [self.num_v_heads, self.num_blocks, TOKEN_BLOCK, TOKEN_BLOCK]- }-}+const MAX_TREE_SIZE: u32 = 2 * TOKEN_BLOCK;pub struct MetalDeltaNetTreeVerify {arguments: TreeVerifyNewArguments,- prefix: <MetalKernels as Kernels>::BuildTreePrefixKernel,- gram: <MetalKernels as Kernels>::BuildTreeGramKernel,- solve: <MetalKernels as Kernels>::TreeUpdateSolveKernel,- out: <MetalKernels as Kernels>::BuildTreeOutKernel,+ kernels: [<MetalKernels as Kernels>::TreeVerifyKernel; 2],}impl DeltaNetTreeVerify for MetalDeltaNetTreeVerify {@@ -76,23 +31,12 @@ impl DeltaNetTreeVerify for MetalDeltaNetTreeVerify {context: &MetalContext,arguments: &TreeVerifyNewArguments,) -> Result<Self, MetalError> {- let use_mxu = arguments.data_type == DataType::BF16 && context.supports_mxu;- let transposed_h0 = !use_mxu- && context.gpu_core_count < LARGE_MIN_GPU_CORES- && context.apple_gpu_family <= MTLGPUFamily::Apple8;Ok(Self {arguments: *arguments,- prefix: <MetalKernels as Kernels>::BuildTreePrefixKernel::new(context)?,- gram: <MetalKernels as Kernels>::BuildTreeGramKernel::new(context, arguments.data_type, use_mxu, true)?,- solve: <MetalKernels as Kernels>::TreeUpdateSolveKernel::new(context, arguments.data_type, 32, true)?,- out: <MetalKernels as Kernels>::BuildTreeOutKernel::new(- context,- arguments.data_type,- arguments.data_type,- use_mxu,- transposed_h0,- true,- )?,+ kernels: [+ <MetalKernels as Kernels>::TreeVerifyKernel::new(context, arguments.data_type, 1)?,+ <MetalKernels as Kernels>::TreeVerifyKernel::new(context, arguments.data_type, 2)?,+ ],})}@@ -101,79 +45,32 @@ impl DeltaNetTreeVerify for MetalDeltaNetTreeVerify {arguments: TreeVerifyEncodeArguments<'_, Metal>,command_buffer: &mut MetalCommandBufferEncoding,) -> Result<<Metal as Backend>::ScratchBuffer, MetalError> {- let layout = Layout::new(arguments.tree_size, &self.arguments);- let mut prefix =- command_buffer.allocate_scratch_for_shape(&[layout.tree_size, layout.num_v_heads], INNER_DATA_TYPE)?;- let mut a_packed = command_buffer.allocate_scratch_for_shape(&layout.a_packed_shape(), INNER_DATA_TYPE)?;- let mut qkd = command_buffer- .allocate_scratch_for_shape(&[layout.num_v_heads, layout.tree_size, layout.tree_size], INNER_DATA_TYPE)?;- let mut a_inverse = command_buffer.allocate_scratch_for_shape(&layout.a_inverse_shape(), INNER_DATA_TYPE)?;- let mut kh0 = command_buffer- .allocate_scratch_for_shape(&[layout.tree_size, layout.num_v_heads, layout.head_v_dim], INNER_DATA_TYPE)?;- let mut u = command_buffer- .allocate_scratch_for_shape(&[layout.num_v_heads, layout.tree_size, layout.head_v_dim], INNER_DATA_TYPE)?;+ assert!(arguments.tree_size <= MAX_TREE_SIZE, "GDN tree verification supports at most {MAX_TREE_SIZE} nodes");let mut output = command_buffer.allocate_scratch_for_shape(- &[layout.tree_size, layout.num_v_heads, layout.head_v_dim],+ &[arguments.tree_size, self.arguments.num_v_heads, self.arguments.head_v_dim],self.arguments.data_type,)?;-- self.prefix.encode(- arguments.trie,- arguments.log_decay,- &mut prefix,- 1,- arguments.tree_size,- self.arguments.num_v_heads,- command_buffer,- );- self.gram.encode(+ self.kernels[(arguments.tree_size > TOKEN_BLOCK) as usize].encode(arguments.q,arguments.k,- arguments.trie,- &prefix,- arguments.beta,- Some(arguments.h0),- &mut a_packed,- &mut qkd,- &mut a_inverse,- Some(&mut kh0),- 1.0,- 1,- arguments.tree_size,- self.arguments.num_k_heads,- self.arguments.num_v_heads,- self.arguments.head_k_dim,- self.arguments.head_v_dim,- command_buffer,- );- self.solve.encode(- Some(&kh0),arguments.v,- &prefix,+ arguments.trie,+ arguments.log_decay,arguments.beta,- &a_packed,- &a_inverse,- &mut u,- 1,- arguments.tree_size,- self.arguments.num_v_heads,- self.arguments.head_v_dim,- command_buffer,- );- self.out.encode(- arguments.q,- &prefix,- &qkd,- &u,- Some(arguments.h0),+ arguments.h0,+ arguments.in_proj,+ arguments.norm_weight,&mut output,1.0,+ arguments.norm_epsilon,1,arguments.tree_size,self.arguments.num_k_heads,self.arguments.num_v_heads,self.arguments.head_k_dim,self.arguments.head_v_dim,+ arguments.conv_dim,+ arguments.total_proj_dim,command_buffer,);Ok(output)diff --git a/crates/uzu-engine/src/backends/metal/kernel/gdn/tree_verify/out.metal b/crates/uzu-engine/src/backends/metal/kernel/gdn/tree_verify/out.metaldeleted file mode 100644index e00132f9d..000000000--- a/crates/uzu-engine/src/backends/metal/kernel/gdn/tree_verify/out.metal+++ /dev/null@@ -1,156 +0,0 @@-#include <metal_stdlib>-#include "../../common/defines.h"-#include "../../common/dsl.h"-#include "../../common/thread_context.h"-#include "../../matmul/common/fragment.h"-#include "../../matmul/common/mxu_fragment/ops.h"-#include "../../matmul/common/simdgroup_fragment_ops.h"--using namespace metal;-using namespace uzu::matmul;--#define MATMUL_MXU_ROWS 16u-#define MATMUL_SIMDGROUP_ROWS 8u-#define MATMUL_COLS 32u-#define SIMDGROUPS_PER_TG 4u--// Build tree output for one (batch, value-head, row, value) tile.-//-// q: [B, T, Hg, K], with query head hq = hv / (HV / Hg)-// prefix: [B, T, HV] path log-decay prefix from BuildTreePrefix-// qkd: [B, HV, T, T] from BuildTreeGram-// u: [B, HV, T, V]-// h0: [B, HV, V, K]-//-// o[row, value] = exp(prefix[row]) * scale * dot(q[row], h0[value])-// + sum_j qkd[row, j] * u[j, value]-template <typename QKT, typename OutputT, bool use_mxu, bool transposed_h0>-VARIANTS(QKT, float, bfloat)-VARIANTS(OutputT, float, bfloat)-VARIANTS(use_mxu, false, true)-VARIANTS(transposed_h0, false, true)-CONSTRAINT(!(use_mxu && transposed_h0))-PUBLIC KERNEL(BuildTreeOut)(- const device QKT* q,- const device float* prefix,- const device float* qkd,- const device float* u,- const device float* h0 OPTIONAL(use_h0),- device OutputT* o,- constant const float& scale,- constant const uint& batch_size,- constant const uint& tree_size,- constant const uint& qk_heads,- constant const uint& value_heads,- constant const uint& head_k_dim,- constant const uint& head_v_dim,- threadgroup uint u_tile_scratch OPTIONAL(transposed_h0)[MATMUL_SIMDGROUP_ROWS * MATMUL_COLS],- const bool use_h0 SPECIALIZE,- const ThreadContext thread_context,- const uint v_tile_idx GROUPS(head_v_dim.div_ceil(MATMUL_COLS)),- const uint row_tile_group_idx GROUPS(tree_size.div_ceil(- if use_mxu {- SIMDGROUPS_PER_TG * MATMUL_MXU_ROWS- } else {- SIMDGROUPS_PER_TG * MATMUL_SIMDGROUP_ROWS- }- )),- const uint batch_value_head_idx GROUPS(batch_size * value_heads),- const uint tid THREADS(SIMDGROUPS_PER_TG * METAL_SIMD_SIZE)-) {- // Relaxed MPP diverges from FP32 across layers.- using Ops = metal::conditional_t<use_mxu, MxuStrictFragmentOps, SimdgroupFragmentOps>;- using H0Read = metal::conditional_t<transposed_h0, ReadTranspose, ReadDirect>;- constexpr ushort ROWS = Ops::FRAGMENT_ROWS;- constexpr ushort COL_FRAGMENTS = MATMUL_COLS / Ops::FRAGMENT_COLS;- using AccFragment = Fragment<float, 1, COL_FRAGMENTS, Ops>;- using QFragment = OperandFragment<float, 1, 1, Ops>;- using H0Fragment = OperandFragment<float, 1, COL_FRAGMENTS, Ops, H0Read>;- using QkdFragment = OperandFragment<float, 1, 1, Ops>;- using UFragment = OperandFragment<float, 1, COL_FRAGMENTS, Ops>;- threadgroup float* u_tile = reinterpret_cast<threadgroup float*>(u_tile_scratch);-- const uint batch_idx = batch_value_head_idx / value_heads;- const uint value_head_idx = batch_value_head_idx - batch_idx * value_heads;- const uint qk_head_idx = value_head_idx / (value_heads / qk_heads);- const uint row_base = (row_tile_group_idx * SIMDGROUPS_PER_TG + thread_context.simdgroup_index) * ROWS;- const uint value_base = v_tile_idx * MATMUL_COLS;-- const short valid_rows = short(min(uint(ROWS), tree_size - min(row_base, tree_size)));- const short valid_cols = short(min(MATMUL_COLS, head_v_dim - min(value_base, head_v_dim)));-- AccFragment acc;- acc.clear();-- if (use_h0) {- const uint q_base = (batch_idx * tree_size * qk_heads + row_base * qk_heads + qk_head_idx) * head_k_dim;- const uint h0_base = ((batch_idx * value_heads + value_head_idx) * head_v_dim + value_base) * head_k_dim;- for (uint k0 = 0; k0 < head_k_dim; k0 += ROWS) {- const short valid_k = short(min(uint(ROWS), head_k_dim - k0));- QFragment q_frag;- H0Fragment h0_frag;- q_frag.load_from(- thread_context.simd_lane_id,- fragment_source(q + q_base + k0, int(qk_heads * head_k_dim)).bounded(valid_rows, valid_k)- );- if constexpr (transposed_h0) {- h0_frag.load_from(- thread_context.simd_lane_id,- fragment_source(h0 + h0_base + k0, int(head_k_dim)).bounded(valid_cols, valid_k)- );- } else {- h0_frag.load_from(- thread_context.simd_lane_id,- fragment_source(h0 + h0_base + k0, 1, int(head_k_dim)).bounded(valid_k, valid_cols)- );- }- fragment_mma(acc, q_frag, h0_frag);- }-- acc.map_coords(thread_context.simd_lane_id, [&](short row, short, float value) {- const uint global_row = row_base + uint(row);- const float row_scale =- global_row < tree_size- ? exp(prefix[(batch_idx * tree_size + global_row) * value_heads + value_head_idx]) * scale- : 0.0f;- return value * row_scale;- });- }-- const uint qkd_base = (batch_idx * value_heads + value_head_idx) * tree_size * tree_size + row_base * tree_size;- const uint u_base = (batch_idx * value_heads + value_head_idx) * tree_size * head_v_dim + value_base;- for (uint j0 = 0; j0 < tree_size; j0 += ROWS) {- const short valid_j = short(min(uint(ROWS), tree_size - j0));- QkdFragment qkd_frag;- UFragment u_frag;- qkd_frag.load_from(- thread_context.simd_lane_id,- fragment_source(qkd + qkd_base + j0, int(tree_size)).bounded(valid_rows, valid_j)- );- if constexpr (transposed_h0) {- for (uint index = tid; index < uint(ROWS) * MATMUL_COLS; index += SIMDGROUPS_PER_TG * METAL_SIMD_SIZE) {- const uint local_j = index / MATMUL_COLS;- const uint local_col = index - local_j * MATMUL_COLS;- const bool in_bounds = local_j < uint(valid_j) && local_col < uint(valid_cols);- u_tile[index] = in_bounds ? u[u_base + (j0 + local_j) * head_v_dim + local_col] : 0.0f;- }- threadgroup_barrier(mem_flags::mem_threadgroup);- u_frag.load_from(- thread_context.simd_lane_id,- fragment_source(u_tile, int(MATMUL_COLS)).bounded(valid_j, valid_cols)- );- fragment_mma(acc, qkd_frag, u_frag);- threadgroup_barrier(mem_flags::mem_threadgroup);- } else {- u_frag.load_from(- thread_context.simd_lane_id,- fragment_source(u + u_base + j0 * head_v_dim, int(head_v_dim)).bounded(valid_j, valid_cols)- );- fragment_mma(acc, qkd_frag, u_frag);- }- }-- device OutputT* out_tile =- o + ((batch_idx * tree_size + row_base) * value_heads + value_head_idx) * head_v_dim + value_base;- acc.store_safe(thread_context.simd_lane_id, out_tile, int(value_heads * head_v_dim), short2(valid_cols, valid_rows));-}diff --git a/crates/uzu-engine/src/backends/metal/kernel/gdn/tree_verify/prefix.metal b/crates/uzu-engine/src/backends/metal/kernel/gdn/tree_verify/prefix.metaldeleted file mode 100644index 6fe092adb..000000000--- a/crates/uzu-engine/src/backends/metal/kernel/gdn/tree_verify/prefix.metal+++ /dev/null@@ -1,50 +0,0 @@-#include <metal_stdlib>-#include "../../common/defines.h"-#include "../../common/dsl.h"-#include "../../common/thread_context.h"-#include "../../generated/trie.h"--#define ROWS_PER_THREADGROUP 4u--using namespace metal;-using namespace uzu::trie;--// trie: [B, T] TrieNode — col is an ancestor-or-self of row iff-// trie[col].trie_start <= row <= trie[col].trie_end.-// log_decay: [B, T, HV]-// prefix: [B, T, HV] fp32-PUBLIC KERNEL(BuildTreePrefix)(- const device TrieNode* trie,- const device float* log_decay,- device float* prefix,- constant const uint& batch_size,- constant const uint& tree_size,- constant const uint& value_heads,- const ThreadContext thread_context,- const uint batch_idx GROUPS(batch_size),- const uint row_idx GROUPS(tree_size),- const uint head_group_idx GROUPS(value_heads.div_ceil(ROWS_PER_THREADGROUP)),- const uint tid THREADS(ROWS_PER_THREADGROUP* METAL_SIMD_SIZE)-) {- const uint simd_group = thread_context.simdgroup_index;- const uint lane_id = thread_context.simd_lane_id;- const uint head_idx = head_group_idx * ROWS_PER_THREADGROUP + simd_group;- if (head_idx >= value_heads) {- return;- }-- const uint batch_offset = batch_idx * tree_size * value_heads;- const uint trie_offset = batch_idx * tree_size;- float partial = 0.0f;- for (uint col_idx = lane_id; col_idx < tree_size; col_idx += METAL_SIMD_SIZE) {- const TrieNode node = trie[trie_offset + col_idx];- if (row_idx >= node.trie_start && row_idx <= node.trie_end) {- partial += log_decay[batch_offset + col_idx * value_heads + head_idx];- }- }-- const float sum = simd_sum(partial);- if (lane_id == 0) {- prefix[batch_offset + row_idx * value_heads + head_idx] = sum;- }-}diff --git a/crates/uzu-engine/src/backends/metal/kernel/gdn/tree_verify/tree_gram.metal b/crates/uzu-engine/src/backends/metal/kernel/gdn/tree_verify/tree_gram.metaldeleted file mode 100644index a8741f1fc..000000000--- a/crates/uzu-engine/src/backends/metal/kernel/gdn/tree_verify/tree_gram.metal+++ /dev/null@@ -1,291 +0,0 @@-#include <metal_stdlib>-#include "../../common/defines.h"-#include "../../common/dsl.h"-#include "../../common/thread_context.h"-#include "../../generated/trie.h"-#include "../../matmul/common/fragment.h"-#include "../../matmul/common/mxu_fragment/ops.h"-#include "../../matmul/common/simdgroup_fragment_ops.h"-#include "../common/gram.h"-#include "../common/tri_inv.h"--using namespace metal;-using namespace uzu::matmul;-using namespace uzu::trie;--#define ROW_TILE 16u-#define COL_TILE 32u-#define NUM_SIMDGROUPS 4u-#define INVALID_ROW 0xffffffffu--// kh0 = k @ h0[batch]^T for one row-tile, walking head_v_dim in COL_TILE-// chunks from dv_start by dv_stride.-template <typename T, typename Ops, typename AccFragment, typename LeftFragment, typename RightFragment>-METAL_FUNC void tree_kh0(- const device T* k_rows,- const device float* h0,- device float* kh0,- const uint batch_idx,- const uint value_head_idx,- const uint row_base,- const uint tile_rows,- const bool full_rows,- const uint tree_size,- const uint value_heads,- const uint head_k_dim,- const uint head_v_dim,- const uint qk_stride,- const uint dv_start,- const uint dv_stride,- const ushort lane-) {- const device float* h0_head = h0 + (batch_idx * value_heads + value_head_idx) * head_v_dim * head_k_dim;- device float* kh0_rows =- kh0 + (batch_idx * tree_size + row_base) * value_heads * head_v_dim + value_head_idx * head_v_dim;- const uint kh0_row_stride = value_heads * head_v_dim;-- for (uint dv_base = dv_start; dv_base < head_v_dim; dv_base += dv_stride) {- const uint tile_dvs = min(uint(COL_TILE), head_v_dim - dv_base);- AccFragment kh0_acc;- kh0_acc.clear();-- const device float* h0_tile = h0_head + dv_base * head_k_dim;- const bool full_dvs = tile_dvs == COL_TILE;-- for (uint kb = 0; kb < head_k_dim; kb += Ops::FRAGMENT_ROWS) {- LeftFragment k_left;- RightFragment h0_right;- k_left.load_maybe_bounded(lane, k_rows + kb, qk_stride, full_rows, tile_rows, Ops::FRAGMENT_ROWS);- h0_right.load_maybe_bounded(lane, h0_tile + kb, head_k_dim, full_dvs, tile_dvs, Ops::FRAGMENT_ROWS);- fragment_mma(kh0_acc, k_left, h0_right);- }-- kh0_acc.store_maybe_bounded(- lane,- kh0_rows + dv_base,- kh0_row_stride,- full_rows && full_dvs,- short2(tile_dvs, tile_rows)- );- }-}--// Builds the tree-verify gram products for one (batch, value-head, row-tile,-// col-tile-group) strip; each of the threadgroup's simdgroups owns one 16x32-// column tile (above-diagonal tiles are just zero-filled in qkd) and the row's-// simdgroups share the kh0 dv chunks round-robin:-// a_packed: A[row, col] = beta[row] * exp(prefix[row] - prefix[col]) * dot(k[row], k[col])-// for proper ancestors (trie interval test), packed block-pair tiles-// [B*HV, NB, ceil(NB/2), 16, 32] f32; only lower-triangle tiles written-// qkd: scale * exp(prefix[row] - prefix[col]) * dot(q[row], k[col]) for-// ancestor-or-self, dense [B*HV, T, T] f32-// a_inv: (I + A)^-1 per diagonal block, compact [B*HV, NB, 16, 16]-// kh0: k @ h0[batch]^T, [B, T, HV, head_v_dim] f32-template <typename T, bool USE_MXU>-VARIANTS(T, float, bfloat)-VARIANTS(USE_MXU, false, true)-PUBLIC KERNEL(BuildTreeGram)(- const device T* q,- const device T* k,- const device TrieNode* trie,- const device float* prefix,- const device float* beta,- const device float* h0 OPTIONAL(use_h0),- device float* a_packed,- device float* qkd,- device float* a_inv,- device float* kh0 OPTIONAL(use_h0),- constant const float& scale,- constant const uint& batch_size,- constant const uint& tree_size,- constant const uint& k_heads,- constant const uint& value_heads,- constant const uint& head_k_dim,- constant const uint& head_v_dim,- threadgroup float diag_a_tile[ROW_TILE * ROW_TILE],- threadgroup float row_prefix[ROW_TILE],- threadgroup float row_beta[ROW_TILE],- threadgroup uint row_token[ROW_TILE],- threadgroup float col_prefix[NUM_SIMDGROUPS * COL_TILE],- threadgroup uint col_trie_start[NUM_SIMDGROUPS * COL_TILE],- threadgroup uint col_trie_end[NUM_SIMDGROUPS * COL_TILE],- const bool use_h0 SPECIALIZE,- const ThreadContext thread_context,- const uint batch_idx GROUPS(batch_size),- const uint value_head_idx GROUPS(value_heads),- const uint tile_idx GROUPS(- tree_size.div_ceil(ROW_TILE) * tree_size.div_ceil(COL_TILE).div_ceil(NUM_SIMDGROUPS)- ),- const uint thread_idx THREADS(NUM_SIMDGROUPS * METAL_SIMD_SIZE)-) {- using Ops = metal::conditional_t<USE_MXU, MxuFragmentOps<>, SimdgroupFragmentOps>;- constexpr ushort ROW_FRAGMENTS = ROW_TILE / Ops::FRAGMENT_ROWS;- constexpr ushort COL_FRAGMENTS = COL_TILE / Ops::FRAGMENT_COLS;- using AccFragment = Fragment<float, ROW_FRAGMENTS, COL_FRAGMENTS, Ops>;- using LeftFragment = OperandFragment<float, ROW_FRAGMENTS, 1, Ops>;- using RightFragment = OperandFragment<float, 1, COL_FRAGMENTS, Ops, ReadTranspose>;- static_assert(COL_TILE == METAL_SIMD_SIZE, "COL_TILE must match the SIMD width");-- const uint key_head_idx = value_head_idx / (value_heads / k_heads);- const uint qk_stride = k_heads * head_k_dim;- const uint qk_base = (batch_idx * tree_size * k_heads + key_head_idx) * head_k_dim;- const uint prefix_base = batch_idx * tree_size * value_heads + value_head_idx;- const uint trie_base = batch_idx * tree_size;- const uint mat_base = (batch_idx * value_heads + value_head_idx) * tree_size * tree_size;-- const uint col_tiles = div_ceil(tree_size, COL_TILE);- const uint col_tile_groups = div_ceil(col_tiles, NUM_SIMDGROUPS);- const uint row_tile_idx = tile_idx / col_tile_groups;- const uint col_tile_group_idx = tile_idx - row_tile_idx * col_tile_groups;- const uint row_base = row_tile_idx * ROW_TILE;- const uint tile_rows = min(ROW_TILE, tree_size - row_base);- const bool full_rows = row_base + ROW_TILE <= tree_size;- const uint in_band_col_tiles = min(row_base / COL_TILE + 1, col_tiles);- const uint diag_col_tile = row_base / COL_TILE;- const uint diag_col_offset = row_base - diag_col_tile * COL_TILE;- const ushort lane = thread_context.simd_lane_id;- const uint simdgroup_idx = thread_context.simdgroup_index;- const uint col_slice = simdgroup_idx * COL_TILE;- threadgroup float* sg_col_prefix = col_prefix + col_slice;- threadgroup uint* sg_col_trie_start = col_trie_start + col_slice;- threadgroup uint* sg_col_trie_end = col_trie_end + col_slice;-- if (thread_idx < ROW_TILE) {- const uint token = row_base + thread_idx;- if (token < tree_size) {- row_token[thread_idx] = token;- row_prefix[thread_idx] = prefix[prefix_base + token * value_heads];- row_beta[thread_idx] = beta[prefix_base + token * value_heads];- } else {- row_token[thread_idx] = INVALID_ROW;- row_prefix[thread_idx] = 0.0f;- row_beta[thread_idx] = 0.0f;- }- }- threadgroup_barrier(mem_flags::mem_threadgroup);-- const uint num_blocks = div_ceil(tree_size, ROW_TILE);- const uint num_col_pairs = div_ceil(num_blocks, 2u);-- const device T* k_rows = k + qk_base + row_base * qk_stride;- const device T* q_rows = q + qk_base + row_base * qk_stride;-- const uint col_tile_idx = col_tile_group_idx * NUM_SIMDGROUPS + simdgroup_idx;- if (col_tile_idx < in_band_col_tiles) {- const uint col_base = col_tile_idx * COL_TILE;- const uint tile_cols = min(COL_TILE, tree_size - col_base);- const uint tile_base = mat_base + row_base * tree_size + col_base;-- const uint col_token = col_base + lane;- if (col_token < tree_size) {- const TrieNode node = trie[trie_base + col_token];- sg_col_trie_start[lane] = node.trie_start;- sg_col_trie_end[lane] = node.trie_end;- sg_col_prefix[lane] = prefix[prefix_base + col_token * value_heads];- } else {- sg_col_trie_start[lane] = 1;- sg_col_trie_end[lane] = 0;- sg_col_prefix[lane] = 0.0f;- }-- const device T* k_cols = k + qk_base + col_base * qk_stride;- const bool full_cols = col_base + COL_TILE <= tree_size;-- AccFragment kk_acc;- AccFragment qk_acc;- kk_acc.clear();- qk_acc.clear();- for (uint kb = 0; kb < head_k_dim; kb += Ops::FRAGMENT_ROWS) {- accumulate_dual_gram_tile<AccFragment, LeftFragment, RightFragment>(- kk_acc,- qk_acc,- k_rows + kb,- q_rows + kb,- k_cols + kb,- int(qk_stride),- ushort(tile_rows),- ushort(tile_cols),- Ops::FRAGMENT_ROWS,- full_rows,- full_cols,- lane- );- }-- const bool has_diag = col_tile_idx == diag_col_tile;- simdgroup_barrier(mem_flags::mem_threadgroup);-- AccFragment::zip_for_each_coord(- lane,- [&](ushort local_row, ushort local_col, thread float& kk, thread float& qk_dot) {- const uint row_idx = row_token[local_row];- const uint col_idx = col_base + local_col;- const bool in_subtree = row_idx >= sg_col_trie_start[local_col] && row_idx <= sg_col_trie_end[local_col];- const float decay = exp(row_prefix[local_row] - sg_col_prefix[local_col]);- const float a_value = in_subtree && row_idx != col_idx ? row_beta[local_row] * decay * kk : 0.0f;- const float qkd_value = in_subtree ? scale * decay * qk_dot : 0.0f;- if (has_diag && local_row < tile_rows && local_col >= diag_col_offset) {- const uint diag_col = local_col - diag_col_offset;- if (diag_col < tile_rows) {- diag_a_tile[local_row * ROW_TILE + diag_col] = a_value;- }- }- kk = a_value;- qk_dot = qkd_value;- },- kk_acc,- qk_acc- );-- device float* a_tile =- a_packed +- (((batch_idx * value_heads + value_head_idx) * num_blocks + row_tile_idx) * num_col_pairs + col_tile_idx) *- (ROW_TILE * COL_TILE);- device float* qkd_tile = qkd + tile_base;- kk_acc.store_maybe_bounded(lane, a_tile, COL_TILE, full_rows, short2(COL_TILE, tile_rows));- qk_acc.store_maybe_bounded(lane, qkd_tile, tree_size, full_rows && full_cols, short2(tile_cols, tile_rows));-- if (has_diag) {- // diag_a_tile is written and read only by this simdgroup.- simdgroup_barrier(mem_flags::mem_threadgroup);- device float* a_inv_block =- a_inv + ((batch_idx * value_heads + value_head_idx) * num_blocks + row_tile_idx) * (ROW_TILE * ROW_TILE);- invert_lower_triangular_block<ROW_TILE>(a_inv_block, diag_a_tile, tile_rows, lane);- }- } else if (col_tile_idx < col_tiles) {- // Above-diagonal zero fill; COL_TILE == METAL_SIMD_SIZE, so each- // lane owns one column and walks rows by pointer increment.- const uint col_base = col_tile_idx * COL_TILE;- const uint tile_cols = min(COL_TILE, tree_size - col_base);- if (lane < tile_cols) {- device float* qkd_ptr = qkd + mat_base + row_base * tree_size + col_base + lane;- for (uint row = 0; row < tile_rows; ++row, qkd_ptr += tree_size) {- *qkd_ptr = 0.0f;- }- }- }-- // kh0 dv chunks are distributed round-robin over this row tile's simdgroups- // (across its col-tile-group threadgroups); chunking also bounds registers.- if (use_h0) {- const uint row_simdgroups = col_tile_groups * NUM_SIMDGROUPS;- tree_kh0<T, Ops, AccFragment, LeftFragment, RightFragment>(- k_rows,- h0,- kh0,- batch_idx,- value_head_idx,- row_base,- tile_rows,- full_rows,- tree_size,- value_heads,- head_k_dim,- head_v_dim,- qk_stride,- col_tile_idx * COL_TILE,- row_simdgroups * COL_TILE,- lane- );- }-}diff --git a/crates/uzu-engine/src/backends/metal/kernel/gdn/tree_verify/tree_update_solve.metal b/crates/uzu-engine/src/backends/metal/kernel/gdn/tree_verify/tree_update_solve.metaldeleted file mode 100644index cd84bc13a..000000000--- a/crates/uzu-engine/src/backends/metal/kernel/gdn/tree_verify/tree_update_solve.metal+++ /dev/null@@ -1,174 +0,0 @@-#include <metal_stdlib>-#include "../../common/defines.h"-#include "../../common/dsl.h"-#include "../../matmul/common/fragment.h"-#include "../../matmul/common/simdgroup_fragment_ops.h"--using namespace metal;-using namespace uzu::matmul;--// Block forward substitution for (I + A) U = beta * (v - exp(prefix) * kh0).-// Per 16-token block i, serially: acc_i = beta * (v_i - exp(prefix_i) * kh0_i);-// acc_i -= A[i, j] @ U[j] for all j < i; U[i] = Ainv[i] @ acc_i.-//-// Shapes (NB = ceil(T/16)):-// kh0 [B, T, HV, head_v_dim] f32, from BuildTreeGram (absent when !use_h0)-// v [B, T, HV, head_v_dim] in T-// prefix, beta [B, T, HV] f32-// A [B*HV, NB, ceil(NB/2), 16, 32] f32 packed pair tiles from BuildTreeGram-// (block (i,j) at pair j/2, column (j%2)*16)-// Ainv [B*HV, NB, 16, 16] f32 diagonal-block inverses from BuildTreeGram-// U [B*HV, T, head_v_dim] f32 (output)-//-// Grid: one simdgroup per (batch, value head, BV value-dim tile); the history is-// consumed one packed pair tile ([16,32] A x [32,BV] U) per fragment_mma plus a-// single-block tail when i is odd.-// A and Ainv are kept in f32: the (I + A)^-1 forward-substitution cascade is-// precision-sensitive.-template <typename T, uint BV>-VARIANTS(T, float, bfloat)-VARIANTS(BV, 16, 32)-PUBLIC KERNEL(TreeUpdateSolve)(- const device float* kh0 OPTIONAL(use_h0),- const device T* v,- const device float* prefix,- const device float* beta,- const device float* a_packed,- const device float* a_inv,- device float* u,- constant const uint& batch_size,- constant const uint& tree_size,- constant const uint& value_heads,- constant const uint& head_v_dim,- const bool use_h0 SPECIALIZE,- const uint batch_idx GROUPS(batch_size),- const uint value_head_idx GROUPS(value_heads),- const uint value_tile_idx GROUPS(head_v_dim.div_ceil(BV)),- const uint lane_idx THREADS(METAL_SIMD_SIZE)-) {- constexpr uint BT = 16;- // MXU was within 5% of SIMD-group at T=32..512.- using FragmentOps = SimdgroupFragmentOps;- constexpr ushort TOKEN_FRAGMENTS = BT / FragmentOps::FRAGMENT_ROWS;- constexpr ushort VALUE_FRAGMENTS = BV / FragmentOps::FRAGMENT_COLS;- using TileFragment = Fragment<float, TOKEN_FRAGMENTS, VALUE_FRAGMENTS, FragmentOps>;- using BlockMatrixFragment = OperandFragment<float, TOKEN_FRAGMENTS, TOKEN_FRAGMENTS, FragmentOps>;- using ValueTileFragment = OperandFragment<float, TOKEN_FRAGMENTS, VALUE_FRAGMENTS, FragmentOps>;- using PairMatrixFragment = OperandFragment<float, TOKEN_FRAGMENTS, 2 * TOKEN_FRAGMENTS, FragmentOps>;- using PairValueFragment = OperandFragment<float, 2 * TOKEN_FRAGMENTS, VALUE_FRAGMENTS, FragmentOps>;-- static_assert(BV % FragmentOps::FRAGMENT_COLS == 0, "BV must be a multiple of the fragment column size");-- const uint batch_value_head_idx = batch_idx * value_heads + value_head_idx;-- const uint value_dim_base = value_tile_idx * BV;- const uint tile_value_cols = min(BV, head_v_dim - value_dim_base);- const uint num_blocks = div_ceil(tree_size, BT);-- const uint value_token_stride = value_heads * head_v_dim;- const uint scalar_token_stride = value_heads;-- const device float* kh0_head_tile =- kh0 + (batch_idx * tree_size * value_heads + value_head_idx) * head_v_dim + value_dim_base;- const device T* value_head_tile =- v + (batch_idx * tree_size * value_heads + value_head_idx) * head_v_dim + value_dim_base;- const device float* prefix_head = prefix + batch_idx * tree_size * value_heads + value_head_idx;- const device float* beta_head = beta + batch_idx * tree_size * value_heads + value_head_idx;- const uint num_col_pairs = div_ceil(num_blocks, 2u);- const device float* a_blocks = a_packed + batch_value_head_idx * num_blocks * num_col_pairs * (BT * 2 * BT);- const device float* a_inv_blocks = a_inv + batch_value_head_idx * num_blocks * (BT * BT);- device float* u_head_tile = u + batch_value_head_idx * tree_size * head_v_dim + value_dim_base;-- for (uint block_idx = 0; block_idx < num_blocks; ++block_idx) {- const uint row_base = block_idx * BT;- const uint tile_rows = min(BT, tree_size - row_base);-- TileFragment acc;-- if (use_h0) {- const device float* kh0_block = kh0_head_tile + row_base * value_token_stride;- acc.load_maybe_bounded(- lane_idx,- kh0_block,- value_token_stride,- tile_rows == BT && tile_value_cols == BV,- tile_rows,- tile_value_cols- );- } else {- acc.clear();- }-- // prefix/beta/exp are per-row: one lane computes them, consumers read via simd_shuffle.- float lane_beta = 0.0f;- float lane_decay = 0.0f;- if (lane_idx < tile_rows) {- const uint token_idx = row_base + lane_idx;- lane_beta = beta_head[token_idx * scalar_token_stride];- lane_decay = exp(prefix_head[token_idx * scalar_token_stride]);- }-- TileFragment v_frag;- const device T* v_block = value_head_tile + row_base * value_token_stride;- v_frag.load_maybe_bounded(- lane_idx,- v_block,- value_token_stride,- tile_rows == BT && tile_value_cols == BV,- tile_rows,- tile_value_cols- );-- TileFragment::zip_for_each_coord(- lane_idx,- [&](ushort local_row_idx, ushort, thread float& acc_value, thread float& v_value) {- const float beta_row = simd_shuffle(lane_beta, local_row_idx);- const float decay_row = simd_shuffle(lane_decay, local_row_idx);- acc_value = beta_row * (v_value - decay_row * acc_value);- },- acc,- v_frag- );-- const uint num_full_pairs = block_idx / 2;- for (uint pair_idx = 0; pair_idx < num_full_pairs; ++pair_idx) {- PairMatrixFragment a_frag;- PairValueFragment u_prev_frag;-- const device float* a_pair = a_blocks + (block_idx * num_col_pairs + pair_idx) * (BT * 2 * BT);- const device float* u_prev_block = u_head_tile + pair_idx * 2 * BT * head_v_dim;- const bool full = tile_rows == BT && tile_value_cols == BV;- a_frag.load_maybe_bounded(lane_idx, a_pair, 2 * BT, full, tile_rows, 2 * BT);- u_prev_frag.load_maybe_bounded(lane_idx, u_prev_block, head_v_dim, full, 2 * BT, tile_value_cols);- a_frag.map([](float value) { return -value; });- fragment_mma(acc, a_frag, u_prev_frag);- }-- for (uint prev_block_idx = num_full_pairs * 2; prev_block_idx < block_idx; ++prev_block_idx) {- const uint prev_row_base = prev_block_idx * BT;- BlockMatrixFragment a_frag;- ValueTileFragment u_prev_frag;-- const device float* a_block =- a_blocks + (block_idx * num_col_pairs + prev_block_idx / 2) * (BT * 2 * BT) + (prev_block_idx % 2) * BT;- a_frag.load_from(lane_idx, fragment_source(a_block, 2 * BT).bounded(tile_rows, BT));- const device float* u_prev_block = u_head_tile + prev_row_base * head_v_dim;- u_prev_frag.load_from(lane_idx, fragment_source(u_prev_block, head_v_dim).bounded(BT, tile_value_cols));- a_frag.map([](float value) { return -value; });- fragment_mma(acc, a_frag, u_prev_frag);- }-- BlockMatrixFragment inv_frag;- TileFragment solved;- solved.clear();-- const device float* inv_block = a_inv_blocks + block_idx * (BT * BT);- inv_frag.load_from(lane_idx, fragment_source(inv_block, BT).bounded(tile_rows, tile_rows));- fragment_mma(solved, inv_frag, acc);- solved.store_safe(lane_idx, u_head_tile + row_base * head_v_dim, head_v_dim, short2(tile_value_cols, tile_rows));-- // The threadgroup is a single simdgroup, so the block-serial U dependency- // only needs a simdgroup-scoped device-memory fence.- simdgroup_barrier(mem_flags::mem_device);- }-}diff --git a/crates/uzu-engine/src/backends/metal/kernel/gdn/tree_verify/tree_verify.metal b/crates/uzu-engine/src/backends/metal/kernel/gdn/tree_verify/tree_verify.metalnew file mode 100644index 000000000..edb0f38dc--- /dev/null+++ b/crates/uzu-engine/src/backends/metal/kernel/gdn/tree_verify/tree_verify.metal@@ -0,0 +1,291 @@+#include <metal_stdlib>+#include "../../activation/activations.h"+#include "../../common/defines.h"+#include "../../common/dsl.h"+#include "../../common/integral_constant.h"+#include "../../common/thread_context.h"+#include "../../generated/trie.h"+#include "../../matmul/common/fragment.h"+#include "../../matmul/common/simdgroup_fragment_ops.h"+#include "../common/tri_inv.h"++using namespace metal;+using namespace uzu::matmul;+using namespace uzu::trie;++#define HEAD_V_DIM 128u+#define HEAD_K_DIM 128u+#define BLOCK_SIZE 16u+#define OWNERS 8u+#define OWNER_COLS (HEAD_V_DIM / OWNERS)+#define OUT_STRIDE (HEAD_V_DIM + 1u)++using GramAccFragment = Fragment<float, 2, 2, SimdgroupFragmentOps>;++// Verifies a whole tree (NB = 1: up to 16 nodes, NB = 2: up to 32) per (batch, value head). Simdgroups 0 .. NB - 1+// build the grams and invert I + A; the other eight own 16 value columns each.+template <typename T, uint NB>+VARIANTS(T, float, bfloat)+VARIANTS(NB, 1, 2)+PUBLIC KERNEL(TreeVerify)(+ const device T* q,+ const device T* k,+ const device T* v,+ const device TrieNode* trie,+ const device float* log_decay,+ const device float* beta,+ const device float* h0,+ const device T* in_proj,+ const device float* norm_weight,+ device T* o,+ constant const float& scale,+ constant const float& norm_epsilon,+ constant const uint& batch_size,+ constant const uint& tree_size,+ constant const uint& num_k_heads,+ constant const uint& num_v_heads,+ constant const uint& head_k_dim,+ constant const uint& head_v_dim,+ constant const uint& conv_dim,+ constant const uint& total_proj_dim,+ threadgroup T out_tile[NB * BLOCK_SIZE * OUT_STRIDE],+ threadgroup float a_tiles[NB * NB * BLOCK_SIZE * BLOCK_SIZE],+ threadgroup float qkd_tile[NB * NB * BLOCK_SIZE * BLOCK_SIZE],+ threadgroup float diag_inverses[NB * BLOCK_SIZE * BLOCK_SIZE],+ threadgroup float row_prefix[NB * BLOCK_SIZE],+ threadgroup float row_beta[NB * BLOCK_SIZE],+ threadgroup float row_decay[NB * BLOCK_SIZE],+ threadgroup uint trie_start[NB * BLOCK_SIZE],+ threadgroup uint trie_end[NB * BLOCK_SIZE],+ const ThreadContext thread_context,+ const uint batch_idx GROUPS(batch_size),+ const uint hv_idx GROUPS(num_v_heads),+ const uint thread_idx THREADS((OWNERS + NB) * METAL_SIMD_SIZE)+) {+ using Ops = SimdgroupFragmentOps;+ constexpr uint TOKENS = NB * BLOCK_SIZE;+ constexpr uint SIMDGROUPS = OWNERS + NB;+ constexpr ushort TOKEN_FRAGMENTS = TOKENS / Ops::FRAGMENT_COLS;+ using OwnerFragment = Fragment<float, OWNER_COLS / Ops::FRAGMENT_ROWS, TOKEN_FRAGMENTS, Ops>;+ using BlockRightFragment = OperandFragment<float, TOKEN_FRAGMENTS, TOKEN_FRAGMENTS, Ops, ReadTranspose>;++ const ushort lane = thread_context.simd_lane_id;+ const uint simdgroup_idx = thread_context.simdgroup_index;+ const bool is_owner = simdgroup_idx >= NB;+ // DeltaNet::new rejects head_k_dim != 128; NB = 1 takes the constant so the K guards fold away.+ const uint k_dim = NB == 1 ? HEAD_K_DIM : head_k_dim;+ const int qk_stride = num_k_heads * k_dim;+ const uint qk_base = (batch_idx * tree_size * num_k_heads + hv_idx / (num_v_heads / num_k_heads)) * k_dim;+ const uint scalar_base = batch_idx * tree_size * num_v_heads + hv_idx;+ const uint value_token_stride = num_v_heads * head_v_dim;+ const uint value_base = batch_idx * tree_size * value_token_stride + hv_idx * head_v_dim;+ const uint dv_base = (simdgroup_idx - NB) * OWNER_COLS;+ const short rows_left = short(tree_size);+ const auto load_rows = [&](auto full, thread auto& frag, const device T* base, const short rows) UZU_ALWAYS_INLINE {+ auto source = fragment_source(base, qk_stride);+ if constexpr (decltype(full)::value) {+ frag.load_from(lane, source);+ } else {+ frag.load_from(lane, source.bounded(rows, Ops::FRAGMENT_ROWS));+ }+ };++ OwnerFragment v_frag;+ if (is_owner) {+ v_frag.load_from(+ lane,+ fragment_source(v + value_base + dv_base, 1, int(value_token_stride)).bounded(OWNER_COLS, rows_left)+ );+ }++ {+ const bool lane_valid = lane < tree_size;+ const TrieNode node = lane_valid ? trie[batch_idx * tree_size + lane] : TrieNode{1, 0, 0};+ const float lane_log_decay = lane_valid ? log_decay[scalar_base + lane * num_v_heads] : 0.0f;+ if (simdgroup_idx == 0 && lane < TOKENS) {+ trie_start[lane] = node.trie_start;+ trie_end[lane] = node.trie_end;+ }+ for (uint row = simdgroup_idx; row < TOKENS; row += SIMDGROUPS) {+ const float sum = simd_sum((row >= node.trie_start && row <= node.trie_end) ? lane_log_decay : 0.0f);+ if (lane == 0) {+ const bool row_valid = row < tree_size;+ row_prefix[row] = row_valid ? sum : 0.0f;+ row_decay[row] = row_valid ? exp(sum) : 1.0f;+ row_beta[row] = row_valid ? beta[scalar_base + row * num_v_heads] : 0.0f;+ }+ }+ }+ threadgroup_barrier(mem_flags::mem_threadgroup);++ OwnerFragment kh0;+ OwnerFragment qh0;+ kh0.clear();+ qh0.clear();+ if (!is_owner) {+ // Owners read all of qkd, so its upper blocks are zeroed.+ auto build_gram_blocks = [&](auto builder) UZU_ALWAYS_INLINE {+ constexpr uint B = decltype(builder)::value;+ GramAccFragment diag_a;+ GramAccFragment diag_q;+ GramAccFragment extra;+ const device T* k_rows = k + qk_base + B * BLOCK_SIZE * qk_stride;+ const device T* q_rows = q + qk_base + B * BLOCK_SIZE * qk_stride;+ const short rows = rows_left - short(B * BLOCK_SIZE);+ uzu::dispatch_bool(rows_left >= short(TOKENS), [&](auto full) UZU_ALWAYS_INLINE {+ using GramLeftFragment = OperandFragment<float, 2, 1, Ops>;+ using GramRightFragment = OperandFragment<float, 1, 2, Ops, ReadTranspose>;+ diag_a.clear();+ diag_q.clear();+ extra.clear();+ _Pragma("clang loop unroll_count(2)") for (ushort step = 0; step < HEAD_K_DIM / Ops::FRAGMENT_ROWS; ++step) {+ const uint kb = step * Ops::FRAGMENT_ROWS;+ if (kb < k_dim) {+ GramLeftFragment k_left;+ GramLeftFragment q_left;+ GramRightFragment k_right;+ load_rows(full, k_left, k_rows + kb, rows);+ load_rows(full, q_left, q_rows + kb, rows);+ load_rows(full, k_right, k_rows + kb, rows);+ fragment_mma(diag_a, k_left, k_right);+ fragment_mma(diag_q, q_left, k_right);+ if constexpr (NB == 2 && B == 0) {+ GramLeftFragment k_below;+ load_rows(full, k_below, k_rows + BLOCK_SIZE * qk_stride + kb, rows - BLOCK_SIZE);+ fragment_mma(extra, k_below, k_right);+ } else if constexpr (NB == 2) {+ GramRightFragment k_first;+ load_rows(full, k_first, k + qk_base + kb, rows_left);+ fragment_mma(extra, q_left, k_first);+ }+ }+ }+ });+ auto store = [&](thread GramAccFragment& block, const bool is_qkd, const uint i, const uint j) UZU_ALWAYS_INLINE {+ block.map_coords(lane, [&](short r, short c, float value) UZU_ALWAYS_INLINE {+ const uint row = i * BLOCK_SIZE + r;+ const uint col = j * BLOCK_SIZE + c;+ const bool in_subtree = row >= trie_start[col] && row <= trie_end[col];+ const float decay = exp(row_prefix[row] - row_prefix[col]);+ if (is_qkd) {+ return in_subtree ? scale * decay * value : 0.0f;+ }+ return in_subtree && row != col ? row_beta[row] * decay * value : 0.0f;+ });+ block.store(lane, (is_qkd ? qkd_tile : a_tiles) + i * BLOCK_SIZE * TOKENS + j * BLOCK_SIZE, int(TOKENS));+ };+ store(diag_a, false, B, B);+ store(diag_q, true, B, B);+ if constexpr (NB == 2) {+ store(extra, B == 1, 1, 0);+ if constexpr (B == 0) {+ GramAccFragment zero;+ zero.clear();+ zero.store(lane, qkd_tile + BLOCK_SIZE, int(TOKENS));+ }+ }+ };+ if (simdgroup_idx == 0) {+ build_gram_blocks(metal::integral_constant<uint, 0>{});+ } else if constexpr (NB == 2) {+ build_gram_blocks(metal::integral_constant<uint, 1>{});+ }+ simdgroup_barrier(mem_flags::mem_threadgroup);+ invert_lower_triangular_block<BLOCK_SIZE, TOKENS>(+ diag_inverses + simdgroup_idx * (BLOCK_SIZE * BLOCK_SIZE),+ a_tiles + simdgroup_idx * BLOCK_SIZE * (TOKENS + 1),+ tree_size - simdgroup_idx * BLOCK_SIZE,+ lane+ );+ } else {+ const device float* h0_rows = h0 + ((batch_idx * num_v_heads + hv_idx) * head_v_dim + dv_base) * k_dim;+ uzu::dispatch_bool(rows_left >= short(TOKENS), [&](auto full) UZU_ALWAYS_INLINE {+ using TokenRightFragment = OperandFragment<float, 1, TOKEN_FRAGMENTS, Ops, ReadTranspose>;+ METAL_PRAGMA_UNROLL+ for (ushort step = 0; step < HEAD_K_DIM / Ops::FRAGMENT_ROWS; ++step) {+ const uint kb = step * Ops::FRAGMENT_ROWS;+ if (kb < k_dim) {+ OperandFragment<float, 2, 1, Ops> h0_frag;+ TokenRightFragment k_right;+ TokenRightFragment q_right;+ h0_frag.load_from(lane, fragment_source(h0_rows + kb, int(k_dim)));+ load_rows(full, k_right, k + qk_base + kb, rows_left);+ load_rows(full, q_right, q + qk_base + kb, rows_left);+ fragment_mma(kh0, h0_frag, k_right);+ fragment_mma(qh0, h0_frag, q_right);+ }+ }+ });+ }+ threadgroup_barrier(mem_flags::mem_threadgroup);++ // Builder b solves column slice b in place over A; only builder 0 reads the off-diagonal block it overwrites.+ const threadgroup float* ainv = diag_inverses;+ if constexpr (NB > 1) {+ ainv = a_tiles;+ if (!is_owner) {+ solve_lower_triangular_slice<BLOCK_SIZE, NB, float>(+ [&](uint b)+ UZU_ALWAYS_INLINE { return static_cast<const threadgroup float*>(a_tiles + b * BLOCK_SIZE * TOKENS); },+ [](const threadgroup float* row, uint m) UZU_ALWAYS_INLINE { return row + m * BLOCK_SIZE; },+ TOKENS,+ diag_inverses,+ [&](uint b, thread GramAccFragment& t_block) UZU_ALWAYS_INLINE {+ t_block.store(lane, a_tiles + b * BLOCK_SIZE * TOKENS + simdgroup_idx * BLOCK_SIZE, int(TOKENS));+ },+ simdgroup_idx,+ lane+ );+ }+ threadgroup_barrier(mem_flags::mem_threadgroup);+ }++ if (is_owner) {+ OwnerFragment::zip_for_each_coord(+ lane,+ [&](ushort, ushort token, thread float& kh0_value, thread float& v_value)+ UZU_ALWAYS_INLINE { kh0_value = row_beta[token] * (v_value - row_decay[token] * kh0_value); },+ kh0,+ v_frag+ );+ BlockRightFragment ainv_right;+ ainv_right.load_from(lane, fragment_source(ainv, int(TOKENS)));+ OwnerFragment u;+ u.clear();+ fragment_mma(u, kh0, ainv_right);++ qh0.map_coords(lane, [&](short, short token, float value) UZU_ALWAYS_INLINE {+ return value * (row_decay[token] * scale);+ });+ BlockRightFragment qkd_right;+ qkd_right.load_from(lane, fragment_source(qkd_tile, int(TOKENS)));+ fragment_mma(qh0, u, qkd_right);+ // Rounded to T before the norm.+ qh0.map_coords(lane, [&](short value_col, short token, float value) UZU_ALWAYS_INLINE {+ out_tile[token * OUT_STRIDE + dv_base + value_col] = static_cast<T>(value);+ return value;+ });+ }+ threadgroup_barrier(mem_flags::mem_threadgroup);++ using T4 = metal::vec<T, 4>;+ const uint dv = lane * 4;+ for (uint token = simdgroup_idx; token < tree_size; token += SIMDGROUPS) {+ float values[4];+ float square_sum = 0.0f;+ METAL_PRAGMA_UNROLL+ for (uint j = 0; j < 4; ++j) {+ values[j] = float(out_tile[token * OUT_STRIDE + dv + j]);+ square_sum += values[j] * values[j];+ }+ const float inv_rms = rsqrt(simd_sum(square_sum) / float(head_v_dim) + norm_epsilon);+ const device T* z_row =+ in_proj + (batch_idx * tree_size + token) * total_proj_dim + conv_dim + hv_idx * head_v_dim + dv;+ T4 result;+ METAL_PRAGMA_UNROLL+ for (uint j = 0; j < 4; ++j) {+ result[j] = static_cast<T>(values[j] * inv_rms * norm_weight[dv + j] * activate_silu(float(z_row[j])));+ }+ *reinterpret_cast<device T4*>(o + value_base + token * value_token_stride + dv) = result;+ }+}diff --git a/crates/uzu-engine/src/encodable_block/mixer/delta_net.rs b/crates/uzu-engine/src/encodable_block/mixer/delta_net.rsindex 66efc68e6..0011ca082 100644--- a/crates/uzu-engine/src/encodable_block/mixer/delta_net.rs+++ b/crates/uzu-engine/src/encodable_block/mixer/delta_net.rs@@ -410,7 +410,7 @@ impl<B: Backend> DeltaNet<B> {command_buffer,);- let mut delta_output = tree_verify+ let delta_output = tree_verify.encode(TreeVerifyEncodeArguments {q: &q,@@ -420,24 +420,16 @@ impl<B: Backend> DeltaNet<B> {log_decay: &log_decay,beta: &beta,h0: &state.ssm_state,+ in_proj: &tree_projected,+ norm_weight: &self.norm_weight,+ conv_dim: self.conv_dim,+ total_proj_dim: self.total_proj_dim,+ norm_epsilon: self.norm_epsilon,tree_size,},command_buffer,).map_err(MixerEncodeError::Backend)?;- self.delta_net_norm_gate.encode(- &mut delta_output,- &tree_projected,- &self.norm_weight,- self.num_heads,- self.value_head_dim,- self.value_dim,- self.conv_dim,- self.total_proj_dim,- self.norm_epsilon,- tree_size,- command_buffer,- );let output = self.out_projection.encode(delta_output, tree_size, command_buffer)?;state.suffix_status = Some(DeltaNetSuffixStatus::Tree {diff --git a/crates/uzu-engine/src/encodable_block/mixer/delta_net/tree_verify.rs b/crates/uzu-engine/src/encodable_block/mixer/delta_net/tree_verify.rsindex 223aceff9..ceb52e9e8 100644--- a/crates/uzu-engine/src/encodable_block/mixer/delta_net/tree_verify.rs+++ b/crates/uzu-engine/src/encodable_block/mixer/delta_net/tree_verify.rs@@ -18,6 +18,11 @@ pub struct TreeVerifyEncodeArguments<'a, B: Backend> {pub log_decay: &'a B::ScratchBuffer,pub beta: &'a B::ScratchBuffer,pub h0: &'a B::GlobalBuffer,+ pub in_proj: &'a B::ScratchBuffer,+ pub norm_weight: &'a B::GlobalBuffer,+ pub conv_dim: u32,+ pub total_proj_dim: u32,+ pub norm_epsilon: f32,pub tree_size: u32,}diff --git a/crates/uzu-engine/unit/backends/common/kernel/gdn/tree_verify/gram.md b/crates/uzu-engine/unit/backends/common/kernel/gdn/tree_verify/gram.mddeleted file mode 100644index 37d968de7..000000000--- a/crates/uzu-engine/unit/backends/common/kernel/gdn/tree_verify/gram.md+++ /dev/null@@ -1,114 +0,0 @@-# GDN Tree Gram Benchmarks--Local reference on M5 Max, `Hg=16 HV=48 K=128`, BF16 inputs, Criterion-`sample_size=10`, 2026-06-26.--Run through the existing `uzu_bench` lib harness:--```bash-cargo bench -p uzu-engine --lib -- "BuildTreeGram/.*/B1_T64"-cargo bench -p uzu-engine --lib -- "BuildTreeGram"-```--The current bench uses the repo-standard `iter_encode_loop_named` GPU-time-harness. The tables below are historical per-dispatch wall-time notes from the-optimization pass.--## Current--Current Ainv path is the scalar 16x16 column solve. The block-wise variant had-isolated MXU large-batch wins, but the simpler code is kept.--Final median time after full-tile unsafe load/store fast paths and-zero-initialized Ainv columns:--| B | Simdgroup T33 | Simdgroup T49 | Simdgroup T64 | Simdgroup T128 | Simdgroup T256 | Simdgroup T512 |-|---|---------------|---------------|---------------|----------------|----------------|----------------|-| 1 | 175.24 us | 177.65 us | 173.69 us | 201.87 us | 301.10 us | 679.74 us |-| 2 | 179.83 us | 183.96 us | 179.33 us | 246.44 us | 426.66 us | 1.1229 ms |-| 4 | 197.03 us | 218.57 us | 210.26 us | 299.20 us | 689.34 us | 2.0174 ms |-| 8 | 234.17 us | 262.53 us | 257.28 us | 438.20 us | 1.1379 ms | 3.8258 ms |--| B | MXU T33 | MXU T49 | MXU T64 | MXU T128 | MXU T256 | MXU T512 |-|---|---------|---------|---------|----------|----------|----------|-| 1 | 173.09 us | 172.65 us | 173.77 us | 183.50 us | 257.54 us | 448.55 us |-| 2 | 173.33 us | 173.81 us | 174.56 us | 201.17 us | 291.87 us | 747.12 us |-| 4 | 175.41 us | 182.72 us | 183.16 us | 258.75 us | 457.23 us | 1.3470 ms |-| 8 | 195.12 us | 221.28 us | 219.26 us | 327.69 us | 830.41 us | 2.7338 ms |--## Variants--B1/T64 variant check, all correctness-tested:--| Variant | Simdgroup | MXU | Kept |-|---|---:|---:|---|-| Baseline | 195.98 us | 180.58 us | no |-| zip-map only | 188.41 us | 177.34 us | no |-| zip-map + staged metadata | 182.08 us | 174.79 us | yes |-| staged + factored exp | 182.08 us | 174.72 us | no, no real gain |-| upper-tile skip only | 192.61 us | 182.10 us | no, MXU regressed |-| staged + factored exp + upper skip | 182.82 us | 175.35 us | no |-| staged metadata + full-tile unsafe loads | 165.53 us | 163.81 us | yes |--Pre epilogue-staging diagonal forward-substitution median time:--| B | MXU T33 | MXU T64 | MXU T128 | MXU T256 |-|---|---------|---------|----------|----------|-| 1 | 173.99 us | 179.70 us | 205.85 us | 301.06 us |-| 2 | 174.77 us | 183.59 us | 250.96 us | 421.89 us |-| 4 | 179.39 us | 217.04 us | 311.54 us | 672.27 us |-| 8 | 212.27 us | 258.01 us | 444.67 us | 1.1549 ms |--| B | Simdgroup T33 | Simdgroup T64 | Simdgroup T128 | Simdgroup T256 |-|---|---------------|---------------|----------------|----------------|-| 1 | 181.29 us | 193.49 us | 246.31 us | 424.49 us |-| 2 | 181.72 us | 198.02 us | 300.56 us | 639.55 us |-| 4 | 199.13 us | 251.43 us | 416.97 us | 1.0192 ms |-| 8 | 256.82 us | 307.09 us | 653.66 us | 1.8139 ms |--After co-walking `kk_acc`/`qk_acc` and staging per-tile prefix/beta/trie-metadata:--| B | MXU T64 | MXU T128 | MXU T256 | MXU T512 |-|---|---------|----------|----------|----------|-| 1 | 175.22 us | 189.40 us | 259.47 us | 487.78 us |-| 2 | 178.05 us | 215.88 us | 338.31 us | 822.90 us |-| 4 | 194.95 us | 264.85 us | 514.95 us | 1.4937 ms |-| 8 | 228.32 us | 355.72 us | 889.52 us | 2.8619 ms |--| B | Simdgroup T64 | Simdgroup T128 | Simdgroup T256 | Simdgroup T512 |-|---|---------------|----------------|----------------|----------------|-| 1 | 183.25 us | 222.01 us | 360.24 us | 883.53 us |-| 2 | 189.73 us | 274.71 us | 529.30 us | 1.5476 ms |-| 4 | 232.40 us | 378.08 us | 901.71 us | 2.8888 ms |-| 8 | 280.61 us | 546.91 us | 1.5795 ms | 5.5700 ms |--## Rejected Ainv Paths--| Variant | Simdgroup B1 T64 | Simdgroup B8 T256 | MXU B1 T64 | MXU B8 T128 | MXU B8 T256 |-|---|---:|---:|---:|---:|---:|-| scalar column Ainv | 173.69 us | 1.1379 ms | 173.77 us | 327.69 us | 830.41 us |-| 8x8 block Ainv + simdgroup matmul | 174.69 us | 1.1317 ms | 175.36 us | 320.86 us | 814.78 us |--| Variant | Simdgroup B1 T64 | Simdgroup B8 T256 | Simdgroup B8 T512 | MXU B1 T64 | MXU B8 T256 |-|---|---:|---:|---:|---:|---:|-| 8x8 block Ainv + scalar diagonal halves | 174.69 us | 1.1317 ms | 3.8325 ms | 175.36 us | 814.78 us |-| 8x8 block Ainv + Neumann diagonal halves | 174.51 us | 1.1749 ms | 4.0259 ms | 173.10 us | 837.44 us |--## Old Neumann Baseline--Speedup over `0600f367` diagonal Neumann. That old kernel has-`TREE_GRAM_MAX_T=64`, so T128/T256 are current-only.--| B | MXU T33 | MXU T64 | Simdgroup T33 | Simdgroup T64 |-|---|---------|---------|---------------|---------------|-| 1 | 1.04x | 1.06x | 1.01x | 1.01x |-| 2 | 1.04x | 1.23x | 1.03x | 1.17x |-| 4 | 1.17x | 1.20x | 1.10x | 1.05x |-| 8 | 1.21x | 1.25x | 1.02x | 1.12x |--Trace `tree_gram_b1_t64_mxu_debug_source.gputrace` showed low occupancy and-low unit use, not a memory-bandwidth wall: occupancy 7.61%, neural accelerator-utilization 6.95%, ALU utilization 10.74%, instruction-throughput utilization-9.52%. The largest source-line buckets were Gram post-processing, then-fragment loads/MMA, then Ainv fill and the small forward substitution.diff --git a/crates/uzu-engine/unit/backends/common/kernel/gdn/tree_verify/mod.rs b/crates/uzu-engine/unit/backends/common/kernel/gdn/tree_verify/mod.rsindex 027eae27b..1daaf4aa4 100644--- a/crates/uzu-engine/unit/backends/common/kernel/gdn/tree_verify/mod.rs+++ b/crates/uzu-engine/unit/backends/common/kernel/gdn/tree_verify/mod.rs@@ -1,10 +1,3 @@mod conv_scan_test;-mod out_bench;-mod out_test;-mod prefix_bench;-mod prefix_test;mod state_advance_test;-mod tree_gram_bench;-mod tree_gram_test;-mod tree_update_solve_bench;-mod tree_update_solve_test;+mod tree_verify_test;diff --git a/crates/uzu-engine/unit/backends/common/kernel/gdn/tree_verify/out_bench.rs b/crates/uzu-engine/unit/backends/common/kernel/gdn/tree_verify/out_bench.rsdeleted file mode 100644index 3ace7ffea..000000000--- a/crates/uzu-engine/unit/backends/common/kernel/gdn/tree_verify/out_bench.rs+++ /dev/null@@ -1,139 +0,0 @@-#![cfg(backend = "metal")]--use std::time::Duration;--use criterion::{BenchmarkId, Criterion};-use futures::executor::block_on;-use half::bf16;-use num_traits::Float;-use uzu_engine_macros::uzu_bench;--use crate::{- backends::{- common::{Backend, Context, Kernels, kernel::BuildTreeOutKernel},- metal::Metal,- },- data_type::DataTypeRepr,- tests::{- helpers::{create_buffer, create_buffer_with_data},- matmul::iter_encode_loop_named,- },-};--const QK_HEADS: usize = 16;-const VALUE_HEADS: usize = 48;-const HEAD_K_DIM: usize = 128;-const HEAD_V_DIM: usize = 128;-const TREE_OUT_MATMUL_MXU_ROWS: usize = 16;-const TREE_OUT_MATMUL_SIMDGROUP_ROWS: usize = 8;-const TREE_SIZES: &[usize] = &[32, 33, 49, 64, 128, 256, 512];-const BATCH_SIZES: &[usize] = &[1, 2, 4, 8];-const TREE_OUT_VALUE_COLS: usize = 32;-const TREE_OUT_SIMDGROUPS_PER_TG: usize = 4;-const BUILD_TREE_OUT_PATHS: &[(&str, bool, bool, usize)] = &[- ("Simdgroup/H0Direct", false, false, TREE_OUT_MATMUL_SIMDGROUP_ROWS),- ("Simdgroup/H0Transposed", false, true, TREE_OUT_MATMUL_SIMDGROUP_ROWS),- ("MXU/H0Direct", true, false, TREE_OUT_MATMUL_MXU_ROWS),-];--struct TreeOutBuffers {- q: <Metal as Backend>::GlobalBuffer,- prefix: <Metal as Backend>::GlobalBuffer,- qkd: <Metal as Backend>::GlobalBuffer,- u: <Metal as Backend>::GlobalBuffer,- h0: <Metal as Backend>::GlobalBuffer,- o: <Metal as Backend>::GlobalBuffer,-}--fn make_buffers<T: DataTypeRepr + Float>(- context: &<Metal as Backend>::Context,- batch_size: usize,- tree_size: usize,-) -> TreeOutBuffers {- let q_len = batch_size * tree_size * QK_HEADS * HEAD_K_DIM;- let prefix_len = batch_size * tree_size * VALUE_HEADS;- let qkd_len = batch_size * VALUE_HEADS * tree_size * tree_size;- let uv_len = batch_size * VALUE_HEADS * tree_size * HEAD_V_DIM;- let h0_len = batch_size * VALUE_HEADS * HEAD_V_DIM * HEAD_K_DIM;- let q = (0..q_len).map(|i| T::from(((i as f32 * 0.017).sin() * 0.2) + 0.01).unwrap()).collect::<Vec<_>>();- let prefix = (0..prefix_len)- .map(|i| -((i % tree_size) as f32) * 0.01 - ((i % VALUE_HEADS) as f32) * 0.003)- .collect::<Vec<_>>();- let qkd = (0..qkd_len).map(|i| ((i as f32 * 0.013).cos() * 0.03) - 0.01).collect::<Vec<_>>();- let u = (0..uv_len).map(|i| (i as f32 * 0.011).sin() * 0.3 + 0.04).collect::<Vec<_>>();- let h0 = (0..h0_len).map(|i| ((i as f32 * 0.019).cos() * 0.2) - 0.01).collect::<Vec<_>>();-- TreeOutBuffers {- q: create_buffer_with_data::<Metal, T>(context, &q),- prefix: create_buffer_with_data::<Metal, f32>(context, &prefix),- qkd: create_buffer_with_data::<Metal, f32>(context, &qkd),- u: create_buffer_with_data::<Metal, f32>(context, &u),- h0: create_buffer_with_data::<Metal, f32>(context, &h0),- o: create_buffer::<Metal, T>(context, uv_len),- }-}--#[uzu_bench]-fn bench_build_tree_out(c: &mut Criterion) {- let context = block_on(<Metal as Backend>::Context::new()).expect("metal context");- bench_build_tree_out_type::<bf16>(c, &context, "BF16");- bench_build_tree_out_type::<f32>(c, &context, "F32");-}--fn bench_build_tree_out_type<T: DataTypeRepr + Float>(- c: &mut Criterion,- context: &<Metal as Backend>::Context,- data_type_name: &str,-) {- for &(path, use_mxu, transposed_h0, tile_rows) in BUILD_TREE_OUT_PATHS {- if use_mxu && !context.supports_mxu {- continue;- }- let tile_name = format!("Rows{tile_rows}_VCols{TREE_OUT_VALUE_COLS}_SG{TREE_OUT_SIMDGROUPS_PER_TG}");- let mut group =- c.benchmark_group(format!("Metal/Kernel/GDNTreeVerify/BuildTreeOut/{data_type_name}/{path}/{tile_name}"));- group.sample_size(10).warm_up_time(Duration::from_millis(100)).measurement_time(Duration::from_millis(500));-- for &batch_size in BATCH_SIZES {- for &tree_size in TREE_SIZES {- let mut buffers = make_buffers::<T>(context, batch_size, tree_size);- let parameter =- format!("B{batch_size}_T{tree_size}_QK{QK_HEADS}_HV{VALUE_HEADS}_K{HEAD_K_DIM}_V{HEAD_V_DIM}");- let benchmark_path =- format!("Metal/Kernel/GDNTreeVerify/BuildTreeOut/{data_type_name}/{path}/{tile_name}/{parameter}");- let benchmark_id = BenchmarkId::from_parameter(parameter);-- let kernel = <<Metal as Backend>::Kernels as Kernels>::BuildTreeOutKernel::new(- context,- T::data_type(),- T::data_type(),- use_mxu,- transposed_h0,- true,- )- .expect("BuildTreeOutKernel");- group.bench_function(benchmark_id, |bencher| {- iter_encode_loop_named::<Metal, _>(context, bencher, &benchmark_path, |command_buffer| {- kernel.encode(- &buffers.q,- &buffers.prefix,- &buffers.qkd,- &buffers.u,- Some(&buffers.h0),- &mut buffers.o,- (HEAD_K_DIM as f32).sqrt().recip(),- batch_size as u32,- tree_size as u32,- QK_HEADS as u32,- VALUE_HEADS as u32,- HEAD_K_DIM as u32,- HEAD_V_DIM as u32,- command_buffer,- );- });- });- }- }- group.finish();- }-}diff --git a/crates/uzu-engine/unit/backends/common/kernel/gdn/tree_verify/out_test.rs b/crates/uzu-engine/unit/backends/common/kernel/gdn/tree_verify/out_test.rsdeleted file mode 100644index b567d2391..000000000--- a/crates/uzu-engine/unit/backends/common/kernel/gdn/tree_verify/out_test.rs+++ /dev/null@@ -1,184 +0,0 @@-#![cfg(backend = "metal")]--use futures::executor::block_on;-use half::bf16;-use num_traits::Float;-use uzu_engine_macros::uzu_test;--use crate::{- backends::{- common::{- Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferPending, Context, Kernels,- kernel::BuildTreeOutKernel,- },- cpu::Cpu,- metal::Metal,- },- data_type::DataTypeRepr,- tests::{- assert::assert_eq_float,- helpers::{buffer_to_vec, create_buffer, create_buffer_with_data, for_each_non_cpu_backend},- },-};--#[derive(Clone, Copy)]-struct Shape {- batch_size: usize,- tree_size: usize,- qk_heads: usize,- value_heads: usize,- head_k_dim: usize,- head_v_dim: usize,-}--struct Inputs<T> {- q: Vec<T>,- prefix: Vec<f32>,- qkd: Vec<f32>,- u: Vec<f32>,- h0: Vec<f32>,-}--const BUILD_TREE_OUT_PATHS: &[(&str, bool, bool)] = &[- ("Simdgroup/H0Direct/Rows8_VCols32_SG4", false, false),- ("Simdgroup/H0Transposed/Rows8_VCols32_SG4", false, true),- ("MXU/H0Direct/Rows16_VCols32_SG4", true, false),-];--fn make_inputs<T: DataTypeRepr + Float>(shape: Shape) -> Inputs<T> {- let q_len = shape.batch_size * shape.tree_size * shape.qk_heads * shape.head_k_dim;- let prefix_len = shape.batch_size * shape.tree_size * shape.value_heads;- let qkd_len = shape.batch_size * shape.value_heads * shape.tree_size * shape.tree_size;- let u_len = shape.batch_size * shape.value_heads * shape.tree_size * shape.head_v_dim;- let h0_len = shape.batch_size * shape.value_heads * shape.head_v_dim * shape.head_k_dim;-- Inputs {- q: (0..q_len).map(|i| T::from(((i as f32 * 0.017).sin() * 0.2) + 0.01).unwrap()).collect(),- prefix: (0..prefix_len)- .map(|i| -((i % shape.tree_size) as f32) * 0.01 - ((i % shape.value_heads) as f32) * 0.003)- .collect(),- qkd: (0..qkd_len).map(|i| ((i as f32 * 0.013).cos() * 0.1) - 0.02).collect(),- u: (0..u_len).map(|i| ((i as f32 * 0.011).sin() * 0.3) + 0.04).collect(),- h0: (0..h0_len).map(|i| ((i as f32 * 0.019).cos() * 0.2) - 0.01).collect(),- }-}--fn run_build_tree_out<B: Backend, T: DataTypeRepr + Float>(- shape: Shape,- inputs: &Inputs<T>,- use_h0: bool,- use_mxu: bool,- transposed_h0: bool,-) -> Vec<T> {- let context = block_on(B::Context::new()).expect("Failed to create Context");- let kernel = <<B as Backend>::Kernels as Kernels>::BuildTreeOutKernel::new(- &context,- T::data_type(),- T::data_type(),- use_mxu,- transposed_h0,- use_h0,- )- .expect("BuildTreeOutKernel");- let q = create_buffer_with_data::<B, T>(&context, &inputs.q);- let prefix = create_buffer_with_data::<B, f32>(&context, &inputs.prefix);- let qkd = create_buffer_with_data::<B, f32>(&context, &inputs.qkd);- let u = create_buffer_with_data::<B, f32>(&context, &inputs.u);- let h0 = use_h0.then(|| create_buffer_with_data::<B, f32>(&context, &inputs.h0));- let mut o =- create_buffer::<B, T>(&context, shape.batch_size * shape.tree_size * shape.value_heads * shape.head_v_dim);-- let mut command_buffer = context.create_command_buffer(None, None).expect("Failed to create command buffer");- kernel.encode(- &q,- &prefix,- &qkd,- &u,- h0.as_ref(),- &mut o,- (shape.head_k_dim as f32).sqrt().recip(),- shape.batch_size as u32,- shape.tree_size as u32,- shape.qk_heads as u32,- shape.value_heads as u32,- shape.head_k_dim as u32,- shape.head_v_dim as u32,- &mut command_buffer,- );- block_on(command_buffer.end_encoding().submit().wait_until_completed()).unwrap();- buffer_to_vec(&o)-}--fn check_shape<T: DataTypeRepr + Float + std::fmt::Display>(- shape: Shape,- eps: f32,-) {- let inputs = make_inputs::<T>(shape);-- for use_h0 in [false, true] {- let expected = run_build_tree_out::<Cpu, T>(shape, &inputs, use_h0, false, false);- for_each_non_cpu_backend!(|B| {- for &(path, use_mxu, transposed_h0) in BUILD_TREE_OUT_PATHS {- if use_mxu || transposed_h0 && !use_h0 {- continue;- }- let actual = run_build_tree_out::<B, T>(shape, &inputs, use_h0, use_mxu, transposed_h0);- let msg = format!(- "backend {} path {path} use_h0 {use_h0} B{}_T{}_QK{}_HV{}_K{}_V{}",- std::any::type_name::<B>(),- shape.batch_size,- shape.tree_size,- shape.qk_heads,- shape.value_heads,- shape.head_k_dim,- shape.head_v_dim- );- assert_eq_float::<T>(&expected, &actual, eps, &msg);- }- });-- if block_on(<Metal as Backend>::Context::new()).expect("Failed to create Context").supports_mxu {- for &(path, use_mxu, transposed_h0) in BUILD_TREE_OUT_PATHS {- if !use_mxu || transposed_h0 && !use_h0 {- continue;- }- let actual = run_build_tree_out::<Metal, T>(shape, &inputs, use_h0, use_mxu, transposed_h0);- let msg = format!(- "backend {} path {path} use_h0 {use_h0} B{}_T{}_QK{}_HV{}_K{}_V{}",- std::any::type_name::<Metal>(),- shape.batch_size,- shape.tree_size,- shape.qk_heads,- shape.value_heads,- shape.head_k_dim,- shape.head_v_dim- );- assert_eq_float::<T>(&expected, &actual, eps, &msg);- }- }- }-}--#[uzu_test]-fn test_build_tree_out_paths() {- let small = Shape {- batch_size: 2,- tree_size: 17,- qk_heads: 2,- value_heads: 6,- head_k_dim: 32,- head_v_dim: 32,- };- let gdn = Shape {- batch_size: 1,- tree_size: 49,- qk_heads: 16,- value_heads: 48,- head_k_dim: 128,- head_v_dim: 128,- };- for shape in [small, gdn] {- check_shape::<bf16>(shape, 5e-3);- check_shape::<f32>(shape, 5e-4);- }-}diff --git a/crates/uzu-engine/unit/backends/common/kernel/gdn/tree_verify/prefix_bench.rs b/crates/uzu-engine/unit/backends/common/kernel/gdn/tree_verify/prefix_bench.rsdeleted file mode 100644index 6dc0edd79..000000000--- a/crates/uzu-engine/unit/backends/common/kernel/gdn/tree_verify/prefix_bench.rs+++ /dev/null@@ -1,67 +0,0 @@-#![cfg(backend = "metal")]--use std::mem::size_of;--use criterion::{BenchmarkId, Criterion, Throughput};-use uzu_engine_macros::uzu_bench;--use crate::{- backends::{- common::{Backend, Kernels, kernel::BuildTreePrefixKernel},- metal::Metal,- },- tests::{- cold_pool::ColdPool,- helpers::{create_buffer, create_buffer_with_data},- matmul::iter_encode_loop_named,- util::type_short_name,- },-};--struct TreePrefixBuffers {- trie: <Metal as Backend>::GlobalBuffer,- log_decay: <Metal as Backend>::GlobalBuffer,- prefix: <Metal as Backend>::GlobalBuffer,-}--#[uzu_bench]-fn bench_build_tree_prefix(c: &mut Criterion) {- let context = crate::tests::util::shared_metal_context();- let kernel =- <<Metal as Backend>::Kernels as Kernels>::BuildTreePrefixKernel::new(&context).expect("BuildTreePrefixKernel");- let mut group = c.benchmark_group(format!("{}/Kernel/GDNTreeVerify/BuildTreePrefix", type_short_name::<Metal>()));-- for tree_size in [49usize, 64, 128] {- let batch_size = 1usize;- let value_heads = 48usize;- let len = batch_size * tree_size * value_heads;- let trie = (0..tree_size as u32).flat_map(|row| [row, tree_size as u32 - 1, row]).collect::<Vec<_>>();- let log_decay = (0..len).map(|i| -0.001 - (i as f32 * 0.017).sin().abs() * 0.1).collect::<Vec<_>>();- let bytes_per_copy = trie.len() * size_of::<u32>() + len * size_of::<f32>() * 2;- let mut buffers = ColdPool::new(bytes_per_copy, || TreePrefixBuffers {- trie: create_buffer_with_data::<Metal, u32>(&context, &trie),- log_decay: create_buffer_with_data::<Metal, f32>(&context, &log_decay),- prefix: create_buffer::<Metal, f32>(&context, len),- });-- group.throughput(Throughput::Elements((len * tree_size) as u64));- group.bench_function(BenchmarkId::from_parameter(format!("B1_T{tree_size}_HV{value_heads}")), |bencher| {- let benchmark_path = format!(- "{}/Kernel/GDNTreeVerify/BuildTreePrefix/B1_T{tree_size}_HV{value_heads}",- type_short_name::<Metal>()- );- iter_encode_loop_named::<Metal, _>(&context, bencher, &benchmark_path, |command_buffer| {- let buffers = buffers.next_mut();- kernel.encode(- &buffers.trie,- &buffers.log_decay,- &mut buffers.prefix,- batch_size as u32,- tree_size as u32,- value_heads as u32,- command_buffer,- );- });- });- }-}diff --git a/crates/uzu-engine/unit/backends/common/kernel/gdn/tree_verify/prefix_test.rs b/crates/uzu-engine/unit/backends/common/kernel/gdn/tree_verify/prefix_test.rsdeleted file mode 100644index 746507ebe..000000000--- a/crates/uzu-engine/unit/backends/common/kernel/gdn/tree_verify/prefix_test.rs+++ /dev/null@@ -1,49 +0,0 @@-#![cfg(backend = "metal")]--use futures::executor::block_on;-use uzu_engine_macros::uzu_test;--use crate::{- backends::{- common::{- Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferPending, Context, Kernels,- kernel::BuildTreePrefixKernel,- },- cpu::Cpu,- },- tests::{- assert::assert_eq_float,- helpers::{buffer_to_vec, create_buffer, create_buffer_with_data, for_each_non_cpu_backend},- },-};--fn run<B: Backend>(tree_size: usize) -> Vec<f32> {- const HEADS: usize = 5;- let context = block_on(B::Context::new()).unwrap();- let trie = (0..tree_size as u32).flat_map(|node| [node, tree_size as u32 - 1, node]).collect::<Vec<_>>();- let log_decay = (0..tree_size * HEADS).map(|i| -0.001 - i as f32 * 0.0001).collect::<Vec<_>>();- let trie = create_buffer_with_data::<B, u32>(&context, &trie);- let log_decay = create_buffer_with_data::<B, f32>(&context, &log_decay);- let mut prefix = create_buffer::<B, f32>(&context, tree_size * HEADS);- let kernel = <B::Kernels as Kernels>::BuildTreePrefixKernel::new(&context).unwrap();- let mut command_buffer = context.create_command_buffer(None, None).unwrap();- kernel.encode(&trie, &log_decay, &mut prefix, 1, tree_size as u32, HEADS as u32, &mut command_buffer);- block_on(command_buffer.end_encoding().submit().wait_until_completed()).unwrap();- buffer_to_vec(&prefix)-}--#[uzu_test]-fn test_build_tree_prefix() {- const HEADS: usize = 5;- for tree_size in [49, 64, 128] {- let expected = (0..tree_size)- .flat_map(|row| {- (0..HEADS).map(move |head| {- (0..=row).map(|token| -0.001 - (token * HEADS + head) as f32 * 0.0001).sum::<f32>()- })- })- .collect::<Vec<_>>();- assert_eq_float(&expected, &run::<Cpu>(tree_size), 1e-6, "CPU");- for_each_non_cpu_backend!(|B| assert_eq_float(&expected, &run::<B>(tree_size), 1e-6, "backend"));- }-}diff --git a/crates/uzu-engine/unit/backends/common/kernel/gdn/tree_verify/tree_gram_bench.rs b/crates/uzu-engine/unit/backends/common/kernel/gdn/tree_verify/tree_gram_bench.rsdeleted file mode 100644index 81ca5a365..000000000--- a/crates/uzu-engine/unit/backends/common/kernel/gdn/tree_verify/tree_gram_bench.rs+++ /dev/null@@ -1,204 +0,0 @@-#![cfg(backend = "metal")]--use std::{mem::size_of, time::Duration};--use criterion::{BenchmarkId, Criterion};-use futures::executor::block_on;-use half::bf16;-use uzu_engine_macros::uzu_bench;--use crate::{- backends::{- common::{Backend, Context, Kernels, kernel::BuildTreeGramKernel},- metal::Metal,- },- data_type::DataType,- tests::{- cold_pool::ColdPool,- helpers::{create_buffer, create_buffer_with_data},- matmul::iter_encode_loop_named,- },-};--const K_HEADS: usize = 16;-const VALUE_HEADS: usize = 48;-const HEAD_K_DIM: usize = 128;-const HEAD_V_DIM: usize = 128;-const TREE_SIZES: &[usize] = &[32, 33, 49, 64, 128, 256, 512];-const BATCH_SIZES: &[usize] = &[1, 2, 4, 8];--struct TreeGramBuffers {- q: <Metal as Backend>::GlobalBuffer,- k: <Metal as Backend>::GlobalBuffer,- trie: <Metal as Backend>::GlobalBuffer,- prefix: <Metal as Backend>::GlobalBuffer,- beta: <Metal as Backend>::GlobalBuffer,- h0: <Metal as Backend>::GlobalBuffer,- a_packed: <Metal as Backend>::GlobalBuffer,- qkd: <Metal as Backend>::GlobalBuffer,- a_inv: <Metal as Backend>::GlobalBuffer,- kh0: <Metal as Backend>::GlobalBuffer,-}--fn reranker_like_trie(- batch_size: usize,- tree_size: usize,-) -> Vec<u32> {- let mut trie = Vec::with_capacity(batch_size * tree_size * 3);- for b in 0..batch_size {- let mut children = vec![Vec::<usize>::new(); tree_size];- let mut child_count = vec![0usize; tree_size];- for node in 1..tree_size {- let mut parent = (node - 1) / 2;- if child_count[parent] >= 2 {- parent = node.saturating_sub(2 + (b + node) % node);- }- children[parent].push(node);- child_count[parent] += 1;- }-- fn dfs(- node: usize,- children: &[Vec<usize>],- trie: &mut [[u32; 3]],- next: &mut u32,- depth: u32,- ) {- let start = *next;- *next += 1;- let mut end = start;- for &child in &children[node] {- dfs(child, children, trie, next, depth + 1);- end = end.max(trie[child][1]);- }- trie[node] = [start, end, depth];- }-- let mut batch_trie = vec![[0u32; 3]; tree_size];- let mut next = 0;- dfs(0, &children, &mut batch_trie, &mut next, 0);- for node in batch_trie {- trie.extend_from_slice(&node);- }- }- trie-}--fn make_buffers(- context: &<Metal as Backend>::Context,- batch_size: usize,- tree_size: usize,-) -> (TreeGramBuffers, f32) {- let qk_len = batch_size * tree_size * K_HEADS * HEAD_K_DIM;- let scalar_len = batch_size * tree_size * VALUE_HEADS;- let out_len = batch_size * VALUE_HEADS * tree_size * tree_size;- let num_blocks = tree_size.div_ceil(16);- let a_len = batch_size * VALUE_HEADS * num_blocks * num_blocks.div_ceil(2) * 16 * 32;- let a_inv_len = batch_size * VALUE_HEADS * num_blocks * 16 * 16;- let h0_len = batch_size * VALUE_HEADS * HEAD_V_DIM * HEAD_K_DIM;- let kh0_len = batch_size * tree_size * VALUE_HEADS * HEAD_V_DIM;- let trie = reranker_like_trie(batch_size, tree_size);- let q = (0..qk_len).map(|i| bf16::from_f32(((i as f32 * 0.017).sin() * 0.2) + 0.01)).collect::<Vec<_>>();- let k = (0..qk_len).map(|i| bf16::from_f32(((i as f32 * 0.019).cos() * 0.18) - 0.02)).collect::<Vec<_>>();- let prefix = (0..scalar_len)- .map(|i| -((i % tree_size) as f32) * 0.01 - ((i % VALUE_HEADS) as f32) * 0.003)- .collect::<Vec<_>>();- let beta = (0..scalar_len).map(|i| 0.25 + ((i as f32 * 0.013).sin() + 1.0) * 0.2).collect::<Vec<_>>();- let h0 = (0..h0_len).map(|i| ((i as f32 * 0.007).sin() * 0.05) - 0.01).collect::<Vec<_>>();- let scale = (HEAD_K_DIM as f32).sqrt().recip();- (- TreeGramBuffers {- q: create_buffer_with_data::<Metal, bf16>(context, &q),- k: create_buffer_with_data::<Metal, bf16>(context, &k),- trie: create_buffer_with_data::<Metal, u32>(context, &trie),- prefix: create_buffer_with_data::<Metal, f32>(context, &prefix),- beta: create_buffer_with_data::<Metal, f32>(context, &beta),- h0: create_buffer_with_data::<Metal, f32>(context, &h0),- a_packed: create_buffer::<Metal, f32>(context, a_len),- qkd: create_buffer::<Metal, f32>(context, out_len),- a_inv: create_buffer::<Metal, f32>(context, a_inv_len),- kh0: create_buffer::<Metal, f32>(context, kh0_len),- },- scale,- )-}--fn buffers_bytes(- batch_size: usize,- tree_size: usize,-) -> usize {- let qk_len = batch_size * tree_size * K_HEADS * HEAD_K_DIM;- let scalar_len = batch_size * tree_size * VALUE_HEADS;- let out_len = batch_size * VALUE_HEADS * tree_size * tree_size;- let num_blocks = tree_size.div_ceil(16);- let a_len = batch_size * VALUE_HEADS * num_blocks * num_blocks.div_ceil(2) * 16 * 32;- let a_inv_len = batch_size * VALUE_HEADS * num_blocks * 16 * 16;- let h0_len = batch_size * VALUE_HEADS * HEAD_V_DIM * HEAD_K_DIM;- let kh0_len = batch_size * tree_size * VALUE_HEADS * HEAD_V_DIM;- qk_len * 2 * size_of::<bf16>()- + h0_len * size_of::<f32>()- + batch_size * tree_size * 3 * size_of::<u32>()- + scalar_len * size_of::<f32>() * 2- + (out_len + a_len + a_inv_len + kh0_len) * size_of::<f32>()-}--#[uzu_bench]-fn bench_build_tree_gram(c: &mut Criterion) {- let context = block_on(<Metal as Backend>::Context::new()).expect("metal context");- let kernel_paths = if context.supports_mxu {- &[("Simdgroup", false), ("MXU", true)][..]- } else {- &[("Simdgroup", false)][..]- };-- for &(kernel_path, use_mxu) in kernel_paths {- let mut group = c.benchmark_group(format!("Metal/Kernel/GDNTreeVerify/BuildTreeGram/{kernel_path}"));- group.sample_size(20).warm_up_time(Duration::from_millis(500)).measurement_time(Duration::from_secs(2));-- for &batch_size in BATCH_SIZES {- for &tree_size in TREE_SIZES {- let kernel = <<Metal as Backend>::Kernels as Kernels>::BuildTreeGramKernel::new(- &context,- DataType::BF16,- use_mxu,- true,- )- .expect("BuildTreeGramKernel");- let scale = (HEAD_K_DIM as f32).sqrt().recip();- let parameter = format!("B{batch_size}_T{tree_size}_Hg{K_HEADS}_HV{VALUE_HEADS}_K{HEAD_K_DIM}");- let benchmark_path = format!("Metal/Kernel/GDNTreeVerify/BuildTreeGram/{kernel_path}/{parameter}");- let benchmark_id = BenchmarkId::from_parameter(parameter);-- let mut buffers = ColdPool::new(buffers_bytes(batch_size, tree_size), || {- make_buffers(&context, batch_size, tree_size).0- });- group.bench_function(benchmark_id, |bencher| {- iter_encode_loop_named::<Metal, _>(context.as_ref(), bencher, &benchmark_path, |command_buffer| {- let buffers = buffers.next_mut();- kernel.encode(- &buffers.q,- &buffers.k,- &buffers.trie,- &buffers.prefix,- &buffers.beta,- Some(&buffers.h0),- &mut buffers.a_packed,- &mut buffers.qkd,- &mut buffers.a_inv,- Some(&mut buffers.kh0),- scale,- batch_size as u32,- tree_size as u32,- K_HEADS as u32,- VALUE_HEADS as u32,- HEAD_K_DIM as u32,- HEAD_V_DIM as u32,- command_buffer,- );- });- });- }- }- group.finish();- }-}diff --git a/crates/uzu-engine/unit/backends/common/kernel/gdn/tree_verify/tree_gram_test.rs b/crates/uzu-engine/unit/backends/common/kernel/gdn/tree_verify/tree_gram_test.rsdeleted file mode 100644index 686159bf6..000000000--- a/crates/uzu-engine/unit/backends/common/kernel/gdn/tree_verify/tree_gram_test.rs+++ /dev/null@@ -1,183 +0,0 @@-#![cfg(backend = "metal")]--use futures::executor::block_on;-use half::bf16;-use num_traits::Float;-use uzu_engine_macros::uzu_test;--use crate::{- backends::{- common::{- Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferPending, Context, Kernels,- kernel::BuildTreeGramKernel,- },- cpu::Cpu,- metal::Metal,- },- data_type::{DataType, DataTypeRepr},- tests::{- assert::assert_eq_float,- helpers::{buffer_to_vec, create_buffer, create_buffer_with_data, for_each_non_cpu_backend},- },-};--const BATCH_SIZE: usize = 2;-const K_HEADS: usize = 2;-const VALUE_HEADS: usize = 6;-const HEAD_K_DIM: usize = 128;-// 80 = two full 32-wide kh0 dv chunks plus a ragged 16-wide one.-const HEAD_V_DIM: usize = 80;--struct Inputs {- trie: Vec<u32>,- q: Vec<f32>,- k: Vec<f32>,- prefix: Vec<f32>,- beta: Vec<f32>,- h0: Vec<f32>,-}--fn make_inputs(tree_size: usize) -> Inputs {- let q_len = BATCH_SIZE * tree_size * K_HEADS * HEAD_K_DIM;- let scalar_len = BATCH_SIZE * tree_size * VALUE_HEADS;- let h0_len = BATCH_SIZE * VALUE_HEADS * HEAD_V_DIM * HEAD_K_DIM;- Inputs {- trie: build_trie(tree_size),- q: (0..q_len).map(|i| ((i as f32 * 0.017).sin() * 0.2) + 0.01).collect(),- k: (0..q_len).map(|i| ((i as f32 * 0.019).cos() * 0.18) - 0.02).collect(),- prefix: (0..scalar_len)- .map(|i| -((i % tree_size) as f32) * 0.01 - ((i % VALUE_HEADS) as f32) * 0.003)- .collect(),- beta: (0..scalar_len).map(|i| 0.25 + ((i as f32 * 0.013).sin() + 1.0) * 0.2).collect(),- h0: (0..h0_len).map(|i| ((i as f32 * 0.007).sin() * 0.05) - 0.01).collect(),- }-}--fn get_output<B: Backend, T: DataTypeRepr + Float>(- q: &[T],- k: &[T],- trie: &[u32],- prefix: &[f32],- beta: &[f32],- h0: &[f32],- tree_size: usize,- scale: f32,- use_mxu: bool,-) -> (Vec<f32>, Vec<f32>, Vec<f32>, Vec<f32>) {- let context = block_on(B::Context::new()).expect("Failed to create Context");- let kernel =- <<B as Backend>::Kernels as Kernels>::BuildTreeGramKernel::new(&context, T::data_type(), use_mxu, true)- .expect("Failed to create BuildTreeGramKernel");-- let output_len = BATCH_SIZE * VALUE_HEADS * tree_size * tree_size;- let num_blocks = tree_size.div_ceil(16);- let a_len = BATCH_SIZE * VALUE_HEADS * num_blocks * num_blocks.div_ceil(2) * 16 * 32;- let a_inv_len = BATCH_SIZE * VALUE_HEADS * num_blocks * 16 * 16;- let kh0_len = BATCH_SIZE * tree_size * VALUE_HEADS * HEAD_V_DIM;- let q = create_buffer_with_data::<B, T>(&context, q);- let k = create_buffer_with_data::<B, T>(&context, k);- let trie = create_buffer_with_data::<B, u32>(&context, trie);- let prefix = create_buffer_with_data::<B, f32>(&context, prefix);- let beta = create_buffer_with_data::<B, f32>(&context, beta);- let h0 = create_buffer_with_data::<B, f32>(&context, h0);- // The GPU only writes packed-A tiles touching the block lower triangle;- // zero-init so CPU and GPU buffers stay comparable elsewhere.- let mut a_packed = create_buffer_with_data::<B, f32>(&context, &vec![0.0f32; a_len]);- let mut qkd = create_buffer::<B, f32>(&context, output_len);- let mut a_inv = create_buffer::<B, f32>(&context, a_inv_len);- let mut kh0 = create_buffer::<B, f32>(&context, kh0_len);-- let mut command_buffer = context.create_command_buffer(None, None).expect("Failed to create command buffer");- kernel.encode(- &q,- &k,- &trie,- &prefix,- &beta,- Some(&h0),- &mut a_packed,- &mut qkd,- &mut a_inv,- Some(&mut kh0),- scale,- BATCH_SIZE as u32,- tree_size as u32,- K_HEADS as u32,- VALUE_HEADS as u32,- HEAD_K_DIM as u32,- HEAD_V_DIM as u32,- &mut command_buffer,- );- block_on(command_buffer.end_encoding().submit().wait_until_completed()).unwrap();-- (buffer_to_vec(&a_packed), buffer_to_vec(&qkd), buffer_to_vec(&a_inv), buffer_to_vec(&kh0))-}--fn check_type<T: DataTypeRepr + Float>(- inputs: &Inputs,- tree_size: usize,- scale: f32,-) {- let q = inputs.q.iter().map(|&value| T::from(value).unwrap()).collect::<Vec<_>>();- let k = inputs.k.iter().map(|&value| T::from(value).unwrap()).collect::<Vec<_>>();- let expected =- get_output::<Cpu, T>(&q, &k, &inputs.trie, &inputs.prefix, &inputs.beta, &inputs.h0, tree_size, scale, false);-- for_each_non_cpu_backend!(|B| {- let actual =- get_output::<B, T>(&q, &k, &inputs.trie, &inputs.prefix, &inputs.beta, &inputs.h0, tree_size, scale, false);- let msg = format!("backend {} simdgroup tree_size {tree_size}", std::any::type_name::<B>());- assert_eq_float::<f32>(&expected.0, &actual.0, 2e-4, &format!("a_packed {msg}"));- assert_eq_float::<f32>(&expected.1, &actual.1, 2e-4, &format!("qkd {msg}"));- assert_eq_float::<f32>(&expected.2, &actual.2, 5e-4, &format!("a_inv {msg}"));- assert_eq_float::<f32>(&expected.3, &actual.3, 2e-4, &format!("kh0 {msg}"));- });-- if block_on(<Metal as Backend>::Context::new()).expect("Failed to create Context").supports_mxu {- let actual = get_output::<Metal, T>(- &q,- &k,- &inputs.trie,- &inputs.prefix,- &inputs.beta,- &inputs.h0,- tree_size,- scale,- true,- );- let msg = format!("backend {} path MXU tree_size {tree_size}", std::any::type_name::<Metal>());- let (a_eps, qkd_eps, inv_eps, kh0_eps) = if T::data_type() == DataType::BF16 {- (1e-5, 1e-6, 1e-5, 5e-4)- } else {- (5e-3, 1e-3, 8e-3, 1e-3)- };- assert_eq_float::<f32>(&expected.0, &actual.0, a_eps, &format!("a_packed {msg}"));- assert_eq_float::<f32>(&expected.1, &actual.1, qkd_eps, &format!("qkd {msg}"));- assert_eq_float::<f32>(&expected.2, &actual.2, inv_eps, &format!("a_inv {msg}"));- assert_eq_float::<f32>(&expected.3, &actual.3, kh0_eps, &format!("kh0 {msg}"));- }-}--fn build_trie(tree_size: usize) -> Vec<u32> {- let last = tree_size as u32 - 1;- let mut trie = Vec::with_capacity(BATCH_SIZE * tree_size * 3);- for i in 0..tree_size as u32 {- trie.extend_from_slice(&[i, last, i]);- }- trie.extend_from_slice(&[0, last, 0]);- for i in 1..tree_size as u32 {- trie.extend_from_slice(&[i, i, 1]);- }- trie-}--#[uzu_test]-fn test_build_tree_gram_matches_cpu() {- let scale = (HEAD_K_DIM as f32).sqrt().recip();-- for tree_size in [17, 64, 128] {- let i = make_inputs(tree_size);- check_type::<f32>(&i, tree_size, scale);- check_type::<bf16>(&i, tree_size, scale);- }-}diff --git a/crates/uzu-engine/unit/backends/common/kernel/gdn/tree_verify/tree_update_solve_bench.rs b/crates/uzu-engine/unit/backends/common/kernel/gdn/tree_verify/tree_update_solve_bench.rsdeleted file mode 100644index 9bab2be4f..000000000--- a/crates/uzu-engine/unit/backends/common/kernel/gdn/tree_verify/tree_update_solve_bench.rs+++ /dev/null@@ -1,225 +0,0 @@-#![cfg(backend = "metal")]--use std::{mem::size_of, time::Duration};--use criterion::{BenchmarkId, Criterion, Throughput};-use futures::executor::block_on;-use half::bf16;-use uzu_engine_macros::uzu_bench;--use crate::{- backends::{- common::{Backend, CommandBuffer, Context, Kernels, kernel::TreeUpdateSolveKernel},- metal::Metal,- },- data_type::DataType,- tests::{- cold_pool::ColdPool,- helpers::{create_buffer, create_buffer_with_data},- matmul::iter_encode_loop_named,- },-};--const NUM_V_HEADS: usize = 48;-const HEAD_V_DIM: usize = 128;-const BT: usize = 16;-const BVS: &[usize] = &[16, 32];--const BENCH_SHAPES: &[(usize, usize)] = &[- (1, 32),- (1, 33),- (1, 49),- (1, 64),- (1, 128),- (1, 256),- (1, 512),- (2, 33),- (2, 49),- (2, 64),- (2, 128),- (2, 256),- (2, 512),- (4, 33),- (4, 49),- (4, 64),- (4, 128),- (4, 256),- (4, 512),- (8, 33),- (8, 49),- (8, 64),- (8, 128),- (8, 256),- (8, 512),-];--struct TreeUpdateSolveBuffers {- kh0: <Metal as Backend>::GlobalBuffer,- v: <Metal as Backend>::GlobalBuffer,- prefix: <Metal as Backend>::GlobalBuffer,- beta: <Metal as Backend>::GlobalBuffer,- a: <Metal as Backend>::GlobalBuffer,- a_inv: <Metal as Backend>::GlobalBuffer,- u: <Metal as Backend>::GlobalBuffer,-}--fn matrix_value(- batch_head_matrix_idx: usize,- row: usize,- col: usize,- tree_size: usize,-) -> f32 {- if col < row {- ((batch_head_matrix_idx * tree_size * tree_size + row * tree_size + col) as f32 * 0.011).sin() * 0.01- } else {- 0.0- }-}--// Compact [B * HV, ceil(T/BT), BT, BT] diagonal blocks.-fn inverse_value(- batch_head_matrix_idx: usize,- block: usize,- row: usize,- col: usize,- tree_size: usize,-) -> f32 {- if row == col {- 1.0- } else if col < row {- -matrix_value(batch_head_matrix_idx, block * BT + row, block * BT + col, tree_size)- } else {- 0.0- }-}--fn make_buffers(- context: &<Metal as Backend>::Context,- batch_size: usize,- tree_size: usize,-) -> TreeUpdateSolveBuffers {- let v_len = batch_size * tree_size * NUM_V_HEADS * HEAD_V_DIM;- let scalar_len = batch_size * tree_size * NUM_V_HEADS;- let num_blocks = tree_size.div_ceil(BT);- let num_col_pairs = num_blocks.div_ceil(2);- let a_len = batch_size * NUM_V_HEADS * num_blocks * num_col_pairs * BT * 2 * BT;- let u_len = batch_size * NUM_V_HEADS * tree_size * HEAD_V_DIM;-- let kh0 = (0..v_len).map(|i| ((i as f32 * 0.019).sin() * 0.2) + 0.01).collect::<Vec<f32>>();- let v = (0..v_len).map(|i| bf16::from_f32(((i as f32 * 0.017).cos() * 0.18) - 0.02)).collect::<Vec<_>>();- let prefix = (0..scalar_len)- .map(|i| -((i % tree_size) as f32) * 0.01 - ((i % NUM_V_HEADS) as f32) * 0.003)- .collect::<Vec<_>>();- let beta = (0..scalar_len).map(|i| 0.25 + ((i as f32 * 0.013).sin() + 1.0) * 0.2).collect::<Vec<_>>();- // Packed [B * HV, NB, ceil(NB/2), BT, 2*BT] block-pair tiles.- let a_f32 = (0..a_len)- .map(|i| {- let local_col = i % (2 * BT);- let local_row = (i / (2 * BT)) % BT;- let pair = (i / (BT * 2 * BT)) % num_col_pairs;- let block = (i / (BT * 2 * BT * num_col_pairs)) % num_blocks;- let batch_head_matrix_idx = i / (BT * 2 * BT * num_col_pairs * num_blocks);- let row = block * BT + local_row;- let col = pair * 2 * BT + local_col;- if col < tree_size && row < tree_size && col / BT < block {- matrix_value(batch_head_matrix_idx, row, col, tree_size)- } else {- 0.0- }- })- .collect::<Vec<_>>();- let a = create_buffer_with_data::<Metal, f32>(context, &a_f32);- let inv_len = batch_size * NUM_V_HEADS * tree_size.div_ceil(BT) * BT * BT;- let a_inv = (0..inv_len)- .map(|i| {- let col = i % BT;- let row = (i / BT) % BT;- let block = (i / (BT * BT)) % tree_size.div_ceil(BT);- let batch_head_matrix_idx = i / (BT * BT * tree_size.div_ceil(BT));- inverse_value(batch_head_matrix_idx, block, row, col, tree_size)- })- .collect::<Vec<_>>();- TreeUpdateSolveBuffers {- kh0: create_buffer_with_data::<Metal, f32>(context, &kh0),- v: create_buffer_with_data::<Metal, bf16>(context, &v),- prefix: create_buffer_with_data::<Metal, f32>(context, &prefix),- beta: create_buffer_with_data::<Metal, f32>(context, &beta),- a,- a_inv: create_buffer_with_data::<Metal, f32>(context, &a_inv),- u: create_buffer::<Metal, f32>(context, u_len),- }-}--fn buffers_bytes(- batch_size: usize,- tree_size: usize,-) -> usize {- let v_len = batch_size * tree_size * NUM_V_HEADS * HEAD_V_DIM;- let scalar_len = batch_size * tree_size * NUM_V_HEADS;- let num_blocks = tree_size.div_ceil(BT);- let a_len = batch_size * NUM_V_HEADS * num_blocks * num_blocks.div_ceil(2) * BT * 2 * BT;- let u_len = batch_size * NUM_V_HEADS * tree_size * HEAD_V_DIM;-- let inv_len = batch_size * NUM_V_HEADS * num_blocks * BT * BT;- v_len * size_of::<bf16>() + (scalar_len * 2 + a_len + inv_len + u_len + v_len) * size_of::<f32>()-}--#[uzu_bench]-fn bench_tree_update_solve(c: &mut Criterion) {- let context = block_on(<Metal as Backend>::Context::new()).expect("metal context");- for &bv in BVS {- let kernel = <<Metal as Backend>::Kernels as Kernels>::TreeUpdateSolveKernel::new(- &context,- DataType::BF16,- bv as u32,- true,- )- .expect("TreeUpdateSolveKernel");-- let mut group = c.benchmark_group(format!("Metal/Kernel/GDNTreeVerify/TreeUpdateSolve/Simdgroup/BV{bv}"));- // 500ms warmup ramps GPU clocks before measuring (µs-scale benches never- // ramp otherwise and medians become run-composition artifacts).- group.sample_size(20).warm_up_time(Duration::from_millis(500)).measurement_time(Duration::from_secs(2));-- for &(batch_size, tree_size) in BENCH_SHAPES {- let parameter = format!("B{batch_size}_T{tree_size}_HV{NUM_V_HEADS}_V{HEAD_V_DIM}_BV{bv}");- let benchmark_path = format!("Metal/Kernel/GDNTreeVerify/TreeUpdateSolve/Simdgroup/BV{bv}/{parameter}");- let benchmark_id = BenchmarkId::from_parameter(parameter);- let mut buffers =- ColdPool::new(buffers_bytes(batch_size, tree_size), || make_buffers(&context, batch_size, tree_size));-- group.throughput(Throughput::Elements((batch_size * tree_size * NUM_V_HEADS * HEAD_V_DIM) as u64));- group.bench_function(benchmark_id, |bencher| {- iter_encode_loop_named::<Metal, _>(context.as_ref(), bencher, &benchmark_path, |command_buffer| {- let buffers = buffers.next_mut();- encode(&kernel, buffers, batch_size, tree_size, command_buffer);- });- });- }- group.finish();- crate::tests::metrics::wait_gpu_cooldown();- }-}--fn encode(- kernel: &impl TreeUpdateSolveKernel<Backend = Metal>,- buffers: &mut TreeUpdateSolveBuffers,- batch_size: usize,- tree_size: usize,- command_buffer: &mut <<Metal as Backend>::CommandBuffer as CommandBuffer>::Encoding,-) {- kernel.encode(- Some(&buffers.kh0),- &buffers.v,- &buffers.prefix,- &buffers.beta,- &buffers.a,- &buffers.a_inv,- &mut buffers.u,- batch_size as u32,- tree_size as u32,- NUM_V_HEADS as u32,- HEAD_V_DIM as u32,- command_buffer,- );-}diff --git a/crates/uzu-engine/unit/backends/common/kernel/gdn/tree_verify/tree_update_solve_test.rs b/crates/uzu-engine/unit/backends/common/kernel/gdn/tree_verify/tree_update_solve_test.rsdeleted file mode 100644index ba955dd6f..000000000--- a/crates/uzu-engine/unit/backends/common/kernel/gdn/tree_verify/tree_update_solve_test.rs+++ /dev/null@@ -1,205 +0,0 @@-#![cfg(backend = "metal")]--use futures::executor::block_on;-use half::bf16;-use uzu_engine_macros::uzu_test;--use crate::{- backends::{- common::{- Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferPending, Context, Kernels,- kernel::TreeUpdateSolveKernel,- },- cpu::Cpu,- },- data_type::{DataType, DataTypeRepr},- tests::{- assert::assert_eq_float,- helpers::{buffer_to_vec, create_buffer, create_buffer_with_data, for_each_non_cpu_backend},- },-};--const BT: u32 = 16;-const BVS: &[u32] = &[16, 32];--#[derive(Clone, Copy)]-struct SolveCase {- name: &'static str,- batch_size: u32,- tree_size: u32,- num_v_heads: u32,- head_v_dim: u32,-}--const CASES: &[SolveCase] = &[- SolveCase {- name: "single_full_block",- batch_size: 1,- tree_size: 16,- num_v_heads: 1,- head_v_dim: 16,- },- SolveCase {- name: "ragged_second_block",- batch_size: 1,- tree_size: 17,- num_v_heads: 2,- head_v_dim: 20,- },- SolveCase {- name: "common_small_tree",- batch_size: 1,- tree_size: 33,- num_v_heads: 4,- head_v_dim: 128,- },- SolveCase {- name: "ragged_four_blocks",- batch_size: 1,- tree_size: 49,- num_v_heads: 4,- head_v_dim: 128,- },- SolveCase {- // NB=7: exercises pair_idx >= 1, tail blocks >= 4, and a ragged final block.- name: "ragged_seven_blocks",- batch_size: 1,- tree_size: 97,- num_v_heads: 2,- head_v_dim: 64,- },- SolveCase {- // NB=16: the large-tree regime this kernel exists for; num_full_pairs- // reaches 7, so the deep packed-pair accumulation is exercised.- name: "large_tree",- batch_size: 1,- tree_size: 256,- num_v_heads: 2,- head_v_dim: 64,- },- SolveCase {- name: "batched",- batch_size: 2,- tree_size: 16,- num_v_heads: 2,- head_v_dim: 16,- },-];--fn run_case<B: Backend, T: DataTypeRepr + Copy>(- case: SolveCase,- bv: u32,- use_h0: bool,- data_type: DataType,- cast: impl Fn(f32) -> T + Copy,-) -> Vec<f32> {- let context = block_on(B::Context::new()).expect("context");- let kernel = <<B as Backend>::Kernels as Kernels>::TreeUpdateSolveKernel::new(&context, data_type, bv, use_h0)- .expect("kernel");-- let batch_size = case.batch_size as usize;- let tree_size = case.tree_size as usize;- let num_v_heads = case.num_v_heads as usize;- let head_v_dim = case.head_v_dim as usize;-- let v_len = batch_size * tree_size * num_v_heads * head_v_dim;- let scalar_len = batch_size * tree_size * num_v_heads;- let num_blocks = tree_size.div_ceil(BT as usize);- let num_col_pairs = num_blocks.div_ceil(2);- let a_len = batch_size * num_v_heads * num_blocks * num_col_pairs * (BT * 2 * BT) as usize;- let inv_len = batch_size * num_v_heads * num_blocks * (BT * BT) as usize;- let u_len = batch_size * num_v_heads * tree_size * head_v_dim;-- let kh0 = (0..v_len).map(|i| ((i as f32 * 0.019).sin() * 0.2) + 0.01).collect::<Vec<f32>>();- let v = (0..v_len).map(|i| cast(((i as f32 * 0.017).cos() * 0.18) - 0.02)).collect::<Vec<_>>();- let prefix = (0..scalar_len)- .map(|i| -((i % tree_size) as f32) * 0.01 - ((i % num_v_heads) as f32) * 0.003)- .collect::<Vec<_>>();- let beta = (0..scalar_len).map(|i| 0.25 + ((i as f32 * 0.013).sin() + 1.0) * 0.2).collect::<Vec<_>>();- // Packed [B * HV, NB, ceil(NB/2), BT, 2*BT] block-pair tiles; strictly-lower- // blocks get values, everything else zero (the kernel never reads it).- let a_f32 = (0..a_len)- .map(|i| {- let local_col = i % (2 * BT as usize);- let local_row = (i / (2 * BT as usize)) % BT as usize;- let pair = (i / (BT * 2 * BT) as usize) % num_col_pairs;- let block = (i / ((BT * 2 * BT) as usize * num_col_pairs)) % num_blocks;- let row = block * BT as usize + local_row;- let col = pair * 2 * BT as usize + local_col;- if col < tree_size && row < tree_size && col / (BT as usize) < block {- // ~0.05 per entry: with the dense lower-triangle fill this puts the- // per-row A magnitude near the real l2-normed-key / softplus-decay regime.- ((i as f32 * 0.011).sin()) * 0.05- } else {- 0.0- }- })- .collect::<Vec<_>>();- // Compact [B * HV, ceil(T/BT), BT, BT] diagonal blocks.- let a_inv = (0..inv_len)- .map(|i| {- let col = i % BT as usize;- let row = (i / BT as usize) % BT as usize;- if row == col {- 1.0- } else if col < row {- -((i as f32 * 0.011).sin()) * 0.05- } else {- 0.0- }- })- .collect::<Vec<_>>();- let kh0 = create_buffer_with_data::<B, f32>(&context, &kh0);- let v = create_buffer_with_data::<B, T>(&context, &v);- let prefix = create_buffer_with_data::<B, f32>(&context, &prefix);- let beta = create_buffer_with_data::<B, f32>(&context, &beta);- let a = create_buffer_with_data::<B, f32>(&context, &a_f32);- let a_inv = create_buffer_with_data::<B, f32>(&context, &a_inv);- let mut u = create_buffer::<B, f32>(&context, u_len);-- let mut command_buffer = context.create_command_buffer(None, None).expect("command buffer");- kernel.encode(- use_h0.then_some(&kh0),- &v,- &prefix,- &beta,- &a,- &a_inv,- &mut u,- case.batch_size,- case.tree_size,- case.num_v_heads,- case.head_v_dim,- &mut command_buffer,- );- block_on(command_buffer.end_encoding().submit().wait_until_completed()).unwrap();-- buffer_to_vec(&u)-}--#[uzu_test]-fn test_tree_update_solve_cases() {- for &bv in BVS {- for &use_h0 in &[true, false] {- for &case in CASES {- let expected = run_case::<Cpu, f32>(case, bv, use_h0, DataType::F32, |x| x);- for_each_non_cpu_backend!(|B| {- let output = run_case::<B, f32>(case, bv, use_h0, DataType::F32, |x| x);- assert_eq_float(&expected, &output, 1e-4, &format!("{} BV{bv} h0={use_h0}", case.name));- });- }- }- }-- for &bv in BVS {- for &use_h0 in &[true, false] {- for &case in CASES {- let expected = run_case::<Cpu, bf16>(case, bv, use_h0, DataType::BF16, bf16::from_f32);- for_each_non_cpu_backend!(|B| {- let output = run_case::<B, bf16>(case, bv, use_h0, DataType::BF16, bf16::from_f32);- assert_eq_float(&expected, &output, 5e-4, &format!("{} BV{bv} h0={use_h0}", case.name));- });- }- }- }-}diff --git a/crates/uzu-engine/unit/backends/common/kernel/gdn/tree_verify/tree_verify_test.rs b/crates/uzu-engine/unit/backends/common/kernel/gdn/tree_verify/tree_verify_test.rsnew file mode 100644index 000000000..b99a2e72f--- /dev/null+++ b/crates/uzu-engine/unit/backends/common/kernel/gdn/tree_verify/tree_verify_test.rs@@ -0,0 +1,89 @@+#![cfg(backend = "metal")]++use futures::executor::block_on;+use half::bf16;+use num_traits::Float;+use uzu_engine_macros::uzu_test;++use crate::{+ backends::{+ common::{+ Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferPending, Context, Kernels,+ kernel::TreeVerifyKernel,+ },+ cpu::Cpu,+ metal::Metal,+ },+ data_type::DataTypeRepr,+ tests::{+ assert::assert_eq_float,+ helpers::{buffer_to_vec, create_buffer_with_data},+ },+};++const K_HEADS: usize = 2;+const V_HEADS: usize = 6;+const DIM: usize = 128;+const CONV_DIM: usize = 5;+const PROJ_DIM: usize = CONV_DIM + V_HEADS * DIM + 3;++fn wave<T: Float>(+ len: usize,+ step: f32,+ amplitude: f32,+ bias: f32,+) -> Vec<T> {+ (0..len).map(|i| T::from((i as f32 * step).sin() * amplitude + bias).unwrap()).collect()+}++fn run<B: Backend, T: DataTypeRepr + Float>(n: usize) -> Vec<T> {+ let context = block_on(B::Context::new()).expect("context");+ let kernel = <B::Kernels as Kernels>::TreeVerifyKernel::new(&context, T::data_type(), (n > 16) as u32 + 1).unwrap();+ let q = create_buffer_with_data::<B, T>(&context, &wave(n * K_HEADS * DIM, 0.017, 0.2, 0.01));+ let k = create_buffer_with_data::<B, T>(&context, &wave(n * K_HEADS * DIM, 0.019, 0.18, -0.02));+ let v = create_buffer_with_data::<B, T>(&context, &wave(n * V_HEADS * DIM, 0.023, 0.5, 0.03));+ let in_proj = create_buffer_with_data::<B, T>(&context, &wave(n * PROJ_DIM, 0.031, 1.5, -0.1));+ let log_decay = create_buffer_with_data::<B, f32>(&context, &wave(n * V_HEADS, 1.7, 0.2, -0.25));+ let beta = create_buffer_with_data::<B, f32>(&context, &wave(n * V_HEADS, 0.013, 0.2, 0.45));+ let h0 = create_buffer_with_data::<B, f32>(&context, &wave(V_HEADS * DIM * DIM, 0.007, 0.05, -0.01));+ let norm_weight = create_buffer_with_data::<B, f32>(&context, &wave(DIM, 0.05, 0.2, 1.0));+ // Caterpillar: odd nodes are leaves, even nodes the spine, so every 16-node block branches.+ let trie: Vec<u32> = (0..n as u32).flat_map(|node| [node, [n as u32 - 1, node][node as usize % 2], 0]).collect();+ let trie = create_buffer_with_data::<B, u32>(&context, &trie);+ let mut o = create_buffer_with_data::<B, T>(&context, &vec![T::zero(); n * V_HEADS * DIM]);++ let mut command_buffer = context.create_command_buffer(None, None).unwrap();+ kernel.encode(+ &q,+ &k,+ &v,+ &trie,+ &log_decay,+ &beta,+ &h0,+ &in_proj,+ &norm_weight,+ &mut o,+ 1.0,+ 1e-6,+ 1,+ n as u32,+ K_HEADS as u32,+ V_HEADS as u32,+ DIM as u32,+ DIM as u32,+ CONV_DIM as u32,+ PROJ_DIM as u32,+ &mut command_buffer,+ );+ block_on(command_buffer.end_encoding().submit().wait_until_completed()).unwrap();+ buffer_to_vec(&o)+}++#[uzu_test]+fn test_tree_verify() {+ for n in [1, 16, 17, 32] {+ assert_eq_float(&run::<Cpu, f32>(n), &run::<Metal, f32>(n), 5e-4, "f32");+ assert_eq_float(&run::<Cpu, bf16>(n), &run::<Metal, bf16>(n), 1.6e-2, "bf16");+ }+}diff --git a/crates/uzu-engine/unit/encodable_block/delta_net_tree_verify_bench.rs b/crates/uzu-engine/unit/encodable_block/delta_net_tree_verify_bench.rsindex ae23c82f3..485d21534 100644--- a/crates/uzu-engine/unit/encodable_block/delta_net_tree_verify_bench.rs+++ b/crates/uzu-engine/unit/encodable_block/delta_net_tree_verify_bench.rs@@ -38,10 +38,13 @@ fn bench_delta_net_tree_verify(c: &mut Criterion) {<<Metal as Backend>::Kernels as Kernels>::DeltaNetTreeVerify::new(context.as_ref(), &arguments).unwrap();eprintln!("GDN tree verification PSO construction: {:.2} ms", started.elapsed().as_secs_f64() * 1e3);let h0 = create_buffer_with_data::<Metal, f32>(&context, &vec![0.001; V_HEADS * HEAD_DIM * HEAD_DIM]);+ let norm_weight = create_buffer_with_data::<Metal, f32>(&context, &vec![1.0; HEAD_DIM]);+ const CONV_DIM: usize = 2 * K_HEADS * HEAD_DIM + V_HEADS * HEAD_DIM;+ const TOTAL_PROJ_DIM: usize = CONV_DIM + V_HEADS * HEAD_DIM + 2 * V_HEADS;let mut group = c.benchmark_group("Metal/Kernel/GDNTreeVerify");group.sample_size(20).warm_up_time(Duration::from_millis(300)).measurement_time(Duration::from_secs(1));- for tree_size in [32usize, 49, 64, 128] {+ for tree_size in [16usize, 32] {let mut command_buffer = context.create_command_buffer(None, None).unwrap();let mut q = command_buffer.allocate_scratch(tree_size * K_HEADS * HEAD_DIM * size_of::<bf16>()).unwrap();q.copyin(&vec![bf16::from_f32(0.01); tree_size * K_HEADS * HEAD_DIM]);@@ -49,6 +52,8 @@ fn bench_delta_net_tree_verify(c: &mut Criterion) {k.copyin(&vec![bf16::from_f32(0.02); tree_size * K_HEADS * HEAD_DIM]);let mut v = command_buffer.allocate_scratch(tree_size * V_HEADS * HEAD_DIM * size_of::<bf16>()).unwrap();v.copyin(&vec![bf16::from_f32(0.03); tree_size * V_HEADS * HEAD_DIM]);+ let mut in_proj = command_buffer.allocate_scratch(tree_size * TOTAL_PROJ_DIM * size_of::<bf16>()).unwrap();+ in_proj.copyin(&vec![bf16::from_f32(0.04); tree_size * TOTAL_PROJ_DIM]);let trie_nodes = (0..tree_size as u32).flat_map(|node| [node, tree_size as u32 - 1, node]).collect::<Vec<_>>();let mut log_decay = command_buffer.allocate_scratch(tree_size * V_HEADS * size_of::<f32>()).unwrap();log_decay.copyin(&vec![-0.01f32; tree_size * V_HEADS]);@@ -68,6 +73,11 @@ fn bench_delta_net_tree_verify(c: &mut Criterion) {log_decay: &log_decay,beta: &beta,h0: &h0,+ in_proj: &in_proj,+ norm_weight: &norm_weight,+ conv_dim: CONV_DIM as u32,+ total_proj_dim: TOTAL_PROJ_DIM as u32,+ norm_epsilon: 1e-6,tree_size: tree_size as u32,},command_buffer,