Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

Fuse gdn tree verify (#996)

CCYeh43e9fe3dd88d
diff --git a/crates/uzu-engine/src/backends/cpu/kernel/gdn/tree_verify/mod.rs b/crates/uzu-engine/src/backends/cpu/kernel/gdn/tree_verify/mod.rs
index e4b9b6cf5..f7359462b 100644
--- a/crates/uzu-engine/src/backends/cpu/kernel/gdn/tree_verify/mod.rs
+++ b/crates/uzu-engine/src/backends/cpu/kernel/gdn/tree_verify/mod.rs
@@ -1,6 +1,3 @@
pub mod conv_scan;
-pub mod out;
-pub mod prefix;
pub mod state_advance;
-pub mod tree_gram;
-pub mod tree_update_solve;
+pub mod tree_verify;
diff --git a/crates/uzu-engine/src/backends/cpu/kernel/gdn/tree_verify/out.rs b/crates/uzu-engine/src/backends/cpu/kernel/gdn/tree_verify/out.rs
deleted file mode 100644
index fde31f84b..000000000
--- a/crates/uzu-engine/src/backends/cpu/kernel/gdn/tree_verify/out.rs
+++ /dev/null
@@ -1,80 +0,0 @@
-use half::bf16;
-use num_traits::Float;
-use uzu_engine_macros::kernel;
-
-use crate::data_type::DataTypeRepr;
-
-#[kernel(BuildTreeOut)]
-#[variants(QKT, f32, bf16)]
-#[variants(OutputT, f32, bf16)]
-pub fn build_tree_out<QKT: DataTypeRepr + Float, OutputT: DataTypeRepr + Float>(
- q: *const QKT,
- prefix: *const f32,
- qkd: *const f32,
- u: *const f32,
- #[optional(use_h0)] h0: Option<*const f32>,
- o: *mut OutputT,
- scale: f32,
- batch_size: u32,
- tree_size: u32,
- qk_heads: u32,
- value_heads: u32,
- head_k_dim: u32,
- head_v_dim: u32,
- #[allow(unused)]
- #[specialize]
- use_mxu: bool,
- #[allow(unused)]
- #[specialize]
- transposed_h0: bool,
- #[specialize] use_h0: bool,
-) {
- let batch_size = batch_size as usize;
- let tree_size = tree_size as usize;
- let qk_heads = qk_heads as usize;
- let value_heads = value_heads as usize;
- let head_k_dim = head_k_dim as usize;
- let head_v_dim = head_v_dim as usize;
- let value_heads_per_qk_head = value_heads / qk_heads;
- let h0 = h0.unwrap_or(std::ptr::null());
-
- for batch in 0..batch_size {
- for value_head in 0..value_heads {
- let qk_head = value_head / value_heads_per_qk_head;
- let q_head_base = (batch * tree_size * qk_heads + qk_head) * head_k_dim;
- let prefix_base = batch * tree_size * value_heads + value_head;
- let qkd_base = (batch * value_heads + value_head) * tree_size * tree_size;
- let u_base = (batch * value_heads + value_head) * tree_size * head_v_dim;
- let out_base = ((batch * tree_size) * value_heads + value_head) * head_v_dim;
-
- for row in 0..tree_size {
- let q_row = q_head_base + row * qk_heads * head_k_dim;
- for value_col in 0..head_v_dim {
- let mut acc = 0.0f32;
-
- if use_h0 {
- let h0_base = ((batch * value_heads + value_head) * head_v_dim + value_col) * head_k_dim;
- let mut dot = 0.0f32;
- for dim in 0..head_k_dim {
- let qv = unsafe { (*q.add(q_row + dim)).to_f32().unwrap() };
- let hv = unsafe { *h0.add(h0_base + dim) };
- dot += qv * hv;
- }
- let prefix_row = unsafe { *prefix.add(prefix_base + row * value_heads) };
- acc += prefix_row.exp() * scale * dot;
- }
-
- for col in 0..tree_size {
- let qkd_value = unsafe { *qkd.add(qkd_base + row * tree_size + col) };
- let u_value = unsafe { *u.add(u_base + col * head_v_dim + value_col) };
- acc += qkd_value * u_value;
- }
-
- unsafe {
- *o.add(out_base + row * value_heads * head_v_dim + value_col) = OutputT::from(acc).unwrap();
- }
- }
- }
- }
- }
-}
diff --git a/crates/uzu-engine/src/backends/cpu/kernel/gdn/tree_verify/prefix.rs b/crates/uzu-engine/src/backends/cpu/kernel/gdn/tree_verify/prefix.rs
deleted file mode 100644
index 37c2b0332..000000000
--- a/crates/uzu-engine/src/backends/cpu/kernel/gdn/tree_verify/prefix.rs
+++ /dev/null
@@ -1,39 +0,0 @@
-use uzu_engine_macros::kernel;
-
-use crate::backends::common::gpu_types::trie::TrieNode;
-
-#[kernel(BuildTreePrefix)]
-pub fn build_tree_prefix(
- trie: *const TrieNode,
- log_decay: *const f32,
- prefix: *mut f32,
- batch_size: u32,
- tree_size: u32,
- value_heads: u32,
-) {
- let batch_size = batch_size as usize;
- let tree_size = tree_size as usize;
- let value_heads = value_heads as usize;
-
- for batch in 0..batch_size {
- let trie_batch = batch * tree_size;
- let batch_offset = batch * tree_size * value_heads;
-
- for row in 0..tree_size {
- let row_idx = row as u32;
- for head in 0..value_heads {
- let mut sum = 0.0f32;
- for col in 0..tree_size {
- // col is an ancestor-or-self of row iff row falls in col's
- // DFS subtree interval [trie_start, trie_end].
- let node = unsafe { *trie.add(trie_batch + col) };
- if row_idx < node.trie_start || row_idx > node.trie_end {
- continue;
- }
- sum += unsafe { *log_decay.add(batch_offset + col * value_heads + head) };
- }
- unsafe { *prefix.add(batch_offset + row * value_heads + head) = sum };
- }
- }
- }
-}
diff --git a/crates/uzu-engine/src/backends/cpu/kernel/gdn/tree_verify/tree_gram.rs b/crates/uzu-engine/src/backends/cpu/kernel/gdn/tree_verify/tree_gram.rs
deleted file mode 100644
index 8488e2ed7..000000000
--- a/crates/uzu-engine/src/backends/cpu/kernel/gdn/tree_verify/tree_gram.rs
+++ /dev/null
@@ -1,158 +0,0 @@
-use half::bf16;
-use num_traits::Float;
-use uzu_engine_macros::kernel;
-
-use crate::{backends::common::gpu_types::trie::TrieNode, data_type::DataTypeRepr};
-
-const BLOCK: usize = 16;
-
-#[kernel(BuildTreeGram)]
-#[variants(T, f32, bf16)]
-#[variants(USE_MXU, false)]
-pub fn build_tree_gram<T: DataTypeRepr + Float, const USE_MXU: bool>(
- q: *const T,
- k: *const T,
- trie: *const TrieNode,
- prefix: *const f32,
- beta: *const f32,
- #[optional(use_h0)] h0: Option<*const f32>,
- a_packed: *mut f32,
- qkd: *mut f32,
- a_inv: *mut f32,
- #[optional(use_h0)] kh0: Option<*mut f32>,
- scale: f32,
- batch_size: u32,
- tree_size: u32,
- k_heads: u32,
- value_heads: u32,
- head_k_dim: u32,
- head_v_dim: u32,
- #[specialize] use_h0: bool,
-) {
- let batch_size = batch_size as usize;
- let tree_size = tree_size as usize;
- let k_heads = k_heads as usize;
- let value_heads = value_heads as usize;
- let head_k_dim = head_k_dim as usize;
- let head_v_dim = head_v_dim as usize;
-
- let value_heads_per_key_head = value_heads / k_heads;
- let num_blocks = tree_size.div_ceil(BLOCK);
- let num_col_pairs = num_blocks.div_ceil(2);
-
- for batch in 0..batch_size {
- for hv in 0..value_heads {
- let hk = hv / value_heads_per_key_head;
-
- // qkd[row, col] = scale * exp(prefix[row] - prefix[col]) * dot(q[row], k[col])
- // for ancestor-or-self, else 0. Dense [T, T].
- let mat_base = (batch * value_heads + hv) * tree_size * tree_size;
- for row in 0..tree_size {
- let prefix_row = unsafe { *prefix.add((batch * tree_size + row) * value_heads + hv) };
- let row_off = ((batch * tree_size + row) * k_heads + hk) * head_k_dim;
- for col in 0..tree_size {
- let out = mat_base + row * tree_size + col;
- let node = unsafe { *trie.add(batch * tree_size + col) };
- if (row as u32) < node.trie_start || (row as u32) > node.trie_end {
- unsafe { *qkd.add(out) = 0.0 };
- continue;
- }
- let col_off = ((batch * tree_size + col) * k_heads + hk) * head_k_dim;
- let mut qk = 0.0f32;
- for d in 0..head_k_dim {
- qk += unsafe { (*q.add(row_off + d)).to_f32().unwrap() }
- * unsafe { (*k.add(col_off + d)).to_f32().unwrap() };
- }
- let prefix_col = unsafe { *prefix.add((batch * tree_size + col) * value_heads + hv) };
- unsafe { *qkd.add(out) = (prefix_row - prefix_col).exp() * scale * qk };
- }
- }
-
- // A[row, col] = beta[row] * exp(prefix[row] - prefix[col]) * dot(k[row], k[col])
- // for proper ancestors, else 0; packed [NB, ceil(NB/2), 16, 32] tiles,
- // only pairs touching the block lower triangle (same tiles the GPU writes).
- let a_base = (batch * value_heads + hv) * num_blocks * num_col_pairs * BLOCK * 2 * BLOCK;
- for block in 0..num_blocks {
- for pair in 0..=block / 2 {
- let tile_base = a_base + (block * num_col_pairs + pair) * BLOCK * 2 * BLOCK;
- for local_row in 0..BLOCK {
- for local_col in 0..2 * BLOCK {
- let row = block * BLOCK + local_row;
- let col = pair * 2 * BLOCK + local_col;
- let mut value = 0.0f32;
- if row != col && row < tree_size && col < tree_size {
- let node = unsafe { *trie.add(batch * tree_size + col) };
- if (row as u32) >= node.trie_start && (row as u32) <= node.trie_end {
- let row_off = ((batch * tree_size + row) * k_heads + hk) * head_k_dim;
- let col_off = ((batch * tree_size + col) * k_heads + hk) * head_k_dim;
- let mut kk = 0.0f32;
- for d in 0..head_k_dim {
- kk += unsafe { (*k.add(row_off + d)).to_f32().unwrap() }
- * unsafe { (*k.add(col_off + d)).to_f32().unwrap() };
- }
- let prefix_row =
- unsafe { *prefix.add((batch * tree_size + row) * value_heads + hv) };
- let prefix_col =
- unsafe { *prefix.add((batch * tree_size + col) * value_heads + hv) };
- let beta_row = unsafe { *beta.add((batch * tree_size + row) * value_heads + hv) };
- value = beta_row * (prefix_row - prefix_col).exp() * kk;
- }
- }
- unsafe { *a_packed.add(tile_base + local_row * 2 * BLOCK + local_col) = value };
- }
- }
- }
- }
-
- // a_inv = (I + A_diag)^-1 per block, compact [NB, 16, 16], identity-padded;
- // A is f32 so the diagonal values are read back from a_packed unrounded.
- for block in 0..num_blocks {
- let block_size = BLOCK.min(tree_size - block * BLOCK);
- let block_base = ((batch * value_heads + hv) * num_blocks + block) * BLOCK * BLOCK;
- let diag_tile = a_base + (block * num_col_pairs + block / 2) * BLOCK * 2 * BLOCK;
- let diag_col = (block % 2) * BLOCK;
- for row in 0..BLOCK {
- for col in 0..BLOCK {
- unsafe {
- *a_inv.add(block_base + row * BLOCK + col) = if row == col {
- 1.0
- } else {
- 0.0
- }
- };
- }
- }
- for row in 0..block_size {
- for col in 0..row {
- let mut sum = 0.0f32;
- for prev_row in col..row {
- sum += unsafe { *a_packed.add(diag_tile + row * 2 * BLOCK + diag_col + prev_row) }
- * unsafe { *a_inv.add(block_base + prev_row * BLOCK + col) };
- }
- unsafe { *a_inv.add(block_base + row * BLOCK + col) = -sum };
- }
- }
- }
-
- // kh0[b, token, hv, dv] = dot(k[b, token, hk], h0[b, hv, dv])
- if !use_h0 {
- continue;
- }
- let h0 = h0.unwrap();
- let kh0 = kh0.unwrap();
- let h0_head = (batch * value_heads + hv) * head_v_dim * head_k_dim;
- for token in 0..tree_size {
- let k_off = ((batch * tree_size + token) * k_heads + hk) * head_k_dim;
- let kh0_off = ((batch * tree_size + token) * value_heads + hv) * head_v_dim;
- for dv in 0..head_v_dim {
- let mut sum = 0.0f32;
- for d in 0..head_k_dim {
- sum += unsafe { (*k.add(k_off + d)).to_f32().unwrap() }
- * unsafe { *h0.add(h0_head + dv * head_k_dim + d) };
- }
- unsafe { *kh0.add(kh0_off + dv) = sum };
- }
- }
- }
- }
-}
diff --git a/crates/uzu-engine/src/backends/cpu/kernel/gdn/tree_verify/tree_update_solve.rs b/crates/uzu-engine/src/backends/cpu/kernel/gdn/tree_verify/tree_update_solve.rs
deleted file mode 100644
index fcc41335f..000000000
--- a/crates/uzu-engine/src/backends/cpu/kernel/gdn/tree_verify/tree_update_solve.rs
+++ /dev/null
@@ -1,122 +0,0 @@
-use half::bf16;
-use num_traits::Float;
-use uzu_engine_macros::kernel;
-
-use crate::data_type::DataTypeRepr;
-
-/// Solves chunked GDN update coefficients: `(I + A) U = beta * (v - exp(prefix) * kh0)`.
-/// `kh0 = k @ h0^T` is precomputed by BuildTreeGram.
-#[kernel(TreeUpdateSolve)]
-#[variants(T, f32, bf16)]
-#[variants(BV, 16, 32)]
-pub fn tree_update_solve<T: DataTypeRepr + Float, const BV: u32>(
- #[optional(use_h0)] kh0: Option<*const f32>,
- v: *const T,
- prefix: *const f32,
- beta: *const f32,
- a_packed: *const f32,
- a_inv: *const f32,
- u: *mut f32,
- batch_size: u32,
- tree_size: u32,
- value_heads: u32,
- head_v_dim: u32,
- #[specialize] use_h0: bool,
-) {
- let batch_size = batch_size as usize;
- let tree_size = tree_size as usize;
- let value_heads = value_heads as usize;
- let head_v_dim = head_v_dim as usize;
- let block_size = 16usize;
-
- if batch_size == 0 || tree_size == 0 || value_heads == 0 || head_v_dim == 0 {
- return;
- }
-
- let num_blocks = tree_size.div_ceil(block_size);
- let num_col_pairs = num_blocks.div_ceil(2);
-
- let kh0 = kh0.unwrap_or(std::ptr::null());
- for batch in 0..batch_size {
- for hv in 0..value_heads {
- let batch_value_head_idx = batch * value_heads + hv;
-
- for block in 0..num_blocks {
- let token_base = block * block_size;
- let mut acc = vec![0.0f32; block_size * head_v_dim];
-
- for local_token in 0..block_size {
- let token = token_base + local_token;
- if token >= tree_size {
- continue;
- }
-
- let prefix_idx = (batch * tree_size + token) * value_heads + hv;
- let beta_val = unsafe { *beta.add(prefix_idx) };
- let decay_from_h0 = unsafe { *prefix.add(prefix_idx) }.exp();
-
- for dv in 0..head_v_dim {
- let v_idx = (((batch * tree_size + token) * value_heads + hv) * head_v_dim) + dv;
- let v_val = unsafe { (*v.add(v_idx)).to_f32().unwrap() };
-
- let kh0_val = if use_h0 {
- unsafe { *kh0.add(v_idx) }
- } else {
- 0.0
- };
-
- acc[local_token * head_v_dim + dv] = beta_val * (v_val - decay_from_h0 * kh0_val);
- }
- }
-
- for prev_token in 0..token_base {
- let prev_block = prev_token / block_size;
- let prev_local = prev_token % block_size;
- for local_token in 0..block_size {
- let token = token_base + local_token;
- if token >= tree_size {
- continue;
- }
-
- // A is packed [B*HV, NB, ceil(NB/2), 16, 32] block-pair tiles.
- let a_idx = ((batch_value_head_idx * num_blocks + block) * num_col_pairs + prev_block / 2)
- * (block_size * 2 * block_size)
- + local_token * (2 * block_size)
- + (prev_block % 2) * block_size
- + prev_local;
- let a_val = unsafe { *a_packed.add(a_idx) };
- for dv in 0..head_v_dim {
- let u_prev_idx = (batch_value_head_idx * tree_size + prev_token) * head_v_dim + dv;
- acc[local_token * head_v_dim + dv] -= a_val * unsafe { *u.add(u_prev_idx) };
- }
- }
- }
-
- for local_token in 0..block_size {
- let token = token_base + local_token;
- if token >= tree_size {
- continue;
- }
-
- for dv in 0..head_v_dim {
- let mut sum = 0.0f32;
- for local_prev_token in 0..block_size {
- let source_token = token_base + local_prev_token;
- if source_token >= tree_size {
- continue;
- }
- // Ainv is compact [B * HV, ceil(T/BT), BT, BT] diagonal blocks.
- let inv_idx = ((batch_value_head_idx * num_blocks + block) * block_size + local_token)
- * block_size
- + local_prev_token;
- sum += unsafe { *a_inv.add(inv_idx) } * acc[local_prev_token * head_v_dim + dv];
- }
-
- let u_idx = (batch_value_head_idx * tree_size + token) * head_v_dim + dv;
- unsafe { *u.add(u_idx) = sum };
- }
- }
- }
- }
- }
-}
diff --git a/crates/uzu-engine/src/backends/cpu/kernel/gdn/tree_verify/tree_verify.rs b/crates/uzu-engine/src/backends/cpu/kernel/gdn/tree_verify/tree_verify.rs
new file mode 100644
index 000000000..8c59896af
--- /dev/null
+++ b/crates/uzu-engine/src/backends/cpu/kernel/gdn/tree_verify/tree_verify.rs
@@ -0,0 +1,105 @@
+use half::bf16;
+use num_traits::Float;
+use uzu_engine_macros::kernel;
+
+use crate::{backends::common::gpu_types::trie::TrieNode, data_type::DataTypeRepr};
+
+#[kernel(TreeVerify)]
+#[variants(T, f32, bf16)]
+#[variants(NB, 1, 2)]
+pub fn tree_verify<T: DataTypeRepr + Float, const NB: u32>(
+ q: *const T,
+ k: *const T,
+ v: *const T,
+ trie: *const TrieNode,
+ log_decay: *const f32,
+ beta: *const f32,
+ h0: *const f32,
+ in_proj: *const T,
+ norm_weight: *const f32,
+ o: *mut T,
+ scale: f32,
+ norm_epsilon: f32,
+ batch_size: u32,
+ tree_size: u32,
+ num_k_heads: u32,
+ num_v_heads: u32,
+ head_k_dim: u32,
+ head_v_dim: u32,
+ conv_dim: u32,
+ total_proj_dim: u32,
+) {
+ let batch_size = batch_size as usize;
+ let tree_size = tree_size as usize;
+ let num_k_heads = num_k_heads as usize;
+ let num_v_heads = num_v_heads as usize;
+ let head_k_dim = head_k_dim as usize;
+ let head_v_dim = head_v_dim as usize;
+ assert!(tree_size <= 16 * NB as usize);
+
+ let load = |ptr: *const T, index: usize| unsafe { (*ptr.add(index)).to_f32().unwrap() };
+ for batch in 0..batch_size {
+ for hv_idx in 0..num_v_heads {
+ let hk_idx = hv_idx / (num_v_heads / num_k_heads);
+ let qk_off = |token: usize| ((batch * tree_size + token) * num_k_heads + hk_idx) * head_k_dim;
+ let scalar = |token: usize| (batch * tree_size + token) * num_v_heads + hv_idx;
+ let nodes: Vec<TrieNode> = (0..tree_size).map(|c| unsafe { *trie.add(batch * tree_size + c) }).collect();
+ let in_subtree =
+ |row: usize, col: usize| (row as u32) >= nodes[col].trie_start && (row as u32) <= nodes[col].trie_end;
+
+ let prefix: Vec<f32> = (0..tree_size)
+ .map(|row| {
+ (0..tree_size).filter(|&c| in_subtree(row, c)).map(|c| unsafe { *log_decay.add(scalar(c)) }).sum()
+ })
+ .collect();
+ let dot =
+ |a: &dyn Fn(usize) -> f32, b: &dyn Fn(usize) -> f32| (0..head_k_dim).map(|d| a(d) * b(d)).sum::<f32>();
+ let mut a = vec![0.0f32; tree_size * tree_size];
+ let mut qkd = vec![0.0f32; tree_size * tree_size];
+ for row in 0..tree_size {
+ for col in 0..tree_size {
+ if !in_subtree(row, col) {
+ continue;
+ }
+ let decay = (prefix[row] - prefix[col]).exp();
+ qkd[row * tree_size + col] =
+ scale * decay * dot(&|d| load(q, qk_off(row) + d), &|d| load(k, qk_off(col) + d));
+ if row != col {
+ let kk = dot(&|d| load(k, qk_off(row) + d), &|d| load(k, qk_off(col) + d));
+ a[row * tree_size + col] = unsafe { *beta.add(scalar(row)) } * decay * kk;
+ }
+ }
+ }
+
+ let h0_base = (batch * num_v_heads + hv_idx) * head_v_dim * head_k_dim;
+ for dv in 0..head_v_dim {
+ let h0_row = |d: usize| unsafe { *h0.add(h0_base + dv * head_k_dim + d) };
+ let mut u = vec![0.0f32; tree_size];
+ for row in 0..tree_size {
+ let kh0 = dot(&|d| load(k, qk_off(row) + d), &h0_row);
+ let v_value = load(v, scalar(row) * head_v_dim + dv);
+ let rhs = unsafe { *beta.add(scalar(row)) } * (v_value - prefix[row].exp() * kh0);
+ u[row] = rhs - (0..row).map(|c| a[row * tree_size + c] * u[c]).sum::<f32>();
+ }
+ for row in 0..tree_size {
+ let qh0 = dot(&|d| load(q, qk_off(row) + d), &h0_row);
+ let value = prefix[row].exp() * scale * qh0
+ + (0..tree_size).map(|c| qkd[row * tree_size + c] * u[c]).sum::<f32>();
+ unsafe { *o.add(scalar(row) * head_v_dim + dv) = T::from(value).unwrap() };
+ }
+ }
+ }
+ }
+ super::super::norm_gate::delta_net_norm_gate::<T>(
+ o,
+ in_proj,
+ norm_weight,
+ num_v_heads as u32,
+ head_v_dim as u32,
+ (num_v_heads * head_v_dim) as u32,
+ conv_dim,
+ total_proj_dim,
+ norm_epsilon,
+ (batch_size * tree_size) as u32,
+ );
+}
diff --git a/crates/uzu-engine/src/backends/metal/kernel/gdn/chunked/precompute.metal b/crates/uzu-engine/src/backends/metal/kernel/gdn/chunked/precompute.metal
index df8e83249..c60a91ca2 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/gdn/chunked/precompute.metal
+++ b/crates/uzu-engine/src/backends/metal/kernel/gdn/chunked/precompute.metal
@@ -60,8 +60,6 @@ KERNEL(DeltaNetChunkedPrecompute)(
using GramRightFragment = OperandFragment<float, 1, GRAM_COL_FRAGMENTS, Ops, ReadTranspose>;
using TileFragment = Fragment<float, ROW_FRAGMENTS, ROW_FRAGMENTS, Ops>;
- using MatrixFragment = OperandFragment<float, ROW_FRAGMENTS, ROW_FRAGMENTS, Ops>;
- using ValueFragment = OperandFragment<bfloat, ROW_FRAGMENTS, ROW_FRAGMENTS, Ops>;
static_assert(HEAD_K_DIM % Ops::FRAGMENT_ROWS == 0, "HEAD_K_DIM must align to fragment K");
constexpr uint num_blocks = CHUNK_SIZE / BLOCK_SIZE;
@@ -211,50 +209,20 @@ KERNEL(DeltaNetChunkedPrecompute)(
threadgroup_barrier(mem_flags::mem_threadgroup);
// Solve this simdgroup's 16-column slice of T; blocks above the slice are zero, solved blocks stay in registers.
- const uint slice_base = block_idx * BLOCK_SIZE;
- device bfloat* t_head = t_out + head_base * CHUNK_SIZE * CHUNK_SIZE + slice_base;
- ValueFragment t_blocks[num_blocks];
- METAL_PRAGMA_UNROLL
- for (uint b = 0; b < num_blocks; ++b) {
- TileFragment t_block;
- t_block.clear();
- if (b >= block_idx) {
- const threadgroup float* b_tiles = a_tiles + tile_base(b) * (BLOCK_SIZE * COL_TILE);
- TileFragment acc_t;
- acc_t.map_coords(lane, [&](short local_row, short local_col, float) UZU_ALWAYS_INLINE {
- return b * BLOCK_SIZE + uint(local_row) == slice_base + uint(local_col) ? 1.0f : 0.0f;
- });
- METAL_PRAGMA_UNROLL
- for (uint prev_block_idx = 0; prev_block_idx < b; ++prev_block_idx) {
- if (prev_block_idx >= block_idx) {
- MatrixFragment a_frag;
- a_frag.load_from(
- lane,
- fragment_source(
- b_tiles + (prev_block_idx / 2) * (BLOCK_SIZE * COL_TILE) + (prev_block_idx % 2) * BLOCK_SIZE,
- COL_TILE
- )
- );
- a_frag.map([](float value) UZU_ALWAYS_INLINE { return -value; });
- fragment_mma(acc_t, a_frag, t_blocks[prev_block_idx]);
- }
- }
- MatrixFragment inv_frag;
- inv_frag.load_from(
- lane,
- fragment_source(
- static_cast<const threadgroup float*>(diag_inverses + b * (BLOCK_SIZE * BLOCK_SIZE)),
- BLOCK_SIZE
- )
- );
- fragment_mma(t_block, inv_frag, acc_t);
- }
- t_block.store(lane, t_head + b * BLOCK_SIZE * CHUNK_SIZE, CHUNK_SIZE);
- METAL_PRAGMA_UNROLL
- for (ushort e = 0; e < ValueFragment::ELEMENTS_PER_FRAGMENT; ++e) {
- t_blocks[b].elements()[e] = bfloat(t_block.elements()[e]);
- }
- }
+ device bfloat* t_head = t_out + head_base * CHUNK_SIZE * CHUNK_SIZE + block_idx * BLOCK_SIZE;
+ solve_lower_triangular_slice<BLOCK_SIZE, num_blocks, bfloat>(
+ [&](uint b) UZU_ALWAYS_INLINE {
+ return static_cast<const threadgroup float*>(a_tiles + tile_base(b) * (BLOCK_SIZE * COL_TILE));
+ },
+ [](const threadgroup float* row, uint m)
+ UZU_ALWAYS_INLINE { return row + (m / 2) * (BLOCK_SIZE * COL_TILE) + (m % 2) * BLOCK_SIZE; },
+ COL_TILE,
+ diag_inverses,
+ [&](uint b, thread TileFragment& t_block)
+ UZU_ALWAYS_INLINE { t_block.store(lane, t_head + b * BLOCK_SIZE * CHUNK_SIZE, CHUNK_SIZE); },
+ block_idx,
+ lane
+ );
threadgroup_barrier(mem_flags::mem_threadgroup);
}
diff --git a/crates/uzu-engine/src/backends/metal/kernel/gdn/common/tri_inv.h b/crates/uzu-engine/src/backends/metal/kernel/gdn/common/tri_inv.h
index 947fc5922..9960d5bdc 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/gdn/common/tri_inv.h
+++ b/crates/uzu-engine/src/backends/metal/kernel/gdn/common/tri_inv.h
@@ -1,9 +1,12 @@
#pragma once
#include "../../common/defines.h"
+#include "../../matmul/common/fragment.h"
+#include "../../matmul/common/simdgroup_fragment_ops.h"
#include <metal_stdlib>
using namespace metal;
+using namespace uzu::matmul;
template <uint BLOCK, uint A_STRIDE = BLOCK, typename OutPtr>
METAL_FUNC void invert_lower_triangular_block(
@@ -39,3 +42,51 @@ METAL_FUNC void invert_lower_triangular_block(
a_inv_block[row * BLOCK + col] = inverse_col[row];
}
}
+
+template <uint BLOCK, uint NB, typename CarryT, typename ARow, typename ABlock, typename StoreBlock>
+METAL_FUNC void solve_lower_triangular_slice(
+ ARow a_row,
+ ABlock a_block,
+ const uint a_stride,
+ const threadgroup float* diag_inverses,
+ StoreBlock store_block,
+ const uint slice,
+ const uint lane
+) {
+ using Ops = SimdgroupFragmentOps;
+ constexpr ushort ROW_FRAGMENTS = BLOCK / Ops::FRAGMENT_ROWS;
+ using TileFragment = Fragment<float, ROW_FRAGMENTS, ROW_FRAGMENTS, Ops>;
+ using MatrixFragment = OperandFragment<float, ROW_FRAGMENTS, ROW_FRAGMENTS, Ops>;
+ using CarryFragment = OperandFragment<CarryT, ROW_FRAGMENTS, ROW_FRAGMENTS, Ops>;
+
+ CarryFragment t_blocks[NB];
+ METAL_PRAGMA_UNROLL
+ for (uint b = 0; b < NB; ++b) {
+ TileFragment t_block;
+ t_block.clear();
+ if (b >= slice) {
+ const threadgroup float* b_tiles = a_row(b);
+ TileFragment acc_t;
+ acc_t.map_coords(lane, [&](short local_row, short local_col, float) UZU_ALWAYS_INLINE {
+ return b * BLOCK + uint(local_row) == slice * BLOCK + uint(local_col) ? 1.0f : 0.0f;
+ });
+ METAL_PRAGMA_UNROLL
+ for (uint m = 0; m < b; ++m) {
+ if (m >= slice) {
+ MatrixFragment a_frag;
+ a_frag.load_from(lane, fragment_source(a_block(b_tiles, m), a_stride));
+ a_frag.map([](float value) UZU_ALWAYS_INLINE { return -value; });
+ fragment_mma(acc_t, a_frag, t_blocks[m]);
+ }
+ }
+ MatrixFragment inv_frag;
+ inv_frag.load_from(lane, fragment_source(diag_inverses + b * (BLOCK * BLOCK), BLOCK));
+ fragment_mma(t_block, inv_frag, acc_t);
+ }
+ store_block(b, t_block);
+ METAL_PRAGMA_UNROLL
+ for (ushort e = 0; e < CarryFragment::ELEMENTS_PER_FRAGMENT; ++e) {
+ t_blocks[b].elements()[e] = CarryT(t_block.elements()[e]);
+ }
+ }
+}
diff --git a/crates/uzu-engine/src/backends/metal/kernel/gdn/tree_verify.rs b/crates/uzu-engine/src/backends/metal/kernel/gdn/tree_verify.rs
index aa830c87d..20f3d6f64 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/gdn/tree_verify.rs
+++ b/crates/uzu-engine/src/backends/metal/kernel/gdn/tree_verify.rs
@@ -1,68 +1,23 @@
-use metal::MTLGPUFamily;
-
use crate::{
backends::{
common::{
Backend, CommandBufferEncoding, Kernels,
- kernel::{
- BuildTreeGramKernel, BuildTreeOutKernel, BuildTreePrefixKernel, TreeUpdateSolveKernel,
- delta_net_tree_verify::DeltaNetTreeVerify,
- },
+ kernel::{TreeVerifyKernel, delta_net_tree_verify::DeltaNetTreeVerify},
},
metal::{
- Metal,
- command_buffer::MetalCommandBufferEncoding,
- context::{LARGE_MIN_GPU_CORES, MetalContext},
- error::MetalError,
+ Metal, command_buffer::MetalCommandBufferEncoding, context::MetalContext, error::MetalError,
kernel::MetalKernels,
},
},
- data_type::DataType,
encodable_block::mixer::delta_net::tree_verify::{TreeVerifyEncodeArguments, TreeVerifyNewArguments},
};
const TOKEN_BLOCK: u32 = 16;
-const BLOCK_PAIR_WIDTH: u32 = 2 * TOKEN_BLOCK;
-const INNER_DATA_TYPE: DataType = DataType::F32;
-
-struct Layout {
- tree_size: u32,
- num_blocks: u32,
- num_block_pairs: u32,
- num_v_heads: u32,
- head_v_dim: u32,
-}
-
-impl Layout {
- const fn new(
- tree_size: u32,
- arguments: &TreeVerifyNewArguments,
- ) -> Self {
- let num_blocks = tree_size.div_ceil(TOKEN_BLOCK);
- Self {
- tree_size,
- num_blocks,
- num_block_pairs: num_blocks.div_ceil(2),
- num_v_heads: arguments.num_v_heads,
- head_v_dim: arguments.head_v_dim,
- }
- }
-
- const fn a_packed_shape(&self) -> [u32; 5] {
- [self.num_v_heads, self.num_blocks, self.num_block_pairs, TOKEN_BLOCK, BLOCK_PAIR_WIDTH]
- }
-
- const fn a_inverse_shape(&self) -> [u32; 4] {
- [self.num_v_heads, self.num_blocks, TOKEN_BLOCK, TOKEN_BLOCK]
- }
-}
+const MAX_TREE_SIZE: u32 = 2 * TOKEN_BLOCK;
pub struct MetalDeltaNetTreeVerify {
arguments: TreeVerifyNewArguments,
- prefix: <MetalKernels as Kernels>::BuildTreePrefixKernel,
- gram: <MetalKernels as Kernels>::BuildTreeGramKernel,
- solve: <MetalKernels as Kernels>::TreeUpdateSolveKernel,
- out: <MetalKernels as Kernels>::BuildTreeOutKernel,
+ kernels: [<MetalKernels as Kernels>::TreeVerifyKernel; 2],
}
impl DeltaNetTreeVerify for MetalDeltaNetTreeVerify {
@@ -76,23 +31,12 @@ impl DeltaNetTreeVerify for MetalDeltaNetTreeVerify {
context: &MetalContext,
arguments: &TreeVerifyNewArguments,
) -> Result<Self, MetalError> {
- let use_mxu = arguments.data_type == DataType::BF16 && context.supports_mxu;
- let transposed_h0 = !use_mxu
- && context.gpu_core_count < LARGE_MIN_GPU_CORES
- && context.apple_gpu_family <= MTLGPUFamily::Apple8;
Ok(Self {
arguments: *arguments,
- prefix: <MetalKernels as Kernels>::BuildTreePrefixKernel::new(context)?,
- gram: <MetalKernels as Kernels>::BuildTreeGramKernel::new(context, arguments.data_type, use_mxu, true)?,
- solve: <MetalKernels as Kernels>::TreeUpdateSolveKernel::new(context, arguments.data_type, 32, true)?,
- out: <MetalKernels as Kernels>::BuildTreeOutKernel::new(
- context,
- arguments.data_type,
- arguments.data_type,
- use_mxu,
- transposed_h0,
- true,
- )?,
+ kernels: [
+ <MetalKernels as Kernels>::TreeVerifyKernel::new(context, arguments.data_type, 1)?,
+ <MetalKernels as Kernels>::TreeVerifyKernel::new(context, arguments.data_type, 2)?,
+ ],
})
}
@@ -101,79 +45,32 @@ impl DeltaNetTreeVerify for MetalDeltaNetTreeVerify {
arguments: TreeVerifyEncodeArguments<'_, Metal>,
command_buffer: &mut MetalCommandBufferEncoding,
) -> Result<<Metal as Backend>::ScratchBuffer, MetalError> {
- let layout = Layout::new(arguments.tree_size, &self.arguments);
- let mut prefix =
- command_buffer.allocate_scratch_for_shape(&[layout.tree_size, layout.num_v_heads], INNER_DATA_TYPE)?;
- let mut a_packed = command_buffer.allocate_scratch_for_shape(&layout.a_packed_shape(), INNER_DATA_TYPE)?;
- let mut qkd = command_buffer
- .allocate_scratch_for_shape(&[layout.num_v_heads, layout.tree_size, layout.tree_size], INNER_DATA_TYPE)?;
- let mut a_inverse = command_buffer.allocate_scratch_for_shape(&layout.a_inverse_shape(), INNER_DATA_TYPE)?;
- let mut kh0 = command_buffer
- .allocate_scratch_for_shape(&[layout.tree_size, layout.num_v_heads, layout.head_v_dim], INNER_DATA_TYPE)?;
- let mut u = command_buffer
- .allocate_scratch_for_shape(&[layout.num_v_heads, layout.tree_size, layout.head_v_dim], INNER_DATA_TYPE)?;
+ assert!(arguments.tree_size <= MAX_TREE_SIZE, "GDN tree verification supports at most {MAX_TREE_SIZE} nodes");
let mut output = command_buffer.allocate_scratch_for_shape(
- &[layout.tree_size, layout.num_v_heads, layout.head_v_dim],
+ &[arguments.tree_size, self.arguments.num_v_heads, self.arguments.head_v_dim],
self.arguments.data_type,
)?;
-
- self.prefix.encode(
- arguments.trie,
- arguments.log_decay,
- &mut prefix,
- 1,
- arguments.tree_size,
- self.arguments.num_v_heads,
- command_buffer,
- );
- self.gram.encode(
+ self.kernels[(arguments.tree_size > TOKEN_BLOCK) as usize].encode(
arguments.q,
arguments.k,
- arguments.trie,
- &prefix,
- arguments.beta,
- Some(arguments.h0),
- &mut a_packed,
- &mut qkd,
- &mut a_inverse,
- Some(&mut kh0),
- 1.0,
- 1,
- arguments.tree_size,
- self.arguments.num_k_heads,
- self.arguments.num_v_heads,
- self.arguments.head_k_dim,
- self.arguments.head_v_dim,
- command_buffer,
- );
- self.solve.encode(
- Some(&kh0),
arguments.v,
- &prefix,
+ arguments.trie,
+ arguments.log_decay,
arguments.beta,
- &a_packed,
- &a_inverse,
- &mut u,
- 1,
- arguments.tree_size,
- self.arguments.num_v_heads,
- self.arguments.head_v_dim,
- command_buffer,
- );
- self.out.encode(
- arguments.q,
- &prefix,
- &qkd,
- &u,
- Some(arguments.h0),
+ arguments.h0,
+ arguments.in_proj,
+ arguments.norm_weight,
&mut output,
1.0,
+ arguments.norm_epsilon,
1,
arguments.tree_size,
self.arguments.num_k_heads,
self.arguments.num_v_heads,
self.arguments.head_k_dim,
self.arguments.head_v_dim,
+ arguments.conv_dim,
+ arguments.total_proj_dim,
command_buffer,
);
Ok(output)
diff --git a/crates/uzu-engine/src/backends/metal/kernel/gdn/tree_verify/out.metal b/crates/uzu-engine/src/backends/metal/kernel/gdn/tree_verify/out.metal
deleted file mode 100644
index e00132f9d..000000000
--- a/crates/uzu-engine/src/backends/metal/kernel/gdn/tree_verify/out.metal
+++ /dev/null
@@ -1,156 +0,0 @@
-#include <metal_stdlib>
-#include "../../common/defines.h"
-#include "../../common/dsl.h"
-#include "../../common/thread_context.h"
-#include "../../matmul/common/fragment.h"
-#include "../../matmul/common/mxu_fragment/ops.h"
-#include "../../matmul/common/simdgroup_fragment_ops.h"
-
-using namespace metal;
-using namespace uzu::matmul;
-
-#define MATMUL_MXU_ROWS 16u
-#define MATMUL_SIMDGROUP_ROWS 8u
-#define MATMUL_COLS 32u
-#define SIMDGROUPS_PER_TG 4u
-
-// Build tree output for one (batch, value-head, row, value) tile.
-//
-// q: [B, T, Hg, K], with query head hq = hv / (HV / Hg)
-// prefix: [B, T, HV] path log-decay prefix from BuildTreePrefix
-// qkd: [B, HV, T, T] from BuildTreeGram
-// u: [B, HV, T, V]
-// h0: [B, HV, V, K]
-//
-// o[row, value] = exp(prefix[row]) * scale * dot(q[row], h0[value])
-// + sum_j qkd[row, j] * u[j, value]
-template <typename QKT, typename OutputT, bool use_mxu, bool transposed_h0>
-VARIANTS(QKT, float, bfloat)
-VARIANTS(OutputT, float, bfloat)
-VARIANTS(use_mxu, false, true)
-VARIANTS(transposed_h0, false, true)
-CONSTRAINT(!(use_mxu && transposed_h0))
-PUBLIC KERNEL(BuildTreeOut)(
- const device QKT* q,
- const device float* prefix,
- const device float* qkd,
- const device float* u,
- const device float* h0 OPTIONAL(use_h0),
- device OutputT* o,
- constant const float& scale,
- constant const uint& batch_size,
- constant const uint& tree_size,
- constant const uint& qk_heads,
- constant const uint& value_heads,
- constant const uint& head_k_dim,
- constant const uint& head_v_dim,
- threadgroup uint u_tile_scratch OPTIONAL(transposed_h0)[MATMUL_SIMDGROUP_ROWS * MATMUL_COLS],
- const bool use_h0 SPECIALIZE,
- const ThreadContext thread_context,
- const uint v_tile_idx GROUPS(head_v_dim.div_ceil(MATMUL_COLS)),
- const uint row_tile_group_idx GROUPS(tree_size.div_ceil(
- if use_mxu {
- SIMDGROUPS_PER_TG * MATMUL_MXU_ROWS
- } else {
- SIMDGROUPS_PER_TG * MATMUL_SIMDGROUP_ROWS
- }
- )),
- const uint batch_value_head_idx GROUPS(batch_size * value_heads),
- const uint tid THREADS(SIMDGROUPS_PER_TG * METAL_SIMD_SIZE)
-) {
- // Relaxed MPP diverges from FP32 across layers.
- using Ops = metal::conditional_t<use_mxu, MxuStrictFragmentOps, SimdgroupFragmentOps>;
- using H0Read = metal::conditional_t<transposed_h0, ReadTranspose, ReadDirect>;
- constexpr ushort ROWS = Ops::FRAGMENT_ROWS;
- constexpr ushort COL_FRAGMENTS = MATMUL_COLS / Ops::FRAGMENT_COLS;
- using AccFragment = Fragment<float, 1, COL_FRAGMENTS, Ops>;
- using QFragment = OperandFragment<float, 1, 1, Ops>;
- using H0Fragment = OperandFragment<float, 1, COL_FRAGMENTS, Ops, H0Read>;
- using QkdFragment = OperandFragment<float, 1, 1, Ops>;
- using UFragment = OperandFragment<float, 1, COL_FRAGMENTS, Ops>;
- threadgroup float* u_tile = reinterpret_cast<threadgroup float*>(u_tile_scratch);
-
- const uint batch_idx = batch_value_head_idx / value_heads;
- const uint value_head_idx = batch_value_head_idx - batch_idx * value_heads;
- const uint qk_head_idx = value_head_idx / (value_heads / qk_heads);
- const uint row_base = (row_tile_group_idx * SIMDGROUPS_PER_TG + thread_context.simdgroup_index) * ROWS;
- const uint value_base = v_tile_idx * MATMUL_COLS;
-
- const short valid_rows = short(min(uint(ROWS), tree_size - min(row_base, tree_size)));
- const short valid_cols = short(min(MATMUL_COLS, head_v_dim - min(value_base, head_v_dim)));
-
- AccFragment acc;
- acc.clear();
-
- if (use_h0) {
- const uint q_base = (batch_idx * tree_size * qk_heads + row_base * qk_heads + qk_head_idx) * head_k_dim;
- const uint h0_base = ((batch_idx * value_heads + value_head_idx) * head_v_dim + value_base) * head_k_dim;
- for (uint k0 = 0; k0 < head_k_dim; k0 += ROWS) {
- const short valid_k = short(min(uint(ROWS), head_k_dim - k0));
- QFragment q_frag;
- H0Fragment h0_frag;
- q_frag.load_from(
- thread_context.simd_lane_id,
- fragment_source(q + q_base + k0, int(qk_heads * head_k_dim)).bounded(valid_rows, valid_k)
- );
- if constexpr (transposed_h0) {
- h0_frag.load_from(
- thread_context.simd_lane_id,
- fragment_source(h0 + h0_base + k0, int(head_k_dim)).bounded(valid_cols, valid_k)
- );
- } else {
- h0_frag.load_from(
- thread_context.simd_lane_id,
- fragment_source(h0 + h0_base + k0, 1, int(head_k_dim)).bounded(valid_k, valid_cols)
- );
- }
- fragment_mma(acc, q_frag, h0_frag);
- }
-
- acc.map_coords(thread_context.simd_lane_id, [&](short row, short, float value) {
- const uint global_row = row_base + uint(row);
- const float row_scale =
- global_row < tree_size
- ? exp(prefix[(batch_idx * tree_size + global_row) * value_heads + value_head_idx]) * scale
- : 0.0f;
- return value * row_scale;
- });
- }
-
- const uint qkd_base = (batch_idx * value_heads + value_head_idx) * tree_size * tree_size + row_base * tree_size;
- const uint u_base = (batch_idx * value_heads + value_head_idx) * tree_size * head_v_dim + value_base;
- for (uint j0 = 0; j0 < tree_size; j0 += ROWS) {
- const short valid_j = short(min(uint(ROWS), tree_size - j0));
- QkdFragment qkd_frag;
- UFragment u_frag;
- qkd_frag.load_from(
- thread_context.simd_lane_id,
- fragment_source(qkd + qkd_base + j0, int(tree_size)).bounded(valid_rows, valid_j)
- );
- if constexpr (transposed_h0) {
- for (uint index = tid; index < uint(ROWS) * MATMUL_COLS; index += SIMDGROUPS_PER_TG * METAL_SIMD_SIZE) {
- const uint local_j = index / MATMUL_COLS;
- const uint local_col = index - local_j * MATMUL_COLS;
- const bool in_bounds = local_j < uint(valid_j) && local_col < uint(valid_cols);
- u_tile[index] = in_bounds ? u[u_base + (j0 + local_j) * head_v_dim + local_col] : 0.0f;
- }
- threadgroup_barrier(mem_flags::mem_threadgroup);
- u_frag.load_from(
- thread_context.simd_lane_id,
- fragment_source(u_tile, int(MATMUL_COLS)).bounded(valid_j, valid_cols)
- );
- fragment_mma(acc, qkd_frag, u_frag);
- threadgroup_barrier(mem_flags::mem_threadgroup);
- } else {
- u_frag.load_from(
- thread_context.simd_lane_id,
- fragment_source(u + u_base + j0 * head_v_dim, int(head_v_dim)).bounded(valid_j, valid_cols)
- );
- fragment_mma(acc, qkd_frag, u_frag);
- }
- }
-
- device OutputT* out_tile =
- o + ((batch_idx * tree_size + row_base) * value_heads + value_head_idx) * head_v_dim + value_base;
- acc.store_safe(thread_context.simd_lane_id, out_tile, int(value_heads * head_v_dim), short2(valid_cols, valid_rows));
-}
diff --git a/crates/uzu-engine/src/backends/metal/kernel/gdn/tree_verify/prefix.metal b/crates/uzu-engine/src/backends/metal/kernel/gdn/tree_verify/prefix.metal
deleted file mode 100644
index 6fe092adb..000000000
--- a/crates/uzu-engine/src/backends/metal/kernel/gdn/tree_verify/prefix.metal
+++ /dev/null
@@ -1,50 +0,0 @@
-#include <metal_stdlib>
-#include "../../common/defines.h"
-#include "../../common/dsl.h"
-#include "../../common/thread_context.h"
-#include "../../generated/trie.h"
-
-#define ROWS_PER_THREADGROUP 4u
-
-using namespace metal;
-using namespace uzu::trie;
-
-// trie: [B, T] TrieNode — col is an ancestor-or-self of row iff
-// trie[col].trie_start <= row <= trie[col].trie_end.
-// log_decay: [B, T, HV]
-// prefix: [B, T, HV] fp32
-PUBLIC KERNEL(BuildTreePrefix)(
- const device TrieNode* trie,
- const device float* log_decay,
- device float* prefix,
- constant const uint& batch_size,
- constant const uint& tree_size,
- constant const uint& value_heads,
- const ThreadContext thread_context,
- const uint batch_idx GROUPS(batch_size),
- const uint row_idx GROUPS(tree_size),
- const uint head_group_idx GROUPS(value_heads.div_ceil(ROWS_PER_THREADGROUP)),
- const uint tid THREADS(ROWS_PER_THREADGROUP* METAL_SIMD_SIZE)
-) {
- const uint simd_group = thread_context.simdgroup_index;
- const uint lane_id = thread_context.simd_lane_id;
- const uint head_idx = head_group_idx * ROWS_PER_THREADGROUP + simd_group;
- if (head_idx >= value_heads) {
- return;
- }
-
- const uint batch_offset = batch_idx * tree_size * value_heads;
- const uint trie_offset = batch_idx * tree_size;
- float partial = 0.0f;
- for (uint col_idx = lane_id; col_idx < tree_size; col_idx += METAL_SIMD_SIZE) {
- const TrieNode node = trie[trie_offset + col_idx];
- if (row_idx >= node.trie_start && row_idx <= node.trie_end) {
- partial += log_decay[batch_offset + col_idx * value_heads + head_idx];
- }
- }
-
- const float sum = simd_sum(partial);
- if (lane_id == 0) {
- prefix[batch_offset + row_idx * value_heads + head_idx] = sum;
- }
-}
diff --git a/crates/uzu-engine/src/backends/metal/kernel/gdn/tree_verify/tree_gram.metal b/crates/uzu-engine/src/backends/metal/kernel/gdn/tree_verify/tree_gram.metal
deleted file mode 100644
index a8741f1fc..000000000
--- a/crates/uzu-engine/src/backends/metal/kernel/gdn/tree_verify/tree_gram.metal
+++ /dev/null
@@ -1,291 +0,0 @@
-#include <metal_stdlib>
-#include "../../common/defines.h"
-#include "../../common/dsl.h"
-#include "../../common/thread_context.h"
-#include "../../generated/trie.h"
-#include "../../matmul/common/fragment.h"
-#include "../../matmul/common/mxu_fragment/ops.h"
-#include "../../matmul/common/simdgroup_fragment_ops.h"
-#include "../common/gram.h"
-#include "../common/tri_inv.h"
-
-using namespace metal;
-using namespace uzu::matmul;
-using namespace uzu::trie;
-
-#define ROW_TILE 16u
-#define COL_TILE 32u
-#define NUM_SIMDGROUPS 4u
-#define INVALID_ROW 0xffffffffu
-
-// kh0 = k @ h0[batch]^T for one row-tile, walking head_v_dim in COL_TILE
-// chunks from dv_start by dv_stride.
-template <typename T, typename Ops, typename AccFragment, typename LeftFragment, typename RightFragment>
-METAL_FUNC void tree_kh0(
- const device T* k_rows,
- const device float* h0,
- device float* kh0,
- const uint batch_idx,
- const uint value_head_idx,
- const uint row_base,
- const uint tile_rows,
- const bool full_rows,
- const uint tree_size,
- const uint value_heads,
- const uint head_k_dim,
- const uint head_v_dim,
- const uint qk_stride,
- const uint dv_start,
- const uint dv_stride,
- const ushort lane
-) {
- const device float* h0_head = h0 + (batch_idx * value_heads + value_head_idx) * head_v_dim * head_k_dim;
- device float* kh0_rows =
- kh0 + (batch_idx * tree_size + row_base) * value_heads * head_v_dim + value_head_idx * head_v_dim;
- const uint kh0_row_stride = value_heads * head_v_dim;
-
- for (uint dv_base = dv_start; dv_base < head_v_dim; dv_base += dv_stride) {
- const uint tile_dvs = min(uint(COL_TILE), head_v_dim - dv_base);
- AccFragment kh0_acc;
- kh0_acc.clear();
-
- const device float* h0_tile = h0_head + dv_base * head_k_dim;
- const bool full_dvs = tile_dvs == COL_TILE;
-
- for (uint kb = 0; kb < head_k_dim; kb += Ops::FRAGMENT_ROWS) {
- LeftFragment k_left;
- RightFragment h0_right;
- k_left.load_maybe_bounded(lane, k_rows + kb, qk_stride, full_rows, tile_rows, Ops::FRAGMENT_ROWS);
- h0_right.load_maybe_bounded(lane, h0_tile + kb, head_k_dim, full_dvs, tile_dvs, Ops::FRAGMENT_ROWS);
- fragment_mma(kh0_acc, k_left, h0_right);
- }
-
- kh0_acc.store_maybe_bounded(
- lane,
- kh0_rows + dv_base,
- kh0_row_stride,
- full_rows && full_dvs,
- short2(tile_dvs, tile_rows)
- );
- }
-}
-
-// Builds the tree-verify gram products for one (batch, value-head, row-tile,
-// col-tile-group) strip; each of the threadgroup's simdgroups owns one 16x32
-// column tile (above-diagonal tiles are just zero-filled in qkd) and the row's
-// simdgroups share the kh0 dv chunks round-robin:
-// a_packed: A[row, col] = beta[row] * exp(prefix[row] - prefix[col]) * dot(k[row], k[col])
-// for proper ancestors (trie interval test), packed block-pair tiles
-// [B*HV, NB, ceil(NB/2), 16, 32] f32; only lower-triangle tiles written
-// qkd: scale * exp(prefix[row] - prefix[col]) * dot(q[row], k[col]) for
-// ancestor-or-self, dense [B*HV, T, T] f32
-// a_inv: (I + A)^-1 per diagonal block, compact [B*HV, NB, 16, 16]
-// kh0: k @ h0[batch]^T, [B, T, HV, head_v_dim] f32
-template <typename T, bool USE_MXU>
-VARIANTS(T, float, bfloat)
-VARIANTS(USE_MXU, false, true)
-PUBLIC KERNEL(BuildTreeGram)(
- const device T* q,
- const device T* k,
- const device TrieNode* trie,
- const device float* prefix,
- const device float* beta,
- const device float* h0 OPTIONAL(use_h0),
- device float* a_packed,
- device float* qkd,
- device float* a_inv,
- device float* kh0 OPTIONAL(use_h0),
- constant const float& scale,
- constant const uint& batch_size,
- constant const uint& tree_size,
- constant const uint& k_heads,
- constant const uint& value_heads,
- constant const uint& head_k_dim,
- constant const uint& head_v_dim,
- threadgroup float diag_a_tile[ROW_TILE * ROW_TILE],
- threadgroup float row_prefix[ROW_TILE],
- threadgroup float row_beta[ROW_TILE],
- threadgroup uint row_token[ROW_TILE],
- threadgroup float col_prefix[NUM_SIMDGROUPS * COL_TILE],
- threadgroup uint col_trie_start[NUM_SIMDGROUPS * COL_TILE],
- threadgroup uint col_trie_end[NUM_SIMDGROUPS * COL_TILE],
- const bool use_h0 SPECIALIZE,
- const ThreadContext thread_context,
- const uint batch_idx GROUPS(batch_size),
- const uint value_head_idx GROUPS(value_heads),
- const uint tile_idx GROUPS(
- tree_size.div_ceil(ROW_TILE) * tree_size.div_ceil(COL_TILE).div_ceil(NUM_SIMDGROUPS)
- ),
- const uint thread_idx THREADS(NUM_SIMDGROUPS * METAL_SIMD_SIZE)
-) {
- using Ops = metal::conditional_t<USE_MXU, MxuFragmentOps<>, SimdgroupFragmentOps>;
- constexpr ushort ROW_FRAGMENTS = ROW_TILE / Ops::FRAGMENT_ROWS;
- constexpr ushort COL_FRAGMENTS = COL_TILE / Ops::FRAGMENT_COLS;
- using AccFragment = Fragment<float, ROW_FRAGMENTS, COL_FRAGMENTS, Ops>;
- using LeftFragment = OperandFragment<float, ROW_FRAGMENTS, 1, Ops>;
- using RightFragment = OperandFragment<float, 1, COL_FRAGMENTS, Ops, ReadTranspose>;
- static_assert(COL_TILE == METAL_SIMD_SIZE, "COL_TILE must match the SIMD width");
-
- const uint key_head_idx = value_head_idx / (value_heads / k_heads);
- const uint qk_stride = k_heads * head_k_dim;
- const uint qk_base = (batch_idx * tree_size * k_heads + key_head_idx) * head_k_dim;
- const uint prefix_base = batch_idx * tree_size * value_heads + value_head_idx;
- const uint trie_base = batch_idx * tree_size;
- const uint mat_base = (batch_idx * value_heads + value_head_idx) * tree_size * tree_size;
-
- const uint col_tiles = div_ceil(tree_size, COL_TILE);
- const uint col_tile_groups = div_ceil(col_tiles, NUM_SIMDGROUPS);
- const uint row_tile_idx = tile_idx / col_tile_groups;
- const uint col_tile_group_idx = tile_idx - row_tile_idx * col_tile_groups;
- const uint row_base = row_tile_idx * ROW_TILE;
- const uint tile_rows = min(ROW_TILE, tree_size - row_base);
- const bool full_rows = row_base + ROW_TILE <= tree_size;
- const uint in_band_col_tiles = min(row_base / COL_TILE + 1, col_tiles);
- const uint diag_col_tile = row_base / COL_TILE;
- const uint diag_col_offset = row_base - diag_col_tile * COL_TILE;
- const ushort lane = thread_context.simd_lane_id;
- const uint simdgroup_idx = thread_context.simdgroup_index;
- const uint col_slice = simdgroup_idx * COL_TILE;
- threadgroup float* sg_col_prefix = col_prefix + col_slice;
- threadgroup uint* sg_col_trie_start = col_trie_start + col_slice;
- threadgroup uint* sg_col_trie_end = col_trie_end + col_slice;
-
- if (thread_idx < ROW_TILE) {
- const uint token = row_base + thread_idx;
- if (token < tree_size) {
- row_token[thread_idx] = token;
- row_prefix[thread_idx] = prefix[prefix_base + token * value_heads];
- row_beta[thread_idx] = beta[prefix_base + token * value_heads];
- } else {
- row_token[thread_idx] = INVALID_ROW;
- row_prefix[thread_idx] = 0.0f;
- row_beta[thread_idx] = 0.0f;
- }
- }
- threadgroup_barrier(mem_flags::mem_threadgroup);
-
- const uint num_blocks = div_ceil(tree_size, ROW_TILE);
- const uint num_col_pairs = div_ceil(num_blocks, 2u);
-
- const device T* k_rows = k + qk_base + row_base * qk_stride;
- const device T* q_rows = q + qk_base + row_base * qk_stride;
-
- const uint col_tile_idx = col_tile_group_idx * NUM_SIMDGROUPS + simdgroup_idx;
- if (col_tile_idx < in_band_col_tiles) {
- const uint col_base = col_tile_idx * COL_TILE;
- const uint tile_cols = min(COL_TILE, tree_size - col_base);
- const uint tile_base = mat_base + row_base * tree_size + col_base;
-
- const uint col_token = col_base + lane;
- if (col_token < tree_size) {
- const TrieNode node = trie[trie_base + col_token];
- sg_col_trie_start[lane] = node.trie_start;
- sg_col_trie_end[lane] = node.trie_end;
- sg_col_prefix[lane] = prefix[prefix_base + col_token * value_heads];
- } else {
- sg_col_trie_start[lane] = 1;
- sg_col_trie_end[lane] = 0;
- sg_col_prefix[lane] = 0.0f;
- }
-
- const device T* k_cols = k + qk_base + col_base * qk_stride;
- const bool full_cols = col_base + COL_TILE <= tree_size;
-
- AccFragment kk_acc;
- AccFragment qk_acc;
- kk_acc.clear();
- qk_acc.clear();
- for (uint kb = 0; kb < head_k_dim; kb += Ops::FRAGMENT_ROWS) {
- accumulate_dual_gram_tile<AccFragment, LeftFragment, RightFragment>(
- kk_acc,
- qk_acc,
- k_rows + kb,
- q_rows + kb,
- k_cols + kb,
- int(qk_stride),
- ushort(tile_rows),
- ushort(tile_cols),
- Ops::FRAGMENT_ROWS,
- full_rows,
- full_cols,
- lane
- );
- }
-
- const bool has_diag = col_tile_idx == diag_col_tile;
- simdgroup_barrier(mem_flags::mem_threadgroup);
-
- AccFragment::zip_for_each_coord(
- lane,
- [&](ushort local_row, ushort local_col, thread float& kk, thread float& qk_dot) {
- const uint row_idx = row_token[local_row];
- const uint col_idx = col_base + local_col;
- const bool in_subtree = row_idx >= sg_col_trie_start[local_col] && row_idx <= sg_col_trie_end[local_col];
- const float decay = exp(row_prefix[local_row] - sg_col_prefix[local_col]);
- const float a_value = in_subtree && row_idx != col_idx ? row_beta[local_row] * decay * kk : 0.0f;
- const float qkd_value = in_subtree ? scale * decay * qk_dot : 0.0f;
- if (has_diag && local_row < tile_rows && local_col >= diag_col_offset) {
- const uint diag_col = local_col - diag_col_offset;
- if (diag_col < tile_rows) {
- diag_a_tile[local_row * ROW_TILE + diag_col] = a_value;
- }
- }
- kk = a_value;
- qk_dot = qkd_value;
- },
- kk_acc,
- qk_acc
- );
-
- device float* a_tile =
- a_packed +
- (((batch_idx * value_heads + value_head_idx) * num_blocks + row_tile_idx) * num_col_pairs + col_tile_idx) *
- (ROW_TILE * COL_TILE);
- device float* qkd_tile = qkd + tile_base;
- kk_acc.store_maybe_bounded(lane, a_tile, COL_TILE, full_rows, short2(COL_TILE, tile_rows));
- qk_acc.store_maybe_bounded(lane, qkd_tile, tree_size, full_rows && full_cols, short2(tile_cols, tile_rows));
-
- if (has_diag) {
- // diag_a_tile is written and read only by this simdgroup.
- simdgroup_barrier(mem_flags::mem_threadgroup);
- device float* a_inv_block =
- a_inv + ((batch_idx * value_heads + value_head_idx) * num_blocks + row_tile_idx) * (ROW_TILE * ROW_TILE);
- invert_lower_triangular_block<ROW_TILE>(a_inv_block, diag_a_tile, tile_rows, lane);
- }
- } else if (col_tile_idx < col_tiles) {
- // Above-diagonal zero fill; COL_TILE == METAL_SIMD_SIZE, so each
- // lane owns one column and walks rows by pointer increment.
- const uint col_base = col_tile_idx * COL_TILE;
- const uint tile_cols = min(COL_TILE, tree_size - col_base);
- if (lane < tile_cols) {
- device float* qkd_ptr = qkd + mat_base + row_base * tree_size + col_base + lane;
- for (uint row = 0; row < tile_rows; ++row, qkd_ptr += tree_size) {
- *qkd_ptr = 0.0f;
- }
- }
- }
-
- // kh0 dv chunks are distributed round-robin over this row tile's simdgroups
- // (across its col-tile-group threadgroups); chunking also bounds registers.
- if (use_h0) {
- const uint row_simdgroups = col_tile_groups * NUM_SIMDGROUPS;
- tree_kh0<T, Ops, AccFragment, LeftFragment, RightFragment>(
- k_rows,
- h0,
- kh0,
- batch_idx,
- value_head_idx,
- row_base,
- tile_rows,
- full_rows,
- tree_size,
- value_heads,
- head_k_dim,
- head_v_dim,
- qk_stride,
- col_tile_idx * COL_TILE,
- row_simdgroups * COL_TILE,
- lane
- );
- }
-}
diff --git a/crates/uzu-engine/src/backends/metal/kernel/gdn/tree_verify/tree_update_solve.metal b/crates/uzu-engine/src/backends/metal/kernel/gdn/tree_verify/tree_update_solve.metal
deleted file mode 100644
index cd84bc13a..000000000
--- a/crates/uzu-engine/src/backends/metal/kernel/gdn/tree_verify/tree_update_solve.metal
+++ /dev/null
@@ -1,174 +0,0 @@
-#include <metal_stdlib>
-#include "../../common/defines.h"
-#include "../../common/dsl.h"
-#include "../../matmul/common/fragment.h"
-#include "../../matmul/common/simdgroup_fragment_ops.h"
-
-using namespace metal;
-using namespace uzu::matmul;
-
-// Block forward substitution for (I + A) U = beta * (v - exp(prefix) * kh0).
-// Per 16-token block i, serially: acc_i = beta * (v_i - exp(prefix_i) * kh0_i);
-// acc_i -= A[i, j] @ U[j] for all j < i; U[i] = Ainv[i] @ acc_i.
-//
-// Shapes (NB = ceil(T/16)):
-// kh0 [B, T, HV, head_v_dim] f32, from BuildTreeGram (absent when !use_h0)
-// v [B, T, HV, head_v_dim] in T
-// prefix, beta [B, T, HV] f32
-// A [B*HV, NB, ceil(NB/2), 16, 32] f32 packed pair tiles from BuildTreeGram
-// (block (i,j) at pair j/2, column (j%2)*16)
-// Ainv [B*HV, NB, 16, 16] f32 diagonal-block inverses from BuildTreeGram
-// U [B*HV, T, head_v_dim] f32 (output)
-//
-// Grid: one simdgroup per (batch, value head, BV value-dim tile); the history is
-// consumed one packed pair tile ([16,32] A x [32,BV] U) per fragment_mma plus a
-// single-block tail when i is odd.
-// A and Ainv are kept in f32: the (I + A)^-1 forward-substitution cascade is
-// precision-sensitive.
-template <typename T, uint BV>
-VARIANTS(T, float, bfloat)
-VARIANTS(BV, 16, 32)
-PUBLIC KERNEL(TreeUpdateSolve)(
- const device float* kh0 OPTIONAL(use_h0),
- const device T* v,
- const device float* prefix,
- const device float* beta,
- const device float* a_packed,
- const device float* a_inv,
- device float* u,
- constant const uint& batch_size,
- constant const uint& tree_size,
- constant const uint& value_heads,
- constant const uint& head_v_dim,
- const bool use_h0 SPECIALIZE,
- const uint batch_idx GROUPS(batch_size),
- const uint value_head_idx GROUPS(value_heads),
- const uint value_tile_idx GROUPS(head_v_dim.div_ceil(BV)),
- const uint lane_idx THREADS(METAL_SIMD_SIZE)
-) {
- constexpr uint BT = 16;
- // MXU was within 5% of SIMD-group at T=32..512.
- using FragmentOps = SimdgroupFragmentOps;
- constexpr ushort TOKEN_FRAGMENTS = BT / FragmentOps::FRAGMENT_ROWS;
- constexpr ushort VALUE_FRAGMENTS = BV / FragmentOps::FRAGMENT_COLS;
- using TileFragment = Fragment<float, TOKEN_FRAGMENTS, VALUE_FRAGMENTS, FragmentOps>;
- using BlockMatrixFragment = OperandFragment<float, TOKEN_FRAGMENTS, TOKEN_FRAGMENTS, FragmentOps>;
- using ValueTileFragment = OperandFragment<float, TOKEN_FRAGMENTS, VALUE_FRAGMENTS, FragmentOps>;
- using PairMatrixFragment = OperandFragment<float, TOKEN_FRAGMENTS, 2 * TOKEN_FRAGMENTS, FragmentOps>;
- using PairValueFragment = OperandFragment<float, 2 * TOKEN_FRAGMENTS, VALUE_FRAGMENTS, FragmentOps>;
-
- static_assert(BV % FragmentOps::FRAGMENT_COLS == 0, "BV must be a multiple of the fragment column size");
-
- const uint batch_value_head_idx = batch_idx * value_heads + value_head_idx;
-
- const uint value_dim_base = value_tile_idx * BV;
- const uint tile_value_cols = min(BV, head_v_dim - value_dim_base);
- const uint num_blocks = div_ceil(tree_size, BT);
-
- const uint value_token_stride = value_heads * head_v_dim;
- const uint scalar_token_stride = value_heads;
-
- const device float* kh0_head_tile =
- kh0 + (batch_idx * tree_size * value_heads + value_head_idx) * head_v_dim + value_dim_base;
- const device T* value_head_tile =
- v + (batch_idx * tree_size * value_heads + value_head_idx) * head_v_dim + value_dim_base;
- const device float* prefix_head = prefix + batch_idx * tree_size * value_heads + value_head_idx;
- const device float* beta_head = beta + batch_idx * tree_size * value_heads + value_head_idx;
- const uint num_col_pairs = div_ceil(num_blocks, 2u);
- const device float* a_blocks = a_packed + batch_value_head_idx * num_blocks * num_col_pairs * (BT * 2 * BT);
- const device float* a_inv_blocks = a_inv + batch_value_head_idx * num_blocks * (BT * BT);
- device float* u_head_tile = u + batch_value_head_idx * tree_size * head_v_dim + value_dim_base;
-
- for (uint block_idx = 0; block_idx < num_blocks; ++block_idx) {
- const uint row_base = block_idx * BT;
- const uint tile_rows = min(BT, tree_size - row_base);
-
- TileFragment acc;
-
- if (use_h0) {
- const device float* kh0_block = kh0_head_tile + row_base * value_token_stride;
- acc.load_maybe_bounded(
- lane_idx,
- kh0_block,
- value_token_stride,
- tile_rows == BT && tile_value_cols == BV,
- tile_rows,
- tile_value_cols
- );
- } else {
- acc.clear();
- }
-
- // prefix/beta/exp are per-row: one lane computes them, consumers read via simd_shuffle.
- float lane_beta = 0.0f;
- float lane_decay = 0.0f;
- if (lane_idx < tile_rows) {
- const uint token_idx = row_base + lane_idx;
- lane_beta = beta_head[token_idx * scalar_token_stride];
- lane_decay = exp(prefix_head[token_idx * scalar_token_stride]);
- }
-
- TileFragment v_frag;
- const device T* v_block = value_head_tile + row_base * value_token_stride;
- v_frag.load_maybe_bounded(
- lane_idx,
- v_block,
- value_token_stride,
- tile_rows == BT && tile_value_cols == BV,
- tile_rows,
- tile_value_cols
- );
-
- TileFragment::zip_for_each_coord(
- lane_idx,
- [&](ushort local_row_idx, ushort, thread float& acc_value, thread float& v_value) {
- const float beta_row = simd_shuffle(lane_beta, local_row_idx);
- const float decay_row = simd_shuffle(lane_decay, local_row_idx);
- acc_value = beta_row * (v_value - decay_row * acc_value);
- },
- acc,
- v_frag
- );
-
- const uint num_full_pairs = block_idx / 2;
- for (uint pair_idx = 0; pair_idx < num_full_pairs; ++pair_idx) {
- PairMatrixFragment a_frag;
- PairValueFragment u_prev_frag;
-
- const device float* a_pair = a_blocks + (block_idx * num_col_pairs + pair_idx) * (BT * 2 * BT);
- const device float* u_prev_block = u_head_tile + pair_idx * 2 * BT * head_v_dim;
- const bool full = tile_rows == BT && tile_value_cols == BV;
- a_frag.load_maybe_bounded(lane_idx, a_pair, 2 * BT, full, tile_rows, 2 * BT);
- u_prev_frag.load_maybe_bounded(lane_idx, u_prev_block, head_v_dim, full, 2 * BT, tile_value_cols);
- a_frag.map([](float value) { return -value; });
- fragment_mma(acc, a_frag, u_prev_frag);
- }
-
- for (uint prev_block_idx = num_full_pairs * 2; prev_block_idx < block_idx; ++prev_block_idx) {
- const uint prev_row_base = prev_block_idx * BT;
- BlockMatrixFragment a_frag;
- ValueTileFragment u_prev_frag;
-
- const device float* a_block =
- a_blocks + (block_idx * num_col_pairs + prev_block_idx / 2) * (BT * 2 * BT) + (prev_block_idx % 2) * BT;
- a_frag.load_from(lane_idx, fragment_source(a_block, 2 * BT).bounded(tile_rows, BT));
- const device float* u_prev_block = u_head_tile + prev_row_base * head_v_dim;
- u_prev_frag.load_from(lane_idx, fragment_source(u_prev_block, head_v_dim).bounded(BT, tile_value_cols));
- a_frag.map([](float value) { return -value; });
- fragment_mma(acc, a_frag, u_prev_frag);
- }
-
- BlockMatrixFragment inv_frag;
- TileFragment solved;
- solved.clear();
-
- const device float* inv_block = a_inv_blocks + block_idx * (BT * BT);
- inv_frag.load_from(lane_idx, fragment_source(inv_block, BT).bounded(tile_rows, tile_rows));
- fragment_mma(solved, inv_frag, acc);
- solved.store_safe(lane_idx, u_head_tile + row_base * head_v_dim, head_v_dim, short2(tile_value_cols, tile_rows));
-
- // The threadgroup is a single simdgroup, so the block-serial U dependency
- // only needs a simdgroup-scoped device-memory fence.
- simdgroup_barrier(mem_flags::mem_device);
- }
-}
diff --git a/crates/uzu-engine/src/backends/metal/kernel/gdn/tree_verify/tree_verify.metal b/crates/uzu-engine/src/backends/metal/kernel/gdn/tree_verify/tree_verify.metal
new file mode 100644
index 000000000..edb0f38dc
--- /dev/null
+++ b/crates/uzu-engine/src/backends/metal/kernel/gdn/tree_verify/tree_verify.metal
@@ -0,0 +1,291 @@
+#include <metal_stdlib>
+#include "../../activation/activations.h"
+#include "../../common/defines.h"
+#include "../../common/dsl.h"
+#include "../../common/integral_constant.h"
+#include "../../common/thread_context.h"
+#include "../../generated/trie.h"
+#include "../../matmul/common/fragment.h"
+#include "../../matmul/common/simdgroup_fragment_ops.h"
+#include "../common/tri_inv.h"
+
+using namespace metal;
+using namespace uzu::matmul;
+using namespace uzu::trie;
+
+#define HEAD_V_DIM 128u
+#define HEAD_K_DIM 128u
+#define BLOCK_SIZE 16u
+#define OWNERS 8u
+#define OWNER_COLS (HEAD_V_DIM / OWNERS)
+#define OUT_STRIDE (HEAD_V_DIM + 1u)
+
+using GramAccFragment = Fragment<float, 2, 2, SimdgroupFragmentOps>;
+
+// Verifies a whole tree (NB = 1: up to 16 nodes, NB = 2: up to 32) per (batch, value head). Simdgroups 0 .. NB - 1
+// build the grams and invert I + A; the other eight own 16 value columns each.
+template <typename T, uint NB>
+VARIANTS(T, float, bfloat)
+VARIANTS(NB, 1, 2)
+PUBLIC KERNEL(TreeVerify)(
+ const device T* q,
+ const device T* k,
+ const device T* v,
+ const device TrieNode* trie,
+ const device float* log_decay,
+ const device float* beta,
+ const device float* h0,
+ const device T* in_proj,
+ const device float* norm_weight,
+ device T* o,
+ constant const float& scale,
+ constant const float& norm_epsilon,
+ constant const uint& batch_size,
+ constant const uint& tree_size,
+ constant const uint& num_k_heads,
+ constant const uint& num_v_heads,
+ constant const uint& head_k_dim,
+ constant const uint& head_v_dim,
+ constant const uint& conv_dim,
+ constant const uint& total_proj_dim,
+ threadgroup T out_tile[NB * BLOCK_SIZE * OUT_STRIDE],
+ threadgroup float a_tiles[NB * NB * BLOCK_SIZE * BLOCK_SIZE],
+ threadgroup float qkd_tile[NB * NB * BLOCK_SIZE * BLOCK_SIZE],
+ threadgroup float diag_inverses[NB * BLOCK_SIZE * BLOCK_SIZE],
+ threadgroup float row_prefix[NB * BLOCK_SIZE],
+ threadgroup float row_beta[NB * BLOCK_SIZE],
+ threadgroup float row_decay[NB * BLOCK_SIZE],
+ threadgroup uint trie_start[NB * BLOCK_SIZE],
+ threadgroup uint trie_end[NB * BLOCK_SIZE],
+ const ThreadContext thread_context,
+ const uint batch_idx GROUPS(batch_size),
+ const uint hv_idx GROUPS(num_v_heads),
+ const uint thread_idx THREADS((OWNERS + NB) * METAL_SIMD_SIZE)
+) {
+ using Ops = SimdgroupFragmentOps;
+ constexpr uint TOKENS = NB * BLOCK_SIZE;
+ constexpr uint SIMDGROUPS = OWNERS + NB;
+ constexpr ushort TOKEN_FRAGMENTS = TOKENS / Ops::FRAGMENT_COLS;
+ using OwnerFragment = Fragment<float, OWNER_COLS / Ops::FRAGMENT_ROWS, TOKEN_FRAGMENTS, Ops>;
+ using BlockRightFragment = OperandFragment<float, TOKEN_FRAGMENTS, TOKEN_FRAGMENTS, Ops, ReadTranspose>;
+
+ const ushort lane = thread_context.simd_lane_id;
+ const uint simdgroup_idx = thread_context.simdgroup_index;
+ const bool is_owner = simdgroup_idx >= NB;
+ // DeltaNet::new rejects head_k_dim != 128; NB = 1 takes the constant so the K guards fold away.
+ const uint k_dim = NB == 1 ? HEAD_K_DIM : head_k_dim;
+ const int qk_stride = num_k_heads * k_dim;
+ const uint qk_base = (batch_idx * tree_size * num_k_heads + hv_idx / (num_v_heads / num_k_heads)) * k_dim;
+ const uint scalar_base = batch_idx * tree_size * num_v_heads + hv_idx;
+ const uint value_token_stride = num_v_heads * head_v_dim;
+ const uint value_base = batch_idx * tree_size * value_token_stride + hv_idx * head_v_dim;
+ const uint dv_base = (simdgroup_idx - NB) * OWNER_COLS;
+ const short rows_left = short(tree_size);
+ const auto load_rows = [&](auto full, thread auto& frag, const device T* base, const short rows) UZU_ALWAYS_INLINE {
+ auto source = fragment_source(base, qk_stride);
+ if constexpr (decltype(full)::value) {
+ frag.load_from(lane, source);
+ } else {
+ frag.load_from(lane, source.bounded(rows, Ops::FRAGMENT_ROWS));
+ }
+ };
+
+ OwnerFragment v_frag;
+ if (is_owner) {
+ v_frag.load_from(
+ lane,
+ fragment_source(v + value_base + dv_base, 1, int(value_token_stride)).bounded(OWNER_COLS, rows_left)
+ );
+ }
+
+ {
+ const bool lane_valid = lane < tree_size;
+ const TrieNode node = lane_valid ? trie[batch_idx * tree_size + lane] : TrieNode{1, 0, 0};
+ const float lane_log_decay = lane_valid ? log_decay[scalar_base + lane * num_v_heads] : 0.0f;
+ if (simdgroup_idx == 0 && lane < TOKENS) {
+ trie_start[lane] = node.trie_start;
+ trie_end[lane] = node.trie_end;
+ }
+ for (uint row = simdgroup_idx; row < TOKENS; row += SIMDGROUPS) {
+ const float sum = simd_sum((row >= node.trie_start && row <= node.trie_end) ? lane_log_decay : 0.0f);
+ if (lane == 0) {
+ const bool row_valid = row < tree_size;
+ row_prefix[row] = row_valid ? sum : 0.0f;
+ row_decay[row] = row_valid ? exp(sum) : 1.0f;
+ row_beta[row] = row_valid ? beta[scalar_base + row * num_v_heads] : 0.0f;
+ }
+ }
+ }
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+
+ OwnerFragment kh0;
+ OwnerFragment qh0;
+ kh0.clear();
+ qh0.clear();
+ if (!is_owner) {
+ // Owners read all of qkd, so its upper blocks are zeroed.
+ auto build_gram_blocks = [&](auto builder) UZU_ALWAYS_INLINE {
+ constexpr uint B = decltype(builder)::value;
+ GramAccFragment diag_a;
+ GramAccFragment diag_q;
+ GramAccFragment extra;
+ const device T* k_rows = k + qk_base + B * BLOCK_SIZE * qk_stride;
+ const device T* q_rows = q + qk_base + B * BLOCK_SIZE * qk_stride;
+ const short rows = rows_left - short(B * BLOCK_SIZE);
+ uzu::dispatch_bool(rows_left >= short(TOKENS), [&](auto full) UZU_ALWAYS_INLINE {
+ using GramLeftFragment = OperandFragment<float, 2, 1, Ops>;
+ using GramRightFragment = OperandFragment<float, 1, 2, Ops, ReadTranspose>;
+ diag_a.clear();
+ diag_q.clear();
+ extra.clear();
+ _Pragma("clang loop unroll_count(2)") for (ushort step = 0; step < HEAD_K_DIM / Ops::FRAGMENT_ROWS; ++step) {
+ const uint kb = step * Ops::FRAGMENT_ROWS;
+ if (kb < k_dim) {
+ GramLeftFragment k_left;
+ GramLeftFragment q_left;
+ GramRightFragment k_right;
+ load_rows(full, k_left, k_rows + kb, rows);
+ load_rows(full, q_left, q_rows + kb, rows);
+ load_rows(full, k_right, k_rows + kb, rows);
+ fragment_mma(diag_a, k_left, k_right);
+ fragment_mma(diag_q, q_left, k_right);
+ if constexpr (NB == 2 && B == 0) {
+ GramLeftFragment k_below;
+ load_rows(full, k_below, k_rows + BLOCK_SIZE * qk_stride + kb, rows - BLOCK_SIZE);
+ fragment_mma(extra, k_below, k_right);
+ } else if constexpr (NB == 2) {
+ GramRightFragment k_first;
+ load_rows(full, k_first, k + qk_base + kb, rows_left);
+ fragment_mma(extra, q_left, k_first);
+ }
+ }
+ }
+ });
+ auto store = [&](thread GramAccFragment& block, const bool is_qkd, const uint i, const uint j) UZU_ALWAYS_INLINE {
+ block.map_coords(lane, [&](short r, short c, float value) UZU_ALWAYS_INLINE {
+ const uint row = i * BLOCK_SIZE + r;
+ const uint col = j * BLOCK_SIZE + c;
+ const bool in_subtree = row >= trie_start[col] && row <= trie_end[col];
+ const float decay = exp(row_prefix[row] - row_prefix[col]);
+ if (is_qkd) {
+ return in_subtree ? scale * decay * value : 0.0f;
+ }
+ return in_subtree && row != col ? row_beta[row] * decay * value : 0.0f;
+ });
+ block.store(lane, (is_qkd ? qkd_tile : a_tiles) + i * BLOCK_SIZE * TOKENS + j * BLOCK_SIZE, int(TOKENS));
+ };
+ store(diag_a, false, B, B);
+ store(diag_q, true, B, B);
+ if constexpr (NB == 2) {
+ store(extra, B == 1, 1, 0);
+ if constexpr (B == 0) {
+ GramAccFragment zero;
+ zero.clear();
+ zero.store(lane, qkd_tile + BLOCK_SIZE, int(TOKENS));
+ }
+ }
+ };
+ if (simdgroup_idx == 0) {
+ build_gram_blocks(metal::integral_constant<uint, 0>{});
+ } else if constexpr (NB == 2) {
+ build_gram_blocks(metal::integral_constant<uint, 1>{});
+ }
+ simdgroup_barrier(mem_flags::mem_threadgroup);
+ invert_lower_triangular_block<BLOCK_SIZE, TOKENS>(
+ diag_inverses + simdgroup_idx * (BLOCK_SIZE * BLOCK_SIZE),
+ a_tiles + simdgroup_idx * BLOCK_SIZE * (TOKENS + 1),
+ tree_size - simdgroup_idx * BLOCK_SIZE,
+ lane
+ );
+ } else {
+ const device float* h0_rows = h0 + ((batch_idx * num_v_heads + hv_idx) * head_v_dim + dv_base) * k_dim;
+ uzu::dispatch_bool(rows_left >= short(TOKENS), [&](auto full) UZU_ALWAYS_INLINE {
+ using TokenRightFragment = OperandFragment<float, 1, TOKEN_FRAGMENTS, Ops, ReadTranspose>;
+ METAL_PRAGMA_UNROLL
+ for (ushort step = 0; step < HEAD_K_DIM / Ops::FRAGMENT_ROWS; ++step) {
+ const uint kb = step * Ops::FRAGMENT_ROWS;
+ if (kb < k_dim) {
+ OperandFragment<float, 2, 1, Ops> h0_frag;
+ TokenRightFragment k_right;
+ TokenRightFragment q_right;
+ h0_frag.load_from(lane, fragment_source(h0_rows + kb, int(k_dim)));
+ load_rows(full, k_right, k + qk_base + kb, rows_left);
+ load_rows(full, q_right, q + qk_base + kb, rows_left);
+ fragment_mma(kh0, h0_frag, k_right);
+ fragment_mma(qh0, h0_frag, q_right);
+ }
+ }
+ });
+ }
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+
+ // Builder b solves column slice b in place over A; only builder 0 reads the off-diagonal block it overwrites.
+ const threadgroup float* ainv = diag_inverses;
+ if constexpr (NB > 1) {
+ ainv = a_tiles;
+ if (!is_owner) {
+ solve_lower_triangular_slice<BLOCK_SIZE, NB, float>(
+ [&](uint b)
+ UZU_ALWAYS_INLINE { return static_cast<const threadgroup float*>(a_tiles + b * BLOCK_SIZE * TOKENS); },
+ [](const threadgroup float* row, uint m) UZU_ALWAYS_INLINE { return row + m * BLOCK_SIZE; },
+ TOKENS,
+ diag_inverses,
+ [&](uint b, thread GramAccFragment& t_block) UZU_ALWAYS_INLINE {
+ t_block.store(lane, a_tiles + b * BLOCK_SIZE * TOKENS + simdgroup_idx * BLOCK_SIZE, int(TOKENS));
+ },
+ simdgroup_idx,
+ lane
+ );
+ }
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ }
+
+ if (is_owner) {
+ OwnerFragment::zip_for_each_coord(
+ lane,
+ [&](ushort, ushort token, thread float& kh0_value, thread float& v_value)
+ UZU_ALWAYS_INLINE { kh0_value = row_beta[token] * (v_value - row_decay[token] * kh0_value); },
+ kh0,
+ v_frag
+ );
+ BlockRightFragment ainv_right;
+ ainv_right.load_from(lane, fragment_source(ainv, int(TOKENS)));
+ OwnerFragment u;
+ u.clear();
+ fragment_mma(u, kh0, ainv_right);
+
+ qh0.map_coords(lane, [&](short, short token, float value) UZU_ALWAYS_INLINE {
+ return value * (row_decay[token] * scale);
+ });
+ BlockRightFragment qkd_right;
+ qkd_right.load_from(lane, fragment_source(qkd_tile, int(TOKENS)));
+ fragment_mma(qh0, u, qkd_right);
+ // Rounded to T before the norm.
+ qh0.map_coords(lane, [&](short value_col, short token, float value) UZU_ALWAYS_INLINE {
+ out_tile[token * OUT_STRIDE + dv_base + value_col] = static_cast<T>(value);
+ return value;
+ });
+ }
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+
+ using T4 = metal::vec<T, 4>;
+ const uint dv = lane * 4;
+ for (uint token = simdgroup_idx; token < tree_size; token += SIMDGROUPS) {
+ float values[4];
+ float square_sum = 0.0f;
+ METAL_PRAGMA_UNROLL
+ for (uint j = 0; j < 4; ++j) {
+ values[j] = float(out_tile[token * OUT_STRIDE + dv + j]);
+ square_sum += values[j] * values[j];
+ }
+ const float inv_rms = rsqrt(simd_sum(square_sum) / float(head_v_dim) + norm_epsilon);
+ const device T* z_row =
+ in_proj + (batch_idx * tree_size + token) * total_proj_dim + conv_dim + hv_idx * head_v_dim + dv;
+ T4 result;
+ METAL_PRAGMA_UNROLL
+ for (uint j = 0; j < 4; ++j) {
+ result[j] = static_cast<T>(values[j] * inv_rms * norm_weight[dv + j] * activate_silu(float(z_row[j])));
+ }
+ *reinterpret_cast<device T4*>(o + value_base + token * value_token_stride + dv) = result;
+ }
+}
diff --git a/crates/uzu-engine/src/encodable_block/mixer/delta_net.rs b/crates/uzu-engine/src/encodable_block/mixer/delta_net.rs
index 66efc68e6..0011ca082 100644
--- a/crates/uzu-engine/src/encodable_block/mixer/delta_net.rs
+++ b/crates/uzu-engine/src/encodable_block/mixer/delta_net.rs
@@ -410,7 +410,7 @@ impl<B: Backend> DeltaNet<B> {
command_buffer,
);
- let mut delta_output = tree_verify
+ let delta_output = tree_verify
.encode(
TreeVerifyEncodeArguments {
q: &q,
@@ -420,24 +420,16 @@ impl<B: Backend> DeltaNet<B> {
log_decay: &log_decay,
beta: &beta,
h0: &state.ssm_state,
+ in_proj: &tree_projected,
+ norm_weight: &self.norm_weight,
+ conv_dim: self.conv_dim,
+ total_proj_dim: self.total_proj_dim,
+ norm_epsilon: self.norm_epsilon,
tree_size,
},
command_buffer,
)
.map_err(MixerEncodeError::Backend)?;
- self.delta_net_norm_gate.encode(
- &mut delta_output,
- &tree_projected,
- &self.norm_weight,
- self.num_heads,
- self.value_head_dim,
- self.value_dim,
- self.conv_dim,
- self.total_proj_dim,
- self.norm_epsilon,
- tree_size,
- command_buffer,
- );
let output = self.out_projection.encode(delta_output, tree_size, command_buffer)?;
state.suffix_status = Some(DeltaNetSuffixStatus::Tree {
diff --git a/crates/uzu-engine/src/encodable_block/mixer/delta_net/tree_verify.rs b/crates/uzu-engine/src/encodable_block/mixer/delta_net/tree_verify.rs
index 223aceff9..ceb52e9e8 100644
--- a/crates/uzu-engine/src/encodable_block/mixer/delta_net/tree_verify.rs
+++ b/crates/uzu-engine/src/encodable_block/mixer/delta_net/tree_verify.rs
@@ -18,6 +18,11 @@ pub struct TreeVerifyEncodeArguments<'a, B: Backend> {
pub log_decay: &'a B::ScratchBuffer,
pub beta: &'a B::ScratchBuffer,
pub h0: &'a B::GlobalBuffer,
+ pub in_proj: &'a B::ScratchBuffer,
+ pub norm_weight: &'a B::GlobalBuffer,
+ pub conv_dim: u32,
+ pub total_proj_dim: u32,
+ pub norm_epsilon: f32,
pub tree_size: u32,
}
diff --git a/crates/uzu-engine/unit/backends/common/kernel/gdn/tree_verify/gram.md b/crates/uzu-engine/unit/backends/common/kernel/gdn/tree_verify/gram.md
deleted file mode 100644
index 37d968de7..000000000
--- a/crates/uzu-engine/unit/backends/common/kernel/gdn/tree_verify/gram.md
+++ /dev/null
@@ -1,114 +0,0 @@
-# GDN Tree Gram Benchmarks
-
-Local reference on M5 Max, `Hg=16 HV=48 K=128`, BF16 inputs, Criterion
-`sample_size=10`, 2026-06-26.
-
-Run through the existing `uzu_bench` lib harness:
-
-```bash
-cargo bench -p uzu-engine --lib -- "BuildTreeGram/.*/B1_T64"
-cargo bench -p uzu-engine --lib -- "BuildTreeGram"
-```
-
-The current bench uses the repo-standard `iter_encode_loop_named` GPU-time
-harness. The tables below are historical per-dispatch wall-time notes from the
-optimization pass.
-
-## Current
-
-Current Ainv path is the scalar 16x16 column solve. The block-wise variant had
-isolated MXU large-batch wins, but the simpler code is kept.
-
-Final median time after full-tile unsafe load/store fast paths and
-zero-initialized Ainv columns:
-
-| B | Simdgroup T33 | Simdgroup T49 | Simdgroup T64 | Simdgroup T128 | Simdgroup T256 | Simdgroup T512 |
-|---|---------------|---------------|---------------|----------------|----------------|----------------|
-| 1 | 175.24 us | 177.65 us | 173.69 us | 201.87 us | 301.10 us | 679.74 us |
-| 2 | 179.83 us | 183.96 us | 179.33 us | 246.44 us | 426.66 us | 1.1229 ms |
-| 4 | 197.03 us | 218.57 us | 210.26 us | 299.20 us | 689.34 us | 2.0174 ms |
-| 8 | 234.17 us | 262.53 us | 257.28 us | 438.20 us | 1.1379 ms | 3.8258 ms |
-
-| B | MXU T33 | MXU T49 | MXU T64 | MXU T128 | MXU T256 | MXU T512 |
-|---|---------|---------|---------|----------|----------|----------|
-| 1 | 173.09 us | 172.65 us | 173.77 us | 183.50 us | 257.54 us | 448.55 us |
-| 2 | 173.33 us | 173.81 us | 174.56 us | 201.17 us | 291.87 us | 747.12 us |
-| 4 | 175.41 us | 182.72 us | 183.16 us | 258.75 us | 457.23 us | 1.3470 ms |
-| 8 | 195.12 us | 221.28 us | 219.26 us | 327.69 us | 830.41 us | 2.7338 ms |
-
-## Variants
-
-B1/T64 variant check, all correctness-tested:
-
-| Variant | Simdgroup | MXU | Kept |
-|---|---:|---:|---|
-| Baseline | 195.98 us | 180.58 us | no |
-| zip-map only | 188.41 us | 177.34 us | no |
-| zip-map + staged metadata | 182.08 us | 174.79 us | yes |
-| staged + factored exp | 182.08 us | 174.72 us | no, no real gain |
-| upper-tile skip only | 192.61 us | 182.10 us | no, MXU regressed |
-| staged + factored exp + upper skip | 182.82 us | 175.35 us | no |
-| staged metadata + full-tile unsafe loads | 165.53 us | 163.81 us | yes |
-
-Pre epilogue-staging diagonal forward-substitution median time:
-
-| B | MXU T33 | MXU T64 | MXU T128 | MXU T256 |
-|---|---------|---------|----------|----------|
-| 1 | 173.99 us | 179.70 us | 205.85 us | 301.06 us |
-| 2 | 174.77 us | 183.59 us | 250.96 us | 421.89 us |
-| 4 | 179.39 us | 217.04 us | 311.54 us | 672.27 us |
-| 8 | 212.27 us | 258.01 us | 444.67 us | 1.1549 ms |
-
-| B | Simdgroup T33 | Simdgroup T64 | Simdgroup T128 | Simdgroup T256 |
-|---|---------------|---------------|----------------|----------------|
-| 1 | 181.29 us | 193.49 us | 246.31 us | 424.49 us |
-| 2 | 181.72 us | 198.02 us | 300.56 us | 639.55 us |
-| 4 | 199.13 us | 251.43 us | 416.97 us | 1.0192 ms |
-| 8 | 256.82 us | 307.09 us | 653.66 us | 1.8139 ms |
-
-After co-walking `kk_acc`/`qk_acc` and staging per-tile prefix/beta/trie
-metadata:
-
-| B | MXU T64 | MXU T128 | MXU T256 | MXU T512 |
-|---|---------|----------|----------|----------|
-| 1 | 175.22 us | 189.40 us | 259.47 us | 487.78 us |
-| 2 | 178.05 us | 215.88 us | 338.31 us | 822.90 us |
-| 4 | 194.95 us | 264.85 us | 514.95 us | 1.4937 ms |
-| 8 | 228.32 us | 355.72 us | 889.52 us | 2.8619 ms |
-
-| B | Simdgroup T64 | Simdgroup T128 | Simdgroup T256 | Simdgroup T512 |
-|---|---------------|----------------|----------------|----------------|
-| 1 | 183.25 us | 222.01 us | 360.24 us | 883.53 us |
-| 2 | 189.73 us | 274.71 us | 529.30 us | 1.5476 ms |
-| 4 | 232.40 us | 378.08 us | 901.71 us | 2.8888 ms |
-| 8 | 280.61 us | 546.91 us | 1.5795 ms | 5.5700 ms |
-
-## Rejected Ainv Paths
-
-| Variant | Simdgroup B1 T64 | Simdgroup B8 T256 | MXU B1 T64 | MXU B8 T128 | MXU B8 T256 |
-|---|---:|---:|---:|---:|---:|
-| scalar column Ainv | 173.69 us | 1.1379 ms | 173.77 us | 327.69 us | 830.41 us |
-| 8x8 block Ainv + simdgroup matmul | 174.69 us | 1.1317 ms | 175.36 us | 320.86 us | 814.78 us |
-
-| Variant | Simdgroup B1 T64 | Simdgroup B8 T256 | Simdgroup B8 T512 | MXU B1 T64 | MXU B8 T256 |
-|---|---:|---:|---:|---:|---:|
-| 8x8 block Ainv + scalar diagonal halves | 174.69 us | 1.1317 ms | 3.8325 ms | 175.36 us | 814.78 us |
-| 8x8 block Ainv + Neumann diagonal halves | 174.51 us | 1.1749 ms | 4.0259 ms | 173.10 us | 837.44 us |
-
-## Old Neumann Baseline
-
-Speedup over `0600f367` diagonal Neumann. That old kernel has
-`TREE_GRAM_MAX_T=64`, so T128/T256 are current-only.
-
-| B | MXU T33 | MXU T64 | Simdgroup T33 | Simdgroup T64 |
-|---|---------|---------|---------------|---------------|
-| 1 | 1.04x | 1.06x | 1.01x | 1.01x |
-| 2 | 1.04x | 1.23x | 1.03x | 1.17x |
-| 4 | 1.17x | 1.20x | 1.10x | 1.05x |
-| 8 | 1.21x | 1.25x | 1.02x | 1.12x |
-
-Trace `tree_gram_b1_t64_mxu_debug_source.gputrace` showed low occupancy and
-low unit use, not a memory-bandwidth wall: occupancy 7.61%, neural accelerator
-utilization 6.95%, ALU utilization 10.74%, instruction-throughput utilization
-9.52%. The largest source-line buckets were Gram post-processing, then
-fragment loads/MMA, then Ainv fill and the small forward substitution.
diff --git a/crates/uzu-engine/unit/backends/common/kernel/gdn/tree_verify/mod.rs b/crates/uzu-engine/unit/backends/common/kernel/gdn/tree_verify/mod.rs
index 027eae27b..1daaf4aa4 100644
--- a/crates/uzu-engine/unit/backends/common/kernel/gdn/tree_verify/mod.rs
+++ b/crates/uzu-engine/unit/backends/common/kernel/gdn/tree_verify/mod.rs
@@ -1,10 +1,3 @@
mod conv_scan_test;
-mod out_bench;
-mod out_test;
-mod prefix_bench;
-mod prefix_test;
mod state_advance_test;
-mod tree_gram_bench;
-mod tree_gram_test;
-mod tree_update_solve_bench;
-mod tree_update_solve_test;
+mod tree_verify_test;
diff --git a/crates/uzu-engine/unit/backends/common/kernel/gdn/tree_verify/out_bench.rs b/crates/uzu-engine/unit/backends/common/kernel/gdn/tree_verify/out_bench.rs
deleted file mode 100644
index 3ace7ffea..000000000
--- a/crates/uzu-engine/unit/backends/common/kernel/gdn/tree_verify/out_bench.rs
+++ /dev/null
@@ -1,139 +0,0 @@
-#![cfg(backend = "metal")]
-
-use std::time::Duration;
-
-use criterion::{BenchmarkId, Criterion};
-use futures::executor::block_on;
-use half::bf16;
-use num_traits::Float;
-use uzu_engine_macros::uzu_bench;
-
-use crate::{
- backends::{
- common::{Backend, Context, Kernels, kernel::BuildTreeOutKernel},
- metal::Metal,
- },
- data_type::DataTypeRepr,
- tests::{
- helpers::{create_buffer, create_buffer_with_data},
- matmul::iter_encode_loop_named,
- },
-};
-
-const QK_HEADS: usize = 16;
-const VALUE_HEADS: usize = 48;
-const HEAD_K_DIM: usize = 128;
-const HEAD_V_DIM: usize = 128;
-const TREE_OUT_MATMUL_MXU_ROWS: usize = 16;
-const TREE_OUT_MATMUL_SIMDGROUP_ROWS: usize = 8;
-const TREE_SIZES: &[usize] = &[32, 33, 49, 64, 128, 256, 512];
-const BATCH_SIZES: &[usize] = &[1, 2, 4, 8];
-const TREE_OUT_VALUE_COLS: usize = 32;
-const TREE_OUT_SIMDGROUPS_PER_TG: usize = 4;
-const BUILD_TREE_OUT_PATHS: &[(&str, bool, bool, usize)] = &[
- ("Simdgroup/H0Direct", false, false, TREE_OUT_MATMUL_SIMDGROUP_ROWS),
- ("Simdgroup/H0Transposed", false, true, TREE_OUT_MATMUL_SIMDGROUP_ROWS),
- ("MXU/H0Direct", true, false, TREE_OUT_MATMUL_MXU_ROWS),
-];
-
-struct TreeOutBuffers {
- q: <Metal as Backend>::GlobalBuffer,
- prefix: <Metal as Backend>::GlobalBuffer,
- qkd: <Metal as Backend>::GlobalBuffer,
- u: <Metal as Backend>::GlobalBuffer,
- h0: <Metal as Backend>::GlobalBuffer,
- o: <Metal as Backend>::GlobalBuffer,
-}
-
-fn make_buffers<T: DataTypeRepr + Float>(
- context: &<Metal as Backend>::Context,
- batch_size: usize,
- tree_size: usize,
-) -> TreeOutBuffers {
- let q_len = batch_size * tree_size * QK_HEADS * HEAD_K_DIM;
- let prefix_len = batch_size * tree_size * VALUE_HEADS;
- let qkd_len = batch_size * VALUE_HEADS * tree_size * tree_size;
- let uv_len = batch_size * VALUE_HEADS * tree_size * HEAD_V_DIM;
- let h0_len = batch_size * VALUE_HEADS * HEAD_V_DIM * HEAD_K_DIM;
- let q = (0..q_len).map(|i| T::from(((i as f32 * 0.017).sin() * 0.2) + 0.01).unwrap()).collect::<Vec<_>>();
- let prefix = (0..prefix_len)
- .map(|i| -((i % tree_size) as f32) * 0.01 - ((i % VALUE_HEADS) as f32) * 0.003)
- .collect::<Vec<_>>();
- let qkd = (0..qkd_len).map(|i| ((i as f32 * 0.013).cos() * 0.03) - 0.01).collect::<Vec<_>>();
- let u = (0..uv_len).map(|i| (i as f32 * 0.011).sin() * 0.3 + 0.04).collect::<Vec<_>>();
- let h0 = (0..h0_len).map(|i| ((i as f32 * 0.019).cos() * 0.2) - 0.01).collect::<Vec<_>>();
-
- TreeOutBuffers {
- q: create_buffer_with_data::<Metal, T>(context, &q),
- prefix: create_buffer_with_data::<Metal, f32>(context, &prefix),
- qkd: create_buffer_with_data::<Metal, f32>(context, &qkd),
- u: create_buffer_with_data::<Metal, f32>(context, &u),
- h0: create_buffer_with_data::<Metal, f32>(context, &h0),
- o: create_buffer::<Metal, T>(context, uv_len),
- }
-}
-
-#[uzu_bench]
-fn bench_build_tree_out(c: &mut Criterion) {
- let context = block_on(<Metal as Backend>::Context::new()).expect("metal context");
- bench_build_tree_out_type::<bf16>(c, &context, "BF16");
- bench_build_tree_out_type::<f32>(c, &context, "F32");
-}
-
-fn bench_build_tree_out_type<T: DataTypeRepr + Float>(
- c: &mut Criterion,
- context: &<Metal as Backend>::Context,
- data_type_name: &str,
-) {
- for &(path, use_mxu, transposed_h0, tile_rows) in BUILD_TREE_OUT_PATHS {
- if use_mxu && !context.supports_mxu {
- continue;
- }
- let tile_name = format!("Rows{tile_rows}_VCols{TREE_OUT_VALUE_COLS}_SG{TREE_OUT_SIMDGROUPS_PER_TG}");
- let mut group =
- c.benchmark_group(format!("Metal/Kernel/GDNTreeVerify/BuildTreeOut/{data_type_name}/{path}/{tile_name}"));
- group.sample_size(10).warm_up_time(Duration::from_millis(100)).measurement_time(Duration::from_millis(500));
-
- for &batch_size in BATCH_SIZES {
- for &tree_size in TREE_SIZES {
- let mut buffers = make_buffers::<T>(context, batch_size, tree_size);
- let parameter =
- format!("B{batch_size}_T{tree_size}_QK{QK_HEADS}_HV{VALUE_HEADS}_K{HEAD_K_DIM}_V{HEAD_V_DIM}");
- let benchmark_path =
- format!("Metal/Kernel/GDNTreeVerify/BuildTreeOut/{data_type_name}/{path}/{tile_name}/{parameter}");
- let benchmark_id = BenchmarkId::from_parameter(parameter);
-
- let kernel = <<Metal as Backend>::Kernels as Kernels>::BuildTreeOutKernel::new(
- context,
- T::data_type(),
- T::data_type(),
- use_mxu,
- transposed_h0,
- true,
- )
- .expect("BuildTreeOutKernel");
- group.bench_function(benchmark_id, |bencher| {
- iter_encode_loop_named::<Metal, _>(context, bencher, &benchmark_path, |command_buffer| {
- kernel.encode(
- &buffers.q,
- &buffers.prefix,
- &buffers.qkd,
- &buffers.u,
- Some(&buffers.h0),
- &mut buffers.o,
- (HEAD_K_DIM as f32).sqrt().recip(),
- batch_size as u32,
- tree_size as u32,
- QK_HEADS as u32,
- VALUE_HEADS as u32,
- HEAD_K_DIM as u32,
- HEAD_V_DIM as u32,
- command_buffer,
- );
- });
- });
- }
- }
- group.finish();
- }
-}
diff --git a/crates/uzu-engine/unit/backends/common/kernel/gdn/tree_verify/out_test.rs b/crates/uzu-engine/unit/backends/common/kernel/gdn/tree_verify/out_test.rs
deleted file mode 100644
index b567d2391..000000000
--- a/crates/uzu-engine/unit/backends/common/kernel/gdn/tree_verify/out_test.rs
+++ /dev/null
@@ -1,184 +0,0 @@
-#![cfg(backend = "metal")]
-
-use futures::executor::block_on;
-use half::bf16;
-use num_traits::Float;
-use uzu_engine_macros::uzu_test;
-
-use crate::{
- backends::{
- common::{
- Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferPending, Context, Kernels,
- kernel::BuildTreeOutKernel,
- },
- cpu::Cpu,
- metal::Metal,
- },
- data_type::DataTypeRepr,
- tests::{
- assert::assert_eq_float,
- helpers::{buffer_to_vec, create_buffer, create_buffer_with_data, for_each_non_cpu_backend},
- },
-};
-
-#[derive(Clone, Copy)]
-struct Shape {
- batch_size: usize,
- tree_size: usize,
- qk_heads: usize,
- value_heads: usize,
- head_k_dim: usize,
- head_v_dim: usize,
-}
-
-struct Inputs<T> {
- q: Vec<T>,
- prefix: Vec<f32>,
- qkd: Vec<f32>,
- u: Vec<f32>,
- h0: Vec<f32>,
-}
-
-const BUILD_TREE_OUT_PATHS: &[(&str, bool, bool)] = &[
- ("Simdgroup/H0Direct/Rows8_VCols32_SG4", false, false),
- ("Simdgroup/H0Transposed/Rows8_VCols32_SG4", false, true),
- ("MXU/H0Direct/Rows16_VCols32_SG4", true, false),
-];
-
-fn make_inputs<T: DataTypeRepr + Float>(shape: Shape) -> Inputs<T> {
- let q_len = shape.batch_size * shape.tree_size * shape.qk_heads * shape.head_k_dim;
- let prefix_len = shape.batch_size * shape.tree_size * shape.value_heads;
- let qkd_len = shape.batch_size * shape.value_heads * shape.tree_size * shape.tree_size;
- let u_len = shape.batch_size * shape.value_heads * shape.tree_size * shape.head_v_dim;
- let h0_len = shape.batch_size * shape.value_heads * shape.head_v_dim * shape.head_k_dim;
-
- Inputs {
- q: (0..q_len).map(|i| T::from(((i as f32 * 0.017).sin() * 0.2) + 0.01).unwrap()).collect(),
- prefix: (0..prefix_len)
- .map(|i| -((i % shape.tree_size) as f32) * 0.01 - ((i % shape.value_heads) as f32) * 0.003)
- .collect(),
- qkd: (0..qkd_len).map(|i| ((i as f32 * 0.013).cos() * 0.1) - 0.02).collect(),
- u: (0..u_len).map(|i| ((i as f32 * 0.011).sin() * 0.3) + 0.04).collect(),
- h0: (0..h0_len).map(|i| ((i as f32 * 0.019).cos() * 0.2) - 0.01).collect(),
- }
-}
-
-fn run_build_tree_out<B: Backend, T: DataTypeRepr + Float>(
- shape: Shape,
- inputs: &Inputs<T>,
- use_h0: bool,
- use_mxu: bool,
- transposed_h0: bool,
-) -> Vec<T> {
- let context = block_on(B::Context::new()).expect("Failed to create Context");
- let kernel = <<B as Backend>::Kernels as Kernels>::BuildTreeOutKernel::new(
- &context,
- T::data_type(),
- T::data_type(),
- use_mxu,
- transposed_h0,
- use_h0,
- )
- .expect("BuildTreeOutKernel");
- let q = create_buffer_with_data::<B, T>(&context, &inputs.q);
- let prefix = create_buffer_with_data::<B, f32>(&context, &inputs.prefix);
- let qkd = create_buffer_with_data::<B, f32>(&context, &inputs.qkd);
- let u = create_buffer_with_data::<B, f32>(&context, &inputs.u);
- let h0 = use_h0.then(|| create_buffer_with_data::<B, f32>(&context, &inputs.h0));
- let mut o =
- create_buffer::<B, T>(&context, shape.batch_size * shape.tree_size * shape.value_heads * shape.head_v_dim);
-
- let mut command_buffer = context.create_command_buffer(None, None).expect("Failed to create command buffer");
- kernel.encode(
- &q,
- &prefix,
- &qkd,
- &u,
- h0.as_ref(),
- &mut o,
- (shape.head_k_dim as f32).sqrt().recip(),
- shape.batch_size as u32,
- shape.tree_size as u32,
- shape.qk_heads as u32,
- shape.value_heads as u32,
- shape.head_k_dim as u32,
- shape.head_v_dim as u32,
- &mut command_buffer,
- );
- block_on(command_buffer.end_encoding().submit().wait_until_completed()).unwrap();
- buffer_to_vec(&o)
-}
-
-fn check_shape<T: DataTypeRepr + Float + std::fmt::Display>(
- shape: Shape,
- eps: f32,
-) {
- let inputs = make_inputs::<T>(shape);
-
- for use_h0 in [false, true] {
- let expected = run_build_tree_out::<Cpu, T>(shape, &inputs, use_h0, false, false);
- for_each_non_cpu_backend!(|B| {
- for &(path, use_mxu, transposed_h0) in BUILD_TREE_OUT_PATHS {
- if use_mxu || transposed_h0 && !use_h0 {
- continue;
- }
- let actual = run_build_tree_out::<B, T>(shape, &inputs, use_h0, use_mxu, transposed_h0);
- let msg = format!(
- "backend {} path {path} use_h0 {use_h0} B{}_T{}_QK{}_HV{}_K{}_V{}",
- std::any::type_name::<B>(),
- shape.batch_size,
- shape.tree_size,
- shape.qk_heads,
- shape.value_heads,
- shape.head_k_dim,
- shape.head_v_dim
- );
- assert_eq_float::<T>(&expected, &actual, eps, &msg);
- }
- });
-
- if block_on(<Metal as Backend>::Context::new()).expect("Failed to create Context").supports_mxu {
- for &(path, use_mxu, transposed_h0) in BUILD_TREE_OUT_PATHS {
- if !use_mxu || transposed_h0 && !use_h0 {
- continue;
- }
- let actual = run_build_tree_out::<Metal, T>(shape, &inputs, use_h0, use_mxu, transposed_h0);
- let msg = format!(
- "backend {} path {path} use_h0 {use_h0} B{}_T{}_QK{}_HV{}_K{}_V{}",
- std::any::type_name::<Metal>(),
- shape.batch_size,
- shape.tree_size,
- shape.qk_heads,
- shape.value_heads,
- shape.head_k_dim,
- shape.head_v_dim
- );
- assert_eq_float::<T>(&expected, &actual, eps, &msg);
- }
- }
- }
-}
-
-#[uzu_test]
-fn test_build_tree_out_paths() {
- let small = Shape {
- batch_size: 2,
- tree_size: 17,
- qk_heads: 2,
- value_heads: 6,
- head_k_dim: 32,
- head_v_dim: 32,
- };
- let gdn = Shape {
- batch_size: 1,
- tree_size: 49,
- qk_heads: 16,
- value_heads: 48,
- head_k_dim: 128,
- head_v_dim: 128,
- };
- for shape in [small, gdn] {
- check_shape::<bf16>(shape, 5e-3);
- check_shape::<f32>(shape, 5e-4);
- }
-}
diff --git a/crates/uzu-engine/unit/backends/common/kernel/gdn/tree_verify/prefix_bench.rs b/crates/uzu-engine/unit/backends/common/kernel/gdn/tree_verify/prefix_bench.rs
deleted file mode 100644
index 6dc0edd79..000000000
--- a/crates/uzu-engine/unit/backends/common/kernel/gdn/tree_verify/prefix_bench.rs
+++ /dev/null
@@ -1,67 +0,0 @@
-#![cfg(backend = "metal")]
-
-use std::mem::size_of;
-
-use criterion::{BenchmarkId, Criterion, Throughput};
-use uzu_engine_macros::uzu_bench;
-
-use crate::{
- backends::{
- common::{Backend, Kernels, kernel::BuildTreePrefixKernel},
- metal::Metal,
- },
- tests::{
- cold_pool::ColdPool,
- helpers::{create_buffer, create_buffer_with_data},
- matmul::iter_encode_loop_named,
- util::type_short_name,
- },
-};
-
-struct TreePrefixBuffers {
- trie: <Metal as Backend>::GlobalBuffer,
- log_decay: <Metal as Backend>::GlobalBuffer,
- prefix: <Metal as Backend>::GlobalBuffer,
-}
-
-#[uzu_bench]
-fn bench_build_tree_prefix(c: &mut Criterion) {
- let context = crate::tests::util::shared_metal_context();
- let kernel =
- <<Metal as Backend>::Kernels as Kernels>::BuildTreePrefixKernel::new(&context).expect("BuildTreePrefixKernel");
- let mut group = c.benchmark_group(format!("{}/Kernel/GDNTreeVerify/BuildTreePrefix", type_short_name::<Metal>()));
-
- for tree_size in [49usize, 64, 128] {
- let batch_size = 1usize;
- let value_heads = 48usize;
- let len = batch_size * tree_size * value_heads;
- let trie = (0..tree_size as u32).flat_map(|row| [row, tree_size as u32 - 1, row]).collect::<Vec<_>>();
- let log_decay = (0..len).map(|i| -0.001 - (i as f32 * 0.017).sin().abs() * 0.1).collect::<Vec<_>>();
- let bytes_per_copy = trie.len() * size_of::<u32>() + len * size_of::<f32>() * 2;
- let mut buffers = ColdPool::new(bytes_per_copy, || TreePrefixBuffers {
- trie: create_buffer_with_data::<Metal, u32>(&context, &trie),
- log_decay: create_buffer_with_data::<Metal, f32>(&context, &log_decay),
- prefix: create_buffer::<Metal, f32>(&context, len),
- });
-
- group.throughput(Throughput::Elements((len * tree_size) as u64));
- group.bench_function(BenchmarkId::from_parameter(format!("B1_T{tree_size}_HV{value_heads}")), |bencher| {
- let benchmark_path = format!(
- "{}/Kernel/GDNTreeVerify/BuildTreePrefix/B1_T{tree_size}_HV{value_heads}",
- type_short_name::<Metal>()
- );
- iter_encode_loop_named::<Metal, _>(&context, bencher, &benchmark_path, |command_buffer| {
- let buffers = buffers.next_mut();
- kernel.encode(
- &buffers.trie,
- &buffers.log_decay,
- &mut buffers.prefix,
- batch_size as u32,
- tree_size as u32,
- value_heads as u32,
- command_buffer,
- );
- });
- });
- }
-}
diff --git a/crates/uzu-engine/unit/backends/common/kernel/gdn/tree_verify/prefix_test.rs b/crates/uzu-engine/unit/backends/common/kernel/gdn/tree_verify/prefix_test.rs
deleted file mode 100644
index 746507ebe..000000000
--- a/crates/uzu-engine/unit/backends/common/kernel/gdn/tree_verify/prefix_test.rs
+++ /dev/null
@@ -1,49 +0,0 @@
-#![cfg(backend = "metal")]
-
-use futures::executor::block_on;
-use uzu_engine_macros::uzu_test;
-
-use crate::{
- backends::{
- common::{
- Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferPending, Context, Kernels,
- kernel::BuildTreePrefixKernel,
- },
- cpu::Cpu,
- },
- tests::{
- assert::assert_eq_float,
- helpers::{buffer_to_vec, create_buffer, create_buffer_with_data, for_each_non_cpu_backend},
- },
-};
-
-fn run<B: Backend>(tree_size: usize) -> Vec<f32> {
- const HEADS: usize = 5;
- let context = block_on(B::Context::new()).unwrap();
- let trie = (0..tree_size as u32).flat_map(|node| [node, tree_size as u32 - 1, node]).collect::<Vec<_>>();
- let log_decay = (0..tree_size * HEADS).map(|i| -0.001 - i as f32 * 0.0001).collect::<Vec<_>>();
- let trie = create_buffer_with_data::<B, u32>(&context, &trie);
- let log_decay = create_buffer_with_data::<B, f32>(&context, &log_decay);
- let mut prefix = create_buffer::<B, f32>(&context, tree_size * HEADS);
- let kernel = <B::Kernels as Kernels>::BuildTreePrefixKernel::new(&context).unwrap();
- let mut command_buffer = context.create_command_buffer(None, None).unwrap();
- kernel.encode(&trie, &log_decay, &mut prefix, 1, tree_size as u32, HEADS as u32, &mut command_buffer);
- block_on(command_buffer.end_encoding().submit().wait_until_completed()).unwrap();
- buffer_to_vec(&prefix)
-}
-
-#[uzu_test]
-fn test_build_tree_prefix() {
- const HEADS: usize = 5;
- for tree_size in [49, 64, 128] {
- let expected = (0..tree_size)
- .flat_map(|row| {
- (0..HEADS).map(move |head| {
- (0..=row).map(|token| -0.001 - (token * HEADS + head) as f32 * 0.0001).sum::<f32>()
- })
- })
- .collect::<Vec<_>>();
- assert_eq_float(&expected, &run::<Cpu>(tree_size), 1e-6, "CPU");
- for_each_non_cpu_backend!(|B| assert_eq_float(&expected, &run::<B>(tree_size), 1e-6, "backend"));
- }
-}
diff --git a/crates/uzu-engine/unit/backends/common/kernel/gdn/tree_verify/tree_gram_bench.rs b/crates/uzu-engine/unit/backends/common/kernel/gdn/tree_verify/tree_gram_bench.rs
deleted file mode 100644
index 81ca5a365..000000000
--- a/crates/uzu-engine/unit/backends/common/kernel/gdn/tree_verify/tree_gram_bench.rs
+++ /dev/null
@@ -1,204 +0,0 @@
-#![cfg(backend = "metal")]
-
-use std::{mem::size_of, time::Duration};
-
-use criterion::{BenchmarkId, Criterion};
-use futures::executor::block_on;
-use half::bf16;
-use uzu_engine_macros::uzu_bench;
-
-use crate::{
- backends::{
- common::{Backend, Context, Kernels, kernel::BuildTreeGramKernel},
- metal::Metal,
- },
- data_type::DataType,
- tests::{
- cold_pool::ColdPool,
- helpers::{create_buffer, create_buffer_with_data},
- matmul::iter_encode_loop_named,
- },
-};
-
-const K_HEADS: usize = 16;
-const VALUE_HEADS: usize = 48;
-const HEAD_K_DIM: usize = 128;
-const HEAD_V_DIM: usize = 128;
-const TREE_SIZES: &[usize] = &[32, 33, 49, 64, 128, 256, 512];
-const BATCH_SIZES: &[usize] = &[1, 2, 4, 8];
-
-struct TreeGramBuffers {
- q: <Metal as Backend>::GlobalBuffer,
- k: <Metal as Backend>::GlobalBuffer,
- trie: <Metal as Backend>::GlobalBuffer,
- prefix: <Metal as Backend>::GlobalBuffer,
- beta: <Metal as Backend>::GlobalBuffer,
- h0: <Metal as Backend>::GlobalBuffer,
- a_packed: <Metal as Backend>::GlobalBuffer,
- qkd: <Metal as Backend>::GlobalBuffer,
- a_inv: <Metal as Backend>::GlobalBuffer,
- kh0: <Metal as Backend>::GlobalBuffer,
-}
-
-fn reranker_like_trie(
- batch_size: usize,
- tree_size: usize,
-) -> Vec<u32> {
- let mut trie = Vec::with_capacity(batch_size * tree_size * 3);
- for b in 0..batch_size {
- let mut children = vec![Vec::<usize>::new(); tree_size];
- let mut child_count = vec![0usize; tree_size];
- for node in 1..tree_size {
- let mut parent = (node - 1) / 2;
- if child_count[parent] >= 2 {
- parent = node.saturating_sub(2 + (b + node) % node);
- }
- children[parent].push(node);
- child_count[parent] += 1;
- }
-
- fn dfs(
- node: usize,
- children: &[Vec<usize>],
- trie: &mut [[u32; 3]],
- next: &mut u32,
- depth: u32,
- ) {
- let start = *next;
- *next += 1;
- let mut end = start;
- for &child in &children[node] {
- dfs(child, children, trie, next, depth + 1);
- end = end.max(trie[child][1]);
- }
- trie[node] = [start, end, depth];
- }
-
- let mut batch_trie = vec![[0u32; 3]; tree_size];
- let mut next = 0;
- dfs(0, &children, &mut batch_trie, &mut next, 0);
- for node in batch_trie {
- trie.extend_from_slice(&node);
- }
- }
- trie
-}
-
-fn make_buffers(
- context: &<Metal as Backend>::Context,
- batch_size: usize,
- tree_size: usize,
-) -> (TreeGramBuffers, f32) {
- let qk_len = batch_size * tree_size * K_HEADS * HEAD_K_DIM;
- let scalar_len = batch_size * tree_size * VALUE_HEADS;
- let out_len = batch_size * VALUE_HEADS * tree_size * tree_size;
- let num_blocks = tree_size.div_ceil(16);
- let a_len = batch_size * VALUE_HEADS * num_blocks * num_blocks.div_ceil(2) * 16 * 32;
- let a_inv_len = batch_size * VALUE_HEADS * num_blocks * 16 * 16;
- let h0_len = batch_size * VALUE_HEADS * HEAD_V_DIM * HEAD_K_DIM;
- let kh0_len = batch_size * tree_size * VALUE_HEADS * HEAD_V_DIM;
- let trie = reranker_like_trie(batch_size, tree_size);
- let q = (0..qk_len).map(|i| bf16::from_f32(((i as f32 * 0.017).sin() * 0.2) + 0.01)).collect::<Vec<_>>();
- let k = (0..qk_len).map(|i| bf16::from_f32(((i as f32 * 0.019).cos() * 0.18) - 0.02)).collect::<Vec<_>>();
- let prefix = (0..scalar_len)
- .map(|i| -((i % tree_size) as f32) * 0.01 - ((i % VALUE_HEADS) as f32) * 0.003)
- .collect::<Vec<_>>();
- let beta = (0..scalar_len).map(|i| 0.25 + ((i as f32 * 0.013).sin() + 1.0) * 0.2).collect::<Vec<_>>();
- let h0 = (0..h0_len).map(|i| ((i as f32 * 0.007).sin() * 0.05) - 0.01).collect::<Vec<_>>();
- let scale = (HEAD_K_DIM as f32).sqrt().recip();
- (
- TreeGramBuffers {
- q: create_buffer_with_data::<Metal, bf16>(context, &q),
- k: create_buffer_with_data::<Metal, bf16>(context, &k),
- trie: create_buffer_with_data::<Metal, u32>(context, &trie),
- prefix: create_buffer_with_data::<Metal, f32>(context, &prefix),
- beta: create_buffer_with_data::<Metal, f32>(context, &beta),
- h0: create_buffer_with_data::<Metal, f32>(context, &h0),
- a_packed: create_buffer::<Metal, f32>(context, a_len),
- qkd: create_buffer::<Metal, f32>(context, out_len),
- a_inv: create_buffer::<Metal, f32>(context, a_inv_len),
- kh0: create_buffer::<Metal, f32>(context, kh0_len),
- },
- scale,
- )
-}
-
-fn buffers_bytes(
- batch_size: usize,
- tree_size: usize,
-) -> usize {
- let qk_len = batch_size * tree_size * K_HEADS * HEAD_K_DIM;
- let scalar_len = batch_size * tree_size * VALUE_HEADS;
- let out_len = batch_size * VALUE_HEADS * tree_size * tree_size;
- let num_blocks = tree_size.div_ceil(16);
- let a_len = batch_size * VALUE_HEADS * num_blocks * num_blocks.div_ceil(2) * 16 * 32;
- let a_inv_len = batch_size * VALUE_HEADS * num_blocks * 16 * 16;
- let h0_len = batch_size * VALUE_HEADS * HEAD_V_DIM * HEAD_K_DIM;
- let kh0_len = batch_size * tree_size * VALUE_HEADS * HEAD_V_DIM;
- qk_len * 2 * size_of::<bf16>()
- + h0_len * size_of::<f32>()
- + batch_size * tree_size * 3 * size_of::<u32>()
- + scalar_len * size_of::<f32>() * 2
- + (out_len + a_len + a_inv_len + kh0_len) * size_of::<f32>()
-}
-
-#[uzu_bench]
-fn bench_build_tree_gram(c: &mut Criterion) {
- let context = block_on(<Metal as Backend>::Context::new()).expect("metal context");
- let kernel_paths = if context.supports_mxu {
- &[("Simdgroup", false), ("MXU", true)][..]
- } else {
- &[("Simdgroup", false)][..]
- };
-
- for &(kernel_path, use_mxu) in kernel_paths {
- let mut group = c.benchmark_group(format!("Metal/Kernel/GDNTreeVerify/BuildTreeGram/{kernel_path}"));
- group.sample_size(20).warm_up_time(Duration::from_millis(500)).measurement_time(Duration::from_secs(2));
-
- for &batch_size in BATCH_SIZES {
- for &tree_size in TREE_SIZES {
- let kernel = <<Metal as Backend>::Kernels as Kernels>::BuildTreeGramKernel::new(
- &context,
- DataType::BF16,
- use_mxu,
- true,
- )
- .expect("BuildTreeGramKernel");
- let scale = (HEAD_K_DIM as f32).sqrt().recip();
- let parameter = format!("B{batch_size}_T{tree_size}_Hg{K_HEADS}_HV{VALUE_HEADS}_K{HEAD_K_DIM}");
- let benchmark_path = format!("Metal/Kernel/GDNTreeVerify/BuildTreeGram/{kernel_path}/{parameter}");
- let benchmark_id = BenchmarkId::from_parameter(parameter);
-
- let mut buffers = ColdPool::new(buffers_bytes(batch_size, tree_size), || {
- make_buffers(&context, batch_size, tree_size).0
- });
- group.bench_function(benchmark_id, |bencher| {
- iter_encode_loop_named::<Metal, _>(context.as_ref(), bencher, &benchmark_path, |command_buffer| {
- let buffers = buffers.next_mut();
- kernel.encode(
- &buffers.q,
- &buffers.k,
- &buffers.trie,
- &buffers.prefix,
- &buffers.beta,
- Some(&buffers.h0),
- &mut buffers.a_packed,
- &mut buffers.qkd,
- &mut buffers.a_inv,
- Some(&mut buffers.kh0),
- scale,
- batch_size as u32,
- tree_size as u32,
- K_HEADS as u32,
- VALUE_HEADS as u32,
- HEAD_K_DIM as u32,
- HEAD_V_DIM as u32,
- command_buffer,
- );
- });
- });
- }
- }
- group.finish();
- }
-}
diff --git a/crates/uzu-engine/unit/backends/common/kernel/gdn/tree_verify/tree_gram_test.rs b/crates/uzu-engine/unit/backends/common/kernel/gdn/tree_verify/tree_gram_test.rs
deleted file mode 100644
index 686159bf6..000000000
--- a/crates/uzu-engine/unit/backends/common/kernel/gdn/tree_verify/tree_gram_test.rs
+++ /dev/null
@@ -1,183 +0,0 @@
-#![cfg(backend = "metal")]
-
-use futures::executor::block_on;
-use half::bf16;
-use num_traits::Float;
-use uzu_engine_macros::uzu_test;
-
-use crate::{
- backends::{
- common::{
- Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferPending, Context, Kernels,
- kernel::BuildTreeGramKernel,
- },
- cpu::Cpu,
- metal::Metal,
- },
- data_type::{DataType, DataTypeRepr},
- tests::{
- assert::assert_eq_float,
- helpers::{buffer_to_vec, create_buffer, create_buffer_with_data, for_each_non_cpu_backend},
- },
-};
-
-const BATCH_SIZE: usize = 2;
-const K_HEADS: usize = 2;
-const VALUE_HEADS: usize = 6;
-const HEAD_K_DIM: usize = 128;
-// 80 = two full 32-wide kh0 dv chunks plus a ragged 16-wide one.
-const HEAD_V_DIM: usize = 80;
-
-struct Inputs {
- trie: Vec<u32>,
- q: Vec<f32>,
- k: Vec<f32>,
- prefix: Vec<f32>,
- beta: Vec<f32>,
- h0: Vec<f32>,
-}
-
-fn make_inputs(tree_size: usize) -> Inputs {
- let q_len = BATCH_SIZE * tree_size * K_HEADS * HEAD_K_DIM;
- let scalar_len = BATCH_SIZE * tree_size * VALUE_HEADS;
- let h0_len = BATCH_SIZE * VALUE_HEADS * HEAD_V_DIM * HEAD_K_DIM;
- Inputs {
- trie: build_trie(tree_size),
- q: (0..q_len).map(|i| ((i as f32 * 0.017).sin() * 0.2) + 0.01).collect(),
- k: (0..q_len).map(|i| ((i as f32 * 0.019).cos() * 0.18) - 0.02).collect(),
- prefix: (0..scalar_len)
- .map(|i| -((i % tree_size) as f32) * 0.01 - ((i % VALUE_HEADS) as f32) * 0.003)
- .collect(),
- beta: (0..scalar_len).map(|i| 0.25 + ((i as f32 * 0.013).sin() + 1.0) * 0.2).collect(),
- h0: (0..h0_len).map(|i| ((i as f32 * 0.007).sin() * 0.05) - 0.01).collect(),
- }
-}
-
-fn get_output<B: Backend, T: DataTypeRepr + Float>(
- q: &[T],
- k: &[T],
- trie: &[u32],
- prefix: &[f32],
- beta: &[f32],
- h0: &[f32],
- tree_size: usize,
- scale: f32,
- use_mxu: bool,
-) -> (Vec<f32>, Vec<f32>, Vec<f32>, Vec<f32>) {
- let context = block_on(B::Context::new()).expect("Failed to create Context");
- let kernel =
- <<B as Backend>::Kernels as Kernels>::BuildTreeGramKernel::new(&context, T::data_type(), use_mxu, true)
- .expect("Failed to create BuildTreeGramKernel");
-
- let output_len = BATCH_SIZE * VALUE_HEADS * tree_size * tree_size;
- let num_blocks = tree_size.div_ceil(16);
- let a_len = BATCH_SIZE * VALUE_HEADS * num_blocks * num_blocks.div_ceil(2) * 16 * 32;
- let a_inv_len = BATCH_SIZE * VALUE_HEADS * num_blocks * 16 * 16;
- let kh0_len = BATCH_SIZE * tree_size * VALUE_HEADS * HEAD_V_DIM;
- let q = create_buffer_with_data::<B, T>(&context, q);
- let k = create_buffer_with_data::<B, T>(&context, k);
- let trie = create_buffer_with_data::<B, u32>(&context, trie);
- let prefix = create_buffer_with_data::<B, f32>(&context, prefix);
- let beta = create_buffer_with_data::<B, f32>(&context, beta);
- let h0 = create_buffer_with_data::<B, f32>(&context, h0);
- // The GPU only writes packed-A tiles touching the block lower triangle;
- // zero-init so CPU and GPU buffers stay comparable elsewhere.
- let mut a_packed = create_buffer_with_data::<B, f32>(&context, &vec![0.0f32; a_len]);
- let mut qkd = create_buffer::<B, f32>(&context, output_len);
- let mut a_inv = create_buffer::<B, f32>(&context, a_inv_len);
- let mut kh0 = create_buffer::<B, f32>(&context, kh0_len);
-
- let mut command_buffer = context.create_command_buffer(None, None).expect("Failed to create command buffer");
- kernel.encode(
- &q,
- &k,
- &trie,
- &prefix,
- &beta,
- Some(&h0),
- &mut a_packed,
- &mut qkd,
- &mut a_inv,
- Some(&mut kh0),
- scale,
- BATCH_SIZE as u32,
- tree_size as u32,
- K_HEADS as u32,
- VALUE_HEADS as u32,
- HEAD_K_DIM as u32,
- HEAD_V_DIM as u32,
- &mut command_buffer,
- );
- block_on(command_buffer.end_encoding().submit().wait_until_completed()).unwrap();
-
- (buffer_to_vec(&a_packed), buffer_to_vec(&qkd), buffer_to_vec(&a_inv), buffer_to_vec(&kh0))
-}
-
-fn check_type<T: DataTypeRepr + Float>(
- inputs: &Inputs,
- tree_size: usize,
- scale: f32,
-) {
- let q = inputs.q.iter().map(|&value| T::from(value).unwrap()).collect::<Vec<_>>();
- let k = inputs.k.iter().map(|&value| T::from(value).unwrap()).collect::<Vec<_>>();
- let expected =
- get_output::<Cpu, T>(&q, &k, &inputs.trie, &inputs.prefix, &inputs.beta, &inputs.h0, tree_size, scale, false);
-
- for_each_non_cpu_backend!(|B| {
- let actual =
- get_output::<B, T>(&q, &k, &inputs.trie, &inputs.prefix, &inputs.beta, &inputs.h0, tree_size, scale, false);
- let msg = format!("backend {} simdgroup tree_size {tree_size}", std::any::type_name::<B>());
- assert_eq_float::<f32>(&expected.0, &actual.0, 2e-4, &format!("a_packed {msg}"));
- assert_eq_float::<f32>(&expected.1, &actual.1, 2e-4, &format!("qkd {msg}"));
- assert_eq_float::<f32>(&expected.2, &actual.2, 5e-4, &format!("a_inv {msg}"));
- assert_eq_float::<f32>(&expected.3, &actual.3, 2e-4, &format!("kh0 {msg}"));
- });
-
- if block_on(<Metal as Backend>::Context::new()).expect("Failed to create Context").supports_mxu {
- let actual = get_output::<Metal, T>(
- &q,
- &k,
- &inputs.trie,
- &inputs.prefix,
- &inputs.beta,
- &inputs.h0,
- tree_size,
- scale,
- true,
- );
- let msg = format!("backend {} path MXU tree_size {tree_size}", std::any::type_name::<Metal>());
- let (a_eps, qkd_eps, inv_eps, kh0_eps) = if T::data_type() == DataType::BF16 {
- (1e-5, 1e-6, 1e-5, 5e-4)
- } else {
- (5e-3, 1e-3, 8e-3, 1e-3)
- };
- assert_eq_float::<f32>(&expected.0, &actual.0, a_eps, &format!("a_packed {msg}"));
- assert_eq_float::<f32>(&expected.1, &actual.1, qkd_eps, &format!("qkd {msg}"));
- assert_eq_float::<f32>(&expected.2, &actual.2, inv_eps, &format!("a_inv {msg}"));
- assert_eq_float::<f32>(&expected.3, &actual.3, kh0_eps, &format!("kh0 {msg}"));
- }
-}
-
-fn build_trie(tree_size: usize) -> Vec<u32> {
- let last = tree_size as u32 - 1;
- let mut trie = Vec::with_capacity(BATCH_SIZE * tree_size * 3);
- for i in 0..tree_size as u32 {
- trie.extend_from_slice(&[i, last, i]);
- }
- trie.extend_from_slice(&[0, last, 0]);
- for i in 1..tree_size as u32 {
- trie.extend_from_slice(&[i, i, 1]);
- }
- trie
-}
-
-#[uzu_test]
-fn test_build_tree_gram_matches_cpu() {
- let scale = (HEAD_K_DIM as f32).sqrt().recip();
-
- for tree_size in [17, 64, 128] {
- let i = make_inputs(tree_size);
- check_type::<f32>(&i, tree_size, scale);
- check_type::<bf16>(&i, tree_size, scale);
- }
-}
diff --git a/crates/uzu-engine/unit/backends/common/kernel/gdn/tree_verify/tree_update_solve_bench.rs b/crates/uzu-engine/unit/backends/common/kernel/gdn/tree_verify/tree_update_solve_bench.rs
deleted file mode 100644
index 9bab2be4f..000000000
--- a/crates/uzu-engine/unit/backends/common/kernel/gdn/tree_verify/tree_update_solve_bench.rs
+++ /dev/null
@@ -1,225 +0,0 @@
-#![cfg(backend = "metal")]
-
-use std::{mem::size_of, time::Duration};
-
-use criterion::{BenchmarkId, Criterion, Throughput};
-use futures::executor::block_on;
-use half::bf16;
-use uzu_engine_macros::uzu_bench;
-
-use crate::{
- backends::{
- common::{Backend, CommandBuffer, Context, Kernels, kernel::TreeUpdateSolveKernel},
- metal::Metal,
- },
- data_type::DataType,
- tests::{
- cold_pool::ColdPool,
- helpers::{create_buffer, create_buffer_with_data},
- matmul::iter_encode_loop_named,
- },
-};
-
-const NUM_V_HEADS: usize = 48;
-const HEAD_V_DIM: usize = 128;
-const BT: usize = 16;
-const BVS: &[usize] = &[16, 32];
-
-const BENCH_SHAPES: &[(usize, usize)] = &[
- (1, 32),
- (1, 33),
- (1, 49),
- (1, 64),
- (1, 128),
- (1, 256),
- (1, 512),
- (2, 33),
- (2, 49),
- (2, 64),
- (2, 128),
- (2, 256),
- (2, 512),
- (4, 33),
- (4, 49),
- (4, 64),
- (4, 128),
- (4, 256),
- (4, 512),
- (8, 33),
- (8, 49),
- (8, 64),
- (8, 128),
- (8, 256),
- (8, 512),
-];
-
-struct TreeUpdateSolveBuffers {
- kh0: <Metal as Backend>::GlobalBuffer,
- v: <Metal as Backend>::GlobalBuffer,
- prefix: <Metal as Backend>::GlobalBuffer,
- beta: <Metal as Backend>::GlobalBuffer,
- a: <Metal as Backend>::GlobalBuffer,
- a_inv: <Metal as Backend>::GlobalBuffer,
- u: <Metal as Backend>::GlobalBuffer,
-}
-
-fn matrix_value(
- batch_head_matrix_idx: usize,
- row: usize,
- col: usize,
- tree_size: usize,
-) -> f32 {
- if col < row {
- ((batch_head_matrix_idx * tree_size * tree_size + row * tree_size + col) as f32 * 0.011).sin() * 0.01
- } else {
- 0.0
- }
-}
-
-// Compact [B * HV, ceil(T/BT), BT, BT] diagonal blocks.
-fn inverse_value(
- batch_head_matrix_idx: usize,
- block: usize,
- row: usize,
- col: usize,
- tree_size: usize,
-) -> f32 {
- if row == col {
- 1.0
- } else if col < row {
- -matrix_value(batch_head_matrix_idx, block * BT + row, block * BT + col, tree_size)
- } else {
- 0.0
- }
-}
-
-fn make_buffers(
- context: &<Metal as Backend>::Context,
- batch_size: usize,
- tree_size: usize,
-) -> TreeUpdateSolveBuffers {
- let v_len = batch_size * tree_size * NUM_V_HEADS * HEAD_V_DIM;
- let scalar_len = batch_size * tree_size * NUM_V_HEADS;
- let num_blocks = tree_size.div_ceil(BT);
- let num_col_pairs = num_blocks.div_ceil(2);
- let a_len = batch_size * NUM_V_HEADS * num_blocks * num_col_pairs * BT * 2 * BT;
- let u_len = batch_size * NUM_V_HEADS * tree_size * HEAD_V_DIM;
-
- let kh0 = (0..v_len).map(|i| ((i as f32 * 0.019).sin() * 0.2) + 0.01).collect::<Vec<f32>>();
- let v = (0..v_len).map(|i| bf16::from_f32(((i as f32 * 0.017).cos() * 0.18) - 0.02)).collect::<Vec<_>>();
- let prefix = (0..scalar_len)
- .map(|i| -((i % tree_size) as f32) * 0.01 - ((i % NUM_V_HEADS) as f32) * 0.003)
- .collect::<Vec<_>>();
- let beta = (0..scalar_len).map(|i| 0.25 + ((i as f32 * 0.013).sin() + 1.0) * 0.2).collect::<Vec<_>>();
- // Packed [B * HV, NB, ceil(NB/2), BT, 2*BT] block-pair tiles.
- let a_f32 = (0..a_len)
- .map(|i| {
- let local_col = i % (2 * BT);
- let local_row = (i / (2 * BT)) % BT;
- let pair = (i / (BT * 2 * BT)) % num_col_pairs;
- let block = (i / (BT * 2 * BT * num_col_pairs)) % num_blocks;
- let batch_head_matrix_idx = i / (BT * 2 * BT * num_col_pairs * num_blocks);
- let row = block * BT + local_row;
- let col = pair * 2 * BT + local_col;
- if col < tree_size && row < tree_size && col / BT < block {
- matrix_value(batch_head_matrix_idx, row, col, tree_size)
- } else {
- 0.0
- }
- })
- .collect::<Vec<_>>();
- let a = create_buffer_with_data::<Metal, f32>(context, &a_f32);
- let inv_len = batch_size * NUM_V_HEADS * tree_size.div_ceil(BT) * BT * BT;
- let a_inv = (0..inv_len)
- .map(|i| {
- let col = i % BT;
- let row = (i / BT) % BT;
- let block = (i / (BT * BT)) % tree_size.div_ceil(BT);
- let batch_head_matrix_idx = i / (BT * BT * tree_size.div_ceil(BT));
- inverse_value(batch_head_matrix_idx, block, row, col, tree_size)
- })
- .collect::<Vec<_>>();
- TreeUpdateSolveBuffers {
- kh0: create_buffer_with_data::<Metal, f32>(context, &kh0),
- v: create_buffer_with_data::<Metal, bf16>(context, &v),
- prefix: create_buffer_with_data::<Metal, f32>(context, &prefix),
- beta: create_buffer_with_data::<Metal, f32>(context, &beta),
- a,
- a_inv: create_buffer_with_data::<Metal, f32>(context, &a_inv),
- u: create_buffer::<Metal, f32>(context, u_len),
- }
-}
-
-fn buffers_bytes(
- batch_size: usize,
- tree_size: usize,
-) -> usize {
- let v_len = batch_size * tree_size * NUM_V_HEADS * HEAD_V_DIM;
- let scalar_len = batch_size * tree_size * NUM_V_HEADS;
- let num_blocks = tree_size.div_ceil(BT);
- let a_len = batch_size * NUM_V_HEADS * num_blocks * num_blocks.div_ceil(2) * BT * 2 * BT;
- let u_len = batch_size * NUM_V_HEADS * tree_size * HEAD_V_DIM;
-
- let inv_len = batch_size * NUM_V_HEADS * num_blocks * BT * BT;
- v_len * size_of::<bf16>() + (scalar_len * 2 + a_len + inv_len + u_len + v_len) * size_of::<f32>()
-}
-
-#[uzu_bench]
-fn bench_tree_update_solve(c: &mut Criterion) {
- let context = block_on(<Metal as Backend>::Context::new()).expect("metal context");
- for &bv in BVS {
- let kernel = <<Metal as Backend>::Kernels as Kernels>::TreeUpdateSolveKernel::new(
- &context,
- DataType::BF16,
- bv as u32,
- true,
- )
- .expect("TreeUpdateSolveKernel");
-
- let mut group = c.benchmark_group(format!("Metal/Kernel/GDNTreeVerify/TreeUpdateSolve/Simdgroup/BV{bv}"));
- // 500ms warmup ramps GPU clocks before measuring (µs-scale benches never
- // ramp otherwise and medians become run-composition artifacts).
- group.sample_size(20).warm_up_time(Duration::from_millis(500)).measurement_time(Duration::from_secs(2));
-
- for &(batch_size, tree_size) in BENCH_SHAPES {
- let parameter = format!("B{batch_size}_T{tree_size}_HV{NUM_V_HEADS}_V{HEAD_V_DIM}_BV{bv}");
- let benchmark_path = format!("Metal/Kernel/GDNTreeVerify/TreeUpdateSolve/Simdgroup/BV{bv}/{parameter}");
- let benchmark_id = BenchmarkId::from_parameter(parameter);
- let mut buffers =
- ColdPool::new(buffers_bytes(batch_size, tree_size), || make_buffers(&context, batch_size, tree_size));
-
- group.throughput(Throughput::Elements((batch_size * tree_size * NUM_V_HEADS * HEAD_V_DIM) as u64));
- group.bench_function(benchmark_id, |bencher| {
- iter_encode_loop_named::<Metal, _>(context.as_ref(), bencher, &benchmark_path, |command_buffer| {
- let buffers = buffers.next_mut();
- encode(&kernel, buffers, batch_size, tree_size, command_buffer);
- });
- });
- }
- group.finish();
- crate::tests::metrics::wait_gpu_cooldown();
- }
-}
-
-fn encode(
- kernel: &impl TreeUpdateSolveKernel<Backend = Metal>,
- buffers: &mut TreeUpdateSolveBuffers,
- batch_size: usize,
- tree_size: usize,
- command_buffer: &mut <<Metal as Backend>::CommandBuffer as CommandBuffer>::Encoding,
-) {
- kernel.encode(
- Some(&buffers.kh0),
- &buffers.v,
- &buffers.prefix,
- &buffers.beta,
- &buffers.a,
- &buffers.a_inv,
- &mut buffers.u,
- batch_size as u32,
- tree_size as u32,
- NUM_V_HEADS as u32,
- HEAD_V_DIM as u32,
- command_buffer,
- );
-}
diff --git a/crates/uzu-engine/unit/backends/common/kernel/gdn/tree_verify/tree_update_solve_test.rs b/crates/uzu-engine/unit/backends/common/kernel/gdn/tree_verify/tree_update_solve_test.rs
deleted file mode 100644
index ba955dd6f..000000000
--- a/crates/uzu-engine/unit/backends/common/kernel/gdn/tree_verify/tree_update_solve_test.rs
+++ /dev/null
@@ -1,205 +0,0 @@
-#![cfg(backend = "metal")]
-
-use futures::executor::block_on;
-use half::bf16;
-use uzu_engine_macros::uzu_test;
-
-use crate::{
- backends::{
- common::{
- Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferPending, Context, Kernels,
- kernel::TreeUpdateSolveKernel,
- },
- cpu::Cpu,
- },
- data_type::{DataType, DataTypeRepr},
- tests::{
- assert::assert_eq_float,
- helpers::{buffer_to_vec, create_buffer, create_buffer_with_data, for_each_non_cpu_backend},
- },
-};
-
-const BT: u32 = 16;
-const BVS: &[u32] = &[16, 32];
-
-#[derive(Clone, Copy)]
-struct SolveCase {
- name: &'static str,
- batch_size: u32,
- tree_size: u32,
- num_v_heads: u32,
- head_v_dim: u32,
-}
-
-const CASES: &[SolveCase] = &[
- SolveCase {
- name: "single_full_block",
- batch_size: 1,
- tree_size: 16,
- num_v_heads: 1,
- head_v_dim: 16,
- },
- SolveCase {
- name: "ragged_second_block",
- batch_size: 1,
- tree_size: 17,
- num_v_heads: 2,
- head_v_dim: 20,
- },
- SolveCase {
- name: "common_small_tree",
- batch_size: 1,
- tree_size: 33,
- num_v_heads: 4,
- head_v_dim: 128,
- },
- SolveCase {
- name: "ragged_four_blocks",
- batch_size: 1,
- tree_size: 49,
- num_v_heads: 4,
- head_v_dim: 128,
- },
- SolveCase {
- // NB=7: exercises pair_idx >= 1, tail blocks >= 4, and a ragged final block.
- name: "ragged_seven_blocks",
- batch_size: 1,
- tree_size: 97,
- num_v_heads: 2,
- head_v_dim: 64,
- },
- SolveCase {
- // NB=16: the large-tree regime this kernel exists for; num_full_pairs
- // reaches 7, so the deep packed-pair accumulation is exercised.
- name: "large_tree",
- batch_size: 1,
- tree_size: 256,
- num_v_heads: 2,
- head_v_dim: 64,
- },
- SolveCase {
- name: "batched",
- batch_size: 2,
- tree_size: 16,
- num_v_heads: 2,
- head_v_dim: 16,
- },
-];
-
-fn run_case<B: Backend, T: DataTypeRepr + Copy>(
- case: SolveCase,
- bv: u32,
- use_h0: bool,
- data_type: DataType,
- cast: impl Fn(f32) -> T + Copy,
-) -> Vec<f32> {
- let context = block_on(B::Context::new()).expect("context");
- let kernel = <<B as Backend>::Kernels as Kernels>::TreeUpdateSolveKernel::new(&context, data_type, bv, use_h0)
- .expect("kernel");
-
- let batch_size = case.batch_size as usize;
- let tree_size = case.tree_size as usize;
- let num_v_heads = case.num_v_heads as usize;
- let head_v_dim = case.head_v_dim as usize;
-
- let v_len = batch_size * tree_size * num_v_heads * head_v_dim;
- let scalar_len = batch_size * tree_size * num_v_heads;
- let num_blocks = tree_size.div_ceil(BT as usize);
- let num_col_pairs = num_blocks.div_ceil(2);
- let a_len = batch_size * num_v_heads * num_blocks * num_col_pairs * (BT * 2 * BT) as usize;
- let inv_len = batch_size * num_v_heads * num_blocks * (BT * BT) as usize;
- let u_len = batch_size * num_v_heads * tree_size * head_v_dim;
-
- let kh0 = (0..v_len).map(|i| ((i as f32 * 0.019).sin() * 0.2) + 0.01).collect::<Vec<f32>>();
- let v = (0..v_len).map(|i| cast(((i as f32 * 0.017).cos() * 0.18) - 0.02)).collect::<Vec<_>>();
- let prefix = (0..scalar_len)
- .map(|i| -((i % tree_size) as f32) * 0.01 - ((i % num_v_heads) as f32) * 0.003)
- .collect::<Vec<_>>();
- let beta = (0..scalar_len).map(|i| 0.25 + ((i as f32 * 0.013).sin() + 1.0) * 0.2).collect::<Vec<_>>();
- // Packed [B * HV, NB, ceil(NB/2), BT, 2*BT] block-pair tiles; strictly-lower
- // blocks get values, everything else zero (the kernel never reads it).
- let a_f32 = (0..a_len)
- .map(|i| {
- let local_col = i % (2 * BT as usize);
- let local_row = (i / (2 * BT as usize)) % BT as usize;
- let pair = (i / (BT * 2 * BT) as usize) % num_col_pairs;
- let block = (i / ((BT * 2 * BT) as usize * num_col_pairs)) % num_blocks;
- let row = block * BT as usize + local_row;
- let col = pair * 2 * BT as usize + local_col;
- if col < tree_size && row < tree_size && col / (BT as usize) < block {
- // ~0.05 per entry: with the dense lower-triangle fill this puts the
- // per-row A magnitude near the real l2-normed-key / softplus-decay regime.
- ((i as f32 * 0.011).sin()) * 0.05
- } else {
- 0.0
- }
- })
- .collect::<Vec<_>>();
- // Compact [B * HV, ceil(T/BT), BT, BT] diagonal blocks.
- let a_inv = (0..inv_len)
- .map(|i| {
- let col = i % BT as usize;
- let row = (i / BT as usize) % BT as usize;
- if row == col {
- 1.0
- } else if col < row {
- -((i as f32 * 0.011).sin()) * 0.05
- } else {
- 0.0
- }
- })
- .collect::<Vec<_>>();
- let kh0 = create_buffer_with_data::<B, f32>(&context, &kh0);
- let v = create_buffer_with_data::<B, T>(&context, &v);
- let prefix = create_buffer_with_data::<B, f32>(&context, &prefix);
- let beta = create_buffer_with_data::<B, f32>(&context, &beta);
- let a = create_buffer_with_data::<B, f32>(&context, &a_f32);
- let a_inv = create_buffer_with_data::<B, f32>(&context, &a_inv);
- let mut u = create_buffer::<B, f32>(&context, u_len);
-
- let mut command_buffer = context.create_command_buffer(None, None).expect("command buffer");
- kernel.encode(
- use_h0.then_some(&kh0),
- &v,
- &prefix,
- &beta,
- &a,
- &a_inv,
- &mut u,
- case.batch_size,
- case.tree_size,
- case.num_v_heads,
- case.head_v_dim,
- &mut command_buffer,
- );
- block_on(command_buffer.end_encoding().submit().wait_until_completed()).unwrap();
-
- buffer_to_vec(&u)
-}
-
-#[uzu_test]
-fn test_tree_update_solve_cases() {
- for &bv in BVS {
- for &use_h0 in &[true, false] {
- for &case in CASES {
- let expected = run_case::<Cpu, f32>(case, bv, use_h0, DataType::F32, |x| x);
- for_each_non_cpu_backend!(|B| {
- let output = run_case::<B, f32>(case, bv, use_h0, DataType::F32, |x| x);
- assert_eq_float(&expected, &output, 1e-4, &format!("{} BV{bv} h0={use_h0}", case.name));
- });
- }
- }
- }
-
- for &bv in BVS {
- for &use_h0 in &[true, false] {
- for &case in CASES {
- let expected = run_case::<Cpu, bf16>(case, bv, use_h0, DataType::BF16, bf16::from_f32);
- for_each_non_cpu_backend!(|B| {
- let output = run_case::<B, bf16>(case, bv, use_h0, DataType::BF16, bf16::from_f32);
- assert_eq_float(&expected, &output, 5e-4, &format!("{} BV{bv} h0={use_h0}", case.name));
- });
- }
- }
- }
-}
diff --git a/crates/uzu-engine/unit/backends/common/kernel/gdn/tree_verify/tree_verify_test.rs b/crates/uzu-engine/unit/backends/common/kernel/gdn/tree_verify/tree_verify_test.rs
new file mode 100644
index 000000000..b99a2e72f
--- /dev/null
+++ b/crates/uzu-engine/unit/backends/common/kernel/gdn/tree_verify/tree_verify_test.rs
@@ -0,0 +1,89 @@
+#![cfg(backend = "metal")]
+
+use futures::executor::block_on;
+use half::bf16;
+use num_traits::Float;
+use uzu_engine_macros::uzu_test;
+
+use crate::{
+ backends::{
+ common::{
+ Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferPending, Context, Kernels,
+ kernel::TreeVerifyKernel,
+ },
+ cpu::Cpu,
+ metal::Metal,
+ },
+ data_type::DataTypeRepr,
+ tests::{
+ assert::assert_eq_float,
+ helpers::{buffer_to_vec, create_buffer_with_data},
+ },
+};
+
+const K_HEADS: usize = 2;
+const V_HEADS: usize = 6;
+const DIM: usize = 128;
+const CONV_DIM: usize = 5;
+const PROJ_DIM: usize = CONV_DIM + V_HEADS * DIM + 3;
+
+fn wave<T: Float>(
+ len: usize,
+ step: f32,
+ amplitude: f32,
+ bias: f32,
+) -> Vec<T> {
+ (0..len).map(|i| T::from((i as f32 * step).sin() * amplitude + bias).unwrap()).collect()
+}
+
+fn run<B: Backend, T: DataTypeRepr + Float>(n: usize) -> Vec<T> {
+ let context = block_on(B::Context::new()).expect("context");
+ let kernel = <B::Kernels as Kernels>::TreeVerifyKernel::new(&context, T::data_type(), (n > 16) as u32 + 1).unwrap();
+ let q = create_buffer_with_data::<B, T>(&context, &wave(n * K_HEADS * DIM, 0.017, 0.2, 0.01));
+ let k = create_buffer_with_data::<B, T>(&context, &wave(n * K_HEADS * DIM, 0.019, 0.18, -0.02));
+ let v = create_buffer_with_data::<B, T>(&context, &wave(n * V_HEADS * DIM, 0.023, 0.5, 0.03));
+ let in_proj = create_buffer_with_data::<B, T>(&context, &wave(n * PROJ_DIM, 0.031, 1.5, -0.1));
+ let log_decay = create_buffer_with_data::<B, f32>(&context, &wave(n * V_HEADS, 1.7, 0.2, -0.25));
+ let beta = create_buffer_with_data::<B, f32>(&context, &wave(n * V_HEADS, 0.013, 0.2, 0.45));
+ let h0 = create_buffer_with_data::<B, f32>(&context, &wave(V_HEADS * DIM * DIM, 0.007, 0.05, -0.01));
+ let norm_weight = create_buffer_with_data::<B, f32>(&context, &wave(DIM, 0.05, 0.2, 1.0));
+ // Caterpillar: odd nodes are leaves, even nodes the spine, so every 16-node block branches.
+ let trie: Vec<u32> = (0..n as u32).flat_map(|node| [node, [n as u32 - 1, node][node as usize % 2], 0]).collect();
+ let trie = create_buffer_with_data::<B, u32>(&context, &trie);
+ let mut o = create_buffer_with_data::<B, T>(&context, &vec![T::zero(); n * V_HEADS * DIM]);
+
+ let mut command_buffer = context.create_command_buffer(None, None).unwrap();
+ kernel.encode(
+ &q,
+ &k,
+ &v,
+ &trie,
+ &log_decay,
+ &beta,
+ &h0,
+ &in_proj,
+ &norm_weight,
+ &mut o,
+ 1.0,
+ 1e-6,
+ 1,
+ n as u32,
+ K_HEADS as u32,
+ V_HEADS as u32,
+ DIM as u32,
+ DIM as u32,
+ CONV_DIM as u32,
+ PROJ_DIM as u32,
+ &mut command_buffer,
+ );
+ block_on(command_buffer.end_encoding().submit().wait_until_completed()).unwrap();
+ buffer_to_vec(&o)
+}
+
+#[uzu_test]
+fn test_tree_verify() {
+ for n in [1, 16, 17, 32] {
+ assert_eq_float(&run::<Cpu, f32>(n), &run::<Metal, f32>(n), 5e-4, "f32");
+ assert_eq_float(&run::<Cpu, bf16>(n), &run::<Metal, bf16>(n), 1.6e-2, "bf16");
+ }
+}
diff --git a/crates/uzu-engine/unit/encodable_block/delta_net_tree_verify_bench.rs b/crates/uzu-engine/unit/encodable_block/delta_net_tree_verify_bench.rs
index ae23c82f3..485d21534 100644
--- a/crates/uzu-engine/unit/encodable_block/delta_net_tree_verify_bench.rs
+++ b/crates/uzu-engine/unit/encodable_block/delta_net_tree_verify_bench.rs
@@ -38,10 +38,13 @@ fn bench_delta_net_tree_verify(c: &mut Criterion) {
<<Metal as Backend>::Kernels as Kernels>::DeltaNetTreeVerify::new(context.as_ref(), &arguments).unwrap();
eprintln!("GDN tree verification PSO construction: {:.2} ms", started.elapsed().as_secs_f64() * 1e3);
let h0 = create_buffer_with_data::<Metal, f32>(&context, &vec![0.001; V_HEADS * HEAD_DIM * HEAD_DIM]);
+ let norm_weight = create_buffer_with_data::<Metal, f32>(&context, &vec![1.0; HEAD_DIM]);
+ const CONV_DIM: usize = 2 * K_HEADS * HEAD_DIM + V_HEADS * HEAD_DIM;
+ const TOTAL_PROJ_DIM: usize = CONV_DIM + V_HEADS * HEAD_DIM + 2 * V_HEADS;
let mut group = c.benchmark_group("Metal/Kernel/GDNTreeVerify");
group.sample_size(20).warm_up_time(Duration::from_millis(300)).measurement_time(Duration::from_secs(1));
- for tree_size in [32usize, 49, 64, 128] {
+ for tree_size in [16usize, 32] {
let mut command_buffer = context.create_command_buffer(None, None).unwrap();
let mut q = command_buffer.allocate_scratch(tree_size * K_HEADS * HEAD_DIM * size_of::<bf16>()).unwrap();
q.copyin(&vec![bf16::from_f32(0.01); tree_size * K_HEADS * HEAD_DIM]);
@@ -49,6 +52,8 @@ fn bench_delta_net_tree_verify(c: &mut Criterion) {
k.copyin(&vec![bf16::from_f32(0.02); tree_size * K_HEADS * HEAD_DIM]);
let mut v = command_buffer.allocate_scratch(tree_size * V_HEADS * HEAD_DIM * size_of::<bf16>()).unwrap();
v.copyin(&vec![bf16::from_f32(0.03); tree_size * V_HEADS * HEAD_DIM]);
+ let mut in_proj = command_buffer.allocate_scratch(tree_size * TOTAL_PROJ_DIM * size_of::<bf16>()).unwrap();
+ in_proj.copyin(&vec![bf16::from_f32(0.04); tree_size * TOTAL_PROJ_DIM]);
let trie_nodes = (0..tree_size as u32).flat_map(|node| [node, tree_size as u32 - 1, node]).collect::<Vec<_>>();
let mut log_decay = command_buffer.allocate_scratch(tree_size * V_HEADS * size_of::<f32>()).unwrap();
log_decay.copyin(&vec![-0.01f32; tree_size * V_HEADS]);
@@ -68,6 +73,11 @@ fn bench_delta_net_tree_verify(c: &mut Criterion) {
log_decay: &log_decay,
beta: &beta,
h0: &h0,
+ in_proj: &in_proj,
+ norm_weight: &norm_weight,
+ conv_dim: CONV_DIM as u32,
+ total_proj_dim: TOTAL_PROJ_DIM as u32,
+ norm_epsilon: 1e-6,
tree_size: tree_size as u32,
},
command_buffer,