Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

GDN speculation (#609)

CCYeh8377b7657bbd
diff --git a/crates/backend-uzu/src/backends/common/kernel/delta_net_tree_verify.rs b/crates/backend-uzu/src/backends/common/kernel/delta_net_tree_verify.rs
new file mode 100644
index 000000000..e9d14bae9
--- /dev/null
+++ b/crates/backend-uzu/src/backends/common/kernel/delta_net_tree_verify.rs
@@ -0,0 +1,42 @@
+use std::convert::Infallible;
+
+use crate::{
+ backends::common::{Allocation, Backend, Encoder, Kernels},
+ encodable_block::mixer::delta_net::tree_verify::{TreeVerifyEncodeArguments, TreeVerifyNewArguments},
+};
+
+pub trait DeltaNetTreeVerify<B: Backend<Kernels: Kernels<DeltaNetTreeVerify = Self>>>: Sized {
+ fn is_supported(context: &B::Context) -> bool;
+
+ fn new(
+ context: &B::Context,
+ arguments: &TreeVerifyNewArguments,
+ ) -> Result<Self, B::Error>;
+
+ fn encode(
+ &self,
+ arguments: TreeVerifyEncodeArguments<'_, B>,
+ encoder: &mut Encoder<B>,
+ ) -> Result<Allocation<B>, B::Error>;
+}
+
+impl<B: Backend<Kernels: Kernels<DeltaNetTreeVerify = Infallible>>> DeltaNetTreeVerify<B> for Infallible {
+ fn is_supported(_context: &B::Context) -> bool {
+ false
+ }
+
+ fn new(
+ _context: &B::Context,
+ _arguments: &TreeVerifyNewArguments,
+ ) -> Result<Self, B::Error> {
+ unreachable!("unsupported DeltaNet tree verifier should not be constructed")
+ }
+
+ fn encode(
+ &self,
+ _arguments: TreeVerifyEncodeArguments<'_, B>,
+ _encoder: &mut Encoder<B>,
+ ) -> Result<Allocation<B>, B::Error> {
+ unreachable!("unsupported DeltaNet tree verifier cannot encode")
+ }
+}
diff --git a/crates/backend-uzu/src/backends/common/kernel/mod.rs b/crates/backend-uzu/src/backends/common/kernel/mod.rs
index 59cfe359f..9d0d9c6d3 100644
--- a/crates/backend-uzu/src/backends/common/kernel/mod.rs
+++ b/crates/backend-uzu/src/backends/common/kernel/mod.rs
@@ -2,6 +2,7 @@ use crate::backends::common::Backend;
pub mod attention_gemm;
pub mod delta_net_chunked_prefill;
+pub mod delta_net_tree_verify;
pub mod matmul;
include!(concat!(env!("OUT_DIR"), "/traits.rs"));
@@ -12,6 +13,7 @@ pub trait Kernels: Sized {
autogen_kernels!();
type AttentionGemmCore: attention_gemm::AttentionGemmCore<Self::Backend>;
type DeltaNetChunkedPrefill: delta_net_chunked_prefill::DeltaNetChunkedPrefill<Self::Backend>;
+ type DeltaNetTreeVerify: delta_net_tree_verify::DeltaNetTreeVerify<Self::Backend>;
type MatmulKernel: matmul::MatmulKernel<Backend = Self::Backend>;
}
diff --git a/crates/backend-uzu/src/backends/cpu/kernel/gdn/conv_scan.rs b/crates/backend-uzu/src/backends/cpu/kernel/gdn/conv_scan.rs
index 36bc40ea5..984f53d0d 100644
--- a/crates/backend-uzu/src/backends/cpu/kernel/gdn/conv_scan.rs
+++ b/crates/backend-uzu/src/backends/cpu/kernel/gdn/conv_scan.rs
@@ -1,4 +1,4 @@
-use half::{bf16, f16};
+use half::bf16;
use num_traits::Float;
use proc_macros::kernel;
@@ -14,7 +14,7 @@ use crate::{array::ArrayElement, backends::common::gpu_types::ActivationType};
// in_proj: [suffix_len, out_stride] — first conv_dim channels overwritten
// state_out: [conv_dim, state_stride]
#[kernel(DeltaNetConvScan)]
-#[variants(T, f32, f16, bf16)]
+#[variants(T, f32, bf16)]
pub fn delta_net_conv_scan<T: ArrayElement + Float>(
conv_padded: *const f32,
conv_weight: *const f32,
diff --git a/crates/backend-uzu/src/backends/cpu/kernel/gdn/conv_update.rs b/crates/backend-uzu/src/backends/cpu/kernel/gdn/conv_update.rs
index 3da868b84..203b54f31 100644
--- a/crates/backend-uzu/src/backends/cpu/kernel/gdn/conv_update.rs
+++ b/crates/backend-uzu/src/backends/cpu/kernel/gdn/conv_update.rs
@@ -1,4 +1,4 @@
-use half::{bf16, f16};
+use half::bf16;
use num_traits::Float;
use proc_macros::kernel;
@@ -6,7 +6,7 @@ use crate::{array::ArrayElement, backends::common::gpu_types::ActivationType};
// Single-token causal conv1d with SiLU, in-place.
#[kernel(DeltaNetConvUpdate)]
-#[variants(T, f32, f16, bf16)]
+#[variants(T, f32, bf16)]
pub fn delta_net_conv_update<T: ArrayElement + Float>(
conv_weight: *const f32,
#[optional(has_bias)] bias: Option<*const f32>,
diff --git a/crates/backend-uzu/src/backends/cpu/kernel/gdn/norm_gate.rs b/crates/backend-uzu/src/backends/cpu/kernel/gdn/norm_gate.rs
index eca1bf120..e305e5c00 100644
--- a/crates/backend-uzu/src/backends/cpu/kernel/gdn/norm_gate.rs
+++ b/crates/backend-uzu/src/backends/cpu/kernel/gdn/norm_gate.rs
@@ -1,4 +1,4 @@
-use half::{bf16, f16};
+use half::bf16;
use num_traits::Float;
use proc_macros::kernel;
@@ -14,7 +14,7 @@ use crate::{array::ArrayElement, backends::common::gpu_types::ActivationType};
// where inv_rms = rsqrt(mean(in_out^2) + epsilon)
// and z is at in_proj[token * total_proj_dim + conv_dim + hv * head_v_dim + i]
#[kernel(DeltaNetNormGate)]
-#[variants(T, f32, f16, bf16)]
+#[variants(T, f32, bf16)]
pub fn delta_net_norm_gate<T: ArrayElement + Float>(
in_out: *mut T,
in_proj: *const T,
diff --git a/crates/backend-uzu/src/backends/cpu/kernel/gdn/prefill.rs b/crates/backend-uzu/src/backends/cpu/kernel/gdn/prefill.rs
index 12a1a8c9f..7d7fab7e9 100644
--- a/crates/backend-uzu/src/backends/cpu/kernel/gdn/prefill.rs
+++ b/crates/backend-uzu/src/backends/cpu/kernel/gdn/prefill.rs
@@ -1,11 +1,11 @@
-use half::{bf16, f16};
+use half::bf16;
use num_traits::Float;
use proc_macros::kernel;
use crate::array::ArrayElement;
#[kernel(DeltaNetPrefill)]
-#[variants(T, f32, f16, bf16)]
+#[variants(T, f32, bf16)]
#[variants(HEAD_K_DIM, 128)]
pub fn delta_net_prefill<T: ArrayElement + Float, const HEAD_K_DIM: u32>(
q_norm: *const f32,
diff --git a/crates/backend-uzu/src/backends/cpu/kernel/gdn/prefill_prep.rs b/crates/backend-uzu/src/backends/cpu/kernel/gdn/prefill_prep.rs
index 00a2f601b..a87bbbc2d 100644
--- a/crates/backend-uzu/src/backends/cpu/kernel/gdn/prefill_prep.rs
+++ b/crates/backend-uzu/src/backends/cpu/kernel/gdn/prefill_prep.rs
@@ -1,4 +1,4 @@
-use half::{bf16, f16};
+use half::bf16;
use num_traits::Float;
use proc_macros::kernel;
@@ -6,14 +6,16 @@ use crate::array::ArrayElement;
// Pre-compute L2-normalized q/k, beta, and decay for all tokens.
#[kernel(DeltaNetPrefillPrep)]
-#[variants(T, f32, f16, bf16)]
+#[variants(T, f32, bf16)]
+#[variants(QKT, f32, bf16)]
#[variants(HEAD_K_DIM, 128)]
-pub fn delta_net_prefill_prep<T: ArrayElement + Float, const HEAD_K_DIM: u32>(
+pub fn delta_net_prefill_prep<T: ArrayElement + Float, QKT: ArrayElement + Float, const HEAD_K_DIM: u32>(
in_proj: *const T,
- a_log: *const T,
- dt_bias: *const T,
- q_norm_out: *mut f32,
- k_norm_out: *mut f32,
+ a_log: *const f32,
+ dt_bias: *const f32,
+ q_norm_out: *mut QKT,
+ k_norm_out: *mut QKT,
+ #[optional(write_compact_v)] compact_v_out: Option<*mut T>,
beta_out: *mut f32,
decay_out: *mut f32,
num_v_heads: u32,
@@ -22,7 +24,10 @@ pub fn delta_net_prefill_prep<T: ArrayElement + Float, const HEAD_K_DIM: u32>(
value_dim: u32,
suffix_len: u32,
#[specialize] write_log_decay: bool,
+ #[specialize] write_compact_v: bool,
) {
+ assert_eq!(compact_v_out.is_some(), write_compact_v, "compact V output presence mismatch");
+
let num_v_heads = num_v_heads as usize;
let num_k_heads = num_k_heads as usize;
let head_k_dim = HEAD_K_DIM as usize;
@@ -36,6 +41,14 @@ pub fn delta_net_prefill_prep<T: ArrayElement + Float, const HEAD_K_DIM: u32>(
for token in 0..suffix_len {
let tok_offset = token * total_proj_dim;
+ if let Some(compact_v_out) = compact_v_out {
+ unsafe {
+ in_proj
+ .add(tok_offset + 2 * key_dim)
+ .copy_to_nonoverlapping(compact_v_out.add(token * value_dim), value_dim)
+ };
+ }
+
for hk in 0..num_k_heads {
// Load and L2-normalize q
let q_off = tok_offset + hk * head_k_dim;
@@ -48,7 +61,9 @@ pub fn delta_net_prefill_prep<T: ArrayElement + Float, const HEAD_K_DIM: u32>(
let q_scale = 1.0 / (head_k_dim as f32).sqrt();
for j in 0..head_k_dim {
let v = unsafe { (*in_proj.add(q_off + j)).to_f32().unwrap() };
- unsafe { *q_norm_out.add(token * key_dim + hk * head_k_dim + j) = v * q_inv * q_scale };
+ unsafe {
+ *q_norm_out.add(token * key_dim + hk * head_k_dim + j) = QKT::from(v * q_inv * q_scale).unwrap()
+ };
}
// Load and L2-normalize k
@@ -61,7 +76,7 @@ pub fn delta_net_prefill_prep<T: ArrayElement + Float, const HEAD_K_DIM: u32>(
let k_inv = 1.0 / (k_sq + 1e-6).sqrt();
for j in 0..head_k_dim {
let v = unsafe { (*in_proj.add(k_off + j)).to_f32().unwrap() };
- unsafe { *k_norm_out.add(token * key_dim + hk * head_k_dim + j) = v * k_inv };
+ unsafe { *k_norm_out.add(token * key_dim + hk * head_k_dim + j) = QKT::from(v * k_inv).unwrap() };
}
// Beta and decay for each v-head of this k-head
@@ -71,8 +86,8 @@ pub fn delta_net_prefill_prep<T: ArrayElement + Float, const HEAD_K_DIM: u32>(
let beta_raw = unsafe { (*in_proj.add(tok_offset + conv_dim + value_dim + hv)).to_f32().unwrap() };
let beta = 1.0 / (1.0 + (-beta_raw).exp());
- let a_log_val = unsafe { (*a_log.add(hv)).to_f32().unwrap() };
- let dt_bias_val = unsafe { (*dt_bias.add(hv)).to_f32().unwrap() };
+ let a_log_val = unsafe { *a_log.add(hv) };
+ let dt_bias_val = unsafe { *dt_bias.add(hv) };
let a_raw =
unsafe { (*in_proj.add(tok_offset + conv_dim + value_dim + num_v_heads + hv)).to_f32().unwrap() };
let sp_in = a_raw + dt_bias_val;
diff --git a/crates/backend-uzu/src/backends/cpu/kernel/gdn/tree_verify/mod.rs b/crates/backend-uzu/src/backends/cpu/kernel/gdn/tree_verify/mod.rs
index fd25fad36..e4b9b6cf5 100644
--- a/crates/backend-uzu/src/backends/cpu/kernel/gdn/tree_verify/mod.rs
+++ b/crates/backend-uzu/src/backends/cpu/kernel/gdn/tree_verify/mod.rs
@@ -1,6 +1,6 @@
pub mod conv_scan;
pub mod out;
-pub mod prefix_beta;
+pub mod prefix;
pub mod state_advance;
pub mod tree_gram;
pub mod tree_update_solve;
diff --git a/crates/backend-uzu/src/backends/cpu/kernel/gdn/tree_verify/out.rs b/crates/backend-uzu/src/backends/cpu/kernel/gdn/tree_verify/out.rs
index d3ac41635..c18492cc3 100644
--- a/crates/backend-uzu/src/backends/cpu/kernel/gdn/tree_verify/out.rs
+++ b/crates/backend-uzu/src/backends/cpu/kernel/gdn/tree_verify/out.rs
@@ -5,15 +5,16 @@ use proc_macros::kernel;
use crate::array::ArrayElement;
#[kernel(BuildTreeOut)]
-#[variants(T, f32, bf16)]
-pub fn build_tree_out<T: ArrayElement + Float>(
- q: *const T,
+#[variants(QKT, f32, bf16)]
+#[variants(OutputT, f32, bf16)]
+pub fn build_tree_out<QKT: ArrayElement + Float, OutputT: ArrayElement + Float>(
+ q: *const QKT,
prefix: *const f32,
qkd: *const f32,
- u: *const T,
+ u: *const f32,
#[optional(use_h0)] h0: Option<*const f32>,
#[optional(use_h0)] h0_indices: Option<*const i32>,
- o: *mut T,
+ o: *mut OutputT,
scale: f32,
batch_size: u32,
tree_size: u32,
@@ -75,12 +76,12 @@ pub fn build_tree_out<T: ArrayElement + Float>(
for col in 0..tree_size {
let qkd_value = unsafe { *qkd.add(qkd_base + row * tree_size + col) };
- let u_value = unsafe { (*u.add(u_base + col * head_v_dim + value_col)).to_f32().unwrap() };
+ let u_value = unsafe { *u.add(u_base + col * head_v_dim + value_col) };
acc += qkd_value * u_value;
}
unsafe {
- *o.add(out_base + row * value_heads * head_v_dim + value_col) = T::from(acc).unwrap();
+ *o.add(out_base + row * value_heads * head_v_dim + value_col) = OutputT::from(acc).unwrap();
}
}
}
diff --git a/crates/backend-uzu/src/backends/cpu/kernel/gdn/tree_verify/prefix.rs b/crates/backend-uzu/src/backends/cpu/kernel/gdn/tree_verify/prefix.rs
new file mode 100644
index 000000000..d225a61a2
--- /dev/null
+++ b/crates/backend-uzu/src/backends/cpu/kernel/gdn/tree_verify/prefix.rs
@@ -0,0 +1,39 @@
+use proc_macros::kernel;
+
+use crate::backends::common::gpu_types::trie::TrieNode;
+
+#[kernel(BuildTreePrefix)]
+pub fn build_tree_prefix(
+ trie: *const TrieNode,
+ log_decay: *const f32,
+ prefix: *mut f32,
+ batch_size: u32,
+ tree_size: u32,
+ value_heads: u32,
+) {
+ let batch_size = batch_size as usize;
+ let tree_size = tree_size as usize;
+ let value_heads = value_heads as usize;
+
+ for batch in 0..batch_size {
+ let trie_batch = batch * tree_size;
+ let batch_offset = batch * tree_size * value_heads;
+
+ for row in 0..tree_size {
+ let row_idx = row as u32;
+ for head in 0..value_heads {
+ let mut sum = 0.0f32;
+ for col in 0..tree_size {
+ // col is an ancestor-or-self of row iff row falls in col's
+ // DFS subtree interval [trie_start, trie_end].
+ let node = unsafe { *trie.add(trie_batch + col) };
+ if row_idx < node.trie_start || row_idx > node.trie_end {
+ continue;
+ }
+ sum += unsafe { *log_decay.add(batch_offset + col * value_heads + head) };
+ }
+ unsafe { *prefix.add(batch_offset + row * value_heads + head) = sum };
+ }
+ }
+ }
+}
diff --git a/crates/backend-uzu/src/backends/cpu/kernel/gdn/tree_verify/prefix_beta.rs b/crates/backend-uzu/src/backends/cpu/kernel/gdn/tree_verify/prefix_beta.rs
deleted file mode 100644
index 086119bd2..000000000
--- a/crates/backend-uzu/src/backends/cpu/kernel/gdn/tree_verify/prefix_beta.rs
+++ /dev/null
@@ -1,58 +0,0 @@
-use half::{bf16, f16};
-use num_traits::Float;
-use proc_macros::kernel;
-
-use crate::{
- array::ArrayElement,
- backends::common::gpu_types::{ActivationType, trie::TrieNode},
-};
-
-#[kernel(BuildPrefixBeta)]
-#[variants(T, f32, f16, bf16)]
-pub fn build_prefix_beta<T: ArrayElement + Float>(
- trie: *const TrieNode,
- a_transposed: *const T,
- b: *const T,
- a_log: *const f32,
- dt_bias: *const f32,
- prefix: *mut f32,
- beta: *mut f32,
- batch_size: u32,
- tree_size: u32,
- value_heads: u32,
-) {
- let batch_size = batch_size as usize;
- let tree_size = tree_size as usize;
- let value_heads = value_heads as usize;
-
- for batch in 0..batch_size {
- let trie_batch = batch * tree_size;
- let batch_offset = batch * tree_size * value_heads;
-
- for row in 0..tree_size {
- let row_idx = row as u32;
- for head in 0..value_heads {
- let out_idx = batch_offset + row * value_heads + head;
- let b_val = unsafe { *b.add(out_idx) }.to_f32().unwrap();
- unsafe { *beta.add(out_idx) = 1.0 / (1.0 + (-b_val).exp()) };
-
- let dt = unsafe { *dt_bias.add(head) };
- let scale = unsafe { *a_log.add(head) }.exp();
-
- let mut sum = 0.0f32;
- for col in 0..tree_size {
- // col is an ancestor-or-self of row iff row falls in col's
- // DFS subtree interval [trie_start, trie_end].
- let node = unsafe { *trie.add(trie_batch + col) };
- if row_idx < node.trie_start || row_idx > node.trie_end {
- continue;
- }
- let a_val = unsafe { *a_transposed.add(batch_offset + head * tree_size + col) }.to_f32().unwrap();
- let sp = ActivationType::SOFTPLUS.activate(a_val + dt);
- sum -= scale * sp;
- }
- unsafe { *prefix.add(out_idx) = sum };
- }
- }
- }
-}
diff --git a/crates/backend-uzu/src/backends/cpu/kernel/gdn/tree_verify/state_advance.rs b/crates/backend-uzu/src/backends/cpu/kernel/gdn/tree_verify/state_advance.rs
index 81ba1b006..095970463 100644
--- a/crates/backend-uzu/src/backends/cpu/kernel/gdn/tree_verify/state_advance.rs
+++ b/crates/backend-uzu/src/backends/cpu/kernel/gdn/tree_verify/state_advance.rs
@@ -8,15 +8,15 @@ use crate::array::ArrayElement;
#[variants(T, f32, bf16)]
#[variants(HEAD_K_DIM, 128)]
pub fn state_advance<T: ArrayElement + Float, const HEAD_K_DIM: u32>(
- k_norm: *const f32,
+ k_norm: *const T,
v: *const T,
log_decay_buf: *const f32,
beta_buf: *const f32,
accepted_indices: *const u32,
state: *mut f32,
accepted_len: u32,
- num_v_heads: u32,
- num_k_heads: u32,
+ #[specialize] num_v_heads: u32,
+ #[specialize] num_k_heads: u32,
) {
let accepted_len = accepted_len as usize;
let num_v_heads = num_v_heads as usize;
@@ -42,13 +42,16 @@ pub fn state_advance<T: ArrayElement + Float, const HEAD_K_DIM: u32>(
for dk_idx in 0..head_k_dim {
let state_element_ptr = unsafe { state.add(state_row_offset + dk_idx) };
unsafe { *state_element_ptr *= decay };
- kv_mem += unsafe { *state_element_ptr * *k_norm.add(k_offset + dk_idx) };
+ kv_mem += unsafe { *state_element_ptr * (*k_norm.add(k_offset + dk_idx)).to_f32().unwrap() };
}
let v_value =
unsafe { (*v.add(tree_idx * value_dim + hv_idx * head_v_dim + dv_idx)).to_f32().unwrap() };
let delta = beta * (v_value - kv_mem);
for dk_idx in 0..head_k_dim {
- unsafe { *state.add(state_row_offset + dk_idx) += *k_norm.add(k_offset + dk_idx) * delta };
+ unsafe {
+ *state.add(state_row_offset + dk_idx) +=
+ (*k_norm.add(k_offset + dk_idx)).to_f32().unwrap() * delta
+ };
}
}
}
diff --git a/crates/backend-uzu/src/backends/cpu/kernel/gdn/tree_verify/tree_gram.rs b/crates/backend-uzu/src/backends/cpu/kernel/gdn/tree_verify/tree_gram.rs
index 7cbf882ea..ae1dce607 100644
--- a/crates/backend-uzu/src/backends/cpu/kernel/gdn/tree_verify/tree_gram.rs
+++ b/crates/backend-uzu/src/backends/cpu/kernel/gdn/tree_verify/tree_gram.rs
@@ -15,7 +15,7 @@ pub fn build_tree_gram<T: ArrayElement + Float, const USE_MXU: bool>(
trie: *const TrieNode,
prefix: *const f32,
beta: *const f32,
- #[optional(use_h0)] h0: Option<*const T>,
+ #[optional(use_h0)] h0: Option<*const f32>,
#[optional(use_h0)] h0_idx: Option<*const i32>,
a_packed: *mut f32,
qkd: *mut f32,
@@ -152,7 +152,7 @@ pub fn build_tree_gram<T: ArrayElement + Float, const USE_MXU: bool>(
let mut sum = 0.0f32;
for d in 0..head_k_dim {
sum += unsafe { (*k.add(k_off + d)).to_f32().unwrap() }
- * unsafe { (*h0.add(h0_head + dv * head_k_dim + d)).to_f32().unwrap() };
+ * unsafe { *h0.add(h0_head + dv * head_k_dim + d) };
}
unsafe { *kh0.add(kh0_off + dv) = sum };
}
diff --git a/crates/backend-uzu/src/backends/cpu/kernel/gdn/tree_verify/tree_update_solve.rs b/crates/backend-uzu/src/backends/cpu/kernel/gdn/tree_verify/tree_update_solve.rs
index 0d25d0a30..ef46d2c38 100644
--- a/crates/backend-uzu/src/backends/cpu/kernel/gdn/tree_verify/tree_update_solve.rs
+++ b/crates/backend-uzu/src/backends/cpu/kernel/gdn/tree_verify/tree_update_solve.rs
@@ -9,8 +9,7 @@ use crate::array::ArrayElement;
#[kernel(TreeUpdateSolve)]
#[variants(T, f32, bf16)]
#[variants(BV, 16, 32)]
-#[variants(USE_MXU, false)]
-pub fn tree_update_solve<T: ArrayElement + Float, const BV: u32, const USE_MXU: bool>(
+pub fn tree_update_solve<T: ArrayElement + Float, const BV: u32>(
#[optional(use_h0)] kh0: Option<*const f32>,
v: *const T,
prefix: *const f32,
diff --git a/crates/backend-uzu/src/backends/cpu/kernel/gdn/update.rs b/crates/backend-uzu/src/backends/cpu/kernel/gdn/update.rs
index 9ea29360f..26b323d5b 100644
--- a/crates/backend-uzu/src/backends/cpu/kernel/gdn/update.rs
+++ b/crates/backend-uzu/src/backends/cpu/kernel/gdn/update.rs
@@ -1,4 +1,4 @@
-use half::{bf16, f16};
+use half::bf16;
use num_traits::Float;
use proc_macros::kernel;
@@ -11,7 +11,7 @@ use crate::{array::ArrayElement, backends::common::gpu_types::ActivationType};
// 3. For each v-dim: read state, compute output, update state with delta rule
// 4. RMSNorm output, gate with SiLU(z), write result
#[kernel(DeltaNetUpdate)]
-#[variants(T, f32, f16, bf16)]
+#[variants(T, f32, bf16)]
#[variants(HEAD_K_DIM, 128)]
pub fn delta_net_update<T: ArrayElement + Float, const HEAD_K_DIM: u32>(
in_proj: *const T,
diff --git a/crates/backend-uzu/src/backends/cpu/kernel/mod.rs b/crates/backend-uzu/src/backends/cpu/kernel/mod.rs
index 3d903e94d..60227949b 100644
--- a/crates/backend-uzu/src/backends/cpu/kernel/mod.rs
+++ b/crates/backend-uzu/src/backends/cpu/kernel/mod.rs
@@ -33,5 +33,6 @@ impl Kernels for CpuKernels {
autogen_kernels!();
type AttentionGemmCore = Infallible;
type DeltaNetChunkedPrefill = Infallible;
+ type DeltaNetTreeVerify = Infallible;
type MatmulKernel = matmul::MatmulCpuKernel;
}
diff --git a/crates/backend-uzu/src/backends/metal/kernel/gdn/chunked/mod.rs b/crates/backend-uzu/src/backends/metal/kernel/gdn/chunked/mod.rs
index 0ec83e804..937f0c00a 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/gdn/chunked/mod.rs
+++ b/crates/backend-uzu/src/backends/metal/kernel/gdn/chunked/mod.rs
@@ -54,7 +54,7 @@ impl DeltaNetChunkedPrefill<Metal> for MetalDeltaNetChunkedPrefill {
Ok(Some(Self {
min_t: MXU_MIN_T,
- prep: DeltaNetPrefillPrepMetalKernel::new(context, outer_data_type, head_dim, true)?,
+ prep: DeltaNetPrefillPrepMetalKernel::new(context, outer_data_type, DataType::F32, head_dim, true, false)?,
cumsum: DeltaNetChunkedCumsumMetalKernel::new(context, CHUNK_SIZE as u32)?,
gram: DeltaNetChunkedGramMetalKernel::new(context, head_dim, CHUNK_SIZE as u32)?,
a_diag_inv: DeltaNetChunkedADiagInvMetalKernel::new(context, CHUNK_SIZE as u32)?,
@@ -124,6 +124,7 @@ impl DeltaNetChunkedPrefill<Metal> for MetalDeltaNetChunkedPrefill {
args.dt_bias,
&mut q_norm,
&mut k_norm,
+ None::<&mut crate::backends::common::Allocation<Metal>>,
&mut beta,
&mut log_decay,
args.num_heads,
diff --git a/crates/backend-uzu/src/backends/metal/kernel/gdn/conv_scan.metal b/crates/backend-uzu/src/backends/metal/kernel/gdn/conv_scan.metal
index 8aa1f1d0b..44aebdbe0 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/gdn/conv_scan.metal
+++ b/crates/backend-uzu/src/backends/metal/kernel/gdn/conv_scan.metal
@@ -15,7 +15,7 @@ using namespace metal;
// state_out: [conv_dim, state_stride]
template <typename T>
-VARIANTS(T, float, half, bfloat)
+VARIANTS(T, float, bfloat)
PUBLIC KERNEL(DeltaNetConvScan)(
device const float* conv_padded,
device const float* conv_weight,
diff --git a/crates/backend-uzu/src/backends/metal/kernel/gdn/conv_update.metal b/crates/backend-uzu/src/backends/metal/kernel/gdn/conv_update.metal
index d9450a0f6..3547af513 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/gdn/conv_update.metal
+++ b/crates/backend-uzu/src/backends/metal/kernel/gdn/conv_update.metal
@@ -6,7 +6,7 @@ using namespace metal;
// Single-token causal conv1d with SiLU, in-place.
template <typename T>
-VARIANTS(T, float, half, bfloat)
+VARIANTS(T, float, bfloat)
PUBLIC KERNEL(DeltaNetConvUpdate)(
device const float* conv_weight,
device const float* bias OPTIONAL(has_bias),
diff --git a/crates/backend-uzu/src/backends/metal/kernel/gdn/mod.rs b/crates/backend-uzu/src/backends/metal/kernel/gdn/mod.rs
index 60c62629e..2082e1623 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/gdn/mod.rs
+++ b/crates/backend-uzu/src/backends/metal/kernel/gdn/mod.rs
@@ -1 +1,2 @@
pub mod chunked;
+pub mod tree_verify;
diff --git a/crates/backend-uzu/src/backends/metal/kernel/gdn/norm_gate.metal b/crates/backend-uzu/src/backends/metal/kernel/gdn/norm_gate.metal
index 7933c475f..281c153ff 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/gdn/norm_gate.metal
+++ b/crates/backend-uzu/src/backends/metal/kernel/gdn/norm_gate.metal
@@ -15,7 +15,7 @@ using namespace metal;
#define HEAD_V_DIM 128
template <typename T>
-VARIANTS(T, float, half, bfloat)
+VARIANTS(T, float, bfloat)
PUBLIC KERNEL(DeltaNetNormGate)(
device T* in_out,
device const T* in_proj,
diff --git a/crates/backend-uzu/src/backends/metal/kernel/gdn/prefill.metal b/crates/backend-uzu/src/backends/metal/kernel/gdn/prefill.metal
index 8f24c8c02..82db56b15 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/gdn/prefill.metal
+++ b/crates/backend-uzu/src/backends/metal/kernel/gdn/prefill.metal
@@ -13,7 +13,7 @@ using namespace metal;
static_assert(PREFILL_THREADS % METAL_SIMD_SIZE == 0, "PREFILL_THREADS must be a multiple of METAL_SIMD_SIZE");
template <typename T, uint HEAD_K_DIM>
-VARIANTS(T, float, half, bfloat)
+VARIANTS(T, float, bfloat)
VARIANTS(HEAD_K_DIM, 128)
PUBLIC KERNEL(DeltaNetPrefill)(
device const float* q_norm,
diff --git a/crates/backend-uzu/src/backends/metal/kernel/gdn/prefill_prep.metal b/crates/backend-uzu/src/backends/metal/kernel/gdn/prefill_prep.metal
index ee6f9cd92..50ed4b4d0 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/gdn/prefill_prep.metal
+++ b/crates/backend-uzu/src/backends/metal/kernel/gdn/prefill_prep.metal
@@ -5,15 +5,17 @@
using namespace metal;
-template <typename T, uint HEAD_K_DIM>
-VARIANTS(T, float, half, bfloat)
+template <typename T, typename QKT, uint HEAD_K_DIM>
+VARIANTS(T, float, bfloat)
+VARIANTS(QKT, float, bfloat)
VARIANTS(HEAD_K_DIM, 128)
PUBLIC KERNEL(DeltaNetPrefillPrep)(
device const T* in_proj,
device const float* a_log,
device const float* dt_bias,
- device float* q_norm_out,
- device float* k_norm_out,
+ device QKT* q_norm_out,
+ device QKT* k_norm_out,
+ device T* compact_v_out OPTIONAL(write_compact_v),
device float* beta_out,
device float* decay_out,
constant const uint& num_v_heads,
@@ -24,6 +26,7 @@ PUBLIC KERNEL(DeltaNetPrefillPrep)(
const uint token_idx GROUPS(suffix_len),
const uint hk_idx GROUPS(num_k_heads),
const bool write_log_decay SPECIALIZE,
+ const bool write_compact_v SPECIALIZE,
const uint lane THREADS(METAL_SIMD_SIZE)
) {
static_assert(HEAD_K_DIM % METAL_SIMD_SIZE == 0, "HEAD_K_DIM must be a multiple of METAL_SIMD_SIZE");
@@ -57,8 +60,8 @@ PUBLIC KERNEL(DeltaNetPrefillPrep)(
const uint out_base = token_idx * key_dim + hk_idx * HEAD_K_DIM;
for (uint i = 0; i < elems_per_thread; ++i) {
const uint idx = lane + i * METAL_SIMD_SIZE;
- q_norm_out[out_base + idx] = q_vals[i] * q_inv_norm * q_scale;
- k_norm_out[out_base + idx] = k_vals[i] * k_inv_norm;
+ q_norm_out[out_base + idx] = static_cast<QKT>(q_vals[i] * q_inv_norm * q_scale);
+ k_norm_out[out_base + idx] = static_cast<QKT>(k_vals[i] * k_inv_norm);
}
for (uint group = lane; group < groups_per_head; group += METAL_SIMD_SIZE) {
@@ -74,4 +77,15 @@ PUBLIC KERNEL(DeltaNetPrefillPrep)(
decay_out[token_idx * num_v_heads + hv] = fast::exp(log_decay);
}
}
+
+ if (write_compact_v) {
+ const uint head_v_dim = value_dim / num_v_heads;
+ for (uint group = 0; group < groups_per_head; ++group) {
+ const uint hv = hk_idx * groups_per_head + group;
+ for (uint dv = lane; dv < head_v_dim; dv += METAL_SIMD_SIZE) {
+ const uint v_idx = hv * head_v_dim + dv;
+ compact_v_out[token_idx * value_dim + v_idx] = in_proj[tok_offset + 2 * key_dim + v_idx];
+ }
+ }
+ }
}
diff --git a/crates/backend-uzu/src/backends/metal/kernel/gdn/tree_verify.rs b/crates/backend-uzu/src/backends/metal/kernel/gdn/tree_verify.rs
new file mode 100644
index 000000000..c786faad6
--- /dev/null
+++ b/crates/backend-uzu/src/backends/metal/kernel/gdn/tree_verify.rs
@@ -0,0 +1,166 @@
+use crate::{
+ array::size_for_shape,
+ backends::{
+ common::{
+ Allocation, Encoder, Kernels,
+ kernel::{
+ BuildTreeGramKernel, BuildTreeOutKernel, BuildTreePrefixKernel, TreeUpdateSolveKernel,
+ delta_net_tree_verify::DeltaNetTreeVerify,
+ },
+ },
+ metal::{Metal, MetalContext, device_tier::DeviceTier, error::MetalError, kernel::MetalKernels},
+ },
+ data_type::DataType,
+ encodable_block::mixer::delta_net::tree_verify::{TreeVerifyEncodeArguments, TreeVerifyNewArguments},
+};
+
+const TOKEN_BLOCK: usize = 16;
+const BLOCK_PAIR_WIDTH: usize = 2 * TOKEN_BLOCK;
+
+struct Layout {
+ tree_size: usize,
+ num_blocks: usize,
+ num_block_pairs: usize,
+ num_v_heads: usize,
+ head_v_dim: usize,
+}
+
+impl Layout {
+ const fn new(
+ tree_size: usize,
+ arguments: &TreeVerifyNewArguments,
+ ) -> Self {
+ let num_blocks = tree_size.div_ceil(TOKEN_BLOCK);
+ Self {
+ tree_size,
+ num_blocks,
+ num_block_pairs: num_blocks.div_ceil(2),
+ num_v_heads: arguments.num_v_heads,
+ head_v_dim: arguments.head_v_dim,
+ }
+ }
+
+ const fn a_packed_shape(&self) -> [usize; 5] {
+ [self.num_v_heads, self.num_blocks, self.num_block_pairs, TOKEN_BLOCK, BLOCK_PAIR_WIDTH]
+ }
+
+ const fn a_inverse_shape(&self) -> [usize; 4] {
+ [self.num_v_heads, self.num_blocks, TOKEN_BLOCK, TOKEN_BLOCK]
+ }
+}
+
+pub struct MetalDeltaNetTreeVerify {
+ arguments: TreeVerifyNewArguments,
+ prefix: <MetalKernels as Kernels>::BuildTreePrefixKernel,
+ gram: <MetalKernels as Kernels>::BuildTreeGramKernel,
+ solve: <MetalKernels as Kernels>::TreeUpdateSolveKernel,
+ out: <MetalKernels as Kernels>::BuildTreeOutKernel,
+}
+
+impl DeltaNetTreeVerify<Metal> for MetalDeltaNetTreeVerify {
+ fn is_supported(_context: &MetalContext) -> bool {
+ true
+ }
+
+ fn new(
+ context: &MetalContext,
+ arguments: &TreeVerifyNewArguments,
+ ) -> Result<Self, MetalError> {
+ let use_mxu = arguments.data_type == DataType::BF16 && context.supports_mxu();
+ let transposed_h0 =
+ !use_mxu && matches!(context.device_tier(), DeviceTier::SmallLegacy | DeviceTier::SmallApple8);
+ Ok(Self {
+ arguments: *arguments,
+ prefix: <MetalKernels as Kernels>::BuildTreePrefixKernel::new(context)?,
+ gram: <MetalKernels as Kernels>::BuildTreeGramKernel::new(context, arguments.data_type, use_mxu, true)?,
+ solve: <MetalKernels as Kernels>::TreeUpdateSolveKernel::new(context, arguments.data_type, 32, true)?,
+ out: <MetalKernels as Kernels>::BuildTreeOutKernel::new(
+ context,
+ arguments.data_type,
+ arguments.data_type,
+ use_mxu,
+ transposed_h0,
+ true,
+ )?,
+ })
+ }
+
+ fn encode(
+ &self,
+ arguments: TreeVerifyEncodeArguments<'_, Metal>,
+ encoder: &mut Encoder<Metal>,
+ ) -> Result<Allocation<Metal>, MetalError> {
+ let layout = Layout::new(arguments.tree_size, &self.arguments);
+ let mut h0_indices = encoder.allocate_constant(DataType::I32.size_in_bytes())?;
+ h0_indices.copyin(&[0i32]);
+ let mut scratch = |shape: &[usize], data_type| encoder.allocate_scratch(size_for_shape(shape, data_type));
+ let mut prefix = scratch(&[layout.tree_size, layout.num_v_heads], DataType::F32)?;
+ let mut a_packed = scratch(&layout.a_packed_shape(), DataType::F32)?;
+ let mut qkd = scratch(&[layout.num_v_heads, layout.tree_size, layout.tree_size], DataType::F32)?;
+ let mut a_inverse = scratch(&layout.a_inverse_shape(), DataType::F32)?;
+ let mut kh0 = scratch(&[layout.tree_size, layout.num_v_heads, layout.head_v_dim], DataType::F32)?;
+ let mut u = scratch(&[layout.num_v_heads, layout.tree_size, layout.head_v_dim], DataType::F32)?;
+ let mut output = scratch(&[layout.tree_size, layout.num_v_heads, layout.head_v_dim], self.arguments.data_type)?;
+ let tree_size = arguments.tree_size.try_into().expect("tree size exceeds u32");
+ let num_k_heads = self.arguments.num_k_heads.try_into().expect("K head count exceeds u32");
+ let num_v_heads = self.arguments.num_v_heads.try_into().expect("V head count exceeds u32");
+ let head_k_dim = self.arguments.head_k_dim.try_into().expect("K head dimension exceeds u32");
+ let head_v_dim = self.arguments.head_v_dim.try_into().expect("V head dimension exceeds u32");
+
+ self.prefix.encode(arguments.trie, arguments.log_decay, &mut prefix, 1, tree_size, num_v_heads, encoder);
+ self.gram.encode(
+ arguments.q,
+ arguments.k,
+ arguments.trie,
+ &prefix,
+ arguments.beta,
+ Some(arguments.h0),
+ Some(&h0_indices),
+ &mut a_packed,
+ &mut qkd,
+ &mut a_inverse,
+ Some(&mut kh0),
+ 1.0,
+ 1,
+ tree_size,
+ num_k_heads,
+ num_v_heads,
+ head_k_dim,
+ head_v_dim,
+ encoder,
+ );
+ self.solve.encode(
+ Some(&kh0),
+ arguments.v,
+ &prefix,
+ arguments.beta,
+ &a_packed,
+ &a_inverse,
+ Some(&h0_indices),
+ &mut u,
+ 1,
+ tree_size,
+ num_v_heads,
+ head_v_dim,
+ encoder,
+ );
+ self.out.encode(
+ arguments.q,
+ &prefix,
+ &qkd,
+ &u,
+ Some(arguments.h0),
+ Some(&h0_indices),
+ &mut output,
+ 1.0,
+ 1,
+ tree_size,
+ num_k_heads,
+ num_v_heads,
+ head_k_dim,
+ head_v_dim,
+ encoder,
+ );
+ Ok(output)
+ }
+}
diff --git a/crates/backend-uzu/src/backends/metal/kernel/gdn/tree_verify/build_tree_out_dispatch_helper.rs b/crates/backend-uzu/src/backends/metal/kernel/gdn/tree_verify/build_tree_out_dispatch_helper.rs
deleted file mode 100644
index a57e2f54e..000000000
--- a/crates/backend-uzu/src/backends/metal/kernel/gdn/tree_verify/build_tree_out_dispatch_helper.rs
+++ /dev/null
@@ -1,11 +0,0 @@
-#![allow(dead_code)]
-
-use crate::backends::metal::device_tier::DeviceTier;
-
-pub(crate) fn build_tree_out_transposed_h0(
- use_mxu: bool,
- device_tier: DeviceTier,
- use_h0: bool,
-) -> bool {
- use_h0 && !use_mxu && matches!(device_tier, DeviceTier::SmallLegacy | DeviceTier::SmallApple8)
-}
diff --git a/crates/backend-uzu/src/backends/metal/kernel/gdn/tree_verify/out.metal b/crates/backend-uzu/src/backends/metal/kernel/gdn/tree_verify/out.metal
index 1a6446c7e..310aa8050 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/gdn/tree_verify/out.metal
+++ b/crates/backend-uzu/src/backends/metal/kernel/gdn/tree_verify/out.metal
@@ -17,29 +17,27 @@ using namespace uzu::matmul;
// Build tree output for one (batch, value-head, row, value) tile.
//
// q: [B, T, Hg, K], with query head hq = hv / (HV / Hg)
-// prefix: [B, T, HV] path log-decay prefix from BuildPrefixBeta
+// prefix: [B, T, HV] path log-decay prefix from BuildTreePrefix
// qkd: [B, HV, T, T] from BuildTreeGram
// u: [B, HV, T, V]
// h0: [H0, HV, V, K]
//
// o[row, value] = exp(prefix[row]) * scale * dot(q[row], h0[value])
// + sum_j qkd[row, j] * u[j, value]
-template <
- typename T,
- bool use_mxu,
- bool transposed_h0>
-VARIANTS(T, float, bfloat)
+template <typename QKT, typename OutputT, bool use_mxu, bool transposed_h0>
+VARIANTS(QKT, float, bfloat)
+VARIANTS(OutputT, float, bfloat)
VARIANTS(use_mxu, false, true)
VARIANTS(transposed_h0, false, true)
CONSTRAINT(!(use_mxu && transposed_h0))
PUBLIC KERNEL(BuildTreeOut)(
- const device T* q,
+ const device QKT* q,
const device float* prefix,
const device float* qkd,
- const device T* u,
+ const device float* u,
const device float* h0 OPTIONAL(use_h0),
const device int* h0_indices OPTIONAL(use_h0),
- device T* o,
+ device OutputT* o,
constant const float& scale,
constant const uint& batch_size,
constant const uint& tree_size,
@@ -61,17 +59,17 @@ PUBLIC KERNEL(BuildTreeOut)(
const uint batch_value_head_idx GROUPS(batch_size * value_heads),
const uint tid THREADS(SIMDGROUPS_PER_TG * METAL_SIMD_SIZE)
) {
- using Ops = metal::conditional_t<use_mxu, MxuFragmentOps<>, SimdgroupFragmentOps>;
- using InputType = metal::conditional_t<use_mxu, T, float>;
+ // Relaxed MPP diverges from FP32 across layers.
+ using Ops = metal::conditional_t<use_mxu, MxuStrictFragmentOps, SimdgroupFragmentOps>;
using H0Read = metal::conditional_t<transposed_h0, ReadTranspose, ReadDirect>;
constexpr ushort ROWS = Ops::FRAGMENT_ROWS;
constexpr ushort COL_FRAGMENTS = MATMUL_COLS / Ops::FRAGMENT_COLS;
using AccFragment = Fragment<float, 1, COL_FRAGMENTS, Ops>;
- using QFragment = OperandFragment<InputType, 1, 1, Ops>;
+ using QFragment = OperandFragment<float, 1, 1, Ops>;
using H0Fragment = OperandFragment<float, 1, COL_FRAGMENTS, Ops, H0Read>;
using QkdFragment = OperandFragment<float, 1, 1, Ops>;
- using UFragment = OperandFragment<InputType, 1, COL_FRAGMENTS, Ops>;
- threadgroup T* u_tile = reinterpret_cast<threadgroup T*>(u_tile_scratch);
+ using UFragment = OperandFragment<float, 1, COL_FRAGMENTS, Ops>;
+ threadgroup float* u_tile = reinterpret_cast<threadgroup float*>(u_tile_scratch);
const uint batch_idx = batch_value_head_idx / value_heads;
const uint value_head_idx = batch_value_head_idx - batch_idx * value_heads;
@@ -136,7 +134,7 @@ PUBLIC KERNEL(BuildTreeOut)(
const uint local_j = index / MATMUL_COLS;
const uint local_col = index - local_j * MATMUL_COLS;
const bool in_bounds = local_j < uint(valid_j) && local_col < uint(valid_cols);
- u_tile[index] = in_bounds ? u[u_base + (j0 + local_j) * head_v_dim + local_col] : T(0);
+ u_tile[index] = in_bounds ? u[u_base + (j0 + local_j) * head_v_dim + local_col] : 0.0f;
}
threadgroup_barrier(mem_flags::mem_threadgroup);
u_frag.load_from(
@@ -154,7 +152,7 @@ PUBLIC KERNEL(BuildTreeOut)(
}
}
- device T* out_tile =
+ device OutputT* out_tile =
o + ((batch_idx * tree_size + row_base) * value_heads + value_head_idx) * head_v_dim + value_base;
acc.store_safe(thread_context.simd_lane_id, out_tile, int(value_heads * head_v_dim), short2(valid_cols, valid_rows));
}
diff --git a/crates/backend-uzu/src/backends/metal/kernel/gdn/tree_verify/prefix_beta.metal b/crates/backend-uzu/src/backends/metal/kernel/gdn/tree_verify/prefix.metal
similarity index 62%
rename from crates/backend-uzu/src/backends/metal/kernel/gdn/tree_verify/prefix_beta.metal
rename to crates/backend-uzu/src/backends/metal/kernel/gdn/tree_verify/prefix.metal
index e498d5aeb..6fe092adb 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/gdn/tree_verify/prefix_beta.metal
+++ b/crates/backend-uzu/src/backends/metal/kernel/gdn/tree_verify/prefix.metal
@@ -1,5 +1,4 @@
#include <metal_stdlib>
-#include "../../activation/activations.h"
#include "../../common/defines.h"
#include "../../common/dsl.h"
#include "../../common/thread_context.h"
@@ -12,19 +11,12 @@ using namespace uzu::trie;
// trie: [B, T] TrieNode — col is an ancestor-or-self of row iff
// trie[col].trie_start <= row <= trie[col].trie_end.
-// a_transposed: [B, HV, T]
-// b: [B, T, HV]
-// prefix,beta: [B, T, HV] fp32
-template <typename T>
-VARIANTS(T, float, half, bfloat)
-PUBLIC KERNEL(BuildPrefixBeta)(
+// log_decay: [B, T, HV]
+// prefix: [B, T, HV] fp32
+PUBLIC KERNEL(BuildTreePrefix)(
const device TrieNode* trie,
- const device T* a_transposed,
- const device T* b,
- const device float* a_log,
- const device float* dt_bias,
+ const device float* log_decay,
device float* prefix,
- device float* beta,
constant const uint& batch_size,
constant const uint& tree_size,
constant const uint& value_heads,
@@ -32,7 +24,7 @@ PUBLIC KERNEL(BuildPrefixBeta)(
const uint batch_idx GROUPS(batch_size),
const uint row_idx GROUPS(tree_size),
const uint head_group_idx GROUPS(value_heads.div_ceil(ROWS_PER_THREADGROUP)),
- const uint tid THREADS(ROWS_PER_THREADGROUP * METAL_SIMD_SIZE)
+ const uint tid THREADS(ROWS_PER_THREADGROUP* METAL_SIMD_SIZE)
) {
const uint simd_group = thread_context.simdgroup_index;
const uint lane_id = thread_context.simd_lane_id;
@@ -43,23 +35,16 @@ PUBLIC KERNEL(BuildPrefixBeta)(
const uint batch_offset = batch_idx * tree_size * value_heads;
const uint trie_offset = batch_idx * tree_size;
- const uint out_idx = batch_offset + row_idx * value_heads + head_idx;
-
- const float scale = exp(float(a_log[head_idx]));
- const float dt = float(dt_bias[head_idx]);
float partial = 0.0f;
for (uint col_idx = lane_id; col_idx < tree_size; col_idx += METAL_SIMD_SIZE) {
const TrieNode node = trie[trie_offset + col_idx];
if (row_idx >= node.trie_start && row_idx <= node.trie_end) {
- const uint a_idx = batch_offset + head_idx * tree_size + col_idx;
- partial -= scale * activate_softplus(float(a_transposed[a_idx]) + dt);
+ partial += log_decay[batch_offset + col_idx * value_heads + head_idx];
}
}
const float sum = simd_sum(partial);
if (lane_id == 0) {
- const float b_val = float(b[out_idx]);
- beta[out_idx] = 1.0f / (1.0f + exp(-b_val));
- prefix[out_idx] = sum;
+ prefix[batch_offset + row_idx * value_heads + head_idx] = sum;
}
}
diff --git a/crates/backend-uzu/src/backends/metal/kernel/gdn/tree_verify/state_advance.metal b/crates/backend-uzu/src/backends/metal/kernel/gdn/tree_verify/state_advance.metal
index 46d8f63dd..9b6c3afe6 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/gdn/tree_verify/state_advance.metal
+++ b/crates/backend-uzu/src/backends/metal/kernel/gdn/tree_verify/state_advance.metal
@@ -15,7 +15,7 @@ VARIANTS(T, float, bfloat)
VARIANTS(HEAD_K_DIM, 128)
PUBLIC KERNEL(StateAdvance)(
// [tree_size, num_k_heads, HEAD_K_DIM]
- device const float* k_norm,
+ device const T* k_norm,
// [tree_size, num_v_heads, HEAD_K_DIM]
device const T* v,
// [tree_size, num_v_heads]
@@ -27,8 +27,8 @@ PUBLIC KERNEL(StateAdvance)(
// [num_v_heads, HEAD_K_DIM, HEAD_K_DIM], input/output
device float* state,
constant const uint& accepted_len,
- constant const uint& num_v_heads,
- constant const uint& num_k_heads,
+ const uint num_v_heads SPECIALIZE,
+ const uint num_k_heads SPECIALIZE,
const ThreadContext thread_context,
const uint hv_idx GROUPS(num_v_heads),
const uint dv_group_idx GROUPS(STATE_ADVANCE_DV_GROUPS),
@@ -60,7 +60,7 @@ PUBLIC KERNEL(StateAdvance)(
const float decay = fast::exp(log_decay_buf[tree_head_offset]);
const float beta = beta_buf[tree_head_offset];
const uint k_offset = tree_idx * key_dim + hk_idx * HEAD_K_DIM + dk_base;
- const float4 k = *reinterpret_cast<device const float4*>(k_norm + k_offset);
+ const float4 k = float4(*reinterpret_cast<const device vec<T, 4>*>(k_norm + k_offset));
state_rows[0] *= decay;
state_rows[1] *= decay;
diff --git a/crates/backend-uzu/src/backends/metal/kernel/gdn/tree_verify/tree_gram.metal b/crates/backend-uzu/src/backends/metal/kernel/gdn/tree_verify/tree_gram.metal
index dc2f1821c..75a681e98 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/gdn/tree_verify/tree_gram.metal
+++ b/crates/backend-uzu/src/backends/metal/kernel/gdn/tree_verify/tree_gram.metal
@@ -23,7 +23,7 @@ using namespace uzu::trie;
template <typename T, typename Ops, typename AccFragment, typename LeftFragment, typename RightFragment>
METAL_FUNC void tree_kh0(
const device T* k_rows,
- const device T* h0,
+ const device float* h0,
const device int* h0_idx,
device float* kh0,
const uint batch_idx,
@@ -44,7 +44,7 @@ METAL_FUNC void tree_kh0(
if (h0_slot < 0) {
return;
}
- const device T* h0_head = h0 + (uint(h0_slot) * value_heads + value_head_idx) * head_v_dim * head_k_dim;
+ const device float* h0_head = h0 + (uint(h0_slot) * value_heads + value_head_idx) * head_v_dim * head_k_dim;
device float* kh0_rows =
kh0 + (batch_idx * tree_size + row_base) * value_heads * head_v_dim + value_head_idx * head_v_dim;
const uint kh0_row_stride = value_heads * head_v_dim;
@@ -54,7 +54,7 @@ METAL_FUNC void tree_kh0(
AccFragment kh0_acc;
kh0_acc.clear();
- const device T* h0_tile = h0_head + dv_base * head_k_dim;
+ const device float* h0_tile = h0_head + dv_base * head_k_dim;
const bool full_dvs = tile_dvs == COL_TILE;
for (uint kb = 0; kb < head_k_dim; kb += Ops::FRAGMENT_ROWS) {
@@ -96,7 +96,7 @@ PUBLIC KERNEL(BuildTreeGram)(
const device TrieNode* trie,
const device float* prefix,
const device float* beta,
- const device T* h0 OPTIONAL(use_h0),
+ const device float* h0 OPTIONAL(use_h0),
const device int* h0_idx OPTIONAL(use_h0),
device float* a_packed,
device float* qkd,
@@ -128,10 +128,9 @@ PUBLIC KERNEL(BuildTreeGram)(
using Ops = metal::conditional_t<USE_MXU, MxuFragmentOps<>, SimdgroupFragmentOps>;
constexpr ushort ROW_FRAGMENTS = ROW_TILE / Ops::FRAGMENT_ROWS;
constexpr ushort COL_FRAGMENTS = COL_TILE / Ops::FRAGMENT_COLS;
- using InputType = metal::conditional_t<USE_MXU, T, float>;
using AccFragment = Fragment<float, ROW_FRAGMENTS, COL_FRAGMENTS, Ops>;
- using LeftFragment = OperandFragment<InputType, ROW_FRAGMENTS, 1, Ops>;
- using RightFragment = OperandFragment<InputType, 1, COL_FRAGMENTS, Ops, ReadTranspose>;
+ using LeftFragment = OperandFragment<float, ROW_FRAGMENTS, 1, Ops>;
+ using RightFragment = OperandFragment<float, 1, COL_FRAGMENTS, Ops, ReadTranspose>;
static_assert(COL_TILE == METAL_SIMD_SIZE, "COL_TILE must match the SIMD width");
const uint key_head_idx = value_head_idx / (value_heads / k_heads);
diff --git a/crates/backend-uzu/src/backends/metal/kernel/gdn/tree_verify/tree_update_solve.metal b/crates/backend-uzu/src/backends/metal/kernel/gdn/tree_verify/tree_update_solve.metal
index 8e8627c2e..a3c98aa93 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/gdn/tree_verify/tree_update_solve.metal
+++ b/crates/backend-uzu/src/backends/metal/kernel/gdn/tree_verify/tree_update_solve.metal
@@ -2,7 +2,6 @@
#include "../../common/defines.h"
#include "../../common/dsl.h"
#include "../../matmul/common/fragment.h"
-#include "../../matmul/common/mxu_fragment_ops.h"
#include "../../matmul/common/simdgroup_fragment_ops.h"
using namespace metal;
@@ -27,12 +26,9 @@ using namespace uzu::matmul;
// single-block tail when i is odd.
// A and Ainv are kept in f32: the (I + A)^-1 forward-substitution cascade is
// precision-sensitive.
-template <typename T, uint BV, bool USE_MXU>
+template <typename T, uint BV>
VARIANTS(T, float, bfloat)
VARIANTS(BV, 16, 32)
-VARIANTS(USE_MXU, false, true)
-CONSTRAINT(!USE_MXU || T != "float")
-CONSTRAINT(!USE_MXU || BV >= 32)
PUBLIC KERNEL(TreeUpdateSolve)(
const device float* kh0 OPTIONAL(use_h0),
const device T* v,
@@ -53,7 +49,8 @@ PUBLIC KERNEL(TreeUpdateSolve)(
const uint lane_idx THREADS(METAL_SIMD_SIZE)
) {
constexpr uint BT = 16;
- using FragmentOps = metal::conditional_t<USE_MXU, MxuFragmentOps<>, SimdgroupFragmentOps>;
+ // MXU was within 5% of SIMD-group at T=32..512.
+ using FragmentOps = SimdgroupFragmentOps;
constexpr ushort TOKEN_FRAGMENTS = BT / FragmentOps::FRAGMENT_ROWS;
constexpr ushort VALUE_FRAGMENTS = BV / FragmentOps::FRAGMENT_COLS;
using TileFragment = Fragment<float, TOKEN_FRAGMENTS, VALUE_FRAGMENTS, FragmentOps>;
diff --git a/crates/backend-uzu/src/backends/metal/kernel/gdn/tree_verify/tree_update_solve_dispatch_helper.rs b/crates/backend-uzu/src/backends/metal/kernel/gdn/tree_verify/tree_update_solve_dispatch_helper.rs
deleted file mode 100644
index d3e5c0905..000000000
--- a/crates/backend-uzu/src/backends/metal/kernel/gdn/tree_verify/tree_update_solve_dispatch_helper.rs
+++ /dev/null
@@ -1,55 +0,0 @@
-#![allow(dead_code)]
-
-#[derive(Debug, Clone, Copy, PartialEq, Eq)]
-pub(crate) struct TreeUpdateSolveVariant {
- pub use_mxu: bool,
- pub bv: u32,
-}
-
-/// Fastest TreeUpdateSolve variant per device + shape, from the 9-chip fleet sweep
-/// (raw table in .claude/plans/solve_u_kernel_optimization.md). The kernel is a
-/// latency-bound serial recurrence at B=1 small-T, where the narrower BV16 wins on
-/// occupancy; the tree size at which the wider BV32 takes over grows with GPU
-/// width. MXU chips switch to the matrix unit past that same B=1 small-T corner.
-pub(crate) fn tree_update_solve_variant(
- supports_mxu: bool,
- gpu_core_count: u32,
- batch_size: u32,
- tree_size: u32,
-) -> TreeUpdateSolveVariant {
- if supports_mxu {
- let small_corner = batch_size == 1 && tree_size <= 64;
- return TreeUpdateSolveVariant {
- use_mxu: !small_corner,
- bv: if small_corner {
- 16
- } else {
- 32
- },
- };
- }
- let bv16 = if gpu_core_count >= 30 {
- batch_size == 1 || (batch_size <= 8 && tree_size <= 64)
- } else {
- let bv16_max_tree = if gpu_core_count >= 10 {
- 128
- } else if gpu_core_count >= 8 {
- 33
- } else {
- 0
- };
- batch_size == 1 && tree_size <= bv16_max_tree
- };
- TreeUpdateSolveVariant {
- use_mxu: false,
- bv: if bv16 {
- 16
- } else {
- 32
- },
- }
-}
-
-#[cfg(test)]
-#[path = "../../../../../../tests/unit/backends/metal/tree_update_solve_dispatch_test.rs"]
-mod tests;
diff --git a/crates/backend-uzu/src/backends/metal/kernel/gdn/update.metal b/crates/backend-uzu/src/backends/metal/kernel/gdn/update.metal
index c99b2add8..b4ef42ce9 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/gdn/update.metal
+++ b/crates/backend-uzu/src/backends/metal/kernel/gdn/update.metal
@@ -14,7 +14,7 @@ using namespace metal;
// One simd group per output dim dv; 32 lanes cover Dk (coalesced state IO,
// reduction via simd_sum). Activations are model dtype T; state stays float.
template <typename T, uint HEAD_K_DIM>
-VARIANTS(T, float, half, bfloat)
+VARIANTS(T, float, bfloat)
VARIANTS(HEAD_K_DIM, 128)
PUBLIC KERNEL(DeltaNetUpdate)(
device const T* in_proj,
diff --git a/crates/backend-uzu/src/backends/metal/kernel/mod.rs b/crates/backend-uzu/src/backends/metal/kernel/mod.rs
index f4a414d40..e3d1126b3 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/mod.rs
+++ b/crates/backend-uzu/src/backends/metal/kernel/mod.rs
@@ -7,12 +7,8 @@ use crate::backends::{
};
pub mod attention;
-#[path = "gdn/tree_verify/build_tree_out_dispatch_helper.rs"]
-mod build_tree_out_dispatch_helper;
pub mod gdn;
pub mod matmul;
-#[path = "gdn/tree_verify/tree_update_solve_dispatch_helper.rs"]
-mod tree_update_solve_dispatch_helper;
pub const MTLB: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/default.metallib"));
@@ -26,5 +22,6 @@ impl Kernels for MetalKernels {
autogen_kernels!();
type AttentionGemmCore = attention::AttentionGemmMetalCore;
type DeltaNetChunkedPrefill = gdn::chunked::MetalDeltaNetChunkedPrefill;
+ type DeltaNetTreeVerify = gdn::tree_verify::MetalDeltaNetTreeVerify;
type MatmulKernel = matmul::MatmulMetalKernel;
}
diff --git a/crates/backend-uzu/src/encodable_block/mixer/delta_net.rs b/crates/backend-uzu/src/encodable_block/mixer/delta_net.rs
index f511818a5..0a15d89c9 100644
--- a/crates/backend-uzu/src/encodable_block/mixer/delta_net.rs
+++ b/crates/backend-uzu/src/encodable_block/mixer/delta_net.rs
@@ -1,13 +1,18 @@
+use std::mem::size_of;
+
use thiserror::Error;
use crate::{
array::size_for_shape,
backends::common::{
Allocation, AllocationType, Backend, Context, Encoder, Kernels,
+ gpu_types::trie::TrieNode,
kernel::{
- Conv1dPackKernel, DeltaNetConvScanKernel, DeltaNetConvUpdateKernel, DeltaNetNormGateKernel,
- DeltaNetPrefillKernel, DeltaNetPrefillPrepKernel, DeltaNetUpdateKernel,
+ Conv1dPackKernel, ConvTreeScanKernel, DeltaNetConvScanKernel, DeltaNetConvUpdateKernel,
+ DeltaNetNormGateKernel, DeltaNetPrefillKernel, DeltaNetPrefillPrepKernel, DeltaNetUpdateKernel,
+ StateAdvanceKernel,
delta_net_chunked_prefill::{DeltaNetChunkedPrefill, DeltaNetChunkedPrefillArgs},
+ delta_net_tree_verify::DeltaNetTreeVerify as DeltaNetTreeVerifyTrait,
},
},
config::token_mixer::delta_net::DeltaNetConfig,
@@ -15,16 +20,37 @@ use crate::{
encodable_block::{
batch_topology::BatchTopology,
linear::{Linear, LinearBlockError},
- mixer::{Mixer, MixerState, attention::rope::PrecalculatedRoPE},
+ mixer::{
+ Mixer, MixerState,
+ attention::rope::PrecalculatedRoPE,
+ delta_net::tree_verify::{TreeVerifyEncodeArguments, TreeVerifyNewArguments},
+ },
},
parameters::{ParameterLoaderError, ParameterTree},
utils::maybe_mut::MaybeMut,
};
+pub(crate) mod tree_verify;
+
+enum DeltaNetSuffixStatus<B: Backend> {
+ Flat {
+ suffix_length: usize,
+ },
+ Tree {
+ conv_states: Allocation<B>,
+ k: Allocation<B>,
+ v: Allocation<B>,
+ log_decay: Allocation<B>,
+ beta: Allocation<B>,
+ parents: Box<[i32]>,
+ },
+}
+
pub struct DeltaNetState<B: Backend> {
conv_state: Allocation<B>,
ssm_state: Allocation<B>,
- suffix_length: Option<usize>,
+ suffix_status: Option<DeltaNetSuffixStatus<B>>,
+ state_advance: <B::Kernels as Kernels>::StateAdvanceKernel,
}
impl<B: Backend> MixerState<B> for DeltaNetState<B> {
@@ -40,9 +66,53 @@ impl<B: Backend> MixerState<B> for DeltaNetState<B> {
fn encode_accept(
&mut self,
accepted_indices: &[usize],
- _encoder: &mut Encoder<B>,
+ encoder: &mut Encoder<B>,
) -> Result<(), B::Error> {
- assert!(self.suffix_length.take() == Some(*accepted_indices.last().unwrap() + 1));
+ let suffix_status = self.suffix_status.take().expect("delta net state has no suffix to accept");
+ let accepted_index = *accepted_indices.last().expect("delta net state attempted to accept zero indices");
+
+ match suffix_status {
+ DeltaNetSuffixStatus::Flat {
+ suffix_length,
+ } => {
+ assert!(accepted_index == suffix_length - 1, "attempted a partial flat delta net accept");
+ },
+ DeltaNetSuffixStatus::Tree {
+ conv_states,
+ k,
+ v,
+ log_decay,
+ beta,
+ parents,
+ } => {
+ assert!(accepted_indices.iter().all(|&index| index < parents.len()));
+ assert_eq!(parents[accepted_indices[0]], -1);
+ assert!(accepted_indices.windows(2).all(|edge| parents[edge[1]] == edge[0] as i32));
+
+ let conv_state_size = self.conv_state.size();
+ encoder.encode_copy(
+ &conv_states,
+ accepted_index * conv_state_size..(accepted_index + 1) * conv_state_size,
+ &mut self.conv_state,
+ ..,
+ );
+
+ let accepted_indices = accepted_indices.iter().map(|&index| index as u32).collect::<Box<[_]>>();
+ let mut accepted_indices_buffer =
+ encoder.allocate_constant(accepted_indices.len() * DataType::U32.size_in_bytes())?;
+ accepted_indices_buffer.copyin(&accepted_indices);
+ self.state_advance.encode(
+ &k,
+ &v,
+ &log_decay,
+ &beta,
+ &accepted_indices_buffer,
+ &mut self.ssm_state,
+ accepted_indices.len() as u32,
+ encoder,
+ );
+ },
+ }
Ok(())
}
}
@@ -64,14 +134,17 @@ pub struct DeltaNet<B: Backend> {
conv_update: <B::Kernels as Kernels>::DeltaNetConvUpdateKernel,
conv_pack: <B::Kernels as Kernels>::Conv1dPackKernel,
conv_scan: <B::Kernels as Kernels>::DeltaNetConvScanKernel,
+ conv_tree_scan: <B::Kernels as Kernels>::ConvTreeScanKernel,
a_log: Allocation<B>,
dt_bias: Allocation<B>,
norm_weight: Allocation<B>,
norm_epsilon: f32,
delta_net_update: <B::Kernels as Kernels>::DeltaNetUpdateKernel,
delta_net_prefill_prep: <B::Kernels as Kernels>::DeltaNetPrefillPrepKernel,
+ delta_net_tree_prep: <B::Kernels as Kernels>::DeltaNetPrefillPrepKernel,
delta_net_prefill: <B::Kernels as Kernels>::DeltaNetPrefillKernel,
delta_net_norm_gate: <B::Kernels as Kernels>::DeltaNetNormGateKernel,
+ tree_verify: Option<<B::Kernels as Kernels>::DeltaNetTreeVerify>,
chunked: Option<<B::Kernels as Kernels>::DeltaNetChunkedPrefill>,
out_projection: Box<dyn Linear<B>>,
}
@@ -152,6 +225,13 @@ impl<B: Backend> DeltaNet<B> {
let conv_scan =
<B::Kernels as Kernels>::DeltaNetConvScanKernel::new(context, outer_data_type, conv_config.has_biases)
.map_err(DeltaNetNewError::Backend)?;
+ let conv_tree_scan = <B::Kernels as Kernels>::ConvTreeScanKernel::new(
+ context,
+ outer_data_type,
+ config.kernel_size as u32,
+ conv_config.has_biases,
+ )
+ .map_err(DeltaNetNewError::Backend)?;
let a_log = parameter_tree.leaf("a_log")?.validate(&[config.num_heads], inner_data_type)?.read_allocation()?;
let dt_bias =
@@ -167,8 +247,19 @@ impl<B: Backend> DeltaNet<B> {
let delta_net_prefill_prep = <B::Kernels as Kernels>::DeltaNetPrefillPrepKernel::new(
context,
outer_data_type,
+ DataType::F32,
config.head_dim as u32,
false,
+ false,
+ )
+ .map_err(DeltaNetNewError::Backend)?;
+ let delta_net_tree_prep = <B::Kernels as Kernels>::DeltaNetPrefillPrepKernel::new(
+ context,
+ outer_data_type,
+ outer_data_type,
+ config.head_dim as u32,
+ true,
+ true,
)
.map_err(DeltaNetNewError::Backend)?;
let delta_net_prefill =
@@ -176,6 +267,24 @@ impl<B: Backend> DeltaNet<B> {
.map_err(DeltaNetNewError::Backend)?;
let delta_net_norm_gate = <B::Kernels as Kernels>::DeltaNetNormGateKernel::new(context, outer_data_type)
.map_err(DeltaNetNewError::Backend)?;
+ let tree_verify =
+ if <<B::Kernels as Kernels>::DeltaNetTreeVerify as DeltaNetTreeVerifyTrait<B>>::is_supported(context) {
+ Some(
+ <<B::Kernels as Kernels>::DeltaNetTreeVerify as DeltaNetTreeVerifyTrait<B>>::new(
+ context,
+ &TreeVerifyNewArguments {
+ data_type: outer_data_type,
+ num_k_heads: config.num_groups,
+ num_v_heads: config.num_heads,
+ head_k_dim: config.head_dim,
+ head_v_dim: config.value_head_dim,
+ },
+ )
+ .map_err(DeltaNetNewError::Backend)?,
+ )
+ } else {
+ None
+ };
let chunked =
<B::Kernels as Kernels>::DeltaNetChunkedPrefill::new(context, outer_data_type, config.head_dim as u32)
@@ -210,25 +319,123 @@ impl<B: Backend> DeltaNet<B> {
conv_update,
conv_pack,
conv_scan,
+ conv_tree_scan,
a_log,
dt_bias,
norm_weight,
norm_epsilon,
delta_net_update,
delta_net_prefill_prep,
+ delta_net_tree_prep,
delta_net_prefill,
delta_net_norm_gate,
+ tree_verify,
chunked,
out_projection,
},
in_projection_input_hadamard_factors,
))
}
+
+ fn encode_tree_verify(
+ &self,
+ in_projected: Allocation<B>,
+ batch_dim: &BatchTopology,
+ state: &mut DeltaNetState<B>,
+ encoder: &mut Encoder<B>,
+ ) -> Result<Allocation<B>, B::Error> {
+ let tree_verify = self.tree_verify.as_ref().expect("DeltaNet tree verification is unsupported");
+ let tree_size = batch_dim.size();
+ let mut parents = encoder.allocate_constant(tree_size * DataType::I32.size_in_bytes())?;
+ parents.copyin(batch_dim.parents());
+ let mut trie = encoder.allocate_constant(tree_size * size_of::<TrieNode>())?;
+ trie.copyin(batch_dim.nodes());
+
+ let mut conv_states = encoder
+ .allocate_scratch(size_for_shape(&[tree_size, self.conv_dim, self.kernel_size - 1], DataType::F32))?;
+ let mut k = encoder.allocate_scratch(size_for_shape(&[tree_size, self.key_dim], self.outer_data_type))?;
+ let mut v = encoder.allocate_scratch(size_for_shape(&[tree_size, self.value_dim], self.outer_data_type))?;
+ let mut beta = encoder.allocate_scratch(size_for_shape(&[tree_size, self.num_heads], DataType::F32))?;
+ let mut log_decay = encoder.allocate_scratch(size_for_shape(&[tree_size, self.num_heads], DataType::F32))?;
+
+ let mut scratch = |shape: &[usize], data_type| encoder.allocate_scratch(size_for_shape(shape, data_type));
+ let mut tree_projected = scratch(&[tree_size, self.total_proj_dim], self.outer_data_type)?;
+ let mut q = scratch(&[tree_size, self.key_dim], self.outer_data_type)?;
+
+ self.conv_tree_scan.encode(
+ &in_projected,
+ &self.conv_weight,
+ self.conv_bias.as_ref(),
+ &state.conv_state,
+ &parents,
+ &mut tree_projected,
+ &mut conv_states,
+ tree_size as u32,
+ self.total_proj_dim as u32,
+ self.conv_dim as u32,
+ encoder,
+ );
+
+ self.delta_net_tree_prep.encode(
+ &tree_projected,
+ &self.a_log,
+ &self.dt_bias,
+ &mut q,
+ &mut k,
+ Some(&mut v),
+ &mut beta,
+ &mut log_decay,
+ self.num_heads as u32,
+ self.num_groups as u32,
+ self.key_dim as u32,
+ self.value_dim as u32,
+ tree_size as u32,
+ encoder,
+ );
+
+ let mut delta_output = tree_verify.encode(
+ TreeVerifyEncodeArguments {
+ q: &q,
+ k: &k,
+ v: &v,
+ trie: &trie,
+ log_decay: &log_decay,
+ beta: &beta,
+ h0: &state.ssm_state,
+ tree_size,
+ },
+ encoder,
+ )?;
+ self.delta_net_norm_gate.encode(
+ &mut delta_output,
+ &tree_projected,
+ &self.norm_weight,
+ self.num_heads as u32,
+ self.value_head_dim as u32,
+ self.value_dim as u32,
+ self.conv_dim as u32,
+ self.total_proj_dim as u32,
+ self.norm_epsilon,
+ tree_size as u32,
+ encoder,
+ );
+
+ let output = self.out_projection.encode(delta_output, tree_size, encoder)?;
+ state.suffix_status = Some(DeltaNetSuffixStatus::Tree {
+ conv_states,
+ k,
+ v,
+ log_decay,
+ beta,
+ parents: batch_dim.parents().into(),
+ });
+ Ok(output)
+ }
}
impl<B: Backend> Mixer<B> for DeltaNet<B> {
fn speculation_supported(&self) -> bool {
- false
+ self.tree_verify.is_some()
}
fn max_context_length(&self) -> Option<usize> {
@@ -255,10 +462,19 @@ impl<B: Backend> Mixer<B> for DeltaNet<B> {
zero_encoder.encode_fill(&mut ssm_state, 0);
zero_encoder.end_encoding().submit().wait_until_completed()?;
+ let state_advance = <B::Kernels as Kernels>::StateAdvanceKernel::new(
+ context,
+ self.outer_data_type,
+ self.head_dim as u32,
+ self.num_heads as u32,
+ self.num_groups as u32,
+ )?;
+
Ok(Box::new(DeltaNetState {
conv_state,
ssm_state,
- suffix_length: None,
+ suffix_status: None,
+ state_advance,
}))
}
@@ -272,20 +488,20 @@ impl<B: Backend> Mixer<B> for DeltaNet<B> {
) -> Result<Allocation<B>, B::Error> {
assert!(precalculated_rope.is_none(), "unexpected rope for delta net mixer");
- if !batch_dim.full_accept() {
- panic!("delta net doesn't support speculation");
- }
-
let state = state.expect("delta net requires state");
let state = state.downcast::<DeltaNetState<B>>().expect("incorrect type of delta net state");
let MaybeMut::Mut(state) = state else {
panic!("delta net doesn't support immutable state");
};
- assert!(state.suffix_length.is_none(), "delta net called with state with unaccepted tokens");
+ assert!(state.suffix_status.is_none(), "delta net called with state with an unaccepted suffix");
let mut in_projected = self.in_projection.encode(hidden, batch_dim.size(), encoder)?;
+ if !batch_dim.full_accept() {
+ return self.encode_tree_verify(in_projected, batch_dim, state, encoder);
+ }
+
let mut delta_output =
encoder.allocate_scratch(size_for_shape(&[batch_dim.size(), self.value_dim], self.outer_data_type))?;
if batch_dim.size() == 1 {
@@ -376,6 +592,7 @@ impl<B: Backend> Mixer<B> for DeltaNet<B> {
&self.dt_bias,
&mut prep_q_norm,
&mut prep_k_norm,
+ None::<&mut Allocation<B>>,
&mut prep_beta,
&mut prep_decay,
self.num_heads as u32,
@@ -418,7 +635,9 @@ impl<B: Backend> Mixer<B> for DeltaNet<B> {
);
}
- state.suffix_length = Some(batch_dim.size());
+ state.suffix_status = Some(DeltaNetSuffixStatus::Flat {
+ suffix_length: batch_dim.size(),
+ });
self.out_projection.encode(delta_output, batch_dim.size(), encoder)
}
diff --git a/crates/backend-uzu/src/encodable_block/mixer/delta_net/tree_verify.rs b/crates/backend-uzu/src/encodable_block/mixer/delta_net/tree_verify.rs
new file mode 100644
index 000000000..4f8b6f7a7
--- /dev/null
+++ b/crates/backend-uzu/src/encodable_block/mixer/delta_net/tree_verify.rs
@@ -0,0 +1,29 @@
+use crate::{
+ backends::common::{Allocation, Backend},
+ data_type::DataType,
+};
+
+#[derive(Clone, Copy)]
+pub struct TreeVerifyNewArguments {
+ pub data_type: DataType,
+ pub num_k_heads: usize,
+ pub num_v_heads: usize,
+ pub head_k_dim: usize,
+ pub head_v_dim: usize,
+}
+
+#[derive(Clone, Copy)]
+pub struct TreeVerifyEncodeArguments<'a, B: Backend> {
+ pub q: &'a Allocation<B>,
+ pub k: &'a Allocation<B>,
+ pub v: &'a Allocation<B>,
+ pub trie: &'a Allocation<B>,
+ pub log_decay: &'a Allocation<B>,
+ pub beta: &'a Allocation<B>,
+ pub h0: &'a Allocation<B>,
+ pub tree_size: usize,
+}
+
+#[cfg(test)]
+#[path = "../../../../unit/encodable_block/delta_net_tree_verify_bench.rs"]
+mod tests;
diff --git a/crates/backend-uzu/tests/unit/backends/metal/tree_update_solve_dispatch_test.rs b/crates/backend-uzu/tests/unit/backends/metal/tree_update_solve_dispatch_test.rs
deleted file mode 100644
index db98fffd7..000000000
--- a/crates/backend-uzu/tests/unit/backends/metal/tree_update_solve_dispatch_test.rs
+++ /dev/null
@@ -1,35 +0,0 @@
-use proc_macros::uzu_test;
-
-use super::tree_update_solve_variant;
-
-fn variant(
- supports_mxu: bool,
- cores: u32,
- batch: u32,
- tree: u32,
-) -> (bool, u32) {
- let v = tree_update_solve_variant(supports_mxu, cores, batch, tree);
- (v.use_mxu, v.bv)
-}
-
-#[uzu_test]
-fn tree_update_solve_dispatch_matches_fleet() {
- // MXU chips (M5+): simdgroup BV16 only at the B=1 small-T corner, MXU BV32 else.
- assert_eq!(variant(true, 40, 1, 64), (false, 16));
- assert_eq!(variant(true, 40, 1, 128), (true, 32));
- assert_eq!(variant(true, 40, 8, 256), (true, 32));
-
- // Max/Ultra (M3/M4 Max, 40): BV16 across B=1 and B<=8 tiny trees, BV32 beyond.
- assert_eq!(variant(false, 40, 1, 256), (false, 16));
- assert_eq!(variant(false, 40, 8, 64), (false, 16));
- assert_eq!(variant(false, 40, 8, 256), (false, 32));
-
- // Mid (M2/M4/M4 Pro, 10-20): BV16 through B=1 T<=128.
- assert_eq!(variant(false, 20, 1, 128), (false, 16));
- assert_eq!(variant(false, 10, 1, 256), (false, 32));
-
- // Narrow (M1, 8): BV16 only at the tiniest tree. Phone-class (A18, 5): never.
- assert_eq!(variant(false, 8, 1, 33), (false, 16));
- assert_eq!(variant(false, 8, 1, 64), (false, 32));
- assert_eq!(variant(false, 5, 1, 33), (false, 32));
-}
diff --git a/crates/backend-uzu/unit/backends/common/kernel/gdn/delta_net_test.rs b/crates/backend-uzu/unit/backends/common/kernel/gdn/delta_net_test.rs
index afd333ab2..78971cc1e 100644
--- a/crates/backend-uzu/unit/backends/common/kernel/gdn/delta_net_test.rs
+++ b/crates/backend-uzu/unit/backends/common/kernel/gdn/delta_net_test.rs
@@ -329,8 +329,10 @@ fn run_prefill_with_norm_gate_typed<T: ArrayElement>(
let prep_k = <<Metal as Backend>::Kernels as Kernels>::DeltaNetPrefillPrepKernel::new(
&context,
T::data_type(),
+ DataType::F32,
head_k_dim as u32,
false,
+ false,
)
.unwrap();
let prefill_k = <<Metal as Backend>::Kernels as Kernels>::DeltaNetPrefillKernel::new(
@@ -349,6 +351,7 @@ fn run_prefill_with_norm_gate_typed<T: ArrayElement>(
&dt_bias_array,
&mut q_norm_array,
&mut k_norm_array,
+ None::<&mut crate::backends::common::Allocation<Metal>>,
&mut beta_array,
&mut decay_array,
num_v_heads as u32,
@@ -491,25 +494,29 @@ fn test_delta_net_prefill_prep() {
let conv_dim = 2 * key_dim + value_dim;
let total_proj_dim = conv_dim + value_dim + num_v_heads + num_v_heads;
- let in_proj: Vec<f32> = (0..suffix_len * total_proj_dim).map(|i| ((i % 37) as f32) * 0.02 - 0.3).collect();
+ let in_proj: Vec<bf16> =
+ (0..suffix_len * total_proj_dim).map(|i| bf16::from_f32(((i % 37) as f32) * 0.02 - 0.3)).collect();
let a_log: Vec<f32> = (0..num_v_heads).map(|i| -1.5 + (i as f32) * 0.05).collect();
let dt_bias: Vec<f32> = (0..num_v_heads).map(|i| 0.3 + (i as f32) * 0.02).collect();
// CPU reference via Kernels trait
let cpu_ctx = <Cpu as Backend>::Context::new().expect("cpu context");
- let cpu_in_proj = alloc_allocation_with_data::<Cpu, f32>(&cpu_ctx, &in_proj);
+ let cpu_in_proj = alloc_allocation_with_data::<Cpu, bf16>(&cpu_ctx, &in_proj);
let cpu_a_log = alloc_allocation_with_data::<Cpu, f32>(&cpu_ctx, &a_log);
let cpu_dt_bias = alloc_allocation_with_data::<Cpu, f32>(&cpu_ctx, &dt_bias);
- let mut cpu_q = alloc_allocation::<Cpu, f32>(&cpu_ctx, suffix_len * key_dim);
- let mut cpu_k = alloc_allocation::<Cpu, f32>(&cpu_ctx, suffix_len * key_dim);
+ let mut cpu_q = alloc_allocation::<Cpu, bf16>(&cpu_ctx, suffix_len * key_dim);
+ let mut cpu_k = alloc_allocation::<Cpu, bf16>(&cpu_ctx, suffix_len * key_dim);
+ let mut cpu_v = alloc_allocation::<Cpu, bf16>(&cpu_ctx, suffix_len * value_dim);
let mut cpu_beta = alloc_allocation::<Cpu, f32>(&cpu_ctx, suffix_len * num_v_heads);
let mut cpu_decay = alloc_allocation::<Cpu, f32>(&cpu_ctx, suffix_len * num_v_heads);
let cpu_prep = <<Cpu as Backend>::Kernels as Kernels>::DeltaNetPrefillPrepKernel::new(
&cpu_ctx,
- DataType::F32,
+ DataType::BF16,
+ DataType::BF16,
head_k_dim as u32,
false,
+ true,
)
.unwrap();
let mut cpu_enc = Encoder::new(cpu_ctx.as_ref()).expect("encoder");
@@ -519,6 +526,7 @@ fn test_delta_net_prefill_prep() {
&cpu_dt_bias,
&mut cpu_q,
&mut cpu_k,
+ Some(&mut cpu_v),
&mut cpu_beta,
&mut cpu_decay,
num_v_heads as u32,
@@ -530,27 +538,36 @@ fn test_delta_net_prefill_prep() {
);
cpu_enc.end_encoding().submit().wait_until_completed().unwrap();
- let ref_q: Vec<f32> = allocation_to_vec(&cpu_q);
- let ref_k: Vec<f32> = allocation_to_vec(&cpu_k);
+ let ref_q = allocation_to_vec::<Cpu, bf16>(&cpu_q).into_iter().map(f32::from).collect::<Vec<_>>();
+ let ref_k = allocation_to_vec::<Cpu, bf16>(&cpu_k).into_iter().map(f32::from).collect::<Vec<_>>();
+ let ref_v: Vec<bf16> = allocation_to_vec(&cpu_v);
let ref_beta: Vec<f32> = allocation_to_vec(&cpu_beta);
let ref_decay: Vec<f32> = allocation_to_vec(&cpu_decay);
+ let expected_v = in_proj
+ .chunks_exact(total_proj_dim)
+ .flat_map(|row| row[2 * key_dim..2 * key_dim + value_dim].iter().copied())
+ .collect::<Vec<_>>();
+ assert_eq!(ref_v, expected_v);
// Metal
let context = <Metal as Backend>::Context::new().expect("context");
- let in_proj_array = alloc_allocation_with_data::<Metal, f32>(&context, &in_proj);
+ let in_proj_array = alloc_allocation_with_data::<Metal, bf16>(&context, &in_proj);
let a_log_array = alloc_allocation_with_data::<Metal, f32>(&context, &a_log);
let dt_bias_array = alloc_allocation_with_data::<Metal, f32>(&context, &dt_bias);
- let mut q_norm_array = alloc_allocation::<Metal, f32>(&context, suffix_len * key_dim);
- let mut k_norm_array = alloc_allocation::<Metal, f32>(&context, suffix_len * key_dim);
+ let mut q_norm_array = alloc_allocation::<Metal, bf16>(&context, suffix_len * key_dim);
+ let mut k_norm_array = alloc_allocation::<Metal, bf16>(&context, suffix_len * key_dim);
+ let mut compact_v_array = alloc_allocation::<Metal, bf16>(&context, suffix_len * value_dim);
let mut beta_array = alloc_allocation::<Metal, f32>(&context, suffix_len * num_v_heads);
let mut decay_array = alloc_allocation::<Metal, f32>(&context, suffix_len * num_v_heads);
let prep_k = <<Metal as Backend>::Kernels as Kernels>::DeltaNetPrefillPrepKernel::new(
&context,
- DataType::F32,
+ DataType::BF16,
+ DataType::BF16,
head_k_dim as u32,
false,
+ true,
)
.unwrap();
@@ -561,6 +578,7 @@ fn test_delta_net_prefill_prep() {
&dt_bias_array,
&mut q_norm_array,
&mut k_norm_array,
+ Some(&mut compact_v_array),
&mut beta_array,
&mut decay_array,
num_v_heads as u32,
@@ -572,13 +590,15 @@ fn test_delta_net_prefill_prep() {
);
encoder.end_encoding().submit().wait_until_completed().unwrap();
- let gpu_q: Vec<f32> = allocation_to_vec(&q_norm_array);
- let gpu_k: Vec<f32> = allocation_to_vec(&k_norm_array);
+ let gpu_q = allocation_to_vec::<Metal, bf16>(&q_norm_array).into_iter().map(f32::from).collect::<Vec<_>>();
+ let gpu_k = allocation_to_vec::<Metal, bf16>(&k_norm_array).into_iter().map(f32::from).collect::<Vec<_>>();
+ let gpu_v: Vec<bf16> = allocation_to_vec(&compact_v_array);
let gpu_beta: Vec<f32> = allocation_to_vec(&beta_array);
let gpu_decay: Vec<f32> = allocation_to_vec(&decay_array);
assert_close(&gpu_q, &ref_q, 1e-4, 1e-3, "prep q_norm");
assert_close(&gpu_k, &ref_k, 1e-4, 1e-3, "prep k_norm");
+ assert_eq!(gpu_v, ref_v);
assert_close(&gpu_beta, &ref_beta, 1e-4, 1e-3, "prep beta");
assert_close(&gpu_decay, &ref_decay, 1e-4, 1e-3, "prep decay");
}
@@ -622,8 +642,10 @@ fn bench_delta_net_prefill() {
let prep_k = <<Metal as Backend>::Kernels as Kernels>::DeltaNetPrefillPrepKernel::new(
&context,
DataType::F32,
+ DataType::F32,
head_k_dim as u32,
false,
+ false,
)
.unwrap();
let prefill_k = <<Metal as Backend>::Kernels as Kernels>::DeltaNetPrefillKernel::new(
@@ -649,6 +671,7 @@ fn bench_delta_net_prefill() {
&dt_bias_array,
&mut q_norm_array,
&mut k_norm_array,
+ None::<&mut crate::backends::common::Allocation<Metal>>,
&mut beta_array,
&mut decay_array,
num_v_heads as u32,
@@ -674,6 +697,7 @@ fn bench_delta_net_prefill() {
&dt_bias_array,
&mut q_norm_array,
&mut k_norm_array,
+ None::<&mut crate::backends::common::Allocation<Metal>>,
&mut beta_array,
&mut decay_array,
num_v_heads as u32,
diff --git a/crates/backend-uzu/unit/backends/common/kernel/gdn/tree_verify/mod.rs b/crates/backend-uzu/unit/backends/common/kernel/gdn/tree_verify/mod.rs
index 0416236e8..027eae27b 100644
--- a/crates/backend-uzu/unit/backends/common/kernel/gdn/tree_verify/mod.rs
+++ b/crates/backend-uzu/unit/backends/common/kernel/gdn/tree_verify/mod.rs
@@ -1,8 +1,8 @@
mod conv_scan_test;
mod out_bench;
mod out_test;
-mod prefix_beta_bench;
-mod prefix_beta_test;
+mod prefix_bench;
+mod prefix_test;
mod state_advance_test;
mod tree_gram_bench;
mod tree_gram_test;
diff --git a/crates/backend-uzu/unit/backends/common/kernel/gdn/tree_verify/out_bench.rs b/crates/backend-uzu/unit/backends/common/kernel/gdn/tree_verify/out_bench.rs
index 1c7c0b260..2942a30d4 100644
--- a/crates/backend-uzu/unit/backends/common/kernel/gdn/tree_verify/out_bench.rs
+++ b/crates/backend-uzu/unit/backends/common/kernel/gdn/tree_verify/out_bench.rs
@@ -25,7 +25,7 @@ const HEAD_K_DIM: usize = 128;
const HEAD_V_DIM: usize = 128;
const TREE_OUT_MATMUL_MXU_ROWS: usize = 16;
const TREE_OUT_MATMUL_SIMDGROUP_ROWS: usize = 8;
-const TREE_SIZES: &[usize] = &[33, 49, 64, 128, 256, 512];
+const TREE_SIZES: &[usize] = &[32, 33, 49, 64, 128, 256, 512];
const BATCH_SIZES: &[usize] = &[1, 2, 4, 8];
const TREE_OUT_VALUE_COLS: usize = 32;
const TREE_OUT_SIMDGROUPS_PER_TG: usize = 4;
@@ -60,7 +60,7 @@ fn make_buffers<T: ArrayElement + Float>(
.map(|i| -((i % tree_size) as f32) * 0.01 - ((i % VALUE_HEADS) as f32) * 0.003)
.collect::<Vec<_>>();
let qkd = (0..qkd_len).map(|i| ((i as f32 * 0.013).cos() * 0.03) - 0.01).collect::<Vec<_>>();
- let u = (0..uv_len).map(|i| T::from(((i as f32 * 0.011).sin() * 0.3) + 0.04).unwrap()).collect::<Vec<_>>();
+ let u = (0..uv_len).map(|i| (i as f32 * 0.011).sin() * 0.3 + 0.04).collect::<Vec<_>>();
let h0 = (0..h0_len).map(|i| ((i as f32 * 0.019).cos() * 0.2) - 0.01).collect::<Vec<_>>();
let h0_indices = (0..batch_size as i32).collect::<Vec<_>>();
@@ -68,7 +68,7 @@ fn make_buffers<T: ArrayElement + Float>(
q: alloc_allocation_with_data::<Metal, T>(context, &q),
prefix: alloc_allocation_with_data::<Metal, f32>(context, &prefix),
qkd: alloc_allocation_with_data::<Metal, f32>(context, &qkd),
- u: alloc_allocation_with_data::<Metal, T>(context, &u),
+ u: alloc_allocation_with_data::<Metal, f32>(context, &u),
h0: alloc_allocation_with_data::<Metal, f32>(context, &h0),
h0_indices: alloc_allocation_with_data::<Metal, i32>(context, &h0_indices),
o: alloc_allocation::<Metal, T>(context, uv_len),
@@ -109,6 +109,7 @@ fn bench_build_tree_out_type<T: ArrayElement + Float>(
let kernel = <<Metal as Backend>::Kernels as Kernels>::BuildTreeOutKernel::new(
context,
T::data_type(),
+ T::data_type(),
use_mxu,
transposed_h0,
true,
diff --git a/crates/backend-uzu/unit/backends/common/kernel/gdn/tree_verify/out_test.rs b/crates/backend-uzu/unit/backends/common/kernel/gdn/tree_verify/out_test.rs
index 4a13e9b48..da362a9c4 100644
--- a/crates/backend-uzu/unit/backends/common/kernel/gdn/tree_verify/out_test.rs
+++ b/crates/backend-uzu/unit/backends/common/kernel/gdn/tree_verify/out_test.rs
@@ -32,7 +32,7 @@ struct Inputs<T> {
q: Vec<T>,
prefix: Vec<f32>,
qkd: Vec<f32>,
- u: Vec<T>,
+ u: Vec<f32>,
h0: Vec<f32>,
h0_indices: Vec<i32>,
}
@@ -57,7 +57,7 @@ fn make_inputs<T: ArrayElement + Float>(shape: Shape) -> Inputs<T> {
.map(|i| -((i % shape.tree_size) as f32) * 0.01 - ((i % shape.value_heads) as f32) * 0.003)
.collect(),
qkd: (0..qkd_len).map(|i| ((i as f32 * 0.013).cos() * 0.1) - 0.02).collect(),
- u: (0..u_len).map(|i| T::from(((i as f32 * 0.011).sin() * 0.3) + 0.04).unwrap()).collect(),
+ u: (0..u_len).map(|i| ((i as f32 * 0.011).sin() * 0.3) + 0.04).collect(),
h0: (0..h0_len).map(|i| ((i as f32 * 0.019).cos() * 0.2) - 0.01).collect(),
h0_indices: (0..shape.batch_size)
.map(|i| {
@@ -82,6 +82,7 @@ fn run_build_tree_out<B: Backend, T: ArrayElement + Float>(
let kernel = <<B as Backend>::Kernels as Kernels>::BuildTreeOutKernel::new(
&context,
T::data_type(),
+ T::data_type(),
use_mxu,
transposed_h0,
use_h0,
@@ -90,7 +91,7 @@ fn run_build_tree_out<B: Backend, T: ArrayElement + Float>(
let q = alloc_allocation_with_data::<B, T>(&context, &inputs.q);
let prefix = alloc_allocation_with_data::<B, f32>(&context, &inputs.prefix);
let qkd = alloc_allocation_with_data::<B, f32>(&context, &inputs.qkd);
- let u = alloc_allocation_with_data::<B, T>(&context, &inputs.u);
+ let u = alloc_allocation_with_data::<B, f32>(&context, &inputs.u);
let h0 = use_h0.then(|| alloc_allocation_with_data::<B, f32>(&context, &inputs.h0));
let h0_indices = use_h0.then(|| alloc_allocation_with_data::<B, i32>(&context, &inputs.h0_indices));
let mut o =
@@ -187,7 +188,7 @@ fn test_build_tree_out_paths() {
head_v_dim: 128,
};
for shape in [small, gdn] {
- check_shape::<bf16>(shape, 0.08);
- check_shape::<f32>(shape, 5e-3);
+ check_shape::<bf16>(shape, 5e-3);
+ check_shape::<f32>(shape, 5e-4);
}
}
diff --git a/crates/backend-uzu/unit/backends/common/kernel/gdn/tree_verify/prefix_bench.rs b/crates/backend-uzu/unit/backends/common/kernel/gdn/tree_verify/prefix_bench.rs
new file mode 100644
index 000000000..6aa242546
--- /dev/null
+++ b/crates/backend-uzu/unit/backends/common/kernel/gdn/tree_verify/prefix_bench.rs
@@ -0,0 +1,67 @@
+#![cfg(metal_backend)]
+
+use std::mem::size_of;
+
+use criterion::{BenchmarkId, Criterion, Throughput};
+use proc_macros::uzu_bench;
+
+use crate::{
+ backends::{
+ common::{Allocation, Backend, Kernels, kernel::BuildTreePrefixKernel},
+ metal::Metal,
+ },
+ tests::{
+ cold_pool::ColdPool,
+ helpers::{alloc_allocation, alloc_allocation_with_data},
+ matmul::iter_encode_loop_named,
+ util::type_short_name,
+ },
+};
+
+struct TreePrefixBuffers {
+ trie: Allocation<Metal>,
+ log_decay: Allocation<Metal>,
+ prefix: Allocation<Metal>,
+}
+
+#[uzu_bench]
+fn bench_build_tree_prefix(c: &mut Criterion) {
+ let context = crate::tests::util::shared_metal_context();
+ let kernel =
+ <<Metal as Backend>::Kernels as Kernels>::BuildTreePrefixKernel::new(&context).expect("BuildTreePrefixKernel");
+ let mut group = c.benchmark_group(format!("{}/Kernel/GDNTreeVerify/BuildTreePrefix", type_short_name::<Metal>()));
+
+ for tree_size in [49usize, 64, 128] {
+ let batch_size = 1usize;
+ let value_heads = 48usize;
+ let len = batch_size * tree_size * value_heads;
+ let trie = (0..tree_size as u32).flat_map(|row| [row, tree_size as u32 - 1, row]).collect::<Vec<_>>();
+ let log_decay = (0..len).map(|i| -0.001 - (i as f32 * 0.017).sin().abs() * 0.1).collect::<Vec<_>>();
+ let bytes_per_copy = trie.len() * size_of::<u32>() + len * size_of::<f32>() * 2;
+ let mut buffers = ColdPool::new(bytes_per_copy, || TreePrefixBuffers {
+ trie: alloc_allocation_with_data::<Metal, u32>(&context, &trie),
+ log_decay: alloc_allocation_with_data::<Metal, f32>(&context, &log_decay),
+ prefix: alloc_allocation::<Metal, f32>(&context, len),
+ });
+
+ group.throughput(Throughput::Elements((len * tree_size) as u64));
+ group.bench_function(BenchmarkId::from_parameter(format!("B1_T{tree_size}_HV{value_heads}")), |bencher| {
+ let benchmark_path = format!(
+ "{}/Kernel/GDNTreeVerify/BuildTreePrefix/B1_T{tree_size}_HV{value_heads}",
+ type_short_name::<Metal>()
+ );
+ iter_encode_loop_named::<Metal, _>(&context, bencher, &benchmark_path, |encoder| {
+ let buffers = buffers.next_mut();
+ kernel.encode(
+ &buffers.trie,
+ &buffers.log_decay,
+ &mut buffers.prefix,
+ batch_size as u32,
+ tree_size as u32,
+ value_heads as u32,
+ encoder,
+ );
+ });
+ });
+ }
+}
diff --git a/crates/backend-uzu/unit/backends/common/kernel/gdn/tree_verify/prefix_beta_bench.rs b/crates/backend-uzu/unit/backends/common/kernel/gdn/tree_verify/prefix_beta_bench.rs
deleted file mode 100644
index dbde90093..000000000
--- a/crates/backend-uzu/unit/backends/common/kernel/gdn/tree_verify/prefix_beta_bench.rs
+++ /dev/null
@@ -1,92 +0,0 @@
-#![cfg(metal_backend)]
-
-use std::mem::size_of;
-
-use criterion::{BenchmarkId, Criterion, Throughput};
-use proc_macros::uzu_bench;
-
-use crate::{
- backends::{
- common::{Allocation, Backend, Kernels, kernel::BuildPrefixBetaKernel},
- metal::Metal,
- },
- data_type::DataType,
- tests::{
- cold_pool::ColdPool,
- helpers::{alloc_allocation, alloc_allocation_with_data},
- matmul::iter_encode_loop_named,
- util::type_short_name,
- },
-};
-
-// chain trie: node `row` covers [row, T-1].
-fn chain_trie(tree_size: usize) -> Vec<u32> {
- let mut trie = Vec::with_capacity(tree_size * 3);
- for row in 0..tree_size as u32 {
- trie.extend_from_slice(&[row, tree_size as u32 - 1, row]);
- }
- trie
-}
-
-struct PrefixBetaBuffers {
- trie: Allocation<Metal>,
- a_transposed: Allocation<Metal>,
- b: Allocation<Metal>,
- prefix: Allocation<Metal>,
- beta: Allocation<Metal>,
-}
-
-#[uzu_bench]
-fn bench_build_prefix_beta(c: &mut Criterion) {
- let context = crate::tests::util::shared_metal_context();
- let kernel = <<Metal as Backend>::Kernels as Kernels>::BuildPrefixBetaKernel::new(&context, DataType::F32)
- .expect("BuildPrefixBetaKernel");
-
- let mut group = c.benchmark_group(format!("{}/Kernel/GDNTreeVerify/BuildPrefixBeta", type_short_name::<Metal>()));
-
- for tree_size in [49usize, 64, 129] {
- let batch_size = 1usize;
- let value_heads = 48usize;
- let len = batch_size * tree_size * value_heads;
- let trie = chain_trie(tree_size);
- let a_transposed = (0..len).map(|i| (i as f32 * 0.17).sin() - 0.4).collect::<Vec<_>>();
- let b_data = (0..len).map(|i| (i as f32 * 0.11).cos() * 0.8).collect::<Vec<_>>();
- let a_log = (0..value_heads).map(|i| -0.2 + i as f32 * 0.03).collect::<Vec<_>>();
- let dt_bias = (0..value_heads).map(|i| 0.1 - i as f32 * 0.02).collect::<Vec<_>>();
-
- let a_log = alloc_allocation_with_data::<Metal, f32>(&context, &a_log);
- let dt_bias = alloc_allocation_with_data::<Metal, f32>(&context, &dt_bias);
- let bytes_per_copy = trie.len() * size_of::<u32>() + len * size_of::<f32>() * 4;
- let mut buffers = ColdPool::new(bytes_per_copy, || PrefixBetaBuffers {
- trie: alloc_allocation_with_data::<Metal, u32>(&context, &trie),
- a_transposed: alloc_allocation_with_data::<Metal, f32>(&context, &a_transposed),
- b: alloc_allocation_with_data::<Metal, f32>(&context, &b_data),
- prefix: alloc_allocation::<Metal, f32>(&context, len),
- beta: alloc_allocation::<Metal, f32>(&context, len),
- });
-
- group.throughput(Throughput::Elements((len * tree_size) as u64));
- group.bench_function(BenchmarkId::from_parameter(format!("B1_T{tree_size}_HV{value_heads}")), |bencher| {
- let benchmark_path = format!(
- "{}/Kernel/GDNTreeVerify/BuildPrefixBeta/B1_T{tree_size}_HV{value_heads}",
- type_short_name::<Metal>()
- );
- iter_encode_loop_named::<Metal, _>(&context, bencher, &benchmark_path, |encoder| {
- let buffers = buffers.next_mut();
- kernel.encode(
- &buffers.trie,
- &buffers.a_transposed,
- &buffers.b,
- &a_log,
- &dt_bias,
- &mut buffers.prefix,
- &mut buffers.beta,
- batch_size as u32,
- tree_size as u32,
- value_heads as u32,
- encoder,
- );
- });
- });
- }
-}
diff --git a/crates/backend-uzu/unit/backends/common/kernel/gdn/tree_verify/prefix_beta_test.rs b/crates/backend-uzu/unit/backends/common/kernel/gdn/tree_verify/prefix_beta_test.rs
deleted file mode 100644
index 980ef8936..000000000
--- a/crates/backend-uzu/unit/backends/common/kernel/gdn/tree_verify/prefix_beta_test.rs
+++ /dev/null
@@ -1,112 +0,0 @@
-use proc_macros::uzu_test;
-use test_runner::for_each_non_cpu_backend;
-
-use crate::{
- backends::{
- common::{Backend, Context, Encoder, Kernels, kernel::BuildPrefixBetaKernel},
- cpu::Cpu,
- },
- data_type::DataType,
- tests::{
- assert::assert_eq_float,
- helpers::{alloc_allocation, alloc_allocation_with_data, allocation_to_vec},
- },
-};
-
-fn get_output<B: Backend>(
- trie: &[u32],
- a_transposed: &[f32],
- b: &[f32],
- a_log: &[f32],
- dt_bias: &[f32],
- batch_size: u32,
- tree_size: u32,
- value_heads: u32,
-) -> (Vec<f32>, Vec<f32>) {
- let context = B::Context::new().expect("Failed to create Context");
- let kernel = <<B as Backend>::Kernels as Kernels>::BuildPrefixBetaKernel::new(&context, DataType::F32)
- .expect("Failed to create BuildPrefixBetaKernel");
-
- let output_len = a_transposed.len();
- let trie = alloc_allocation_with_data::<B, u32>(&context, trie);
- let a_transposed = alloc_allocation_with_data::<B, f32>(&context, a_transposed);
- let b = alloc_allocation_with_data::<B, f32>(&context, b);
- let a_log = alloc_allocation_with_data::<B, f32>(&context, a_log);
- let dt_bias = alloc_allocation_with_data::<B, f32>(&context, dt_bias);
- let mut prefix = alloc_allocation::<B, f32>(&context, output_len);
- let mut beta = alloc_allocation::<B, f32>(&context, output_len);
-
- let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");
- kernel.encode(
- &trie,
- &a_transposed,
- &b,
- &a_log,
- &dt_bias,
- &mut prefix,
- &mut beta,
- batch_size,
- tree_size,
- value_heads,
- &mut encoder,
- );
- encoder.end_encoding().submit().wait_until_completed().unwrap();
-
- (allocation_to_vec(&prefix), allocation_to_vec(&beta))
-}
-
-// chain (batch 0) + broom (batch 1); the broom's varied [start, end] exercises
-// trie_end, which the chain (end always T-1) can't.
-fn build_trie(tree_size: usize) -> Vec<u32> {
- let last = tree_size as u32 - 1;
- let mut trie = Vec::with_capacity(2 * tree_size * 3);
- for i in 0..tree_size as u32 {
- trie.extend_from_slice(&[i, last, i]);
- }
- trie.extend_from_slice(&[0, last, 0]);
- for i in 1..tree_size as u32 {
- trie.extend_from_slice(&[i, i, 1]);
- }
- trie
-}
-
-#[uzu_test]
-fn test_build_prefix_beta_matches_cpu() {
- let batch_size = 2;
- let value_heads = 5;
-
- for tree_size in [5, 33, 129] {
- let len = batch_size * tree_size * value_heads;
- let trie = build_trie(tree_size);
- let a_transposed = (0..len).map(|i| (i as f32 * 0.13).sin() - 0.4).collect::<Vec<_>>();
- let b = (0..len).map(|i| (i as f32 * 0.11).cos() * 0.8).collect::<Vec<_>>();
- let a_log = (0..value_heads).map(|i| -0.2 + i as f32 * 0.03).collect::<Vec<_>>();
- let dt_bias = (0..value_heads).map(|i| 0.1 - i as f32 * 0.02).collect::<Vec<_>>();
- let expected = get_output::<Cpu>(
- &trie,
- &a_transposed,
- &b,
- &a_log,
- &dt_bias,
- batch_size as u32,
- tree_size as u32,
- value_heads as u32,
- );
-
- for_each_non_cpu_backend!(|B| {
- let output = get_output::<B>(
- &trie,
- &a_transposed,
- &b,
- &a_log,
- &dt_bias,
- batch_size as u32,
- tree_size as u32,
- value_heads as u32,
- );
- let msg = format!("backend {} tree_size {tree_size}", std::any::type_name::<B>());
- assert_eq_float::<f32>(&expected.0, &output.0, 1e-4, &format!("prefix {msg}"));
- assert_eq_float::<f32>(&expected.1, &output.1, 1e-5, &format!("beta {msg}"));
- });
- }
-}
diff --git a/crates/backend-uzu/unit/backends/common/kernel/gdn/tree_verify/prefix_test.rs b/crates/backend-uzu/unit/backends/common/kernel/gdn/tree_verify/prefix_test.rs
new file mode 100644
index 000000000..6392d7508
--- /dev/null
+++ b/crates/backend-uzu/unit/backends/common/kernel/gdn/tree_verify/prefix_test.rs
@@ -0,0 +1,46 @@
+#![cfg(metal_backend)]
+
+use proc_macros::uzu_test;
+use test_runner::for_each_non_cpu_backend;
+
+use crate::{
+ backends::{
+ common::{Backend, Context, Encoder, Kernels, kernel::BuildTreePrefixKernel},
+ cpu::Cpu,
+ },
+ tests::{
+ assert::assert_eq_float,
+ helpers::{alloc_allocation, alloc_allocation_with_data, allocation_to_vec},
+ },
+};
+
+fn run<B: Backend>(tree_size: usize) -> Vec<f32> {
+ const HEADS: usize = 5;
+ let context = B::Context::new().unwrap();
+ let trie = (0..tree_size as u32).flat_map(|node| [node, tree_size as u32 - 1, node]).collect::<Vec<_>>();
+ let log_decay = (0..tree_size * HEADS).map(|i| -0.001 - i as f32 * 0.0001).collect::<Vec<_>>();
+ let trie = alloc_allocation_with_data::<B, u32>(&context, &trie);
+ let log_decay = alloc_allocation_with_data::<B, f32>(&context, &log_decay);
+ let mut prefix = alloc_allocation::<B, f32>(&context, tree_size * HEADS);
+ let kernel = <B::Kernels as Kernels>::BuildTreePrefixKernel::new(&context).unwrap();
+ let mut encoder = Encoder::new(context.as_ref()).unwrap();
+ kernel.encode(&trie, &log_decay, &mut prefix, 1, tree_size as u32, HEADS as u32, &mut encoder);
+ encoder.end_encoding().submit().wait_until_completed().unwrap();
+ allocation_to_vec(&prefix)
+}
+
+#[uzu_test]
+fn test_build_tree_prefix() {
+ const HEADS: usize = 5;
+ for tree_size in [49, 64, 128] {
+ let expected = (0..tree_size)
+ .flat_map(|row| {
+ (0..HEADS).map(move |head| {
+ (0..=row).map(|token| -0.001 - (token * HEADS + head) as f32 * 0.0001).sum::<f32>()
+ })
+ })
+ .collect::<Vec<_>>();
+ assert_eq_float(&expected, &run::<Cpu>(tree_size), 1e-6, "CPU");
+ for_each_non_cpu_backend!(|B| assert_eq_float(&expected, &run::<B>(tree_size), 1e-6, "backend"));
+ }
+}
diff --git a/crates/backend-uzu/unit/backends/common/kernel/gdn/tree_verify/state_advance_test.rs b/crates/backend-uzu/unit/backends/common/kernel/gdn/tree_verify/state_advance_test.rs
index f40a39e1d..1b141d5b2 100644
--- a/crates/backend-uzu/unit/backends/common/kernel/gdn/tree_verify/state_advance_test.rs
+++ b/crates/backend-uzu/unit/backends/common/kernel/gdn/tree_verify/state_advance_test.rs
@@ -36,20 +36,25 @@ const TEST_NUM_K_HEADS: usize = 2;
fn run<B: Backend, T: ArrayElement + Float>(accepted_indices: &[u32]) -> Vec<f32> {
let context = B::Context::new().expect("context");
- let kernel =
- <<B as Backend>::Kernels as Kernels>::StateAdvanceKernel::new(&context, T::data_type(), HEAD_DIM as u32)
- .expect("kernel");
+ let kernel = <<B as Backend>::Kernels as Kernels>::StateAdvanceKernel::new(
+ &context,
+ T::data_type(),
+ HEAD_DIM as u32,
+ TEST_NUM_V_HEADS as u32,
+ TEST_NUM_K_HEADS as u32,
+ )
+ .expect("kernel");
let key_len = TREE_SIZE * TEST_NUM_K_HEADS * HEAD_DIM;
let value_len = TREE_SIZE * TEST_NUM_V_HEADS * HEAD_DIM;
let scalar_len = TREE_SIZE * TEST_NUM_V_HEADS;
let state_len = TEST_NUM_V_HEADS * HEAD_DIM * HEAD_DIM;
- let k_norm = (0..key_len).map(|i| (i % 17) as f32 * 0.002 - 0.016).collect::<Vec<_>>();
+ let k_norm = (0..key_len).map(|i| T::from((i % 17) as f32 * 0.002 - 0.016).unwrap()).collect::<Vec<_>>();
let v = (0..value_len).map(|i| T::from((i % 19) as f32 * 0.003 - 0.027).unwrap()).collect::<Vec<_>>();
let log_decay = (0..scalar_len).map(|i| -0.01 - (i % 7) as f32 * 0.002).collect::<Vec<_>>();
let beta = (0..scalar_len).map(|i| 0.2 + (i % 5) as f32 * 0.03).collect::<Vec<_>>();
let initial_state = (0..state_len).map(|i| (i % 23) as f32 * 0.001 - 0.011).collect::<Vec<_>>();
- let k_norm = alloc_allocation_with_data::<B, f32>(&context, &k_norm);
+ let k_norm = alloc_allocation_with_data::<B, T>(&context, &k_norm);
let v = alloc_allocation_with_data::<B, T>(&context, &v);
let log_decay = alloc_allocation_with_data::<B, f32>(&context, &log_decay);
let beta = alloc_allocation_with_data::<B, f32>(&context, &beta);
@@ -66,8 +71,6 @@ fn run<B: Backend, T: ArrayElement + Float>(accepted_indices: &[u32]) -> Vec<f32
&accepted_indices,
&mut committed_state,
accepted_len as u32,
- TEST_NUM_V_HEADS as u32,
- TEST_NUM_K_HEADS as u32,
&mut encoder,
);
encoder.end_encoding().submit().wait_until_completed().unwrap();
@@ -92,10 +95,18 @@ fn bench_state_advance(c: &mut Criterion) {
const BENCHMARK: &str = "Metal/Kernel/GDNTreeVerify/StateAdvance";
let context = crate::tests::util::shared_metal_context();
- let kernel =
- <<Metal as Backend>::Kernels as Kernels>::StateAdvanceKernel::new(&context, DataType::BF16, HEAD_DIM as u32)
- .expect("kernel");
- let k_norm = alloc_allocation_with_data::<Metal, f32>(&context, &vec![0.001; TREE_SIZE * NUM_K_HEADS * HEAD_DIM]);
+ let kernel = <<Metal as Backend>::Kernels as Kernels>::StateAdvanceKernel::new(
+ &context,
+ DataType::BF16,
+ HEAD_DIM as u32,
+ NUM_V_HEADS as u32,
+ NUM_K_HEADS as u32,
+ )
+ .expect("kernel");
+ let k_norm = alloc_allocation_with_data::<Metal, bf16>(
+ &context,
+ &vec![bf16::from_f32(0.001); TREE_SIZE * NUM_K_HEADS * HEAD_DIM],
+ );
let v = alloc_allocation_with_data::<Metal, bf16>(
&context,
&vec![bf16::from_f32(0.01); TREE_SIZE * NUM_V_HEADS * HEAD_DIM],
@@ -121,8 +132,6 @@ fn bench_state_advance(c: &mut Criterion) {
&accepted_indices,
committed_states.next_mut(),
accepted_len as u32,
- NUM_V_HEADS as u32,
- NUM_K_HEADS as u32,
encoder,
);
});
diff --git a/crates/backend-uzu/unit/backends/common/kernel/gdn/tree_verify/tree_gram_bench.rs b/crates/backend-uzu/unit/backends/common/kernel/gdn/tree_verify/tree_gram_bench.rs
index ff5199ee9..0e01ed629 100644
--- a/crates/backend-uzu/unit/backends/common/kernel/gdn/tree_verify/tree_gram_bench.rs
+++ b/crates/backend-uzu/unit/backends/common/kernel/gdn/tree_verify/tree_gram_bench.rs
@@ -23,7 +23,7 @@ const K_HEADS: usize = 16;
const VALUE_HEADS: usize = 48;
const HEAD_K_DIM: usize = 128;
const HEAD_V_DIM: usize = 128;
-const TREE_SIZES: &[usize] = &[33, 49, 64, 128, 256, 512];
+const TREE_SIZES: &[usize] = &[32, 33, 49, 64, 128, 256, 512];
const BATCH_SIZES: &[usize] = &[1, 2, 4, 8];
struct TreeGramBuffers {
@@ -104,7 +104,7 @@ fn make_buffers(
.map(|i| -((i % tree_size) as f32) * 0.01 - ((i % VALUE_HEADS) as f32) * 0.003)
.collect::<Vec<_>>();
let beta = (0..scalar_len).map(|i| 0.25 + ((i as f32 * 0.013).sin() + 1.0) * 0.2).collect::<Vec<_>>();
- let h0 = (0..h0_len).map(|i| bf16::from_f32(((i as f32 * 0.007).sin() * 0.05) - 0.01)).collect::<Vec<_>>();
+ let h0 = (0..h0_len).map(|i| ((i as f32 * 0.007).sin() * 0.05) - 0.01).collect::<Vec<_>>();
let h0_idx = (0..batch_size).map(|i| i as i32).collect::<Vec<_>>();
let scale = (HEAD_K_DIM as f32).sqrt().recip();
(
@@ -114,7 +114,7 @@ fn make_buffers(
trie: alloc_allocation_with_data::<Metal, u32>(context, &trie),
prefix: alloc_allocation_with_data::<Metal, f32>(context, &prefix),
beta: alloc_allocation_with_data::<Metal, f32>(context, &beta),
- h0: alloc_allocation_with_data::<Metal, bf16>(context, &h0),
+ h0: alloc_allocation_with_data::<Metal, f32>(context, &h0),
h0_idx: alloc_allocation_with_data::<Metal, i32>(context, &h0_idx),
a_packed: alloc_allocation::<Metal, f32>(context, a_len),
qkd: alloc_allocation::<Metal, f32>(context, out_len),
@@ -137,7 +137,8 @@ fn buffers_bytes(
let a_inv_len = batch_size * VALUE_HEADS * num_blocks * 16 * 16;
let h0_len = batch_size * VALUE_HEADS * HEAD_V_DIM * HEAD_K_DIM;
let kh0_len = batch_size * tree_size * VALUE_HEADS * HEAD_V_DIM;
- (qk_len * 2 + h0_len) * size_of::<bf16>()
+ qk_len * 2 * size_of::<bf16>()
+ + h0_len * size_of::<f32>()
+ batch_size * tree_size * 3 * size_of::<u32>()
+ scalar_len * size_of::<f32>() * 2
+ (out_len + a_len + a_inv_len + kh0_len) * size_of::<f32>()
diff --git a/crates/backend-uzu/unit/backends/common/kernel/gdn/tree_verify/tree_gram_test.rs b/crates/backend-uzu/unit/backends/common/kernel/gdn/tree_verify/tree_gram_test.rs
index 627066065..ae7744b2b 100644
--- a/crates/backend-uzu/unit/backends/common/kernel/gdn/tree_verify/tree_gram_test.rs
+++ b/crates/backend-uzu/unit/backends/common/kernel/gdn/tree_verify/tree_gram_test.rs
@@ -1,9 +1,12 @@
#![cfg(metal_backend)]
+use half::bf16;
+use num_traits::Float;
use proc_macros::uzu_test;
use test_runner::for_each_non_cpu_backend;
use crate::{
+ array::ArrayElement,
backends::{
common::{Backend, Context, Encoder, Kernels, kernel::BuildTreeGramKernel},
cpu::Cpu,
@@ -51,9 +54,9 @@ fn make_inputs(tree_size: usize) -> Inputs {
}
}
-fn get_output<B: Backend>(
- q: &[f32],
- k: &[f32],
+fn get_output<B: Backend, T: ArrayElement + Float>(
+ q: &[T],
+ k: &[T],
trie: &[u32],
prefix: &[f32],
beta: &[f32],
@@ -64,16 +67,17 @@ fn get_output<B: Backend>(
use_mxu: bool,
) -> (Vec<f32>, Vec<f32>, Vec<f32>, Vec<f32>) {
let context = B::Context::new().expect("Failed to create Context");
- let kernel = <<B as Backend>::Kernels as Kernels>::BuildTreeGramKernel::new(&context, DataType::F32, use_mxu, true)
- .expect("Failed to create BuildTreeGramKernel");
+ let kernel =
+ <<B as Backend>::Kernels as Kernels>::BuildTreeGramKernel::new(&context, T::data_type(), use_mxu, true)
+ .expect("Failed to create BuildTreeGramKernel");
let output_len = BATCH_SIZE * VALUE_HEADS * tree_size * tree_size;
let num_blocks = tree_size.div_ceil(16);
let a_len = BATCH_SIZE * VALUE_HEADS * num_blocks * num_blocks.div_ceil(2) * 16 * 32;
let a_inv_len = BATCH_SIZE * VALUE_HEADS * num_blocks * 16 * 16;
let kh0_len = BATCH_SIZE * tree_size * VALUE_HEADS * HEAD_V_DIM;
- let q = alloc_allocation_with_data::<B, f32>(&context, q);
- let k = alloc_allocation_with_data::<B, f32>(&context, k);
+ let q = alloc_allocation_with_data::<B, T>(&context, q);
+ let k = alloc_allocation_with_data::<B, T>(&context, k);
let trie = alloc_allocation_with_data::<B, u32>(&context, trie);
let prefix = alloc_allocation_with_data::<B, f32>(&context, prefix);
let beta = alloc_allocation_with_data::<B, f32>(&context, beta);
@@ -115,6 +119,72 @@ fn get_output<B: Backend>(
(allocation_to_vec(&a_packed), allocation_to_vec(&qkd), allocation_to_vec(&a_inv), allocation_to_vec(&kh0))
}
+fn check_type<T: ArrayElement + Float>(
+ inputs: &Inputs,
+ tree_size: usize,
+ scale: f32,
+) {
+ let q = inputs.q.iter().map(|&value| T::from(value).unwrap()).collect::<Vec<_>>();
+ let k = inputs.k.iter().map(|&value| T::from(value).unwrap()).collect::<Vec<_>>();
+ let expected = get_output::<Cpu, T>(
+ &q,
+ &k,
+ &inputs.trie,
+ &inputs.prefix,
+ &inputs.beta,
+ &inputs.h0,
+ &inputs.h0_idx,
+ tree_size,
+ scale,
+ false,
+ );
+
+ for_each_non_cpu_backend!(|B| {
+ let actual = get_output::<B, T>(
+ &q,
+ &k,
+ &inputs.trie,
+ &inputs.prefix,
+ &inputs.beta,
+ &inputs.h0,
+ &inputs.h0_idx,
+ tree_size,
+ scale,
+ false,
+ );
+ let msg = format!("backend {} simdgroup tree_size {tree_size}", std::any::type_name::<B>());
+ assert_eq_float::<f32>(&expected.0, &actual.0, 2e-4, &format!("a_packed {msg}"));
+ assert_eq_float::<f32>(&expected.1, &actual.1, 2e-4, &format!("qkd {msg}"));
+ assert_eq_float::<f32>(&expected.2, &actual.2, 5e-4, &format!("a_inv {msg}"));
+ assert_eq_float::<f32>(&expected.3, &actual.3, 2e-4, &format!("kh0 {msg}"));
+ });
+
+ if <Metal as Backend>::Context::new().expect("Failed to create Context").supports_mxu() {
+ let actual = get_output::<Metal, T>(
+ &q,
+ &k,
+ &inputs.trie,
+ &inputs.prefix,
+ &inputs.beta,
+ &inputs.h0,
+ &inputs.h0_idx,
+ tree_size,
+ scale,
+ true,
+ );
+ let msg = format!("backend {} path MXU tree_size {tree_size}", std::any::type_name::<Metal>());
+ let (a_eps, qkd_eps, inv_eps, kh0_eps) = if T::data_type() == DataType::BF16 {
+ (1e-5, 1e-6, 1e-5, 5e-4)
+ } else {
+ (5e-3, 1e-3, 8e-3, 1e-3)
+ };
+ assert_eq_float::<f32>(&expected.0, &actual.0, a_eps, &format!("a_packed {msg}"));
+ assert_eq_float::<f32>(&expected.1, &actual.1, qkd_eps, &format!("qkd {msg}"));
+ assert_eq_float::<f32>(&expected.2, &actual.2, inv_eps, &format!("a_inv {msg}"));
+ assert_eq_float::<f32>(&expected.3, &actual.3, kh0_eps, &format!("kh0 {msg}"));
+ }
+}
+
fn build_trie(tree_size: usize) -> Vec<u32> {
let last = tree_size as u32 - 1;
let mut trie = Vec::with_capacity(BATCH_SIZE * tree_size * 3);
@@ -134,27 +204,7 @@ fn test_build_tree_gram_matches_cpu() {
for tree_size in [17, 64, 128] {
let i = make_inputs(tree_size);
- let expected =
- get_output::<Cpu>(&i.q, &i.k, &i.trie, &i.prefix, &i.beta, &i.h0, &i.h0_idx, tree_size, scale, false);
-
- for_each_non_cpu_backend!(|B| {
- let actual =
- get_output::<B>(&i.q, &i.k, &i.trie, &i.prefix, &i.beta, &i.h0, &i.h0_idx, tree_size, scale, false);
- let msg = format!("backend {} simdgroup tree_size {tree_size}", std::any::type_name::<B>());
- assert_eq_float::<f32>(&expected.0, &actual.0, 5e-3, &format!("a_packed {msg}"));
- assert_eq_float::<f32>(&expected.1, &actual.1, 5e-3, &format!("qkd {msg}"));
- assert_eq_float::<f32>(&expected.2, &actual.2, 1e-2, &format!("a_inv {msg}"));
- assert_eq_float::<f32>(&expected.3, &actual.3, 5e-3, &format!("kh0 {msg}"));
- });
-
- if <Metal as Backend>::Context::new().expect("Failed to create Context").supports_mxu() {
- let actual =
- get_output::<Metal>(&i.q, &i.k, &i.trie, &i.prefix, &i.beta, &i.h0, &i.h0_idx, tree_size, scale, true);
- let msg = format!("backend {} path MXU tree_size {tree_size}", std::any::type_name::<Metal>());
- assert_eq_float::<f32>(&expected.0, &actual.0, 5e-3, &format!("a_packed {msg}"));
- assert_eq_float::<f32>(&expected.1, &actual.1, 5e-3, &format!("qkd {msg}"));
- assert_eq_float::<f32>(&expected.2, &actual.2, 1e-2, &format!("a_inv {msg}"));
- assert_eq_float::<f32>(&expected.3, &actual.3, 5e-3, &format!("kh0 {msg}"));
- }
+ check_type::<f32>(&i, tree_size, scale);
+ check_type::<bf16>(&i, tree_size, scale);
}
}
diff --git a/crates/backend-uzu/unit/backends/common/kernel/gdn/tree_verify/tree_update_solve_bench.rs b/crates/backend-uzu/unit/backends/common/kernel/gdn/tree_verify/tree_update_solve_bench.rs
index 897d564fb..01cf09285 100644
--- a/crates/backend-uzu/unit/backends/common/kernel/gdn/tree_verify/tree_update_solve_bench.rs
+++ b/crates/backend-uzu/unit/backends/common/kernel/gdn/tree_verify/tree_update_solve_bench.rs
@@ -23,9 +23,9 @@ const NUM_V_HEADS: usize = 48;
const HEAD_V_DIM: usize = 128;
const BT: usize = 16;
const BVS: &[usize] = &[16, 32];
-const MXU_BVS: &[usize] = &[32];
const BENCH_SHAPES: &[(usize, usize)] = &[
+ (1, 32),
(1, 33),
(1, 49),
(1, 64),
@@ -172,51 +172,38 @@ fn buffers_bytes(
#[uzu_bench]
fn bench_tree_update_solve(c: &mut Criterion) {
let context = <Metal as Backend>::Context::new().expect("metal context");
- let kernel_paths = if context.supports_mxu() {
- &[("Simdgroup", false, BVS), ("MXU", true, MXU_BVS)][..]
- } else {
- &[("Simdgroup", false, BVS)][..]
- };
-
- for &(kernel_path, use_mxu, bvs) in kernel_paths {
- for &bv in bvs {
- let kernel = <<Metal as Backend>::Kernels as Kernels>::TreeUpdateSolveKernel::new(
- &context,
- DataType::BF16,
- bv as u32,
- use_mxu,
- true,
- )
- .expect("TreeUpdateSolveKernel");
-
- let mut group =
- c.benchmark_group(format!("Metal/Kernel/GDNTreeVerify/TreeUpdateSolve/{kernel_path}/BV{bv}"));
- // 500ms warmup ramps GPU clocks before measuring (µs-scale benches never
- // ramp otherwise and medians become run-composition artifacts).
- group.sample_size(20).warm_up_time(Duration::from_millis(500)).measurement_time(Duration::from_secs(2));
-
- for &(batch_size, tree_size) in BENCH_SHAPES {
- let benchmark_path = format!(
- "Metal/Kernel/GDNTreeVerify/TreeUpdateSolve/{kernel_path}/BV{bv}/B{batch_size}_T{tree_size}"
- );
- let benchmark_id = BenchmarkId::from_parameter(format!(
- "B{batch_size}_T{tree_size}_HV{NUM_V_HEADS}_V{HEAD_V_DIM}_BV{bv}"
- ));
- let mut buffers = ColdPool::new(buffers_bytes(batch_size, tree_size), || {
- make_buffers(&context, batch_size, tree_size)
- });
-
- group.throughput(Throughput::Elements((batch_size * tree_size * NUM_V_HEADS * HEAD_V_DIM) as u64));
- group.bench_function(benchmark_id, |bencher| {
- iter_encode_loop_named::<Metal, _>(context.as_ref(), bencher, &benchmark_path, |encoder| {
- let buffers = buffers.next_mut();
- encode(&kernel, buffers, batch_size, tree_size, encoder);
- });
+ for &bv in BVS {
+ let kernel = <<Metal as Backend>::Kernels as Kernels>::TreeUpdateSolveKernel::new(
+ &context,
+ DataType::BF16,
+ bv as u32,
+ true,
+ )
+ .expect("TreeUpdateSolveKernel");
+
+ let mut group = c.benchmark_group(format!("Metal/Kernel/GDNTreeVerify/TreeUpdateSolve/Simdgroup/BV{bv}"));
+ // 500ms warmup ramps GPU clocks before measuring (µs-scale benches never
+ // ramp otherwise and medians become run-composition artifacts).
+ group.sample_size(20).warm_up_time(Duration::from_millis(500)).measurement_time(Duration::from_secs(2));
+
+ for &(batch_size, tree_size) in BENCH_SHAPES {
+ let benchmark_path =
+ format!("Metal/Kernel/GDNTreeVerify/TreeUpdateSolve/Simdgroup/BV{bv}/B{batch_size}_T{tree_size}");
+ let benchmark_id =
+ BenchmarkId::from_parameter(format!("B{batch_size}_T{tree_size}_HV{NUM_V_HEADS}_V{HEAD_V_DIM}_BV{bv}"));
+ let mut buffers =
+ ColdPool::new(buffers_bytes(batch_size, tree_size), || make_buffers(&context, batch_size, tree_size));
+
+ group.throughput(Throughput::Elements((batch_size * tree_size * NUM_V_HEADS * HEAD_V_DIM) as u64));
+ group.bench_function(benchmark_id, |bencher| {
+ iter_encode_loop_named::<Metal, _>(context.as_ref(), bencher, &benchmark_path, |encoder| {
+ let buffers = buffers.next_mut();
+ encode(&kernel, buffers, batch_size, tree_size, encoder);
});
- }
- group.finish();
- test_runner::metrics::wait_gpu_cooldown();
+ });
}
+ group.finish();
+ test_runner::metrics::wait_gpu_cooldown();
}
}
diff --git a/crates/backend-uzu/unit/backends/common/kernel/gdn/tree_verify/tree_update_solve_test.rs b/crates/backend-uzu/unit/backends/common/kernel/gdn/tree_verify/tree_update_solve_test.rs
index 45e94396b..66b6db9b9 100644
--- a/crates/backend-uzu/unit/backends/common/kernel/gdn/tree_verify/tree_update_solve_test.rs
+++ b/crates/backend-uzu/unit/backends/common/kernel/gdn/tree_verify/tree_update_solve_test.rs
@@ -9,7 +9,6 @@ use crate::{
backends::{
common::{Backend, Context, Encoder, Kernels, kernel::TreeUpdateSolveKernel},
cpu::Cpu,
- metal::Metal,
},
data_type::DataType,
tests::{
@@ -88,15 +87,13 @@ const CASES: &[SolveCase] = &[
fn run_case<B: Backend, T: ArrayElement + Copy>(
case: SolveCase,
bv: u32,
- use_mxu: bool,
use_h0: bool,
data_type: DataType,
cast: impl Fn(f32) -> T + Copy,
) -> Vec<f32> {
let context = B::Context::new().expect("context");
- let kernel =
- <<B as Backend>::Kernels as Kernels>::TreeUpdateSolveKernel::new(&context, data_type, bv, use_mxu, use_h0)
- .expect("kernel");
+ let kernel = <<B as Backend>::Kernels as Kernels>::TreeUpdateSolveKernel::new(&context, data_type, bv, use_h0)
+ .expect("kernel");
let batch_size = case.batch_size as usize;
let tree_size = case.tree_size as usize;
@@ -191,34 +188,23 @@ fn test_tree_update_solve_cases() {
for &bv in BVS {
for &use_h0 in &[true, false] {
for &case in CASES {
- let expected = run_case::<Cpu, f32>(case, bv, false, use_h0, DataType::F32, |x| x);
+ let expected = run_case::<Cpu, f32>(case, bv, use_h0, DataType::F32, |x| x);
for_each_non_cpu_backend!(|B| {
- let output = run_case::<B, f32>(case, bv, false, use_h0, DataType::F32, |x| x);
+ let output = run_case::<B, f32>(case, bv, use_h0, DataType::F32, |x| x);
assert_eq_float(&expected, &output, 1e-4, &format!("{} BV{bv} h0={use_h0}", case.name));
});
}
}
}
- {
- let supports_mxu = <Metal as Backend>::Context::new().expect("context").supports_mxu();
- let paths: &[(&str, bool)] = if supports_mxu {
- &[("Simdgroup", false), ("MXU", true)]
- } else {
- &[("Simdgroup", false)]
- };
-
- // MXU needs an even number of column fragments or MPP row pairing; BV16
- // (a single 16x16 fragment) is not supported.
- for &bv in &[32] {
- for &use_h0 in &[true, false] {
- for &case in CASES {
- let expected = run_case::<Cpu, bf16>(case, bv, false, use_h0, DataType::BF16, bf16::from_f32);
- for &(path, use_mxu) in paths {
- let output = run_case::<Metal, bf16>(case, bv, use_mxu, use_h0, DataType::BF16, bf16::from_f32);
- assert_eq_float(&expected, &output, 1e-2, &format!("{} {path} BV{bv} h0={use_h0}", case.name));
- }
- }
+ for &bv in BVS {
+ for &use_h0 in &[true, false] {
+ for &case in CASES {
+ let expected = run_case::<Cpu, bf16>(case, bv, use_h0, DataType::BF16, bf16::from_f32);
+ for_each_non_cpu_backend!(|B| {
+ let output = run_case::<B, bf16>(case, bv, use_h0, DataType::BF16, bf16::from_f32);
+ assert_eq_float(&expected, &output, 5e-4, &format!("{} BV{bv} h0={use_h0}", case.name));
+ });
}
}
}
diff --git a/crates/backend-uzu/unit/backends/metal/kernel/gdn/chunked_test.rs b/crates/backend-uzu/unit/backends/metal/kernel/gdn/chunked_test.rs
index 0948a6fb5..220c352b2 100644
--- a/crates/backend-uzu/unit/backends/metal/kernel/gdn/chunked_test.rs
+++ b/crates/backend-uzu/unit/backends/metal/kernel/gdn/chunked_test.rs
@@ -82,8 +82,10 @@ fn run_prefill<T: ArrayElement>(
<BackendKernels<Metal> as Kernels>::DeltaNetPrefillPrepKernel::new(
context,
T::data_type(),
+ DataType::F32,
HEAD_K_DIM as u32,
false,
+ false,
)
.expect("prep")
.encode(
@@ -92,6 +94,7 @@ fn run_prefill<T: ArrayElement>(
&dt_bias,
&mut q,
&mut k,
+ None::<&mut crate::backends::common::Allocation<Metal>>,
&mut beta,
&mut decay,
NUM_V_HEADS as u32,
diff --git a/crates/backend-uzu/unit/encodable_block/delta_net_tree_verify_bench.rs b/crates/backend-uzu/unit/encodable_block/delta_net_tree_verify_bench.rs
new file mode 100644
index 000000000..0910f2e17
--- /dev/null
+++ b/crates/backend-uzu/unit/encodable_block/delta_net_tree_verify_bench.rs
@@ -0,0 +1,80 @@
+#![cfg(metal_backend)]
+
+use std::time::{Duration, Instant};
+
+use criterion::Criterion;
+use half::bf16;
+use proc_macros::uzu_bench;
+
+use crate::{
+ backends::{
+ common::{Backend, Context, Kernels, kernel::delta_net_tree_verify::DeltaNetTreeVerify},
+ metal::Metal,
+ },
+ data_type::DataType,
+ encodable_block::mixer::delta_net::tree_verify::{TreeVerifyEncodeArguments, TreeVerifyNewArguments},
+ tests::{helpers::alloc_allocation_with_data, matmul::iter_encode_loop_named},
+};
+
+#[uzu_bench]
+fn bench_delta_net_tree_verify(c: &mut Criterion) {
+ const K_HEADS: usize = 16;
+ const V_HEADS: usize = 48;
+ const HEAD_DIM: usize = 128;
+ let arguments = TreeVerifyNewArguments {
+ data_type: DataType::BF16,
+ num_k_heads: K_HEADS,
+ num_v_heads: V_HEADS,
+ head_k_dim: HEAD_DIM,
+ head_v_dim: HEAD_DIM,
+ };
+ let context = <Metal as Backend>::Context::new().unwrap();
+ let started = Instant::now();
+ let tree_verify =
+ <<Metal as Backend>::Kernels as Kernels>::DeltaNetTreeVerify::new(context.as_ref(), &arguments).unwrap();
+ eprintln!("GDN tree verification PSO construction: {:.2} ms", started.elapsed().as_secs_f64() * 1e3);
+ let h0 = alloc_allocation_with_data::<Metal, f32>(&context, &vec![0.001; V_HEADS * HEAD_DIM * HEAD_DIM]);
+ let mut group = c.benchmark_group("Metal/Kernel/GDNTreeVerify");
+ group.sample_size(20).warm_up_time(Duration::from_millis(300)).measurement_time(Duration::from_secs(1));
+
+ for tree_size in [32usize, 49, 64, 128] {
+ let q = alloc_allocation_with_data::<Metal, bf16>(
+ &context,
+ &vec![bf16::from_f32(0.01); tree_size * K_HEADS * HEAD_DIM],
+ );
+ let k = alloc_allocation_with_data::<Metal, bf16>(
+ &context,
+ &vec![bf16::from_f32(0.02); tree_size * K_HEADS * HEAD_DIM],
+ );
+ let v = alloc_allocation_with_data::<Metal, bf16>(
+ &context,
+ &vec![bf16::from_f32(0.03); tree_size * V_HEADS * HEAD_DIM],
+ );
+ let trie = alloc_allocation_with_data::<Metal, u32>(
+ &context,
+ &(0..tree_size as u32).flat_map(|node| [node, tree_size as u32 - 1, node]).collect::<Vec<_>>(),
+ );
+ let log_decay = alloc_allocation_with_data::<Metal, f32>(&context, &vec![-0.01; tree_size * V_HEADS]);
+ let beta = alloc_allocation_with_data::<Metal, f32>(&context, &vec![0.2; tree_size * V_HEADS]);
+ let benchmark_path = format!("Metal/Kernel/GDNTreeVerify/T{tree_size}");
+ group.bench_function(format!("T{tree_size}"), |bencher| {
+ iter_encode_loop_named(context.as_ref(), bencher, &benchmark_path, |encoder| {
+ tree_verify
+ .encode(
+ TreeVerifyEncodeArguments {
+ q: &q,
+ k: &k,
+ v: &v,
+ trie: &trie,
+ log_decay: &log_decay,
+ beta: &beta,
+ h0: &h0,
+ tree_size,
+ },
+ encoder,
+ )
+ .unwrap();
+ })
+ });
+ }
+}