Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

Support BuildTreeGram Kernel (#569)

CCYeh5f01024ba4e5
diff --git a/crates/backend-uzu/src/backends/cpu/kernel/gdn_tree_verify/gram.rs b/crates/backend-uzu/src/backends/cpu/kernel/gdn_tree_verify/gram.rs
new file mode 100644
index 000000000..1da6f5688
--- /dev/null
+++ b/crates/backend-uzu/src/backends/cpu/kernel/gdn_tree_verify/gram.rs
@@ -0,0 +1,113 @@
+use half::bf16;
+use num_traits::Float;
+use proc_macros::kernel;
+
+use crate::{array::ArrayElement, backends::common::gpu_types::trie::TrieNode};
+#[kernel(BuildTreeGram)]
+#[variants(T, f32, bf16)]
+#[variants(USE_MXU, false)]
+pub fn build_tree_gram<T: ArrayElement + Float, const USE_MXU: bool>(
+ q: *const T,
+ k: *const T,
+ trie: *const TrieNode,
+ prefix: *const f32,
+ beta: *const f32,
+ a_mat: *mut f32,
+ qkd: *mut f32,
+ ainv: *mut f32,
+ scale: f32,
+ batch_size: u32,
+ tree_size: u32,
+ k_heads: u32,
+ value_heads: u32,
+ head_k_dim: u32,
+) {
+ let batch_size = batch_size as usize;
+ let tree_size = tree_size as usize;
+ let k_heads = k_heads as usize;
+ let value_heads = value_heads as usize;
+ let head_k_dim = head_k_dim as usize;
+
+ let groups_per_head = value_heads / k_heads;
+
+ for batch in 0..batch_size {
+ for hv in 0..value_heads {
+ let hk = hv / groups_per_head;
+ let mat_base = (batch * value_heads + hv) * tree_size * tree_size;
+ let trie_base = batch * tree_size;
+
+ for i in 0..tree_size {
+ let row_idx = i as u32;
+ let prefix_i = unsafe { *prefix.add((batch * tree_size + i) * value_heads + hv) };
+ let beta_i = unsafe { *beta.add((batch * tree_size + i) * value_heads + hv) };
+ let row_off = ((batch * tree_size + i) * k_heads + hk) * head_k_dim;
+ for j in 0..tree_size {
+ let out = mat_base + i * tree_size + j;
+ let node = unsafe { *trie.add(trie_base + j) };
+ let incl = row_idx >= node.trie_start && row_idx <= node.trie_end;
+ if !incl {
+ unsafe {
+ *a_mat.add(out) = 0.0;
+ *qkd.add(out) = 0.0;
+ }
+ continue;
+ }
+ let col_off = ((batch * tree_size + j) * k_heads + hk) * head_k_dim;
+ let mut kk = 0.0f32;
+ let mut qk = 0.0f32;
+ for d in 0..head_k_dim {
+ let ki = unsafe { (*k.add(row_off + d)).to_f32().unwrap() };
+ let kj = unsafe { (*k.add(col_off + d)).to_f32().unwrap() };
+ let qi = unsafe { (*q.add(row_off + d)).to_f32().unwrap() };
+ kk += ki * kj;
+ qk += qi * kj;
+ }
+ let prefix_j = unsafe { *prefix.add((batch * tree_size + j) * value_heads + hv) };
+ let dexp = (prefix_i - prefix_j).exp();
+ unsafe {
+ *a_mat.add(out) = if i != j {
+ beta_i * dexp * kk
+ } else {
+ 0.0
+ };
+ *qkd.add(out) = dexp * scale * qk;
+ }
+ }
+ }
+
+ invert_tree_gram_matrix(a_mat, ainv, mat_base, tree_size);
+ }
+ }
+}
+
+fn invert_tree_gram_matrix(
+ a_mat: *const f32,
+ ainv: *mut f32,
+ mat_base: usize,
+ tree_size: usize,
+) {
+ for i in 0..tree_size {
+ for j in 0..tree_size {
+ unsafe {
+ *ainv.add(mat_base + i * tree_size + j) = if i == j {
+ 1.0
+ } else {
+ 0.0
+ };
+ }
+ }
+ }
+ for i in 0..tree_size {
+ for j in 0..i {
+ let mut s = 0.0f32;
+ for kx in j..i {
+ unsafe {
+ s += *a_mat.add(mat_base + i * tree_size + kx) * *ainv.add(mat_base + kx * tree_size + j);
+ }
+ }
+ unsafe {
+ *ainv.add(mat_base + i * tree_size + j) = -s;
+ }
+ }
+ }
+}
diff --git a/crates/backend-uzu/src/backends/cpu/kernel/gdn_tree_verify/mod.rs b/crates/backend-uzu/src/backends/cpu/kernel/gdn_tree_verify/mod.rs
index 524d1acad..dfd9a7402 100644
--- a/crates/backend-uzu/src/backends/cpu/kernel/gdn_tree_verify/mod.rs
+++ b/crates/backend-uzu/src/backends/cpu/kernel/gdn_tree_verify/mod.rs
@@ -1 +1,2 @@
+pub mod gram;
pub mod prefix_beta;
diff --git a/crates/backend-uzu/src/backends/metal/kernel/gdn_tree_verify/gram.metal b/crates/backend-uzu/src/backends/metal/kernel/gdn_tree_verify/gram.metal
new file mode 100644
index 000000000..d6211a506
--- /dev/null
+++ b/crates/backend-uzu/src/backends/metal/kernel/gdn_tree_verify/gram.metal
@@ -0,0 +1,260 @@
+#include <metal_stdlib>
+#include "../common/defines.h"
+#include "../common/dsl.h"
+#include "../common/thread_context.h"
+#include "../generated/trie.h"
+#include "../matmul/common/fragment.h"
+#include "../matmul/common/mxu_fragment_ops.h"
+#include "../matmul/common/simdgroup_fragment_ops.h"
+
+using namespace metal;
+using namespace uzu::matmul;
+using namespace uzu::trie;
+
+#define TREE_GRAM_ROW_TILE 16u
+#define TREE_GRAM_COL_TILE 32u
+#define TREE_GRAM_THREADS METAL_SIMD_SIZE
+#define TREE_GRAM_INVALID_ROW 0xffffffffu
+
+METAL_FUNC void invert_tree_gram_diagonal_block(
+ device float* ainv,
+ threadgroup const float* diag_a_tile,
+ const uint mat_base,
+ const uint row_base,
+ const uint tree_size,
+ const uint block_size,
+ const uint thread_idx
+);
+
+// Build tree gram matrices for one (batch, value-head) tile.
+//
+// q, k: [B, T, Hg, K], with key head hk = hv / (HV / Hg)
+// trie: [B, T] DFS intervals; col is an ancestor of row when row is in col's interval
+// prefix: [B, T, HV] path log-decay prefix from BuildPrefixBeta
+// beta: [B, T, HV] sigmoid gate from BuildPrefixBeta
+//
+// a_mat[row, col] = beta[row] * exp(prefix[row] - prefix[col]) * dot(k[row], k[col])
+// for proper ancestors only
+// qkd[row, col] = scale * exp(prefix[row] - prefix[col]) * dot(q[row], k[col])
+// for ancestor-or-self
+// ainv = (I + A)^-1 for each diagonal block
+template <typename T, bool USE_MXU>
+VARIANTS(T, float, bfloat)
+VARIANTS(USE_MXU, false, true)
+PUBLIC KERNEL(BuildTreeGram)(
+ const device T* q,
+ const device T* k,
+ const device TrieNode* trie,
+ const device float* prefix,
+ const device float* beta,
+ device float* a_mat,
+ device float* qkd,
+ device float* ainv,
+ constant const float& scale,
+ constant const uint& batch_size,
+ constant const uint& tree_size,
+ constant const uint& k_heads,
+ constant const uint& value_heads,
+ constant const uint& head_k_dim,
+ threadgroup float diag_a_tile[TREE_GRAM_ROW_TILE * TREE_GRAM_ROW_TILE],
+ threadgroup float row_prefix[TREE_GRAM_ROW_TILE],
+ threadgroup float row_beta[TREE_GRAM_ROW_TILE],
+ threadgroup uint row_token[TREE_GRAM_ROW_TILE],
+ threadgroup float col_prefix[TREE_GRAM_COL_TILE],
+ threadgroup uint col_trie_start[TREE_GRAM_COL_TILE],
+ threadgroup uint col_trie_end[TREE_GRAM_COL_TILE],
+ const ThreadContext thread_context,
+ const uint batch_idx GROUPS(batch_size),
+ const uint value_head_idx GROUPS(value_heads),
+ const uint tile_idx GROUPS(tree_size.div_ceil(TREE_GRAM_ROW_TILE) * tree_size.div_ceil(TREE_GRAM_COL_TILE)),
+ const uint thread_idx THREADS(TREE_GRAM_THREADS)
+) {
+ using Ops = metal::conditional_t<USE_MXU, MxuFragmentOps<>, SimdgroupFragmentOps>;
+ constexpr ushort ROW_FRAGMENTS = TREE_GRAM_ROW_TILE / Ops::FRAGMENT_ROWS;
+ constexpr ushort COL_FRAGMENTS = TREE_GRAM_COL_TILE / Ops::FRAGMENT_COLS;
+ using InputType = metal::conditional_t<USE_MXU, T, float>;
+ using AccFragment = Fragment<float, ROW_FRAGMENTS, COL_FRAGMENTS, Ops>;
+ using LeftFragment = OperandFragment<InputType, ROW_FRAGMENTS, 1, Ops>;
+ using RightFragment = OperandFragment<InputType, 1, COL_FRAGMENTS, Ops, ReadTranspose>;
+
+ const uint value_heads_per_key_head = value_heads / k_heads;
+ const uint key_head_idx = value_head_idx / value_heads_per_key_head;
+ const uint qk_stride = k_heads * head_k_dim;
+ const uint qk_base = (batch_idx * tree_size * k_heads + key_head_idx) * head_k_dim;
+ const uint prefix_base = batch_idx * tree_size * value_heads + value_head_idx;
+ const uint trie_base = batch_idx * tree_size;
+ const uint mat_base = (batch_idx * value_heads + value_head_idx) * tree_size * tree_size;
+
+ const uint col_tiles = div_ceil(tree_size, TREE_GRAM_COL_TILE);
+ const uint row_tile_idx = tile_idx / col_tiles;
+ const uint col_tile_idx = tile_idx - row_tile_idx * col_tiles;
+ const uint row_base = row_tile_idx * TREE_GRAM_ROW_TILE;
+ const uint col_base = col_tile_idx * TREE_GRAM_COL_TILE;
+ const ushort lane = thread_context.simd_lane_id;
+
+ if (row_base >= tree_size) {
+ return;
+ }
+
+ const uint tile_rows = min(TREE_GRAM_ROW_TILE, tree_size - row_base);
+ const uint tile_cols = min(TREE_GRAM_COL_TILE, tree_size - col_base);
+ const uint tile_base = mat_base + row_base * tree_size + col_base;
+
+ if (col_base >= row_base + TREE_GRAM_ROW_TILE) {
+ for (uint idx = thread_idx; idx < tile_rows * tile_cols; idx += TREE_GRAM_THREADS) {
+ const uint local_row = idx / tile_cols;
+ const uint local_col = idx - local_row * tile_cols;
+ const uint offset = tile_base + local_row * tree_size + local_col;
+ a_mat[offset] = 0.0f;
+ qkd[offset] = 0.0f;
+ }
+ return;
+ }
+
+ AccFragment kk_acc;
+ AccFragment qk_acc;
+ kk_acc.clear();
+ qk_acc.clear();
+
+ for (uint k_block_start = 0; k_block_start < head_k_dim; k_block_start += Ops::FRAGMENT_ROWS) {
+ const uint k_remaining = head_k_dim - k_block_start;
+ const uint k_tile_size = Ops::FRAGMENT_ROWS;
+ const uint valid_k_cols = min(k_remaining, k_tile_size);
+
+ LeftFragment k_left;
+ LeftFragment q_left;
+ RightFragment k_right;
+
+ const uint qk_row_base = qk_base + row_base * qk_stride + k_block_start;
+ const uint qk_col_base = qk_base + col_base * qk_stride + k_block_start;
+ const device T* k_rows = k + qk_row_base;
+ const device T* q_rows = q + qk_row_base;
+ const device T* k_cols = k + qk_col_base;
+
+ const bool full_k_tile = valid_k_cols == k_tile_size;
+ if (full_k_tile && row_base + TREE_GRAM_ROW_TILE <= tree_size) {
+ k_left.load_from(lane, fragment_source(k_rows, qk_stride));
+ q_left.load_from(lane, fragment_source(q_rows, qk_stride));
+ } else {
+ k_left.load_from(lane, fragment_source(k_rows, qk_stride).bounded(tile_rows, valid_k_cols));
+ q_left.load_from(lane, fragment_source(q_rows, qk_stride).bounded(tile_rows, valid_k_cols));
+ }
+ if (full_k_tile && col_base + TREE_GRAM_COL_TILE <= tree_size) {
+ k_right.load_from(lane, fragment_source(k_cols, qk_stride));
+ } else {
+ k_right.load_from(lane, fragment_source(k_cols, qk_stride).bounded(tile_cols, valid_k_cols));
+ }
+
+ fragment_mma(kk_acc, k_left, k_right);
+ fragment_mma(qk_acc, q_left, k_right);
+ }
+
+ const bool has_diag = col_base <= row_base && row_base < col_base + TREE_GRAM_COL_TILE;
+ const uint diag_col_offset = has_diag ? row_base - col_base : 0;
+ const uint diag_size = min(TREE_GRAM_ROW_TILE, tree_size - row_base);
+
+ if (thread_idx < TREE_GRAM_COL_TILE) {
+ const uint col_token = col_base + thread_idx;
+ if (col_token < tree_size) {
+ const TrieNode node = trie[trie_base + col_token];
+ col_trie_start[thread_idx] = node.trie_start;
+ col_trie_end[thread_idx] = node.trie_end;
+ col_prefix[thread_idx] = prefix[prefix_base + col_token * value_heads];
+ } else {
+ col_trie_start[thread_idx] = 1;
+ col_trie_end[thread_idx] = 0;
+ col_prefix[thread_idx] = 0.0f;
+ }
+ }
+ if (thread_idx < TREE_GRAM_ROW_TILE) {
+ const uint token = row_base + thread_idx;
+ if (token < tree_size) {
+ row_token[thread_idx] = token;
+ row_prefix[thread_idx] = prefix[prefix_base + token * value_heads];
+ row_beta[thread_idx] = beta[prefix_base + token * value_heads];
+ } else {
+ row_token[thread_idx] = TREE_GRAM_INVALID_ROW;
+ row_prefix[thread_idx] = 0.0f;
+ row_beta[thread_idx] = 0.0f;
+ }
+ }
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+
+ AccFragment::zip_for_each_coord(
+ lane,
+ [&](ushort local_row, ushort local_col, thread float& kk, thread float& qk_dot) {
+ const uint row_idx = row_token[local_row];
+ const uint col_idx = col_base + local_col;
+ const bool in_subtree = row_idx >= col_trie_start[local_col] && row_idx <= col_trie_end[local_col];
+ const float decay = exp(row_prefix[local_row] - col_prefix[local_col]);
+ const float a_value = in_subtree && row_idx != col_idx ? row_beta[local_row] * decay * kk : 0.0f;
+ const float qkd_value = in_subtree ? scale * decay * qk_dot : 0.0f;
+ if (has_diag && local_row < diag_size && local_col >= diag_col_offset) {
+ const uint diag_col = local_col - diag_col_offset;
+ if (diag_col < diag_size) {
+ diag_a_tile[local_row * TREE_GRAM_ROW_TILE + diag_col] = a_value;
+ }
+ }
+ kk = a_value;
+ qk_dot = qkd_value;
+ },
+ kk_acc,
+ qk_acc
+ );
+
+ const short2 tile_dims = short2(tile_cols, tile_rows);
+ device float* a_tile = a_mat + tile_base;
+ device float* qkd_tile = qkd + tile_base;
+ if (row_base + TREE_GRAM_ROW_TILE <= tree_size && col_base + TREE_GRAM_COL_TILE <= tree_size) {
+ kk_acc.store(lane, a_tile, tree_size);
+ qk_acc.store(lane, qkd_tile, tree_size);
+ } else {
+ kk_acc.store_safe(lane, a_tile, tree_size, tile_dims);
+ qk_acc.store_safe(lane, qkd_tile, tree_size, tile_dims);
+ }
+
+ if (has_diag) {
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ invert_tree_gram_diagonal_block(ainv, diag_a_tile, mat_base, row_base, tree_size, diag_size, thread_idx);
+ }
+}
+
+METAL_FUNC void invert_tree_gram_diagonal_block(
+ device float* ainv,
+ threadgroup const float* diag_a_tile,
+ const uint mat_base,
+ const uint row_base,
+ const uint tree_size,
+ const uint block_size,
+ const uint thread_idx
+) {
+ if (thread_idx >= block_size) {
+ return;
+ }
+
+ const uint col = thread_idx;
+ float inverse_col[TREE_GRAM_ROW_TILE] = {};
+ inverse_col[col] = 1.0f;
+
+ METAL_PRAGMA_UNROLL
+ for (uint row = 0; row < TREE_GRAM_ROW_TILE; row++) {
+ if (row > col && row < block_size) {
+ float acc = 0.0f;
+ METAL_PRAGMA_UNROLL
+ for (uint prev_row = 0; prev_row < TREE_GRAM_ROW_TILE; prev_row++) {
+ if (prev_row < row) {
+ acc += diag_a_tile[row * TREE_GRAM_ROW_TILE + prev_row] * inverse_col[prev_row];
+ }
+ }
+ inverse_col[row] = -acc;
+ }
+ }
+
+ METAL_PRAGMA_UNROLL
+ for (uint row = 0; row < TREE_GRAM_ROW_TILE; row++) {
+ if (row < block_size) {
+ const uint ainv_offset = mat_base + (row_base + row) * tree_size + row_base + col;
+ ainv[ainv_offset] = inverse_col[row];
+ }
+ }
+}
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/fragment.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/fragment.h
index 1f40431e4..860176bdb 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/fragment.h
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/fragment.h
@@ -73,8 +73,8 @@ struct Fragment {
METAL_CONST ushort ROW_FRAGMENTS = ROW_FRAGMENTS_;
METAL_CONST ushort COL_FRAGMENTS = COL_FRAGMENTS_;
- METAL_CONST ushort FRAGMENT_ROWS = ushort(Ops::FRAGMENT_ROWS);
- METAL_CONST ushort FRAGMENT_COLS = ushort(Ops::FRAGMENT_COLS);
+ METAL_CONST ushort FRAGMENT_ROWS = Ops::FRAGMENT_ROWS;
+ METAL_CONST ushort FRAGMENT_COLS = Ops::FRAGMENT_COLS;
METAL_CONST ushort NUM_FRAGS = ROW_FRAGMENTS * COL_FRAGMENTS;
METAL_CONST ushort ELEMENTS_PER_FRAGMENT = NUM_FRAGS * Ops::ELEMENTS_PER_THREAD;
@@ -127,8 +127,8 @@ struct Fragment {
const short2 position = get_position(simd_lane_id);
for_each_fragment([&](auto fragment_row, auto fragment_col) {
const ushort frag_base = (fragment_row.value * COL_FRAGMENTS + fragment_col.value) * Ops::ELEMENTS_PER_THREAD;
- const short row_base = position.y + short(fragment_row.value) * FRAGMENT_ROWS;
- const short col_base = position.x + short(fragment_col.value) * FRAGMENT_COLS;
+ const short row_base = position.y + fragment_row.value * FRAGMENT_ROWS;
+ const short col_base = position.x + fragment_col.value * FRAGMENT_COLS;
METAL_PRAGMA_UNROLL
for (ushort element_index = 0; element_index < Ops::ELEMENTS_PER_THREAD; ++element_index) {
const short2 element_offset = Ops::get_element_offset(element_index);
@@ -140,6 +140,27 @@ struct Fragment {
});
}
+ template <class Fn, class... Fragments>
+ METAL_FUNC static void zip_for_each_coord(const ushort simd_lane_id, Fn fn, thread Fragments&... fragments) {
+ static_assert(sizeof...(Fragments) > 0, "zip_for_each_coord needs at least one fragment");
+ static_assert((metal::is_same_v<Fragment, Fragments> && ...), "zipped fragments must have the same type");
+
+ const short2 position = get_position(simd_lane_id);
+ for_each_fragment([&](auto fragment_row, auto fragment_col) {
+ const ushort frag_base = (fragment_row.value * COL_FRAGMENTS + fragment_col.value) * Ops::ELEMENTS_PER_THREAD;
+ const short row_base = position.y + fragment_row.value * FRAGMENT_ROWS;
+ const short col_base = position.x + fragment_col.value * FRAGMENT_COLS;
+ METAL_PRAGMA_UNROLL
+ for (ushort element_index = 0; element_index < Ops::ELEMENTS_PER_THREAD; ++element_index) {
+ const short2 element_offset = Ops::get_element_offset(element_index);
+ const short row = row_base + element_offset.y;
+ const short col = col_base + element_offset.x;
+ const ushort index = frag_base + element_index;
+ fn(row, col, fragments.elements()[index]...);
+ }
+ });
+ }
+
// Row lanes differ in bits {0,3} for both simdgroup and MXU layouts.
template <typename Acc, class Fn>
METAL_FUNC void row_reduce(thread Acc* out, const Acc identity, Fn op) thread {
@@ -275,7 +296,7 @@ private:
METAL_CONST bool UNSAFE = false;
template <class Fn>
- METAL_FUNC void for_each_fragment(Fn fn) thread {
+ METAL_FUNC static void for_each_fragment(Fn fn) {
const_for_loop<0, ROW_FRAGMENTS, 1>([&](auto row_index) {
const_for_loop<0, COL_FRAGMENTS, 1>([&](auto col_index) { fn(row_index, col_index); });
});
diff --git a/crates/backend-uzu/unit/backends/common/kernel/gram_bench.rs b/crates/backend-uzu/unit/backends/common/kernel/gram_bench.rs
new file mode 100644
index 000000000..c5e5bb502
--- /dev/null
+++ b/crates/backend-uzu/unit/backends/common/kernel/gram_bench.rs
@@ -0,0 +1,195 @@
+#![cfg(metal_backend)]
+
+use std::{env, mem::size_of, time::Duration};
+
+use criterion::{BenchmarkId, Criterion};
+use half::bf16;
+use proc_macros::uzu_bench;
+
+use crate::{
+ array::ArrayContextExt,
+ backends::{
+ common::{Allocation, Backend, Context, Encoder, Kernels, kernel::BuildTreeGramKernel},
+ metal::Metal,
+ },
+ data_type::DataType,
+ tests::{cold_pool::ColdPool, matmul::iter_encode_loop_named},
+};
+
+const K_HEADS: usize = 16;
+const VALUE_HEADS: usize = 48;
+const HEAD_K_DIM: usize = 128;
+const TREE_SIZES: &[usize] = &[33, 49, 64, 128, 256, 512];
+const BATCH_SIZES: &[usize] = &[1, 2, 4, 8];
+
+struct TreeGramBuffers {
+ q: Allocation<Metal>,
+ k: Allocation<Metal>,
+ trie: Allocation<Metal>,
+ prefix: Allocation<Metal>,
+ beta: Allocation<Metal>,
+ a_mat: Allocation<Metal>,
+ qkd: Allocation<Metal>,
+ ainv: Allocation<Metal>,
+}
+
+fn reranker_like_trie(
+ batch_size: usize,
+ tree_size: usize,
+) -> Vec<u32> {
+ let mut trie = Vec::with_capacity(batch_size * tree_size * 3);
+ for b in 0..batch_size {
+ let mut children = vec![Vec::<usize>::new(); tree_size];
+ let mut child_count = vec![0usize; tree_size];
+ for node in 1..tree_size {
+ let mut parent = (node - 1) / 2;
+ if child_count[parent] >= 2 {
+ parent = node.saturating_sub(2 + (b + node) % node);
+ }
+ children[parent].push(node);
+ child_count[parent] += 1;
+ }
+
+ fn dfs(
+ node: usize,
+ children: &[Vec<usize>],
+ trie: &mut [[u32; 3]],
+ next: &mut u32,
+ depth: u32,
+ ) {
+ let start = *next;
+ *next += 1;
+ let mut end = start;
+ for &child in &children[node] {
+ dfs(child, children, trie, next, depth + 1);
+ end = end.max(trie[child][1]);
+ }
+ trie[node] = [start, end, depth];
+ }
+
+ let mut batch_trie = vec![[0u32; 3]; tree_size];
+ let mut next = 0;
+ dfs(0, &children, &mut batch_trie, &mut next, 0);
+ for node in batch_trie {
+ trie.extend_from_slice(&node);
+ }
+ }
+ trie
+}
+
+fn make_buffers(
+ context: &<Metal as Backend>::Context,
+ batch_size: usize,
+ tree_size: usize,
+) -> (TreeGramBuffers, f32) {
+ let qk_len = batch_size * tree_size * K_HEADS * HEAD_K_DIM;
+ let head_len = batch_size * tree_size * VALUE_HEADS;
+ let out_len = batch_size * VALUE_HEADS * tree_size * tree_size;
+ let trie = reranker_like_trie(batch_size, tree_size);
+ let q = (0..qk_len).map(|i| bf16::from_f32(((i as f32 * 0.017).sin() * 0.2) + 0.01)).collect::<Vec<_>>();
+ let k = (0..qk_len).map(|i| bf16::from_f32(((i as f32 * 0.019).cos() * 0.18) - 0.02)).collect::<Vec<_>>();
+ let prefix = (0..head_len)
+ .map(|i| -((i % tree_size) as f32) * 0.01 - ((i % VALUE_HEADS) as f32) * 0.003)
+ .collect::<Vec<_>>();
+ let beta = (0..head_len).map(|i| 0.25 + ((i as f32 * 0.013).sin() + 1.0) * 0.2).collect::<Vec<_>>();
+ let scale = (HEAD_K_DIM as f32).sqrt().recip();
+ (
+ TreeGramBuffers {
+ q: context.create_array_from(&[q.len()], &q).into_allocation(),
+ k: context.create_array_from(&[k.len()], &k).into_allocation(),
+ trie: context.create_array_from(&[trie.len()], &trie).into_allocation(),
+ prefix: context.create_array_from(&[prefix.len()], &prefix).into_allocation(),
+ beta: context.create_array_from(&[beta.len()], &beta).into_allocation(),
+ a_mat: context.create_array_uninitialized(&[out_len], DataType::F32).into_allocation(),
+ qkd: context.create_array_uninitialized(&[out_len], DataType::F32).into_allocation(),
+ ainv: context.create_array_uninitialized(&[out_len], DataType::F32).into_allocation(),
+ },
+ scale,
+ )
+}
+
+fn buffers_bytes(
+ batch_size: usize,
+ tree_size: usize,
+) -> usize {
+ let qk_len = batch_size * tree_size * K_HEADS * HEAD_K_DIM;
+ let head_len = batch_size * tree_size * VALUE_HEADS;
+ let out_len = batch_size * VALUE_HEADS * tree_size * tree_size;
+ qk_len * size_of::<bf16>() * 2
+ + batch_size * tree_size * 3 * size_of::<u32>()
+ + head_len * size_of::<f32>() * 2
+ + out_len * size_of::<f32>() * 3
+}
+
+#[uzu_bench]
+fn bench_build_tree_gram(c: &mut Criterion) {
+ let context = <Metal as Backend>::Context::new().expect("metal context");
+ let cold_buffers = env::var("UZU_GRAM_COLD_BUFFERS").is_ok();
+ let kernel_paths = if context.supports_mxu() {
+ &[("Simdgroup", false), ("MXU", true)][..]
+ } else {
+ &[("Simdgroup", false)][..]
+ };
+
+ for &(kernel_path, use_mxu) in kernel_paths {
+ let mut group = c.benchmark_group(format!("Metal/Kernel/GDNTreeVerify/BuildTreeGram/{kernel_path}"));
+ group.sample_size(10).warm_up_time(Duration::from_millis(100)).measurement_time(Duration::from_millis(500));
+
+ for &batch_size in BATCH_SIZES {
+ for &tree_size in TREE_SIZES {
+ let kernel = <<Metal as Backend>::Kernels as Kernels>::BuildTreeGramKernel::new(
+ &context,
+ DataType::BF16,
+ use_mxu,
+ )
+ .expect("BuildTreeGramKernel");
+ let scale = (HEAD_K_DIM as f32).sqrt().recip();
+ let benchmark_path =
+ format!("Metal/Kernel/GDNTreeVerify/BuildTreeGram/{kernel_path}/B{batch_size}_T{tree_size}");
+ let benchmark_id = BenchmarkId::from_parameter(format!(
+ "B{batch_size}_T{tree_size}_Hg{K_HEADS}_HV{VALUE_HEADS}_K{HEAD_K_DIM}"
+ ));
+
+ let encode = |buffers: &mut TreeGramBuffers, encoder: &mut Encoder<Metal>| {
+ kernel.encode(
+ &buffers.q,
+ &buffers.k,
+ &buffers.trie,
+ &buffers.prefix,
+ &buffers.beta,
+ &mut buffers.a_mat,
+ &mut buffers.qkd,
+ &mut buffers.ainv,
+ scale,
+ batch_size as u32,
+ tree_size as u32,
+ K_HEADS as u32,
+ VALUE_HEADS as u32,
+ HEAD_K_DIM as u32,
+ encoder,
+ );
+ };
+
+ if cold_buffers {
+ let mut buffers = ColdPool::new(buffers_bytes(batch_size, tree_size), || {
+ make_buffers(&context, batch_size, tree_size).0
+ });
+ group.bench_function(benchmark_id, |bencher| {
+ iter_encode_loop_named::<Metal, _>(context.as_ref(), bencher, &benchmark_path, |encoder| {
+ let buffers = buffers.next_mut();
+ encode(buffers, encoder);
+ });
+ });
+ } else {
+ let (mut buffers, _) = make_buffers(&context, batch_size, tree_size);
+ group.bench_function(benchmark_id, |bencher| {
+ iter_encode_loop_named::<Metal, _>(context.as_ref(), bencher, &benchmark_path, |encoder| {
+ encode(&mut buffers, encoder);
+ });
+ });
+ }
+ }
+ }
+ group.finish();
+ }
+}
diff --git a/crates/backend-uzu/unit/backends/common/kernel/gram_test.rs b/crates/backend-uzu/unit/backends/common/kernel/gram_test.rs
new file mode 100644
index 000000000..697f90968
--- /dev/null
+++ b/crates/backend-uzu/unit/backends/common/kernel/gram_test.rs
@@ -0,0 +1,136 @@
+use proc_macros::uzu_test;
+use test_runner::for_each_non_cpu_backend;
+
+use crate::{
+ array::ArrayContextExt,
+ backends::{
+ common::{Backend, Context, Encoder, Kernels, kernel::BuildTreeGramKernel},
+ cpu::Cpu,
+ },
+ data_type::DataType,
+ tests::{assert::assert_eq_float, helpers::allocation_to_vec},
+};
+
+const BATCH_SIZE: usize = 2;
+const K_HEADS: usize = 2;
+const VALUE_HEADS: usize = 6;
+const HEAD_K_DIM: usize = 128;
+
+fn get_output<B: Backend>(
+ q: &[f32],
+ k: &[f32],
+ trie: &[u32],
+ prefix: &[f32],
+ beta: &[f32],
+ tree_size: usize,
+ scale: f32,
+ use_mxu: bool,
+) -> (Vec<f32>, Vec<f32>, Vec<f32>) {
+ let context = B::Context::new().expect("Failed to create Context");
+ let kernel = <<B as Backend>::Kernels as Kernels>::BuildTreeGramKernel::new(&context, DataType::F32, use_mxu)
+ .expect("Failed to create BuildTreeGramKernel");
+
+ let output_len = BATCH_SIZE * VALUE_HEADS * tree_size * tree_size;
+ let q = context.create_array_from(&[q.len()], q);
+ let k = context.create_array_from(&[k.len()], k);
+ let trie = context.create_array_from(&[trie.len()], trie);
+ let prefix = context.create_array_from(&[prefix.len()], prefix);
+ let beta = context.create_array_from(&[beta.len()], beta);
+ let mut a_mat = context.create_array_uninitialized(&[output_len], DataType::F32).into_allocation();
+ let mut qkd = context.create_array_uninitialized(&[output_len], DataType::F32).into_allocation();
+ let mut ainv = context.create_array_uninitialized(&[output_len], DataType::F32).into_allocation();
+
+ let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");
+ kernel.encode(
+ q.allocation(),
+ k.allocation(),
+ trie.allocation(),
+ prefix.allocation(),
+ beta.allocation(),
+ &mut a_mat,
+ &mut qkd,
+ &mut ainv,
+ scale,
+ BATCH_SIZE as u32,
+ tree_size as u32,
+ K_HEADS as u32,
+ VALUE_HEADS as u32,
+ HEAD_K_DIM as u32,
+ &mut encoder,
+ );
+ encoder.end_encoding().submit().wait_until_completed().unwrap();
+
+ (allocation_to_vec(&a_mat), allocation_to_vec(&qkd), allocation_to_vec(&ainv))
+}
+
+fn build_trie(tree_size: usize) -> Vec<u32> {
+ let last = tree_size as u32 - 1;
+ let mut trie = Vec::with_capacity(BATCH_SIZE * tree_size * 3);
+ for i in 0..tree_size as u32 {
+ trie.extend_from_slice(&[i, last, i]);
+ }
+ trie.extend_from_slice(&[0, last, 0]);
+ for i in 1..tree_size as u32 {
+ trie.extend_from_slice(&[i, i, 1]);
+ }
+ trie
+}
+
+#[uzu_test]
+fn test_build_tree_gram_matches_cpu() {
+ let scale = (HEAD_K_DIM as f32).sqrt().recip();
+
+ for tree_size in [17, 64, 128] {
+ let q_len = BATCH_SIZE * tree_size * K_HEADS * HEAD_K_DIM;
+ let kv_len = BATCH_SIZE * tree_size * VALUE_HEADS;
+ let trie = build_trie(tree_size);
+ let q = (0..q_len).map(|i| ((i as f32 * 0.017).sin() * 0.2) + 0.01).collect::<Vec<_>>();
+ let k = (0..q_len).map(|i| ((i as f32 * 0.019).cos() * 0.18) - 0.02).collect::<Vec<_>>();
+ let prefix = (0..kv_len)
+ .map(|i| -((i % tree_size) as f32) * 0.01 - ((i % VALUE_HEADS) as f32) * 0.003)
+ .collect::<Vec<_>>();
+ let beta = (0..kv_len).map(|i| 0.25 + ((i as f32 * 0.013).sin() + 1.0) * 0.2).collect::<Vec<_>>();
+
+ let expected = get_output::<Cpu>(&q, &k, &trie, &prefix, &beta, tree_size, scale, false);
+
+ for_each_non_cpu_backend!(|B| {
+ let context = <<B as Backend>::Context as Context>::new().expect("Failed to create Context");
+ let paths: &[bool] = if context.supports_mxu() {
+ &[false, true]
+ } else {
+ &[false]
+ };
+
+ for &use_mxu in paths {
+ let actual = get_output::<B>(&q, &k, &trie, &prefix, &beta, tree_size, scale, use_mxu);
+ let path = if use_mxu {
+ "mxu"
+ } else {
+ "simdgroup"
+ };
+ let msg = format!("backend {} {path} tree_size {tree_size}", std::any::type_name::<B>());
+ assert_eq_float::<f32>(&expected.0, &actual.0, 5e-3, &format!("a_mat {msg}"));
+ assert_eq_float::<f32>(&expected.1, &actual.1, 5e-3, &format!("qkd {msg}"));
+
+ for (expected_matrix, actual_matrix) in
+ expected.2.chunks(tree_size * tree_size).zip(actual.2.chunks(tree_size * tree_size))
+ {
+ for block_start in (0..tree_size).step_by(16) {
+ let block_end = (block_start + 16).min(tree_size);
+ for i in block_start..block_end {
+ for j in block_start..block_end {
+ let idx = i * tree_size + j;
+ let expected = expected_matrix[idx];
+ let actual = actual_matrix[idx];
+ assert!(
+ (expected - actual).abs() < 1e-2,
+ "ainv {msg}. Mismatch at matrix index {idx}: expected {expected}, got {actual}"
+ );
+ }
+ }
+ }
+ }
+ }
+ });
+ }
+}
diff --git a/crates/backend-uzu/unit/backends/common/kernel/mod.rs b/crates/backend-uzu/unit/backends/common/kernel/mod.rs
index fc87a59d0..0d9a5e47c 100644
--- a/crates/backend-uzu/unit/backends/common/kernel/mod.rs
+++ b/crates/backend-uzu/unit/backends/common/kernel/mod.rs
@@ -4,6 +4,8 @@ mod audio;
mod delta_net_test;
mod embedding;
mod gated_act_mul_test;
+mod gram_bench;
+mod gram_test;
mod hadamard_transform_test;
mod kv_cache_update_test;
mod layer_norm_test;