A high-performance inference engine for AI models
Support BuildTreeGram Kernel (#569)
CCYeh5f01024ba4e5
diff --git a/crates/backend-uzu/src/backends/cpu/kernel/gdn_tree_verify/gram.rs b/crates/backend-uzu/src/backends/cpu/kernel/gdn_tree_verify/gram.rsnew file mode 100644index 000000000..1da6f5688--- /dev/null+++ b/crates/backend-uzu/src/backends/cpu/kernel/gdn_tree_verify/gram.rs@@ -0,0 +1,113 @@+use half::bf16;+use num_traits::Float;+use proc_macros::kernel;++use crate::{array::ArrayElement, backends::common::gpu_types::trie::TrieNode};+#[kernel(BuildTreeGram)]+#[variants(T, f32, bf16)]+#[variants(USE_MXU, false)]+pub fn build_tree_gram<T: ArrayElement + Float, const USE_MXU: bool>(+ q: *const T,+ k: *const T,+ trie: *const TrieNode,+ prefix: *const f32,+ beta: *const f32,+ a_mat: *mut f32,+ qkd: *mut f32,+ ainv: *mut f32,+ scale: f32,+ batch_size: u32,+ tree_size: u32,+ k_heads: u32,+ value_heads: u32,+ head_k_dim: u32,+) {+ let batch_size = batch_size as usize;+ let tree_size = tree_size as usize;+ let k_heads = k_heads as usize;+ let value_heads = value_heads as usize;+ let head_k_dim = head_k_dim as usize;++ let groups_per_head = value_heads / k_heads;++ for batch in 0..batch_size {+ for hv in 0..value_heads {+ let hk = hv / groups_per_head;+ let mat_base = (batch * value_heads + hv) * tree_size * tree_size;+ let trie_base = batch * tree_size;++ for i in 0..tree_size {+ let row_idx = i as u32;+ let prefix_i = unsafe { *prefix.add((batch * tree_size + i) * value_heads + hv) };+ let beta_i = unsafe { *beta.add((batch * tree_size + i) * value_heads + hv) };+ let row_off = ((batch * tree_size + i) * k_heads + hk) * head_k_dim;+ for j in 0..tree_size {+ let out = mat_base + i * tree_size + j;+ let node = unsafe { *trie.add(trie_base + j) };+ let incl = row_idx >= node.trie_start && row_idx <= node.trie_end;+ if !incl {+ unsafe {+ *a_mat.add(out) = 0.0;+ *qkd.add(out) = 0.0;+ }+ continue;+ }+ let col_off = ((batch * tree_size + j) * k_heads + hk) * head_k_dim;+ let mut kk = 0.0f32;+ let mut qk = 0.0f32;+ for d in 0..head_k_dim {+ let ki = unsafe { (*k.add(row_off + d)).to_f32().unwrap() };+ let kj = unsafe { (*k.add(col_off + d)).to_f32().unwrap() };+ let qi = unsafe { (*q.add(row_off + d)).to_f32().unwrap() };+ kk += ki * kj;+ qk += qi * kj;+ }+ let prefix_j = unsafe { *prefix.add((batch * tree_size + j) * value_heads + hv) };+ let dexp = (prefix_i - prefix_j).exp();+ unsafe {+ *a_mat.add(out) = if i != j {+ beta_i * dexp * kk+ } else {+ 0.0+ };+ *qkd.add(out) = dexp * scale * qk;+ }+ }+ }++ invert_tree_gram_matrix(a_mat, ainv, mat_base, tree_size);+ }+ }+}++fn invert_tree_gram_matrix(+ a_mat: *const f32,+ ainv: *mut f32,+ mat_base: usize,+ tree_size: usize,+) {+ for i in 0..tree_size {+ for j in 0..tree_size {+ unsafe {+ *ainv.add(mat_base + i * tree_size + j) = if i == j {+ 1.0+ } else {+ 0.0+ };+ }+ }+ }+ for i in 0..tree_size {+ for j in 0..i {+ let mut s = 0.0f32;+ for kx in j..i {+ unsafe {+ s += *a_mat.add(mat_base + i * tree_size + kx) * *ainv.add(mat_base + kx * tree_size + j);+ }+ }+ unsafe {+ *ainv.add(mat_base + i * tree_size + j) = -s;+ }+ }+ }+}diff --git a/crates/backend-uzu/src/backends/cpu/kernel/gdn_tree_verify/mod.rs b/crates/backend-uzu/src/backends/cpu/kernel/gdn_tree_verify/mod.rsindex 524d1acad..dfd9a7402 100644--- a/crates/backend-uzu/src/backends/cpu/kernel/gdn_tree_verify/mod.rs+++ b/crates/backend-uzu/src/backends/cpu/kernel/gdn_tree_verify/mod.rs@@ -1 +1,2 @@+pub mod gram;pub mod prefix_beta;diff --git a/crates/backend-uzu/src/backends/metal/kernel/gdn_tree_verify/gram.metal b/crates/backend-uzu/src/backends/metal/kernel/gdn_tree_verify/gram.metalnew file mode 100644index 000000000..d6211a506--- /dev/null+++ b/crates/backend-uzu/src/backends/metal/kernel/gdn_tree_verify/gram.metal@@ -0,0 +1,260 @@+#include <metal_stdlib>+#include "../common/defines.h"+#include "../common/dsl.h"+#include "../common/thread_context.h"+#include "../generated/trie.h"+#include "../matmul/common/fragment.h"+#include "../matmul/common/mxu_fragment_ops.h"+#include "../matmul/common/simdgroup_fragment_ops.h"++using namespace metal;+using namespace uzu::matmul;+using namespace uzu::trie;++#define TREE_GRAM_ROW_TILE 16u+#define TREE_GRAM_COL_TILE 32u+#define TREE_GRAM_THREADS METAL_SIMD_SIZE+#define TREE_GRAM_INVALID_ROW 0xffffffffu++METAL_FUNC void invert_tree_gram_diagonal_block(+ device float* ainv,+ threadgroup const float* diag_a_tile,+ const uint mat_base,+ const uint row_base,+ const uint tree_size,+ const uint block_size,+ const uint thread_idx+);++// Build tree gram matrices for one (batch, value-head) tile.+//+// q, k: [B, T, Hg, K], with key head hk = hv / (HV / Hg)+// trie: [B, T] DFS intervals; col is an ancestor of row when row is in col's interval+// prefix: [B, T, HV] path log-decay prefix from BuildPrefixBeta+// beta: [B, T, HV] sigmoid gate from BuildPrefixBeta+//+// a_mat[row, col] = beta[row] * exp(prefix[row] - prefix[col]) * dot(k[row], k[col])+// for proper ancestors only+// qkd[row, col] = scale * exp(prefix[row] - prefix[col]) * dot(q[row], k[col])+// for ancestor-or-self+// ainv = (I + A)^-1 for each diagonal block+template <typename T, bool USE_MXU>+VARIANTS(T, float, bfloat)+VARIANTS(USE_MXU, false, true)+PUBLIC KERNEL(BuildTreeGram)(+ const device T* q,+ const device T* k,+ const device TrieNode* trie,+ const device float* prefix,+ const device float* beta,+ device float* a_mat,+ device float* qkd,+ device float* ainv,+ constant const float& scale,+ constant const uint& batch_size,+ constant const uint& tree_size,+ constant const uint& k_heads,+ constant const uint& value_heads,+ constant const uint& head_k_dim,+ threadgroup float diag_a_tile[TREE_GRAM_ROW_TILE * TREE_GRAM_ROW_TILE],+ threadgroup float row_prefix[TREE_GRAM_ROW_TILE],+ threadgroup float row_beta[TREE_GRAM_ROW_TILE],+ threadgroup uint row_token[TREE_GRAM_ROW_TILE],+ threadgroup float col_prefix[TREE_GRAM_COL_TILE],+ threadgroup uint col_trie_start[TREE_GRAM_COL_TILE],+ threadgroup uint col_trie_end[TREE_GRAM_COL_TILE],+ const ThreadContext thread_context,+ const uint batch_idx GROUPS(batch_size),+ const uint value_head_idx GROUPS(value_heads),+ const uint tile_idx GROUPS(tree_size.div_ceil(TREE_GRAM_ROW_TILE) * tree_size.div_ceil(TREE_GRAM_COL_TILE)),+ const uint thread_idx THREADS(TREE_GRAM_THREADS)+) {+ using Ops = metal::conditional_t<USE_MXU, MxuFragmentOps<>, SimdgroupFragmentOps>;+ constexpr ushort ROW_FRAGMENTS = TREE_GRAM_ROW_TILE / Ops::FRAGMENT_ROWS;+ constexpr ushort COL_FRAGMENTS = TREE_GRAM_COL_TILE / Ops::FRAGMENT_COLS;+ using InputType = metal::conditional_t<USE_MXU, T, float>;+ using AccFragment = Fragment<float, ROW_FRAGMENTS, COL_FRAGMENTS, Ops>;+ using LeftFragment = OperandFragment<InputType, ROW_FRAGMENTS, 1, Ops>;+ using RightFragment = OperandFragment<InputType, 1, COL_FRAGMENTS, Ops, ReadTranspose>;++ const uint value_heads_per_key_head = value_heads / k_heads;+ const uint key_head_idx = value_head_idx / value_heads_per_key_head;+ const uint qk_stride = k_heads * head_k_dim;+ const uint qk_base = (batch_idx * tree_size * k_heads + key_head_idx) * head_k_dim;+ const uint prefix_base = batch_idx * tree_size * value_heads + value_head_idx;+ const uint trie_base = batch_idx * tree_size;+ const uint mat_base = (batch_idx * value_heads + value_head_idx) * tree_size * tree_size;++ const uint col_tiles = div_ceil(tree_size, TREE_GRAM_COL_TILE);+ const uint row_tile_idx = tile_idx / col_tiles;+ const uint col_tile_idx = tile_idx - row_tile_idx * col_tiles;+ const uint row_base = row_tile_idx * TREE_GRAM_ROW_TILE;+ const uint col_base = col_tile_idx * TREE_GRAM_COL_TILE;+ const ushort lane = thread_context.simd_lane_id;++ if (row_base >= tree_size) {+ return;+ }++ const uint tile_rows = min(TREE_GRAM_ROW_TILE, tree_size - row_base);+ const uint tile_cols = min(TREE_GRAM_COL_TILE, tree_size - col_base);+ const uint tile_base = mat_base + row_base * tree_size + col_base;++ if (col_base >= row_base + TREE_GRAM_ROW_TILE) {+ for (uint idx = thread_idx; idx < tile_rows * tile_cols; idx += TREE_GRAM_THREADS) {+ const uint local_row = idx / tile_cols;+ const uint local_col = idx - local_row * tile_cols;+ const uint offset = tile_base + local_row * tree_size + local_col;+ a_mat[offset] = 0.0f;+ qkd[offset] = 0.0f;+ }+ return;+ }++ AccFragment kk_acc;+ AccFragment qk_acc;+ kk_acc.clear();+ qk_acc.clear();++ for (uint k_block_start = 0; k_block_start < head_k_dim; k_block_start += Ops::FRAGMENT_ROWS) {+ const uint k_remaining = head_k_dim - k_block_start;+ const uint k_tile_size = Ops::FRAGMENT_ROWS;+ const uint valid_k_cols = min(k_remaining, k_tile_size);++ LeftFragment k_left;+ LeftFragment q_left;+ RightFragment k_right;++ const uint qk_row_base = qk_base + row_base * qk_stride + k_block_start;+ const uint qk_col_base = qk_base + col_base * qk_stride + k_block_start;+ const device T* k_rows = k + qk_row_base;+ const device T* q_rows = q + qk_row_base;+ const device T* k_cols = k + qk_col_base;++ const bool full_k_tile = valid_k_cols == k_tile_size;+ if (full_k_tile && row_base + TREE_GRAM_ROW_TILE <= tree_size) {+ k_left.load_from(lane, fragment_source(k_rows, qk_stride));+ q_left.load_from(lane, fragment_source(q_rows, qk_stride));+ } else {+ k_left.load_from(lane, fragment_source(k_rows, qk_stride).bounded(tile_rows, valid_k_cols));+ q_left.load_from(lane, fragment_source(q_rows, qk_stride).bounded(tile_rows, valid_k_cols));+ }+ if (full_k_tile && col_base + TREE_GRAM_COL_TILE <= tree_size) {+ k_right.load_from(lane, fragment_source(k_cols, qk_stride));+ } else {+ k_right.load_from(lane, fragment_source(k_cols, qk_stride).bounded(tile_cols, valid_k_cols));+ }++ fragment_mma(kk_acc, k_left, k_right);+ fragment_mma(qk_acc, q_left, k_right);+ }++ const bool has_diag = col_base <= row_base && row_base < col_base + TREE_GRAM_COL_TILE;+ const uint diag_col_offset = has_diag ? row_base - col_base : 0;+ const uint diag_size = min(TREE_GRAM_ROW_TILE, tree_size - row_base);++ if (thread_idx < TREE_GRAM_COL_TILE) {+ const uint col_token = col_base + thread_idx;+ if (col_token < tree_size) {+ const TrieNode node = trie[trie_base + col_token];+ col_trie_start[thread_idx] = node.trie_start;+ col_trie_end[thread_idx] = node.trie_end;+ col_prefix[thread_idx] = prefix[prefix_base + col_token * value_heads];+ } else {+ col_trie_start[thread_idx] = 1;+ col_trie_end[thread_idx] = 0;+ col_prefix[thread_idx] = 0.0f;+ }+ }+ if (thread_idx < TREE_GRAM_ROW_TILE) {+ const uint token = row_base + thread_idx;+ if (token < tree_size) {+ row_token[thread_idx] = token;+ row_prefix[thread_idx] = prefix[prefix_base + token * value_heads];+ row_beta[thread_idx] = beta[prefix_base + token * value_heads];+ } else {+ row_token[thread_idx] = TREE_GRAM_INVALID_ROW;+ row_prefix[thread_idx] = 0.0f;+ row_beta[thread_idx] = 0.0f;+ }+ }+ threadgroup_barrier(mem_flags::mem_threadgroup);++ AccFragment::zip_for_each_coord(+ lane,+ [&](ushort local_row, ushort local_col, thread float& kk, thread float& qk_dot) {+ const uint row_idx = row_token[local_row];+ const uint col_idx = col_base + local_col;+ const bool in_subtree = row_idx >= col_trie_start[local_col] && row_idx <= col_trie_end[local_col];+ const float decay = exp(row_prefix[local_row] - col_prefix[local_col]);+ const float a_value = in_subtree && row_idx != col_idx ? row_beta[local_row] * decay * kk : 0.0f;+ const float qkd_value = in_subtree ? scale * decay * qk_dot : 0.0f;+ if (has_diag && local_row < diag_size && local_col >= diag_col_offset) {+ const uint diag_col = local_col - diag_col_offset;+ if (diag_col < diag_size) {+ diag_a_tile[local_row * TREE_GRAM_ROW_TILE + diag_col] = a_value;+ }+ }+ kk = a_value;+ qk_dot = qkd_value;+ },+ kk_acc,+ qk_acc+ );++ const short2 tile_dims = short2(tile_cols, tile_rows);+ device float* a_tile = a_mat + tile_base;+ device float* qkd_tile = qkd + tile_base;+ if (row_base + TREE_GRAM_ROW_TILE <= tree_size && col_base + TREE_GRAM_COL_TILE <= tree_size) {+ kk_acc.store(lane, a_tile, tree_size);+ qk_acc.store(lane, qkd_tile, tree_size);+ } else {+ kk_acc.store_safe(lane, a_tile, tree_size, tile_dims);+ qk_acc.store_safe(lane, qkd_tile, tree_size, tile_dims);+ }++ if (has_diag) {+ threadgroup_barrier(mem_flags::mem_threadgroup);+ invert_tree_gram_diagonal_block(ainv, diag_a_tile, mat_base, row_base, tree_size, diag_size, thread_idx);+ }+}++METAL_FUNC void invert_tree_gram_diagonal_block(+ device float* ainv,+ threadgroup const float* diag_a_tile,+ const uint mat_base,+ const uint row_base,+ const uint tree_size,+ const uint block_size,+ const uint thread_idx+) {+ if (thread_idx >= block_size) {+ return;+ }++ const uint col = thread_idx;+ float inverse_col[TREE_GRAM_ROW_TILE] = {};+ inverse_col[col] = 1.0f;++ METAL_PRAGMA_UNROLL+ for (uint row = 0; row < TREE_GRAM_ROW_TILE; row++) {+ if (row > col && row < block_size) {+ float acc = 0.0f;+ METAL_PRAGMA_UNROLL+ for (uint prev_row = 0; prev_row < TREE_GRAM_ROW_TILE; prev_row++) {+ if (prev_row < row) {+ acc += diag_a_tile[row * TREE_GRAM_ROW_TILE + prev_row] * inverse_col[prev_row];+ }+ }+ inverse_col[row] = -acc;+ }+ }++ METAL_PRAGMA_UNROLL+ for (uint row = 0; row < TREE_GRAM_ROW_TILE; row++) {+ if (row < block_size) {+ const uint ainv_offset = mat_base + (row_base + row) * tree_size + row_base + col;+ ainv[ainv_offset] = inverse_col[row];+ }+ }+}diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/fragment.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/fragment.hindex 1f40431e4..860176bdb 100644--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/fragment.h+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/fragment.h@@ -73,8 +73,8 @@ struct Fragment {METAL_CONST ushort ROW_FRAGMENTS = ROW_FRAGMENTS_;METAL_CONST ushort COL_FRAGMENTS = COL_FRAGMENTS_;- METAL_CONST ushort FRAGMENT_ROWS = ushort(Ops::FRAGMENT_ROWS);- METAL_CONST ushort FRAGMENT_COLS = ushort(Ops::FRAGMENT_COLS);+ METAL_CONST ushort FRAGMENT_ROWS = Ops::FRAGMENT_ROWS;+ METAL_CONST ushort FRAGMENT_COLS = Ops::FRAGMENT_COLS;METAL_CONST ushort NUM_FRAGS = ROW_FRAGMENTS * COL_FRAGMENTS;METAL_CONST ushort ELEMENTS_PER_FRAGMENT = NUM_FRAGS * Ops::ELEMENTS_PER_THREAD;@@ -127,8 +127,8 @@ struct Fragment {const short2 position = get_position(simd_lane_id);for_each_fragment([&](auto fragment_row, auto fragment_col) {const ushort frag_base = (fragment_row.value * COL_FRAGMENTS + fragment_col.value) * Ops::ELEMENTS_PER_THREAD;- const short row_base = position.y + short(fragment_row.value) * FRAGMENT_ROWS;- const short col_base = position.x + short(fragment_col.value) * FRAGMENT_COLS;+ const short row_base = position.y + fragment_row.value * FRAGMENT_ROWS;+ const short col_base = position.x + fragment_col.value * FRAGMENT_COLS;METAL_PRAGMA_UNROLLfor (ushort element_index = 0; element_index < Ops::ELEMENTS_PER_THREAD; ++element_index) {const short2 element_offset = Ops::get_element_offset(element_index);@@ -140,6 +140,27 @@ struct Fragment {});}+ template <class Fn, class... Fragments>+ METAL_FUNC static void zip_for_each_coord(const ushort simd_lane_id, Fn fn, thread Fragments&... fragments) {+ static_assert(sizeof...(Fragments) > 0, "zip_for_each_coord needs at least one fragment");+ static_assert((metal::is_same_v<Fragment, Fragments> && ...), "zipped fragments must have the same type");++ const short2 position = get_position(simd_lane_id);+ for_each_fragment([&](auto fragment_row, auto fragment_col) {+ const ushort frag_base = (fragment_row.value * COL_FRAGMENTS + fragment_col.value) * Ops::ELEMENTS_PER_THREAD;+ const short row_base = position.y + fragment_row.value * FRAGMENT_ROWS;+ const short col_base = position.x + fragment_col.value * FRAGMENT_COLS;+ METAL_PRAGMA_UNROLL+ for (ushort element_index = 0; element_index < Ops::ELEMENTS_PER_THREAD; ++element_index) {+ const short2 element_offset = Ops::get_element_offset(element_index);+ const short row = row_base + element_offset.y;+ const short col = col_base + element_offset.x;+ const ushort index = frag_base + element_index;+ fn(row, col, fragments.elements()[index]...);+ }+ });+ }+// Row lanes differ in bits {0,3} for both simdgroup and MXU layouts.template <typename Acc, class Fn>METAL_FUNC void row_reduce(thread Acc* out, const Acc identity, Fn op) thread {@@ -275,7 +296,7 @@ private:METAL_CONST bool UNSAFE = false;template <class Fn>- METAL_FUNC void for_each_fragment(Fn fn) thread {+ METAL_FUNC static void for_each_fragment(Fn fn) {const_for_loop<0, ROW_FRAGMENTS, 1>([&](auto row_index) {const_for_loop<0, COL_FRAGMENTS, 1>([&](auto col_index) { fn(row_index, col_index); });});diff --git a/crates/backend-uzu/unit/backends/common/kernel/gram_bench.rs b/crates/backend-uzu/unit/backends/common/kernel/gram_bench.rsnew file mode 100644index 000000000..c5e5bb502--- /dev/null+++ b/crates/backend-uzu/unit/backends/common/kernel/gram_bench.rs@@ -0,0 +1,195 @@+#![cfg(metal_backend)]++use std::{env, mem::size_of, time::Duration};++use criterion::{BenchmarkId, Criterion};+use half::bf16;+use proc_macros::uzu_bench;++use crate::{+ array::ArrayContextExt,+ backends::{+ common::{Allocation, Backend, Context, Encoder, Kernels, kernel::BuildTreeGramKernel},+ metal::Metal,+ },+ data_type::DataType,+ tests::{cold_pool::ColdPool, matmul::iter_encode_loop_named},+};++const K_HEADS: usize = 16;+const VALUE_HEADS: usize = 48;+const HEAD_K_DIM: usize = 128;+const TREE_SIZES: &[usize] = &[33, 49, 64, 128, 256, 512];+const BATCH_SIZES: &[usize] = &[1, 2, 4, 8];++struct TreeGramBuffers {+ q: Allocation<Metal>,+ k: Allocation<Metal>,+ trie: Allocation<Metal>,+ prefix: Allocation<Metal>,+ beta: Allocation<Metal>,+ a_mat: Allocation<Metal>,+ qkd: Allocation<Metal>,+ ainv: Allocation<Metal>,+}++fn reranker_like_trie(+ batch_size: usize,+ tree_size: usize,+) -> Vec<u32> {+ let mut trie = Vec::with_capacity(batch_size * tree_size * 3);+ for b in 0..batch_size {+ let mut children = vec![Vec::<usize>::new(); tree_size];+ let mut child_count = vec![0usize; tree_size];+ for node in 1..tree_size {+ let mut parent = (node - 1) / 2;+ if child_count[parent] >= 2 {+ parent = node.saturating_sub(2 + (b + node) % node);+ }+ children[parent].push(node);+ child_count[parent] += 1;+ }++ fn dfs(+ node: usize,+ children: &[Vec<usize>],+ trie: &mut [[u32; 3]],+ next: &mut u32,+ depth: u32,+ ) {+ let start = *next;+ *next += 1;+ let mut end = start;+ for &child in &children[node] {+ dfs(child, children, trie, next, depth + 1);+ end = end.max(trie[child][1]);+ }+ trie[node] = [start, end, depth];+ }++ let mut batch_trie = vec![[0u32; 3]; tree_size];+ let mut next = 0;+ dfs(0, &children, &mut batch_trie, &mut next, 0);+ for node in batch_trie {+ trie.extend_from_slice(&node);+ }+ }+ trie+}++fn make_buffers(+ context: &<Metal as Backend>::Context,+ batch_size: usize,+ tree_size: usize,+) -> (TreeGramBuffers, f32) {+ let qk_len = batch_size * tree_size * K_HEADS * HEAD_K_DIM;+ let head_len = batch_size * tree_size * VALUE_HEADS;+ let out_len = batch_size * VALUE_HEADS * tree_size * tree_size;+ let trie = reranker_like_trie(batch_size, tree_size);+ let q = (0..qk_len).map(|i| bf16::from_f32(((i as f32 * 0.017).sin() * 0.2) + 0.01)).collect::<Vec<_>>();+ let k = (0..qk_len).map(|i| bf16::from_f32(((i as f32 * 0.019).cos() * 0.18) - 0.02)).collect::<Vec<_>>();+ let prefix = (0..head_len)+ .map(|i| -((i % tree_size) as f32) * 0.01 - ((i % VALUE_HEADS) as f32) * 0.003)+ .collect::<Vec<_>>();+ let beta = (0..head_len).map(|i| 0.25 + ((i as f32 * 0.013).sin() + 1.0) * 0.2).collect::<Vec<_>>();+ let scale = (HEAD_K_DIM as f32).sqrt().recip();+ (+ TreeGramBuffers {+ q: context.create_array_from(&[q.len()], &q).into_allocation(),+ k: context.create_array_from(&[k.len()], &k).into_allocation(),+ trie: context.create_array_from(&[trie.len()], &trie).into_allocation(),+ prefix: context.create_array_from(&[prefix.len()], &prefix).into_allocation(),+ beta: context.create_array_from(&[beta.len()], &beta).into_allocation(),+ a_mat: context.create_array_uninitialized(&[out_len], DataType::F32).into_allocation(),+ qkd: context.create_array_uninitialized(&[out_len], DataType::F32).into_allocation(),+ ainv: context.create_array_uninitialized(&[out_len], DataType::F32).into_allocation(),+ },+ scale,+ )+}++fn buffers_bytes(+ batch_size: usize,+ tree_size: usize,+) -> usize {+ let qk_len = batch_size * tree_size * K_HEADS * HEAD_K_DIM;+ let head_len = batch_size * tree_size * VALUE_HEADS;+ let out_len = batch_size * VALUE_HEADS * tree_size * tree_size;+ qk_len * size_of::<bf16>() * 2+ + batch_size * tree_size * 3 * size_of::<u32>()+ + head_len * size_of::<f32>() * 2+ + out_len * size_of::<f32>() * 3+}++#[uzu_bench]+fn bench_build_tree_gram(c: &mut Criterion) {+ let context = <Metal as Backend>::Context::new().expect("metal context");+ let cold_buffers = env::var("UZU_GRAM_COLD_BUFFERS").is_ok();+ let kernel_paths = if context.supports_mxu() {+ &[("Simdgroup", false), ("MXU", true)][..]+ } else {+ &[("Simdgroup", false)][..]+ };++ for &(kernel_path, use_mxu) in kernel_paths {+ let mut group = c.benchmark_group(format!("Metal/Kernel/GDNTreeVerify/BuildTreeGram/{kernel_path}"));+ group.sample_size(10).warm_up_time(Duration::from_millis(100)).measurement_time(Duration::from_millis(500));++ for &batch_size in BATCH_SIZES {+ for &tree_size in TREE_SIZES {+ let kernel = <<Metal as Backend>::Kernels as Kernels>::BuildTreeGramKernel::new(+ &context,+ DataType::BF16,+ use_mxu,+ )+ .expect("BuildTreeGramKernel");+ let scale = (HEAD_K_DIM as f32).sqrt().recip();+ let benchmark_path =+ format!("Metal/Kernel/GDNTreeVerify/BuildTreeGram/{kernel_path}/B{batch_size}_T{tree_size}");+ let benchmark_id = BenchmarkId::from_parameter(format!(+ "B{batch_size}_T{tree_size}_Hg{K_HEADS}_HV{VALUE_HEADS}_K{HEAD_K_DIM}"+ ));++ let encode = |buffers: &mut TreeGramBuffers, encoder: &mut Encoder<Metal>| {+ kernel.encode(+ &buffers.q,+ &buffers.k,+ &buffers.trie,+ &buffers.prefix,+ &buffers.beta,+ &mut buffers.a_mat,+ &mut buffers.qkd,+ &mut buffers.ainv,+ scale,+ batch_size as u32,+ tree_size as u32,+ K_HEADS as u32,+ VALUE_HEADS as u32,+ HEAD_K_DIM as u32,+ encoder,+ );+ };++ if cold_buffers {+ let mut buffers = ColdPool::new(buffers_bytes(batch_size, tree_size), || {+ make_buffers(&context, batch_size, tree_size).0+ });+ group.bench_function(benchmark_id, |bencher| {+ iter_encode_loop_named::<Metal, _>(context.as_ref(), bencher, &benchmark_path, |encoder| {+ let buffers = buffers.next_mut();+ encode(buffers, encoder);+ });+ });+ } else {+ let (mut buffers, _) = make_buffers(&context, batch_size, tree_size);+ group.bench_function(benchmark_id, |bencher| {+ iter_encode_loop_named::<Metal, _>(context.as_ref(), bencher, &benchmark_path, |encoder| {+ encode(&mut buffers, encoder);+ });+ });+ }+ }+ }+ group.finish();+ }+}diff --git a/crates/backend-uzu/unit/backends/common/kernel/gram_test.rs b/crates/backend-uzu/unit/backends/common/kernel/gram_test.rsnew file mode 100644index 000000000..697f90968--- /dev/null+++ b/crates/backend-uzu/unit/backends/common/kernel/gram_test.rs@@ -0,0 +1,136 @@+use proc_macros::uzu_test;+use test_runner::for_each_non_cpu_backend;++use crate::{+ array::ArrayContextExt,+ backends::{+ common::{Backend, Context, Encoder, Kernels, kernel::BuildTreeGramKernel},+ cpu::Cpu,+ },+ data_type::DataType,+ tests::{assert::assert_eq_float, helpers::allocation_to_vec},+};++const BATCH_SIZE: usize = 2;+const K_HEADS: usize = 2;+const VALUE_HEADS: usize = 6;+const HEAD_K_DIM: usize = 128;++fn get_output<B: Backend>(+ q: &[f32],+ k: &[f32],+ trie: &[u32],+ prefix: &[f32],+ beta: &[f32],+ tree_size: usize,+ scale: f32,+ use_mxu: bool,+) -> (Vec<f32>, Vec<f32>, Vec<f32>) {+ let context = B::Context::new().expect("Failed to create Context");+ let kernel = <<B as Backend>::Kernels as Kernels>::BuildTreeGramKernel::new(&context, DataType::F32, use_mxu)+ .expect("Failed to create BuildTreeGramKernel");++ let output_len = BATCH_SIZE * VALUE_HEADS * tree_size * tree_size;+ let q = context.create_array_from(&[q.len()], q);+ let k = context.create_array_from(&[k.len()], k);+ let trie = context.create_array_from(&[trie.len()], trie);+ let prefix = context.create_array_from(&[prefix.len()], prefix);+ let beta = context.create_array_from(&[beta.len()], beta);+ let mut a_mat = context.create_array_uninitialized(&[output_len], DataType::F32).into_allocation();+ let mut qkd = context.create_array_uninitialized(&[output_len], DataType::F32).into_allocation();+ let mut ainv = context.create_array_uninitialized(&[output_len], DataType::F32).into_allocation();++ let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");+ kernel.encode(+ q.allocation(),+ k.allocation(),+ trie.allocation(),+ prefix.allocation(),+ beta.allocation(),+ &mut a_mat,+ &mut qkd,+ &mut ainv,+ scale,+ BATCH_SIZE as u32,+ tree_size as u32,+ K_HEADS as u32,+ VALUE_HEADS as u32,+ HEAD_K_DIM as u32,+ &mut encoder,+ );+ encoder.end_encoding().submit().wait_until_completed().unwrap();++ (allocation_to_vec(&a_mat), allocation_to_vec(&qkd), allocation_to_vec(&ainv))+}++fn build_trie(tree_size: usize) -> Vec<u32> {+ let last = tree_size as u32 - 1;+ let mut trie = Vec::with_capacity(BATCH_SIZE * tree_size * 3);+ for i in 0..tree_size as u32 {+ trie.extend_from_slice(&[i, last, i]);+ }+ trie.extend_from_slice(&[0, last, 0]);+ for i in 1..tree_size as u32 {+ trie.extend_from_slice(&[i, i, 1]);+ }+ trie+}++#[uzu_test]+fn test_build_tree_gram_matches_cpu() {+ let scale = (HEAD_K_DIM as f32).sqrt().recip();++ for tree_size in [17, 64, 128] {+ let q_len = BATCH_SIZE * tree_size * K_HEADS * HEAD_K_DIM;+ let kv_len = BATCH_SIZE * tree_size * VALUE_HEADS;+ let trie = build_trie(tree_size);+ let q = (0..q_len).map(|i| ((i as f32 * 0.017).sin() * 0.2) + 0.01).collect::<Vec<_>>();+ let k = (0..q_len).map(|i| ((i as f32 * 0.019).cos() * 0.18) - 0.02).collect::<Vec<_>>();+ let prefix = (0..kv_len)+ .map(|i| -((i % tree_size) as f32) * 0.01 - ((i % VALUE_HEADS) as f32) * 0.003)+ .collect::<Vec<_>>();+ let beta = (0..kv_len).map(|i| 0.25 + ((i as f32 * 0.013).sin() + 1.0) * 0.2).collect::<Vec<_>>();++ let expected = get_output::<Cpu>(&q, &k, &trie, &prefix, &beta, tree_size, scale, false);++ for_each_non_cpu_backend!(|B| {+ let context = <<B as Backend>::Context as Context>::new().expect("Failed to create Context");+ let paths: &[bool] = if context.supports_mxu() {+ &[false, true]+ } else {+ &[false]+ };++ for &use_mxu in paths {+ let actual = get_output::<B>(&q, &k, &trie, &prefix, &beta, tree_size, scale, use_mxu);+ let path = if use_mxu {+ "mxu"+ } else {+ "simdgroup"+ };+ let msg = format!("backend {} {path} tree_size {tree_size}", std::any::type_name::<B>());+ assert_eq_float::<f32>(&expected.0, &actual.0, 5e-3, &format!("a_mat {msg}"));+ assert_eq_float::<f32>(&expected.1, &actual.1, 5e-3, &format!("qkd {msg}"));++ for (expected_matrix, actual_matrix) in+ expected.2.chunks(tree_size * tree_size).zip(actual.2.chunks(tree_size * tree_size))+ {+ for block_start in (0..tree_size).step_by(16) {+ let block_end = (block_start + 16).min(tree_size);+ for i in block_start..block_end {+ for j in block_start..block_end {+ let idx = i * tree_size + j;+ let expected = expected_matrix[idx];+ let actual = actual_matrix[idx];+ assert!(+ (expected - actual).abs() < 1e-2,+ "ainv {msg}. Mismatch at matrix index {idx}: expected {expected}, got {actual}"+ );+ }+ }+ }+ }+ }+ });+ }+}diff --git a/crates/backend-uzu/unit/backends/common/kernel/mod.rs b/crates/backend-uzu/unit/backends/common/kernel/mod.rsindex fc87a59d0..0d9a5e47c 100644--- a/crates/backend-uzu/unit/backends/common/kernel/mod.rs+++ b/crates/backend-uzu/unit/backends/common/kernel/mod.rs@@ -4,6 +4,8 @@ mod audio;mod delta_net_test;mod embedding;mod gated_act_mul_test;+mod gram_bench;+mod gram_test;mod hadamard_transform_test;mod kv_cache_update_test;mod layer_norm_test;