A high-performance inference engine for AI models
Remove gpu fill zeros command (#926)
CCYeh14163e07516c
diff --git a/crates/uzu-engine/src/backends/cpu/kernel/gdn/tree_verify/out.rs b/crates/uzu-engine/src/backends/cpu/kernel/gdn/tree_verify/out.rsindex 1ab67c34f..cbb490f06 100644--- a/crates/uzu-engine/src/backends/cpu/kernel/gdn/tree_verify/out.rs+++ b/crates/uzu-engine/src/backends/cpu/kernel/gdn/tree_verify/out.rs@@ -13,7 +13,6 @@ pub fn build_tree_out<QKT: ArrayElement + Float, OutputT: ArrayElement + Float>(qkd: *const f32,u: *const f32,#[optional(use_h0)] h0: Option<*const f32>,- #[optional(use_h0)] h0_indices: Option<*const i32>,o: *mut OutputT,scale: f32,batch_size: u32,@@ -38,14 +37,8 @@ pub fn build_tree_out<QKT: ArrayElement + Float, OutputT: ArrayElement + Float>(let head_v_dim = head_v_dim as usize;let value_heads_per_qk_head = value_heads / qk_heads;let h0 = h0.unwrap_or(std::ptr::null());- let h0_indices = h0_indices.unwrap_or(std::ptr::null());for batch in 0..batch_size {- let h0_index = if use_h0 {- unsafe { *h0_indices.add(batch) }- } else {- -1- };for value_head in 0..value_heads {let qk_head = value_head / value_heads_per_qk_head;let q_head_base = (batch * tree_size * qk_heads + qk_head) * head_k_dim;@@ -59,9 +52,8 @@ pub fn build_tree_out<QKT: ArrayElement + Float, OutputT: ArrayElement + Float>(for value_col in 0..head_v_dim {let mut acc = 0.0f32;- if use_h0 && h0_index >= 0 {- let h0_base =- ((h0_index as usize * value_heads + value_head) * head_v_dim + value_col) * head_k_dim;+ if use_h0 {+ let h0_base = ((batch * value_heads + value_head) * head_v_dim + value_col) * head_k_dim;let mut dot = 0.0f32;for dim in 0..head_k_dim {let qv = unsafe { (*q.add(q_row + dim)).to_f32().unwrap() };diff --git a/crates/uzu-engine/src/backends/cpu/kernel/gdn/tree_verify/tree_gram.rs b/crates/uzu-engine/src/backends/cpu/kernel/gdn/tree_verify/tree_gram.rsindex 548cb3c8d..9ac1eba10 100644--- a/crates/uzu-engine/src/backends/cpu/kernel/gdn/tree_verify/tree_gram.rs+++ b/crates/uzu-engine/src/backends/cpu/kernel/gdn/tree_verify/tree_gram.rs@@ -16,7 +16,6 @@ pub fn build_tree_gram<T: ArrayElement + Float, const USE_MXU: bool>(prefix: *const f32,beta: *const f32,#[optional(use_h0)] h0: Option<*const f32>,- #[optional(use_h0)] h0_idx: Option<*const i32>,a_packed: *mut f32,qkd: *mut f32,a_inv: *mut f32,@@ -135,27 +134,23 @@ pub fn build_tree_gram<T: ArrayElement + Float, const USE_MXU: bool>(}}- // kh0[b, token, hv, dv] = dot(k[b, token, hk], h0[slot, hv, dv])+ // kh0[b, token, hv, dv] = dot(k[b, token, hk], h0[b, hv, dv])if !use_h0 {continue;}let h0 = h0.unwrap();- let h0_idx = h0_idx.unwrap();let kh0 = kh0.unwrap();- let h0_slot = unsafe { *h0_idx.add(batch) };- if h0_slot >= 0 {- let h0_head = ((h0_slot as usize) * value_heads + hv) * head_v_dim * head_k_dim;- for token in 0..tree_size {- let k_off = ((batch * tree_size + token) * k_heads + hk) * head_k_dim;- let kh0_off = ((batch * tree_size + token) * value_heads + hv) * head_v_dim;- for dv in 0..head_v_dim {- let mut sum = 0.0f32;- for d in 0..head_k_dim {- sum += unsafe { (*k.add(k_off + d)).to_f32().unwrap() }- * unsafe { *h0.add(h0_head + dv * head_k_dim + d) };- }- unsafe { *kh0.add(kh0_off + dv) = sum };+ let h0_head = (batch * value_heads + hv) * head_v_dim * head_k_dim;+ for token in 0..tree_size {+ let k_off = ((batch * tree_size + token) * k_heads + hk) * head_k_dim;+ let kh0_off = ((batch * tree_size + token) * value_heads + hv) * head_v_dim;+ for dv in 0..head_v_dim {+ let mut sum = 0.0f32;+ for d in 0..head_k_dim {+ sum += unsafe { (*k.add(k_off + d)).to_f32().unwrap() }+ * unsafe { *h0.add(h0_head + dv * head_k_dim + d) };}+ unsafe { *kh0.add(kh0_off + dv) = sum };}}}diff --git a/crates/uzu-engine/src/backends/cpu/kernel/gdn/tree_verify/tree_update_solve.rs b/crates/uzu-engine/src/backends/cpu/kernel/gdn/tree_verify/tree_update_solve.rsindex ccb5f011e..f243c1e38 100644--- a/crates/uzu-engine/src/backends/cpu/kernel/gdn/tree_verify/tree_update_solve.rs+++ b/crates/uzu-engine/src/backends/cpu/kernel/gdn/tree_verify/tree_update_solve.rs@@ -16,7 +16,6 @@ pub fn tree_update_solve<T: ArrayElement + Float, const BV: u32>(beta: *const f32,a_packed: *const f32,a_inv: *const f32,- #[optional(use_h0)] h0_idx: Option<*const i32>,u: *mut f32,batch_size: u32,tree_size: u32,@@ -38,15 +37,7 @@ pub fn tree_update_solve<T: ArrayElement + Float, const BV: u32>(let num_col_pairs = num_blocks.div_ceil(2);let kh0 = kh0.unwrap_or(std::ptr::null());- let h0_idx = h0_idx.unwrap_or(std::ptr::null());-for batch in 0..batch_size {- let h0_slot = if use_h0 {- unsafe { *h0_idx.add(batch) }- } else {- -1- };-for hv in 0..value_heads {let batch_value_head_idx = batch * value_heads + hv;@@ -68,7 +59,7 @@ pub fn tree_update_solve<T: ArrayElement + Float, const BV: u32>(let v_idx = (((batch * tree_size + token) * value_heads + hv) * head_v_dim) + dv;let v_val = unsafe { (*v.add(v_idx)).to_f32().unwrap() };- let kh0_val = if h0_slot >= 0 {+ let kh0_val = if use_h0 {unsafe { *kh0.add(v_idx) }} else {0.0diff --git a/crates/uzu-engine/src/backends/metal/kernel/gdn/tree_verify.rs b/crates/uzu-engine/src/backends/metal/kernel/gdn/tree_verify.rsindex 34807d306..62b7d8c4f 100644--- a/crates/uzu-engine/src/backends/metal/kernel/gdn/tree_verify.rs+++ b/crates/uzu-engine/src/backends/metal/kernel/gdn/tree_verify.rs@@ -99,9 +99,6 @@ impl DeltaNetTreeVerify for MetalDeltaNetTreeVerify {command_buffer: &mut MetalCommandBufferEncoding,) -> Result<<Metal as Backend>::ScratchBuffer, MetalError> {let layout = Layout::new(arguments.tree_size, &self.arguments);- let mut h0_indices = command_buffer.allocate_scratch(DataType::I32.size_in_bytes())?;- command_buffer.encode_fill(&mut h0_indices, 0);-let mut prefix =command_buffer.allocate_scratch_for_shape(&[layout.tree_size, layout.num_v_heads], INNER_DATA_TYPE)?;let mut a_packed = command_buffer.allocate_scratch_for_shape(&layout.a_packed_shape(), INNER_DATA_TYPE)?;@@ -133,7 +130,6 @@ impl DeltaNetTreeVerify for MetalDeltaNetTreeVerify {&prefix,arguments.beta,Some(arguments.h0),- Some(&h0_indices),&mut a_packed,&mut qkd,&mut a_inverse,@@ -154,7 +150,6 @@ impl DeltaNetTreeVerify for MetalDeltaNetTreeVerify {arguments.beta,&a_packed,&a_inverse,- Some(&h0_indices),&mut u,1,arguments.tree_size,@@ -168,7 +163,6 @@ impl DeltaNetTreeVerify for MetalDeltaNetTreeVerify {&qkd,&u,Some(arguments.h0),- Some(&h0_indices),&mut output,1.0,1,diff --git a/crates/uzu-engine/src/backends/metal/kernel/gdn/tree_verify/out.metal b/crates/uzu-engine/src/backends/metal/kernel/gdn/tree_verify/out.metalindex 50022cee9..e00132f9d 100644--- a/crates/uzu-engine/src/backends/metal/kernel/gdn/tree_verify/out.metal+++ b/crates/uzu-engine/src/backends/metal/kernel/gdn/tree_verify/out.metal@@ -20,7 +20,7 @@ using namespace uzu::matmul;// prefix: [B, T, HV] path log-decay prefix from BuildTreePrefix// qkd: [B, HV, T, T] from BuildTreeGram// u: [B, HV, T, V]-// h0: [H0, HV, V, K]+// h0: [B, HV, V, K]//// o[row, value] = exp(prefix[row]) * scale * dot(q[row], h0[value])// + sum_j qkd[row, j] * u[j, value]@@ -36,7 +36,6 @@ PUBLIC KERNEL(BuildTreeOut)(const device float* qkd,const device float* u,const device float* h0 OPTIONAL(use_h0),- const device int* h0_indices OPTIONAL(use_h0),device OutputT* o,constant const float& scale,constant const uint& batch_size,@@ -83,10 +82,9 @@ PUBLIC KERNEL(BuildTreeOut)(AccFragment acc;acc.clear();- const int h0_index = use_h0 ? h0_indices[batch_idx] : -1;- if (use_h0 && h0_index >= 0) {+ if (use_h0) {const uint q_base = (batch_idx * tree_size * qk_heads + row_base * qk_heads + qk_head_idx) * head_k_dim;- const uint h0_base = ((uint(h0_index) * value_heads + value_head_idx) * head_v_dim + value_base) * head_k_dim;+ const uint h0_base = ((batch_idx * value_heads + value_head_idx) * head_v_dim + value_base) * head_k_dim;for (uint k0 = 0; k0 < head_k_dim; k0 += ROWS) {const short valid_k = short(min(uint(ROWS), head_k_dim - k0));QFragment q_frag;diff --git a/crates/uzu-engine/src/backends/metal/kernel/gdn/tree_verify/tree_gram.metal b/crates/uzu-engine/src/backends/metal/kernel/gdn/tree_verify/tree_gram.metalindex 54c28d4a4..bef34b39f 100644--- a/crates/uzu-engine/src/backends/metal/kernel/gdn/tree_verify/tree_gram.metal+++ b/crates/uzu-engine/src/backends/metal/kernel/gdn/tree_verify/tree_gram.metal@@ -18,13 +18,12 @@ using namespace uzu::trie;#define NUM_SIMDGROUPS 2u#define INVALID_ROW 0xffffffffu-// kh0 = k @ h0[h0_idx[batch]]^T for one row-tile, walking head_v_dim in COL_TILE-// chunks from dv_start by dv_stride. No-op for a zero initial state.+// kh0 = k @ h0[batch]^T for one row-tile, walking head_v_dim in COL_TILE+// chunks from dv_start by dv_stride.template <typename T, typename Ops, typename AccFragment, typename LeftFragment, typename RightFragment>METAL_FUNC void tree_kh0(const device T* k_rows,const device float* h0,- const device int* h0_idx,device float* kh0,const uint batch_idx,const uint value_head_idx,@@ -40,11 +39,7 @@ METAL_FUNC void tree_kh0(const uint dv_stride,const ushort lane) {- const int h0_slot = h0_idx[batch_idx];- if (h0_slot < 0) {- return;- }- const device float* h0_head = h0 + (uint(h0_slot) * value_heads + value_head_idx) * head_v_dim * head_k_dim;+ const device float* h0_head = h0 + (batch_idx * value_heads + value_head_idx) * head_v_dim * head_k_dim;device float* kh0_rows =kh0 + (batch_idx * tree_size + row_base) * value_heads * head_v_dim + value_head_idx * head_v_dim;const uint kh0_row_stride = value_heads * head_v_dim;@@ -85,8 +80,7 @@ METAL_FUNC void tree_kh0(// qkd: scale * exp(prefix[row] - prefix[col]) * dot(q[row], k[col]) for// ancestor-or-self, dense [B*HV, T, T] f32// a_inv: (I + A)^-1 per diagonal block, compact [B*HV, NB, 16, 16]-// kh0: k @ h0[h0_idx[batch]]^T, [B, T, HV, head_v_dim] f32; skipped when-// h0_idx[batch] < 0+// kh0: k @ h0[batch]^T, [B, T, HV, head_v_dim] f32template <typename T, bool USE_MXU>VARIANTS(T, float, bfloat)VARIANTS(USE_MXU, false, true)@@ -97,7 +91,6 @@ PUBLIC KERNEL(BuildTreeGram)(const device float* prefix,const device float* beta,const device float* h0 OPTIONAL(use_h0),- const device int* h0_idx OPTIONAL(use_h0),device float* a_packed,device float* qkd,device float* a_inv,@@ -279,7 +272,6 @@ PUBLIC KERNEL(BuildTreeGram)(tree_kh0<T, Ops, AccFragment, LeftFragment, RightFragment>(k_rows,h0,- h0_idx,kh0,batch_idx,value_head_idx,diff --git a/crates/uzu-engine/src/backends/metal/kernel/gdn/tree_verify/tree_update_solve.metal b/crates/uzu-engine/src/backends/metal/kernel/gdn/tree_verify/tree_update_solve.metalindex a3c98aa93..cd84bc13a 100644--- a/crates/uzu-engine/src/backends/metal/kernel/gdn/tree_verify/tree_update_solve.metal+++ b/crates/uzu-engine/src/backends/metal/kernel/gdn/tree_verify/tree_update_solve.metal@@ -18,7 +18,6 @@ using namespace uzu::matmul;// A [B*HV, NB, ceil(NB/2), 16, 32] f32 packed pair tiles from BuildTreeGram// (block (i,j) at pair j/2, column (j%2)*16)// Ainv [B*HV, NB, 16, 16] f32 diagonal-block inverses from BuildTreeGram-// h0_idx [B] i32, negative = zero initial state (absent when !use_h0)// U [B*HV, T, head_v_dim] f32 (output)//// Grid: one simdgroup per (batch, value head, BV value-dim tile); the history is@@ -36,7 +35,6 @@ PUBLIC KERNEL(TreeUpdateSolve)(const device float* beta,const device float* a_packed,const device float* a_inv,- const device int* h0_idx OPTIONAL(use_h0),device float* u,constant const uint& batch_size,constant const uint& tree_size,@@ -66,7 +64,6 @@ PUBLIC KERNEL(TreeUpdateSolve)(const uint value_dim_base = value_tile_idx * BV;const uint tile_value_cols = min(BV, head_v_dim - value_dim_base);const uint num_blocks = div_ceil(tree_size, BT);- const int h0_slot = use_h0 ? h0_idx[batch_idx] : -1;const uint value_token_stride = value_heads * head_v_dim;const uint scalar_token_stride = value_heads;@@ -88,7 +85,7 @@ PUBLIC KERNEL(TreeUpdateSolve)(TileFragment acc;- if (h0_slot >= 0) {+ if (use_h0) {const device float* kh0_block = kh0_head_tile + row_base * value_token_stride;acc.load_maybe_bounded(lane_idx,diff --git a/crates/uzu-engine/unit/backends/common/kernel/gdn/tree_verify/out_bench.rs b/crates/uzu-engine/unit/backends/common/kernel/gdn/tree_verify/out_bench.rsindex e21376da3..52ff22aec 100644--- a/crates/uzu-engine/unit/backends/common/kernel/gdn/tree_verify/out_bench.rs+++ b/crates/uzu-engine/unit/backends/common/kernel/gdn/tree_verify/out_bench.rs@@ -41,7 +41,6 @@ struct TreeOutBuffers {qkd: <Metal as Backend>::GlobalBuffer,u: <Metal as Backend>::GlobalBuffer,h0: <Metal as Backend>::GlobalBuffer,- h0_indices: <Metal as Backend>::GlobalBuffer,o: <Metal as Backend>::GlobalBuffer,}@@ -62,7 +61,6 @@ fn make_buffers<T: ArrayElement + Float>(let qkd = (0..qkd_len).map(|i| ((i as f32 * 0.013).cos() * 0.03) - 0.01).collect::<Vec<_>>();let u = (0..uv_len).map(|i| (i as f32 * 0.011).sin() * 0.3 + 0.04).collect::<Vec<_>>();let h0 = (0..h0_len).map(|i| ((i as f32 * 0.019).cos() * 0.2) - 0.01).collect::<Vec<_>>();- let h0_indices = (0..batch_size as i32).collect::<Vec<_>>();TreeOutBuffers {q: create_buffer_with_data::<Metal, T>(context, &q),@@ -70,7 +68,6 @@ fn make_buffers<T: ArrayElement + Float>(qkd: create_buffer_with_data::<Metal, f32>(context, &qkd),u: create_buffer_with_data::<Metal, f32>(context, &u),h0: create_buffer_with_data::<Metal, f32>(context, &h0),- h0_indices: create_buffer_with_data::<Metal, i32>(context, &h0_indices),o: create_buffer::<Metal, T>(context, uv_len),}}@@ -122,7 +119,6 @@ fn bench_build_tree_out_type<T: ArrayElement + Float>(&buffers.qkd,&buffers.u,Some(&buffers.h0),- Some(&buffers.h0_indices),&mut buffers.o,(HEAD_K_DIM as f32).sqrt().recip(),batch_size as u32,diff --git a/crates/uzu-engine/unit/backends/common/kernel/gdn/tree_verify/out_test.rs b/crates/uzu-engine/unit/backends/common/kernel/gdn/tree_verify/out_test.rsindex 195c4e117..698ab0b97 100644--- a/crates/uzu-engine/unit/backends/common/kernel/gdn/tree_verify/out_test.rs+++ b/crates/uzu-engine/unit/backends/common/kernel/gdn/tree_verify/out_test.rs@@ -36,7 +36,6 @@ struct Inputs<T> {qkd: Vec<f32>,u: Vec<f32>,h0: Vec<f32>,- h0_indices: Vec<i32>,}const BUILD_TREE_OUT_PATHS: &[(&str, bool, bool)] = &[@@ -50,8 +49,7 @@ fn make_inputs<T: ArrayElement + Float>(shape: Shape) -> Inputs<T> {let prefix_len = shape.batch_size * shape.tree_size * shape.value_heads;let qkd_len = shape.batch_size * shape.value_heads * shape.tree_size * shape.tree_size;let u_len = shape.batch_size * shape.value_heads * shape.tree_size * shape.head_v_dim;- let h0_pool = shape.batch_size + 1;- let h0_len = h0_pool * shape.value_heads * shape.head_v_dim * shape.head_k_dim;+ let h0_len = shape.batch_size * shape.value_heads * shape.head_v_dim * shape.head_k_dim;Inputs {q: (0..q_len).map(|i| T::from(((i as f32 * 0.017).sin() * 0.2) + 0.01).unwrap()).collect(),@@ -61,15 +59,6 @@ fn make_inputs<T: ArrayElement + Float>(shape: Shape) -> Inputs<T> {qkd: (0..qkd_len).map(|i| ((i as f32 * 0.013).cos() * 0.1) - 0.02).collect(),u: (0..u_len).map(|i| ((i as f32 * 0.011).sin() * 0.3) + 0.04).collect(),h0: (0..h0_len).map(|i| ((i as f32 * 0.019).cos() * 0.2) - 0.01).collect(),- h0_indices: (0..shape.batch_size)- .map(|i| {- if i + 1 == shape.batch_size {- -1- } else {- i as i32- }- })- .collect(),}}@@ -95,7 +84,6 @@ fn run_build_tree_out<B: Backend, T: ArrayElement + Float>(let qkd = create_buffer_with_data::<B, f32>(&context, &inputs.qkd);let u = create_buffer_with_data::<B, f32>(&context, &inputs.u);let h0 = use_h0.then(|| create_buffer_with_data::<B, f32>(&context, &inputs.h0));- let h0_indices = use_h0.then(|| create_buffer_with_data::<B, i32>(&context, &inputs.h0_indices));let mut o =create_buffer::<B, T>(&context, shape.batch_size * shape.tree_size * shape.value_heads * shape.head_v_dim);@@ -106,7 +94,6 @@ fn run_build_tree_out<B: Backend, T: ArrayElement + Float>(&qkd,&u,h0.as_ref(),- h0_indices.as_ref(),&mut o,(shape.head_k_dim as f32).sqrt().recip(),shape.batch_size as u32,diff --git a/crates/uzu-engine/unit/backends/common/kernel/gdn/tree_verify/tree_gram_bench.rs b/crates/uzu-engine/unit/backends/common/kernel/gdn/tree_verify/tree_gram_bench.rsindex 577d8d1f7..733a74811 100644--- a/crates/uzu-engine/unit/backends/common/kernel/gdn/tree_verify/tree_gram_bench.rs+++ b/crates/uzu-engine/unit/backends/common/kernel/gdn/tree_verify/tree_gram_bench.rs@@ -33,7 +33,6 @@ struct TreeGramBuffers {prefix: <Metal as Backend>::GlobalBuffer,beta: <Metal as Backend>::GlobalBuffer,h0: <Metal as Backend>::GlobalBuffer,- h0_idx: <Metal as Backend>::GlobalBuffer,a_packed: <Metal as Backend>::GlobalBuffer,qkd: <Metal as Backend>::GlobalBuffer,a_inv: <Metal as Backend>::GlobalBuffer,@@ -105,7 +104,6 @@ fn make_buffers(.collect::<Vec<_>>();let beta = (0..scalar_len).map(|i| 0.25 + ((i as f32 * 0.013).sin() + 1.0) * 0.2).collect::<Vec<_>>();let h0 = (0..h0_len).map(|i| ((i as f32 * 0.007).sin() * 0.05) - 0.01).collect::<Vec<_>>();- let h0_idx = (0..batch_size).map(|i| i as i32).collect::<Vec<_>>();let scale = (HEAD_K_DIM as f32).sqrt().recip();(TreeGramBuffers {@@ -115,7 +113,6 @@ fn make_buffers(prefix: create_buffer_with_data::<Metal, f32>(context, &prefix),beta: create_buffer_with_data::<Metal, f32>(context, &beta),h0: create_buffer_with_data::<Metal, f32>(context, &h0),- h0_idx: create_buffer_with_data::<Metal, i32>(context, &h0_idx),a_packed: create_buffer::<Metal, f32>(context, a_len),qkd: create_buffer::<Metal, f32>(context, out_len),a_inv: create_buffer::<Metal, f32>(context, a_inv_len),@@ -142,7 +139,6 @@ fn buffers_bytes(+ batch_size * tree_size * 3 * size_of::<u32>()+ scalar_len * size_of::<f32>() * 2+ (out_len + a_len + a_inv_len + kh0_len) * size_of::<f32>()- + batch_size * size_of::<i32>()}#[uzu_bench]@@ -185,7 +181,6 @@ fn bench_build_tree_gram(c: &mut Criterion) {&buffers.prefix,&buffers.beta,Some(&buffers.h0),- Some(&buffers.h0_idx),&mut buffers.a_packed,&mut buffers.qkd,&mut buffers.a_inv,diff --git a/crates/uzu-engine/unit/backends/common/kernel/gdn/tree_verify/tree_gram_test.rs b/crates/uzu-engine/unit/backends/common/kernel/gdn/tree_verify/tree_gram_test.rsindex b38e7602c..502c5a5da 100644--- a/crates/uzu-engine/unit/backends/common/kernel/gdn/tree_verify/tree_gram_test.rs+++ b/crates/uzu-engine/unit/backends/common/kernel/gdn/tree_verify/tree_gram_test.rs@@ -35,7 +35,6 @@ struct Inputs {prefix: Vec<f32>,beta: Vec<f32>,h0: Vec<f32>,- h0_idx: Vec<i32>,}fn make_inputs(tree_size: usize) -> Inputs {@@ -51,8 +50,6 @@ fn make_inputs(tree_size: usize) -> Inputs {.collect(),beta: (0..scalar_len).map(|i| 0.25 + ((i as f32 * 0.013).sin() + 1.0) * 0.2).collect(),h0: (0..h0_len).map(|i| ((i as f32 * 0.007).sin() * 0.05) - 0.01).collect(),- // Batch 1 has no initial state: covers the kh0-skip path.- h0_idx: vec![0, -1],}}@@ -63,7 +60,6 @@ fn get_output<B: Backend, T: ArrayElement + Float>(prefix: &[f32],beta: &[f32],h0: &[f32],- h0_idx: &[i32],tree_size: usize,scale: f32,use_mxu: bool,@@ -84,15 +80,12 @@ fn get_output<B: Backend, T: ArrayElement + Float>(let prefix = create_buffer_with_data::<B, f32>(&context, prefix);let beta = create_buffer_with_data::<B, f32>(&context, beta);let h0 = create_buffer_with_data::<B, f32>(&context, h0);- let h0_idx = create_buffer_with_data::<B, i32>(&context, h0_idx);// The GPU only writes packed-A tiles touching the block lower triangle;// zero-init so CPU and GPU buffers stay comparable elsewhere.let mut a_packed = create_buffer_with_data::<B, f32>(&context, &vec![0.0f32; a_len]);let mut qkd = create_buffer::<B, f32>(&context, output_len);let mut a_inv = create_buffer::<B, f32>(&context, a_inv_len);- // kh0 is only written for batches with h0_idx >= 0; zero-init so CPU and GPU- // outputs stay comparable on the skipped batch.- let mut kh0 = create_buffer_with_data::<B, f32>(&context, &vec![0.0f32; kh0_len]);+ let mut kh0 = create_buffer::<B, f32>(&context, kh0_len);let mut command_buffer = context.create_command_buffer(None, None).expect("Failed to create command buffer");kernel.encode(@@ -102,7 +95,6 @@ fn get_output<B: Backend, T: ArrayElement + Float>(&prefix,&beta,Some(&h0),- Some(&h0_idx),&mut a_packed,&mut qkd,&mut a_inv,@@ -128,32 +120,12 @@ fn check_type<T: ArrayElement + Float>() {let q = inputs.q.iter().map(|&value| T::from(value).unwrap()).collect::<Vec<_>>();let k = inputs.k.iter().map(|&value| T::from(value).unwrap()).collect::<Vec<_>>();- let expected = get_output::<Cpu, T>(- &q,- &k,- &inputs.trie,- &inputs.prefix,- &inputs.beta,- &inputs.h0,- &inputs.h0_idx,- tree_size,- scale,- false,- );+ let expected =+ get_output::<Cpu, T>(&q, &k, &inputs.trie, &inputs.prefix, &inputs.beta, &inputs.h0, tree_size, scale, false);for_each_non_cpu_backend!(|B| {- let actual = get_output::<B, T>(- &q,- &k,- &inputs.trie,- &inputs.prefix,- &inputs.beta,- &inputs.h0,- &inputs.h0_idx,- tree_size,- scale,- false,- );+ let actual =+ get_output::<B, T>(&q, &k, &inputs.trie, &inputs.prefix, &inputs.beta, &inputs.h0, tree_size, scale, false);let msg = format!("backend {} simdgroup tree_size {tree_size}", std::any::type_name::<B>());assert_eq_float::<f32>(&expected.0, &actual.0, 2e-4, &format!("a_packed {msg}"));assert_eq_float::<f32>(&expected.1, &actual.1, 2e-4, &format!("qkd {msg}"));@@ -169,7 +141,6 @@ fn check_type<T: ArrayElement + Float>(&inputs.prefix,&inputs.beta,&inputs.h0,- &inputs.h0_idx,tree_size,scale,true,diff --git a/crates/uzu-engine/unit/backends/common/kernel/gdn/tree_verify/tree_update_solve_bench.rs b/crates/uzu-engine/unit/backends/common/kernel/gdn/tree_verify/tree_update_solve_bench.rsindex 9b3c14c8f..450ead2b8 100644--- a/crates/uzu-engine/unit/backends/common/kernel/gdn/tree_verify/tree_update_solve_bench.rs+++ b/crates/uzu-engine/unit/backends/common/kernel/gdn/tree_verify/tree_update_solve_bench.rs@@ -59,7 +59,6 @@ struct TreeUpdateSolveBuffers {beta: <Metal as Backend>::GlobalBuffer,a: <Metal as Backend>::GlobalBuffer,a_inv: <Metal as Backend>::GlobalBuffer,- h0_idx: <Metal as Backend>::GlobalBuffer,u: <Metal as Backend>::GlobalBuffer,}@@ -139,8 +138,6 @@ fn make_buffers(inverse_value(batch_head_matrix_idx, block, row, col, tree_size)}).collect::<Vec<_>>();- let h0_idx = (0..batch_size).map(|i| i as i32).collect::<Vec<_>>();-TreeUpdateSolveBuffers {kh0: create_buffer_with_data::<Metal, f32>(context, &kh0),v: create_buffer_with_data::<Metal, bf16>(context, &v),@@ -148,7 +145,6 @@ fn make_buffers(beta: create_buffer_with_data::<Metal, f32>(context, &beta),a,a_inv: create_buffer_with_data::<Metal, f32>(context, &a_inv),- h0_idx: create_buffer_with_data::<Metal, i32>(context, &h0_idx),u: create_buffer::<Metal, f32>(context, u_len),}}@@ -164,9 +160,7 @@ fn buffers_bytes(let u_len = batch_size * NUM_V_HEADS * tree_size * HEAD_V_DIM;let inv_len = batch_size * NUM_V_HEADS * num_blocks * BT * BT;- v_len * size_of::<bf16>()- + (scalar_len * 2 + a_len + inv_len + u_len + v_len) * size_of::<f32>()- + batch_size * size_of::<i32>()+ v_len * size_of::<bf16>() + (scalar_len * 2 + a_len + inv_len + u_len + v_len) * size_of::<f32>()}#[uzu_bench]@@ -220,7 +214,6 @@ fn encode(&buffers.beta,&buffers.a,&buffers.a_inv,- Some(&buffers.h0_idx),&mut buffers.u,batch_size as u32,tree_size as u32,diff --git a/crates/uzu-engine/unit/backends/common/kernel/gdn/tree_verify/tree_update_solve_test.rs b/crates/uzu-engine/unit/backends/common/kernel/gdn/tree_verify/tree_update_solve_test.rsindex c340afd7a..ffde218d7 100644--- a/crates/uzu-engine/unit/backends/common/kernel/gdn/tree_verify/tree_update_solve_test.rs+++ b/crates/uzu-engine/unit/backends/common/kernel/gdn/tree_verify/tree_update_solve_test.rs@@ -78,7 +78,7 @@ const CASES: &[SolveCase] = &[head_v_dim: 64,},SolveCase {- name: "batched_optional_h0",+ name: "batched",batch_size: 2,tree_size: 16,num_v_heads: 2,@@ -149,19 +149,12 @@ fn run_case<B: Backend, T: ArrayElement + Copy>(}}).collect::<Vec<_>>();- let h0_idx = if case.name == "batched_optional_h0" {- vec![0, -1]- } else {- (0..batch_size).map(|batch| batch as i32).collect::<Vec<_>>()- };-let kh0 = create_buffer_with_data::<B, f32>(&context, &kh0);let v = create_buffer_with_data::<B, T>(&context, &v);let prefix = create_buffer_with_data::<B, f32>(&context, &prefix);let beta = create_buffer_with_data::<B, f32>(&context, &beta);let a = create_buffer_with_data::<B, f32>(&context, &a_f32);let a_inv = create_buffer_with_data::<B, f32>(&context, &a_inv);- let h0_idx = create_buffer_with_data::<B, i32>(&context, &h0_idx);let mut u = create_buffer::<B, f32>(&context, u_len);let mut command_buffer = context.create_command_buffer(None, None).expect("command buffer");@@ -172,7 +165,6 @@ fn run_case<B: Backend, T: ArrayElement + Copy>(&beta,&a,&a_inv,- use_h0.then_some(&h0_idx),&mut u,case.batch_size,case.tree_size,