Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

Remove gpu fill zeros command (#926)

CCYeh14163e07516c
diff --git a/crates/uzu-engine/src/backends/cpu/kernel/gdn/tree_verify/out.rs b/crates/uzu-engine/src/backends/cpu/kernel/gdn/tree_verify/out.rs
index 1ab67c34f..cbb490f06 100644
--- a/crates/uzu-engine/src/backends/cpu/kernel/gdn/tree_verify/out.rs
+++ b/crates/uzu-engine/src/backends/cpu/kernel/gdn/tree_verify/out.rs
@@ -13,7 +13,6 @@ pub fn build_tree_out<QKT: ArrayElement + Float, OutputT: ArrayElement + Float>(
qkd: *const f32,
u: *const f32,
#[optional(use_h0)] h0: Option<*const f32>,
- #[optional(use_h0)] h0_indices: Option<*const i32>,
o: *mut OutputT,
scale: f32,
batch_size: u32,
@@ -38,14 +37,8 @@ pub fn build_tree_out<QKT: ArrayElement + Float, OutputT: ArrayElement + Float>(
let head_v_dim = head_v_dim as usize;
let value_heads_per_qk_head = value_heads / qk_heads;
let h0 = h0.unwrap_or(std::ptr::null());
- let h0_indices = h0_indices.unwrap_or(std::ptr::null());
for batch in 0..batch_size {
- let h0_index = if use_h0 {
- unsafe { *h0_indices.add(batch) }
- } else {
- -1
- };
for value_head in 0..value_heads {
let qk_head = value_head / value_heads_per_qk_head;
let q_head_base = (batch * tree_size * qk_heads + qk_head) * head_k_dim;
@@ -59,9 +52,8 @@ pub fn build_tree_out<QKT: ArrayElement + Float, OutputT: ArrayElement + Float>(
for value_col in 0..head_v_dim {
let mut acc = 0.0f32;
- if use_h0 && h0_index >= 0 {
- let h0_base =
- ((h0_index as usize * value_heads + value_head) * head_v_dim + value_col) * head_k_dim;
+ if use_h0 {
+ let h0_base = ((batch * value_heads + value_head) * head_v_dim + value_col) * head_k_dim;
let mut dot = 0.0f32;
for dim in 0..head_k_dim {
let qv = unsafe { (*q.add(q_row + dim)).to_f32().unwrap() };
diff --git a/crates/uzu-engine/src/backends/cpu/kernel/gdn/tree_verify/tree_gram.rs b/crates/uzu-engine/src/backends/cpu/kernel/gdn/tree_verify/tree_gram.rs
index 548cb3c8d..9ac1eba10 100644
--- a/crates/uzu-engine/src/backends/cpu/kernel/gdn/tree_verify/tree_gram.rs
+++ b/crates/uzu-engine/src/backends/cpu/kernel/gdn/tree_verify/tree_gram.rs
@@ -16,7 +16,6 @@ pub fn build_tree_gram<T: ArrayElement + Float, const USE_MXU: bool>(
prefix: *const f32,
beta: *const f32,
#[optional(use_h0)] h0: Option<*const f32>,
- #[optional(use_h0)] h0_idx: Option<*const i32>,
a_packed: *mut f32,
qkd: *mut f32,
a_inv: *mut f32,
@@ -135,27 +134,23 @@ pub fn build_tree_gram<T: ArrayElement + Float, const USE_MXU: bool>(
}
}
- // kh0[b, token, hv, dv] = dot(k[b, token, hk], h0[slot, hv, dv])
+ // kh0[b, token, hv, dv] = dot(k[b, token, hk], h0[b, hv, dv])
if !use_h0 {
continue;
}
let h0 = h0.unwrap();
- let h0_idx = h0_idx.unwrap();
let kh0 = kh0.unwrap();
- let h0_slot = unsafe { *h0_idx.add(batch) };
- if h0_slot >= 0 {
- let h0_head = ((h0_slot as usize) * value_heads + hv) * head_v_dim * head_k_dim;
- for token in 0..tree_size {
- let k_off = ((batch * tree_size + token) * k_heads + hk) * head_k_dim;
- let kh0_off = ((batch * tree_size + token) * value_heads + hv) * head_v_dim;
- for dv in 0..head_v_dim {
- let mut sum = 0.0f32;
- for d in 0..head_k_dim {
- sum += unsafe { (*k.add(k_off + d)).to_f32().unwrap() }
- * unsafe { *h0.add(h0_head + dv * head_k_dim + d) };
- }
- unsafe { *kh0.add(kh0_off + dv) = sum };
+ let h0_head = (batch * value_heads + hv) * head_v_dim * head_k_dim;
+ for token in 0..tree_size {
+ let k_off = ((batch * tree_size + token) * k_heads + hk) * head_k_dim;
+ let kh0_off = ((batch * tree_size + token) * value_heads + hv) * head_v_dim;
+ for dv in 0..head_v_dim {
+ let mut sum = 0.0f32;
+ for d in 0..head_k_dim {
+ sum += unsafe { (*k.add(k_off + d)).to_f32().unwrap() }
+ * unsafe { *h0.add(h0_head + dv * head_k_dim + d) };
}
+ unsafe { *kh0.add(kh0_off + dv) = sum };
}
}
}
diff --git a/crates/uzu-engine/src/backends/cpu/kernel/gdn/tree_verify/tree_update_solve.rs b/crates/uzu-engine/src/backends/cpu/kernel/gdn/tree_verify/tree_update_solve.rs
index ccb5f011e..f243c1e38 100644
--- a/crates/uzu-engine/src/backends/cpu/kernel/gdn/tree_verify/tree_update_solve.rs
+++ b/crates/uzu-engine/src/backends/cpu/kernel/gdn/tree_verify/tree_update_solve.rs
@@ -16,7 +16,6 @@ pub fn tree_update_solve<T: ArrayElement + Float, const BV: u32>(
beta: *const f32,
a_packed: *const f32,
a_inv: *const f32,
- #[optional(use_h0)] h0_idx: Option<*const i32>,
u: *mut f32,
batch_size: u32,
tree_size: u32,
@@ -38,15 +37,7 @@ pub fn tree_update_solve<T: ArrayElement + Float, const BV: u32>(
let num_col_pairs = num_blocks.div_ceil(2);
let kh0 = kh0.unwrap_or(std::ptr::null());
- let h0_idx = h0_idx.unwrap_or(std::ptr::null());
-
for batch in 0..batch_size {
- let h0_slot = if use_h0 {
- unsafe { *h0_idx.add(batch) }
- } else {
- -1
- };
-
for hv in 0..value_heads {
let batch_value_head_idx = batch * value_heads + hv;
@@ -68,7 +59,7 @@ pub fn tree_update_solve<T: ArrayElement + Float, const BV: u32>(
let v_idx = (((batch * tree_size + token) * value_heads + hv) * head_v_dim) + dv;
let v_val = unsafe { (*v.add(v_idx)).to_f32().unwrap() };
- let kh0_val = if h0_slot >= 0 {
+ let kh0_val = if use_h0 {
unsafe { *kh0.add(v_idx) }
} else {
0.0
diff --git a/crates/uzu-engine/src/backends/metal/kernel/gdn/tree_verify.rs b/crates/uzu-engine/src/backends/metal/kernel/gdn/tree_verify.rs
index 34807d306..62b7d8c4f 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/gdn/tree_verify.rs
+++ b/crates/uzu-engine/src/backends/metal/kernel/gdn/tree_verify.rs
@@ -99,9 +99,6 @@ impl DeltaNetTreeVerify for MetalDeltaNetTreeVerify {
command_buffer: &mut MetalCommandBufferEncoding,
) -> Result<<Metal as Backend>::ScratchBuffer, MetalError> {
let layout = Layout::new(arguments.tree_size, &self.arguments);
- let mut h0_indices = command_buffer.allocate_scratch(DataType::I32.size_in_bytes())?;
- command_buffer.encode_fill(&mut h0_indices, 0);
-
let mut prefix =
command_buffer.allocate_scratch_for_shape(&[layout.tree_size, layout.num_v_heads], INNER_DATA_TYPE)?;
let mut a_packed = command_buffer.allocate_scratch_for_shape(&layout.a_packed_shape(), INNER_DATA_TYPE)?;
@@ -133,7 +130,6 @@ impl DeltaNetTreeVerify for MetalDeltaNetTreeVerify {
&prefix,
arguments.beta,
Some(arguments.h0),
- Some(&h0_indices),
&mut a_packed,
&mut qkd,
&mut a_inverse,
@@ -154,7 +150,6 @@ impl DeltaNetTreeVerify for MetalDeltaNetTreeVerify {
arguments.beta,
&a_packed,
&a_inverse,
- Some(&h0_indices),
&mut u,
1,
arguments.tree_size,
@@ -168,7 +163,6 @@ impl DeltaNetTreeVerify for MetalDeltaNetTreeVerify {
&qkd,
&u,
Some(arguments.h0),
- Some(&h0_indices),
&mut output,
1.0,
1,
diff --git a/crates/uzu-engine/src/backends/metal/kernel/gdn/tree_verify/out.metal b/crates/uzu-engine/src/backends/metal/kernel/gdn/tree_verify/out.metal
index 50022cee9..e00132f9d 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/gdn/tree_verify/out.metal
+++ b/crates/uzu-engine/src/backends/metal/kernel/gdn/tree_verify/out.metal
@@ -20,7 +20,7 @@ using namespace uzu::matmul;
// prefix: [B, T, HV] path log-decay prefix from BuildTreePrefix
// qkd: [B, HV, T, T] from BuildTreeGram
// u: [B, HV, T, V]
-// h0: [H0, HV, V, K]
+// h0: [B, HV, V, K]
//
// o[row, value] = exp(prefix[row]) * scale * dot(q[row], h0[value])
// + sum_j qkd[row, j] * u[j, value]
@@ -36,7 +36,6 @@ PUBLIC KERNEL(BuildTreeOut)(
const device float* qkd,
const device float* u,
const device float* h0 OPTIONAL(use_h0),
- const device int* h0_indices OPTIONAL(use_h0),
device OutputT* o,
constant const float& scale,
constant const uint& batch_size,
@@ -83,10 +82,9 @@ PUBLIC KERNEL(BuildTreeOut)(
AccFragment acc;
acc.clear();
- const int h0_index = use_h0 ? h0_indices[batch_idx] : -1;
- if (use_h0 && h0_index >= 0) {
+ if (use_h0) {
const uint q_base = (batch_idx * tree_size * qk_heads + row_base * qk_heads + qk_head_idx) * head_k_dim;
- const uint h0_base = ((uint(h0_index) * value_heads + value_head_idx) * head_v_dim + value_base) * head_k_dim;
+ const uint h0_base = ((batch_idx * value_heads + value_head_idx) * head_v_dim + value_base) * head_k_dim;
for (uint k0 = 0; k0 < head_k_dim; k0 += ROWS) {
const short valid_k = short(min(uint(ROWS), head_k_dim - k0));
QFragment q_frag;
diff --git a/crates/uzu-engine/src/backends/metal/kernel/gdn/tree_verify/tree_gram.metal b/crates/uzu-engine/src/backends/metal/kernel/gdn/tree_verify/tree_gram.metal
index 54c28d4a4..bef34b39f 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/gdn/tree_verify/tree_gram.metal
+++ b/crates/uzu-engine/src/backends/metal/kernel/gdn/tree_verify/tree_gram.metal
@@ -18,13 +18,12 @@ using namespace uzu::trie;
#define NUM_SIMDGROUPS 2u
#define INVALID_ROW 0xffffffffu
-// kh0 = k @ h0[h0_idx[batch]]^T for one row-tile, walking head_v_dim in COL_TILE
-// chunks from dv_start by dv_stride. No-op for a zero initial state.
+// kh0 = k @ h0[batch]^T for one row-tile, walking head_v_dim in COL_TILE
+// chunks from dv_start by dv_stride.
template <typename T, typename Ops, typename AccFragment, typename LeftFragment, typename RightFragment>
METAL_FUNC void tree_kh0(
const device T* k_rows,
const device float* h0,
- const device int* h0_idx,
device float* kh0,
const uint batch_idx,
const uint value_head_idx,
@@ -40,11 +39,7 @@ METAL_FUNC void tree_kh0(
const uint dv_stride,
const ushort lane
) {
- const int h0_slot = h0_idx[batch_idx];
- if (h0_slot < 0) {
- return;
- }
- const device float* h0_head = h0 + (uint(h0_slot) * value_heads + value_head_idx) * head_v_dim * head_k_dim;
+ const device float* h0_head = h0 + (batch_idx * value_heads + value_head_idx) * head_v_dim * head_k_dim;
device float* kh0_rows =
kh0 + (batch_idx * tree_size + row_base) * value_heads * head_v_dim + value_head_idx * head_v_dim;
const uint kh0_row_stride = value_heads * head_v_dim;
@@ -85,8 +80,7 @@ METAL_FUNC void tree_kh0(
// qkd: scale * exp(prefix[row] - prefix[col]) * dot(q[row], k[col]) for
// ancestor-or-self, dense [B*HV, T, T] f32
// a_inv: (I + A)^-1 per diagonal block, compact [B*HV, NB, 16, 16]
-// kh0: k @ h0[h0_idx[batch]]^T, [B, T, HV, head_v_dim] f32; skipped when
-// h0_idx[batch] < 0
+// kh0: k @ h0[batch]^T, [B, T, HV, head_v_dim] f32
template <typename T, bool USE_MXU>
VARIANTS(T, float, bfloat)
VARIANTS(USE_MXU, false, true)
@@ -97,7 +91,6 @@ PUBLIC KERNEL(BuildTreeGram)(
const device float* prefix,
const device float* beta,
const device float* h0 OPTIONAL(use_h0),
- const device int* h0_idx OPTIONAL(use_h0),
device float* a_packed,
device float* qkd,
device float* a_inv,
@@ -279,7 +272,6 @@ PUBLIC KERNEL(BuildTreeGram)(
tree_kh0<T, Ops, AccFragment, LeftFragment, RightFragment>(
k_rows,
h0,
- h0_idx,
kh0,
batch_idx,
value_head_idx,
diff --git a/crates/uzu-engine/src/backends/metal/kernel/gdn/tree_verify/tree_update_solve.metal b/crates/uzu-engine/src/backends/metal/kernel/gdn/tree_verify/tree_update_solve.metal
index a3c98aa93..cd84bc13a 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/gdn/tree_verify/tree_update_solve.metal
+++ b/crates/uzu-engine/src/backends/metal/kernel/gdn/tree_verify/tree_update_solve.metal
@@ -18,7 +18,6 @@ using namespace uzu::matmul;
// A [B*HV, NB, ceil(NB/2), 16, 32] f32 packed pair tiles from BuildTreeGram
// (block (i,j) at pair j/2, column (j%2)*16)
// Ainv [B*HV, NB, 16, 16] f32 diagonal-block inverses from BuildTreeGram
-// h0_idx [B] i32, negative = zero initial state (absent when !use_h0)
// U [B*HV, T, head_v_dim] f32 (output)
//
// Grid: one simdgroup per (batch, value head, BV value-dim tile); the history is
@@ -36,7 +35,6 @@ PUBLIC KERNEL(TreeUpdateSolve)(
const device float* beta,
const device float* a_packed,
const device float* a_inv,
- const device int* h0_idx OPTIONAL(use_h0),
device float* u,
constant const uint& batch_size,
constant const uint& tree_size,
@@ -66,7 +64,6 @@ PUBLIC KERNEL(TreeUpdateSolve)(
const uint value_dim_base = value_tile_idx * BV;
const uint tile_value_cols = min(BV, head_v_dim - value_dim_base);
const uint num_blocks = div_ceil(tree_size, BT);
- const int h0_slot = use_h0 ? h0_idx[batch_idx] : -1;
const uint value_token_stride = value_heads * head_v_dim;
const uint scalar_token_stride = value_heads;
@@ -88,7 +85,7 @@ PUBLIC KERNEL(TreeUpdateSolve)(
TileFragment acc;
- if (h0_slot >= 0) {
+ if (use_h0) {
const device float* kh0_block = kh0_head_tile + row_base * value_token_stride;
acc.load_maybe_bounded(
lane_idx,
diff --git a/crates/uzu-engine/unit/backends/common/kernel/gdn/tree_verify/out_bench.rs b/crates/uzu-engine/unit/backends/common/kernel/gdn/tree_verify/out_bench.rs
index e21376da3..52ff22aec 100644
--- a/crates/uzu-engine/unit/backends/common/kernel/gdn/tree_verify/out_bench.rs
+++ b/crates/uzu-engine/unit/backends/common/kernel/gdn/tree_verify/out_bench.rs
@@ -41,7 +41,6 @@ struct TreeOutBuffers {
qkd: <Metal as Backend>::GlobalBuffer,
u: <Metal as Backend>::GlobalBuffer,
h0: <Metal as Backend>::GlobalBuffer,
- h0_indices: <Metal as Backend>::GlobalBuffer,
o: <Metal as Backend>::GlobalBuffer,
}
@@ -62,7 +61,6 @@ fn make_buffers<T: ArrayElement + Float>(
let qkd = (0..qkd_len).map(|i| ((i as f32 * 0.013).cos() * 0.03) - 0.01).collect::<Vec<_>>();
let u = (0..uv_len).map(|i| (i as f32 * 0.011).sin() * 0.3 + 0.04).collect::<Vec<_>>();
let h0 = (0..h0_len).map(|i| ((i as f32 * 0.019).cos() * 0.2) - 0.01).collect::<Vec<_>>();
- let h0_indices = (0..batch_size as i32).collect::<Vec<_>>();
TreeOutBuffers {
q: create_buffer_with_data::<Metal, T>(context, &q),
@@ -70,7 +68,6 @@ fn make_buffers<T: ArrayElement + Float>(
qkd: create_buffer_with_data::<Metal, f32>(context, &qkd),
u: create_buffer_with_data::<Metal, f32>(context, &u),
h0: create_buffer_with_data::<Metal, f32>(context, &h0),
- h0_indices: create_buffer_with_data::<Metal, i32>(context, &h0_indices),
o: create_buffer::<Metal, T>(context, uv_len),
}
}
@@ -122,7 +119,6 @@ fn bench_build_tree_out_type<T: ArrayElement + Float>(
&buffers.qkd,
&buffers.u,
Some(&buffers.h0),
- Some(&buffers.h0_indices),
&mut buffers.o,
(HEAD_K_DIM as f32).sqrt().recip(),
batch_size as u32,
diff --git a/crates/uzu-engine/unit/backends/common/kernel/gdn/tree_verify/out_test.rs b/crates/uzu-engine/unit/backends/common/kernel/gdn/tree_verify/out_test.rs
index 195c4e117..698ab0b97 100644
--- a/crates/uzu-engine/unit/backends/common/kernel/gdn/tree_verify/out_test.rs
+++ b/crates/uzu-engine/unit/backends/common/kernel/gdn/tree_verify/out_test.rs
@@ -36,7 +36,6 @@ struct Inputs<T> {
qkd: Vec<f32>,
u: Vec<f32>,
h0: Vec<f32>,
- h0_indices: Vec<i32>,
}
const BUILD_TREE_OUT_PATHS: &[(&str, bool, bool)] = &[
@@ -50,8 +49,7 @@ fn make_inputs<T: ArrayElement + Float>(shape: Shape) -> Inputs<T> {
let prefix_len = shape.batch_size * shape.tree_size * shape.value_heads;
let qkd_len = shape.batch_size * shape.value_heads * shape.tree_size * shape.tree_size;
let u_len = shape.batch_size * shape.value_heads * shape.tree_size * shape.head_v_dim;
- let h0_pool = shape.batch_size + 1;
- let h0_len = h0_pool * shape.value_heads * shape.head_v_dim * shape.head_k_dim;
+ let h0_len = shape.batch_size * shape.value_heads * shape.head_v_dim * shape.head_k_dim;
Inputs {
q: (0..q_len).map(|i| T::from(((i as f32 * 0.017).sin() * 0.2) + 0.01).unwrap()).collect(),
@@ -61,15 +59,6 @@ fn make_inputs<T: ArrayElement + Float>(shape: Shape) -> Inputs<T> {
qkd: (0..qkd_len).map(|i| ((i as f32 * 0.013).cos() * 0.1) - 0.02).collect(),
u: (0..u_len).map(|i| ((i as f32 * 0.011).sin() * 0.3) + 0.04).collect(),
h0: (0..h0_len).map(|i| ((i as f32 * 0.019).cos() * 0.2) - 0.01).collect(),
- h0_indices: (0..shape.batch_size)
- .map(|i| {
- if i + 1 == shape.batch_size {
- -1
- } else {
- i as i32
- }
- })
- .collect(),
}
}
@@ -95,7 +84,6 @@ fn run_build_tree_out<B: Backend, T: ArrayElement + Float>(
let qkd = create_buffer_with_data::<B, f32>(&context, &inputs.qkd);
let u = create_buffer_with_data::<B, f32>(&context, &inputs.u);
let h0 = use_h0.then(|| create_buffer_with_data::<B, f32>(&context, &inputs.h0));
- let h0_indices = use_h0.then(|| create_buffer_with_data::<B, i32>(&context, &inputs.h0_indices));
let mut o =
create_buffer::<B, T>(&context, shape.batch_size * shape.tree_size * shape.value_heads * shape.head_v_dim);
@@ -106,7 +94,6 @@ fn run_build_tree_out<B: Backend, T: ArrayElement + Float>(
&qkd,
&u,
h0.as_ref(),
- h0_indices.as_ref(),
&mut o,
(shape.head_k_dim as f32).sqrt().recip(),
shape.batch_size as u32,
diff --git a/crates/uzu-engine/unit/backends/common/kernel/gdn/tree_verify/tree_gram_bench.rs b/crates/uzu-engine/unit/backends/common/kernel/gdn/tree_verify/tree_gram_bench.rs
index 577d8d1f7..733a74811 100644
--- a/crates/uzu-engine/unit/backends/common/kernel/gdn/tree_verify/tree_gram_bench.rs
+++ b/crates/uzu-engine/unit/backends/common/kernel/gdn/tree_verify/tree_gram_bench.rs
@@ -33,7 +33,6 @@ struct TreeGramBuffers {
prefix: <Metal as Backend>::GlobalBuffer,
beta: <Metal as Backend>::GlobalBuffer,
h0: <Metal as Backend>::GlobalBuffer,
- h0_idx: <Metal as Backend>::GlobalBuffer,
a_packed: <Metal as Backend>::GlobalBuffer,
qkd: <Metal as Backend>::GlobalBuffer,
a_inv: <Metal as Backend>::GlobalBuffer,
@@ -105,7 +104,6 @@ fn make_buffers(
.collect::<Vec<_>>();
let beta = (0..scalar_len).map(|i| 0.25 + ((i as f32 * 0.013).sin() + 1.0) * 0.2).collect::<Vec<_>>();
let h0 = (0..h0_len).map(|i| ((i as f32 * 0.007).sin() * 0.05) - 0.01).collect::<Vec<_>>();
- let h0_idx = (0..batch_size).map(|i| i as i32).collect::<Vec<_>>();
let scale = (HEAD_K_DIM as f32).sqrt().recip();
(
TreeGramBuffers {
@@ -115,7 +113,6 @@ fn make_buffers(
prefix: create_buffer_with_data::<Metal, f32>(context, &prefix),
beta: create_buffer_with_data::<Metal, f32>(context, &beta),
h0: create_buffer_with_data::<Metal, f32>(context, &h0),
- h0_idx: create_buffer_with_data::<Metal, i32>(context, &h0_idx),
a_packed: create_buffer::<Metal, f32>(context, a_len),
qkd: create_buffer::<Metal, f32>(context, out_len),
a_inv: create_buffer::<Metal, f32>(context, a_inv_len),
@@ -142,7 +139,6 @@ fn buffers_bytes(
+ batch_size * tree_size * 3 * size_of::<u32>()
+ scalar_len * size_of::<f32>() * 2
+ (out_len + a_len + a_inv_len + kh0_len) * size_of::<f32>()
- + batch_size * size_of::<i32>()
}
#[uzu_bench]
@@ -185,7 +181,6 @@ fn bench_build_tree_gram(c: &mut Criterion) {
&buffers.prefix,
&buffers.beta,
Some(&buffers.h0),
- Some(&buffers.h0_idx),
&mut buffers.a_packed,
&mut buffers.qkd,
&mut buffers.a_inv,
diff --git a/crates/uzu-engine/unit/backends/common/kernel/gdn/tree_verify/tree_gram_test.rs b/crates/uzu-engine/unit/backends/common/kernel/gdn/tree_verify/tree_gram_test.rs
index b38e7602c..502c5a5da 100644
--- a/crates/uzu-engine/unit/backends/common/kernel/gdn/tree_verify/tree_gram_test.rs
+++ b/crates/uzu-engine/unit/backends/common/kernel/gdn/tree_verify/tree_gram_test.rs
@@ -35,7 +35,6 @@ struct Inputs {
prefix: Vec<f32>,
beta: Vec<f32>,
h0: Vec<f32>,
- h0_idx: Vec<i32>,
}
fn make_inputs(tree_size: usize) -> Inputs {
@@ -51,8 +50,6 @@ fn make_inputs(tree_size: usize) -> Inputs {
.collect(),
beta: (0..scalar_len).map(|i| 0.25 + ((i as f32 * 0.013).sin() + 1.0) * 0.2).collect(),
h0: (0..h0_len).map(|i| ((i as f32 * 0.007).sin() * 0.05) - 0.01).collect(),
- // Batch 1 has no initial state: covers the kh0-skip path.
- h0_idx: vec![0, -1],
}
}
@@ -63,7 +60,6 @@ fn get_output<B: Backend, T: ArrayElement + Float>(
prefix: &[f32],
beta: &[f32],
h0: &[f32],
- h0_idx: &[i32],
tree_size: usize,
scale: f32,
use_mxu: bool,
@@ -84,15 +80,12 @@ fn get_output<B: Backend, T: ArrayElement + Float>(
let prefix = create_buffer_with_data::<B, f32>(&context, prefix);
let beta = create_buffer_with_data::<B, f32>(&context, beta);
let h0 = create_buffer_with_data::<B, f32>(&context, h0);
- let h0_idx = create_buffer_with_data::<B, i32>(&context, h0_idx);
// The GPU only writes packed-A tiles touching the block lower triangle;
// zero-init so CPU and GPU buffers stay comparable elsewhere.
let mut a_packed = create_buffer_with_data::<B, f32>(&context, &vec![0.0f32; a_len]);
let mut qkd = create_buffer::<B, f32>(&context, output_len);
let mut a_inv = create_buffer::<B, f32>(&context, a_inv_len);
- // kh0 is only written for batches with h0_idx >= 0; zero-init so CPU and GPU
- // outputs stay comparable on the skipped batch.
- let mut kh0 = create_buffer_with_data::<B, f32>(&context, &vec![0.0f32; kh0_len]);
+ let mut kh0 = create_buffer::<B, f32>(&context, kh0_len);
let mut command_buffer = context.create_command_buffer(None, None).expect("Failed to create command buffer");
kernel.encode(
@@ -102,7 +95,6 @@ fn get_output<B: Backend, T: ArrayElement + Float>(
&prefix,
&beta,
Some(&h0),
- Some(&h0_idx),
&mut a_packed,
&mut qkd,
&mut a_inv,
@@ -128,32 +120,12 @@ fn check_type<T: ArrayElement + Float>(
) {
let q = inputs.q.iter().map(|&value| T::from(value).unwrap()).collect::<Vec<_>>();
let k = inputs.k.iter().map(|&value| T::from(value).unwrap()).collect::<Vec<_>>();
- let expected = get_output::<Cpu, T>(
- &q,
- &k,
- &inputs.trie,
- &inputs.prefix,
- &inputs.beta,
- &inputs.h0,
- &inputs.h0_idx,
- tree_size,
- scale,
- false,
- );
+ let expected =
+ get_output::<Cpu, T>(&q, &k, &inputs.trie, &inputs.prefix, &inputs.beta, &inputs.h0, tree_size, scale, false);
for_each_non_cpu_backend!(|B| {
- let actual = get_output::<B, T>(
- &q,
- &k,
- &inputs.trie,
- &inputs.prefix,
- &inputs.beta,
- &inputs.h0,
- &inputs.h0_idx,
- tree_size,
- scale,
- false,
- );
+ let actual =
+ get_output::<B, T>(&q, &k, &inputs.trie, &inputs.prefix, &inputs.beta, &inputs.h0, tree_size, scale, false);
let msg = format!("backend {} simdgroup tree_size {tree_size}", std::any::type_name::<B>());
assert_eq_float::<f32>(&expected.0, &actual.0, 2e-4, &format!("a_packed {msg}"));
assert_eq_float::<f32>(&expected.1, &actual.1, 2e-4, &format!("qkd {msg}"));
@@ -169,7 +141,6 @@ fn check_type<T: ArrayElement + Float>(
&inputs.prefix,
&inputs.beta,
&inputs.h0,
- &inputs.h0_idx,
tree_size,
scale,
true,
diff --git a/crates/uzu-engine/unit/backends/common/kernel/gdn/tree_verify/tree_update_solve_bench.rs b/crates/uzu-engine/unit/backends/common/kernel/gdn/tree_verify/tree_update_solve_bench.rs
index 9b3c14c8f..450ead2b8 100644
--- a/crates/uzu-engine/unit/backends/common/kernel/gdn/tree_verify/tree_update_solve_bench.rs
+++ b/crates/uzu-engine/unit/backends/common/kernel/gdn/tree_verify/tree_update_solve_bench.rs
@@ -59,7 +59,6 @@ struct TreeUpdateSolveBuffers {
beta: <Metal as Backend>::GlobalBuffer,
a: <Metal as Backend>::GlobalBuffer,
a_inv: <Metal as Backend>::GlobalBuffer,
- h0_idx: <Metal as Backend>::GlobalBuffer,
u: <Metal as Backend>::GlobalBuffer,
}
@@ -139,8 +138,6 @@ fn make_buffers(
inverse_value(batch_head_matrix_idx, block, row, col, tree_size)
})
.collect::<Vec<_>>();
- let h0_idx = (0..batch_size).map(|i| i as i32).collect::<Vec<_>>();
-
TreeUpdateSolveBuffers {
kh0: create_buffer_with_data::<Metal, f32>(context, &kh0),
v: create_buffer_with_data::<Metal, bf16>(context, &v),
@@ -148,7 +145,6 @@ fn make_buffers(
beta: create_buffer_with_data::<Metal, f32>(context, &beta),
a,
a_inv: create_buffer_with_data::<Metal, f32>(context, &a_inv),
- h0_idx: create_buffer_with_data::<Metal, i32>(context, &h0_idx),
u: create_buffer::<Metal, f32>(context, u_len),
}
}
@@ -164,9 +160,7 @@ fn buffers_bytes(
let u_len = batch_size * NUM_V_HEADS * tree_size * HEAD_V_DIM;
let inv_len = batch_size * NUM_V_HEADS * num_blocks * BT * BT;
- v_len * size_of::<bf16>()
- + (scalar_len * 2 + a_len + inv_len + u_len + v_len) * size_of::<f32>()
- + batch_size * size_of::<i32>()
+ v_len * size_of::<bf16>() + (scalar_len * 2 + a_len + inv_len + u_len + v_len) * size_of::<f32>()
}
#[uzu_bench]
@@ -220,7 +214,6 @@ fn encode(
&buffers.beta,
&buffers.a,
&buffers.a_inv,
- Some(&buffers.h0_idx),
&mut buffers.u,
batch_size as u32,
tree_size as u32,
diff --git a/crates/uzu-engine/unit/backends/common/kernel/gdn/tree_verify/tree_update_solve_test.rs b/crates/uzu-engine/unit/backends/common/kernel/gdn/tree_verify/tree_update_solve_test.rs
index c340afd7a..ffde218d7 100644
--- a/crates/uzu-engine/unit/backends/common/kernel/gdn/tree_verify/tree_update_solve_test.rs
+++ b/crates/uzu-engine/unit/backends/common/kernel/gdn/tree_verify/tree_update_solve_test.rs
@@ -78,7 +78,7 @@ const CASES: &[SolveCase] = &[
head_v_dim: 64,
},
SolveCase {
- name: "batched_optional_h0",
+ name: "batched",
batch_size: 2,
tree_size: 16,
num_v_heads: 2,
@@ -149,19 +149,12 @@ fn run_case<B: Backend, T: ArrayElement + Copy>(
}
})
.collect::<Vec<_>>();
- let h0_idx = if case.name == "batched_optional_h0" {
- vec![0, -1]
- } else {
- (0..batch_size).map(|batch| batch as i32).collect::<Vec<_>>()
- };
-
let kh0 = create_buffer_with_data::<B, f32>(&context, &kh0);
let v = create_buffer_with_data::<B, T>(&context, &v);
let prefix = create_buffer_with_data::<B, f32>(&context, &prefix);
let beta = create_buffer_with_data::<B, f32>(&context, &beta);
let a = create_buffer_with_data::<B, f32>(&context, &a_f32);
let a_inv = create_buffer_with_data::<B, f32>(&context, &a_inv);
- let h0_idx = create_buffer_with_data::<B, i32>(&context, &h0_idx);
let mut u = create_buffer::<B, f32>(&context, u_len);
let mut command_buffer = context.create_command_buffer(None, None).expect("command buffer");
@@ -172,7 +165,6 @@ fn run_case<B: Backend, T: ArrayElement + Copy>(
&beta,
&a,
&a_inv,
- use_h0.then_some(&h0_idx),
&mut u,
case.batch_size,
case.tree_size,