Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

Weaver speedup(10%) + cleanup (#658)

CCYeh780d4901cbdb
diff --git a/crates/backend-uzu/src/backends/cpu/kernel/attention/ancestor_attention.rs b/crates/backend-uzu/src/backends/cpu/kernel/attention/ancestor_attention.rs
index 0522ecf47..b8d5fc7f2 100644
--- a/crates/backend-uzu/src/backends/cpu/kernel/attention/ancestor_attention.rs
+++ b/crates/backend-uzu/src/backends/cpu/kernel/attention/ancestor_attention.rs
@@ -7,10 +7,11 @@ use super::attention_single_pass::attention_single_pass;
#[variants(HEAD_DIM, 128)]
fn ancestor_attention<const HEAD_DIM: u32>(
prefix_kv: *const bf16,
- node_kv: *const bf16,
+ node_kv: *mut bf16,
current_qkv: *const bf16,
ancestor_indices: *const u32,
ancestor_counts: *const u32,
+ node_indices: *const u32,
output: *mut bf16,
rows: u32,
prefix_length: u32,
@@ -91,6 +92,19 @@ fn ancestor_attention<const HEAD_DIM: u32>(
false,
false,
);
+ if node_capacity > 0 {
+ let node = (*node_indices.add(row) as usize).min(last_node);
+ std::ptr::copy_nonoverlapping(
+ current_row.add(KEY_COMPONENT * model_dim),
+ node_kv.add(node * model_dim),
+ model_dim,
+ );
+ std::ptr::copy_nonoverlapping(
+ current_row.add(VALUE_COMPONENT * model_dim),
+ node_kv.add(node_capacity * model_dim + node * model_dim),
+ model_dim,
+ );
+ }
}
}
}
diff --git a/crates/backend-uzu/src/backends/cpu/kernel/weaver/mod.rs b/crates/backend-uzu/src/backends/cpu/kernel/weaver/mod.rs
index f7c44e340..a199938d1 100644
--- a/crates/backend-uzu/src/backends/cpu/kernel/weaver/mod.rs
+++ b/crates/backend-uzu/src/backends/cpu/kernel/weaver/mod.rs
@@ -1,4 +1,3 @@
-pub mod weaver_frontier_scatter;
+pub mod weaver_frontier_insert_children;
pub mod weaver_frontier_select;
-pub mod weaver_node_cache_write;
pub mod weaver_top_children;
diff --git a/crates/backend-uzu/src/backends/cpu/kernel/weaver/weaver_frontier_scatter.rs b/crates/backend-uzu/src/backends/cpu/kernel/weaver/weaver_frontier_insert_children.rs
similarity index 97%
rename from crates/backend-uzu/src/backends/cpu/kernel/weaver/weaver_frontier_scatter.rs
rename to crates/backend-uzu/src/backends/cpu/kernel/weaver/weaver_frontier_insert_children.rs
index c8f94c7f3..71f6647c8 100644
--- a/crates/backend-uzu/src/backends/cpu/kernel/weaver/weaver_frontier_scatter.rs
+++ b/crates/backend-uzu/src/backends/cpu/kernel/weaver/weaver_frontier_insert_children.rs
@@ -13,8 +13,8 @@ fn top_k_score_key(score: f32) -> u32 {
}
}
-#[kernel(WeaverFrontierScatter)]
-pub fn weaver_frontier_scatter(
+#[kernel(WeaverFrontierInsertChildren)]
+pub fn weaver_frontier_insert_children(
packed_tree: *const u32,
node_metadata: *const u32,
node_valid: *const u32,
diff --git a/crates/backend-uzu/src/backends/cpu/kernel/weaver/weaver_node_cache_write.rs b/crates/backend-uzu/src/backends/cpu/kernel/weaver/weaver_node_cache_write.rs
deleted file mode 100644
index 1f4c177d6..000000000
--- a/crates/backend-uzu/src/backends/cpu/kernel/weaver/weaver_node_cache_write.rs
+++ /dev/null
@@ -1,34 +0,0 @@
-use half::bf16;
-use num_traits::Float;
-use proc_macros::kernel;
-
-use crate::array::ArrayElement;
-
-#[kernel(WeaverNodeCacheWrite)]
-#[variants(T, f32, bf16)]
-pub fn weaver_node_cache_write<T: ArrayElement + Float>(
- current_qkv: *const T,
- node_kv: *mut T,
- node_indices: *const u32,
- model_dim: u32,
- node_capacity: u32,
- total: u32,
-) {
- if node_capacity == 0 {
- return;
- }
- let model_dim = model_dim as usize;
- let qkv_width = 3 * model_dim;
- let last_node = (node_capacity - 1) as usize;
- unsafe {
- for position in 0..total as usize {
- let row = position / (2 * model_dim);
- let offset = position - row * 2 * model_dim;
- let component = offset / model_dim;
- let component_offset = offset % model_dim;
- let node = (*node_indices.add(row) as usize).min(last_node);
- *node_kv.add(component * node_capacity as usize * model_dim + node * model_dim + component_offset) =
- *current_qkv.add(row * qkv_width + (component + 1) * model_dim + component_offset);
- }
- }
-}
diff --git a/crates/backend-uzu/src/backends/metal/kernel/attention/ancestor_attention.metal b/crates/backend-uzu/src/backends/metal/kernel/attention/ancestor_attention.metal
index e615d580f..519d6ec6d 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/attention/ancestor_attention.metal
+++ b/crates/backend-uzu/src/backends/metal/kernel/attention/ancestor_attention.metal
@@ -53,10 +53,11 @@ template <uint HEAD_DIM>
VARIANTS(HEAD_DIM, 128)
PUBLIC KERNEL(AncestorAttention)(
const device bfloat* prefix_kv,
- const device bfloat* node_kv,
+ device bfloat* node_kv,
const device bfloat* current_qkv,
const device uint* ancestor_indices,
const device uint* ancestor_counts,
+ const device uint* node_indices,
device bfloat* output,
constant uint& rows,
constant uint& prefix_length,
@@ -86,7 +87,7 @@ PUBLIC KERNEL(AncestorAttention)(
const uint prefix_value_offset = prefix_length * model_vectors + head_offset;
const uint node_value_offset = node_capacity * model_vectors + head_offset;
const device bfloat4* prefix_vectors = (const device bfloat4*)prefix_kv;
- const device bfloat4* node_vectors = (const device bfloat4*)node_kv;
+ device bfloat4* node_vectors = (device bfloat4*)node_kv;
const device bfloat4* current_row = (const device bfloat4*)current_qkv + row * qkv_vectors;
const float4 query = float4(current_row[head_offset]) * scale;
@@ -127,5 +128,12 @@ PUBLIC KERNEL(AncestorAttention)(
return current_row;
});
+ if (node_capacity > 0u) {
+ const uint node = min(node_indices[row], last_node);
+
+ node_vectors[node * model_vectors + head_offset] = current_row[current_key_offset];
+ node_vectors[node_value_offset + node * model_vectors] = current_row[current_value_offset];
+ }
+
((device bfloat4*)output)[(row * num_heads + head) * vectors_per_head + lane] = bfloat4(values / sum);
}
diff --git a/crates/backend-uzu/src/backends/metal/kernel/weaver/weaver_frontier_scatter.metal b/crates/backend-uzu/src/backends/metal/kernel/weaver/weaver_frontier_insert_children.metal
similarity index 97%
rename from crates/backend-uzu/src/backends/metal/kernel/weaver/weaver_frontier_scatter.metal
rename to crates/backend-uzu/src/backends/metal/kernel/weaver/weaver_frontier_insert_children.metal
index ee2f95c33..d79cf432e 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/weaver/weaver_frontier_scatter.metal
+++ b/crates/backend-uzu/src/backends/metal/kernel/weaver/weaver_frontier_insert_children.metal
@@ -5,7 +5,7 @@
using namespace metal;
-PUBLIC KERNEL(WeaverFrontierScatter)(
+PUBLIC KERNEL(WeaverFrontierInsertChildren)(
const device uint* packed_tree,
const device uint* node_metadata,
const device uint* node_valid,
diff --git a/crates/backend-uzu/src/backends/metal/kernel/weaver/weaver_node_cache_write.metal b/crates/backend-uzu/src/backends/metal/kernel/weaver/weaver_node_cache_write.metal
deleted file mode 100644
index a9b6c6bcf..000000000
--- a/crates/backend-uzu/src/backends/metal/kernel/weaver/weaver_node_cache_write.metal
+++ /dev/null
@@ -1,29 +0,0 @@
-#include <metal_stdlib>
-#include "../common/dsl.h"
-
-// Scatters the key/value halves of the current step's packed QKV rows into their
-// node-cache slots. Kept out of AncestorAttention so that the node arena stays
-// read-only for the whole attention dispatch.
-template <typename T>
-VARIANTS(T, float, bfloat)
-PUBLIC KERNEL(WeaverNodeCacheWrite)(
- const device T* current_qkv,
- device T* node_kv,
- const device uint* node_indices,
- constant uint& model_dim,
- constant uint& node_capacity,
- constant uint& total,
- const uint position AXIS(total, 256)
-) {
- if (node_capacity == 0) {
- return;
- }
- const uint row = position / (2 * model_dim);
- const uint offset = position - row * 2 * model_dim;
- const uint component = offset / model_dim;
- const uint component_offset = offset % model_dim;
- const uint qkv_width = 3 * model_dim;
- const uint node = min(node_indices[row], node_capacity - 1u);
- node_kv[component * node_capacity * model_dim + node * model_dim + component_offset] =
- current_qkv[row * qkv_width + (component + 1) * model_dim + component_offset];
-}
diff --git a/crates/backend-uzu/src/encodable_block/weaver.rs b/crates/backend-uzu/src/encodable_block/weaver.rs
index 4ee117304..b0f928c55 100644
--- a/crates/backend-uzu/src/encodable_block/weaver.rs
+++ b/crates/backend-uzu/src/encodable_block/weaver.rs
@@ -3,14 +3,13 @@ use thiserror::Error;
use crate::{
array::size_for_shape,
backends::common::{
- Allocation, AllocationType, AsBufferRangeMut, Backend, Context, Encoder, Kernels,
+ Allocation, AsBufferRangeMut, Backend, Encoder, Kernels,
gpu_types::weaver::{
CANDIDATES_MAX, FRONTIER_MAX_SLOTS, FRONTIER_NO_WINNER, FrontierIdx, MetadataIdx, TreeIdx,
},
kernel::{
ActivationKernel, AncestorAttentionKernel, AttentionPrepareKernel, TensorAddBiasKernel,
- WeaverFrontierScatterKernel, WeaverFrontierSelectKernel, WeaverNodeCacheWriteKernel,
- WeaverTopChildrenKernel,
+ WeaverFrontierInsertChildrenKernel, WeaverFrontierSelectKernel, WeaverTopChildrenKernel,
},
},
config::{normalization::NormalizationConfig, weaver::WeaverConfig},
@@ -120,7 +119,7 @@ pub struct Weaver<B: Backend> {
node_position_add: <B::Kernels as Kernels>::TensorAddBiasKernel,
top_children: <B::Kernels as Kernels>::WeaverTopChildrenKernel,
frontier_select: <B::Kernels as Kernels>::WeaverFrontierSelectKernel,
- frontier_scatter: <B::Kernels as Kernels>::WeaverFrontierScatterKernel,
+ frontier_insert_children: <B::Kernels as Kernels>::WeaverFrontierInsertChildrenKernel,
model_dim: usize,
target_model_dim: usize,
max_depth: usize,
@@ -133,7 +132,6 @@ struct WeaverLayer<B: Backend> {
attention_prepare: <B::Kernels as Kernels>::AttentionPrepareKernel,
prefix_attention: AttentionCores<B>,
ancestor_attention: <B::Kernels as Kernels>::AncestorAttentionKernel,
- node_cache_write: <B::Kernels as Kernels>::WeaverNodeCacheWriteKernel,
out_projection: Box<dyn Linear<B>>,
// MLP
@@ -275,8 +273,8 @@ impl<B: Backend> Weaver<B> {
<B::Kernels as Kernels>::WeaverTopChildrenKernel::new(context).map_err(WeaverNewError::Backend)?;
let frontier_select =
<B::Kernels as Kernels>::WeaverFrontierSelectKernel::new(context).map_err(WeaverNewError::Backend)?;
- let frontier_scatter =
- <B::Kernels as Kernels>::WeaverFrontierScatterKernel::new(context).map_err(WeaverNewError::Backend)?;
+ let frontier_insert_children = <B::Kernels as Kernels>::WeaverFrontierInsertChildrenKernel::new(context)
+ .map_err(WeaverNewError::Backend)?;
Ok(Self {
token_embedding_norm,
token_embedding_projection,
@@ -290,7 +288,7 @@ impl<B: Backend> Weaver<B> {
node_position_add,
top_children,
frontier_select,
- frontier_scatter,
+ frontier_insert_children,
model_dim: config.model_dim,
target_model_dim: config.target_model_dim,
max_depth: config.max_depth,
@@ -301,7 +299,6 @@ impl<B: Backend> Weaver<B> {
&self,
inputs: WeaverInputs<'_, B>,
shape: TreeShape,
- context: &B::Context,
encoder: &mut Encoder<B>,
) -> Result<EncodedWeaverTree<B>, WeaverEncodeError<B>> {
let tree_slot_count = shape.budget + 1;
@@ -323,7 +320,7 @@ impl<B: Backend> Weaver<B> {
.encode_prefix(inputs.target_hidden, inputs.draft_hidden, lookahead_count, encoder)
.map_err(WeaverEncodeError::Backend)?;
let mut node_cache =
- self.create_node_expansion_kv_cache(tree_slot_count, context).map_err(WeaverEncodeError::Backend)?;
+ self.create_node_expansion_kv_cache(tree_slot_count, encoder).map_err(WeaverEncodeError::Backend)?;
let mut tree_init = vec![0u32; TreeIdx::COUNT * tree_slot_count];
for slot in 0..tree_slot_count {
@@ -395,6 +392,9 @@ impl<B: Backend> Weaver<B> {
encoder,
);
}
+ if batch_start_slot + batch_node_count == tree_slot_count {
+ break; // packed-tree slots filled
+ }
let (batch_candidate_ids, batch_candidate_logits) = if batch_start_slot == 0 {
(inputs.candidates.ids, inputs.candidates.logits)
} else {
@@ -413,7 +413,7 @@ impl<B: Backend> Weaver<B> {
candidates_per_node: inputs.candidates.candidates_per_depth,
depth_seeds: &depth_seeds,
};
- let selected_children = self.encode_node_expansion(
+ let top_children = self.encode_node_expansion(
&prefix_cache,
&nodes,
&candidates,
@@ -422,12 +422,12 @@ impl<B: Backend> Weaver<B> {
inputs.target_embedding,
encoder,
)?;
- self.frontier_scatter.encode(
+ self.frontier_insert_children.encode(
&packed_tree,
&node_metadata,
&node_valid,
- &selected_children.token_ids,
- &selected_children.logprobs,
+ &top_children.token_ids,
+ &top_children.logprobs,
&mut frontier,
frontier_capacity as u32,
tree_slot_count as u32,
@@ -497,13 +497,12 @@ impl<B: Backend> Weaver<B> {
fn create_node_expansion_kv_cache(
&self,
capacity: usize,
- context: &B::Context,
+ encoder: &mut Encoder<B>,
) -> Result<NodeExpansionKvCache<B>, B::Error> {
assert!(capacity > 0, "Weaver node capacity must be positive");
let kv_size = size_for_shape(&[2, capacity, self.model_dim], DATA_TYPE);
- let layers = (0..self.layers.len())
- .map(|_| context.create_allocation(kv_size, AllocationType::Global))
- .collect::<Result<Box<[_]>, _>>()?;
+ let layers =
+ (0..self.layers.len()).map(|_| encoder.allocate_scratch(kv_size)).collect::<Result<Box<[_]>, _>>()?;
Ok(NodeExpansionKvCache {
layers,
capacity: capacity as u32,
@@ -711,8 +710,6 @@ impl<B: Backend> WeaverLayer<B> {
let ancestor_attention =
<B::Kernels as Kernels>::AncestorAttentionKernel::new(context, head_dim as u32, num_heads as u32)
.map_err(WeaverNewError::Backend)?;
- let node_cache_write = <B::Kernels as Kernels>::WeaverNodeCacheWriteKernel::new(context, DATA_TYPE)
- .map_err(WeaverNewError::Backend)?;
Ok(Self {
qkv_projection,
out_projection,
@@ -724,7 +721,6 @@ impl<B: Backend> WeaverLayer<B> {
down_projection,
activation,
ancestor_attention,
- node_cache_write,
attention_scale,
model_dim,
hidden_dim,
@@ -780,9 +776,7 @@ impl<B: Backend> WeaverLayer<B> {
let mut queries =
encoder.allocate_scratch(size_for_shape(&[self.num_heads, token_count, self.head_dim], DATA_TYPE))?;
let kv_plane_bytes = size_for_shape(&[token_count, self.model_dim], DATA_TYPE);
- let mut kv_cache = encoder
- .context()
- .create_allocation(size_for_shape(&[2, token_count, self.model_dim], DATA_TYPE), AllocationType::Global)?;
+ let mut kv_cache = encoder.allocate_scratch(size_for_shape(&[2, token_count, self.model_dim], DATA_TYPE))?;
let (keys, values) = kv_cache.as_buffer_range_mut().split_at(kv_plane_bytes);
self.attention_prepare.encode(
&qkv,
@@ -826,10 +820,11 @@ impl<B: Backend> WeaverLayer<B> {
encoder.allocate_scratch(size_for_shape(&[nodes.count, self.model_dim], DATA_TYPE))?;
self.ancestor_attention.encode(
prefix_kv,
- &*node_kv_cache,
+ node_kv_cache,
&current_qkv,
nodes.ancestor_indices,
ancestor_counts,
+ tree_slot_indices,
&mut attention_output,
nodes.count as u32,
prefix_length as u32,
@@ -838,17 +833,6 @@ impl<B: Backend> WeaverLayer<B> {
self.attention_scale,
encoder,
);
- // Separate dispatch so the node arena is read-only above; the encoder
- // serializes it after the attention that reads the ancestor slots.
- self.node_cache_write.encode(
- &current_qkv,
- node_kv_cache,
- tree_slot_indices,
- self.model_dim as u32,
- node_capacity,
- (nodes.count * 2 * self.model_dim) as u32,
- encoder,
- );
self.encode_post_attention(attention_output, residual_state, nodes.count, encoder)
}
diff --git a/crates/backend-uzu/src/speculators/dflash_speculator.rs b/crates/backend-uzu/src/speculators/dflash_speculator.rs
index 3f9a9d8a3..6ec23c0ad 100644
--- a/crates/backend-uzu/src/speculators/dflash_speculator.rs
+++ b/crates/backend-uzu/src/speculators/dflash_speculator.rs
@@ -186,7 +186,6 @@ impl<B: Backend> DFlashSpeculator<B> {
frontier_width: options.frontier_width,
children_per_node: options.children_per_node,
},
- &self.context,
&mut encoder,
)?;
let completed = encoder.end_encoding().submit().wait_until_completed().map_err(DFlashTreeError::Backend)?;
diff --git a/crates/backend-uzu/tests/unit/backends/common/kernel/attention/ancestor_attention_test.rs b/crates/backend-uzu/tests/unit/backends/common/kernel/attention/ancestor_attention_test.rs
index 04356b975..d07a35363 100644
--- a/crates/backend-uzu/tests/unit/backends/common/kernel/attention/ancestor_attention_test.rs
+++ b/crates/backend-uzu/tests/unit/backends/common/kernel/attention/ancestor_attention_test.rs
@@ -28,6 +28,7 @@ struct Runner<B: Backend> {
current_qkv: Allocation<B>,
ancestor_indices: Allocation<B>,
ancestor_counts: Allocation<B>,
+ node_indices: Allocation<B>,
output: Allocation<B>,
rows: u32,
prefix_length: u32,
@@ -52,6 +53,7 @@ impl<B: Backend> Runner<B> {
.collect::<Vec<_>>()
};
let first_output_node = nodes - rows;
+ let node_indices = (first_output_node..nodes).map(|node| node as u32).collect::<Vec<_>>();
let ancestor_counts = (0..rows).map(|row| (row % (ancestor_stride + 1)) as u32).collect::<Vec<_>>();
let mut ancestor_indices = vec![0; rows * ancestor_stride];
for row in 0..rows {
@@ -71,6 +73,7 @@ impl<B: Backend> Runner<B> {
current_qkv: alloc_allocation_with_data::<B, bf16>(&context, &values(rows * qkv_width, 29)),
ancestor_indices: alloc_allocation_with_data::<B, u32>(&context, &ancestor_indices),
ancestor_counts: alloc_allocation_with_data::<B, u32>(&context, &ancestor_counts),
+ node_indices: alloc_allocation_with_data::<B, u32>(&context, &node_indices),
output: alloc_allocation::<B, bf16>(&context, rows * NUM_HEADS as usize * HEAD_DIM),
context,
kernel,
@@ -89,10 +92,11 @@ impl<B: Backend> Runner<B> {
for _ in 0..repetitions {
self.kernel.encode(
&self.prefix_kv,
- &self.node_kv,
+ &mut self.node_kv,
&self.current_qkv,
&self.ancestor_indices,
&self.ancestor_counts,
+ &self.node_indices,
&mut self.output,
self.rows,
self.prefix_length,
@@ -109,18 +113,17 @@ impl<B: Backend> Runner<B> {
#[uzu_test]
fn ancestor_attention_matches_cpu() {
let mut cpu = Runner::<Cpu>::new(4, 5, 3, 16);
- let initial_nodes = allocation_to_vec::<Cpu, bf16>(&cpu.node_kv);
cpu.encode(1);
let expected_output = allocation_to_vec::<Cpu, bf16>(&cpu.output);
-
- assert_eq!(allocation_to_vec::<Cpu, bf16>(&cpu.node_kv), initial_nodes);
+ let expected_node_kv = allocation_to_vec::<Cpu, bf16>(&cpu.node_kv);
let mut metal = Runner::<Metal>::new(4, 5, 3, 16);
metal.encode(1);
let actual_output = allocation_to_vec::<Metal, bf16>(&metal.output);
+ let actual_node_kv = allocation_to_vec::<Metal, bf16>(&metal.node_kv);
assert_eq_float(&expected_output, &actual_output, 0.02, "AncestorAttention output");
- assert_eq!(allocation_to_vec::<Metal, bf16>(&metal.node_kv), initial_nodes);
+ assert_eq!(actual_node_kv, expected_node_kv);
}
#[uzu_test]
diff --git a/crates/backend-uzu/tests/unit/backends/common/kernel/weaver/mod.rs b/crates/backend-uzu/tests/unit/backends/common/kernel/weaver/mod.rs
index 52d3b29e3..2fb0afeb7 100644
--- a/crates/backend-uzu/tests/unit/backends/common/kernel/weaver/mod.rs
+++ b/crates/backend-uzu/tests/unit/backends/common/kernel/weaver/mod.rs
@@ -1,3 +1,2 @@
mod weaver_frontier_test;
-mod weaver_node_cache_write_test;
mod weaver_top_children_test;
diff --git a/crates/backend-uzu/tests/unit/backends/common/kernel/weaver/weaver_frontier_test.rs b/crates/backend-uzu/tests/unit/backends/common/kernel/weaver/weaver_frontier_test.rs
index 949292fd5..bce06ae78 100644
--- a/crates/backend-uzu/tests/unit/backends/common/kernel/weaver/weaver_frontier_test.rs
+++ b/crates/backend-uzu/tests/unit/backends/common/kernel/weaver/weaver_frontier_test.rs
@@ -5,7 +5,7 @@ use crate::{
common::{
Backend, Encoder, Kernels,
gpu_types::weaver::{FrontierIdx, MetadataIdx, TreeIdx},
- kernel::{WeaverFrontierScatterKernel, WeaverFrontierSelectKernel},
+ kernel::{WeaverFrontierInsertChildrenKernel, WeaverFrontierSelectKernel},
},
cpu::Cpu,
},
@@ -77,7 +77,7 @@ fn select<B: Backend>() -> Vec<u32> {
.collect()
}
-fn scatter<B: Backend>() -> Vec<u32> {
+fn insert_children<B: Backend>() -> Vec<u32> {
let context = create_context::<B>();
let mut tree = vec![0; TreeIdx::COUNT * 4];
tree[TreeIdx::PathLogprobBits as usize * 4..(TreeIdx::PathLogprobBits as usize + 1) * 4]
@@ -91,7 +91,7 @@ fn scatter<B: Backend>() -> Vec<u32> {
let ids = alloc_allocation_with_data::<B, u32>(&context, &(10..19).collect::<Vec<_>>());
let scores = alloc_allocation_with_data::<B, f32>(&context, &[-0.1, -0.2, -0.3, 8.0, 8.0, 8.0, 0.1, 0.2, 0.3]);
let mut frontier = alloc_allocation_with_data::<B, u32>(&context, &[42; FrontierIdx::COUNT * 16]);
- let kernel = <B::Kernels as Kernels>::WeaverFrontierScatterKernel::new(&context).unwrap();
+ let kernel = <B::Kernels as Kernels>::WeaverFrontierInsertChildrenKernel::new(&context).unwrap();
let mut encoder = Encoder::new(context.as_ref()).unwrap();
kernel.encode(&tree, &metadata, &valid, &ids, &scores, &mut frontier, 16, 4, 3, 3, &mut encoder);
encoder.end_encoding().submit().wait_until_completed().unwrap();
@@ -102,6 +102,6 @@ fn scatter<B: Backend>() -> Vec<u32> {
fn weaver_frontier_kernels_match_cpu() {
for_each_non_cpu_backend!(|B| {
assert_eq!(select::<B>(), select::<Cpu>());
- assert_eq!(scatter::<B>(), scatter::<Cpu>());
+ assert_eq!(insert_children::<B>(), insert_children::<Cpu>());
});
}
diff --git a/crates/backend-uzu/tests/unit/backends/common/kernel/weaver/weaver_node_cache_write_test.rs b/crates/backend-uzu/tests/unit/backends/common/kernel/weaver/weaver_node_cache_write_test.rs
deleted file mode 100644
index bd1260d26..000000000
--- a/crates/backend-uzu/tests/unit/backends/common/kernel/weaver/weaver_node_cache_write_test.rs
+++ /dev/null
@@ -1,98 +0,0 @@
-#![cfg(backend = "metal")]
-
-use half::bf16;
-use proc_macros::uzu_test;
-
-use crate::{
- backends::{
- common::{Allocation, Backend, Encoder, Kernels, kernel::WeaverNodeCacheWriteKernel},
- cpu::Cpu,
- metal::Metal,
- },
- data_type::DataType,
- tests::helpers::{alloc_allocation_with_data, allocation_to_vec, create_context},
-};
-
-const MODEL_DIM: usize = 64;
-const ROWS: usize = 4;
-const NODES: usize = 9;
-
-struct Runner<B: Backend> {
- context: std::sync::Arc<B::Context>,
- kernel: <B::Kernels as Kernels>::WeaverNodeCacheWriteKernel,
- current_qkv: Allocation<B>,
- node_kv: Allocation<B>,
- node_indices: Allocation<B>,
-}
-
-fn values(
- length: usize,
- offset: usize,
-) -> Vec<bf16> {
- (0..length).map(|index| bf16::from_f32((((index + offset) * 17 % 251) as f32 - 125.0) / 128.0)).collect()
-}
-
-/// Slots are deliberately non-contiguous and out of order so the scatter is
-/// exercised rather than a coincidentally-contiguous copy.
-const NODE_INDICES: [u32; ROWS] = [7, 1, 8, 4];
-
-impl<B: Backend> Runner<B> {
- fn new() -> Self {
- let qkv_width = 3 * MODEL_DIM;
- let kv_width = 2 * MODEL_DIM;
- let context = create_context::<B>();
- let kernel =
- <B::Kernels as Kernels>::WeaverNodeCacheWriteKernel::new(context.as_ref(), DataType::BF16).unwrap();
- Self {
- current_qkv: alloc_allocation_with_data::<B, bf16>(&context, &values(ROWS * qkv_width, 29)),
- node_kv: alloc_allocation_with_data::<B, bf16>(&context, &values(NODES * kv_width, 11)),
- node_indices: alloc_allocation_with_data::<B, u32>(&context, &NODE_INDICES),
- context,
- kernel,
- }
- }
-
- fn run(&mut self) -> Vec<bf16> {
- let mut encoder = Encoder::new(self.context.as_ref()).unwrap();
- self.kernel.encode(
- &self.current_qkv,
- &mut self.node_kv,
- &self.node_indices,
- MODEL_DIM as u32,
- NODES as u32,
- (ROWS * 2 * MODEL_DIM) as u32,
- &mut encoder,
- );
- encoder.end_encoding().submit().wait_until_completed().unwrap();
- allocation_to_vec::<B, bf16>(&self.node_kv)
- }
-}
-
-#[uzu_test]
-fn weaver_node_cache_write_matches_cpu() {
- let qkv_width = 3 * MODEL_DIM;
-
- let mut cpu = Runner::<Cpu>::new();
- let initial_nodes = allocation_to_vec::<Cpu, bf16>(&cpu.node_kv);
- let current_qkv = allocation_to_vec::<Cpu, bf16>(&cpu.current_qkv);
- let expected = cpu.run();
-
- for component in 0..2 {
- for node in 0..NODES {
- let destination = component * NODES * MODEL_DIM + node * MODEL_DIM;
- match NODE_INDICES.iter().position(|&index| index as usize == node) {
- Some(row) => {
- let source = row * qkv_width + (component + 1) * MODEL_DIM;
- assert_eq!(expected[destination..destination + MODEL_DIM], current_qkv[source..source + MODEL_DIM],);
- },
- None => assert_eq!(
- expected[destination..destination + MODEL_DIM],
- initial_nodes[destination..destination + MODEL_DIM],
- ),
- }
- }
- }
-
- let mut metal = Runner::<Metal>::new();
- assert_eq!(metal.run(), expected);
-}