Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

Weaver cleanup + speedup (#637)

CCYeh50fbd82b772a
diff --git a/crates/backend-uzu/src/backends/common/buffer/arg.rs b/crates/backend-uzu/src/backends/common/buffer/arg.rs
index ad56f62b0..5c383d800 100644
--- a/crates/backend-uzu/src/backends/common/buffer/arg.rs
+++ b/crates/backend-uzu/src/backends/common/buffer/arg.rs
@@ -1,4 +1,4 @@
-use crate::backends::common::{AsBufferRangeMut, AsBufferRangeRef, Backend, Buffer};
+use crate::backends::common::{AsBufferRangeMut, AsBufferRangeRef, Backend, Buffer, BufferRangeMut};
// TODO: This whole thing is horrible and should be unified with AsBufferRangeRef
pub trait BufferArg<'a, B: Backend>: Copy {
@@ -30,6 +30,13 @@ pub trait BufferArgMut<'a, B: Backend> {
fn into_parts(self) -> (&'a dyn Buffer<Backend = B>, usize, usize);
}
+impl<'a, B: Backend, Buf: Buffer<Backend = B>> BufferArgMut<'a, B> for BufferRangeMut<'a, Buf> {
+ fn into_parts(self) -> (&'a dyn Buffer<Backend = B>, usize, usize) {
+ let (buffer, range) = (self.buffer(), self.range());
+ (buffer, range.start, range.len())
+ }
+}
+
impl<'a, B: Backend, T: AsBufferRangeMut<Buffer: Buffer<Backend = B>>> BufferArgMut<'a, B> for &'a mut T {
fn into_parts(self) -> (&'a dyn Buffer<Backend = B>, usize, usize) {
let buffer_range = self.as_buffer_range_mut();
diff --git a/crates/backend-uzu/src/backends/common/buffer/range.rs b/crates/backend-uzu/src/backends/common/buffer/range.rs
index dd0913f39..5887811b3 100644
--- a/crates/backend-uzu/src/backends/common/buffer/range.rs
+++ b/crates/backend-uzu/src/backends/common/buffer/range.rs
@@ -74,6 +74,26 @@ impl<'a, B: Buffer> BufferRangeMut<'a, B> {
}
}
+ pub fn split_at(
+ self,
+ byte_offset: usize,
+ ) -> (Self, Self) {
+ assert!(byte_offset <= self.range.len(), "split offset exceeds buffer range size");
+ let split = self.range.start + byte_offset;
+ let buffer = self.buffer;
+
+ (
+ Self {
+ buffer,
+ range: self.range.start..split,
+ },
+ Self {
+ buffer,
+ range: split..self.range.end,
+ },
+ )
+ }
+
pub fn buffer(&self) -> &'a B {
self.buffer
}
diff --git a/crates/backend-uzu/src/backends/common/gpu_types/weaver.rs b/crates/backend-uzu/src/backends/common/gpu_types/weaver.rs
index e6b058188..09d0e86cd 100644
--- a/crates/backend-uzu/src/backends/common/gpu_types/weaver.rs
+++ b/crates/backend-uzu/src/backends/common/gpu_types/weaver.rs
@@ -2,27 +2,48 @@ pub const CANDIDATES_MAX: usize = 512;
pub const TOP_CHILDREN_THREADS: usize = 256;
pub const TOP_CHILDREN_SIMDGROUPS: usize = 8;
-pub const FRONTIER_LANE_TOKEN: usize = 0;
-pub const FRONTIER_LANE_PARENT: usize = 1;
-pub const FRONTIER_LANE_DEPTH: usize = 2;
-pub const FRONTIER_LANE_CUM: usize = 3;
-pub const FRONTIER_LANE_LOGPROB: usize = 4;
-pub const FRONTIER_LANE_KEY: usize = 5;
-pub const FRONTIER_LANE_ACTIVE: usize = 6;
-pub const FRONTIER_LANE_COUNT: usize = FRONTIER_LANE_ACTIVE + 1;
+#[repr(C)]
+#[derive(Clone, Copy)]
+pub enum FrontierIdx {
+ TokenId,
+ ParentSlot,
+ Depth,
+ PathLogprobBits,
+ EdgeLogprobBits,
+ PathScoreKey,
+ Active,
+}
-pub const TREE_LANE_TOKEN: usize = 0;
-pub const TREE_LANE_PARENT: usize = 1;
-pub const TREE_LANE_DEPTH: usize = 2;
-pub const TREE_LANE_CUM: usize = 3;
-pub const TREE_LANE_LOGPROB: usize = 4;
-pub const TREE_LANE_MASK: usize = 5;
-pub const TREE_LANE_COUNT: usize = TREE_LANE_MASK + 1;
+impl FrontierIdx {
+ pub const COUNT: usize = Self::Active as usize + 1;
+}
-pub const METADATA_LANE_DEPTH: usize = 0;
-pub const METADATA_LANE_ANCESTOR_COUNT: usize = 1;
-pub const METADATA_LANE_NODE_INDEX: usize = 2;
-pub const METADATA_LANE_COUNT: usize = METADATA_LANE_NODE_INDEX + 1;
+#[repr(C)]
+#[derive(Clone, Copy)]
+pub enum TreeIdx {
+ TokenId,
+ ParentSlot,
+ Depth,
+ PathLogprobBits,
+ EdgeLogprobBits,
+ Valid,
+}
+
+impl TreeIdx {
+ pub const COUNT: usize = Self::Valid as usize + 1;
+}
+
+#[repr(C)]
+#[derive(Clone, Copy)]
+pub enum MetadataIdx {
+ Depth,
+ AncestorCount,
+ TreeSlot,
+}
+
+impl MetadataIdx {
+ pub const COUNT: usize = Self::TreeSlot as usize + 1;
+}
pub const FRONTIER_NO_WINNER: u32 = !0;
pub const FRONTIER_SELECT_THREADS: usize = 256;
diff --git a/crates/backend-uzu/src/backends/cpu/kernel/attention/ancestor_attention.rs b/crates/backend-uzu/src/backends/cpu/kernel/attention/ancestor_attention.rs
new file mode 100644
index 000000000..0522ecf47
--- /dev/null
+++ b/crates/backend-uzu/src/backends/cpu/kernel/attention/ancestor_attention.rs
@@ -0,0 +1,96 @@
+use half::bf16;
+use proc_macros::kernel;
+
+use super::attention_single_pass::attention_single_pass;
+
+#[kernel(AncestorAttention)]
+#[variants(HEAD_DIM, 128)]
+fn ancestor_attention<const HEAD_DIM: u32>(
+ prefix_kv: *const bf16,
+ node_kv: *const bf16,
+ current_qkv: *const bf16,
+ ancestor_indices: *const u32,
+ ancestor_counts: *const u32,
+ output: *mut bf16,
+ rows: u32,
+ prefix_length: u32,
+ ancestor_stride: u32,
+ node_capacity: u32,
+ scale: f32,
+ #[specialize] num_heads: u32,
+) {
+ const QKV_COMPONENTS: usize = 3;
+ const KEY_COMPONENT: usize = 1;
+ const VALUE_COMPONENT: usize = 2;
+ let head_dim = HEAD_DIM as usize;
+ let num_heads = num_heads as usize;
+ let model_dim = num_heads * head_dim;
+ let qkv_width = QKV_COMPONENTS * model_dim;
+ let prefix_length = prefix_length as usize;
+ let node_capacity = node_capacity as usize;
+ let last_node = node_capacity.saturating_sub(1);
+
+ for row in 0..rows as usize {
+ unsafe {
+ let current_row = current_qkv.add(row * qkv_width);
+
+ let ancestor_count = *ancestor_counts.add(row) as usize;
+ let length = prefix_length + ancestor_count + 1;
+ let mut keys = vec![bf16::ZERO; length * model_dim];
+ let mut values = vec![bf16::ZERO; length * model_dim];
+ std::ptr::copy_nonoverlapping(prefix_kv, keys.as_mut_ptr(), prefix_length * model_dim);
+ std::ptr::copy_nonoverlapping(
+ prefix_kv.add(prefix_length * model_dim),
+ values.as_mut_ptr(),
+ prefix_length * model_dim,
+ );
+ for offset in 0..ancestor_count {
+ let ancestor = (*ancestor_indices.add(row * ancestor_stride as usize + offset) as usize).min(last_node);
+ std::ptr::copy_nonoverlapping(
+ node_kv.add(ancestor * model_dim),
+ keys.as_mut_ptr().add((prefix_length + offset) * model_dim),
+ model_dim,
+ );
+ std::ptr::copy_nonoverlapping(
+ node_kv.add(node_capacity * model_dim + ancestor * model_dim),
+ values.as_mut_ptr().add((prefix_length + offset) * model_dim),
+ model_dim,
+ );
+ }
+ std::ptr::copy_nonoverlapping(
+ current_row.add(KEY_COMPONENT * model_dim),
+ keys.as_mut_ptr().add((length - 1) * model_dim),
+ model_dim,
+ );
+ std::ptr::copy_nonoverlapping(
+ current_row.add(VALUE_COMPONENT * model_dim),
+ values.as_mut_ptr().add((length - 1) * model_dim),
+ model_dim,
+ );
+ attention_single_pass::<bf16, HEAD_DIM>(
+ current_row,
+ keys.as_ptr(),
+ values.as_ptr(),
+ output.add(row * num_heads * head_dim),
+ 1,
+ length as u32,
+ HEAD_DIM,
+ model_dim as u32,
+ HEAD_DIM,
+ model_dim as u32,
+ None,
+ scale,
+ None,
+ None,
+ None,
+ num_heads as u32,
+ 1,
+ false,
+ false,
+ false,
+ false,
+ false,
+ );
+ }
+ }
+}
diff --git a/crates/backend-uzu/src/backends/cpu/kernel/attention/attention_last_query.rs b/crates/backend-uzu/src/backends/cpu/kernel/attention/attention_last_query.rs
deleted file mode 100644
index dd1e4d075..000000000
--- a/crates/backend-uzu/src/backends/cpu/kernel/attention/attention_last_query.rs
+++ /dev/null
@@ -1,80 +0,0 @@
-use half::bf16;
-use proc_macros::kernel;
-
-use super::attention_single_pass::attention_single_pass;
-
-#[kernel(AttentionLastQuery)]
-#[variants(HEAD_DIM, 128)]
-fn attention_last_query<const HEAD_DIM: u32>(
- prefix_qkv: *const bf16,
- node_qkv: *const bf16,
- current_qkv: *const bf16,
- ancestor_indices: *const u32,
- ancestor_counts: *const u32,
- output: *mut bf16,
- rows: u32,
- prefix_length: u32,
- ancestor_stride: u32,
- node_capacity: u32,
- scale: f32,
- #[specialize] num_heads: u32,
-) {
- const QKV_COMPONENTS: usize = 3;
- const KV_COMPONENTS: usize = 2;
- const KEY_COMPONENT: usize = 1;
- const VALUE_COMPONENT: usize = 2;
- let head_dim = HEAD_DIM as usize;
- let num_heads = num_heads as usize;
- let qkv_width = QKV_COMPONENTS * num_heads * head_dim;
- let kv_offset = KEY_COMPONENT * num_heads * head_dim;
- let kv_width = KV_COMPONENTS * num_heads * head_dim;
- let last_node = node_capacity.saturating_sub(1) as usize;
-
- for row in 0..rows as usize {
- unsafe {
- let current_row = current_qkv.add(row * qkv_width);
-
- let ancestor_count = *ancestor_counts.add(row) as usize;
- let length = prefix_length as usize + ancestor_count + 1;
- let mut sequence = vec![bf16::ZERO; length * qkv_width];
- std::ptr::copy_nonoverlapping(prefix_qkv, sequence.as_mut_ptr(), prefix_length as usize * qkv_width);
- for offset in 0..ancestor_count {
- let ancestor = (*ancestor_indices.add(row * ancestor_stride as usize + offset) as usize).min(last_node);
- std::ptr::copy_nonoverlapping(
- node_qkv.add(ancestor * qkv_width + kv_offset),
- sequence.as_mut_ptr().add((prefix_length as usize + offset) * qkv_width + kv_offset),
- kv_width,
- );
- }
- std::ptr::copy_nonoverlapping(
- current_row.add(kv_offset),
- sequence.as_mut_ptr().add((length - 1) * qkv_width + kv_offset),
- kv_width,
- );
- attention_single_pass::<bf16, HEAD_DIM>(
- current_row,
- sequence.as_ptr().add(kv_offset),
- sequence.as_ptr().add(VALUE_COMPONENT * num_heads * head_dim),
- output.add(row * num_heads * head_dim),
- 1,
- length as u32,
- HEAD_DIM,
- qkv_width as u32,
- HEAD_DIM,
- qkv_width as u32,
- None,
- scale,
- None,
- None,
- None,
- num_heads as u32,
- 1,
- false,
- false,
- false,
- false,
- false,
- );
- }
- }
-}
diff --git a/crates/backend-uzu/src/backends/cpu/kernel/attention/mod.rs b/crates/backend-uzu/src/backends/cpu/kernel/attention/mod.rs
index 813c08dce..052bec54a 100644
--- a/crates/backend-uzu/src/backends/cpu/kernel/attention/mod.rs
+++ b/crates/backend-uzu/src/backends/cpu/kernel/attention/mod.rs
@@ -1,5 +1,5 @@
+pub mod ancestor_attention;
pub mod attention_fallback;
-pub mod attention_last_query;
pub mod attention_prepare;
pub mod attention_single_pass;
pub mod attention_two_pass;
diff --git a/crates/backend-uzu/src/backends/cpu/kernel/normalization/normalization.rs b/crates/backend-uzu/src/backends/cpu/kernel/normalization/normalization.rs
index d15063d4b..854b7b27f 100644
--- a/crates/backend-uzu/src/backends/cpu/kernel/normalization/normalization.rs
+++ b/crates/backend-uzu/src/backends/cpu/kernel/normalization/normalization.rs
@@ -6,17 +6,18 @@ use crate::array::ArrayElement;
#[kernel(Normalization)]
#[variants(InputT, f32, f16, bf16)]
-#[variants(ScaleT, f32, f16, bf16)]
+#[variants(AffineT, f32, f16, bf16)]
#[variants(OutputT, f32, f16, bf16)]
#[variants(AccumT, f32)]
pub fn normalization<
InputT: ArrayElement + Float,
- ScaleT: ArrayElement + Float,
+ AffineT: ArrayElement + Float,
OutputT: ArrayElement + Float,
AccumT: ArrayElement + Float,
>(
#[optional(!in_place)] input: Option<*const InputT>,
- scales: *const ScaleT,
+ scales: *const AffineT,
+ #[optional(has_biases)] biases: Option<*const AffineT>,
output: *mut OutputT,
#[optional(copy_to_shortcut)] shortcut: Option<*mut InputT>,
#[optional(use_hadamard)] hadamard_factors: Option<*const i32>,
@@ -33,9 +34,11 @@ pub fn normalization<
#[specialize] use_hadamard: bool,
#[specialize] scale_residual_sum: bool,
#[specialize] scale_output: bool,
+ #[specialize] has_biases: bool,
) {
assert_eq!(shortcut.is_some(), copy_to_shortcut);
assert_eq!(hadamard_factors.is_some(), use_hadamard);
+ assert_eq!(biases.is_some(), has_biases);
assert!(copy_to_shortcut || !residual_add);
if use_hadamard {
@@ -94,7 +97,7 @@ pub fn normalization<
};
let scale_val = unsafe { AccumT::from(*scales.add(i)).unwrap() };
let normalized: AccumT = (input_val - mean) * rms_inv;
- let result: OutputT = if full_layer {
+ let mut result: OutputT = if full_layer {
// Full-layer: keep everything in accumulation precision
let scale_with_offset: AccumT = scale_val + scale_offset;
OutputT::from(normalized * scale_with_offset).unwrap()
@@ -104,11 +107,13 @@ pub fn normalization<
let scale_with_offset_out = OutputT::from(scale_val + scale_offset).unwrap();
normalized_out * scale_with_offset_out
};
- let result = if scale_output {
- result * OutputT::from(post_layer_scalar).unwrap()
- } else {
- result
- };
+ if has_biases {
+ let bias = unsafe { AccumT::from(*biases.unwrap().add(i)).unwrap() };
+ result = OutputT::from(AccumT::from(result).unwrap() + bias).unwrap();
+ }
+ if scale_output {
+ result = result * OutputT::from(post_layer_scalar).unwrap();
+ }
unsafe { *output.add(batch_offset + i) = result };
}
}
diff --git a/crates/backend-uzu/src/backends/cpu/kernel/weaver/weaver_frontier_scatter.rs b/crates/backend-uzu/src/backends/cpu/kernel/weaver/weaver_frontier_scatter.rs
index b6cab65a3..627b7602d 100644
--- a/crates/backend-uzu/src/backends/cpu/kernel/weaver/weaver_frontier_scatter.rs
+++ b/crates/backend-uzu/src/backends/cpu/kernel/weaver/weaver_frontier_scatter.rs
@@ -1,6 +1,6 @@
use proc_macros::kernel;
-use crate::backends::common::gpu_types::weaver;
+use crate::backends::common::gpu_types::weaver::{FrontierIdx, MetadataIdx, TreeIdx};
const F32_SIGN_BIT: u32 = 1 << (u32::BITS - 1);
@@ -24,43 +24,43 @@ pub fn weaver_frontier_scatter(
capacity: u32,
tree_slots: u32,
rows: u32,
- fanout: u32,
+ children_per_node: u32,
) {
- if capacity == 0 || tree_slots == 0 || fanout == 0 {
+ if capacity == 0 || tree_slots == 0 || children_per_node == 0 {
return;
}
- let (capacity, tree_slots, rows, fanout) = (capacity as usize, tree_slots as usize, rows as usize, fanout as usize);
- let tree = unsafe { std::slice::from_raw_parts(tree, weaver::TREE_LANE_COUNT * tree_slots) };
+ let (capacity, tree_slots, rows, children_per_node) =
+ (capacity as usize, tree_slots as usize, rows as usize, children_per_node as usize);
+ let tree = unsafe { std::slice::from_raw_parts(tree, TreeIdx::COUNT * tree_slots) };
let parent_indices =
- unsafe { std::slice::from_raw_parts(round_metadata.add(weaver::METADATA_LANE_NODE_INDEX * rows), rows) };
+ unsafe { std::slice::from_raw_parts(round_metadata.add(MetadataIdx::TreeSlot as usize * rows), rows) };
let round_valid = unsafe { std::slice::from_raw_parts(round_valid, rows) };
- let child_ids = unsafe { std::slice::from_raw_parts(child_ids, rows * fanout) };
- let child_logprobs = unsafe { std::slice::from_raw_parts(child_logprobs, rows * fanout) };
- let frontier = unsafe { std::slice::from_raw_parts_mut(frontier, weaver::FRONTIER_LANE_COUNT * capacity) };
+ let child_ids = unsafe { std::slice::from_raw_parts(child_ids, rows * children_per_node) };
+ let child_logprobs = unsafe { std::slice::from_raw_parts(child_logprobs, rows * children_per_node) };
+ let frontier = unsafe { std::slice::from_raw_parts_mut(frontier, FrontierIdx::COUNT * capacity) };
- for index in 0..rows * fanout {
- let row = index / fanout;
+ for index in 0..rows * children_per_node {
+ let row = index / children_per_node;
if round_valid[row] == 0 {
continue;
}
let parent = parent_indices[row] as usize;
- let slot = parent * fanout + index % fanout;
+ let slot = parent * children_per_node + index % children_per_node;
if parent >= tree_slots || slot >= capacity {
continue;
}
let logprob = child_logprobs[index];
- let cumulative_logprob = f32::from_bits(tree[weaver::TREE_LANE_CUM * tree_slots + parent]) + logprob;
- let values = [
- child_ids[index],
- parent as u32,
- tree[weaver::TREE_LANE_DEPTH * tree_slots + parent] + 1,
- cumulative_logprob.to_bits(),
- logprob.to_bits(),
- top_k_score_key(cumulative_logprob),
- 1,
- ];
- for (lane, value) in values.into_iter().enumerate() {
- frontier[lane * capacity + slot] = value;
- }
+ let cumulative_logprob =
+ f32::from_bits(tree[TreeIdx::PathLogprobBits as usize * tree_slots + parent]) + logprob;
+ let mut set = |field: FrontierIdx, value| {
+ frontier[field as usize * capacity + slot] = value;
+ };
+ set(FrontierIdx::TokenId, child_ids[index]);
+ set(FrontierIdx::ParentSlot, parent as u32);
+ set(FrontierIdx::Depth, tree[TreeIdx::Depth as usize * tree_slots + parent] + 1);
+ set(FrontierIdx::PathLogprobBits, cumulative_logprob.to_bits());
+ set(FrontierIdx::EdgeLogprobBits, logprob.to_bits());
+ set(FrontierIdx::PathScoreKey, top_k_score_key(cumulative_logprob));
+ set(FrontierIdx::Active, 1);
}
}
diff --git a/crates/backend-uzu/src/backends/cpu/kernel/weaver/weaver_frontier_select.rs b/crates/backend-uzu/src/backends/cpu/kernel/weaver/weaver_frontier_select.rs
index 7e178879a..c90db4461 100644
--- a/crates/backend-uzu/src/backends/cpu/kernel/weaver/weaver_frontier_select.rs
+++ b/crates/backend-uzu/src/backends/cpu/kernel/weaver/weaver_frontier_select.rs
@@ -1,6 +1,8 @@
use proc_macros::kernel;
-use crate::backends::common::gpu_types::weaver;
+use crate::backends::common::gpu_types::weaver::{
+ FRONTIER_MAX_SLOTS, FRONTIER_MAX_WIDTH, FRONTIER_NO_WINNER, FrontierIdx, MetadataIdx, TreeIdx,
+};
#[kernel(WeaverFrontierSelect)]
pub fn weaver_frontier_select(
@@ -26,9 +28,9 @@ pub fn weaver_frontier_select(
candidate_pool_size: u32,
) {
if capacity == 0
- || capacity as usize > weaver::FRONTIER_MAX_SLOTS
+ || capacity as usize > FRONTIER_MAX_SLOTS
|| width == 0
- || width as usize > weaver::FRONTIER_MAX_WIDTH
+ || width as usize > FRONTIER_MAX_WIDTH
|| ancestor_stride == 0
|| max_depth == 0
|| tree_slots == 0
@@ -42,11 +44,11 @@ pub fn weaver_frontier_select(
(capacity as usize, tree_slots as usize, width as usize, ancestor_stride as usize);
let (candidate_pool_rows, candidate_pool_size) = (candidate_pool_rows as usize, candidate_pool_size as usize);
let pool_len = candidate_pool_rows * candidate_pool_size;
- let frontier = unsafe { std::slice::from_raw_parts_mut(frontier, weaver::FRONTIER_LANE_COUNT * capacity) };
- let tree = unsafe { std::slice::from_raw_parts_mut(tree, weaver::TREE_LANE_COUNT * slots) };
+ let frontier = unsafe { std::slice::from_raw_parts_mut(frontier, FrontierIdx::COUNT * capacity) };
+ let tree = unsafe { std::slice::from_raw_parts_mut(tree, TreeIdx::COUNT * slots) };
let slot_ancestors = unsafe { std::slice::from_raw_parts_mut(slot_ancestors, slots * stride) };
let round_token_ids = unsafe { std::slice::from_raw_parts_mut(round_token_ids, width) };
- let round_metadata = unsafe { std::slice::from_raw_parts_mut(round_metadata, weaver::METADATA_LANE_COUNT * width) };
+ let round_metadata = unsafe { std::slice::from_raw_parts_mut(round_metadata, MetadataIdx::COUNT * width) };
let round_ancestors = unsafe { std::slice::from_raw_parts_mut(round_ancestors, width * stride) };
let round_valid = unsafe { std::slice::from_raw_parts_mut(round_valid, width) };
let candidate_pool_ids = unsafe { std::slice::from_raw_parts(candidate_pool_ids, pool_len) };
@@ -58,46 +60,46 @@ pub fn weaver_frontier_select(
for row in 0..width {
let (mut key, mut parent, mut token, mut winner) =
- (0, weaver::FRONTIER_NO_WINNER, weaver::FRONTIER_NO_WINNER, weaver::FRONTIER_NO_WINNER);
+ (0, FRONTIER_NO_WINNER, FRONTIER_NO_WINNER, FRONTIER_NO_WINNER);
for slot in 0..capacity {
- if frontier[weaver::FRONTIER_LANE_ACTIVE * capacity + slot] == 0 {
+ if frontier[FrontierIdx::Active as usize * capacity + slot] == 0 {
continue;
}
let next = (
- frontier[weaver::FRONTIER_LANE_KEY * capacity + slot],
- frontier[weaver::FRONTIER_LANE_PARENT * capacity + slot],
- frontier[weaver::FRONTIER_LANE_TOKEN * capacity + slot],
+ frontier[FrontierIdx::PathScoreKey as usize * capacity + slot],
+ frontier[FrontierIdx::ParentSlot as usize * capacity + slot],
+ frontier[FrontierIdx::TokenId as usize * capacity + slot],
);
if next.0 > key || (next.0 == key && (next.1 < parent || (next.1 == parent && next.2 < token))) {
(key, parent, token, winner) = (next.0, next.1, next.2, slot as u32);
}
}
- let real = winner != weaver::FRONTIER_NO_WINNER;
+ let real = winner != FRONTIER_NO_WINNER;
let winner = if real {
winner as usize
} else {
0
};
let tree_slot = slot_start as usize + row;
- let lane_value = |lane: usize| u32::from(real) * frontier[lane * capacity + winner];
- let token = lane_value(weaver::FRONTIER_LANE_TOKEN);
- let depth = lane_value(weaver::FRONTIER_LANE_DEPTH);
- let cumulative_logprob = lane_value(weaver::FRONTIER_LANE_CUM);
- let logprob = lane_value(weaver::FRONTIER_LANE_LOGPROB);
+ let field_value = |field: FrontierIdx| u32::from(real) * frontier[field as usize * capacity + winner];
+ let token = field_value(FrontierIdx::TokenId);
+ let depth = field_value(FrontierIdx::Depth);
+ let cumulative_logprob = field_value(FrontierIdx::PathLogprobBits);
+ let logprob = field_value(FrontierIdx::EdgeLogprobBits);
- tree[weaver::TREE_LANE_TOKEN * slots + tree_slot] = token;
- tree[weaver::TREE_LANE_PARENT * slots + tree_slot] = if real {
+ tree[TreeIdx::TokenId as usize * slots + tree_slot] = token;
+ tree[TreeIdx::ParentSlot as usize * slots + tree_slot] = if real {
parent
} else {
- weaver::FRONTIER_NO_WINNER
+ FRONTIER_NO_WINNER
};
- tree[weaver::TREE_LANE_DEPTH * slots + tree_slot] = depth;
- tree[weaver::TREE_LANE_CUM * slots + tree_slot] = cumulative_logprob;
- tree[weaver::TREE_LANE_LOGPROB * slots + tree_slot] = logprob;
- tree[weaver::TREE_LANE_MASK * slots + tree_slot] = u32::from(real);
+ tree[TreeIdx::Depth as usize * slots + tree_slot] = depth;
+ tree[TreeIdx::PathLogprobBits as usize * slots + tree_slot] = cumulative_logprob;
+ tree[TreeIdx::EdgeLogprobBits as usize * slots + tree_slot] = logprob;
+ tree[TreeIdx::Valid as usize * slots + tree_slot] = u32::from(real);
if real {
- frontier[weaver::FRONTIER_LANE_ACTIVE * capacity + winner] = 0;
+ frontier[FrontierIdx::Active as usize * capacity + winner] = 0;
}
let parent_slot = if real && (parent as usize) < slots {
@@ -118,9 +120,9 @@ pub fn weaver_frontier_select(
}
round_token_ids[row] = token;
- round_metadata[weaver::METADATA_LANE_DEPTH * width + row] = depth.min(max_depth - 1);
- round_metadata[weaver::METADATA_LANE_ANCESTOR_COUNT * width + row] = depth.min(stride as u32);
- round_metadata[weaver::METADATA_LANE_NODE_INDEX * width + row] = tree_slot as u32;
+ round_metadata[MetadataIdx::Depth as usize * width + row] = depth.min(max_depth - 1);
+ round_metadata[MetadataIdx::AncestorCount as usize * width + row] = depth.min(stride as u32);
+ round_metadata[MetadataIdx::TreeSlot as usize * width + row] = tree_slot as u32;
round_valid[row] = u32::from(real && depth < lookahead_count && depth < max_depth);
// Every row of a round expands the pool for its own depth, so the winner's pool row is
diff --git a/crates/backend-uzu/src/backends/cpu/kernel/weaver/weaver_node_cache_write.rs b/crates/backend-uzu/src/backends/cpu/kernel/weaver/weaver_node_cache_write.rs
index 346d9bdd2..1f4c177d6 100644
--- a/crates/backend-uzu/src/backends/cpu/kernel/weaver/weaver_node_cache_write.rs
+++ b/crates/backend-uzu/src/backends/cpu/kernel/weaver/weaver_node_cache_write.rs
@@ -8,7 +8,7 @@ use crate::array::ArrayElement;
#[variants(T, f32, bf16)]
pub fn weaver_node_cache_write<T: ArrayElement + Float>(
current_qkv: *const T,
- node_qkv: *mut T,
+ node_kv: *mut T,
node_indices: *const u32,
model_dim: u32,
node_capacity: u32,
@@ -24,9 +24,11 @@ pub fn weaver_node_cache_write<T: ArrayElement + Float>(
for position in 0..total as usize {
let row = position / (2 * model_dim);
let offset = position - row * 2 * model_dim;
+ let component = offset / model_dim;
+ let component_offset = offset % model_dim;
let node = (*node_indices.add(row) as usize).min(last_node);
- *node_qkv.add(node * qkv_width + model_dim + offset) =
- *current_qkv.add(row * qkv_width + model_dim + offset);
+ *node_kv.add(component * node_capacity as usize * model_dim + node * model_dim + component_offset) =
+ *current_qkv.add(row * qkv_width + (component + 1) * model_dim + component_offset);
}
}
}
diff --git a/crates/backend-uzu/src/backends/cpu/kernel/weaver/weaver_top_children.rs b/crates/backend-uzu/src/backends/cpu/kernel/weaver/weaver_top_children.rs
index 55b713f3f..24cbcd0eb 100644
--- a/crates/backend-uzu/src/backends/cpu/kernel/weaver/weaver_top_children.rs
+++ b/crates/backend-uzu/src/backends/cpu/kernel/weaver/weaver_top_children.rs
@@ -1,29 +1,29 @@
use half::bf16;
use proc_macros::kernel;
-use crate::backends::common::gpu_types::weaver;
+use crate::backends::common::gpu_types::weaver::CANDIDATES_MAX;
#[kernel(WeaverTopChildren)]
pub fn weaver_top_children(
residual_logits: *const bf16,
- candidate_scores: *const f32,
+ candidate_logits: *const f32,
candidate_ids: *const u32,
- output_ids: *mut u32,
- output_logprobs: *mut f32,
+ output_token_ids: *mut u32,
+ output_model_logprobs: *mut f32,
rows: u32,
candidates: u32,
- children: u32,
+ children_per_node: u32,
) {
let rows = rows as usize;
let candidates = candidates as usize;
- let children = children as usize;
- if candidates == 0 || candidates > weaver::CANDIDATES_MAX || children == 0 || children > candidates {
+ let children_per_node = children_per_node as usize;
+ if candidates == 0 || candidates > CANDIDATES_MAX || children_per_node == 0 || children_per_node > candidates {
return;
}
for row in 0..rows {
let base = row * candidates;
let logits = (0..candidates)
- .map(|index| unsafe { *candidate_scores.add(base + index) + (*residual_logits.add(base + index)).to_f32() })
+ .map(|index| unsafe { *candidate_logits.add(base + index) + (*residual_logits.add(base + index)).to_f32() })
.collect::<Vec<_>>();
let max = logits.iter().copied().fold(f32::NEG_INFINITY, f32::max);
let log_sum = logits.iter().map(|value| (value - max).exp()).sum::<f32>().ln() + max;
@@ -33,10 +33,10 @@ pub fn weaver_top_children(
.total_cmp(&logits[left])
.then_with(|| unsafe { (*candidate_ids.add(base + left)).cmp(&*candidate_ids.add(base + right)) })
});
- for (rank, index) in indices.into_iter().take(children).enumerate() {
+ for (rank, index) in indices.into_iter().take(children_per_node).enumerate() {
unsafe {
- *output_ids.add(row * children + rank) = *candidate_ids.add(base + index);
- *output_logprobs.add(row * children + rank) = logits[index] - log_sum;
+ *output_token_ids.add(row * children_per_node + rank) = *candidate_ids.add(base + index);
+ *output_model_logprobs.add(row * children_per_node + rank) = logits[index] - log_sum;
}
}
}
diff --git a/crates/backend-uzu/src/backends/metal/kernel/attention/attention_last_query.metal b/crates/backend-uzu/src/backends/metal/kernel/attention/ancestor_attention.metal
similarity index 72%
rename from crates/backend-uzu/src/backends/metal/kernel/attention/attention_last_query.metal
rename to crates/backend-uzu/src/backends/metal/kernel/attention/ancestor_attention.metal
index 0f6aabaac..e615d580f 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/attention/attention_last_query.metal
+++ b/crates/backend-uzu/src/backends/metal/kernel/attention/ancestor_attention.metal
@@ -51,9 +51,9 @@ METAL_FUNC void attend_qkv(
template <uint HEAD_DIM>
VARIANTS(HEAD_DIM, 128)
-PUBLIC KERNEL(AttentionLastQuery)(
- const device bfloat* prefix_qkv,
- const device bfloat* node_qkv,
+PUBLIC KERNEL(AncestorAttention)(
+ const device bfloat* prefix_kv,
+ const device bfloat* node_kv,
const device bfloat* current_qkv,
const device uint* ancestor_indices,
const device uint* ancestor_counts,
@@ -78,29 +78,33 @@ PUBLIC KERNEL(AttentionLastQuery)(
const uint row = row_head / num_heads;
const uint head = row_head % num_heads;
const uint lane = thread_context.simd_lane_id;
+ const uint model_vectors = num_heads * vectors_per_head;
const uint qkv_vectors = QKV_COMPONENTS * num_heads * vectors_per_head;
- const uint key_offset = (KEY_COMPONENT * num_heads + head) * vectors_per_head + lane;
- const uint value_offset = (VALUE_COMPONENT * num_heads + head) * vectors_per_head + lane;
- const device bfloat4* prefix_vectors = (const device bfloat4*)prefix_qkv;
- const device bfloat4* node_vectors = (const device bfloat4*)node_qkv;
+ const uint head_offset = head * vectors_per_head + lane;
+ const uint current_key_offset = KEY_COMPONENT * model_vectors + head_offset;
+ const uint current_value_offset = VALUE_COMPONENT * model_vectors + head_offset;
+ const uint prefix_value_offset = prefix_length * model_vectors + head_offset;
+ const uint node_value_offset = node_capacity * model_vectors + head_offset;
+ const device bfloat4* prefix_vectors = (const device bfloat4*)prefix_kv;
+ const device bfloat4* node_vectors = (const device bfloat4*)node_kv;
const device bfloat4* current_row = (const device bfloat4*)current_qkv + row * qkv_vectors;
- const float4 query = float4(current_row[head * vectors_per_head + lane]) * scale;
+ const float4 query = float4(current_row[head_offset]) * scale;
// prefix_length >= 1 (the u0 token): seed the online softmax from position 0.
- float max_score = simd_sum(dot(query, float4(prefix_vectors[key_offset])));
- float4 values = float4(prefix_vectors[value_offset]);
+ float max_score = simd_sum(dot(query, float4(prefix_vectors[head_offset])));
+ float4 values = float4(prefix_vectors[prefix_value_offset]);
float sum = 1.0f;
uint position = 1;
for (; position + unroll_count - 1 < prefix_length; position += unroll_count) {
- attend_qkv<unroll_count>(query, key_offset, value_offset, values, max_score, sum, [&](int step) {
- return prefix_vectors + (position + step) * qkv_vectors;
+ attend_qkv<unroll_count>(query, head_offset, prefix_value_offset, values, max_score, sum, [&](int step) {
+ return prefix_vectors + (position + step) * model_vectors;
});
}
for (; position < prefix_length; ++position) {
- attend_qkv<1>(query, key_offset, value_offset, values, max_score, sum, [&](int) {
- return prefix_vectors + position * qkv_vectors;
+ attend_qkv<1>(query, head_offset, prefix_value_offset, values, max_score, sum, [&](int) {
+ return prefix_vectors + position * model_vectors;
});
}
@@ -109,17 +113,19 @@ PUBLIC KERNEL(AttentionLastQuery)(
const uint last_node = node_capacity == 0 ? 0u : node_capacity - 1u;
uint offset = 0;
for (; offset + unroll_count - 1 < ancestor_count; offset += unroll_count) {
- attend_qkv<unroll_count>(query, key_offset, value_offset, values, max_score, sum, [&](int step) {
- return node_vectors + min(row_ancestors[offset + step], last_node) * qkv_vectors;
+ attend_qkv<unroll_count>(query, head_offset, node_value_offset, values, max_score, sum, [&](int step) {
+ return node_vectors + min(row_ancestors[offset + step], last_node) * model_vectors;
});
}
for (; offset < ancestor_count; ++offset) {
- attend_qkv<1>(query, key_offset, value_offset, values, max_score, sum, [&](int) {
- return node_vectors + min(row_ancestors[offset], last_node) * qkv_vectors;
+ attend_qkv<1>(query, head_offset, node_value_offset, values, max_score, sum, [&](int) {
+ return node_vectors + min(row_ancestors[offset], last_node) * model_vectors;
});
}
- attend_qkv<1>(query, key_offset, value_offset, values, max_score, sum, [&](int) { return current_row; });
+ attend_qkv<1>(query, current_key_offset, current_value_offset, values, max_score, sum, [&](int) {
+ return current_row;
+ });
((device bfloat4*)output)[(row * num_heads + head) * vectors_per_head + lane] = bfloat4(values / sum);
}
diff --git a/crates/backend-uzu/src/backends/metal/kernel/generated/weaver.h b/crates/backend-uzu/src/backends/metal/kernel/generated/weaver.h
index b49c08d42..f453dcc9e 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/generated/weaver.h
+++ b/crates/backend-uzu/src/backends/metal/kernel/generated/weaver.h
@@ -11,43 +11,30 @@ static constant constexpr size_t TOP_CHILDREN_THREADS = 256;
static constant constexpr size_t TOP_CHILDREN_SIMDGROUPS = 8;
-static constant constexpr size_t FRONTIER_LANE_TOKEN = 0;
-
-static constant constexpr size_t FRONTIER_LANE_PARENT = 1;
-
-static constant constexpr size_t FRONTIER_LANE_DEPTH = 2;
-
-static constant constexpr size_t FRONTIER_LANE_CUM = 3;
-
-static constant constexpr size_t FRONTIER_LANE_LOGPROB = 4;
-
-static constant constexpr size_t FRONTIER_LANE_KEY = 5;
-
-static constant constexpr size_t FRONTIER_LANE_ACTIVE = 6;
-
-static constant constexpr size_t FRONTIER_LANE_COUNT = FRONTIER_LANE_ACTIVE + 1;
-
-static constant constexpr size_t TREE_LANE_TOKEN = 0;
-
-static constant constexpr size_t TREE_LANE_PARENT = 1;
-
-static constant constexpr size_t TREE_LANE_DEPTH = 2;
-
-static constant constexpr size_t TREE_LANE_CUM = 3;
-
-static constant constexpr size_t TREE_LANE_LOGPROB = 4;
-
-static constant constexpr size_t TREE_LANE_MASK = 5;
-
-static constant constexpr size_t TREE_LANE_COUNT = TREE_LANE_MASK + 1;
-
-static constant constexpr size_t METADATA_LANE_DEPTH = 0;
-
-static constant constexpr size_t METADATA_LANE_ANCESTOR_COUNT = 1;
-
-static constant constexpr size_t METADATA_LANE_NODE_INDEX = 2;
-
-static constant constexpr size_t METADATA_LANE_COUNT = METADATA_LANE_NODE_INDEX + 1;
+enum class FrontierIdx : uint32_t {
+ TokenId = 0,
+ ParentSlot = 1,
+ Depth = 2,
+ PathLogprobBits = 3,
+ EdgeLogprobBits = 4,
+ PathScoreKey = 5,
+ Active = 6,
+};
+
+enum class TreeIdx : uint32_t {
+ TokenId = 0,
+ ParentSlot = 1,
+ Depth = 2,
+ PathLogprobBits = 3,
+ EdgeLogprobBits = 4,
+ Valid = 5,
+};
+
+enum class MetadataIdx : uint32_t {
+ Depth = 0,
+ AncestorCount = 1,
+ TreeSlot = 2,
+};
static constant constexpr uint32_t FRONTIER_NO_WINNER = ~0;
diff --git a/crates/backend-uzu/src/backends/metal/kernel/mod.rs b/crates/backend-uzu/src/backends/metal/kernel/mod.rs
index f2285865d..429e646c8 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/mod.rs
+++ b/crates/backend-uzu/src/backends/metal/kernel/mod.rs
@@ -3,7 +3,7 @@ use crate::backends::{
Kernels,
gpu_types::{
gemm::{gemm_tiling_simdgroups_per_column, gemm_tiling_simdgroups_per_row},
- weaver,
+ weaver::{FRONTIER_SELECT_THREADS, TOP_CHILDREN_THREADS},
},
},
metal::Metal,
diff --git a/crates/backend-uzu/src/backends/metal/kernel/normalization/normalization.metal b/crates/backend-uzu/src/backends/metal/kernel/normalization/normalization.metal
index 5f1a7ada7..a94191831 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/normalization/normalization.metal
+++ b/crates/backend-uzu/src/backends/metal/kernel/normalization/normalization.metal
@@ -11,14 +11,15 @@ using namespace metal;
// TODO: Are numerics of subtract_mean fine?
-template <typename InputT, typename ScaleT, typename OutputT, typename AccumT>
+template <typename InputT, typename AffineT, typename OutputT, typename AccumT>
VARIANTS(InputT, float, half, bfloat)
-VARIANTS(ScaleT, float, half, bfloat)
+VARIANTS(AffineT, float, half, bfloat)
VARIANTS(OutputT, float, half, bfloat)
VARIANTS(AccumT, float)
PUBLIC KERNEL(Normalization)(
const device InputT* input OPTIONAL(!in_place),
- const device ScaleT* scales,
+ const device AffineT* scales,
+ const device AffineT* biases OPTIONAL(has_biases),
device OutputT* output,
device InputT* shortcut OPTIONAL(copy_to_shortcut),
const device int32_t* hadamard_factors OPTIONAL(use_hadamard),
@@ -35,6 +36,7 @@ PUBLIC KERNEL(Normalization)(
const bool use_hadamard SPECIALIZE,
const bool scale_residual_sum SPECIALIZE,
const bool scale_output SPECIALIZE,
+ const bool has_biases SPECIALIZE,
threadgroup AccumT shared_sum[METAL_SIMD_SIZE],
const ThreadContext thread_context,
const uint batch_idx GROUPS(batch_size),
@@ -121,6 +123,10 @@ PUBLIC KERNEL(Normalization)(
val = static_cast<OutputT>((x - mean) * rms_inv) * static_cast<OutputT>(scale);
}
+ if (has_biases) {
+ val = static_cast<OutputT>(static_cast<AccumT>(val) + static_cast<AccumT>(biases[i]));
+ }
+
if (use_hadamard) {
val = static_cast<OutputT>(simdgroup_input_random_hadamard_transform(
static_cast<ushort>(thread_in_row % METAL_SIMD_SIZE),
diff --git a/crates/backend-uzu/src/backends/metal/kernel/weaver/weaver_frontier.h b/crates/backend-uzu/src/backends/metal/kernel/weaver/weaver_frontier.h
index 9cfecf384..1720ff9e3 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/weaver/weaver_frontier.h
+++ b/crates/backend-uzu/src/backends/metal/kernel/weaver/weaver_frontier.h
@@ -2,4 +2,4 @@
#include "../generated/weaver.h"
-namespace weaver = uzu::weaver;
+using namespace uzu::weaver;
diff --git a/crates/backend-uzu/src/backends/metal/kernel/weaver/weaver_frontier_scatter.metal b/crates/backend-uzu/src/backends/metal/kernel/weaver/weaver_frontier_scatter.metal
index 667591ca7..d5b1ad4da 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/weaver/weaver_frontier_scatter.metal
+++ b/crates/backend-uzu/src/backends/metal/kernel/weaver/weaver_frontier_scatter.metal
@@ -15,35 +15,35 @@ PUBLIC KERNEL(WeaverFrontierScatter)(
constant uint& capacity,
constant uint& tree_slots,
constant uint& rows,
- constant uint& fanout,
- const uint position AXIS(rows* fanout, 64)
+ constant uint& children_per_node,
+ const uint position AXIS(rows* children_per_node, 64)
) {
- if (capacity == 0 || tree_slots == 0 || fanout == 0) {
+ if (capacity == 0 || tree_slots == 0 || children_per_node == 0) {
return;
}
- const uint row = position / fanout, child = position % fanout;
+ const uint row = position / children_per_node, child = position % children_per_node;
if (round_valid[row] == 0u) {
return;
}
- const uint parent = round_metadata[weaver::METADATA_LANE_NODE_INDEX * rows + row];
+ const uint parent = round_metadata[uint(MetadataIdx::TreeSlot) * rows + row];
if (parent >= tree_slots) {
return;
}
- const uint slot = parent * fanout + child;
+ const uint slot = parent * children_per_node + child;
if (slot >= capacity) {
return;
}
- const float logprob = child_logprobs[row * fanout + child];
- const float cumulative_logprob = as_type<float>(tree[weaver::TREE_LANE_CUM * tree_slots + parent]) + logprob;
+ const float logprob = child_logprobs[row * children_per_node + child];
+ const float cumulative_logprob = as_type<float>(tree[uint(TreeIdx::PathLogprobBits) * tree_slots + parent]) + logprob;
- frontier[weaver::FRONTIER_LANE_TOKEN * capacity + slot] = child_ids[row * fanout + child];
- frontier[weaver::FRONTIER_LANE_PARENT * capacity + slot] = parent;
- frontier[weaver::FRONTIER_LANE_DEPTH * capacity + slot] = tree[weaver::TREE_LANE_DEPTH * tree_slots + parent] + 1u;
- frontier[weaver::FRONTIER_LANE_CUM * capacity + slot] = as_type<uint>(cumulative_logprob);
- frontier[weaver::FRONTIER_LANE_LOGPROB * capacity + slot] = as_type<uint>(logprob);
- frontier[weaver::FRONTIER_LANE_KEY * capacity + slot] = top_k_score_key(cumulative_logprob);
- frontier[weaver::FRONTIER_LANE_ACTIVE * capacity + slot] = 1u;
+ frontier[uint(FrontierIdx::TokenId) * capacity + slot] = child_ids[row * children_per_node + child];
+ frontier[uint(FrontierIdx::ParentSlot) * capacity + slot] = parent;
+ frontier[uint(FrontierIdx::Depth) * capacity + slot] = tree[uint(TreeIdx::Depth) * tree_slots + parent] + 1u;
+ frontier[uint(FrontierIdx::PathLogprobBits) * capacity + slot] = as_type<uint>(cumulative_logprob);
+ frontier[uint(FrontierIdx::EdgeLogprobBits) * capacity + slot] = as_type<uint>(logprob);
+ frontier[uint(FrontierIdx::PathScoreKey) * capacity + slot] = top_k_score_key(cumulative_logprob);
+ frontier[uint(FrontierIdx::Active) * capacity + slot] = 1u;
}
diff --git a/crates/backend-uzu/src/backends/metal/kernel/weaver/weaver_frontier_select.metal b/crates/backend-uzu/src/backends/metal/kernel/weaver/weaver_frontier_select.metal
index b58b5b192..0ced94673 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/weaver/weaver_frontier_select.metal
+++ b/crates/backend-uzu/src/backends/metal/kernel/weaver/weaver_frontier_select.metal
@@ -27,34 +27,34 @@ PUBLIC KERNEL(WeaverFrontierSelect)(
constant uint& lookahead_count,
constant uint& candidate_pool_rows,
constant uint& candidate_pool_size,
- threadgroup uint4 reduce[weaver::FRONTIER_SELECT_SIMDGROUPS],
- threadgroup uint winner_slot[weaver::FRONTIER_MAX_WIDTH],
- threadgroup uint row_candidate_pool_row[weaver::FRONTIER_MAX_WIDTH],
+ threadgroup uint4 reduce[FRONTIER_SELECT_SIMDGROUPS],
+ threadgroup uint winner_slot[FRONTIER_MAX_WIDTH],
+ threadgroup uint row_candidate_pool_row[FRONTIER_MAX_WIDTH],
const ThreadContext thread_context,
const uint group_index GROUPS(1),
- const uint lid THREADS(weaver::FRONTIER_SELECT_THREADS)
+ const uint lid THREADS(FRONTIER_SELECT_THREADS)
) {
(void)group_index;
- if (capacity == 0 || capacity > weaver::FRONTIER_MAX_SLOTS || width == 0 || width > weaver::FRONTIER_MAX_WIDTH ||
+ if (capacity == 0 || capacity > FRONTIER_MAX_SLOTS || width == 0 || width > FRONTIER_MAX_WIDTH ||
ancestor_stride == 0 || max_depth == 0 || tree_slots == 0 || slot_start + width > tree_slots ||
candidate_pool_rows == 0 || candidate_pool_size == 0) {
return;
}
- bool entry_active[weaver::FRONTIER_ENTRIES_PER_THREAD];
- for (uint entry = 0; entry < weaver::FRONTIER_ENTRIES_PER_THREAD; ++entry) {
- const uint slot = lid + entry * weaver::FRONTIER_SELECT_THREADS;
- entry_active[entry] = slot < capacity && frontier[weaver::FRONTIER_LANE_ACTIVE * capacity + slot] != 0u;
+ bool entry_active[FRONTIER_ENTRIES_PER_THREAD];
+ for (uint entry = 0; entry < FRONTIER_ENTRIES_PER_THREAD; ++entry) {
+ const uint slot = lid + entry * FRONTIER_SELECT_THREADS;
+ entry_active[entry] = slot < capacity && frontier[uint(FrontierIdx::Active) * capacity + slot] != 0u;
}
for (uint child = 0; child < width; ++child) {
- uint4 local = uint4(0u, weaver::FRONTIER_NO_WINNER, weaver::FRONTIER_NO_WINNER, weaver::FRONTIER_NO_WINNER);
- for (uint entry = 0; entry < weaver::FRONTIER_ENTRIES_PER_THREAD; ++entry) {
- const uint slot = lid + entry * weaver::FRONTIER_SELECT_THREADS;
+ uint4 local = uint4(0u, FRONTIER_NO_WINNER, FRONTIER_NO_WINNER, FRONTIER_NO_WINNER);
+ for (uint entry = 0; entry < FRONTIER_ENTRIES_PER_THREAD; ++entry) {
+ const uint slot = lid + entry * FRONTIER_SELECT_THREADS;
if (entry_active[entry]) {
- const uint key = frontier[weaver::FRONTIER_LANE_KEY * capacity + slot];
- const uint parent = frontier[weaver::FRONTIER_LANE_PARENT * capacity + slot];
- const uint token = frontier[weaver::FRONTIER_LANE_TOKEN * capacity + slot];
+ const uint key = frontier[uint(FrontierIdx::PathScoreKey) * capacity + slot];
+ const uint parent = frontier[uint(FrontierIdx::ParentSlot) * capacity + slot];
+ const uint token = frontier[uint(FrontierIdx::TokenId) * capacity + slot];
if (key > local.x || (key == local.x && (parent < local.y || (parent == local.y && token < local.z)))) {
local = uint4(key, parent, token, slot);
}
@@ -63,24 +63,23 @@ PUBLIC KERNEL(WeaverFrontierSelect)(
uint4 simd;
simd.x = simd_max(local.x);
- simd.y = simd_min(local.x == simd.x ? local.y : weaver::FRONTIER_NO_WINNER);
- simd.z = simd_min(all(local.xy == simd.xy) ? local.z : weaver::FRONTIER_NO_WINNER);
- simd.w = simd_min(all(local.xyz == simd.xyz) ? local.w : weaver::FRONTIER_NO_WINNER);
+ simd.y = simd_min(local.x == simd.x ? local.y : FRONTIER_NO_WINNER);
+ simd.z = simd_min(all(local.xy == simd.xy) ? local.z : FRONTIER_NO_WINNER);
+ simd.w = simd_min(all(local.xyz == simd.xyz) ? local.w : FRONTIER_NO_WINNER);
if (thread_context.simd_lane_id == 0) {
reduce[thread_context.simdgroup_index] = simd;
}
threadgroup_barrier(mem_flags::mem_threadgroup);
if (thread_context.simdgroup_index == 0) {
- const uint4 partial =
- thread_context.simd_lane_id < weaver::FRONTIER_SELECT_SIMDGROUPS
- ? reduce[thread_context.simd_lane_id]
- : uint4(0u, weaver::FRONTIER_NO_WINNER, weaver::FRONTIER_NO_WINNER, weaver::FRONTIER_NO_WINNER);
+ const uint4 partial = thread_context.simd_lane_id < FRONTIER_SELECT_SIMDGROUPS
+ ? reduce[thread_context.simd_lane_id]
+ : uint4(0u, FRONTIER_NO_WINNER, FRONTIER_NO_WINNER, FRONTIER_NO_WINNER);
uint4 selected;
selected.x = simd_max(partial.x);
- selected.y = simd_min(partial.x == selected.x ? partial.y : weaver::FRONTIER_NO_WINNER);
- selected.z = simd_min(all(partial.xy == selected.xy) ? partial.z : weaver::FRONTIER_NO_WINNER);
- selected.w = simd_min(all(partial.xyz == selected.xyz) ? partial.w : weaver::FRONTIER_NO_WINNER);
+ selected.y = simd_min(partial.x == selected.x ? partial.y : FRONTIER_NO_WINNER);
+ selected.z = simd_min(all(partial.xy == selected.xy) ? partial.z : FRONTIER_NO_WINNER);
+ selected.w = simd_min(all(partial.xyz == selected.xyz) ? partial.w : FRONTIER_NO_WINNER);
if (thread_context.simd_lane_id == 0) {
winner_slot[child] = selected.w;
}
@@ -88,32 +87,32 @@ PUBLIC KERNEL(WeaverFrontierSelect)(
threadgroup_barrier(mem_flags::mem_threadgroup);
const uint selected = winner_slot[child];
- for (uint entry = 0; entry < weaver::FRONTIER_ENTRIES_PER_THREAD; ++entry) {
- entry_active[entry] = entry_active[entry] && (lid + entry * weaver::FRONTIER_SELECT_THREADS) != selected;
+ for (uint entry = 0; entry < FRONTIER_ENTRIES_PER_THREAD; ++entry) {
+ entry_active[entry] = entry_active[entry] && (lid + entry * FRONTIER_SELECT_THREADS) != selected;
}
}
if (lid < width) {
const uint row = lid;
const uint slot = winner_slot[row];
- const bool real = slot != weaver::FRONTIER_NO_WINNER;
+ const bool real = slot != FRONTIER_NO_WINNER;
const uint tree_slot = slot_start + row;
- const uint token = real ? frontier[weaver::FRONTIER_LANE_TOKEN * capacity + slot] : 0u;
- const uint parent = real ? frontier[weaver::FRONTIER_LANE_PARENT * capacity + slot] : weaver::FRONTIER_NO_WINNER;
- const uint depth = real ? frontier[weaver::FRONTIER_LANE_DEPTH * capacity + slot] : 0u;
- const uint cumulative_logprob = real ? frontier[weaver::FRONTIER_LANE_CUM * capacity + slot] : 0u;
- const uint logprob = real ? frontier[weaver::FRONTIER_LANE_LOGPROB * capacity + slot] : 0u;
+ const uint token = real ? frontier[uint(FrontierIdx::TokenId) * capacity + slot] : 0u;
+ const uint parent = real ? frontier[uint(FrontierIdx::ParentSlot) * capacity + slot] : FRONTIER_NO_WINNER;
+ const uint depth = real ? frontier[uint(FrontierIdx::Depth) * capacity + slot] : 0u;
+ const uint cumulative_logprob = real ? frontier[uint(FrontierIdx::PathLogprobBits) * capacity + slot] : 0u;
+ const uint logprob = real ? frontier[uint(FrontierIdx::EdgeLogprobBits) * capacity + slot] : 0u;
- tree[weaver::TREE_LANE_TOKEN * tree_slots + tree_slot] = token;
- tree[weaver::TREE_LANE_PARENT * tree_slots + tree_slot] = parent;
- tree[weaver::TREE_LANE_DEPTH * tree_slots + tree_slot] = depth;
- tree[weaver::TREE_LANE_CUM * tree_slots + tree_slot] = cumulative_logprob;
- tree[weaver::TREE_LANE_LOGPROB * tree_slots + tree_slot] = logprob;
- tree[weaver::TREE_LANE_MASK * tree_slots + tree_slot] = real ? 1u : 0u;
+ tree[uint(TreeIdx::TokenId) * tree_slots + tree_slot] = token;
+ tree[uint(TreeIdx::ParentSlot) * tree_slots + tree_slot] = parent;
+ tree[uint(TreeIdx::Depth) * tree_slots + tree_slot] = depth;
+ tree[uint(TreeIdx::PathLogprobBits) * tree_slots + tree_slot] = cumulative_logprob;
+ tree[uint(TreeIdx::EdgeLogprobBits) * tree_slots + tree_slot] = logprob;
+ tree[uint(TreeIdx::Valid) * tree_slots + tree_slot] = real ? 1u : 0u;
if (real) {
- frontier[weaver::FRONTIER_LANE_ACTIVE * capacity + slot] = 0u;
+ frontier[uint(FrontierIdx::Active) * capacity + slot] = 0u;
}
const uint parent_slot = real && parent < tree_slots ? parent : 0u;
@@ -127,9 +126,9 @@ PUBLIC KERNEL(WeaverFrontierSelect)(
}
round_token_ids[row] = token;
- round_metadata[weaver::METADATA_LANE_DEPTH * width + row] = min(depth, max_depth - 1u);
- round_metadata[weaver::METADATA_LANE_ANCESTOR_COUNT * width + row] = min(depth, ancestor_stride);
- round_metadata[weaver::METADATA_LANE_NODE_INDEX * width + row] = tree_slot;
+ round_metadata[uint(MetadataIdx::Depth) * width + row] = min(depth, max_depth - 1u);
+ round_metadata[uint(MetadataIdx::AncestorCount) * width + row] = min(depth, ancestor_stride);
+ round_metadata[uint(MetadataIdx::TreeSlot) * width + row] = tree_slot;
round_valid[row] = real && depth < lookahead_count && depth < max_depth ? 1u : 0u;
row_candidate_pool_row[row] = min(depth, candidate_pool_rows - 1u);
@@ -140,7 +139,7 @@ PUBLIC KERNEL(WeaverFrontierSelect)(
for (uint row = 0; row < width; ++row) {
const uint source = row_candidate_pool_row[row] * candidate_pool_size;
const uint destination = row * candidate_pool_size;
- for (uint candidate = lid; candidate < candidate_pool_size; candidate += weaver::FRONTIER_SELECT_THREADS) {
+ for (uint candidate = lid; candidate < candidate_pool_size; candidate += FRONTIER_SELECT_THREADS) {
round_candidate_ids[destination + candidate] = candidate_pool_ids[source + candidate];
round_candidate_scores[destination + candidate] = candidate_pool_scores[source + candidate];
}
diff --git a/crates/backend-uzu/src/backends/metal/kernel/weaver/weaver_node_cache_write.metal b/crates/backend-uzu/src/backends/metal/kernel/weaver/weaver_node_cache_write.metal
index 4c22b966f..a9b6c6bcf 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/weaver/weaver_node_cache_write.metal
+++ b/crates/backend-uzu/src/backends/metal/kernel/weaver/weaver_node_cache_write.metal
@@ -2,13 +2,13 @@
#include "../common/dsl.h"
// Scatters the key/value halves of the current step's packed QKV rows into their
-// node-cache slots. Kept out of AttentionLastQuery so that the node arena stays
+// node-cache slots. Kept out of AncestorAttention so that the node arena stays
// read-only for the whole attention dispatch.
template <typename T>
VARIANTS(T, float, bfloat)
PUBLIC KERNEL(WeaverNodeCacheWrite)(
const device T* current_qkv,
- device T* node_qkv,
+ device T* node_kv,
const device uint* node_indices,
constant uint& model_dim,
constant uint& node_capacity,
@@ -20,7 +20,10 @@ PUBLIC KERNEL(WeaverNodeCacheWrite)(
}
const uint row = position / (2 * model_dim);
const uint offset = position - row * 2 * model_dim;
+ const uint component = offset / model_dim;
+ const uint component_offset = offset % model_dim;
const uint qkv_width = 3 * model_dim;
const uint node = min(node_indices[row], node_capacity - 1u);
- node_qkv[node * qkv_width + model_dim + offset] = current_qkv[row * qkv_width + model_dim + offset];
+ node_kv[component * node_capacity * model_dim + node * model_dim + component_offset] =
+ current_qkv[row * qkv_width + (component + 1) * model_dim + component_offset];
}
diff --git a/crates/backend-uzu/src/backends/metal/kernel/weaver/weaver_top_children.metal b/crates/backend-uzu/src/backends/metal/kernel/weaver/weaver_top_children.metal
index 84e101ffc..e80de0bc5 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/weaver/weaver_top_children.metal
+++ b/crates/backend-uzu/src/backends/metal/kernel/weaver/weaver_top_children.metal
@@ -13,17 +13,17 @@ METAL_FUNC bool weaver_better(uint score, uint token, uint index, uint best_scor
PUBLIC KERNEL(WeaverTopChildren)(
const device bfloat* residual_logits,
- const device float* candidate_scores,
+ const device float* candidate_logits,
const device uint* candidate_ids,
- device uint* output_ids,
- device float* output_logprobs,
+ device uint* output_token_ids,
+ device float* output_model_logprobs,
constant uint& rows,
constant uint& candidates,
- constant uint& children,
- threadgroup float reduce_float[weaver::TOP_CHILDREN_SIMDGROUPS],
- threadgroup uint reduce_score[weaver::TOP_CHILDREN_SIMDGROUPS],
- threadgroup uint reduce_token[weaver::TOP_CHILDREN_SIMDGROUPS],
- threadgroup uint reduce_index[weaver::TOP_CHILDREN_SIMDGROUPS],
+ constant uint& children_per_node,
+ threadgroup float reduce_float[TOP_CHILDREN_SIMDGROUPS],
+ threadgroup uint reduce_score[TOP_CHILDREN_SIMDGROUPS],
+ threadgroup uint reduce_token[TOP_CHILDREN_SIMDGROUPS],
+ threadgroup uint reduce_index[TOP_CHILDREN_SIMDGROUPS],
threadgroup float& logit_max,
threadgroup float& log_sum,
threadgroup uint& winner_score,
@@ -31,21 +31,21 @@ PUBLIC KERNEL(WeaverTopChildren)(
threadgroup uint& winner_index,
const ThreadContext thread_context,
const uint row GROUPS(rows),
- const uint lid THREADS(weaver::TOP_CHILDREN_THREADS)
+ const uint lid THREADS(TOP_CHILDREN_THREADS)
) {
- if (candidates == 0 || candidates > weaver::CANDIDATES_MAX || children == 0 || children > candidates) {
+ if (candidates == 0 || candidates > CANDIDATES_MAX || children_per_node == 0 || children_per_node > candidates) {
return;
}
const uint base = row * candidates;
const uint first_index = lid;
- const uint second_index = lid + weaver::TOP_CHILDREN_THREADS;
+ const uint second_index = lid + TOP_CHILDREN_THREADS;
const bool first_valid = first_index < candidates;
const bool second_valid = second_index < candidates;
const float first_logit =
- first_valid ? candidate_scores[base + first_index] + float(residual_logits[base + first_index]) : -INFINITY;
+ first_valid ? candidate_logits[base + first_index] + float(residual_logits[base + first_index]) : -INFINITY;
const float second_logit =
- second_valid ? candidate_scores[base + second_index] + float(residual_logits[base + second_index]) : -INFINITY;
+ second_valid ? candidate_logits[base + second_index] + float(residual_logits[base + second_index]) : -INFINITY;
const uint first_token = first_valid ? uint(candidate_ids[base + first_index]) : 0xffffffffu;
const uint second_token = second_valid ? uint(candidate_ids[base + second_index]) : 0xffffffffu;
const uint first_score = first_valid ? top_k_score_key(first_logit) : 0u;
@@ -60,9 +60,8 @@ PUBLIC KERNEL(WeaverTopChildren)(
}
threadgroup_barrier(mem_flags::mem_threadgroup);
if (thread_context.simdgroup_index == 0) {
- const float group_value = thread_context.simd_lane_id < weaver::TOP_CHILDREN_SIMDGROUPS
- ? reduce_float[thread_context.simd_lane_id]
- : -INFINITY;
+ const float group_value =
+ thread_context.simd_lane_id < TOP_CHILDREN_SIMDGROUPS ? reduce_float[thread_context.simd_lane_id] : -INFINITY;
const float maximum = simd_max(group_value);
if (thread_context.simd_lane_id == 0) {
logit_max = maximum;
@@ -78,9 +77,8 @@ PUBLIC KERNEL(WeaverTopChildren)(
}
threadgroup_barrier(mem_flags::mem_threadgroup);
if (thread_context.simdgroup_index == 0) {
- const float group_value = thread_context.simd_lane_id < weaver::TOP_CHILDREN_SIMDGROUPS
- ? reduce_float[thread_context.simd_lane_id]
- : 0.0f;
+ const float group_value =
+ thread_context.simd_lane_id < TOP_CHILDREN_SIMDGROUPS ? reduce_float[thread_context.simd_lane_id] : 0.0f;
const float total = simd_sum(group_value);
if (thread_context.simd_lane_id == 0) {
log_sum = log(total) + logit_max;
@@ -88,7 +86,7 @@ PUBLIC KERNEL(WeaverTopChildren)(
}
threadgroup_barrier(mem_flags::mem_threadgroup);
- for (uint child = 0; child < children; ++child) {
+ for (uint child = 0; child < children_per_node; ++child) {
uint local_score = 0u;
uint local_token = 0xffffffffu;
uint local_index = 0xffffffffu;
@@ -116,7 +114,7 @@ PUBLIC KERNEL(WeaverTopChildren)(
threadgroup_barrier(mem_flags::mem_threadgroup);
if (thread_context.simdgroup_index == 0) {
- const bool lane_valid = thread_context.simd_lane_id < weaver::TOP_CHILDREN_SIMDGROUPS;
+ const bool lane_valid = thread_context.simd_lane_id < TOP_CHILDREN_SIMDGROUPS;
const uint group_score = lane_valid ? reduce_score[thread_context.simd_lane_id] : 0u;
const uint selected_score = simd_max(group_score);
const uint group_token =
@@ -136,8 +134,8 @@ PUBLIC KERNEL(WeaverTopChildren)(
if (lid == 0) {
const float winner_logit = top_k_score_from_key(winner_score);
- output_ids[row * children + child] = winner_token;
- output_logprobs[row * children + child] = winner_logit - log_sum;
+ output_token_ids[row * children_per_node + child] = winner_token;
+ output_model_logprobs[row * children_per_node + child] = winner_logit - log_sum;
}
first_active = first_active && first_index != winner_index;
second_active = second_active && second_index != winner_index;
diff --git a/crates/backend-uzu/src/config/weaver.rs b/crates/backend-uzu/src/config/weaver.rs
index 48f38cb2f..fbf4444e7 100644
--- a/crates/backend-uzu/src/config/weaver.rs
+++ b/crates/backend-uzu/src/config/weaver.rs
@@ -1,6 +1,6 @@
use proc_macros::uzu_config;
-use crate::config::{linear::LinearConfig, normalization::NormalizationConfig};
+use crate::config::normalization::NormalizationConfig;
#[uzu_config]
pub struct WeaverConfig {
@@ -12,6 +12,5 @@ pub struct WeaverConfig {
pub hidden_dim: usize,
pub max_depth: usize,
pub candidate_pool_size: usize,
- pub linear_config: LinearConfig,
pub norm_config: NormalizationConfig,
}
diff --git a/crates/backend-uzu/src/encodable_block/classifier.rs b/crates/backend-uzu/src/encodable_block/classifier.rs
index aab36146c..335869492 100644
--- a/crates/backend-uzu/src/encodable_block/classifier.rs
+++ b/crates/backend-uzu/src/encodable_block/classifier.rs
@@ -8,7 +8,7 @@ use crate::{
encodable_block::{
batch_topology::BatchTopology,
embedding::{Embedding, EmbeddingError},
- normalization::{Normalization, NormalizationNewError, PostLayerScalar},
+ normalization::{Normalization, NormalizationNewError, PostLayerScalar, ShortcutMode},
prediction_head::{PredictionHead, PredictionHeadError},
transformer::{Transformer, TransformerNewError},
},
@@ -62,8 +62,7 @@ impl<B: Backend> Classifier<B> {
let embedding_norm = Normalization::new(
config.transformer_config.model_dim,
None,
- false,
- false,
+ ShortcutMode::None,
PostLayerScalar::None,
data_type,
&config.embedding_norm_config,
diff --git a/crates/backend-uzu/src/encodable_block/dflash.rs b/crates/backend-uzu/src/encodable_block/dflash.rs
index 79f39657a..74c3198da 100644
--- a/crates/backend-uzu/src/encodable_block/dflash.rs
+++ b/crates/backend-uzu/src/encodable_block/dflash.rs
@@ -24,7 +24,7 @@ use crate::{
},
},
mlp::{Mlp, MlpBlockError},
- normalization::{Normalization, NormalizationNewError, PostLayerScalar},
+ normalization::{Normalization, NormalizationNewError, PostLayerScalar, ShortcutMode},
},
parameters::{ParameterLoaderError, ParameterTree},
utils::maybe_mut::MaybeMut,
@@ -92,7 +92,16 @@ fn plain_norm<B: Backend>(
parameter_tree: &ParameterTree<B>,
data_type: DataType,
) -> Result<Normalization<B>, NormalizationNewError<B>> {
- Normalization::new(model_dim, None, false, false, PostLayerScalar::None, data_type, config, parameter_tree, context)
+ Normalization::new(
+ model_dim,
+ None,
+ ShortcutMode::None,
+ PostLayerScalar::None,
+ data_type,
+ config,
+ parameter_tree,
+ context,
+ )
}
impl<B: Backend> DFlashDraft<B> {
diff --git a/crates/backend-uzu/src/encodable_block/normalization.rs b/crates/backend-uzu/src/encodable_block/normalization.rs
index 1b6e14e48..4674084a7 100644
--- a/crates/backend-uzu/src/encodable_block/normalization.rs
+++ b/crates/backend-uzu/src/encodable_block/normalization.rs
@@ -20,6 +20,13 @@ pub enum PostLayerScalar {
ScaleOutput(f32),
}
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum ShortcutMode {
+ None,
+ Copy,
+ Add,
+}
+
#[derive(Debug, Error)]
pub enum NormalizationNewError<B: Backend> {
#[error("Backend error: {0}")]
@@ -32,6 +39,7 @@ pub struct Normalization<B: Backend> {
epsilon: f32,
scale_offset: Option<f32>,
scales: Allocation<B>,
+ biases: Option<Allocation<B>>,
element_count: usize,
hadamard_factors: Option<Allocation<B>>,
post_layer_scalar_value: f32,
@@ -43,18 +51,24 @@ impl<B: Backend> Normalization<B> {
pub fn new(
element_count: usize,
hadamard_factors: Option<Allocation<B>>,
- copy_to_shortcut: bool,
- residual_add: bool,
+ shortcut_mode: ShortcutMode,
post_layer_scalar: PostLayerScalar,
data_type: DataType,
config: &NormalizationConfig,
parameter_tree: &ParameterTree<B>,
context: &B::Context,
) -> Result<Self, NormalizationNewError<B>> {
- assert!(!config.has_biases, "normalization doesn't support biases");
- assert!(copy_to_shortcut || !residual_add, "residual_add requires shortcut");
+ let (copy_to_shortcut, residual_add) = match shortcut_mode {
+ ShortcutMode::None => (false, false),
+ ShortcutMode::Copy => (true, false),
+ ShortcutMode::Add => (true, true),
+ };
let scales = parameter_tree.leaf("scales")?.validate(&[element_count], DataType::F32)?.read_allocation()?;
+ let biases = config
+ .has_biases
+ .then(|| parameter_tree.leaf("biases")?.validate(&[element_count], DataType::F32)?.read_allocation())
+ .transpose()?;
let (scale_residual_sum, scale_output, post_layer_scalar_value) = match post_layer_scalar {
PostLayerScalar::None => (false, false, 1.0),
@@ -76,6 +90,7 @@ impl<B: Backend> Normalization<B> {
hadamard_factors.is_some(),
scale_residual_sum,
scale_output,
+ biases.is_some(),
)
.map_err(NormalizationNewError::Backend)?;
@@ -83,6 +98,7 @@ impl<B: Backend> Normalization<B> {
epsilon: config.epsilon,
scale_offset: config.scale_offset,
scales,
+ biases,
element_count,
hadamard_factors,
post_layer_scalar_value,
@@ -106,6 +122,7 @@ impl<B: Backend> Normalization<B> {
self.kernel.encode(
Some((input, row_offset_bytes)),
&self.scales,
+ self.biases.as_ref(),
&mut output,
shortcut,
self.hadamard_factors.as_ref(),
diff --git a/crates/backend-uzu/src/encodable_block/per_layer_embedding.rs b/crates/backend-uzu/src/encodable_block/per_layer_embedding.rs
index 705a5958f..180982a7c 100644
--- a/crates/backend-uzu/src/encodable_block/per_layer_embedding.rs
+++ b/crates/backend-uzu/src/encodable_block/per_layer_embedding.rs
@@ -15,7 +15,7 @@ use crate::{
data_type::DataType,
encodable_block::{
linear::{Linear, LinearBlockError},
- normalization::{Normalization, NormalizationNewError, PostLayerScalar},
+ normalization::{Normalization, NormalizationNewError, PostLayerScalar, ShortcutMode},
},
parameters::{ParameterLoaderError, ParameterTree},
};
@@ -83,8 +83,7 @@ impl<B: Backend> PerLayerEmbedding<B> {
let projection_norm = Normalization::new(
config.ple_dim,
None,
- false,
- false,
+ ShortcutMode::None,
PostLayerScalar::ScaleOutput(config.input_scale),
data_type,
&projection_norm_config,
@@ -200,8 +199,7 @@ impl<B: Backend> PerLayerEmbeddingProjection<B> {
let norm = Normalization::new(
model_dim,
None,
- false,
- false,
+ ShortcutMode::None,
PostLayerScalar::None,
data_type,
&config.norm_config,
diff --git a/crates/backend-uzu/src/encodable_block/prediction_head.rs b/crates/backend-uzu/src/encodable_block/prediction_head.rs
index 43b45f822..b08c0e5f8 100644
--- a/crates/backend-uzu/src/encodable_block/prediction_head.rs
+++ b/crates/backend-uzu/src/encodable_block/prediction_head.rs
@@ -10,7 +10,7 @@ use crate::{
data_type::DataType,
encodable_block::{
linear::{Linear, LinearBlockError},
- normalization::{Normalization, NormalizationNewError, PostLayerScalar},
+ normalization::{Normalization, NormalizationNewError, PostLayerScalar, ShortcutMode},
},
parameters::{ParameterLoaderError, ParameterTree},
};
@@ -61,8 +61,7 @@ impl<B: Backend> PredictionHead<B> {
let normalization = Normalization::new(
hidden_dim,
None,
- false,
- false,
+ ShortcutMode::None,
PostLayerScalar::None,
data_type,
&config.normalization_config,
diff --git a/crates/backend-uzu/src/encodable_block/transformer.rs b/crates/backend-uzu/src/encodable_block/transformer.rs
index ac31d1bc4..5cfe976ad 100644
--- a/crates/backend-uzu/src/encodable_block/transformer.rs
+++ b/crates/backend-uzu/src/encodable_block/transformer.rs
@@ -9,7 +9,7 @@ use crate::{
encodable_block::{
batch_topology::BatchTopology,
mixer::{MixerState, attention::rope::PrecalculatedRoPE},
- normalization::{Normalization, NormalizationNewError, PostLayerScalar},
+ normalization::{Normalization, NormalizationNewError, PostLayerScalar, ShortcutMode},
transformer_layer::{TransformerLayer, TransformerLayerError},
},
parameters::{ParameterLoaderError, ParameterTree},
@@ -132,8 +132,7 @@ impl<B: Backend> Transformer<B> {
let output_norm = Normalization::new(
transformer_config.model_dim,
output_norm_hadamard_factors,
- true,
- true,
+ ShortcutMode::Add,
PostLayerScalar::None,
data_type,
&transformer_config.output_norm_config,
diff --git a/crates/backend-uzu/src/encodable_block/transformer_layer.rs b/crates/backend-uzu/src/encodable_block/transformer_layer.rs
index a1bfbe756..5527ddcdf 100644
--- a/crates/backend-uzu/src/encodable_block/transformer_layer.rs
+++ b/crates/backend-uzu/src/encodable_block/transformer_layer.rs
@@ -8,7 +8,7 @@ use crate::{
batch_topology::BatchTopology,
mixer::{Mixer, MixerNewError, MixerState, attention::rope::PrecalculatedRoPE},
mlp::{Mlp, MlpBlockError},
- normalization::{Normalization, NormalizationNewError, PostLayerScalar},
+ normalization::{Normalization, NormalizationNewError, PostLayerScalar, ShortcutMode},
per_layer_embedding::PerLayerEmbeddingProjection,
},
parameters::{ParameterLoaderError, ParameterTree},
@@ -94,8 +94,11 @@ impl<B: Backend> TransformerLayer<B> {
Some(Normalization::new(
transformer_config.model_dim,
mixer_hadamard_factors,
- true,
- layer_index > 0,
+ if layer_index > 0 {
+ ShortcutMode::Add
+ } else {
+ ShortcutMode::Copy
+ },
PostLayerScalar::None,
data_type,
pre_mixer_norm_config,
@@ -113,8 +116,7 @@ impl<B: Backend> TransformerLayer<B> {
Some(Normalization::new(
transformer_config.model_dim,
None,
- false,
- false,
+ ShortcutMode::None,
PostLayerScalar::None,
data_type,
norm_config,
@@ -137,8 +139,7 @@ impl<B: Backend> TransformerLayer<B> {
let pre_mlp_norm = Normalization::new(
transformer_config.model_dim,
mlp_input_hadamard_factors,
- true,
- true,
+ ShortcutMode::Add,
residual_sum_scalar,
data_type,
&layer_config.pre_mlp_norm_config,
@@ -150,8 +151,7 @@ impl<B: Backend> TransformerLayer<B> {
Some(Normalization::new(
transformer_config.model_dim,
None,
- false,
- false,
+ ShortcutMode::None,
output_scalar,
data_type,
norm_config,
diff --git a/crates/backend-uzu/src/encodable_block/weaver.rs b/crates/backend-uzu/src/encodable_block/weaver.rs
index a321226c8..c57281cf1 100644
--- a/crates/backend-uzu/src/encodable_block/weaver.rs
+++ b/crates/backend-uzu/src/encodable_block/weaver.rs
@@ -3,11 +3,11 @@ use thiserror::Error;
use crate::{
array::size_for_shape,
backends::common::{
- Allocation, AllocationType, Backend, Context, Encoder, Kernels,
- gpu_types::weaver,
+ Allocation, AllocationType, AsBufferRangeMut, Backend, Context, Encoder, Kernels,
+ gpu_types::weaver::{CANDIDATES_MAX, MetadataIdx},
kernel::{
- ActivationKernel, AttentionLastQueryKernel, AttentionPrepareKernel, TensorAddBiasKernel,
- TensorAddSwapKernel, WeaverNodeCacheWriteKernel, WeaverTopChildrenKernel,
+ ActivationKernel, AncestorAttentionKernel, AttentionPrepareKernel, TensorAddBiasKernel,
+ WeaverNodeCacheWriteKernel, WeaverTopChildrenKernel,
},
},
config::{normalization::NormalizationConfig, weaver::WeaverConfig},
@@ -19,75 +19,88 @@ use crate::{
AttentionStateType,
core::{AttentionCoreEncodeArguments, AttentionCoreNewArguments, AttentionCores},
},
- normalization::{Normalization, NormalizationNewError, PostLayerScalar},
+ normalization::{Normalization, NormalizationNewError, PostLayerScalar, ShortcutMode},
},
parameters::{ParameterLoaderError, ParameterTree},
};
-pub(crate) struct WeaverPrefix<B: Backend> {
- layer_qkv: Box<[Allocation<B>]>,
- pub(crate) length: usize,
+const DATA_TYPE: DataType = DataType::BF16;
+
+pub(crate) struct TopKChildren<B: Backend> {
+ pub(crate) token_ids: Allocation<B>,
+ pub(crate) logprobs: Allocation<B>,
+}
+
+struct PrefixOutput<B: Backend> {
+ hidden: Allocation<B>,
+ kv_cache: Allocation<B>,
+}
+
+struct PrefixAttentionInputs<B: Backend> {
+ queries: Allocation<B>,
+ kv_cache: Allocation<B>,
+}
+
+pub(crate) struct WeaverPrefixKvCache<B: Backend> {
+ layer_kv: Box<[Allocation<B>]>,
+ token_count: usize,
+}
+
+pub(crate) struct WeaverNodeKvCache<B: Backend> {
+ layer_kv: Box<[Allocation<B>]>,
+ node_capacity: u32,
}
pub(crate) struct WeaverStepBatch<'a, B: Backend> {
- pub row_count: usize,
- pub candidate_count: usize,
+ pub node_count: usize,
+ pub candidates_per_node: usize,
pub ancestor_stride: usize,
- pub parent_token_ids: &'a Allocation<B>,
+ pub node_token_ids: &'a Allocation<B>,
pub candidate_ids: &'a Allocation<B>,
- pub candidate_scores: &'a Allocation<B>,
+ pub candidate_logits: &'a Allocation<B>,
pub ancestor_indices: &'a Allocation<B>,
pub node_metadata: &'a Allocation<B>,
}
-pub(crate) struct WeaverNodeState<B: Backend> {
- layer_qkv: Box<[Allocation<B>]>,
-}
-
pub(crate) struct Weaver<B: Backend> {
- embedding_norm: WeaverNorm<B>,
- hidden_state_norm: WeaverNorm<B>,
- embedding_projection: Box<dyn Linear<B>>,
- blocks: Box<[WeaverBlock<B>]>,
- output_norm: WeaverNorm<B>,
+ token_embedding_norm: Normalization<B>,
+ token_embedding_projection: Box<dyn Linear<B>>,
+ hidden_state_norm: Normalization<B>,
hidden_state_projection: Box<dyn Linear<B>>,
- query_projection: Box<dyn Linear<B>>,
+ blocks: Box<[WeaverBlock<B>]>,
+ readout_norm: Normalization<B>,
+ readout_query_projection: Box<dyn Linear<B>>,
position_embeddings: Allocation<B>,
- position_add: <B::Kernels as Kernels>::TensorAddBiasKernel,
- indexed_position_add: <B::Kernels as Kernels>::TensorAddBiasKernel,
+ prefix_position_add: <B::Kernels as Kernels>::TensorAddBiasKernel,
+ node_position_add: <B::Kernels as Kernels>::TensorAddBiasKernel,
top_children: <B::Kernels as Kernels>::WeaverTopChildrenKernel,
model_dim: usize,
target_model_dim: usize,
max_depth: usize,
- data_type: DataType,
-}
-
-struct WeaverNorm<B: Backend> {
- normalization: Normalization<B>,
- biases: Option<Allocation<B>>,
- bias_kernel: Option<<B::Kernels as Kernels>::TensorAddBiasKernel>,
- dimension: usize,
}
struct WeaverBlock<B: Backend> {
+ // Attention
+ pre_attention_norm: Normalization<B>,
qkv_projection: Box<dyn Linear<B>>,
- out_projection: Box<dyn Linear<B>>,
- prefix_attention: AttentionCores<B>,
attention_prepare: <B::Kernels as Kernels>::AttentionPrepareKernel,
- pre_attention_norm: WeaverNorm<B>,
- pre_mlp_norm: WeaverNorm<B>,
+ prefix_attention: AttentionCores<B>,
+ ancestor_attention: <B::Kernels as Kernels>::AncestorAttentionKernel,
+ node_cache_write: <B::Kernels as Kernels>::WeaverNodeCacheWriteKernel,
+ out_projection: Box<dyn Linear<B>>,
+
+ // MLP
+ pre_mlp_norm: Normalization<B>,
up_projection: Box<dyn Linear<B>>,
- down_projection: Box<dyn Linear<B>>,
activation: <B::Kernels as Kernels>::ActivationKernel,
- residual_add: <B::Kernels as Kernels>::TensorAddSwapKernel,
- last_query_attention: <B::Kernels as Kernels>::AttentionLastQueryKernel,
- node_cache_write: <B::Kernels as Kernels>::WeaverNodeCacheWriteKernel,
+ down_projection: Box<dyn Linear<B>>,
+
+ // Geometry
attention_scale: f32,
model_dim: usize,
hidden_dim: usize,
num_heads: usize,
head_dim: usize,
- data_type: DataType,
}
#[derive(Debug, Error)]
@@ -100,9 +113,11 @@ pub enum WeaverNewError<B: Backend> {
Normalization(#[from] NormalizationNewError<B>),
#[error("backend error: {0}")]
Backend(#[source] B::Error),
+ #[error("Weaver requires at least one layer")]
+ InvalidLayerCount,
#[error("model_dim must be divisible by num_heads")]
InvalidHeadConfig,
- #[error("candidate_pool_size must be in 1..={max}, got {0}", max = weaver::CANDIDATES_MAX)]
+ #[error("candidate_pool_size must be in 1..={max}, got {0}", max = CANDIDATES_MAX)]
InvalidCandidatePoolSize(usize),
}
@@ -114,120 +129,48 @@ pub enum WeaverEncodeError<B: Backend> {
Embedding(#[from] EmbeddingError<B>),
}
-fn linear<B: Backend>(
- context: &B::Context,
- parameter_tree: &ParameterTree<B>,
- name: &str,
- input_dim: usize,
- output_dim: usize,
- has_biases: bool,
- data_type: DataType,
-) -> Result<Box<dyn Linear<B>>, WeaverNewError<B>> {
- Ok(<dyn Linear<B>>::new(input_dim, [output_dim], has_biases, context, data_type, &parameter_tree.subtree(name)?)?)
-}
-
-impl<B: Backend> WeaverNorm<B> {
- fn new(
- context: &B::Context,
- dim: usize,
- config: &NormalizationConfig,
- parameter_tree: &ParameterTree<B>,
- data_type: DataType,
- ) -> Result<Self, WeaverNewError<B>> {
- let mut normalization_config = config.clone();
- normalization_config.has_biases = false;
- let normalization = Normalization::new(
- dim,
- None,
- false,
- false,
- PostLayerScalar::None,
- data_type,
- &normalization_config,
- parameter_tree,
- context,
- )?;
- let biases = config
- .has_biases
- .then(|| parameter_tree.leaf("biases")?.validate(&[dim], DataType::F32)?.read_allocation())
- .transpose()?;
- let bias_kernel = biases
- .as_ref()
- .map(|_| <B::Kernels as Kernels>::TensorAddBiasKernel::new(context, data_type, DataType::F32, true, false))
- .transpose()
- .map_err(WeaverNewError::Backend)?;
- Ok(Self {
- normalization,
- biases,
- bias_kernel,
- dimension: dim,
- })
- }
-
- fn encode(
- &self,
- input: &Allocation<B>,
- row_count: usize,
- encoder: &mut Encoder<B>,
- ) -> Result<Allocation<B>, B::Error> {
- let mut output = self.normalization.encode(input, 0, row_count, None, encoder)?;
- if let (Some(biases), Some(kernel)) = (&self.biases, &self.bias_kernel) {
- kernel.encode(
- None::<&Allocation<B>>,
- biases,
- None::<&Allocation<B>>,
- &mut output,
- self.dimension as u32,
- (row_count * self.dimension) as u32,
- encoder,
- );
- }
- Ok(output)
- }
-}
-
impl<B: Backend> Weaver<B> {
- pub(crate) fn create_node_state(
- &self,
- capacity: usize,
- context: &B::Context,
- ) -> Result<WeaverNodeState<B>, WeaverEncodeError<B>> {
- assert!(capacity > 0);
- let qkv_size = size_for_shape(&[capacity, 3, self.model_dim], self.data_type);
- let layer_qkv = (0..self.blocks.len())
- .map(|_| context.create_allocation(qkv_size, AllocationType::Global).map_err(WeaverEncodeError::Backend))
- .collect::<Result<Box<[_]>, _>>()?;
- Ok(WeaverNodeState {
- layer_qkv,
- })
- }
-
pub(crate) fn new(
context: &B::Context,
config: &WeaverConfig,
parameter_tree: &ParameterTree<B>,
- data_type: DataType,
) -> Result<Self, WeaverNewError<B>> {
- assert_eq!(data_type, DataType::BF16, "Weaver only supports BF16");
+ if config.num_layers == 0 {
+ return Err(WeaverNewError::InvalidLayerCount);
+ }
if config.num_heads == 0 || !config.model_dim.is_multiple_of(config.num_heads) {
return Err(WeaverNewError::InvalidHeadConfig);
}
- if config.candidate_pool_size == 0 || config.candidate_pool_size > weaver::CANDIDATES_MAX {
+ if config.candidate_pool_size == 0 || config.candidate_pool_size > CANDIDATES_MAX {
return Err(WeaverNewError::InvalidCandidatePoolSize(config.candidate_pool_size));
}
- let norm = |dim, name: &str| -> Result<_, WeaverNewError<B>> {
- WeaverNorm::new(context, dim, &config.norm_config, &parameter_tree.subtree(name)?, data_type)
- };
- let embedding_norm = norm(config.target_embedding_dim, "embedding_norm")?;
- let hidden_state_norm = norm(config.target_model_dim, "hidden_state_norm")?;
- let embedding_projection = linear(
+ let token_embedding_norm = Normalization::new(
+ config.target_embedding_dim,
+ None,
+ ShortcutMode::None,
+ PostLayerScalar::None,
+ DATA_TYPE,
+ &config.norm_config,
+ &parameter_tree.subtree("embedding_norm")?,
context,
- parameter_tree,
- "embedding_projection",
+ )?;
+ let hidden_state_norm = Normalization::new(
+ config.target_model_dim,
+ None,
+ ShortcutMode::None,
+ PostLayerScalar::None,
+ DATA_TYPE,
+ &config.norm_config,
+ &parameter_tree.subtree("hidden_state_norm")?,
+ context,
+ )?;
+ let token_embedding_projection = <dyn Linear<B>>::new(
config.target_embedding_dim,
- config.model_dim,
+ [config.model_dim],
true,
- data_type,
+ context,
+ DATA_TYPE,
+ &parameter_tree.subtree("embedding_projection")?,
)?;
let blocks_tree = parameter_tree.subtree("blocks")?;
let blocks = (0..config.num_layers)
@@ -237,207 +180,241 @@ impl<B: Backend> Weaver<B> {
config.model_dim,
config.hidden_dim,
config.num_heads,
+ index > 0,
&config.norm_config,
&blocks_tree.subtree(&index.to_string())?,
- data_type,
)
})
.collect::<Result<Box<[_]>, WeaverNewError<B>>>()?;
- let output_norm = norm(config.model_dim, "output_norm")?;
- let hidden_state_projection = linear(
+ let readout_norm = Normalization::new(
+ config.model_dim,
+ None,
+ ShortcutMode::Add,
+ PostLayerScalar::None,
+ DATA_TYPE,
+ &config.norm_config,
+ &parameter_tree.subtree("output_norm")?,
context,
- parameter_tree,
- "hidden_state_projection",
+ )?;
+ let hidden_state_projection = <dyn Linear<B>>::new(
config.target_model_dim,
- config.model_dim,
+ [config.model_dim],
true,
- data_type,
- )?;
- let query_projection = linear(
context,
- parameter_tree,
- "query_projection",
+ DATA_TYPE,
+ &parameter_tree.subtree("hidden_state_projection")?,
+ )?;
+ let readout_query_projection = <dyn Linear<B>>::new(
config.model_dim,
- config.target_model_dim,
+ [config.target_model_dim],
false,
- data_type,
+ context,
+ DATA_TYPE,
+ &parameter_tree.subtree("query_projection")?,
)?;
let position_embeddings = parameter_tree
.leaf("position_embeddings")?
.validate(&[config.max_depth, config.model_dim], DataType::F32)?
.read_allocation()?;
- let position_add =
- <B::Kernels as Kernels>::TensorAddBiasKernel::new(context, data_type, DataType::F32, true, false)
+ let prefix_position_add =
+ <B::Kernels as Kernels>::TensorAddBiasKernel::new(context, DATA_TYPE, DataType::F32, true, false)
.map_err(WeaverNewError::Backend)?;
- let indexed_position_add =
- <B::Kernels as Kernels>::TensorAddBiasKernel::new(context, data_type, DataType::F32, true, true)
+ let node_position_add =
+ <B::Kernels as Kernels>::TensorAddBiasKernel::new(context, DATA_TYPE, DataType::F32, true, true)
.map_err(WeaverNewError::Backend)?;
let top_children =
<B::Kernels as Kernels>::WeaverTopChildrenKernel::new(context).map_err(WeaverNewError::Backend)?;
Ok(Self {
- embedding_norm,
+ token_embedding_norm,
+ token_embedding_projection,
hidden_state_norm,
- embedding_projection,
- blocks,
- output_norm,
hidden_state_projection,
- query_projection,
+ blocks,
+ readout_norm,
+ readout_query_projection,
position_embeddings,
- position_add,
- indexed_position_add,
+ prefix_position_add,
+ node_position_add,
top_children,
model_dim: config.model_dim,
target_model_dim: config.target_model_dim,
max_depth: config.max_depth,
- data_type,
})
}
pub(crate) fn build_prefix(
&self,
target_hidden: &Allocation<B>,
- lookaheads: &Allocation<B>,
- lookahead_offset: usize,
+ draft_hidden: &Allocation<B>,
lookahead_count: usize,
encoder: &mut Encoder<B>,
- ) -> Result<WeaverPrefix<B>, WeaverEncodeError<B>> {
- let length = lookahead_count + 1;
- assert!(lookahead_count <= self.max_depth);
- let row_bytes = self.target_model_dim * self.data_type.size_in_bytes();
- assert!(target_hidden.size() >= row_bytes);
- assert!(lookaheads.size() >= (lookahead_offset + lookahead_count) * row_bytes);
+ ) -> Result<WeaverPrefixKvCache<B>, WeaverEncodeError<B>> {
+ assert!((1..=self.max_depth).contains(&lookahead_count));
+ let token_count = lookahead_count + 1;
+ let hidden_row_bytes = self.target_model_dim * DATA_TYPE.size_in_bytes();
- let context = encoder.context();
- let mut input = encoder
- .allocate_scratch(size_for_shape(&[length, self.target_model_dim], self.data_type))
+ let mut prefix_input = encoder
+ .allocate_scratch(size_for_shape(&[token_count, self.target_model_dim], DATA_TYPE))
+ .map_err(WeaverEncodeError::Backend)?;
+ encoder.encode_copy(target_hidden, 0..hidden_row_bytes, &mut prefix_input, 0..hidden_row_bytes);
+ encoder.encode_copy(
+ draft_hidden,
+ hidden_row_bytes..token_count * hidden_row_bytes,
+ &mut prefix_input,
+ hidden_row_bytes..token_count * hidden_row_bytes,
+ );
+ let normalized_prefix = self
+ .hidden_state_norm
+ .encode(&prefix_input, 0, token_count, None, encoder)
+ .map_err(WeaverEncodeError::Backend)?;
+ let mut hidden = self
+ .hidden_state_projection
+ .encode(normalized_prefix, token_count, encoder)
.map_err(WeaverEncodeError::Backend)?;
- encoder.encode_copy(target_hidden, 0..row_bytes, &mut input, 0..row_bytes);
- if lookahead_count > 0 {
- encoder.encode_copy(
- lookaheads,
- lookahead_offset * row_bytes..(lookahead_offset + lookahead_count) * row_bytes,
- &mut input,
- row_bytes..length * row_bytes,
- );
- }
- let normalized = self.hidden_state_norm.encode(&input, length, encoder).map_err(WeaverEncodeError::Backend)?;
- let mut hidden =
- self.hidden_state_projection.encode(normalized, length, encoder).map_err(WeaverEncodeError::Backend)?;
let position_elements = lookahead_count * self.model_dim;
- self.position_add.encode(
+ self.prefix_position_add.encode(
None::<&Allocation<B>>,
&self.position_embeddings,
None::<&Allocation<B>>,
- (&mut hidden, self.model_dim * self.data_type.size_in_bytes()),
+ (&mut hidden, self.model_dim * DATA_TYPE.size_in_bytes()),
position_elements as u32,
position_elements as u32,
encoder,
);
-
- let mut layer_qkv = Vec::with_capacity(self.blocks.len());
- for block in &self.blocks {
- let (next_hidden, qkv) =
- block.encode_prefix(hidden, length, encoder).map_err(WeaverEncodeError::Backend)?;
- let mut cached_qkv =
- context.create_allocation(qkv.size(), AllocationType::Global).map_err(WeaverEncodeError::Backend)?;
- encoder.encode_copy(&qkv, .., &mut cached_qkv, ..);
- layer_qkv.push(cached_qkv);
+ let (last_block, preceding_blocks) = self.blocks.split_last().expect("Weaver must have at least one block");
+ let mut residual = encoder.allocate_scratch(hidden.size()).map_err(WeaverEncodeError::Backend)?;
+ let mut layer_kv = Vec::with_capacity(self.blocks.len());
+ for block in preceding_blocks {
+ let PrefixOutput {
+ hidden: next_hidden,
+ kv_cache,
+ } = block.encode_prefix(hidden, &mut residual, token_count, encoder).map_err(WeaverEncodeError::Backend)?;
+ layer_kv.push(kv_cache);
hidden = next_hidden;
}
- drop(input);
- drop(hidden);
- Ok(WeaverPrefix {
- layer_qkv: layer_qkv.into_boxed_slice(),
- length,
+ layer_kv.push(
+ last_block
+ .prepare_prefix_attention(&hidden, &mut residual, token_count, encoder)
+ .map_err(WeaverEncodeError::Backend)?
+ .kv_cache,
+ );
+ Ok(WeaverPrefixKvCache {
+ layer_kv: layer_kv.into_boxed_slice(),
+ token_count,
+ })
+ }
+
+ pub(crate) fn create_node_kv_cache(
+ &self,
+ capacity: usize,
+ context: &B::Context,
+ ) -> Result<WeaverNodeKvCache<B>, WeaverEncodeError<B>> {
+ assert!(capacity > 0, "Weaver node capacity must be positive");
+ let node_capacity = u32::try_from(capacity).expect("Weaver node capacity exceeds the kernel limit");
+ let kv_size = size_for_shape(&[2, capacity, self.model_dim], DATA_TYPE);
+ let layer_kv = (0..self.blocks.len())
+ .map(|_| context.create_allocation(kv_size, AllocationType::Global).map_err(WeaverEncodeError::Backend))
+ .collect::<Result<Box<[_]>, _>>()?;
+ Ok(WeaverNodeKvCache {
+ layer_kv,
+ node_capacity,
})
}
pub(crate) fn encode_step_batch(
&self,
- prefix: &WeaverPrefix<B>,
- input: &WeaverStepBatch<'_, B>,
- state: &mut WeaverNodeState<B>,
- children: usize,
+ prefix: &WeaverPrefixKvCache<B>,
+ batch: &WeaverStepBatch<'_, B>,
+ node_cache: &mut WeaverNodeKvCache<B>,
+ children_per_node: usize,
target_embedding: &Embedding<B>,
encoder: &mut Encoder<B>,
- ) -> Result<(Allocation<B>, Allocation<B>), WeaverEncodeError<B>> {
- let rows = input.row_count;
- let candidates = input.candidate_count;
- assert!(rows > 0);
- assert!(candidates > 0 && candidates <= weaver::CANDIDATES_MAX);
- assert!(children > 0 && children <= candidates);
- assert!(input.ancestor_stride > 0);
- let word = DataType::U32.size_in_bytes();
- debug_assert!(input.node_metadata.size() >= weaver::METADATA_LANE_COUNT * rows * word);
- debug_assert!(input.parent_token_ids.size() >= rows * word);
- debug_assert!(input.ancestor_indices.size() >= rows * input.ancestor_stride * word);
-
- let token_embedding = target_embedding.encode_lookup(input.parent_token_ids, rows, encoder)?;
- let embedding_normalized =
- self.embedding_norm.encode(&token_embedding, rows, encoder).map_err(WeaverEncodeError::Backend)?;
- let mut current = self
- .embedding_projection
- .encode(embedding_normalized, rows, encoder)
+ ) -> Result<TopKChildren<B>, WeaverEncodeError<B>> {
+ let token_embedding = target_embedding.encode_lookup(batch.node_token_ids, batch.node_count, encoder)?;
+ let normalized_embedding = self
+ .token_embedding_norm
+ .encode(&token_embedding, 0, batch.node_count, None, encoder)
+ .map_err(WeaverEncodeError::Backend)?;
+ let mut hidden = self
+ .token_embedding_projection
+ .encode(normalized_embedding, batch.node_count, encoder)
.map_err(WeaverEncodeError::Backend)?;
- self.indexed_position_add.encode(
+ self.node_position_add.encode(
None::<&Allocation<B>>,
&self.position_embeddings,
- Some(input.node_metadata),
- &mut current,
+ Some(batch.node_metadata),
+ &mut hidden,
self.model_dim as u32,
- (rows * self.model_dim) as u32,
+ (batch.node_count * self.model_dim) as u32,
encoder,
);
+ let mut residual = encoder.allocate_scratch(hidden.size()).map_err(WeaverEncodeError::Backend)?;
+ let node_capacity = node_cache.node_capacity;
for (layer_index, block) in self.blocks.iter().enumerate() {
- current = block
+ hidden = block
.encode_step(
- current,
- &prefix.layer_qkv[layer_index],
- &mut state.layer_qkv[layer_index],
- input,
- prefix.length,
+ hidden,
+ &mut residual,
+ &prefix.layer_kv[layer_index],
+ &mut node_cache.layer_kv[layer_index],
+ node_capacity,
+ batch,
+ prefix.token_count,
encoder,
)
.map_err(WeaverEncodeError::Backend)?;
}
- self.encode_step_output(&current, input, children, target_embedding, encoder)
+ self.encode_step_output(&hidden, &mut residual, batch, children_per_node, target_embedding, encoder)
}
/// Outputs must outlive the batch.
fn encode_step_output(
&self,
- current: &Allocation<B>,
- step: &WeaverStepBatch<'_, B>,
- children: usize,
+ hidden: &Allocation<B>,
+ residual: &mut Allocation<B>,
+ batch: &WeaverStepBatch<'_, B>,
+ children_per_node: usize,
target_embedding: &Embedding<B>,
encoder: &mut Encoder<B>,
- ) -> Result<(Allocation<B>, Allocation<B>), WeaverEncodeError<B>> {
- let (rows, candidates) = (step.row_count, step.candidate_count);
- let output_normalized = self.output_norm.encode(current, rows, encoder).map_err(WeaverEncodeError::Backend)?;
- let query =
- self.query_projection.encode(output_normalized, rows, encoder).map_err(WeaverEncodeError::Backend)?;
- let candidate_logits =
- target_embedding.encode_readout_sparse(&query, step.candidate_ids, rows, candidates, encoder)?;
- let mut child_ids = encoder
- .allocate_scratch(size_for_shape(&[rows, children], DataType::U32))
+ ) -> Result<TopKChildren<B>, WeaverEncodeError<B>> {
+ let normalized_output = self
+ .readout_norm
+ .encode(hidden, 0, batch.node_count, Some(residual), encoder)
.map_err(WeaverEncodeError::Backend)?;
- let mut child_logprobs = encoder
- .allocate_scratch(size_for_shape(&[rows, children], DataType::F32))
+ let query = self
+ .readout_query_projection
+ .encode(normalized_output, batch.node_count, encoder)
+ .map_err(WeaverEncodeError::Backend)?;
+ let residual_logits = target_embedding.encode_readout_sparse(
+ &query,
+ batch.candidate_ids,
+ batch.node_count,
+ batch.candidates_per_node,
+ encoder,
+ )?;
+ let mut token_ids = encoder
+ .allocate_scratch(size_for_shape(&[batch.node_count, children_per_node], DataType::U32))
+ .map_err(WeaverEncodeError::Backend)?;
+ let mut logprobs = encoder
+ .allocate_scratch(size_for_shape(&[batch.node_count, children_per_node], DataType::F32))
.map_err(WeaverEncodeError::Backend)?;
self.top_children.encode(
- &candidate_logits,
- step.candidate_scores,
- step.candidate_ids,
- &mut child_ids,
- &mut child_logprobs,
- rows as u32,
- candidates as u32,
- children as u32,
+ &residual_logits,
+ batch.candidate_logits,
+ batch.candidate_ids,
+ &mut token_ids,
+ &mut logprobs,
+ batch.node_count as u32,
+ batch.candidates_per_node as u32,
+ children_per_node as u32,
encoder,
);
- Ok((child_ids, child_logprobs))
+ Ok(TopKChildren {
+ token_ids,
+ logprobs,
+ })
}
}
@@ -447,15 +424,28 @@ impl<B: Backend> WeaverBlock<B> {
model_dim: usize,
hidden_dim: usize,
num_heads: usize,
+ add_to_residual: bool,
norm_config: &NormalizationConfig,
parameter_tree: &ParameterTree<B>,
- data_type: DataType,
) -> Result<Self, WeaverNewError<B>> {
let head_dim = model_dim / num_heads;
let attention_scale = 1.0 / (head_dim as f32).sqrt();
- let qkv_projection =
- linear(context, parameter_tree, "qkv_projection", model_dim, 3 * model_dim, false, data_type)?;
- let out_projection = linear(context, parameter_tree, "out_projection", model_dim, model_dim, false, data_type)?;
+ let qkv_projection = <dyn Linear<B>>::new(
+ model_dim,
+ [3 * model_dim],
+ false,
+ context,
+ DATA_TYPE,
+ &parameter_tree.subtree("qkv_projection")?,
+ )?;
+ let out_projection = <dyn Linear<B>>::new(
+ model_dim,
+ [model_dim],
+ false,
+ context,
+ DATA_TYPE,
+ &parameter_tree.subtree("out_projection")?,
+ )?;
let prefix_attention = AttentionCores::new(
AttentionCoreNewArguments {
head_dim,
@@ -467,30 +457,60 @@ impl<B: Backend> WeaverBlock<B> {
is_trie: false,
sliding_window_size: None,
scale: Some(attention_scale),
- data_type,
+ data_type: DATA_TYPE,
},
context,
)
.map_err(WeaverNewError::Backend)?;
let attention_prepare =
- <B::Kernels as Kernels>::AttentionPrepareKernel::new(context, data_type, DataType::F32, true, false)
+ <B::Kernels as Kernels>::AttentionPrepareKernel::new(context, DATA_TYPE, DataType::F32, true, false)
.map_err(WeaverNewError::Backend)?;
- let norm = |name: &str| -> Result<_, WeaverNewError<B>> {
- WeaverNorm::new(context, model_dim, norm_config, &parameter_tree.subtree(name)?, data_type)
- };
- let pre_attention_norm = norm("pre_attention_norm")?;
- let pre_mlp_norm = norm("pre_mlp_norm")?;
- let up_projection = linear(context, parameter_tree, "up_projection", model_dim, hidden_dim, true, data_type)?;
- let down_projection =
- linear(context, parameter_tree, "down_projection", hidden_dim, model_dim, true, data_type)?;
- let activation = <B::Kernels as Kernels>::ActivationKernel::new(context, data_type, true)
+ let pre_attention_norm = Normalization::new(
+ model_dim,
+ None,
+ if add_to_residual {
+ ShortcutMode::Add
+ } else {
+ ShortcutMode::Copy
+ },
+ PostLayerScalar::None,
+ DATA_TYPE,
+ norm_config,
+ &parameter_tree.subtree("pre_attention_norm")?,
+ context,
+ )?;
+ let pre_mlp_norm = Normalization::new(
+ model_dim,
+ None,
+ ShortcutMode::Add,
+ PostLayerScalar::None,
+ DATA_TYPE,
+ norm_config,
+ &parameter_tree.subtree("pre_mlp_norm")?,
+ context,
+ )?;
+ let up_projection = <dyn Linear<B>>::new(
+ model_dim,
+ [hidden_dim],
+ true,
+ context,
+ DATA_TYPE,
+ &parameter_tree.subtree("up_projection")?,
+ )?;
+ let down_projection = <dyn Linear<B>>::new(
+ hidden_dim,
+ [model_dim],
+ true,
+ context,
+ DATA_TYPE,
+ &parameter_tree.subtree("down_projection")?,
+ )?;
+ let activation = <B::Kernels as Kernels>::ActivationKernel::new(context, DATA_TYPE, true)
.map_err(WeaverNewError::Backend)?;
- let residual_add =
- <B::Kernels as Kernels>::TensorAddSwapKernel::new(context, data_type).map_err(WeaverNewError::Backend)?;
- let last_query_attention =
- <B::Kernels as Kernels>::AttentionLastQueryKernel::new(context, head_dim as u32, num_heads as u32)
+ let ancestor_attention =
+ <B::Kernels as Kernels>::AncestorAttentionKernel::new(context, head_dim as u32, num_heads as u32)
.map_err(WeaverNewError::Backend)?;
- let node_cache_write = <B::Kernels as Kernels>::WeaverNodeCacheWriteKernel::new(context, data_type)
+ let node_cache_write = <B::Kernels as Kernels>::WeaverNodeCacheWriteKernel::new(context, DATA_TYPE)
.map_err(WeaverNewError::Backend)?;
Ok(Self {
qkv_projection,
@@ -502,45 +522,114 @@ impl<B: Backend> WeaverBlock<B> {
up_projection,
down_projection,
activation,
- residual_add,
- last_query_attention,
+ ancestor_attention,
node_cache_write,
attention_scale,
model_dim,
hidden_dim,
num_heads,
head_dim,
- data_type,
+ })
+ }
+
+ fn encode_prefix(
+ &self,
+ input: Allocation<B>,
+ residual: &mut Allocation<B>,
+ token_count: usize,
+ encoder: &mut Encoder<B>,
+ ) -> Result<PrefixOutput<B>, B::Error> {
+ let PrefixAttentionInputs {
+ queries,
+ kv_cache,
+ } = self.prepare_prefix_attention(&input, residual, token_count, encoder)?;
+ let state_type = AttentionStateType::Full {
+ length: 0,
+ };
+ let kv_plane_bytes = size_for_shape(&[token_count, self.model_dim], DATA_TYPE);
+ let attention = self.prefix_attention.encode(
+ AttentionCoreEncodeArguments {
+ queries: &queries,
+ keys: &kv_cache,
+ values: (&kv_cache, kv_plane_bytes),
+ suffix_length: token_count,
+ trie: None,
+ sinks: None,
+ state_type: &state_type,
+ },
+ encoder,
+ )?;
+ let hidden = self.encode_post_attention(attention, residual, token_count, encoder)?;
+ Ok(PrefixOutput {
+ hidden,
+ kv_cache,
+ })
+ }
+
+ fn prepare_prefix_attention(
+ &self,
+ input: &Allocation<B>,
+ residual: &mut Allocation<B>,
+ token_count: usize,
+ encoder: &mut Encoder<B>,
+ ) -> Result<PrefixAttentionInputs<B>, B::Error> {
+ let attention_input = self.pre_attention_norm.encode(input, 0, token_count, Some(residual), encoder)?;
+ let qkv = self.qkv_projection.encode(attention_input, token_count, encoder)?;
+ let mut queries =
+ encoder.allocate_scratch(size_for_shape(&[self.num_heads, token_count, self.head_dim], DATA_TYPE))?;
+ let kv_plane_bytes = size_for_shape(&[token_count, self.model_dim], DATA_TYPE);
+ let mut kv = encoder
+ .context()
+ .create_allocation(size_for_shape(&[2, token_count, self.model_dim], DATA_TYPE), AllocationType::Global)?;
+ let (keys, values) = kv.as_buffer_range_mut().split_at(kv_plane_bytes);
+ self.attention_prepare.encode(
+ &qkv,
+ &mut queries,
+ Some(keys),
+ Some(values),
+ None::<&Allocation<B>>,
+ None::<&Allocation<B>>,
+ self.num_heads as u32,
+ Some(self.num_heads as u32),
+ self.head_dim as u32,
+ None,
+ Some(0),
+ token_count as u32,
+ encoder,
+ );
+ Ok(PrefixAttentionInputs {
+ queries,
+ kv_cache: kv,
})
}
fn encode_step(
&self,
- current: Allocation<B>,
- prefix_qkv: &Allocation<B>,
- state_qkv: &mut Allocation<B>,
- step: &WeaverStepBatch<'_, B>,
+ input: Allocation<B>,
+ residual: &mut Allocation<B>,
+ prefix_kv: &Allocation<B>,
+ node_kv_cache: &mut Allocation<B>,
+ node_capacity: u32,
+ batch: &WeaverStepBatch<'_, B>,
prefix_length: usize,
encoder: &mut Encoder<B>,
) -> Result<Allocation<B>, B::Error> {
- let rows = step.row_count;
- let normalized = self.pre_attention_norm.encode(&current, rows, encoder)?;
- let current_qkv = self.qkv_projection.encode(normalized, rows, encoder)?;
- let metadata_row_bytes = rows * DataType::U32.size_in_bytes();
- let ancestor_counts = (step.node_metadata, metadata_row_bytes);
- let node_indices = (step.node_metadata, 2 * metadata_row_bytes);
- let mut attention = encoder.allocate_scratch(size_for_shape(&[rows, self.model_dim], self.data_type))?;
- let node_capacity = (state_qkv.size() / size_for_shape(&[3, self.model_dim], self.data_type)) as u32;
- self.last_query_attention.encode(
- prefix_qkv,
- &*state_qkv,
+ let attention_input = self.pre_attention_norm.encode(&input, 0, batch.node_count, Some(residual), encoder)?;
+ let current_qkv = self.qkv_projection.encode(attention_input, batch.node_count, encoder)?;
+ let metadata_lane_bytes = batch.node_count * DataType::U32.size_in_bytes();
+ let ancestor_counts = (batch.node_metadata, MetadataIdx::AncestorCount as usize * metadata_lane_bytes);
+ let tree_slots = (batch.node_metadata, MetadataIdx::TreeSlot as usize * metadata_lane_bytes);
+ let mut attention = encoder.allocate_scratch(size_for_shape(&[batch.node_count, self.model_dim], DATA_TYPE))?;
+ self.ancestor_attention.encode(
+ prefix_kv,
+ &*node_kv_cache,
&current_qkv,
- step.ancestor_indices,
+ batch.ancestor_indices,
ancestor_counts,
&mut attention,
- rows as u32,
+ batch.node_count as u32,
prefix_length as u32,
- step.ancestor_stride as u32,
+ batch.ancestor_stride as u32,
node_capacity,
self.attention_scale,
encoder,
@@ -549,86 +638,33 @@ impl<B: Backend> WeaverBlock<B> {
// serializes it after the attention that reads the ancestor slots.
self.node_cache_write.encode(
&current_qkv,
- state_qkv,
- node_indices,
+ node_kv_cache,
+ tree_slots,
self.model_dim as u32,
node_capacity,
- (rows * 2 * self.model_dim) as u32,
- encoder,
- );
- attention = self.out_projection.encode(attention, rows, encoder)?;
- let mut residual = current;
- self.residual_add.encode(&mut residual, &mut attention, (rows * self.model_dim) as u32, encoder);
- let normalized = self.pre_mlp_norm.encode(&residual, rows, encoder)?;
- let mut mlp = self.up_projection.encode(normalized, rows, encoder)?;
- self.activation.encode(
- None::<&Allocation<B>>,
- &mut mlp,
- (rows * self.hidden_dim) as u32,
- crate::backends::common::gpu_types::ActivationType::GELUExact,
+ (batch.node_count * 2 * self.model_dim) as u32,
encoder,
);
- let mut output = self.down_projection.encode(mlp, rows, encoder)?;
- self.residual_add.encode(&mut output, &mut residual, (rows * self.model_dim) as u32, encoder);
- Ok(residual)
+ self.encode_post_attention(attention, residual, batch.node_count, encoder)
}
- fn encode_prefix(
+ fn encode_post_attention(
&self,
- hidden: Allocation<B>,
- rows: usize,
+ attention: Allocation<B>,
+ residual: &mut Allocation<B>,
+ row_count: usize,
encoder: &mut Encoder<B>,
- ) -> Result<(Allocation<B>, Allocation<B>), B::Error> {
- let normalized = self.pre_attention_norm.encode(&hidden, rows, encoder)?;
- let qkv = self.qkv_projection.encode(normalized, rows, encoder)?;
- let mut queries =
- encoder.allocate_scratch(size_for_shape(&[self.num_heads, rows, self.head_dim], self.data_type))?;
- let mut keys = encoder.allocate_scratch(size_for_shape(&[rows, self.model_dim], self.data_type))?;
- let mut values = encoder.allocate_scratch(size_for_shape(&[rows, self.model_dim], self.data_type))?;
- self.attention_prepare.encode(
- &qkv,
- &mut queries,
- Some(&mut keys),
- Some(&mut values),
- None::<&Allocation<B>>,
- None::<&Allocation<B>>,
- self.num_heads as u32,
- Some(self.num_heads as u32),
- self.head_dim as u32,
- None,
- Some(0),
- rows as u32,
- encoder,
- );
- let state_type = AttentionStateType::Full {
- length: 0,
- };
- let attention = self.prefix_attention.encode(
- AttentionCoreEncodeArguments {
- queries: &queries,
- keys: &keys,
- values: &values,
- suffix_length: rows,
- trie: None,
- sinks: None,
- state_type: &state_type,
- },
- encoder,
- )?;
- let mut attention = self.out_projection.encode(attention, rows, encoder)?;
- let mut residual = hidden;
- self.residual_add.encode(&mut residual, &mut attention, (rows * self.model_dim) as u32, encoder);
- let normalized = self.pre_mlp_norm.encode(&residual, rows, encoder)?;
- let mut mlp = self.up_projection.encode(normalized, rows, encoder)?;
+ ) -> Result<Allocation<B>, B::Error> {
+ let projected_attention = self.out_projection.encode(attention, row_count, encoder)?;
+ let mlp_input = self.pre_mlp_norm.encode(&projected_attention, 0, row_count, Some(residual), encoder)?;
+ let mut mlp_hidden = self.up_projection.encode(mlp_input, row_count, encoder)?;
self.activation.encode(
None::<&Allocation<B>>,
- &mut mlp,
- (rows * self.hidden_dim) as u32,
+ &mut mlp_hidden,
+ (row_count * self.hidden_dim) as u32,
crate::backends::common::gpu_types::ActivationType::GELUExact,
encoder,
);
- let mut output = self.down_projection.encode(mlp, rows, encoder)?;
- self.residual_add.encode(&mut output, &mut residual, (rows * self.model_dim) as u32, encoder);
- Ok((residual, qkv))
+ self.down_projection.encode(mlp_hidden, row_count, encoder)
}
}
diff --git a/crates/backend-uzu/src/engine/dflash_speculator.rs b/crates/backend-uzu/src/engine/dflash_speculator.rs
index 306f05cce..3fe5dbb4d 100644
--- a/crates/backend-uzu/src/engine/dflash_speculator.rs
+++ b/crates/backend-uzu/src/engine/dflash_speculator.rs
@@ -66,9 +66,7 @@ impl<B: Backend> Engine<B> {
let weaver = speculator_config
.weaver_config
.as_ref()
- .map(|weaver_config| {
- Weaver::new(&*self.context, weaver_config, &speculator_tree.subtree("weaver")?, data_type)
- })
+ .map(|weaver_config| Weaver::new(&*self.context, weaver_config, &speculator_tree.subtree("weaver")?))
.transpose()?;
weight_loader.tree().assert_all_tensors_validated()?;
diff --git a/crates/backend-uzu/src/speculators/dflash_speculator.rs b/crates/backend-uzu/src/speculators/dflash_speculator.rs
index 0e4b58335..20591d36d 100644
--- a/crates/backend-uzu/src/speculators/dflash_speculator.rs
+++ b/crates/backend-uzu/src/speculators/dflash_speculator.rs
@@ -6,7 +6,7 @@ pub use crate::encodable_block::dflash::DFlashState;
use crate::{
backends::common::{
Allocation, AllocationType, Backend, Context, Encoder, Kernels,
- gpu_types::weaver,
+ gpu_types::weaver::{FRONTIER_MAX_SLOTS, FRONTIER_NO_WINNER, FrontierIdx, MetadataIdx, TreeIdx},
kernel::{WeaverFrontierScatterKernel, WeaverFrontierSelectKernel},
},
config::speculator::dflash::DFlashSpeculatorConfig,
@@ -14,7 +14,7 @@ use crate::{
encodable_block::{
dflash::DFlashDraft,
embedding::{Embedding, EmbeddingError},
- weaver::{Weaver, WeaverEncodeError, WeaverNodeState, WeaverPrefix, WeaverStepBatch},
+ weaver::{Weaver, WeaverEncodeError, WeaverNodeKvCache, WeaverPrefixKvCache, WeaverStepBatch},
},
engine::language_model::LanguageModel,
};
@@ -158,14 +158,14 @@ impl<B: Backend> DFlashSpeculator<B> {
let lookahead_count = max_depth.min(block_size.saturating_sub(1));
// The frontier holds one slot per (tree slot, child) pair; the select
// kernel silently no-ops past its capacity.
- if (options.budget + 1) * options.children_per_node > weaver::FRONTIER_MAX_SLOTS || lookahead_count == 0 {
+ if (options.budget + 1) * options.children_per_node > FRONTIER_MAX_SLOTS {
return Err(DFlashTreeError::InvalidOptions);
}
let target_hidden = state.target_output_norm().ok_or(DFlashTreeError::MissingTargetHidden)?;
- let prefix = weaver.build_prefix(target_hidden, &draft_hidden, 1, lookahead_count, &mut encoder)?;
+ let prefix = weaver.build_prefix(target_hidden, &draft_hidden, lookahead_count, &mut encoder)?;
drop(draft_hidden);
drop(draft_logits);
- let mut weaver_state = weaver.create_node_state(options.budget + 1, &self.context)?;
+ let mut weaver_state = weaver.create_node_kv_cache(options.budget + 1, &self.context)?;
let arguments = TreeEncodingArguments {
weaver,
prefix: &prefix,
@@ -310,12 +310,12 @@ impl<B: Backend> DFlashSpeculator<B> {
&self,
encoder: &mut Encoder<B>,
params: &TreeEncodingArguments<'_, B>,
- state: &mut WeaverNodeState<B>,
+ state: &mut WeaverNodeKvCache<B>,
) -> Result<Allocation<B>, DFlashTreeError<B>> {
let context = &*self.context;
let slots = params.options.budget + 1;
- let fanout = params.options.children_per_node;
- let capacity = slots * fanout;
+ let children_per_node = params.options.children_per_node;
+ let capacity = slots * children_per_node;
let width = params.options.frontier_width;
let stride = params.max_depth;
let pool_size = params.pool_size;
@@ -325,16 +325,16 @@ impl<B: Backend> DFlashSpeculator<B> {
let scatter =
<B::Kernels as Kernels>::WeaverFrontierScatterKernel::new(context).map_err(DFlashTreeError::Backend)?;
- let mut tree_values = vec![0u32; weaver::TREE_LANE_COUNT * slots];
+ let mut tree_values = vec![0u32; TreeIdx::COUNT * slots];
for slot in 0..slots {
- tree_values[weaver::TREE_LANE_PARENT * slots + slot] = weaver::FRONTIER_NO_WINNER;
+ tree_values[TreeIdx::ParentSlot as usize * slots + slot] = FRONTIER_NO_WINNER;
}
- tree_values[weaver::TREE_LANE_TOKEN * slots] = params.bonus_token;
- tree_values[weaver::TREE_LANE_MASK * slots] = 1;
+ tree_values[TreeIdx::TokenId as usize * slots] = params.bonus_token;
+ tree_values[TreeIdx::Valid as usize * slots] = 1;
let mut tree = encoder.allocate_constant_from_slice(&tree_values).map_err(DFlashTreeError::Backend)?;
let mut frontier = encoder
- .allocate_constant_from_slice(&vec![0u32; weaver::FRONTIER_LANE_COUNT * capacity])
+ .allocate_constant_from_slice(&vec![0u32; FrontierIdx::COUNT * capacity])
.map_err(DFlashTreeError::Backend)?;
let mut slot_ancestors =
encoder.allocate_constant_from_slice(&vec![0u32; slots * stride]).map_err(DFlashTreeError::Backend)?;
@@ -346,7 +346,7 @@ impl<B: Backend> DFlashSpeculator<B> {
let mut round_token_ids =
encoder.allocate_constant_from_slice(&round_token_id_values).map_err(DFlashTreeError::Backend)?;
let mut round_metadata = encoder
- .allocate_constant_from_slice(&vec![0u32; weaver::METADATA_LANE_COUNT * width])
+ .allocate_constant_from_slice(&vec![0u32; MetadataIdx::COUNT * width])
.map_err(DFlashTreeError::Backend)?;
let mut round_ancestors =
encoder.allocate_constant_from_slice(&vec![0u32; width * stride]).map_err(DFlashTreeError::Backend)?;
@@ -395,20 +395,20 @@ impl<B: Backend> DFlashSpeculator<B> {
(&round_candidate_ids, &round_candidate_scores)
};
let input = WeaverStepBatch {
- row_count: rows,
- candidate_count: pool_size,
+ node_count: rows,
+ candidates_per_node: pool_size,
ancestor_stride: stride,
- parent_token_ids: &round_token_ids,
+ node_token_ids: &round_token_ids,
candidate_ids,
- candidate_scores,
+ candidate_logits: candidate_scores,
ancestor_indices: &round_ancestors,
node_metadata: &round_metadata,
};
- let (child_ids, child_logprobs) = params.weaver.encode_step_batch(
+ let children = params.weaver.encode_step_batch(
params.prefix,
&input,
state,
- fanout,
+ children_per_node,
params.target_embedding,
encoder,
)?;
@@ -416,17 +416,16 @@ impl<B: Backend> DFlashSpeculator<B> {
&tree,
&round_metadata,
&round_valid,
- &child_ids,
- &child_logprobs,
+ &children.token_ids,
+ &children.logprobs,
&mut frontier,
capacity as u32,
slots as u32,
rows as u32,
- fanout as u32,
+ children_per_node as u32,
encoder,
);
- drop(child_ids);
- drop(child_logprobs);
+ drop(children);
slot_start += rows;
}
Ok(tree)
@@ -434,20 +433,16 @@ impl<B: Backend> DFlashSpeculator<B> {
}
fn tree_from_slots(tree: &[u32]) -> Vec<HostTreeNode> {
- assert!(
- tree.len().is_multiple_of(weaver::TREE_LANE_COUNT),
- "tree array must contain {} equal-length lanes",
- weaver::TREE_LANE_COUNT
- );
- let slots = tree.len() / weaver::TREE_LANE_COUNT;
- let lane = |lane: usize, slot: usize| tree[lane * slots + slot];
+ assert!(tree.len().is_multiple_of(TreeIdx::COUNT), "tree array must contain {} equal-length lanes", TreeIdx::COUNT);
+ let slots = tree.len() / TreeIdx::COUNT;
+ let field = |field: TreeIdx, slot: usize| tree[field as usize * slots + slot];
let mut slot_to_node = vec![usize::MAX; slots];
let mut nodes: Vec<HostTreeNode> = Vec::with_capacity(slots);
for slot in 0..slots {
- if lane(weaver::TREE_LANE_MASK, slot) == 0 {
+ if field(TreeIdx::Valid, slot) == 0 {
continue;
}
- let parent_slot = lane(weaver::TREE_LANE_PARENT, slot) as i32;
+ let parent_slot = field(TreeIdx::ParentSlot, slot) as i32;
let parent = (parent_slot >= 0).then(|| {
let parent = slot_to_node[parent_slot as usize];
assert_ne!(parent, usize::MAX, "tree slot {slot} names padding slot {parent_slot} as its parent");
@@ -459,10 +454,10 @@ fn tree_from_slots(tree: &[u32]) -> Vec<HostTreeNode> {
nodes[parent].children.push(index);
}
nodes.push(HostTreeNode {
- token: lane(weaver::TREE_LANE_TOKEN, slot),
+ token: field(TreeIdx::TokenId, slot),
parent,
- depth: lane(weaver::TREE_LANE_DEPTH, slot) as usize,
- logprob: f32::from_bits(lane(weaver::TREE_LANE_LOGPROB, slot)),
+ depth: field(TreeIdx::Depth, slot) as usize,
+ logprob: f32::from_bits(field(TreeIdx::EdgeLogprobBits, slot)),
children: Vec::new(),
});
}
@@ -471,7 +466,7 @@ fn tree_from_slots(tree: &[u32]) -> Vec<HostTreeNode> {
struct TreeEncodingArguments<'a, B: Backend> {
weaver: &'a Weaver<B>,
- prefix: &'a WeaverPrefix<B>,
+ prefix: &'a WeaverPrefixKvCache<B>,
target_embedding: &'a Embedding<B>,
pool_ids: &'a Allocation<B>,
pool_scores: &'a Allocation<B>,
diff --git a/crates/backend-uzu/tests/unit/backends/common/allocator/allocator.rs b/crates/backend-uzu/tests/unit/backends/common/allocator/allocator.rs
index e7e193182..3a5523aa4 100644
--- a/crates/backend-uzu/tests/unit/backends/common/allocator/allocator.rs
+++ b/crates/backend-uzu/tests/unit/backends/common/allocator/allocator.rs
@@ -10,7 +10,10 @@ use proc_macros::uzu_test;
use serde::{Deserialize, Serialize};
use crate::backends::{
- common::{Allocation, AllocationPool, AllocationType, Allocator, Backend, Context},
+ common::{
+ Allocation, AllocationPool, AllocationType, Allocator, AsBufferRangeMut, AsBufferRangeRef, Backend,
+ BufferArgMut, Context,
+ },
metal::Metal,
};
@@ -127,3 +130,18 @@ fn bench_allocator_generation_trace() {
eprintln!("Cpu time: {} ms", elapsed.as_millis());
eprintln!("--------------------");
}
+
+#[uzu_test]
+#[ignore]
+fn allocation_split_at_mut_returns_disjoint_buffer_arguments() {
+ let context = <<Metal as Backend>::Context as Context>::new().unwrap();
+ let mut allocation = context.create_allocation(64, AllocationType::Global).unwrap();
+ let allocation_range = allocation.as_buffer_range_ref().range();
+
+ let (head, tail) = allocation.as_buffer_range_mut().split_at(24);
+ let (_, head_offset, head_length) = BufferArgMut::<'_, Metal>::into_parts(head);
+ let (_, tail_offset, tail_length) = BufferArgMut::<'_, Metal>::into_parts(tail);
+
+ assert_eq!((head_offset, head_length), (allocation_range.start, 24));
+ assert_eq!((tail_offset, tail_length), (allocation_range.start + 24, 40));
+}
diff --git a/crates/backend-uzu/tests/unit/backends/common/kernel/attention/attention_last_query_test.rs b/crates/backend-uzu/tests/unit/backends/common/kernel/attention/ancestor_attention_test.rs
similarity index 77%
rename from crates/backend-uzu/tests/unit/backends/common/kernel/attention/attention_last_query_test.rs
rename to crates/backend-uzu/tests/unit/backends/common/kernel/attention/ancestor_attention_test.rs
index 64c1228ed..e6fcc7656 100644
--- a/crates/backend-uzu/tests/unit/backends/common/kernel/attention/attention_last_query_test.rs
+++ b/crates/backend-uzu/tests/unit/backends/common/kernel/attention/ancestor_attention_test.rs
@@ -7,7 +7,7 @@ use proc_macros::uzu_test;
use crate::{
backends::{
- common::{Allocation, Backend, Encoder, Kernels, kernel::AttentionLastQueryKernel},
+ common::{Allocation, Backend, Encoder, Kernels, kernel::AncestorAttentionKernel},
cpu::Cpu,
metal::Metal,
},
@@ -22,9 +22,9 @@ const NUM_HEADS: u32 = 16;
struct Runner<B: Backend> {
context: Arc<B::Context>,
- kernel: <B::Kernels as Kernels>::AttentionLastQueryKernel,
- prefix_qkv: Allocation<B>,
- node_qkv: Allocation<B>,
+ kernel: <B::Kernels as Kernels>::AncestorAttentionKernel,
+ prefix_kv: Allocation<B>,
+ node_kv: Allocation<B>,
current_qkv: Allocation<B>,
ancestor_indices: Allocation<B>,
ancestor_counts: Allocation<B>,
@@ -43,7 +43,9 @@ impl<B: Backend> Runner<B> {
nodes: usize,
) -> Self {
assert!(rows < nodes && ancestor_stride > 0);
- let packed_width = 3 * NUM_HEADS as usize * HEAD_DIM;
+ let model_dim = NUM_HEADS as usize * HEAD_DIM;
+ let qkv_width = 3 * model_dim;
+ let kv_width = 2 * model_dim;
let values = |length, offset| {
(0..length)
.map(|index| bf16::from_f32((((index + offset) * 17 % 251) as f32 - 125.0) / 128.0))
@@ -61,12 +63,12 @@ impl<B: Backend> Runner<B> {
let context = create_context::<B>();
let kernel =
- <B::Kernels as Kernels>::AttentionLastQueryKernel::new(context.as_ref(), HEAD_DIM as u32, NUM_HEADS)
+ <B::Kernels as Kernels>::AncestorAttentionKernel::new(context.as_ref(), HEAD_DIM as u32, NUM_HEADS)
.unwrap();
Self {
- prefix_qkv: alloc_allocation_with_data::<B, bf16>(&context, &values(prefix_length * packed_width, 0)),
- node_qkv: alloc_allocation_with_data::<B, bf16>(&context, &values(nodes * packed_width, 11)),
- current_qkv: alloc_allocation_with_data::<B, bf16>(&context, &values(rows * packed_width, 29)),
+ prefix_kv: alloc_allocation_with_data::<B, bf16>(&context, &values(prefix_length * kv_width, 0)),
+ node_kv: alloc_allocation_with_data::<B, bf16>(&context, &values(nodes * kv_width, 11)),
+ current_qkv: alloc_allocation_with_data::<B, bf16>(&context, &values(rows * qkv_width, 29)),
ancestor_indices: alloc_allocation_with_data::<B, u32>(&context, &ancestor_indices),
ancestor_counts: alloc_allocation_with_data::<B, u32>(&context, &ancestor_counts),
output: alloc_allocation::<B, bf16>(&context, rows * NUM_HEADS as usize * HEAD_DIM),
@@ -86,8 +88,8 @@ impl<B: Backend> Runner<B> {
let mut encoder = Encoder::new(self.context.as_ref()).unwrap();
for _ in 0..repetitions {
self.kernel.encode(
- &self.prefix_qkv,
- &self.node_qkv,
+ &self.prefix_kv,
+ &self.node_kv,
&self.current_qkv,
&self.ancestor_indices,
&self.ancestor_counts,
@@ -105,25 +107,25 @@ impl<B: Backend> Runner<B> {
}
#[uzu_test]
-fn attention_last_query_matches_cpu() {
+fn ancestor_attention_matches_cpu() {
let mut cpu = Runner::<Cpu>::new(4, 5, 3, 16);
- let initial_nodes = allocation_to_vec::<Cpu, bf16>(&cpu.node_qkv);
+ let initial_nodes = allocation_to_vec::<Cpu, bf16>(&cpu.node_kv);
cpu.encode(1);
let expected_output = allocation_to_vec::<Cpu, bf16>(&cpu.output);
- assert_eq!(allocation_to_vec::<Cpu, bf16>(&cpu.node_qkv), initial_nodes);
+ assert_eq!(allocation_to_vec::<Cpu, bf16>(&cpu.node_kv), initial_nodes);
let mut metal = Runner::<Metal>::new(4, 5, 3, 16);
metal.encode(1);
let actual_output = allocation_to_vec::<Metal, bf16>(&metal.output);
- assert_eq_float(&expected_output, &actual_output, 0.02, "AttentionLastQuery output");
- assert_eq!(allocation_to_vec::<Metal, bf16>(&metal.node_qkv), initial_nodes);
+ assert_eq_float(&expected_output, &actual_output, 0.02, "AncestorAttention output");
+ assert_eq!(allocation_to_vec::<Metal, bf16>(&metal.node_kv), initial_nodes);
}
#[uzu_test]
#[ignore = "benchmark"]
-fn benchmark_attention_last_query() {
+fn benchmark_ancestor_attention() {
const BATCH: u32 = 32;
const SAMPLES: usize = 50;
@@ -135,5 +137,5 @@ fn benchmark_attention_last_query() {
}
let mut samples = (0..SAMPLES).map(|_| run()).collect::<Vec<_>>();
samples.sort_unstable();
- eprintln!("attention_last_query gpu={:?}", samples[SAMPLES / 2]);
+ eprintln!("ancestor_attention gpu={:?}", samples[SAMPLES / 2]);
}
diff --git a/crates/backend-uzu/tests/unit/backends/common/kernel/attention/mod.rs b/crates/backend-uzu/tests/unit/backends/common/kernel/attention/mod.rs
index 0b723476d..2d613a765 100644
--- a/crates/backend-uzu/tests/unit/backends/common/kernel/attention/mod.rs
+++ b/crates/backend-uzu/tests/unit/backends/common/kernel/attention/mod.rs
@@ -1,5 +1,5 @@
+mod ancestor_attention_test;
mod attention_fallback_test;
-mod attention_last_query_test;
mod attention_single_pass_test;
mod attention_two_pass_test;
mod sigmoid_gate_test;
diff --git a/crates/backend-uzu/tests/unit/backends/common/kernel/weaver/weaver_frontier_test.rs b/crates/backend-uzu/tests/unit/backends/common/kernel/weaver/weaver_frontier_test.rs
index a03fdeaaf..5f27d522b 100644
--- a/crates/backend-uzu/tests/unit/backends/common/kernel/weaver/weaver_frontier_test.rs
+++ b/crates/backend-uzu/tests/unit/backends/common/kernel/weaver/weaver_frontier_test.rs
@@ -5,7 +5,7 @@ use crate::{
backends::{
common::{
Backend, Encoder, Kernels,
- gpu_types::weaver,
+ gpu_types::weaver::{FrontierIdx, MetadataIdx, TreeIdx},
kernel::{WeaverFrontierScatterKernel, WeaverFrontierSelectKernel},
},
cpu::Cpu,
@@ -15,7 +15,7 @@ use crate::{
fn select<B: Backend>() -> Vec<u32> {
let context = create_context::<B>();
- let mut frontier = vec![0; weaver::FRONTIER_LANE_COUNT * 8];
+ let mut frontier = vec![0; FrontierIdx::COUNT * 8];
for (slot, (token, parent, depth, cum, key, active)) in [
(9, 1, 1, 0x3f00_0000, 100, 1),
(8, 0, 2, 0x3f00_0001, 100, 1),
@@ -34,7 +34,7 @@ fn select<B: Backend>() -> Vec<u32> {
}
}
let mut frontier = alloc_allocation_with_data::<B, u32>(&context, &frontier);
- let mut tree = alloc_allocation_with_data::<B, u32>(&context, &[55; weaver::TREE_LANE_COUNT * 7]);
+ let mut tree = alloc_allocation_with_data::<B, u32>(&context, &[55; TreeIdx::COUNT * 7]);
let mut slot_ancestors = alloc_allocation_with_data::<B, u32>(&context, &(0u32..7 * 3).collect::<Vec<_>>());
let mut token = alloc_allocation_with_data::<B, u32>(&context, &[66; 4]);
let mut metadata = alloc_allocation_with_data::<B, u32>(&context, &[77; 3 * 4]);
@@ -80,19 +80,18 @@ fn select<B: Backend>() -> Vec<u32> {
fn scatter<B: Backend>() -> Vec<u32> {
let context = create_context::<B>();
- let mut tree = vec![0; weaver::TREE_LANE_COUNT * 4];
- tree[weaver::TREE_LANE_CUM * 4..(weaver::TREE_LANE_CUM + 1) * 4]
+ let mut tree = vec![0; TreeIdx::COUNT * 4];
+ tree[TreeIdx::PathLogprobBits as usize * 4..(TreeIdx::PathLogprobBits as usize + 1) * 4]
.copy_from_slice(&[0.5, -1.0, 2.0, 4.0].map(f32::to_bits));
- tree[weaver::TREE_LANE_DEPTH * 4..(weaver::TREE_LANE_DEPTH + 1) * 4].copy_from_slice(&[0, 2, 4, 6]);
+ tree[TreeIdx::Depth as usize * 4..(TreeIdx::Depth as usize + 1) * 4].copy_from_slice(&[0, 2, 4, 6]);
let tree = alloc_allocation_with_data::<B, u32>(&context, &tree);
- let mut metadata = vec![0; weaver::METADATA_LANE_COUNT * 3];
- metadata[weaver::METADATA_LANE_NODE_INDEX * 3..(weaver::METADATA_LANE_NODE_INDEX + 1) * 3]
- .copy_from_slice(&[1, 3, 0]);
+ let mut metadata = vec![0; MetadataIdx::COUNT * 3];
+ metadata[MetadataIdx::TreeSlot as usize * 3..(MetadataIdx::TreeSlot as usize + 1) * 3].copy_from_slice(&[1, 3, 0]);
let metadata = alloc_allocation_with_data::<B, u32>(&context, &metadata);
let valid = alloc_allocation_with_data::<B, u32>(&context, &[1, 0, 1]);
let ids = alloc_allocation_with_data::<B, u32>(&context, &(10..19).collect::<Vec<_>>());
let scores = alloc_allocation_with_data::<B, f32>(&context, &[-0.1, -0.2, -0.3, 8.0, 8.0, 8.0, 0.1, 0.2, 0.3]);
- let mut frontier = alloc_allocation_with_data::<B, u32>(&context, &[42; weaver::FRONTIER_LANE_COUNT * 16]);
+ let mut frontier = alloc_allocation_with_data::<B, u32>(&context, &[42; FrontierIdx::COUNT * 16]);
let kernel = <B::Kernels as Kernels>::WeaverFrontierScatterKernel::new(&context).unwrap();
let mut encoder = Encoder::new(context.as_ref()).unwrap();
kernel.encode(&tree, &metadata, &valid, &ids, &scores, &mut frontier, 16, 4, 3, 3, &mut encoder);
diff --git a/crates/backend-uzu/tests/unit/backends/common/kernel/weaver/weaver_node_cache_write_test.rs b/crates/backend-uzu/tests/unit/backends/common/kernel/weaver/weaver_node_cache_write_test.rs
index ba6d27ab2..50fdc7078 100644
--- a/crates/backend-uzu/tests/unit/backends/common/kernel/weaver/weaver_node_cache_write_test.rs
+++ b/crates/backend-uzu/tests/unit/backends/common/kernel/weaver/weaver_node_cache_write_test.rs
@@ -21,7 +21,7 @@ struct Runner<B: Backend> {
context: std::sync::Arc<B::Context>,
kernel: <B::Kernels as Kernels>::WeaverNodeCacheWriteKernel,
current_qkv: Allocation<B>,
- node_qkv: Allocation<B>,
+ node_kv: Allocation<B>,
node_indices: Allocation<B>,
}
@@ -38,13 +38,14 @@ const NODE_INDICES: [u32; ROWS] = [7, 1, 8, 4];
impl<B: Backend> Runner<B> {
fn new() -> Self {
- let packed_width = 3 * MODEL_DIM;
+ let qkv_width = 3 * MODEL_DIM;
+ let kv_width = 2 * MODEL_DIM;
let context = create_context::<B>();
let kernel =
<B::Kernels as Kernels>::WeaverNodeCacheWriteKernel::new(context.as_ref(), DataType::BF16).unwrap();
Self {
- current_qkv: alloc_allocation_with_data::<B, bf16>(&context, &values(ROWS * packed_width, 29)),
- node_qkv: alloc_allocation_with_data::<B, bf16>(&context, &values(NODES * packed_width, 11)),
+ current_qkv: alloc_allocation_with_data::<B, bf16>(&context, &values(ROWS * qkv_width, 29)),
+ node_kv: alloc_allocation_with_data::<B, bf16>(&context, &values(NODES * kv_width, 11)),
node_indices: alloc_allocation_with_data::<B, u32>(&context, &NODE_INDICES),
context,
kernel,
@@ -55,7 +56,7 @@ impl<B: Backend> Runner<B> {
let mut encoder = Encoder::new(self.context.as_ref()).unwrap();
self.kernel.encode(
&self.current_qkv,
- &mut self.node_qkv,
+ &mut self.node_kv,
&self.node_indices,
MODEL_DIM as u32,
NODES as u32,
@@ -63,32 +64,32 @@ impl<B: Backend> Runner<B> {
&mut encoder,
);
encoder.end_encoding().submit().wait_until_completed().unwrap();
- allocation_to_vec::<B, bf16>(&self.node_qkv)
+ allocation_to_vec::<B, bf16>(&self.node_kv)
}
}
#[uzu_test]
fn weaver_node_cache_write_matches_cpu() {
- let packed_width = 3 * MODEL_DIM;
+ let qkv_width = 3 * MODEL_DIM;
let mut cpu = Runner::<Cpu>::new();
- let initial_nodes = allocation_to_vec::<Cpu, bf16>(&cpu.node_qkv);
+ let initial_nodes = allocation_to_vec::<Cpu, bf16>(&cpu.node_kv);
let current_qkv = allocation_to_vec::<Cpu, bf16>(&cpu.current_qkv);
let expected = cpu.run();
- // Every written slot takes the K/V halves of its row; the query third and
- // every untouched slot keep their prior contents.
- for node in 0..NODES {
- let base = node * packed_width;
- match NODE_INDICES.iter().position(|&index| index as usize == node) {
- Some(row) => {
- assert_eq!(expected[base..base + MODEL_DIM], initial_nodes[base..base + MODEL_DIM]);
- assert_eq!(
- expected[base + MODEL_DIM..base + packed_width],
- current_qkv[row * packed_width + MODEL_DIM..(row + 1) * packed_width],
- );
- },
- None => assert_eq!(expected[base..base + packed_width], initial_nodes[base..base + packed_width]),
+ for component in 0..2 {
+ for node in 0..NODES {
+ let destination = component * NODES * MODEL_DIM + node * MODEL_DIM;
+ match NODE_INDICES.iter().position(|&index| index as usize == node) {
+ Some(row) => {
+ let source = row * qkv_width + (component + 1) * MODEL_DIM;
+ assert_eq!(expected[destination..destination + MODEL_DIM], current_qkv[source..source + MODEL_DIM],);
+ },
+ None => assert_eq!(
+ expected[destination..destination + MODEL_DIM],
+ initial_nodes[destination..destination + MODEL_DIM],
+ ),
+ }
}
}
diff --git a/crates/backend-uzu/tests/unit/backends/common/kernel/weaver/weaver_top_children_test.rs b/crates/backend-uzu/tests/unit/backends/common/kernel/weaver/weaver_top_children_test.rs
index 047ddceba..773368b25 100644
--- a/crates/backend-uzu/tests/unit/backends/common/kernel/weaver/weaver_top_children_test.rs
+++ b/crates/backend-uzu/tests/unit/backends/common/kernel/weaver/weaver_top_children_test.rs
@@ -15,34 +15,34 @@ const CHILDREN: usize = 8;
fn top_children<B: Backend>(
residual: &[bf16],
- scores: &[f32],
+ candidate_logits: &[f32],
ids: &[u32],
) -> (Vec<u32>, Vec<f32>) {
let rows = residual.len() / CANDIDATES;
assert_eq!(residual.len(), rows * CANDIDATES);
- assert_eq!(scores.len(), rows * CANDIDATES);
+ assert_eq!(candidate_logits.len(), rows * CANDIDATES);
assert_eq!(ids.len(), rows * CANDIDATES);
let context = create_context::<B>();
let residual = alloc_allocation_with_data::<B, bf16>(&context, residual);
- let scores = alloc_allocation_with_data::<B, f32>(&context, scores);
+ let candidate_logits = alloc_allocation_with_data::<B, f32>(&context, candidate_logits);
let ids = alloc_allocation_with_data::<B, u32>(&context, ids);
- let mut output_ids = alloc_allocation::<B, u32>(&context, rows * CHILDREN);
- let mut output_logprobs = alloc_allocation::<B, f32>(&context, rows * CHILDREN);
+ let mut output_token_ids = alloc_allocation::<B, u32>(&context, rows * CHILDREN);
+ let mut output_model_logprobs = alloc_allocation::<B, f32>(&context, rows * CHILDREN);
let kernel = <B::Kernels as Kernels>::WeaverTopChildrenKernel::new(&context).unwrap();
let mut encoder = Encoder::new(context.as_ref()).unwrap();
kernel.encode(
&residual,
- &scores,
+ &candidate_logits,
&ids,
- &mut output_ids,
- &mut output_logprobs,
+ &mut output_token_ids,
+ &mut output_model_logprobs,
rows as u32,
CANDIDATES as u32,
CHILDREN as u32,
&mut encoder,
);
encoder.end_encoding().submit().wait_until_completed().unwrap();
- (allocation_to_vec(&output_ids), allocation_to_vec(&output_logprobs))
+ (allocation_to_vec(&output_token_ids), allocation_to_vec(&output_model_logprobs))
}
#[uzu_test]
@@ -51,15 +51,15 @@ fn weaver_top_children_matches_cpu() {
let residual = (0..ROWS * CANDIDATES)
.map(|index| bf16::from_f32(((index as f32 * 0.017).cos() * 4.0).round() * 0.125))
.collect::<Vec<_>>();
- let scores =
+ let candidate_logits =
(0..ROWS * CANDIDATES).map(|index| ((index as f32 * 0.011).sin() * 3.0).round() * 0.125).collect::<Vec<_>>();
let ids = (0..ROWS)
.flat_map(|row| (0..CANDIDATES).rev().map(move |index| 70_000 + (row * CANDIDATES + index) as u32))
.collect::<Vec<_>>();
for_each_non_cpu_backend!(|B| {
- let expected = top_children::<Cpu>(&residual, &scores, &ids);
- let actual = top_children::<B>(&residual, &scores, &ids);
+ let expected = top_children::<Cpu>(&residual, &candidate_logits, &ids);
+ let actual = top_children::<B>(&residual, &candidate_logits, &ids);
assert_eq!(actual.0, expected.0);
for (actual, expected) in actual.1.iter().zip(expected.1) {
assert!((actual - expected).abs() < 1e-5);