A high-performance inference engine for AI models
Weaver cleanup + speedup (#637)
CCYeh50fbd82b772a
diff --git a/crates/backend-uzu/src/backends/common/buffer/arg.rs b/crates/backend-uzu/src/backends/common/buffer/arg.rsindex ad56f62b0..5c383d800 100644--- a/crates/backend-uzu/src/backends/common/buffer/arg.rs+++ b/crates/backend-uzu/src/backends/common/buffer/arg.rs@@ -1,4 +1,4 @@-use crate::backends::common::{AsBufferRangeMut, AsBufferRangeRef, Backend, Buffer};+use crate::backends::common::{AsBufferRangeMut, AsBufferRangeRef, Backend, Buffer, BufferRangeMut};// TODO: This whole thing is horrible and should be unified with AsBufferRangeRefpub trait BufferArg<'a, B: Backend>: Copy {@@ -30,6 +30,13 @@ pub trait BufferArgMut<'a, B: Backend> {fn into_parts(self) -> (&'a dyn Buffer<Backend = B>, usize, usize);}+impl<'a, B: Backend, Buf: Buffer<Backend = B>> BufferArgMut<'a, B> for BufferRangeMut<'a, Buf> {+ fn into_parts(self) -> (&'a dyn Buffer<Backend = B>, usize, usize) {+ let (buffer, range) = (self.buffer(), self.range());+ (buffer, range.start, range.len())+ }+}+impl<'a, B: Backend, T: AsBufferRangeMut<Buffer: Buffer<Backend = B>>> BufferArgMut<'a, B> for &'a mut T {fn into_parts(self) -> (&'a dyn Buffer<Backend = B>, usize, usize) {let buffer_range = self.as_buffer_range_mut();diff --git a/crates/backend-uzu/src/backends/common/buffer/range.rs b/crates/backend-uzu/src/backends/common/buffer/range.rsindex dd0913f39..5887811b3 100644--- a/crates/backend-uzu/src/backends/common/buffer/range.rs+++ b/crates/backend-uzu/src/backends/common/buffer/range.rs@@ -74,6 +74,26 @@ impl<'a, B: Buffer> BufferRangeMut<'a, B> {}}+ pub fn split_at(+ self,+ byte_offset: usize,+ ) -> (Self, Self) {+ assert!(byte_offset <= self.range.len(), "split offset exceeds buffer range size");+ let split = self.range.start + byte_offset;+ let buffer = self.buffer;++ (+ Self {+ buffer,+ range: self.range.start..split,+ },+ Self {+ buffer,+ range: split..self.range.end,+ },+ )+ }+pub fn buffer(&self) -> &'a B {self.buffer}diff --git a/crates/backend-uzu/src/backends/common/gpu_types/weaver.rs b/crates/backend-uzu/src/backends/common/gpu_types/weaver.rsindex e6b058188..09d0e86cd 100644--- a/crates/backend-uzu/src/backends/common/gpu_types/weaver.rs+++ b/crates/backend-uzu/src/backends/common/gpu_types/weaver.rs@@ -2,27 +2,48 @@ pub const CANDIDATES_MAX: usize = 512;pub const TOP_CHILDREN_THREADS: usize = 256;pub const TOP_CHILDREN_SIMDGROUPS: usize = 8;-pub const FRONTIER_LANE_TOKEN: usize = 0;-pub const FRONTIER_LANE_PARENT: usize = 1;-pub const FRONTIER_LANE_DEPTH: usize = 2;-pub const FRONTIER_LANE_CUM: usize = 3;-pub const FRONTIER_LANE_LOGPROB: usize = 4;-pub const FRONTIER_LANE_KEY: usize = 5;-pub const FRONTIER_LANE_ACTIVE: usize = 6;-pub const FRONTIER_LANE_COUNT: usize = FRONTIER_LANE_ACTIVE + 1;+#[repr(C)]+#[derive(Clone, Copy)]+pub enum FrontierIdx {+ TokenId,+ ParentSlot,+ Depth,+ PathLogprobBits,+ EdgeLogprobBits,+ PathScoreKey,+ Active,+}-pub const TREE_LANE_TOKEN: usize = 0;-pub const TREE_LANE_PARENT: usize = 1;-pub const TREE_LANE_DEPTH: usize = 2;-pub const TREE_LANE_CUM: usize = 3;-pub const TREE_LANE_LOGPROB: usize = 4;-pub const TREE_LANE_MASK: usize = 5;-pub const TREE_LANE_COUNT: usize = TREE_LANE_MASK + 1;+impl FrontierIdx {+ pub const COUNT: usize = Self::Active as usize + 1;+}-pub const METADATA_LANE_DEPTH: usize = 0;-pub const METADATA_LANE_ANCESTOR_COUNT: usize = 1;-pub const METADATA_LANE_NODE_INDEX: usize = 2;-pub const METADATA_LANE_COUNT: usize = METADATA_LANE_NODE_INDEX + 1;+#[repr(C)]+#[derive(Clone, Copy)]+pub enum TreeIdx {+ TokenId,+ ParentSlot,+ Depth,+ PathLogprobBits,+ EdgeLogprobBits,+ Valid,+}++impl TreeIdx {+ pub const COUNT: usize = Self::Valid as usize + 1;+}++#[repr(C)]+#[derive(Clone, Copy)]+pub enum MetadataIdx {+ Depth,+ AncestorCount,+ TreeSlot,+}++impl MetadataIdx {+ pub const COUNT: usize = Self::TreeSlot as usize + 1;+}pub const FRONTIER_NO_WINNER: u32 = !0;pub const FRONTIER_SELECT_THREADS: usize = 256;diff --git a/crates/backend-uzu/src/backends/cpu/kernel/attention/ancestor_attention.rs b/crates/backend-uzu/src/backends/cpu/kernel/attention/ancestor_attention.rsnew file mode 100644index 000000000..0522ecf47--- /dev/null+++ b/crates/backend-uzu/src/backends/cpu/kernel/attention/ancestor_attention.rs@@ -0,0 +1,96 @@+use half::bf16;+use proc_macros::kernel;++use super::attention_single_pass::attention_single_pass;++#[kernel(AncestorAttention)]+#[variants(HEAD_DIM, 128)]+fn ancestor_attention<const HEAD_DIM: u32>(+ prefix_kv: *const bf16,+ node_kv: *const bf16,+ current_qkv: *const bf16,+ ancestor_indices: *const u32,+ ancestor_counts: *const u32,+ output: *mut bf16,+ rows: u32,+ prefix_length: u32,+ ancestor_stride: u32,+ node_capacity: u32,+ scale: f32,+ #[specialize] num_heads: u32,+) {+ const QKV_COMPONENTS: usize = 3;+ const KEY_COMPONENT: usize = 1;+ const VALUE_COMPONENT: usize = 2;+ let head_dim = HEAD_DIM as usize;+ let num_heads = num_heads as usize;+ let model_dim = num_heads * head_dim;+ let qkv_width = QKV_COMPONENTS * model_dim;+ let prefix_length = prefix_length as usize;+ let node_capacity = node_capacity as usize;+ let last_node = node_capacity.saturating_sub(1);++ for row in 0..rows as usize {+ unsafe {+ let current_row = current_qkv.add(row * qkv_width);++ let ancestor_count = *ancestor_counts.add(row) as usize;+ let length = prefix_length + ancestor_count + 1;+ let mut keys = vec![bf16::ZERO; length * model_dim];+ let mut values = vec![bf16::ZERO; length * model_dim];+ std::ptr::copy_nonoverlapping(prefix_kv, keys.as_mut_ptr(), prefix_length * model_dim);+ std::ptr::copy_nonoverlapping(+ prefix_kv.add(prefix_length * model_dim),+ values.as_mut_ptr(),+ prefix_length * model_dim,+ );+ for offset in 0..ancestor_count {+ let ancestor = (*ancestor_indices.add(row * ancestor_stride as usize + offset) as usize).min(last_node);+ std::ptr::copy_nonoverlapping(+ node_kv.add(ancestor * model_dim),+ keys.as_mut_ptr().add((prefix_length + offset) * model_dim),+ model_dim,+ );+ std::ptr::copy_nonoverlapping(+ node_kv.add(node_capacity * model_dim + ancestor * model_dim),+ values.as_mut_ptr().add((prefix_length + offset) * model_dim),+ model_dim,+ );+ }+ std::ptr::copy_nonoverlapping(+ current_row.add(KEY_COMPONENT * model_dim),+ keys.as_mut_ptr().add((length - 1) * model_dim),+ model_dim,+ );+ std::ptr::copy_nonoverlapping(+ current_row.add(VALUE_COMPONENT * model_dim),+ values.as_mut_ptr().add((length - 1) * model_dim),+ model_dim,+ );+ attention_single_pass::<bf16, HEAD_DIM>(+ current_row,+ keys.as_ptr(),+ values.as_ptr(),+ output.add(row * num_heads * head_dim),+ 1,+ length as u32,+ HEAD_DIM,+ model_dim as u32,+ HEAD_DIM,+ model_dim as u32,+ None,+ scale,+ None,+ None,+ None,+ num_heads as u32,+ 1,+ false,+ false,+ false,+ false,+ false,+ );+ }+ }+}diff --git a/crates/backend-uzu/src/backends/cpu/kernel/attention/attention_last_query.rs b/crates/backend-uzu/src/backends/cpu/kernel/attention/attention_last_query.rsdeleted file mode 100644index dd1e4d075..000000000--- a/crates/backend-uzu/src/backends/cpu/kernel/attention/attention_last_query.rs+++ /dev/null@@ -1,80 +0,0 @@-use half::bf16;-use proc_macros::kernel;--use super::attention_single_pass::attention_single_pass;--#[kernel(AttentionLastQuery)]-#[variants(HEAD_DIM, 128)]-fn attention_last_query<const HEAD_DIM: u32>(- prefix_qkv: *const bf16,- node_qkv: *const bf16,- current_qkv: *const bf16,- ancestor_indices: *const u32,- ancestor_counts: *const u32,- output: *mut bf16,- rows: u32,- prefix_length: u32,- ancestor_stride: u32,- node_capacity: u32,- scale: f32,- #[specialize] num_heads: u32,-) {- const QKV_COMPONENTS: usize = 3;- const KV_COMPONENTS: usize = 2;- const KEY_COMPONENT: usize = 1;- const VALUE_COMPONENT: usize = 2;- let head_dim = HEAD_DIM as usize;- let num_heads = num_heads as usize;- let qkv_width = QKV_COMPONENTS * num_heads * head_dim;- let kv_offset = KEY_COMPONENT * num_heads * head_dim;- let kv_width = KV_COMPONENTS * num_heads * head_dim;- let last_node = node_capacity.saturating_sub(1) as usize;-- for row in 0..rows as usize {- unsafe {- let current_row = current_qkv.add(row * qkv_width);-- let ancestor_count = *ancestor_counts.add(row) as usize;- let length = prefix_length as usize + ancestor_count + 1;- let mut sequence = vec![bf16::ZERO; length * qkv_width];- std::ptr::copy_nonoverlapping(prefix_qkv, sequence.as_mut_ptr(), prefix_length as usize * qkv_width);- for offset in 0..ancestor_count {- let ancestor = (*ancestor_indices.add(row * ancestor_stride as usize + offset) as usize).min(last_node);- std::ptr::copy_nonoverlapping(- node_qkv.add(ancestor * qkv_width + kv_offset),- sequence.as_mut_ptr().add((prefix_length as usize + offset) * qkv_width + kv_offset),- kv_width,- );- }- std::ptr::copy_nonoverlapping(- current_row.add(kv_offset),- sequence.as_mut_ptr().add((length - 1) * qkv_width + kv_offset),- kv_width,- );- attention_single_pass::<bf16, HEAD_DIM>(- current_row,- sequence.as_ptr().add(kv_offset),- sequence.as_ptr().add(VALUE_COMPONENT * num_heads * head_dim),- output.add(row * num_heads * head_dim),- 1,- length as u32,- HEAD_DIM,- qkv_width as u32,- HEAD_DIM,- qkv_width as u32,- None,- scale,- None,- None,- None,- num_heads as u32,- 1,- false,- false,- false,- false,- false,- );- }- }-}diff --git a/crates/backend-uzu/src/backends/cpu/kernel/attention/mod.rs b/crates/backend-uzu/src/backends/cpu/kernel/attention/mod.rsindex 813c08dce..052bec54a 100644--- a/crates/backend-uzu/src/backends/cpu/kernel/attention/mod.rs+++ b/crates/backend-uzu/src/backends/cpu/kernel/attention/mod.rs@@ -1,5 +1,5 @@+pub mod ancestor_attention;pub mod attention_fallback;-pub mod attention_last_query;pub mod attention_prepare;pub mod attention_single_pass;pub mod attention_two_pass;diff --git a/crates/backend-uzu/src/backends/cpu/kernel/normalization/normalization.rs b/crates/backend-uzu/src/backends/cpu/kernel/normalization/normalization.rsindex d15063d4b..854b7b27f 100644--- a/crates/backend-uzu/src/backends/cpu/kernel/normalization/normalization.rs+++ b/crates/backend-uzu/src/backends/cpu/kernel/normalization/normalization.rs@@ -6,17 +6,18 @@ use crate::array::ArrayElement;#[kernel(Normalization)]#[variants(InputT, f32, f16, bf16)]-#[variants(ScaleT, f32, f16, bf16)]+#[variants(AffineT, f32, f16, bf16)]#[variants(OutputT, f32, f16, bf16)]#[variants(AccumT, f32)]pub fn normalization<InputT: ArrayElement + Float,- ScaleT: ArrayElement + Float,+ AffineT: ArrayElement + Float,OutputT: ArrayElement + Float,AccumT: ArrayElement + Float,>(#[optional(!in_place)] input: Option<*const InputT>,- scales: *const ScaleT,+ scales: *const AffineT,+ #[optional(has_biases)] biases: Option<*const AffineT>,output: *mut OutputT,#[optional(copy_to_shortcut)] shortcut: Option<*mut InputT>,#[optional(use_hadamard)] hadamard_factors: Option<*const i32>,@@ -33,9 +34,11 @@ pub fn normalization<#[specialize] use_hadamard: bool,#[specialize] scale_residual_sum: bool,#[specialize] scale_output: bool,+ #[specialize] has_biases: bool,) {assert_eq!(shortcut.is_some(), copy_to_shortcut);assert_eq!(hadamard_factors.is_some(), use_hadamard);+ assert_eq!(biases.is_some(), has_biases);assert!(copy_to_shortcut || !residual_add);if use_hadamard {@@ -94,7 +97,7 @@ pub fn normalization<};let scale_val = unsafe { AccumT::from(*scales.add(i)).unwrap() };let normalized: AccumT = (input_val - mean) * rms_inv;- let result: OutputT = if full_layer {+ let mut result: OutputT = if full_layer {// Full-layer: keep everything in accumulation precisionlet scale_with_offset: AccumT = scale_val + scale_offset;OutputT::from(normalized * scale_with_offset).unwrap()@@ -104,11 +107,13 @@ pub fn normalization<let scale_with_offset_out = OutputT::from(scale_val + scale_offset).unwrap();normalized_out * scale_with_offset_out};- let result = if scale_output {- result * OutputT::from(post_layer_scalar).unwrap()- } else {- result- };+ if has_biases {+ let bias = unsafe { AccumT::from(*biases.unwrap().add(i)).unwrap() };+ result = OutputT::from(AccumT::from(result).unwrap() + bias).unwrap();+ }+ if scale_output {+ result = result * OutputT::from(post_layer_scalar).unwrap();+ }unsafe { *output.add(batch_offset + i) = result };}}diff --git a/crates/backend-uzu/src/backends/cpu/kernel/weaver/weaver_frontier_scatter.rs b/crates/backend-uzu/src/backends/cpu/kernel/weaver/weaver_frontier_scatter.rsindex b6cab65a3..627b7602d 100644--- a/crates/backend-uzu/src/backends/cpu/kernel/weaver/weaver_frontier_scatter.rs+++ b/crates/backend-uzu/src/backends/cpu/kernel/weaver/weaver_frontier_scatter.rs@@ -1,6 +1,6 @@use proc_macros::kernel;-use crate::backends::common::gpu_types::weaver;+use crate::backends::common::gpu_types::weaver::{FrontierIdx, MetadataIdx, TreeIdx};const F32_SIGN_BIT: u32 = 1 << (u32::BITS - 1);@@ -24,43 +24,43 @@ pub fn weaver_frontier_scatter(capacity: u32,tree_slots: u32,rows: u32,- fanout: u32,+ children_per_node: u32,) {- if capacity == 0 || tree_slots == 0 || fanout == 0 {+ if capacity == 0 || tree_slots == 0 || children_per_node == 0 {return;}- let (capacity, tree_slots, rows, fanout) = (capacity as usize, tree_slots as usize, rows as usize, fanout as usize);- let tree = unsafe { std::slice::from_raw_parts(tree, weaver::TREE_LANE_COUNT * tree_slots) };+ let (capacity, tree_slots, rows, children_per_node) =+ (capacity as usize, tree_slots as usize, rows as usize, children_per_node as usize);+ let tree = unsafe { std::slice::from_raw_parts(tree, TreeIdx::COUNT * tree_slots) };let parent_indices =- unsafe { std::slice::from_raw_parts(round_metadata.add(weaver::METADATA_LANE_NODE_INDEX * rows), rows) };+ unsafe { std::slice::from_raw_parts(round_metadata.add(MetadataIdx::TreeSlot as usize * rows), rows) };let round_valid = unsafe { std::slice::from_raw_parts(round_valid, rows) };- let child_ids = unsafe { std::slice::from_raw_parts(child_ids, rows * fanout) };- let child_logprobs = unsafe { std::slice::from_raw_parts(child_logprobs, rows * fanout) };- let frontier = unsafe { std::slice::from_raw_parts_mut(frontier, weaver::FRONTIER_LANE_COUNT * capacity) };+ let child_ids = unsafe { std::slice::from_raw_parts(child_ids, rows * children_per_node) };+ let child_logprobs = unsafe { std::slice::from_raw_parts(child_logprobs, rows * children_per_node) };+ let frontier = unsafe { std::slice::from_raw_parts_mut(frontier, FrontierIdx::COUNT * capacity) };- for index in 0..rows * fanout {- let row = index / fanout;+ for index in 0..rows * children_per_node {+ let row = index / children_per_node;if round_valid[row] == 0 {continue;}let parent = parent_indices[row] as usize;- let slot = parent * fanout + index % fanout;+ let slot = parent * children_per_node + index % children_per_node;if parent >= tree_slots || slot >= capacity {continue;}let logprob = child_logprobs[index];- let cumulative_logprob = f32::from_bits(tree[weaver::TREE_LANE_CUM * tree_slots + parent]) + logprob;- let values = [- child_ids[index],- parent as u32,- tree[weaver::TREE_LANE_DEPTH * tree_slots + parent] + 1,- cumulative_logprob.to_bits(),- logprob.to_bits(),- top_k_score_key(cumulative_logprob),- 1,- ];- for (lane, value) in values.into_iter().enumerate() {- frontier[lane * capacity + slot] = value;- }+ let cumulative_logprob =+ f32::from_bits(tree[TreeIdx::PathLogprobBits as usize * tree_slots + parent]) + logprob;+ let mut set = |field: FrontierIdx, value| {+ frontier[field as usize * capacity + slot] = value;+ };+ set(FrontierIdx::TokenId, child_ids[index]);+ set(FrontierIdx::ParentSlot, parent as u32);+ set(FrontierIdx::Depth, tree[TreeIdx::Depth as usize * tree_slots + parent] + 1);+ set(FrontierIdx::PathLogprobBits, cumulative_logprob.to_bits());+ set(FrontierIdx::EdgeLogprobBits, logprob.to_bits());+ set(FrontierIdx::PathScoreKey, top_k_score_key(cumulative_logprob));+ set(FrontierIdx::Active, 1);}}diff --git a/crates/backend-uzu/src/backends/cpu/kernel/weaver/weaver_frontier_select.rs b/crates/backend-uzu/src/backends/cpu/kernel/weaver/weaver_frontier_select.rsindex 7e178879a..c90db4461 100644--- a/crates/backend-uzu/src/backends/cpu/kernel/weaver/weaver_frontier_select.rs+++ b/crates/backend-uzu/src/backends/cpu/kernel/weaver/weaver_frontier_select.rs@@ -1,6 +1,8 @@use proc_macros::kernel;-use crate::backends::common::gpu_types::weaver;+use crate::backends::common::gpu_types::weaver::{+ FRONTIER_MAX_SLOTS, FRONTIER_MAX_WIDTH, FRONTIER_NO_WINNER, FrontierIdx, MetadataIdx, TreeIdx,+};#[kernel(WeaverFrontierSelect)]pub fn weaver_frontier_select(@@ -26,9 +28,9 @@ pub fn weaver_frontier_select(candidate_pool_size: u32,) {if capacity == 0- || capacity as usize > weaver::FRONTIER_MAX_SLOTS+ || capacity as usize > FRONTIER_MAX_SLOTS|| width == 0- || width as usize > weaver::FRONTIER_MAX_WIDTH+ || width as usize > FRONTIER_MAX_WIDTH|| ancestor_stride == 0|| max_depth == 0|| tree_slots == 0@@ -42,11 +44,11 @@ pub fn weaver_frontier_select((capacity as usize, tree_slots as usize, width as usize, ancestor_stride as usize);let (candidate_pool_rows, candidate_pool_size) = (candidate_pool_rows as usize, candidate_pool_size as usize);let pool_len = candidate_pool_rows * candidate_pool_size;- let frontier = unsafe { std::slice::from_raw_parts_mut(frontier, weaver::FRONTIER_LANE_COUNT * capacity) };- let tree = unsafe { std::slice::from_raw_parts_mut(tree, weaver::TREE_LANE_COUNT * slots) };+ let frontier = unsafe { std::slice::from_raw_parts_mut(frontier, FrontierIdx::COUNT * capacity) };+ let tree = unsafe { std::slice::from_raw_parts_mut(tree, TreeIdx::COUNT * slots) };let slot_ancestors = unsafe { std::slice::from_raw_parts_mut(slot_ancestors, slots * stride) };let round_token_ids = unsafe { std::slice::from_raw_parts_mut(round_token_ids, width) };- let round_metadata = unsafe { std::slice::from_raw_parts_mut(round_metadata, weaver::METADATA_LANE_COUNT * width) };+ let round_metadata = unsafe { std::slice::from_raw_parts_mut(round_metadata, MetadataIdx::COUNT * width) };let round_ancestors = unsafe { std::slice::from_raw_parts_mut(round_ancestors, width * stride) };let round_valid = unsafe { std::slice::from_raw_parts_mut(round_valid, width) };let candidate_pool_ids = unsafe { std::slice::from_raw_parts(candidate_pool_ids, pool_len) };@@ -58,46 +60,46 @@ pub fn weaver_frontier_select(for row in 0..width {let (mut key, mut parent, mut token, mut winner) =- (0, weaver::FRONTIER_NO_WINNER, weaver::FRONTIER_NO_WINNER, weaver::FRONTIER_NO_WINNER);+ (0, FRONTIER_NO_WINNER, FRONTIER_NO_WINNER, FRONTIER_NO_WINNER);for slot in 0..capacity {- if frontier[weaver::FRONTIER_LANE_ACTIVE * capacity + slot] == 0 {+ if frontier[FrontierIdx::Active as usize * capacity + slot] == 0 {continue;}let next = (- frontier[weaver::FRONTIER_LANE_KEY * capacity + slot],- frontier[weaver::FRONTIER_LANE_PARENT * capacity + slot],- frontier[weaver::FRONTIER_LANE_TOKEN * capacity + slot],+ frontier[FrontierIdx::PathScoreKey as usize * capacity + slot],+ frontier[FrontierIdx::ParentSlot as usize * capacity + slot],+ frontier[FrontierIdx::TokenId as usize * capacity + slot],);if next.0 > key || (next.0 == key && (next.1 < parent || (next.1 == parent && next.2 < token))) {(key, parent, token, winner) = (next.0, next.1, next.2, slot as u32);}}- let real = winner != weaver::FRONTIER_NO_WINNER;+ let real = winner != FRONTIER_NO_WINNER;let winner = if real {winner as usize} else {0};let tree_slot = slot_start as usize + row;- let lane_value = |lane: usize| u32::from(real) * frontier[lane * capacity + winner];- let token = lane_value(weaver::FRONTIER_LANE_TOKEN);- let depth = lane_value(weaver::FRONTIER_LANE_DEPTH);- let cumulative_logprob = lane_value(weaver::FRONTIER_LANE_CUM);- let logprob = lane_value(weaver::FRONTIER_LANE_LOGPROB);+ let field_value = |field: FrontierIdx| u32::from(real) * frontier[field as usize * capacity + winner];+ let token = field_value(FrontierIdx::TokenId);+ let depth = field_value(FrontierIdx::Depth);+ let cumulative_logprob = field_value(FrontierIdx::PathLogprobBits);+ let logprob = field_value(FrontierIdx::EdgeLogprobBits);- tree[weaver::TREE_LANE_TOKEN * slots + tree_slot] = token;- tree[weaver::TREE_LANE_PARENT * slots + tree_slot] = if real {+ tree[TreeIdx::TokenId as usize * slots + tree_slot] = token;+ tree[TreeIdx::ParentSlot as usize * slots + tree_slot] = if real {parent} else {- weaver::FRONTIER_NO_WINNER+ FRONTIER_NO_WINNER};- tree[weaver::TREE_LANE_DEPTH * slots + tree_slot] = depth;- tree[weaver::TREE_LANE_CUM * slots + tree_slot] = cumulative_logprob;- tree[weaver::TREE_LANE_LOGPROB * slots + tree_slot] = logprob;- tree[weaver::TREE_LANE_MASK * slots + tree_slot] = u32::from(real);+ tree[TreeIdx::Depth as usize * slots + tree_slot] = depth;+ tree[TreeIdx::PathLogprobBits as usize * slots + tree_slot] = cumulative_logprob;+ tree[TreeIdx::EdgeLogprobBits as usize * slots + tree_slot] = logprob;+ tree[TreeIdx::Valid as usize * slots + tree_slot] = u32::from(real);if real {- frontier[weaver::FRONTIER_LANE_ACTIVE * capacity + winner] = 0;+ frontier[FrontierIdx::Active as usize * capacity + winner] = 0;}let parent_slot = if real && (parent as usize) < slots {@@ -118,9 +120,9 @@ pub fn weaver_frontier_select(}round_token_ids[row] = token;- round_metadata[weaver::METADATA_LANE_DEPTH * width + row] = depth.min(max_depth - 1);- round_metadata[weaver::METADATA_LANE_ANCESTOR_COUNT * width + row] = depth.min(stride as u32);- round_metadata[weaver::METADATA_LANE_NODE_INDEX * width + row] = tree_slot as u32;+ round_metadata[MetadataIdx::Depth as usize * width + row] = depth.min(max_depth - 1);+ round_metadata[MetadataIdx::AncestorCount as usize * width + row] = depth.min(stride as u32);+ round_metadata[MetadataIdx::TreeSlot as usize * width + row] = tree_slot as u32;round_valid[row] = u32::from(real && depth < lookahead_count && depth < max_depth);// Every row of a round expands the pool for its own depth, so the winner's pool row isdiff --git a/crates/backend-uzu/src/backends/cpu/kernel/weaver/weaver_node_cache_write.rs b/crates/backend-uzu/src/backends/cpu/kernel/weaver/weaver_node_cache_write.rsindex 346d9bdd2..1f4c177d6 100644--- a/crates/backend-uzu/src/backends/cpu/kernel/weaver/weaver_node_cache_write.rs+++ b/crates/backend-uzu/src/backends/cpu/kernel/weaver/weaver_node_cache_write.rs@@ -8,7 +8,7 @@ use crate::array::ArrayElement;#[variants(T, f32, bf16)]pub fn weaver_node_cache_write<T: ArrayElement + Float>(current_qkv: *const T,- node_qkv: *mut T,+ node_kv: *mut T,node_indices: *const u32,model_dim: u32,node_capacity: u32,@@ -24,9 +24,11 @@ pub fn weaver_node_cache_write<T: ArrayElement + Float>(for position in 0..total as usize {let row = position / (2 * model_dim);let offset = position - row * 2 * model_dim;+ let component = offset / model_dim;+ let component_offset = offset % model_dim;let node = (*node_indices.add(row) as usize).min(last_node);- *node_qkv.add(node * qkv_width + model_dim + offset) =- *current_qkv.add(row * qkv_width + model_dim + offset);+ *node_kv.add(component * node_capacity as usize * model_dim + node * model_dim + component_offset) =+ *current_qkv.add(row * qkv_width + (component + 1) * model_dim + component_offset);}}}diff --git a/crates/backend-uzu/src/backends/cpu/kernel/weaver/weaver_top_children.rs b/crates/backend-uzu/src/backends/cpu/kernel/weaver/weaver_top_children.rsindex 55b713f3f..24cbcd0eb 100644--- a/crates/backend-uzu/src/backends/cpu/kernel/weaver/weaver_top_children.rs+++ b/crates/backend-uzu/src/backends/cpu/kernel/weaver/weaver_top_children.rs@@ -1,29 +1,29 @@use half::bf16;use proc_macros::kernel;-use crate::backends::common::gpu_types::weaver;+use crate::backends::common::gpu_types::weaver::CANDIDATES_MAX;#[kernel(WeaverTopChildren)]pub fn weaver_top_children(residual_logits: *const bf16,- candidate_scores: *const f32,+ candidate_logits: *const f32,candidate_ids: *const u32,- output_ids: *mut u32,- output_logprobs: *mut f32,+ output_token_ids: *mut u32,+ output_model_logprobs: *mut f32,rows: u32,candidates: u32,- children: u32,+ children_per_node: u32,) {let rows = rows as usize;let candidates = candidates as usize;- let children = children as usize;- if candidates == 0 || candidates > weaver::CANDIDATES_MAX || children == 0 || children > candidates {+ let children_per_node = children_per_node as usize;+ if candidates == 0 || candidates > CANDIDATES_MAX || children_per_node == 0 || children_per_node > candidates {return;}for row in 0..rows {let base = row * candidates;let logits = (0..candidates)- .map(|index| unsafe { *candidate_scores.add(base + index) + (*residual_logits.add(base + index)).to_f32() })+ .map(|index| unsafe { *candidate_logits.add(base + index) + (*residual_logits.add(base + index)).to_f32() }).collect::<Vec<_>>();let max = logits.iter().copied().fold(f32::NEG_INFINITY, f32::max);let log_sum = logits.iter().map(|value| (value - max).exp()).sum::<f32>().ln() + max;@@ -33,10 +33,10 @@ pub fn weaver_top_children(.total_cmp(&logits[left]).then_with(|| unsafe { (*candidate_ids.add(base + left)).cmp(&*candidate_ids.add(base + right)) })});- for (rank, index) in indices.into_iter().take(children).enumerate() {+ for (rank, index) in indices.into_iter().take(children_per_node).enumerate() {unsafe {- *output_ids.add(row * children + rank) = *candidate_ids.add(base + index);- *output_logprobs.add(row * children + rank) = logits[index] - log_sum;+ *output_token_ids.add(row * children_per_node + rank) = *candidate_ids.add(base + index);+ *output_model_logprobs.add(row * children_per_node + rank) = logits[index] - log_sum;}}}diff --git a/crates/backend-uzu/src/backends/metal/kernel/attention/attention_last_query.metal b/crates/backend-uzu/src/backends/metal/kernel/attention/ancestor_attention.metalsimilarity index 72%rename from crates/backend-uzu/src/backends/metal/kernel/attention/attention_last_query.metalrename to crates/backend-uzu/src/backends/metal/kernel/attention/ancestor_attention.metalindex 0f6aabaac..e615d580f 100644--- a/crates/backend-uzu/src/backends/metal/kernel/attention/attention_last_query.metal+++ b/crates/backend-uzu/src/backends/metal/kernel/attention/ancestor_attention.metal@@ -51,9 +51,9 @@ METAL_FUNC void attend_qkv(template <uint HEAD_DIM>VARIANTS(HEAD_DIM, 128)-PUBLIC KERNEL(AttentionLastQuery)(- const device bfloat* prefix_qkv,- const device bfloat* node_qkv,+PUBLIC KERNEL(AncestorAttention)(+ const device bfloat* prefix_kv,+ const device bfloat* node_kv,const device bfloat* current_qkv,const device uint* ancestor_indices,const device uint* ancestor_counts,@@ -78,29 +78,33 @@ PUBLIC KERNEL(AttentionLastQuery)(const uint row = row_head / num_heads;const uint head = row_head % num_heads;const uint lane = thread_context.simd_lane_id;+ const uint model_vectors = num_heads * vectors_per_head;const uint qkv_vectors = QKV_COMPONENTS * num_heads * vectors_per_head;- const uint key_offset = (KEY_COMPONENT * num_heads + head) * vectors_per_head + lane;- const uint value_offset = (VALUE_COMPONENT * num_heads + head) * vectors_per_head + lane;- const device bfloat4* prefix_vectors = (const device bfloat4*)prefix_qkv;- const device bfloat4* node_vectors = (const device bfloat4*)node_qkv;+ const uint head_offset = head * vectors_per_head + lane;+ const uint current_key_offset = KEY_COMPONENT * model_vectors + head_offset;+ const uint current_value_offset = VALUE_COMPONENT * model_vectors + head_offset;+ const uint prefix_value_offset = prefix_length * model_vectors + head_offset;+ const uint node_value_offset = node_capacity * model_vectors + head_offset;+ const device bfloat4* prefix_vectors = (const device bfloat4*)prefix_kv;+ const device bfloat4* node_vectors = (const device bfloat4*)node_kv;const device bfloat4* current_row = (const device bfloat4*)current_qkv + row * qkv_vectors;- const float4 query = float4(current_row[head * vectors_per_head + lane]) * scale;+ const float4 query = float4(current_row[head_offset]) * scale;// prefix_length >= 1 (the u0 token): seed the online softmax from position 0.- float max_score = simd_sum(dot(query, float4(prefix_vectors[key_offset])));- float4 values = float4(prefix_vectors[value_offset]);+ float max_score = simd_sum(dot(query, float4(prefix_vectors[head_offset])));+ float4 values = float4(prefix_vectors[prefix_value_offset]);float sum = 1.0f;uint position = 1;for (; position + unroll_count - 1 < prefix_length; position += unroll_count) {- attend_qkv<unroll_count>(query, key_offset, value_offset, values, max_score, sum, [&](int step) {- return prefix_vectors + (position + step) * qkv_vectors;+ attend_qkv<unroll_count>(query, head_offset, prefix_value_offset, values, max_score, sum, [&](int step) {+ return prefix_vectors + (position + step) * model_vectors;});}for (; position < prefix_length; ++position) {- attend_qkv<1>(query, key_offset, value_offset, values, max_score, sum, [&](int) {- return prefix_vectors + position * qkv_vectors;+ attend_qkv<1>(query, head_offset, prefix_value_offset, values, max_score, sum, [&](int) {+ return prefix_vectors + position * model_vectors;});}@@ -109,17 +113,19 @@ PUBLIC KERNEL(AttentionLastQuery)(const uint last_node = node_capacity == 0 ? 0u : node_capacity - 1u;uint offset = 0;for (; offset + unroll_count - 1 < ancestor_count; offset += unroll_count) {- attend_qkv<unroll_count>(query, key_offset, value_offset, values, max_score, sum, [&](int step) {- return node_vectors + min(row_ancestors[offset + step], last_node) * qkv_vectors;+ attend_qkv<unroll_count>(query, head_offset, node_value_offset, values, max_score, sum, [&](int step) {+ return node_vectors + min(row_ancestors[offset + step], last_node) * model_vectors;});}for (; offset < ancestor_count; ++offset) {- attend_qkv<1>(query, key_offset, value_offset, values, max_score, sum, [&](int) {- return node_vectors + min(row_ancestors[offset], last_node) * qkv_vectors;+ attend_qkv<1>(query, head_offset, node_value_offset, values, max_score, sum, [&](int) {+ return node_vectors + min(row_ancestors[offset], last_node) * model_vectors;});}- attend_qkv<1>(query, key_offset, value_offset, values, max_score, sum, [&](int) { return current_row; });+ attend_qkv<1>(query, current_key_offset, current_value_offset, values, max_score, sum, [&](int) {+ return current_row;+ });((device bfloat4*)output)[(row * num_heads + head) * vectors_per_head + lane] = bfloat4(values / sum);}diff --git a/crates/backend-uzu/src/backends/metal/kernel/generated/weaver.h b/crates/backend-uzu/src/backends/metal/kernel/generated/weaver.hindex b49c08d42..f453dcc9e 100644--- a/crates/backend-uzu/src/backends/metal/kernel/generated/weaver.h+++ b/crates/backend-uzu/src/backends/metal/kernel/generated/weaver.h@@ -11,43 +11,30 @@ static constant constexpr size_t TOP_CHILDREN_THREADS = 256;static constant constexpr size_t TOP_CHILDREN_SIMDGROUPS = 8;-static constant constexpr size_t FRONTIER_LANE_TOKEN = 0;--static constant constexpr size_t FRONTIER_LANE_PARENT = 1;--static constant constexpr size_t FRONTIER_LANE_DEPTH = 2;--static constant constexpr size_t FRONTIER_LANE_CUM = 3;--static constant constexpr size_t FRONTIER_LANE_LOGPROB = 4;--static constant constexpr size_t FRONTIER_LANE_KEY = 5;--static constant constexpr size_t FRONTIER_LANE_ACTIVE = 6;--static constant constexpr size_t FRONTIER_LANE_COUNT = FRONTIER_LANE_ACTIVE + 1;--static constant constexpr size_t TREE_LANE_TOKEN = 0;--static constant constexpr size_t TREE_LANE_PARENT = 1;--static constant constexpr size_t TREE_LANE_DEPTH = 2;--static constant constexpr size_t TREE_LANE_CUM = 3;--static constant constexpr size_t TREE_LANE_LOGPROB = 4;--static constant constexpr size_t TREE_LANE_MASK = 5;--static constant constexpr size_t TREE_LANE_COUNT = TREE_LANE_MASK + 1;--static constant constexpr size_t METADATA_LANE_DEPTH = 0;--static constant constexpr size_t METADATA_LANE_ANCESTOR_COUNT = 1;--static constant constexpr size_t METADATA_LANE_NODE_INDEX = 2;--static constant constexpr size_t METADATA_LANE_COUNT = METADATA_LANE_NODE_INDEX + 1;+enum class FrontierIdx : uint32_t {+ TokenId = 0,+ ParentSlot = 1,+ Depth = 2,+ PathLogprobBits = 3,+ EdgeLogprobBits = 4,+ PathScoreKey = 5,+ Active = 6,+};++enum class TreeIdx : uint32_t {+ TokenId = 0,+ ParentSlot = 1,+ Depth = 2,+ PathLogprobBits = 3,+ EdgeLogprobBits = 4,+ Valid = 5,+};++enum class MetadataIdx : uint32_t {+ Depth = 0,+ AncestorCount = 1,+ TreeSlot = 2,+};static constant constexpr uint32_t FRONTIER_NO_WINNER = ~0;diff --git a/crates/backend-uzu/src/backends/metal/kernel/mod.rs b/crates/backend-uzu/src/backends/metal/kernel/mod.rsindex f2285865d..429e646c8 100644--- a/crates/backend-uzu/src/backends/metal/kernel/mod.rs+++ b/crates/backend-uzu/src/backends/metal/kernel/mod.rs@@ -3,7 +3,7 @@ use crate::backends::{Kernels,gpu_types::{gemm::{gemm_tiling_simdgroups_per_column, gemm_tiling_simdgroups_per_row},- weaver,+ weaver::{FRONTIER_SELECT_THREADS, TOP_CHILDREN_THREADS},},},metal::Metal,diff --git a/crates/backend-uzu/src/backends/metal/kernel/normalization/normalization.metal b/crates/backend-uzu/src/backends/metal/kernel/normalization/normalization.metalindex 5f1a7ada7..a94191831 100644--- a/crates/backend-uzu/src/backends/metal/kernel/normalization/normalization.metal+++ b/crates/backend-uzu/src/backends/metal/kernel/normalization/normalization.metal@@ -11,14 +11,15 @@ using namespace metal;// TODO: Are numerics of subtract_mean fine?-template <typename InputT, typename ScaleT, typename OutputT, typename AccumT>+template <typename InputT, typename AffineT, typename OutputT, typename AccumT>VARIANTS(InputT, float, half, bfloat)-VARIANTS(ScaleT, float, half, bfloat)+VARIANTS(AffineT, float, half, bfloat)VARIANTS(OutputT, float, half, bfloat)VARIANTS(AccumT, float)PUBLIC KERNEL(Normalization)(const device InputT* input OPTIONAL(!in_place),- const device ScaleT* scales,+ const device AffineT* scales,+ const device AffineT* biases OPTIONAL(has_biases),device OutputT* output,device InputT* shortcut OPTIONAL(copy_to_shortcut),const device int32_t* hadamard_factors OPTIONAL(use_hadamard),@@ -35,6 +36,7 @@ PUBLIC KERNEL(Normalization)(const bool use_hadamard SPECIALIZE,const bool scale_residual_sum SPECIALIZE,const bool scale_output SPECIALIZE,+ const bool has_biases SPECIALIZE,threadgroup AccumT shared_sum[METAL_SIMD_SIZE],const ThreadContext thread_context,const uint batch_idx GROUPS(batch_size),@@ -121,6 +123,10 @@ PUBLIC KERNEL(Normalization)(val = static_cast<OutputT>((x - mean) * rms_inv) * static_cast<OutputT>(scale);}+ if (has_biases) {+ val = static_cast<OutputT>(static_cast<AccumT>(val) + static_cast<AccumT>(biases[i]));+ }+if (use_hadamard) {val = static_cast<OutputT>(simdgroup_input_random_hadamard_transform(static_cast<ushort>(thread_in_row % METAL_SIMD_SIZE),diff --git a/crates/backend-uzu/src/backends/metal/kernel/weaver/weaver_frontier.h b/crates/backend-uzu/src/backends/metal/kernel/weaver/weaver_frontier.hindex 9cfecf384..1720ff9e3 100644--- a/crates/backend-uzu/src/backends/metal/kernel/weaver/weaver_frontier.h+++ b/crates/backend-uzu/src/backends/metal/kernel/weaver/weaver_frontier.h@@ -2,4 +2,4 @@#include "../generated/weaver.h"-namespace weaver = uzu::weaver;+using namespace uzu::weaver;diff --git a/crates/backend-uzu/src/backends/metal/kernel/weaver/weaver_frontier_scatter.metal b/crates/backend-uzu/src/backends/metal/kernel/weaver/weaver_frontier_scatter.metalindex 667591ca7..d5b1ad4da 100644--- a/crates/backend-uzu/src/backends/metal/kernel/weaver/weaver_frontier_scatter.metal+++ b/crates/backend-uzu/src/backends/metal/kernel/weaver/weaver_frontier_scatter.metal@@ -15,35 +15,35 @@ PUBLIC KERNEL(WeaverFrontierScatter)(constant uint& capacity,constant uint& tree_slots,constant uint& rows,- constant uint& fanout,- const uint position AXIS(rows* fanout, 64)+ constant uint& children_per_node,+ const uint position AXIS(rows* children_per_node, 64)) {- if (capacity == 0 || tree_slots == 0 || fanout == 0) {+ if (capacity == 0 || tree_slots == 0 || children_per_node == 0) {return;}- const uint row = position / fanout, child = position % fanout;+ const uint row = position / children_per_node, child = position % children_per_node;if (round_valid[row] == 0u) {return;}- const uint parent = round_metadata[weaver::METADATA_LANE_NODE_INDEX * rows + row];+ const uint parent = round_metadata[uint(MetadataIdx::TreeSlot) * rows + row];if (parent >= tree_slots) {return;}- const uint slot = parent * fanout + child;+ const uint slot = parent * children_per_node + child;if (slot >= capacity) {return;}- const float logprob = child_logprobs[row * fanout + child];- const float cumulative_logprob = as_type<float>(tree[weaver::TREE_LANE_CUM * tree_slots + parent]) + logprob;+ const float logprob = child_logprobs[row * children_per_node + child];+ const float cumulative_logprob = as_type<float>(tree[uint(TreeIdx::PathLogprobBits) * tree_slots + parent]) + logprob;- frontier[weaver::FRONTIER_LANE_TOKEN * capacity + slot] = child_ids[row * fanout + child];- frontier[weaver::FRONTIER_LANE_PARENT * capacity + slot] = parent;- frontier[weaver::FRONTIER_LANE_DEPTH * capacity + slot] = tree[weaver::TREE_LANE_DEPTH * tree_slots + parent] + 1u;- frontier[weaver::FRONTIER_LANE_CUM * capacity + slot] = as_type<uint>(cumulative_logprob);- frontier[weaver::FRONTIER_LANE_LOGPROB * capacity + slot] = as_type<uint>(logprob);- frontier[weaver::FRONTIER_LANE_KEY * capacity + slot] = top_k_score_key(cumulative_logprob);- frontier[weaver::FRONTIER_LANE_ACTIVE * capacity + slot] = 1u;+ frontier[uint(FrontierIdx::TokenId) * capacity + slot] = child_ids[row * children_per_node + child];+ frontier[uint(FrontierIdx::ParentSlot) * capacity + slot] = parent;+ frontier[uint(FrontierIdx::Depth) * capacity + slot] = tree[uint(TreeIdx::Depth) * tree_slots + parent] + 1u;+ frontier[uint(FrontierIdx::PathLogprobBits) * capacity + slot] = as_type<uint>(cumulative_logprob);+ frontier[uint(FrontierIdx::EdgeLogprobBits) * capacity + slot] = as_type<uint>(logprob);+ frontier[uint(FrontierIdx::PathScoreKey) * capacity + slot] = top_k_score_key(cumulative_logprob);+ frontier[uint(FrontierIdx::Active) * capacity + slot] = 1u;}diff --git a/crates/backend-uzu/src/backends/metal/kernel/weaver/weaver_frontier_select.metal b/crates/backend-uzu/src/backends/metal/kernel/weaver/weaver_frontier_select.metalindex b58b5b192..0ced94673 100644--- a/crates/backend-uzu/src/backends/metal/kernel/weaver/weaver_frontier_select.metal+++ b/crates/backend-uzu/src/backends/metal/kernel/weaver/weaver_frontier_select.metal@@ -27,34 +27,34 @@ PUBLIC KERNEL(WeaverFrontierSelect)(constant uint& lookahead_count,constant uint& candidate_pool_rows,constant uint& candidate_pool_size,- threadgroup uint4 reduce[weaver::FRONTIER_SELECT_SIMDGROUPS],- threadgroup uint winner_slot[weaver::FRONTIER_MAX_WIDTH],- threadgroup uint row_candidate_pool_row[weaver::FRONTIER_MAX_WIDTH],+ threadgroup uint4 reduce[FRONTIER_SELECT_SIMDGROUPS],+ threadgroup uint winner_slot[FRONTIER_MAX_WIDTH],+ threadgroup uint row_candidate_pool_row[FRONTIER_MAX_WIDTH],const ThreadContext thread_context,const uint group_index GROUPS(1),- const uint lid THREADS(weaver::FRONTIER_SELECT_THREADS)+ const uint lid THREADS(FRONTIER_SELECT_THREADS)) {(void)group_index;- if (capacity == 0 || capacity > weaver::FRONTIER_MAX_SLOTS || width == 0 || width > weaver::FRONTIER_MAX_WIDTH ||+ if (capacity == 0 || capacity > FRONTIER_MAX_SLOTS || width == 0 || width > FRONTIER_MAX_WIDTH ||ancestor_stride == 0 || max_depth == 0 || tree_slots == 0 || slot_start + width > tree_slots ||candidate_pool_rows == 0 || candidate_pool_size == 0) {return;}- bool entry_active[weaver::FRONTIER_ENTRIES_PER_THREAD];- for (uint entry = 0; entry < weaver::FRONTIER_ENTRIES_PER_THREAD; ++entry) {- const uint slot = lid + entry * weaver::FRONTIER_SELECT_THREADS;- entry_active[entry] = slot < capacity && frontier[weaver::FRONTIER_LANE_ACTIVE * capacity + slot] != 0u;+ bool entry_active[FRONTIER_ENTRIES_PER_THREAD];+ for (uint entry = 0; entry < FRONTIER_ENTRIES_PER_THREAD; ++entry) {+ const uint slot = lid + entry * FRONTIER_SELECT_THREADS;+ entry_active[entry] = slot < capacity && frontier[uint(FrontierIdx::Active) * capacity + slot] != 0u;}for (uint child = 0; child < width; ++child) {- uint4 local = uint4(0u, weaver::FRONTIER_NO_WINNER, weaver::FRONTIER_NO_WINNER, weaver::FRONTIER_NO_WINNER);- for (uint entry = 0; entry < weaver::FRONTIER_ENTRIES_PER_THREAD; ++entry) {- const uint slot = lid + entry * weaver::FRONTIER_SELECT_THREADS;+ uint4 local = uint4(0u, FRONTIER_NO_WINNER, FRONTIER_NO_WINNER, FRONTIER_NO_WINNER);+ for (uint entry = 0; entry < FRONTIER_ENTRIES_PER_THREAD; ++entry) {+ const uint slot = lid + entry * FRONTIER_SELECT_THREADS;if (entry_active[entry]) {- const uint key = frontier[weaver::FRONTIER_LANE_KEY * capacity + slot];- const uint parent = frontier[weaver::FRONTIER_LANE_PARENT * capacity + slot];- const uint token = frontier[weaver::FRONTIER_LANE_TOKEN * capacity + slot];+ const uint key = frontier[uint(FrontierIdx::PathScoreKey) * capacity + slot];+ const uint parent = frontier[uint(FrontierIdx::ParentSlot) * capacity + slot];+ const uint token = frontier[uint(FrontierIdx::TokenId) * capacity + slot];if (key > local.x || (key == local.x && (parent < local.y || (parent == local.y && token < local.z)))) {local = uint4(key, parent, token, slot);}@@ -63,24 +63,23 @@ PUBLIC KERNEL(WeaverFrontierSelect)(uint4 simd;simd.x = simd_max(local.x);- simd.y = simd_min(local.x == simd.x ? local.y : weaver::FRONTIER_NO_WINNER);- simd.z = simd_min(all(local.xy == simd.xy) ? local.z : weaver::FRONTIER_NO_WINNER);- simd.w = simd_min(all(local.xyz == simd.xyz) ? local.w : weaver::FRONTIER_NO_WINNER);+ simd.y = simd_min(local.x == simd.x ? local.y : FRONTIER_NO_WINNER);+ simd.z = simd_min(all(local.xy == simd.xy) ? local.z : FRONTIER_NO_WINNER);+ simd.w = simd_min(all(local.xyz == simd.xyz) ? local.w : FRONTIER_NO_WINNER);if (thread_context.simd_lane_id == 0) {reduce[thread_context.simdgroup_index] = simd;}threadgroup_barrier(mem_flags::mem_threadgroup);if (thread_context.simdgroup_index == 0) {- const uint4 partial =- thread_context.simd_lane_id < weaver::FRONTIER_SELECT_SIMDGROUPS- ? reduce[thread_context.simd_lane_id]- : uint4(0u, weaver::FRONTIER_NO_WINNER, weaver::FRONTIER_NO_WINNER, weaver::FRONTIER_NO_WINNER);+ const uint4 partial = thread_context.simd_lane_id < FRONTIER_SELECT_SIMDGROUPS+ ? reduce[thread_context.simd_lane_id]+ : uint4(0u, FRONTIER_NO_WINNER, FRONTIER_NO_WINNER, FRONTIER_NO_WINNER);uint4 selected;selected.x = simd_max(partial.x);- selected.y = simd_min(partial.x == selected.x ? partial.y : weaver::FRONTIER_NO_WINNER);- selected.z = simd_min(all(partial.xy == selected.xy) ? partial.z : weaver::FRONTIER_NO_WINNER);- selected.w = simd_min(all(partial.xyz == selected.xyz) ? partial.w : weaver::FRONTIER_NO_WINNER);+ selected.y = simd_min(partial.x == selected.x ? partial.y : FRONTIER_NO_WINNER);+ selected.z = simd_min(all(partial.xy == selected.xy) ? partial.z : FRONTIER_NO_WINNER);+ selected.w = simd_min(all(partial.xyz == selected.xyz) ? partial.w : FRONTIER_NO_WINNER);if (thread_context.simd_lane_id == 0) {winner_slot[child] = selected.w;}@@ -88,32 +87,32 @@ PUBLIC KERNEL(WeaverFrontierSelect)(threadgroup_barrier(mem_flags::mem_threadgroup);const uint selected = winner_slot[child];- for (uint entry = 0; entry < weaver::FRONTIER_ENTRIES_PER_THREAD; ++entry) {- entry_active[entry] = entry_active[entry] && (lid + entry * weaver::FRONTIER_SELECT_THREADS) != selected;+ for (uint entry = 0; entry < FRONTIER_ENTRIES_PER_THREAD; ++entry) {+ entry_active[entry] = entry_active[entry] && (lid + entry * FRONTIER_SELECT_THREADS) != selected;}}if (lid < width) {const uint row = lid;const uint slot = winner_slot[row];- const bool real = slot != weaver::FRONTIER_NO_WINNER;+ const bool real = slot != FRONTIER_NO_WINNER;const uint tree_slot = slot_start + row;- const uint token = real ? frontier[weaver::FRONTIER_LANE_TOKEN * capacity + slot] : 0u;- const uint parent = real ? frontier[weaver::FRONTIER_LANE_PARENT * capacity + slot] : weaver::FRONTIER_NO_WINNER;- const uint depth = real ? frontier[weaver::FRONTIER_LANE_DEPTH * capacity + slot] : 0u;- const uint cumulative_logprob = real ? frontier[weaver::FRONTIER_LANE_CUM * capacity + slot] : 0u;- const uint logprob = real ? frontier[weaver::FRONTIER_LANE_LOGPROB * capacity + slot] : 0u;+ const uint token = real ? frontier[uint(FrontierIdx::TokenId) * capacity + slot] : 0u;+ const uint parent = real ? frontier[uint(FrontierIdx::ParentSlot) * capacity + slot] : FRONTIER_NO_WINNER;+ const uint depth = real ? frontier[uint(FrontierIdx::Depth) * capacity + slot] : 0u;+ const uint cumulative_logprob = real ? frontier[uint(FrontierIdx::PathLogprobBits) * capacity + slot] : 0u;+ const uint logprob = real ? frontier[uint(FrontierIdx::EdgeLogprobBits) * capacity + slot] : 0u;- tree[weaver::TREE_LANE_TOKEN * tree_slots + tree_slot] = token;- tree[weaver::TREE_LANE_PARENT * tree_slots + tree_slot] = parent;- tree[weaver::TREE_LANE_DEPTH * tree_slots + tree_slot] = depth;- tree[weaver::TREE_LANE_CUM * tree_slots + tree_slot] = cumulative_logprob;- tree[weaver::TREE_LANE_LOGPROB * tree_slots + tree_slot] = logprob;- tree[weaver::TREE_LANE_MASK * tree_slots + tree_slot] = real ? 1u : 0u;+ tree[uint(TreeIdx::TokenId) * tree_slots + tree_slot] = token;+ tree[uint(TreeIdx::ParentSlot) * tree_slots + tree_slot] = parent;+ tree[uint(TreeIdx::Depth) * tree_slots + tree_slot] = depth;+ tree[uint(TreeIdx::PathLogprobBits) * tree_slots + tree_slot] = cumulative_logprob;+ tree[uint(TreeIdx::EdgeLogprobBits) * tree_slots + tree_slot] = logprob;+ tree[uint(TreeIdx::Valid) * tree_slots + tree_slot] = real ? 1u : 0u;if (real) {- frontier[weaver::FRONTIER_LANE_ACTIVE * capacity + slot] = 0u;+ frontier[uint(FrontierIdx::Active) * capacity + slot] = 0u;}const uint parent_slot = real && parent < tree_slots ? parent : 0u;@@ -127,9 +126,9 @@ PUBLIC KERNEL(WeaverFrontierSelect)(}round_token_ids[row] = token;- round_metadata[weaver::METADATA_LANE_DEPTH * width + row] = min(depth, max_depth - 1u);- round_metadata[weaver::METADATA_LANE_ANCESTOR_COUNT * width + row] = min(depth, ancestor_stride);- round_metadata[weaver::METADATA_LANE_NODE_INDEX * width + row] = tree_slot;+ round_metadata[uint(MetadataIdx::Depth) * width + row] = min(depth, max_depth - 1u);+ round_metadata[uint(MetadataIdx::AncestorCount) * width + row] = min(depth, ancestor_stride);+ round_metadata[uint(MetadataIdx::TreeSlot) * width + row] = tree_slot;round_valid[row] = real && depth < lookahead_count && depth < max_depth ? 1u : 0u;row_candidate_pool_row[row] = min(depth, candidate_pool_rows - 1u);@@ -140,7 +139,7 @@ PUBLIC KERNEL(WeaverFrontierSelect)(for (uint row = 0; row < width; ++row) {const uint source = row_candidate_pool_row[row] * candidate_pool_size;const uint destination = row * candidate_pool_size;- for (uint candidate = lid; candidate < candidate_pool_size; candidate += weaver::FRONTIER_SELECT_THREADS) {+ for (uint candidate = lid; candidate < candidate_pool_size; candidate += FRONTIER_SELECT_THREADS) {round_candidate_ids[destination + candidate] = candidate_pool_ids[source + candidate];round_candidate_scores[destination + candidate] = candidate_pool_scores[source + candidate];}diff --git a/crates/backend-uzu/src/backends/metal/kernel/weaver/weaver_node_cache_write.metal b/crates/backend-uzu/src/backends/metal/kernel/weaver/weaver_node_cache_write.metalindex 4c22b966f..a9b6c6bcf 100644--- a/crates/backend-uzu/src/backends/metal/kernel/weaver/weaver_node_cache_write.metal+++ b/crates/backend-uzu/src/backends/metal/kernel/weaver/weaver_node_cache_write.metal@@ -2,13 +2,13 @@#include "../common/dsl.h"// Scatters the key/value halves of the current step's packed QKV rows into their-// node-cache slots. Kept out of AttentionLastQuery so that the node arena stays+// node-cache slots. Kept out of AncestorAttention so that the node arena stays// read-only for the whole attention dispatch.template <typename T>VARIANTS(T, float, bfloat)PUBLIC KERNEL(WeaverNodeCacheWrite)(const device T* current_qkv,- device T* node_qkv,+ device T* node_kv,const device uint* node_indices,constant uint& model_dim,constant uint& node_capacity,@@ -20,7 +20,10 @@ PUBLIC KERNEL(WeaverNodeCacheWrite)(}const uint row = position / (2 * model_dim);const uint offset = position - row * 2 * model_dim;+ const uint component = offset / model_dim;+ const uint component_offset = offset % model_dim;const uint qkv_width = 3 * model_dim;const uint node = min(node_indices[row], node_capacity - 1u);- node_qkv[node * qkv_width + model_dim + offset] = current_qkv[row * qkv_width + model_dim + offset];+ node_kv[component * node_capacity * model_dim + node * model_dim + component_offset] =+ current_qkv[row * qkv_width + (component + 1) * model_dim + component_offset];}diff --git a/crates/backend-uzu/src/backends/metal/kernel/weaver/weaver_top_children.metal b/crates/backend-uzu/src/backends/metal/kernel/weaver/weaver_top_children.metalindex 84e101ffc..e80de0bc5 100644--- a/crates/backend-uzu/src/backends/metal/kernel/weaver/weaver_top_children.metal+++ b/crates/backend-uzu/src/backends/metal/kernel/weaver/weaver_top_children.metal@@ -13,17 +13,17 @@ METAL_FUNC bool weaver_better(uint score, uint token, uint index, uint best_scorPUBLIC KERNEL(WeaverTopChildren)(const device bfloat* residual_logits,- const device float* candidate_scores,+ const device float* candidate_logits,const device uint* candidate_ids,- device uint* output_ids,- device float* output_logprobs,+ device uint* output_token_ids,+ device float* output_model_logprobs,constant uint& rows,constant uint& candidates,- constant uint& children,- threadgroup float reduce_float[weaver::TOP_CHILDREN_SIMDGROUPS],- threadgroup uint reduce_score[weaver::TOP_CHILDREN_SIMDGROUPS],- threadgroup uint reduce_token[weaver::TOP_CHILDREN_SIMDGROUPS],- threadgroup uint reduce_index[weaver::TOP_CHILDREN_SIMDGROUPS],+ constant uint& children_per_node,+ threadgroup float reduce_float[TOP_CHILDREN_SIMDGROUPS],+ threadgroup uint reduce_score[TOP_CHILDREN_SIMDGROUPS],+ threadgroup uint reduce_token[TOP_CHILDREN_SIMDGROUPS],+ threadgroup uint reduce_index[TOP_CHILDREN_SIMDGROUPS],threadgroup float& logit_max,threadgroup float& log_sum,threadgroup uint& winner_score,@@ -31,21 +31,21 @@ PUBLIC KERNEL(WeaverTopChildren)(threadgroup uint& winner_index,const ThreadContext thread_context,const uint row GROUPS(rows),- const uint lid THREADS(weaver::TOP_CHILDREN_THREADS)+ const uint lid THREADS(TOP_CHILDREN_THREADS)) {- if (candidates == 0 || candidates > weaver::CANDIDATES_MAX || children == 0 || children > candidates) {+ if (candidates == 0 || candidates > CANDIDATES_MAX || children_per_node == 0 || children_per_node > candidates) {return;}const uint base = row * candidates;const uint first_index = lid;- const uint second_index = lid + weaver::TOP_CHILDREN_THREADS;+ const uint second_index = lid + TOP_CHILDREN_THREADS;const bool first_valid = first_index < candidates;const bool second_valid = second_index < candidates;const float first_logit =- first_valid ? candidate_scores[base + first_index] + float(residual_logits[base + first_index]) : -INFINITY;+ first_valid ? candidate_logits[base + first_index] + float(residual_logits[base + first_index]) : -INFINITY;const float second_logit =- second_valid ? candidate_scores[base + second_index] + float(residual_logits[base + second_index]) : -INFINITY;+ second_valid ? candidate_logits[base + second_index] + float(residual_logits[base + second_index]) : -INFINITY;const uint first_token = first_valid ? uint(candidate_ids[base + first_index]) : 0xffffffffu;const uint second_token = second_valid ? uint(candidate_ids[base + second_index]) : 0xffffffffu;const uint first_score = first_valid ? top_k_score_key(first_logit) : 0u;@@ -60,9 +60,8 @@ PUBLIC KERNEL(WeaverTopChildren)(}threadgroup_barrier(mem_flags::mem_threadgroup);if (thread_context.simdgroup_index == 0) {- const float group_value = thread_context.simd_lane_id < weaver::TOP_CHILDREN_SIMDGROUPS- ? reduce_float[thread_context.simd_lane_id]- : -INFINITY;+ const float group_value =+ thread_context.simd_lane_id < TOP_CHILDREN_SIMDGROUPS ? reduce_float[thread_context.simd_lane_id] : -INFINITY;const float maximum = simd_max(group_value);if (thread_context.simd_lane_id == 0) {logit_max = maximum;@@ -78,9 +77,8 @@ PUBLIC KERNEL(WeaverTopChildren)(}threadgroup_barrier(mem_flags::mem_threadgroup);if (thread_context.simdgroup_index == 0) {- const float group_value = thread_context.simd_lane_id < weaver::TOP_CHILDREN_SIMDGROUPS- ? reduce_float[thread_context.simd_lane_id]- : 0.0f;+ const float group_value =+ thread_context.simd_lane_id < TOP_CHILDREN_SIMDGROUPS ? reduce_float[thread_context.simd_lane_id] : 0.0f;const float total = simd_sum(group_value);if (thread_context.simd_lane_id == 0) {log_sum = log(total) + logit_max;@@ -88,7 +86,7 @@ PUBLIC KERNEL(WeaverTopChildren)(}threadgroup_barrier(mem_flags::mem_threadgroup);- for (uint child = 0; child < children; ++child) {+ for (uint child = 0; child < children_per_node; ++child) {uint local_score = 0u;uint local_token = 0xffffffffu;uint local_index = 0xffffffffu;@@ -116,7 +114,7 @@ PUBLIC KERNEL(WeaverTopChildren)(threadgroup_barrier(mem_flags::mem_threadgroup);if (thread_context.simdgroup_index == 0) {- const bool lane_valid = thread_context.simd_lane_id < weaver::TOP_CHILDREN_SIMDGROUPS;+ const bool lane_valid = thread_context.simd_lane_id < TOP_CHILDREN_SIMDGROUPS;const uint group_score = lane_valid ? reduce_score[thread_context.simd_lane_id] : 0u;const uint selected_score = simd_max(group_score);const uint group_token =@@ -136,8 +134,8 @@ PUBLIC KERNEL(WeaverTopChildren)(if (lid == 0) {const float winner_logit = top_k_score_from_key(winner_score);- output_ids[row * children + child] = winner_token;- output_logprobs[row * children + child] = winner_logit - log_sum;+ output_token_ids[row * children_per_node + child] = winner_token;+ output_model_logprobs[row * children_per_node + child] = winner_logit - log_sum;}first_active = first_active && first_index != winner_index;second_active = second_active && second_index != winner_index;diff --git a/crates/backend-uzu/src/config/weaver.rs b/crates/backend-uzu/src/config/weaver.rsindex 48f38cb2f..fbf4444e7 100644--- a/crates/backend-uzu/src/config/weaver.rs+++ b/crates/backend-uzu/src/config/weaver.rs@@ -1,6 +1,6 @@use proc_macros::uzu_config;-use crate::config::{linear::LinearConfig, normalization::NormalizationConfig};+use crate::config::normalization::NormalizationConfig;#[uzu_config]pub struct WeaverConfig {@@ -12,6 +12,5 @@ pub struct WeaverConfig {pub hidden_dim: usize,pub max_depth: usize,pub candidate_pool_size: usize,- pub linear_config: LinearConfig,pub norm_config: NormalizationConfig,}diff --git a/crates/backend-uzu/src/encodable_block/classifier.rs b/crates/backend-uzu/src/encodable_block/classifier.rsindex aab36146c..335869492 100644--- a/crates/backend-uzu/src/encodable_block/classifier.rs+++ b/crates/backend-uzu/src/encodable_block/classifier.rs@@ -8,7 +8,7 @@ use crate::{encodable_block::{batch_topology::BatchTopology,embedding::{Embedding, EmbeddingError},- normalization::{Normalization, NormalizationNewError, PostLayerScalar},+ normalization::{Normalization, NormalizationNewError, PostLayerScalar, ShortcutMode},prediction_head::{PredictionHead, PredictionHeadError},transformer::{Transformer, TransformerNewError},},@@ -62,8 +62,7 @@ impl<B: Backend> Classifier<B> {let embedding_norm = Normalization::new(config.transformer_config.model_dim,None,- false,- false,+ ShortcutMode::None,PostLayerScalar::None,data_type,&config.embedding_norm_config,diff --git a/crates/backend-uzu/src/encodable_block/dflash.rs b/crates/backend-uzu/src/encodable_block/dflash.rsindex 79f39657a..74c3198da 100644--- a/crates/backend-uzu/src/encodable_block/dflash.rs+++ b/crates/backend-uzu/src/encodable_block/dflash.rs@@ -24,7 +24,7 @@ use crate::{},},mlp::{Mlp, MlpBlockError},- normalization::{Normalization, NormalizationNewError, PostLayerScalar},+ normalization::{Normalization, NormalizationNewError, PostLayerScalar, ShortcutMode},},parameters::{ParameterLoaderError, ParameterTree},utils::maybe_mut::MaybeMut,@@ -92,7 +92,16 @@ fn plain_norm<B: Backend>(parameter_tree: &ParameterTree<B>,data_type: DataType,) -> Result<Normalization<B>, NormalizationNewError<B>> {- Normalization::new(model_dim, None, false, false, PostLayerScalar::None, data_type, config, parameter_tree, context)+ Normalization::new(+ model_dim,+ None,+ ShortcutMode::None,+ PostLayerScalar::None,+ data_type,+ config,+ parameter_tree,+ context,+ )}impl<B: Backend> DFlashDraft<B> {diff --git a/crates/backend-uzu/src/encodable_block/normalization.rs b/crates/backend-uzu/src/encodable_block/normalization.rsindex 1b6e14e48..4674084a7 100644--- a/crates/backend-uzu/src/encodable_block/normalization.rs+++ b/crates/backend-uzu/src/encodable_block/normalization.rs@@ -20,6 +20,13 @@ pub enum PostLayerScalar {ScaleOutput(f32),}+#[derive(Debug, Clone, Copy, PartialEq, Eq)]+pub enum ShortcutMode {+ None,+ Copy,+ Add,+}+#[derive(Debug, Error)]pub enum NormalizationNewError<B: Backend> {#[error("Backend error: {0}")]@@ -32,6 +39,7 @@ pub struct Normalization<B: Backend> {epsilon: f32,scale_offset: Option<f32>,scales: Allocation<B>,+ biases: Option<Allocation<B>>,element_count: usize,hadamard_factors: Option<Allocation<B>>,post_layer_scalar_value: f32,@@ -43,18 +51,24 @@ impl<B: Backend> Normalization<B> {pub fn new(element_count: usize,hadamard_factors: Option<Allocation<B>>,- copy_to_shortcut: bool,- residual_add: bool,+ shortcut_mode: ShortcutMode,post_layer_scalar: PostLayerScalar,data_type: DataType,config: &NormalizationConfig,parameter_tree: &ParameterTree<B>,context: &B::Context,) -> Result<Self, NormalizationNewError<B>> {- assert!(!config.has_biases, "normalization doesn't support biases");- assert!(copy_to_shortcut || !residual_add, "residual_add requires shortcut");+ let (copy_to_shortcut, residual_add) = match shortcut_mode {+ ShortcutMode::None => (false, false),+ ShortcutMode::Copy => (true, false),+ ShortcutMode::Add => (true, true),+ };let scales = parameter_tree.leaf("scales")?.validate(&[element_count], DataType::F32)?.read_allocation()?;+ let biases = config+ .has_biases+ .then(|| parameter_tree.leaf("biases")?.validate(&[element_count], DataType::F32)?.read_allocation())+ .transpose()?;let (scale_residual_sum, scale_output, post_layer_scalar_value) = match post_layer_scalar {PostLayerScalar::None => (false, false, 1.0),@@ -76,6 +90,7 @@ impl<B: Backend> Normalization<B> {hadamard_factors.is_some(),scale_residual_sum,scale_output,+ biases.is_some(),).map_err(NormalizationNewError::Backend)?;@@ -83,6 +98,7 @@ impl<B: Backend> Normalization<B> {epsilon: config.epsilon,scale_offset: config.scale_offset,scales,+ biases,element_count,hadamard_factors,post_layer_scalar_value,@@ -106,6 +122,7 @@ impl<B: Backend> Normalization<B> {self.kernel.encode(Some((input, row_offset_bytes)),&self.scales,+ self.biases.as_ref(),&mut output,shortcut,self.hadamard_factors.as_ref(),diff --git a/crates/backend-uzu/src/encodable_block/per_layer_embedding.rs b/crates/backend-uzu/src/encodable_block/per_layer_embedding.rsindex 705a5958f..180982a7c 100644--- a/crates/backend-uzu/src/encodable_block/per_layer_embedding.rs+++ b/crates/backend-uzu/src/encodable_block/per_layer_embedding.rs@@ -15,7 +15,7 @@ use crate::{data_type::DataType,encodable_block::{linear::{Linear, LinearBlockError},- normalization::{Normalization, NormalizationNewError, PostLayerScalar},+ normalization::{Normalization, NormalizationNewError, PostLayerScalar, ShortcutMode},},parameters::{ParameterLoaderError, ParameterTree},};@@ -83,8 +83,7 @@ impl<B: Backend> PerLayerEmbedding<B> {let projection_norm = Normalization::new(config.ple_dim,None,- false,- false,+ ShortcutMode::None,PostLayerScalar::ScaleOutput(config.input_scale),data_type,&projection_norm_config,@@ -200,8 +199,7 @@ impl<B: Backend> PerLayerEmbeddingProjection<B> {let norm = Normalization::new(model_dim,None,- false,- false,+ ShortcutMode::None,PostLayerScalar::None,data_type,&config.norm_config,diff --git a/crates/backend-uzu/src/encodable_block/prediction_head.rs b/crates/backend-uzu/src/encodable_block/prediction_head.rsindex 43b45f822..b08c0e5f8 100644--- a/crates/backend-uzu/src/encodable_block/prediction_head.rs+++ b/crates/backend-uzu/src/encodable_block/prediction_head.rs@@ -10,7 +10,7 @@ use crate::{data_type::DataType,encodable_block::{linear::{Linear, LinearBlockError},- normalization::{Normalization, NormalizationNewError, PostLayerScalar},+ normalization::{Normalization, NormalizationNewError, PostLayerScalar, ShortcutMode},},parameters::{ParameterLoaderError, ParameterTree},};@@ -61,8 +61,7 @@ impl<B: Backend> PredictionHead<B> {let normalization = Normalization::new(hidden_dim,None,- false,- false,+ ShortcutMode::None,PostLayerScalar::None,data_type,&config.normalization_config,diff --git a/crates/backend-uzu/src/encodable_block/transformer.rs b/crates/backend-uzu/src/encodable_block/transformer.rsindex ac31d1bc4..5cfe976ad 100644--- a/crates/backend-uzu/src/encodable_block/transformer.rs+++ b/crates/backend-uzu/src/encodable_block/transformer.rs@@ -9,7 +9,7 @@ use crate::{encodable_block::{batch_topology::BatchTopology,mixer::{MixerState, attention::rope::PrecalculatedRoPE},- normalization::{Normalization, NormalizationNewError, PostLayerScalar},+ normalization::{Normalization, NormalizationNewError, PostLayerScalar, ShortcutMode},transformer_layer::{TransformerLayer, TransformerLayerError},},parameters::{ParameterLoaderError, ParameterTree},@@ -132,8 +132,7 @@ impl<B: Backend> Transformer<B> {let output_norm = Normalization::new(transformer_config.model_dim,output_norm_hadamard_factors,- true,- true,+ ShortcutMode::Add,PostLayerScalar::None,data_type,&transformer_config.output_norm_config,diff --git a/crates/backend-uzu/src/encodable_block/transformer_layer.rs b/crates/backend-uzu/src/encodable_block/transformer_layer.rsindex a1bfbe756..5527ddcdf 100644--- a/crates/backend-uzu/src/encodable_block/transformer_layer.rs+++ b/crates/backend-uzu/src/encodable_block/transformer_layer.rs@@ -8,7 +8,7 @@ use crate::{batch_topology::BatchTopology,mixer::{Mixer, MixerNewError, MixerState, attention::rope::PrecalculatedRoPE},mlp::{Mlp, MlpBlockError},- normalization::{Normalization, NormalizationNewError, PostLayerScalar},+ normalization::{Normalization, NormalizationNewError, PostLayerScalar, ShortcutMode},per_layer_embedding::PerLayerEmbeddingProjection,},parameters::{ParameterLoaderError, ParameterTree},@@ -94,8 +94,11 @@ impl<B: Backend> TransformerLayer<B> {Some(Normalization::new(transformer_config.model_dim,mixer_hadamard_factors,- true,- layer_index > 0,+ if layer_index > 0 {+ ShortcutMode::Add+ } else {+ ShortcutMode::Copy+ },PostLayerScalar::None,data_type,pre_mixer_norm_config,@@ -113,8 +116,7 @@ impl<B: Backend> TransformerLayer<B> {Some(Normalization::new(transformer_config.model_dim,None,- false,- false,+ ShortcutMode::None,PostLayerScalar::None,data_type,norm_config,@@ -137,8 +139,7 @@ impl<B: Backend> TransformerLayer<B> {let pre_mlp_norm = Normalization::new(transformer_config.model_dim,mlp_input_hadamard_factors,- true,- true,+ ShortcutMode::Add,residual_sum_scalar,data_type,&layer_config.pre_mlp_norm_config,@@ -150,8 +151,7 @@ impl<B: Backend> TransformerLayer<B> {Some(Normalization::new(transformer_config.model_dim,None,- false,- false,+ ShortcutMode::None,output_scalar,data_type,norm_config,diff --git a/crates/backend-uzu/src/encodable_block/weaver.rs b/crates/backend-uzu/src/encodable_block/weaver.rsindex a321226c8..c57281cf1 100644--- a/crates/backend-uzu/src/encodable_block/weaver.rs+++ b/crates/backend-uzu/src/encodable_block/weaver.rs@@ -3,11 +3,11 @@ use thiserror::Error;use crate::{array::size_for_shape,backends::common::{- Allocation, AllocationType, Backend, Context, Encoder, Kernels,- gpu_types::weaver,+ Allocation, AllocationType, AsBufferRangeMut, Backend, Context, Encoder, Kernels,+ gpu_types::weaver::{CANDIDATES_MAX, MetadataIdx},kernel::{- ActivationKernel, AttentionLastQueryKernel, AttentionPrepareKernel, TensorAddBiasKernel,- TensorAddSwapKernel, WeaverNodeCacheWriteKernel, WeaverTopChildrenKernel,+ ActivationKernel, AncestorAttentionKernel, AttentionPrepareKernel, TensorAddBiasKernel,+ WeaverNodeCacheWriteKernel, WeaverTopChildrenKernel,},},config::{normalization::NormalizationConfig, weaver::WeaverConfig},@@ -19,75 +19,88 @@ use crate::{AttentionStateType,core::{AttentionCoreEncodeArguments, AttentionCoreNewArguments, AttentionCores},},- normalization::{Normalization, NormalizationNewError, PostLayerScalar},+ normalization::{Normalization, NormalizationNewError, PostLayerScalar, ShortcutMode},},parameters::{ParameterLoaderError, ParameterTree},};-pub(crate) struct WeaverPrefix<B: Backend> {- layer_qkv: Box<[Allocation<B>]>,- pub(crate) length: usize,+const DATA_TYPE: DataType = DataType::BF16;++pub(crate) struct TopKChildren<B: Backend> {+ pub(crate) token_ids: Allocation<B>,+ pub(crate) logprobs: Allocation<B>,+}++struct PrefixOutput<B: Backend> {+ hidden: Allocation<B>,+ kv_cache: Allocation<B>,+}++struct PrefixAttentionInputs<B: Backend> {+ queries: Allocation<B>,+ kv_cache: Allocation<B>,+}++pub(crate) struct WeaverPrefixKvCache<B: Backend> {+ layer_kv: Box<[Allocation<B>]>,+ token_count: usize,+}++pub(crate) struct WeaverNodeKvCache<B: Backend> {+ layer_kv: Box<[Allocation<B>]>,+ node_capacity: u32,}pub(crate) struct WeaverStepBatch<'a, B: Backend> {- pub row_count: usize,- pub candidate_count: usize,+ pub node_count: usize,+ pub candidates_per_node: usize,pub ancestor_stride: usize,- pub parent_token_ids: &'a Allocation<B>,+ pub node_token_ids: &'a Allocation<B>,pub candidate_ids: &'a Allocation<B>,- pub candidate_scores: &'a Allocation<B>,+ pub candidate_logits: &'a Allocation<B>,pub ancestor_indices: &'a Allocation<B>,pub node_metadata: &'a Allocation<B>,}-pub(crate) struct WeaverNodeState<B: Backend> {- layer_qkv: Box<[Allocation<B>]>,-}-pub(crate) struct Weaver<B: Backend> {- embedding_norm: WeaverNorm<B>,- hidden_state_norm: WeaverNorm<B>,- embedding_projection: Box<dyn Linear<B>>,- blocks: Box<[WeaverBlock<B>]>,- output_norm: WeaverNorm<B>,+ token_embedding_norm: Normalization<B>,+ token_embedding_projection: Box<dyn Linear<B>>,+ hidden_state_norm: Normalization<B>,hidden_state_projection: Box<dyn Linear<B>>,- query_projection: Box<dyn Linear<B>>,+ blocks: Box<[WeaverBlock<B>]>,+ readout_norm: Normalization<B>,+ readout_query_projection: Box<dyn Linear<B>>,position_embeddings: Allocation<B>,- position_add: <B::Kernels as Kernels>::TensorAddBiasKernel,- indexed_position_add: <B::Kernels as Kernels>::TensorAddBiasKernel,+ prefix_position_add: <B::Kernels as Kernels>::TensorAddBiasKernel,+ node_position_add: <B::Kernels as Kernels>::TensorAddBiasKernel,top_children: <B::Kernels as Kernels>::WeaverTopChildrenKernel,model_dim: usize,target_model_dim: usize,max_depth: usize,- data_type: DataType,-}--struct WeaverNorm<B: Backend> {- normalization: Normalization<B>,- biases: Option<Allocation<B>>,- bias_kernel: Option<<B::Kernels as Kernels>::TensorAddBiasKernel>,- dimension: usize,}struct WeaverBlock<B: Backend> {+ // Attention+ pre_attention_norm: Normalization<B>,qkv_projection: Box<dyn Linear<B>>,- out_projection: Box<dyn Linear<B>>,- prefix_attention: AttentionCores<B>,attention_prepare: <B::Kernels as Kernels>::AttentionPrepareKernel,- pre_attention_norm: WeaverNorm<B>,- pre_mlp_norm: WeaverNorm<B>,+ prefix_attention: AttentionCores<B>,+ ancestor_attention: <B::Kernels as Kernels>::AncestorAttentionKernel,+ node_cache_write: <B::Kernels as Kernels>::WeaverNodeCacheWriteKernel,+ out_projection: Box<dyn Linear<B>>,++ // MLP+ pre_mlp_norm: Normalization<B>,up_projection: Box<dyn Linear<B>>,- down_projection: Box<dyn Linear<B>>,activation: <B::Kernels as Kernels>::ActivationKernel,- residual_add: <B::Kernels as Kernels>::TensorAddSwapKernel,- last_query_attention: <B::Kernels as Kernels>::AttentionLastQueryKernel,- node_cache_write: <B::Kernels as Kernels>::WeaverNodeCacheWriteKernel,+ down_projection: Box<dyn Linear<B>>,++ // Geometryattention_scale: f32,model_dim: usize,hidden_dim: usize,num_heads: usize,head_dim: usize,- data_type: DataType,}#[derive(Debug, Error)]@@ -100,9 +113,11 @@ pub enum WeaverNewError<B: Backend> {Normalization(#[from] NormalizationNewError<B>),#[error("backend error: {0}")]Backend(#[source] B::Error),+ #[error("Weaver requires at least one layer")]+ InvalidLayerCount,#[error("model_dim must be divisible by num_heads")]InvalidHeadConfig,- #[error("candidate_pool_size must be in 1..={max}, got {0}", max = weaver::CANDIDATES_MAX)]+ #[error("candidate_pool_size must be in 1..={max}, got {0}", max = CANDIDATES_MAX)]InvalidCandidatePoolSize(usize),}@@ -114,120 +129,48 @@ pub enum WeaverEncodeError<B: Backend> {Embedding(#[from] EmbeddingError<B>),}-fn linear<B: Backend>(- context: &B::Context,- parameter_tree: &ParameterTree<B>,- name: &str,- input_dim: usize,- output_dim: usize,- has_biases: bool,- data_type: DataType,-) -> Result<Box<dyn Linear<B>>, WeaverNewError<B>> {- Ok(<dyn Linear<B>>::new(input_dim, [output_dim], has_biases, context, data_type, ¶meter_tree.subtree(name)?)?)-}--impl<B: Backend> WeaverNorm<B> {- fn new(- context: &B::Context,- dim: usize,- config: &NormalizationConfig,- parameter_tree: &ParameterTree<B>,- data_type: DataType,- ) -> Result<Self, WeaverNewError<B>> {- let mut normalization_config = config.clone();- normalization_config.has_biases = false;- let normalization = Normalization::new(- dim,- None,- false,- false,- PostLayerScalar::None,- data_type,- &normalization_config,- parameter_tree,- context,- )?;- let biases = config- .has_biases- .then(|| parameter_tree.leaf("biases")?.validate(&[dim], DataType::F32)?.read_allocation())- .transpose()?;- let bias_kernel = biases- .as_ref()- .map(|_| <B::Kernels as Kernels>::TensorAddBiasKernel::new(context, data_type, DataType::F32, true, false))- .transpose()- .map_err(WeaverNewError::Backend)?;- Ok(Self {- normalization,- biases,- bias_kernel,- dimension: dim,- })- }-- fn encode(- &self,- input: &Allocation<B>,- row_count: usize,- encoder: &mut Encoder<B>,- ) -> Result<Allocation<B>, B::Error> {- let mut output = self.normalization.encode(input, 0, row_count, None, encoder)?;- if let (Some(biases), Some(kernel)) = (&self.biases, &self.bias_kernel) {- kernel.encode(- None::<&Allocation<B>>,- biases,- None::<&Allocation<B>>,- &mut output,- self.dimension as u32,- (row_count * self.dimension) as u32,- encoder,- );- }- Ok(output)- }-}-impl<B: Backend> Weaver<B> {- pub(crate) fn create_node_state(- &self,- capacity: usize,- context: &B::Context,- ) -> Result<WeaverNodeState<B>, WeaverEncodeError<B>> {- assert!(capacity > 0);- let qkv_size = size_for_shape(&[capacity, 3, self.model_dim], self.data_type);- let layer_qkv = (0..self.blocks.len())- .map(|_| context.create_allocation(qkv_size, AllocationType::Global).map_err(WeaverEncodeError::Backend))- .collect::<Result<Box<[_]>, _>>()?;- Ok(WeaverNodeState {- layer_qkv,- })- }-pub(crate) fn new(context: &B::Context,config: &WeaverConfig,parameter_tree: &ParameterTree<B>,- data_type: DataType,) -> Result<Self, WeaverNewError<B>> {- assert_eq!(data_type, DataType::BF16, "Weaver only supports BF16");+ if config.num_layers == 0 {+ return Err(WeaverNewError::InvalidLayerCount);+ }if config.num_heads == 0 || !config.model_dim.is_multiple_of(config.num_heads) {return Err(WeaverNewError::InvalidHeadConfig);}- if config.candidate_pool_size == 0 || config.candidate_pool_size > weaver::CANDIDATES_MAX {+ if config.candidate_pool_size == 0 || config.candidate_pool_size > CANDIDATES_MAX {return Err(WeaverNewError::InvalidCandidatePoolSize(config.candidate_pool_size));}- let norm = |dim, name: &str| -> Result<_, WeaverNewError<B>> {- WeaverNorm::new(context, dim, &config.norm_config, ¶meter_tree.subtree(name)?, data_type)- };- let embedding_norm = norm(config.target_embedding_dim, "embedding_norm")?;- let hidden_state_norm = norm(config.target_model_dim, "hidden_state_norm")?;- let embedding_projection = linear(+ let token_embedding_norm = Normalization::new(+ config.target_embedding_dim,+ None,+ ShortcutMode::None,+ PostLayerScalar::None,+ DATA_TYPE,+ &config.norm_config,+ ¶meter_tree.subtree("embedding_norm")?,context,- parameter_tree,- "embedding_projection",+ )?;+ let hidden_state_norm = Normalization::new(+ config.target_model_dim,+ None,+ ShortcutMode::None,+ PostLayerScalar::None,+ DATA_TYPE,+ &config.norm_config,+ ¶meter_tree.subtree("hidden_state_norm")?,+ context,+ )?;+ let token_embedding_projection = <dyn Linear<B>>::new(config.target_embedding_dim,- config.model_dim,+ [config.model_dim],true,- data_type,+ context,+ DATA_TYPE,+ ¶meter_tree.subtree("embedding_projection")?,)?;let blocks_tree = parameter_tree.subtree("blocks")?;let blocks = (0..config.num_layers)@@ -237,207 +180,241 @@ impl<B: Backend> Weaver<B> {config.model_dim,config.hidden_dim,config.num_heads,+ index > 0,&config.norm_config,&blocks_tree.subtree(&index.to_string())?,- data_type,)}).collect::<Result<Box<[_]>, WeaverNewError<B>>>()?;- let output_norm = norm(config.model_dim, "output_norm")?;- let hidden_state_projection = linear(+ let readout_norm = Normalization::new(+ config.model_dim,+ None,+ ShortcutMode::Add,+ PostLayerScalar::None,+ DATA_TYPE,+ &config.norm_config,+ ¶meter_tree.subtree("output_norm")?,context,- parameter_tree,- "hidden_state_projection",+ )?;+ let hidden_state_projection = <dyn Linear<B>>::new(config.target_model_dim,- config.model_dim,+ [config.model_dim],true,- data_type,- )?;- let query_projection = linear(context,- parameter_tree,- "query_projection",+ DATA_TYPE,+ ¶meter_tree.subtree("hidden_state_projection")?,+ )?;+ let readout_query_projection = <dyn Linear<B>>::new(config.model_dim,- config.target_model_dim,+ [config.target_model_dim],false,- data_type,+ context,+ DATA_TYPE,+ ¶meter_tree.subtree("query_projection")?,)?;let position_embeddings = parameter_tree.leaf("position_embeddings")?.validate(&[config.max_depth, config.model_dim], DataType::F32)?.read_allocation()?;- let position_add =- <B::Kernels as Kernels>::TensorAddBiasKernel::new(context, data_type, DataType::F32, true, false)+ let prefix_position_add =+ <B::Kernels as Kernels>::TensorAddBiasKernel::new(context, DATA_TYPE, DataType::F32, true, false).map_err(WeaverNewError::Backend)?;- let indexed_position_add =- <B::Kernels as Kernels>::TensorAddBiasKernel::new(context, data_type, DataType::F32, true, true)+ let node_position_add =+ <B::Kernels as Kernels>::TensorAddBiasKernel::new(context, DATA_TYPE, DataType::F32, true, true).map_err(WeaverNewError::Backend)?;let top_children =<B::Kernels as Kernels>::WeaverTopChildrenKernel::new(context).map_err(WeaverNewError::Backend)?;Ok(Self {- embedding_norm,+ token_embedding_norm,+ token_embedding_projection,hidden_state_norm,- embedding_projection,- blocks,- output_norm,hidden_state_projection,- query_projection,+ blocks,+ readout_norm,+ readout_query_projection,position_embeddings,- position_add,- indexed_position_add,+ prefix_position_add,+ node_position_add,top_children,model_dim: config.model_dim,target_model_dim: config.target_model_dim,max_depth: config.max_depth,- data_type,})}pub(crate) fn build_prefix(&self,target_hidden: &Allocation<B>,- lookaheads: &Allocation<B>,- lookahead_offset: usize,+ draft_hidden: &Allocation<B>,lookahead_count: usize,encoder: &mut Encoder<B>,- ) -> Result<WeaverPrefix<B>, WeaverEncodeError<B>> {- let length = lookahead_count + 1;- assert!(lookahead_count <= self.max_depth);- let row_bytes = self.target_model_dim * self.data_type.size_in_bytes();- assert!(target_hidden.size() >= row_bytes);- assert!(lookaheads.size() >= (lookahead_offset + lookahead_count) * row_bytes);+ ) -> Result<WeaverPrefixKvCache<B>, WeaverEncodeError<B>> {+ assert!((1..=self.max_depth).contains(&lookahead_count));+ let token_count = lookahead_count + 1;+ let hidden_row_bytes = self.target_model_dim * DATA_TYPE.size_in_bytes();- let context = encoder.context();- let mut input = encoder- .allocate_scratch(size_for_shape(&[length, self.target_model_dim], self.data_type))+ let mut prefix_input = encoder+ .allocate_scratch(size_for_shape(&[token_count, self.target_model_dim], DATA_TYPE))+ .map_err(WeaverEncodeError::Backend)?;+ encoder.encode_copy(target_hidden, 0..hidden_row_bytes, &mut prefix_input, 0..hidden_row_bytes);+ encoder.encode_copy(+ draft_hidden,+ hidden_row_bytes..token_count * hidden_row_bytes,+ &mut prefix_input,+ hidden_row_bytes..token_count * hidden_row_bytes,+ );+ let normalized_prefix = self+ .hidden_state_norm+ .encode(&prefix_input, 0, token_count, None, encoder)+ .map_err(WeaverEncodeError::Backend)?;+ let mut hidden = self+ .hidden_state_projection+ .encode(normalized_prefix, token_count, encoder).map_err(WeaverEncodeError::Backend)?;- encoder.encode_copy(target_hidden, 0..row_bytes, &mut input, 0..row_bytes);- if lookahead_count > 0 {- encoder.encode_copy(- lookaheads,- lookahead_offset * row_bytes..(lookahead_offset + lookahead_count) * row_bytes,- &mut input,- row_bytes..length * row_bytes,- );- }- let normalized = self.hidden_state_norm.encode(&input, length, encoder).map_err(WeaverEncodeError::Backend)?;- let mut hidden =- self.hidden_state_projection.encode(normalized, length, encoder).map_err(WeaverEncodeError::Backend)?;let position_elements = lookahead_count * self.model_dim;- self.position_add.encode(+ self.prefix_position_add.encode(None::<&Allocation<B>>,&self.position_embeddings,None::<&Allocation<B>>,- (&mut hidden, self.model_dim * self.data_type.size_in_bytes()),+ (&mut hidden, self.model_dim * DATA_TYPE.size_in_bytes()),position_elements as u32,position_elements as u32,encoder,);-- let mut layer_qkv = Vec::with_capacity(self.blocks.len());- for block in &self.blocks {- let (next_hidden, qkv) =- block.encode_prefix(hidden, length, encoder).map_err(WeaverEncodeError::Backend)?;- let mut cached_qkv =- context.create_allocation(qkv.size(), AllocationType::Global).map_err(WeaverEncodeError::Backend)?;- encoder.encode_copy(&qkv, .., &mut cached_qkv, ..);- layer_qkv.push(cached_qkv);+ let (last_block, preceding_blocks) = self.blocks.split_last().expect("Weaver must have at least one block");+ let mut residual = encoder.allocate_scratch(hidden.size()).map_err(WeaverEncodeError::Backend)?;+ let mut layer_kv = Vec::with_capacity(self.blocks.len());+ for block in preceding_blocks {+ let PrefixOutput {+ hidden: next_hidden,+ kv_cache,+ } = block.encode_prefix(hidden, &mut residual, token_count, encoder).map_err(WeaverEncodeError::Backend)?;+ layer_kv.push(kv_cache);hidden = next_hidden;}- drop(input);- drop(hidden);- Ok(WeaverPrefix {- layer_qkv: layer_qkv.into_boxed_slice(),- length,+ layer_kv.push(+ last_block+ .prepare_prefix_attention(&hidden, &mut residual, token_count, encoder)+ .map_err(WeaverEncodeError::Backend)?+ .kv_cache,+ );+ Ok(WeaverPrefixKvCache {+ layer_kv: layer_kv.into_boxed_slice(),+ token_count,+ })+ }++ pub(crate) fn create_node_kv_cache(+ &self,+ capacity: usize,+ context: &B::Context,+ ) -> Result<WeaverNodeKvCache<B>, WeaverEncodeError<B>> {+ assert!(capacity > 0, "Weaver node capacity must be positive");+ let node_capacity = u32::try_from(capacity).expect("Weaver node capacity exceeds the kernel limit");+ let kv_size = size_for_shape(&[2, capacity, self.model_dim], DATA_TYPE);+ let layer_kv = (0..self.blocks.len())+ .map(|_| context.create_allocation(kv_size, AllocationType::Global).map_err(WeaverEncodeError::Backend))+ .collect::<Result<Box<[_]>, _>>()?;+ Ok(WeaverNodeKvCache {+ layer_kv,+ node_capacity,})}pub(crate) fn encode_step_batch(&self,- prefix: &WeaverPrefix<B>,- input: &WeaverStepBatch<'_, B>,- state: &mut WeaverNodeState<B>,- children: usize,+ prefix: &WeaverPrefixKvCache<B>,+ batch: &WeaverStepBatch<'_, B>,+ node_cache: &mut WeaverNodeKvCache<B>,+ children_per_node: usize,target_embedding: &Embedding<B>,encoder: &mut Encoder<B>,- ) -> Result<(Allocation<B>, Allocation<B>), WeaverEncodeError<B>> {- let rows = input.row_count;- let candidates = input.candidate_count;- assert!(rows > 0);- assert!(candidates > 0 && candidates <= weaver::CANDIDATES_MAX);- assert!(children > 0 && children <= candidates);- assert!(input.ancestor_stride > 0);- let word = DataType::U32.size_in_bytes();- debug_assert!(input.node_metadata.size() >= weaver::METADATA_LANE_COUNT * rows * word);- debug_assert!(input.parent_token_ids.size() >= rows * word);- debug_assert!(input.ancestor_indices.size() >= rows * input.ancestor_stride * word);-- let token_embedding = target_embedding.encode_lookup(input.parent_token_ids, rows, encoder)?;- let embedding_normalized =- self.embedding_norm.encode(&token_embedding, rows, encoder).map_err(WeaverEncodeError::Backend)?;- let mut current = self- .embedding_projection- .encode(embedding_normalized, rows, encoder)+ ) -> Result<TopKChildren<B>, WeaverEncodeError<B>> {+ let token_embedding = target_embedding.encode_lookup(batch.node_token_ids, batch.node_count, encoder)?;+ let normalized_embedding = self+ .token_embedding_norm+ .encode(&token_embedding, 0, batch.node_count, None, encoder)+ .map_err(WeaverEncodeError::Backend)?;+ let mut hidden = self+ .token_embedding_projection+ .encode(normalized_embedding, batch.node_count, encoder).map_err(WeaverEncodeError::Backend)?;- self.indexed_position_add.encode(+ self.node_position_add.encode(None::<&Allocation<B>>,&self.position_embeddings,- Some(input.node_metadata),- &mut current,+ Some(batch.node_metadata),+ &mut hidden,self.model_dim as u32,- (rows * self.model_dim) as u32,+ (batch.node_count * self.model_dim) as u32,encoder,);+ let mut residual = encoder.allocate_scratch(hidden.size()).map_err(WeaverEncodeError::Backend)?;+ let node_capacity = node_cache.node_capacity;for (layer_index, block) in self.blocks.iter().enumerate() {- current = block+ hidden = block.encode_step(- current,- &prefix.layer_qkv[layer_index],- &mut state.layer_qkv[layer_index],- input,- prefix.length,+ hidden,+ &mut residual,+ &prefix.layer_kv[layer_index],+ &mut node_cache.layer_kv[layer_index],+ node_capacity,+ batch,+ prefix.token_count,encoder,).map_err(WeaverEncodeError::Backend)?;}- self.encode_step_output(¤t, input, children, target_embedding, encoder)+ self.encode_step_output(&hidden, &mut residual, batch, children_per_node, target_embedding, encoder)}/// Outputs must outlive the batch.fn encode_step_output(&self,- current: &Allocation<B>,- step: &WeaverStepBatch<'_, B>,- children: usize,+ hidden: &Allocation<B>,+ residual: &mut Allocation<B>,+ batch: &WeaverStepBatch<'_, B>,+ children_per_node: usize,target_embedding: &Embedding<B>,encoder: &mut Encoder<B>,- ) -> Result<(Allocation<B>, Allocation<B>), WeaverEncodeError<B>> {- let (rows, candidates) = (step.row_count, step.candidate_count);- let output_normalized = self.output_norm.encode(current, rows, encoder).map_err(WeaverEncodeError::Backend)?;- let query =- self.query_projection.encode(output_normalized, rows, encoder).map_err(WeaverEncodeError::Backend)?;- let candidate_logits =- target_embedding.encode_readout_sparse(&query, step.candidate_ids, rows, candidates, encoder)?;- let mut child_ids = encoder- .allocate_scratch(size_for_shape(&[rows, children], DataType::U32))+ ) -> Result<TopKChildren<B>, WeaverEncodeError<B>> {+ let normalized_output = self+ .readout_norm+ .encode(hidden, 0, batch.node_count, Some(residual), encoder).map_err(WeaverEncodeError::Backend)?;- let mut child_logprobs = encoder- .allocate_scratch(size_for_shape(&[rows, children], DataType::F32))+ let query = self+ .readout_query_projection+ .encode(normalized_output, batch.node_count, encoder)+ .map_err(WeaverEncodeError::Backend)?;+ let residual_logits = target_embedding.encode_readout_sparse(+ &query,+ batch.candidate_ids,+ batch.node_count,+ batch.candidates_per_node,+ encoder,+ )?;+ let mut token_ids = encoder+ .allocate_scratch(size_for_shape(&[batch.node_count, children_per_node], DataType::U32))+ .map_err(WeaverEncodeError::Backend)?;+ let mut logprobs = encoder+ .allocate_scratch(size_for_shape(&[batch.node_count, children_per_node], DataType::F32)).map_err(WeaverEncodeError::Backend)?;self.top_children.encode(- &candidate_logits,- step.candidate_scores,- step.candidate_ids,- &mut child_ids,- &mut child_logprobs,- rows as u32,- candidates as u32,- children as u32,+ &residual_logits,+ batch.candidate_logits,+ batch.candidate_ids,+ &mut token_ids,+ &mut logprobs,+ batch.node_count as u32,+ batch.candidates_per_node as u32,+ children_per_node as u32,encoder,);- Ok((child_ids, child_logprobs))+ Ok(TopKChildren {+ token_ids,+ logprobs,+ })}}@@ -447,15 +424,28 @@ impl<B: Backend> WeaverBlock<B> {model_dim: usize,hidden_dim: usize,num_heads: usize,+ add_to_residual: bool,norm_config: &NormalizationConfig,parameter_tree: &ParameterTree<B>,- data_type: DataType,) -> Result<Self, WeaverNewError<B>> {let head_dim = model_dim / num_heads;let attention_scale = 1.0 / (head_dim as f32).sqrt();- let qkv_projection =- linear(context, parameter_tree, "qkv_projection", model_dim, 3 * model_dim, false, data_type)?;- let out_projection = linear(context, parameter_tree, "out_projection", model_dim, model_dim, false, data_type)?;+ let qkv_projection = <dyn Linear<B>>::new(+ model_dim,+ [3 * model_dim],+ false,+ context,+ DATA_TYPE,+ ¶meter_tree.subtree("qkv_projection")?,+ )?;+ let out_projection = <dyn Linear<B>>::new(+ model_dim,+ [model_dim],+ false,+ context,+ DATA_TYPE,+ ¶meter_tree.subtree("out_projection")?,+ )?;let prefix_attention = AttentionCores::new(AttentionCoreNewArguments {head_dim,@@ -467,30 +457,60 @@ impl<B: Backend> WeaverBlock<B> {is_trie: false,sliding_window_size: None,scale: Some(attention_scale),- data_type,+ data_type: DATA_TYPE,},context,).map_err(WeaverNewError::Backend)?;let attention_prepare =- <B::Kernels as Kernels>::AttentionPrepareKernel::new(context, data_type, DataType::F32, true, false)+ <B::Kernels as Kernels>::AttentionPrepareKernel::new(context, DATA_TYPE, DataType::F32, true, false).map_err(WeaverNewError::Backend)?;- let norm = |name: &str| -> Result<_, WeaverNewError<B>> {- WeaverNorm::new(context, model_dim, norm_config, ¶meter_tree.subtree(name)?, data_type)- };- let pre_attention_norm = norm("pre_attention_norm")?;- let pre_mlp_norm = norm("pre_mlp_norm")?;- let up_projection = linear(context, parameter_tree, "up_projection", model_dim, hidden_dim, true, data_type)?;- let down_projection =- linear(context, parameter_tree, "down_projection", hidden_dim, model_dim, true, data_type)?;- let activation = <B::Kernels as Kernels>::ActivationKernel::new(context, data_type, true)+ let pre_attention_norm = Normalization::new(+ model_dim,+ None,+ if add_to_residual {+ ShortcutMode::Add+ } else {+ ShortcutMode::Copy+ },+ PostLayerScalar::None,+ DATA_TYPE,+ norm_config,+ ¶meter_tree.subtree("pre_attention_norm")?,+ context,+ )?;+ let pre_mlp_norm = Normalization::new(+ model_dim,+ None,+ ShortcutMode::Add,+ PostLayerScalar::None,+ DATA_TYPE,+ norm_config,+ ¶meter_tree.subtree("pre_mlp_norm")?,+ context,+ )?;+ let up_projection = <dyn Linear<B>>::new(+ model_dim,+ [hidden_dim],+ true,+ context,+ DATA_TYPE,+ ¶meter_tree.subtree("up_projection")?,+ )?;+ let down_projection = <dyn Linear<B>>::new(+ hidden_dim,+ [model_dim],+ true,+ context,+ DATA_TYPE,+ ¶meter_tree.subtree("down_projection")?,+ )?;+ let activation = <B::Kernels as Kernels>::ActivationKernel::new(context, DATA_TYPE, true).map_err(WeaverNewError::Backend)?;- let residual_add =- <B::Kernels as Kernels>::TensorAddSwapKernel::new(context, data_type).map_err(WeaverNewError::Backend)?;- let last_query_attention =- <B::Kernels as Kernels>::AttentionLastQueryKernel::new(context, head_dim as u32, num_heads as u32)+ let ancestor_attention =+ <B::Kernels as Kernels>::AncestorAttentionKernel::new(context, head_dim as u32, num_heads as u32).map_err(WeaverNewError::Backend)?;- let node_cache_write = <B::Kernels as Kernels>::WeaverNodeCacheWriteKernel::new(context, data_type)+ let node_cache_write = <B::Kernels as Kernels>::WeaverNodeCacheWriteKernel::new(context, DATA_TYPE).map_err(WeaverNewError::Backend)?;Ok(Self {qkv_projection,@@ -502,45 +522,114 @@ impl<B: Backend> WeaverBlock<B> {up_projection,down_projection,activation,- residual_add,- last_query_attention,+ ancestor_attention,node_cache_write,attention_scale,model_dim,hidden_dim,num_heads,head_dim,- data_type,+ })+ }++ fn encode_prefix(+ &self,+ input: Allocation<B>,+ residual: &mut Allocation<B>,+ token_count: usize,+ encoder: &mut Encoder<B>,+ ) -> Result<PrefixOutput<B>, B::Error> {+ let PrefixAttentionInputs {+ queries,+ kv_cache,+ } = self.prepare_prefix_attention(&input, residual, token_count, encoder)?;+ let state_type = AttentionStateType::Full {+ length: 0,+ };+ let kv_plane_bytes = size_for_shape(&[token_count, self.model_dim], DATA_TYPE);+ let attention = self.prefix_attention.encode(+ AttentionCoreEncodeArguments {+ queries: &queries,+ keys: &kv_cache,+ values: (&kv_cache, kv_plane_bytes),+ suffix_length: token_count,+ trie: None,+ sinks: None,+ state_type: &state_type,+ },+ encoder,+ )?;+ let hidden = self.encode_post_attention(attention, residual, token_count, encoder)?;+ Ok(PrefixOutput {+ hidden,+ kv_cache,+ })+ }++ fn prepare_prefix_attention(+ &self,+ input: &Allocation<B>,+ residual: &mut Allocation<B>,+ token_count: usize,+ encoder: &mut Encoder<B>,+ ) -> Result<PrefixAttentionInputs<B>, B::Error> {+ let attention_input = self.pre_attention_norm.encode(input, 0, token_count, Some(residual), encoder)?;+ let qkv = self.qkv_projection.encode(attention_input, token_count, encoder)?;+ let mut queries =+ encoder.allocate_scratch(size_for_shape(&[self.num_heads, token_count, self.head_dim], DATA_TYPE))?;+ let kv_plane_bytes = size_for_shape(&[token_count, self.model_dim], DATA_TYPE);+ let mut kv = encoder+ .context()+ .create_allocation(size_for_shape(&[2, token_count, self.model_dim], DATA_TYPE), AllocationType::Global)?;+ let (keys, values) = kv.as_buffer_range_mut().split_at(kv_plane_bytes);+ self.attention_prepare.encode(+ &qkv,+ &mut queries,+ Some(keys),+ Some(values),+ None::<&Allocation<B>>,+ None::<&Allocation<B>>,+ self.num_heads as u32,+ Some(self.num_heads as u32),+ self.head_dim as u32,+ None,+ Some(0),+ token_count as u32,+ encoder,+ );+ Ok(PrefixAttentionInputs {+ queries,+ kv_cache: kv,})}fn encode_step(&self,- current: Allocation<B>,- prefix_qkv: &Allocation<B>,- state_qkv: &mut Allocation<B>,- step: &WeaverStepBatch<'_, B>,+ input: Allocation<B>,+ residual: &mut Allocation<B>,+ prefix_kv: &Allocation<B>,+ node_kv_cache: &mut Allocation<B>,+ node_capacity: u32,+ batch: &WeaverStepBatch<'_, B>,prefix_length: usize,encoder: &mut Encoder<B>,) -> Result<Allocation<B>, B::Error> {- let rows = step.row_count;- let normalized = self.pre_attention_norm.encode(¤t, rows, encoder)?;- let current_qkv = self.qkv_projection.encode(normalized, rows, encoder)?;- let metadata_row_bytes = rows * DataType::U32.size_in_bytes();- let ancestor_counts = (step.node_metadata, metadata_row_bytes);- let node_indices = (step.node_metadata, 2 * metadata_row_bytes);- let mut attention = encoder.allocate_scratch(size_for_shape(&[rows, self.model_dim], self.data_type))?;- let node_capacity = (state_qkv.size() / size_for_shape(&[3, self.model_dim], self.data_type)) as u32;- self.last_query_attention.encode(- prefix_qkv,- &*state_qkv,+ let attention_input = self.pre_attention_norm.encode(&input, 0, batch.node_count, Some(residual), encoder)?;+ let current_qkv = self.qkv_projection.encode(attention_input, batch.node_count, encoder)?;+ let metadata_lane_bytes = batch.node_count * DataType::U32.size_in_bytes();+ let ancestor_counts = (batch.node_metadata, MetadataIdx::AncestorCount as usize * metadata_lane_bytes);+ let tree_slots = (batch.node_metadata, MetadataIdx::TreeSlot as usize * metadata_lane_bytes);+ let mut attention = encoder.allocate_scratch(size_for_shape(&[batch.node_count, self.model_dim], DATA_TYPE))?;+ self.ancestor_attention.encode(+ prefix_kv,+ &*node_kv_cache,¤t_qkv,- step.ancestor_indices,+ batch.ancestor_indices,ancestor_counts,&mut attention,- rows as u32,+ batch.node_count as u32,prefix_length as u32,- step.ancestor_stride as u32,+ batch.ancestor_stride as u32,node_capacity,self.attention_scale,encoder,@@ -549,86 +638,33 @@ impl<B: Backend> WeaverBlock<B> {// serializes it after the attention that reads the ancestor slots.self.node_cache_write.encode(¤t_qkv,- state_qkv,- node_indices,+ node_kv_cache,+ tree_slots,self.model_dim as u32,node_capacity,- (rows * 2 * self.model_dim) as u32,- encoder,- );- attention = self.out_projection.encode(attention, rows, encoder)?;- let mut residual = current;- self.residual_add.encode(&mut residual, &mut attention, (rows * self.model_dim) as u32, encoder);- let normalized = self.pre_mlp_norm.encode(&residual, rows, encoder)?;- let mut mlp = self.up_projection.encode(normalized, rows, encoder)?;- self.activation.encode(- None::<&Allocation<B>>,- &mut mlp,- (rows * self.hidden_dim) as u32,- crate::backends::common::gpu_types::ActivationType::GELUExact,+ (batch.node_count * 2 * self.model_dim) as u32,encoder,);- let mut output = self.down_projection.encode(mlp, rows, encoder)?;- self.residual_add.encode(&mut output, &mut residual, (rows * self.model_dim) as u32, encoder);- Ok(residual)+ self.encode_post_attention(attention, residual, batch.node_count, encoder)}- fn encode_prefix(+ fn encode_post_attention(&self,- hidden: Allocation<B>,- rows: usize,+ attention: Allocation<B>,+ residual: &mut Allocation<B>,+ row_count: usize,encoder: &mut Encoder<B>,- ) -> Result<(Allocation<B>, Allocation<B>), B::Error> {- let normalized = self.pre_attention_norm.encode(&hidden, rows, encoder)?;- let qkv = self.qkv_projection.encode(normalized, rows, encoder)?;- let mut queries =- encoder.allocate_scratch(size_for_shape(&[self.num_heads, rows, self.head_dim], self.data_type))?;- let mut keys = encoder.allocate_scratch(size_for_shape(&[rows, self.model_dim], self.data_type))?;- let mut values = encoder.allocate_scratch(size_for_shape(&[rows, self.model_dim], self.data_type))?;- self.attention_prepare.encode(- &qkv,- &mut queries,- Some(&mut keys),- Some(&mut values),- None::<&Allocation<B>>,- None::<&Allocation<B>>,- self.num_heads as u32,- Some(self.num_heads as u32),- self.head_dim as u32,- None,- Some(0),- rows as u32,- encoder,- );- let state_type = AttentionStateType::Full {- length: 0,- };- let attention = self.prefix_attention.encode(- AttentionCoreEncodeArguments {- queries: &queries,- keys: &keys,- values: &values,- suffix_length: rows,- trie: None,- sinks: None,- state_type: &state_type,- },- encoder,- )?;- let mut attention = self.out_projection.encode(attention, rows, encoder)?;- let mut residual = hidden;- self.residual_add.encode(&mut residual, &mut attention, (rows * self.model_dim) as u32, encoder);- let normalized = self.pre_mlp_norm.encode(&residual, rows, encoder)?;- let mut mlp = self.up_projection.encode(normalized, rows, encoder)?;+ ) -> Result<Allocation<B>, B::Error> {+ let projected_attention = self.out_projection.encode(attention, row_count, encoder)?;+ let mlp_input = self.pre_mlp_norm.encode(&projected_attention, 0, row_count, Some(residual), encoder)?;+ let mut mlp_hidden = self.up_projection.encode(mlp_input, row_count, encoder)?;self.activation.encode(None::<&Allocation<B>>,- &mut mlp,- (rows * self.hidden_dim) as u32,+ &mut mlp_hidden,+ (row_count * self.hidden_dim) as u32,crate::backends::common::gpu_types::ActivationType::GELUExact,encoder,);- let mut output = self.down_projection.encode(mlp, rows, encoder)?;- self.residual_add.encode(&mut output, &mut residual, (rows * self.model_dim) as u32, encoder);- Ok((residual, qkv))+ self.down_projection.encode(mlp_hidden, row_count, encoder)}}diff --git a/crates/backend-uzu/src/engine/dflash_speculator.rs b/crates/backend-uzu/src/engine/dflash_speculator.rsindex 306f05cce..3fe5dbb4d 100644--- a/crates/backend-uzu/src/engine/dflash_speculator.rs+++ b/crates/backend-uzu/src/engine/dflash_speculator.rs@@ -66,9 +66,7 @@ impl<B: Backend> Engine<B> {let weaver = speculator_config.weaver_config.as_ref()- .map(|weaver_config| {- Weaver::new(&*self.context, weaver_config, &speculator_tree.subtree("weaver")?, data_type)- })+ .map(|weaver_config| Weaver::new(&*self.context, weaver_config, &speculator_tree.subtree("weaver")?)).transpose()?;weight_loader.tree().assert_all_tensors_validated()?;diff --git a/crates/backend-uzu/src/speculators/dflash_speculator.rs b/crates/backend-uzu/src/speculators/dflash_speculator.rsindex 0e4b58335..20591d36d 100644--- a/crates/backend-uzu/src/speculators/dflash_speculator.rs+++ b/crates/backend-uzu/src/speculators/dflash_speculator.rs@@ -6,7 +6,7 @@ pub use crate::encodable_block::dflash::DFlashState;use crate::{backends::common::{Allocation, AllocationType, Backend, Context, Encoder, Kernels,- gpu_types::weaver,+ gpu_types::weaver::{FRONTIER_MAX_SLOTS, FRONTIER_NO_WINNER, FrontierIdx, MetadataIdx, TreeIdx},kernel::{WeaverFrontierScatterKernel, WeaverFrontierSelectKernel},},config::speculator::dflash::DFlashSpeculatorConfig,@@ -14,7 +14,7 @@ use crate::{encodable_block::{dflash::DFlashDraft,embedding::{Embedding, EmbeddingError},- weaver::{Weaver, WeaverEncodeError, WeaverNodeState, WeaverPrefix, WeaverStepBatch},+ weaver::{Weaver, WeaverEncodeError, WeaverNodeKvCache, WeaverPrefixKvCache, WeaverStepBatch},},engine::language_model::LanguageModel,};@@ -158,14 +158,14 @@ impl<B: Backend> DFlashSpeculator<B> {let lookahead_count = max_depth.min(block_size.saturating_sub(1));// The frontier holds one slot per (tree slot, child) pair; the select// kernel silently no-ops past its capacity.- if (options.budget + 1) * options.children_per_node > weaver::FRONTIER_MAX_SLOTS || lookahead_count == 0 {+ if (options.budget + 1) * options.children_per_node > FRONTIER_MAX_SLOTS {return Err(DFlashTreeError::InvalidOptions);}let target_hidden = state.target_output_norm().ok_or(DFlashTreeError::MissingTargetHidden)?;- let prefix = weaver.build_prefix(target_hidden, &draft_hidden, 1, lookahead_count, &mut encoder)?;+ let prefix = weaver.build_prefix(target_hidden, &draft_hidden, lookahead_count, &mut encoder)?;drop(draft_hidden);drop(draft_logits);- let mut weaver_state = weaver.create_node_state(options.budget + 1, &self.context)?;+ let mut weaver_state = weaver.create_node_kv_cache(options.budget + 1, &self.context)?;let arguments = TreeEncodingArguments {weaver,prefix: &prefix,@@ -310,12 +310,12 @@ impl<B: Backend> DFlashSpeculator<B> {&self,encoder: &mut Encoder<B>,params: &TreeEncodingArguments<'_, B>,- state: &mut WeaverNodeState<B>,+ state: &mut WeaverNodeKvCache<B>,) -> Result<Allocation<B>, DFlashTreeError<B>> {let context = &*self.context;let slots = params.options.budget + 1;- let fanout = params.options.children_per_node;- let capacity = slots * fanout;+ let children_per_node = params.options.children_per_node;+ let capacity = slots * children_per_node;let width = params.options.frontier_width;let stride = params.max_depth;let pool_size = params.pool_size;@@ -325,16 +325,16 @@ impl<B: Backend> DFlashSpeculator<B> {let scatter =<B::Kernels as Kernels>::WeaverFrontierScatterKernel::new(context).map_err(DFlashTreeError::Backend)?;- let mut tree_values = vec![0u32; weaver::TREE_LANE_COUNT * slots];+ let mut tree_values = vec![0u32; TreeIdx::COUNT * slots];for slot in 0..slots {- tree_values[weaver::TREE_LANE_PARENT * slots + slot] = weaver::FRONTIER_NO_WINNER;+ tree_values[TreeIdx::ParentSlot as usize * slots + slot] = FRONTIER_NO_WINNER;}- tree_values[weaver::TREE_LANE_TOKEN * slots] = params.bonus_token;- tree_values[weaver::TREE_LANE_MASK * slots] = 1;+ tree_values[TreeIdx::TokenId as usize * slots] = params.bonus_token;+ tree_values[TreeIdx::Valid as usize * slots] = 1;let mut tree = encoder.allocate_constant_from_slice(&tree_values).map_err(DFlashTreeError::Backend)?;let mut frontier = encoder- .allocate_constant_from_slice(&vec![0u32; weaver::FRONTIER_LANE_COUNT * capacity])+ .allocate_constant_from_slice(&vec![0u32; FrontierIdx::COUNT * capacity]).map_err(DFlashTreeError::Backend)?;let mut slot_ancestors =encoder.allocate_constant_from_slice(&vec![0u32; slots * stride]).map_err(DFlashTreeError::Backend)?;@@ -346,7 +346,7 @@ impl<B: Backend> DFlashSpeculator<B> {let mut round_token_ids =encoder.allocate_constant_from_slice(&round_token_id_values).map_err(DFlashTreeError::Backend)?;let mut round_metadata = encoder- .allocate_constant_from_slice(&vec![0u32; weaver::METADATA_LANE_COUNT * width])+ .allocate_constant_from_slice(&vec![0u32; MetadataIdx::COUNT * width]).map_err(DFlashTreeError::Backend)?;let mut round_ancestors =encoder.allocate_constant_from_slice(&vec![0u32; width * stride]).map_err(DFlashTreeError::Backend)?;@@ -395,20 +395,20 @@ impl<B: Backend> DFlashSpeculator<B> {(&round_candidate_ids, &round_candidate_scores)};let input = WeaverStepBatch {- row_count: rows,- candidate_count: pool_size,+ node_count: rows,+ candidates_per_node: pool_size,ancestor_stride: stride,- parent_token_ids: &round_token_ids,+ node_token_ids: &round_token_ids,candidate_ids,- candidate_scores,+ candidate_logits: candidate_scores,ancestor_indices: &round_ancestors,node_metadata: &round_metadata,};- let (child_ids, child_logprobs) = params.weaver.encode_step_batch(+ let children = params.weaver.encode_step_batch(params.prefix,&input,state,- fanout,+ children_per_node,params.target_embedding,encoder,)?;@@ -416,17 +416,16 @@ impl<B: Backend> DFlashSpeculator<B> {&tree,&round_metadata,&round_valid,- &child_ids,- &child_logprobs,+ &children.token_ids,+ &children.logprobs,&mut frontier,capacity as u32,slots as u32,rows as u32,- fanout as u32,+ children_per_node as u32,encoder,);- drop(child_ids);- drop(child_logprobs);+ drop(children);slot_start += rows;}Ok(tree)@@ -434,20 +433,16 @@ impl<B: Backend> DFlashSpeculator<B> {}fn tree_from_slots(tree: &[u32]) -> Vec<HostTreeNode> {- assert!(- tree.len().is_multiple_of(weaver::TREE_LANE_COUNT),- "tree array must contain {} equal-length lanes",- weaver::TREE_LANE_COUNT- );- let slots = tree.len() / weaver::TREE_LANE_COUNT;- let lane = |lane: usize, slot: usize| tree[lane * slots + slot];+ assert!(tree.len().is_multiple_of(TreeIdx::COUNT), "tree array must contain {} equal-length lanes", TreeIdx::COUNT);+ let slots = tree.len() / TreeIdx::COUNT;+ let field = |field: TreeIdx, slot: usize| tree[field as usize * slots + slot];let mut slot_to_node = vec![usize::MAX; slots];let mut nodes: Vec<HostTreeNode> = Vec::with_capacity(slots);for slot in 0..slots {- if lane(weaver::TREE_LANE_MASK, slot) == 0 {+ if field(TreeIdx::Valid, slot) == 0 {continue;}- let parent_slot = lane(weaver::TREE_LANE_PARENT, slot) as i32;+ let parent_slot = field(TreeIdx::ParentSlot, slot) as i32;let parent = (parent_slot >= 0).then(|| {let parent = slot_to_node[parent_slot as usize];assert_ne!(parent, usize::MAX, "tree slot {slot} names padding slot {parent_slot} as its parent");@@ -459,10 +454,10 @@ fn tree_from_slots(tree: &[u32]) -> Vec<HostTreeNode> {nodes[parent].children.push(index);}nodes.push(HostTreeNode {- token: lane(weaver::TREE_LANE_TOKEN, slot),+ token: field(TreeIdx::TokenId, slot),parent,- depth: lane(weaver::TREE_LANE_DEPTH, slot) as usize,- logprob: f32::from_bits(lane(weaver::TREE_LANE_LOGPROB, slot)),+ depth: field(TreeIdx::Depth, slot) as usize,+ logprob: f32::from_bits(field(TreeIdx::EdgeLogprobBits, slot)),children: Vec::new(),});}@@ -471,7 +466,7 @@ fn tree_from_slots(tree: &[u32]) -> Vec<HostTreeNode> {struct TreeEncodingArguments<'a, B: Backend> {weaver: &'a Weaver<B>,- prefix: &'a WeaverPrefix<B>,+ prefix: &'a WeaverPrefixKvCache<B>,target_embedding: &'a Embedding<B>,pool_ids: &'a Allocation<B>,pool_scores: &'a Allocation<B>,diff --git a/crates/backend-uzu/tests/unit/backends/common/allocator/allocator.rs b/crates/backend-uzu/tests/unit/backends/common/allocator/allocator.rsindex e7e193182..3a5523aa4 100644--- a/crates/backend-uzu/tests/unit/backends/common/allocator/allocator.rs+++ b/crates/backend-uzu/tests/unit/backends/common/allocator/allocator.rs@@ -10,7 +10,10 @@ use proc_macros::uzu_test;use serde::{Deserialize, Serialize};use crate::backends::{- common::{Allocation, AllocationPool, AllocationType, Allocator, Backend, Context},+ common::{+ Allocation, AllocationPool, AllocationType, Allocator, AsBufferRangeMut, AsBufferRangeRef, Backend,+ BufferArgMut, Context,+ },metal::Metal,};@@ -127,3 +130,18 @@ fn bench_allocator_generation_trace() {eprintln!("Cpu time: {} ms", elapsed.as_millis());eprintln!("--------------------");}++#[uzu_test]+#[ignore]+fn allocation_split_at_mut_returns_disjoint_buffer_arguments() {+ let context = <<Metal as Backend>::Context as Context>::new().unwrap();+ let mut allocation = context.create_allocation(64, AllocationType::Global).unwrap();+ let allocation_range = allocation.as_buffer_range_ref().range();++ let (head, tail) = allocation.as_buffer_range_mut().split_at(24);+ let (_, head_offset, head_length) = BufferArgMut::<'_, Metal>::into_parts(head);+ let (_, tail_offset, tail_length) = BufferArgMut::<'_, Metal>::into_parts(tail);++ assert_eq!((head_offset, head_length), (allocation_range.start, 24));+ assert_eq!((tail_offset, tail_length), (allocation_range.start + 24, 40));+}diff --git a/crates/backend-uzu/tests/unit/backends/common/kernel/attention/attention_last_query_test.rs b/crates/backend-uzu/tests/unit/backends/common/kernel/attention/ancestor_attention_test.rssimilarity index 77%rename from crates/backend-uzu/tests/unit/backends/common/kernel/attention/attention_last_query_test.rsrename to crates/backend-uzu/tests/unit/backends/common/kernel/attention/ancestor_attention_test.rsindex 64c1228ed..e6fcc7656 100644--- a/crates/backend-uzu/tests/unit/backends/common/kernel/attention/attention_last_query_test.rs+++ b/crates/backend-uzu/tests/unit/backends/common/kernel/attention/ancestor_attention_test.rs@@ -7,7 +7,7 @@ use proc_macros::uzu_test;use crate::{backends::{- common::{Allocation, Backend, Encoder, Kernels, kernel::AttentionLastQueryKernel},+ common::{Allocation, Backend, Encoder, Kernels, kernel::AncestorAttentionKernel},cpu::Cpu,metal::Metal,},@@ -22,9 +22,9 @@ const NUM_HEADS: u32 = 16;struct Runner<B: Backend> {context: Arc<B::Context>,- kernel: <B::Kernels as Kernels>::AttentionLastQueryKernel,- prefix_qkv: Allocation<B>,- node_qkv: Allocation<B>,+ kernel: <B::Kernels as Kernels>::AncestorAttentionKernel,+ prefix_kv: Allocation<B>,+ node_kv: Allocation<B>,current_qkv: Allocation<B>,ancestor_indices: Allocation<B>,ancestor_counts: Allocation<B>,@@ -43,7 +43,9 @@ impl<B: Backend> Runner<B> {nodes: usize,) -> Self {assert!(rows < nodes && ancestor_stride > 0);- let packed_width = 3 * NUM_HEADS as usize * HEAD_DIM;+ let model_dim = NUM_HEADS as usize * HEAD_DIM;+ let qkv_width = 3 * model_dim;+ let kv_width = 2 * model_dim;let values = |length, offset| {(0..length).map(|index| bf16::from_f32((((index + offset) * 17 % 251) as f32 - 125.0) / 128.0))@@ -61,12 +63,12 @@ impl<B: Backend> Runner<B> {let context = create_context::<B>();let kernel =- <B::Kernels as Kernels>::AttentionLastQueryKernel::new(context.as_ref(), HEAD_DIM as u32, NUM_HEADS)+ <B::Kernels as Kernels>::AncestorAttentionKernel::new(context.as_ref(), HEAD_DIM as u32, NUM_HEADS).unwrap();Self {- prefix_qkv: alloc_allocation_with_data::<B, bf16>(&context, &values(prefix_length * packed_width, 0)),- node_qkv: alloc_allocation_with_data::<B, bf16>(&context, &values(nodes * packed_width, 11)),- current_qkv: alloc_allocation_with_data::<B, bf16>(&context, &values(rows * packed_width, 29)),+ prefix_kv: alloc_allocation_with_data::<B, bf16>(&context, &values(prefix_length * kv_width, 0)),+ node_kv: alloc_allocation_with_data::<B, bf16>(&context, &values(nodes * kv_width, 11)),+ current_qkv: alloc_allocation_with_data::<B, bf16>(&context, &values(rows * qkv_width, 29)),ancestor_indices: alloc_allocation_with_data::<B, u32>(&context, &ancestor_indices),ancestor_counts: alloc_allocation_with_data::<B, u32>(&context, &ancestor_counts),output: alloc_allocation::<B, bf16>(&context, rows * NUM_HEADS as usize * HEAD_DIM),@@ -86,8 +88,8 @@ impl<B: Backend> Runner<B> {let mut encoder = Encoder::new(self.context.as_ref()).unwrap();for _ in 0..repetitions {self.kernel.encode(- &self.prefix_qkv,- &self.node_qkv,+ &self.prefix_kv,+ &self.node_kv,&self.current_qkv,&self.ancestor_indices,&self.ancestor_counts,@@ -105,25 +107,25 @@ impl<B: Backend> Runner<B> {}#[uzu_test]-fn attention_last_query_matches_cpu() {+fn ancestor_attention_matches_cpu() {let mut cpu = Runner::<Cpu>::new(4, 5, 3, 16);- let initial_nodes = allocation_to_vec::<Cpu, bf16>(&cpu.node_qkv);+ let initial_nodes = allocation_to_vec::<Cpu, bf16>(&cpu.node_kv);cpu.encode(1);let expected_output = allocation_to_vec::<Cpu, bf16>(&cpu.output);- assert_eq!(allocation_to_vec::<Cpu, bf16>(&cpu.node_qkv), initial_nodes);+ assert_eq!(allocation_to_vec::<Cpu, bf16>(&cpu.node_kv), initial_nodes);let mut metal = Runner::<Metal>::new(4, 5, 3, 16);metal.encode(1);let actual_output = allocation_to_vec::<Metal, bf16>(&metal.output);- assert_eq_float(&expected_output, &actual_output, 0.02, "AttentionLastQuery output");- assert_eq!(allocation_to_vec::<Metal, bf16>(&metal.node_qkv), initial_nodes);+ assert_eq_float(&expected_output, &actual_output, 0.02, "AncestorAttention output");+ assert_eq!(allocation_to_vec::<Metal, bf16>(&metal.node_kv), initial_nodes);}#[uzu_test]#[ignore = "benchmark"]-fn benchmark_attention_last_query() {+fn benchmark_ancestor_attention() {const BATCH: u32 = 32;const SAMPLES: usize = 50;@@ -135,5 +137,5 @@ fn benchmark_attention_last_query() {}let mut samples = (0..SAMPLES).map(|_| run()).collect::<Vec<_>>();samples.sort_unstable();- eprintln!("attention_last_query gpu={:?}", samples[SAMPLES / 2]);+ eprintln!("ancestor_attention gpu={:?}", samples[SAMPLES / 2]);}diff --git a/crates/backend-uzu/tests/unit/backends/common/kernel/attention/mod.rs b/crates/backend-uzu/tests/unit/backends/common/kernel/attention/mod.rsindex 0b723476d..2d613a765 100644--- a/crates/backend-uzu/tests/unit/backends/common/kernel/attention/mod.rs+++ b/crates/backend-uzu/tests/unit/backends/common/kernel/attention/mod.rs@@ -1,5 +1,5 @@+mod ancestor_attention_test;mod attention_fallback_test;-mod attention_last_query_test;mod attention_single_pass_test;mod attention_two_pass_test;mod sigmoid_gate_test;diff --git a/crates/backend-uzu/tests/unit/backends/common/kernel/weaver/weaver_frontier_test.rs b/crates/backend-uzu/tests/unit/backends/common/kernel/weaver/weaver_frontier_test.rsindex a03fdeaaf..5f27d522b 100644--- a/crates/backend-uzu/tests/unit/backends/common/kernel/weaver/weaver_frontier_test.rs+++ b/crates/backend-uzu/tests/unit/backends/common/kernel/weaver/weaver_frontier_test.rs@@ -5,7 +5,7 @@ use crate::{backends::{common::{Backend, Encoder, Kernels,- gpu_types::weaver,+ gpu_types::weaver::{FrontierIdx, MetadataIdx, TreeIdx},kernel::{WeaverFrontierScatterKernel, WeaverFrontierSelectKernel},},cpu::Cpu,@@ -15,7 +15,7 @@ use crate::{fn select<B: Backend>() -> Vec<u32> {let context = create_context::<B>();- let mut frontier = vec![0; weaver::FRONTIER_LANE_COUNT * 8];+ let mut frontier = vec![0; FrontierIdx::COUNT * 8];for (slot, (token, parent, depth, cum, key, active)) in [(9, 1, 1, 0x3f00_0000, 100, 1),(8, 0, 2, 0x3f00_0001, 100, 1),@@ -34,7 +34,7 @@ fn select<B: Backend>() -> Vec<u32> {}}let mut frontier = alloc_allocation_with_data::<B, u32>(&context, &frontier);- let mut tree = alloc_allocation_with_data::<B, u32>(&context, &[55; weaver::TREE_LANE_COUNT * 7]);+ let mut tree = alloc_allocation_with_data::<B, u32>(&context, &[55; TreeIdx::COUNT * 7]);let mut slot_ancestors = alloc_allocation_with_data::<B, u32>(&context, &(0u32..7 * 3).collect::<Vec<_>>());let mut token = alloc_allocation_with_data::<B, u32>(&context, &[66; 4]);let mut metadata = alloc_allocation_with_data::<B, u32>(&context, &[77; 3 * 4]);@@ -80,19 +80,18 @@ fn select<B: Backend>() -> Vec<u32> {fn scatter<B: Backend>() -> Vec<u32> {let context = create_context::<B>();- let mut tree = vec![0; weaver::TREE_LANE_COUNT * 4];- tree[weaver::TREE_LANE_CUM * 4..(weaver::TREE_LANE_CUM + 1) * 4]+ let mut tree = vec![0; TreeIdx::COUNT * 4];+ tree[TreeIdx::PathLogprobBits as usize * 4..(TreeIdx::PathLogprobBits as usize + 1) * 4].copy_from_slice(&[0.5, -1.0, 2.0, 4.0].map(f32::to_bits));- tree[weaver::TREE_LANE_DEPTH * 4..(weaver::TREE_LANE_DEPTH + 1) * 4].copy_from_slice(&[0, 2, 4, 6]);+ tree[TreeIdx::Depth as usize * 4..(TreeIdx::Depth as usize + 1) * 4].copy_from_slice(&[0, 2, 4, 6]);let tree = alloc_allocation_with_data::<B, u32>(&context, &tree);- let mut metadata = vec![0; weaver::METADATA_LANE_COUNT * 3];- metadata[weaver::METADATA_LANE_NODE_INDEX * 3..(weaver::METADATA_LANE_NODE_INDEX + 1) * 3]- .copy_from_slice(&[1, 3, 0]);+ let mut metadata = vec![0; MetadataIdx::COUNT * 3];+ metadata[MetadataIdx::TreeSlot as usize * 3..(MetadataIdx::TreeSlot as usize + 1) * 3].copy_from_slice(&[1, 3, 0]);let metadata = alloc_allocation_with_data::<B, u32>(&context, &metadata);let valid = alloc_allocation_with_data::<B, u32>(&context, &[1, 0, 1]);let ids = alloc_allocation_with_data::<B, u32>(&context, &(10..19).collect::<Vec<_>>());let scores = alloc_allocation_with_data::<B, f32>(&context, &[-0.1, -0.2, -0.3, 8.0, 8.0, 8.0, 0.1, 0.2, 0.3]);- let mut frontier = alloc_allocation_with_data::<B, u32>(&context, &[42; weaver::FRONTIER_LANE_COUNT * 16]);+ let mut frontier = alloc_allocation_with_data::<B, u32>(&context, &[42; FrontierIdx::COUNT * 16]);let kernel = <B::Kernels as Kernels>::WeaverFrontierScatterKernel::new(&context).unwrap();let mut encoder = Encoder::new(context.as_ref()).unwrap();kernel.encode(&tree, &metadata, &valid, &ids, &scores, &mut frontier, 16, 4, 3, 3, &mut encoder);diff --git a/crates/backend-uzu/tests/unit/backends/common/kernel/weaver/weaver_node_cache_write_test.rs b/crates/backend-uzu/tests/unit/backends/common/kernel/weaver/weaver_node_cache_write_test.rsindex ba6d27ab2..50fdc7078 100644--- a/crates/backend-uzu/tests/unit/backends/common/kernel/weaver/weaver_node_cache_write_test.rs+++ b/crates/backend-uzu/tests/unit/backends/common/kernel/weaver/weaver_node_cache_write_test.rs@@ -21,7 +21,7 @@ struct Runner<B: Backend> {context: std::sync::Arc<B::Context>,kernel: <B::Kernels as Kernels>::WeaverNodeCacheWriteKernel,current_qkv: Allocation<B>,- node_qkv: Allocation<B>,+ node_kv: Allocation<B>,node_indices: Allocation<B>,}@@ -38,13 +38,14 @@ const NODE_INDICES: [u32; ROWS] = [7, 1, 8, 4];impl<B: Backend> Runner<B> {fn new() -> Self {- let packed_width = 3 * MODEL_DIM;+ let qkv_width = 3 * MODEL_DIM;+ let kv_width = 2 * MODEL_DIM;let context = create_context::<B>();let kernel =<B::Kernels as Kernels>::WeaverNodeCacheWriteKernel::new(context.as_ref(), DataType::BF16).unwrap();Self {- current_qkv: alloc_allocation_with_data::<B, bf16>(&context, &values(ROWS * packed_width, 29)),- node_qkv: alloc_allocation_with_data::<B, bf16>(&context, &values(NODES * packed_width, 11)),+ current_qkv: alloc_allocation_with_data::<B, bf16>(&context, &values(ROWS * qkv_width, 29)),+ node_kv: alloc_allocation_with_data::<B, bf16>(&context, &values(NODES * kv_width, 11)),node_indices: alloc_allocation_with_data::<B, u32>(&context, &NODE_INDICES),context,kernel,@@ -55,7 +56,7 @@ impl<B: Backend> Runner<B> {let mut encoder = Encoder::new(self.context.as_ref()).unwrap();self.kernel.encode(&self.current_qkv,- &mut self.node_qkv,+ &mut self.node_kv,&self.node_indices,MODEL_DIM as u32,NODES as u32,@@ -63,32 +64,32 @@ impl<B: Backend> Runner<B> {&mut encoder,);encoder.end_encoding().submit().wait_until_completed().unwrap();- allocation_to_vec::<B, bf16>(&self.node_qkv)+ allocation_to_vec::<B, bf16>(&self.node_kv)}}#[uzu_test]fn weaver_node_cache_write_matches_cpu() {- let packed_width = 3 * MODEL_DIM;+ let qkv_width = 3 * MODEL_DIM;let mut cpu = Runner::<Cpu>::new();- let initial_nodes = allocation_to_vec::<Cpu, bf16>(&cpu.node_qkv);+ let initial_nodes = allocation_to_vec::<Cpu, bf16>(&cpu.node_kv);let current_qkv = allocation_to_vec::<Cpu, bf16>(&cpu.current_qkv);let expected = cpu.run();- // Every written slot takes the K/V halves of its row; the query third and- // every untouched slot keep their prior contents.- for node in 0..NODES {- let base = node * packed_width;- match NODE_INDICES.iter().position(|&index| index as usize == node) {- Some(row) => {- assert_eq!(expected[base..base + MODEL_DIM], initial_nodes[base..base + MODEL_DIM]);- assert_eq!(- expected[base + MODEL_DIM..base + packed_width],- current_qkv[row * packed_width + MODEL_DIM..(row + 1) * packed_width],- );- },- None => assert_eq!(expected[base..base + packed_width], initial_nodes[base..base + packed_width]),+ for component in 0..2 {+ for node in 0..NODES {+ let destination = component * NODES * MODEL_DIM + node * MODEL_DIM;+ match NODE_INDICES.iter().position(|&index| index as usize == node) {+ Some(row) => {+ let source = row * qkv_width + (component + 1) * MODEL_DIM;+ assert_eq!(expected[destination..destination + MODEL_DIM], current_qkv[source..source + MODEL_DIM],);+ },+ None => assert_eq!(+ expected[destination..destination + MODEL_DIM],+ initial_nodes[destination..destination + MODEL_DIM],+ ),+ }}}diff --git a/crates/backend-uzu/tests/unit/backends/common/kernel/weaver/weaver_top_children_test.rs b/crates/backend-uzu/tests/unit/backends/common/kernel/weaver/weaver_top_children_test.rsindex 047ddceba..773368b25 100644--- a/crates/backend-uzu/tests/unit/backends/common/kernel/weaver/weaver_top_children_test.rs+++ b/crates/backend-uzu/tests/unit/backends/common/kernel/weaver/weaver_top_children_test.rs@@ -15,34 +15,34 @@ const CHILDREN: usize = 8;fn top_children<B: Backend>(residual: &[bf16],- scores: &[f32],+ candidate_logits: &[f32],ids: &[u32],) -> (Vec<u32>, Vec<f32>) {let rows = residual.len() / CANDIDATES;assert_eq!(residual.len(), rows * CANDIDATES);- assert_eq!(scores.len(), rows * CANDIDATES);+ assert_eq!(candidate_logits.len(), rows * CANDIDATES);assert_eq!(ids.len(), rows * CANDIDATES);let context = create_context::<B>();let residual = alloc_allocation_with_data::<B, bf16>(&context, residual);- let scores = alloc_allocation_with_data::<B, f32>(&context, scores);+ let candidate_logits = alloc_allocation_with_data::<B, f32>(&context, candidate_logits);let ids = alloc_allocation_with_data::<B, u32>(&context, ids);- let mut output_ids = alloc_allocation::<B, u32>(&context, rows * CHILDREN);- let mut output_logprobs = alloc_allocation::<B, f32>(&context, rows * CHILDREN);+ let mut output_token_ids = alloc_allocation::<B, u32>(&context, rows * CHILDREN);+ let mut output_model_logprobs = alloc_allocation::<B, f32>(&context, rows * CHILDREN);let kernel = <B::Kernels as Kernels>::WeaverTopChildrenKernel::new(&context).unwrap();let mut encoder = Encoder::new(context.as_ref()).unwrap();kernel.encode(&residual,- &scores,+ &candidate_logits,&ids,- &mut output_ids,- &mut output_logprobs,+ &mut output_token_ids,+ &mut output_model_logprobs,rows as u32,CANDIDATES as u32,CHILDREN as u32,&mut encoder,);encoder.end_encoding().submit().wait_until_completed().unwrap();- (allocation_to_vec(&output_ids), allocation_to_vec(&output_logprobs))+ (allocation_to_vec(&output_token_ids), allocation_to_vec(&output_model_logprobs))}#[uzu_test]@@ -51,15 +51,15 @@ fn weaver_top_children_matches_cpu() {let residual = (0..ROWS * CANDIDATES).map(|index| bf16::from_f32(((index as f32 * 0.017).cos() * 4.0).round() * 0.125)).collect::<Vec<_>>();- let scores =+ let candidate_logits =(0..ROWS * CANDIDATES).map(|index| ((index as f32 * 0.011).sin() * 3.0).round() * 0.125).collect::<Vec<_>>();let ids = (0..ROWS).flat_map(|row| (0..CANDIDATES).rev().map(move |index| 70_000 + (row * CANDIDATES + index) as u32)).collect::<Vec<_>>();for_each_non_cpu_backend!(|B| {- let expected = top_children::<Cpu>(&residual, &scores, &ids);- let actual = top_children::<B>(&residual, &scores, &ids);+ let expected = top_children::<Cpu>(&residual, &candidate_logits, &ids);+ let actual = top_children::<B>(&residual, &candidate_logits, &ids);assert_eq!(actual.0, expected.0);for (actual, expected) in actual.1.iter().zip(expected.1) {assert!((actual - expected).abs() < 1e-5);