A high-performance inference engine for AI models
Research tree construction (#736)
uuuvn3deb0224b7de
diff --git a/crates/backend-uzu/src/backends/common/gpu_types/weaver.rs b/crates/backend-uzu/src/backends/common/gpu_types/weaver.rsindex bada0a1ec..5e568b863 100644--- a/crates/backend-uzu/src/backends/common/gpu_types/weaver.rs+++ b/crates/backend-uzu/src/backends/common/gpu_types/weaver.rs@@ -51,3 +51,4 @@ pub const FRONTIER_SELECT_SIMDGROUPS: u32 = 8;pub const FRONTIER_ENTRIES_PER_THREAD: u32 = 8;pub const FRONTIER_MAX_SLOTS: u32 = 2048;pub const FRONTIER_MAX_WIDTH: u32 = 32;+pub const PADDING_DEPTH: u32 = 0;diff --git a/crates/backend-uzu/src/backends/cpu/kernel/attention/ancestor_attention.rs b/crates/backend-uzu/src/backends/cpu/kernel/attention/ancestor_attention.rsindex 586c9adda..f4d33f19f 100644--- a/crates/backend-uzu/src/backends/cpu/kernel/attention/ancestor_attention.rs+++ b/crates/backend-uzu/src/backends/cpu/kernel/attention/ancestor_attention.rs@@ -42,7 +42,8 @@ fn ancestor_attention<const HEAD_DIM: u32>(let current_row = current_qkv.add(row * qkv_width);let depth = *node_metadata.add(MetadataIdx::Depth as usize * row_count + row);- let position = depth.min(max_depth - 1) as usize + 1;+ assert!(depth < max_depth, "node metadata depth must be rope-safe");+ let position = depth as usize + 1;let rotate = |component: usize| {let mut rotated = vec![bf16::ZERO; model_dim];for head in 0..num_heads {diff --git a/crates/backend-uzu/src/backends/cpu/kernel/weaver/weaver_frontier_select.rs b/crates/backend-uzu/src/backends/cpu/kernel/weaver/weaver_frontier_select.rsindex 365ce8620..08a616961 100644--- a/crates/backend-uzu/src/backends/cpu/kernel/weaver/weaver_frontier_select.rs+++ b/crates/backend-uzu/src/backends/cpu/kernel/weaver/weaver_frontier_select.rs@@ -1,7 +1,7 @@use proc_macros::kernel;use crate::backends::common::gpu_types::weaver::{- FRONTIER_MAX_SLOTS, FRONTIER_MAX_WIDTH, FRONTIER_NO_WINNER, FrontierIdx, MetadataIdx, TreeIdx,+ FRONTIER_MAX_SLOTS, FRONTIER_MAX_WIDTH, FRONTIER_NO_WINNER, FrontierIdx, MetadataIdx, PADDING_DEPTH, TreeIdx,};#[kernel(WeaverFrontierSelect)]@@ -66,8 +66,11 @@ pub fn weaver_frontier_select(if frontier[FrontierIdx::Active as usize * frontier_capacity + slot] == 0 {continue;}+ let score_key = frontier[FrontierIdx::PathScoreKey as usize * frontier_capacity + slot];+ let expandable =+ u32::from(frontier[FrontierIdx::Depth as usize * frontier_capacity + slot] < lookahead_count);let next = (- frontier[FrontierIdx::PathScoreKey as usize * frontier_capacity + slot],+ (expandable << 31) | (score_key >> 1),frontier[FrontierIdx::ParentSlot as usize * frontier_capacity + slot],frontier[FrontierIdx::TokenId as usize * frontier_capacity + slot],);@@ -121,16 +124,21 @@ pub fn weaver_frontier_select(}node_token_ids[node] = token;- node_metadata[MetadataIdx::Depth as usize * node_count + node] = depth.min(max_depth - 1);+ let expandable = depth < lookahead_count;+ node_metadata[MetadataIdx::Depth as usize * node_count + node] = if expandable {+ depth+ } else {+ PADDING_DEPTH+ };node_metadata[MetadataIdx::AncestorCount as usize * node_count + node] = depth;node_metadata[MetadataIdx::TreeSlot as usize * node_count + node] = tree_slot as u32;- node_valid[node] = u32::from(real && depth < lookahead_count);+ node_valid[node] = u32::from(real && expandable);- // Every node expands the candidate pool for its own depth.- let candidate_depth = (depth as usize).min(candidate_depth_count - 1);- let source = candidate_depth * candidates_per_depth..(candidate_depth + 1) * candidates_per_depth;- let destination = node * candidates_per_depth..(node + 1) * candidates_per_depth;- node_candidate_ids[destination.clone()].copy_from_slice(&candidate_pool_ids[source.clone()]);- node_candidate_logits[destination].copy_from_slice(&candidate_pool_logits[source]);+ if (depth as usize) < candidate_depth_count {+ let source = depth as usize * candidates_per_depth..(depth as usize + 1) * candidates_per_depth;+ let destination = node * candidates_per_depth..(node + 1) * candidates_per_depth;+ node_candidate_ids[destination.clone()].copy_from_slice(&candidate_pool_ids[source.clone()]);+ node_candidate_logits[destination].copy_from_slice(&candidate_pool_logits[source]);+ }}}diff --git a/crates/backend-uzu/src/backends/metal/kernel/attention/ancestor_attention.metal b/crates/backend-uzu/src/backends/metal/kernel/attention/ancestor_attention.metalindex a7079dac6..2900c14b7 100644--- a/crates/backend-uzu/src/backends/metal/kernel/attention/ancestor_attention.metal+++ b/crates/backend-uzu/src/backends/metal/kernel/attention/ancestor_attention.metal@@ -101,7 +101,7 @@ PUBLIC KERNEL(AncestorAttention)(// ask it directly instead of reading the buffer a second time.static_assert(vectors_per_head == METAL_SIMD_SIZE, "one lane per head vector");const uint depth = node_metadata[uint(MetadataIdx::Depth) * rows + row];- const uint rope_position = min(depth, max_depth - 1u) + 1u;+ const uint rope_position = depth + 1u;const device float4* cosine_row = (const device float4*)(cosines + rope_position * HEAD_DIM);const device float4* sine_row = (const device float4*)(sines + rope_position * HEAD_DIM);const float4 cosine = cosine_row[lane];diff --git a/crates/backend-uzu/src/backends/metal/kernel/generated/weaver.h b/crates/backend-uzu/src/backends/metal/kernel/generated/weaver.hindex fd799486f..173fdfaf4 100644--- a/crates/backend-uzu/src/backends/metal/kernel/generated/weaver.h+++ b/crates/backend-uzu/src/backends/metal/kernel/generated/weaver.h@@ -47,4 +47,6 @@ static constant constexpr uint32_t FRONTIER_ENTRIES_PER_THREAD = 8;static constant constexpr uint32_t FRONTIER_MAX_SLOTS = 2048;static constant constexpr uint32_t FRONTIER_MAX_WIDTH = 32;++static constant constexpr uint32_t PADDING_DEPTH = 0;} // namespace uzu::weaverdiff --git a/crates/backend-uzu/src/backends/metal/kernel/weaver/weaver_frontier_select.metal b/crates/backend-uzu/src/backends/metal/kernel/weaver/weaver_frontier_select.metalindex 3f4044580..591601ff2 100644--- a/crates/backend-uzu/src/backends/metal/kernel/weaver/weaver_frontier_select.metal+++ b/crates/backend-uzu/src/backends/metal/kernel/weaver/weaver_frontier_select.metal@@ -53,11 +53,14 @@ PUBLIC KERNEL(WeaverFrontierSelect)(for (uint entry = 0; entry < FRONTIER_ENTRIES_PER_THREAD; ++entry) {const uint slot = lid + entry * FRONTIER_SELECT_THREADS;if (entry_active[entry]) {+ const uint depth = frontier[uint(FrontierIdx::Depth) * frontier_capacity + slot];const uint key = frontier[uint(FrontierIdx::PathScoreKey) * frontier_capacity + slot];const uint parent = frontier[uint(FrontierIdx::ParentSlot) * frontier_capacity + slot];const uint token = frontier[uint(FrontierIdx::TokenId) * frontier_capacity + slot];- if (key > local.x || (key == local.x && (parent < local.y || (parent == local.y && token < local.z)))) {- local = uint4(key, parent, token, slot);+ const uint packed_key = ((depth < lookahead_count ? 1u : 0u) << 31) | (key >> 1);+ if (packed_key > local.x ||+ (packed_key == local.x && (parent < local.y || (parent == local.y && token < local.z)))) {+ local = uint4(packed_key, parent, token, slot);}}}@@ -127,17 +130,20 @@ PUBLIC KERNEL(WeaverFrontierSelect)(}node_token_ids[node] = token;- node_metadata[uint(MetadataIdx::Depth) * node_count + node] = min(depth, max_depth - 1u);+ node_metadata[uint(MetadataIdx::Depth) * node_count + node] = depth < lookahead_count ? depth : PADDING_DEPTH;node_metadata[uint(MetadataIdx::AncestorCount) * node_count + node] = depth;node_metadata[uint(MetadataIdx::TreeSlot) * node_count + node] = tree_slot;node_valid[node] = real && depth < lookahead_count ? 1u : 0u;- node_candidate_depth[node] = min(depth, candidate_depth_count - 1u);+ node_candidate_depth[node] = depth;}threadgroup_barrier(mem_flags::mem_threadgroup);for (uint node = 0; node < node_count; ++node) {+ if (node_candidate_depth[node] >= candidate_depth_count) {+ continue;+ }const uint source = node_candidate_depth[node] * candidates_per_depth;const uint destination = node * candidates_per_depth;for (uint candidate = lid; candidate < candidates_per_depth; candidate += FRONTIER_SELECT_THREADS) {diff --git a/crates/backend-uzu/src/encodable_block/dflash.rs b/crates/backend-uzu/src/encodable_block/dflash.rsindex 1eba2a76e..4f73f9249 100644--- a/crates/backend-uzu/src/encodable_block/dflash.rs+++ b/crates/backend-uzu/src/encodable_block/dflash.rs@@ -93,13 +93,8 @@ impl<B: Backend> DFlash<B> {) -> Result<Self, DFlashNewError<B>> {let mask_token_id = config.mask_token_id as u32;assert!(config.block_size <= ATTENTION_SUFFIX_CAPACITY, "DFlash block_size exceeds attention suffix capacity");- assert_eq!(- config.num_target_layers,- config.target_layer_ids.len() as u32,- "num_target_layers must match target_layer_ids length"- );let target_feature_projection = <dyn Linear<B>>::new(- config.model_dim * config.num_target_layers,+ config.model_dim * config.target_layer_ids.len() as u32,[config.model_dim],false,context,@@ -171,7 +166,7 @@ impl<B: Backend> DFlash<B> {max_context_length: *config.rope_config.max_sequence_length(),block_size: config.block_size,mask_token_id,- target_feature_input_dim: config.model_dim * config.num_target_layers,+ target_feature_input_dim: config.model_dim * config.target_layer_ids.len() as u32,data_type,})}diff --git a/crates/backend-uzu/src/encodable_block/weaver.rs b/crates/backend-uzu/src/encodable_block/weaver.rsindex e347f15b9..bda6fcc37 100644--- a/crates/backend-uzu/src/encodable_block/weaver.rs+++ b/crates/backend-uzu/src/encodable_block/weaver.rs@@ -26,52 +26,83 @@ use crate::{parameters::ParameterTree,};-pub(crate) const DATA_TYPE: DataType = DataType::BF16;-pub(crate) const ROPE_DATA_TYPE: DataType = DataType::F32;+pub const DATA_TYPE: DataType = DataType::BF16;+pub const ROPE_DATA_TYPE: DataType = DataType::F32;pub struct WeaverTreeShape {- pub budget: u32,- pub depth: u32,+ pub tree_budget: u32,+ pub max_depth: u32,+ pub dflash_depth: u32,+ pub rounds: u32,pub expand_per_round: u32,pub expand_width: u32,}+impl WeaverTreeShape {+ fn slot_count(&self) -> u32 {+ 1 + self.rounds.saturating_sub(1) * self.expand_per_round+ }+}+pub struct EncodedWeaverTree<B: Backend> {packed_tree: Allocation<B>,+ frontier: Allocation<B>,}pub struct ProposalNode {pub token_id: u32,pub depth: u32,+ pub logprob: f32,pub child_indices: Vec<usize>,}impl<B: Backend> EncodedWeaverTree<B> {pub fn read_nodes(self) -> Vec<ProposalNode> {let packed_tree = &self.packed_tree.copyout::<u32>();+ let frontier = &self.frontier.copyout::<u32>();let tree_slot_count = packed_tree.len() / TreeIdx::COUNT;- let field = |field: TreeIdx, slot: usize| packed_tree[field as usize * tree_slot_count + slot];- let mut slot_to_node_index = vec![usize::MAX; tree_slot_count];- let mut nodes: Vec<ProposalNode> = Vec::with_capacity(tree_slot_count);+ let frontier_capacity = frontier.len() / FrontierIdx::COUNT;+ let tree_field = |field: TreeIdx, slot: usize| packed_tree[field as usize * tree_slot_count + slot];+ let frontier_field = |field: FrontierIdx, slot: usize| frontier[field as usize * frontier_capacity + slot];++ let mut slot_to_index = vec![usize::MAX; tree_slot_count];+ let mut nodes: Vec<ProposalNode> = Vec::new();for slot in 0..tree_slot_count {- if field(TreeIdx::Valid, slot) == 0 {+ if tree_field(TreeIdx::Valid, slot) == 0 {continue;}- let parent_slot = field(TreeIdx::ParentSlot, slot) as i32;+ let parent_slot = tree_field(TreeIdx::ParentSlot, slot) as i32;let parent = (parent_slot >= 0).then(|| {- let parent = slot_to_node_index[parent_slot as usize];+ let parent = slot_to_index[parent_slot as usize];assert_ne!(parent, usize::MAX, "tree slot {slot} names padding slot {parent_slot} as its parent");parent});let index = nodes.len();- slot_to_node_index[slot] = index;+ slot_to_index[slot] = index;if let Some(parent) = parent {nodes[parent].child_indices.push(index);}nodes.push(ProposalNode {- token_id: field(TreeIdx::TokenId, slot),- depth: field(TreeIdx::Depth, slot),+ token_id: tree_field(TreeIdx::TokenId, slot),+ depth: tree_field(TreeIdx::Depth, slot),+ logprob: f32::from_bits(tree_field(TreeIdx::EdgeLogprobBits, slot)),+ child_indices: Vec::new(),+ });+ }+ for slot in 0..frontier_capacity {+ if frontier_field(FrontierIdx::Active, slot) == 0 {+ continue;+ }+ let parent_slot = frontier_field(FrontierIdx::ParentSlot, slot) as usize;+ let parent = slot_to_index[parent_slot];+ assert_ne!(parent, usize::MAX, "frontier slot {slot} names padding slot {parent_slot} as its parent");+ let index = nodes.len();+ nodes[parent].child_indices.push(index);+ nodes.push(ProposalNode {+ token_id: frontier_field(FrontierIdx::TokenId, slot),+ depth: frontier_field(FrontierIdx::Depth, slot),+ logprob: f32::from_bits(frontier_field(FrontierIdx::EdgeLogprobBits, slot)),child_indices: Vec::new(),});}@@ -255,85 +286,34 @@ impl<B: Backend> Weaver<B> {self.max_depth}- pub fn encode_tree(+ fn encode_prefix(&self,target_hidden: &Allocation<B>,draft_hidden: &Allocation<B>,- target_embedding: &Embedding<B>,- logits: &Allocation<B>,- depth_seeds: &[u64],- root_token_id: u32,- shape: WeaverTreeShape,+ rope: &PrecalculatedRoPE<B>,+ depth: u32,encoder: &mut Encoder<B>,- ) -> Result<EncodedWeaverTree<B>, WeaverEncodeError<B>> {- encoder.push_debug_group("weaver tree");-- let tree_slot_count = shape.budget;- let ancestor_stride = self.max_depth;- if shape.budget == 0- || shape.depth < 2- || shape.depth > self.max_depth + 1- || shape.expand_per_round == 0- || shape.expand_per_round > FRONTIER_MAX_WIDTH- || shape.expand_width == 0- || shape.expand_width > self.candidate_pool_size- || tree_slot_count > FRONTIER_MAX_SLOTS / shape.expand_width- || depth_seeds.len() as u32 != self.max_depth- {- return Err(WeaverEncodeError::InvalidTreeInput);- }- let frontier_capacity = tree_slot_count * shape.expand_width;- // `depth` counts the root: nodes expand up to `depth - 1` edges from it.- let lookahead_count = shape.depth - 1;+ ) -> Result<Vec<Allocation<B>>, WeaverEncodeError<B>> {+ encoder.push_debug_group("weaver prefix");- // Rank the draft logits: the top `candidate_pool_size` tokens per- // lookahead row form the candidate pool node expansions draw from.- let vocab_size = target_embedding.vocab_size();- assert!(- logits.size() >= size_for_shape(&[lookahead_count, vocab_size], DataType::F32),- "draft logits do not cover the lookahead rows"- );- let mut candidate_ids = encoder- .allocate_scratch(size_for_shape(&[lookahead_count, self.candidate_pool_size], DataType::U32))- .map_err(WeaverEncodeError::Backend)?;- let mut candidate_logits = encoder- .allocate_scratch(size_for_shape(&[lookahead_count, self.candidate_pool_size], DataType::F32))- .map_err(WeaverEncodeError::Backend)?;- self.top_k- .encode(- logits,- &mut candidate_ids,- &mut candidate_logits,- lookahead_count,- self.candidate_pool_size,- encoder,- )- .map_err(WeaverEncodeError::Backend)?;-- let rope_positions = (0..=self.max_depth).collect::<Box<[_]>>();- let rope = PrecalculatedRoPE::precalculate(&self.rope_config, &rope_positions, encoder)- .map_err(WeaverEncodeError::Backend)?;-- // Prefix pass: run the target token and one draft row per non-root- // chain node through every layer, collecting per-layer KV caches.let hidden_row_bytes = size_for_shape(&[self.target_model_dim], DATA_TYPE);let mut prefix_hidden = encoder- .allocate_scratch(size_for_shape(&[shape.depth, self.target_model_dim], DATA_TYPE))+ .allocate_scratch(size_for_shape(&[depth, self.target_model_dim], DATA_TYPE)).map_err(WeaverEncodeError::Backend)?;encoder.encode_copy(target_hidden, 0..hidden_row_bytes, &mut prefix_hidden, 0..hidden_row_bytes);encoder.encode_copy(draft_hidden,- hidden_row_bytes..shape.depth as usize * hidden_row_bytes,+ hidden_row_bytes..depth as usize * hidden_row_bytes,&mut prefix_hidden,- hidden_row_bytes..shape.depth as usize * hidden_row_bytes,+ hidden_row_bytes..depth as usize * hidden_row_bytes,);let normalized_prefix = self.hidden_state_norm- .encode(&prefix_hidden, 0, shape.depth, None, encoder)+ .encode(&prefix_hidden, 0, depth, None, encoder).map_err(WeaverEncodeError::Backend)?;let mut residual_input = self.hidden_state_projection- .encode(normalized_prefix, shape.depth, encoder)+ .encode(normalized_prefix, depth, encoder).map_err(WeaverEncodeError::Backend)?;let (last_layer, preceding_layers) = self.layers.split_last().expect("Weaver must have at least one layer");let mut residual_state = encoder.allocate_scratch(residual_input.size()).map_err(WeaverEncodeError::Backend)?;@@ -343,12 +323,12 @@ impl<B: Backend> Weaver<B> {queries,kv_cache,} = layer- .encode_prefix_attention(&residual_input, &mut residual_state, &rope, shape.depth, encoder)+ .encode_prefix_attention(&residual_input, &mut residual_state, rope, depth, encoder).map_err(WeaverEncodeError::Backend)?;let state_type = AttentionStateType::Full {length: 0,};- let kv_plane_bytes = size_for_shape(&[shape.depth, self.model_dim], DATA_TYPE);+ let kv_plane_bytes = size_for_shape(&[depth, self.model_dim], DATA_TYPE);let attention_output = layer.prefix_attention.encode(@@ -356,7 +336,7 @@ impl<B: Backend> Weaver<B> {queries: &queries,keys: &kv_cache,values: (&kv_cache, kv_plane_bytes),- suffix_length: shape.depth,+ suffix_length: depth,trie: None,sinks: None,state_type: &state_type,@@ -365,17 +345,245 @@ impl<B: Backend> Weaver<B> {).map_err(WeaverEncodeError::Backend)?;residual_input = layer- .encode_post_attention(attention_output, &mut residual_state, shape.depth, encoder)+ .encode_post_attention(attention_output, &mut residual_state, depth, encoder).map_err(WeaverEncodeError::Backend)?;prefix_kv_layers.push(kv_cache);}prefix_kv_layers.push(last_layer- .encode_prefix_attention(&residual_input, &mut residual_state, &rope, shape.depth, encoder)+ .encode_prefix_attention(&residual_input, &mut residual_state, rope, depth, encoder).map_err(WeaverEncodeError::Backend)?.kv_cache,);+ encoder.pop_debug_group();++ Ok(prefix_kv_layers)+ }++ fn encode_step(+ &self,+ target_embedding: &Embedding<B>,+ prefix_kv_layers: &[Allocation<B>],+ rope: &PrecalculatedRoPE<B>,+ node_kv_layers: &mut [Allocation<B>],+ packed_tree: &mut Allocation<B>,+ frontier: &mut Allocation<B>,+ slot_ancestors: &mut Allocation<B>,+ node_token_ids: &mut Allocation<B>,+ node_metadata: &mut Allocation<B>,+ node_ancestor_indices: &mut Allocation<B>,+ node_valid: &mut Allocation<B>,+ node_candidate_ids: &mut Allocation<B>,+ node_candidate_logits: &mut Allocation<B>,+ depth_seeds_buffer: &Allocation<B>,+ candidate_ids: &Allocation<B>,+ candidate_logits: &Allocation<B>,+ shape: &WeaverTreeShape,+ batch_node_count: u32,+ batch_start_slot: u32,+ encoder: &mut Encoder<B>,+ ) -> Result<(), WeaverEncodeError<B>> {+ let tree_slot_count = shape.slot_count();+ let ancestor_stride = self.max_depth;+ let frontier_capacity = tree_slot_count * shape.expand_width;++ if batch_start_slot > 0 {+ self.frontier_select.encode(+ &mut *frontier,+ &mut *packed_tree,+ &mut *slot_ancestors,+ &mut *node_token_ids,+ &mut *node_metadata,+ &mut *node_ancestor_indices,+ &mut *node_valid,+ candidate_ids,+ candidate_logits,+ &mut *node_candidate_ids,+ &mut *node_candidate_logits,+ frontier_capacity,+ tree_slot_count,+ batch_node_count,+ batch_start_slot,+ ancestor_stride,+ self.max_depth,+ shape.max_depth - 1,+ shape.dflash_depth - 1,+ self.candidate_pool_size,+ encoder,+ );+ }+ let (batch_candidate_ids, batch_candidate_logits) = if batch_start_slot == 0 {+ (candidate_ids, candidate_logits)+ } else {+ (&*node_candidate_ids, &*node_candidate_logits)+ };++ // Node expansion: embed the batch's tokens, run every layer against+ // the prefix KV and each node's ancestors, then pick its children.+ let token_embedding = target_embedding.encode_lookup(&*node_token_ids, batch_node_count, encoder)?;+ let normalized_embedding = self+ .token_embedding_norm+ .encode(&token_embedding, 0, batch_node_count, None, encoder)+ .map_err(WeaverEncodeError::Backend)?;+ let mut residual_input = self+ .token_embedding_projection+ .encode(normalized_embedding, batch_node_count, encoder)+ .map_err(WeaverEncodeError::Backend)?;+ let mut residual_state = encoder.allocate_scratch(residual_input.size()).map_err(WeaverEncodeError::Backend)?;+ let metadata_field_bytes = size_for_shape(&[batch_node_count], DataType::U32);+ for (layer_index, layer) in self.layers.iter().enumerate() {+ let attention_input = layer+ .pre_attention_norm+ .encode(&residual_input, 0, batch_node_count, Some(&mut residual_state), encoder)+ .map_err(WeaverEncodeError::Backend)?;+ let current_qkv = layer+ .qkv_projection+ .encode(attention_input, batch_node_count, encoder)+ .map_err(WeaverEncodeError::Backend)?;+ let mut attention_output = encoder+ .allocate_scratch(size_for_shape(&[batch_node_count, self.model_dim], DATA_TYPE))+ .map_err(WeaverEncodeError::Backend)?;+ layer.ancestor_attention.encode(+ &prefix_kv_layers[layer_index],+ &mut node_kv_layers[layer_index],+ ¤t_qkv,+ &rope.cosines,+ &rope.sines,+ &*node_metadata,+ &*node_ancestor_indices,+ (&*node_metadata, MetadataIdx::AncestorCount as usize * metadata_field_bytes),+ (&*node_metadata, MetadataIdx::TreeSlot as usize * metadata_field_bytes),+ &mut attention_output,+ batch_node_count,+ shape.dflash_depth,+ ancestor_stride,+ tree_slot_count,+ layer.max_depth,+ layer.attention_scale,+ encoder,+ );+ residual_input = layer+ .encode_post_attention(attention_output, &mut residual_state, batch_node_count, encoder)+ .map_err(WeaverEncodeError::Backend)?;+ }++ let normalized_output = self+ .readout_norm+ .encode(&residual_input, 0, batch_node_count, Some(&mut residual_state), encoder)+ .map_err(WeaverEncodeError::Backend)?;+ let query = self+ .readout_query_projection+ .encode(normalized_output, batch_node_count, encoder)+ .map_err(WeaverEncodeError::Backend)?;+ let logit_residuals = target_embedding.encode_readout_sparse(+ &query,+ batch_candidate_ids,+ batch_node_count,+ self.candidate_pool_size,+ encoder,+ )?;+ let mut child_token_ids = encoder+ .allocate_scratch(size_for_shape(&[batch_node_count, shape.expand_width], DataType::U32))+ .map_err(WeaverEncodeError::Backend)?;+ let mut child_logprobs = encoder+ .allocate_scratch(size_for_shape(&[batch_node_count, shape.expand_width], DataType::F32))+ .map_err(WeaverEncodeError::Backend)?;+ self.top_children.encode(+ &logit_residuals,+ batch_candidate_logits,+ batch_candidate_ids,+ depth_seeds_buffer,+ &*node_metadata,+ &mut child_token_ids,+ &mut child_logprobs,+ batch_node_count,+ self.candidate_pool_size,+ shape.expand_width,+ target_embedding.vocab_size(),+ encoder,+ );++ self.frontier_insert_children.encode(+ &*packed_tree,+ &*node_metadata,+ &*node_valid,+ &child_token_ids,+ &child_logprobs,+ frontier,+ frontier_capacity,+ tree_slot_count,+ batch_node_count,+ shape.expand_width,+ encoder,+ );++ Ok(())+ }++ pub fn encode_tree(+ &self,+ target_hidden: &Allocation<B>,+ draft_hidden: &Allocation<B>,+ target_embedding: &Embedding<B>,+ logits: &Allocation<B>,+ depth_seeds: &[u64],+ root_token_id: u32,+ shape: WeaverTreeShape,+ encoder: &mut Encoder<B>,+ ) -> Result<EncodedWeaverTree<B>, WeaverEncodeError<B>> {+ encoder.push_debug_group("weaver tree");++ let tree_slot_count = shape.slot_count();+ let ancestor_stride = self.max_depth;+ if shape.tree_budget == 0+ || shape.rounds == 0+ || shape.max_depth < 2+ || shape.max_depth > self.max_depth + 1+ || shape.dflash_depth < shape.max_depth+ || shape.dflash_depth > self.max_depth + 1+ || shape.expand_per_round == 0+ || shape.expand_per_round > FRONTIER_MAX_WIDTH+ || shape.expand_width == 0+ || shape.expand_width > self.candidate_pool_size+ || tree_slot_count > FRONTIER_MAX_SLOTS / shape.expand_width+ || depth_seeds.len() as u32 != self.max_depth+ {+ return Err(WeaverEncodeError::InvalidTreeInput);+ }+ let frontier_capacity = tree_slot_count * shape.expand_width;+ let pool_depth_count = shape.dflash_depth - 1;++ // Rank the draft logits: the top `candidate_pool_size` tokens per+ // lookahead row form the candidate pool node expansions draw from.+ let vocab_size = target_embedding.vocab_size();+ assert!(+ logits.size() >= size_for_shape(&[pool_depth_count, vocab_size], DataType::F32),+ "draft logits do not cover the lookahead rows"+ );+ let mut candidate_ids = encoder+ .allocate_scratch(size_for_shape(&[pool_depth_count, self.candidate_pool_size], DataType::U32))+ .map_err(WeaverEncodeError::Backend)?;+ let mut candidate_logits = encoder+ .allocate_scratch(size_for_shape(&[pool_depth_count, self.candidate_pool_size], DataType::F32))+ .map_err(WeaverEncodeError::Backend)?;+ self.top_k+ .encode(+ logits,+ &mut candidate_ids,+ &mut candidate_logits,+ pool_depth_count,+ self.candidate_pool_size,+ encoder,+ )+ .map_err(WeaverEncodeError::Backend)?;++ let rope_positions = (0..=self.max_depth).collect::<Box<[_]>>();+ let rope = PrecalculatedRoPE::precalculate(&self.rope_config, &rope_positions, encoder)+ .map_err(WeaverEncodeError::Backend)?;++ let prefix_kv_layers = self.encode_prefix(target_hidden, draft_hidden, &rope, shape.dflash_depth, encoder)?;+// Per-layer KV cache for tree nodes, one slot per packed-tree slot.let node_kv_size = size_for_shape(&[2, tree_slot_count, self.model_dim], DATA_TYPE);let mut node_kv_layers = (0..self.layers.len())@@ -425,145 +633,36 @@ impl<B: Backend> Weaver<B> {encoder.allocate_constant_from_slice(depth_seeds).map_err(WeaverEncodeError::Backend)?;let mut batch_start_slot = 0;- while batch_start_slot < tree_slot_count {- let batch_node_count = if batch_start_slot == 0 {+ for round in 0..shape.rounds {+ let batch_node_count = if round == 0 {1} else {- shape.expand_per_round.min(tree_slot_count - batch_start_slot)+ shape.expand_per_round};- if batch_start_slot > 0 {- self.frontier_select.encode(- &mut frontier,- &mut packed_tree,- &mut slot_ancestors,- &mut node_token_ids,- &mut node_metadata,- &mut node_ancestor_indices,- &mut node_valid,- &candidate_ids,- &candidate_logits,- &mut node_candidate_ids,- &mut node_candidate_logits,- frontier_capacity,- tree_slot_count,- batch_node_count,- batch_start_slot,- ancestor_stride,- self.max_depth,- lookahead_count,- lookahead_count,- self.candidate_pool_size,- encoder,- );- }- if batch_start_slot + batch_node_count == tree_slot_count {- break; // packed-tree slots filled- }- let (batch_candidate_ids, batch_candidate_logits) = if batch_start_slot == 0 {- (&candidate_ids, &candidate_logits)- } else {- (&node_candidate_ids, &node_candidate_logits)- };-- // Node expansion: embed the batch's tokens, run every layer against- // the prefix KV and each node's ancestors, then pick its children.- let token_embedding = target_embedding.encode_lookup(&node_token_ids, batch_node_count, encoder)?;- let normalized_embedding = self- .token_embedding_norm- .encode(&token_embedding, 0, batch_node_count, None, encoder)- .map_err(WeaverEncodeError::Backend)?;- let mut residual_input = self- .token_embedding_projection- .encode(normalized_embedding, batch_node_count, encoder)- .map_err(WeaverEncodeError::Backend)?;- let mut residual_state =- encoder.allocate_scratch(residual_input.size()).map_err(WeaverEncodeError::Backend)?;- let metadata_field_bytes = size_for_shape(&[batch_node_count], DataType::U32);- for (layer_index, layer) in self.layers.iter().enumerate() {- let attention_input = layer- .pre_attention_norm- .encode(&residual_input, 0, batch_node_count, Some(&mut residual_state), encoder)- .map_err(WeaverEncodeError::Backend)?;- let current_qkv = layer- .qkv_projection- .encode(attention_input, batch_node_count, encoder)- .map_err(WeaverEncodeError::Backend)?;- let mut attention_output = encoder- .allocate_scratch(size_for_shape(&[batch_node_count, self.model_dim], DATA_TYPE))- .map_err(WeaverEncodeError::Backend)?;- layer.ancestor_attention.encode(- &prefix_kv_layers[layer_index],- &mut node_kv_layers[layer_index],- ¤t_qkv,- &rope.cosines,- &rope.sines,- &node_metadata,- &node_ancestor_indices,- (&node_metadata, MetadataIdx::AncestorCount as usize * metadata_field_bytes),- (&node_metadata, MetadataIdx::TreeSlot as usize * metadata_field_bytes),- &mut attention_output,- batch_node_count,- shape.depth,- ancestor_stride,- tree_slot_count,- layer.max_depth,- layer.attention_scale,- encoder,- );- residual_input = layer- .encode_post_attention(attention_output, &mut residual_state, batch_node_count, encoder)- .map_err(WeaverEncodeError::Backend)?;- }-- let normalized_output = self- .readout_norm- .encode(&residual_input, 0, batch_node_count, Some(&mut residual_state), encoder)- .map_err(WeaverEncodeError::Backend)?;- let query = self- .readout_query_projection- .encode(normalized_output, batch_node_count, encoder)- .map_err(WeaverEncodeError::Backend)?;- let logit_residuals = target_embedding.encode_readout_sparse(- &query,- batch_candidate_ids,- batch_node_count,- self.candidate_pool_size,- encoder,- )?;- let mut child_token_ids = encoder- .allocate_scratch(size_for_shape(&[batch_node_count, shape.expand_width], DataType::U32))- .map_err(WeaverEncodeError::Backend)?;- let mut child_logprobs = encoder- .allocate_scratch(size_for_shape(&[batch_node_count, shape.expand_width], DataType::F32))- .map_err(WeaverEncodeError::Backend)?;- self.top_children.encode(- &logit_residuals,- batch_candidate_logits,- batch_candidate_ids,- &depth_seeds_buffer,- &node_metadata,- &mut child_token_ids,- &mut child_logprobs,- batch_node_count,- self.candidate_pool_size,- shape.expand_width,- target_embedding.vocab_size(),- encoder,- );-- self.frontier_insert_children.encode(- &packed_tree,- &node_metadata,- &node_valid,- &child_token_ids,- &child_logprobs,+ encoder.push_debug_group("weaver step");+ self.encode_step(+ target_embedding,+ &prefix_kv_layers,+ &rope,+ &mut node_kv_layers,+ &mut packed_tree,&mut frontier,- frontier_capacity,- tree_slot_count,+ &mut slot_ancestors,+ &mut node_token_ids,+ &mut node_metadata,+ &mut node_ancestor_indices,+ &mut node_valid,+ &mut node_candidate_ids,+ &mut node_candidate_logits,+ &depth_seeds_buffer,+ &candidate_ids,+ &candidate_logits,+ &shape,batch_node_count,- shape.expand_width,+ batch_start_slot,encoder,- );+ )?;+ encoder.pop_debug_group();batch_start_slot += batch_node_count;}@@ -571,6 +670,7 @@ impl<B: Backend> Weaver<B> {Ok(EncodedWeaverTree {packed_tree,+ frontier,})}}diff --git a/crates/backend-uzu/src/encodable_block/weaver_layer.rs b/crates/backend-uzu/src/encodable_block/weaver_layer.rsindex 139a57834..dacfd8d11 100644--- a/crates/backend-uzu/src/encodable_block/weaver_layer.rs+++ b/crates/backend-uzu/src/encodable_block/weaver_layer.rs@@ -28,23 +28,18 @@ pub struct PreparedPrefixAttention<B: Backend> {}pub struct WeaverLayer<B: Backend> {- // Attentionpub pre_attention_norm: Normalization<B>,pub qkv_projection: Box<dyn Linear<B>>,- pub attention_prepare: <B::Kernels as Kernels>::AttentionPrepareKernel,+ attention_prepare: <B::Kernels as Kernels>::AttentionPrepareKernel,pub prefix_attention: AttentionCores<B>,pub ancestor_attention: <B::Kernels as Kernels>::AncestorAttentionKernel,- pub out_projection: Box<dyn Linear<B>>,-- // MLP- pub pre_mlp_norm: Normalization<B>,- pub mlp: Box<dyn Mlp<B>>,-- // Geometry+ out_projection: Box<dyn Linear<B>>,+ pre_mlp_norm: Normalization<B>,+ mlp: Box<dyn Mlp<B>>,pub attention_scale: f32,- pub model_dim: u32,- pub num_heads: u32,- pub head_dim: u32,+ model_dim: u32,+ num_heads: u32,+ head_dim: u32,pub max_depth: u32,}diff --git a/crates/backend-uzu/src/engine/language_model/stream/stream.rs b/crates/backend-uzu/src/engine/language_model/stream/stream.rsindex 2bad7060b..1f85bf6db 100644--- a/crates/backend-uzu/src/engine/language_model/stream/stream.rs+++ b/crates/backend-uzu/src/engine/language_model/stream/stream.rs@@ -334,7 +334,7 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {metrics.num_tokens_accepted += 1;DecodingState::ForwardPassPending(DecodingStatePending {- input_trie: TrieNode::new(0, 0),+ input_trie: TrieNode::new(0, 0, 0.0),full_accept: true,pending,capture_span,@@ -547,13 +547,7 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {None};- let full_batch_size = self.model.speculator.as_ref().map_or(1, |speculator| {- if speculator.has_weaver() {- 32- } else {- 16- }- });+ let full_batch_size = 16;let speculation_batch = self.model_state.max_context_length@@ -576,10 +570,12 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {root_token as u32,self.model.decoder.embedding(),DFlashTfmTreeShape {- budget: speculation_batch,+ tree_budget: speculation_batch,+ max_tree_depth: 16,+ dflash_depth_override: None,construction_method: if speculator.has_weaver() {DFlashTfmTreeConstructionMethod::Weaver {- depth: 16,+ rounds: 16,expand_per_round: 4,expand_width: 4,}@@ -601,7 +597,7 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {} => (*token, None),ForwardPassChaining::InFlight(pending) => (0, Some(&pending.output_tokens)),};- (TrieNode::new(token, self.model_state.prng.derive(context_length as u64)), chain_copy, true)+ (TrieNode::new(token, self.model_state.prng.derive(context_length as u64), 0.0), chain_copy, true)};let input_flat_trie = input_trie.linearize();diff --git a/crates/backend-uzu/src/speculators/dflash_tfm.rs b/crates/backend-uzu/src/speculators/dflash_tfm.rsindex 1d9a0ae92..9ee920a2a 100644--- a/crates/backend-uzu/src/speculators/dflash_tfm.rs+++ b/crates/backend-uzu/src/speculators/dflash_tfm.rs@@ -56,14 +56,16 @@ pub enum DFlashSpeculatorLoadError<B: Backend> {pub enum DFlashTfmTreeConstructionMethod {Argmax,Weaver {- depth: u32,+ rounds: u32,expand_per_round: u32,expand_width: u32,},}pub struct DFlashTfmTreeShape {- pub budget: u32,+ pub tree_budget: u32,+ pub max_tree_depth: u32,+ pub dflash_depth_override: Option<u32>,pub construction_method: DFlashTfmTreeConstructionMethod,}@@ -153,7 +155,15 @@ impl<B: Backend> DFlashTfmSpeculator<B> {prng: &PRng,allocation_pool: Arc<AllocationPool<B>>,) -> Result<TrieNode, DFlashTreeError<B>> {- assert!(shape.budget >= 2, "tree budget needs at least a root and one draft token");+ assert!(shape.tree_budget >= 2, "tree budget needs at least a root and one draft token");++ let block_size = self.dflash.block_size();+ let dflash_depth = shape.dflash_depth_override.unwrap_or(block_size);+ if !(2..=block_size).contains(&dflash_depth) {+ return Err(DFlashTreeError::InvalidTreeShape(format!(+ "dflash depth {dflash_depth} is outside 2..={block_size}"+ )));+ }let root_position = state.context_length();@@ -162,26 +172,27 @@ impl<B: Backend> DFlashTfmSpeculator<B> {let nodes = match shape.construction_method {DFlashTfmTreeConstructionMethod::Argmax => {- if shape.budget > self.dflash.block_size() {+ if shape.tree_budget > dflash_depth {return Err(DFlashTreeError::InvalidTreeShape(format!(- "argmax chain of {} nodes needs {} draft rows, block size is {}",- shape.budget,- shape.budget - 1,- self.dflash.block_size()+ "argmax chain of {} nodes needs {} draft rows, dflash depth is {}",+ shape.tree_budget,+ shape.tree_budget - 1,+ dflash_depth)));}- let chain_length = shape.budget - 1;- let mut nodes = Vec::with_capacity(shape.budget as usize);+ let chain_length = shape.tree_budget - 1;+ let mut nodes = Vec::with_capacity(shape.tree_budget as usize);nodes.push(ProposalNode {token_id: target_output_token,depth: 0,+ logprob: 0.0,child_indices: vec![1],});let dflash_output = self.dflash.encode_draft(state,target_output_token,target_embedding,- shape.budget,+ dflash_depth,&mut encoder,)?;let topology_nodes = (0..chain_length)@@ -213,6 +224,7 @@ impl<B: Backend> DFlashTfmSpeculator<B> {nodes.extend(tokens.into_iter().zip(1u32..).map(|(token_id, depth)| ProposalNode {token_id,depth,+ logprob: 0.0,child_indices: if depth < chain_length {vec![depth as usize + 1]} else {@@ -222,33 +234,35 @@ impl<B: Backend> DFlashTfmSpeculator<B> {nodes},DFlashTfmTreeConstructionMethod::Weaver {- depth,+ rounds,expand_per_round,expand_width,} => {let weaver =self.weaver.as_ref().expect("weaver tree construction requires a speculator with weaver weights");- if depth < 2 {- return Err(DFlashTreeError::InvalidTreeShape(format!(- "tree depth {depth} needs at least a root and one level"- )));- }- // `depth` counts the root; the weaver's `max_depth` counts edges.- if depth > weaver.max_depth() + 1 {+ // `max_depth` counts the root; the weaver's `max_depth` counts edges.+ if shape.max_tree_depth < 2 || shape.max_tree_depth > weaver.max_depth() + 1 {return Err(DFlashTreeError::InvalidTreeShape(format!(- "requested tree depth {depth} exceeds weaver max depth {}",+ "tree max_depth {} is outside 2..={}",+ shape.max_tree_depth,weaver.max_depth() + 1)));}- if depth > self.dflash.block_size() {+ if shape.max_tree_depth > dflash_depth {return Err(DFlashTreeError::InvalidTreeShape(format!(- "tree of depth {depth} needs {} draft rows, block size is {}",- depth - 1,- self.dflash.block_size()+ "tree of max_depth {} needs {} draft rows, dflash depth is {}",+ shape.max_tree_depth,+ shape.max_tree_depth - 1,+ dflash_depth)));}- let dflash_output =- self.dflash.encode_draft(state, target_output_token, target_embedding, depth, &mut encoder)?;+ let dflash_output = self.dflash.encode_draft(+ state,+ target_output_token,+ target_embedding,+ dflash_depth,+ &mut encoder,+ )?;let depth_seeds = (0..weaver.max_depth()).map(|depth| prng.derive(root_position as u64 + depth as u64)).collect::<Box<[u64]>>();@@ -260,8 +274,10 @@ impl<B: Backend> DFlashTfmSpeculator<B> {&depth_seeds,target_output_token,WeaverTreeShape {- budget: shape.budget,- depth,+ tree_budget: shape.tree_budget,+ max_depth: shape.max_tree_depth,+ dflash_depth,+ rounds,expand_per_round,expand_width,},@@ -283,8 +299,11 @@ impl<B: Backend> DFlashTfmSpeculator<B> {prng: &PRng,) -> TrieNode {let node = &nodes[index];- let mut trie_node =- TrieNode::new(node.token_id as u64, prng.derive(root_position as u64 + node.depth as u64));+ let mut trie_node = TrieNode::new(+ node.token_id as u64,+ prng.derive(root_position as u64 + node.depth as u64),+ node.logprob,+ );for &child_index in &node.child_indices {#[cfg(grammar)]if let Some(grammar) = grammar.as_mut()@@ -312,13 +331,15 @@ impl<B: Backend> DFlashTfmSpeculator<B> {trie_node}- Ok(recursive_build(+ let mut trie = recursive_build(&nodes,0,root_position,#[cfg(grammar)]grammar,prng,- ))+ );+ trie.prune_to_budget(shape.tree_budget as usize);+ Ok(trie)}}diff --git a/crates/backend-uzu/src/trie.rs b/crates/backend-uzu/src/trie.rsindex f56c26b8e..5c1086f60 100644--- a/crates/backend-uzu/src/trie.rs+++ b/crates/backend-uzu/src/trie.rs@@ -26,6 +26,7 @@ pub enum TrieAcceptError {pub struct TrieNode {token: u64,seed: u64,+ logprob: f32,next: Vec<TrieNode>,}@@ -45,10 +46,12 @@ impl TrieNode {pub fn new(token: u64,seed: u64,+ logprob: f32,) -> Self {Self {token,seed,+ logprob,next: Vec::new(),}}@@ -77,6 +80,63 @@ impl TrieNode {self.token}+ #[cfg(test)]+ pub fn logprob(&self) -> f32 {+ self.logprob+ }++ #[cfg(test)]+ pub fn node_count(&self) -> usize {+ 1 + self.next.iter().map(TrieNode::node_count).sum::<usize>()+ }++ pub fn prune_to_budget(+ &mut self,+ budget: usize,+ ) {+ assert!(budget > 0, "budget must keep at least the root");++ fn collect_logprobs(+ node: &TrieNode,+ parent_logprob: f32,+ logprobs: &mut Vec<f32>,+ ) {+ let logprob = parent_logprob + node.logprob;+ logprobs.push(logprob);+ for child in &node.next {+ collect_logprobs(child, logprob, logprobs);+ }+ }+ let mut logprobs = Vec::new();+ collect_logprobs(self, 0.0, &mut logprobs);+ if budget >= logprobs.len() {+ return;+ }++ let mut order: Box<[usize]> = (0..logprobs.len()).collect();+ order.sort_by(|&a, &b| logprobs[b].total_cmp(&logprobs[a]));+ let mut kept = vec![false; logprobs.len()];+ for &index in order.iter().take(budget) {+ kept[index] = true;+ }++ fn prune(+ node: &mut TrieNode,+ kept: &[bool],+ cursor: &mut usize,+ ) {+ *cursor += 1;+ let mut children = std::mem::take(&mut node.next);+ children.retain_mut(|child| {+ let child_index = *cursor;+ prune(child, kept, cursor);+ kept[child_index]+ });+ node.next = children;+ }+ prune(self, &kept, &mut 0);+ }+pub fn flat(prefix_length: usize,tokens: &[u64],@@ -84,11 +144,11 @@ impl TrieNode {) -> Self {assert!(!tokens.is_empty(), "need seed node");- let mut root = TrieNode::new(tokens[0], prng.derive(prefix_length as u64));+ let mut root = TrieNode::new(tokens[0], prng.derive(prefix_length as u64), 0.0);let mut leaf = &mut root;for (index, token) in tokens.iter().copied().enumerate().skip(1) {- leaf.add(TrieNode::new(token, prng.derive((prefix_length + index) as u64))).unwrap();+ leaf.add(TrieNode::new(token, prng.derive((prefix_length + index) as u64), 0.0)).unwrap();leaf = &mut leaf.next[0];}diff --git a/crates/backend-uzu/tests/unit/backends/common/kernel/attention/ancestor_attention_test.rs b/crates/backend-uzu/tests/unit/backends/common/kernel/attention/ancestor_attention_test.rsindex 3f8703407..b3d4e42d7 100644--- a/crates/backend-uzu/tests/unit/backends/common/kernel/attention/ancestor_attention_test.rs+++ b/crates/backend-uzu/tests/unit/backends/common/kernel/attention/ancestor_attention_test.rs@@ -84,7 +84,7 @@ impl<B: Backend> Runner<B> {}let mut node_metadata = vec![0u32; rows * MetadataIdx::COUNT];for row in 0..rows {- node_metadata[MetadataIdx::Depth as usize * rows + row] = (row as u32 * 3) % (MAX_DEPTH + 2);+ node_metadata[MetadataIdx::Depth as usize * rows + row] = (row as u32 * 3) % MAX_DEPTH;}let context = create_context::<B>();diff --git a/crates/backend-uzu/tests/unit/backends/common/kernel/weaver/weaver_frontier_test.rs b/crates/backend-uzu/tests/unit/backends/common/kernel/weaver/weaver_frontier_test.rsindex bce06ae78..259fccd9c 100644--- a/crates/backend-uzu/tests/unit/backends/common/kernel/weaver/weaver_frontier_test.rs+++ b/crates/backend-uzu/tests/unit/backends/common/kernel/weaver/weaver_frontier_test.rs@@ -20,7 +20,7 @@ fn select<B: Backend>() -> Vec<u32> {(8, 0, 2, 0x3f00_0001, 100, 1),(7, 0, 2, 0x3f00_0002, 100, 1),(7, 0, 2, 0x3f00_0003, 100, 1),- (2, 1, 3, 0x3f00_0004, 90, 1),+ (2, 1, 3, 0x3f00_0004, 300, 1),(0, 0, 0, 0x3f00_0005, 200, 0),(4, 1, 3, 0x3f00_0006, 80, 1),(5, 1, 1, 0x3f00_0007, 70, 1),diff --git a/crates/backend-uzu/tests/unit/trie_test.rs b/crates/backend-uzu/tests/unit/trie_test.rsindex 335cfde35..25f5fb53d 100644--- a/crates/backend-uzu/tests/unit/trie_test.rs+++ b/crates/backend-uzu/tests/unit/trie_test.rs@@ -10,9 +10,9 @@ fn verify_sprout(assert_eq!(flat_trie.len(), 1);assert_eq!(flat_trie.index(trie_root), Some(0));- assert_eq!(flat_trie.index(&TrieNode::new(1, 0)), None);- assert_eq!(flat_trie.index(&TrieNode::new(0, 1)), None);- assert_eq!(flat_trie.index(&TrieNode::new(0, 0)), None);+ assert_eq!(flat_trie.index(&TrieNode::new(1, 0, 0.0)), None);+ assert_eq!(flat_trie.index(&TrieNode::new(0, 1, 0.0)), None);+ assert_eq!(flat_trie.index(&TrieNode::new(0, 0, 0.0)), None);assert_eq!(flat_trie.token_ids().collect::<Vec<u64>>(), vec![0]);assert_eq!(flat_trie.token_subtrie_ranges().map(|node| node.height as usize).collect::<Vec<_>>(), vec![0]);assert_eq!(flat_trie.token_seeds().collect::<Vec<u64>>(), vec![expected_seed]);@@ -20,7 +20,7 @@ fn verify_sprout(#[uzu_test]fn test_trie_manual_sprout() {- let trie_root = TrieNode::new(0, 0);+ let trie_root = TrieNode::new(0, 0, 0.0);verify_sprout(&trie_root, 0);}@@ -62,13 +62,13 @@ fn verify_stick(fn test_trie_manual_stick() {let rng = PRng::new(0);- let mut trie_stick = TrieNode::new(9, rng.derive(9));+ let mut trie_stick = TrieNode::new(9, rng.derive(9), 0.0);for i in (1..9u64).rev() {- let mut trie_parent = TrieNode::new(i, rng.derive(i));+ let mut trie_parent = TrieNode::new(i, rng.derive(i), 0.0);trie_parent.add(trie_stick).unwrap();trie_stick = trie_parent;}- let mut trie_root = TrieNode::new(0, rng.derive(0));+ let mut trie_root = TrieNode::new(0, rng.derive(0), 0.0);trie_root.add(trie_stick).unwrap();verify_stick(&trie_root, &rng);@@ -108,14 +108,14 @@ fn verify_bush(#[uzu_test]fn test_trie_manual_bush() {let rng = PRng::new(0);- let mut trie_root = TrieNode::new(0, rng.derive(0));+ let mut trie_root = TrieNode::new(0, rng.derive(0), 0.0);- assert!(trie_root.add(TrieNode::new(1, rng.derive(1))).is_ok());- assert!(trie_root.add(TrieNode::new(1, rng.derive(1))).is_err());- assert!(trie_root.add(TrieNode::new(1, 10)).is_err());+ assert!(trie_root.add(TrieNode::new(1, rng.derive(1), 0.0)).is_ok());+ assert!(trie_root.add(TrieNode::new(1, rng.derive(1), 0.0)).is_err());+ assert!(trie_root.add(TrieNode::new(1, 10, 0.0)).is_err());- assert!(trie_root.add(TrieNode::new(2, rng.derive(1))).is_ok());- assert!(trie_root.add(TrieNode::new(3, rng.derive(1))).is_ok());+ assert!(trie_root.add(TrieNode::new(2, rng.derive(1), 0.0)).is_ok());+ assert!(trie_root.add(TrieNode::new(3, rng.derive(1), 0.0)).is_ok());verify_bush(&trie_root, &rng);}@@ -159,21 +159,75 @@ fn verify_tree(#[uzu_test]fn test_trie_manual_tree() {let rng = PRng::new(0);- let mut trie_root = TrieNode::new(0, rng.derive(0));+ let mut trie_root = TrieNode::new(0, rng.derive(0), 0.0);- assert!(trie_root.add(TrieNode::new(1, rng.derive(1))).is_ok());- assert!(trie_root.add(TrieNode::new(1, rng.derive(1))).is_err());- assert!(trie_root.add(TrieNode::new(1, 10)).is_err());+ assert!(trie_root.add(TrieNode::new(1, rng.derive(1), 0.0)).is_ok());+ assert!(trie_root.add(TrieNode::new(1, rng.derive(1), 0.0)).is_err());+ assert!(trie_root.add(TrieNode::new(1, 10, 0.0)).is_err());- let mut mid_b = TrieNode::new(2, rng.derive(1));- assert!(mid_b.add(TrieNode::new(10, rng.derive(2))).is_ok());+ let mut mid_b = TrieNode::new(2, rng.derive(1), 0.0);+ assert!(mid_b.add(TrieNode::new(10, rng.derive(2), 0.0)).is_ok());- let mut mid_c = TrieNode::new(3, rng.derive(1));- assert!(mid_c.add(TrieNode::new(20, rng.derive(2))).is_ok());- assert!(mid_c.add(TrieNode::new(21, rng.derive(2))).is_ok());+ let mut mid_c = TrieNode::new(3, rng.derive(1), 0.0);+ assert!(mid_c.add(TrieNode::new(20, rng.derive(2), 0.0)).is_ok());+ assert!(mid_c.add(TrieNode::new(21, rng.derive(2), 0.0)).is_ok());assert!(trie_root.add(mid_b).is_ok());assert!(trie_root.add(mid_c).is_ok());verify_tree(&trie_root, &rng)}++fn verify_pruned(+ trie_root: &TrieNode,+ expected_tokens: &[u64],+) {+ assert_eq!(trie_root.node_count(), expected_tokens.len());+ assert_eq!(trie_root.linearize().token_ids().collect::<Vec<u64>>(), expected_tokens);+}++fn sample_tree() -> TrieNode {+ let mut root = TrieNode::new(0, 0, 0.0);+ let mut mid_a = TrieNode::new(1, 1, -0.1);+ mid_a.add(TrieNode::new(4, 2, -0.4)).unwrap();+ let mut mid_b = TrieNode::new(2, 1, -0.2);+ mid_b.add(TrieNode::new(5, 2, -2.8)).unwrap();+ root.add(mid_a).unwrap();+ root.add(mid_b).unwrap();+ root.add(TrieNode::new(3, 1, -0.3)).unwrap();+ root+}++#[uzu_test]+fn test_trie_prune_to_budget() {+ let mut trie = sample_tree();+ trie.prune_to_budget(4);+ verify_pruned(&trie, &[0, 1, 2, 3]);+ assert_eq!(trie.get(1).unwrap().logprob(), -0.1);+ assert_eq!(trie.get(2).unwrap().logprob(), -0.2);+ assert_eq!(trie.get(3).unwrap().logprob(), -0.3);++ let mut trie = sample_tree();+ trie.prune_to_budget(2);+ verify_pruned(&trie, &[0, 1]);++ let mut trie = sample_tree();+ trie.prune_to_budget(6);+ verify_pruned(&trie, &[0, 1, 4, 2, 5, 3]);++ let mut trie = sample_tree();+ trie.prune_to_budget(100);+ verify_pruned(&trie, &[0, 1, 4, 2, 5, 3]);+}++#[uzu_test]+fn test_trie_prune_to_budget_tie_keeps_parent() {+ let mut root = TrieNode::new(0, 0, 0.0);+ let mut child = TrieNode::new(1, 1, 0.0);+ child.add(TrieNode::new(2, 2, 0.0)).unwrap();+ root.add(child).unwrap();+ root.add(TrieNode::new(3, 1, 0.0)).unwrap();++ root.prune_to_budget(2);+ verify_pruned(&root, &[0, 1]);+}