Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

Research tree construction (#736)

uuuvn3deb0224b7de
diff --git a/crates/backend-uzu/src/backends/common/gpu_types/weaver.rs b/crates/backend-uzu/src/backends/common/gpu_types/weaver.rs
index bada0a1ec..5e568b863 100644
--- a/crates/backend-uzu/src/backends/common/gpu_types/weaver.rs
+++ b/crates/backend-uzu/src/backends/common/gpu_types/weaver.rs
@@ -51,3 +51,4 @@ pub const FRONTIER_SELECT_SIMDGROUPS: u32 = 8;
pub const FRONTIER_ENTRIES_PER_THREAD: u32 = 8;
pub const FRONTIER_MAX_SLOTS: u32 = 2048;
pub const FRONTIER_MAX_WIDTH: u32 = 32;
+pub const PADDING_DEPTH: u32 = 0;
diff --git a/crates/backend-uzu/src/backends/cpu/kernel/attention/ancestor_attention.rs b/crates/backend-uzu/src/backends/cpu/kernel/attention/ancestor_attention.rs
index 586c9adda..f4d33f19f 100644
--- a/crates/backend-uzu/src/backends/cpu/kernel/attention/ancestor_attention.rs
+++ b/crates/backend-uzu/src/backends/cpu/kernel/attention/ancestor_attention.rs
@@ -42,7 +42,8 @@ fn ancestor_attention<const HEAD_DIM: u32>(
let current_row = current_qkv.add(row * qkv_width);
let depth = *node_metadata.add(MetadataIdx::Depth as usize * row_count + row);
- let position = depth.min(max_depth - 1) as usize + 1;
+ assert!(depth < max_depth, "node metadata depth must be rope-safe");
+ let position = depth as usize + 1;
let rotate = |component: usize| {
let mut rotated = vec![bf16::ZERO; model_dim];
for head in 0..num_heads {
diff --git a/crates/backend-uzu/src/backends/cpu/kernel/weaver/weaver_frontier_select.rs b/crates/backend-uzu/src/backends/cpu/kernel/weaver/weaver_frontier_select.rs
index 365ce8620..08a616961 100644
--- a/crates/backend-uzu/src/backends/cpu/kernel/weaver/weaver_frontier_select.rs
+++ b/crates/backend-uzu/src/backends/cpu/kernel/weaver/weaver_frontier_select.rs
@@ -1,7 +1,7 @@
use proc_macros::kernel;
use crate::backends::common::gpu_types::weaver::{
- FRONTIER_MAX_SLOTS, FRONTIER_MAX_WIDTH, FRONTIER_NO_WINNER, FrontierIdx, MetadataIdx, TreeIdx,
+ FRONTIER_MAX_SLOTS, FRONTIER_MAX_WIDTH, FRONTIER_NO_WINNER, FrontierIdx, MetadataIdx, PADDING_DEPTH, TreeIdx,
};
#[kernel(WeaverFrontierSelect)]
@@ -66,8 +66,11 @@ pub fn weaver_frontier_select(
if frontier[FrontierIdx::Active as usize * frontier_capacity + slot] == 0 {
continue;
}
+ let score_key = frontier[FrontierIdx::PathScoreKey as usize * frontier_capacity + slot];
+ let expandable =
+ u32::from(frontier[FrontierIdx::Depth as usize * frontier_capacity + slot] < lookahead_count);
let next = (
- frontier[FrontierIdx::PathScoreKey as usize * frontier_capacity + slot],
+ (expandable << 31) | (score_key >> 1),
frontier[FrontierIdx::ParentSlot as usize * frontier_capacity + slot],
frontier[FrontierIdx::TokenId as usize * frontier_capacity + slot],
);
@@ -121,16 +124,21 @@ pub fn weaver_frontier_select(
}
node_token_ids[node] = token;
- node_metadata[MetadataIdx::Depth as usize * node_count + node] = depth.min(max_depth - 1);
+ let expandable = depth < lookahead_count;
+ node_metadata[MetadataIdx::Depth as usize * node_count + node] = if expandable {
+ depth
+ } else {
+ PADDING_DEPTH
+ };
node_metadata[MetadataIdx::AncestorCount as usize * node_count + node] = depth;
node_metadata[MetadataIdx::TreeSlot as usize * node_count + node] = tree_slot as u32;
- node_valid[node] = u32::from(real && depth < lookahead_count);
+ node_valid[node] = u32::from(real && expandable);
- // Every node expands the candidate pool for its own depth.
- let candidate_depth = (depth as usize).min(candidate_depth_count - 1);
- let source = candidate_depth * candidates_per_depth..(candidate_depth + 1) * candidates_per_depth;
- let destination = node * candidates_per_depth..(node + 1) * candidates_per_depth;
- node_candidate_ids[destination.clone()].copy_from_slice(&candidate_pool_ids[source.clone()]);
- node_candidate_logits[destination].copy_from_slice(&candidate_pool_logits[source]);
+ if (depth as usize) < candidate_depth_count {
+ let source = depth as usize * candidates_per_depth..(depth as usize + 1) * candidates_per_depth;
+ let destination = node * candidates_per_depth..(node + 1) * candidates_per_depth;
+ node_candidate_ids[destination.clone()].copy_from_slice(&candidate_pool_ids[source.clone()]);
+ node_candidate_logits[destination].copy_from_slice(&candidate_pool_logits[source]);
+ }
}
}
diff --git a/crates/backend-uzu/src/backends/metal/kernel/attention/ancestor_attention.metal b/crates/backend-uzu/src/backends/metal/kernel/attention/ancestor_attention.metal
index a7079dac6..2900c14b7 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/attention/ancestor_attention.metal
+++ b/crates/backend-uzu/src/backends/metal/kernel/attention/ancestor_attention.metal
@@ -101,7 +101,7 @@ PUBLIC KERNEL(AncestorAttention)(
// ask it directly instead of reading the buffer a second time.
static_assert(vectors_per_head == METAL_SIMD_SIZE, "one lane per head vector");
const uint depth = node_metadata[uint(MetadataIdx::Depth) * rows + row];
- const uint rope_position = min(depth, max_depth - 1u) + 1u;
+ const uint rope_position = depth + 1u;
const device float4* cosine_row = (const device float4*)(cosines + rope_position * HEAD_DIM);
const device float4* sine_row = (const device float4*)(sines + rope_position * HEAD_DIM);
const float4 cosine = cosine_row[lane];
diff --git a/crates/backend-uzu/src/backends/metal/kernel/generated/weaver.h b/crates/backend-uzu/src/backends/metal/kernel/generated/weaver.h
index fd799486f..173fdfaf4 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/generated/weaver.h
+++ b/crates/backend-uzu/src/backends/metal/kernel/generated/weaver.h
@@ -47,4 +47,6 @@ static constant constexpr uint32_t FRONTIER_ENTRIES_PER_THREAD = 8;
static constant constexpr uint32_t FRONTIER_MAX_SLOTS = 2048;
static constant constexpr uint32_t FRONTIER_MAX_WIDTH = 32;
+
+static constant constexpr uint32_t PADDING_DEPTH = 0;
} // namespace uzu::weaver
diff --git a/crates/backend-uzu/src/backends/metal/kernel/weaver/weaver_frontier_select.metal b/crates/backend-uzu/src/backends/metal/kernel/weaver/weaver_frontier_select.metal
index 3f4044580..591601ff2 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/weaver/weaver_frontier_select.metal
+++ b/crates/backend-uzu/src/backends/metal/kernel/weaver/weaver_frontier_select.metal
@@ -53,11 +53,14 @@ PUBLIC KERNEL(WeaverFrontierSelect)(
for (uint entry = 0; entry < FRONTIER_ENTRIES_PER_THREAD; ++entry) {
const uint slot = lid + entry * FRONTIER_SELECT_THREADS;
if (entry_active[entry]) {
+ const uint depth = frontier[uint(FrontierIdx::Depth) * frontier_capacity + slot];
const uint key = frontier[uint(FrontierIdx::PathScoreKey) * frontier_capacity + slot];
const uint parent = frontier[uint(FrontierIdx::ParentSlot) * frontier_capacity + slot];
const uint token = frontier[uint(FrontierIdx::TokenId) * frontier_capacity + slot];
- if (key > local.x || (key == local.x && (parent < local.y || (parent == local.y && token < local.z)))) {
- local = uint4(key, parent, token, slot);
+ const uint packed_key = ((depth < lookahead_count ? 1u : 0u) << 31) | (key >> 1);
+ if (packed_key > local.x ||
+ (packed_key == local.x && (parent < local.y || (parent == local.y && token < local.z)))) {
+ local = uint4(packed_key, parent, token, slot);
}
}
}
@@ -127,17 +130,20 @@ PUBLIC KERNEL(WeaverFrontierSelect)(
}
node_token_ids[node] = token;
- node_metadata[uint(MetadataIdx::Depth) * node_count + node] = min(depth, max_depth - 1u);
+ node_metadata[uint(MetadataIdx::Depth) * node_count + node] = depth < lookahead_count ? depth : PADDING_DEPTH;
node_metadata[uint(MetadataIdx::AncestorCount) * node_count + node] = depth;
node_metadata[uint(MetadataIdx::TreeSlot) * node_count + node] = tree_slot;
node_valid[node] = real && depth < lookahead_count ? 1u : 0u;
- node_candidate_depth[node] = min(depth, candidate_depth_count - 1u);
+ node_candidate_depth[node] = depth;
}
threadgroup_barrier(mem_flags::mem_threadgroup);
for (uint node = 0; node < node_count; ++node) {
+ if (node_candidate_depth[node] >= candidate_depth_count) {
+ continue;
+ }
const uint source = node_candidate_depth[node] * candidates_per_depth;
const uint destination = node * candidates_per_depth;
for (uint candidate = lid; candidate < candidates_per_depth; candidate += FRONTIER_SELECT_THREADS) {
diff --git a/crates/backend-uzu/src/encodable_block/dflash.rs b/crates/backend-uzu/src/encodable_block/dflash.rs
index 1eba2a76e..4f73f9249 100644
--- a/crates/backend-uzu/src/encodable_block/dflash.rs
+++ b/crates/backend-uzu/src/encodable_block/dflash.rs
@@ -93,13 +93,8 @@ impl<B: Backend> DFlash<B> {
) -> Result<Self, DFlashNewError<B>> {
let mask_token_id = config.mask_token_id as u32;
assert!(config.block_size <= ATTENTION_SUFFIX_CAPACITY, "DFlash block_size exceeds attention suffix capacity");
- assert_eq!(
- config.num_target_layers,
- config.target_layer_ids.len() as u32,
- "num_target_layers must match target_layer_ids length"
- );
let target_feature_projection = <dyn Linear<B>>::new(
- config.model_dim * config.num_target_layers,
+ config.model_dim * config.target_layer_ids.len() as u32,
[config.model_dim],
false,
context,
@@ -171,7 +166,7 @@ impl<B: Backend> DFlash<B> {
max_context_length: *config.rope_config.max_sequence_length(),
block_size: config.block_size,
mask_token_id,
- target_feature_input_dim: config.model_dim * config.num_target_layers,
+ target_feature_input_dim: config.model_dim * config.target_layer_ids.len() as u32,
data_type,
})
}
diff --git a/crates/backend-uzu/src/encodable_block/weaver.rs b/crates/backend-uzu/src/encodable_block/weaver.rs
index e347f15b9..bda6fcc37 100644
--- a/crates/backend-uzu/src/encodable_block/weaver.rs
+++ b/crates/backend-uzu/src/encodable_block/weaver.rs
@@ -26,52 +26,83 @@ use crate::{
parameters::ParameterTree,
};
-pub(crate) const DATA_TYPE: DataType = DataType::BF16;
-pub(crate) const ROPE_DATA_TYPE: DataType = DataType::F32;
+pub const DATA_TYPE: DataType = DataType::BF16;
+pub const ROPE_DATA_TYPE: DataType = DataType::F32;
pub struct WeaverTreeShape {
- pub budget: u32,
- pub depth: u32,
+ pub tree_budget: u32,
+ pub max_depth: u32,
+ pub dflash_depth: u32,
+ pub rounds: u32,
pub expand_per_round: u32,
pub expand_width: u32,
}
+impl WeaverTreeShape {
+ fn slot_count(&self) -> u32 {
+ 1 + self.rounds.saturating_sub(1) * self.expand_per_round
+ }
+}
+
pub struct EncodedWeaverTree<B: Backend> {
packed_tree: Allocation<B>,
+ frontier: Allocation<B>,
}
pub struct ProposalNode {
pub token_id: u32,
pub depth: u32,
+ pub logprob: f32,
pub child_indices: Vec<usize>,
}
impl<B: Backend> EncodedWeaverTree<B> {
pub fn read_nodes(self) -> Vec<ProposalNode> {
let packed_tree = &self.packed_tree.copyout::<u32>();
+ let frontier = &self.frontier.copyout::<u32>();
let tree_slot_count = packed_tree.len() / TreeIdx::COUNT;
- let field = |field: TreeIdx, slot: usize| packed_tree[field as usize * tree_slot_count + slot];
- let mut slot_to_node_index = vec![usize::MAX; tree_slot_count];
- let mut nodes: Vec<ProposalNode> = Vec::with_capacity(tree_slot_count);
+ let frontier_capacity = frontier.len() / FrontierIdx::COUNT;
+ let tree_field = |field: TreeIdx, slot: usize| packed_tree[field as usize * tree_slot_count + slot];
+ let frontier_field = |field: FrontierIdx, slot: usize| frontier[field as usize * frontier_capacity + slot];
+
+ let mut slot_to_index = vec![usize::MAX; tree_slot_count];
+ let mut nodes: Vec<ProposalNode> = Vec::new();
for slot in 0..tree_slot_count {
- if field(TreeIdx::Valid, slot) == 0 {
+ if tree_field(TreeIdx::Valid, slot) == 0 {
continue;
}
- let parent_slot = field(TreeIdx::ParentSlot, slot) as i32;
+ let parent_slot = tree_field(TreeIdx::ParentSlot, slot) as i32;
let parent = (parent_slot >= 0).then(|| {
- let parent = slot_to_node_index[parent_slot as usize];
+ let parent = slot_to_index[parent_slot as usize];
assert_ne!(parent, usize::MAX, "tree slot {slot} names padding slot {parent_slot} as its parent");
parent
});
let index = nodes.len();
- slot_to_node_index[slot] = index;
+ slot_to_index[slot] = index;
if let Some(parent) = parent {
nodes[parent].child_indices.push(index);
}
nodes.push(ProposalNode {
- token_id: field(TreeIdx::TokenId, slot),
- depth: field(TreeIdx::Depth, slot),
+ token_id: tree_field(TreeIdx::TokenId, slot),
+ depth: tree_field(TreeIdx::Depth, slot),
+ logprob: f32::from_bits(tree_field(TreeIdx::EdgeLogprobBits, slot)),
+ child_indices: Vec::new(),
+ });
+ }
+ for slot in 0..frontier_capacity {
+ if frontier_field(FrontierIdx::Active, slot) == 0 {
+ continue;
+ }
+ let parent_slot = frontier_field(FrontierIdx::ParentSlot, slot) as usize;
+ let parent = slot_to_index[parent_slot];
+ assert_ne!(parent, usize::MAX, "frontier slot {slot} names padding slot {parent_slot} as its parent");
+ let index = nodes.len();
+ nodes[parent].child_indices.push(index);
+ nodes.push(ProposalNode {
+ token_id: frontier_field(FrontierIdx::TokenId, slot),
+ depth: frontier_field(FrontierIdx::Depth, slot),
+ logprob: f32::from_bits(frontier_field(FrontierIdx::EdgeLogprobBits, slot)),
child_indices: Vec::new(),
});
}
@@ -255,85 +286,34 @@ impl<B: Backend> Weaver<B> {
self.max_depth
}
- pub fn encode_tree(
+ fn encode_prefix(
&self,
target_hidden: &Allocation<B>,
draft_hidden: &Allocation<B>,
- target_embedding: &Embedding<B>,
- logits: &Allocation<B>,
- depth_seeds: &[u64],
- root_token_id: u32,
- shape: WeaverTreeShape,
+ rope: &PrecalculatedRoPE<B>,
+ depth: u32,
encoder: &mut Encoder<B>,
- ) -> Result<EncodedWeaverTree<B>, WeaverEncodeError<B>> {
- encoder.push_debug_group("weaver tree");
-
- let tree_slot_count = shape.budget;
- let ancestor_stride = self.max_depth;
- if shape.budget == 0
- || shape.depth < 2
- || shape.depth > self.max_depth + 1
- || shape.expand_per_round == 0
- || shape.expand_per_round > FRONTIER_MAX_WIDTH
- || shape.expand_width == 0
- || shape.expand_width > self.candidate_pool_size
- || tree_slot_count > FRONTIER_MAX_SLOTS / shape.expand_width
- || depth_seeds.len() as u32 != self.max_depth
- {
- return Err(WeaverEncodeError::InvalidTreeInput);
- }
- let frontier_capacity = tree_slot_count * shape.expand_width;
- // `depth` counts the root: nodes expand up to `depth - 1` edges from it.
- let lookahead_count = shape.depth - 1;
+ ) -> Result<Vec<Allocation<B>>, WeaverEncodeError<B>> {
+ encoder.push_debug_group("weaver prefix");
- // Rank the draft logits: the top `candidate_pool_size` tokens per
- // lookahead row form the candidate pool node expansions draw from.
- let vocab_size = target_embedding.vocab_size();
- assert!(
- logits.size() >= size_for_shape(&[lookahead_count, vocab_size], DataType::F32),
- "draft logits do not cover the lookahead rows"
- );
- let mut candidate_ids = encoder
- .allocate_scratch(size_for_shape(&[lookahead_count, self.candidate_pool_size], DataType::U32))
- .map_err(WeaverEncodeError::Backend)?;
- let mut candidate_logits = encoder
- .allocate_scratch(size_for_shape(&[lookahead_count, self.candidate_pool_size], DataType::F32))
- .map_err(WeaverEncodeError::Backend)?;
- self.top_k
- .encode(
- logits,
- &mut candidate_ids,
- &mut candidate_logits,
- lookahead_count,
- self.candidate_pool_size,
- encoder,
- )
- .map_err(WeaverEncodeError::Backend)?;
-
- let rope_positions = (0..=self.max_depth).collect::<Box<[_]>>();
- let rope = PrecalculatedRoPE::precalculate(&self.rope_config, &rope_positions, encoder)
- .map_err(WeaverEncodeError::Backend)?;
-
- // Prefix pass: run the target token and one draft row per non-root
- // chain node through every layer, collecting per-layer KV caches.
let hidden_row_bytes = size_for_shape(&[self.target_model_dim], DATA_TYPE);
let mut prefix_hidden = encoder
- .allocate_scratch(size_for_shape(&[shape.depth, self.target_model_dim], DATA_TYPE))
+ .allocate_scratch(size_for_shape(&[depth, self.target_model_dim], DATA_TYPE))
.map_err(WeaverEncodeError::Backend)?;
encoder.encode_copy(target_hidden, 0..hidden_row_bytes, &mut prefix_hidden, 0..hidden_row_bytes);
encoder.encode_copy(
draft_hidden,
- hidden_row_bytes..shape.depth as usize * hidden_row_bytes,
+ hidden_row_bytes..depth as usize * hidden_row_bytes,
&mut prefix_hidden,
- hidden_row_bytes..shape.depth as usize * hidden_row_bytes,
+ hidden_row_bytes..depth as usize * hidden_row_bytes,
);
let normalized_prefix = self
.hidden_state_norm
- .encode(&prefix_hidden, 0, shape.depth, None, encoder)
+ .encode(&prefix_hidden, 0, depth, None, encoder)
.map_err(WeaverEncodeError::Backend)?;
let mut residual_input = self
.hidden_state_projection
- .encode(normalized_prefix, shape.depth, encoder)
+ .encode(normalized_prefix, depth, encoder)
.map_err(WeaverEncodeError::Backend)?;
let (last_layer, preceding_layers) = self.layers.split_last().expect("Weaver must have at least one layer");
let mut residual_state = encoder.allocate_scratch(residual_input.size()).map_err(WeaverEncodeError::Backend)?;
@@ -343,12 +323,12 @@ impl<B: Backend> Weaver<B> {
queries,
kv_cache,
} = layer
- .encode_prefix_attention(&residual_input, &mut residual_state, &rope, shape.depth, encoder)
+ .encode_prefix_attention(&residual_input, &mut residual_state, rope, depth, encoder)
.map_err(WeaverEncodeError::Backend)?;
let state_type = AttentionStateType::Full {
length: 0,
};
- let kv_plane_bytes = size_for_shape(&[shape.depth, self.model_dim], DATA_TYPE);
+ let kv_plane_bytes = size_for_shape(&[depth, self.model_dim], DATA_TYPE);
let attention_output = layer
.prefix_attention
.encode(
@@ -356,7 +336,7 @@ impl<B: Backend> Weaver<B> {
queries: &queries,
keys: &kv_cache,
values: (&kv_cache, kv_plane_bytes),
- suffix_length: shape.depth,
+ suffix_length: depth,
trie: None,
sinks: None,
state_type: &state_type,
@@ -365,17 +345,245 @@ impl<B: Backend> Weaver<B> {
)
.map_err(WeaverEncodeError::Backend)?;
residual_input = layer
- .encode_post_attention(attention_output, &mut residual_state, shape.depth, encoder)
+ .encode_post_attention(attention_output, &mut residual_state, depth, encoder)
.map_err(WeaverEncodeError::Backend)?;
prefix_kv_layers.push(kv_cache);
}
prefix_kv_layers.push(
last_layer
- .encode_prefix_attention(&residual_input, &mut residual_state, &rope, shape.depth, encoder)
+ .encode_prefix_attention(&residual_input, &mut residual_state, rope, depth, encoder)
.map_err(WeaverEncodeError::Backend)?
.kv_cache,
);
+ encoder.pop_debug_group();
+
+ Ok(prefix_kv_layers)
+ }
+
+ fn encode_step(
+ &self,
+ target_embedding: &Embedding<B>,
+ prefix_kv_layers: &[Allocation<B>],
+ rope: &PrecalculatedRoPE<B>,
+ node_kv_layers: &mut [Allocation<B>],
+ packed_tree: &mut Allocation<B>,
+ frontier: &mut Allocation<B>,
+ slot_ancestors: &mut Allocation<B>,
+ node_token_ids: &mut Allocation<B>,
+ node_metadata: &mut Allocation<B>,
+ node_ancestor_indices: &mut Allocation<B>,
+ node_valid: &mut Allocation<B>,
+ node_candidate_ids: &mut Allocation<B>,
+ node_candidate_logits: &mut Allocation<B>,
+ depth_seeds_buffer: &Allocation<B>,
+ candidate_ids: &Allocation<B>,
+ candidate_logits: &Allocation<B>,
+ shape: &WeaverTreeShape,
+ batch_node_count: u32,
+ batch_start_slot: u32,
+ encoder: &mut Encoder<B>,
+ ) -> Result<(), WeaverEncodeError<B>> {
+ let tree_slot_count = shape.slot_count();
+ let ancestor_stride = self.max_depth;
+ let frontier_capacity = tree_slot_count * shape.expand_width;
+
+ if batch_start_slot > 0 {
+ self.frontier_select.encode(
+ &mut *frontier,
+ &mut *packed_tree,
+ &mut *slot_ancestors,
+ &mut *node_token_ids,
+ &mut *node_metadata,
+ &mut *node_ancestor_indices,
+ &mut *node_valid,
+ candidate_ids,
+ candidate_logits,
+ &mut *node_candidate_ids,
+ &mut *node_candidate_logits,
+ frontier_capacity,
+ tree_slot_count,
+ batch_node_count,
+ batch_start_slot,
+ ancestor_stride,
+ self.max_depth,
+ shape.max_depth - 1,
+ shape.dflash_depth - 1,
+ self.candidate_pool_size,
+ encoder,
+ );
+ }
+ let (batch_candidate_ids, batch_candidate_logits) = if batch_start_slot == 0 {
+ (candidate_ids, candidate_logits)
+ } else {
+ (&*node_candidate_ids, &*node_candidate_logits)
+ };
+
+ // Node expansion: embed the batch's tokens, run every layer against
+ // the prefix KV and each node's ancestors, then pick its children.
+ let token_embedding = target_embedding.encode_lookup(&*node_token_ids, batch_node_count, encoder)?;
+ let normalized_embedding = self
+ .token_embedding_norm
+ .encode(&token_embedding, 0, batch_node_count, None, encoder)
+ .map_err(WeaverEncodeError::Backend)?;
+ let mut residual_input = self
+ .token_embedding_projection
+ .encode(normalized_embedding, batch_node_count, encoder)
+ .map_err(WeaverEncodeError::Backend)?;
+ let mut residual_state = encoder.allocate_scratch(residual_input.size()).map_err(WeaverEncodeError::Backend)?;
+ let metadata_field_bytes = size_for_shape(&[batch_node_count], DataType::U32);
+ for (layer_index, layer) in self.layers.iter().enumerate() {
+ let attention_input = layer
+ .pre_attention_norm
+ .encode(&residual_input, 0, batch_node_count, Some(&mut residual_state), encoder)
+ .map_err(WeaverEncodeError::Backend)?;
+ let current_qkv = layer
+ .qkv_projection
+ .encode(attention_input, batch_node_count, encoder)
+ .map_err(WeaverEncodeError::Backend)?;
+ let mut attention_output = encoder
+ .allocate_scratch(size_for_shape(&[batch_node_count, self.model_dim], DATA_TYPE))
+ .map_err(WeaverEncodeError::Backend)?;
+ layer.ancestor_attention.encode(
+ &prefix_kv_layers[layer_index],
+ &mut node_kv_layers[layer_index],
+ &current_qkv,
+ &rope.cosines,
+ &rope.sines,
+ &*node_metadata,
+ &*node_ancestor_indices,
+ (&*node_metadata, MetadataIdx::AncestorCount as usize * metadata_field_bytes),
+ (&*node_metadata, MetadataIdx::TreeSlot as usize * metadata_field_bytes),
+ &mut attention_output,
+ batch_node_count,
+ shape.dflash_depth,
+ ancestor_stride,
+ tree_slot_count,
+ layer.max_depth,
+ layer.attention_scale,
+ encoder,
+ );
+ residual_input = layer
+ .encode_post_attention(attention_output, &mut residual_state, batch_node_count, encoder)
+ .map_err(WeaverEncodeError::Backend)?;
+ }
+
+ let normalized_output = self
+ .readout_norm
+ .encode(&residual_input, 0, batch_node_count, Some(&mut residual_state), encoder)
+ .map_err(WeaverEncodeError::Backend)?;
+ let query = self
+ .readout_query_projection
+ .encode(normalized_output, batch_node_count, encoder)
+ .map_err(WeaverEncodeError::Backend)?;
+ let logit_residuals = target_embedding.encode_readout_sparse(
+ &query,
+ batch_candidate_ids,
+ batch_node_count,
+ self.candidate_pool_size,
+ encoder,
+ )?;
+ let mut child_token_ids = encoder
+ .allocate_scratch(size_for_shape(&[batch_node_count, shape.expand_width], DataType::U32))
+ .map_err(WeaverEncodeError::Backend)?;
+ let mut child_logprobs = encoder
+ .allocate_scratch(size_for_shape(&[batch_node_count, shape.expand_width], DataType::F32))
+ .map_err(WeaverEncodeError::Backend)?;
+ self.top_children.encode(
+ &logit_residuals,
+ batch_candidate_logits,
+ batch_candidate_ids,
+ depth_seeds_buffer,
+ &*node_metadata,
+ &mut child_token_ids,
+ &mut child_logprobs,
+ batch_node_count,
+ self.candidate_pool_size,
+ shape.expand_width,
+ target_embedding.vocab_size(),
+ encoder,
+ );
+
+ self.frontier_insert_children.encode(
+ &*packed_tree,
+ &*node_metadata,
+ &*node_valid,
+ &child_token_ids,
+ &child_logprobs,
+ frontier,
+ frontier_capacity,
+ tree_slot_count,
+ batch_node_count,
+ shape.expand_width,
+ encoder,
+ );
+
+ Ok(())
+ }
+
+ pub fn encode_tree(
+ &self,
+ target_hidden: &Allocation<B>,
+ draft_hidden: &Allocation<B>,
+ target_embedding: &Embedding<B>,
+ logits: &Allocation<B>,
+ depth_seeds: &[u64],
+ root_token_id: u32,
+ shape: WeaverTreeShape,
+ encoder: &mut Encoder<B>,
+ ) -> Result<EncodedWeaverTree<B>, WeaverEncodeError<B>> {
+ encoder.push_debug_group("weaver tree");
+
+ let tree_slot_count = shape.slot_count();
+ let ancestor_stride = self.max_depth;
+ if shape.tree_budget == 0
+ || shape.rounds == 0
+ || shape.max_depth < 2
+ || shape.max_depth > self.max_depth + 1
+ || shape.dflash_depth < shape.max_depth
+ || shape.dflash_depth > self.max_depth + 1
+ || shape.expand_per_round == 0
+ || shape.expand_per_round > FRONTIER_MAX_WIDTH
+ || shape.expand_width == 0
+ || shape.expand_width > self.candidate_pool_size
+ || tree_slot_count > FRONTIER_MAX_SLOTS / shape.expand_width
+ || depth_seeds.len() as u32 != self.max_depth
+ {
+ return Err(WeaverEncodeError::InvalidTreeInput);
+ }
+ let frontier_capacity = tree_slot_count * shape.expand_width;
+ let pool_depth_count = shape.dflash_depth - 1;
+
+ // Rank the draft logits: the top `candidate_pool_size` tokens per
+ // lookahead row form the candidate pool node expansions draw from.
+ let vocab_size = target_embedding.vocab_size();
+ assert!(
+ logits.size() >= size_for_shape(&[pool_depth_count, vocab_size], DataType::F32),
+ "draft logits do not cover the lookahead rows"
+ );
+ let mut candidate_ids = encoder
+ .allocate_scratch(size_for_shape(&[pool_depth_count, self.candidate_pool_size], DataType::U32))
+ .map_err(WeaverEncodeError::Backend)?;
+ let mut candidate_logits = encoder
+ .allocate_scratch(size_for_shape(&[pool_depth_count, self.candidate_pool_size], DataType::F32))
+ .map_err(WeaverEncodeError::Backend)?;
+ self.top_k
+ .encode(
+ logits,
+ &mut candidate_ids,
+ &mut candidate_logits,
+ pool_depth_count,
+ self.candidate_pool_size,
+ encoder,
+ )
+ .map_err(WeaverEncodeError::Backend)?;
+
+ let rope_positions = (0..=self.max_depth).collect::<Box<[_]>>();
+ let rope = PrecalculatedRoPE::precalculate(&self.rope_config, &rope_positions, encoder)
+ .map_err(WeaverEncodeError::Backend)?;
+
+ let prefix_kv_layers = self.encode_prefix(target_hidden, draft_hidden, &rope, shape.dflash_depth, encoder)?;
+
// Per-layer KV cache for tree nodes, one slot per packed-tree slot.
let node_kv_size = size_for_shape(&[2, tree_slot_count, self.model_dim], DATA_TYPE);
let mut node_kv_layers = (0..self.layers.len())
@@ -425,145 +633,36 @@ impl<B: Backend> Weaver<B> {
encoder.allocate_constant_from_slice(depth_seeds).map_err(WeaverEncodeError::Backend)?;
let mut batch_start_slot = 0;
- while batch_start_slot < tree_slot_count {
- let batch_node_count = if batch_start_slot == 0 {
+ for round in 0..shape.rounds {
+ let batch_node_count = if round == 0 {
1
} else {
- shape.expand_per_round.min(tree_slot_count - batch_start_slot)
+ shape.expand_per_round
};
- if batch_start_slot > 0 {
- self.frontier_select.encode(
- &mut frontier,
- &mut packed_tree,
- &mut slot_ancestors,
- &mut node_token_ids,
- &mut node_metadata,
- &mut node_ancestor_indices,
- &mut node_valid,
- &candidate_ids,
- &candidate_logits,
- &mut node_candidate_ids,
- &mut node_candidate_logits,
- frontier_capacity,
- tree_slot_count,
- batch_node_count,
- batch_start_slot,
- ancestor_stride,
- self.max_depth,
- lookahead_count,
- lookahead_count,
- self.candidate_pool_size,
- encoder,
- );
- }
- if batch_start_slot + batch_node_count == tree_slot_count {
- break; // packed-tree slots filled
- }
- let (batch_candidate_ids, batch_candidate_logits) = if batch_start_slot == 0 {
- (&candidate_ids, &candidate_logits)
- } else {
- (&node_candidate_ids, &node_candidate_logits)
- };
-
- // Node expansion: embed the batch's tokens, run every layer against
- // the prefix KV and each node's ancestors, then pick its children.
- let token_embedding = target_embedding.encode_lookup(&node_token_ids, batch_node_count, encoder)?;
- let normalized_embedding = self
- .token_embedding_norm
- .encode(&token_embedding, 0, batch_node_count, None, encoder)
- .map_err(WeaverEncodeError::Backend)?;
- let mut residual_input = self
- .token_embedding_projection
- .encode(normalized_embedding, batch_node_count, encoder)
- .map_err(WeaverEncodeError::Backend)?;
- let mut residual_state =
- encoder.allocate_scratch(residual_input.size()).map_err(WeaverEncodeError::Backend)?;
- let metadata_field_bytes = size_for_shape(&[batch_node_count], DataType::U32);
- for (layer_index, layer) in self.layers.iter().enumerate() {
- let attention_input = layer
- .pre_attention_norm
- .encode(&residual_input, 0, batch_node_count, Some(&mut residual_state), encoder)
- .map_err(WeaverEncodeError::Backend)?;
- let current_qkv = layer
- .qkv_projection
- .encode(attention_input, batch_node_count, encoder)
- .map_err(WeaverEncodeError::Backend)?;
- let mut attention_output = encoder
- .allocate_scratch(size_for_shape(&[batch_node_count, self.model_dim], DATA_TYPE))
- .map_err(WeaverEncodeError::Backend)?;
- layer.ancestor_attention.encode(
- &prefix_kv_layers[layer_index],
- &mut node_kv_layers[layer_index],
- &current_qkv,
- &rope.cosines,
- &rope.sines,
- &node_metadata,
- &node_ancestor_indices,
- (&node_metadata, MetadataIdx::AncestorCount as usize * metadata_field_bytes),
- (&node_metadata, MetadataIdx::TreeSlot as usize * metadata_field_bytes),
- &mut attention_output,
- batch_node_count,
- shape.depth,
- ancestor_stride,
- tree_slot_count,
- layer.max_depth,
- layer.attention_scale,
- encoder,
- );
- residual_input = layer
- .encode_post_attention(attention_output, &mut residual_state, batch_node_count, encoder)
- .map_err(WeaverEncodeError::Backend)?;
- }
-
- let normalized_output = self
- .readout_norm
- .encode(&residual_input, 0, batch_node_count, Some(&mut residual_state), encoder)
- .map_err(WeaverEncodeError::Backend)?;
- let query = self
- .readout_query_projection
- .encode(normalized_output, batch_node_count, encoder)
- .map_err(WeaverEncodeError::Backend)?;
- let logit_residuals = target_embedding.encode_readout_sparse(
- &query,
- batch_candidate_ids,
- batch_node_count,
- self.candidate_pool_size,
- encoder,
- )?;
- let mut child_token_ids = encoder
- .allocate_scratch(size_for_shape(&[batch_node_count, shape.expand_width], DataType::U32))
- .map_err(WeaverEncodeError::Backend)?;
- let mut child_logprobs = encoder
- .allocate_scratch(size_for_shape(&[batch_node_count, shape.expand_width], DataType::F32))
- .map_err(WeaverEncodeError::Backend)?;
- self.top_children.encode(
- &logit_residuals,
- batch_candidate_logits,
- batch_candidate_ids,
- &depth_seeds_buffer,
- &node_metadata,
- &mut child_token_ids,
- &mut child_logprobs,
- batch_node_count,
- self.candidate_pool_size,
- shape.expand_width,
- target_embedding.vocab_size(),
- encoder,
- );
-
- self.frontier_insert_children.encode(
- &packed_tree,
- &node_metadata,
- &node_valid,
- &child_token_ids,
- &child_logprobs,
+ encoder.push_debug_group("weaver step");
+ self.encode_step(
+ target_embedding,
+ &prefix_kv_layers,
+ &rope,
+ &mut node_kv_layers,
+ &mut packed_tree,
&mut frontier,
- frontier_capacity,
- tree_slot_count,
+ &mut slot_ancestors,
+ &mut node_token_ids,
+ &mut node_metadata,
+ &mut node_ancestor_indices,
+ &mut node_valid,
+ &mut node_candidate_ids,
+ &mut node_candidate_logits,
+ &depth_seeds_buffer,
+ &candidate_ids,
+ &candidate_logits,
+ &shape,
batch_node_count,
- shape.expand_width,
+ batch_start_slot,
encoder,
- );
+ )?;
+ encoder.pop_debug_group();
batch_start_slot += batch_node_count;
}
@@ -571,6 +670,7 @@ impl<B: Backend> Weaver<B> {
Ok(EncodedWeaverTree {
packed_tree,
+ frontier,
})
}
}
diff --git a/crates/backend-uzu/src/encodable_block/weaver_layer.rs b/crates/backend-uzu/src/encodable_block/weaver_layer.rs
index 139a57834..dacfd8d11 100644
--- a/crates/backend-uzu/src/encodable_block/weaver_layer.rs
+++ b/crates/backend-uzu/src/encodable_block/weaver_layer.rs
@@ -28,23 +28,18 @@ pub struct PreparedPrefixAttention<B: Backend> {
}
pub struct WeaverLayer<B: Backend> {
- // Attention
pub pre_attention_norm: Normalization<B>,
pub qkv_projection: Box<dyn Linear<B>>,
- pub attention_prepare: <B::Kernels as Kernels>::AttentionPrepareKernel,
+ attention_prepare: <B::Kernels as Kernels>::AttentionPrepareKernel,
pub prefix_attention: AttentionCores<B>,
pub ancestor_attention: <B::Kernels as Kernels>::AncestorAttentionKernel,
- pub out_projection: Box<dyn Linear<B>>,
-
- // MLP
- pub pre_mlp_norm: Normalization<B>,
- pub mlp: Box<dyn Mlp<B>>,
-
- // Geometry
+ out_projection: Box<dyn Linear<B>>,
+ pre_mlp_norm: Normalization<B>,
+ mlp: Box<dyn Mlp<B>>,
pub attention_scale: f32,
- pub model_dim: u32,
- pub num_heads: u32,
- pub head_dim: u32,
+ model_dim: u32,
+ num_heads: u32,
+ head_dim: u32,
pub max_depth: u32,
}
diff --git a/crates/backend-uzu/src/engine/language_model/stream/stream.rs b/crates/backend-uzu/src/engine/language_model/stream/stream.rs
index 2bad7060b..1f85bf6db 100644
--- a/crates/backend-uzu/src/engine/language_model/stream/stream.rs
+++ b/crates/backend-uzu/src/engine/language_model/stream/stream.rs
@@ -334,7 +334,7 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {
metrics.num_tokens_accepted += 1;
DecodingState::ForwardPassPending(DecodingStatePending {
- input_trie: TrieNode::new(0, 0),
+ input_trie: TrieNode::new(0, 0, 0.0),
full_accept: true,
pending,
capture_span,
@@ -547,13 +547,7 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {
None
};
- let full_batch_size = self.model.speculator.as_ref().map_or(1, |speculator| {
- if speculator.has_weaver() {
- 32
- } else {
- 16
- }
- });
+ let full_batch_size = 16;
let speculation_batch = self
.model_state
.max_context_length
@@ -576,10 +570,12 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {
root_token as u32,
self.model.decoder.embedding(),
DFlashTfmTreeShape {
- budget: speculation_batch,
+ tree_budget: speculation_batch,
+ max_tree_depth: 16,
+ dflash_depth_override: None,
construction_method: if speculator.has_weaver() {
DFlashTfmTreeConstructionMethod::Weaver {
- depth: 16,
+ rounds: 16,
expand_per_round: 4,
expand_width: 4,
}
@@ -601,7 +597,7 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {
} => (*token, None),
ForwardPassChaining::InFlight(pending) => (0, Some(&pending.output_tokens)),
};
- (TrieNode::new(token, self.model_state.prng.derive(context_length as u64)), chain_copy, true)
+ (TrieNode::new(token, self.model_state.prng.derive(context_length as u64), 0.0), chain_copy, true)
};
let input_flat_trie = input_trie.linearize();
diff --git a/crates/backend-uzu/src/speculators/dflash_tfm.rs b/crates/backend-uzu/src/speculators/dflash_tfm.rs
index 1d9a0ae92..9ee920a2a 100644
--- a/crates/backend-uzu/src/speculators/dflash_tfm.rs
+++ b/crates/backend-uzu/src/speculators/dflash_tfm.rs
@@ -56,14 +56,16 @@ pub enum DFlashSpeculatorLoadError<B: Backend> {
pub enum DFlashTfmTreeConstructionMethod {
Argmax,
Weaver {
- depth: u32,
+ rounds: u32,
expand_per_round: u32,
expand_width: u32,
},
}
pub struct DFlashTfmTreeShape {
- pub budget: u32,
+ pub tree_budget: u32,
+ pub max_tree_depth: u32,
+ pub dflash_depth_override: Option<u32>,
pub construction_method: DFlashTfmTreeConstructionMethod,
}
@@ -153,7 +155,15 @@ impl<B: Backend> DFlashTfmSpeculator<B> {
prng: &PRng,
allocation_pool: Arc<AllocationPool<B>>,
) -> Result<TrieNode, DFlashTreeError<B>> {
- assert!(shape.budget >= 2, "tree budget needs at least a root and one draft token");
+ assert!(shape.tree_budget >= 2, "tree budget needs at least a root and one draft token");
+
+ let block_size = self.dflash.block_size();
+ let dflash_depth = shape.dflash_depth_override.unwrap_or(block_size);
+ if !(2..=block_size).contains(&dflash_depth) {
+ return Err(DFlashTreeError::InvalidTreeShape(format!(
+ "dflash depth {dflash_depth} is outside 2..={block_size}"
+ )));
+ }
let root_position = state.context_length();
@@ -162,26 +172,27 @@ impl<B: Backend> DFlashTfmSpeculator<B> {
let nodes = match shape.construction_method {
DFlashTfmTreeConstructionMethod::Argmax => {
- if shape.budget > self.dflash.block_size() {
+ if shape.tree_budget > dflash_depth {
return Err(DFlashTreeError::InvalidTreeShape(format!(
- "argmax chain of {} nodes needs {} draft rows, block size is {}",
- shape.budget,
- shape.budget - 1,
- self.dflash.block_size()
+ "argmax chain of {} nodes needs {} draft rows, dflash depth is {}",
+ shape.tree_budget,
+ shape.tree_budget - 1,
+ dflash_depth
)));
}
- let chain_length = shape.budget - 1;
- let mut nodes = Vec::with_capacity(shape.budget as usize);
+ let chain_length = shape.tree_budget - 1;
+ let mut nodes = Vec::with_capacity(shape.tree_budget as usize);
nodes.push(ProposalNode {
token_id: target_output_token,
depth: 0,
+ logprob: 0.0,
child_indices: vec![1],
});
let dflash_output = self.dflash.encode_draft(
state,
target_output_token,
target_embedding,
- shape.budget,
+ dflash_depth,
&mut encoder,
)?;
let topology_nodes = (0..chain_length)
@@ -213,6 +224,7 @@ impl<B: Backend> DFlashTfmSpeculator<B> {
nodes.extend(tokens.into_iter().zip(1u32..).map(|(token_id, depth)| ProposalNode {
token_id,
depth,
+ logprob: 0.0,
child_indices: if depth < chain_length {
vec![depth as usize + 1]
} else {
@@ -222,33 +234,35 @@ impl<B: Backend> DFlashTfmSpeculator<B> {
nodes
},
DFlashTfmTreeConstructionMethod::Weaver {
- depth,
+ rounds,
expand_per_round,
expand_width,
} => {
let weaver =
self.weaver.as_ref().expect("weaver tree construction requires a speculator with weaver weights");
- if depth < 2 {
- return Err(DFlashTreeError::InvalidTreeShape(format!(
- "tree depth {depth} needs at least a root and one level"
- )));
- }
- // `depth` counts the root; the weaver's `max_depth` counts edges.
- if depth > weaver.max_depth() + 1 {
+ // `max_depth` counts the root; the weaver's `max_depth` counts edges.
+ if shape.max_tree_depth < 2 || shape.max_tree_depth > weaver.max_depth() + 1 {
return Err(DFlashTreeError::InvalidTreeShape(format!(
- "requested tree depth {depth} exceeds weaver max depth {}",
+ "tree max_depth {} is outside 2..={}",
+ shape.max_tree_depth,
weaver.max_depth() + 1
)));
}
- if depth > self.dflash.block_size() {
+ if shape.max_tree_depth > dflash_depth {
return Err(DFlashTreeError::InvalidTreeShape(format!(
- "tree of depth {depth} needs {} draft rows, block size is {}",
- depth - 1,
- self.dflash.block_size()
+ "tree of max_depth {} needs {} draft rows, dflash depth is {}",
+ shape.max_tree_depth,
+ shape.max_tree_depth - 1,
+ dflash_depth
)));
}
- let dflash_output =
- self.dflash.encode_draft(state, target_output_token, target_embedding, depth, &mut encoder)?;
+ let dflash_output = self.dflash.encode_draft(
+ state,
+ target_output_token,
+ target_embedding,
+ dflash_depth,
+ &mut encoder,
+ )?;
let depth_seeds = (0..weaver.max_depth())
.map(|depth| prng.derive(root_position as u64 + depth as u64))
.collect::<Box<[u64]>>();
@@ -260,8 +274,10 @@ impl<B: Backend> DFlashTfmSpeculator<B> {
&depth_seeds,
target_output_token,
WeaverTreeShape {
- budget: shape.budget,
- depth,
+ tree_budget: shape.tree_budget,
+ max_depth: shape.max_tree_depth,
+ dflash_depth,
+ rounds,
expand_per_round,
expand_width,
},
@@ -283,8 +299,11 @@ impl<B: Backend> DFlashTfmSpeculator<B> {
prng: &PRng,
) -> TrieNode {
let node = &nodes[index];
- let mut trie_node =
- TrieNode::new(node.token_id as u64, prng.derive(root_position as u64 + node.depth as u64));
+ let mut trie_node = TrieNode::new(
+ node.token_id as u64,
+ prng.derive(root_position as u64 + node.depth as u64),
+ node.logprob,
+ );
for &child_index in &node.child_indices {
#[cfg(grammar)]
if let Some(grammar) = grammar.as_mut()
@@ -312,13 +331,15 @@ impl<B: Backend> DFlashTfmSpeculator<B> {
trie_node
}
- Ok(recursive_build(
+ let mut trie = recursive_build(
&nodes,
0,
root_position,
#[cfg(grammar)]
grammar,
prng,
- ))
+ );
+ trie.prune_to_budget(shape.tree_budget as usize);
+ Ok(trie)
}
}
diff --git a/crates/backend-uzu/src/trie.rs b/crates/backend-uzu/src/trie.rs
index f56c26b8e..5c1086f60 100644
--- a/crates/backend-uzu/src/trie.rs
+++ b/crates/backend-uzu/src/trie.rs
@@ -26,6 +26,7 @@ pub enum TrieAcceptError {
pub struct TrieNode {
token: u64,
seed: u64,
+ logprob: f32,
next: Vec<TrieNode>,
}
@@ -45,10 +46,12 @@ impl TrieNode {
pub fn new(
token: u64,
seed: u64,
+ logprob: f32,
) -> Self {
Self {
token,
seed,
+ logprob,
next: Vec::new(),
}
}
@@ -77,6 +80,63 @@ impl TrieNode {
self.token
}
+ #[cfg(test)]
+ pub fn logprob(&self) -> f32 {
+ self.logprob
+ }
+
+ #[cfg(test)]
+ pub fn node_count(&self) -> usize {
+ 1 + self.next.iter().map(TrieNode::node_count).sum::<usize>()
+ }
+
+ pub fn prune_to_budget(
+ &mut self,
+ budget: usize,
+ ) {
+ assert!(budget > 0, "budget must keep at least the root");
+
+ fn collect_logprobs(
+ node: &TrieNode,
+ parent_logprob: f32,
+ logprobs: &mut Vec<f32>,
+ ) {
+ let logprob = parent_logprob + node.logprob;
+ logprobs.push(logprob);
+ for child in &node.next {
+ collect_logprobs(child, logprob, logprobs);
+ }
+ }
+ let mut logprobs = Vec::new();
+ collect_logprobs(self, 0.0, &mut logprobs);
+ if budget >= logprobs.len() {
+ return;
+ }
+
+ let mut order: Box<[usize]> = (0..logprobs.len()).collect();
+ order.sort_by(|&a, &b| logprobs[b].total_cmp(&logprobs[a]));
+ let mut kept = vec![false; logprobs.len()];
+ for &index in order.iter().take(budget) {
+ kept[index] = true;
+ }
+
+ fn prune(
+ node: &mut TrieNode,
+ kept: &[bool],
+ cursor: &mut usize,
+ ) {
+ *cursor += 1;
+ let mut children = std::mem::take(&mut node.next);
+ children.retain_mut(|child| {
+ let child_index = *cursor;
+ prune(child, kept, cursor);
+ kept[child_index]
+ });
+ node.next = children;
+ }
+ prune(self, &kept, &mut 0);
+ }
+
pub fn flat(
prefix_length: usize,
tokens: &[u64],
@@ -84,11 +144,11 @@ impl TrieNode {
) -> Self {
assert!(!tokens.is_empty(), "need seed node");
- let mut root = TrieNode::new(tokens[0], prng.derive(prefix_length as u64));
+ let mut root = TrieNode::new(tokens[0], prng.derive(prefix_length as u64), 0.0);
let mut leaf = &mut root;
for (index, token) in tokens.iter().copied().enumerate().skip(1) {
- leaf.add(TrieNode::new(token, prng.derive((prefix_length + index) as u64))).unwrap();
+ leaf.add(TrieNode::new(token, prng.derive((prefix_length + index) as u64), 0.0)).unwrap();
leaf = &mut leaf.next[0];
}
diff --git a/crates/backend-uzu/tests/unit/backends/common/kernel/attention/ancestor_attention_test.rs b/crates/backend-uzu/tests/unit/backends/common/kernel/attention/ancestor_attention_test.rs
index 3f8703407..b3d4e42d7 100644
--- a/crates/backend-uzu/tests/unit/backends/common/kernel/attention/ancestor_attention_test.rs
+++ b/crates/backend-uzu/tests/unit/backends/common/kernel/attention/ancestor_attention_test.rs
@@ -84,7 +84,7 @@ impl<B: Backend> Runner<B> {
}
let mut node_metadata = vec![0u32; rows * MetadataIdx::COUNT];
for row in 0..rows {
- node_metadata[MetadataIdx::Depth as usize * rows + row] = (row as u32 * 3) % (MAX_DEPTH + 2);
+ node_metadata[MetadataIdx::Depth as usize * rows + row] = (row as u32 * 3) % MAX_DEPTH;
}
let context = create_context::<B>();
diff --git a/crates/backend-uzu/tests/unit/backends/common/kernel/weaver/weaver_frontier_test.rs b/crates/backend-uzu/tests/unit/backends/common/kernel/weaver/weaver_frontier_test.rs
index bce06ae78..259fccd9c 100644
--- a/crates/backend-uzu/tests/unit/backends/common/kernel/weaver/weaver_frontier_test.rs
+++ b/crates/backend-uzu/tests/unit/backends/common/kernel/weaver/weaver_frontier_test.rs
@@ -20,7 +20,7 @@ fn select<B: Backend>() -> Vec<u32> {
(8, 0, 2, 0x3f00_0001, 100, 1),
(7, 0, 2, 0x3f00_0002, 100, 1),
(7, 0, 2, 0x3f00_0003, 100, 1),
- (2, 1, 3, 0x3f00_0004, 90, 1),
+ (2, 1, 3, 0x3f00_0004, 300, 1),
(0, 0, 0, 0x3f00_0005, 200, 0),
(4, 1, 3, 0x3f00_0006, 80, 1),
(5, 1, 1, 0x3f00_0007, 70, 1),
diff --git a/crates/backend-uzu/tests/unit/trie_test.rs b/crates/backend-uzu/tests/unit/trie_test.rs
index 335cfde35..25f5fb53d 100644
--- a/crates/backend-uzu/tests/unit/trie_test.rs
+++ b/crates/backend-uzu/tests/unit/trie_test.rs
@@ -10,9 +10,9 @@ fn verify_sprout(
assert_eq!(flat_trie.len(), 1);
assert_eq!(flat_trie.index(trie_root), Some(0));
- assert_eq!(flat_trie.index(&TrieNode::new(1, 0)), None);
- assert_eq!(flat_trie.index(&TrieNode::new(0, 1)), None);
- assert_eq!(flat_trie.index(&TrieNode::new(0, 0)), None);
+ assert_eq!(flat_trie.index(&TrieNode::new(1, 0, 0.0)), None);
+ assert_eq!(flat_trie.index(&TrieNode::new(0, 1, 0.0)), None);
+ assert_eq!(flat_trie.index(&TrieNode::new(0, 0, 0.0)), None);
assert_eq!(flat_trie.token_ids().collect::<Vec<u64>>(), vec![0]);
assert_eq!(flat_trie.token_subtrie_ranges().map(|node| node.height as usize).collect::<Vec<_>>(), vec![0]);
assert_eq!(flat_trie.token_seeds().collect::<Vec<u64>>(), vec![expected_seed]);
@@ -20,7 +20,7 @@ fn verify_sprout(
#[uzu_test]
fn test_trie_manual_sprout() {
- let trie_root = TrieNode::new(0, 0);
+ let trie_root = TrieNode::new(0, 0, 0.0);
verify_sprout(&trie_root, 0);
}
@@ -62,13 +62,13 @@ fn verify_stick(
fn test_trie_manual_stick() {
let rng = PRng::new(0);
- let mut trie_stick = TrieNode::new(9, rng.derive(9));
+ let mut trie_stick = TrieNode::new(9, rng.derive(9), 0.0);
for i in (1..9u64).rev() {
- let mut trie_parent = TrieNode::new(i, rng.derive(i));
+ let mut trie_parent = TrieNode::new(i, rng.derive(i), 0.0);
trie_parent.add(trie_stick).unwrap();
trie_stick = trie_parent;
}
- let mut trie_root = TrieNode::new(0, rng.derive(0));
+ let mut trie_root = TrieNode::new(0, rng.derive(0), 0.0);
trie_root.add(trie_stick).unwrap();
verify_stick(&trie_root, &rng);
@@ -108,14 +108,14 @@ fn verify_bush(
#[uzu_test]
fn test_trie_manual_bush() {
let rng = PRng::new(0);
- let mut trie_root = TrieNode::new(0, rng.derive(0));
+ let mut trie_root = TrieNode::new(0, rng.derive(0), 0.0);
- assert!(trie_root.add(TrieNode::new(1, rng.derive(1))).is_ok());
- assert!(trie_root.add(TrieNode::new(1, rng.derive(1))).is_err());
- assert!(trie_root.add(TrieNode::new(1, 10)).is_err());
+ assert!(trie_root.add(TrieNode::new(1, rng.derive(1), 0.0)).is_ok());
+ assert!(trie_root.add(TrieNode::new(1, rng.derive(1), 0.0)).is_err());
+ assert!(trie_root.add(TrieNode::new(1, 10, 0.0)).is_err());
- assert!(trie_root.add(TrieNode::new(2, rng.derive(1))).is_ok());
- assert!(trie_root.add(TrieNode::new(3, rng.derive(1))).is_ok());
+ assert!(trie_root.add(TrieNode::new(2, rng.derive(1), 0.0)).is_ok());
+ assert!(trie_root.add(TrieNode::new(3, rng.derive(1), 0.0)).is_ok());
verify_bush(&trie_root, &rng);
}
@@ -159,21 +159,75 @@ fn verify_tree(
#[uzu_test]
fn test_trie_manual_tree() {
let rng = PRng::new(0);
- let mut trie_root = TrieNode::new(0, rng.derive(0));
+ let mut trie_root = TrieNode::new(0, rng.derive(0), 0.0);
- assert!(trie_root.add(TrieNode::new(1, rng.derive(1))).is_ok());
- assert!(trie_root.add(TrieNode::new(1, rng.derive(1))).is_err());
- assert!(trie_root.add(TrieNode::new(1, 10)).is_err());
+ assert!(trie_root.add(TrieNode::new(1, rng.derive(1), 0.0)).is_ok());
+ assert!(trie_root.add(TrieNode::new(1, rng.derive(1), 0.0)).is_err());
+ assert!(trie_root.add(TrieNode::new(1, 10, 0.0)).is_err());
- let mut mid_b = TrieNode::new(2, rng.derive(1));
- assert!(mid_b.add(TrieNode::new(10, rng.derive(2))).is_ok());
+ let mut mid_b = TrieNode::new(2, rng.derive(1), 0.0);
+ assert!(mid_b.add(TrieNode::new(10, rng.derive(2), 0.0)).is_ok());
- let mut mid_c = TrieNode::new(3, rng.derive(1));
- assert!(mid_c.add(TrieNode::new(20, rng.derive(2))).is_ok());
- assert!(mid_c.add(TrieNode::new(21, rng.derive(2))).is_ok());
+ let mut mid_c = TrieNode::new(3, rng.derive(1), 0.0);
+ assert!(mid_c.add(TrieNode::new(20, rng.derive(2), 0.0)).is_ok());
+ assert!(mid_c.add(TrieNode::new(21, rng.derive(2), 0.0)).is_ok());
assert!(trie_root.add(mid_b).is_ok());
assert!(trie_root.add(mid_c).is_ok());
verify_tree(&trie_root, &rng)
}
+
+fn verify_pruned(
+ trie_root: &TrieNode,
+ expected_tokens: &[u64],
+) {
+ assert_eq!(trie_root.node_count(), expected_tokens.len());
+ assert_eq!(trie_root.linearize().token_ids().collect::<Vec<u64>>(), expected_tokens);
+}
+
+fn sample_tree() -> TrieNode {
+ let mut root = TrieNode::new(0, 0, 0.0);
+ let mut mid_a = TrieNode::new(1, 1, -0.1);
+ mid_a.add(TrieNode::new(4, 2, -0.4)).unwrap();
+ let mut mid_b = TrieNode::new(2, 1, -0.2);
+ mid_b.add(TrieNode::new(5, 2, -2.8)).unwrap();
+ root.add(mid_a).unwrap();
+ root.add(mid_b).unwrap();
+ root.add(TrieNode::new(3, 1, -0.3)).unwrap();
+ root
+}
+
+#[uzu_test]
+fn test_trie_prune_to_budget() {
+ let mut trie = sample_tree();
+ trie.prune_to_budget(4);
+ verify_pruned(&trie, &[0, 1, 2, 3]);
+ assert_eq!(trie.get(1).unwrap().logprob(), -0.1);
+ assert_eq!(trie.get(2).unwrap().logprob(), -0.2);
+ assert_eq!(trie.get(3).unwrap().logprob(), -0.3);
+
+ let mut trie = sample_tree();
+ trie.prune_to_budget(2);
+ verify_pruned(&trie, &[0, 1]);
+
+ let mut trie = sample_tree();
+ trie.prune_to_budget(6);
+ verify_pruned(&trie, &[0, 1, 4, 2, 5, 3]);
+
+ let mut trie = sample_tree();
+ trie.prune_to_budget(100);
+ verify_pruned(&trie, &[0, 1, 4, 2, 5, 3]);
+}
+
+#[uzu_test]
+fn test_trie_prune_to_budget_tie_keeps_parent() {
+ let mut root = TrieNode::new(0, 0, 0.0);
+ let mut child = TrieNode::new(1, 1, 0.0);
+ child.add(TrieNode::new(2, 2, 0.0)).unwrap();
+ root.add(child).unwrap();
+ root.add(TrieNode::new(3, 1, 0.0)).unwrap();
+
+ root.prune_to_budget(2);
+ verify_pruned(&root, &[0, 1]);
+}