Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

trie rewrite and guuumbel speculative decoding (#50)

uuuvnfcfdfa2ba3aa
diff --git a/Cargo.lock b/Cargo.lock
index b12c69953..19f001918 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -2648,6 +2648,7 @@ dependencies = [
"clap",
"half",
"is_close",
+ "itertools",
"libc",
"metal",
"minijinja",
diff --git a/crates/uzu/Cargo.toml b/crates/uzu/Cargo.toml
index 5becfbbac..a1c60081b 100644
--- a/crates/uzu/Cargo.toml
+++ b/crates/uzu/Cargo.toml
@@ -26,6 +26,7 @@ minijinja-contrib = { version = "2.12.0", features = ["pycompat"] }
clap = { version = "4.5.47", features = ["derive"] }
rand = "0.9.2"
sysinfo = "0.37.0"
+itertools = "0.14.0"
[dev-dependencies]
is_close = "0.1.3"
diff --git a/crates/uzu/src/backends/metal/forward_pass/state.rs b/crates/uzu/src/backends/metal/forward_pass/state.rs
index 8ddb134ee..6fb3f027d 100644
--- a/crates/uzu/src/backends/metal/forward_pass/state.rs
+++ b/crates/uzu/src/backends/metal/forward_pass/state.rs
@@ -937,9 +937,9 @@ impl ForwardPassState {
shared_buffers: Rc<RefCell<SharedBuffers>>,
token_ids: &[u64],
token_positions: &[usize],
+ token_seeds: &[u64],
active_suffix_length: usize,
is_prefilling: bool,
- token_seeds: &[u64],
trace: bool,
external_bias_fn: Option<&dyn Fn(usize, usize) -> bool>,
skip_token_ids_copy: bool,
diff --git a/crates/uzu/src/generator/generator.rs b/crates/uzu/src/generator/generator.rs
index 7b47ff0f6..1ada08fb6 100644
--- a/crates/uzu/src/generator/generator.rs
+++ b/crates/uzu/src/generator/generator.rs
@@ -1,6 +1,9 @@
-use std::{collections::HashMap, path::Path, sync::Arc, time::Instant};
+use std::{
+ collections::HashMap, iter::repeat_n, path::Path, sync::Arc, time::Instant,
+};
use block::ConcreteBlock;
+use itertools::izip;
use mpsgraph::CommandBuffer;
use super::{
@@ -15,12 +18,12 @@ use crate::{
ForwardPassState, INVALID_POSITION,
encodable_with_state::{EncodableWithState, EncodingParameters},
},
- linearizer::trie::{TokenTrie, TrieCreationConfig},
session::{
config::DecodingConfig,
parameter::{ConfigResolvableValue, SamplingMethod},
types::Error,
},
+ trie::{TrieCreationConfig, TrieNode},
utils::env_utils::MetalEnvVar,
};
@@ -73,86 +76,76 @@ impl Generator {
self.tokens.extend(tokens.clone());
+ let tokens_length = tokens.len();
+
let prefill_step_size = self
.decoding_config
.prefill_step_size
.resolve(&self.context.model_config);
-
- let tokens_length = tokens.len();
- let number_of_prefill_steps =
- (tokens_length as f32 / prefill_step_size as f32).ceil() as usize;
- let total_prefill_tokens_count =
- number_of_prefill_steps * prefill_step_size;
- let unused_tokens_count = total_prefill_tokens_count - tokens_length;
+ let prefill_steps = tokens_length.div_ceil(prefill_step_size);
+ let prefill_size = prefill_steps * prefill_step_size;
let speculator = &self.decoding_config.speculator_config.speculator;
- let speculated_suffix = TokenTrie::from_speculator(
+
+ let suffix_length = prefill_size - tokens_length;
+ let suffix_root = TrieNode::from_speculator(
&tokens,
&mut self.context.next_seed,
- false,
speculator.as_ref(),
&TrieCreationConfig::default(),
- unused_tokens_count,
- )
- .linearize(0, unused_tokens_count);
-
- let zero_padding_tokens: Vec<u64> =
- vec![0; unused_tokens_count - speculated_suffix.tokens.len()];
-
- let padded_tokens = [
- &tokens[..],
- &speculated_suffix.tokens[..],
- &zero_padding_tokens[..],
- ]
- .concat();
-
- let mut padded_positions: Vec<usize> =
- (prefix_offset..prefix_offset + tokens_length).collect();
- padded_positions.extend(
- speculated_suffix
- .indices
- .iter()
- .map(|index| index + prefix_offset + tokens_length),
+ suffix_length + 1,
);
- let padding_count =
- unused_tokens_count - speculated_suffix.tokens.len();
- padded_positions
- .extend(std::iter::repeat(INVALID_POSITION).take(padding_count));
-
- let zero_padding_left_seeds: Vec<u64> = vec![0; tokens_length];
- let zero_padding_right_seeds: Vec<u64> =
- vec![0; unused_tokens_count - speculated_suffix.seeds.len()];
- let padded_seeds = [
- &zero_padding_left_seeds[..],
- &speculated_suffix.seeds[..],
- &zero_padding_right_seeds[..],
- ]
- .concat();
+ let flat_trie = suffix_root.linearize();
+ let active_suffix_length = flat_trie.len() - 1;
+
+ let token_ids = tokens
+ .iter()
+ .copied()
+ .take(tokens_length - 1)
+ .chain(flat_trie.token_ids())
+ .chain(repeat_n(0, suffix_length - active_suffix_length))
+ .collect::<Box<[u64]>>();
+
+ let token_positions = (prefix_offset
+ ..prefix_offset + tokens_length - 1)
+ .chain(flat_trie.token_positions().map(|trie_position| {
+ prefix_offset + tokens_length - 1 + trie_position
+ }))
+ .chain(repeat_n(
+ INVALID_POSITION,
+ suffix_length - active_suffix_length,
+ ))
+ .collect::<Box<[usize]>>();
+
+ let token_seeds = repeat_n(0, tokens_length - 1)
+ .chain(flat_trie.token_seeds())
+ .chain(repeat_n(0, suffix_length - active_suffix_length))
+ .collect::<Box<[u64]>>();
let mut last_state: Option<ForwardPassState> = None;
let mut run_times: Vec<f64> = Vec::new();
// Process each prefill step and update the KV cache.
- for step in 0..number_of_prefill_steps {
+ for (step, (step_token_ids, step_token_positions, step_token_seeds)) in
+ izip!(
+ token_ids.chunks(prefill_step_size),
+ token_positions.chunks(prefill_step_size),
+ token_seeds.chunks(prefill_step_size)
+ )
+ .enumerate()
+ {
let tokens_start_index = step * prefill_step_size;
let tokens_end_index = tokens_start_index + prefill_step_size;
- let tokens_for_step =
- &padded_tokens[tokens_start_index..tokens_end_index];
- let positions_for_step =
- &padded_positions[tokens_start_index..tokens_end_index];
- let active_suffix_length = positions_for_step
+ let active_suffix_length = step_token_positions
.iter()
.position(|&pos| pos == INVALID_POSITION)
.unwrap_or(prefill_step_size);
- let seeds_for_step =
- &padded_seeds[tokens_start_index..tokens_end_index];
- let is_last_prefill_step = step == number_of_prefill_steps - 1;
+ let is_last_prefill_step = step == prefill_steps - 1;
let should_sample_after_step =
sample_suffix && is_last_prefill_step;
- let is_first_prefill = step == 0;
let should_capture =
- self.gpu_capture.should_capture_prefill(is_first_prefill);
+ self.gpu_capture.should_capture_prefill(step == 0);
if should_capture {
let _ = self
@@ -165,9 +158,9 @@ impl Generator {
});
let task = GeneratorRunTask {
- token_ids: tokens_for_step.to_vec(),
- token_positions: positions_for_step.to_vec(),
- token_seeds: seeds_for_step.to_vec(),
+ token_ids: step_token_ids,
+ token_positions: step_token_positions,
+ token_seeds: step_token_seeds,
expected_number_of_new_tokens: prefill_step_size,
active_suffix_length,
is_prefilling: !should_sample_after_step,
@@ -196,7 +189,7 @@ impl Generator {
..step_end_token_index)
.map(|idx| idx + prefix_offset)
.collect();
- if step == number_of_prefill_steps - 1 && sample_suffix {
+ if step == prefill_steps - 1 && sample_suffix {
// Exclude the last token because it belongs to the suffix for sampling.
positions_for_step.pop();
}
@@ -237,42 +230,22 @@ impl Generator {
}
let sampled_tokens = self.sample(&mut final_state)?;
- let last_suffix_start =
- prefill_step_size * (number_of_prefill_steps - 1);
-
- let sampling_row = (tokens_length - last_suffix_start) - 1;
- let mut current_token_index: isize = -1;
- let mut accepted_token_indices: Vec<usize> = vec![sampling_row];
- let mut accepted_tokens: Vec<u64> = Vec::new();
- loop {
- let current_index_in_window =
- ((current_token_index + tokens_length as isize)
- % prefill_step_size as isize) as usize;
- let new_token = sampled_tokens[current_index_in_window];
- accepted_tokens.push(new_token);
-
- if let Some(map) =
- speculated_suffix.transition_map.get(&current_token_index)
- {
- if let Some(&next_token_index) = map.get(&new_token) {
- accepted_token_indices.push(next_token_index as usize);
- current_token_index = next_token_index;
- } else {
- break;
- }
- } else {
- break;
- }
- }
+ let last_suffix_start = prefill_step_size * (prefill_steps - 1);
+ let suffix_root_index = (tokens_length - last_suffix_start) - 1;
+
+ let (accepted_tokens, accepted_token_indices) =
+ flat_trie.accept(&sampled_tokens[suffix_root_index..]);
self.update_cache_layers(
- &accepted_token_indices,
+ &accepted_token_indices
+ .into_iter()
+ .map(|p| suffix_root_index + p)
+ .collect::<Box<[usize]>>(),
Some(last_suffix_start),
false,
);
self.tokens.extend(accepted_tokens.clone());
-
self.sync_prefix();
Ok(PrefillResult {
@@ -286,45 +259,43 @@ impl Generator {
sampling_method: SamplingMethod,
) -> Result<GenerateResult, Error> {
let speculator = &self.decoding_config.speculator_config.speculator;
- let speculated_suffix = TokenTrie::from_speculator(
+
+ let suffix_length = self.decoding_config.generate_suffix_length();
+ let suffix_root = TrieNode::from_speculator(
&self.tokens,
&mut self.context.next_seed,
- true,
speculator.as_ref(),
&TrieCreationConfig::default(),
- self.decoding_config.generate_suffix_length(),
- )
- .linearize(0, self.decoding_config.generate_suffix_length());
-
- let expected_suffix_length =
- self.decoding_config.generate_suffix_length();
- let unused_tokens_count =
- expected_suffix_length - speculated_suffix.tokens.len();
+ suffix_length,
+ );
- let zero_padding_tokens: Vec<u64> = vec![0; unused_tokens_count];
- let padded_tokens =
- [speculated_suffix.tokens.clone(), zero_padding_tokens].concat();
+ let flat_trie = suffix_root.linearize();
+ let active_suffix_length = flat_trie.len();
- let zero_padding_seeds: Vec<u64> = vec![0; unused_tokens_count];
- let padded_seeds =
- [speculated_suffix.seeds.clone(), zero_padding_seeds].concat();
+ let token_ids = flat_trie
+ .token_ids()
+ .chain(repeat_n(0, suffix_length - active_suffix_length))
+ .collect::<Box<[u64]>>();
let start_position = self.tokens.len() - 1;
-
- let padded_positions: Vec<usize> = speculated_suffix
- .indices
- .iter()
- .map(|idx| idx + start_position)
- .chain(
- std::iter::repeat(INVALID_POSITION).take(unused_tokens_count),
- )
- .collect();
- let active_suffix_length = speculated_suffix.tokens.len();
+ let token_positions = flat_trie
+ .token_positions()
+ .map(|trie_position| start_position + trie_position)
+ .chain(repeat_n(
+ INVALID_POSITION,
+ suffix_length - active_suffix_length,
+ ))
+ .collect::<Box<[usize]>>();
+
+ let token_seeds = flat_trie
+ .token_seeds()
+ .chain(repeat_n(0, suffix_length - active_suffix_length))
+ .collect::<Box<[u64]>>();
let task = GeneratorRunTask {
- token_ids: padded_tokens,
- token_positions: padded_positions,
- token_seeds: padded_seeds,
+ token_ids: &token_ids,
+ token_positions: &token_positions,
+ token_seeds: &token_seeds,
expected_number_of_new_tokens: 1,
active_suffix_length,
is_prefilling: false,
@@ -339,27 +310,10 @@ impl Generator {
let sampled_tokens = self.sample(&mut state)?;
- let mut accepted_token_indices = Vec::new();
- let mut accepted_tokens = Vec::new();
- let mut current_token_index: isize = 0;
- loop {
- let new_token = sampled_tokens[current_token_index as usize];
- accepted_tokens.push(new_token);
- if let Some(map) =
- speculated_suffix.transition_map.get(&(current_token_index))
- {
- if let Some(&next_token_index) = map.get(&new_token) {
- accepted_token_indices.push(next_token_index as usize);
- current_token_index = next_token_index;
- } else {
- break;
- }
- } else {
- break;
- }
- }
+ let (accepted_tokens, accepted_token_indices) =
+ flat_trie.accept(&sampled_tokens);
- self.update_cache_layers(&accepted_token_indices, None, false);
+ self.update_cache_layers(&accepted_token_indices[1..], None, false);
self.tokens.extend(accepted_tokens.clone());
self.sync_prefix();
@@ -393,9 +347,9 @@ impl Generator {
suffix_length: usize,
) {
let task = GeneratorRunTask {
- token_ids: vec![0; suffix_length],
- token_positions: (0..suffix_length).collect::<Vec<usize>>(),
- token_seeds: vec![0; suffix_length],
+ token_ids: &repeat_n(0, suffix_length).collect::<Box<[u64]>>(),
+ token_positions: &(0..suffix_length).collect::<Box<[usize]>>(),
+ token_seeds: &repeat_n(0, suffix_length).collect::<Box<[u64]>>(),
expected_number_of_new_tokens: suffix_length,
active_suffix_length: suffix_length,
is_prefilling: false,
@@ -618,9 +572,9 @@ impl Generator {
let last_token = *self.tokens.last().ok_or(Error::PrefillFailed)?;
let task = GeneratorRunTask {
- token_ids: vec![last_token],
- token_positions: vec![0], // Ignored, using async buffer
- token_seeds: vec![0], // Ignored, using async buffer
+ token_ids: &[last_token],
+ token_positions: &[0], // Ignored, using async buffer
+ token_seeds: &[0], // Ignored, using async buffer
expected_number_of_new_tokens: 1,
active_suffix_length: 1,
is_prefilling: false,
@@ -638,9 +592,9 @@ impl Generator {
self.context.shared_buffers.clone(),
&task.token_ids,
&task.token_positions,
+ &task.token_seeds,
task.active_suffix_length,
task.is_prefilling,
- &task.token_seeds,
false,
None,
is_continuation,
diff --git a/crates/uzu/src/generator/tasks.rs b/crates/uzu/src/generator/tasks.rs
index 9d97f175d..dcf7fb170 100644
--- a/crates/uzu/src/generator/tasks.rs
+++ b/crates/uzu/src/generator/tasks.rs
@@ -33,21 +33,21 @@ impl GeneratorEncodedTask {
}
#[derive(Debug, Clone)]
-pub struct GeneratorRunTask {
- pub token_ids: Vec<u64>,
- pub token_positions: Vec<usize>,
- pub token_seeds: Vec<u64>,
+pub struct GeneratorRunTask<'a> {
+ pub token_ids: &'a [u64],
+ pub token_positions: &'a [usize],
+ pub token_seeds: &'a [u64],
pub expected_number_of_new_tokens: usize,
pub active_suffix_length: usize,
pub is_prefilling: bool,
}
-impl GeneratorRunTask {
+impl<'a> GeneratorRunTask<'a> {
pub fn speculate_next_task(&self) -> Self {
GeneratorRunTask {
- token_ids: self.token_ids.clone(),
- token_positions: self.token_positions.clone(),
- token_seeds: self.token_seeds.clone(),
+ token_ids: self.token_ids,
+ token_positions: self.token_positions,
+ token_seeds: self.token_seeds,
expected_number_of_new_tokens: self.expected_number_of_new_tokens,
active_suffix_length: self.active_suffix_length,
is_prefilling: self.is_prefilling,
@@ -76,11 +76,11 @@ impl GeneratorRunTask {
&context.scratch_buffers,
context.cache_layers.clone(),
context.shared_buffers.clone(),
- &self.token_ids,
- &self.token_positions,
+ self.token_ids,
+ self.token_positions,
+ self.token_seeds,
self.active_suffix_length,
self.is_prefilling,
- &self.token_seeds,
false,
external_bias_fn,
false,
diff --git a/crates/uzu/src/lib.rs b/crates/uzu/src/lib.rs
index 8a9f80912..e28e2d944 100644
--- a/crates/uzu/src/lib.rs
+++ b/crates/uzu/src/lib.rs
@@ -1,11 +1,11 @@
pub mod backends;
pub mod config;
pub mod generator;
-pub mod linearizer;
pub mod parameters;
pub mod session;
pub mod speculators;
pub mod tracer;
+pub mod trie;
pub mod utils;
pub use utils::*;
pub mod device;
diff --git a/crates/uzu/src/linearizer/mod.rs b/crates/uzu/src/linearizer/mod.rs
deleted file mode 100644
index 95655d93b..000000000
--- a/crates/uzu/src/linearizer/mod.rs
+++ /dev/null
@@ -1,8 +0,0 @@
-pub mod node;
-pub mod speculated_suffix;
-pub mod transition;
-pub mod trie;
-
-#[cfg(test)]
-mod tests_node;
-mod tests_trie;
diff --git a/crates/uzu/src/linearizer/node.rs b/crates/uzu/src/linearizer/node.rs
deleted file mode 100644
index 62df6cb1d..000000000
--- a/crates/uzu/src/linearizer/node.rs
+++ /dev/null
@@ -1,80 +0,0 @@
-use super::transition::Transition;
-
-#[derive(Debug)]
-pub struct Node {
- transitions: Vec<Transition>,
-}
-
-impl Default for Node {
- fn default() -> Self {
- Self::new()
- }
-}
-
-impl Node {
- pub fn new() -> Self {
- Self {
- transitions: Vec::new(),
- }
- }
-
- pub fn has_transition(
- &self,
- token: u64,
- ) -> bool {
- self.transitions.iter().any(|t| t.token() == token)
- }
-
- pub fn get_next(
- &self,
- token: u64,
- ) -> Option<&Node> {
- self.transitions.iter().find(|t| t.token() == token).map(|t| &t.node)
- }
-
- pub fn get_or_insert_next(
- &mut self,
- token: u64,
- seed: u64,
- ) -> &mut Node {
- if let Some(pos) =
- self.transitions.iter().position(|t| t.token() == token)
- {
- return &mut self.transitions[pos].node;
- }
-
- self.transitions.push(Transition::new(token, seed));
- &mut self.transitions.last_mut().unwrap().node
- }
-
- pub fn dfs_with_path<F>(
- &self,
- path: &mut Vec<(isize, u64, u64)>,
- callback: &mut F,
- ) -> isize
- where
- F: FnMut(&[(isize, u64, u64)]),
- {
- let mut node_index = path.last().map(|(idx, _, _)| *idx).unwrap_or(-1);
-
- for transition in self.transitions.iter() {
- node_index += 1;
- path.push((node_index, transition.token(), transition.seed()));
- callback(path);
- node_index = transition.node.dfs_with_path(path, callback);
- path.pop();
- }
-
- node_index
- }
-
- pub fn dfs<F>(
- &self,
- mut callback: F,
- ) where
- F: FnMut(&[(isize, u64, u64)]),
- {
- let mut path: Vec<(isize, u64, u64)> = Vec::new();
- self.dfs_with_path(&mut path, &mut callback);
- }
-}
diff --git a/crates/uzu/src/linearizer/speculated_suffix.rs b/crates/uzu/src/linearizer/speculated_suffix.rs
deleted file mode 100644
index 2f3874036..000000000
--- a/crates/uzu/src/linearizer/speculated_suffix.rs
+++ /dev/null
@@ -1,9 +0,0 @@
-use std::collections::HashMap;
-
-#[derive(Debug)]
-pub struct SpeculatedSuffix {
- pub tokens: Vec<u64>,
- pub seeds: Vec<u64>,
- pub indices: Vec<usize>,
- pub transition_map: HashMap<isize, HashMap<u64, isize>>,
-}
diff --git a/crates/uzu/src/linearizer/tests_node.rs b/crates/uzu/src/linearizer/tests_node.rs
deleted file mode 100644
index 9f4680d76..000000000
--- a/crates/uzu/src/linearizer/tests_node.rs
+++ /dev/null
@@ -1,267 +0,0 @@
-#[cfg(test)]
-mod tests {
- use std::collections::HashSet;
-
- use super::super::node::Node;
-
- #[test]
- fn test_new_node_has_no_transitions() {
- let node = Node::new();
- for i in 0..10 {
- assert!(!node.has_transition(i));
- }
- }
-
- #[test]
- fn test_has_transition() {
- let mut node = Node::new();
- node.get_or_insert_next(42, 0);
-
- assert!(node.has_transition(42));
- assert!(!node.has_transition(43));
- }
-
- #[test]
- fn test_get_next() {
- let mut node = Node::new();
- node.get_or_insert_next(1, 0);
-
- assert!(node.get_next(1).is_some());
- assert!(node.get_next(2).is_none());
- }
-
- #[test]
- fn test_get_or_insert_next() {
- let mut node = Node::new();
-
- // First insertion should create a new node
- let node1 = node.get_or_insert_next(1, 0);
- assert!(!node1.has_transition(99));
-
- // Add a transition to the child node
- node1.get_or_insert_next(99, 0);
-
- // Get the same node again
- let node1_again = node.get_or_insert_next(1, 0);
-
- // Verify it's the same node by checking it has the transition we added
- assert!(node1_again.has_transition(99));
-
- // Different token should give a different node
- let node2 = node.get_or_insert_next(2, 0);
- assert!(!node2.has_transition(99));
- }
-
- #[test]
- fn test_multiple_transitions() {
- let mut node = Node::new();
-
- // Insert several transitions
- for i in 0..5 {
- node.get_or_insert_next(i, 0);
- }
-
- // Verify all transitions exist
- for i in 0..5 {
- assert!(node.has_transition(i));
- }
-
- // Verify non-existent transitions
- for i in 5..10 {
- assert!(!node.has_transition(i));
- }
- }
-
- #[test]
- fn test_dfs_with_path_empty_node() {
- let node = Node::new();
- let mut path: Vec<(isize, u64, u64)> = Vec::new();
- let mut visited = Vec::new();
-
- node.dfs_with_path(&mut path, &mut |current_path: &[(
- isize,
- u64,
- u64,
- )]| {
- let tokens: Vec<u64> =
- current_path.iter().map(|(_, token, _)| *token).collect();
- visited.push(tokens);
- });
-
- assert!(visited.is_empty());
- }
-
- #[test]
- fn test_dfs_with_path_single_node() {
- let mut node = Node::new();
- node.get_or_insert_next(42, 0);
-
- let mut path: Vec<(isize, u64, u64)> = Vec::new();
- let mut visited = Vec::new();
-
- node.dfs_with_path(&mut path, &mut |current_path: &[(
- isize,
- u64,
- u64,
- )]| {
- let tokens: Vec<u64> =
- current_path.iter().map(|(_, token, _)| *token).collect();
- visited.push(tokens);
- });
-
- assert_eq!(visited.len(), 1);
- assert_eq!(visited[0], vec![42]);
- }
-
- #[test]
- fn test_dfs_with_path_complex_tree() {
- let mut root = Node::new();
-
- // Build a sample trie: [1 -> 2 -> 3], [1 -> 2 -> 4], [1 -> 5], [6]
- let node1 = root.get_or_insert_next(1, 0);
- let node2 = node1.get_or_insert_next(2, 0);
- node2.get_or_insert_next(3, 0);
- node2.get_or_insert_next(4, 0);
- node1.get_or_insert_next(5, 0);
- root.get_or_insert_next(6, 0);
-
- let mut path: Vec<(isize, u64, u64)> = Vec::new();
- let mut visited = Vec::new();
-
- root.dfs_with_path(&mut path, &mut |current_path: &[(
- isize,
- u64,
- u64,
- )]| {
- let tokens: Vec<u64> =
- current_path.iter().map(|(_, token, _)| *token).collect();
- visited.push(tokens);
- });
-
- // Check all expected paths were visited
- assert_eq!(visited.len(), 6);
- assert!(visited.contains(&vec![1]));
- assert!(visited.contains(&vec![1, 2]));
- assert!(visited.contains(&vec![1, 2, 3]));
- assert!(visited.contains(&vec![1, 2, 4]));
- assert!(visited.contains(&vec![1, 5]));
- assert!(visited.contains(&vec![6]));
- }
-
- #[test]
- fn test_dfs_path_indices() {
- let mut root = Node::new();
-
- // Build a simple trie: [1], [2]
- root.get_or_insert_next(1, 0);
- root.get_or_insert_next(2, 0);
-
- let mut path: Vec<(isize, u64, u64)> = Vec::new();
- let mut indices = Vec::new();
-
- root.dfs_with_path(&mut path, &mut |current_path: &[(
- isize,
- u64,
- u64,
- )]| {
- if !current_path.is_empty() {
- indices.push(current_path.last().unwrap().0);
- }
- });
-
- // Check indices are unique and as expected
- assert_eq!(indices.len(), 2);
- assert_eq!(indices[0], 0);
- assert_eq!(indices[1], 1);
- }
-
- #[test]
- fn test_simplified_dfs_empty_node() {
- let node = Node::new();
- let mut visited = Vec::new();
-
- node.dfs(|current_path: &[(isize, u64, u64)]| {
- let tokens: Vec<u64> =
- current_path.iter().map(|(_, token, _)| *token).collect();
- visited.push(tokens);
- });
-
- assert!(visited.is_empty());
- }
-
- #[test]
- fn test_simplified_dfs_complex_tree() {
- let mut root = Node::new();
-
- // Build a sample trie: [1 -> 2 -> 3], [1 -> 2 -> 4], [1 -> 5], [6]
- let node1 = root.get_or_insert_next(1, 0);
- let node2 = node1.get_or_insert_next(2, 0);
- node2.get_or_insert_next(3, 0);
- node2.get_or_insert_next(4, 0);
- node1.get_or_insert_next(5, 0);
- root.get_or_insert_next(6, 0);
-
- let mut visited = Vec::new();
-
- root.dfs(|current_path: &[(isize, u64, u64)]| {
- let tokens: Vec<u64> =
- current_path.iter().map(|(_, token, _)| *token).collect();
- visited.push(tokens);
- });
-
- // Check all expected paths were visited
- assert_eq!(visited.len(), 6);
- assert!(visited.contains(&vec![1]));
- assert!(visited.contains(&vec![1, 2]));
- assert!(visited.contains(&vec![1, 2, 3]));
- assert!(visited.contains(&vec![1, 2, 4]));
- assert!(visited.contains(&vec![1, 5]));
- assert!(visited.contains(&vec![6]));
- }
-
- #[test]
- fn test_both_dfs_methods_equivalent() {
- let mut root = Node::new();
-
- // Build a more complex trie
- let node1 = root.get_or_insert_next(10, 0);
- let node2 = node1.get_or_insert_next(20, 0);
- node2.get_or_insert_next(30, 0);
- node2.get_or_insert_next(40, 0);
- node1.get_or_insert_next(50, 0);
- let node3 = root.get_or_insert_next(60, 0);
- node3.get_or_insert_next(70, 0);
-
- // Get results from dfs_with_path
- let mut path: Vec<(isize, u64, u64)> = Vec::new();
- let mut visited1 = Vec::new();
-
- root.dfs_with_path(&mut path, &mut |current_path: &[(
- isize,
- u64,
- u64,
- )]| {
- let tokens: Vec<u64> =
- current_path.iter().map(|(_, token, _)| *token).collect();
- visited1.push(tokens);
- });
-
- // Get results from simplified dfs
- let mut visited2 = Vec::new();
-
- root.dfs(|current_path: &[(isize, u64, u64)]| {
- let tokens: Vec<u64> =
- current_path.iter().map(|(_, token, _)| *token).collect();
- visited2.push(tokens);
- });
-
- // Check both methods give the same results
- assert_eq!(visited1.len(), visited2.len());
-
- // Convert to sets to ignore order differences
- let set1: HashSet<Vec<u64>> = visited1.into_iter().collect();
- let set2: HashSet<Vec<u64>> = visited2.into_iter().collect();
-
- assert_eq!(set1, set2);
- }
-}
diff --git a/crates/uzu/src/linearizer/tests_trie.rs b/crates/uzu/src/linearizer/tests_trie.rs
deleted file mode 100644
index f44f8c04d..000000000
--- a/crates/uzu/src/linearizer/tests_trie.rs
+++ /dev/null
@@ -1,297 +0,0 @@
-#[cfg(test)]
-mod tests {
- use std::collections::HashMap;
-
- use crate::{
- generator::rng::DerivableSeed,
- linearizer::trie::{TokenTrie, TrieCreationConfig},
- speculators::speculator::Speculator,
- };
-
- struct StaticSpeculator {
- responses: HashMap<Vec<u64>, HashMap<u64, f32>>,
- }
-
- impl StaticSpeculator {
- fn new(responses: HashMap<Vec<u64>, HashMap<u64, f32>>) -> Self {
- Self {
- responses,
- }
- }
- }
-
- impl Speculator for StaticSpeculator {
- fn speculate(
- &self,
- prefix: &[u64],
- ) -> HashMap<u64, f32> {
- self.responses.get(prefix).cloned().unwrap_or_default()
- }
- }
-
- #[test]
- fn test_from_speculator_applies_prob_cutoff() {
- let mut responses = HashMap::new();
- responses.insert(
- vec![42],
- HashMap::from([(1, 0.9_f32), (2, 0.05_f32), (3, 0.2_f32)]),
- );
- responses.insert(vec![42, 1], HashMap::new());
-
- let speculator = StaticSpeculator::new(responses);
- let mut seed = DerivableSeed::new(1);
-
- let trie = TokenTrie::from_speculator(
- &[42],
- &mut seed,
- false,
- &speculator,
- &TrieCreationConfig::default(),
- 5,
- );
- let suffix = trie.linearize(0, 5);
-
- assert_eq!(suffix.tokens, vec![1, 3]);
- assert!(!suffix.tokens.contains(&2));
- assert_eq!(suffix.transition_map.len(), 3);
- }
-
- #[test]
- fn test_from_speculator_respects_top_k_limit() {
- let mut responses = HashMap::new();
- responses.insert(vec![5], HashMap::from([(9, 0.8_f32), (10, 0.7_f32)]));
- responses.insert(vec![5, 9], HashMap::from([(11, 0.6_f32)]));
- responses.insert(vec![5, 9, 11], HashMap::new());
-
- let speculator = StaticSpeculator::new(responses);
- let mut seed = DerivableSeed::new(7);
-
- let trie = TokenTrie::from_speculator(
- &[5],
- &mut seed,
- false,
- &speculator,
- &TrieCreationConfig {
- top_k: 1,
- prob_cutoff: 0.0,
- },
- 3,
- );
-
- let suffix = trie.linearize(0, 4);
-
- assert_eq!(suffix.tokens, vec![9, 11]);
- assert!(!suffix.tokens.contains(&10));
- }
-
- #[test]
- fn test_from_speculator_includes_last_prefix_token_and_seeds() {
- let mut responses = HashMap::new();
- responses.insert(vec![2, 4], HashMap::from([(8, 0.9_f32)]));
- responses.insert(vec![2, 4, 8], HashMap::new());
-
- let speculator = StaticSpeculator::new(responses);
-
- let mut seed = DerivableSeed::new(99);
- let mut expected_seed_stream = DerivableSeed::new(99);
- let expected_seeds =
- vec![expected_seed_stream.next(), expected_seed_stream.next()];
-
- let trie = TokenTrie::from_speculator(
- &[2, 4],
- &mut seed,
- true,
- &speculator,
- &TrieCreationConfig::default(),
- 2,
- );
-
- let suffix = trie.linearize(0, 3);
-
- assert_eq!(suffix.tokens, vec![4, 8]);
- assert_eq!(suffix.seeds, expected_seeds);
- assert_eq!(suffix.indices, vec![0, 1]);
- }
-
- #[test]
- fn test_from_speculator_stops_at_max_length() {
- let mut responses = HashMap::new();
- responses.insert(
- vec![1],
- HashMap::from([(5, 0.9_f32), (6, 0.8_f32), (7, 0.7_f32)]),
- );
- responses
- .insert(vec![1, 5], HashMap::from([(8, 0.6_f32), (9, 0.5_f32)]));
- responses.insert(vec![1, 5, 8], HashMap::from([(10, 0.4_f32)]));
-
- let speculator = StaticSpeculator::new(responses);
- let mut seed = DerivableSeed::new(1234);
-
- let trie = TokenTrie::from_speculator(
- &[1],
- &mut seed,
- false,
- &speculator,
- &TrieCreationConfig::default(),
- 3,
- );
-
- let suffix = trie.linearize(0, 5);
-
- assert_eq!(suffix.tokens.len(), 3);
- assert!(suffix.tokens.contains(&5));
- assert!(suffix.tokens.contains(&6));
- assert!(suffix.tokens.contains(&8));
- assert!(!suffix.tokens.contains(&9));
- }
-
- #[test]
- fn test_from_speculator_with_empty_responses() {
- let speculator = StaticSpeculator::new(HashMap::new());
-
- let mut seed = DerivableSeed::new(555);
- let mut expected_seed_stream = DerivableSeed::new(555);
- let expected_first_seed = expected_seed_stream.next();
-
- let trie = TokenTrie::from_speculator(
- &[12, 13],
- &mut seed,
- true,
- &speculator,
- &TrieCreationConfig::default(),
- 4,
- );
-
- let suffix = trie.linearize(0, 5);
-
- // Only the last prefix token should be included; no speculated tokens.
- assert_eq!(suffix.tokens, vec![13]);
- assert_eq!(suffix.indices, vec![0]);
- assert_eq!(suffix.seeds, vec![expected_first_seed]);
-
- assert_eq!(suffix.transition_map.len(), 2);
- let root_transitions = &suffix.transition_map[&-1];
- assert_eq!(root_transitions.len(), 1);
- assert_eq!(root_transitions.get(&13), Some(&0));
- }
-
- #[test]
- fn test_linearize_empty_trie() {
- let trie = TokenTrie::new();
- let suffix = trie.linearize(0, 10);
-
- assert!(suffix.tokens.is_empty());
- assert!(suffix.indices.is_empty());
- assert!(suffix.transition_map.len() == 1);
- }
-
- #[test]
- fn test_linearize_single_sequence() {
- let mut trie = TokenTrie::new();
- trie.insert(&[1, 2, 3]);
-
- let suffix = trie.linearize(0, 5);
-
- assert_eq!(suffix.tokens.len(), 3);
- assert!(suffix.tokens.contains(&1));
- assert!(suffix.tokens.contains(&2));
- assert!(suffix.tokens.contains(&3));
-
- assert_eq!(suffix.transition_map.len(), 4);
-
- let root_transitions = &suffix.transition_map[&-1];
- assert!(root_transitions.contains_key(&1));
-
- let token1_idx = *root_transitions.get(&1).unwrap();
- let token1_transitions = &suffix.transition_map[&token1_idx];
- assert!(token1_transitions.contains_key(&2));
-
- let token2_idx = *token1_transitions.get(&2).unwrap();
- let token2_transitions = &suffix.transition_map[&token2_idx];
- assert!(token2_transitions.contains_key(&3));
- }
-
- #[test]
- fn test_linearize_branching_trie() {
- let mut trie = TokenTrie::new();
- trie.insert(&[1, 2, 3]);
- trie.insert(&[1, 2, 4]);
- trie.insert(&[5, 6]);
-
- let suffix = trie.linearize(10, 10);
-
- assert!(suffix.tokens.contains(&1));
- assert!(suffix.tokens.contains(&2));
- assert!(suffix.tokens.contains(&3));
- assert!(suffix.tokens.contains(&4));
- assert!(suffix.tokens.contains(&5));
- assert!(suffix.tokens.contains(&6));
-
- assert_eq!(suffix.tokens.len(), 6);
-
- assert_eq!(suffix.transition_map.len(), 7);
-
- let root_transitions = &suffix.transition_map[&-1];
- assert!(root_transitions.contains_key(&1));
- assert!(root_transitions.contains_key(&5));
-
- let token1_idx = *root_transitions.get(&1).unwrap();
- let token1_transitions = &suffix.transition_map[&token1_idx];
- assert!(token1_transitions.contains_key(&2));
-
- let token2_idx = *token1_transitions.get(&2).unwrap();
- let token2_transitions = &suffix.transition_map[&token2_idx];
- assert!(token2_transitions.contains_key(&3));
- assert!(token2_transitions.contains_key(&4));
-
- let token5_idx = *root_transitions.get(&5).unwrap();
- let token5_transitions = &suffix.transition_map[&token5_idx];
- assert!(token5_transitions.contains_key(&6));
- }
-
- #[test]
- fn test_linearize_max_length() {
- let mut trie = TokenTrie::new();
- trie.insert(&[1, 2, 3, 4, 5, 6, 7, 8, 9, 10]);
-
- let suffix = trie.linearize(0, 5);
-
- assert_eq!(suffix.tokens.len(), 5);
- assert_eq!(suffix.indices.len(), 5);
- assert_eq!(suffix.transition_map.len(), 6);
-
- for i in 1..=5 {
- assert!(suffix.tokens.contains(&i));
- }
-
- let root_transitions = &suffix.transition_map[&-1];
- assert!(root_transitions.contains_key(&1));
-
- let mut current_idx = *root_transitions.get(&1).unwrap();
- for i in 2..=4 {
- let transitions = &suffix.transition_map[&current_idx];
- assert!(transitions.contains_key(&i));
- current_idx = *transitions.get(&i).unwrap();
- }
- }
-
- #[test]
- fn test_linearize_start_index() {
- let mut trie = TokenTrie::new();
- trie.insert(&[1, 2, 3]);
-
- let suffix = trie.linearize(100, 10);
-
- assert_eq!(suffix.tokens.len(), 3);
- assert!(suffix.tokens.contains(&1));
- assert!(suffix.tokens.contains(&2));
- assert!(suffix.tokens.contains(&3));
-
- for idx in suffix.indices.iter() {
- assert!(*idx >= 100 && *idx <= 102);
- }
-
- let root_transitions = &suffix.transition_map[&-1];
- assert!(root_transitions.contains_key(&1));
- }
-}
diff --git a/crates/uzu/src/linearizer/transition.rs b/crates/uzu/src/linearizer/transition.rs
deleted file mode 100644
index 76426fe6c..000000000
--- a/crates/uzu/src/linearizer/transition.rs
+++ /dev/null
@@ -1,29 +0,0 @@
-use super::node::Node;
-
-#[derive(Debug)]
-pub struct Transition {
- token: u64,
- seed: u64,
- pub node: Node,
-}
-
-impl Transition {
- pub fn new(
- token: u64,
- seed: u64,
- ) -> Self {
- Self {
- token,
- seed,
- node: Node::new(),
- }
- }
-
- pub fn token(&self) -> u64 {
- self.token
- }
-
- pub fn seed(&self) -> u64 {
- self.seed
- }
-}
diff --git a/crates/uzu/src/linearizer/trie.rs b/crates/uzu/src/linearizer/trie.rs
deleted file mode 100644
index a0486e9f1..000000000
--- a/crates/uzu/src/linearizer/trie.rs
+++ /dev/null
@@ -1,169 +0,0 @@
-use std::collections::HashMap;
-
-use crate::{
- generator::rng::DerivableSeed, speculators::speculator::Speculator,
-};
-
-use super::{node::Node, speculated_suffix::SpeculatedSuffix};
-
-pub struct TrieCreationConfig {
- pub top_k: usize,
- pub prob_cutoff: f32,
-}
-
-impl Default for TrieCreationConfig {
- fn default() -> Self {
- TrieCreationConfig {
- top_k: 2,
- prob_cutoff: 0.1,
- }
- }
-}
-
-#[derive(Debug)]
-pub struct TokenTrie {
- root: Node,
-}
-
-impl Default for TokenTrie {
- fn default() -> Self {
- Self::new()
- }
-}
-
-impl TokenTrie {
- pub fn new() -> Self {
- Self {
- root: Node::new(),
- }
- }
-
- pub fn from_sequences(sequences: &[Vec<u64>]) -> Self {
- let mut trie = Self::new();
- for sequence in sequences {
- trie.insert(sequence);
- }
- trie
- }
-
- pub fn from_speculator(
- prefix: &[u64],
- seed: &mut DerivableSeed,
- include_last_prefix_token: bool,
- speculator: &dyn Speculator,
- creation_config: &TrieCreationConfig,
- max_length: usize,
- ) -> Self {
- let mut trie = Self::new();
- let mut current_node = &mut trie.root;
- let mut trie_size = 0;
-
- if include_last_prefix_token {
- current_node = current_node
- .get_or_insert_next(*prefix.last().unwrap(), seed.next());
- trie_size += 1;
- }
-
- let mut speculation_prefix = prefix.to_vec();
- while trie_size < max_length {
- let speculation_result = speculator.speculate(&speculation_prefix);
-
- let mut arr =
- speculation_result.into_iter().collect::<Vec<(u64, f32)>>();
- arr.sort_by(|(_, a), (_, b)| f32::total_cmp(b, a));
-
- let arr = arr
- .into_iter()
- .enumerate()
- .filter(|&(i, (_, p))| {
- i == 0 || p > creation_config.prob_cutoff
- })
- .map(|(_, x)| x)
- .take(usize::min(creation_config.top_k, max_length - trie_size))
- .collect::<Vec<(u64, f32)>>();
-
- let Some(&(top_token, _)) = arr.first() else {
- break;
- };
-
- trie_size += arr.len();
- for &(tok, _) in arr.iter() {
- current_node.get_or_insert_next(tok, seed.next());
- }
-
- current_node = current_node.get_or_insert_next(top_token, 0);
- speculation_prefix.push(top_token);
- }
-
- trie
- }
-
- pub fn insert<'a, I>(
- &mut self,
- sequence: I,
- ) where
- I: IntoIterator<Item = &'a u64>,
- {
- let mut current_node = &mut self.root;
- for &token in sequence {
- current_node = current_node.get_or_insert_next(token, 0);
- }
- }
-
- pub fn linearize(
- &self,
- start_index: usize,
- max_length: usize,
- ) -> SpeculatedSuffix {
- let mut tokens = Vec::new();
- let mut seeds = Vec::new();
- let mut indices = Vec::new();
-
- self.root.dfs(|path| {
- if tokens.len() >= max_length {
- return;
- }
- if !path.is_empty() {
- tokens.push(path.last().unwrap().1);
- seeds.push(path.last().unwrap().2);
- indices.push(start_index + path.len() - 1);
- }
- });
-
- let tokens_len = tokens.len();
-
- let mut transition_map: HashMap<
- isize,
- std::collections::HashMap<u64, isize>,
- > = HashMap::new();
- for index in -1..(tokens_len as isize) {
- transition_map.insert(index, HashMap::new());
- }
-
- self.root.dfs(|path| {
- if path.is_empty()
- || path.last().unwrap().0 >= (max_length as isize)
- {
- return;
- }
-
- let (current_index, current_token, _) = *path.last().unwrap();
- let parent_index = if path.len() > 1 {
- path[path.len() - 2].0
- } else {
- -1
- };
-
- if let Some(transitions) = transition_map.get_mut(&parent_index) {
- transitions.insert(current_token, current_index);
- }
- });
-
- SpeculatedSuffix {
- tokens,
- indices,
- seeds,
- transition_map,
- }
- }
-}
diff --git a/crates/uzu/src/runners/decoder_runner.rs b/crates/uzu/src/runners/decoder_runner.rs
index e740cdabb..caade4528 100644
--- a/crates/uzu/src/runners/decoder_runner.rs
+++ b/crates/uzu/src/runners/decoder_runner.rs
@@ -127,9 +127,9 @@ impl DecoderTestContext {
shared_buffers.clone(),
&token_ids,
&token_positions,
+ &token_seeds,
token_ids.len(),
false,
- &token_seeds,
false,
None,
false,
diff --git a/crates/uzu/src/tracer/tracer.rs b/crates/uzu/src/tracer/tracer.rs
index 3d6bc88a4..d9c0e6f5f 100644
--- a/crates/uzu/src/tracer/tracer.rs
+++ b/crates/uzu/src/tracer/tracer.rs
@@ -315,9 +315,9 @@ impl Tracer {
self.generator_context.shared_buffers.clone(),
&token_ids,
&token_positions,
+ &token_seeds,
token_ids.len(),
false,
- &token_seeds,
true,
None,
false,
diff --git a/crates/uzu/src/trie.rs b/crates/uzu/src/trie.rs
new file mode 100644
index 000000000..7bede11fc
--- /dev/null
+++ b/crates/uzu/src/trie.rs
@@ -0,0 +1,225 @@
+use crate::{
+ generator::{gumbel::speculator_sample, rng::DerivableSeed},
+ speculators::speculator::Speculator,
+};
+
+use thiserror::Error;
+
+#[derive(Debug, Error)]
+pub enum TrieError {
+ #[error("child with the same token id is already present")]
+ DuplicateTokenId,
+}
+
+pub struct TrieCreationConfig {
+ pub width: usize,
+}
+
+impl Default for TrieCreationConfig {
+ fn default() -> Self {
+ TrieCreationConfig {
+ width: 4,
+ }
+ }
+}
+
+#[derive(Debug)]
+pub struct TrieNode {
+ token: u64,
+ seed: u64,
+ next: Vec<TrieNode>,
+}
+
+#[derive(Debug)]
+pub struct FlatTrie<'a> {
+ tokens: Box<[(&'a TrieNode, usize)]>,
+}
+
+impl TrieNode {
+ pub fn new(
+ token: u64,
+ seed: u64,
+ ) -> Self {
+ Self {
+ token,
+ seed,
+ next: Vec::new(),
+ }
+ }
+
+ pub fn add(
+ &mut self,
+ next: TrieNode,
+ ) -> Result<usize, TrieError> {
+ if self.next.iter().any(|n| n.token == next.token) {
+ return Err(TrieError::DuplicateTokenId);
+ }
+
+ self.next.push(next);
+ Ok(self.next.len() - 1)
+ }
+
+ pub fn get(
+ &self,
+ token: u64,
+ ) -> Option<&TrieNode> {
+ self.next.iter().find(|n| n.token == token)
+ }
+
+ pub fn get_mut(
+ &mut self,
+ token: u64,
+ ) -> Option<&mut TrieNode> {
+ self.next.iter_mut().find(|n| n.token == token)
+ }
+
+ pub fn token(&self) -> u64 {
+ self.token
+ }
+
+ pub fn seed(&self) -> u64 {
+ self.seed
+ }
+
+ pub fn from_speculator(
+ prefix: &[u64],
+ next_seed: &mut DerivableSeed,
+ speculator: &dyn Speculator,
+ creation_config: &TrieCreationConfig,
+ max_length: usize,
+ ) -> Self {
+ assert!(max_length >= 1, "can't have zero sized trie");
+ assert!(prefix.len() >= 1, "need seed node");
+
+ let mut speculated_suffix = prefix.to_vec();
+
+ let mut length = 1;
+ let mut root = Self::new(*prefix.last().unwrap(), next_seed.next());
+
+ let mut cur_node = &mut root;
+ let mut cur_node_width = 0;
+ let mut cur_node_speculator_weights =
+ speculator.speculate(&speculated_suffix);
+
+ let mut next_node = None;
+
+ while length < max_length {
+ // Guuumbel speculator trick: both speculator and llm sample via gumbel max trick using the same noise for increased acceptance rate
+ if let Some(next_speculated_token) =
+ speculator_sample(cur_node.seed(), &cur_node_speculator_weights)
+ {
+ // Add speculated token to the trie
+ let leaf_node =
+ Self::new(next_speculated_token, next_seed.next());
+ cur_node.add(leaf_node).unwrap();
+
+ // If this is the first node we sampled (most likely to be selected after gumbel noise application) - set it as the next one
+ if next_node.is_none() {
+ next_node = Some(next_speculated_token);
+ }
+
+ // Remove the token so that the next iteration samples the second most likely after gumbel noise application token
+ cur_node_speculator_weights.remove(&next_speculated_token);
+
+ length += 1;
+ cur_node_width += 1;
+ if cur_node_width >= creation_config.width {
+ cur_node_speculator_weights.clear();
+ }
+ } else if let Some(next_node_token) = next_node.take() {
+ // Out of speculated tokens for this node, move onto the likeliest next node
+ speculated_suffix.push(next_node_token);
+ cur_node = cur_node.get_mut(next_node_token).unwrap();
+ cur_node_width = 0;
+ cur_node_speculator_weights =
+ speculator.speculate(&speculated_suffix);
+ continue;
+ } else {
+ // Dead end, exit
+ break;
+ };
+ }
+
+ root
+ }
+
+ pub fn linearize(&self) -> FlatTrie<'_> {
+ let mut tokens = vec![(self, 0)];
+
+ let mut stack = vec![(self, 0)];
+ while let Some((cur_node, next_child_idx)) = stack.last_mut() {
+ let Some(next_node) = cur_node.next.get(*next_child_idx) else {
+ stack.pop();
+ continue;
+ };
+ *next_child_idx += 1;
+
+ tokens.push((next_node, stack.len()));
+
+ if !next_node.next.is_empty() {
+ stack.push((next_node, 0));
+ }
+ }
+
+ FlatTrie::new(tokens.into_boxed_slice())
+ }
+}
+
+impl<'a> FlatTrie<'a> {
+ pub fn new(tokens: Box<[(&'a TrieNode, usize)]>) -> Self {
+ Self {
+ tokens,
+ }
+ }
+
+ pub fn len(&self) -> usize {
+ self.tokens.len()
+ }
+
+ pub fn token_ids(&self) -> impl Iterator<Item = u64> {
+ self.tokens.iter().map(|&(n, _p)| n.token)
+ }
+
+ pub fn token_positions(&self) -> impl Iterator<Item = usize> {
+ self.tokens.iter().map(|&(_n, p)| p)
+ }
+
+ pub fn token_seeds(&self) -> impl Iterator<Item = u64> {
+ self.tokens.iter().map(|&(n, _p)| n.seed)
+ }
+
+ pub fn root(&self) -> Option<&TrieNode> {
+ self.tokens.first().map(|&(n, _p)| n)
+ }
+
+ pub fn index(
+ &self,
+ node: &'a TrieNode,
+ ) -> Option<usize> {
+ self.tokens.iter().position(|&(n, _)| std::ptr::eq(n, node))
+ }
+
+ pub fn accept(
+ &self,
+ sampled_tokens: &[u64],
+ ) -> (Vec<u64>, Vec<usize>) {
+ let mut current_token = self.root().unwrap();
+ let mut accepted_tokens = Vec::new();
+ let mut accepted_token_indices = Vec::new();
+ loop {
+ let current_token_index = self.index(current_token).unwrap();
+ let current_token_id = sampled_tokens[current_token_index];
+
+ accepted_token_indices.push(current_token_index);
+ accepted_tokens.push(current_token_id);
+
+ let Some(next_token) = current_token.get(current_token_id) else {
+ break;
+ };
+
+ current_token = next_token;
+ }
+
+ (accepted_tokens, accepted_token_indices)
+ }
+}
diff --git a/crates/uzu/tests/trie_test.rs b/crates/uzu/tests/trie_test.rs
new file mode 100644
index 000000000..6bf8faa38
--- /dev/null
+++ b/crates/uzu/tests/trie_test.rs
@@ -0,0 +1,285 @@
+use std::collections::{HashMap, HashSet};
+
+use uzu::{
+ generator::rng::DerivableSeed,
+ speculators::{empty_speculator::EmptySpeculator, speculator::Speculator},
+ trie::{TrieCreationConfig, TrieNode},
+};
+
+struct StaticSpeculator {
+ responses: HashMap<Vec<u64>, HashMap<u64, f32>>,
+}
+
+impl StaticSpeculator {
+ fn new(responses: HashMap<Vec<u64>, HashMap<u64, f32>>) -> Self {
+ Self {
+ responses,
+ }
+ }
+}
+
+impl Speculator for StaticSpeculator {
+ fn speculate(
+ &self,
+ prefix: &[u64],
+ ) -> HashMap<u64, f32> {
+ self.responses.get(prefix).cloned().unwrap_or_default()
+ }
+}
+
+fn verify_sprout(
+ trie_root: &TrieNode,
+ expected_seed: u64,
+) {
+ let flat_trie = trie_root.linearize();
+
+ assert_eq!(flat_trie.len(), 1);
+ assert_eq!(flat_trie.index(&trie_root), Some(0));
+ assert_eq!(flat_trie.index(&TrieNode::new(1, 0)), None);
+ assert_eq!(flat_trie.index(&TrieNode::new(0, 1)), None);
+ assert_eq!(flat_trie.index(&TrieNode::new(0, 0)), None);
+ assert_eq!(flat_trie.token_ids().collect::<Vec<u64>>(), vec![0]);
+ assert_eq!(flat_trie.token_positions().collect::<Vec<usize>>(), vec![0]);
+ assert_eq!(
+ flat_trie.token_seeds().collect::<Vec<u64>>(),
+ vec![expected_seed]
+ );
+}
+
+#[test]
+fn test_trie_manual_sprout() {
+ let trie_root = TrieNode::new(0, 0);
+
+ verify_sprout(&trie_root, 0);
+}
+
+#[test]
+fn test_trie_from_speculator_sprout() {
+ let mut trie_rng = DerivableSeed::new(42);
+ let mut ref_rng = DerivableSeed::new(42);
+
+ let speculator = EmptySpeculator;
+
+ let trie_root = TrieNode::from_speculator(
+ &[0],
+ &mut trie_rng,
+ &speculator,
+ &TrieCreationConfig {
+ width: 5,
+ },
+ 10,
+ );
+
+ verify_sprout(&trie_root, ref_rng.next());
+}
+
+fn verify_stick(
+ trie_root: &TrieNode,
+ mut expected_seeds: HashSet<u64>,
+) {
+ let flat_trie = trie_root.linearize();
+
+ assert_eq!(flat_trie.len(), 10);
+ let token_ids = flat_trie.token_ids().collect::<Vec<u64>>();
+ let token_positions = flat_trie.token_positions().collect::<Vec<usize>>();
+ let token_seeds = flat_trie.token_seeds().collect::<Vec<u64>>();
+ assert_eq!(token_ids.len(), 10);
+ assert_eq!(token_positions.len(), 10);
+ assert_eq!(token_seeds.len(), 10);
+
+ let mut cur_node = trie_root;
+
+ let position = flat_trie.index(cur_node).unwrap();
+ assert_eq!(token_ids[position], 0);
+ assert_eq!(token_positions[position], 0);
+ assert!(expected_seeds.remove(&token_seeds[position]));
+
+ for i in 1..10 {
+ cur_node = cur_node.get(i as u64).unwrap();
+ assert_eq!(cur_node.token(), i as u64);
+ assert!(expected_seeds.remove(&cur_node.seed()));
+
+ let position = flat_trie.index(cur_node).unwrap();
+ assert_eq!(token_ids[position], cur_node.token());
+ assert_eq!(token_positions[position], i as usize);
+ assert_eq!(token_seeds[position], cur_node.seed());
+ }
+
+ assert!(expected_seeds.is_empty());
+}
+
+#[test]
+fn test_trie_manual_stick() {
+ let mut trie_root = TrieNode::new(0, 0);
+
+ let mut trie_leaf = &mut trie_root;
+ for (token, seed) in std::iter::zip(1..10, 1..10) {
+ trie_leaf.add(TrieNode::new(token, seed)).unwrap();
+ trie_leaf = trie_leaf.get_mut(token).unwrap();
+ }
+
+ verify_stick(&trie_root, (0..10).collect());
+}
+
+#[test]
+fn test_trie_from_speculator_stick() {
+ let mut trie_rng = DerivableSeed::new(42);
+ let mut ref_rng = DerivableSeed::new(42);
+
+ let mut hs = HashMap::new();
+ for i in 1..10 {
+ hs.insert(
+ (0..i).collect(),
+ vec![(i, 0.988), (888, 0.001), (999, 0.001)].into_iter().collect(),
+ );
+ }
+ let speculator = StaticSpeculator::new(hs);
+
+ let trie_root = TrieNode::from_speculator(
+ &[0],
+ &mut trie_rng,
+ &speculator,
+ &TrieCreationConfig {
+ width: 1,
+ },
+ 10,
+ );
+
+ verify_stick(
+ &trie_root,
+ std::iter::from_fn(|| Some(ref_rng.next())).take(10).collect(),
+ );
+}
+
+fn verify_bush(
+ trie_root: &TrieNode,
+ mut expected_seeds: HashSet<u64>,
+) {
+ let flat_trie = trie_root.linearize();
+
+ assert_eq!(flat_trie.len(), 4);
+ let token_ids = flat_trie.token_ids().collect::<Vec<u64>>();
+ let token_positions = flat_trie.token_positions().collect::<Vec<usize>>();
+ let token_seeds = flat_trie.token_seeds().collect::<Vec<u64>>();
+ assert_eq!(token_ids.len(), 4);
+ assert_eq!(token_positions.len(), 4);
+ assert_eq!(token_seeds.len(), 4);
+
+ let root_position = flat_trie.index(&trie_root).unwrap();
+ assert_eq!(token_ids[root_position], 0);
+ assert_eq!(token_positions[root_position], 0);
+ assert!(expected_seeds.remove(&token_seeds[root_position]));
+
+ for leaf_token in [1, 2, 3] {
+ let leaf = trie_root.get(leaf_token).unwrap();
+ assert_eq!(leaf.token(), leaf_token);
+ assert!(expected_seeds.remove(&leaf.seed()));
+
+ let position = flat_trie.index(&leaf).unwrap();
+ assert_eq!(token_ids[position], leaf.token());
+ assert_eq!(token_positions[position], 1);
+ assert_eq!(token_seeds[position], leaf.seed());
+ }
+
+ assert!(expected_seeds.is_empty());
+}
+
+#[test]
+fn test_trie_manual_bush() {
+ let mut trie_root = TrieNode::new(0, 0);
+
+ assert!(trie_root.add(TrieNode::new(1, 1)).is_ok());
+ assert!(trie_root.add(TrieNode::new(1, 1)).is_err());
+ assert!(trie_root.add(TrieNode::new(1, 10)).is_err());
+
+ assert!(trie_root.add(TrieNode::new(2, 2)).is_ok());
+ assert!(trie_root.add(TrieNode::new(3, 3)).is_ok());
+
+ verify_bush(&trie_root, (0..4).into_iter().collect());
+}
+
+#[test]
+fn test_trie_from_speculator_bush() {
+ let mut trie_rng = DerivableSeed::new(42);
+ let mut ref_rng = DerivableSeed::new(42);
+
+ let mut hs = HashMap::new();
+ hs.insert(
+ vec![0],
+ vec![(1, 0.33), (2, 0.33), (3, 0.33), (888, 0.005), (999, 0.005)]
+ .into_iter()
+ .collect(),
+ );
+ let speculator = StaticSpeculator::new(hs);
+
+ let trie_root = TrieNode::from_speculator(
+ &[0],
+ &mut trie_rng,
+ &speculator,
+ &TrieCreationConfig {
+ width: 3,
+ },
+ 10,
+ );
+
+ verify_bush(
+ &trie_root,
+ std::iter::from_fn(|| Some(ref_rng.next())).take(4).collect(),
+ );
+}
+
+fn verify_tree(trie_root: &TrieNode) {
+ let flat_trie = trie_root.linearize();
+
+ assert_eq!(flat_trie.len(), 7);
+ let token_ids = flat_trie.token_ids().collect::<Vec<u64>>();
+ let token_positions = flat_trie.token_positions().collect::<Vec<usize>>();
+ let token_seeds = flat_trie.token_seeds().collect::<Vec<u64>>();
+ assert_eq!(token_ids.len(), 7);
+ assert_eq!(token_positions.len(), 7);
+ assert_eq!(token_seeds.len(), 7);
+
+ let root_position = flat_trie.index(&trie_root).unwrap();
+ assert_eq!(token_ids[root_position], 0);
+ assert_eq!(token_positions[root_position], 0);
+ assert_eq!(token_seeds[root_position], 0);
+
+ for (mid_token, mid_seed) in [(1, 1), (2, 1), (3, 2)] {
+ let leaf = trie_root.get(mid_token).unwrap();
+ let position = flat_trie.index(&leaf).unwrap();
+ assert_eq!(token_ids[position], mid_token);
+ assert_eq!(token_positions[position], 1);
+ assert_eq!(token_seeds[position], mid_seed);
+ }
+
+ for (mid_token, leaf_token, leaf_seed) in
+ [(2, 10, 15), (3, 20, 25), (3, 21, 26)]
+ {
+ let leaf = trie_root.get(mid_token).unwrap().get(leaf_token).unwrap();
+ let leaf_position = flat_trie.index(leaf).unwrap();
+ assert_eq!(token_ids[leaf_position], leaf_token);
+ assert_eq!(token_positions[leaf_position], 2);
+ assert_eq!(token_seeds[leaf_position], leaf_seed);
+ }
+}
+
+#[test]
+fn test_trie_manual_tree() {
+ let mut trie_root = TrieNode::new(0, 0);
+
+ assert!(trie_root.add(TrieNode::new(1, 1)).is_ok());
+ assert!(trie_root.add(TrieNode::new(1, 1)).is_err());
+ assert!(trie_root.add(TrieNode::new(1, 10)).is_err());
+
+ assert!(trie_root.add(TrieNode::new(2, 1)).is_ok());
+ assert!(trie_root.add(TrieNode::new(3, 2)).is_ok());
+
+ let mid_b = trie_root.get_mut(2).unwrap();
+ assert!(mid_b.add(TrieNode::new(10, 15)).is_ok());
+
+ let mid_c = trie_root.get_mut(3).unwrap();
+ assert!(mid_c.add(TrieNode::new(20, 25)).is_ok());
+ assert!(mid_c.add(TrieNode::new(21, 26)).is_ok());
+
+ verify_tree(&trie_root)
+}