A high-performance inference engine for AI models
trie rewrite and guuumbel speculative decoding (#50)
uuuvnfcfdfa2ba3aa
diff --git a/Cargo.lock b/Cargo.lockindex b12c69953..19f001918 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -2648,6 +2648,7 @@ dependencies = ["clap","half","is_close",+ "itertools","libc","metal","minijinja",diff --git a/crates/uzu/Cargo.toml b/crates/uzu/Cargo.tomlindex 5becfbbac..a1c60081b 100644--- a/crates/uzu/Cargo.toml+++ b/crates/uzu/Cargo.toml@@ -26,6 +26,7 @@ minijinja-contrib = { version = "2.12.0", features = ["pycompat"] }clap = { version = "4.5.47", features = ["derive"] }rand = "0.9.2"sysinfo = "0.37.0"+itertools = "0.14.0"[dev-dependencies]is_close = "0.1.3"diff --git a/crates/uzu/src/backends/metal/forward_pass/state.rs b/crates/uzu/src/backends/metal/forward_pass/state.rsindex 8ddb134ee..6fb3f027d 100644--- a/crates/uzu/src/backends/metal/forward_pass/state.rs+++ b/crates/uzu/src/backends/metal/forward_pass/state.rs@@ -937,9 +937,9 @@ impl ForwardPassState {shared_buffers: Rc<RefCell<SharedBuffers>>,token_ids: &[u64],token_positions: &[usize],+ token_seeds: &[u64],active_suffix_length: usize,is_prefilling: bool,- token_seeds: &[u64],trace: bool,external_bias_fn: Option<&dyn Fn(usize, usize) -> bool>,skip_token_ids_copy: bool,diff --git a/crates/uzu/src/generator/generator.rs b/crates/uzu/src/generator/generator.rsindex 7b47ff0f6..1ada08fb6 100644--- a/crates/uzu/src/generator/generator.rs+++ b/crates/uzu/src/generator/generator.rs@@ -1,6 +1,9 @@-use std::{collections::HashMap, path::Path, sync::Arc, time::Instant};+use std::{+ collections::HashMap, iter::repeat_n, path::Path, sync::Arc, time::Instant,+};use block::ConcreteBlock;+use itertools::izip;use mpsgraph::CommandBuffer;use super::{@@ -15,12 +18,12 @@ use crate::{ForwardPassState, INVALID_POSITION,encodable_with_state::{EncodableWithState, EncodingParameters},},- linearizer::trie::{TokenTrie, TrieCreationConfig},session::{config::DecodingConfig,parameter::{ConfigResolvableValue, SamplingMethod},types::Error,},+ trie::{TrieCreationConfig, TrieNode},utils::env_utils::MetalEnvVar,};@@ -73,86 +76,76 @@ impl Generator {self.tokens.extend(tokens.clone());+ let tokens_length = tokens.len();+let prefill_step_size = self.decoding_config.prefill_step_size.resolve(&self.context.model_config);-- let tokens_length = tokens.len();- let number_of_prefill_steps =- (tokens_length as f32 / prefill_step_size as f32).ceil() as usize;- let total_prefill_tokens_count =- number_of_prefill_steps * prefill_step_size;- let unused_tokens_count = total_prefill_tokens_count - tokens_length;+ let prefill_steps = tokens_length.div_ceil(prefill_step_size);+ let prefill_size = prefill_steps * prefill_step_size;let speculator = &self.decoding_config.speculator_config.speculator;- let speculated_suffix = TokenTrie::from_speculator(++ let suffix_length = prefill_size - tokens_length;+ let suffix_root = TrieNode::from_speculator(&tokens,&mut self.context.next_seed,- false,speculator.as_ref(),&TrieCreationConfig::default(),- unused_tokens_count,- )- .linearize(0, unused_tokens_count);-- let zero_padding_tokens: Vec<u64> =- vec![0; unused_tokens_count - speculated_suffix.tokens.len()];-- let padded_tokens = [- &tokens[..],- &speculated_suffix.tokens[..],- &zero_padding_tokens[..],- ]- .concat();-- let mut padded_positions: Vec<usize> =- (prefix_offset..prefix_offset + tokens_length).collect();- padded_positions.extend(- speculated_suffix- .indices- .iter()- .map(|index| index + prefix_offset + tokens_length),+ suffix_length + 1,);- let padding_count =- unused_tokens_count - speculated_suffix.tokens.len();- padded_positions- .extend(std::iter::repeat(INVALID_POSITION).take(padding_count));-- let zero_padding_left_seeds: Vec<u64> = vec![0; tokens_length];- let zero_padding_right_seeds: Vec<u64> =- vec![0; unused_tokens_count - speculated_suffix.seeds.len()];- let padded_seeds = [- &zero_padding_left_seeds[..],- &speculated_suffix.seeds[..],- &zero_padding_right_seeds[..],- ]- .concat();+ let flat_trie = suffix_root.linearize();+ let active_suffix_length = flat_trie.len() - 1;++ let token_ids = tokens+ .iter()+ .copied()+ .take(tokens_length - 1)+ .chain(flat_trie.token_ids())+ .chain(repeat_n(0, suffix_length - active_suffix_length))+ .collect::<Box<[u64]>>();++ let token_positions = (prefix_offset+ ..prefix_offset + tokens_length - 1)+ .chain(flat_trie.token_positions().map(|trie_position| {+ prefix_offset + tokens_length - 1 + trie_position+ }))+ .chain(repeat_n(+ INVALID_POSITION,+ suffix_length - active_suffix_length,+ ))+ .collect::<Box<[usize]>>();++ let token_seeds = repeat_n(0, tokens_length - 1)+ .chain(flat_trie.token_seeds())+ .chain(repeat_n(0, suffix_length - active_suffix_length))+ .collect::<Box<[u64]>>();let mut last_state: Option<ForwardPassState> = None;let mut run_times: Vec<f64> = Vec::new();// Process each prefill step and update the KV cache.- for step in 0..number_of_prefill_steps {+ for (step, (step_token_ids, step_token_positions, step_token_seeds)) in+ izip!(+ token_ids.chunks(prefill_step_size),+ token_positions.chunks(prefill_step_size),+ token_seeds.chunks(prefill_step_size)+ )+ .enumerate()+ {let tokens_start_index = step * prefill_step_size;let tokens_end_index = tokens_start_index + prefill_step_size;- let tokens_for_step =- &padded_tokens[tokens_start_index..tokens_end_index];- let positions_for_step =- &padded_positions[tokens_start_index..tokens_end_index];- let active_suffix_length = positions_for_step+ let active_suffix_length = step_token_positions.iter().position(|&pos| pos == INVALID_POSITION).unwrap_or(prefill_step_size);- let seeds_for_step =- &padded_seeds[tokens_start_index..tokens_end_index];- let is_last_prefill_step = step == number_of_prefill_steps - 1;+ let is_last_prefill_step = step == prefill_steps - 1;let should_sample_after_step =sample_suffix && is_last_prefill_step;- let is_first_prefill = step == 0;let should_capture =- self.gpu_capture.should_capture_prefill(is_first_prefill);+ self.gpu_capture.should_capture_prefill(step == 0);if should_capture {let _ = self@@ -165,9 +158,9 @@ impl Generator {});let task = GeneratorRunTask {- token_ids: tokens_for_step.to_vec(),- token_positions: positions_for_step.to_vec(),- token_seeds: seeds_for_step.to_vec(),+ token_ids: step_token_ids,+ token_positions: step_token_positions,+ token_seeds: step_token_seeds,expected_number_of_new_tokens: prefill_step_size,active_suffix_length,is_prefilling: !should_sample_after_step,@@ -196,7 +189,7 @@ impl Generator {..step_end_token_index).map(|idx| idx + prefix_offset).collect();- if step == number_of_prefill_steps - 1 && sample_suffix {+ if step == prefill_steps - 1 && sample_suffix {// Exclude the last token because it belongs to the suffix for sampling.positions_for_step.pop();}@@ -237,42 +230,22 @@ impl Generator {}let sampled_tokens = self.sample(&mut final_state)?;- let last_suffix_start =- prefill_step_size * (number_of_prefill_steps - 1);-- let sampling_row = (tokens_length - last_suffix_start) - 1;- let mut current_token_index: isize = -1;- let mut accepted_token_indices: Vec<usize> = vec![sampling_row];- let mut accepted_tokens: Vec<u64> = Vec::new();- loop {- let current_index_in_window =- ((current_token_index + tokens_length as isize)- % prefill_step_size as isize) as usize;- let new_token = sampled_tokens[current_index_in_window];- accepted_tokens.push(new_token);-- if let Some(map) =- speculated_suffix.transition_map.get(¤t_token_index)- {- if let Some(&next_token_index) = map.get(&new_token) {- accepted_token_indices.push(next_token_index as usize);- current_token_index = next_token_index;- } else {- break;- }- } else {- break;- }- }+ let last_suffix_start = prefill_step_size * (prefill_steps - 1);+ let suffix_root_index = (tokens_length - last_suffix_start) - 1;++ let (accepted_tokens, accepted_token_indices) =+ flat_trie.accept(&sampled_tokens[suffix_root_index..]);self.update_cache_layers(- &accepted_token_indices,+ &accepted_token_indices+ .into_iter()+ .map(|p| suffix_root_index + p)+ .collect::<Box<[usize]>>(),Some(last_suffix_start),false,);self.tokens.extend(accepted_tokens.clone());-self.sync_prefix();Ok(PrefillResult {@@ -286,45 +259,43 @@ impl Generator {sampling_method: SamplingMethod,) -> Result<GenerateResult, Error> {let speculator = &self.decoding_config.speculator_config.speculator;- let speculated_suffix = TokenTrie::from_speculator(++ let suffix_length = self.decoding_config.generate_suffix_length();+ let suffix_root = TrieNode::from_speculator(&self.tokens,&mut self.context.next_seed,- true,speculator.as_ref(),&TrieCreationConfig::default(),- self.decoding_config.generate_suffix_length(),- )- .linearize(0, self.decoding_config.generate_suffix_length());-- let expected_suffix_length =- self.decoding_config.generate_suffix_length();- let unused_tokens_count =- expected_suffix_length - speculated_suffix.tokens.len();+ suffix_length,+ );- let zero_padding_tokens: Vec<u64> = vec![0; unused_tokens_count];- let padded_tokens =- [speculated_suffix.tokens.clone(), zero_padding_tokens].concat();+ let flat_trie = suffix_root.linearize();+ let active_suffix_length = flat_trie.len();- let zero_padding_seeds: Vec<u64> = vec![0; unused_tokens_count];- let padded_seeds =- [speculated_suffix.seeds.clone(), zero_padding_seeds].concat();+ let token_ids = flat_trie+ .token_ids()+ .chain(repeat_n(0, suffix_length - active_suffix_length))+ .collect::<Box<[u64]>>();let start_position = self.tokens.len() - 1;-- let padded_positions: Vec<usize> = speculated_suffix- .indices- .iter()- .map(|idx| idx + start_position)- .chain(- std::iter::repeat(INVALID_POSITION).take(unused_tokens_count),- )- .collect();- let active_suffix_length = speculated_suffix.tokens.len();+ let token_positions = flat_trie+ .token_positions()+ .map(|trie_position| start_position + trie_position)+ .chain(repeat_n(+ INVALID_POSITION,+ suffix_length - active_suffix_length,+ ))+ .collect::<Box<[usize]>>();++ let token_seeds = flat_trie+ .token_seeds()+ .chain(repeat_n(0, suffix_length - active_suffix_length))+ .collect::<Box<[u64]>>();let task = GeneratorRunTask {- token_ids: padded_tokens,- token_positions: padded_positions,- token_seeds: padded_seeds,+ token_ids: &token_ids,+ token_positions: &token_positions,+ token_seeds: &token_seeds,expected_number_of_new_tokens: 1,active_suffix_length,is_prefilling: false,@@ -339,27 +310,10 @@ impl Generator {let sampled_tokens = self.sample(&mut state)?;- let mut accepted_token_indices = Vec::new();- let mut accepted_tokens = Vec::new();- let mut current_token_index: isize = 0;- loop {- let new_token = sampled_tokens[current_token_index as usize];- accepted_tokens.push(new_token);- if let Some(map) =- speculated_suffix.transition_map.get(&(current_token_index))- {- if let Some(&next_token_index) = map.get(&new_token) {- accepted_token_indices.push(next_token_index as usize);- current_token_index = next_token_index;- } else {- break;- }- } else {- break;- }- }+ let (accepted_tokens, accepted_token_indices) =+ flat_trie.accept(&sampled_tokens);- self.update_cache_layers(&accepted_token_indices, None, false);+ self.update_cache_layers(&accepted_token_indices[1..], None, false);self.tokens.extend(accepted_tokens.clone());self.sync_prefix();@@ -393,9 +347,9 @@ impl Generator {suffix_length: usize,) {let task = GeneratorRunTask {- token_ids: vec![0; suffix_length],- token_positions: (0..suffix_length).collect::<Vec<usize>>(),- token_seeds: vec![0; suffix_length],+ token_ids: &repeat_n(0, suffix_length).collect::<Box<[u64]>>(),+ token_positions: &(0..suffix_length).collect::<Box<[usize]>>(),+ token_seeds: &repeat_n(0, suffix_length).collect::<Box<[u64]>>(),expected_number_of_new_tokens: suffix_length,active_suffix_length: suffix_length,is_prefilling: false,@@ -618,9 +572,9 @@ impl Generator {let last_token = *self.tokens.last().ok_or(Error::PrefillFailed)?;let task = GeneratorRunTask {- token_ids: vec![last_token],- token_positions: vec![0], // Ignored, using async buffer- token_seeds: vec![0], // Ignored, using async buffer+ token_ids: &[last_token],+ token_positions: &[0], // Ignored, using async buffer+ token_seeds: &[0], // Ignored, using async bufferexpected_number_of_new_tokens: 1,active_suffix_length: 1,is_prefilling: false,@@ -638,9 +592,9 @@ impl Generator {self.context.shared_buffers.clone(),&task.token_ids,&task.token_positions,+ &task.token_seeds,task.active_suffix_length,task.is_prefilling,- &task.token_seeds,false,None,is_continuation,diff --git a/crates/uzu/src/generator/tasks.rs b/crates/uzu/src/generator/tasks.rsindex 9d97f175d..dcf7fb170 100644--- a/crates/uzu/src/generator/tasks.rs+++ b/crates/uzu/src/generator/tasks.rs@@ -33,21 +33,21 @@ impl GeneratorEncodedTask {}#[derive(Debug, Clone)]-pub struct GeneratorRunTask {- pub token_ids: Vec<u64>,- pub token_positions: Vec<usize>,- pub token_seeds: Vec<u64>,+pub struct GeneratorRunTask<'a> {+ pub token_ids: &'a [u64],+ pub token_positions: &'a [usize],+ pub token_seeds: &'a [u64],pub expected_number_of_new_tokens: usize,pub active_suffix_length: usize,pub is_prefilling: bool,}-impl GeneratorRunTask {+impl<'a> GeneratorRunTask<'a> {pub fn speculate_next_task(&self) -> Self {GeneratorRunTask {- token_ids: self.token_ids.clone(),- token_positions: self.token_positions.clone(),- token_seeds: self.token_seeds.clone(),+ token_ids: self.token_ids,+ token_positions: self.token_positions,+ token_seeds: self.token_seeds,expected_number_of_new_tokens: self.expected_number_of_new_tokens,active_suffix_length: self.active_suffix_length,is_prefilling: self.is_prefilling,@@ -76,11 +76,11 @@ impl GeneratorRunTask {&context.scratch_buffers,context.cache_layers.clone(),context.shared_buffers.clone(),- &self.token_ids,- &self.token_positions,+ self.token_ids,+ self.token_positions,+ self.token_seeds,self.active_suffix_length,self.is_prefilling,- &self.token_seeds,false,external_bias_fn,false,diff --git a/crates/uzu/src/lib.rs b/crates/uzu/src/lib.rsindex 8a9f80912..e28e2d944 100644--- a/crates/uzu/src/lib.rs+++ b/crates/uzu/src/lib.rs@@ -1,11 +1,11 @@pub mod backends;pub mod config;pub mod generator;-pub mod linearizer;pub mod parameters;pub mod session;pub mod speculators;pub mod tracer;+pub mod trie;pub mod utils;pub use utils::*;pub mod device;diff --git a/crates/uzu/src/linearizer/mod.rs b/crates/uzu/src/linearizer/mod.rsdeleted file mode 100644index 95655d93b..000000000--- a/crates/uzu/src/linearizer/mod.rs+++ /dev/null@@ -1,8 +0,0 @@-pub mod node;-pub mod speculated_suffix;-pub mod transition;-pub mod trie;--#[cfg(test)]-mod tests_node;-mod tests_trie;diff --git a/crates/uzu/src/linearizer/node.rs b/crates/uzu/src/linearizer/node.rsdeleted file mode 100644index 62df6cb1d..000000000--- a/crates/uzu/src/linearizer/node.rs+++ /dev/null@@ -1,80 +0,0 @@-use super::transition::Transition;--#[derive(Debug)]-pub struct Node {- transitions: Vec<Transition>,-}--impl Default for Node {- fn default() -> Self {- Self::new()- }-}--impl Node {- pub fn new() -> Self {- Self {- transitions: Vec::new(),- }- }-- pub fn has_transition(- &self,- token: u64,- ) -> bool {- self.transitions.iter().any(|t| t.token() == token)- }-- pub fn get_next(- &self,- token: u64,- ) -> Option<&Node> {- self.transitions.iter().find(|t| t.token() == token).map(|t| &t.node)- }-- pub fn get_or_insert_next(- &mut self,- token: u64,- seed: u64,- ) -> &mut Node {- if let Some(pos) =- self.transitions.iter().position(|t| t.token() == token)- {- return &mut self.transitions[pos].node;- }-- self.transitions.push(Transition::new(token, seed));- &mut self.transitions.last_mut().unwrap().node- }-- pub fn dfs_with_path<F>(- &self,- path: &mut Vec<(isize, u64, u64)>,- callback: &mut F,- ) -> isize- where- F: FnMut(&[(isize, u64, u64)]),- {- let mut node_index = path.last().map(|(idx, _, _)| *idx).unwrap_or(-1);-- for transition in self.transitions.iter() {- node_index += 1;- path.push((node_index, transition.token(), transition.seed()));- callback(path);- node_index = transition.node.dfs_with_path(path, callback);- path.pop();- }-- node_index- }-- pub fn dfs<F>(- &self,- mut callback: F,- ) where- F: FnMut(&[(isize, u64, u64)]),- {- let mut path: Vec<(isize, u64, u64)> = Vec::new();- self.dfs_with_path(&mut path, &mut callback);- }-}diff --git a/crates/uzu/src/linearizer/speculated_suffix.rs b/crates/uzu/src/linearizer/speculated_suffix.rsdeleted file mode 100644index 2f3874036..000000000--- a/crates/uzu/src/linearizer/speculated_suffix.rs+++ /dev/null@@ -1,9 +0,0 @@-use std::collections::HashMap;--#[derive(Debug)]-pub struct SpeculatedSuffix {- pub tokens: Vec<u64>,- pub seeds: Vec<u64>,- pub indices: Vec<usize>,- pub transition_map: HashMap<isize, HashMap<u64, isize>>,-}diff --git a/crates/uzu/src/linearizer/tests_node.rs b/crates/uzu/src/linearizer/tests_node.rsdeleted file mode 100644index 9f4680d76..000000000--- a/crates/uzu/src/linearizer/tests_node.rs+++ /dev/null@@ -1,267 +0,0 @@-#[cfg(test)]-mod tests {- use std::collections::HashSet;-- use super::super::node::Node;-- #[test]- fn test_new_node_has_no_transitions() {- let node = Node::new();- for i in 0..10 {- assert!(!node.has_transition(i));- }- }-- #[test]- fn test_has_transition() {- let mut node = Node::new();- node.get_or_insert_next(42, 0);-- assert!(node.has_transition(42));- assert!(!node.has_transition(43));- }-- #[test]- fn test_get_next() {- let mut node = Node::new();- node.get_or_insert_next(1, 0);-- assert!(node.get_next(1).is_some());- assert!(node.get_next(2).is_none());- }-- #[test]- fn test_get_or_insert_next() {- let mut node = Node::new();-- // First insertion should create a new node- let node1 = node.get_or_insert_next(1, 0);- assert!(!node1.has_transition(99));-- // Add a transition to the child node- node1.get_or_insert_next(99, 0);-- // Get the same node again- let node1_again = node.get_or_insert_next(1, 0);-- // Verify it's the same node by checking it has the transition we added- assert!(node1_again.has_transition(99));-- // Different token should give a different node- let node2 = node.get_or_insert_next(2, 0);- assert!(!node2.has_transition(99));- }-- #[test]- fn test_multiple_transitions() {- let mut node = Node::new();-- // Insert several transitions- for i in 0..5 {- node.get_or_insert_next(i, 0);- }-- // Verify all transitions exist- for i in 0..5 {- assert!(node.has_transition(i));- }-- // Verify non-existent transitions- for i in 5..10 {- assert!(!node.has_transition(i));- }- }-- #[test]- fn test_dfs_with_path_empty_node() {- let node = Node::new();- let mut path: Vec<(isize, u64, u64)> = Vec::new();- let mut visited = Vec::new();-- node.dfs_with_path(&mut path, &mut |current_path: &[(- isize,- u64,- u64,- )]| {- let tokens: Vec<u64> =- current_path.iter().map(|(_, token, _)| *token).collect();- visited.push(tokens);- });-- assert!(visited.is_empty());- }-- #[test]- fn test_dfs_with_path_single_node() {- let mut node = Node::new();- node.get_or_insert_next(42, 0);-- let mut path: Vec<(isize, u64, u64)> = Vec::new();- let mut visited = Vec::new();-- node.dfs_with_path(&mut path, &mut |current_path: &[(- isize,- u64,- u64,- )]| {- let tokens: Vec<u64> =- current_path.iter().map(|(_, token, _)| *token).collect();- visited.push(tokens);- });-- assert_eq!(visited.len(), 1);- assert_eq!(visited[0], vec![42]);- }-- #[test]- fn test_dfs_with_path_complex_tree() {- let mut root = Node::new();-- // Build a sample trie: [1 -> 2 -> 3], [1 -> 2 -> 4], [1 -> 5], [6]- let node1 = root.get_or_insert_next(1, 0);- let node2 = node1.get_or_insert_next(2, 0);- node2.get_or_insert_next(3, 0);- node2.get_or_insert_next(4, 0);- node1.get_or_insert_next(5, 0);- root.get_or_insert_next(6, 0);-- let mut path: Vec<(isize, u64, u64)> = Vec::new();- let mut visited = Vec::new();-- root.dfs_with_path(&mut path, &mut |current_path: &[(- isize,- u64,- u64,- )]| {- let tokens: Vec<u64> =- current_path.iter().map(|(_, token, _)| *token).collect();- visited.push(tokens);- });-- // Check all expected paths were visited- assert_eq!(visited.len(), 6);- assert!(visited.contains(&vec![1]));- assert!(visited.contains(&vec![1, 2]));- assert!(visited.contains(&vec![1, 2, 3]));- assert!(visited.contains(&vec![1, 2, 4]));- assert!(visited.contains(&vec![1, 5]));- assert!(visited.contains(&vec![6]));- }-- #[test]- fn test_dfs_path_indices() {- let mut root = Node::new();-- // Build a simple trie: [1], [2]- root.get_or_insert_next(1, 0);- root.get_or_insert_next(2, 0);-- let mut path: Vec<(isize, u64, u64)> = Vec::new();- let mut indices = Vec::new();-- root.dfs_with_path(&mut path, &mut |current_path: &[(- isize,- u64,- u64,- )]| {- if !current_path.is_empty() {- indices.push(current_path.last().unwrap().0);- }- });-- // Check indices are unique and as expected- assert_eq!(indices.len(), 2);- assert_eq!(indices[0], 0);- assert_eq!(indices[1], 1);- }-- #[test]- fn test_simplified_dfs_empty_node() {- let node = Node::new();- let mut visited = Vec::new();-- node.dfs(|current_path: &[(isize, u64, u64)]| {- let tokens: Vec<u64> =- current_path.iter().map(|(_, token, _)| *token).collect();- visited.push(tokens);- });-- assert!(visited.is_empty());- }-- #[test]- fn test_simplified_dfs_complex_tree() {- let mut root = Node::new();-- // Build a sample trie: [1 -> 2 -> 3], [1 -> 2 -> 4], [1 -> 5], [6]- let node1 = root.get_or_insert_next(1, 0);- let node2 = node1.get_or_insert_next(2, 0);- node2.get_or_insert_next(3, 0);- node2.get_or_insert_next(4, 0);- node1.get_or_insert_next(5, 0);- root.get_or_insert_next(6, 0);-- let mut visited = Vec::new();-- root.dfs(|current_path: &[(isize, u64, u64)]| {- let tokens: Vec<u64> =- current_path.iter().map(|(_, token, _)| *token).collect();- visited.push(tokens);- });-- // Check all expected paths were visited- assert_eq!(visited.len(), 6);- assert!(visited.contains(&vec![1]));- assert!(visited.contains(&vec![1, 2]));- assert!(visited.contains(&vec![1, 2, 3]));- assert!(visited.contains(&vec![1, 2, 4]));- assert!(visited.contains(&vec![1, 5]));- assert!(visited.contains(&vec![6]));- }-- #[test]- fn test_both_dfs_methods_equivalent() {- let mut root = Node::new();-- // Build a more complex trie- let node1 = root.get_or_insert_next(10, 0);- let node2 = node1.get_or_insert_next(20, 0);- node2.get_or_insert_next(30, 0);- node2.get_or_insert_next(40, 0);- node1.get_or_insert_next(50, 0);- let node3 = root.get_or_insert_next(60, 0);- node3.get_or_insert_next(70, 0);-- // Get results from dfs_with_path- let mut path: Vec<(isize, u64, u64)> = Vec::new();- let mut visited1 = Vec::new();-- root.dfs_with_path(&mut path, &mut |current_path: &[(- isize,- u64,- u64,- )]| {- let tokens: Vec<u64> =- current_path.iter().map(|(_, token, _)| *token).collect();- visited1.push(tokens);- });-- // Get results from simplified dfs- let mut visited2 = Vec::new();-- root.dfs(|current_path: &[(isize, u64, u64)]| {- let tokens: Vec<u64> =- current_path.iter().map(|(_, token, _)| *token).collect();- visited2.push(tokens);- });-- // Check both methods give the same results- assert_eq!(visited1.len(), visited2.len());-- // Convert to sets to ignore order differences- let set1: HashSet<Vec<u64>> = visited1.into_iter().collect();- let set2: HashSet<Vec<u64>> = visited2.into_iter().collect();-- assert_eq!(set1, set2);- }-}diff --git a/crates/uzu/src/linearizer/tests_trie.rs b/crates/uzu/src/linearizer/tests_trie.rsdeleted file mode 100644index f44f8c04d..000000000--- a/crates/uzu/src/linearizer/tests_trie.rs+++ /dev/null@@ -1,297 +0,0 @@-#[cfg(test)]-mod tests {- use std::collections::HashMap;-- use crate::{- generator::rng::DerivableSeed,- linearizer::trie::{TokenTrie, TrieCreationConfig},- speculators::speculator::Speculator,- };-- struct StaticSpeculator {- responses: HashMap<Vec<u64>, HashMap<u64, f32>>,- }-- impl StaticSpeculator {- fn new(responses: HashMap<Vec<u64>, HashMap<u64, f32>>) -> Self {- Self {- responses,- }- }- }-- impl Speculator for StaticSpeculator {- fn speculate(- &self,- prefix: &[u64],- ) -> HashMap<u64, f32> {- self.responses.get(prefix).cloned().unwrap_or_default()- }- }-- #[test]- fn test_from_speculator_applies_prob_cutoff() {- let mut responses = HashMap::new();- responses.insert(- vec![42],- HashMap::from([(1, 0.9_f32), (2, 0.05_f32), (3, 0.2_f32)]),- );- responses.insert(vec![42, 1], HashMap::new());-- let speculator = StaticSpeculator::new(responses);- let mut seed = DerivableSeed::new(1);-- let trie = TokenTrie::from_speculator(- &[42],- &mut seed,- false,- &speculator,- &TrieCreationConfig::default(),- 5,- );- let suffix = trie.linearize(0, 5);-- assert_eq!(suffix.tokens, vec![1, 3]);- assert!(!suffix.tokens.contains(&2));- assert_eq!(suffix.transition_map.len(), 3);- }-- #[test]- fn test_from_speculator_respects_top_k_limit() {- let mut responses = HashMap::new();- responses.insert(vec![5], HashMap::from([(9, 0.8_f32), (10, 0.7_f32)]));- responses.insert(vec![5, 9], HashMap::from([(11, 0.6_f32)]));- responses.insert(vec![5, 9, 11], HashMap::new());-- let speculator = StaticSpeculator::new(responses);- let mut seed = DerivableSeed::new(7);-- let trie = TokenTrie::from_speculator(- &[5],- &mut seed,- false,- &speculator,- &TrieCreationConfig {- top_k: 1,- prob_cutoff: 0.0,- },- 3,- );-- let suffix = trie.linearize(0, 4);-- assert_eq!(suffix.tokens, vec![9, 11]);- assert!(!suffix.tokens.contains(&10));- }-- #[test]- fn test_from_speculator_includes_last_prefix_token_and_seeds() {- let mut responses = HashMap::new();- responses.insert(vec![2, 4], HashMap::from([(8, 0.9_f32)]));- responses.insert(vec![2, 4, 8], HashMap::new());-- let speculator = StaticSpeculator::new(responses);-- let mut seed = DerivableSeed::new(99);- let mut expected_seed_stream = DerivableSeed::new(99);- let expected_seeds =- vec![expected_seed_stream.next(), expected_seed_stream.next()];-- let trie = TokenTrie::from_speculator(- &[2, 4],- &mut seed,- true,- &speculator,- &TrieCreationConfig::default(),- 2,- );-- let suffix = trie.linearize(0, 3);-- assert_eq!(suffix.tokens, vec![4, 8]);- assert_eq!(suffix.seeds, expected_seeds);- assert_eq!(suffix.indices, vec![0, 1]);- }-- #[test]- fn test_from_speculator_stops_at_max_length() {- let mut responses = HashMap::new();- responses.insert(- vec![1],- HashMap::from([(5, 0.9_f32), (6, 0.8_f32), (7, 0.7_f32)]),- );- responses- .insert(vec![1, 5], HashMap::from([(8, 0.6_f32), (9, 0.5_f32)]));- responses.insert(vec![1, 5, 8], HashMap::from([(10, 0.4_f32)]));-- let speculator = StaticSpeculator::new(responses);- let mut seed = DerivableSeed::new(1234);-- let trie = TokenTrie::from_speculator(- &[1],- &mut seed,- false,- &speculator,- &TrieCreationConfig::default(),- 3,- );-- let suffix = trie.linearize(0, 5);-- assert_eq!(suffix.tokens.len(), 3);- assert!(suffix.tokens.contains(&5));- assert!(suffix.tokens.contains(&6));- assert!(suffix.tokens.contains(&8));- assert!(!suffix.tokens.contains(&9));- }-- #[test]- fn test_from_speculator_with_empty_responses() {- let speculator = StaticSpeculator::new(HashMap::new());-- let mut seed = DerivableSeed::new(555);- let mut expected_seed_stream = DerivableSeed::new(555);- let expected_first_seed = expected_seed_stream.next();-- let trie = TokenTrie::from_speculator(- &[12, 13],- &mut seed,- true,- &speculator,- &TrieCreationConfig::default(),- 4,- );-- let suffix = trie.linearize(0, 5);-- // Only the last prefix token should be included; no speculated tokens.- assert_eq!(suffix.tokens, vec![13]);- assert_eq!(suffix.indices, vec![0]);- assert_eq!(suffix.seeds, vec![expected_first_seed]);-- assert_eq!(suffix.transition_map.len(), 2);- let root_transitions = &suffix.transition_map[&-1];- assert_eq!(root_transitions.len(), 1);- assert_eq!(root_transitions.get(&13), Some(&0));- }-- #[test]- fn test_linearize_empty_trie() {- let trie = TokenTrie::new();- let suffix = trie.linearize(0, 10);-- assert!(suffix.tokens.is_empty());- assert!(suffix.indices.is_empty());- assert!(suffix.transition_map.len() == 1);- }-- #[test]- fn test_linearize_single_sequence() {- let mut trie = TokenTrie::new();- trie.insert(&[1, 2, 3]);-- let suffix = trie.linearize(0, 5);-- assert_eq!(suffix.tokens.len(), 3);- assert!(suffix.tokens.contains(&1));- assert!(suffix.tokens.contains(&2));- assert!(suffix.tokens.contains(&3));-- assert_eq!(suffix.transition_map.len(), 4);-- let root_transitions = &suffix.transition_map[&-1];- assert!(root_transitions.contains_key(&1));-- let token1_idx = *root_transitions.get(&1).unwrap();- let token1_transitions = &suffix.transition_map[&token1_idx];- assert!(token1_transitions.contains_key(&2));-- let token2_idx = *token1_transitions.get(&2).unwrap();- let token2_transitions = &suffix.transition_map[&token2_idx];- assert!(token2_transitions.contains_key(&3));- }-- #[test]- fn test_linearize_branching_trie() {- let mut trie = TokenTrie::new();- trie.insert(&[1, 2, 3]);- trie.insert(&[1, 2, 4]);- trie.insert(&[5, 6]);-- let suffix = trie.linearize(10, 10);-- assert!(suffix.tokens.contains(&1));- assert!(suffix.tokens.contains(&2));- assert!(suffix.tokens.contains(&3));- assert!(suffix.tokens.contains(&4));- assert!(suffix.tokens.contains(&5));- assert!(suffix.tokens.contains(&6));-- assert_eq!(suffix.tokens.len(), 6);-- assert_eq!(suffix.transition_map.len(), 7);-- let root_transitions = &suffix.transition_map[&-1];- assert!(root_transitions.contains_key(&1));- assert!(root_transitions.contains_key(&5));-- let token1_idx = *root_transitions.get(&1).unwrap();- let token1_transitions = &suffix.transition_map[&token1_idx];- assert!(token1_transitions.contains_key(&2));-- let token2_idx = *token1_transitions.get(&2).unwrap();- let token2_transitions = &suffix.transition_map[&token2_idx];- assert!(token2_transitions.contains_key(&3));- assert!(token2_transitions.contains_key(&4));-- let token5_idx = *root_transitions.get(&5).unwrap();- let token5_transitions = &suffix.transition_map[&token5_idx];- assert!(token5_transitions.contains_key(&6));- }-- #[test]- fn test_linearize_max_length() {- let mut trie = TokenTrie::new();- trie.insert(&[1, 2, 3, 4, 5, 6, 7, 8, 9, 10]);-- let suffix = trie.linearize(0, 5);-- assert_eq!(suffix.tokens.len(), 5);- assert_eq!(suffix.indices.len(), 5);- assert_eq!(suffix.transition_map.len(), 6);-- for i in 1..=5 {- assert!(suffix.tokens.contains(&i));- }-- let root_transitions = &suffix.transition_map[&-1];- assert!(root_transitions.contains_key(&1));-- let mut current_idx = *root_transitions.get(&1).unwrap();- for i in 2..=4 {- let transitions = &suffix.transition_map[¤t_idx];- assert!(transitions.contains_key(&i));- current_idx = *transitions.get(&i).unwrap();- }- }-- #[test]- fn test_linearize_start_index() {- let mut trie = TokenTrie::new();- trie.insert(&[1, 2, 3]);-- let suffix = trie.linearize(100, 10);-- assert_eq!(suffix.tokens.len(), 3);- assert!(suffix.tokens.contains(&1));- assert!(suffix.tokens.contains(&2));- assert!(suffix.tokens.contains(&3));-- for idx in suffix.indices.iter() {- assert!(*idx >= 100 && *idx <= 102);- }-- let root_transitions = &suffix.transition_map[&-1];- assert!(root_transitions.contains_key(&1));- }-}diff --git a/crates/uzu/src/linearizer/transition.rs b/crates/uzu/src/linearizer/transition.rsdeleted file mode 100644index 76426fe6c..000000000--- a/crates/uzu/src/linearizer/transition.rs+++ /dev/null@@ -1,29 +0,0 @@-use super::node::Node;--#[derive(Debug)]-pub struct Transition {- token: u64,- seed: u64,- pub node: Node,-}--impl Transition {- pub fn new(- token: u64,- seed: u64,- ) -> Self {- Self {- token,- seed,- node: Node::new(),- }- }-- pub fn token(&self) -> u64 {- self.token- }-- pub fn seed(&self) -> u64 {- self.seed- }-}diff --git a/crates/uzu/src/linearizer/trie.rs b/crates/uzu/src/linearizer/trie.rsdeleted file mode 100644index a0486e9f1..000000000--- a/crates/uzu/src/linearizer/trie.rs+++ /dev/null@@ -1,169 +0,0 @@-use std::collections::HashMap;--use crate::{- generator::rng::DerivableSeed, speculators::speculator::Speculator,-};--use super::{node::Node, speculated_suffix::SpeculatedSuffix};--pub struct TrieCreationConfig {- pub top_k: usize,- pub prob_cutoff: f32,-}--impl Default for TrieCreationConfig {- fn default() -> Self {- TrieCreationConfig {- top_k: 2,- prob_cutoff: 0.1,- }- }-}--#[derive(Debug)]-pub struct TokenTrie {- root: Node,-}--impl Default for TokenTrie {- fn default() -> Self {- Self::new()- }-}--impl TokenTrie {- pub fn new() -> Self {- Self {- root: Node::new(),- }- }-- pub fn from_sequences(sequences: &[Vec<u64>]) -> Self {- let mut trie = Self::new();- for sequence in sequences {- trie.insert(sequence);- }- trie- }-- pub fn from_speculator(- prefix: &[u64],- seed: &mut DerivableSeed,- include_last_prefix_token: bool,- speculator: &dyn Speculator,- creation_config: &TrieCreationConfig,- max_length: usize,- ) -> Self {- let mut trie = Self::new();- let mut current_node = &mut trie.root;- let mut trie_size = 0;-- if include_last_prefix_token {- current_node = current_node- .get_or_insert_next(*prefix.last().unwrap(), seed.next());- trie_size += 1;- }-- let mut speculation_prefix = prefix.to_vec();- while trie_size < max_length {- let speculation_result = speculator.speculate(&speculation_prefix);-- let mut arr =- speculation_result.into_iter().collect::<Vec<(u64, f32)>>();- arr.sort_by(|(_, a), (_, b)| f32::total_cmp(b, a));-- let arr = arr- .into_iter()- .enumerate()- .filter(|&(i, (_, p))| {- i == 0 || p > creation_config.prob_cutoff- })- .map(|(_, x)| x)- .take(usize::min(creation_config.top_k, max_length - trie_size))- .collect::<Vec<(u64, f32)>>();-- let Some(&(top_token, _)) = arr.first() else {- break;- };-- trie_size += arr.len();- for &(tok, _) in arr.iter() {- current_node.get_or_insert_next(tok, seed.next());- }-- current_node = current_node.get_or_insert_next(top_token, 0);- speculation_prefix.push(top_token);- }-- trie- }-- pub fn insert<'a, I>(- &mut self,- sequence: I,- ) where- I: IntoIterator<Item = &'a u64>,- {- let mut current_node = &mut self.root;- for &token in sequence {- current_node = current_node.get_or_insert_next(token, 0);- }- }-- pub fn linearize(- &self,- start_index: usize,- max_length: usize,- ) -> SpeculatedSuffix {- let mut tokens = Vec::new();- let mut seeds = Vec::new();- let mut indices = Vec::new();-- self.root.dfs(|path| {- if tokens.len() >= max_length {- return;- }- if !path.is_empty() {- tokens.push(path.last().unwrap().1);- seeds.push(path.last().unwrap().2);- indices.push(start_index + path.len() - 1);- }- });-- let tokens_len = tokens.len();-- let mut transition_map: HashMap<- isize,- std::collections::HashMap<u64, isize>,- > = HashMap::new();- for index in -1..(tokens_len as isize) {- transition_map.insert(index, HashMap::new());- }-- self.root.dfs(|path| {- if path.is_empty()- || path.last().unwrap().0 >= (max_length as isize)- {- return;- }-- let (current_index, current_token, _) = *path.last().unwrap();- let parent_index = if path.len() > 1 {- path[path.len() - 2].0- } else {- -1- };-- if let Some(transitions) = transition_map.get_mut(&parent_index) {- transitions.insert(current_token, current_index);- }- });-- SpeculatedSuffix {- tokens,- indices,- seeds,- transition_map,- }- }-}diff --git a/crates/uzu/src/runners/decoder_runner.rs b/crates/uzu/src/runners/decoder_runner.rsindex e740cdabb..caade4528 100644--- a/crates/uzu/src/runners/decoder_runner.rs+++ b/crates/uzu/src/runners/decoder_runner.rs@@ -127,9 +127,9 @@ impl DecoderTestContext {shared_buffers.clone(),&token_ids,&token_positions,+ &token_seeds,token_ids.len(),false,- &token_seeds,false,None,false,diff --git a/crates/uzu/src/tracer/tracer.rs b/crates/uzu/src/tracer/tracer.rsindex 3d6bc88a4..d9c0e6f5f 100644--- a/crates/uzu/src/tracer/tracer.rs+++ b/crates/uzu/src/tracer/tracer.rs@@ -315,9 +315,9 @@ impl Tracer {self.generator_context.shared_buffers.clone(),&token_ids,&token_positions,+ &token_seeds,token_ids.len(),false,- &token_seeds,true,None,false,diff --git a/crates/uzu/src/trie.rs b/crates/uzu/src/trie.rsnew file mode 100644index 000000000..7bede11fc--- /dev/null+++ b/crates/uzu/src/trie.rs@@ -0,0 +1,225 @@+use crate::{+ generator::{gumbel::speculator_sample, rng::DerivableSeed},+ speculators::speculator::Speculator,+};++use thiserror::Error;++#[derive(Debug, Error)]+pub enum TrieError {+ #[error("child with the same token id is already present")]+ DuplicateTokenId,+}++pub struct TrieCreationConfig {+ pub width: usize,+}++impl Default for TrieCreationConfig {+ fn default() -> Self {+ TrieCreationConfig {+ width: 4,+ }+ }+}++#[derive(Debug)]+pub struct TrieNode {+ token: u64,+ seed: u64,+ next: Vec<TrieNode>,+}++#[derive(Debug)]+pub struct FlatTrie<'a> {+ tokens: Box<[(&'a TrieNode, usize)]>,+}++impl TrieNode {+ pub fn new(+ token: u64,+ seed: u64,+ ) -> Self {+ Self {+ token,+ seed,+ next: Vec::new(),+ }+ }++ pub fn add(+ &mut self,+ next: TrieNode,+ ) -> Result<usize, TrieError> {+ if self.next.iter().any(|n| n.token == next.token) {+ return Err(TrieError::DuplicateTokenId);+ }++ self.next.push(next);+ Ok(self.next.len() - 1)+ }++ pub fn get(+ &self,+ token: u64,+ ) -> Option<&TrieNode> {+ self.next.iter().find(|n| n.token == token)+ }++ pub fn get_mut(+ &mut self,+ token: u64,+ ) -> Option<&mut TrieNode> {+ self.next.iter_mut().find(|n| n.token == token)+ }++ pub fn token(&self) -> u64 {+ self.token+ }++ pub fn seed(&self) -> u64 {+ self.seed+ }++ pub fn from_speculator(+ prefix: &[u64],+ next_seed: &mut DerivableSeed,+ speculator: &dyn Speculator,+ creation_config: &TrieCreationConfig,+ max_length: usize,+ ) -> Self {+ assert!(max_length >= 1, "can't have zero sized trie");+ assert!(prefix.len() >= 1, "need seed node");++ let mut speculated_suffix = prefix.to_vec();++ let mut length = 1;+ let mut root = Self::new(*prefix.last().unwrap(), next_seed.next());++ let mut cur_node = &mut root;+ let mut cur_node_width = 0;+ let mut cur_node_speculator_weights =+ speculator.speculate(&speculated_suffix);++ let mut next_node = None;++ while length < max_length {+ // Guuumbel speculator trick: both speculator and llm sample via gumbel max trick using the same noise for increased acceptance rate+ if let Some(next_speculated_token) =+ speculator_sample(cur_node.seed(), &cur_node_speculator_weights)+ {+ // Add speculated token to the trie+ let leaf_node =+ Self::new(next_speculated_token, next_seed.next());+ cur_node.add(leaf_node).unwrap();++ // If this is the first node we sampled (most likely to be selected after gumbel noise application) - set it as the next one+ if next_node.is_none() {+ next_node = Some(next_speculated_token);+ }++ // Remove the token so that the next iteration samples the second most likely after gumbel noise application token+ cur_node_speculator_weights.remove(&next_speculated_token);++ length += 1;+ cur_node_width += 1;+ if cur_node_width >= creation_config.width {+ cur_node_speculator_weights.clear();+ }+ } else if let Some(next_node_token) = next_node.take() {+ // Out of speculated tokens for this node, move onto the likeliest next node+ speculated_suffix.push(next_node_token);+ cur_node = cur_node.get_mut(next_node_token).unwrap();+ cur_node_width = 0;+ cur_node_speculator_weights =+ speculator.speculate(&speculated_suffix);+ continue;+ } else {+ // Dead end, exit+ break;+ };+ }++ root+ }++ pub fn linearize(&self) -> FlatTrie<'_> {+ let mut tokens = vec![(self, 0)];++ let mut stack = vec![(self, 0)];+ while let Some((cur_node, next_child_idx)) = stack.last_mut() {+ let Some(next_node) = cur_node.next.get(*next_child_idx) else {+ stack.pop();+ continue;+ };+ *next_child_idx += 1;++ tokens.push((next_node, stack.len()));++ if !next_node.next.is_empty() {+ stack.push((next_node, 0));+ }+ }++ FlatTrie::new(tokens.into_boxed_slice())+ }+}++impl<'a> FlatTrie<'a> {+ pub fn new(tokens: Box<[(&'a TrieNode, usize)]>) -> Self {+ Self {+ tokens,+ }+ }++ pub fn len(&self) -> usize {+ self.tokens.len()+ }++ pub fn token_ids(&self) -> impl Iterator<Item = u64> {+ self.tokens.iter().map(|&(n, _p)| n.token)+ }++ pub fn token_positions(&self) -> impl Iterator<Item = usize> {+ self.tokens.iter().map(|&(_n, p)| p)+ }++ pub fn token_seeds(&self) -> impl Iterator<Item = u64> {+ self.tokens.iter().map(|&(n, _p)| n.seed)+ }++ pub fn root(&self) -> Option<&TrieNode> {+ self.tokens.first().map(|&(n, _p)| n)+ }++ pub fn index(+ &self,+ node: &'a TrieNode,+ ) -> Option<usize> {+ self.tokens.iter().position(|&(n, _)| std::ptr::eq(n, node))+ }++ pub fn accept(+ &self,+ sampled_tokens: &[u64],+ ) -> (Vec<u64>, Vec<usize>) {+ let mut current_token = self.root().unwrap();+ let mut accepted_tokens = Vec::new();+ let mut accepted_token_indices = Vec::new();+ loop {+ let current_token_index = self.index(current_token).unwrap();+ let current_token_id = sampled_tokens[current_token_index];++ accepted_token_indices.push(current_token_index);+ accepted_tokens.push(current_token_id);++ let Some(next_token) = current_token.get(current_token_id) else {+ break;+ };++ current_token = next_token;+ }++ (accepted_tokens, accepted_token_indices)+ }+}diff --git a/crates/uzu/tests/trie_test.rs b/crates/uzu/tests/trie_test.rsnew file mode 100644index 000000000..6bf8faa38--- /dev/null+++ b/crates/uzu/tests/trie_test.rs@@ -0,0 +1,285 @@+use std::collections::{HashMap, HashSet};++use uzu::{+ generator::rng::DerivableSeed,+ speculators::{empty_speculator::EmptySpeculator, speculator::Speculator},+ trie::{TrieCreationConfig, TrieNode},+};++struct StaticSpeculator {+ responses: HashMap<Vec<u64>, HashMap<u64, f32>>,+}++impl StaticSpeculator {+ fn new(responses: HashMap<Vec<u64>, HashMap<u64, f32>>) -> Self {+ Self {+ responses,+ }+ }+}++impl Speculator for StaticSpeculator {+ fn speculate(+ &self,+ prefix: &[u64],+ ) -> HashMap<u64, f32> {+ self.responses.get(prefix).cloned().unwrap_or_default()+ }+}++fn verify_sprout(+ trie_root: &TrieNode,+ expected_seed: u64,+) {+ let flat_trie = trie_root.linearize();++ assert_eq!(flat_trie.len(), 1);+ assert_eq!(flat_trie.index(&trie_root), Some(0));+ assert_eq!(flat_trie.index(&TrieNode::new(1, 0)), None);+ assert_eq!(flat_trie.index(&TrieNode::new(0, 1)), None);+ assert_eq!(flat_trie.index(&TrieNode::new(0, 0)), None);+ assert_eq!(flat_trie.token_ids().collect::<Vec<u64>>(), vec![0]);+ assert_eq!(flat_trie.token_positions().collect::<Vec<usize>>(), vec![0]);+ assert_eq!(+ flat_trie.token_seeds().collect::<Vec<u64>>(),+ vec![expected_seed]+ );+}++#[test]+fn test_trie_manual_sprout() {+ let trie_root = TrieNode::new(0, 0);++ verify_sprout(&trie_root, 0);+}++#[test]+fn test_trie_from_speculator_sprout() {+ let mut trie_rng = DerivableSeed::new(42);+ let mut ref_rng = DerivableSeed::new(42);++ let speculator = EmptySpeculator;++ let trie_root = TrieNode::from_speculator(+ &[0],+ &mut trie_rng,+ &speculator,+ &TrieCreationConfig {+ width: 5,+ },+ 10,+ );++ verify_sprout(&trie_root, ref_rng.next());+}++fn verify_stick(+ trie_root: &TrieNode,+ mut expected_seeds: HashSet<u64>,+) {+ let flat_trie = trie_root.linearize();++ assert_eq!(flat_trie.len(), 10);+ let token_ids = flat_trie.token_ids().collect::<Vec<u64>>();+ let token_positions = flat_trie.token_positions().collect::<Vec<usize>>();+ let token_seeds = flat_trie.token_seeds().collect::<Vec<u64>>();+ assert_eq!(token_ids.len(), 10);+ assert_eq!(token_positions.len(), 10);+ assert_eq!(token_seeds.len(), 10);++ let mut cur_node = trie_root;++ let position = flat_trie.index(cur_node).unwrap();+ assert_eq!(token_ids[position], 0);+ assert_eq!(token_positions[position], 0);+ assert!(expected_seeds.remove(&token_seeds[position]));++ for i in 1..10 {+ cur_node = cur_node.get(i as u64).unwrap();+ assert_eq!(cur_node.token(), i as u64);+ assert!(expected_seeds.remove(&cur_node.seed()));++ let position = flat_trie.index(cur_node).unwrap();+ assert_eq!(token_ids[position], cur_node.token());+ assert_eq!(token_positions[position], i as usize);+ assert_eq!(token_seeds[position], cur_node.seed());+ }++ assert!(expected_seeds.is_empty());+}++#[test]+fn test_trie_manual_stick() {+ let mut trie_root = TrieNode::new(0, 0);++ let mut trie_leaf = &mut trie_root;+ for (token, seed) in std::iter::zip(1..10, 1..10) {+ trie_leaf.add(TrieNode::new(token, seed)).unwrap();+ trie_leaf = trie_leaf.get_mut(token).unwrap();+ }++ verify_stick(&trie_root, (0..10).collect());+}++#[test]+fn test_trie_from_speculator_stick() {+ let mut trie_rng = DerivableSeed::new(42);+ let mut ref_rng = DerivableSeed::new(42);++ let mut hs = HashMap::new();+ for i in 1..10 {+ hs.insert(+ (0..i).collect(),+ vec![(i, 0.988), (888, 0.001), (999, 0.001)].into_iter().collect(),+ );+ }+ let speculator = StaticSpeculator::new(hs);++ let trie_root = TrieNode::from_speculator(+ &[0],+ &mut trie_rng,+ &speculator,+ &TrieCreationConfig {+ width: 1,+ },+ 10,+ );++ verify_stick(+ &trie_root,+ std::iter::from_fn(|| Some(ref_rng.next())).take(10).collect(),+ );+}++fn verify_bush(+ trie_root: &TrieNode,+ mut expected_seeds: HashSet<u64>,+) {+ let flat_trie = trie_root.linearize();++ assert_eq!(flat_trie.len(), 4);+ let token_ids = flat_trie.token_ids().collect::<Vec<u64>>();+ let token_positions = flat_trie.token_positions().collect::<Vec<usize>>();+ let token_seeds = flat_trie.token_seeds().collect::<Vec<u64>>();+ assert_eq!(token_ids.len(), 4);+ assert_eq!(token_positions.len(), 4);+ assert_eq!(token_seeds.len(), 4);++ let root_position = flat_trie.index(&trie_root).unwrap();+ assert_eq!(token_ids[root_position], 0);+ assert_eq!(token_positions[root_position], 0);+ assert!(expected_seeds.remove(&token_seeds[root_position]));++ for leaf_token in [1, 2, 3] {+ let leaf = trie_root.get(leaf_token).unwrap();+ assert_eq!(leaf.token(), leaf_token);+ assert!(expected_seeds.remove(&leaf.seed()));++ let position = flat_trie.index(&leaf).unwrap();+ assert_eq!(token_ids[position], leaf.token());+ assert_eq!(token_positions[position], 1);+ assert_eq!(token_seeds[position], leaf.seed());+ }++ assert!(expected_seeds.is_empty());+}++#[test]+fn test_trie_manual_bush() {+ let mut trie_root = TrieNode::new(0, 0);++ assert!(trie_root.add(TrieNode::new(1, 1)).is_ok());+ assert!(trie_root.add(TrieNode::new(1, 1)).is_err());+ assert!(trie_root.add(TrieNode::new(1, 10)).is_err());++ assert!(trie_root.add(TrieNode::new(2, 2)).is_ok());+ assert!(trie_root.add(TrieNode::new(3, 3)).is_ok());++ verify_bush(&trie_root, (0..4).into_iter().collect());+}++#[test]+fn test_trie_from_speculator_bush() {+ let mut trie_rng = DerivableSeed::new(42);+ let mut ref_rng = DerivableSeed::new(42);++ let mut hs = HashMap::new();+ hs.insert(+ vec![0],+ vec![(1, 0.33), (2, 0.33), (3, 0.33), (888, 0.005), (999, 0.005)]+ .into_iter()+ .collect(),+ );+ let speculator = StaticSpeculator::new(hs);++ let trie_root = TrieNode::from_speculator(+ &[0],+ &mut trie_rng,+ &speculator,+ &TrieCreationConfig {+ width: 3,+ },+ 10,+ );++ verify_bush(+ &trie_root,+ std::iter::from_fn(|| Some(ref_rng.next())).take(4).collect(),+ );+}++fn verify_tree(trie_root: &TrieNode) {+ let flat_trie = trie_root.linearize();++ assert_eq!(flat_trie.len(), 7);+ let token_ids = flat_trie.token_ids().collect::<Vec<u64>>();+ let token_positions = flat_trie.token_positions().collect::<Vec<usize>>();+ let token_seeds = flat_trie.token_seeds().collect::<Vec<u64>>();+ assert_eq!(token_ids.len(), 7);+ assert_eq!(token_positions.len(), 7);+ assert_eq!(token_seeds.len(), 7);++ let root_position = flat_trie.index(&trie_root).unwrap();+ assert_eq!(token_ids[root_position], 0);+ assert_eq!(token_positions[root_position], 0);+ assert_eq!(token_seeds[root_position], 0);++ for (mid_token, mid_seed) in [(1, 1), (2, 1), (3, 2)] {+ let leaf = trie_root.get(mid_token).unwrap();+ let position = flat_trie.index(&leaf).unwrap();+ assert_eq!(token_ids[position], mid_token);+ assert_eq!(token_positions[position], 1);+ assert_eq!(token_seeds[position], mid_seed);+ }++ for (mid_token, leaf_token, leaf_seed) in+ [(2, 10, 15), (3, 20, 25), (3, 21, 26)]+ {+ let leaf = trie_root.get(mid_token).unwrap().get(leaf_token).unwrap();+ let leaf_position = flat_trie.index(leaf).unwrap();+ assert_eq!(token_ids[leaf_position], leaf_token);+ assert_eq!(token_positions[leaf_position], 2);+ assert_eq!(token_seeds[leaf_position], leaf_seed);+ }+}++#[test]+fn test_trie_manual_tree() {+ let mut trie_root = TrieNode::new(0, 0);++ assert!(trie_root.add(TrieNode::new(1, 1)).is_ok());+ assert!(trie_root.add(TrieNode::new(1, 1)).is_err());+ assert!(trie_root.add(TrieNode::new(1, 10)).is_err());++ assert!(trie_root.add(TrieNode::new(2, 1)).is_ok());+ assert!(trie_root.add(TrieNode::new(3, 2)).is_ok());++ let mid_b = trie_root.get_mut(2).unwrap();+ assert!(mid_b.add(TrieNode::new(10, 15)).is_ok());++ let mid_c = trie_root.get_mut(3).unwrap();+ assert!(mid_c.add(TrieNode::new(20, 25)).is_ok());+ assert!(mid_c.add(TrieNode::new(21, 26)).is_ok());++ verify_tree(&trie_root)+}