Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

remove old trie (#73)

uuuvn9cb3eba5522a
diff --git a/crates/uzu/src/lib.rs b/crates/uzu/src/lib.rs
index 645f4f965..aac000ee8 100644
--- a/crates/uzu/src/lib.rs
+++ b/crates/uzu/src/lib.rs
@@ -2,7 +2,6 @@ pub mod backends;
pub mod classifier;
pub mod config;
pub mod language_model;
-pub mod linearizer;
pub mod parameters;
pub mod prelude;
pub mod session;
diff --git a/crates/uzu/src/linearizer/mod.rs b/crates/uzu/src/linearizer/mod.rs
deleted file mode 100644
index 95655d93b..000000000
--- a/crates/uzu/src/linearizer/mod.rs
+++ /dev/null
@@ -1,8 +0,0 @@
-pub mod node;
-pub mod speculated_suffix;
-pub mod transition;
-pub mod trie;
-
-#[cfg(test)]
-mod tests_node;
-mod tests_trie;
diff --git a/crates/uzu/src/linearizer/node.rs b/crates/uzu/src/linearizer/node.rs
deleted file mode 100644
index 62df6cb1d..000000000
--- a/crates/uzu/src/linearizer/node.rs
+++ /dev/null
@@ -1,80 +0,0 @@
-use super::transition::Transition;
-
-#[derive(Debug)]
-pub struct Node {
- transitions: Vec<Transition>,
-}
-
-impl Default for Node {
- fn default() -> Self {
- Self::new()
- }
-}
-
-impl Node {
- pub fn new() -> Self {
- Self {
- transitions: Vec::new(),
- }
- }
-
- pub fn has_transition(
- &self,
- token: u64,
- ) -> bool {
- self.transitions.iter().any(|t| t.token() == token)
- }
-
- pub fn get_next(
- &self,
- token: u64,
- ) -> Option<&Node> {
- self.transitions.iter().find(|t| t.token() == token).map(|t| &t.node)
- }
-
- pub fn get_or_insert_next(
- &mut self,
- token: u64,
- seed: u64,
- ) -> &mut Node {
- if let Some(pos) =
- self.transitions.iter().position(|t| t.token() == token)
- {
- return &mut self.transitions[pos].node;
- }
-
- self.transitions.push(Transition::new(token, seed));
- &mut self.transitions.last_mut().unwrap().node
- }
-
- pub fn dfs_with_path<F>(
- &self,
- path: &mut Vec<(isize, u64, u64)>,
- callback: &mut F,
- ) -> isize
- where
- F: FnMut(&[(isize, u64, u64)]),
- {
- let mut node_index = path.last().map(|(idx, _, _)| *idx).unwrap_or(-1);
-
- for transition in self.transitions.iter() {
- node_index += 1;
- path.push((node_index, transition.token(), transition.seed()));
- callback(path);
- node_index = transition.node.dfs_with_path(path, callback);
- path.pop();
- }
-
- node_index
- }
-
- pub fn dfs<F>(
- &self,
- mut callback: F,
- ) where
- F: FnMut(&[(isize, u64, u64)]),
- {
- let mut path: Vec<(isize, u64, u64)> = Vec::new();
- self.dfs_with_path(&mut path, &mut callback);
- }
-}
diff --git a/crates/uzu/src/linearizer/speculated_suffix.rs b/crates/uzu/src/linearizer/speculated_suffix.rs
deleted file mode 100644
index 2f3874036..000000000
--- a/crates/uzu/src/linearizer/speculated_suffix.rs
+++ /dev/null
@@ -1,9 +0,0 @@
-use std::collections::HashMap;
-
-#[derive(Debug)]
-pub struct SpeculatedSuffix {
- pub tokens: Vec<u64>,
- pub seeds: Vec<u64>,
- pub indices: Vec<usize>,
- pub transition_map: HashMap<isize, HashMap<u64, isize>>,
-}
diff --git a/crates/uzu/src/linearizer/tests_node.rs b/crates/uzu/src/linearizer/tests_node.rs
deleted file mode 100644
index 9f4680d76..000000000
--- a/crates/uzu/src/linearizer/tests_node.rs
+++ /dev/null
@@ -1,267 +0,0 @@
-#[cfg(test)]
-mod tests {
- use std::collections::HashSet;
-
- use super::super::node::Node;
-
- #[test]
- fn test_new_node_has_no_transitions() {
- let node = Node::new();
- for i in 0..10 {
- assert!(!node.has_transition(i));
- }
- }
-
- #[test]
- fn test_has_transition() {
- let mut node = Node::new();
- node.get_or_insert_next(42, 0);
-
- assert!(node.has_transition(42));
- assert!(!node.has_transition(43));
- }
-
- #[test]
- fn test_get_next() {
- let mut node = Node::new();
- node.get_or_insert_next(1, 0);
-
- assert!(node.get_next(1).is_some());
- assert!(node.get_next(2).is_none());
- }
-
- #[test]
- fn test_get_or_insert_next() {
- let mut node = Node::new();
-
- // First insertion should create a new node
- let node1 = node.get_or_insert_next(1, 0);
- assert!(!node1.has_transition(99));
-
- // Add a transition to the child node
- node1.get_or_insert_next(99, 0);
-
- // Get the same node again
- let node1_again = node.get_or_insert_next(1, 0);
-
- // Verify it's the same node by checking it has the transition we added
- assert!(node1_again.has_transition(99));
-
- // Different token should give a different node
- let node2 = node.get_or_insert_next(2, 0);
- assert!(!node2.has_transition(99));
- }
-
- #[test]
- fn test_multiple_transitions() {
- let mut node = Node::new();
-
- // Insert several transitions
- for i in 0..5 {
- node.get_or_insert_next(i, 0);
- }
-
- // Verify all transitions exist
- for i in 0..5 {
- assert!(node.has_transition(i));
- }
-
- // Verify non-existent transitions
- for i in 5..10 {
- assert!(!node.has_transition(i));
- }
- }
-
- #[test]
- fn test_dfs_with_path_empty_node() {
- let node = Node::new();
- let mut path: Vec<(isize, u64, u64)> = Vec::new();
- let mut visited = Vec::new();
-
- node.dfs_with_path(&mut path, &mut |current_path: &[(
- isize,
- u64,
- u64,
- )]| {
- let tokens: Vec<u64> =
- current_path.iter().map(|(_, token, _)| *token).collect();
- visited.push(tokens);
- });
-
- assert!(visited.is_empty());
- }
-
- #[test]
- fn test_dfs_with_path_single_node() {
- let mut node = Node::new();
- node.get_or_insert_next(42, 0);
-
- let mut path: Vec<(isize, u64, u64)> = Vec::new();
- let mut visited = Vec::new();
-
- node.dfs_with_path(&mut path, &mut |current_path: &[(
- isize,
- u64,
- u64,
- )]| {
- let tokens: Vec<u64> =
- current_path.iter().map(|(_, token, _)| *token).collect();
- visited.push(tokens);
- });
-
- assert_eq!(visited.len(), 1);
- assert_eq!(visited[0], vec![42]);
- }
-
- #[test]
- fn test_dfs_with_path_complex_tree() {
- let mut root = Node::new();
-
- // Build a sample trie: [1 -> 2 -> 3], [1 -> 2 -> 4], [1 -> 5], [6]
- let node1 = root.get_or_insert_next(1, 0);
- let node2 = node1.get_or_insert_next(2, 0);
- node2.get_or_insert_next(3, 0);
- node2.get_or_insert_next(4, 0);
- node1.get_or_insert_next(5, 0);
- root.get_or_insert_next(6, 0);
-
- let mut path: Vec<(isize, u64, u64)> = Vec::new();
- let mut visited = Vec::new();
-
- root.dfs_with_path(&mut path, &mut |current_path: &[(
- isize,
- u64,
- u64,
- )]| {
- let tokens: Vec<u64> =
- current_path.iter().map(|(_, token, _)| *token).collect();
- visited.push(tokens);
- });
-
- // Check all expected paths were visited
- assert_eq!(visited.len(), 6);
- assert!(visited.contains(&vec![1]));
- assert!(visited.contains(&vec![1, 2]));
- assert!(visited.contains(&vec![1, 2, 3]));
- assert!(visited.contains(&vec![1, 2, 4]));
- assert!(visited.contains(&vec![1, 5]));
- assert!(visited.contains(&vec![6]));
- }
-
- #[test]
- fn test_dfs_path_indices() {
- let mut root = Node::new();
-
- // Build a simple trie: [1], [2]
- root.get_or_insert_next(1, 0);
- root.get_or_insert_next(2, 0);
-
- let mut path: Vec<(isize, u64, u64)> = Vec::new();
- let mut indices = Vec::new();
-
- root.dfs_with_path(&mut path, &mut |current_path: &[(
- isize,
- u64,
- u64,
- )]| {
- if !current_path.is_empty() {
- indices.push(current_path.last().unwrap().0);
- }
- });
-
- // Check indices are unique and as expected
- assert_eq!(indices.len(), 2);
- assert_eq!(indices[0], 0);
- assert_eq!(indices[1], 1);
- }
-
- #[test]
- fn test_simplified_dfs_empty_node() {
- let node = Node::new();
- let mut visited = Vec::new();
-
- node.dfs(|current_path: &[(isize, u64, u64)]| {
- let tokens: Vec<u64> =
- current_path.iter().map(|(_, token, _)| *token).collect();
- visited.push(tokens);
- });
-
- assert!(visited.is_empty());
- }
-
- #[test]
- fn test_simplified_dfs_complex_tree() {
- let mut root = Node::new();
-
- // Build a sample trie: [1 -> 2 -> 3], [1 -> 2 -> 4], [1 -> 5], [6]
- let node1 = root.get_or_insert_next(1, 0);
- let node2 = node1.get_or_insert_next(2, 0);
- node2.get_or_insert_next(3, 0);
- node2.get_or_insert_next(4, 0);
- node1.get_or_insert_next(5, 0);
- root.get_or_insert_next(6, 0);
-
- let mut visited = Vec::new();
-
- root.dfs(|current_path: &[(isize, u64, u64)]| {
- let tokens: Vec<u64> =
- current_path.iter().map(|(_, token, _)| *token).collect();
- visited.push(tokens);
- });
-
- // Check all expected paths were visited
- assert_eq!(visited.len(), 6);
- assert!(visited.contains(&vec![1]));
- assert!(visited.contains(&vec![1, 2]));
- assert!(visited.contains(&vec![1, 2, 3]));
- assert!(visited.contains(&vec![1, 2, 4]));
- assert!(visited.contains(&vec![1, 5]));
- assert!(visited.contains(&vec![6]));
- }
-
- #[test]
- fn test_both_dfs_methods_equivalent() {
- let mut root = Node::new();
-
- // Build a more complex trie
- let node1 = root.get_or_insert_next(10, 0);
- let node2 = node1.get_or_insert_next(20, 0);
- node2.get_or_insert_next(30, 0);
- node2.get_or_insert_next(40, 0);
- node1.get_or_insert_next(50, 0);
- let node3 = root.get_or_insert_next(60, 0);
- node3.get_or_insert_next(70, 0);
-
- // Get results from dfs_with_path
- let mut path: Vec<(isize, u64, u64)> = Vec::new();
- let mut visited1 = Vec::new();
-
- root.dfs_with_path(&mut path, &mut |current_path: &[(
- isize,
- u64,
- u64,
- )]| {
- let tokens: Vec<u64> =
- current_path.iter().map(|(_, token, _)| *token).collect();
- visited1.push(tokens);
- });
-
- // Get results from simplified dfs
- let mut visited2 = Vec::new();
-
- root.dfs(|current_path: &[(isize, u64, u64)]| {
- let tokens: Vec<u64> =
- current_path.iter().map(|(_, token, _)| *token).collect();
- visited2.push(tokens);
- });
-
- // Check both methods give the same results
- assert_eq!(visited1.len(), visited2.len());
-
- // Convert to sets to ignore order differences
- let set1: HashSet<Vec<u64>> = visited1.into_iter().collect();
- let set2: HashSet<Vec<u64>> = visited2.into_iter().collect();
-
- assert_eq!(set1, set2);
- }
-}
diff --git a/crates/uzu/src/linearizer/tests_trie.rs b/crates/uzu/src/linearizer/tests_trie.rs
deleted file mode 100644
index 9896fb206..000000000
--- a/crates/uzu/src/linearizer/tests_trie.rs
+++ /dev/null
@@ -1,297 +0,0 @@
-#[cfg(test)]
-mod tests {
- use std::collections::HashMap;
-
- use crate::{
- language_model::rng::DerivableSeed,
- linearizer::trie::{TokenTrie, TrieCreationConfig},
- speculators::speculator::Speculator,
- };
-
- struct StaticSpeculator {
- responses: HashMap<Vec<u64>, HashMap<u64, f32>>,
- }
-
- impl StaticSpeculator {
- fn new(responses: HashMap<Vec<u64>, HashMap<u64, f32>>) -> Self {
- Self {
- responses,
- }
- }
- }
-
- impl Speculator for StaticSpeculator {
- fn speculate(
- &self,
- prefix: &[u64],
- ) -> HashMap<u64, f32> {
- self.responses.get(prefix).cloned().unwrap_or_default()
- }
- }
-
- #[test]
- fn test_from_speculator_applies_prob_cutoff() {
- let mut responses = HashMap::new();
- responses.insert(
- vec![42],
- HashMap::from([(1, 0.9_f32), (2, 0.05_f32), (3, 0.2_f32)]),
- );
- responses.insert(vec![42, 1], HashMap::new());
-
- let speculator = StaticSpeculator::new(responses);
- let mut seed = DerivableSeed::new(1);
-
- let trie = TokenTrie::from_speculator(
- &[42],
- &mut seed,
- false,
- &speculator,
- &TrieCreationConfig::default(),
- 5,
- );
- let suffix = trie.linearize(0, 5);
-
- assert_eq!(suffix.tokens, vec![1, 3]);
- assert!(!suffix.tokens.contains(&2));
- assert_eq!(suffix.transition_map.len(), 3);
- }
-
- #[test]
- fn test_from_speculator_respects_top_k_limit() {
- let mut responses = HashMap::new();
- responses.insert(vec![5], HashMap::from([(9, 0.8_f32), (10, 0.7_f32)]));
- responses.insert(vec![5, 9], HashMap::from([(11, 0.6_f32)]));
- responses.insert(vec![5, 9, 11], HashMap::new());
-
- let speculator = StaticSpeculator::new(responses);
- let mut seed = DerivableSeed::new(7);
-
- let trie = TokenTrie::from_speculator(
- &[5],
- &mut seed,
- false,
- &speculator,
- &TrieCreationConfig {
- top_k: 1,
- prob_cutoff: 0.0,
- },
- 3,
- );
-
- let suffix = trie.linearize(0, 4);
-
- assert_eq!(suffix.tokens, vec![9, 11]);
- assert!(!suffix.tokens.contains(&10));
- }
-
- #[test]
- fn test_from_speculator_includes_last_prefix_token_and_seeds() {
- let mut responses = HashMap::new();
- responses.insert(vec![2, 4], HashMap::from([(8, 0.9_f32)]));
- responses.insert(vec![2, 4, 8], HashMap::new());
-
- let speculator = StaticSpeculator::new(responses);
-
- let mut seed = DerivableSeed::new(99);
- let mut expected_seed_stream = DerivableSeed::new(99);
- let expected_seeds =
- vec![expected_seed_stream.next(), expected_seed_stream.next()];
-
- let trie = TokenTrie::from_speculator(
- &[2, 4],
- &mut seed,
- true,
- &speculator,
- &TrieCreationConfig::default(),
- 2,
- );
-
- let suffix = trie.linearize(0, 3);
-
- assert_eq!(suffix.tokens, vec![4, 8]);
- assert_eq!(suffix.seeds, expected_seeds);
- assert_eq!(suffix.indices, vec![0, 1]);
- }
-
- #[test]
- fn test_from_speculator_stops_at_max_length() {
- let mut responses = HashMap::new();
- responses.insert(
- vec![1],
- HashMap::from([(5, 0.9_f32), (6, 0.8_f32), (7, 0.7_f32)]),
- );
- responses
- .insert(vec![1, 5], HashMap::from([(8, 0.6_f32), (9, 0.5_f32)]));
- responses.insert(vec![1, 5, 8], HashMap::from([(10, 0.4_f32)]));
-
- let speculator = StaticSpeculator::new(responses);
- let mut seed = DerivableSeed::new(1234);
-
- let trie = TokenTrie::from_speculator(
- &[1],
- &mut seed,
- false,
- &speculator,
- &TrieCreationConfig::default(),
- 3,
- );
-
- let suffix = trie.linearize(0, 5);
-
- assert_eq!(suffix.tokens.len(), 3);
- assert!(suffix.tokens.contains(&5));
- assert!(suffix.tokens.contains(&6));
- assert!(suffix.tokens.contains(&8));
- assert!(!suffix.tokens.contains(&9));
- }
-
- #[test]
- fn test_from_speculator_with_empty_responses() {
- let speculator = StaticSpeculator::new(HashMap::new());
-
- let mut seed = DerivableSeed::new(555);
- let mut expected_seed_stream = DerivableSeed::new(555);
- let expected_first_seed = expected_seed_stream.next();
-
- let trie = TokenTrie::from_speculator(
- &[12, 13],
- &mut seed,
- true,
- &speculator,
- &TrieCreationConfig::default(),
- 4,
- );
-
- let suffix = trie.linearize(0, 5);
-
- // Only the last prefix token should be included; no speculated tokens.
- assert_eq!(suffix.tokens, vec![13]);
- assert_eq!(suffix.indices, vec![0]);
- assert_eq!(suffix.seeds, vec![expected_first_seed]);
-
- assert_eq!(suffix.transition_map.len(), 2);
- let root_transitions = &suffix.transition_map[&-1];
- assert_eq!(root_transitions.len(), 1);
- assert_eq!(root_transitions.get(&13), Some(&0));
- }
-
- #[test]
- fn test_linearize_empty_trie() {
- let trie = TokenTrie::new();
- let suffix = trie.linearize(0, 10);
-
- assert!(suffix.tokens.is_empty());
- assert!(suffix.indices.is_empty());
- assert!(suffix.transition_map.len() == 1);
- }
-
- #[test]
- fn test_linearize_single_sequence() {
- let mut trie = TokenTrie::new();
- trie.insert(&[1, 2, 3]);
-
- let suffix = trie.linearize(0, 5);
-
- assert_eq!(suffix.tokens.len(), 3);
- assert!(suffix.tokens.contains(&1));
- assert!(suffix.tokens.contains(&2));
- assert!(suffix.tokens.contains(&3));
-
- assert_eq!(suffix.transition_map.len(), 4);
-
- let root_transitions = &suffix.transition_map[&-1];
- assert!(root_transitions.contains_key(&1));
-
- let token1_idx = *root_transitions.get(&1).unwrap();
- let token1_transitions = &suffix.transition_map[&token1_idx];
- assert!(token1_transitions.contains_key(&2));
-
- let token2_idx = *token1_transitions.get(&2).unwrap();
- let token2_transitions = &suffix.transition_map[&token2_idx];
- assert!(token2_transitions.contains_key(&3));
- }
-
- #[test]
- fn test_linearize_branching_trie() {
- let mut trie = TokenTrie::new();
- trie.insert(&[1, 2, 3]);
- trie.insert(&[1, 2, 4]);
- trie.insert(&[5, 6]);
-
- let suffix = trie.linearize(10, 10);
-
- assert!(suffix.tokens.contains(&1));
- assert!(suffix.tokens.contains(&2));
- assert!(suffix.tokens.contains(&3));
- assert!(suffix.tokens.contains(&4));
- assert!(suffix.tokens.contains(&5));
- assert!(suffix.tokens.contains(&6));
-
- assert_eq!(suffix.tokens.len(), 6);
-
- assert_eq!(suffix.transition_map.len(), 7);
-
- let root_transitions = &suffix.transition_map[&-1];
- assert!(root_transitions.contains_key(&1));
- assert!(root_transitions.contains_key(&5));
-
- let token1_idx = *root_transitions.get(&1).unwrap();
- let token1_transitions = &suffix.transition_map[&token1_idx];
- assert!(token1_transitions.contains_key(&2));
-
- let token2_idx = *token1_transitions.get(&2).unwrap();
- let token2_transitions = &suffix.transition_map[&token2_idx];
- assert!(token2_transitions.contains_key(&3));
- assert!(token2_transitions.contains_key(&4));
-
- let token5_idx = *root_transitions.get(&5).unwrap();
- let token5_transitions = &suffix.transition_map[&token5_idx];
- assert!(token5_transitions.contains_key(&6));
- }
-
- #[test]
- fn test_linearize_max_length() {
- let mut trie = TokenTrie::new();
- trie.insert(&[1, 2, 3, 4, 5, 6, 7, 8, 9, 10]);
-
- let suffix = trie.linearize(0, 5);
-
- assert_eq!(suffix.tokens.len(), 5);
- assert_eq!(suffix.indices.len(), 5);
- assert_eq!(suffix.transition_map.len(), 6);
-
- for i in 1..=5 {
- assert!(suffix.tokens.contains(&i));
- }
-
- let root_transitions = &suffix.transition_map[&-1];
- assert!(root_transitions.contains_key(&1));
-
- let mut current_idx = *root_transitions.get(&1).unwrap();
- for i in 2..=4 {
- let transitions = &suffix.transition_map[&current_idx];
- assert!(transitions.contains_key(&i));
- current_idx = *transitions.get(&i).unwrap();
- }
- }
-
- #[test]
- fn test_linearize_start_index() {
- let mut trie = TokenTrie::new();
- trie.insert(&[1, 2, 3]);
-
- let suffix = trie.linearize(100, 10);
-
- assert_eq!(suffix.tokens.len(), 3);
- assert!(suffix.tokens.contains(&1));
- assert!(suffix.tokens.contains(&2));
- assert!(suffix.tokens.contains(&3));
-
- for idx in suffix.indices.iter() {
- assert!(*idx >= 100 && *idx <= 102);
- }
-
- let root_transitions = &suffix.transition_map[&-1];
- assert!(root_transitions.contains_key(&1));
- }
-}
diff --git a/crates/uzu/src/linearizer/transition.rs b/crates/uzu/src/linearizer/transition.rs
deleted file mode 100644
index 76426fe6c..000000000
--- a/crates/uzu/src/linearizer/transition.rs
+++ /dev/null
@@ -1,29 +0,0 @@
-use super::node::Node;
-
-#[derive(Debug)]
-pub struct Transition {
- token: u64,
- seed: u64,
- pub node: Node,
-}
-
-impl Transition {
- pub fn new(
- token: u64,
- seed: u64,
- ) -> Self {
- Self {
- token,
- seed,
- node: Node::new(),
- }
- }
-
- pub fn token(&self) -> u64 {
- self.token
- }
-
- pub fn seed(&self) -> u64 {
- self.seed
- }
-}
diff --git a/crates/uzu/src/linearizer/trie.rs b/crates/uzu/src/linearizer/trie.rs
deleted file mode 100644
index 8aa2f86f0..000000000
--- a/crates/uzu/src/linearizer/trie.rs
+++ /dev/null
@@ -1,168 +0,0 @@
-use std::collections::HashMap;
-
-use super::{node::Node, speculated_suffix::SpeculatedSuffix};
-use crate::{
- language_model::rng::DerivableSeed, speculators::speculator::Speculator,
-};
-
-pub struct TrieCreationConfig {
- pub top_k: usize,
- pub prob_cutoff: f32,
-}
-
-impl Default for TrieCreationConfig {
- fn default() -> Self {
- TrieCreationConfig {
- top_k: 2,
- prob_cutoff: 0.1,
- }
- }
-}
-
-#[derive(Debug)]
-pub struct TokenTrie {
- root: Node,
-}
-
-impl Default for TokenTrie {
- fn default() -> Self {
- Self::new()
- }
-}
-
-impl TokenTrie {
- pub fn new() -> Self {
- Self {
- root: Node::new(),
- }
- }
-
- pub fn from_sequences(sequences: &[Vec<u64>]) -> Self {
- let mut trie = Self::new();
- for sequence in sequences {
- trie.insert(sequence);
- }
- trie
- }
-
- pub fn from_speculator(
- prefix: &[u64],
- seed: &mut DerivableSeed,
- include_last_prefix_token: bool,
- speculator: &dyn Speculator,
- creation_config: &TrieCreationConfig,
- max_length: usize,
- ) -> Self {
- let mut trie = Self::new();
- let mut current_node = &mut trie.root;
- let mut trie_size = 0;
-
- if include_last_prefix_token {
- current_node = current_node
- .get_or_insert_next(*prefix.last().unwrap(), seed.next());
- trie_size += 1;
- }
-
- let mut speculation_prefix = prefix.to_vec();
- while trie_size < max_length {
- let speculation_result = speculator.speculate(&speculation_prefix);
-
- let mut arr =
- speculation_result.into_iter().collect::<Vec<(u64, f32)>>();
- arr.sort_by(|(_, a), (_, b)| f32::total_cmp(b, a));
-
- let arr = arr
- .into_iter()
- .enumerate()
- .filter(|&(i, (_, p))| {
- i == 0 || p > creation_config.prob_cutoff
- })
- .map(|(_, x)| x)
- .take(usize::min(creation_config.top_k, max_length - trie_size))
- .collect::<Vec<(u64, f32)>>();
-
- let Some(&(top_token, _)) = arr.first() else {
- break;
- };
-
- trie_size += arr.len();
- for &(tok, _) in arr.iter() {
- current_node.get_or_insert_next(tok, seed.next());
- }
-
- current_node = current_node.get_or_insert_next(top_token, 0);
- speculation_prefix.push(top_token);
- }
-
- trie
- }
-
- pub fn insert<'a, I>(
- &mut self,
- sequence: I,
- ) where
- I: IntoIterator<Item = &'a u64>,
- {
- let mut current_node = &mut self.root;
- for &token in sequence {
- current_node = current_node.get_or_insert_next(token, 0);
- }
- }
-
- pub fn linearize(
- &self,
- start_index: usize,
- max_length: usize,
- ) -> SpeculatedSuffix {
- let mut tokens = Vec::new();
- let mut seeds = Vec::new();
- let mut indices = Vec::new();
-
- self.root.dfs(|path| {
- if tokens.len() >= max_length {
- return;
- }
- if !path.is_empty() {
- tokens.push(path.last().unwrap().1);
- seeds.push(path.last().unwrap().2);
- indices.push(start_index + path.len() - 1);
- }
- });
-
- let tokens_len = tokens.len();
-
- let mut transition_map: HashMap<
- isize,
- std::collections::HashMap<u64, isize>,
- > = HashMap::new();
- for index in -1..(tokens_len as isize) {
- transition_map.insert(index, HashMap::new());
- }
-
- self.root.dfs(|path| {
- if path.is_empty()
- || path.last().unwrap().0 >= (max_length as isize)
- {
- return;
- }
-
- let (current_index, current_token, _) = *path.last().unwrap();
- let parent_index = if path.len() > 1 {
- path[path.len() - 2].0
- } else {
- -1
- };
-
- if let Some(transitions) = transition_map.get_mut(&parent_index) {
- transitions.insert(current_token, current_index);
- }
- });
-
- SpeculatedSuffix {
- tokens,
- indices,
- seeds,
- transition_map,
- }
- }
-}
diff --git a/crates/uzu/src/prelude.rs b/crates/uzu/src/prelude.rs
index e177c2921..c9ce4126a 100644
--- a/crates/uzu/src/prelude.rs
+++ b/crates/uzu/src/prelude.rs
@@ -4,7 +4,6 @@
//! frequently used types into scope.
// Constants & Utilities
-// Linearizer
// Session Core
pub use crate::session::ChatSession;
// Session Config
@@ -22,6 +21,6 @@ pub use crate::session::types::{
// Speculators
pub use crate::speculators::speculator::Speculator;
pub use crate::{
- VERSION, linearizer::trie::TokenTrie, session::ClassificationSession,
+ VERSION, session::ClassificationSession,
speculators::empty_speculator::EmptySpeculator, storage_path,
};