A high-performance inference engine for AI models
new speculator api (#36)
uuuvn269ecf1a35c6
diff --git a/crates/uzu/src/generator/generator.rs b/crates/uzu/src/generator/generator.rsindex ba9234cc5..5cfc3773c 100644--- a/crates/uzu/src/generator/generator.rs+++ b/crates/uzu/src/generator/generator.rs@@ -14,7 +14,7 @@ use crate::{encodable_with_state::{EncodableWithState, EncodingParameters},kv_cache::INVALID_POSITION,},- linearizer::trie::TokenTrie,+ linearizer::trie::{TokenTrie, TrieCreationConfig},session::{config::DecodingConfig,parameter::{ConfigResolvableValue, SamplingMethod},@@ -83,9 +83,14 @@ impl Generator {let unused_tokens_count = total_prefill_tokens_count - tokens_length;let speculator = &self.decoding_config.speculator_config.speculator;- let proposals = speculator.generate_proposals(&tokens);- let speculated_suffix = TokenTrie::from_sequences(&proposals)- .linearize(0, unused_tokens_count);+ let speculated_suffix = TokenTrie::from_speculator(+ &tokens,+ false,+ speculator.as_ref(),+ &TrieCreationConfig::default(),+ unused_tokens_count,+ )+ .linearize(0, unused_tokens_count);let zero_padding_tokens: Vec<u64> =vec![0; unused_tokens_count - speculated_suffix.tokens.len()];@@ -222,20 +227,15 @@ impl Generator {&mut self,sampling_method: SamplingMethod,) -> Result<GenerateResult, Error> {- let last_token = self.tokens.last().ok_or(Error::GenerateFailed)?;-let speculator = &self.decoding_config.speculator_config.speculator;- let mut proposals: Vec<Vec<u64>> = speculator- .generate_proposals(&self.tokens)- .into_iter()- .map(|v| std::iter::once(*last_token).chain(v).collect())- .collect();- if proposals.is_empty() {- proposals = vec![vec![*last_token]];- }-- let speculated_suffix = TokenTrie::from_sequences(&proposals)- .linearize(0, self.decoding_config.generate_suffix_length());+ let speculated_suffix = TokenTrie::from_speculator(+ &self.tokens,+ true,+ speculator.as_ref(),+ &TrieCreationConfig::default(),+ self.decoding_config.generate_suffix_length(),+ )+ .linearize(0, self.decoding_config.generate_suffix_length());let expected_suffix_length =self.decoding_config.generate_suffix_length();diff --git a/crates/uzu/src/linearizer/trie.rs b/crates/uzu/src/linearizer/trie.rsindex 19ae414db..6e93350f3 100644--- a/crates/uzu/src/linearizer/trie.rs+++ b/crates/uzu/src/linearizer/trie.rs@@ -1,7 +1,23 @@use std::collections::HashMap;+use crate::speculators::speculator::Speculator;+use super::{node::Node, speculated_suffix::SpeculatedSuffix};+pub struct TrieCreationConfig {+ pub top_k: usize,+ pub prob_cutoff: f32,+}++impl Default for TrieCreationConfig {+ fn default() -> Self {+ TrieCreationConfig {+ top_k: 2,+ prob_cutoff: 0.1,+ }+ }+}+#[derive(Debug)]pub struct TokenTrie {root: Node,@@ -20,6 +36,57 @@ impl TokenTrie {}}+ pub fn from_speculator(+ prefix: &[u64],+ include_last_prefix_token: bool,+ speculator: &dyn Speculator,+ creation_config: &TrieCreationConfig,+ max_length: usize,+ ) -> Self {+ let mut trie = Self::new();+ let mut current_node = &mut trie.root;+ let mut trie_size = 0;++ if include_last_prefix_token {+ current_node =+ current_node.get_or_insert_next(*prefix.last().unwrap());+ trie_size += 1;+ }++ let mut speculation_prefix = prefix.to_vec();+ while trie_size < max_length {+ let speculation_result = speculator.speculate(&speculation_prefix);++ let mut arr =+ speculation_result.into_iter().collect::<Vec<(u64, f32)>>();+ arr.sort_by(|(_, a), (_, b)| f32::total_cmp(b, a));++ let arr = arr+ .into_iter()+ .enumerate()+ .filter(|&(i, (_, p))| {+ i == 0 || p > creation_config.prob_cutoff+ })+ .map(|(_, x)| x)+ .take(usize::min(creation_config.top_k, max_length - trie_size))+ .collect::<Vec<(u64, f32)>>();++ let Some(&(top_token, _)) = arr.first() else {+ break;+ };++ trie_size += arr.len();+ for &(tok, _) in arr.iter() {+ current_node.get_or_insert_next(tok);+ }++ current_node = current_node.get_or_insert_next(top_token);+ speculation_prefix.push(top_token);+ }++ trie+ }+pub fn from_sequences(sequences: &[Vec<u64>]) -> Self {let mut trie = Self::new();for sequence in sequences {diff --git a/crates/uzu/src/speculators/empty_speculator.rs b/crates/uzu/src/speculators/empty_speculator.rsindex 8f302f721..cc680a085 100644--- a/crates/uzu/src/speculators/empty_speculator.rs+++ b/crates/uzu/src/speculators/empty_speculator.rs@@ -1,13 +1,14 @@use super::speculator::Speculator;+use std::collections::HashMap;#[derive(Default, Debug, Clone, Copy)]pub struct EmptySpeculator;impl Speculator for EmptySpeculator {- fn generate_proposals(+ fn speculate(&self,_prefix: &[u64],- ) -> Vec<Vec<u64>> {- Vec::new()+ ) -> HashMap<u64, f32> {+ HashMap::new()}}diff --git a/crates/uzu/src/speculators/speculator.rs b/crates/uzu/src/speculators/speculator.rsindex 80a80964f..cdba2c610 100644--- a/crates/uzu/src/speculators/speculator.rs+++ b/crates/uzu/src/speculators/speculator.rs@@ -1,6 +1,8 @@+use std::collections::HashMap;+pub trait Speculator: Send + Sync {- fn generate_proposals(+ fn speculate(&self,prefix: &[u64],- ) -> Vec<Vec<u64>>;+ ) -> HashMap<u64, f32>;}