Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

new speculator api (#36)

uuuvn269ecf1a35c6
diff --git a/crates/uzu/src/generator/generator.rs b/crates/uzu/src/generator/generator.rs
index ba9234cc5..5cfc3773c 100644
--- a/crates/uzu/src/generator/generator.rs
+++ b/crates/uzu/src/generator/generator.rs
@@ -14,7 +14,7 @@ use crate::{
encodable_with_state::{EncodableWithState, EncodingParameters},
kv_cache::INVALID_POSITION,
},
- linearizer::trie::TokenTrie,
+ linearizer::trie::{TokenTrie, TrieCreationConfig},
session::{
config::DecodingConfig,
parameter::{ConfigResolvableValue, SamplingMethod},
@@ -83,9 +83,14 @@ impl Generator {
let unused_tokens_count = total_prefill_tokens_count - tokens_length;
let speculator = &self.decoding_config.speculator_config.speculator;
- let proposals = speculator.generate_proposals(&tokens);
- let speculated_suffix = TokenTrie::from_sequences(&proposals)
- .linearize(0, unused_tokens_count);
+ let speculated_suffix = TokenTrie::from_speculator(
+ &tokens,
+ false,
+ speculator.as_ref(),
+ &TrieCreationConfig::default(),
+ unused_tokens_count,
+ )
+ .linearize(0, unused_tokens_count);
let zero_padding_tokens: Vec<u64> =
vec![0; unused_tokens_count - speculated_suffix.tokens.len()];
@@ -222,20 +227,15 @@ impl Generator {
&mut self,
sampling_method: SamplingMethod,
) -> Result<GenerateResult, Error> {
- let last_token = self.tokens.last().ok_or(Error::GenerateFailed)?;
-
let speculator = &self.decoding_config.speculator_config.speculator;
- let mut proposals: Vec<Vec<u64>> = speculator
- .generate_proposals(&self.tokens)
- .into_iter()
- .map(|v| std::iter::once(*last_token).chain(v).collect())
- .collect();
- if proposals.is_empty() {
- proposals = vec![vec![*last_token]];
- }
-
- let speculated_suffix = TokenTrie::from_sequences(&proposals)
- .linearize(0, self.decoding_config.generate_suffix_length());
+ let speculated_suffix = TokenTrie::from_speculator(
+ &self.tokens,
+ true,
+ speculator.as_ref(),
+ &TrieCreationConfig::default(),
+ self.decoding_config.generate_suffix_length(),
+ )
+ .linearize(0, self.decoding_config.generate_suffix_length());
let expected_suffix_length =
self.decoding_config.generate_suffix_length();
diff --git a/crates/uzu/src/linearizer/trie.rs b/crates/uzu/src/linearizer/trie.rs
index 19ae414db..6e93350f3 100644
--- a/crates/uzu/src/linearizer/trie.rs
+++ b/crates/uzu/src/linearizer/trie.rs
@@ -1,7 +1,23 @@
use std::collections::HashMap;
+use crate::speculators::speculator::Speculator;
+
use super::{node::Node, speculated_suffix::SpeculatedSuffix};
+pub struct TrieCreationConfig {
+ pub top_k: usize,
+ pub prob_cutoff: f32,
+}
+
+impl Default for TrieCreationConfig {
+ fn default() -> Self {
+ TrieCreationConfig {
+ top_k: 2,
+ prob_cutoff: 0.1,
+ }
+ }
+}
+
#[derive(Debug)]
pub struct TokenTrie {
root: Node,
@@ -20,6 +36,57 @@ impl TokenTrie {
}
}
+ pub fn from_speculator(
+ prefix: &[u64],
+ include_last_prefix_token: bool,
+ speculator: &dyn Speculator,
+ creation_config: &TrieCreationConfig,
+ max_length: usize,
+ ) -> Self {
+ let mut trie = Self::new();
+ let mut current_node = &mut trie.root;
+ let mut trie_size = 0;
+
+ if include_last_prefix_token {
+ current_node =
+ current_node.get_or_insert_next(*prefix.last().unwrap());
+ trie_size += 1;
+ }
+
+ let mut speculation_prefix = prefix.to_vec();
+ while trie_size < max_length {
+ let speculation_result = speculator.speculate(&speculation_prefix);
+
+ let mut arr =
+ speculation_result.into_iter().collect::<Vec<(u64, f32)>>();
+ arr.sort_by(|(_, a), (_, b)| f32::total_cmp(b, a));
+
+ let arr = arr
+ .into_iter()
+ .enumerate()
+ .filter(|&(i, (_, p))| {
+ i == 0 || p > creation_config.prob_cutoff
+ })
+ .map(|(_, x)| x)
+ .take(usize::min(creation_config.top_k, max_length - trie_size))
+ .collect::<Vec<(u64, f32)>>();
+
+ let Some(&(top_token, _)) = arr.first() else {
+ break;
+ };
+
+ trie_size += arr.len();
+ for &(tok, _) in arr.iter() {
+ current_node.get_or_insert_next(tok);
+ }
+
+ current_node = current_node.get_or_insert_next(top_token);
+ speculation_prefix.push(top_token);
+ }
+
+ trie
+ }
+
pub fn from_sequences(sequences: &[Vec<u64>]) -> Self {
let mut trie = Self::new();
for sequence in sequences {
diff --git a/crates/uzu/src/speculators/empty_speculator.rs b/crates/uzu/src/speculators/empty_speculator.rs
index 8f302f721..cc680a085 100644
--- a/crates/uzu/src/speculators/empty_speculator.rs
+++ b/crates/uzu/src/speculators/empty_speculator.rs
@@ -1,13 +1,14 @@
use super::speculator::Speculator;
+use std::collections::HashMap;
#[derive(Default, Debug, Clone, Copy)]
pub struct EmptySpeculator;
impl Speculator for EmptySpeculator {
- fn generate_proposals(
+ fn speculate(
&self,
_prefix: &[u64],
- ) -> Vec<Vec<u64>> {
- Vec::new()
+ ) -> HashMap<u64, f32> {
+ HashMap::new()
}
}
diff --git a/crates/uzu/src/speculators/speculator.rs b/crates/uzu/src/speculators/speculator.rs
index 80a80964f..cdba2c610 100644
--- a/crates/uzu/src/speculators/speculator.rs
+++ b/crates/uzu/src/speculators/speculator.rs
@@ -1,6 +1,8 @@
+use std::collections::HashMap;
+
pub trait Speculator: Send + Sync {
- fn generate_proposals(
+ fn speculate(
&self,
prefix: &[u64],
- ) -> Vec<Vec<u64>>;
+ ) -> HashMap<u64, f32>;
}