Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

Text prefix match (#778)

uuuvnc3d14d1f9905
diff --git a/crates/legacy/hanashi/src/chat/hanashi/mod.rs b/crates/legacy/hanashi/src/chat/hanashi/mod.rs
index fc2e1e899..6bcb94e3d 100644
--- a/crates/legacy/hanashi/src/chat/hanashi/mod.rs
+++ b/crates/legacy/hanashi/src/chat/hanashi/mod.rs
@@ -116,10 +116,10 @@ impl EncodingTrait for HanashiEncodingImpl {
let text_encoding = self.tokenizer.encode(text, false).map_err(|_| Error::UnableToEncodeText)?;
for token_id in text_encoding.get_ids() {
let token = self.resolve_token(*token_id, true)?;
- self.push_token_to_parser(&token)?;
+ self.push_token_to_parser(&token, true)?;
self.state.tokens.push(token);
}
-
+ self.parser.flush_extraction();
self.update_messages_from_parser_state()?;
Ok(())
}
@@ -130,7 +130,7 @@ impl EncodingTrait for HanashiEncodingImpl {
) -> Result<(), Self::Error> {
for token_id in &token_ids {
let token = self.resolve_token(*token_id, true)?;
- self.push_token_to_parser(&token)?;
+ self.push_token_to_parser(&token, false)?;
self.state.tokens.push(token);
}
@@ -148,14 +148,28 @@ impl EncodingTrait for HanashiEncodingImpl {
}
impl HanashiEncodingImpl {
+ pub fn tokenize(
+ &self,
+ text: &str,
+ ) -> Result<Vec<TokenId>, Error> {
+ let encoding = self.tokenizer.encode(text, false).map_err(|_| Error::UnableToEncodeText)?;
+ Ok(encoding.get_ids().to_vec())
+ }
+
fn push_token_to_parser(
&mut self,
token: &Token,
+ defer_extraction: bool,
) -> Result<(), Error> {
if token.is_special && !self.framing_tokens.contains(&token.value) {
return Ok(());
}
- self.parser.push(&token.clone().to_parser_token())?;
+ let parser_token = token.clone().to_parser_token();
+ if defer_extraction {
+ self.parser.push_bulk(&parser_token)?;
+ } else {
+ self.parser.push(&parser_token)?;
+ }
Ok(())
}
diff --git a/crates/legacy/hanashi/src/chat/harmony/mod.rs b/crates/legacy/hanashi/src/chat/harmony/mod.rs
index 6de279409..46a87c333 100644
--- a/crates/legacy/hanashi/src/chat/harmony/mod.rs
+++ b/crates/legacy/hanashi/src/chat/harmony/mod.rs
@@ -203,6 +203,13 @@ impl EncodingTrait for HarmonyEncodingImpl {
}
impl HarmonyEncodingImpl {
+ pub fn tokenize(
+ &self,
+ text: &str,
+ ) -> Result<Vec<TokenId>, Error> {
+ Ok(self.encoding.tokenizer().encode_with_special_tokens(text))
+ }
+
fn process(
&mut self,
token_id: TokenId,
diff --git a/crates/legacy/hanashi/src/chat/mod.rs b/crates/legacy/hanashi/src/chat/mod.rs
index c5847b8a4..f3abb2a11 100644
--- a/crates/legacy/hanashi/src/chat/mod.rs
+++ b/crates/legacy/hanashi/src/chat/mod.rs
@@ -37,6 +37,15 @@ pub enum Encoding {
Harmony(HarmonyEncodingImpl),
}
+impl Encoding {
+ pub fn tokenize(
+ &self,
+ text: &str,
+ ) -> Result<Vec<TokenId>, Error> {
+ dispatch!(self, tokenize, text)
+ }
+}
+
impl EncodingTrait for Encoding {
type Config = EncodingConfig;
type Input = Vec<ChatMessage>;
diff --git a/crates/legacy/nagare/src/chat/token.rs b/crates/legacy/nagare/src/chat/token.rs
index cc43418b8..02502ac4f 100644
--- a/crates/legacy/nagare/src/chat/token.rs
+++ b/crates/legacy/nagare/src/chat/token.rs
@@ -139,17 +139,18 @@ impl Session {
},
};
- // if new_all_tokens = curr_all_tokens + suffix, then just encode suffix,
- // else reset state and encode all tokens
- let mut reset = new_all_tokens.len() <= curr_all_tokens.len();
- if !reset {
- for i in 0..curr_all_tokens.len() {
- if new_all_tokens[i] != curr_all_tokens[i].id as u64 {
- reset = true;
- break;
- }
- }
- }
+ // The engine state can only be kept whole or reset, so reuse it whenever the session's
+ // text is a prefix of the newly rendered text — even if tokenizations differ, as sampled
+ // replies are not canonically tokenized — and prefill only the raw-tokenized text suffix.
+ let curr_text = curr_all_tokens.iter().fold(String::new(), |mut text, token| {
+ text.push_str(&token.value);
+ text
+ });
+ let new_text = self.encoding.state().tokens.iter().fold(String::new(), |mut text, token| {
+ text.push_str(&token.value);
+ text
+ });
+ let reset = !new_text.starts_with(&curr_text);
let cached_tokens_input = if reset {
0
} else {
@@ -161,7 +162,14 @@ impl Session {
}
new_all_tokens
} else {
- new_all_tokens[curr_all_tokens.len()..].to_vec()
+ match self.encoding.tokenize(&new_text[curr_text.len()..]) {
+ Ok(suffix_tokens) => suffix_tokens.into_iter().map(u64::from).collect(),
+ Err(err) => {
+ return error_stream(ChatSessionError::Backend {
+ message: err.to_string(),
+ });
+ },
+ }
};
let instance = self.instance.as_ref();
@@ -376,12 +384,11 @@ impl StreamingState<'_> {
&self,
last_stat: bool,
) -> ChatReplyStats {
- let speculator_stats = if let Some(metrics) = self.metrics.as_ref()
- && metrics.num_decode_forward_passes > 0
- {
- Some(ChatReplySpeculatorStats {
- tokens_per_forward_pass: metrics.num_tokens_accepted as f64 / metrics.num_decode_forward_passes as f64,
- num_decode_forward_passes: metrics.num_decode_forward_passes as u32,
+ let speculator_stats = if let Some(metrics) = self.metrics.as_ref() {
+ let num_forward_passes = metrics.num_prefill_forward_passes + metrics.num_decode_forward_passes;
+ (num_forward_passes > 0).then(|| ChatReplySpeculatorStats {
+ tokens_per_forward_pass: metrics.num_tokens_accepted as f64 / num_forward_passes as f64,
+ num_decode_forward_passes: num_forward_passes as u32,
})
} else {
None
diff --git a/crates/legacy/token-stream-parser/src/token_stream/parser.rs b/crates/legacy/token-stream-parser/src/token_stream/parser.rs
index 2371f89dd..371e0836e 100644
--- a/crates/legacy/token-stream-parser/src/token_stream/parser.rs
+++ b/crates/legacy/token-stream-parser/src/token_stream/parser.rs
@@ -46,16 +46,7 @@ impl Parser for TokenStreamParser {
&mut self,
input: &Token,
) -> Result<(), TokenStreamParserError> {
- let event = match self.framing.push(input) {
- Ok(event) => event,
- Err(infallible) => match infallible {},
- };
- self.reduction.push(&event)?;
- match self.extraction.push(self.reduction.state()) {
- Ok(()) => {},
- Err(infallible) => match infallible {},
- }
- Ok(())
+ self.push_inner(input, true)
}
fn state(&self) -> &ExtractionParserState {
@@ -70,6 +61,39 @@ impl Parser for TokenStreamParser {
}
impl TokenStreamParser {
+ fn push_inner(
+ &mut self,
+ input: &Token,
+ extract: bool,
+ ) -> Result<(), TokenStreamParserError> {
+ let event = match self.framing.push(input) {
+ Ok(event) => event,
+ Err(infallible) => match infallible {},
+ };
+ self.reduction.push(&event)?;
+ if extract {
+ self.flush_extraction();
+ }
+ Ok(())
+ }
+
+ /// Framing and reduction only: extraction output is recomputed from the whole reduction
+ /// state on every push, which is quadratic over bulk loads. Call `flush_extraction` once
+ /// after the last `push_bulk`.
+ pub fn push_bulk(
+ &mut self,
+ input: &Token,
+ ) -> Result<(), TokenStreamParserError> {
+ self.push_inner(input, false)
+ }
+
+ pub fn flush_extraction(&mut self) {
+ match self.extraction.push(self.reduction.state()) {
+ Ok(()) => {},
+ Err(infallible) => match infallible {},
+ }
+ }
+
/// Exposes `value` to every transformation pipeline as `$<name>`.
pub fn set_variable(
&mut self,