A high-performance inference engine for AI models
Text prefix match (#778)
uuuvnc3d14d1f9905
diff --git a/crates/legacy/hanashi/src/chat/hanashi/mod.rs b/crates/legacy/hanashi/src/chat/hanashi/mod.rsindex fc2e1e899..6bcb94e3d 100644--- a/crates/legacy/hanashi/src/chat/hanashi/mod.rs+++ b/crates/legacy/hanashi/src/chat/hanashi/mod.rs@@ -116,10 +116,10 @@ impl EncodingTrait for HanashiEncodingImpl {let text_encoding = self.tokenizer.encode(text, false).map_err(|_| Error::UnableToEncodeText)?;for token_id in text_encoding.get_ids() {let token = self.resolve_token(*token_id, true)?;- self.push_token_to_parser(&token)?;+ self.push_token_to_parser(&token, true)?;self.state.tokens.push(token);}-+ self.parser.flush_extraction();self.update_messages_from_parser_state()?;Ok(())}@@ -130,7 +130,7 @@ impl EncodingTrait for HanashiEncodingImpl {) -> Result<(), Self::Error> {for token_id in &token_ids {let token = self.resolve_token(*token_id, true)?;- self.push_token_to_parser(&token)?;+ self.push_token_to_parser(&token, false)?;self.state.tokens.push(token);}@@ -148,14 +148,28 @@ impl EncodingTrait for HanashiEncodingImpl {}impl HanashiEncodingImpl {+ pub fn tokenize(+ &self,+ text: &str,+ ) -> Result<Vec<TokenId>, Error> {+ let encoding = self.tokenizer.encode(text, false).map_err(|_| Error::UnableToEncodeText)?;+ Ok(encoding.get_ids().to_vec())+ }+fn push_token_to_parser(&mut self,token: &Token,+ defer_extraction: bool,) -> Result<(), Error> {if token.is_special && !self.framing_tokens.contains(&token.value) {return Ok(());}- self.parser.push(&token.clone().to_parser_token())?;+ let parser_token = token.clone().to_parser_token();+ if defer_extraction {+ self.parser.push_bulk(&parser_token)?;+ } else {+ self.parser.push(&parser_token)?;+ }Ok(())}diff --git a/crates/legacy/hanashi/src/chat/harmony/mod.rs b/crates/legacy/hanashi/src/chat/harmony/mod.rsindex 6de279409..46a87c333 100644--- a/crates/legacy/hanashi/src/chat/harmony/mod.rs+++ b/crates/legacy/hanashi/src/chat/harmony/mod.rs@@ -203,6 +203,13 @@ impl EncodingTrait for HarmonyEncodingImpl {}impl HarmonyEncodingImpl {+ pub fn tokenize(+ &self,+ text: &str,+ ) -> Result<Vec<TokenId>, Error> {+ Ok(self.encoding.tokenizer().encode_with_special_tokens(text))+ }+fn process(&mut self,token_id: TokenId,diff --git a/crates/legacy/hanashi/src/chat/mod.rs b/crates/legacy/hanashi/src/chat/mod.rsindex c5847b8a4..f3abb2a11 100644--- a/crates/legacy/hanashi/src/chat/mod.rs+++ b/crates/legacy/hanashi/src/chat/mod.rs@@ -37,6 +37,15 @@ pub enum Encoding {Harmony(HarmonyEncodingImpl),}+impl Encoding {+ pub fn tokenize(+ &self,+ text: &str,+ ) -> Result<Vec<TokenId>, Error> {+ dispatch!(self, tokenize, text)+ }+}+impl EncodingTrait for Encoding {type Config = EncodingConfig;type Input = Vec<ChatMessage>;diff --git a/crates/legacy/nagare/src/chat/token.rs b/crates/legacy/nagare/src/chat/token.rsindex cc43418b8..02502ac4f 100644--- a/crates/legacy/nagare/src/chat/token.rs+++ b/crates/legacy/nagare/src/chat/token.rs@@ -139,17 +139,18 @@ impl Session {},};- // if new_all_tokens = curr_all_tokens + suffix, then just encode suffix,- // else reset state and encode all tokens- let mut reset = new_all_tokens.len() <= curr_all_tokens.len();- if !reset {- for i in 0..curr_all_tokens.len() {- if new_all_tokens[i] != curr_all_tokens[i].id as u64 {- reset = true;- break;- }- }- }+ // The engine state can only be kept whole or reset, so reuse it whenever the session's+ // text is a prefix of the newly rendered text — even if tokenizations differ, as sampled+ // replies are not canonically tokenized — and prefill only the raw-tokenized text suffix.+ let curr_text = curr_all_tokens.iter().fold(String::new(), |mut text, token| {+ text.push_str(&token.value);+ text+ });+ let new_text = self.encoding.state().tokens.iter().fold(String::new(), |mut text, token| {+ text.push_str(&token.value);+ text+ });+ let reset = !new_text.starts_with(&curr_text);let cached_tokens_input = if reset {0} else {@@ -161,7 +162,14 @@ impl Session {}new_all_tokens} else {- new_all_tokens[curr_all_tokens.len()..].to_vec()+ match self.encoding.tokenize(&new_text[curr_text.len()..]) {+ Ok(suffix_tokens) => suffix_tokens.into_iter().map(u64::from).collect(),+ Err(err) => {+ return error_stream(ChatSessionError::Backend {+ message: err.to_string(),+ });+ },+ }};let instance = self.instance.as_ref();@@ -376,12 +384,11 @@ impl StreamingState<'_> {&self,last_stat: bool,) -> ChatReplyStats {- let speculator_stats = if let Some(metrics) = self.metrics.as_ref()- && metrics.num_decode_forward_passes > 0- {- Some(ChatReplySpeculatorStats {- tokens_per_forward_pass: metrics.num_tokens_accepted as f64 / metrics.num_decode_forward_passes as f64,- num_decode_forward_passes: metrics.num_decode_forward_passes as u32,+ let speculator_stats = if let Some(metrics) = self.metrics.as_ref() {+ let num_forward_passes = metrics.num_prefill_forward_passes + metrics.num_decode_forward_passes;+ (num_forward_passes > 0).then(|| ChatReplySpeculatorStats {+ tokens_per_forward_pass: metrics.num_tokens_accepted as f64 / num_forward_passes as f64,+ num_decode_forward_passes: num_forward_passes as u32,})} else {Nonediff --git a/crates/legacy/token-stream-parser/src/token_stream/parser.rs b/crates/legacy/token-stream-parser/src/token_stream/parser.rsindex 2371f89dd..371e0836e 100644--- a/crates/legacy/token-stream-parser/src/token_stream/parser.rs+++ b/crates/legacy/token-stream-parser/src/token_stream/parser.rs@@ -46,16 +46,7 @@ impl Parser for TokenStreamParser {&mut self,input: &Token,) -> Result<(), TokenStreamParserError> {- let event = match self.framing.push(input) {- Ok(event) => event,- Err(infallible) => match infallible {},- };- self.reduction.push(&event)?;- match self.extraction.push(self.reduction.state()) {- Ok(()) => {},- Err(infallible) => match infallible {},- }- Ok(())+ self.push_inner(input, true)}fn state(&self) -> &ExtractionParserState {@@ -70,6 +61,39 @@ impl Parser for TokenStreamParser {}impl TokenStreamParser {+ fn push_inner(+ &mut self,+ input: &Token,+ extract: bool,+ ) -> Result<(), TokenStreamParserError> {+ let event = match self.framing.push(input) {+ Ok(event) => event,+ Err(infallible) => match infallible {},+ };+ self.reduction.push(&event)?;+ if extract {+ self.flush_extraction();+ }+ Ok(())+ }++ /// Framing and reduction only: extraction output is recomputed from the whole reduction+ /// state on every push, which is quadratic over bulk loads. Call `flush_extraction` once+ /// after the last `push_bulk`.+ pub fn push_bulk(+ &mut self,+ input: &Token,+ ) -> Result<(), TokenStreamParserError> {+ self.push_inner(input, false)+ }++ pub fn flush_extraction(&mut self) {+ match self.extraction.push(self.reduction.state()) {+ Ok(()) => {},+ Err(infallible) => match infallible {},+ }+ }+/// Exposes `value` to every transformation pipeline as `$<name>`.pub fn set_variable(&mut self,