A high-performance inference engine for AI models
Remove leading whitespace from "end of thinking" tag (#1010)
Aleksandr Golokoz6715ff93a91f
diff --git a/crates/legacy/uzu/src/engine/bridge/chat_token_backend.rs b/crates/legacy/uzu/src/engine/bridge/chat_token_backend.rsindex 6beb189e0..d0cfbbf77 100644--- a/crates/legacy/uzu/src/engine/bridge/chat_token_backend.rs+++ b/crates/legacy/uzu/src/engine/bridge/chat_token_backend.rs@@ -73,7 +73,8 @@ impl<B: Backend> UzuChatTokenBackendInstance<B> {};let max_context_length = get_max_context_length(&model, config.context_length.clone());#[cfg(feature = "capability-grammar")]- let grammar_trigger_token_sequence = grammar_trigger_token_sequence(&model);+ let grammar_trigger_token_sequence =+ grammar_trigger_token_sequence(model.end_of_thinking_tag(), model.tokenizer());Ok(Self {engine,diff --git a/crates/legacy/uzu/src/engine/bridge/helpers.rs b/crates/legacy/uzu/src/engine/bridge/helpers.rsindex 94b2c336e..e29413c95 100644--- a/crates/legacy/uzu/src/engine/bridge/helpers.rs+++ b/crates/legacy/uzu/src/engine/bridge/helpers.rs@@ -48,11 +48,29 @@ pub fn get_grammar(/// The token sequence of the model's end-of-thinking tag. A reasoning model/// opens every reply with reasoning tokens that no grammar can accept, so the/// grammar must stay disengaged until this sequence has been generated.+/// Leading whitespace-only tokens are excluded because BPE can merge them with+/// preceding whitespace in both the prompt and generated reasoning. Tokens that+/// fuse whitespace with marker text retain their original encoding.#[cfg(feature = "capability-grammar")]-pub fn grammar_trigger_token_sequence<B: Backend>(model: &LanguageModel<B>) -> Option<Vec<u64>> {- let tag = model.end_of_thinking_tag()?;+pub fn grammar_trigger_token_sequence(+ end_of_thinking_tag: Option<&str>,+ tokenizer: &Tokenizer,+) -> Option<Vec<u64>> {+ let tag = end_of_thinking_tag?;+ if tag.trim().is_empty() {+ return None;+ }++ let encoding = tokenizer.encode(tag, false).ok()?;+ let token_ids = encoding.get_ids();+ let leading_whitespace_tokens = token_ids+ .iter()+ .take_while(|&&token_id| {+ tokenizer.decode(&[token_id], false).is_ok_and(|text| !text.is_empty() && text.trim().is_empty())+ })+ .count();let token_ids: Vec<u64> =- model.tokenizer().encode(tag, false).ok()?.get_ids().iter().map(|token_id| u64::from(*token_id)).collect();+ token_ids[leading_whitespace_tokens..].iter().map(|token_id| u64::from(*token_id)).collect();(!token_ids.is_empty()).then_some(token_ids)}@@ -160,9 +178,14 @@ fn get_custom_sampling_method(method: &ShojiSamplingMethod) -> Result<UzuSamplin#[cfg(all(test, feature = "capability-grammar"))]mod tests {- use tokenizers::{Tokenizer, decoders::fuse::Fuse, models::wordlevel::WordLevel};+ use tokenizers::{+ AddedToken, Tokenizer,+ decoders::fuse::Fuse,+ models::{bpe::BPE, wordlevel::WordLevel},+ pre_tokenizers::byte_level::ByteLevel,+ };- use super::grammar_trigger_token_sequence_for_prompt;+ use super::{ShojiGrammar, get_grammar, grammar_trigger_token_sequence, grammar_trigger_token_sequence_for_prompt};fn tokenizer() -> Tokenizer {let model = WordLevel::builder()@@ -196,6 +219,152 @@ mod tests {assert_eq!(grammar_trigger_token_sequence_for_prompt(Some(&[1]), &[1, 2, 3], &tokenizer), Some(vec![1]));}++ // Like Qwen's tokenizer, merge adjacent newlines while keeping the thinking+ // tags as special tokens. No model download is needed to reproduce the bug.+ fn bpe_tokenizer() -> Tokenizer {+ let vocab = [+ ("<unk>", 0),+ ("Ċ", 1),+ ("ĊĊ", 2),+ ("<think>", 3),+ ("</think>", 4),+ ("Y", 5),+ ("E", 6),+ ("S", 7),+ ("N", 8),+ ("O", 9),+ ("{", 10),+ ("}", 11),+ ("<eos>", 12),+ ("Ġ", 13),+ ("ĠY", 14),+ ]+ .map(|(token, id)| (token.to_string(), id));+ let model = BPE::builder()+ .vocab_and_merges(vocab, vec![("Ċ".into(), "Ċ".into()), ("Ġ".into(), "Y".into())])+ .build()+ .unwrap();+ let mut tokenizer = Tokenizer::new(model);+ tokenizer.with_pre_tokenizer(Some(ByteLevel::new(false, false, true)));+ tokenizer.with_decoder(Some(ByteLevel::default()));+ tokenizer.add_special_tokens(&[+ AddedToken::from("<think>", true),+ AddedToken::from("</think>", true),+ AddedToken::from("<eos>", true),+ ]);+ tokenizer+ }++ fn encode(+ tokenizer: &Tokenizer,+ text: &str,+ ) -> Vec<u64> {+ tokenizer.encode(text, false).unwrap().get_ids().iter().map(|id| u64::from(*id)).collect()+ }++ fn grammars() -> [ShojiGrammar; 3] {+ [+ ShojiGrammar::Regex {+ pattern: "(YES|NO)".into(),+ },+ ShojiGrammar::JsonAny {},+ ShojiGrammar::JsonSchema {+ schema: r#"{"type":"object","additionalProperties":false}"#.into(),+ },+ ]+ }++ #[test]+ fn grammar_engages_when_prompt_closes_reasoning_with_merged_newlines() {+ let tokenizer = bpe_tokenizer();+ assert_eq!(encode(&tokenizer, "\n</think>"), [1, 4]);+ let prompt = encode(&tokenizer, "<think>\n\n</think>\n\n");+ assert_eq!(prompt, [3, 2, 4, 2]);+ let trigger = grammar_trigger_token_sequence(Some("\n</think>"), &tokenizer);+ let trigger = grammar_trigger_token_sequence_for_prompt(trigger.as_deref(), &prompt, &tokenizer);+ assert_eq!(trigger, None);++ for config in grammars() {+ let mut grammar = get_grammar(config, &tokenizer, &[12], trigger.clone()).unwrap();+ assert!(grammar.next_bitmask(&mut [0]));+ }+ }++ #[test]+ fn grammar_engages_after_generated_trigger_with_merged_newlines() {+ let tokenizer = bpe_tokenizer();+ let trigger = grammar_trigger_token_sequence(Some("\n</think>"), &tokenizer);+ let prompt = encode(&tokenizer, "<think>\n");+ let trigger = grammar_trigger_token_sequence_for_prompt(trigger.as_deref(), &prompt, &tokenizer);++ for config in grammars() {+ let mut grammar = get_grammar(config, &tokenizer, &[12], trigger.clone()).unwrap();+ assert!(!grammar.next_bitmask(&mut [0]));+ grammar.accept_token(2).unwrap(); // merged newlines during reasoning+ assert!(!grammar.next_bitmask(&mut [0]));+ grammar.accept_token(4).unwrap(); // </think>+ assert!(grammar.next_bitmask(&mut [0]));+ // Speculative rollback across the trigger must restore reasoning.+ grammar.rollback(1);+ assert!(!grammar.next_bitmask(&mut [0]));+ grammar.accept_token(4).unwrap();+ assert!(grammar.next_bitmask(&mut [0]));+ }+ }++ #[test]+ fn grammar_trigger_preserves_multi_token_tags() {+ let tokenizer = bpe_tokenizer();+ let trigger = grammar_trigger_token_sequence(Some("\n</think><think>"), &tokenizer);+ assert_eq!(trigger, Some(vec![4, 3]));+ let prompt = encode(&tokenizer, "<think>\n\n</think><think>\n\n");+ assert_eq!(grammar_trigger_token_sequence_for_prompt(trigger.as_deref(), &prompt, &tokenizer), None);+ let history = encode(&tokenizer, "</think><think>\nYES<think>\n");+ assert_eq!(grammar_trigger_token_sequence_for_prompt(trigger.as_deref(), &history, &tokenizer), trigger);+ }++ #[test]+ fn grammar_trigger_preserves_whitespace_fused_with_marker_text() {+ let tokenizer = bpe_tokenizer();+ let expected = encode(&tokenizer, " YES");+ assert_eq!(expected, [14, 6, 7]);+ assert_ne!(expected, encode(&tokenizer, "YES"));++ // Preserve the fused space even when a separate newline precedes it.+ for tag in [" YES", "\n YES"] {+ let trigger = grammar_trigger_token_sequence(Some(tag), &tokenizer);+ assert_eq!(trigger.as_deref(), Some(expected.as_slice()));++ let prompt = encode(&tokenizer, "<think>\n\n YES\n\n");+ assert_eq!(grammar_trigger_token_sequence_for_prompt(trigger.as_deref(), &prompt, &tokenizer), None);++ let prompt = encode(&tokenizer, "<think>\n");+ let trigger = grammar_trigger_token_sequence_for_prompt(trigger.as_deref(), &prompt, &tokenizer);+ let mut grammar = get_grammar(+ ShojiGrammar::Regex {+ pattern: "(YES|NO)".into(),+ },+ &tokenizer,+ &[12],+ trigger,+ )+ .unwrap();+ for token in encode(&tokenizer, "\n\n YES") {+ assert!(!grammar.next_bitmask(&mut [0]));+ grammar.accept_token(token).unwrap();+ }+ assert!(grammar.next_bitmask(&mut [0]));+ }+ }++ #[test]+ fn grammar_trigger_ignores_missing_or_whitespace_only_tags() {+ let tokenizer = bpe_tokenizer();+ for tag in [None, Some(""), Some("\n\n")] {+ assert_eq!(grammar_trigger_token_sequence(tag, &tokenizer), None);+ }+ }}#[cfg(test)]