Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

Remove leading whitespace from "end of thinking" tag (#1010)

Aleksandr Golokoz6715ff93a91f
diff --git a/crates/legacy/uzu/src/engine/bridge/chat_token_backend.rs b/crates/legacy/uzu/src/engine/bridge/chat_token_backend.rs
index 6beb189e0..d0cfbbf77 100644
--- a/crates/legacy/uzu/src/engine/bridge/chat_token_backend.rs
+++ b/crates/legacy/uzu/src/engine/bridge/chat_token_backend.rs
@@ -73,7 +73,8 @@ impl<B: Backend> UzuChatTokenBackendInstance<B> {
};
let max_context_length = get_max_context_length(&model, config.context_length.clone());
#[cfg(feature = "capability-grammar")]
- let grammar_trigger_token_sequence = grammar_trigger_token_sequence(&model);
+ let grammar_trigger_token_sequence =
+ grammar_trigger_token_sequence(model.end_of_thinking_tag(), model.tokenizer());
Ok(Self {
engine,
diff --git a/crates/legacy/uzu/src/engine/bridge/helpers.rs b/crates/legacy/uzu/src/engine/bridge/helpers.rs
index 94b2c336e..e29413c95 100644
--- a/crates/legacy/uzu/src/engine/bridge/helpers.rs
+++ b/crates/legacy/uzu/src/engine/bridge/helpers.rs
@@ -48,11 +48,29 @@ pub fn get_grammar(
/// The token sequence of the model's end-of-thinking tag. A reasoning model
/// opens every reply with reasoning tokens that no grammar can accept, so the
/// grammar must stay disengaged until this sequence has been generated.
+/// Leading whitespace-only tokens are excluded because BPE can merge them with
+/// preceding whitespace in both the prompt and generated reasoning. Tokens that
+/// fuse whitespace with marker text retain their original encoding.
#[cfg(feature = "capability-grammar")]
-pub fn grammar_trigger_token_sequence<B: Backend>(model: &LanguageModel<B>) -> Option<Vec<u64>> {
- let tag = model.end_of_thinking_tag()?;
+pub fn grammar_trigger_token_sequence(
+ end_of_thinking_tag: Option<&str>,
+ tokenizer: &Tokenizer,
+) -> Option<Vec<u64>> {
+ let tag = end_of_thinking_tag?;
+ if tag.trim().is_empty() {
+ return None;
+ }
+
+ let encoding = tokenizer.encode(tag, false).ok()?;
+ let token_ids = encoding.get_ids();
+ let leading_whitespace_tokens = token_ids
+ .iter()
+ .take_while(|&&token_id| {
+ tokenizer.decode(&[token_id], false).is_ok_and(|text| !text.is_empty() && text.trim().is_empty())
+ })
+ .count();
let token_ids: Vec<u64> =
- model.tokenizer().encode(tag, false).ok()?.get_ids().iter().map(|token_id| u64::from(*token_id)).collect();
+ token_ids[leading_whitespace_tokens..].iter().map(|token_id| u64::from(*token_id)).collect();
(!token_ids.is_empty()).then_some(token_ids)
}
@@ -160,9 +178,14 @@ fn get_custom_sampling_method(method: &ShojiSamplingMethod) -> Result<UzuSamplin
#[cfg(all(test, feature = "capability-grammar"))]
mod tests {
- use tokenizers::{Tokenizer, decoders::fuse::Fuse, models::wordlevel::WordLevel};
+ use tokenizers::{
+ AddedToken, Tokenizer,
+ decoders::fuse::Fuse,
+ models::{bpe::BPE, wordlevel::WordLevel},
+ pre_tokenizers::byte_level::ByteLevel,
+ };
- use super::grammar_trigger_token_sequence_for_prompt;
+ use super::{ShojiGrammar, get_grammar, grammar_trigger_token_sequence, grammar_trigger_token_sequence_for_prompt};
fn tokenizer() -> Tokenizer {
let model = WordLevel::builder()
@@ -196,6 +219,152 @@ mod tests {
assert_eq!(grammar_trigger_token_sequence_for_prompt(Some(&[1]), &[1, 2, 3], &tokenizer), Some(vec![1]));
}
+
+ // Like Qwen's tokenizer, merge adjacent newlines while keeping the thinking
+ // tags as special tokens. No model download is needed to reproduce the bug.
+ fn bpe_tokenizer() -> Tokenizer {
+ let vocab = [
+ ("<unk>", 0),
+ ("Ċ", 1),
+ ("ĊĊ", 2),
+ ("<think>", 3),
+ ("</think>", 4),
+ ("Y", 5),
+ ("E", 6),
+ ("S", 7),
+ ("N", 8),
+ ("O", 9),
+ ("{", 10),
+ ("}", 11),
+ ("<eos>", 12),
+ ("Ġ", 13),
+ ("ĠY", 14),
+ ]
+ .map(|(token, id)| (token.to_string(), id));
+ let model = BPE::builder()
+ .vocab_and_merges(vocab, vec![("Ċ".into(), "Ċ".into()), ("Ġ".into(), "Y".into())])
+ .build()
+ .unwrap();
+ let mut tokenizer = Tokenizer::new(model);
+ tokenizer.with_pre_tokenizer(Some(ByteLevel::new(false, false, true)));
+ tokenizer.with_decoder(Some(ByteLevel::default()));
+ tokenizer.add_special_tokens(&[
+ AddedToken::from("<think>", true),
+ AddedToken::from("</think>", true),
+ AddedToken::from("<eos>", true),
+ ]);
+ tokenizer
+ }
+
+ fn encode(
+ tokenizer: &Tokenizer,
+ text: &str,
+ ) -> Vec<u64> {
+ tokenizer.encode(text, false).unwrap().get_ids().iter().map(|id| u64::from(*id)).collect()
+ }
+
+ fn grammars() -> [ShojiGrammar; 3] {
+ [
+ ShojiGrammar::Regex {
+ pattern: "(YES|NO)".into(),
+ },
+ ShojiGrammar::JsonAny {},
+ ShojiGrammar::JsonSchema {
+ schema: r#"{"type":"object","additionalProperties":false}"#.into(),
+ },
+ ]
+ }
+
+ #[test]
+ fn grammar_engages_when_prompt_closes_reasoning_with_merged_newlines() {
+ let tokenizer = bpe_tokenizer();
+ assert_eq!(encode(&tokenizer, "\n</think>"), [1, 4]);
+ let prompt = encode(&tokenizer, "<think>\n\n</think>\n\n");
+ assert_eq!(prompt, [3, 2, 4, 2]);
+ let trigger = grammar_trigger_token_sequence(Some("\n</think>"), &tokenizer);
+ let trigger = grammar_trigger_token_sequence_for_prompt(trigger.as_deref(), &prompt, &tokenizer);
+ assert_eq!(trigger, None);
+
+ for config in grammars() {
+ let mut grammar = get_grammar(config, &tokenizer, &[12], trigger.clone()).unwrap();
+ assert!(grammar.next_bitmask(&mut [0]));
+ }
+ }
+
+ #[test]
+ fn grammar_engages_after_generated_trigger_with_merged_newlines() {
+ let tokenizer = bpe_tokenizer();
+ let trigger = grammar_trigger_token_sequence(Some("\n</think>"), &tokenizer);
+ let prompt = encode(&tokenizer, "<think>\n");
+ let trigger = grammar_trigger_token_sequence_for_prompt(trigger.as_deref(), &prompt, &tokenizer);
+
+ for config in grammars() {
+ let mut grammar = get_grammar(config, &tokenizer, &[12], trigger.clone()).unwrap();
+ assert!(!grammar.next_bitmask(&mut [0]));
+ grammar.accept_token(2).unwrap(); // merged newlines during reasoning
+ assert!(!grammar.next_bitmask(&mut [0]));
+ grammar.accept_token(4).unwrap(); // </think>
+ assert!(grammar.next_bitmask(&mut [0]));
+ // Speculative rollback across the trigger must restore reasoning.
+ grammar.rollback(1);
+ assert!(!grammar.next_bitmask(&mut [0]));
+ grammar.accept_token(4).unwrap();
+ assert!(grammar.next_bitmask(&mut [0]));
+ }
+ }
+
+ #[test]
+ fn grammar_trigger_preserves_multi_token_tags() {
+ let tokenizer = bpe_tokenizer();
+ let trigger = grammar_trigger_token_sequence(Some("\n</think><think>"), &tokenizer);
+ assert_eq!(trigger, Some(vec![4, 3]));
+ let prompt = encode(&tokenizer, "<think>\n\n</think><think>\n\n");
+ assert_eq!(grammar_trigger_token_sequence_for_prompt(trigger.as_deref(), &prompt, &tokenizer), None);
+ let history = encode(&tokenizer, "</think><think>\nYES<think>\n");
+ assert_eq!(grammar_trigger_token_sequence_for_prompt(trigger.as_deref(), &history, &tokenizer), trigger);
+ }
+
+ #[test]
+ fn grammar_trigger_preserves_whitespace_fused_with_marker_text() {
+ let tokenizer = bpe_tokenizer();
+ let expected = encode(&tokenizer, " YES");
+ assert_eq!(expected, [14, 6, 7]);
+ assert_ne!(expected, encode(&tokenizer, "YES"));
+
+ // Preserve the fused space even when a separate newline precedes it.
+ for tag in [" YES", "\n YES"] {
+ let trigger = grammar_trigger_token_sequence(Some(tag), &tokenizer);
+ assert_eq!(trigger.as_deref(), Some(expected.as_slice()));
+
+ let prompt = encode(&tokenizer, "<think>\n\n YES\n\n");
+ assert_eq!(grammar_trigger_token_sequence_for_prompt(trigger.as_deref(), &prompt, &tokenizer), None);
+
+ let prompt = encode(&tokenizer, "<think>\n");
+ let trigger = grammar_trigger_token_sequence_for_prompt(trigger.as_deref(), &prompt, &tokenizer);
+ let mut grammar = get_grammar(
+ ShojiGrammar::Regex {
+ pattern: "(YES|NO)".into(),
+ },
+ &tokenizer,
+ &[12],
+ trigger,
+ )
+ .unwrap();
+ for token in encode(&tokenizer, "\n\n YES") {
+ assert!(!grammar.next_bitmask(&mut [0]));
+ grammar.accept_token(token).unwrap();
+ }
+ assert!(grammar.next_bitmask(&mut [0]));
+ }
+ }
+
+ #[test]
+ fn grammar_trigger_ignores_missing_or_whitespace_only_tags() {
+ let tokenizer = bpe_tokenizer();
+ for tag in [None, Some(""), Some("\n\n")] {
+ assert_eq!(grammar_trigger_token_sequence(tag, &tokenizer), None);
+ }
+ }
}
#[cfg(test)]