Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

Allow multiple assistant messages (#836)

Aleksandr Golokoz4c1fa7d58825
diff --git a/crates/legacy/hanashi/configs/ordering/default.json b/crates/legacy/hanashi/configs/ordering/default.json
index d69a7deb4..598a1a0ff 100644
--- a/crates/legacy/hanashi/configs/ordering/default.json
+++ b/crates/legacy/hanashi/configs/ordering/default.json
@@ -17,6 +17,7 @@
"user"
],
"assistant": [
+ "assistant",
"tool",
"user"
],
diff --git a/crates/legacy/hanashi/src/chat/hanashi/mod.rs b/crates/legacy/hanashi/src/chat/hanashi/mod.rs
index c528b6272..05f22aa47 100644
--- a/crates/legacy/hanashi/src/chat/hanashi/mod.rs
+++ b/crates/legacy/hanashi/src/chat/hanashi/mod.rs
@@ -12,7 +12,7 @@ use shoji::types::{
basic::{Token, TokenId},
session::chat::{ChatContentBlock, ChatMessage, ChatModelCapabilities, ChatRole},
};
-use token_stream_parser::{Parser as _, token_stream::TokenStreamParser};
+use token_stream_parser::{Parser as _, ParserState as _, token_stream::TokenStreamParser};
use tokenizers::{Tokenizer, step_decode_stream};
use self::{
@@ -100,16 +100,6 @@ impl EncodingTrait for HanashiEncodingImpl {
}
self.state.messages.extend(messages.clone());
- // let transformation pipelines gate tool-call extraction on whether tools were declared
- let tools_declared = self
- .state
- .messages
- .iter()
- .any(|message| message.content.iter().any(|block| matches!(block, ChatContentBlock::Tools { .. })));
- if tools_declared {
- self.parser.set_variable("tools", serde_json::Value::Bool(true));
- }
-
let bos_token = self.config.tokens.bos_token_id.and_then(|token_id| self.resolve_token(token_id, false).ok());
let eos_token = self.config.tokens.eos_token_id.and_then(|token_id| self.resolve_token(token_id, false).ok());
let text = self.renderer.render(&messages, true, bos_token, eos_token, None)?;
@@ -120,6 +110,38 @@ impl EncodingTrait for HanashiEncodingImpl {
self.push_token_to_parser(&token, true)?;
self.state.tokens.push(token);
}
+ if self.state.messages.last().is_some_and(|message| message.role == (ChatRole::Assistant {})) {
+ // Parse from the generation prompt's open frame so assistant-merging
+ // transformations cannot include history in the new reply.
+ let prompt_tokens: Vec<_> = self
+ .parser
+ .reduction()
+ .state()
+ .sections
+ .last()
+ .ok_or(SynchronizationError::Desynchronization)?
+ .tokens()
+ .into_iter()
+ .cloned()
+ .collect();
+ self.parser.reset();
+ for token in &prompt_tokens {
+ self.parser.push_bulk(token)?;
+ }
+
+ // Reserve a separate reply so synchronization preserves the final history message.
+ self.validator.validate_next(&ChatRole::Assistant {})?;
+ self.state.messages.push(ChatMessage::assistant());
+ }
+ // Restore tool context after any parser reset, before extracting the reply.
+ let tools_declared = self
+ .state
+ .messages
+ .iter()
+ .any(|message| message.content.iter().any(|block| matches!(block, ChatContentBlock::Tools { .. })));
+ if tools_declared {
+ self.parser.set_variable("tools", serde_json::Value::Bool(true));
+ }
self.parser.flush_extraction();
self.update_messages_from_parser_state()?;
Ok(())
diff --git a/crates/legacy/hanashi/tests/test_consecutive_assistants.rs b/crates/legacy/hanashi/tests/test_consecutive_assistants.rs
new file mode 100644
index 000000000..11d375f6b
--- /dev/null
+++ b/crates/legacy/hanashi/tests/test_consecutive_assistants.rs
@@ -0,0 +1,220 @@
+use std::sync::Arc;
+
+use hanashi::{
+ Encoding,
+ chat::hanashi::{HanashiEncodingImpl, config::HanashiConfig},
+};
+use shoji::types::{
+ basic::{ToolDescription, ToolFunction, ToolNamespace},
+ session::chat::{ChatMessage, ChatRole},
+};
+use tokenizers::{
+ AddedToken, Tokenizer,
+ models::bpe::{BPE, Vocab},
+ pre_tokenizers::byte_level::ByteLevel,
+};
+
+fn encoding(config: HanashiConfig) -> HanashiEncodingImpl {
+ let mut config = config.resolve().unwrap();
+ // A byte tokenizer exercises rendering and streaming without downloaded model assets.
+ let vocab: Vocab =
+ ByteLevel::alphabet().into_iter().enumerate().map(|(id, c)| (c.to_string(), id as u32)).collect();
+ let mut tokenizer = Tokenizer::new(BPE::builder().vocab_and_merges(vocab, vec![]).build().unwrap());
+ tokenizer.with_pre_tokenizer(Some(ByteLevel::default().add_prefix_space(false)));
+ tokenizer.with_decoder(Some(ByteLevel::default()));
+ let mut special_tokens = config.parsing.framing_config().tokens;
+ special_tokens.push("<bos>".into());
+ tokenizer
+ .add_special_tokens(&special_tokens.into_iter().map(|token| AddedToken::from(token, true)).collect::<Vec<_>>());
+ tokenizer.add_tokens(&["system", "user", "assistant"].map(|role| AddedToken::from(role, false).single_word(true)));
+ config.tokens.bos_token_id = config.tokens.bos_token_id.map(|_| tokenizer.token_to_id("<bos>").unwrap());
+ HanashiEncodingImpl::new(
+ HanashiConfig::Custom {
+ config,
+ },
+ Arc::new(tokenizer),
+ )
+ .unwrap()
+}
+
+fn compaction_history() -> Vec<ChatMessage> {
+ vec![
+ ChatMessage::user().with_text("Original task".into()),
+ ChatMessage::assistant().with_text("Compacted session summary".into()),
+ ChatMessage::assistant()
+ .with_reasoning("Retained reasoning".into())
+ .with_text("Retained recent assistant turn".into()),
+ ]
+}
+
+fn assert_reply_preserves_history(
+ encoding: &mut HanashiEncodingImpl,
+ history: &[ChatMessage],
+ completion: &str,
+) {
+ let prompt = encoding.state().text();
+ assert_eq!(&encoding.state().messages[..history.len()], history);
+ assert_eq!(encoding.state().messages.len(), history.len() + 1);
+ assert_eq!(encoding.state().messages.last().unwrap().role, ChatRole::Assistant {});
+ assert!(encoding.state().messages.last().unwrap().text().unwrap_or_default().is_empty());
+ assert!(encoding.state().messages.last().unwrap().reasoning().unwrap_or_default().is_empty());
+
+ for token in encoding.tokenize(completion).unwrap() {
+ encoding.decode(vec![token]).unwrap();
+ assert_eq!(&encoding.state().messages[..history.len()], history);
+ assert_eq!(encoding.state().messages.len(), history.len() + 1);
+ let reply = encoding.state().messages.last().unwrap();
+ assert!("Resumed successfully.".starts_with(reply.text().unwrap_or_default().trim()), "{reply:?}");
+ assert!("Done thinking.".starts_with(reply.reasoning().unwrap_or_default().trim()), "{reply:?}");
+ }
+ assert_eq!(&encoding.state().messages[..history.len()], history);
+ assert_eq!(encoding.state().messages.len(), history.len() + 1);
+ assert_eq!(encoding.state().messages.last().unwrap().text().as_deref(), Some("Resumed successfully."));
+ assert_eq!(encoding.state().text(), format!("{prompt}{completion}"));
+}
+
+#[test]
+fn compaction_history_accepts_consecutive_assistants() {
+ for extra_assistant in [false, true] {
+ let mut encoding = encoding(HanashiConfig::Qwen36);
+ let mut history = compaction_history();
+ if extra_assistant {
+ history.push(ChatMessage::assistant().with_text("Another retained turn".into()));
+ }
+ history.push(ChatMessage::user().with_text("Continue".into()));
+ encoding.encode(history.clone()).unwrap();
+
+ let prompt = encoding.state().text();
+ assert!(prompt.contains(concat!(
+ "<|im_start|>assistant\nCompacted session summary<|im_end|>\n",
+ "<|im_start|>assistant\nRetained recent assistant turn<|im_end|>"
+ )));
+ assert_reply_preserves_history(
+ &mut encoding,
+ &history,
+ "Done thinking.</think>\n\nResumed successfully.<|im_end|>",
+ );
+ }
+}
+
+#[test]
+fn history_ending_with_assistant_keeps_a_separate_reply() {
+ for config in [HanashiConfig::Qwen36, HanashiConfig::Qwen3Instruct] {
+ let completion = match config {
+ HanashiConfig::Qwen36 => "Done thinking.</think>\n\nResumed successfully.<|im_end|>",
+ _ => "Resumed successfully.<|im_end|>",
+ };
+ let mut encoding = encoding(config);
+ // Even a single historical assistant must survive the new generation prompt.
+ let history = vec![
+ ChatMessage::user().with_text("Original task".into()),
+ ChatMessage::assistant().with_text("Summary".into()),
+ ];
+ encoding.encode(history.clone()).unwrap();
+ assert_reply_preserves_history(&mut encoding, &history, completion);
+
+ encoding.reset().unwrap();
+ let mut history = compaction_history();
+ history[2] = ChatMessage::assistant().with_text("Retained recent assistant turn".into());
+ encoding.encode(history.clone()).unwrap();
+ assert_reply_preserves_history(&mut encoding, &history, completion);
+ }
+}
+
+fn assert_merging_parser_keeps_a_separate_reply(
+ config: HanashiConfig,
+ completion: &str,
+) {
+ let mut encoding = encoding(config);
+ for history in [
+ vec![
+ ChatMessage::user().with_text("Original task".into()),
+ ChatMessage::assistant().with_text("Summary".into()),
+ ],
+ compaction_history(),
+ ] {
+ encoding.reset().unwrap();
+ encoding.encode(history.clone()).unwrap();
+ assert_reply_preserves_history(&mut encoding, &history, completion);
+ assert_eq!(encoding.state().messages.last().unwrap().reasoning().as_deref(), Some("Done thinking."));
+
+ // Start another reply on the same parser, preserving the reply just generated too.
+ let next_messages = vec![
+ ChatMessage::user().with_text("Continue".into()),
+ ChatMessage::assistant().with_text("Another retained turn".into()),
+ ];
+ let mut history = encoding.state().messages.clone();
+ history.extend(next_messages.clone());
+ encoding.encode(next_messages).unwrap();
+ assert_reply_preserves_history(&mut encoding, &history, completion);
+
+ let next_message = ChatMessage::user().with_text("Continue again".into());
+ let mut history = encoding.state().messages.clone();
+ history.push(next_message.clone());
+ encoding.encode(vec![next_message]).unwrap();
+ assert_reply_preserves_history(&mut encoding, &history, completion);
+ }
+}
+
+#[test]
+fn gpt_oss_history_ending_with_assistant_keeps_a_separate_reply() {
+ assert_merging_parser_keeps_a_separate_reply(
+ HanashiConfig::GptOss,
+ concat!(
+ "<|channel|>analysis<|message|>Done thinking.<|end|>",
+ "<|start|>assistant<|channel|>final<|message|>Resumed successfully.<|return|>"
+ ),
+ );
+}
+
+#[test]
+fn muse_glimmer_history_ending_with_assistant_keeps_a_separate_reply() {
+ assert_merging_parser_keeps_a_separate_reply(
+ HanashiConfig::MuseGlimmer,
+ concat!(
+ " to=self<|message|>Done thinking.<|eom|>",
+ "<|start|>assistant to=user<|message|>Resumed successfully.<|eot|>"
+ ),
+ );
+}
+
+#[test]
+fn history_ending_with_assistant_preserves_tool_context() {
+ let mut encoding = encoding(HanashiConfig::Llama32);
+ // Llama's bare-JSON tool-call extraction depends on the parser's `tools` variable.
+ for tools_declared in [true, false] {
+ encoding.reset().unwrap();
+ let mut history = vec![ChatMessage::system().with_text("You are a helpful assistant".into())];
+ if tools_declared {
+ history.push(ChatMessage::developer().with_tool_namespaces(vec![ToolNamespace {
+ name: "functions".into(),
+ description: None,
+ tools: vec![ToolDescription::Function {
+ tool_function: ToolFunction {
+ name: "get_current_time".into(),
+ description: "Returns the current time".into(),
+ parameters: None,
+ return_definition: None,
+ },
+ }],
+ }]));
+ }
+ history.push(ChatMessage::user().with_text("What time is it?".into()));
+ history.push(ChatMessage::assistant().with_text("Retained answer".into()));
+ encoding.encode(history.clone()).unwrap();
+
+ let completion = r#"{"name": "get_current_time", "parameters": {"timezone": "UTC"}}<|eom_id|>"#;
+ encoding.decode(encoding.tokenize(completion).unwrap()).unwrap();
+ assert_eq!(&encoding.state().messages[..history.len()], history);
+ assert_eq!(encoding.state().messages.len(), history.len() + 1);
+ let reply = encoding.state().messages.last().unwrap();
+ let calls = reply.tool_calls();
+ if tools_declared {
+ assert_eq!(calls.len(), 1);
+ assert_eq!(calls[0].name, "get_current_time");
+ } else {
+ assert!(calls.is_empty());
+ assert!(reply.text().unwrap().contains("\"parameters\""));
+ }
+ }
+}