Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

Fixes (#854)

Aleksandr Golokoz56105c3a98e1
diff --git a/crates/legacy/cli/src/bench/runner.rs b/crates/legacy/cli/src/bench/runner.rs
index ffeeb2294..ef1b7a9e1 100644
--- a/crates/legacy/cli/src/bench/runner.rs
+++ b/crates/legacy/cli/src/bench/runner.rs
@@ -1,5 +1,5 @@
use std::{
- path::{Path, PathBuf},
+ path::Path,
time::{SystemTime, UNIX_EPOCH},
};
@@ -42,10 +42,13 @@ impl BenchRunner {
mut progress: Option<F>,
) -> Result<Vec<BenchResult>> {
let messages = self.task.to_chat_messages()?;
- let model_path_string = self.model_path.trim_end_matches('/').to_string();
- let model_path = PathBuf::from(&model_path_string);
- let parent_path = model_path.parent().map(|p| p.to_string_lossy().into_owned()).unwrap_or_default();
- let engine_config = EngineConfig::default().with_local_path(parent_path);
+ let model_path = Path::new(&self.model_path)
+ .canonicalize()
+ .with_context(|| format!("Can not open model path: {}", self.model_path))?;
+ let model_path_string = model_path.to_string_lossy().into_owned();
+ // Path lookup already handles unregistered directories. Registering the
+ // parent would duplicate models that are also in the download cache.
+ let engine_config = EngineConfig::default();
let engine = Engine::new(engine_config).await.with_context(|| "Can not create engine".to_string())?;
let mut model = engine
diff --git a/crates/legacy/hanashi/src/chat/hanashi/mod.rs b/crates/legacy/hanashi/src/chat/hanashi/mod.rs
index 05f22aa47..e6c1f4f8c 100644
--- a/crates/legacy/hanashi/src/chat/hanashi/mod.rs
+++ b/crates/legacy/hanashi/src/chat/hanashi/mod.rs
@@ -110,9 +110,12 @@ impl EncodingTrait for HanashiEncodingImpl {
self.push_token_to_parser(&token, true)?;
self.state.tokens.push(token);
}
- if self.state.messages.last().is_some_and(|message| message.role == (ChatRole::Assistant {})) {
- // Parse from the generation prompt's open frame so assistant-merging
- // transformations cannot include history in the new reply.
+ if self.state.messages.last().is_some_and(|message| message.role != (ChatRole::Tool {})) {
+ // Parse from the generation prompt's open frame. Re-parsing history
+ // can reinterpret literal markup as tool results or merge assistant
+ // turns, so it must not determine the boundary of the new reply.
+ // Tool continuations can share a frame with the preceding call and
+ // result (e.g. FunctionGemma), and still need that parser context.
let prompt_tokens: Vec<_> = self
.parser
.reduction()
diff --git a/crates/legacy/hanashi/tests/test_consecutive_assistants.rs b/crates/legacy/hanashi/tests/test_consecutive_assistants.rs
index 11d375f6b..70c48de33 100644
--- a/crates/legacy/hanashi/tests/test_consecutive_assistants.rs
+++ b/crates/legacy/hanashi/tests/test_consecutive_assistants.rs
@@ -5,8 +5,8 @@ use hanashi::{
chat::hanashi::{HanashiEncodingImpl, config::HanashiConfig},
};
use shoji::types::{
- basic::{ToolDescription, ToolFunction, ToolNamespace},
- session::chat::{ChatMessage, ChatRole},
+ basic::{ToolDescription, ToolFunction, ToolNamespace, Value},
+ session::chat::{ChatContentBlock, ChatMessage, ChatRole},
};
use tokenizers::{
AddedToken, Tokenizer,
@@ -26,7 +26,10 @@ fn encoding(config: HanashiConfig) -> HanashiEncodingImpl {
special_tokens.push("<bos>".into());
tokenizer
.add_special_tokens(&special_tokens.into_iter().map(|token| AddedToken::from(token, true)).collect::<Vec<_>>());
- tokenizer.add_tokens(&["system", "user", "assistant"].map(|role| AddedToken::from(role, false).single_word(true)));
+ tokenizer.add_tokens(
+ &["system", "developer", "user", "assistant", "model", "tool"]
+ .map(|role| AddedToken::from(role, false).single_word(true)),
+ );
config.tokens.bos_token_id = config.tokens.bos_token_id.map(|_| tokenizer.token_to_id("<bos>").unwrap());
HanashiEncodingImpl::new(
HanashiConfig::Custom {
@@ -97,6 +100,42 @@ fn compaction_history_accepts_consecutive_assistants() {
}
}
+#[test]
+fn literal_tool_response_in_user_message_preserves_history() {
+ for config in [HanashiConfig::Qwen3, HanashiConfig::Qwen35, HanashiConfig::Qwen36, HanashiConfig::Qwen38] {
+ let completion = match config {
+ HanashiConfig::Qwen3 => "<think>Done thinking.</think>\n\nResumed successfully.<|im_end|>",
+ _ => "Done thinking.</think>\n\nResumed successfully.<|im_end|>",
+ };
+ let mut encoding = encoding(config);
+ for content in [
+ "<tool_response>\nSunny.\n</tool_response>",
+ "Explain <tool_response>\nSunny.\n</tool_response> please.",
+ "<tool_response>\n{\"weather\":\"sunny\"}\n</tool_response>",
+ ] {
+ encoding.reset().unwrap();
+ let history = vec![
+ ChatMessage::user().with_text("Check the weather.".into()),
+ ChatMessage::assistant().with_text("Checking.".into()),
+ ChatMessage::user().with_text(content.into()),
+ ];
+ encoding.encode(history.clone()).unwrap();
+ assert!(encoding.state().text().contains(content));
+ assert_reply_preserves_history(&mut encoding, &history, completion);
+
+ let next_messages = vec![
+ ChatMessage::user().with_text("Check again.".into()),
+ ChatMessage::assistant().with_text("Checking again.".into()),
+ ChatMessage::user().with_text(content.into()),
+ ];
+ let mut history = encoding.state().messages.clone();
+ history.extend(next_messages.clone());
+ encoding.encode(next_messages).unwrap();
+ assert_reply_preserves_history(&mut encoding, &history, completion);
+ }
+ }
+}
+
#[test]
fn history_ending_with_assistant_keeps_a_separate_reply() {
for config in [HanashiConfig::Qwen36, HanashiConfig::Qwen3Instruct] {
@@ -218,3 +257,67 @@ fn history_ending_with_assistant_preserves_tool_context() {
}
}
}
+
+#[test]
+fn tool_result_continues_after_an_isolated_reply() {
+ for (config, call, completion) in [
+ (
+ HanashiConfig::Qwen38,
+ "</think>\n\n<tool_call>\n<function=get_weather>\n</function>\n</tool_call><|im_end|>",
+ "</think>\n\nSunny.<|im_end|>",
+ ),
+ (
+ HanashiConfig::FunctionGemma,
+ "<start_function_call>call:get_weather{}<end_function_call>",
+ "Sunny.<end_of_turn>",
+ ),
+ ] {
+ let needs_developer_text = matches!(config, HanashiConfig::FunctionGemma);
+ let mut encoding = encoding(config);
+ let mut history = vec![
+ ChatMessage::developer().with_tool_namespaces(vec![ToolNamespace {
+ name: "functions".into(),
+ description: None,
+ tools: vec![ToolDescription::Function {
+ tool_function: ToolFunction {
+ name: "get_weather".into(),
+ description: "Get the weather".into(),
+ parameters: Some(Value::from(serde_json::json!({"type": "object", "properties": {}}))),
+ return_definition: None,
+ },
+ }],
+ }]),
+ ChatMessage::user().with_text("Check the weather.".into()),
+ ];
+ if needs_developer_text {
+ history[0] = history[0]
+ .clone()
+ .with_text("You are a model that can do function calling with the following functions".into());
+ }
+ encoding.encode(history.clone()).unwrap();
+ for token in encoding.tokenize(call).unwrap() {
+ encoding.decode(vec![token]).unwrap();
+ }
+ assert_eq!(&encoding.state().messages[..history.len()], &history);
+ let reply = encoding.state().messages.last().unwrap();
+ assert_eq!(reply.tool_calls().len(), 1);
+ assert_eq!(reply.tool_calls()[0].name, "get_weather");
+
+ let mut replay = encoding.state().messages.clone();
+ replay.push(ChatMessage::tool().with_block(ChatContentBlock::ToolCallResult {
+ identifier: None,
+ name: Some("get_weather".into()),
+ value: Value::from(serde_json::json!("Sunny.")),
+ }));
+ encoding.reset().unwrap();
+ encoding.encode(replay).unwrap();
+ for token in encoding.tokenize(completion).unwrap() {
+ encoding.decode(vec![token]).unwrap();
+ }
+ assert_eq!(&encoding.state().messages[..history.len()], &history);
+ let reply = encoding.state().messages.last().unwrap();
+ assert_eq!(reply.role, ChatRole::Assistant {});
+ assert_eq!(reply.text().as_deref(), Some("Sunny."));
+ assert!(reply.tool_calls().is_empty());
+ }
+}