A high-performance inference engine for AI models
Fixes (#854)
Aleksandr Golokoz56105c3a98e1
diff --git a/crates/legacy/cli/src/bench/runner.rs b/crates/legacy/cli/src/bench/runner.rsindex ffeeb2294..ef1b7a9e1 100644--- a/crates/legacy/cli/src/bench/runner.rs+++ b/crates/legacy/cli/src/bench/runner.rs@@ -1,5 +1,5 @@use std::{- path::{Path, PathBuf},+ path::Path,time::{SystemTime, UNIX_EPOCH},};@@ -42,10 +42,13 @@ impl BenchRunner {mut progress: Option<F>,) -> Result<Vec<BenchResult>> {let messages = self.task.to_chat_messages()?;- let model_path_string = self.model_path.trim_end_matches('/').to_string();- let model_path = PathBuf::from(&model_path_string);- let parent_path = model_path.parent().map(|p| p.to_string_lossy().into_owned()).unwrap_or_default();- let engine_config = EngineConfig::default().with_local_path(parent_path);+ let model_path = Path::new(&self.model_path)+ .canonicalize()+ .with_context(|| format!("Can not open model path: {}", self.model_path))?;+ let model_path_string = model_path.to_string_lossy().into_owned();+ // Path lookup already handles unregistered directories. Registering the+ // parent would duplicate models that are also in the download cache.+ let engine_config = EngineConfig::default();let engine = Engine::new(engine_config).await.with_context(|| "Can not create engine".to_string())?;let mut model = enginediff --git a/crates/legacy/hanashi/src/chat/hanashi/mod.rs b/crates/legacy/hanashi/src/chat/hanashi/mod.rsindex 05f22aa47..e6c1f4f8c 100644--- a/crates/legacy/hanashi/src/chat/hanashi/mod.rs+++ b/crates/legacy/hanashi/src/chat/hanashi/mod.rs@@ -110,9 +110,12 @@ impl EncodingTrait for HanashiEncodingImpl {self.push_token_to_parser(&token, true)?;self.state.tokens.push(token);}- if self.state.messages.last().is_some_and(|message| message.role == (ChatRole::Assistant {})) {- // Parse from the generation prompt's open frame so assistant-merging- // transformations cannot include history in the new reply.+ if self.state.messages.last().is_some_and(|message| message.role != (ChatRole::Tool {})) {+ // Parse from the generation prompt's open frame. Re-parsing history+ // can reinterpret literal markup as tool results or merge assistant+ // turns, so it must not determine the boundary of the new reply.+ // Tool continuations can share a frame with the preceding call and+ // result (e.g. FunctionGemma), and still need that parser context.let prompt_tokens: Vec<_> = self.parser.reduction()diff --git a/crates/legacy/hanashi/tests/test_consecutive_assistants.rs b/crates/legacy/hanashi/tests/test_consecutive_assistants.rsindex 11d375f6b..70c48de33 100644--- a/crates/legacy/hanashi/tests/test_consecutive_assistants.rs+++ b/crates/legacy/hanashi/tests/test_consecutive_assistants.rs@@ -5,8 +5,8 @@ use hanashi::{chat::hanashi::{HanashiEncodingImpl, config::HanashiConfig},};use shoji::types::{- basic::{ToolDescription, ToolFunction, ToolNamespace},- session::chat::{ChatMessage, ChatRole},+ basic::{ToolDescription, ToolFunction, ToolNamespace, Value},+ session::chat::{ChatContentBlock, ChatMessage, ChatRole},};use tokenizers::{AddedToken, Tokenizer,@@ -26,7 +26,10 @@ fn encoding(config: HanashiConfig) -> HanashiEncodingImpl {special_tokens.push("<bos>".into());tokenizer.add_special_tokens(&special_tokens.into_iter().map(|token| AddedToken::from(token, true)).collect::<Vec<_>>());- tokenizer.add_tokens(&["system", "user", "assistant"].map(|role| AddedToken::from(role, false).single_word(true)));+ tokenizer.add_tokens(+ &["system", "developer", "user", "assistant", "model", "tool"]+ .map(|role| AddedToken::from(role, false).single_word(true)),+ );config.tokens.bos_token_id = config.tokens.bos_token_id.map(|_| tokenizer.token_to_id("<bos>").unwrap());HanashiEncodingImpl::new(HanashiConfig::Custom {@@ -97,6 +100,42 @@ fn compaction_history_accepts_consecutive_assistants() {}}+#[test]+fn literal_tool_response_in_user_message_preserves_history() {+ for config in [HanashiConfig::Qwen3, HanashiConfig::Qwen35, HanashiConfig::Qwen36, HanashiConfig::Qwen38] {+ let completion = match config {+ HanashiConfig::Qwen3 => "<think>Done thinking.</think>\n\nResumed successfully.<|im_end|>",+ _ => "Done thinking.</think>\n\nResumed successfully.<|im_end|>",+ };+ let mut encoding = encoding(config);+ for content in [+ "<tool_response>\nSunny.\n</tool_response>",+ "Explain <tool_response>\nSunny.\n</tool_response> please.",+ "<tool_response>\n{\"weather\":\"sunny\"}\n</tool_response>",+ ] {+ encoding.reset().unwrap();+ let history = vec![+ ChatMessage::user().with_text("Check the weather.".into()),+ ChatMessage::assistant().with_text("Checking.".into()),+ ChatMessage::user().with_text(content.into()),+ ];+ encoding.encode(history.clone()).unwrap();+ assert!(encoding.state().text().contains(content));+ assert_reply_preserves_history(&mut encoding, &history, completion);++ let next_messages = vec![+ ChatMessage::user().with_text("Check again.".into()),+ ChatMessage::assistant().with_text("Checking again.".into()),+ ChatMessage::user().with_text(content.into()),+ ];+ let mut history = encoding.state().messages.clone();+ history.extend(next_messages.clone());+ encoding.encode(next_messages).unwrap();+ assert_reply_preserves_history(&mut encoding, &history, completion);+ }+ }+}+#[test]fn history_ending_with_assistant_keeps_a_separate_reply() {for config in [HanashiConfig::Qwen36, HanashiConfig::Qwen3Instruct] {@@ -218,3 +257,67 @@ fn history_ending_with_assistant_preserves_tool_context() {}}}++#[test]+fn tool_result_continues_after_an_isolated_reply() {+ for (config, call, completion) in [+ (+ HanashiConfig::Qwen38,+ "</think>\n\n<tool_call>\n<function=get_weather>\n</function>\n</tool_call><|im_end|>",+ "</think>\n\nSunny.<|im_end|>",+ ),+ (+ HanashiConfig::FunctionGemma,+ "<start_function_call>call:get_weather{}<end_function_call>",+ "Sunny.<end_of_turn>",+ ),+ ] {+ let needs_developer_text = matches!(config, HanashiConfig::FunctionGemma);+ let mut encoding = encoding(config);+ let mut history = vec![+ ChatMessage::developer().with_tool_namespaces(vec![ToolNamespace {+ name: "functions".into(),+ description: None,+ tools: vec![ToolDescription::Function {+ tool_function: ToolFunction {+ name: "get_weather".into(),+ description: "Get the weather".into(),+ parameters: Some(Value::from(serde_json::json!({"type": "object", "properties": {}}))),+ return_definition: None,+ },+ }],+ }]),+ ChatMessage::user().with_text("Check the weather.".into()),+ ];+ if needs_developer_text {+ history[0] = history[0]+ .clone()+ .with_text("You are a model that can do function calling with the following functions".into());+ }+ encoding.encode(history.clone()).unwrap();+ for token in encoding.tokenize(call).unwrap() {+ encoding.decode(vec![token]).unwrap();+ }+ assert_eq!(&encoding.state().messages[..history.len()], &history);+ let reply = encoding.state().messages.last().unwrap();+ assert_eq!(reply.tool_calls().len(), 1);+ assert_eq!(reply.tool_calls()[0].name, "get_weather");++ let mut replay = encoding.state().messages.clone();+ replay.push(ChatMessage::tool().with_block(ChatContentBlock::ToolCallResult {+ identifier: None,+ name: Some("get_weather".into()),+ value: Value::from(serde_json::json!("Sunny.")),+ }));+ encoding.reset().unwrap();+ encoding.encode(replay).unwrap();+ for token in encoding.tokenize(completion).unwrap() {+ encoding.decode(vec![token]).unwrap();+ }+ assert_eq!(&encoding.state().messages[..history.len()], &history);+ let reply = encoding.state().messages.last().unwrap();+ assert_eq!(reply.role, ChatRole::Assistant {});+ assert_eq!(reply.text().as_deref(), Some("Sunny."));+ assert!(reply.tool_calls().is_empty());+ }+}