Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

Rust tool calls (#639)

Aleksandr Golokozd647cf9a1cfe
diff --git a/Cargo.lock b/Cargo.lock
index 6db68c14f..63d10689d 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -3760,6 +3760,7 @@ dependencies = [
"napi",
"napi-build",
"napi-derive",
+ "proc-macros",
"pyo3",
"pyo3-async-runtimes",
"pyo3-build-config",
@@ -3770,6 +3771,7 @@ dependencies = [
"serde_json",
"shoji",
"thiserror 2.0.18",
+ "tokenizers",
"tokio",
"tokio-util",
"tracing",
@@ -4673,8 +4675,10 @@ dependencies = [
name = "proc-macros"
version = "0.5.14"
dependencies = [
+ "proc-macro-crate",
"proc-macro2",
"quote",
+ "serde_derive_internals",
"syn 2.0.118",
]
diff --git a/Cargo.toml b/Cargo.toml
index 2cacd2223..c1af03faa 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -66,6 +66,7 @@ xgrammar = { package = "xgrammar-rs", git = "https://github.com/trymirai/xgramma
half = { version = "2.7", features = ["bytemuck", "num-traits"] }
ndarray = "0.17"
serde = { version = "1.0", features = ["derive"] }
+serde_derive_internals = "0.29"
serde_json = "1.0"
monostate = "0.1"
thiserror = "2.0"
@@ -115,6 +116,7 @@ tokio-util = { version = "0.7.17", default-features = false, features = ["rt"] }
blake3 = "1.8"
quote = "1.0"
proc-macro2 = "1.0"
+proc-macro-crate = "3.5"
syn = { version = "2.0", features = ["full", "fold"] }
prettyplease = "0.2"
rhai = "1.24.0"
diff --git a/bindings/python/uzu/__init__.pyi b/bindings/python/uzu/__init__.pyi
index deb19f14c..75cbaa798 100644
--- a/bindings/python/uzu/__init__.pyi
+++ b/bindings/python/uzu/__init__.pyi
@@ -314,12 +314,19 @@ class ChatReplyConfig:
def sampling_policy(self) -> SamplingPolicy: ...
@property
def grammar(self) -> typing.Optional[Grammar]: ...
- def __new__(cls, token_limit: typing.Optional[builtins.int], sampling_policy: SamplingPolicy, grammar: typing.Optional[Grammar]) -> ChatReplyConfig: ...
+ @property
+ def tool_turn_limit(self) -> typing.Optional[builtins.int]:
+ r"""
+ Maximum number of automatic tool-call turns per reply.
+ `None` falls back to the session default.
+ """
+ def __new__(cls, token_limit: typing.Optional[builtins.int], sampling_policy: SamplingPolicy, grammar: typing.Optional[Grammar], tool_turn_limit: typing.Optional[builtins.int]) -> ChatReplyConfig: ...
@staticmethod
def create() -> ChatReplyConfig: ...
def with_token_limit(self, token_limit: typing.Optional[builtins.int]) -> ChatReplyConfig: ...
def with_sampling_policy(self, sampling_policy: SamplingPolicy) -> ChatReplyConfig: ...
def with_sampling_method(self, sampling_method: SamplingMethod) -> ChatReplyConfig: ...
+ def with_tool_turn_limit(self, tool_turn_limit: typing.Optional[builtins.int]) -> ChatReplyConfig: ...
def with_grammar(self, grammar: typing.Optional[Grammar]) -> ChatReplyConfig: ...
@typing.final
diff --git a/bindings/swift/Sources/Uzu/Generated/uzu/nagare.swift b/bindings/swift/Sources/Uzu/Generated/uzu/nagare.swift
index ec1537b2d..29203a0fa 100644
--- a/bindings/swift/Sources/Uzu/Generated/uzu/nagare.swift
+++ b/bindings/swift/Sources/Uzu/Generated/uzu/nagare.swift
@@ -416,6 +416,22 @@ fileprivate final class UniffiHandleMap<T>: @unchecked Sendable {
// Public interface members begin here.
+#if swift(>=5.8)
+@_documentation(visibility: private)
+#endif
+fileprivate struct FfiConverterUInt32: FfiConverterPrimitive {
+ typealias FfiType = UInt32
+ typealias SwiftType = UInt32
+
+ public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> UInt32 {
+ return try lift(readInt(&buf))
+ }
+
+ public static func write(_ value: SwiftType, into buf: inout [UInt8]) {
+ writeInt(&buf, lower(value))
+ }
+}
+
#if swift(>=5.8)
@_documentation(visibility: private)
#endif
@@ -1255,6 +1271,8 @@ public enum ChatSessionError: Swift.Error, Equatable, Hashable, Codable, Foundat
case UnsupportedModel
case UnableToPerformOperationInCurrentState
case NoResponse
+ case ToolTurnLimitExceeded(limit: UInt32
+ )
@@ -1293,6 +1311,9 @@ public struct FfiConverterTypeChatSessionError: FfiConverterRustBuffer {
case 3: return .UnsupportedModel
case 4: return .UnableToPerformOperationInCurrentState
case 5: return .NoResponse
+ case 6: return .ToolTurnLimitExceeded(
+ limit: try FfiConverterUInt32.read(from: &buf)
+ )
default: throw UniffiInternalError.unexpectedEnumCase
}
@@ -1326,6 +1347,11 @@ public struct FfiConverterTypeChatSessionError: FfiConverterRustBuffer {
case .NoResponse:
writeInt(&buf, Int32(5))
+
+ case let .ToolTurnLimitExceeded(limit):
+ writeInt(&buf, Int32(6))
+ FfiConverterUInt32.write(limit, into: &buf)
+
}
}
}
diff --git a/bindings/swift/Sources/Uzu/Generated/uzu/shoji.swift b/bindings/swift/Sources/Uzu/Generated/uzu/shoji.swift
index 146d6d5b6..f0ae80bb4 100644
--- a/bindings/swift/Sources/Uzu/Generated/uzu/shoji.swift
+++ b/bindings/swift/Sources/Uzu/Generated/uzu/shoji.swift
@@ -1066,13 +1066,23 @@ public struct ChatReplyConfig: Equatable, Hashable, Codable {
public var tokenLimit: UInt32?
public var samplingPolicy: SamplingPolicy
public var grammar: Grammar?
+ /**
+ * Maximum number of automatic tool-call turns per reply.
+ * `None` falls back to the session default.
+ */
+ public var toolTurnLimit: UInt32?
// Default memberwise initializers are never public by default, so we
// declare one manually.
- public init(tokenLimit: UInt32?, samplingPolicy: SamplingPolicy, grammar: Grammar?) {
+ public init(tokenLimit: UInt32?, samplingPolicy: SamplingPolicy, grammar: Grammar?,
+ /**
+ * Maximum number of automatic tool-call turns per reply.
+ * `None` falls back to the session default.
+ */toolTurnLimit: UInt32?) {
self.tokenLimit = tokenLimit
self.samplingPolicy = samplingPolicy
self.grammar = grammar
+ self.toolTurnLimit = toolTurnLimit
}
@@ -1112,6 +1122,15 @@ public func withTokenLimit(tokenLimit: UInt32?) -> ChatReplyConfig {
})
}
+public func withToolTurnLimit(toolTurnLimit: UInt32?) -> ChatReplyConfig {
+ return try! FfiConverterTypeChatReplyConfig_lift(try! rustCall() {
+ uniffi_shoji_fn_method_chatreplyconfig_with_tool_turn_limit(
+ FfiConverterTypeChatReplyConfig_lower(self),
+ FfiConverterOptionUInt32.lower(toolTurnLimit),$0
+ )
+})
+}
+
}
@@ -1129,7 +1148,8 @@ public struct FfiConverterTypeChatReplyConfig: FfiConverterRustBuffer {
try ChatReplyConfig(
tokenLimit: FfiConverterOptionUInt32.read(from: &buf),
samplingPolicy: FfiConverterTypeSamplingPolicy.read(from: &buf),
- grammar: FfiConverterOptionTypeGrammar.read(from: &buf)
+ grammar: FfiConverterOptionTypeGrammar.read(from: &buf),
+ toolTurnLimit: FfiConverterOptionUInt32.read(from: &buf)
)
}
@@ -1137,6 +1157,7 @@ public struct FfiConverterTypeChatReplyConfig: FfiConverterRustBuffer {
FfiConverterOptionUInt32.write(value.tokenLimit, into: &buf)
FfiConverterTypeSamplingPolicy.write(value.samplingPolicy, into: &buf)
FfiConverterOptionTypeGrammar.write(value.grammar, into: &buf)
+ FfiConverterOptionUInt32.write(value.toolTurnLimit, into: &buf)
}
}
diff --git a/bindings/typescript/src/napi/index.d.mts b/bindings/typescript/src/napi/index.d.mts
index eedca445e..f8dad3205 100644
--- a/bindings/typescript/src/napi/index.d.mts
+++ b/bindings/typescript/src/napi/index.d.mts
@@ -69,6 +69,7 @@ export type ChatSessionError =
| { type: 'UnsupportedModel' }
| { type: 'UnableToPerformOperationInCurrentState' }
| { type: 'NoResponse' }
+ | { type: 'ToolTurnLimitExceeded', limit: number }
export declare const enum ChatSessionState {
Idle = 'Idle',
@@ -255,12 +256,18 @@ export declare class ChatReplyConfig {
tokenLimit?: number
samplingPolicy: SamplingPolicy
grammar?: Grammar
- constructor(tokenLimit?: number, samplingPolicy: SamplingPolicy, grammar?: Grammar)
+ /**
+ * Maximum number of automatic tool-call turns per reply.
+ * `None` falls back to the session default.
+ */
+ toolTurnLimit?: number
+ constructor(tokenLimit?: number, samplingPolicy: SamplingPolicy, grammar?: Grammar, toolTurnLimit?: number)
static create(): ChatReplyConfig
withTokenLimit(tokenLimit?: number | undefined | null): ChatReplyConfig
withSamplingPolicy(samplingPolicy: SamplingPolicy): ChatReplyConfig
withSamplingMethod(samplingMethod: SamplingMethod): ChatReplyConfig
+ withToolTurnLimit(toolTurnLimit?: number | undefined | null): ChatReplyConfig
withGrammar(grammar?: Grammar | undefined | null): ChatReplyConfig
}
diff --git a/bindings/typescript/src/napi/index.d.ts b/bindings/typescript/src/napi/index.d.ts
index eedca445e..f8dad3205 100644
--- a/bindings/typescript/src/napi/index.d.ts
+++ b/bindings/typescript/src/napi/index.d.ts
@@ -69,6 +69,7 @@ export type ChatSessionError =
| { type: 'UnsupportedModel' }
| { type: 'UnableToPerformOperationInCurrentState' }
| { type: 'NoResponse' }
+ | { type: 'ToolTurnLimitExceeded', limit: number }
export declare const enum ChatSessionState {
Idle = 'Idle',
@@ -255,12 +256,18 @@ export declare class ChatReplyConfig {
tokenLimit?: number
samplingPolicy: SamplingPolicy
grammar?: Grammar
- constructor(tokenLimit?: number, samplingPolicy: SamplingPolicy, grammar?: Grammar)
+ /**
+ * Maximum number of automatic tool-call turns per reply.
+ * `None` falls back to the session default.
+ */
+ toolTurnLimit?: number
+ constructor(tokenLimit?: number, samplingPolicy: SamplingPolicy, grammar?: Grammar, toolTurnLimit?: number)
static create(): ChatReplyConfig
withTokenLimit(tokenLimit?: number | undefined | null): ChatReplyConfig
withSamplingPolicy(samplingPolicy: SamplingPolicy): ChatReplyConfig
withSamplingMethod(samplingMethod: SamplingMethod): ChatReplyConfig
+ withToolTurnLimit(toolTurnLimit?: number | undefined | null): ChatReplyConfig
withGrammar(grammar?: Grammar | undefined | null): ChatReplyConfig
}
diff --git a/crates/backend-remote/src/openai/bridging/chat/request.rs b/crates/backend-remote/src/openai/bridging/chat/request.rs
index 860944b0f..2d4a4e49d 100644
--- a/crates/backend-remote/src/openai/bridging/chat/request.rs
+++ b/crates/backend-remote/src/openai/bridging/chat/request.rs
@@ -3,7 +3,7 @@ use async_openai::types::chat::{
};
use shoji::types::{
basic::{SamplingMethod, SamplingPolicy},
- session::chat::{ChatMessage, ChatMessageList, ChatReplyConfig},
+ session::chat::{ChatContentBlock, ChatMessage, ChatMessageList, ChatReplyConfig, ChatRole},
};
use crate::openai::{
@@ -16,7 +16,11 @@ pub fn build(
config: &ChatReplyConfig,
messages: Vec<ChatMessage>,
) -> Result<CreateChatCompletionRequest, Error> {
- let completion_messages = messages.iter().map(chat::message::build).collect::<Result<Vec<_>, _>>()?;
+ let completion_messages = messages
+ .iter()
+ .filter(|message| !is_tool_definition_message(message))
+ .map(chat::message::build)
+ .collect::<Result<Vec<_>, _>>()?;
let tools = messages
.tool_namespaces()
@@ -69,3 +73,9 @@ pub fn build(
Ok(request)
}
+
+fn is_tool_definition_message(message: &ChatMessage) -> bool {
+ message.role == ChatRole::Developer {}
+ && !message.content.is_empty()
+ && message.content.iter().all(|content| matches!(content, ChatContentBlock::Tools { .. }))
+}
diff --git a/crates/backend-remote/src/openai/stream_state.rs b/crates/backend-remote/src/openai/stream_state.rs
index 5a508df07..f7c125af5 100644
--- a/crates/backend-remote/src/openai/stream_state.rs
+++ b/crates/backend-remote/src/openai/stream_state.rs
@@ -91,7 +91,11 @@ impl StreamState {
}
if let Some(reason) = chunk.finish_reason {
- self.finish_reason = Some(reason);
+ if matches!(reason, ChatReplyFinishReason::Stop) && !self.tool_calls.is_empty() {
+ self.finish_reason = Some(ChatReplyFinishReason::ToolCalls)
+ } else {
+ self.finish_reason = Some(reason);
+ }
processed = true;
}
diff --git a/crates/backend-uzu/src/bridge/backend.rs b/crates/backend-uzu/src/bridge/backend.rs
index a8e3dda60..ce0da6bee 100644
--- a/crates/backend-uzu/src/bridge/backend.rs
+++ b/crates/backend-uzu/src/bridge/backend.rs
@@ -1,4 +1,4 @@
-use std::pin::Pin;
+use std::{pin::Pin, sync::Arc};
use shoji::{
traits::{
@@ -52,7 +52,7 @@ impl ChatTokenBackend for UzuLlmBackend {
&'a self,
reference: String,
config: ChatConfig,
- #[cfg_attr(not(grammar), allow(unused_variables))] tokenizer: &'a Tokenizer,
+ #[cfg_attr(not(grammar), allow(unused_variables))] tokenizer: Arc<Tokenizer>,
) -> Pin<Box<dyn Future<Output = Result<Box<dyn ChatTokenInstance>, BackendError>> + Send + 'a>> {
Box::pin(async move {
let instance = select_backend!(
diff --git a/crates/backend-uzu/src/bridge/chat_token_backend.rs b/crates/backend-uzu/src/bridge/chat_token_backend.rs
index eae510420..3d9a09300 100644
--- a/crates/backend-uzu/src/bridge/chat_token_backend.rs
+++ b/crates/backend-uzu/src/bridge/chat_token_backend.rs
@@ -48,7 +48,7 @@ pub struct UzuChatTokenBackendInstance<B: Backend> {
model: Arc<Mutex<LanguageModel<B>>>,
config: ChatConfig,
#[cfg(grammar)]
- tokenizer: Tokenizer,
+ tokenizer: Arc<Tokenizer>,
stop_token_ids: Vec<i32>,
max_context_length: Option<usize>,
}
@@ -57,14 +57,13 @@ impl<B: Backend> UzuChatTokenBackendInstance<B> {
pub fn new(
model_path: String,
config: ChatConfig,
- #[cfg(grammar)] tokenizer: &Tokenizer,
+ #[cfg(grammar)] tokenizer: Arc<Tokenizer>,
) -> Result<Self, BackendError> {
let engine = Engine::<B>::new().map_err(|err| err.to_string())?;
let model_path = PathBuf::from(model_path);
let model = engine.load_language_model(&model_path).map_err(|err| err.to_string())?;
let stop_token_ids = model.generation_config().stop_token_ids.iter().map(|id| *id as i32).collect();
-
let max_context_length = get_max_context_length(&model, config.context_length.clone());
Ok(Self {
@@ -72,7 +71,7 @@ impl<B: Backend> UzuChatTokenBackendInstance<B> {
model: Arc::new(Mutex::new(model)),
config,
#[cfg(grammar)]
- tokenizer: tokenizer.clone(),
+ tokenizer,
stop_token_ids,
max_context_length,
})
@@ -120,7 +119,7 @@ impl<B: Backend> BackendInstance for UzuChatTokenBackendInstance<B> {
#[cfg(grammar)]
let grammar = if let Some(grammar_config) = config.grammar {
- match get_grammar(grammar_config, &self.tokenizer, &self.stop_token_ids) {
+ match get_grammar(grammar_config, self.tokenizer.as_ref(), &self.stop_token_ids) {
Ok(grammar) => Some(grammar),
Err(err) => {
return Box::pin(NoMetricsStream::new(error_stream(err.to_string())));
diff --git a/crates/hanashi/configs/parsing/functiongemma.json b/crates/hanashi/configs/parsing/functiongemma.json
index ce9a040d1..31600e2f5 100644
--- a/crates/hanashi/configs/parsing/functiongemma.json
+++ b/crates/hanashi/configs/parsing/functiongemma.json
@@ -25,6 +25,14 @@
"close_tokens": [
"<end_function_call>"
]
+ },
+ {
+ "type": "bounded",
+ "name": "tool_call_result",
+ "open_token": "<start_function_response>",
+ "close_tokens": [
+ "<end_function_response>"
+ ]
}
]
}
@@ -55,6 +63,29 @@
}
]
}
+ ],
+ "tool_call_result": [
+ {
+ "type": "on",
+ "field": "$finished",
+ "with": "$value",
+ "do": [
+ {
+ "type": "regex_replace",
+ "pattern": "^response:(\\w+)\\{([\\s\\S]*)\\}",
+ "template": "{\"name\": \"$1\", \"value\": {$2}}"
+ },
+ {
+ "type": "regex_replace",
+ "pattern": "<escape>",
+ "template": "\""
+ },
+ {
+ "type": "parse_json",
+ "repair": true
+ }
+ ]
+ }
]
}
}
diff --git a/crates/hanashi/configs/parsing/llama-3.json b/crates/hanashi/configs/parsing/llama-3.json
index d6204ee35..5ba4e744f 100644
--- a/crates/hanashi/configs/parsing/llama-3.json
+++ b/crates/hanashi/configs/parsing/llama-3.json
@@ -12,8 +12,7 @@
"groups": [
{
"type": "greedy",
- "name": "role",
- "capturing_limit": 1
+ "name": "role"
},
{
"type": "open",
@@ -32,6 +31,140 @@
]
},
"transformation": {
+ "content": [
+ {
+ "type": "switch",
+ "key": [],
+ "cases": [
+ {
+ "when": {
+ "type": "and",
+ "conditions": [
+ {
+ "type": "field",
+ "key": "$tools",
+ "condition": {
+ "type": "equals",
+ "value": true
+ }
+ },
+ {
+ "type": "field",
+ "key": "$close",
+ "condition": {
+ "type": "equals",
+ "value": "<|eom_id|>"
+ }
+ }
+ ]
+ },
+ "then": [
+ {
+ "type": "on",
+ "field": "$finished",
+ "with": "$value",
+ "do": [
+ {
+ "type": "switch",
+ "key": [
+ {
+ "type": "first"
+ },
+ {
+ "type": "get",
+ "key": "value"
+ },
+ {
+ "type": "regex_replace",
+ "pattern": "^\\s+",
+ "template": ""
+ }
+ ],
+ "cases": [
+ {
+ "when": {
+ "type": "and",
+ "conditions": [
+ {
+ "type": "starts_with",
+ "value": "{"
+ },
+ {
+ "type": "contains",
+ "value": "\"name\""
+ },
+ {
+ "type": "contains",
+ "value": "\"parameters\""
+ }
+ ]
+ },
+ "then": [
+ {
+ "type": "first"
+ },
+ {
+ "type": "get",
+ "key": "value"
+ },
+ {
+ "type": "regex_replace",
+ "pattern": "\\{\\s*\"\\s*\\}",
+ "template": "{}"
+ },
+ {
+ "type": "split_top_level",
+ "separator": ";"
+ },
+ {
+ "type": "each",
+ "apply": [
+ {
+ "type": "regex_replace",
+ "pattern": "\"type\"\\s*:\\s*\"function\"\\s*,\\s*\"function\"\\s*:\\s*(?:\\{\\s*\"name\"\\s*:)?",
+ "template": "\"name\":"
+ },
+ {
+ "type": "regex_replace",
+ "pattern": "\"parameters\"",
+ "template": "\"arguments\""
+ },
+ {
+ "type": "parse_json",
+ "repair": true
+ },
+ {
+ "type": "object",
+ "fields": {
+ "type": [
+ {
+ "type": "literal",
+ "value": "tool_call"
+ }
+ ],
+ "value": []
+ }
+ }
+ ]
+ }
+ ]
+ }
+ ],
+ "default": []
+ }
+ ]
+ }
+ ]
+ }
+ ],
+ "default": [
+ {
+ "type": "get",
+ "key": "$value"
+ }
+ ]
+ }
+ ],
"tool_call": [
{
"type": "on",
@@ -72,8 +205,13 @@
},
"then": [
{
- "type": "regex_find_all",
- "pattern": "([^;]+)"
+ "type": "regex_replace",
+ "pattern": "\\{\\s*\"\\s*\\}",
+ "template": "{}"
+ },
+ {
+ "type": "split_top_level",
+ "separator": ";"
},
{
"type": "each",
diff --git a/crates/hanashi/configs/parsing/qwen3.5.json b/crates/hanashi/configs/parsing/qwen3.5.json
index d2022f50f..cf3980016 100644
--- a/crates/hanashi/configs/parsing/qwen3.5.json
+++ b/crates/hanashi/configs/parsing/qwen3.5.json
@@ -21,6 +21,7 @@
{
"type": "bounded",
"name": "reasoning",
+ "closes_on_sibling": true,
"open_token": "<think>",
"close_tokens": [
"</think>"
@@ -33,6 +34,14 @@
"close_tokens": [
"</tool_call>"
]
+ },
+ {
+ "type": "bounded",
+ "name": "tool_call_result",
+ "open_token": "<tool_response>",
+ "close_tokens": [
+ "</tool_response>"
+ ]
}
]
}
diff --git a/crates/hanashi/configs/parsing/qwen3.6.json b/crates/hanashi/configs/parsing/qwen3.6.json
index d2022f50f..cf3980016 100644
--- a/crates/hanashi/configs/parsing/qwen3.6.json
+++ b/crates/hanashi/configs/parsing/qwen3.6.json
@@ -21,6 +21,7 @@
{
"type": "bounded",
"name": "reasoning",
+ "closes_on_sibling": true,
"open_token": "<think>",
"close_tokens": [
"</think>"
@@ -33,6 +34,14 @@
"close_tokens": [
"</tool_call>"
]
+ },
+ {
+ "type": "bounded",
+ "name": "tool_call_result",
+ "open_token": "<tool_response>",
+ "close_tokens": [
+ "</tool_response>"
+ ]
}
]
}
diff --git a/crates/hanashi/configs/parsing/qwen3.json b/crates/hanashi/configs/parsing/qwen3.json
index 82b49bf95..d76e5a670 100644
--- a/crates/hanashi/configs/parsing/qwen3.json
+++ b/crates/hanashi/configs/parsing/qwen3.json
@@ -21,6 +21,7 @@
{
"type": "bounded",
"name": "reasoning",
+ "closes_on_sibling": true,
"open_token": "<think>",
"close_tokens": [
"</think>"
@@ -33,6 +34,14 @@
"close_tokens": [
"</tool_call>"
]
+ },
+ {
+ "type": "bounded",
+ "name": "tool_call_result",
+ "open_token": "<tool_response>",
+ "close_tokens": [
+ "</tool_response>"
+ ]
}
]
}
diff --git a/crates/hanashi/configs/rendering/functiongemma.json b/crates/hanashi/configs/rendering/functiongemma.json
index ab6490f88..27a240484 100644
--- a/crates/hanashi/configs/rendering/functiongemma.json
+++ b/crates/hanashi/configs/rendering/functiongemma.json
@@ -1,6 +1,6 @@
{
"jinja": {
- "template": "{%- macro format_parameters(properties, required) -%}\n {%- set standard_keys = ['description', 'type', 'properties', 'required', 'nullable'] -%}\n {%- set ns = namespace(found_first=false) -%}\n {%- for key, value in properties | dictsort -%}\n {%- if key not in standard_keys -%}\n {%- if ns.found_first %},{% endif -%}\n {%- set ns.found_first = true -%}\n {{- key }}:{description:<escape>{{ value['description'] }}<escape>\n {%- if value['type'] | upper == 'STRING' -%}\n {%- if value['enum'] -%}\n ,enum:{{ format_argument(value['enum']) }}\n {%- endif -%}\n {%- elif value['type'] | upper == 'OBJECT' -%}\n ,properties:{\n {%- if value['properties'] is defined and value['properties'] is mapping -%}\n {{- format_parameters(value['properties'], value['required'] | default([])) -}}\n {%- elif value is mapping -%}\n {{- format_parameters(value, value['required'] | default([])) -}}\n {%- endif -%}\n }\n {%- if value['required'] -%}\n ,required:[\n {%- for item in value['required'] | default([]) -%}\n <escape>{{- item -}}<escape>\n {%- if not loop.last %},{% endif -%}\n {%- endfor -%}\n ]\n {%- endif -%}\n {%- elif value['type'] | upper == 'ARRAY' -%}\n {%- if value['items'] is mapping and value['items'] -%}\n ,items:{\n {%- set ns_items = namespace(found_first=false) -%}\n {%- for item_key, item_value in value['items'] | dictsort -%}\n {%- if item_value is not none -%}\n {%- if ns_items.found_first %},{% endif -%}\n {%- set ns_items.found_first = true -%}\n {%- if item_key == 'properties' -%}\n properties:{\n {%- if item_value is mapping -%}\n {{- format_parameters(item_value, value['items']['required'] | default([])) -}}\n {%- endif -%}\n }\n {%- elif item_key == 'required' -%}\n required:[\n {%- for req_item in item_value -%}\n <escape>{{- req_item -}}<escape>\n {%- if not loop.last %},{% endif -%}\n {%- endfor -%}\n ]\n {%- elif item_key == 'type' -%}\n {%- if item_value is string -%}\n type:{{ format_argument(item_value | upper) }}\n {%- else -%}\n type:{{ format_argument(item_value | map('upper') | list) }}\n {%- endif -%}\n {%- else -%}\n {{ item_key }}:{{ format_argument(item_value) }}\n {%- endif -%}\n {%- endif -%}\n {%- endfor -%}\n }\n {%- endif -%}\n {%- endif -%}\n ,type:<escape>{{ value['type'] | upper }}<escape>}\n {%- endif -%}\n {%- endfor -%}\n{%- endmacro -%}\n{% macro format_function_declaration(tool_data) -%}\ndeclaration:{{- tool_data['function']['name'] -}}\n{description:<escape>{{- tool_data['function']['description'] -}}<escape>\n{%- set params = tool_data['function']['parameters'] -%}\n{%- if params -%}\n ,parameters:{\n {%- if params['properties'] -%}\n properties:{ {{- format_parameters(params['properties'], params['required']) -}} },\n {%- endif -%}\n {%- if params['required'] -%}\n required:[\n {%- for item in params['required'] -%}\n <escape>{{- item -}}<escape>\n {{- ',' if not loop.last -}}\n {%- endfor -%}\n ],\n {%- endif -%}\n {%- if params['type'] -%}\n type:<escape>{{- params['type'] | upper -}}<escape>}\n {%- endif -%}\n{%- endif -%}\n}\n{%- endmacro -%}\n{% macro format_argument(argument, escape_keys=True) -%}\n{%- if argument is string -%}\n {{- '<escape>' + argument + '<escape>' -}}\n{%- elif argument is boolean -%}\n {%- if argument -%}\n {{- 'true' -}}\n {%- else -%}\n {{- 'false' -}}\n {%- endif -%}\n{%- elif argument is mapping -%}\n {{- '{' -}}\n {%- set ns = namespace(found_first=false) -%}\n {%- for key, value in argument | dictsort -%}\n {%- if ns.found_first %},{% endif -%}\n {%- set ns.found_first = true -%}\n {%- if escape_keys -%}\n {{- '<escape>' + key + '<escape>' -}}\n {%- else -%}\n {{- key -}}\n {%- endif -%}\n :{{- format_argument(value, escape_keys=escape_keys) -}}\n {%- endfor -%}\n {{- '}' -}}\n{%- elif argument is sequence -%}\n {{- '[' -}}\n {%- for item in argument -%}\n {{- format_argument(item, escape_keys=escape_keys) -}}\n {%- if not loop.last %},{% endif -%}\n {%- endfor -%}\n {{- ']' -}}\n{%- else -%}\n {{- argument -}}\n{%- endif -%}\n{%- endmacro -%}\n{{ bos_token }}\n{%- set ns = namespace(prev_message_type=None) -%}\n{#- Tool Declarations -#}\n{%- set loop_messages = messages -%}\n{%- if tools or messages[0]['role'] == 'system' or messages[0]['role'] == 'developer' -%}\n {{- '<start_of_turn>developer\\n' -}}\n {%- if messages[0]['role'] == 'system' or messages[0]['role'] == 'developer' -%}\n {%- if messages[0]['content'] is string -%}\n {{- messages[0]['content'] | trim -}}\n {%- elif messages[0]['content'] is sequence -%}\n {%- for item in messages[0]['content'] -%}\n {%- if item['type'] == 'text' -%}\n {{- item['text'] | trim -}}\n {%- endif -%}\n {%- endfor -%}\n {%- endif -%}\n {%- set loop_messages = messages[1:] -%}\n {%- endif -%}\n {%- if tools -%}\n {%- for tool in tools %}\n {{- '<start_function_declaration>' -}}\n {{- format_function_declaration(tool) | trim }}\n {{- '<end_function_declaration>' -}}\n {%- endfor %}\n {%- endif -%}\n {{- '<end_of_turn>\\n' }}\n{%- endif %}\n{#- Loop through messages. -#}\n{%- for message in loop_messages -%}\n {%- if (message['role'] == 'assistant') -%}\n {#- Rename \"assistant\" to \"model\". -#}\n {%- set role = \"model\" -%}\n {%- else -%}\n {%- set role = message['role'] -%}\n {%- endif -%}\n {%- if role != 'tool' -%}\n {%- if ns.prev_message_type != 'tool_response' -%}\n {{- '<start_of_turn>' + role + '\\n' }}\n {%- endif -%}\n {%- set ns.prev_message_type = None -%}\n {%- if 'content' in message and message['content'] is not none -%}\n {%- if message['content'] is string -%}\n {{ message['content'] | trim }}\n {%- elif message['content'] is sequence -%}\n {%- for item in message['content'] -%}\n {%- if item['type'] == 'image' -%}\n {{ '<start_of_image>' }}\n {%- elif item['type'] == 'text' -%}\n {{ item['text'] | trim }}\n {%- endif -%}\n {%- endfor -%}\n {%- else -%}\n {{ raise_exception(\"Invalid content type in user/assistant message\") }}\n {%- endif -%}\n {%- set ns.prev_message_type = 'content' -%}\n {%- endif -%}\n {%- if 'tool_calls' in message and message['tool_calls'] and message['tool_calls'] is iterable -%}\n {#- Tool Calls -#}\n {%- for tool_call in message['tool_calls'] -%}\n {% set function = tool_call['function'] %}\n {{- '<start_function_call>call:' + function['name'] + '{' -}}\n {%- if 'arguments' in function -%}\n {%- if function['arguments'] is mapping -%}\n {%- set ns = namespace(found_first=false) -%}\n {%- for key, value in function['arguments'] | dictsort -%}\n {%- if ns.found_first %},{% endif -%}\n {%- set ns.found_first = true -%}\n {{- key -}}:{{- format_argument(value, escape_keys=False) -}}\n {%- endfor -%}\n {%- elif function['arguments'] is string -%}\n {# This handles string-JSON, just in case #}\n {{ function['arguments'] }}\n {%- endif %}\n {%- endif -%}\n {{- '}<end_function_call>' -}}\n {%- endfor -%}\n {%- if loop.last -%}\n {{ '<start_function_response>' }}\n {%- endif -%}\n {%- set ns.prev_message_type = 'tool_call' -%}\n {%- endif -%}\n {%- else -%}\n {#- Tool Responses -#}\n {%- if 'content' in message and message['content'] -%}\n {%- if message['content'] is mapping -%}\n {%- if 'name' in message['content'] and 'response' in message['content'] -%}\n {{ '<start_function_response>response:' + message['content']['name'] | trim + '{' }}\n {%- set response_ns = namespace(found_first=false) -%}\n {%- for key, value in message['content']['response'] | dictsort -%}\n {%- if response_ns.found_first %},{% endif -%}\n {%- set response_ns.found_first = true -%}\n {{- key -}}:{{- format_argument(value, escape_keys=False) -}}\n {%- endfor -%}\n {{- '}<end_function_response>' -}}\n {%- elif 'name' in message -%}\n {{ '<start_function_response>response:' + message['name'] | trim + '{' }}\n {%- set response_ns = namespace(found_first=false) -%}\n {%- for key, value in message['content'] | dictsort -%}\n {%- if response_ns.found_first %},{% endif -%}\n {%- set response_ns.found_first = true -%}\n {{- key -}}:{{- format_argument(value, escape_keys=False) -}}\n {%- endfor -%}\n {{- '}<end_function_response>' -}}\n {%- else -%}\n {{ raise_exception(\"Invalid tool response mapping: must contain 'name' and 'response' keys, or 'name' must be in the message.\") }}\n {%- endif -%}\n {%- elif message['content'] is string -%}\n {%- if 'name' in message -%}\n {{ '<start_function_response>response:' + message['name'] | trim + '{value:' + format_argument(message['content'], escape_keys=False) + '}<end_function_response>' }}\n {%- else -%}\n {{ raise_exception(\"Invalid tool response: 'name' must be provided.\") }}\n {%- endif -%}\n {%- elif message['content'] is sequence -%}\n {%- for item in message['content'] -%}\n {%- if item is mapping -%}\n {%- if 'name' in item and 'response' in item -%}\n {{ '<start_function_response>response:' + item['name'] | trim + '{' }}\n {%- set response_ns = namespace(found_first=false) -%}\n {%- for key, value in item['response'] | dictsort -%}\n {%- if response_ns.found_first %},{% endif -%}\n {%- set response_ns.found_first = true -%}\n {{- key -}}:{{- format_argument(value, escape_keys=False) -}}\n {%- endfor -%}\n {{- '}<end_function_response>' -}}\n {%- elif 'name' in message -%}\n {{ '<start_function_response>response:' + message['name'] | trim + '{' }}\n {%- set response_ns = namespace(found_first=false) -%}\n {%- for key, value in item | dictsort -%}\n {%- if response_ns.found_first %},{% endif -%}\n {%- set response_ns.found_first = true -%}\n {{- key -}}:{{- format_argument(value, escape_keys=False) -}}\n {%- endfor -%}\n {{- '}<end_function_response>' -}}\n {%- else -%}\n {{ raise_exception(\"Invalid tool response mapping: must contain 'name' and 'response' keys, or 'name' must be in the message.\") }}\n {%- endif -%}\n {%- else -%}\n {{ raise_exception(\"Invalid tool response message: multiple responses must all be mappings\") }}\n {%- endif -%}\n {%- endfor -%}\n {%- else -%}\n {{ raise_exception(\"Invalid content type in tool message: must be mapping, sequence of mappings, or string.\") }}\n {%- endif -%}\n {%- endif -%}\n {%- set ns.prev_message_type = 'tool_response' -%}\n {%- endif -%}\n {%- if ns.prev_message_type not in ['tool_call', 'tool_response'] -%}\n {{ '<end_of_turn>\\n' }}\n {%- endif -%}\n{%- endfor -%}\n{%- if add_generation_prompt -%}\n {%- if ns.prev_message_type != 'tool_response' -%}\n {{- '<start_of_turn>model\\n' -}}\n {%- endif -%}\n{%- endif -%}\n",
+ "template": "{%- macro format_parameters(properties, required) -%}\n {%- set standard_keys = ['description', 'type', 'properties', 'required', 'nullable'] -%}\n {%- set ns = namespace(found_first=false) -%}\n {%- for key, value in properties | dictsort -%}\n {%- if key not in standard_keys -%}\n {%- if ns.found_first %},{% endif -%}\n {%- set ns.found_first = true -%}\n {{- key }}:{description:<escape>{{ value['description'] }}<escape>\n {%- if value['type'] | upper == 'STRING' -%}\n {%- if value['enum'] -%}\n ,enum:{{ format_argument(value['enum']) }}\n {%- endif -%}\n {%- elif value['type'] | upper == 'OBJECT' -%}\n ,properties:{\n {%- if value['properties'] is defined and value['properties'] is mapping -%}\n {{- format_parameters(value['properties'], value['required'] | default([])) -}}\n {%- elif value is mapping -%}\n {{- format_parameters(value, value['required'] | default([])) -}}\n {%- endif -%}\n }\n {%- if value['required'] -%}\n ,required:[\n {%- for item in value['required'] | default([]) -%}\n <escape>{{- item -}}<escape>\n {%- if not loop.last %},{% endif -%}\n {%- endfor -%}\n ]\n {%- endif -%}\n {%- elif value['type'] | upper == 'ARRAY' -%}\n {%- if value['items'] is mapping and value['items'] -%}\n ,items:{\n {%- set ns_items = namespace(found_first=false) -%}\n {%- for item_key, item_value in value['items'] | dictsort -%}\n {%- if item_value is not none -%}\n {%- if ns_items.found_first %},{% endif -%}\n {%- set ns_items.found_first = true -%}\n {%- if item_key == 'properties' -%}\n properties:{\n {%- if item_value is mapping -%}\n {{- format_parameters(item_value, value['items']['required'] | default([])) -}}\n {%- endif -%}\n }\n {%- elif item_key == 'required' -%}\n required:[\n {%- for req_item in item_value -%}\n <escape>{{- req_item -}}<escape>\n {%- if not loop.last %},{% endif -%}\n {%- endfor -%}\n ]\n {%- elif item_key == 'type' -%}\n {%- if item_value is string -%}\n type:{{ format_argument(item_value | upper) }}\n {%- else -%}\n type:{{ format_argument(item_value | map('upper') | list) }}\n {%- endif -%}\n {%- else -%}\n {{ item_key }}:{{ format_argument(item_value) }}\n {%- endif -%}\n {%- endif -%}\n {%- endfor -%}\n }\n {%- endif -%}\n {%- endif -%}\n ,type:<escape>{{ value['type'] | upper }}<escape>}\n {%- endif -%}\n {%- endfor -%}\n{%- endmacro -%}\n{% macro format_function_declaration(tool_data) -%}\ndeclaration:{{- tool_data['function']['name'] -}}\n{description:<escape>{{- tool_data['function']['description'] -}}<escape>\n{%- set params = tool_data['function']['parameters'] -%}\n{%- if params -%}\n ,parameters:{\n {%- if params['properties'] -%}\n properties:{ {{- format_parameters(params['properties'], params['required']) -}} },\n {%- endif -%}\n {%- if params['required'] -%}\n required:[\n {%- for item in params['required'] -%}\n <escape>{{- item -}}<escape>\n {{- ',' if not loop.last -}}\n {%- endfor -%}\n ],\n {%- endif -%}\n {%- if params['type'] -%}\n type:<escape>{{- params['type'] | upper -}}<escape>}\n {%- endif -%}\n{%- endif -%}\n}\n{%- endmacro -%}\n{% macro format_argument(argument, escape_keys=True) -%}\n{%- if argument is string -%}\n {{- '<escape>' + argument + '<escape>' -}}\n{%- elif argument is boolean -%}\n {%- if argument -%}\n {{- 'true' -}}\n {%- else -%}\n {{- 'false' -}}\n {%- endif -%}\n{%- elif argument is mapping -%}\n {{- '{' -}}\n {%- set ns = namespace(found_first=false) -%}\n {%- for key, value in argument | dictsort -%}\n {%- if ns.found_first %},{% endif -%}\n {%- set ns.found_first = true -%}\n {%- if escape_keys -%}\n {{- '<escape>' + key + '<escape>' -}}\n {%- else -%}\n {{- key -}}\n {%- endif -%}\n :{{- format_argument(value, escape_keys=escape_keys) -}}\n {%- endfor -%}\n {{- '}' -}}\n{%- elif argument is sequence -%}\n {{- '[' -}}\n {%- for item in argument -%}\n {{- format_argument(item, escape_keys=escape_keys) -}}\n {%- if not loop.last %},{% endif -%}\n {%- endfor -%}\n {{- ']' -}}\n{%- else -%}\n {{- argument -}}\n{%- endif -%}\n{%- endmacro -%}\n{{ bos_token }}\n{%- set ns = namespace(prev_message_type=None) -%}\n{#- Tool Declarations -#}\n{%- set loop_messages = messages -%}\n{%- if tools or messages[0]['role'] == 'system' or messages[0]['role'] == 'developer' -%}\n {{- '<start_of_turn>developer\\n' -}}\n {%- if messages[0]['role'] == 'system' or messages[0]['role'] == 'developer' -%}\n {%- if messages[0]['content'] is string -%}\n {{- messages[0]['content'] | trim -}}\n {%- elif messages[0]['content'] is sequence -%}\n {%- for item in messages[0]['content'] -%}\n {%- if item['type'] == 'text' -%}\n {{- item['text'] | trim -}}\n {%- endif -%}\n {%- endfor -%}\n {%- endif -%}\n {%- set loop_messages = messages[1:] -%}\n {%- endif -%}\n {%- if tools -%}\n {%- for tool in tools %}\n {{- '<start_function_declaration>' -}}\n {{- format_function_declaration(tool) | trim }}\n {{- '<end_function_declaration>' -}}\n {%- endfor %}\n {%- endif -%}\n {{- '<end_of_turn>\\n' }}\n{%- endif %}\n{#- Loop through messages. -#}\n{%- for message in loop_messages -%}\n {%- if (message['role'] == 'assistant') -%}\n {#- Rename \"assistant\" to \"model\". -#}\n {%- set role = \"model\" -%}\n {%- else -%}\n {%- set role = message['role'] -%}\n {%- endif -%}\n {%- if role != 'tool' -%}\n {%- if ns.prev_message_type != 'tool_response' -%}\n {{- '<start_of_turn>' + role + '\\n' }}\n {%- endif -%}\n {%- set ns.prev_message_type = None -%}\n {%- if 'content' in message and message['content'] is not none -%}\n {%- if message['content'] is string -%}\n {{ message['content'] | trim }}\n {%- elif message['content'] is sequence -%}\n {%- for item in message['content'] -%}\n {%- if item['type'] == 'image' -%}\n {{ '<start_of_image>' }}\n {%- elif item['type'] == 'text' -%}\n {{ item['text'] | trim }}\n {%- endif -%}\n {%- endfor -%}\n {%- else -%}\n {{ raise_exception(\"Invalid content type in user/assistant message\") }}\n {%- endif -%}\n {%- set ns.prev_message_type = 'content' -%}\n {%- endif -%}\n {%- if 'tool_calls' in message and message['tool_calls'] and message['tool_calls'] is iterable -%}\n {#- Tool Calls -#}\n {%- for tool_call in message['tool_calls'] -%}\n {% set function = tool_call['function'] if tool_call['function'] is defined else tool_call %}\n {{- '<start_function_call>call:' + function['name'] + '{' -}}\n {%- if 'arguments' in function -%}\n {%- if function['arguments'] is mapping -%}\n {%- set ns = namespace(found_first=false) -%}\n {%- for key, value in function['arguments'] | dictsort -%}\n {%- if ns.found_first %},{% endif -%}\n {%- set ns.found_first = true -%}\n {{- key -}}:{{- format_argument(value, escape_keys=False) -}}\n {%- endfor -%}\n {%- elif function['arguments'] is string -%}\n {# This handles string-JSON, just in case #}\n {{ function['arguments'] }}\n {%- endif %}\n {%- endif -%}\n {{- '}<end_function_call>' -}}\n {%- endfor -%}\n {%- if loop.last -%}\n {{ '<start_function_response>' }}\n {%- endif -%}\n {%- set ns.prev_message_type = 'tool_call' -%}\n {%- endif -%}\n {%- else -%}\n {#- Tool Responses -#}\n {%- if 'content' in message and message['content'] -%}\n {%- if message['content'] is mapping -%}\n {%- if 'name' in message['content'] and 'response' in message['content'] -%}\n {{ '<start_function_response>response:' + message['content']['name'] | trim + '{' }}\n {%- set response_ns = namespace(found_first=false) -%}\n {%- for key, value in message['content']['response'] | dictsort -%}\n {%- if response_ns.found_first %},{% endif -%}\n {%- set response_ns.found_first = true -%}\n {{- key -}}:{{- format_argument(value, escape_keys=False) -}}\n {%- endfor -%}\n {{- '}<end_function_response>' -}}\n {%- elif 'name' in message -%}\n {{ '<start_function_response>response:' + message['name'] | trim + '{' }}\n {%- set response_ns = namespace(found_first=false) -%}\n {%- for key, value in message['content'] | dictsort -%}\n {%- if response_ns.found_first %},{% endif -%}\n {%- set response_ns.found_first = true -%}\n {{- key -}}:{{- format_argument(value, escape_keys=False) -}}\n {%- endfor -%}\n {{- '}<end_function_response>' -}}\n {%- else -%}\n {{ raise_exception(\"Invalid tool response mapping: must contain 'name' and 'response' keys, or 'name' must be in the message.\") }}\n {%- endif -%}\n {%- elif message['content'] is string -%}\n {%- if 'name' in message -%}\n {{ '<start_function_response>response:' + message['name'] | trim + '{value:' + format_argument(message['content'], escape_keys=False) + '}<end_function_response>' }}\n {%- else -%}\n {{ raise_exception(\"Invalid tool response: 'name' must be provided.\") }}\n {%- endif -%}\n {%- elif message['content'] is sequence -%}\n {%- for item in message['content'] -%}\n {%- if item is mapping -%}\n {%- if 'name' in item and 'response' in item -%}\n {{ '<start_function_response>response:' + item['name'] | trim + '{' }}\n {%- set response_ns = namespace(found_first=false) -%}\n {%- for key, value in item['response'] | dictsort -%}\n {%- if response_ns.found_first %},{% endif -%}\n {%- set response_ns.found_first = true -%}\n {{- key -}}:{{- format_argument(value, escape_keys=False) -}}\n {%- endfor -%}\n {{- '}<end_function_response>' -}}\n {%- elif 'name' in message -%}\n {{ '<start_function_response>response:' + message['name'] | trim + '{' }}\n {%- set response_ns = namespace(found_first=false) -%}\n {%- for key, value in item | dictsort -%}\n {%- if response_ns.found_first %},{% endif -%}\n {%- set response_ns.found_first = true -%}\n {{- key -}}:{{- format_argument(value, escape_keys=False) -}}\n {%- endfor -%}\n {{- '}<end_function_response>' -}}\n {%- else -%}\n {{ raise_exception(\"Invalid tool response mapping: must contain 'name' and 'response' keys, or 'name' must be in the message.\") }}\n {%- endif -%}\n {%- else -%}\n {{ raise_exception(\"Invalid tool response message: multiple responses must all be mappings\") }}\n {%- endif -%}\n {%- endfor -%}\n {%- else -%}\n {{ raise_exception(\"Invalid content type in tool message: must be mapping, sequence of mappings, or string.\") }}\n {%- endif -%}\n {%- endif -%}\n {%- set ns.prev_message_type = 'tool_response' -%}\n {%- endif -%}\n {%- if ns.prev_message_type not in ['tool_call', 'tool_response'] -%}\n {{ '<end_of_turn>\\n' }}\n {%- endif -%}\n{%- endfor -%}\n{%- if add_generation_prompt -%}\n {%- if ns.prev_message_type != 'tool_response' -%}\n {{- '<start_of_turn>model\\n' -}}\n {%- endif -%}\n{%- endif -%}\n",
"required_functions": [],
"preamble_control_key": "add_generation_prompt",
"bos_token_key": "bos_token"
@@ -9,7 +9,11 @@
"role_key": "role",
"type_key": "type",
"text_key": "text",
- "url_key": "url"
+ "url_key": "url",
+ "custom_keys": {
+ "tool_call_result_name": "name",
+ "tool_call_result_response": "response"
+ }
},
"rendering": {
"developer": {
@@ -50,6 +54,7 @@
]
},
"tool_calls": {
+ "disable_raw": true,
"blocks": [
"tool_call",
"tool_call_candidate"
diff --git a/crates/hanashi/configs/rendering/gemma-4.json b/crates/hanashi/configs/rendering/gemma-4.json
index beb0faee7..6ca0ee6d0 100644
--- a/crates/hanashi/configs/rendering/gemma-4.json
+++ b/crates/hanashi/configs/rendering/gemma-4.json
@@ -62,6 +62,7 @@
"block": "reasoning"
},
"tool_calls": {
+ "disable_raw": true,
"blocks": [
"tool_call",
"tool_call_candidate"
diff --git a/crates/hanashi/configs/rendering/gpt-oss.json b/crates/hanashi/configs/rendering/gpt-oss.json
index b86b75441..75df0dce1 100644
--- a/crates/hanashi/configs/rendering/gpt-oss.json
+++ b/crates/hanashi/configs/rendering/gpt-oss.json
@@ -82,6 +82,7 @@
"block": "reasoning"
},
"tool_calls": {
+ "disable_raw": true,
"blocks": [
"tool_call",
"tool_call_candidate"
diff --git a/crates/hanashi/configs/rendering/llama-3.2.json b/crates/hanashi/configs/rendering/llama-3.2.json
index 87a593e2a..1677ae8a6 100644
--- a/crates/hanashi/configs/rendering/llama-3.2.json
+++ b/crates/hanashi/configs/rendering/llama-3.2.json
@@ -1,6 +1,6 @@
{
"jinja": {
- "template": "{{- bos_token }}\n{%- if custom_tools is defined %}\n {%- set tools = custom_tools %}\n{%- endif %}\n{%- if not tools_in_user_message is defined %}\n {%- set tools_in_user_message = true %}\n{%- endif %}\n{%- if not date_string is defined %}\n {%- if strftime_now is defined %}\n {%- set date_string = strftime_now(\"%d %b %Y\") %}\n {%- else %}\n {%- set date_string = \"26 Jul 2024\" %}\n {%- endif %}\n{%- endif %}\n{%- if not tools is defined %}\n {%- set tools = none %}\n{%- endif %}\n\n{#- This block extracts the system message, so we can slot it into the right place. #}\n{%- if messages[0]['role'] == 'system' %}\n {%- set system_message = messages[0]['content']|trim %}\n {%- set messages = messages[1:] %}\n{%- else %}\n {%- set system_message = \"\" %}\n{%- endif %}\n\n{#- System message #}\n{{- \"<|start_header_id|>system<|end_header_id|>\\n\\n\" }}\n{%- if tools is not none %}\n {{- \"Environment: ipython\\n\" }}\n{%- endif %}\n{{- \"Cutting Knowledge Date: December 2023\\n\" }}\n{{- \"Today Date: \" + date_string + \"\\n\\n\" }}\n{%- if tools is not none and not tools_in_user_message %}\n {{- \"You have access to the following functions. To call a function, please respond with JSON for a function call.\" }}\n {{- 'Respond in the format {\"name\": function name, \"parameters\": dictionary of argument name and its value}.' }}\n {{- \"Do not use variables.\\n\\n\" }}\n {%- for t in tools %}\n {{- t | tojson(indent=4) }}\n {{- \"\\n\\n\" }}\n {%- endfor %}\n{%- endif %}\n{{- system_message }}\n{{- \"<|eot_id|>\" }}\n\n{#- Custom tools are passed in a user message with some extra guidance #}\n{%- if tools_in_user_message and not tools is none %}\n {#- Extract the first user message so we can plug it in here #}\n {%- if messages | length != 0 %}\n {%- set first_user_message = messages[0]['content']|trim %}\n {%- set messages = messages[1:] %}\n {%- else %}\n {{- raise_exception(\"Cannot put tools in the first user message when there's no first user message!\") }}\n{%- endif %}\n {{- '<|start_header_id|>user<|end_header_id|>\\n\\n' -}}\n {{- \"Given the following functions, please respond with a JSON for a function call \" }}\n {{- \"with its proper arguments that best answers the given prompt.\\n\\n\" }}\n {{- 'Respond in the format {\"name\": function name, \"parameters\": dictionary of argument name and its value}.' }}\n {{- \"Do not use variables.\\n\\n\" }}\n {%- for t in tools %}\n {{- t | tojson(indent=4) }}\n {{- \"\\n\\n\" }}\n {%- endfor %}\n {{- first_user_message + \"<|eot_id|>\"}}\n{%- endif %}\n\n{%- for message in messages %}\n {%- if not (message.role == 'ipython' or message.role == 'tool' or 'tool_calls' in message) %}\n {{- '<|start_header_id|>' + message['role'] + '<|end_header_id|>\\n\\n'+ message['content'] | trim + '<|eot_id|>' }}\n {%- elif 'tool_calls' in message %}\n {%- if not message.tool_calls|length == 1 %}\n {{- raise_exception(\"This model only supports single tool-calls at once!\") }}\n {%- endif %}\n {%- set tool_call = message.tool_calls[0].function %}\n {{- '<|start_header_id|>assistant<|end_header_id|>\\n\\n' -}}\n {{- '{\"name\": \"' + tool_call.name + '\", ' }}\n {{- '\"parameters\": ' }}\n {{- tool_call.arguments | tojson }}\n {{- \"}\" }}\n {{- \"<|eot_id|>\" }}\n {%- elif message.role == \"tool\" or message.role == \"ipython\" %}\n {{- \"<|start_header_id|>ipython<|end_header_id|>\\n\\n\" }}\n {%- if message.content is mapping or message.content is iterable %}\n {{- message.content | tojson }}\n {%- else %}\n {{- message.content }}\n {%- endif %}\n {{- \"<|eot_id|>\" }}\n {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n {{- '<|start_header_id|>assistant<|end_header_id|>\\n\\n' }}\n{%- endif %}\n",
+ "template": "{{- bos_token }}\n{%- if custom_tools is defined %}\n {%- set tools = custom_tools %}\n{%- endif %}\n{%- if not tools_in_user_message is defined %}\n {%- set tools_in_user_message = true %}\n{%- endif %}\n{%- if not date_string is defined %}\n {%- if strftime_now is defined %}\n {%- set date_string = strftime_now(\"%d %b %Y\") %}\n {%- else %}\n {%- set date_string = \"26 Jul 2024\" %}\n {%- endif %}\n{%- endif %}\n{%- if not tools is defined %}\n {%- set tools = none %}\n{%- endif %}\n\n{#- This block extracts the system message, so we can slot it into the right place. #}\n{%- if messages[0]['role'] == 'system' %}\n {%- set system_message = messages[0]['content']|trim %}\n {%- set messages = messages[1:] %}\n{%- else %}\n {%- set system_message = \"\" %}\n{%- endif %}\n\n{#- System message #}\n{{- \"<|start_header_id|>system<|end_header_id|>\\n\\n\" }}\n{%- if tools is not none %}\n {{- \"Environment: ipython\\n\" }}\n{%- endif %}\n{{- \"Cutting Knowledge Date: December 2023\\n\" }}\n{{- \"Today Date: \" + date_string + \"\\n\\n\" }}\n{%- if tools is not none and not tools_in_user_message %}\n {{- \"You have access to the following functions. To call a function, please respond with JSON for a function call.\" }}\n {{- 'Respond in the format {\"name\": function name, \"parameters\": dictionary of argument name and its value}.' }}\n {{- \"Do not use variables.\\n\\n\" }}\n {%- for t in tools %}\n {{- t | tojson(indent=4) }}\n {{- \"\\n\\n\" }}\n {%- endfor %}\n{%- endif %}\n{{- system_message }}\n{{- \"<|eot_id|>\" }}\n\n{#- Custom tools are passed in a user message with some extra guidance #}\n{%- if tools_in_user_message and not tools is none %}\n {#- Extract the first user message so we can plug it in here #}\n {%- if messages | length != 0 %}\n {%- set first_user_message = messages[0]['content']|trim %}\n {%- set messages = messages[1:] %}\n {%- else %}\n {{- raise_exception(\"Cannot put tools in the first user message when there's no first user message!\") }}\n{%- endif %}\n {{- '<|start_header_id|>user<|end_header_id|>\\n\\n' -}}\n {{- \"Given the following functions, please respond with a JSON for a function call \" }}\n {{- \"with its proper arguments that best answers the given prompt.\\n\\n\" }}\n {{- 'Respond in the format {\"name\": function name, \"parameters\": dictionary of argument name and its value}.' }}\n {{- \"Do not use variables.\\n\\n\" }}\n {%- for t in tools %}\n {{- t | tojson(indent=4) }}\n {{- \"\\n\\n\" }}\n {%- endfor %}\n {{- first_user_message + \"<|eot_id|>\"}}\n{%- endif %}\n\n{%- for message in messages %}\n {%- if not (message.role == 'ipython' or message.role == 'tool' or 'tool_calls' in message) %}\n {{- '<|start_header_id|>' + message['role'] + '<|end_header_id|>\\n\\n'+ message['content'] | trim + '<|eot_id|>' }}\n {%- elif 'tool_calls' in message %}\n {{- '<|start_header_id|>assistant<|end_header_id|>\\n\\n' -}}\n {%- for tool_call in message.tool_calls %}\n {%- if tool_call.function is defined %}\n {%- set tool_call = tool_call.function %}\n {%- endif %}\n {{- '{\"name\": \"' + tool_call.name + '\", ' }}\n {{- '\"parameters\": ' }}\n {{- tool_call.arguments | tojson }}\n {{- \"}\" }}\n {%- if not loop.last %}\n {{- \"; \" }}\n {%- endif %}\n {%- endfor %}\n {{- \"<|eot_id|>\" }}\n {%- elif message.role == \"tool\" or message.role == \"ipython\" %}\n {{- \"<|start_header_id|>ipython<|end_header_id|>\\n\\n\" }}\n {%- if message.content is not string and (message.content is mapping or message.content is iterable) %}\n {{- message.content | tojson }}\n {%- else %}\n {{- message.content }}\n {%- endif %}\n {{- \"<|eot_id|>\" }}\n {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n {{- '<|start_header_id|>assistant<|end_header_id|>\\n\\n' }}\n{%- endif %}\n",
"required_functions": [
"strftime_now"
],
@@ -53,11 +53,11 @@
]
},
"tool_calls": {
+ "disable_raw": true,
"blocks": [
"tool_call",
"tool_call_candidate"
- ],
- "limit": 1
+ ]
}
}
},
diff --git a/crates/hanashi/configs/rendering/qwen3-instruct.json b/crates/hanashi/configs/rendering/qwen3-instruct.json
index 3a24ca009..faf50df85 100644
--- a/crates/hanashi/configs/rendering/qwen3-instruct.json
+++ b/crates/hanashi/configs/rendering/qwen3-instruct.json
@@ -1,6 +1,6 @@
{
"jinja": {
- "template": "{%- if tools %}\n {{- '<|im_start|>system\\n' }}\n {%- if messages[0].role == 'system' %}\n {{- messages[0].content + '\\n\\n' }}\n {%- endif %}\n {{- \"# Tools\\n\\nYou may call one or more functions to assist with the user query.\\n\\nYou are provided with function signatures within <tools></tools> XML tags:\\n<tools>\" }}\n {%- for tool in tools %}\n {{- \"\\n\" }}\n {{- tool | tojson }}\n {%- endfor %}\n {{- \"\\n</tools>\\n\\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\\n<tool_call>\\n{\\\"name\\\": <function-name>, \\\"arguments\\\": <args-json-object>}\\n</tool_call><|im_end|>\\n\" }}\n{%- else %}\n {%- if messages[0].role == 'system' %}\n {{- '<|im_start|>system\\n' + messages[0].content + '<|im_end|>\\n' }}\n {%- endif %}\n{%- endif %}\n{%- for message in messages %}\n {%- if message.content is string %}\n {%- set content = message.content %}\n {%- else %}\n {%- set content = '' %}\n {%- endif %}\n {%- if (message.role == \"user\") or (message.role == \"system\" and not loop.first) %}\n {{- '<|im_start|>' + message.role + '\\n' + content + '<|im_end|>' + '\\n' }}\n {%- elif message.role == \"assistant\" %}\n {{- '<|im_start|>' + message.role + '\\n' + content }}\n {%- if message.tool_calls %}\n {%- for tool_call in message.tool_calls %}\n {%- if (loop.first and content) or (not loop.first) %}\n {{- '\\n' }}\n {%- endif %}\n {%- if tool_call.function %}\n {%- set tool_call = tool_call.function %}\n {%- endif %}\n {{- '<tool_call>\\n{\"name\": \"' }}\n {{- tool_call.name }}\n {{- '\", \"arguments\": ' }}\n {%- if tool_call.arguments is string %}\n {{- tool_call.arguments }}\n {%- else %}\n {{- tool_call.arguments | tojson }}\n {%- endif %}\n {{- '}\\n</tool_call>' }}\n {%- endfor %}\n {%- endif %}\n {{- '<|im_end|>\\n' }}\n {%- elif message.role == \"tool\" %}\n {%- if loop.first or (messages[loop.index0 - 1].role != \"tool\") %}\n {{- '<|im_start|>user' }}\n {%- endif %}\n {{- '\\n<tool_response>\\n' }}\n {{- content }}\n {{- '\\n</tool_response>' }}\n {%- if loop.last or (messages[loop.index0 + 1].role != \"tool\") %}\n {{- '<|im_end|>\\n' }}\n {%- endif %}\n {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n {{- '<|im_start|>assistant\\n' }}\n{%- endif %}",
+ "template": "{%- if tools %}\n {{- '<|im_start|>system\\n' }}\n {%- if messages[0].role == 'system' %}\n {{- messages[0].content + '\\n\\n' }}\n {%- endif %}\n {{- \"# Tools\\n\\nYou may call one or more functions to assist with the user query.\\n\\nYou are provided with function signatures within <tools></tools> XML tags:\\n<tools>\" }}\n {%- for tool in tools %}\n {{- \"\\n\" }}\n {{- tool | tojson }}\n {%- endfor %}\n {{- \"\\n</tools>\\n\\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\\n<tool_call>\\n{\\\"name\\\": <function-name>, \\\"arguments\\\": <args-json-object>}\\n</tool_call><|im_end|>\\n\" }}\n{%- else %}\n {%- if messages[0].role == 'system' %}\n {{- '<|im_start|>system\\n' + messages[0].content + '<|im_end|>\\n' }}\n {%- endif %}\n{%- endif %}\n{%- for message in messages %}\n {%- if message.content is string %}\n {%- set content = message.content %}\n {%- elif message.content is mapping %}\n {%- set content = message.content | tojson %}\n {%- else %}\n {%- set content = '' %}\n {%- endif %}\n {%- if (message.role == \"user\") or (message.role == \"system\" and not loop.first) %}\n {{- '<|im_start|>' + message.role + '\\n' + content + '<|im_end|>' + '\\n' }}\n {%- elif message.role == \"assistant\" %}\n {{- '<|im_start|>' + message.role + '\\n' + content }}\n {%- if message.tool_calls %}\n {%- for tool_call in message.tool_calls %}\n {%- if (loop.first and content) or (not loop.first) %}\n {{- '\\n' }}\n {%- endif %}\n {%- if tool_call.function %}\n {%- set tool_call = tool_call.function %}\n {%- endif %}\n {{- '<tool_call>\\n{\"name\": \"' }}\n {{- tool_call.name }}\n {{- '\", \"arguments\": ' }}\n {%- if tool_call.arguments is string %}\n {{- tool_call.arguments }}\n {%- else %}\n {{- tool_call.arguments | tojson }}\n {%- endif %}\n {{- '}\\n</tool_call>' }}\n {%- endfor %}\n {%- endif %}\n {{- '<|im_end|>\\n' }}\n {%- elif message.role == \"tool\" %}\n {%- if loop.first or (messages[loop.index0 - 1].role != \"tool\") %}\n {{- '<|im_start|>user' }}\n {%- endif %}\n {{- '\\n<tool_response>\\n' }}\n {{- content }}\n {{- '\\n</tool_response>' }}\n {%- if loop.last or (messages[loop.index0 + 1].role != \"tool\") %}\n {{- '<|im_end|>\\n' }}\n {%- endif %}\n {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n {{- '<|im_start|>assistant\\n' }}\n{%- endif %}",
"required_functions": [],
"preamble_control_key": "add_generation_prompt"
},
@@ -50,6 +50,7 @@
]
},
"tool_calls": {
+ "disable_raw": true,
"blocks": [
"tool_call",
"tool_call_candidate"
diff --git a/crates/hanashi/configs/rendering/qwen3-thinking.json b/crates/hanashi/configs/rendering/qwen3-thinking.json
index 1fbbe10e8..23af5a87b 100644
--- a/crates/hanashi/configs/rendering/qwen3-thinking.json
+++ b/crates/hanashi/configs/rendering/qwen3-thinking.json
@@ -1,6 +1,6 @@
{
"jinja": {
- "template": "{%- if tools %}\n {{- '<|im_start|>system\\n' }}\n {%- if messages[0].role == 'system' %}\n {{- messages[0].content + '\\n\\n' }}\n {%- endif %}\n {{- \"# Tools\\n\\nYou may call one or more functions to assist with the user query.\\n\\nYou are provided with function signatures within <tools></tools> XML tags:\\n<tools>\" }}\n {%- for tool in tools %}\n {{- \"\\n\" }}\n {{- tool | tojson }}\n {%- endfor %}\n {{- \"\\n</tools>\\n\\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\\n<tool_call>\\n{\\\"name\\\": <function-name>, \\\"arguments\\\": <args-json-object>}\\n</tool_call><|im_end|>\\n\" }}\n{%- else %}\n {%- if messages[0].role == 'system' %}\n {{- '<|im_start|>system\\n' + messages[0].content + '<|im_end|>\\n' }}\n {%- endif %}\n{%- endif %}\n{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}\n{%- for message in messages[::-1] %}\n {%- set index = (messages|length - 1) - loop.index0 %}\n {%- if ns.multi_step_tool and message.role == \"user\" and message.content is string and not(message.content.startswith('<tool_response>') and message.content.endswith('</tool_response>')) %}\n {%- set ns.multi_step_tool = false %}\n {%- set ns.last_query_index = index %}\n {%- endif %}\n{%- endfor %}\n{%- for message in messages %}\n {%- if message.content is string %}\n {%- set content = message.content %}\n {%- else %}\n {%- set content = '' %}\n {%- endif %}\n {%- if (message.role == \"user\") or (message.role == \"system\" and not loop.first) %}\n {{- '<|im_start|>' + message.role + '\\n' + content + '<|im_end|>' + '\\n' }}\n {%- elif message.role == \"assistant\" %}\n {%- set reasoning_content = '' %}\n {%- if message.reasoning_content is string %}\n {%- set reasoning_content = message.reasoning_content %}\n {%- else %}\n {%- if '</think>' in content %}\n {%- set reasoning_content = content.split('</think>')[0].rstrip('\\n').split('<think>')[-1].lstrip('\\n') %}\n {%- set content = content.split('</think>')[-1].lstrip('\\n') %}\n {%- endif %}\n {%- endif %}\n {%- if loop.index0 > ns.last_query_index %}\n {%- if loop.last or (not loop.last and reasoning_content) %}\n {{- '<|im_start|>' + message.role + '\\n<think>\\n' + reasoning_content.strip('\\n') + '\\n</think>\\n\\n' + content.lstrip('\\n') }}\n {%- else %}\n {{- '<|im_start|>' + message.role + '\\n' + content }}\n {%- endif %}\n {%- else %}\n {{- '<|im_start|>' + message.role + '\\n' + content }}\n {%- endif %}\n {%- if message.tool_calls %}\n {%- for tool_call in message.tool_calls %}\n {%- if (loop.first and content) or (not loop.first) %}\n {{- '\\n' }}\n {%- endif %}\n {%- if tool_call.function %}\n {%- set tool_call = tool_call.function %}\n {%- endif %}\n {{- '<tool_call>\\n{\"name\": \"' }}\n {{- tool_call.name }}\n {{- '\", \"arguments\": ' }}\n {%- if tool_call.arguments is string %}\n {{- tool_call.arguments }}\n {%- else %}\n {{- tool_call.arguments | tojson }}\n {%- endif %}\n {{- '}\\n</tool_call>' }}\n {%- endfor %}\n {%- endif %}\n {{- '<|im_end|>\\n' }}\n {%- elif message.role == \"tool\" %}\n {%- if loop.first or (messages[loop.index0 - 1].role != \"tool\") %}\n {{- '<|im_start|>user' }}\n {%- endif %}\n {{- '\\n<tool_response>\\n' }}\n {{- content }}\n {{- '\\n</tool_response>' }}\n {%- if loop.last or (messages[loop.index0 + 1].role != \"tool\") %}\n {{- '<|im_end|>\\n' }}\n {%- endif %}\n {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n {{- '<|im_start|>assistant\\n<think>\\n' }}\n{%- endif %}",
+ "template": "{%- if tools %}\n {{- '<|im_start|>system\\n' }}\n {%- if messages[0].role == 'system' %}\n {{- messages[0].content + '\\n\\n' }}\n {%- endif %}\n {{- \"# Tools\\n\\nYou may call one or more functions to assist with the user query.\\n\\nYou are provided with function signatures within <tools></tools> XML tags:\\n<tools>\" }}\n {%- for tool in tools %}\n {{- \"\\n\" }}\n {{- tool | tojson }}\n {%- endfor %}\n {{- \"\\n</tools>\\n\\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\\n<tool_call>\\n{\\\"name\\\": <function-name>, \\\"arguments\\\": <args-json-object>}\\n</tool_call><|im_end|>\\n\" }}\n{%- else %}\n {%- if messages[0].role == 'system' %}\n {{- '<|im_start|>system\\n' + messages[0].content + '<|im_end|>\\n' }}\n {%- endif %}\n{%- endif %}\n{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}\n{%- for message in messages[::-1] %}\n {%- set index = (messages|length - 1) - loop.index0 %}\n {%- if ns.multi_step_tool and message.role == \"user\" and message.content is string and not(message.content.startswith('<tool_response>') and message.content.endswith('</tool_response>')) %}\n {%- set ns.multi_step_tool = false %}\n {%- set ns.last_query_index = index %}\n {%- endif %}\n{%- endfor %}\n{%- for message in messages %}\n {%- if message.content is string %}\n {%- set content = message.content %}\n {%- elif message.content is mapping %}\n {%- set content = message.content | tojson %}\n {%- else %}\n {%- set content = '' %}\n {%- endif %}\n {%- if (message.role == \"user\") or (message.role == \"system\" and not loop.first) %}\n {{- '<|im_start|>' + message.role + '\\n' + content + '<|im_end|>' + '\\n' }}\n {%- elif message.role == \"assistant\" %}\n {%- set reasoning_content = '' %}\n {%- if message.reasoning_content is string %}\n {%- set reasoning_content = message.reasoning_content %}\n {%- else %}\n {%- if '</think>' in content %}\n {%- set reasoning_content = content.split('</think>')[0].rstrip('\\n').split('<think>')[-1].lstrip('\\n') %}\n {%- set content = content.split('</think>')[-1].lstrip('\\n') %}\n {%- endif %}\n {%- endif %}\n {%- if loop.index0 > ns.last_query_index %}\n {%- if loop.last or (not loop.last and reasoning_content) %}\n {{- '<|im_start|>' + message.role + '\\n<think>\\n' + reasoning_content.strip('\\n') + '\\n</think>\\n\\n' + content.lstrip('\\n') }}\n {%- else %}\n {{- '<|im_start|>' + message.role + '\\n' + content }}\n {%- endif %}\n {%- else %}\n {{- '<|im_start|>' + message.role + '\\n' + content }}\n {%- endif %}\n {%- if message.tool_calls %}\n {%- for tool_call in message.tool_calls %}\n {%- if (loop.first and content) or (not loop.first) %}\n {{- '\\n' }}\n {%- endif %}\n {%- if tool_call.function %}\n {%- set tool_call = tool_call.function %}\n {%- endif %}\n {{- '<tool_call>\\n{\"name\": \"' }}\n {{- tool_call.name }}\n {{- '\", \"arguments\": ' }}\n {%- if tool_call.arguments is string %}\n {{- tool_call.arguments }}\n {%- else %}\n {{- tool_call.arguments | tojson }}\n {%- endif %}\n {{- '}\\n</tool_call>' }}\n {%- endfor %}\n {%- endif %}\n {{- '<|im_end|>\\n' }}\n {%- elif message.role == \"tool\" %}\n {%- if loop.first or (messages[loop.index0 - 1].role != \"tool\") %}\n {{- '<|im_start|>user' }}\n {%- endif %}\n {{- '\\n<tool_response>\\n' }}\n {{- content }}\n {{- '\\n</tool_response>' }}\n {%- if loop.last or (messages[loop.index0 + 1].role != \"tool\") %}\n {{- '<|im_end|>\\n' }}\n {%- endif %}\n {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n {{- '<|im_start|>assistant\\n<think>\\n' }}\n{%- endif %}",
"required_functions": [],
"preamble_control_key": "add_generation_prompt"
},
@@ -53,6 +53,7 @@
"block": "reasoning"
},
"tool_calls": {
+ "disable_raw": true,
"blocks": [
"tool_call",
"tool_call_candidate"
diff --git a/crates/hanashi/configs/rendering/qwen3.5.json b/crates/hanashi/configs/rendering/qwen3.5.json
index 695e1a359..dd9d1a0b7 100644
--- a/crates/hanashi/configs/rendering/qwen3.5.json
+++ b/crates/hanashi/configs/rendering/qwen3.5.json
@@ -1,6 +1,6 @@
{
"jinja": {
- "template": "{%- set image_count = namespace(value=0) %}\n{%- set video_count = namespace(value=0) %}\n{%- macro render_content(content, do_vision_count, is_system_content=false) %}\n {%- if content is string %}\n {{- content }}\n {%- elif content is iterable and content is not mapping %}\n {%- for item in content %}\n {%- if 'image' in item or 'image_url' in item or item.type == 'image' %}\n {%- if is_system_content %}\n {{- raise_exception('System message cannot contain images.') }}\n {%- endif %}\n {%- if do_vision_count %}\n {%- set image_count.value = image_count.value + 1 %}\n {%- endif %}\n {%- if add_vision_id %}\n {{- 'Picture ' ~ image_count.value ~ ': ' }}\n {%- endif %}\n {{- '<|vision_start|><|image_pad|><|vision_end|>' }}\n {%- elif 'video' in item or item.type == 'video' %}\n {%- if is_system_content %}\n {{- raise_exception('System message cannot contain videos.') }}\n {%- endif %}\n {%- if do_vision_count %}\n {%- set video_count.value = video_count.value + 1 %}\n {%- endif %}\n {%- if add_vision_id %}\n {{- 'Video ' ~ video_count.value ~ ': ' }}\n {%- endif %}\n {{- '<|vision_start|><|video_pad|><|vision_end|>' }}\n {%- elif 'text' in item %}\n {{- item.text }}\n {%- else %}\n {{- raise_exception('Unexpected item type in content.') }}\n {%- endif %}\n {%- endfor %}\n {%- elif content is none or content is undefined %}\n {{- '' }}\n {%- else %}\n {{- raise_exception('Unexpected content type.') }}\n {%- endif %}\n{%- endmacro %}\n{%- if not messages %}\n {{- raise_exception('No messages provided.') }}\n{%- endif %}\n{%- if tools and tools is iterable and tools is not mapping %}\n {{- '<|im_start|>system\\n' }}\n {{- \"# Tools\\n\\nYou have access to the following functions:\\n\\n<tools>\" }}\n {%- for tool in tools %}\n {{- \"\\n\" }}\n {{- tool | tojson }}\n {%- endfor %}\n {{- \"\\n</tools>\" }}\n {{- '\\n\\nIf you choose to call a function ONLY reply in the following format with NO suffix:\\n\\n<tool_call>\\n<function=example_function_name>\\n<parameter=example_parameter_1>\\nvalue_1\\n</parameter>\\n<parameter=example_parameter_2>\\nThis is the value for the second parameter\\nthat can span\\nmultiple lines\\n</parameter>\\n</function>\\n</tool_call>\\n\\n<IMPORTANT>\\nReminder:\\n- Function calls MUST follow the specified format: an inner <function=...></function> block must be nested within <tool_call></tool_call> XML tags\\n- Required parameters MUST be specified\\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\\n</IMPORTANT>' }}\n {%- if messages[0].role == 'system' %}\n {%- set content = render_content(messages[0].content, false, true)|trim %}\n {%- if content %}\n {{- '\\n\\n' + content }}\n {%- endif %}\n {%- endif %}\n {{- '<|im_end|>\\n' }}\n{%- else %}\n {%- if messages[0].role == 'system' %}\n {%- set content = render_content(messages[0].content, false, true)|trim %}\n {{- '<|im_start|>system\\n' + content + '<|im_end|>\\n' }}\n {%- endif %}\n{%- endif %}\n{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}\n{%- for message in messages[::-1] %}\n {%- set index = (messages|length - 1) - loop.index0 %}\n {%- if ns.multi_step_tool and message.role == \"user\" %}\n {%- set content = render_content(message.content, false)|trim %}\n {%- if not(content.startswith('<tool_response>') and content.endswith('</tool_response>')) %}\n {%- set ns.multi_step_tool = false %}\n {%- set ns.last_query_index = index %}\n {%- endif %}\n {%- endif %}\n{%- endfor %}\n{%- if ns.multi_step_tool %}\n {{- raise_exception('No user query found in messages.') }}\n{%- endif %}\n{%- for message in messages %}\n {%- set content = render_content(message.content, true)|trim %}\n {%- if message.role == \"system\" %}\n {%- if not loop.first %}\n {{- raise_exception('System message must be at the beginning.') }}\n {%- endif %}\n {%- elif message.role == \"user\" %}\n {{- '<|im_start|>' + message.role + '\\n' + content + '<|im_end|>' + '\\n' }}\n {%- elif message.role == \"assistant\" %}\n {%- set reasoning_content = '' %}\n {%- if message.reasoning_content is string %}\n {%- set reasoning_content = message.reasoning_content %}\n {%- else %}\n {%- if '</think>' in content %}\n {%- set reasoning_content = content.split('</think>')[0].rstrip('\\n').split('<think>')[-1].lstrip('\\n') %}\n {%- set content = content.split('</think>')[-1].lstrip('\\n') %}\n {%- endif %}\n {%- endif %}\n {%- set reasoning_content = reasoning_content|trim %}\n {%- if loop.index0 > ns.last_query_index %}\n {{- '<|im_start|>' + message.role + '\\n<think>\\n' + reasoning_content + '\\n</think>\\n\\n' + content }}\n {%- else %}\n {{- '<|im_start|>' + message.role + '\\n' + content }}\n {%- endif %}\n {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %}\n {%- for tool_call in message.tool_calls %}\n {%- if tool_call.function is defined %}\n {%- set tool_call = tool_call.function %}\n {%- endif %}\n {%- if loop.first %}\n {%- if content|trim %}\n {{- '\\n\\n<tool_call>\\n<function=' + tool_call.name + '>\\n' }}\n {%- else %}\n {{- '<tool_call>\\n<function=' + tool_call.name + '>\\n' }}\n {%- endif %}\n {%- else %}\n {{- '\\n<tool_call>\\n<function=' + tool_call.name + '>\\n' }}\n {%- endif %}\n {%- if tool_call.arguments is defined %}\n {%- for args_name, args_value in tool_call.arguments|items %}\n {{- '<parameter=' + args_name + '>\\n' }}\n {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %}\n {{- args_value }}\n {{- '\\n</parameter>\\n' }}\n {%- endfor %}\n {%- endif %}\n {{- '</function>\\n</tool_call>' }}\n {%- endfor %}\n {%- endif %}\n {{- '<|im_end|>\\n' }}\n {%- elif message.role == \"tool\" %}\n {%- if loop.previtem and loop.previtem.role != \"tool\" %}\n {{- '<|im_start|>user' }}\n {%- endif %}\n {{- '\\n<tool_response>\\n' }}\n {{- content }}\n {{- '\\n</tool_response>' }}\n {%- if not loop.last and loop.nextitem.role != \"tool\" %}\n {{- '<|im_end|>\\n' }}\n {%- elif loop.last %}\n {{- '<|im_end|>\\n' }}\n {%- endif %}\n {%- else %}\n {{- raise_exception('Unexpected message role.') }}\n {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n {{- '<|im_start|>assistant\\n' }}\n {%- if enable_thinking is defined and enable_thinking is true %}\n {{- '<think>\\n' }}\n {%- else %}\n {{- '<think>\\n\\n</think>\\n\\n' }}\n {%- endif %}\n{%- endif %}",
+ "template": "{%- set image_count = namespace(value=0) %}\n{%- set video_count = namespace(value=0) %}\n{%- macro render_content(content, do_vision_count, is_system_content=false) %}\n {%- if content is string %}\n {{- content }}\n {%- elif content is mapping %}\n {{- content | tojson }}\n {%- elif content is iterable and content is not mapping %}\n {%- for item in content %}\n {%- if 'image' in item or 'image_url' in item or item.type == 'image' %}\n {%- if is_system_content %}\n {{- raise_exception('System message cannot contain images.') }}\n {%- endif %}\n {%- if do_vision_count %}\n {%- set image_count.value = image_count.value + 1 %}\n {%- endif %}\n {%- if add_vision_id %}\n {{- 'Picture ' ~ image_count.value ~ ': ' }}\n {%- endif %}\n {{- '<|vision_start|><|image_pad|><|vision_end|>' }}\n {%- elif 'video' in item or item.type == 'video' %}\n {%- if is_system_content %}\n {{- raise_exception('System message cannot contain videos.') }}\n {%- endif %}\n {%- if do_vision_count %}\n {%- set video_count.value = video_count.value + 1 %}\n {%- endif %}\n {%- if add_vision_id %}\n {{- 'Video ' ~ video_count.value ~ ': ' }}\n {%- endif %}\n {{- '<|vision_start|><|video_pad|><|vision_end|>' }}\n {%- elif 'text' in item %}\n {{- item.text }}\n {%- else %}\n {{- raise_exception('Unexpected item type in content.') }}\n {%- endif %}\n {%- endfor %}\n {%- elif content is none or content is undefined %}\n {{- '' }}\n {%- else %}\n {{- content | tojson }}\n {%- endif %}\n{%- endmacro %}\n{%- if not messages %}\n {{- raise_exception('No messages provided.') }}\n{%- endif %}\n{%- if tools and tools is iterable and tools is not mapping %}\n {{- '<|im_start|>system\\n' }}\n {{- \"# Tools\\n\\nYou have access to the following functions:\\n\\n<tools>\" }}\n {%- for tool in tools %}\n {{- \"\\n\" }}\n {{- tool | tojson }}\n {%- endfor %}\n {{- \"\\n</tools>\" }}\n {{- '\\n\\nIf you choose to call a function ONLY reply in the following format with NO suffix:\\n\\n<tool_call>\\n<function=example_function_name>\\n<parameter=example_parameter_1>\\nvalue_1\\n</parameter>\\n<parameter=example_parameter_2>\\nThis is the value for the second parameter\\nthat can span\\nmultiple lines\\n</parameter>\\n</function>\\n</tool_call>\\n\\n<IMPORTANT>\\nReminder:\\n- Function calls MUST follow the specified format: an inner <function=...></function> block must be nested within <tool_call></tool_call> XML tags\\n- Required parameters MUST be specified\\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\\n</IMPORTANT>' }}\n {%- if messages[0].role == 'system' %}\n {%- set content = render_content(messages[0].content, false, true)|trim %}\n {%- if content %}\n {{- '\\n\\n' + content }}\n {%- endif %}\n {%- endif %}\n {{- '<|im_end|>\\n' }}\n{%- else %}\n {%- if messages[0].role == 'system' %}\n {%- set content = render_content(messages[0].content, false, true)|trim %}\n {{- '<|im_start|>system\\n' + content + '<|im_end|>\\n' }}\n {%- endif %}\n{%- endif %}\n{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}\n{%- for message in messages[::-1] %}\n {%- set index = (messages|length - 1) - loop.index0 %}\n {%- if ns.multi_step_tool and message.role == \"user\" %}\n {%- set content = render_content(message.content, false)|trim %}\n {%- if not(content.startswith('<tool_response>') and content.endswith('</tool_response>')) %}\n {%- set ns.multi_step_tool = false %}\n {%- set ns.last_query_index = index %}\n {%- endif %}\n {%- endif %}\n{%- endfor %}\n{%- if ns.multi_step_tool %}\n {{- raise_exception('No user query found in messages.') }}\n{%- endif %}\n{%- for message in messages %}\n {%- set content = render_content(message.content, true)|trim %}\n {%- if message.role == \"system\" %}\n {%- if not loop.first %}\n {{- raise_exception('System message must be at the beginning.') }}\n {%- endif %}\n {%- elif message.role == \"user\" %}\n {{- '<|im_start|>' + message.role + '\\n' + content + '<|im_end|>' + '\\n' }}\n {%- elif message.role == \"assistant\" %}\n {%- set reasoning_content = '' %}\n {%- if message.reasoning_content is string %}\n {%- set reasoning_content = message.reasoning_content %}\n {%- else %}\n {%- if '</think>' in content %}\n {%- set reasoning_content = content.split('</think>')[0].rstrip('\\n').split('<think>')[-1].lstrip('\\n') %}\n {%- set content = content.split('</think>')[-1].lstrip('\\n') %}\n {%- endif %}\n {%- endif %}\n {%- set reasoning_content = reasoning_content|trim %}\n {%- if loop.index0 > ns.last_query_index %}\n {{- '<|im_start|>' + message.role + '\\n<think>\\n' + reasoning_content + '\\n</think>\\n\\n' + content }}\n {%- else %}\n {{- '<|im_start|>' + message.role + '\\n' + content }}\n {%- endif %}\n {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %}\n {%- for tool_call in message.tool_calls %}\n {%- if tool_call.function is defined %}\n {%- set tool_call = tool_call.function %}\n {%- endif %}\n {%- if loop.first %}\n {%- if content|trim %}\n {{- '\\n\\n<tool_call>\\n<function=' + tool_call.name + '>\\n' }}\n {%- else %}\n {{- '<tool_call>\\n<function=' + tool_call.name + '>\\n' }}\n {%- endif %}\n {%- else %}\n {{- '\\n<tool_call>\\n<function=' + tool_call.name + '>\\n' }}\n {%- endif %}\n {%- if tool_call.arguments is defined %}\n {%- for args_name, args_value in tool_call.arguments|items %}\n {{- '<parameter=' + args_name + '>\\n' }}\n {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %}\n {{- args_value }}\n {{- '\\n</parameter>\\n' }}\n {%- endfor %}\n {%- endif %}\n {{- '</function>\\n</tool_call>' }}\n {%- endfor %}\n {%- endif %}\n {{- '<|im_end|>\\n' }}\n {%- elif message.role == \"tool\" %}\n {%- if loop.previtem and loop.previtem.role != \"tool\" %}\n {{- '<|im_start|>user' }}\n {%- endif %}\n {{- '\\n<tool_response>\\n' }}\n {{- content }}\n {{- '\\n</tool_response>' }}\n {%- if not loop.last and loop.nextitem.role != \"tool\" %}\n {{- '<|im_end|>\\n' }}\n {%- elif loop.last %}\n {{- '<|im_end|>\\n' }}\n {%- endif %}\n {%- else %}\n {{- raise_exception('Unexpected message role.') }}\n {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n {{- '<|im_start|>assistant\\n' }}\n {%- if enable_thinking is defined and enable_thinking is true %}\n {{- '<think>\\n' }}\n {%- else %}\n {{- '<think>\\n\\n</think>\\n\\n' }}\n {%- endif %}\n{%- endif %}",
"required_functions": [],
"preamble_control_key": "add_generation_prompt"
},
@@ -61,6 +61,7 @@
"block": "reasoning"
},
"tool_calls": {
+ "disable_raw": true,
"blocks": [
"tool_call",
"tool_call_candidate"
@@ -81,4 +82,4 @@
}
}
}
-}
\ No newline at end of file
+}
diff --git a/crates/hanashi/configs/rendering/qwen3.6.json b/crates/hanashi/configs/rendering/qwen3.6.json
index 695e1a359..677465e54 100644
--- a/crates/hanashi/configs/rendering/qwen3.6.json
+++ b/crates/hanashi/configs/rendering/qwen3.6.json
@@ -1,6 +1,6 @@
{
"jinja": {
- "template": "{%- set image_count = namespace(value=0) %}\n{%- set video_count = namespace(value=0) %}\n{%- macro render_content(content, do_vision_count, is_system_content=false) %}\n {%- if content is string %}\n {{- content }}\n {%- elif content is iterable and content is not mapping %}\n {%- for item in content %}\n {%- if 'image' in item or 'image_url' in item or item.type == 'image' %}\n {%- if is_system_content %}\n {{- raise_exception('System message cannot contain images.') }}\n {%- endif %}\n {%- if do_vision_count %}\n {%- set image_count.value = image_count.value + 1 %}\n {%- endif %}\n {%- if add_vision_id %}\n {{- 'Picture ' ~ image_count.value ~ ': ' }}\n {%- endif %}\n {{- '<|vision_start|><|image_pad|><|vision_end|>' }}\n {%- elif 'video' in item or item.type == 'video' %}\n {%- if is_system_content %}\n {{- raise_exception('System message cannot contain videos.') }}\n {%- endif %}\n {%- if do_vision_count %}\n {%- set video_count.value = video_count.value + 1 %}\n {%- endif %}\n {%- if add_vision_id %}\n {{- 'Video ' ~ video_count.value ~ ': ' }}\n {%- endif %}\n {{- '<|vision_start|><|video_pad|><|vision_end|>' }}\n {%- elif 'text' in item %}\n {{- item.text }}\n {%- else %}\n {{- raise_exception('Unexpected item type in content.') }}\n {%- endif %}\n {%- endfor %}\n {%- elif content is none or content is undefined %}\n {{- '' }}\n {%- else %}\n {{- raise_exception('Unexpected content type.') }}\n {%- endif %}\n{%- endmacro %}\n{%- if not messages %}\n {{- raise_exception('No messages provided.') }}\n{%- endif %}\n{%- if tools and tools is iterable and tools is not mapping %}\n {{- '<|im_start|>system\\n' }}\n {{- \"# Tools\\n\\nYou have access to the following functions:\\n\\n<tools>\" }}\n {%- for tool in tools %}\n {{- \"\\n\" }}\n {{- tool | tojson }}\n {%- endfor %}\n {{- \"\\n</tools>\" }}\n {{- '\\n\\nIf you choose to call a function ONLY reply in the following format with NO suffix:\\n\\n<tool_call>\\n<function=example_function_name>\\n<parameter=example_parameter_1>\\nvalue_1\\n</parameter>\\n<parameter=example_parameter_2>\\nThis is the value for the second parameter\\nthat can span\\nmultiple lines\\n</parameter>\\n</function>\\n</tool_call>\\n\\n<IMPORTANT>\\nReminder:\\n- Function calls MUST follow the specified format: an inner <function=...></function> block must be nested within <tool_call></tool_call> XML tags\\n- Required parameters MUST be specified\\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\\n</IMPORTANT>' }}\n {%- if messages[0].role == 'system' %}\n {%- set content = render_content(messages[0].content, false, true)|trim %}\n {%- if content %}\n {{- '\\n\\n' + content }}\n {%- endif %}\n {%- endif %}\n {{- '<|im_end|>\\n' }}\n{%- else %}\n {%- if messages[0].role == 'system' %}\n {%- set content = render_content(messages[0].content, false, true)|trim %}\n {{- '<|im_start|>system\\n' + content + '<|im_end|>\\n' }}\n {%- endif %}\n{%- endif %}\n{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}\n{%- for message in messages[::-1] %}\n {%- set index = (messages|length - 1) - loop.index0 %}\n {%- if ns.multi_step_tool and message.role == \"user\" %}\n {%- set content = render_content(message.content, false)|trim %}\n {%- if not(content.startswith('<tool_response>') and content.endswith('</tool_response>')) %}\n {%- set ns.multi_step_tool = false %}\n {%- set ns.last_query_index = index %}\n {%- endif %}\n {%- endif %}\n{%- endfor %}\n{%- if ns.multi_step_tool %}\n {{- raise_exception('No user query found in messages.') }}\n{%- endif %}\n{%- for message in messages %}\n {%- set content = render_content(message.content, true)|trim %}\n {%- if message.role == \"system\" %}\n {%- if not loop.first %}\n {{- raise_exception('System message must be at the beginning.') }}\n {%- endif %}\n {%- elif message.role == \"user\" %}\n {{- '<|im_start|>' + message.role + '\\n' + content + '<|im_end|>' + '\\n' }}\n {%- elif message.role == \"assistant\" %}\n {%- set reasoning_content = '' %}\n {%- if message.reasoning_content is string %}\n {%- set reasoning_content = message.reasoning_content %}\n {%- else %}\n {%- if '</think>' in content %}\n {%- set reasoning_content = content.split('</think>')[0].rstrip('\\n').split('<think>')[-1].lstrip('\\n') %}\n {%- set content = content.split('</think>')[-1].lstrip('\\n') %}\n {%- endif %}\n {%- endif %}\n {%- set reasoning_content = reasoning_content|trim %}\n {%- if loop.index0 > ns.last_query_index %}\n {{- '<|im_start|>' + message.role + '\\n<think>\\n' + reasoning_content + '\\n</think>\\n\\n' + content }}\n {%- else %}\n {{- '<|im_start|>' + message.role + '\\n' + content }}\n {%- endif %}\n {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %}\n {%- for tool_call in message.tool_calls %}\n {%- if tool_call.function is defined %}\n {%- set tool_call = tool_call.function %}\n {%- endif %}\n {%- if loop.first %}\n {%- if content|trim %}\n {{- '\\n\\n<tool_call>\\n<function=' + tool_call.name + '>\\n' }}\n {%- else %}\n {{- '<tool_call>\\n<function=' + tool_call.name + '>\\n' }}\n {%- endif %}\n {%- else %}\n {{- '\\n<tool_call>\\n<function=' + tool_call.name + '>\\n' }}\n {%- endif %}\n {%- if tool_call.arguments is defined %}\n {%- for args_name, args_value in tool_call.arguments|items %}\n {{- '<parameter=' + args_name + '>\\n' }}\n {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %}\n {{- args_value }}\n {{- '\\n</parameter>\\n' }}\n {%- endfor %}\n {%- endif %}\n {{- '</function>\\n</tool_call>' }}\n {%- endfor %}\n {%- endif %}\n {{- '<|im_end|>\\n' }}\n {%- elif message.role == \"tool\" %}\n {%- if loop.previtem and loop.previtem.role != \"tool\" %}\n {{- '<|im_start|>user' }}\n {%- endif %}\n {{- '\\n<tool_response>\\n' }}\n {{- content }}\n {{- '\\n</tool_response>' }}\n {%- if not loop.last and loop.nextitem.role != \"tool\" %}\n {{- '<|im_end|>\\n' }}\n {%- elif loop.last %}\n {{- '<|im_end|>\\n' }}\n {%- endif %}\n {%- else %}\n {{- raise_exception('Unexpected message role.') }}\n {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n {{- '<|im_start|>assistant\\n' }}\n {%- if enable_thinking is defined and enable_thinking is true %}\n {{- '<think>\\n' }}\n {%- else %}\n {{- '<think>\\n\\n</think>\\n\\n' }}\n {%- endif %}\n{%- endif %}",
+ "template": "{%- set image_count = namespace(value=0) %}\n{%- set video_count = namespace(value=0) %}\n{%- macro render_content(content, do_vision_count, is_system_content=false) %}\n {%- if content is string %}\n {{- content }}\n {%- elif content is mapping %}\n {{- content | tojson }}\n {%- elif content is iterable and content is not mapping %}\n {%- for item in content %}\n {%- if 'image' in item or 'image_url' in item or item.type == 'image' %}\n {%- if is_system_content %}\n {{- raise_exception('System message cannot contain images.') }}\n {%- endif %}\n {%- if do_vision_count %}\n {%- set image_count.value = image_count.value + 1 %}\n {%- endif %}\n {%- if add_vision_id %}\n {{- 'Picture ' ~ image_count.value ~ ': ' }}\n {%- endif %}\n {{- '<|vision_start|><|image_pad|><|vision_end|>' }}\n {%- elif 'video' in item or item.type == 'video' %}\n {%- if is_system_content %}\n {{- raise_exception('System message cannot contain videos.') }}\n {%- endif %}\n {%- if do_vision_count %}\n {%- set video_count.value = video_count.value + 1 %}\n {%- endif %}\n {%- if add_vision_id %}\n {{- 'Video ' ~ video_count.value ~ ': ' }}\n {%- endif %}\n {{- '<|vision_start|><|video_pad|><|vision_end|>' }}\n {%- elif 'text' in item %}\n {{- item.text }}\n {%- else %}\n {{- raise_exception('Unexpected item type in content.') }}\n {%- endif %}\n {%- endfor %}\n {%- elif content is none or content is undefined %}\n {{- '' }}\n {%- else %}\n {{- raise_exception('Unexpected content type.') }}\n {%- endif %}\n{%- endmacro %}\n{%- if not messages %}\n {{- raise_exception('No messages provided.') }}\n{%- endif %}\n{%- if tools and tools is iterable and tools is not mapping %}\n {{- '<|im_start|>system\\n' }}\n {{- \"# Tools\\n\\nYou have access to the following functions:\\n\\n<tools>\" }}\n {%- for tool in tools %}\n {{- \"\\n\" }}\n {{- tool | tojson }}\n {%- endfor %}\n {{- \"\\n</tools>\" }}\n {{- '\\n\\nIf you choose to call a function ONLY reply in the following format with NO suffix:\\n\\n<tool_call>\\n<function=example_function_name>\\n<parameter=example_parameter_1>\\nvalue_1\\n</parameter>\\n<parameter=example_parameter_2>\\nThis is the value for the second parameter\\nthat can span\\nmultiple lines\\n</parameter>\\n</function>\\n</tool_call>\\n\\n<IMPORTANT>\\nReminder:\\n- Function calls MUST follow the specified format: an inner <function=...></function> block must be nested within <tool_call></tool_call> XML tags\\n- Required parameters MUST be specified\\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\\n</IMPORTANT>' }}\n {%- if messages[0].role == 'system' %}\n {%- set content = render_content(messages[0].content, false, true)|trim %}\n {%- if content %}\n {{- '\\n\\n' + content }}\n {%- endif %}\n {%- endif %}\n {{- '<|im_end|>\\n' }}\n{%- else %}\n {%- if messages[0].role == 'system' %}\n {%- set content = render_content(messages[0].content, false, true)|trim %}\n {{- '<|im_start|>system\\n' + content + '<|im_end|>\\n' }}\n {%- endif %}\n{%- endif %}\n{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}\n{%- for message in messages[::-1] %}\n {%- set index = (messages|length - 1) - loop.index0 %}\n {%- if ns.multi_step_tool and message.role == \"user\" %}\n {%- set content = render_content(message.content, false)|trim %}\n {%- if not(content.startswith('<tool_response>') and content.endswith('</tool_response>')) %}\n {%- set ns.multi_step_tool = false %}\n {%- set ns.last_query_index = index %}\n {%- endif %}\n {%- endif %}\n{%- endfor %}\n{%- if ns.multi_step_tool %}\n {{- raise_exception('No user query found in messages.') }}\n{%- endif %}\n{%- for message in messages %}\n {%- set content = render_content(message.content, true)|trim %}\n {%- if message.role == \"system\" %}\n {%- if not loop.first %}\n {{- raise_exception('System message must be at the beginning.') }}\n {%- endif %}\n {%- elif message.role == \"user\" %}\n {{- '<|im_start|>' + message.role + '\\n' + content + '<|im_end|>' + '\\n' }}\n {%- elif message.role == \"assistant\" %}\n {%- set reasoning_content = '' %}\n {%- if message.reasoning_content is string %}\n {%- set reasoning_content = message.reasoning_content %}\n {%- else %}\n {%- if '</think>' in content %}\n {%- set reasoning_content = content.split('</think>')[0].rstrip('\\n').split('<think>')[-1].lstrip('\\n') %}\n {%- set content = content.split('</think>')[-1].lstrip('\\n') %}\n {%- endif %}\n {%- endif %}\n {%- set reasoning_content = reasoning_content|trim %}\n {%- if loop.index0 > ns.last_query_index %}\n {{- '<|im_start|>' + message.role + '\\n<think>\\n' + reasoning_content + '\\n</think>\\n\\n' + content }}\n {%- else %}\n {{- '<|im_start|>' + message.role + '\\n' + content }}\n {%- endif %}\n {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %}\n {%- for tool_call in message.tool_calls %}\n {%- if tool_call.function is defined %}\n {%- set tool_call = tool_call.function %}\n {%- endif %}\n {%- if loop.first %}\n {%- if content|trim %}\n {{- '\\n\\n<tool_call>\\n<function=' + tool_call.name + '>\\n' }}\n {%- else %}\n {{- '<tool_call>\\n<function=' + tool_call.name + '>\\n' }}\n {%- endif %}\n {%- else %}\n {{- '\\n<tool_call>\\n<function=' + tool_call.name + '>\\n' }}\n {%- endif %}\n {%- if tool_call.arguments is defined %}\n {%- for args_name, args_value in tool_call.arguments|items %}\n {{- '<parameter=' + args_name + '>\\n' }}\n {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %}\n {{- args_value }}\n {{- '\\n</parameter>\\n' }}\n {%- endfor %}\n {%- endif %}\n {{- '</function>\\n</tool_call>' }}\n {%- endfor %}\n {%- endif %}\n {{- '<|im_end|>\\n' }}\n {%- elif message.role == \"tool\" %}\n {%- if loop.previtem and loop.previtem.role != \"tool\" %}\n {{- '<|im_start|>user' }}\n {%- endif %}\n {{- '\\n<tool_response>\\n' }}\n {{- content }}\n {{- '\\n</tool_response>' }}\n {%- if not loop.last and loop.nextitem.role != \"tool\" %}\n {{- '<|im_end|>\\n' }}\n {%- elif loop.last %}\n {{- '<|im_end|>\\n' }}\n {%- endif %}\n {%- else %}\n {{- raise_exception('Unexpected message role.') }}\n {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n {{- '<|im_start|>assistant\\n' }}\n {%- if enable_thinking is defined and enable_thinking is true %}\n {{- '<think>\\n' }}\n {%- else %}\n {{- '<think>\\n\\n</think>\\n\\n' }}\n {%- endif %}\n{%- endif %}",
"required_functions": [],
"preamble_control_key": "add_generation_prompt"
},
@@ -61,6 +61,7 @@
"block": "reasoning"
},
"tool_calls": {
+ "disable_raw": true,
"blocks": [
"tool_call",
"tool_call_candidate"
diff --git a/crates/hanashi/configs/rendering/qwen3.json b/crates/hanashi/configs/rendering/qwen3.json
index c5e892966..7d097bbf3 100644
--- a/crates/hanashi/configs/rendering/qwen3.json
+++ b/crates/hanashi/configs/rendering/qwen3.json
@@ -1,6 +1,6 @@
{
"jinja": {
- "template": "{%- if tools %}\n {{- '<|im_start|>system\\n' }}\n {%- if messages[0].role == 'system' %}\n {{- messages[0].content + '\\n\\n' }}\n {%- endif %}\n {{- \"# Tools\\n\\nYou may call one or more functions to assist with the user query.\\n\\nYou are provided with function signatures within <tools></tools> XML tags:\\n<tools>\" }}\n {%- for tool in tools %}\n {{- \"\\n\" }}\n {{- tool | tojson }}\n {%- endfor %}\n {{- \"\\n</tools>\\n\\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\\n<tool_call>\\n{\\\"name\\\": <function-name>, \\\"arguments\\\": <args-json-object>}\\n</tool_call><|im_end|>\\n\" }}\n{%- else %}\n {%- if messages[0].role == 'system' %}\n {{- '<|im_start|>system\\n' + messages[0].content + '<|im_end|>\\n' }}\n {%- endif %}\n{%- endif %}\n{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}\n{%- for message in messages[::-1] %}\n {%- set index = (messages|length - 1) - loop.index0 %}\n {%- if ns.multi_step_tool and message.role == \"user\" and message.content is string and not(message.content.startswith('<tool_response>') and message.content.endswith('</tool_response>')) %}\n {%- set ns.multi_step_tool = false %}\n {%- set ns.last_query_index = index %}\n {%- endif %}\n{%- endfor %}\n{%- for message in messages %}\n {%- if message.content is string %}\n {%- set content = message.content %}\n {%- else %}\n {%- set content = '' %}\n {%- endif %}\n {%- if (message.role == \"user\") or (message.role == \"system\" and not loop.first) %}\n {{- '<|im_start|>' + message.role + '\\n' + content + '<|im_end|>' + '\\n' }}\n {%- elif message.role == \"assistant\" %}\n {%- set reasoning_content = '' %}\n {%- if message.reasoning_content is string %}\n {%- set reasoning_content = message.reasoning_content %}\n {%- else %}\n {%- if '</think>' in content %}\n {%- set reasoning_content = content.split('</think>')[0].rstrip('\\n').split('<think>')[-1].lstrip('\\n') %}\n {%- set content = content.split('</think>')[-1].lstrip('\\n') %}\n {%- endif %}\n {%- endif %}\n {%- if loop.index0 > ns.last_query_index %}\n {%- if loop.last or (not loop.last and reasoning_content) %}\n {{- '<|im_start|>' + message.role + '\\n<think>\\n' + reasoning_content.strip('\\n') + '\\n</think>\\n\\n' + content.lstrip('\\n') }}\n {%- else %}\n {{- '<|im_start|>' + message.role + '\\n' + content }}\n {%- endif %}\n {%- else %}\n {{- '<|im_start|>' + message.role + '\\n' + content }}\n {%- endif %}\n {%- if message.tool_calls %}\n {%- for tool_call in message.tool_calls %}\n {%- if (loop.first and content) or (not loop.first) %}\n {{- '\\n' }}\n {%- endif %}\n {%- if tool_call.function %}\n {%- set tool_call = tool_call.function %}\n {%- endif %}\n {{- '<tool_call>\\n{\"name\": \"' }}\n {{- tool_call.name }}\n {{- '\", \"arguments\": ' }}\n {%- if tool_call.arguments is string %}\n {{- tool_call.arguments }}\n {%- else %}\n {{- tool_call.arguments | tojson }}\n {%- endif %}\n {{- '}\\n</tool_call>' }}\n {%- endfor %}\n {%- endif %}\n {{- '<|im_end|>\\n' }}\n {%- elif message.role == \"tool\" %}\n {%- if loop.first or (messages[loop.index0 - 1].role != \"tool\") %}\n {{- '<|im_start|>user' }}\n {%- endif %}\n {{- '\\n<tool_response>\\n' }}\n {{- content }}\n {{- '\\n</tool_response>' }}\n {%- if loop.last or (messages[loop.index0 + 1].role != \"tool\") %}\n {{- '<|im_end|>\\n' }}\n {%- endif %}\n {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n {{- '<|im_start|>assistant\\n' }}\n {%- if enable_thinking is defined and enable_thinking is false %}\n {{- '<think>\\n\\n</think>\\n\\n' }}\n {%- endif %}\n{%- endif %}",
+ "template": "{%- if tools %}\n {{- '<|im_start|>system\\n' }}\n {%- if messages[0].role == 'system' %}\n {{- messages[0].content + '\\n\\n' }}\n {%- endif %}\n {{- \"# Tools\\n\\nYou may call one or more functions to assist with the user query.\\n\\nYou are provided with function signatures within <tools></tools> XML tags:\\n<tools>\" }}\n {%- for tool in tools %}\n {{- \"\\n\" }}\n {{- tool | tojson }}\n {%- endfor %}\n {{- \"\\n</tools>\\n\\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\\n<tool_call>\\n{\\\"name\\\": <function-name>, \\\"arguments\\\": <args-json-object>}\\n</tool_call><|im_end|>\\n\" }}\n{%- else %}\n {%- if messages[0].role == 'system' %}\n {{- '<|im_start|>system\\n' + messages[0].content + '<|im_end|>\\n' }}\n {%- endif %}\n{%- endif %}\n{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}\n{%- for message in messages[::-1] %}\n {%- set index = (messages|length - 1) - loop.index0 %}\n {%- if ns.multi_step_tool and message.role == \"user\" and message.content is string and not(message.content.startswith('<tool_response>') and message.content.endswith('</tool_response>')) %}\n {%- set ns.multi_step_tool = false %}\n {%- set ns.last_query_index = index %}\n {%- endif %}\n{%- endfor %}\n{%- for message in messages %}\n {%- if message.content is string %}\n {%- set content = message.content %}\n {%- elif message.content is mapping %}\n {%- set content = message.content | tojson %}\n {%- else %}\n {%- set content = '' %}\n {%- endif %}\n {%- if (message.role == \"user\") or (message.role == \"system\" and not loop.first) %}\n {{- '<|im_start|>' + message.role + '\\n' + content + '<|im_end|>' + '\\n' }}\n {%- elif message.role == \"assistant\" %}\n {%- set reasoning_content = '' %}\n {%- if message.reasoning_content is string %}\n {%- set reasoning_content = message.reasoning_content %}\n {%- else %}\n {%- if '</think>' in content %}\n {%- set reasoning_content = content.split('</think>')[0].rstrip('\\n').split('<think>')[-1].lstrip('\\n') %}\n {%- set content = content.split('</think>')[-1].lstrip('\\n') %}\n {%- endif %}\n {%- endif %}\n {%- if loop.index0 > ns.last_query_index %}\n {%- if loop.last or (not loop.last and reasoning_content) %}\n {{- '<|im_start|>' + message.role + '\\n<think>\\n' + reasoning_content.strip('\\n') + '\\n</think>\\n\\n' + content.lstrip('\\n') }}\n {%- else %}\n {{- '<|im_start|>' + message.role + '\\n' + content }}\n {%- endif %}\n {%- else %}\n {{- '<|im_start|>' + message.role + '\\n' + content }}\n {%- endif %}\n {%- if message.tool_calls %}\n {%- for tool_call in message.tool_calls %}\n {%- if (loop.first and content) or (not loop.first) %}\n {{- '\\n' }}\n {%- endif %}\n {%- if tool_call.function %}\n {%- set tool_call = tool_call.function %}\n {%- endif %}\n {{- '<tool_call>\\n{\"name\": \"' }}\n {{- tool_call.name }}\n {{- '\", \"arguments\": ' }}\n {%- if tool_call.arguments is string %}\n {{- tool_call.arguments }}\n {%- else %}\n {{- tool_call.arguments | tojson }}\n {%- endif %}\n {{- '}\\n</tool_call>' }}\n {%- endfor %}\n {%- endif %}\n {{- '<|im_end|>\\n' }}\n {%- elif message.role == \"tool\" %}\n {%- if loop.first or (messages[loop.index0 - 1].role != \"tool\") %}\n {{- '<|im_start|>user' }}\n {%- endif %}\n {{- '\\n<tool_response>\\n' }}\n {{- content }}\n {{- '\\n</tool_response>' }}\n {%- if loop.last or (messages[loop.index0 + 1].role != \"tool\") %}\n {{- '<|im_end|>\\n' }}\n {%- endif %}\n {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n {{- '<|im_start|>assistant\\n' }}\n {%- if enable_thinking is defined and enable_thinking is false %}\n {{- '<think>\\n\\n</think>\\n\\n' }}\n {%- endif %}\n{%- endif %}",
"required_functions": [],
"preamble_control_key": "add_generation_prompt"
},
@@ -61,6 +61,7 @@
"block": "reasoning"
},
"tool_calls": {
+ "disable_raw": true,
"blocks": [
"tool_call",
"tool_call_candidate"
diff --git a/crates/hanashi/src/chat/context.rs b/crates/hanashi/src/chat/context.rs
index 01a8da1c8..65ccf98ca 100644
--- a/crates/hanashi/src/chat/context.rs
+++ b/crates/hanashi/src/chat/context.rs
@@ -10,8 +10,3 @@ pub enum TokenizerLocation {
path: String,
},
}
-
-#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
-pub struct Context {
- pub tokenizer_location: TokenizerLocation,
-}
diff --git a/crates/hanashi/src/chat/hanashi/messages/canonical/message.rs b/crates/hanashi/src/chat/hanashi/messages/canonical/message.rs
index b59acda22..fb44d50a7 100644
--- a/crates/hanashi/src/chat/hanashi/messages/canonical/message.rs
+++ b/crates/hanashi/src/chat/hanashi/messages/canonical/message.rs
@@ -132,9 +132,34 @@ impl Message {
} => serde_json::to_value(value.clone()),
OriginalContentBlock::ToolCallResult {
identifier: _,
- name: _,
+ name,
value,
- } => serde_json::to_value(value.clone()),
+ } => {
+ let name_key = config.custom_keys.get("tool_call_result_name");
+ let response_key = config.custom_keys.get("tool_call_result_response");
+ match (name_key, response_key, name) {
+ (Some(name_key), Some(response_key), Some(name)) => {
+ let response =
+ serde_json::to_value(value.clone()).map_err(|error| Error::SerializationFailed {
+ reason: error.to_string(),
+ })?;
+ let response = if response.is_object() {
+ response
+ } else {
+ serde_json::to_value(IndexMap::from([("value".to_string(), response)])).map_err(
+ |error| Error::SerializationFailed {
+ reason: error.to_string(),
+ },
+ )?
+ };
+ serde_json::to_value(IndexMap::from([
+ (name_key.clone(), Value::String(name.clone())),
+ (response_key.clone(), response),
+ ]))
+ },
+ _ => serde_json::to_value(value.clone()),
+ }
+ },
OriginalContentBlock::Translation {
payload: input,
source_language_code,
diff --git a/crates/hanashi/src/chat/hanashi/messages/streamed/content.rs b/crates/hanashi/src/chat/hanashi/messages/streamed/content.rs
index c68363bc6..6e34be1a4 100644
--- a/crates/hanashi/src/chat/hanashi/messages/streamed/content.rs
+++ b/crates/hanashi/src/chat/hanashi/messages/streamed/content.rs
@@ -7,6 +7,10 @@ use shoji::types::{
use crate::chat::hanashi::messages::streamed::Section;
+const NAME_KEY: &str = "name";
+const VALUE_KEY: &str = "value";
+const RETURN_KEY: &str = "return";
+
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
#[serde(untagged)]
pub enum Content {
@@ -38,6 +42,7 @@ impl Content {
let mut reasoning_parts: Vec<String> = Vec::new();
let mut text_parts: Vec<String> = Vec::new();
let mut tool_calls: Vec<ChatContentBlock> = Vec::new();
+ let mut tool_call_results: Vec<ChatContentBlock> = Vec::new();
for section in sections {
match section {
@@ -50,15 +55,23 @@ impl Content {
Section::ToolCall {
value: Some(value),
} => {
- let block = match serde_json::from_value::<ToolCall>(value.clone()) {
- Ok(tool_call) => ChatContentBlock::ToolCall {
- value: tool_call,
- },
- Err(_) => ChatContentBlock::ToolCallCandidate {
- value: value.into(),
+ match serde_json::from_value::<ToolCall>(value.clone()) {
+ Ok(value) => tool_calls.push(ChatContentBlock::ToolCall {
+ value,
+ }),
+ // Llama 3.2 may echo a result as a pseudo call such as `{"time":"17:03","return":"time"}`.
+ // Only treat that positively identified shape as text;
+ // malformed calls must remain candidates so the turn retains its ToolCalls finish reason.
+ Err(_) => {
+ if is_tool_call_result_echo(&value) {
+ text_parts.push(value.to_string());
+ } else {
+ tool_calls.push(ChatContentBlock::ToolCallCandidate {
+ value: value.into(),
+ });
+ }
},
- };
- tool_calls.push(block);
+ }
},
Section::ToolCall {
value: None,
@@ -67,6 +80,16 @@ impl Content {
value: Value::Null.into(),
});
},
+ Section::ToolCallResult {
+ value: Some(value),
+ } => {
+ let (name, value) = tool_call_result_parts(value);
+ tool_call_results.push(ChatContentBlock::ToolCallResult {
+ identifier: None,
+ name,
+ value: value.into(),
+ });
+ },
_ => {},
}
}
@@ -85,8 +108,37 @@ impl Content {
});
}
blocks.extend(tool_calls);
+ blocks.extend(tool_call_results);
blocks
},
}
}
}
+
+fn is_tool_call_result_echo(value: &Value) -> bool {
+ let Some(object) = value.as_object() else {
+ return false;
+ };
+ let Some(Value::String(returned_field)) = object.get(RETURN_KEY) else {
+ return false;
+ };
+ returned_field != RETURN_KEY && object.contains_key(returned_field)
+}
+
+// Parsers may emit tool results as `{"name": ..., "value": ...}` (e.g. functiongemma, gemma-4);
+// renderers need the function name to reproduce the `response:<name>{...}` form.
+fn tool_call_result_parts(value: Value) -> (Option<String>, Value) {
+ match value {
+ Value::Object(mut map)
+ if map.len() == 2 && map.get(NAME_KEY).is_some_and(Value::is_string) && map.contains_key(VALUE_KEY) =>
+ {
+ let name = match map.remove(NAME_KEY) {
+ Some(Value::String(name)) => Some(name),
+ _ => None,
+ };
+ let value = map.remove(VALUE_KEY).unwrap_or(Value::Null);
+ (name, value)
+ },
+ other => (None, other),
+ }
+}
diff --git a/crates/hanashi/src/chat/hanashi/mod.rs b/crates/hanashi/src/chat/hanashi/mod.rs
index dfb853e15..fc2e1e899 100644
--- a/crates/hanashi/src/chat/hanashi/mod.rs
+++ b/crates/hanashi/src/chat/hanashi/mod.rs
@@ -5,28 +5,35 @@ mod ordering;
pub mod renderer;
mod token;
+use std::{collections::HashSet, sync::Arc};
+
pub use error::Error;
use shoji::types::{
basic::{Token, TokenId},
- session::chat::{ChatContentBlock, ChatMessage},
+ session::chat::{ChatContentBlock, ChatMessage, ChatModelCapabilities, ChatRole},
};
use token_stream_parser::{Parser as _, token_stream::TokenStreamParser};
use tokenizers::{Tokenizer, step_decode_stream};
-use self::{config::HanashiResolvedConfig, messages::streamed::Message as StreamedMessage, ordering::Validator};
+use self::{
+ config::HanashiResolvedConfig,
+ messages::streamed::{Content as StreamedContent, Message as StreamedMessage, Section as StreamedSection},
+ ordering::Validator,
+};
use crate::{
Encoding as EncodingTrait,
chat::{
- Context, State, SynchronizationError, SynchronizationResult,
+ State, SynchronizationError, SynchronizationResult,
hanashi::{config::HanashiConfig, messages::rendered::FieldConfig, renderer::Renderer, token::ToParserToken},
},
- util::tokenizer::load_tokenizer,
};
-pub struct Encoding {
+pub struct HanashiEncodingImpl {
+ capabilities: ChatModelCapabilities,
config: HanashiResolvedConfig,
- tokenizer: Tokenizer,
+ tokenizer: Arc<Tokenizer>,
parser: TokenStreamParser,
+ framing_tokens: HashSet<String>,
renderer: Renderer,
validator: Validator,
@@ -36,27 +43,22 @@ pub struct Encoding {
tokenizer_decode_prefix_index: usize,
}
-impl EncodingTrait for Encoding {
- type Config = HanashiConfig;
- type Context = Context;
- type Input = Vec<ChatMessage>;
- type Output = Vec<TokenId>;
- type State = State;
- type Error = Error;
-
- fn new(
- config: Self::Config,
- context: Self::Context,
- ) -> Result<Self, Self::Error> {
+impl HanashiEncodingImpl {
+ pub fn new(
+ config: HanashiConfig,
+ tokenizer: Arc<Tokenizer>,
+ ) -> Result<Self, Error> {
let resolved_config = config.resolve()?;
- let tokenizer = load_tokenizer(&context.tokenizer_location).map_err(|_| Error::UnableToLoadTokenizer)?;
let parser = TokenStreamParser::new(resolved_config.parsing.clone())?;
+ let framing_tokens = resolved_config.parsing.framing_config().tokens.into_iter().collect();
let renderer = Renderer::new(resolved_config.rendering.clone());
let validator = Validator::new(resolved_config.ordering.clone());
Ok(Self {
+ capabilities: config.capabilities()?,
config: resolved_config,
tokenizer,
parser,
+ framing_tokens,
renderer,
validator,
state: State::default(),
@@ -65,6 +67,14 @@ impl EncodingTrait for Encoding {
tokenizer_decode_prefix_index: 0,
})
}
+}
+
+impl EncodingTrait for HanashiEncodingImpl {
+ type Config = HanashiConfig;
+ type Input = Vec<ChatMessage>;
+ type Output = Vec<TokenId>;
+ type State = State;
+ type Error = Error;
fn state(&self) -> &Self::State {
&self.state
@@ -90,13 +100,23 @@ impl EncodingTrait for Encoding {
}
self.state.messages.extend(messages.clone());
+ // let transformation pipelines gate tool-call extraction on whether tools were declared
+ let tools_declared = self
+ .state
+ .messages
+ .iter()
+ .any(|message| message.content.iter().any(|block| matches!(block, ChatContentBlock::Tools { .. })));
+ if tools_declared {
+ self.parser.set_variable("tools", serde_json::Value::Bool(true));
+ }
+
let bos_token = self.config.tokens.bos_token_id.and_then(|token_id| self.resolve_token(token_id, false).ok());
let eos_token = self.config.tokens.eos_token_id.and_then(|token_id| self.resolve_token(token_id, false).ok());
let text = self.renderer.render(&messages, true, bos_token, eos_token, None)?;
let text_encoding = self.tokenizer.encode(text, false).map_err(|_| Error::UnableToEncodeText)?;
for token_id in text_encoding.get_ids() {
let token = self.resolve_token(*token_id, true)?;
- self.parser.push(&token.clone().to_parser_token())?;
+ self.push_token_to_parser(&token)?;
self.state.tokens.push(token);
}
@@ -110,7 +130,7 @@ impl EncodingTrait for Encoding {
) -> Result<(), Self::Error> {
for token_id in &token_ids {
let token = self.resolve_token(*token_id, true)?;
- self.parser.push(&token.clone().to_parser_token())?;
+ self.push_token_to_parser(&token)?;
self.state.tokens.push(token);
}
@@ -118,12 +138,27 @@ impl EncodingTrait for Encoding {
Ok(())
}
- fn tokenizer(&self) -> Option<&Tokenizer> {
- Some(&self.tokenizer)
+ fn supports_tool_calls(&self) -> bool {
+ self.capabilities.supports_tools
+ }
+
+ fn supports_multiple_tool_calls(&self) -> bool {
+ self.capabilities.supports_multiple_tool_calls
}
}
-impl Encoding {
+impl HanashiEncodingImpl {
+ fn push_token_to_parser(
+ &mut self,
+ token: &Token,
+ ) -> Result<(), Error> {
+ if token.is_special && !self.framing_tokens.contains(&token.value) {
+ return Ok(());
+ }
+ self.parser.push(&token.clone().to_parser_token())?;
+ Ok(())
+ }
+
fn fill_default_content(
&self,
messages: &[ChatMessage],
@@ -165,19 +200,75 @@ impl Encoding {
Ok(modified_messages)
}
+ fn message_from_sections(
+ sections: Vec<StreamedSection>,
+ role: ChatRole,
+ ) -> ChatMessage {
+ ChatMessage::from(StreamedMessage {
+ role,
+ content: Some(StreamedContent::Sections(sections)),
+ })
+ }
+
fn update_messages_from_parser_state(&mut self) -> Result<(), Error> {
let value = self.parser.state().value.clone();
+ let messages =
+ serde_json::from_value::<Vec<StreamedMessage>>(value).map_err(|_| Error::InvalidStreamedContent)?;
let rendering_config = &self.config.rendering;
- let streamed_messages: Vec<ChatMessage> = serde_json::from_value::<Vec<StreamedMessage>>(value)
- .map_err(|_| Error::InvalidStreamedContent)?
- .into_iter()
- .map(|streamed_message| {
- let role = rendering_config.get_role_by_name(&streamed_message.role.to_string());
- let mut message = ChatMessage::from(streamed_message);
- message.role = role;
- message
- })
- .collect();
+
+ let mut streamed_messages: Vec<ChatMessage> = Vec::new();
+ for msg in messages {
+ let role = rendering_config.get_role_by_name(&msg.role.to_string());
+ // templates may render tool results inside another role's turn (e.g. qwen renders them as `<|im_start|>user\n<tool_response>...`,
+ // functiongemma keeps calls, responses and the follow-up reply in one model turn),
+ // so tool result sections are split into standalone tool messages in stream order
+ match msg.content {
+ Some(StreamedContent::Sections(sections))
+ if sections.iter().any(|section| {
+ matches!(
+ section,
+ StreamedSection::ToolCallResult {
+ value: Some(serde_json::Value::Object(_))
+ }
+ )
+ }) =>
+ {
+ let mut pending: Vec<StreamedSection> = Vec::new();
+ for section in sections {
+ if matches!(
+ section,
+ StreamedSection::ToolCallResult {
+ value: Some(serde_json::Value::Object(_))
+ }
+ ) {
+ let message = Self::message_from_sections(std::mem::take(&mut pending), role.clone());
+ if !message.content.is_empty() {
+ streamed_messages.push(message);
+ }
+ streamed_messages.push(Self::message_from_sections(vec![section], ChatRole::Tool {}));
+ } else {
+ pending.push(section);
+ }
+ }
+ let message = Self::message_from_sections(pending, role.clone());
+ if !message.content.is_empty() {
+ streamed_messages.push(message);
+ }
+ },
+ content => {
+ let mut message = ChatMessage::from(StreamedMessage {
+ role: role.clone(),
+ content,
+ });
+ if !message.content.is_empty()
+ && message.content.iter().all(|block| matches!(block, ChatContentBlock::ToolCallResult { .. }))
+ {
+ message.role = ChatRole::Tool {};
+ }
+ streamed_messages.push(message);
+ },
+ }
+ }
let result = self.state.synchronize_messages(&streamed_messages)?;
if result == SynchronizationResult::Inserted {
diff --git a/crates/hanashi/src/chat/hanashi/renderer/config/jinja_function.rs b/crates/hanashi/src/chat/hanashi/renderer/config/jinja_function.rs
index e7f323853..9afb8d987 100644
--- a/crates/hanashi/src/chat/hanashi/renderer/config/jinja_function.rs
+++ b/crates/hanashi/src/chat/hanashi/renderer/config/jinja_function.rs
@@ -5,6 +5,8 @@ use std::{
use serde::{Deserialize, Deserializer, Serialize, Serializer, de};
+use crate::chat::hanashi::renderer::STRFTIME_NOW_FUNCTION_NAME;
+
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
pub enum JinjaFunction {
StrftimeNow,
@@ -15,7 +17,7 @@ impl FromStr for JinjaFunction {
fn from_str(s: &str) -> Result<Self, Self::Err> {
match s {
- "strftime_now" => Ok(JinjaFunction::StrftimeNow),
+ name if name == STRFTIME_NOW_FUNCTION_NAME => Ok(JinjaFunction::StrftimeNow),
other => Err(format!("Unknown jinja function: {other}")),
}
}
@@ -27,7 +29,7 @@ impl Display for JinjaFunction {
formatter: &mut fmt::Formatter<'_>,
) -> fmt::Result {
let name = match self {
- JinjaFunction::StrftimeNow => "strftime_now",
+ JinjaFunction::StrftimeNow => STRFTIME_NOW_FUNCTION_NAME,
};
write!(formatter, "{name}")
}
diff --git a/crates/hanashi/src/chat/hanashi/renderer/mod.rs b/crates/hanashi/src/chat/hanashi/renderer/mod.rs
index fbd84bfd1..b2dc28065 100644
--- a/crates/hanashi/src/chat/hanashi/renderer/mod.rs
+++ b/crates/hanashi/src/chat/hanashi/renderer/mod.rs
@@ -13,6 +13,9 @@ use shoji::types::{basic::Token, session::chat::ChatMessage};
use crate::chat::hanashi::messages::rendered::Message as RenderedMessage;
pub static TEMPLATE_NAME: &str = "chat_template";
+pub static STRFTIME_NOW_FUNCTION_NAME: &str = "strftime_now";
+pub static RAISE_EXCEPTION_FUNCTION_NAME: &str = "raise_exception";
+pub static TOJSON_FILTER_NAME: &str = "tojson";
pub struct Renderer {
config: RendererConfig,
@@ -43,8 +46,8 @@ impl Renderer {
},
);
}
- environment.add_function("raise_exception", raise_exception);
- environment.add_filter("tojson", to_json);
+ environment.add_function(RAISE_EXCEPTION_FUNCTION_NAME, raise_exception);
+ environment.add_filter(TOJSON_FILTER_NAME, to_json);
environment
.add_template(TEMPLATE_NAME, self.config.jinja.template.as_str())
.map_err(|_| Error::InvalidTemplate)?;
diff --git a/crates/hanashi/src/chat/harmony/bridging/message.rs b/crates/hanashi/src/chat/harmony/bridging/message.rs
index 1aab9633a..e2d1b6952 100644
--- a/crates/hanashi/src/chat/harmony/bridging/message.rs
+++ b/crates/hanashi/src/chat/harmony/bridging/message.rs
@@ -2,58 +2,61 @@ use std::collections::HashMap;
use openai_harmony::chat::{
Author as ExternalAuthor, Content as ExternalContent, DeveloperContent as ExternalDeveloperContent,
- Message as ExternalMessage, Role as ExternalRole, SystemContent as ExternalSystemContent, TextContent,
+ Message as ExternalMessage, Role as ExternalRole, SystemContent as ExternalSystemContent,
};
use shoji::types::{
basic::{ReasoningEffort, ToolCall, ToolNamespace, Value},
session::chat::{ChatContentBlock, ChatMessage, ChatMessageMetadata, ChatRole},
};
-use crate::chat::harmony::bridging::{Error, FromHarmony, ToHarmony};
+use crate::chat::{
+ harmony::{
+ CHANNEL_ANALYSIS, CHANNEL_COMMENTARY, CHANNEL_FINAL, CONTENT_TYPE_JSON, ROLE_ASSISTANT,
+ bridging::{Error, FromHarmony, ToHarmony},
+ },
+ strftime_now,
+};
const FUNCTIONS_NAMESPACE: &str = "functions";
-const CHANNEL_ANALYSIS: &str = "analysis";
-const CHANNEL_COMMENTARY: &str = "commentary";
-const CHANNEL_FINAL: &str = "final";
-const CONTENT_TYPE_JSON: &str = "json";
-const RECIPIENT_ASSISTANT: &str = "assistant";
+const RECIPIENT_ASSISTANT: &str = ROLE_ASSISTANT;
const BUILTIN_BROWSER: &str = "browser";
const BUILTIN_PYTHON: &str = "python";
pub fn bridge_messages_to_harmony(messages: &[ChatMessage]) -> Result<Vec<ExternalMessage>, Error> {
let mut result = Vec::new();
+ // Plain system text becomes developer `# Instructions`, matching the reference gpt-oss chat template.
+ // The harmony system message always carries the meta preamble (identity, reasoning effort, valid channels) the model was trained to expect.
+ let mut pending_instructions: Vec<String> = Vec::new();
for message in messages {
+ if !pending_instructions.is_empty() && !matches!(message.role, ChatRole::System {} | ChatRole::Developer {}) {
+ result.push(developer_message(std::mem::take(&mut pending_instructions).join("")));
+ }
+
match &message.role {
ChatRole::System {} => {
- let mut system_filled = false;
let mut system_content = ExternalSystemContent::default();
- let mut text_parts: Vec<String> = Vec::new();
for block in &message.content {
match block {
ChatContentBlock::Identity {
value,
} => {
- system_filled = true;
system_content.model_identity = Some(value.clone());
},
ChatContentBlock::ReasoningEffort {
value,
} => {
- system_filled = true;
system_content.reasoning_effort = Some((*value).to_harmony()?);
},
ChatContentBlock::ConversationStartDate {
value,
} => {
- system_filled = true;
system_content.conversation_start_date = Some(value.clone());
},
ChatContentBlock::KnowledgeCutoff {
value,
} => {
- system_filled = true;
system_content.knowledge_cutoff = Some(value.clone());
},
ChatContentBlock::BuiltinTools {
@@ -62,11 +65,9 @@ pub fn bridge_messages_to_harmony(messages: &[ChatMessage]) -> Result<Vec<Extern
for name in names {
match name.as_str() {
BUILTIN_BROWSER => {
- system_filled = true;
system_content = system_content.with_browser_tool();
},
BUILTIN_PYTHON => {
- system_filled = true;
system_content = system_content.with_python_tool();
},
_ => {
@@ -80,7 +81,7 @@ pub fn bridge_messages_to_harmony(messages: &[ChatMessage]) -> Result<Vec<Extern
ChatContentBlock::Text {
value,
} => {
- text_parts.push(value.clone());
+ pending_instructions.push(value.clone());
},
other => {
return Err(Error::UnsupportedContentBlock {
@@ -91,27 +92,22 @@ pub fn bridge_messages_to_harmony(messages: &[ChatMessage]) -> Result<Vec<Extern
}
}
- let mut content: Vec<ExternalContent> = vec![];
- if system_filled {
- content.push(ExternalContent::SystemContent(system_content));
- }
- if !text_parts.is_empty() {
- content.push(ExternalContent::Text(TextContent {
- text: text_parts.join(""),
- }));
+ // the reference gpt-oss chat template always includes the current date, so default it here the same way it does (strftime_now("%Y-%m-%d"))
+ if system_content.conversation_start_date.is_none() {
+ system_content.conversation_start_date = Some(strftime_now("%Y-%m-%d".to_string()));
}
result.push(ExternalMessage {
author: ExternalAuthor::from(ExternalRole::System),
recipient: None,
- content,
+ content: vec![ExternalContent::SystemContent(system_content)],
channel: None,
content_type: None,
});
},
ChatRole::Developer {} => {
let mut developer_content = ExternalDeveloperContent::default();
- let mut text_parts: Vec<String> = Vec::new();
+ let mut text_parts: Vec<String> = std::mem::take(&mut pending_instructions);
for block in &message.content {
match block {
@@ -218,9 +214,15 @@ pub fn bridge_messages_to_harmony(messages: &[ChatMessage]) -> Result<Vec<Extern
}
if !text_parts.is_empty() {
+ // text next to a tool call is a commentary-channel preamble, not a final reply
+ let channel = if tool_calls.is_empty() {
+ CHANNEL_FINAL
+ } else {
+ CHANNEL_COMMENTARY
+ };
result.push(
ExternalMessage::from_role_and_content(ExternalRole::Assistant, text_parts.join(""))
- .with_channel(CHANNEL_FINAL),
+ .with_channel(channel),
);
}
@@ -290,9 +292,27 @@ pub fn bridge_messages_to_harmony(messages: &[ChatMessage]) -> Result<Vec<Extern
}
}
+ if !pending_instructions.is_empty() {
+ result.push(developer_message(pending_instructions.join("")));
+ }
+
Ok(result)
}
+fn developer_message(instructions: String) -> ExternalMessage {
+ let developer_content = ExternalDeveloperContent {
+ instructions: Some(instructions),
+ ..Default::default()
+ };
+ ExternalMessage {
+ author: ExternalAuthor::from(ExternalRole::Developer),
+ recipient: None,
+ content: vec![ExternalContent::DeveloperContent(developer_content)],
+ channel: None,
+ content_type: None,
+ }
+}
+
pub fn bridge_messages_from_harmony(messages: &[ExternalMessage]) -> Result<Vec<ChatMessage>, Error> {
let mut result = Vec::new();
let mut index = 0;
@@ -429,25 +449,32 @@ pub fn bridge_messages_from_harmony(messages: &[ExternalMessage]) -> Result<Vec<
});
},
Some(CHANNEL_COMMENTARY) => {
- let recipient =
- assistant_message.recipient.as_deref().ok_or_else(|| Error::SerializationFailed {
- message: "Tool call missing recipient".to_string(),
- })?;
- let name = strip_namespace_prefix(recipient);
-
- match serde_json::from_str::<serde_json::Value>(&text) {
- Ok(json_value) => {
- content.push(ChatContentBlock::ToolCall {
- value: ToolCall {
- identifier: None,
- name: name.to_string(),
- arguments: Value::from(json_value),
+ match assistant_message.recipient.as_deref() {
+ Some(recipient) => {
+ let name = strip_namespace_prefix(recipient);
+
+ match serde_json::from_str::<serde_json::Value>(&text) {
+ Ok(json_value) => {
+ content.push(ChatContentBlock::ToolCall {
+ value: ToolCall {
+ identifier: None,
+ name: name.to_string(),
+ arguments: Value::from(json_value),
+ },
+ });
},
- });
+ Err(_) => {
+ content.push(ChatContentBlock::ToolCallCandidate {
+ value: Value::from(serde_json::Value::String(text)),
+ });
+ },
+ }
},
- Err(_) => {
- content.push(ChatContentBlock::ToolCallCandidate {
- value: Value::from(serde_json::Value::String(text)),
+ // commentary without a recipient is a user-visible preamble
+ // (e.g. an action plan announced before calling tools)
+ None => {
+ content.push(ChatContentBlock::Text {
+ value: text,
});
},
}
diff --git a/crates/hanashi/src/chat/harmony/mod.rs b/crates/hanashi/src/chat/harmony/mod.rs
index beb3dcb47..6de279409 100644
--- a/crates/hanashi/src/chat/harmony/mod.rs
+++ b/crates/hanashi/src/chat/harmony/mod.rs
@@ -15,37 +15,71 @@ use openai_harmony::{
};
use shoji::types::{
basic::{Token, TokenId},
- session::chat::ChatMessage,
+ session::chat::{ChatMessage, ChatModelCapabilities},
};
-use tokenizers::Tokenizer;
use crate::{
Encoding as EncodingTrait,
- chat::{Context, State, TokenizerLocation},
+ chat::{State, TokenizerLocation},
};
-pub struct Encoding {
+const TOKEN_START: &str = "<|start|>";
+const TOKEN_MESSAGE: &str = "<|message|>";
+const TOKEN_CHANNEL: &str = "<|channel|>";
+const TOKEN_CONSTRAIN: &str = "<|constrain|>";
+const ROLE_ASSISTANT: &str = "assistant";
+const CONTENT_TYPE_JSON: &str = "json";
+const CHANNEL_ANALYSIS: &str = "analysis";
+const CHANNEL_COMMENTARY: &str = "commentary";
+const CHANNEL_FINAL: &str = "final";
+const CHANNELS: [&str; 3] = [CHANNEL_ANALYSIS, CHANNEL_COMMENTARY, CHANNEL_FINAL];
+
+/// Ids of the harmony formatting tokens the decode-side header repair needs to recognize.
+struct SpecialTokens {
+ start: TokenId,
+ message: TokenId,
+ channel: TokenId,
+ stop: Vec<TokenId>,
+}
+
+impl SpecialTokens {
+ fn resolve(encoding: &HarmonyEncoding) -> Result<Self, Error> {
+ let single = |literal: &str| -> Result<TokenId, Error> {
+ match encoding.tokenizer().encode_with_special_tokens(literal).as_slice() {
+ [token_id] => Ok(*token_id),
+ _ => Err(Error::UnableToLoadEncoding),
+ }
+ };
+ Ok(Self {
+ start: single(TOKEN_START)?,
+ message: single(TOKEN_MESSAGE)?,
+ channel: single(TOKEN_CHANNEL)?,
+ stop: encoding.stop_tokens().map_err(|_| Error::UnableToLoadEncoding)?.into_iter().collect(),
+ })
+ }
+}
+
+pub struct HarmonyEncodingImpl {
+ capabilities: ChatModelCapabilities,
encoding: HarmonyEncoding,
parser: StreamableParser,
state: State,
+ completion_message_start: usize,
+ special_tokens: SpecialTokens,
+ // Some while the current message header is being decoded (from decoding start or `<|start|>` until `<|message|>`);
+ // header tokens are held back from the parser so a malformed header can be repaired before the parser sees it.
+ header_buffer: Option<Vec<Token>>,
}
-impl EncodingTrait for Encoding {
- type Config = HarmonyConfig;
- type Context = Context;
- type Input = Vec<ChatMessage>;
- type Output = Vec<TokenId>;
- type State = State;
- type Error = Error;
-
- fn new(
- config: Self::Config,
- context: Self::Context,
- ) -> Result<Self, Self::Error> {
+impl HarmonyEncodingImpl {
+ pub fn new(
+ config: HarmonyConfig,
+ tokenizer_location: TokenizerLocation,
+ ) -> Result<Self, Error> {
let TokenizerLocation::Directory {
path,
..
- } = &context.tokenizer_location
+ } = &tokenizer_location
else {
return Err(Error::ExpectedDirectoryTokenizerLocation);
};
@@ -58,12 +92,25 @@ impl EncodingTrait for Encoding {
};
let encoding = load_harmony_encoding(encoding_name).map_err(|_| Error::UnableToLoadEncoding)?;
let parser = StreamableParser::new(encoding.clone(), None).map_err(|_| Error::UnableToLoadEncoding)?;
+ let special_tokens = SpecialTokens::resolve(&encoding)?;
Ok(Self {
+ capabilities: config.capabilities(),
encoding,
parser,
state: State::default(),
+ completion_message_start: 0,
+ special_tokens,
+ header_buffer: None,
})
}
+}
+
+impl EncodingTrait for HarmonyEncodingImpl {
+ type Config = HarmonyConfig;
+ type Input = Vec<ChatMessage>;
+ type Output = Vec<TokenId>;
+ type State = State;
+ type Error = Error;
fn state(&self) -> &Self::State {
&self.state
@@ -72,6 +119,8 @@ impl EncodingTrait for Encoding {
fn reset(&mut self) -> Result<(), Self::Error> {
self.parser = StreamableParser::new(self.encoding.clone(), None).map_err(|_| Error::UnableToLoadEncoding)?;
self.state = State::default();
+ self.completion_message_start = 0;
+ self.header_buffer = None;
Ok(())
}
@@ -80,21 +129,26 @@ impl EncodingTrait for Encoding {
messages: Self::Input,
) -> Result<(), Self::Error> {
self.state.messages.extend(messages.clone());
+ self.completion_message_start = self.state.messages.len();
+ // Keep an empty completion in state until decoding replaces it. Cancellation
+ // or an immediate token limit can render state before decode is ever called.
+ self.state.messages.push(ChatMessage::assistant());
+ // The rendered prompt ends with `<|start|>assistant`; parse only the generated continuation of that header.
+ self.parser = StreamableParser::new(self.encoding.clone(), Some(HarmonyRole::Assistant))
+ .map_err(|_| Error::UnableToLoadEncoding)?;
+ self.header_buffer = Some(Vec::new());
let conversation = HarmonyConversation::from_messages(bridge_messages_to_harmony(&messages)?);
let token_ids = self
.encoding
.render_conversation_for_completion(&conversation, HarmonyRole::Assistant, None)
.map_err(|_| Error::UnableToRenderConversation)?;
+ let token_ids = canonicalize_tool_call_headers(&self.encoding, token_ids, &self.special_tokens)?;
for token_id in token_ids {
let token = resolve_token(&self.encoding, token_id)?;
- self.parser.process(token_id).map_err(|error| Error::ParserError {
- reason: error.to_string(),
- })?;
self.state.tokens.push(token);
}
- self.update_messages_from_parser_state()?;
Ok(())
}
@@ -104,9 +158,34 @@ impl EncodingTrait for Encoding {
) -> Result<(), Self::Error> {
for token_id in token_ids {
let token = resolve_token(&self.encoding, token_id)?;
- self.parser.process(token_id).map_err(|error| Error::ParserError {
- reason: error.to_string(),
- })?;
+
+ if self.header_buffer.is_some() {
+ if token_id == self.special_tokens.message {
+ // header complete: repair it if needed, then release it to the parser
+ let header_tokens = self.header_buffer.take().unwrap_or_default();
+ let header_token_ids = repair_header(&self.encoding, &header_tokens, &self.special_tokens)
+ .unwrap_or_else(|| header_tokens.iter().map(|header_token| header_token.id).collect());
+ for header_token_id in header_token_ids {
+ self.process(header_token_id)?;
+ }
+ self.process(token_id)?;
+ } else if self.special_tokens.stop.contains(&token_id) {
+ // message ended without `<|message|>`: replay verbatim and let the parser decide
+ let header_tokens = self.header_buffer.take().unwrap_or_default();
+ for header_token in header_tokens {
+ self.process(header_token.id)?;
+ }
+ self.process(token_id)?;
+ } else if let Some(header_tokens) = &mut self.header_buffer {
+ header_tokens.push(token.clone());
+ }
+ } else {
+ self.process(token_id)?;
+ if token_id == self.special_tokens.start {
+ self.header_buffer = Some(Vec::new());
+ }
+ }
+
self.state.tokens.push(token);
}
@@ -114,12 +193,26 @@ impl EncodingTrait for Encoding {
Ok(())
}
- fn tokenizer(&self) -> Option<&Tokenizer> {
- None
+ fn supports_tool_calls(&self) -> bool {
+ self.capabilities.supports_tools
+ }
+
+ fn supports_multiple_tool_calls(&self) -> bool {
+ self.capabilities.supports_multiple_tool_calls
}
}
-impl Encoding {
+impl HarmonyEncodingImpl {
+ fn process(
+ &mut self,
+ token_id: TokenId,
+ ) -> Result<(), Error> {
+ self.parser.process(token_id).map_err(|error| Error::ParserError {
+ reason: error.to_string(),
+ })?;
+ Ok(())
+ }
+
fn update_messages_from_parser_state(&mut self) -> Result<(), Error> {
let mut streamed_harmony_messages = self.parser.messages().to_vec();
@@ -140,12 +233,118 @@ impl Encoding {
}
let streamed_messages = bridge_messages_from_harmony(&streamed_harmony_messages)?;
- self.state.synchronize_messages(&streamed_messages)?;
+ self.state.messages.truncate(self.completion_message_start);
+ self.state.messages.extend(streamed_messages);
Ok(())
}
}
+/// openai_harmony (like the community HF chat template) re-renders an assistant tool-call header as
+/// `assistant to=functions.x<|channel|>commentary json`, but the model itself generates — and OpenAI's harmony
+/// reference replays — `assistant<|channel|>commentary to=functions.x <|constrain|>json`. Replaying the model's
+/// turns in the non-native order makes it progressively garble its own tool-call headers over multi-turn tool
+/// exchanges (repeated `to=` fragments, misplaced channel markers), so rewrite each re-rendered tool-call header
+/// into the form the model natively emits. Tool-result headers (`functions.x to=assistant<|channel|>commentary`)
+/// already match the reference and stay untouched.
+fn canonicalize_tool_call_headers(
+ encoding: &HarmonyEncoding,
+ token_ids: Vec<TokenId>,
+ special_tokens: &SpecialTokens,
+) -> Result<Vec<TokenId>, Error> {
+ let mut result: Vec<TokenId> = Vec::with_capacity(token_ids.len());
+ let mut index = 0;
+ while index < token_ids.len() {
+ let token_id = token_ids[index];
+ result.push(token_id);
+ index += 1;
+ if token_id != special_tokens.start {
+ continue;
+ }
+
+ let Some(header_length) = token_ids[index..].iter().position(|id| *id == special_tokens.message) else {
+ continue;
+ };
+ let header_text = token_ids[index..index + header_length]
+ .iter()
+ .map(|id| resolve_token(encoding, *id).map(|token| token.value))
+ .collect::<Result<String, Error>>()?;
+
+ let Some(rest) = header_text.strip_prefix("assistant to=") else {
+ continue;
+ };
+ let Some((recipient, after_channel)) = rest.split_once(TOKEN_CHANNEL) else {
+ continue;
+ };
+ let channel_end = after_channel.find(|c: char| c.is_whitespace() || c == '<').unwrap_or(after_channel.len());
+ let channel = &after_channel[..channel_end];
+ if recipient.is_empty() || channel.is_empty() {
+ continue;
+ }
+
+ let header =
+ format!("{ROLE_ASSISTANT}{TOKEN_CHANNEL}{channel} to={recipient} {TOKEN_CONSTRAIN}{CONTENT_TYPE_JSON}");
+ result.extend(encoding.tokenizer().encode_with_special_tokens(&header));
+ index += header_length;
+ }
+ Ok(result)
+}
+
+/// gpt-oss sometimes drops or misplaces the `<|channel|>` token when it opens a tool-call header right after a tool response,
+/// emitting e.g. `commentary to=functions.x<|channel|>commentary<|constrain|>json` straight after `<|start|>assistant`.
+/// When a header is that recognizable quirk — the channel name first, followed only by fragments we can attribute
+/// (a recipient, a repeated channel declaration, a content type) — rebuild the canonical header the parser expects.
+/// Any other shape returns None and is replayed verbatim, keeping the parser's strict behavior for genuinely garbled output.
+fn repair_header(
+ encoding: &HarmonyEncoding,
+ header_tokens: &[Token],
+ special_tokens: &SpecialTokens,
+) -> Option<Vec<TokenId>> {
+ if header_tokens.first().is_some_and(|token| token.id == special_tokens.channel) {
+ return None;
+ }
+
+ let header_text: String = header_tokens.iter().map(|token| token.value.as_str()).collect();
+ let text = header_text.strip_prefix(ROLE_ASSISTANT).unwrap_or(&header_text).trim_start();
+
+ let channel = CHANNELS.into_iter().find(|channel| {
+ text.strip_prefix(channel).is_some_and(|rest| rest.is_empty() || rest.starts_with(' ') || rest.starts_with('<'))
+ })?;
+
+ let mut recipient: Option<&str> = None;
+ let mut remainder = text[channel.len()..].trim_start();
+ while !remainder.is_empty() {
+ if let Some(rest) = remainder.strip_prefix("to=") {
+ let end = rest.find(|c: char| c.is_whitespace() || c == '<').unwrap_or(rest.len());
+ if end == 0 {
+ return None;
+ }
+ if recipient.is_none() {
+ recipient = Some(&rest[..end]);
+ }
+ remainder = &rest[end..];
+ } else if let Some(rest) = remainder.strip_prefix(TOKEN_CHANNEL) {
+ let end = rest.find(|c: char| c.is_whitespace() || c == '<').unwrap_or(rest.len());
+ if &rest[..end] != channel {
+ return None;
+ }
+ remainder = &rest[end..];
+ } else if let Some(rest) = remainder.strip_prefix(TOKEN_CONSTRAIN) {
+ let end = rest.find(|c: char| c.is_whitespace() || c == '<').unwrap_or(rest.len());
+ remainder = &rest[end..];
+ } else {
+ remainder = remainder.strip_prefix(CONTENT_TYPE_JSON)?;
+ }
+ remainder = remainder.trim_start();
+ }
+
+ let header = match recipient {
+ Some(recipient) => format!("{TOKEN_CHANNEL}{channel} to={recipient} {TOKEN_CONSTRAIN}{CONTENT_TYPE_JSON}"),
+ None => format!("{TOKEN_CHANNEL}{channel}"),
+ };
+ Some(encoding.tokenizer().encode_with_special_tokens(&header))
+}
+
fn resolve_token(
encoding: &HarmonyEncoding,
token_id: TokenId,
diff --git a/crates/hanashi/src/chat/mod.rs b/crates/hanashi/src/chat/mod.rs
index 33db6365e..c5847b8a4 100644
--- a/crates/hanashi/src/chat/mod.rs
+++ b/crates/hanashi/src/chat/mod.rs
@@ -6,16 +6,15 @@ pub mod harmony;
mod state;
pub use config::EncodingConfig;
-pub use context::{Context, TokenizerLocation};
+pub use context::TokenizerLocation;
pub use error::Error;
pub use hanashi::renderer::strftime_now;
use shoji::types::{basic::TokenId, session::chat::ChatMessage};
pub use state::{State, SynchronizationError, SynchronizationResult};
-use tokenizers::Tokenizer;
use crate::{
Encoding as EncodingTrait,
- chat::{hanashi::Encoding as HanashiEncoding, harmony::Encoding as HarmonyEncoding},
+ chat::{hanashi::HanashiEncodingImpl, harmony::HarmonyEncodingImpl},
};
macro_rules! dispatch {
@@ -34,32 +33,17 @@ macro_rules! dispatch {
}
pub enum Encoding {
- Hanashi(HanashiEncoding),
- Harmony(HarmonyEncoding),
+ Hanashi(HanashiEncodingImpl),
+ Harmony(HarmonyEncodingImpl),
}
impl EncodingTrait for Encoding {
type Config = EncodingConfig;
- type Context = Context;
type Input = Vec<ChatMessage>;
type Output = Vec<TokenId>;
type State = State;
type Error = Error;
- fn new(
- config: Self::Config,
- context: Self::Context,
- ) -> Result<Self, Self::Error> {
- match config {
- EncodingConfig::Hanashi {
- config,
- } => Ok(Encoding::Hanashi(HanashiEncoding::new(config, context)?)),
- EncodingConfig::Harmony {
- config,
- } => Ok(Encoding::Harmony(HarmonyEncoding::new(config, context)?)),
- }
- }
-
fn state(&self) -> &Self::State {
dispatch!(infallible self, state)
}
@@ -82,7 +66,11 @@ impl EncodingTrait for Encoding {
dispatch!(self, decode, value)
}
- fn tokenizer(&self) -> Option<&Tokenizer> {
- dispatch!(infallible self, tokenizer)
+ fn supports_tool_calls(&self) -> bool {
+ dispatch!(infallible self, supports_tool_calls)
+ }
+
+ fn supports_multiple_tool_calls(&self) -> bool {
+ dispatch!(infallible self, supports_multiple_tool_calls)
}
}
diff --git a/crates/hanashi/src/classification/encoding.rs b/crates/hanashi/src/classification/encoding.rs
index fc5f1d6ba..8060080fc 100644
--- a/crates/hanashi/src/classification/encoding.rs
+++ b/crates/hanashi/src/classification/encoding.rs
@@ -11,7 +11,10 @@ use crate::{
TokenizerLocation,
hanashi::{
Error,
- renderer::{TEMPLATE_NAME, raise_exception, to_json},
+ renderer::{
+ RAISE_EXCEPTION_FUNCTION_NAME, STRFTIME_NOW_FUNCTION_NAME, TEMPLATE_NAME, TOJSON_FILTER_NAME,
+ raise_exception, to_json,
+ },
},
strftime_now,
},
@@ -67,9 +70,9 @@ impl ClassificationEncoding {
) -> Result<Vec<u32>, Error> {
let mut environment = Environment::new();
environment.set_unknown_method_callback(unknown_method_callback);
- environment.add_function("strftime_now", strftime_now);
- environment.add_function("raise_exception", raise_exception);
- environment.add_filter("tojson", to_json);
+ environment.add_function(STRFTIME_NOW_FUNCTION_NAME, strftime_now);
+ environment.add_function(RAISE_EXCEPTION_FUNCTION_NAME, raise_exception);
+ environment.add_filter(TOJSON_FILTER_NAME, to_json);
environment
.add_template(TEMPLATE_NAME, config.prompt_template.as_str())
.map_err(|_| Error::UnableToEncodeText)?;
diff --git a/crates/hanashi/src/lib.rs b/crates/hanashi/src/lib.rs
index 2307a6180..ecca3db9c 100644
--- a/crates/hanashi/src/lib.rs
+++ b/crates/hanashi/src/lib.rs
@@ -1,37 +1,33 @@
-use tokenizers::Tokenizer;
-
mod classification;
mod util;
pub mod chat;
pub use classification::encoding::ClassificationEncoding;
+pub use util::tokenizer::load_tokenizer;
pub trait Encoding {
type Config;
- type Context;
type Input;
type Output;
type State;
type Error;
- fn new(
- config: Self::Config,
- context: Self::Context,
- ) -> Result<Self, Self::Error>
- where
- Self: Sized;
fn state(&self) -> &Self::State;
+
fn reset(&mut self) -> Result<(), Self::Error>;
fn encode(
&mut self,
value: Self::Input,
) -> Result<(), Self::Error>;
+
fn decode(
&mut self,
value: Self::Output,
) -> Result<(), Self::Error>;
- fn tokenizer(&self) -> Option<&Tokenizer>;
+ fn supports_tool_calls(&self) -> bool;
+
+ fn supports_multiple_tool_calls(&self) -> bool;
}
diff --git a/crates/hanashi/tests/helpers/mod.rs b/crates/hanashi/tests/helpers/mod.rs
index 1d46d95b3..b4b736702 100644
--- a/crates/hanashi/tests/helpers/mod.rs
+++ b/crates/hanashi/tests/helpers/mod.rs
@@ -1,7 +1,10 @@
#![allow(dead_code)]
-use std::{collections::HashMap, path::PathBuf};
+use std::{collections::HashMap, path::PathBuf, sync::Arc};
+use hanashi::chat::{
+ Encoding, EncodingConfig, TokenizerLocation, hanashi::HanashiEncodingImpl, harmony::HarmonyEncodingImpl,
+};
use serde::Deserialize;
use shoji::types::{
basic::{ReasoningEffort, ToolDescription, ToolNamespace, TranslationPayload},
@@ -90,6 +93,26 @@ pub fn load_tokenizer(model_name: &str) -> Tokenizer {
.unwrap_or_else(|error| panic!("Failed to load tokenizer {}: {error}", path.display()))
}
+pub fn build_encoding(
+ config: EncodingConfig,
+ model_name: &str,
+) -> Encoding {
+ match config {
+ EncodingConfig::Hanashi {
+ config,
+ } => Encoding::Hanashi(HanashiEncodingImpl::new(config, Arc::new(load_tokenizer(model_name))).unwrap()),
+ EncodingConfig::Harmony {
+ config,
+ } => {
+ let tokenizer_location = TokenizerLocation::Directory {
+ path: tokenizer_directory(model_name).to_str().unwrap().to_string(),
+ name: None,
+ };
+ Encoding::Harmony(HarmonyEncodingImpl::new(config, tokenizer_location).unwrap())
+ },
+ }
+}
+
fn resolve_reasoning_effort(context: &HashMap<String, serde_json::Value>) -> Option<ReasoningEffort> {
let value = context.get("enable_thinking")?;
match value.as_bool() {
diff --git a/crates/hanashi/tests/test_encoding.rs b/crates/hanashi/tests/test_encoding.rs
index 2be5ae10d..4cffd87dd 100644
--- a/crates/hanashi/tests/test_encoding.rs
+++ b/crates/hanashi/tests/test_encoding.rs
@@ -4,16 +4,14 @@ use std::collections::HashMap;
use hanashi::{
Encoding as EncodingTrait,
- chat::{
- Context, Encoding, EncodingConfig, TokenizerLocation, hanashi::config::HanashiConfig, harmony::HarmonyConfig,
- },
+ chat::{EncodingConfig, hanashi::config::HanashiConfig, harmony::HarmonyConfig},
};
use helpers::{
- build_messages, load_registry, load_response_test_data, load_tokenizer, normalize_pattern, response_path,
- tokenizer_directory,
+ build_encoding, build_messages, load_registry, load_response_test_data, load_tokenizer, normalize_pattern,
+ response_path, tokenizer_directory,
};
use shoji::types::{
- basic::ReasoningEffort,
+ basic::{ReasoningEffort, Token, ToolCall, ToolDescription, ToolFunction, ToolNamespace, Value},
session::chat::{ChatContentBlock, ChatMessage, ChatMessageMetadata, ChatRole},
};
@@ -64,13 +62,7 @@ fn run_encoding_test(
let tokenizer = load_tokenizer(&model.name());
let test_data = load_response_test_data(&model.name());
- let context = Context {
- tokenizer_location: TokenizerLocation::Directory {
- path: tokenizer_directory.to_str().unwrap().to_string(),
- name: None,
- },
- };
- let mut encoding = Encoding::new(expected_config.clone(), context).unwrap();
+ let mut encoding = build_encoding(expected_config.clone(), &model.name());
for data in &test_data {
let messages = match messages_normalizer {
@@ -236,11 +228,278 @@ fn test_encoding_gpt_oss() {
);
}
+#[test]
+fn test_encoding_gpt_oss_initializes_empty_completion_message() {
+ let tokenizer_directory = tokenizer_directory("openai_gpt-oss-20b");
+ if !tokenizer_directory.exists() {
+ return;
+ }
+
+ let mut encoding = build_encoding(harmony(HarmonyConfig::GptOss), "openai_gpt-oss-20b");
+ encoding.encode(vec![ChatMessage::user().with_text("Hello".to_string())]).unwrap();
+
+ let completion = encoding.state().messages.last().unwrap();
+ assert_eq!(completion.role, ChatRole::Assistant {});
+ assert!(completion.content.is_empty());
+}
+
+/// Multi-turn tool calling: prior tool calls must be re-rendered in the form the model itself
+/// generates — `<|channel|>commentary to=functions.x <|constrain|>json<|message|>…<|call|>` — not the
+/// `to=` -before-channel, plain-`json` form the community HF chat template (and openai_harmony's
+/// renderer) produce. Replaying the model's turns in the non-native order makes gpt-oss-20b
+/// progressively garble its own tool-call headers over multi-turn tool exchanges (repeated `to=`
+/// fragments, misplaced channel markers) until the reply loop dead-ends; with native-order replay the
+/// same conversation runs to a clean final answer. Tool-result headers
+/// (`functions.x to=assistant<|channel|>commentary`) match the model's expectations as rendered.
+#[test]
+fn test_encoding_gpt_oss_tool_call_rerender() {
+ let tokenizer_directory = tokenizer_directory("openai_gpt-oss-20b");
+ if !tokenizer_directory.exists() {
+ return;
+ }
+
+ let mut encoding = build_encoding(harmony(HarmonyConfig::GptOss), "openai_gpt-oss-20b");
+
+ let messages = vec![
+ ChatMessage::system().with_text("You are a helpful assistant".to_string()).with_block(
+ ChatContentBlock::ConversationStartDate {
+ value: "2026-07-23".to_string(),
+ },
+ ),
+ ChatMessage::user().with_text("What time is it now?".to_string()),
+ ChatMessage::assistant().with_reasoning("Need the current time.".to_string()).with_tool_call(ToolCall {
+ identifier: None,
+ name: "get_current_time".to_string(),
+ arguments: Value::from(serde_json::json!({})),
+ }),
+ ChatMessage::tool().with_block(ChatContentBlock::ToolCallResult {
+ identifier: None,
+ name: Some("get_current_time".to_string()),
+ value: Value::from(serde_json::json!({"time": "17:03"})),
+ }),
+ ];
+ encoding.encode(messages).unwrap();
+
+ let expected = concat!(
+ "<|start|>system<|message|>You are ChatGPT, a large language model trained by OpenAI.\n",
+ "Knowledge cutoff: 2024-06\n",
+ "Current date: 2026-07-23\n",
+ "\n",
+ "Reasoning: medium\n",
+ "\n",
+ "# Valid channels: analysis, commentary, final. Channel must be included for every message.<|end|>",
+ "<|start|>developer<|message|># Instructions\n",
+ "\n",
+ "You are a helpful assistant<|end|>",
+ "<|start|>user<|message|>What time is it now?<|end|>",
+ "<|start|>assistant<|channel|>analysis<|message|>Need the current time.<|end|>",
+ "<|start|>assistant<|channel|>commentary to=functions.get_current_time <|constrain|>json<|message|>{}<|call|>",
+ "<|start|>functions.get_current_time to=assistant<|channel|>commentary<|message|>{\"time\":\"17:03\"}<|end|>",
+ "<|start|>assistant",
+ );
+ assert_eq!(encoding.state().text(), expected);
+}
+
+/// gpt-oss sometimes drops or misplaces the `<|channel|>` token when it opens a tool-call header
+/// right after a tool response (e.g. `commentary to=functions.x<|channel|>commentary
+/// <|constrain|>json<|message|>` straight after `<|start|>assistant`). The decoder must repair
+/// such headers instead of failing the whole turn, while keeping the parser strict for anything
+/// it cannot attribute.
+#[test]
+fn test_decoding_gpt_oss_malformed_tool_call_header() {
+ let tokenizer_directory = tokenizer_directory("openai_gpt-oss-20b");
+ if !tokenizer_directory.exists() {
+ return;
+ }
+ let tokenizer = load_tokenizer("openai_gpt-oss-20b");
+
+ let mut encoding = build_encoding(harmony(HarmonyConfig::GptOss), "openai_gpt-oss-20b");
+
+ let cases: &[(&str, Option<&str>)] = &[
+ // the exact quirk observed from gpt-oss-20b: channel name first, marker misplaced
+ (
+ "commentary to=functions.get_current_location<|channel|>commentary <|constrain|>json<|message|>{}<|call|>",
+ Some("get_current_location"),
+ ),
+ // channel marker missing entirely
+ ("commentary to=functions.get_current_time <|constrain|>json<|message|>{}<|call|>", Some("get_current_time")),
+ // plain-text content type, no marker
+ ("commentary to=functions.get_current_time json<|message|>{}<|call|>", Some("get_current_time")),
+ // well-formed headers must pass through untouched
+ (
+ "<|channel|>commentary to=functions.get_current_time <|constrain|>json<|message|>{}<|call|>",
+ Some("get_current_time"),
+ ),
+ (
+ "<|channel|>analysis<|message|>Need the time.<|end|>\
+ <|start|>assistant to=functions.get_current_time<|channel|>commentary <|constrain|>json<|message|>{}<|call|>",
+ Some("get_current_time"),
+ ),
+ ("<|channel|>final<|message|>It is 17:03.<|return|>", None),
+ ];
+
+ for (completion, expected_tool_call) in cases {
+ encoding.reset().unwrap();
+ encoding.encode(vec![ChatMessage::user().with_text("What time is it now?".to_string())]).unwrap();
+
+ let token_ids = tokenizer.encode(*completion, false).unwrap().get_ids().to_vec();
+ for token_id in token_ids {
+ encoding.decode(vec![token_id]).unwrap_or_else(|error| panic!("Failed to decode {completion:?}: {error}"));
+ }
+
+ let assistant_message = encoding.state().messages.last().unwrap();
+ assert_eq!(assistant_message.role, ChatRole::Assistant {}, "for completion {completion:?}");
+ let tool_call_names = assistant_message
+ .content
+ .iter()
+ .filter_map(|block| match block {
+ ChatContentBlock::ToolCall {
+ value,
+ } => Some(value.name.as_str()),
+ _ => None,
+ })
+ .collect::<Vec<_>>();
+ match expected_tool_call {
+ Some(name) => assert_eq!(tool_call_names, vec![*name], "for completion {completion:?}"),
+ None => assert!(tool_call_names.is_empty(), "for completion {completion:?}"),
+ }
+ }
+
+ // a header the repair cannot attribute must still fail strict parsing
+ encoding.reset().unwrap();
+ encoding.encode(vec![ChatMessage::user().with_text("What time is it now?".to_string())]).unwrap();
+ let garbled = "commentary nonsense to=functions.get_current_time<|message|>{}<|call|>";
+ let token_ids = tokenizer.encode(garbled, false).unwrap().get_ids().to_vec();
+ let result = token_ids.into_iter().try_for_each(|token_id| encoding.decode(vec![token_id]));
+ assert!(result.is_err(), "Expected garbled header {garbled:?} to fail decoding");
+}
+
#[test]
fn test_encoding_llama_32() {
run_encoding_test(&hanashi(HanashiConfig::Llama32), Some(r"Today Date: (.+?)\n"), None, false, false, None);
}
+/// After a tool response, Llama 3.2 1B sometimes replies by echoing the result JSON as a new
+/// pseudo tool call (`<|python_tag|>{"time": ...}<|eom_id|>`). Once such a section finishes it
+/// parses into an object that is not a tool call; it must surface as plain text so the turn ends
+/// with a normal reply instead of dead-ending on a tool-call candidate.
+#[test]
+fn test_decoding_llama_32_malformed_tool_call() {
+ let tokenizer_directory = tokenizer_directory("meta-llama_Llama-3.2-1B-Instruct");
+ if !tokenizer_directory.exists() {
+ return;
+ }
+ let tokenizer = load_tokenizer("meta-llama_Llama-3.2-1B-Instruct");
+
+ let mut encoding = build_encoding(hanashi(HanashiConfig::Llama32), "meta-llama_Llama-3.2-1B-Instruct");
+
+ // (completion, expected tool call, expected text fragment)
+ let cases: &[(&str, Option<&str>, Option<&str>)] = &[
+ // proper tool call: unchanged behavior
+ (
+ "<|python_tag|>{\"name\": \"get_current_time\", \"parameters\": {}}<|eom_id|>",
+ Some("get_current_time"),
+ None,
+ ),
+ // tool-result echo: must become text, not a tool-call candidate
+ ("<|python_tag|>{\"time\": \"17:03\", \"return\": \"time\"}<|eom_id|>", None, Some("17:03")),
+ ];
+
+ for (completion, expected_tool_call, expected_fragment) in cases {
+ encoding.reset().unwrap();
+ encoding
+ .encode(vec![
+ ChatMessage::system().with_text("You are a helpful assistant".to_string()),
+ ChatMessage::user().with_text("What is the time now?".to_string()),
+ ])
+ .unwrap();
+
+ let token_ids = tokenizer.encode(*completion, false).unwrap().get_ids().to_vec();
+ for token_id in token_ids {
+ encoding.decode(vec![token_id]).unwrap_or_else(|error| panic!("Failed to decode {completion:?}: {error}"));
+ }
+
+ let assistant_message = encoding.state().messages.last().unwrap();
+ assert_eq!(assistant_message.role, ChatRole::Assistant {}, "for completion {completion:?}");
+ let tool_call_names = assistant_message.tool_calls().iter().map(|call| call.name.clone()).collect::<Vec<_>>();
+ let has_candidates =
+ assistant_message.content.iter().any(|block| matches!(block, ChatContentBlock::ToolCallCandidate { .. }));
+ match expected_tool_call {
+ Some(name) => assert_eq!(tool_call_names, vec![name.to_string()], "for completion {completion:?}"),
+ None => {
+ assert!(tool_call_names.is_empty(), "for completion {completion:?}");
+ assert!(!has_candidates, "Expected no tool-call candidates for completion {completion:?}");
+ },
+ }
+ if let Some(fragment) = expected_fragment {
+ let text = assistant_message.text().unwrap_or_default();
+ assert!(
+ text.contains(fragment),
+ "Expected text with {fragment:?} for completion {completion:?}, got {text:?}"
+ );
+ }
+ }
+}
+
+/// Llama 3 emits content-level tool calls as bare JSON with "name"/"parameters" — the same shape as an ordinary reply
+/// that happens to contain a function descriptor. The bare-JSON fallback must only rewrite content into a tool call
+/// when the conversation actually declared tools AND the turn ended with <|eom_id|> (the "expect a tool response"
+/// stop token). A turn ending with <|eot_id|> is a final answer even if it quotes a call-shaped JSON — treating it
+/// as a call sends the session into an endless call/result loop. Otherwise the JSON must survive as plain text.
+#[test]
+fn test_decoding_llama_32_content_tool_call_requires_tools() {
+ let tokenizer_directory = tokenizer_directory("meta-llama_Llama-3.2-1B-Instruct");
+ if !tokenizer_directory.exists() {
+ return;
+ }
+ let tokenizer = load_tokenizer("meta-llama_Llama-3.2-1B-Instruct");
+
+ let call_json = "{\"name\": \"get_current_time\", \"parameters\": {\"timezone\": \"UTC\"}}";
+ let tools_message = ChatMessage::developer().with_tool_namespaces(vec![ToolNamespace {
+ name: "functions".to_string(),
+ description: None,
+ tools: vec![ToolDescription::Function {
+ tool_function: ToolFunction {
+ name: "get_current_time".to_string(),
+ description: "Returns the current time".to_string(),
+ parameters: None,
+ return_definition: None,
+ },
+ }],
+ }]);
+
+ // (tools declared, stop token, expected tool calls)
+ let cases: &[(bool, &str, &[&str])] = &[
+ (false, "<|eom_id|>", &[]),
+ (false, "<|eot_id|>", &[]),
+ (true, "<|eom_id|>", &["get_current_time"]),
+ (true, "<|eot_id|>", &[]),
+ ];
+ for (tools_declared, stop_token, expected_tool_calls) in cases {
+ let completion = format!("{call_json}{stop_token}");
+ let mut encoding = build_encoding(hanashi(HanashiConfig::Llama32), "meta-llama_Llama-3.2-1B-Instruct");
+ let mut messages = vec![ChatMessage::system().with_text("You are a helpful assistant".to_string())];
+ if *tools_declared {
+ messages.push(tools_message.clone());
+ }
+ messages.push(ChatMessage::user().with_text("Describe the time tool as JSON".to_string()));
+ encoding.encode(messages).unwrap();
+
+ for token_id in tokenizer.encode(completion.as_str(), false).unwrap().get_ids().to_vec() {
+ encoding.decode(vec![token_id]).unwrap();
+ }
+
+ let assistant_message = encoding.state().messages.last().unwrap();
+ assert_eq!(assistant_message.role, ChatRole::Assistant {}, "for completion {completion:?}");
+ let tool_call_names = assistant_message.tool_calls().iter().map(|call| call.name.clone()).collect::<Vec<_>>();
+ assert_eq!(tool_call_names, *expected_tool_calls, "for completion {completion:?}");
+ if expected_tool_calls.is_empty() {
+ let text = assistant_message.text().unwrap_or_default();
+ assert!(text.contains("\"parameters\""), "Expected raw JSON to stay text, got {text:?}");
+ }
+ }
+}
+
#[test]
fn test_encoding_qwen3() {
run_encoding_test(&hanashi(HanashiConfig::Qwen3), None, None, false, false, None);
@@ -313,3 +572,39 @@ fn test_encoding_functiongemma() {
}),
);
}
+
+#[test]
+fn test_rendering_functiongemma_non_object_tool_results() {
+ let config = HanashiConfig::FunctionGemma.resolve().unwrap();
+ let renderer = hanashi::chat::hanashi::renderer::Renderer::new(config.rendering);
+ let cases = [
+ (serde_json::json!(42), "response:get_value{value:42}"),
+ (
+ serde_json::json!(["first", "second"]),
+ "response:get_value{value:[<escape>first<escape>,<escape>second<escape>]}",
+ ),
+ ];
+
+ for (value, expected) in cases {
+ let message = ChatMessage::tool().with_block(ChatContentBlock::ToolCallResult {
+ identifier: None,
+ name: Some("get_value".to_string()),
+ value: Value::from(value),
+ });
+ let rendered = renderer
+ .render(
+ &[message],
+ true,
+ Some(Token {
+ id: 0,
+ value: "<bos>".to_string(),
+ is_special: true,
+ }),
+ None,
+ None,
+ )
+ .unwrap();
+
+ assert!(rendered.contains(expected), "rendered prompt did not preserve the tool result: {rendered}");
+ }
+}
diff --git a/crates/json-transform/src/execution/operation.rs b/crates/json-transform/src/execution/operation.rs
index 0bc68182b..1dc440828 100644
--- a/crates/json-transform/src/execution/operation.rs
+++ b/crates/json-transform/src/execution/operation.rs
@@ -150,6 +150,11 @@ pub enum Operation {
#[serde(default)]
regex_engine: RegexEngine,
},
+ /// Split a string at separators outside quoted strings and nested structures.
+ /// String → Array<String>
+ SplitTopLevel {
+ separator: char,
+ },
/// Parse a JSON string into a Value. With `repair: true`, attempts to fix malformed JSON.
/// String → Value
ParseJson {
@@ -238,6 +243,9 @@ impl fmt::Display for Operation {
Self::RegexFindAll {
..
} => write!(formatter, "regex_find_all"),
+ Self::SplitTopLevel {
+ separator,
+ } => write!(formatter, "split_top_level({separator})"),
Self::ParseJson {
repair,
} => {
@@ -373,6 +381,9 @@ impl Operation {
pattern,
regex_engine,
} => string::execute_regex_find_all(pattern, regex_engine, input),
+ Self::SplitTopLevel {
+ separator,
+ } => string::execute_split_top_level(*separator, input),
Self::ParseJson {
repair,
} => string::execute_parse_json(*repair, input),
diff --git a/crates/json-transform/src/execution/string.rs b/crates/json-transform/src/execution/string.rs
index c785b4397..573dbbbac 100644
--- a/crates/json-transform/src/execution/string.rs
+++ b/crates/json-transform/src/execution/string.rs
@@ -56,6 +56,54 @@ pub fn execute_regex_find_all(
Ok(Value::Array(matches))
}
+#[tracing::instrument(skip(input))]
+pub fn execute_split_top_level(
+ separator: char,
+ input: Value,
+) -> Result<Value, TransformError> {
+ let Value::String(text) = input else {
+ return Ok(Value::Null);
+ };
+
+ let mut values = Vec::new();
+ let mut start = 0;
+ let mut depth = 0usize;
+ let mut quote = None;
+ let mut escaped = false;
+
+ for (index, character) in text.char_indices() {
+ if let Some(quote_character) = quote {
+ if escaped {
+ escaped = false;
+ } else if character == '\\' {
+ escaped = true;
+ } else if character == quote_character {
+ quote = None;
+ }
+ continue;
+ }
+
+ match character {
+ '"' | '\'' => quote = Some(character),
+ '{' | '[' | '(' => depth += 1,
+ '}' | ']' | ')' => depth = depth.saturating_sub(1),
+ _ if character == separator && depth == 0 => {
+ if start < index {
+ values.push(Value::String(text[start..index].to_string()));
+ }
+ start = index + character.len_utf8();
+ },
+ _ => {},
+ }
+ }
+
+ if start < text.len() {
+ values.push(Value::String(text[start..].to_string()));
+ }
+
+ Ok(Value::Array(values))
+}
+
#[tracing::instrument(skip(input))]
pub fn execute_parse_json(
repair: bool,
diff --git a/crates/json-transform/tests/test_json_transform_string.rs b/crates/json-transform/tests/test_json_transform_string.rs
index b49bed735..c1c776e60 100644
--- a/crates/json-transform/tests/test_json_transform_string.rs
+++ b/crates/json-transform/tests/test_json_transform_string.rs
@@ -216,6 +216,33 @@ fn test_regex_find_all_with_each_parse_json() {
assert_eq!(result, json!([{"city": "London"}, {"city": "Tokyo"}]));
}
+// SplitTopLevel
+
+#[test]
+fn test_split_top_level_ignores_separators_in_json_strings() {
+ let result = execute_root(
+ vec![
+ Operation::SplitTopLevel {
+ separator: ';',
+ },
+ Operation::Each {
+ apply: vec![Operation::ParseJson {
+ repair: false,
+ }],
+ },
+ ],
+ json!(r#"{"query":"a;b"}; {"query":"c\";d","nested":{"value":"e;f"}}"#),
+ )
+ .unwrap();
+ assert_eq!(
+ result,
+ json!([
+ {"query": "a;b"},
+ {"query": "c\";d", "nested": {"value": "e;f"}}
+ ])
+ );
+}
+
// ParseJson
#[test]
diff --git a/crates/nagare/Cargo.toml b/crates/nagare/Cargo.toml
index 0e106bc69..c10399daa 100644
--- a/crates/nagare/Cargo.toml
+++ b/crates/nagare/Cargo.toml
@@ -52,6 +52,7 @@ bindings-wasm = [
[dependencies]
bindings-types.workspace = true
bindings.workspace = true
+proc-macros.workspace = true
shoji.workspace = true
hanashi.workspace = true
serde.workspace = true
@@ -63,6 +64,7 @@ indexmap.workspace = true
reqwest.workspace = true
futures.workspace = true
tokio-util.workspace = true
+tokenizers.workspace = true
tokio = { workspace = true, features = ["full"] }
uniffi = { workspace = true, optional = true }
napi = { workspace = true, optional = true }
diff --git a/crates/nagare/src/chat/error.rs b/crates/nagare/src/chat/error.rs
index 255fafbdd..41aa5c5b2 100644
--- a/crates/nagare/src/chat/error.rs
+++ b/crates/nagare/src/chat/error.rs
@@ -18,4 +18,8 @@ pub enum ChatSessionError {
UnableToPerformOperationInCurrentState {},
#[error("No response")]
NoResponse {},
+ #[error("Tool turn limit exceeded: {limit}")]
+ ToolTurnLimitExceeded {
+ limit: u32,
+ },
}
diff --git a/crates/nagare/src/chat/message.rs b/crates/nagare/src/chat/message.rs
index 332b4f769..8299a66f2 100644
--- a/crates/nagare/src/chat/message.rs
+++ b/crates/nagare/src/chat/message.rs
@@ -61,4 +61,12 @@ impl Session {
pub fn peak_memory_usage(&self) -> Option<usize> {
self.instance.peak_memory_usage()
}
+
+ pub fn supports_tool_calls(&self) -> bool {
+ true
+ }
+
+ pub fn supports_multiple_tool_calls(&self) -> bool {
+ true
+ }
}
diff --git a/crates/nagare/src/chat/mod.rs b/crates/nagare/src/chat/mod.rs
index e4fcc3a71..c5c384b74 100644
--- a/crates/nagare/src/chat/mod.rs
+++ b/crates/nagare/src/chat/mod.rs
@@ -2,10 +2,10 @@ mod error;
pub mod message;
pub mod token;
-use std::sync::Arc;
+use std::{panic::AssertUnwindSafe, sync::Arc};
pub use error::ChatSessionError;
-use futures::StreamExt;
+use futures::{FutureExt, StreamExt};
use indexmap::IndexMap;
use serde::{Deserialize, Serialize};
use shoji::{
@@ -14,14 +14,23 @@ use shoji::{
backend::chat_message::{Output as BackendOutput, ToolCallState},
},
types::{
- basic::{CancelToken, ToolCall, Value},
+ basic::{CancelToken, ToolCall, ToolDescription, ToolNamespace, Value},
model::{Model, ModelSpecialization},
- session::chat::{ChatConfig, ChatMessage, ChatReply, ChatReplyConfig},
+ session::chat::{
+ ChatConfig, ChatContentBlock, ChatMessage, ChatReply, ChatReplyConfig, ChatReplyFinishReason, ChatRole,
+ },
},
};
-use tokio::sync::{Mutex, mpsc};
+use tokio::sync::{Mutex, mpsc, mpsc::UnboundedSender};
+use tokio_util::sync::CancellationToken;
+use uuid::Uuid;
-use crate::telemetry::{Telemetry, TelemetryEvent};
+use crate::{
+ telemetry::{Telemetry, TelemetryEvent},
+ tool::{func_def::ToolDescriptor, registry::ToolRegistry},
+};
+
+const DEFAULT_TOOL_TURN_LIMIT: u32 = 10;
#[bindings::export(Enumeration)]
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
@@ -83,6 +92,20 @@ impl Instance {
Instance::Message(session) => session.peak_memory_usage(),
}
}
+
+ pub fn supports_tool_calls(&self) -> bool {
+ match self {
+ Instance::Token(session) => session.supports_tool_calls(),
+ Instance::Message(session) => session.supports_tool_calls(),
+ }
+ }
+
+ pub fn supports_multiple_tool_calls(&self) -> bool {
+ match self {
+ Instance::Token(session) => session.supports_multiple_tool_calls(),
+ Instance::Message(session) => session.supports_multiple_tool_calls(),
+ }
+ }
}
#[bindings::export(Class)]
@@ -93,6 +116,7 @@ pub struct ChatSession {
messages: Arc<Mutex<Vec<ChatMessage>>>,
model_id: String,
telemetry: Telemetry,
+ tool_registry: Option<Arc<Mutex<ToolRegistry>>>,
}
impl ChatSession {
@@ -123,18 +147,377 @@ impl ChatSession {
message: error.to_string(),
})??;
+ let supports_tool_calls = instance.supports_tool_calls();
Ok(Self {
instance: Arc::new(Mutex::new(instance)),
state: Arc::new(Mutex::new(ChatSessionState::Idle)),
messages: Arc::new(Mutex::new(Vec::new())),
model_id,
telemetry,
+ tool_registry: supports_tool_calls.then(|| Arc::new(Mutex::new(ToolRegistry::new()))),
})
}
pub async fn peak_memory_usage(&self) -> Option<usize> {
self.instance.lock().await.peak_memory_usage()
}
+
+ pub async fn add_tool_function(
+ &mut self,
+ function: impl Into<ToolDescriptor>,
+ ) -> Result<(), ChatSessionError> {
+ self.add_tool_function_definitions(vec![function.into()]).await
+ }
+
+ pub async fn add_tool_descriptor(
+ &mut self,
+ definition: ToolDescriptor,
+ ) -> Result<(), ChatSessionError> {
+ self.add_tool_function_definitions(vec![definition]).await
+ }
+
+ pub async fn add_tool_function_definitions(
+ &mut self,
+ definitions: Vec<ToolDescriptor>,
+ ) -> Result<(), ChatSessionError> {
+ if definitions.is_empty() {
+ return Ok(());
+ }
+ let Some(registry) = self.tool_registry.as_ref() else {
+ return Err(ChatSessionError::Backend {
+ message: "Tool calls are not supported by this model".to_string(),
+ });
+ };
+
+ // register first so the owned set covers both prior and incoming definitions;
+ // an incoming name must displace a stale caller-provided declaration too
+ let owned_namespaces = {
+ let mut registry_guard = registry.lock().await;
+ for def in definitions {
+ registry_guard.add_function(def)
+ }
+ registry_guard.get_namespaces()
+ };
+
+ let mut messages_guard = self.messages.lock().await;
+ let messages_empty = messages_guard.is_empty();
+ if !messages_empty {
+ messages_guard.retain_mut(|message| {
+ if message.role == (ChatRole::Developer {}) {
+ let original_len = message.content.len();
+ // strip only registry-owned declarations; callers may declare
+ // tools they execute themselves, which must stay visible
+ message.content.retain_mut(|content| {
+ let ChatContentBlock::Tools {
+ namespaces,
+ } = content
+ else {
+ return true;
+ };
+ remove_owned_tools(namespaces, &owned_namespaces);
+ !namespaces.is_empty()
+ });
+ original_len == message.content.len() || !message.content.is_empty()
+ } else {
+ true
+ }
+ });
+ }
+ drop(messages_guard);
+
+ if !messages_empty {
+ let mut instance_guard = self.instance.lock().await;
+ match &mut *instance_guard {
+ Instance::Token(session) => session.reset().await?,
+ Instance::Message(session) => session.reset().await?,
+ };
+ }
+
+ Ok(())
+ }
+
+ async fn send_input(
+ &self,
+ sender: UnboundedSender<Result<Vec<ChatReply>, ChatSessionError>>,
+ input: Vec<ChatMessage>,
+ config: ChatReplyConfig,
+ cancel_token: CancellationToken,
+ ) {
+ // prepare all messages
+ let all_messages = {
+ let mut messages_guard = self.messages.lock().await;
+ messages_guard.extend(input);
+
+ // register tools if needed
+ if let Some(ref registry) = self.tool_registry {
+ let namespaces = registry.lock().await.get_namespaces();
+ if !namespaces.is_empty() {
+ merge_registered_tools(&mut messages_guard, namespaces);
+ }
+ }
+
+ messages_guard.clone()
+ };
+
+ self.telemetry.report(TelemetryEvent::ModelInferenceStarted {
+ model_id: self.model_id.clone(),
+ });
+
+ let mut outputs: IndexMap<u32, ChatReply> = IndexMap::new();
+ let mut error_value: Option<serde_json::Value> = None;
+ let mut generated_tool_call_identifiers: Vec<Option<String>> = Vec::new();
+
+ let mut instance = self.instance.lock().await;
+ let mut stream = match &mut *instance {
+ Instance::Token(session) => session.stream(&all_messages, config, cancel_token).await,
+ Instance::Message(session) => session.stream(&all_messages, config, cancel_token),
+ };
+ while let Some(partial_output) = stream.next().await {
+ match partial_output {
+ Ok(backend_output) => {
+ // prepare reply
+ let message = build_message(&backend_output, &mut generated_tool_call_identifiers);
+ let finish_reason = backend_output.finish_reason;
+ let output = ChatReply {
+ message: message.clone(),
+ stats: backend_output.stats.clone(),
+ finish_reason: finish_reason.clone(),
+ };
+
+ // add output to messages list
+ let mut messages_guard = self.messages.lock().await;
+ let is_new = outputs.insert(0, output).is_none();
+ if is_new {
+ messages_guard.push(message.clone());
+ } else if let Some(last) = messages_guard.last_mut() {
+ *last = message.clone();
+ }
+ drop(messages_guard);
+
+ // send new output
+ if sender.send(Ok(outputs.values().cloned().collect())).is_err() {
+ break;
+ }
+ if finish_reason.is_some() {
+ break;
+ }
+ },
+ Err(error) => {
+ error_value = Some(serde_json::json!({ "message": error.to_string() }));
+ let _ = sender.send(Err(error));
+ break;
+ },
+ }
+ }
+ drop(stream);
+ drop(instance);
+
+ // telemetry report result
+ if let Some(error) = error_value {
+ self.telemetry.report(TelemetryEvent::ModelInferenceFailed {
+ error,
+ });
+ } else if let Some(last) = outputs.values().last() {
+ self.telemetry.report(TelemetryEvent::ModelInferenceFinished {
+ model_id: self.model_id.clone(),
+ stats: last.stats.clone(),
+ });
+ }
+ }
+
+ async fn execute_turn(
+ &self,
+ sender: UnboundedSender<Result<Vec<ChatReply>, ChatSessionError>>,
+ input: Vec<ChatMessage>,
+ config: ChatReplyConfig,
+ cancel_token: CancellationToken,
+ ) {
+ // check state
+ if !self.try_transition(ChatSessionState::Idle, ChatSessionState::Generation).await {
+ let _ = sender.send(Err(ChatSessionError::UnableToPerformOperationInCurrentState {}));
+ return;
+ }
+
+ let tool_turn_limit = config.tool_turn_limit.unwrap_or(DEFAULT_TOOL_TURN_LIMIT);
+ let mut tool_turns: u32 = 0;
+ let mut next_input = input;
+
+ 'turns: loop {
+ let (turn_sender, mut turn_receiver) =
+ mpsc::unbounded_channel::<Result<Vec<ChatReply>, ChatSessionError>>();
+ let turn_input = std::mem::take(&mut next_input);
+ let turn_config = config.clone();
+ let turn_cancel_token = cancel_token.clone();
+ let session = self.clone();
+ tokio::spawn(
+ async move { session.send_input(turn_sender, turn_input, turn_config, turn_cancel_token).await },
+ );
+
+ // send message
+ let mut turn_replies: Vec<ChatReply> = Vec::new();
+ while let Some(result) = turn_receiver.recv().await {
+ match result {
+ Ok(replies) => {
+ turn_replies = replies;
+ if sender.send(Ok(turn_replies.clone())).is_err() {
+ break 'turns;
+ }
+ },
+ Err(error) => {
+ let _ = sender.send(Err(error));
+ break 'turns;
+ },
+ }
+ }
+
+ // check for tool calls and execute if needed
+ if let Some(last_reply) = turn_replies.last()
+ && last_reply.finish_reason == Some(ChatReplyFinishReason::ToolCalls)
+ && !cancel_token.is_cancelled()
+ {
+ let mut tool_calls = last_reply.message.tool_calls();
+ // a candidate is a call the parser could not finalize; executing only the
+ // finished subset would run a partial batch, so hand the turn back to the caller
+ let has_unfinished_candidates = last_reply
+ .message
+ .content
+ .iter()
+ .any(|block| matches!(block, ChatContentBlock::ToolCallCandidate { .. }));
+
+ let supports_multiple_tool_calls = self.instance.lock().await.supports_multiple_tool_calls();
+ if !supports_multiple_tool_calls && tool_calls.len() > 1 {
+ tool_calls.truncate(1);
+ // Sanitize history before any early return below. The caller may
+ // own one of the calls, but single-call templates still cannot
+ // re-encode an assistant message containing the extras.
+ if let Some(last_message) = self.messages.lock().await.last_mut() {
+ retain_first_tool_call(last_message);
+ }
+ }
+
+ if has_unfinished_candidates || !self.has_registered_tool_functions(&tool_calls).await {
+ break;
+ }
+
+ if tool_turns >= tool_turn_limit {
+ let _ = sender.send(Err(ChatSessionError::ToolTurnLimitExceeded {
+ limit: tool_turn_limit,
+ }));
+ break;
+ }
+ tool_turns += 1;
+
+ if self.try_transition(ChatSessionState::Generation, ChatSessionState::ToolCalling).await {
+ let tool_messages = tokio::select! {
+ tool_messages = self.execute_tool_calls(tool_calls) => tool_messages,
+ _ = cancel_token.cancelled() => break,
+ };
+ if self.try_transition(ChatSessionState::ToolCalling, ChatSessionState::Generation).await {
+ if !tool_messages.is_empty() && !cancel_token.is_cancelled() {
+ next_input = tool_messages;
+ } else {
+ break;
+ }
+ } else {
+ return;
+ }
+ } else {
+ return;
+ }
+ } else {
+ break;
+ }
+ }
+
+ // a canceled turn can leave the stored assistant tool-call reply without a matching tool result;
+ // drop it so the history stays a valid tool-call sequence
+ if cancel_token.is_cancelled() {
+ let mut messages_guard = self.messages.lock().await;
+ let unresolved = messages_guard.last().is_some_and(|message| {
+ message.role == (ChatRole::Assistant {})
+ && message.content.iter().any(|block| {
+ matches!(block, ChatContentBlock::ToolCall { .. } | ChatContentBlock::ToolCallCandidate { .. })
+ })
+ });
+ if unresolved {
+ messages_guard.pop();
+ }
+ }
+
+ *self.state.lock().await = ChatSessionState::Idle;
+ }
+
+ async fn has_registered_tool_functions(
+ &self,
+ tool_calls: &[ToolCall],
+ ) -> bool {
+ let Some(ref registry) = self.tool_registry else {
+ return false;
+ };
+ let registry_guard = registry.lock().await;
+ !tool_calls.is_empty() && tool_calls.iter().all(|call| registry_guard.get_function(&call.name).is_some())
+ }
+
+ async fn execute_tool_calls(
+ &self,
+ tool_calls: Vec<ToolCall>,
+ ) -> Vec<ChatMessage> {
+ if tool_calls.is_empty() {
+ return vec![];
+ }
+ let Some(ref registry) = self.tool_registry else {
+ return vec![];
+ };
+
+ let mut tool_messages: Vec<ChatMessage> = Vec::with_capacity(tool_calls.len());
+ for call in tool_calls {
+ let func = {
+ let registry_guard = registry.lock().await;
+ registry_guard.get_function(&call.name).cloned()
+ };
+ let value = if let Some(func) = func {
+ // a panicking tool must not unwind the turn task: the stream would close as a success while the session stays stuck in ToolCalling
+ match AssertUnwindSafe(func.execute(call.arguments)).catch_unwind().await {
+ Ok(Ok(value)) => value,
+ Ok(Err(err)) => Value::from(serde_json::json!({
+ "error": err.to_string(),
+ })),
+ Err(panic) => Value::from(serde_json::json!({
+ "error": format!("Tool '{}' panicked: {}", call.name, panic_message(panic.as_ref())),
+ })),
+ }
+ } else {
+ Value::from(serde_json::json!({
+ "error": format!("Unknown function: {}", call.name),
+ }))
+ };
+
+ // chat templates expect tool results as plain text, one message per call
+ let tool_message = ChatMessage::tool().with_block(ChatContentBlock::ToolCallResult {
+ identifier: call.identifier.clone(),
+ name: Some(call.name.clone()),
+ value,
+ });
+ tool_messages.push(tool_message);
+ }
+ tool_messages
+ }
+
+ /// Atomically switches state from `from` to `to`.
+ /// Returns false (leaving state untouched) if the current state differs.
+ async fn try_transition(
+ &self,
+ from: ChatSessionState,
+ to: ChatSessionState,
+ ) -> bool {
+ let mut state_guard = self.state.lock().await;
+ if *state_guard == from {
+ *state_guard = to;
+ true
+ } else {
+ false
+ }
+ }
}
#[bindings::export(Implementation)]
@@ -205,105 +588,10 @@ impl ChatSession {
config: ChatReplyConfig,
) -> ChatSessionStream {
let cancel_token_to_return = CancelToken::new();
- let (sender, receiver) = mpsc::unbounded_channel::<Result<Vec<ChatReply>, ChatSessionError>>();
-
- let instance = self.instance.clone();
- let state = self.state.clone();
- let messages = self.messages.clone();
- let telemetry = self.telemetry.clone();
- let model_id = self.model_id.clone();
let cancel_token = cancel_token_to_return.inner().clone();
-
- tokio::spawn(async move {
- {
- let mut state = state.lock().await;
- match *state {
- ChatSessionState::Idle => {
- *state = ChatSessionState::Generation;
- },
- ChatSessionState::Generation | ChatSessionState::Resetting | ChatSessionState::ToolCalling => {
- let _ = sender.send(Err(ChatSessionError::UnableToPerformOperationInCurrentState {}));
- return;
- },
- }
- drop(state);
- }
-
- telemetry.report(TelemetryEvent::ModelInferenceStarted {
- model_id: model_id.clone(),
- });
-
- let all_messages = {
- let mut messages = messages.lock().await;
- messages.extend(input);
- messages.clone()
- };
-
- let mut outputs: IndexMap<u32, ChatReply> = IndexMap::new();
-
- let mut instance = instance.lock().await;
- let mut stream = match &mut *instance {
- Instance::Token(session) => session.stream(&all_messages, config, cancel_token).await,
- Instance::Message(session) => session.stream(&all_messages, config, cancel_token),
- };
-
- let turn_index: u32 = 0;
- let mut errored = false;
- while let Some(partial_output) = stream.next().await {
- match partial_output {
- Ok(backend_output) => {
- let message = build_message(&backend_output);
- let finish_reason = backend_output.finish_reason;
- let output = ChatReply {
- message: message.clone(),
- stats: backend_output.stats.clone(),
- finish_reason: finish_reason.clone(),
- };
- let is_new = outputs.insert(turn_index, output).is_none();
-
- {
- let mut messages = messages.lock().await;
- if is_new {
- messages.push(message.clone());
- } else if let Some(last) = messages.last_mut() {
- *last = message.clone();
- }
- }
-
- if sender.send(Ok(outputs.values().cloned().collect())).is_err() {
- break;
- }
- if finish_reason.is_some() {
- break;
- }
- },
- Err(error) => {
- errored = true;
- telemetry.report(TelemetryEvent::ModelInferenceFailed {
- error: serde_json::json!({ "message": error.to_string() }),
- });
- let _ = sender.send(Err(error));
- break;
- },
- }
- }
-
- drop(stream);
- drop(instance);
-
- {
- let mut state = state.lock().await;
- *state = ChatSessionState::Idle;
- }
-
- if !errored && let Some(last) = outputs.values().last() {
- telemetry.report(TelemetryEvent::ModelInferenceFinished {
- model_id,
- stats: last.stats.clone(),
- });
- }
- });
-
+ let (sender, receiver) = mpsc::unbounded_channel::<Result<Vec<ChatReply>, ChatSessionError>>();
+ let session = self.clone();
+ tokio::spawn(async move { session.execute_turn(sender, input, config, cancel_token).await });
ChatSessionStream {
receiver: Arc::new(Mutex::new(receiver)),
cancel_token: cancel_token_to_return,
@@ -311,7 +599,10 @@ impl ChatSession {
}
}
-fn build_message(output: &BackendOutput) -> ChatMessage {
+fn build_message(
+ output: &BackendOutput,
+ generated_tool_call_identifiers: &mut Vec<Option<String>>,
+) -> ChatMessage {
let mut message = ChatMessage::assistant();
if let Some(reasoning) = &output.reasoning {
message = message.with_reasoning(reasoning.clone());
@@ -319,17 +610,133 @@ fn build_message(output: &BackendOutput) -> ChatMessage {
if let Some(text) = &output.text {
message = message.with_text(text.clone());
}
- for tool_call in &output.tool_calls {
+ for (index, tool_call) in output.tool_calls.iter().enumerate() {
message = match tool_call {
ToolCallState::Candidate(candidate) => {
message.with_tool_call_candidate(Value::from(serde_json::Value::String(candidate.clone())))
},
- ToolCallState::Finished(tool_call) => message.with_tool_call(ToolCall {
- identifier: tool_call.identifier.clone(),
- name: tool_call.name.clone(),
- arguments: tool_call.arguments.clone(),
- }),
+ ToolCallState::Finished(tool_call) => {
+ let identifier = tool_call.identifier.clone().or_else(|| {
+ if generated_tool_call_identifiers.len() <= index {
+ generated_tool_call_identifiers.resize(index + 1, None);
+ }
+ Some(
+ generated_tool_call_identifiers[index]
+ .get_or_insert_with(|| Uuid::new_v4().to_string())
+ .clone(),
+ )
+ });
+ message.with_tool_call(ToolCall {
+ identifier,
+ name: tool_call.name.clone(),
+ arguments: tool_call.arguments.clone(),
+ })
+ },
};
}
message
}
+
+fn find_tools_definitions(messages: &mut [ChatMessage]) -> Option<&mut Vec<ToolNamespace>> {
+ messages.iter_mut().filter(|msg| msg.role == ChatRole::Developer {}).find_map(|msg| {
+ msg.content.iter_mut().find_map(|content| match content {
+ ChatContentBlock::Tools {
+ namespaces,
+ } => Some(namespaces),
+ _ => None,
+ })
+ })
+}
+
+fn merge_registered_tools(
+ messages: &mut Vec<ChatMessage>,
+ namespaces: Vec<ToolNamespace>,
+) {
+ if let Some(existing) = find_tools_definitions(messages) {
+ merge_tool_namespaces(existing, namespaces);
+ return;
+ }
+
+ if let Some(developer_message) = messages.iter_mut().find(|message| message.role == ChatRole::Developer {}) {
+ developer_message.content.push(ChatContentBlock::Tools {
+ namespaces,
+ });
+ return;
+ }
+
+ let position = messages.iter().position(|message| message.role == ChatRole::System {});
+ let tools_message = ChatMessage::developer().with_tool_namespaces(namespaces);
+ messages.insert(position.map(|position| position + 1).unwrap_or(0), tools_message);
+}
+
+fn panic_message(payload: &(dyn std::any::Any + Send)) -> &str {
+ payload
+ .downcast_ref::<&str>()
+ .copied()
+ .or_else(|| payload.downcast_ref::<String>().map(String::as_str))
+ .unwrap_or("unknown panic")
+}
+
+fn retain_first_tool_call(message: &mut ChatMessage) {
+ let mut tool_call_seen = false;
+ message.content.retain(|block| match block {
+ ChatContentBlock::ToolCall {
+ ..
+ }
+ | ChatContentBlock::ToolCallCandidate {
+ ..
+ } => !std::mem::replace(&mut tool_call_seen, true),
+ _ => true,
+ });
+}
+
+fn remove_owned_tools(
+ namespaces: &mut Vec<ToolNamespace>,
+ owned: &[ToolNamespace],
+) {
+ for namespace in namespaces.iter_mut() {
+ let Some(owned_namespace) = owned.iter().find(|owned_ns| owned_ns.name == namespace.name) else {
+ continue;
+ };
+ namespace.tools.retain(|tool| {
+ let ToolDescription::Function {
+ tool_function,
+ } = tool;
+ !owned_namespace.tools.iter().any(|owned_tool| {
+ let ToolDescription::Function {
+ tool_function: owned_function,
+ } = owned_tool;
+ owned_function.name == tool_function.name
+ })
+ });
+ }
+ namespaces.retain(|namespace| !namespace.tools.is_empty());
+}
+
+fn merge_tool_namespaces(
+ existing: &mut Vec<ToolNamespace>,
+ registered: Vec<ToolNamespace>,
+) {
+ for namespace in registered {
+ let Some(target) = existing.iter_mut().find(|existing_ns| existing_ns.name == namespace.name) else {
+ existing.push(namespace);
+ continue;
+ };
+ for tool in namespace.tools {
+ let ToolDescription::Function {
+ tool_function,
+ } = &tool;
+ let existing = target.tools.iter_mut().find(|existing_tool| {
+ let ToolDescription::Function {
+ tool_function: existing_function,
+ } = existing_tool;
+ existing_function.name == tool_function.name
+ });
+ if let Some(existing) = existing {
+ *existing = tool;
+ } else {
+ target.tools.push(tool);
+ }
+ }
+ }
+}
diff --git a/crates/nagare/src/chat/token.rs b/crates/nagare/src/chat/token.rs
index 4660a5310..626494736 100644
--- a/crates/nagare/src/chat/token.rs
+++ b/crates/nagare/src/chat/token.rs
@@ -1,10 +1,16 @@
use std::{
+ io,
pin::Pin,
+ sync::Arc,
time::{Duration, Instant},
};
use futures::{Stream, StreamExt, stream};
-use hanashi::{Encoding as EncodingTrait, chat::Encoding};
+use hanashi::{
+ Encoding as EncodingTrait,
+ chat::{Encoding, EncodingConfig, TokenizerLocation, hanashi::HanashiEncodingImpl, harmony::HarmonyEncodingImpl},
+ load_tokenizer,
+};
use shoji::{
traits::{
State,
@@ -29,10 +35,7 @@ use tokio_util::sync::CancellationToken;
use crate::{
chat::ChatSessionError,
- util::{
- helpers::{build_encoding, error_stream},
- power::PowerRecorder,
- },
+ util::{helpers::error_stream, power::PowerRecorder},
};
pub struct Session {
@@ -51,17 +54,63 @@ impl Session {
reference: String,
model: &Model,
) -> Result<Self, ChatSessionError> {
- let encoding = build_encoding(reference.clone(), model).map_err(|err| ChatSessionError::Loading {
- message: err.to_string(),
- })?;
- let tokenizer = encoding.tokenizer().ok_or_else(|| ChatSessionError::Loading {
- message: "tokenizer is empty".to_string(),
- })?;
+ let encoding_configs = model
+ .encodings
+ .iter()
+ .map(|value| serde_json::from_str::<EncodingConfig>(value.json.as_str()).map_err(io::Error::other))
+ .collect::<Result<Vec<_>, _>>()
+ .map_err(|err| ChatSessionError::Loading {
+ message: format!("Failed to parse encoding config: {err}"),
+ })?;
- let instance =
- backend.instance(reference, config, tokenizer).await.map_err(|error| ChatSessionError::Backend {
+ let encoding_config: Option<EncodingConfig> = if encoding_configs.is_empty() {
+ None
+ } else if encoding_configs.len() == 1 {
+ encoding_configs.first().cloned()
+ } else {
+ let harmony_config =
+ encoding_configs.iter().find(|config| matches!(config, EncodingConfig::Harmony { .. }));
+ if harmony_config.is_some() {
+ harmony_config.cloned()
+ } else {
+ encoding_configs.first().cloned()
+ }
+ };
+
+ let tokenizer_location = TokenizerLocation::Directory {
+ path: reference.clone(),
+ name: None,
+ };
+ let tokenizer = load_tokenizer(&tokenizer_location).map_err(|err| ChatSessionError::Loading {
+ message: format!("Failed to initialize tokenizer: {err}"),
+ })?;
+ let tokenizer = Arc::new(tokenizer);
+
+ let encoding = match encoding_config {
+ Some(EncodingConfig::Hanashi {
+ config,
+ }) => HanashiEncodingImpl::new(config, tokenizer.clone()).map(Encoding::Hanashi).map_err(|err| {
+ ChatSessionError::Loading {
+ message: format!("can not create harmony encoding: {err}"),
+ }
+ }),
+ Some(EncodingConfig::Harmony {
+ config,
+ }) => HarmonyEncodingImpl::new(config, tokenizer_location).map(Encoding::Harmony).map_err(|err| {
+ ChatSessionError::Loading {
+ message: format!("can not create harmony encoding: {err}"),
+ }
+ }),
+ None => Err(ChatSessionError::Loading {
+ message: "can not get encoding config".to_string(),
+ }),
+ }?;
+
+ let instance = backend.instance(reference.clone(), config, tokenizer).await.map_err(|error| {
+ ChatSessionError::Backend {
message: error.to_string(),
- })?;
+ }
+ })?;
let state = instance.state().await.map_err(|error| ChatSessionError::Backend {
message: error.to_string(),
})?;
@@ -256,7 +305,15 @@ impl Session {
} => Some(ToolCallState::Candidate(value.json.clone())),
_ => None,
})
- .collect();
+ .collect::<Vec<_>>();
+
+ let finish_reason = if let Some(ChatReplyFinishReason::Stop) = finish_reason
+ && !tool_calls.is_empty()
+ {
+ Some(ChatReplyFinishReason::ToolCalls)
+ } else {
+ finish_reason
+ };
Output {
reasoning: message.reasoning(),
@@ -266,6 +323,14 @@ impl Session {
stats: state.get_stats(have_finish_reason),
}
}
+
+ pub fn supports_tool_calls(&self) -> bool {
+ self.encoding.supports_tool_calls()
+ }
+
+ pub fn supports_multiple_tool_calls(&self) -> bool {
+ self.encoding.supports_multiple_tool_calls()
+ }
}
struct StreamingState<'a> {
diff --git a/crates/nagare/src/lib.rs b/crates/nagare/src/lib.rs
index 65698a0d0..92ec15fac 100644
--- a/crates/nagare/src/lib.rs
+++ b/crates/nagare/src/lib.rs
@@ -6,4 +6,5 @@ pub mod chat;
pub mod classification;
pub mod telemetry;
pub mod text_to_speech;
+pub mod tool;
mod util;
diff --git a/crates/nagare/src/tool/func_def.rs b/crates/nagare/src/tool/func_def.rs
new file mode 100644
index 000000000..24c8c20b6
--- /dev/null
+++ b/crates/nagare/src/tool/func_def.rs
@@ -0,0 +1,144 @@
+use std::{error::Error, future::Future, sync::Arc};
+
+pub use shoji::types::basic::Value;
+
+pub type ErrorFuture = Box<dyn Error + Send + Sync>;
+pub type FunctionFuture = dyn Fn(Value) -> Box<dyn Future<Output = Result<Value, ErrorFuture>> + Send> + Send + Sync;
+
+#[derive(Clone)]
+pub struct ToolDescriptor {
+ pub name: String,
+ pub description: String,
+ pub parameters: Option<Value>,
+ pub return_definition: Option<Value>,
+ func: Arc<FunctionFuture>,
+}
+
+impl ToolDescriptor {
+ pub fn new(
+ name: String,
+ description: String,
+ parameters: Option<Value>,
+ return_definition: Option<Value>,
+ func: Box<FunctionFuture>,
+ ) -> Self {
+ Self {
+ name,
+ description,
+ parameters,
+ return_definition,
+ func: Arc::new(func),
+ }
+ }
+
+ pub async fn execute(
+ &self,
+ args: Value,
+ ) -> Result<Value, ErrorFuture> {
+ let args = self.coerce_arguments(args);
+ Box::into_pin((self.func)(args)).await
+ }
+
+ // Small models often mistype scalar tool arguments (e.g. Llama 3.2 1B passes "37" for a number parameter);
+ // coerce argument values to their schema-declared scalar types instead of failing the call — an error result
+ // makes such models retry the same call indefinitely.
+ fn coerce_arguments(
+ &self,
+ args: Value,
+ ) -> Value {
+ let Some(parameters) = &self.parameters else {
+ return args;
+ };
+ let Ok(schema) = serde_json::Value::try_from(parameters.clone()) else {
+ return args;
+ };
+ let Ok(json) = serde_json::Value::try_from(args.clone()) else {
+ return args;
+ };
+ Value::from(coerce_to_schema(json, &schema))
+ }
+}
+
+/// Runtime surface of the `uzu_tool_function`/`uzu_tool_closure` expansions;
+/// public so generated code relies on ordinary API instead of hidden re-exports.
+pub fn parse_arguments(args: Value) -> Result<serde_json::Value, ErrorFuture> {
+ Ok(serde_json::Value::try_from(args)?)
+}
+
+pub fn extract_argument<T: serde::de::DeserializeOwned>(
+ args: &serde_json::Value,
+ name: &str,
+ tool_name: &str,
+) -> Result<T, ErrorFuture> {
+ serde_json::from_value(args.get(name).cloned().unwrap_or(serde_json::Value::Null))
+ .map_err(|error| format!("invalid value for parameter `{name}` of tool `{tool_name}`: {error}").into())
+}
+
+pub fn serialize_result<T: serde::Serialize>(result: &T) -> Result<Value, ErrorFuture> {
+ Ok(Value::from(serde_json::to_value(result)?))
+}
+
+pub fn null_result() -> Value {
+ Value::from(serde_json::Value::Null)
+}
+
+fn coerce_to_schema(
+ value: serde_json::Value,
+ schema: &serde_json::Value,
+) -> serde_json::Value {
+ use serde_json::Value as Json;
+
+ match schema.get("type").and_then(Json::as_str) {
+ Some("object") => match value {
+ Json::Object(map) => Json::Object(
+ map.into_iter()
+ .map(|(key, value)| {
+ let value = match schema.get("properties").and_then(|properties| properties.get(&key)) {
+ Some(property_schema) => coerce_to_schema(value, property_schema),
+ None => value,
+ };
+ (key, value)
+ })
+ .collect(),
+ ),
+ other => other,
+ },
+ Some("array") => match (value, schema.get("items")) {
+ (Json::Array(items), Some(item_schema)) => {
+ Json::Array(items.into_iter().map(|item| coerce_to_schema(item, item_schema)).collect())
+ },
+ (other, _) => other,
+ },
+ Some("number") => match &value {
+ Json::String(text) => match text.trim().parse::<f64>().ok().and_then(serde_json::Number::from_f64) {
+ Some(number) => Json::Number(number),
+ None => value,
+ },
+ _ => value,
+ },
+ Some("integer") => match &value {
+ Json::String(text) => {
+ let text = text.trim();
+ match text.parse::<i64>() {
+ Ok(number) => Json::Number(number.into()),
+ Err(_) => match text.parse::<u64>() {
+ Ok(number) => Json::Number(number.into()),
+ Err(_) => value,
+ },
+ }
+ },
+ _ => value,
+ },
+ Some("boolean") => match &value {
+ Json::String(text) if text.trim().eq_ignore_ascii_case("true") => Json::Bool(true),
+ Json::String(text) if text.trim().eq_ignore_ascii_case("false") => Json::Bool(false),
+ _ => value,
+ },
+ Some("string") => match value {
+ Json::Number(number) => Json::String(number.to_string()),
+ Json::Bool(boolean) => Json::String(boolean.to_string()),
+ other => other,
+ },
+ _ => value,
+ }
+}
diff --git a/crates/nagare/src/tool/mod.rs b/crates/nagare/src/tool/mod.rs
new file mode 100644
index 000000000..386eb3239
--- /dev/null
+++ b/crates/nagare/src/tool/mod.rs
@@ -0,0 +1,5 @@
+pub mod func_def;
+pub mod registry;
+pub mod schema;
+
+pub use proc_macros::{uzu_tool_closure, uzu_tool_function};
diff --git a/crates/nagare/src/tool/registry.rs b/crates/nagare/src/tool/registry.rs
new file mode 100644
index 000000000..d336bd389
--- /dev/null
+++ b/crates/nagare/src/tool/registry.rs
@@ -0,0 +1,69 @@
+use indexmap::IndexMap;
+use shoji::types::basic::{ToolDescription, ToolFunction, ToolNamespace};
+
+use crate::tool::func_def::ToolDescriptor;
+
+#[derive(Clone)]
+pub struct ToolRegistry {
+ functions: IndexMap<String, ToolDescriptor>,
+}
+
+impl ToolRegistry {
+ pub fn new() -> Self {
+ Self {
+ functions: IndexMap::new(),
+ }
+ }
+
+ pub fn add_function(
+ &mut self,
+ function: ToolDescriptor,
+ ) {
+ self.functions.insert(function.name.to_string(), function);
+ }
+
+ pub fn get_function(
+ &self,
+ name: &str,
+ ) -> Option<&ToolDescriptor> {
+ self.functions.get(name)
+ }
+
+ pub fn get_namespaces(&self) -> Vec<ToolNamespace> {
+ let mut namespaces = Vec::new();
+
+ let tools_functions = self
+ .functions
+ .values()
+ .map(|func_def| ToolDescription::Function {
+ tool_function: ToolFunction {
+ name: func_def.name.to_string(),
+ description: func_def.description.to_string(),
+ parameters: func_def.parameters.clone().or_else(|| {
+ Some(
+ serde_json::json!({
+ "type": "object",
+ "properties": {},
+ "required": []
+ })
+ .into(),
+ )
+ }),
+ // No chat format documents a "return" schema in tool declarations
+ // (harmony drops it too). Small models mimic any extra key they see:
+ // Llama 3.2 1B answers with {"return": ...} echoes instead of text.
+ return_definition: None,
+ },
+ })
+ .collect::<Vec<_>>();
+ if !tools_functions.is_empty() {
+ namespaces.push(ToolNamespace {
+ name: "functions".to_string(),
+ description: None,
+ tools: tools_functions,
+ });
+ }
+
+ namespaces
+ }
+}
diff --git a/crates/nagare/src/tool/schema.rs b/crates/nagare/src/tool/schema.rs
new file mode 100644
index 000000000..d6113ec08
--- /dev/null
+++ b/crates/nagare/src/tool/schema.rs
@@ -0,0 +1,181 @@
+use indexmap::IndexMap;
+pub use proc_macros::UzuToolSchema;
+use serde::{Deserialize, Serialize};
+use shoji::types::basic::Value;
+
+pub trait ToolSchema {
+ fn json_schema() -> JsonSchema;
+}
+
+macro_rules! impl_tool_schema {
+ ($constructor:ident => $($ty:ty),+) => {
+ $(
+ impl ToolSchema for $ty {
+ fn json_schema() -> JsonSchema {
+ JsonSchema::$constructor()
+ }
+ }
+ )+
+ };
+}
+
+impl_tool_schema!(string => String);
+impl_tool_schema!(number => f32, f64);
+impl_tool_schema!(integer => i8, i16, i32, i64, isize, u8, u16, u32, u64, usize);
+impl_tool_schema!(boolean => bool);
+
+impl<T: ToolSchema> ToolSchema for Vec<T> {
+ fn json_schema() -> JsonSchema {
+ JsonSchema::array(T::json_schema())
+ }
+}
+
+impl<T: ToolSchema> ToolSchema for Option<T> {
+ fn json_schema() -> JsonSchema {
+ T::json_schema()
+ }
+}
+
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
+pub struct JsonSchema {
+ #[serde(flatten)]
+ pub schema_type: JsonSchemaType,
+ #[serde(skip_serializing_if = "Option::is_none")]
+ pub description: Option<String>,
+}
+
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
+#[serde(tag = "type", rename_all = "lowercase")]
+pub enum JsonSchemaType {
+ String {
+ #[serde(rename = "enum", skip_serializing_if = "Option::is_none")]
+ enum_values: Option<Vec<String>>,
+ },
+ Number,
+ Integer,
+ Boolean,
+ Array {
+ items: Box<JsonSchema>,
+ },
+ Object {
+ #[serde(default)]
+ properties: IndexMap<String, JsonSchema>,
+ #[serde(default)]
+ required: Vec<String>,
+ #[serde(rename = "additionalProperties", skip_serializing_if = "Option::is_none")]
+ additional_properties: Option<bool>,
+ },
+}
+
+impl JsonSchema {
+ pub fn string() -> Self {
+ Self {
+ schema_type: JsonSchemaType::String {
+ enum_values: None,
+ },
+ description: None,
+ }
+ }
+
+ pub fn number() -> Self {
+ Self {
+ schema_type: JsonSchemaType::Number,
+ description: None,
+ }
+ }
+
+ pub fn integer() -> Self {
+ Self {
+ schema_type: JsonSchemaType::Integer,
+ description: None,
+ }
+ }
+
+ pub fn boolean() -> Self {
+ Self {
+ schema_type: JsonSchemaType::Boolean,
+ description: None,
+ }
+ }
+
+ pub fn array(items: JsonSchema) -> Self {
+ Self {
+ schema_type: JsonSchemaType::Array {
+ items: Box::new(items),
+ },
+ description: None,
+ }
+ }
+
+ pub fn object<PK, P, RK, R>(
+ properties: P,
+ required: R,
+ ) -> Self
+ where
+ PK: Into<String>,
+ P: IntoIterator<Item = (PK, JsonSchema)>,
+ RK: Into<String>,
+ R: IntoIterator<Item = RK>,
+ {
+ Self {
+ schema_type: JsonSchemaType::Object {
+ properties: properties.into_iter().map(|(key, schema)| (key.into(), schema)).collect(),
+ required: required.into_iter().map(Into::into).collect(),
+ additional_properties: None,
+ },
+ description: None,
+ }
+ }
+
+ pub fn empty_object() -> Self {
+ Self {
+ schema_type: JsonSchemaType::Object {
+ properties: IndexMap::new(),
+ required: Vec::new(),
+ additional_properties: None,
+ },
+ description: None,
+ }
+ }
+
+ pub fn with_additional_properties(
+ mut self,
+ allowed: bool,
+ ) -> Self {
+ if let JsonSchemaType::Object {
+ additional_properties,
+ ..
+ } = &mut self.schema_type
+ {
+ *additional_properties = Some(allowed);
+ }
+ self
+ }
+
+ pub fn with_description(
+ mut self,
+ description: impl Into<String>,
+ ) -> Self {
+ self.description = Some(description.into());
+ self
+ }
+
+ pub fn with_enum_values(
+ mut self,
+ values: Vec<String>,
+ ) -> Self {
+ if let JsonSchemaType::String {
+ enum_values,
+ } = &mut self.schema_type
+ {
+ *enum_values = Some(values);
+ }
+ self
+ }
+
+ pub fn to_value(&self) -> Value {
+ Value {
+ json: serde_json::to_string(self).expect("JsonSchema serialization is infallible"),
+ }
+ }
+}
diff --git a/crates/nagare/src/util/helpers.rs b/crates/nagare/src/util/helpers.rs
index 018ebb2a8..08e2598a5 100644
--- a/crates/nagare/src/util/helpers.rs
+++ b/crates/nagare/src/util/helpers.rs
@@ -1,11 +1,7 @@
-use std::{io, pin::Pin};
+use std::pin::Pin;
use futures::{Stream, stream};
-use hanashi::{
- Encoding as EncodingTrait,
- chat::{Context, Encoding, EncodingConfig, TokenizerLocation},
-};
-use shoji::{traits::backend::chat_message::Output, types::model::Model};
+use shoji::traits::backend::chat_message::Output;
use crate::chat::ChatSessionError;
@@ -14,38 +10,3 @@ pub fn error_stream<'a>(
) -> Pin<Box<dyn Stream<Item = Result<Output, ChatSessionError>> + Send + 'a>> {
Box::pin(stream::once(async move { Err(err) }))
}
-
-pub fn build_encoding(
- reference: String,
- model: &Model,
-) -> Result<Encoding, io::Error> {
- let encoding_configs = model
- .encodings
- .iter()
- .map(|value| serde_json::from_str::<EncodingConfig>(value.json.as_str()).map_err(io::Error::other))
- .collect::<Result<Vec<_>, _>>()?;
-
- let config: Option<EncodingConfig> = if encoding_configs.is_empty() {
- None
- } else if encoding_configs.len() == 1 {
- encoding_configs.first().cloned()
- } else {
- let hanashi_config = encoding_configs.iter().find(|config| matches!(config, EncodingConfig::Hanashi { .. }));
- if hanashi_config.is_some() {
- hanashi_config.cloned()
- } else {
- encoding_configs.first().cloned()
- }
- };
- let Some(config) = config else {
- return Err(io::Error::other("can not get encoding config"));
- };
-
- let encoding_context = Context {
- tokenizer_location: TokenizerLocation::Directory {
- path: reference,
- name: Some("tokenizer.json".to_string()),
- },
- };
- Encoding::new(config, encoding_context).map_err(io::Error::other)
-}
diff --git a/crates/proc-macros/Cargo.toml b/crates/proc-macros/Cargo.toml
index 299e28512..6654b3d99 100644
--- a/crates/proc-macros/Cargo.toml
+++ b/crates/proc-macros/Cargo.toml
@@ -13,7 +13,9 @@ proc-macro = true
[dependencies]
proc-macro2.workspace = true
+proc-macro-crate.workspace = true
quote.workspace = true
+serde_derive_internals.workspace = true
syn.workspace = true
[lints]
diff --git a/crates/proc-macros/src/lib.rs b/crates/proc-macros/src/lib.rs
index fae235a29..e71852e77 100644
--- a/crates/proc-macros/src/lib.rs
+++ b/crates/proc-macros/src/lib.rs
@@ -1,6 +1,7 @@
mod dsl;
mod uzu_config;
mod uzu_test;
+mod uzu_tool;
use proc_macro::TokenStream;
@@ -46,3 +47,22 @@ pub fn uzu_bench(
) -> TokenStream {
uzu_test::uzu_bench(args, input)
}
+
+// Tool calls
+#[proc_macro]
+pub fn uzu_tool_closure(input: TokenStream) -> TokenStream {
+ uzu_tool::uzu_tool_closure(input)
+}
+
+#[proc_macro_attribute]
+pub fn uzu_tool_function(
+ args: TokenStream,
+ input: TokenStream,
+) -> TokenStream {
+ uzu_tool::uzu_tool_function(args, input)
+}
+
+#[proc_macro_derive(UzuToolSchema, attributes(serde))]
+pub fn uzu_tool_schema(input: TokenStream) -> TokenStream {
+ uzu_tool::uzu_derive_tool_schema(input)
+}
diff --git a/crates/proc-macros/src/uzu_tool/closure.rs b/crates/proc-macros/src/uzu_tool/closure.rs
new file mode 100644
index 000000000..c04a37e5e
--- /dev/null
+++ b/crates/proc-macros/src/uzu_tool/closure.rs
@@ -0,0 +1,174 @@
+use proc_macro::TokenStream;
+use proc_macro2::TokenStream as TokenStream2;
+use quote::quote;
+use syn::{
+ Attribute, Error, ExprClosure, Pat, ReturnType, Token,
+ ext::IdentExt,
+ parse::{Parse, ParseStream},
+ parse_macro_input,
+};
+
+use crate::uzu_tool::util::{
+ Param, arg_parsing_tokens, doc_string, is_option, is_unit, nagare_path, parameters_tokens, result_ok_type,
+ return_definition_tokens,
+};
+
+/// Input of `uzu_tool_closure!`: optional doc comments, a tool name, a colon, and a closure.
+struct ToolClosure {
+ attrs: Vec<Attribute>,
+ name: syn::Ident,
+ closure: ExprClosure,
+}
+
+impl Parse for ToolClosure {
+ fn parse(input: ParseStream) -> syn::Result<Self> {
+ let attrs = input.call(Attribute::parse_outer)?;
+ let name = input.call(syn::Ident::parse_any)?;
+ input.parse::<Token![:]>()?;
+ let closure: ExprClosure = input.parse()?;
+ if !input.is_empty() {
+ return Err(input.error("unexpected tokens after the tool closure"));
+ }
+ Ok(Self {
+ attrs,
+ name,
+ closure,
+ })
+ }
+}
+
+pub fn uzu_tool_closure(input: TokenStream) -> TokenStream {
+ let tool = parse_macro_input!(input as ToolClosure);
+ match expand_tool_closure(tool) {
+ Ok(tokens) => tokens.into(),
+ Err(error) => error.to_compile_error().into(),
+ }
+}
+
+fn expand_tool_closure(tool: ToolClosure) -> syn::Result<TokenStream2> {
+ let nagare = nagare_path()?;
+ let ToolClosure {
+ attrs,
+ name,
+ closure,
+ } = tool;
+
+ if let Some(constness) = &closure.constness {
+ return Err(Error::new_spanned(constness, "uzu_tool_closure! does not support `const` closures"));
+ }
+ if let Some(movability) = &closure.movability {
+ return Err(Error::new_spanned(movability, "uzu_tool_closure! does not support `static` closures"));
+ }
+ if let Some(lifetimes) = &closure.lifetimes {
+ return Err(Error::new_spanned(lifetimes, "uzu_tool_closure! does not support `for<...>` binders"));
+ }
+
+ let name_str = name.unraw().to_string();
+ let description = doc_string(&attrs);
+ let is_async = closure.asyncness.is_some();
+
+ let mut params = Vec::new();
+ for input in &closure.inputs {
+ let Pat::Type(pat_type) = input else {
+ return Err(Error::new_spanned(
+ input,
+ "uzu_tool_closure! parameters must have explicit types, e.g. `city: String`",
+ ));
+ };
+ let Pat::Ident(pat_ident) = &*pat_type.pat else {
+ return Err(Error::new_spanned(&pat_type.pat, "uzu_tool_closure! parameters must be plain identifiers"));
+ };
+ if pat_ident.subpat.is_some() {
+ return Err(Error::new_spanned(pat_ident, "uzu_tool_closure! parameters must be plain identifiers"));
+ }
+ params.push(Param {
+ ident: pat_ident.ident.clone(),
+ ty: (*pat_type.ty).clone(),
+ description: doc_string(&pat_type.attrs),
+ required: !is_option(&pat_type.ty),
+ });
+ }
+
+ // Without a return type annotation the result is serialized as-is (`()` becomes `null`) and no return
+ // schema is published; `Result` unwrapping requires an explicit `-> Result<T, E>` annotation.
+ let (return_type, ok_type, is_result) = match &closure.output {
+ ReturnType::Default => (None, None, false),
+ ReturnType::Type(_, ty) => match result_ok_type(ty) {
+ Some(ok) => (Some((**ty).clone()), (!is_unit(ok)).then(|| ok.clone()), true),
+ None => (Some((**ty).clone()), (!is_unit(ty)).then(|| (**ty).clone()), false),
+ },
+ };
+
+ let parameters = parameters_tokens(&params, &nagare);
+ let return_definition = return_definition_tokens(ok_type.as_ref(), &nagare);
+ let arg_parsing = arg_parsing_tokens(&params, &name_str, &nagare);
+
+ let body = &closure.body;
+ let closure_params = params.iter().map(|param| {
+ let ident = &param.ident;
+ let ty = &param.ty;
+ quote!(#ident: #ty)
+ });
+ // An async body is rewritten as a closure returning an `async move` block, so each invocation produces
+ // an owned future; the invoker clones the closure per call, which only requires the captures to be
+ // `Clone + Send + Sync + 'static`.
+ let func = if is_async {
+ quote!(move |#(#closure_params),*| async move { #body })
+ } else {
+ quote!(move |#(#closure_params),*| { #body })
+ };
+
+ let arg_idents = params.iter().map(|param| &param.ident);
+ let mut call = quote!(__uzu_tool_func(#(#arg_idents),*));
+ if is_async {
+ call = quote!(#call.await);
+ }
+ let result_stmt = match &return_type {
+ Some(ty) => quote!(let __uzu_result: #ty = #call;),
+ None => quote!(let __uzu_result = #call;),
+ };
+ let unwrap_stmt = if is_result {
+ quote! {
+ let __uzu_result =
+ __uzu_result.map_err(|error| -> #nagare::tool::func_def::ErrorFuture { error.into() })?;
+ }
+ } else {
+ quote!()
+ };
+ let serialize_result = return_type.is_none() || ok_type.is_some();
+ let output_expr = if serialize_result {
+ quote! {
+ #nagare::tool::func_def::serialize_result(&__uzu_result)
+ }
+ } else {
+ quote! {
+ let _ = __uzu_result;
+ ::core::result::Result::<
+ #nagare::tool::func_def::Value,
+ #nagare::tool::func_def::ErrorFuture,
+ >::Ok(#nagare::tool::func_def::null_result())
+ }
+ };
+
+ Ok(quote! {
+ {
+ let __uzu_tool_func = #func;
+ #nagare::tool::func_def::ToolDescriptor::new(
+ ::std::string::String::from(#name_str),
+ ::std::string::String::from(#description),
+ #parameters,
+ #return_definition,
+ ::std::boxed::Box::new(move |args| {
+ let __uzu_tool_func = ::core::clone::Clone::clone(&__uzu_tool_func);
+ ::std::boxed::Box::new(async move {
+ let args = #nagare::tool::func_def::parse_arguments(args)?;
+ #(#arg_parsing)*
+ #result_stmt
+ #unwrap_stmt
+ #output_expr
+ })
+ }),
+ )
+ }
+ })
+}
diff --git a/crates/proc-macros/src/uzu_tool/function.rs b/crates/proc-macros/src/uzu_tool/function.rs
new file mode 100644
index 000000000..09d24d7c0
--- /dev/null
+++ b/crates/proc-macros/src/uzu_tool/function.rs
@@ -0,0 +1,135 @@
+use proc_macro::TokenStream;
+use quote::{format_ident, quote};
+use syn::{
+ __private::TokenStream2, Attribute, Error, FnArg, ItemFn, Pat, ReturnType, ext::IdentExt, parse_macro_input,
+};
+
+use crate::uzu_tool::util::{
+ Param, arg_parsing_tokens, doc_string, is_option, is_unit, nagare_path, parameters_tokens, result_ok_type,
+ return_definition_tokens,
+};
+
+pub fn uzu_tool_function(
+ _args: TokenStream,
+ input: TokenStream,
+) -> TokenStream {
+ let func = parse_macro_input!(input as ItemFn);
+ match expand_tool_function(func) {
+ Ok(tokens) => tokens.into(),
+ Err(error) => error.to_compile_error().into(),
+ }
+}
+
+fn expand_tool_function(mut func: ItemFn) -> syn::Result<TokenStream2> {
+ let nagare = nagare_path()?;
+
+ if !func.sig.generics.params.is_empty() {
+ return Err(Error::new_spanned(&func.sig.generics, "#[tool_function] does not support generic functions"));
+ }
+
+ let vis = func.vis.clone();
+ let name = func.sig.ident.clone();
+ let name_str = name.unraw().to_string();
+ let description = doc_string(&func.attrs);
+ let doc_attrs: Vec<Attribute> = func.attrs.iter().filter(|attr| attr.path().is_ident("doc")).cloned().collect();
+ let is_async = func.sig.asyncness.is_some();
+
+ let mut params = Vec::new();
+ for input in &mut func.sig.inputs {
+ match input {
+ FnArg::Receiver(receiver) => {
+ return Err(Error::new_spanned(receiver, "#[tool_function] does not support `self` parameters"));
+ },
+ FnArg::Typed(arg) => {
+ let Pat::Ident(pat) = &*arg.pat else {
+ return Err(Error::new_spanned(&arg.pat, "#[tool_function] parameters must be plain identifiers"));
+ };
+ let description = doc_string(&arg.attrs);
+ arg.attrs.retain(|attr| !attr.path().is_ident("doc"));
+ params.push(Param {
+ ident: pat.ident.clone(),
+ ty: (*arg.ty).clone(),
+ description,
+ required: !is_option(&arg.ty),
+ });
+ },
+ }
+ }
+
+ let (ok_type, is_result) = match &func.sig.output {
+ ReturnType::Default => (None, false),
+ ReturnType::Type(_, ty) => match result_ok_type(ty) {
+ Some(ok) => ((!is_unit(ok)).then(|| ok.clone()), true),
+ None => ((!is_unit(ty)).then(|| (**ty).clone()), false),
+ },
+ };
+
+ let mut call_fn = func.clone();
+ call_fn.sig.ident = format_ident!("call");
+ call_fn.attrs.retain(|attr| !attr.path().is_ident("doc"));
+
+ let parameters = parameters_tokens(&params, &nagare);
+ let return_definition = return_definition_tokens(ok_type.as_ref(), &nagare);
+ let arg_parsing = arg_parsing_tokens(&params, &name_str, &nagare);
+
+ let arg_idents = params.iter().map(|param| &param.ident);
+ let mut call = quote!(Self::call(#(#arg_idents),*));
+ if is_async {
+ call = quote!(#call.await);
+ }
+ let map_err = quote!(.map_err(|error| -> #nagare::tool::func_def::ErrorFuture { error.into() })?);
+ let call_stmt = match (&ok_type, is_result) {
+ (Some(_), true) => quote!(let result = #call #map_err;),
+ (Some(_), false) => quote!(let result = #call;),
+ (None, true) => quote!(#call #map_err;),
+ (None, false) => quote!(#call;),
+ };
+ let output_expr = match &ok_type {
+ Some(_) => quote! {
+ #nagare::tool::func_def::serialize_result(&result)
+ },
+ None => quote! {
+ ::core::result::Result::Ok(#nagare::tool::func_def::null_result())
+ },
+ };
+
+ Ok(quote! {
+ #(#doc_attrs)*
+ #[allow(non_camel_case_types)]
+ #[derive(Clone, Copy)]
+ #vis struct #name;
+
+ impl #name {
+ #call_fn
+
+ #vis fn definition() -> #nagare::tool::func_def::ToolDescriptor {
+ #nagare::tool::func_def::ToolDescriptor::new(
+ ::std::string::String::from(#name_str),
+ ::std::string::String::from(#description),
+ #parameters,
+ #return_definition,
+ ::std::boxed::Box::new(|args| ::std::boxed::Box::new(Self::invoke(args))),
+ )
+ }
+
+ async fn invoke(
+ args: #nagare::tool::func_def::Value,
+ ) -> ::core::result::Result<
+ #nagare::tool::func_def::Value,
+ #nagare::tool::func_def::ErrorFuture,
+ >
+ {
+ let args = #nagare::tool::func_def::parse_arguments(args)?;
+ #(#arg_parsing)*
+ #call_stmt
+ #output_expr
+ }
+ }
+
+ impl ::core::convert::From<#name> for #nagare::tool::func_def::ToolDescriptor {
+ fn from(_: #name) -> Self {
+ #name::definition()
+ }
+ }
+ })
+}
diff --git a/crates/proc-macros/src/uzu_tool/mod.rs b/crates/proc-macros/src/uzu_tool/mod.rs
new file mode 100644
index 000000000..f33cdf7d0
--- /dev/null
+++ b/crates/proc-macros/src/uzu_tool/mod.rs
@@ -0,0 +1,8 @@
+mod closure;
+mod function;
+mod schema;
+mod util;
+
+pub use closure::uzu_tool_closure;
+pub use function::uzu_tool_function;
+pub use schema::uzu_derive_tool_schema;
diff --git a/crates/proc-macros/src/uzu_tool/schema.rs b/crates/proc-macros/src/uzu_tool/schema.rs
new file mode 100644
index 000000000..790fcd2a1
--- /dev/null
+++ b/crates/proc-macros/src/uzu_tool/schema.rs
@@ -0,0 +1,150 @@
+use proc_macro::TokenStream;
+use quote::quote;
+use serde_derive_internals::{
+ Ctxt, Derive,
+ ast::{Container as SerdeContainer, Data as SerdeData},
+};
+use syn::{__private::TokenStream2, Data, DeriveInput, Error, Fields, parse_macro_input};
+
+use crate::uzu_tool::util::{doc_string, is_option, nagare_path};
+
+pub fn uzu_derive_tool_schema(input: TokenStream) -> TokenStream {
+ let input = parse_macro_input!(input as DeriveInput);
+ match expand_tool_schema(input) {
+ Ok(tokens) => tokens.into(),
+ Err(error) => error.to_compile_error().into(),
+ }
+}
+
+fn expand_tool_schema(input: DeriveInput) -> syn::Result<TokenStream2> {
+ let nagare = nagare_path()?;
+
+ let Data::Struct(data) = &input.data else {
+ return Err(Error::new_spanned(&input.ident, "#[derive(UzuToolSchema)] only supports structs"));
+ };
+ let Fields::Named(_) = &data.fields else {
+ return Err(Error::new_spanned(
+ &input.ident,
+ "#[derive(UzuToolSchema)] only supports structs with named fields",
+ ));
+ };
+
+ let serde_context = Ctxt::new();
+ let serde_container = SerdeContainer::from_ast(&serde_context, &input, Derive::Deserialize);
+ serde_context.check()?;
+ let Some(serde_container) = serde_container else {
+ return Err(Error::new_spanned(&input.ident, "failed to parse Serde attributes"));
+ };
+ let SerdeData::Struct(_, serde_fields) = &serde_container.data else {
+ unreachable!();
+ };
+
+ if serde_container.attrs.transparent()
+ || serde_container.attrs.type_from().is_some()
+ || serde_container.attrs.type_try_from().is_some()
+ || serde_container.attrs.type_into().is_some()
+ || serde_container.attrs.remote().is_some()
+ {
+ return Err(Error::new_spanned(
+ &input.ident,
+ "UzuToolSchema does not support Serde container attributes that change the serialized representation",
+ ));
+ }
+
+ let schema_fields = serde_fields
+ .iter()
+ .filter_map(|field| {
+ let attrs = &field.attrs;
+ if attrs.skip_serializing() != attrs.skip_deserializing() {
+ return Some(Err(Error::new_spanned(
+ field.original,
+ "UzuToolSchema requires fields to be skipped in both Serialize and Deserialize; use #[serde(skip)]",
+ )));
+ }
+ if attrs.skip_serializing() {
+ return None;
+ }
+ if attrs.flatten() {
+ return Some(Err(Error::new_spanned(
+ field.original,
+ "UzuToolSchema does not support #[serde(flatten)]",
+ )));
+ }
+ if attrs.skip_serializing_if().is_some() {
+ return Some(Err(Error::new_spanned(
+ field.original,
+ "UzuToolSchema does not support #[serde(skip_serializing_if = \"...\")]",
+ )));
+ }
+ if attrs.serialize_with().is_some() || attrs.deserialize_with().is_some() || attrs.getter().is_some() {
+ return Some(Err(Error::new_spanned(
+ field.original,
+ "UzuToolSchema does not support Serde field attributes that change the serialized representation",
+ )));
+ }
+
+ let serialize_name = attrs.name().serialize_name();
+ let deserialize_name = attrs.name().deserialize_name();
+ if serialize_name != deserialize_name {
+ return Some(Err(Error::new_spanned(
+ field.original,
+ "UzuToolSchema requires matching Serialize and Deserialize field names",
+ )));
+ }
+
+ let required =
+ !is_option(field.ty) && attrs.default().is_none() && serde_container.attrs.default().is_none();
+ Some(Ok((field.original, serialize_name.to_string(), required)))
+ })
+ .collect::<syn::Result<Vec<_>>>()?;
+
+ let ident = &input.ident;
+ let mut generics = input.generics.clone();
+ for (field, _, _) in &schema_fields {
+ let ty = &field.ty;
+ generics.make_where_clause().predicates.push(syn::parse_quote!(#ty: #nagare::tool::schema::ToolSchema));
+ }
+ let (impl_generics, ty_generics, where_clause) = generics.split_for_impl();
+
+ let schema = if schema_fields.is_empty() {
+ quote!(#nagare::tool::schema::JsonSchema::empty_object())
+ } else {
+ let properties = schema_fields.iter().map(|(field, name, _)| {
+ let ty = &field.ty;
+ let schema = quote!(<#ty as #nagare::tool::schema::ToolSchema>::json_schema());
+ let description = doc_string(&field.attrs);
+ if description.is_empty() {
+ quote!((#name, #schema))
+ } else {
+ quote!((#name, #schema.with_description(#description)))
+ }
+ });
+ let required = schema_fields.iter().filter(|(_, _, required)| *required).map(|(_, name, _)| name);
+ quote! {
+ #nagare::tool::schema::JsonSchema::object(
+ [#(#properties),*],
+ ::std::vec::Vec::<&str>::from([#(#required),*]),
+ )
+ }
+ };
+ let with_additional_properties = if serde_container.attrs.deny_unknown_fields() {
+ quote!(.with_additional_properties(false))
+ } else {
+ quote!()
+ };
+
+ let struct_description = doc_string(&input.attrs);
+ let with_description = if struct_description.is_empty() {
+ quote!()
+ } else {
+ quote!(.with_description(#struct_description))
+ };
+
+ Ok(quote! {
+ impl #impl_generics #nagare::tool::schema::ToolSchema for #ident #ty_generics #where_clause {
+ fn json_schema() -> #nagare::tool::schema::JsonSchema {
+ #schema #with_additional_properties #with_description
+ }
+ }
+ })
+}
diff --git a/crates/proc-macros/src/uzu_tool/util.rs b/crates/proc-macros/src/uzu_tool/util.rs
new file mode 100644
index 000000000..e4d094405
--- /dev/null
+++ b/crates/proc-macros/src/uzu_tool/util.rs
@@ -0,0 +1,149 @@
+use proc_macro_crate::{FoundCrate, crate_name};
+use proc_macro2::Span;
+use quote::{format_ident, quote};
+use syn::{
+ __private::TokenStream2, Attribute, Error, Expr, ExprLit, GenericArgument, Lit, Meta, PathArguments, Type,
+ ext::IdentExt,
+};
+
+pub struct Param {
+ pub ident: syn::Ident,
+ pub ty: Type,
+ pub description: String,
+ pub required: bool,
+}
+
+pub fn arg_parsing_tokens(
+ params: &[Param],
+ tool_name: &str,
+ nagare: &TokenStream2,
+) -> Vec<TokenStream2> {
+ params
+ .iter()
+ .map(|param| {
+ let ident = &param.ident;
+ let ty = &param.ty;
+ let arg_name = param.ident.unraw().to_string();
+ quote! {
+ let #ident: #ty = #nagare::tool::func_def::extract_argument(&args, #arg_name, #tool_name)?;
+ }
+ })
+ .collect()
+}
+
+pub fn doc_string(attrs: &[Attribute]) -> String {
+ let lines: Vec<String> = attrs
+ .iter()
+ .filter_map(|attr| {
+ if !attr.path().is_ident("doc") {
+ return None;
+ }
+ if let Meta::NameValue(name_value) = &attr.meta
+ && let Expr::Lit(ExprLit {
+ lit: Lit::Str(literal),
+ ..
+ }) = &name_value.value
+ {
+ return Some(literal.value().trim().to_string());
+ }
+ None
+ })
+ .collect();
+ lines.join("\n").trim().to_string()
+}
+
+pub fn nagare_path() -> syn::Result<TokenStream2> {
+ match crate_name("nagare") {
+ Ok(FoundCrate::Itself) => Ok(quote!(crate)),
+ Ok(FoundCrate::Name(name)) => {
+ let ident = format_ident!("{name}");
+ Ok(quote!(::#ident))
+ },
+ Err(nagare_error) => match crate_name("uzu") {
+ Ok(FoundCrate::Itself) => Ok(quote!(crate::session)),
+ Ok(FoundCrate::Name(name)) => {
+ let ident = format_ident!("{name}");
+ Ok(quote!(::#ident::session))
+ },
+ Err(uzu_error) => Err(Error::new(
+ Span::call_site(),
+ format!(
+ "either `nagare` or `uzu` must be a direct dependency to use the tool macros: \
+ failed to find `nagare` ({nagare_error}); failed to find `uzu` ({uzu_error})"
+ ),
+ )),
+ },
+ }
+}
+
+pub fn is_option(ty: &Type) -> bool {
+ if let Type::Path(type_path) = ty
+ && let Some(segment) = type_path.path.segments.last()
+ {
+ return segment.ident == "Option";
+ }
+ false
+}
+
+pub fn is_unit(ty: &Type) -> bool {
+ matches!(ty, Type::Tuple(tuple) if tuple.elems.is_empty())
+}
+
+pub fn parameters_tokens(
+ params: &[Param],
+ nagare: &TokenStream2,
+) -> TokenStream2 {
+ if params.is_empty() {
+ return quote!(::core::option::Option::None);
+ }
+ let properties = params.iter().map(|param| {
+ let name = param.ident.unraw().to_string();
+ let ty = &param.ty;
+ let schema = quote!(<#ty as #nagare::tool::schema::ToolSchema>::json_schema());
+ if param.description.is_empty() {
+ quote!((#name, #schema))
+ } else {
+ let description = &param.description;
+ quote!((#name, #schema.with_description(#description)))
+ }
+ });
+ let required = params.iter().filter(|param| param.required).map(|param| param.ident.unraw().to_string());
+ quote! {
+ ::core::option::Option::Some(
+ #nagare::tool::schema::JsonSchema::object(
+ [#(#properties),*],
+ ::std::vec::Vec::<&str>::from([#(#required),*]),
+ )
+ .to_value(),
+ )
+ }
+}
+
+pub fn result_ok_type(ty: &Type) -> Option<&Type> {
+ let Type::Path(type_path) = ty else {
+ return None;
+ };
+ let segment = type_path.path.segments.last()?;
+ if segment.ident != "Result" {
+ return None;
+ }
+ let PathArguments::AngleBracketed(arguments) = &segment.arguments else {
+ return None;
+ };
+ arguments.args.iter().find_map(|argument| match argument {
+ GenericArgument::Type(ty) => Some(ty),
+ _ => None,
+ })
+}
+
+pub fn return_definition_tokens(
+ ok_type: Option<&Type>,
+ nagare: &TokenStream2,
+) -> TokenStream2 {
+ match ok_type {
+ Some(ty) => quote! {
+ ::core::option::Option::Some(<#ty as #nagare::tool::schema::ToolSchema>::json_schema().to_value())
+ },
+ None => quote!(::core::option::Option::None),
+ }
+}
diff --git a/crates/shoji/src/traits/backend/chat_token.rs b/crates/shoji/src/traits/backend/chat_token.rs
index f63de81ee..6901c4f9d 100644
--- a/crates/shoji/src/traits/backend/chat_token.rs
+++ b/crates/shoji/src/traits/backend/chat_token.rs
@@ -1,4 +1,4 @@
-use std::pin::Pin;
+use std::{pin::Pin, sync::Arc};
use tokenizers::Tokenizer;
@@ -30,7 +30,7 @@ pub trait Backend: Send + Sync {
&'a self,
reference: String,
config: ChatConfig,
- tokenizer: &'a Tokenizer,
+ tokenizer: Arc<Tokenizer>,
) -> Pin<Box<dyn Future<Output = Result<Box<dyn Instance>, Error>> + Send + 'a>>;
}
diff --git a/crates/shoji/src/types/basic/tool_function.rs b/crates/shoji/src/types/basic/tool_function.rs
index fdb521a69..02de8de5e 100644
--- a/crates/shoji/src/types/basic/tool_function.rs
+++ b/crates/shoji/src/types/basic/tool_function.rs
@@ -8,7 +8,8 @@ use crate::types::basic::Value;
pub struct ToolFunction {
pub name: String,
pub description: String,
+ #[serde(skip_serializing_if = "Option::is_none")]
pub parameters: Option<Value>,
- #[serde(rename = "return")]
+ #[serde(rename = "return", skip_serializing_if = "Option::is_none")]
pub return_definition: Option<Value>,
}
diff --git a/crates/shoji/src/types/session/chat/reply/config.rs b/crates/shoji/src/types/session/chat/reply/config.rs
index eb33a5b6d..6824b5c05 100644
--- a/crates/shoji/src/types/session/chat/reply/config.rs
+++ b/crates/shoji/src/types/session/chat/reply/config.rs
@@ -8,6 +8,9 @@ pub struct ChatReplyConfig {
pub token_limit: Option<u32>,
pub sampling_policy: SamplingPolicy,
pub grammar: Option<Grammar>,
+ /// Maximum number of automatic tool-call turns per reply.
+ /// `None` falls back to the session default.
+ pub tool_turn_limit: Option<u32>,
}
#[bindings::export(Implementation)]
@@ -55,6 +58,17 @@ impl ChatReplyConfig {
}
}
+ #[bindings::export(Method)]
+ pub fn with_tool_turn_limit(
+ &self,
+ tool_turn_limit: Option<u32>,
+ ) -> Self {
+ Self {
+ tool_turn_limit,
+ ..self.clone()
+ }
+ }
+
#[bindings::export(Method)]
pub fn with_grammar(
&self,
diff --git a/crates/token-stream-parser/src/extraction/parser.rs b/crates/token-stream-parser/src/extraction/parser.rs
index aa442d3f3..ec846be85 100644
--- a/crates/token-stream-parser/src/extraction/parser.rs
+++ b/crates/token-stream-parser/src/extraction/parser.rs
@@ -50,3 +50,14 @@ impl Parser for ExtractionParser {
self.tree.reset();
}
}
+
+impl ExtractionParser {
+ /// Exposes `value` to every transformation pipeline as `$<name>`.
+ pub fn set_variable(
+ &mut self,
+ name: &str,
+ value: serde_json::Value,
+ ) {
+ self.tree.set_variable(name, value);
+ }
+}
diff --git a/crates/token-stream-parser/src/extraction/resolver.rs b/crates/token-stream-parser/src/extraction/resolver.rs
index 8a51f9b77..a22037675 100644
--- a/crates/token-stream-parser/src/extraction/resolver.rs
+++ b/crates/token-stream-parser/src/extraction/resolver.rs
@@ -15,6 +15,7 @@ const PIPELINE_NAME_ROOT: &str = "root";
const PIPELINE_INPUT_NAME: &str = "$name";
const PIPELINE_INPUT_FINISHED: &str = "$finished";
const PIPELINE_INPUT_VALUE: &str = "$value";
+const PIPELINE_INPUT_CLOSE: &str = "$close";
type ExtractionParserGroupPath = Vec<usize>;
@@ -23,6 +24,7 @@ pub(crate) struct ExtractionParserResolver {
cache: HashMap<ExtractionParserGroupPath, Value>,
sections_compose_groups: HashSet<String>,
schema: Option<TransformSchema>,
+ variables: serde_json::Map<String, Value>,
}
impl ExtractionParserResolver {
@@ -34,11 +36,24 @@ impl ExtractionParserResolver {
cache: HashMap::new(),
sections_compose_groups,
schema,
+ variables: serde_json::Map::new(),
}
}
+ /// Exposes `value` to every pipeline as `$<name>`, alongside the built-in inputs.
+ pub fn set_variable(
+ &mut self,
+ name: &str,
+ value: Value,
+ ) {
+ // cached group values may have been computed with the old variable value
+ self.cache.clear();
+ self.variables.insert(format!("${name}"), value);
+ }
+
pub fn reset(&mut self) {
self.cache.clear();
+ self.variables.clear();
}
/// Compute output value from reduction state
@@ -47,7 +62,7 @@ impl ExtractionParserResolver {
&mut self,
reduction_state: &ReductionParserState,
) -> Value {
- let child_values = self.compute_children(&reduction_state.sections, &vec![]);
+ let child_values = self.compute_children(&reduction_state.sections, &vec![], None);
let root_values: Vec<Value> = child_values
.into_iter()
@@ -57,16 +72,19 @@ impl ExtractionParserResolver {
.collect();
let composed = Value::Array(root_values);
- execute_named_pipeline(PIPELINE_NAME_ROOT, false, composed, &self.schema)
+ execute_named_pipeline(PIPELINE_NAME_ROOT, false, composed, &self.schema, &self.variables, None)
}
}
impl ExtractionParserResolver {
- /// Compute values for a list of reduction sections
+ /// Compute values for a list of reduction sections.
+ /// `inherited_close` is the closing token of the nearest ancestor group: nested groups are
+ /// closed implicitly (with no token of their own) when an ancestor's close token arrives.
fn compute_children(
&mut self,
sections: &[ReductionParserSection],
parent_path: &ExtractionParserGroupPath,
+ inherited_close: Option<&str>,
) -> Vec<(String, Value)> {
let mut results = Vec::new();
@@ -79,6 +97,7 @@ impl ExtractionParserResolver {
},
ReductionParserSection::Group {
name,
+ close,
finished,
sections: group_sections,
..
@@ -86,7 +105,8 @@ impl ExtractionParserResolver {
let mut group_path = parent_path.clone();
group_path.push(section_index);
- let value = self.compute_group(name, group_sections, *finished, &group_path);
+ let close = close.as_ref().map(|token| token.value.as_str()).or(inherited_close);
+ let value = self.compute_group(name, group_sections, *finished, &group_path, close);
results.push((name.clone(), value));
},
}
@@ -103,18 +123,19 @@ impl ExtractionParserResolver {
sections: &[ReductionParserSection],
finished: bool,
path: &ExtractionParserGroupPath,
+ close: Option<&str>,
) -> Value {
if let Some(cached) = self.cache.get(path) {
tracing::trace!("cached");
return cached.clone();
}
- let child_values = self.compute_children(sections, path);
+ let child_values = self.compute_children(sections, path, close);
let sections_compose = self.sections_compose_groups.contains(name);
let composed = auto_compose(child_values, sections_compose);
- let value = execute_named_pipeline(name, finished, composed, &self.schema);
+ let value = execute_named_pipeline(name, finished, composed, &self.schema, &self.variables, close);
if finished {
tracing::trace!(group = name, "finalized");
@@ -190,6 +211,8 @@ fn execute_named_pipeline(
finished: bool,
composed_value: Value,
schema: &Option<TransformSchema>,
+ variables: &serde_json::Map<String, Value>,
+ close: Option<&str>,
) -> Value {
let Some(schema) = schema else {
return composed_value;
@@ -200,10 +223,14 @@ fn execute_named_pipeline(
let _span = tracing::info_span!("execute_named_pipeline", name = name, finished = finished,).entered();
let fallback = composed_value.clone();
- let mut pipeline_input_map = serde_json::Map::new();
+ let mut pipeline_input_map = variables.clone();
pipeline_input_map.insert(PIPELINE_INPUT_NAME.to_string(), Value::String(name.to_string()));
pipeline_input_map.insert(PIPELINE_INPUT_FINISHED.to_string(), Value::Bool(finished));
pipeline_input_map.insert(PIPELINE_INPUT_VALUE.to_string(), composed_value);
+ pipeline_input_map.insert(
+ PIPELINE_INPUT_CLOSE.to_string(),
+ close.map(|token| Value::String(token.to_string())).unwrap_or(Value::Null),
+ );
let pipeline_input = Value::Object(pipeline_input_map);
schema.execute(name, pipeline_input).unwrap_or(fallback)
}
diff --git a/crates/token-stream-parser/src/reduction/config.rs b/crates/token-stream-parser/src/reduction/config.rs
index 2a94752a6..070b86f79 100644
--- a/crates/token-stream-parser/src/reduction/config.rs
+++ b/crates/token-stream-parser/src/reduction/config.rs
@@ -12,6 +12,8 @@ pub enum ReductionParserGroup {
open_token: TokenValue,
close_tokens: Vec<TokenValue>,
#[serde(default)]
+ closes_on_sibling: bool,
+ #[serde(default)]
groups: Vec<ReductionParserGroup>,
},
Open {
diff --git a/crates/token-stream-parser/src/reduction/stack.rs b/crates/token-stream-parser/src/reduction/stack.rs
index 95aa0430b..b8ea87518 100644
--- a/crates/token-stream-parser/src/reduction/stack.rs
+++ b/crates/token-stream-parser/src/reduction/stack.rs
@@ -60,8 +60,9 @@ impl ReductionParserStackEntry {
fn closes_on_sibling(&self) -> bool {
match &self.definition {
ReductionParserGroup::Bounded {
+ closes_on_sibling,
..
- } => false,
+ } => *closes_on_sibling,
ReductionParserGroup::Open {
..
} => true,
diff --git a/crates/token-stream-parser/src/token_stream/parser.rs b/crates/token-stream-parser/src/token_stream/parser.rs
index 3b927e20c..2371f89dd 100644
--- a/crates/token-stream-parser/src/token_stream/parser.rs
+++ b/crates/token-stream-parser/src/token_stream/parser.rs
@@ -70,6 +70,15 @@ impl Parser for TokenStreamParser {
}
impl TokenStreamParser {
+ /// Exposes `value` to every transformation pipeline as `$<name>`.
+ pub fn set_variable(
+ &mut self,
+ name: &str,
+ value: serde_json::Value,
+ ) {
+ self.extraction.set_variable(name, value);
+ }
+
pub fn framing(&self) -> &FramingParser {
&self.framing
}
diff --git a/crates/token-stream-parser/tests/test_token_stream_parser_llama3.rs b/crates/token-stream-parser/tests/test_token_stream_parser_llama3.rs
index 7af20b6b5..12f95c0a1 100644
--- a/crates/token-stream-parser/tests/test_token_stream_parser_llama3.rs
+++ b/crates/token-stream-parser/tests/test_token_stream_parser_llama3.rs
@@ -191,6 +191,31 @@ fn test_token_stream_parser_llama3_multiple_tool_calls() {
);
}
+#[test]
+fn test_token_stream_parser_llama3_tool_call_with_semicolon_argument() {
+ let suite = llama3_suite();
+ let data = TestData {
+ prompt: "<|start_header_id|>user<|end_header_id|>\n\nSearch<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n".into(),
+ completion: "<|python_tag|>{\"name\": \"search\", \"parameters\": {\"query\": \"a;b\"}}<|eom_id|>".into(),
+ };
+
+ run_parser_test(
+ &suite,
+ &data,
+ &TestExpectations {
+ framing: None,
+ reduction: None,
+ extraction: suite.expect_extraction(json!([
+ {"role": "user", "content": [{"type": "$text", "value": "\n\nSearch"}]},
+ {"role": "assistant", "content": [
+ {"type": "$text", "value": "\n\n"},
+ {"type": "tool_call", "value": {"name": "search", "arguments": {"query": "a;b"}}}
+ ]}
+ ])),
+ },
+ );
+}
+
#[test]
fn test_token_stream_parser_llama3_builtin_tool_call() {
let suite = llama3_suite();
diff --git a/crates/token-stream-parser/tests/test_token_stream_parser_qwen3_5.rs b/crates/token-stream-parser/tests/test_token_stream_parser_qwen3_5.rs
new file mode 100644
index 000000000..3527777b8
--- /dev/null
+++ b/crates/token-stream-parser/tests/test_token_stream_parser_qwen3_5.rs
@@ -0,0 +1,74 @@
+mod helpers;
+
+use helpers::{TestData, TestExpectations, TestSuite, init_tracing_for_tests, run_parser_test};
+use serde_json::json;
+
+fn qwen35_suite() -> TestSuite {
+ init_tracing_for_tests();
+ TestSuite::load("qwen3.5", "Qwen_Qwen3.5-0.8B")
+}
+
+#[test]
+fn test_token_stream_parser_qwen35_tool_call_preserves_scalar_arguments() {
+ let suite = qwen35_suite();
+ let data = TestData {
+ prompt: "<|im_start|>user\nTemp?<|im_end|>\n<|im_start|>assistant\n".into(),
+ completion: "<tool_call>\n<function=get_current_temperature>\n<parameter=latitude>\n51.5074\n</parameter>\n<parameter=longitude>\n-0.1278\n</parameter>\n<parameter=days>\n3\n</parameter>\n<parameter=metric>\ntrue\n</parameter>\n<parameter=unit>\ncelsius\n</parameter>\n</function>\n</tool_call><|im_end|>".into(),
+ };
+
+ run_parser_test(
+ &suite,
+ &data,
+ &TestExpectations {
+ framing: None,
+ reduction: None,
+ extraction: suite.expect_extraction(json!([
+ {"role": "user", "content": [{"type": "$text", "value": "\nTemp?"}]},
+ {"role": "assistant", "content": [
+ {"type": "$text", "value": "\n"},
+ {"type": "tool_call", "value": {
+ "name": "get_current_temperature",
+ "arguments": {
+ "latitude": "51.5074",
+ "longitude": "-0.1278",
+ "days": "3",
+ "metric": "true",
+ "unit": "celsius"
+ }
+ }}
+ ]}
+ ])),
+ },
+ );
+}
+
+#[test]
+fn test_token_stream_parser_qwen35_preserves_ambiguous_string_arguments() {
+ let suite = qwen35_suite();
+ let data = TestData {
+ prompt: "<|im_start|>user\nLook up IDs<|im_end|>\n<|im_start|>assistant\n".into(),
+ completion: "<tool_call>\n<function=lookup>\n<parameter=nullable_id>\nnull\n</parameter>\n<parameter=padded_id>\n00123\n</parameter>\n</function>\n</tool_call><|im_end|>".into(),
+ };
+
+ run_parser_test(
+ &suite,
+ &data,
+ &TestExpectations {
+ framing: None,
+ reduction: None,
+ extraction: suite.expect_extraction(json!([
+ {"role": "user", "content": [{"type": "$text", "value": "\nLook up IDs"}]},
+ {"role": "assistant", "content": [
+ {"type": "$text", "value": "\n"},
+ {"type": "tool_call", "value": {
+ "name": "lookup",
+ "arguments": {
+ "nullable_id": "null",
+ "padded_id": "00123"
+ }
+ }}
+ ]}
+ ])),
+ },
+ );
+}
diff --git a/crates/uzu/Cargo.toml b/crates/uzu/Cargo.toml
index 85766f04b..1d8d4a96f 100644
--- a/crates/uzu/Cargo.toml
+++ b/crates/uzu/Cargo.toml
@@ -132,6 +132,10 @@ serde_json.workspace = true
mock-registry.workspace = true
rstest.workspace = true
+[[test]]
+name = "tool"
+path = "tests/tool/mod.rs"
+
[[test]]
name = "unit"
path = "tests/unit/test_entry.rs"
diff --git a/crates/uzu/examples/tool_calls.rs b/crates/uzu/examples/tool_calls.rs
new file mode 100644
index 000000000..c5d58c8fb
--- /dev/null
+++ b/crates/uzu/examples/tool_calls.rs
@@ -0,0 +1,70 @@
+use nagare::tool::{func_def::ErrorFuture, schema::UzuToolSchema, uzu_tool_closure, uzu_tool_function};
+use serde::{Deserialize, Serialize};
+use shoji::types::{
+ basic::{SamplingMethod, SamplingPolicy},
+ session::chat::{ChatConfig, ChatMessage, ChatReplyConfig},
+};
+use uzu::engine::{Engine, EngineConfig};
+
+/// A geographic coordinate.
+#[derive(Serialize, Deserialize, UzuToolSchema)]
+struct Coordinate {
+ /// Latitude in decimal degrees.
+ latitude: f64,
+ /// Longitude in decimal degrees.
+ longitude: f64,
+}
+
+/// Returns current location in coordinates
+#[uzu_tool_function]
+fn get_current_location() -> Result<Coordinate, ErrorFuture> {
+ Ok(Coordinate {
+ latitude: 51.5074,
+ longitude: -0.1278,
+ })
+}
+
+#[tokio::main]
+async fn main() -> Result<(), Box<dyn std::error::Error>> {
+ let engine = Engine::new(EngineConfig::default()).await?;
+ let model = engine.model("mlx-community/Qwen3.5-9B-MLX-8bit".to_string()).await?.ok_or("Model not found")?;
+ let downloader = engine.download(&model).await?;
+ while let Some(update) = downloader.next().await {
+ println!("Download progress: {}", update.progress());
+ }
+
+ let mut session = engine.chat(model, ChatConfig::default()).await?;
+ session.add_tool_function(get_current_location).await?;
+ session
+ .add_tool_descriptor(uzu_tool_closure! {
+ /// Returns temperature in provided location
+ get_current_temperature: |
+ /// Latitude in decimal degrees.
+ _latitude: f64,
+ /// Longitude in decimal degrees.
+ _longitude: f64
+ | -> Result<f64, ErrorFuture> {
+ Ok(25.0)
+ }
+ })
+ .await?;
+
+ let messages = vec![
+ ChatMessage::system().with_text("You are a helpful assistant".to_string()),
+ ChatMessage::user().with_text("What temperature is it now at my location?".to_string()),
+ ];
+
+ let config = ChatReplyConfig {
+ sampling_policy: SamplingPolicy::Custom {
+ method: SamplingMethod::Greedy {},
+ },
+ ..ChatReplyConfig::default()
+ };
+ let replies = session.reply(messages.clone(), config).await?;
+ if let Some(reply) = replies.last() {
+ println!("Reasoning: {}", reply.message.reasoning().unwrap_or_default());
+ println!("Text: {}", reply.message.text().unwrap_or_default());
+ }
+
+ Ok(())
+}
diff --git a/crates/uzu/tests/tool/calls.rs b/crates/uzu/tests/tool/calls.rs
new file mode 100644
index 000000000..9cea90081
--- /dev/null
+++ b/crates/uzu/tests/tool/calls.rs
@@ -0,0 +1,245 @@
+#![cfg(not(target_family = "wasm"))]
+
+use serde::Serialize;
+use uzu::{
+ engine::{Engine, EngineConfig},
+ session::tool::{func_def::ToolDescriptor, schema::UzuToolSchema, uzu_tool_function},
+ types::{
+ basic::{SamplingMethod, SamplingPolicy},
+ session::chat::{ChatConfig, ChatMessage, ChatReplyConfig, ChatReplyFinishReason, ChatRole},
+ },
+};
+
+/// A geographic coordinate.
+#[derive(Serialize, UzuToolSchema)]
+struct Coordinate {
+ /// Latitude in decimal degrees.
+ latitude: f64,
+ /// Longitude in decimal degrees.
+ longitude: f64,
+}
+
+/// The current temperature at the requested location.
+#[derive(Serialize, UzuToolSchema)]
+struct Temperature {
+ /// Current temperature in degrees Celsius.
+ temperature: f64,
+}
+
+/// The current time.
+#[derive(Serialize, UzuToolSchema)]
+struct Time {
+ /// Current time.
+ time: String,
+}
+
+/// Get the user's current geographic location as latitude and longitude coordinates.
+#[uzu_tool_function]
+fn get_current_location() -> Coordinate {
+ Coordinate {
+ latitude: 59.938784,
+ longitude: 30.314997,
+ }
+}
+
+/// Get the current temperature at the given geographic coordinates.
+#[uzu_tool_function]
+fn get_current_temperature(
+ /// Latitude in decimal degrees.
+ latitude: f64,
+ /// Longitude in decimal degrees.
+ longitude: f64,
+) -> Temperature {
+ let _ = (latitude, longitude);
+ Temperature {
+ temperature: 25.0,
+ }
+}
+
+/// Get the current time in 24h format.
+#[uzu_tool_function]
+fn get_current_time() -> Time {
+ Time {
+ time: "17:03".to_string(),
+ }
+}
+
+struct TestCase {
+ /// User prompt sent to the model.
+ prompt: &'static str,
+ /// Tools registered for this case.
+ tools: &'static [fn() -> ToolDescriptor],
+ /// Tools that must be called (and produce results) during the turn.
+ expected_tools: &'static [&'static str],
+ /// Fragments that must appear in the final reply text.
+ expected_fragments: &'static [&'static str],
+}
+
+/// Inputs and expected outputs that every model must pass.
+const TEST_CASES: &[TestCase] = &[
+ TestCase {
+ prompt: "What is the time now in 24-hour format?",
+ tools: &[get_current_time::definition],
+ expected_tools: &["get_current_time"],
+ expected_fragments: &["17:03"],
+ },
+ TestCase {
+ prompt: "What is the temperature at my current location?",
+ tools: &[get_current_location::definition, get_current_temperature::definition],
+ expected_tools: &["get_current_location", "get_current_temperature"],
+ // "25" instead of "25.9": some models round the tool result when phrasing the reply
+ // (gpt-oss-20b answers "25 °C"), and "25" still matches replies quoting "25.9" exactly.
+ expected_fragments: &["25"],
+ },
+ TestCase {
+ prompt: "What time is it now in 24-hour format and what is the temperature at my current location?",
+ tools: &[get_current_time::definition, get_current_location::definition, get_current_temperature::definition],
+ expected_tools: &["get_current_time", "get_current_location", "get_current_temperature"],
+ expected_fragments: &["17:03", "25"],
+ },
+];
+
+async fn run_tool_calls_test(
+ model_id: &str,
+ with_system_message: bool,
+ single_tool_call_per_turn: bool,
+ cases: &[TestCase],
+) {
+ let engine = Engine::new(EngineConfig::default()).await.unwrap();
+ let model =
+ engine.model(model_id.to_string()).await.unwrap().unwrap_or_else(|| panic!("Model {model_id} not found"));
+ println!("Loading {model_id}...");
+ let download = engine.download(&model).await.unwrap();
+ while download.next().await.is_some() {}
+
+ for case in cases {
+ println!("Prompt: {}", case.prompt);
+
+ let mut session = engine.chat(model.clone(), ChatConfig::default()).await.unwrap();
+ session
+ .add_tool_function_definitions(case.tools.iter().map(|definition| definition()).collect())
+ .await
+ .unwrap();
+
+ let mut messages = Vec::new();
+ if with_system_message {
+ messages.push(ChatMessage::system().with_text("You are a helpful assistant".to_string()));
+ }
+ messages.push(ChatMessage::user().with_text(case.prompt.to_string()));
+
+ // Greedy sampling keeps these tests deterministic; with the default stochastic sampling
+ // the models occasionally skip a tool call, hallucinate that part of the answer, or
+ // finish without any text in the final reply.
+ let reply_config = ChatReplyConfig {
+ sampling_policy: SamplingPolicy::Custom {
+ method: SamplingMethod::Greedy {},
+ },
+ ..ChatReplyConfig::default()
+ };
+ let replies = session.reply(messages, reply_config).await.unwrap();
+ let final_reply = replies.last().expect("Expected at least one reply");
+ assert_eq!(
+ final_reply.finish_reason,
+ Some(ChatReplyFinishReason::Stop),
+ "Expected the final reply to finish with Stop for prompt {:?}",
+ case.prompt
+ );
+ let final_text = final_reply.message.text().unwrap_or_default();
+ assert!(!final_text.is_empty(), "Expected the final reply to contain text for prompt {:?}", case.prompt);
+ for fragment in case.expected_fragments {
+ assert!(
+ final_text.contains(fragment),
+ "Expected the final reply for prompt {:?} to contain {fragment:?}, got: {final_text:?}",
+ case.prompt
+ );
+ }
+
+ let history = session.messages().await;
+ let tool_calls = history
+ .iter()
+ .filter(|message| matches!(message.role, ChatRole::Assistant {}))
+ .flat_map(|message| message.tool_calls())
+ .collect::<Vec<_>>();
+ let tool_call_results = history
+ .iter()
+ .filter(|message| matches!(message.role, ChatRole::Tool {}))
+ .flat_map(|message| message.tool_call_results())
+ .collect::<Vec<_>>();
+
+ let called_names = tool_calls.iter().map(|call| call.name.as_str()).collect::<Vec<_>>();
+ assert!(!tool_calls.is_empty(), "Expected at least one tool call for prompt {:?}", case.prompt);
+ assert!(!tool_call_results.is_empty(), "Expected at least one tool call result for prompt {:?}", case.prompt);
+ for expected in case.expected_tools {
+ assert!(
+ called_names.contains(expected),
+ "Expected tool {expected} to be called for prompt {:?}, called tools: {called_names:?}",
+ case.prompt
+ );
+ assert!(
+ tool_call_results.iter().any(|(_, name, _)| name.as_deref() == Some(*expected)),
+ "Expected a result for tool {expected} for prompt {:?}",
+ case.prompt
+ );
+ }
+
+ if single_tool_call_per_turn {
+ for message in history.iter().filter(|message| matches!(message.role, ChatRole::Assistant {})) {
+ assert!(
+ message.tool_calls().len() <= 1,
+ "Expected at most one tool call per assistant turn for prompt {:?}, got {:?}",
+ case.prompt,
+ message.tool_calls().iter().map(|call| call.name.clone()).collect::<Vec<_>>()
+ );
+ }
+ }
+ }
+}
+
+#[ignore]
+#[tokio::test]
+async fn functiongemma_270m_it() {
+ // FunctionGemma 270M only handles single-step tool calls out of the box:
+ // for the prompts that require chaining (get_current_location -> get_current_temperature) it invents coordinates or
+ // asks the user for them instead of calling get_current_location first.
+ // Per the model card, multi-step use cases require task-specific fine-tuning.
+ run_tool_calls_test("google/functiongemma-270m-it", false, true, &TEST_CASES[..1]).await;
+}
+
+#[ignore]
+#[tokio::test]
+async fn gpt_oss_20b() {
+ run_tool_calls_test("openai/gpt-oss-20b", true, false, TEST_CASES).await;
+}
+
+#[ignore]
+#[tokio::test]
+async fn lfm2_350m() {
+ run_tool_calls_test("LiquidAI/LFM2-350M", true, false, &TEST_CASES[..1]).await;
+}
+
+#[ignore]
+#[tokio::test]
+async fn lfm2_5_350m() {
+ run_tool_calls_test("LiquidAI/LFM2.5-350M", true, false, &TEST_CASES[..1]).await;
+}
+
+#[ignore]
+#[tokio::test]
+async fn llama_3_2_1b_instruct() {
+ // Like FunctionGemma, Llama 3.2 1B only handles single-step tool calls: for the prompts that
+ // require chaining (get_current_location -> get_current_temperature) it invents coordinates
+ // (e.g. latitude "37") instead of calling get_current_location first.
+ run_tool_calls_test("meta-llama/Llama-3.2-1B-Instruct", true, false, &TEST_CASES[..1]).await;
+}
+
+#[ignore]
+#[tokio::test]
+async fn qwen3_1_7b() {
+ run_tool_calls_test("Qwen/Qwen3-1.7B", true, false, TEST_CASES).await;
+}
+
+#[ignore]
+#[tokio::test]
+async fn qwen3_5_0_8b() {
+ run_tool_calls_test("Qwen/Qwen3.5-0.8B", true, false, TEST_CASES).await;
+}
diff --git a/crates/uzu/tests/tool/macro_closure.rs b/crates/uzu/tests/tool/macro_closure.rs
new file mode 100644
index 000000000..456c793cb
--- /dev/null
+++ b/crates/uzu/tests/tool/macro_closure.rs
@@ -0,0 +1,166 @@
+#![cfg(not(target_family = "wasm"))]
+
+use std::sync::{
+ Arc,
+ atomic::{AtomicI64, Ordering},
+};
+
+use nagare::tool::{func_def::ToolDescriptor, uzu_tool_closure};
+
+#[tokio::test]
+async fn closure_with_captured_state() {
+ let counter = Arc::new(AtomicI64::new(0));
+ let captured = counter.clone();
+ let definition: ToolDescriptor = uzu_tool_closure! {
+ /// Add an amount to the running total.
+ accumulate: |
+ /// Amount to add to the total.
+ amount: i64,
+ | -> i64 {
+ captured.fetch_add(amount, Ordering::SeqCst) + amount
+ }
+ };
+
+ assert_eq!(definition.name, "accumulate");
+ assert_eq!(definition.description, "Add an amount to the running total.");
+
+ let parameters: serde_json::Value = serde_json::from_str(&definition.parameters.as_ref().unwrap().json).unwrap();
+ assert_eq!(
+ parameters,
+ serde_json::json!({
+ "type": "object",
+ "properties": {
+ "amount": { "type": "integer", "description": "Amount to add to the total." }
+ },
+ "required": ["amount"]
+ })
+ );
+
+ let return_definition: serde_json::Value =
+ serde_json::from_str(&definition.return_definition.as_ref().unwrap().json).unwrap();
+ assert_eq!(return_definition, serde_json::json!({ "type": "integer" }));
+
+ let result: serde_json::Value =
+ definition.execute(serde_json::json!({ "amount": 40 }).into()).await.unwrap().try_into().unwrap();
+ assert_eq!(result, serde_json::json!(40));
+
+ let result: serde_json::Value =
+ definition.execute(serde_json::json!({ "amount": 2 }).into()).await.unwrap().try_into().unwrap();
+ assert_eq!(result, serde_json::json!(42));
+
+ assert_eq!(counter.load(Ordering::SeqCst), 42);
+
+ let error = definition.execute(serde_json::json!({ "amount": "oops" }).into()).await.unwrap_err();
+ assert!(error.to_string().contains("amount"), "unexpected error: {error}");
+}
+
+#[tokio::test]
+async fn async_closure_with_optional_parameter() {
+ let greeting = String::from("Hello");
+ let definition = uzu_tool_closure! {
+ /// Greet the given person.
+ greet: async |
+ /// Name of the person to greet.
+ name: String,
+ /// Optional punctuation to end the greeting with.
+ punctuation: Option<String>,
+ | -> String {
+ let punctuation = punctuation.unwrap_or_else(|| "!".to_string());
+ format!("{greeting}, {name}{punctuation}")
+ }
+ };
+
+ assert_eq!(definition.name, "greet");
+ assert_eq!(definition.description, "Greet the given person.");
+
+ let parameters: serde_json::Value = serde_json::from_str(&definition.parameters.as_ref().unwrap().json).unwrap();
+ assert_eq!(
+ parameters,
+ serde_json::json!({
+ "type": "object",
+ "properties": {
+ "name": { "type": "string", "description": "Name of the person to greet." },
+ "punctuation": {
+ "type": "string",
+ "description": "Optional punctuation to end the greeting with."
+ }
+ },
+ "required": ["name"]
+ })
+ );
+
+ let result: serde_json::Value =
+ definition.execute(serde_json::json!({ "name": "Ada" }).into()).await.unwrap().try_into().unwrap();
+ assert_eq!(result, serde_json::json!("Hello, Ada!"));
+
+ let result: serde_json::Value = definition
+ .execute(serde_json::json!({ "name": "Ada", "punctuation": "?" }).into())
+ .await
+ .unwrap()
+ .try_into()
+ .unwrap();
+ assert_eq!(result, serde_json::json!("Hello, Ada?"));
+}
+
+#[tokio::test]
+async fn result_closure_propagates_errors() {
+ let definition = uzu_tool_closure! {
+ /// Divide the dividend by the divisor.
+ divide: |dividend: f64, divisor: f64| -> Result<f64, Box<dyn std::error::Error + Send + Sync>> {
+ if divisor == 0.0 {
+ Err("division by zero".into())
+ } else {
+ Ok(dividend / divisor)
+ }
+ }
+ };
+
+ let return_definition: serde_json::Value =
+ serde_json::from_str(&definition.return_definition.as_ref().unwrap().json).unwrap();
+ assert_eq!(return_definition, serde_json::json!({ "type": "number" }));
+
+ let result: serde_json::Value = definition
+ .execute(serde_json::json!({ "dividend": 10.0, "divisor": 4.0 }).into())
+ .await
+ .unwrap()
+ .try_into()
+ .unwrap();
+ assert_eq!(result, serde_json::json!(2.5));
+
+ let error = definition.execute(serde_json::json!({ "dividend": 1.0, "divisor": 0.0 }).into()).await.unwrap_err();
+ assert_eq!(error.to_string(), "division by zero");
+}
+
+#[tokio::test]
+async fn closure_without_parameters_or_annotation() {
+ let definition = uzu_tool_closure! {
+ /// Report the answer.
+ answer: || 42
+ };
+
+ assert_eq!(definition.name, "answer");
+ assert!(definition.parameters.is_none());
+ assert!(definition.return_definition.is_none());
+
+ let result: serde_json::Value = definition.execute(serde_json::json!({}).into()).await.unwrap().try_into().unwrap();
+ assert_eq!(result, serde_json::json!(42));
+}
+
+#[tokio::test]
+async fn unit_closure_returns_null() {
+ let flag = Arc::new(AtomicI64::new(0));
+ let captured = flag.clone();
+ let definition = uzu_tool_closure! {
+ /// Reset the session state.
+ reset: || {
+ captured.store(1, Ordering::SeqCst);
+ }
+ };
+
+ assert!(definition.parameters.is_none());
+ assert!(definition.return_definition.is_none());
+
+ let result: serde_json::Value = definition.execute(serde_json::json!({}).into()).await.unwrap().try_into().unwrap();
+ assert_eq!(result, serde_json::Value::Null);
+ assert_eq!(flag.load(Ordering::SeqCst), 1);
+}
diff --git a/crates/uzu/tests/tool/macro_function.rs b/crates/uzu/tests/tool/macro_function.rs
new file mode 100644
index 000000000..c62bb0f04
--- /dev/null
+++ b/crates/uzu/tests/tool/macro_function.rs
@@ -0,0 +1,427 @@
+#![cfg(not(target_family = "wasm"))]
+
+use nagare::tool::{func_def::ToolDescriptor, schema::UzuToolSchema, uzu_tool_function};
+use serde::{Deserialize, Serialize};
+
+/// The current weather at the requested location.
+#[derive(Serialize, UzuToolSchema)]
+struct Weather {
+ /// Temperature in degrees Celsius.
+ temperature: f64,
+ /// Optional human-readable summary.
+ summary: Option<String>,
+}
+
+/// Get the current weather for the given geographic coordinates.
+#[uzu_tool_function]
+fn get_weather(
+ /// Latitude in decimal degrees.
+ latitude: f64,
+ /// Longitude in decimal degrees.
+ longitude: f64,
+) -> Result<Weather, Box<dyn std::error::Error + Send + Sync>> {
+ Ok(Weather {
+ temperature: latitude + longitude,
+ summary: None,
+ })
+}
+
+/// An async tool without parameters.
+#[uzu_tool_function]
+async fn current_time() -> String {
+ "17:03".to_string()
+}
+
+#[tokio::test]
+async fn check_generated_definition() {
+ let definition: ToolDescriptor = get_weather.into();
+ assert_eq!(definition.name, "get_weather");
+ assert_eq!(definition.description, "Get the current weather for the given geographic coordinates.");
+
+ let parameters: serde_json::Value = serde_json::from_str(&definition.parameters.as_ref().unwrap().json).unwrap();
+ assert_eq!(
+ parameters,
+ serde_json::json!({
+ "type": "object",
+ "properties": {
+ "latitude": { "type": "number", "description": "Latitude in decimal degrees." },
+ "longitude": { "type": "number", "description": "Longitude in decimal degrees." }
+ },
+ "required": ["latitude", "longitude"]
+ })
+ );
+
+ let return_definition: serde_json::Value =
+ serde_json::from_str(&definition.return_definition.as_ref().unwrap().json).unwrap();
+ assert_eq!(
+ return_definition,
+ serde_json::json!({
+ "type": "object",
+ "properties": {
+ "temperature": { "type": "number", "description": "Temperature in degrees Celsius." },
+ "summary": { "type": "string", "description": "Optional human-readable summary." }
+ },
+ "required": ["temperature"],
+ "description": "The current weather at the requested location."
+ })
+ );
+
+ let result = definition.execute(serde_json::json!({ "latitude": 10.5, "longitude": 20.25 }).into()).await.unwrap();
+ let result: serde_json::Value = result.try_into().unwrap();
+ assert_eq!(result, serde_json::json!({ "temperature": 30.75, "summary": null }));
+
+ let error = definition.execute(serde_json::json!({ "latitude": "oops" }).into()).await.unwrap_err();
+ assert!(error.to_string().contains("latitude"), "unexpected error: {error}");
+
+ let time: ToolDescriptor = current_time.into();
+ assert_eq!(time.name, "current_time");
+ assert!(time.parameters.is_none());
+ let time_schema: serde_json::Value = serde_json::from_str(&time.return_definition.as_ref().unwrap().json).unwrap();
+ assert_eq!(time_schema, serde_json::json!({ "type": "string" }));
+ let result: serde_json::Value = time.execute(serde_json::json!({}).into()).await.unwrap().try_into().unwrap();
+ assert_eq!(result, serde_json::json!("17:03"));
+}
+
+/// A geographic coordinate.
+#[derive(Serialize, Deserialize, UzuToolSchema)]
+struct Coordinate {
+ /// Latitude in decimal degrees.
+ latitude: f64,
+ /// Longitude in decimal degrees.
+ longitude: f64,
+}
+
+/// A weather forecast request.
+#[derive(Deserialize, UzuToolSchema)]
+struct ForecastRequest {
+ /// Location to forecast.
+ location: Coordinate,
+ /// Number of days ahead.
+ days: u32,
+}
+
+/// The forecast for a single day.
+#[derive(Serialize, UzuToolSchema)]
+struct DayForecast {
+ /// Temperature in degrees Celsius.
+ temperature: f64,
+ /// Chance of rain from 0 to 1.
+ rain_probability: f64,
+}
+
+/// A forecast for the requested location.
+#[derive(Serialize, UzuToolSchema)]
+struct Forecast {
+ location: Coordinate,
+ /// Forecasts per day.
+ days: Vec<DayForecast>,
+}
+
+/// Get the weather forecast for a location.
+#[uzu_tool_function]
+fn get_forecast(request: ForecastRequest) -> Forecast {
+ Forecast {
+ location: request.location,
+ days: (0..request.days)
+ .map(|day| DayForecast {
+ temperature: 20.0 + day as f64,
+ rain_probability: 0.25,
+ })
+ .collect(),
+ }
+}
+
+#[tokio::test]
+async fn structured_input_and_output() {
+ let definition: ToolDescriptor = get_forecast.into();
+ assert_eq!(definition.name, "get_forecast");
+ assert_eq!(definition.description, "Get the weather forecast for a location.");
+
+ let coordinate_properties = serde_json::json!({
+ "latitude": { "type": "number", "description": "Latitude in decimal degrees." },
+ "longitude": { "type": "number", "description": "Longitude in decimal degrees." }
+ });
+
+ let parameters: serde_json::Value = serde_json::from_str(&definition.parameters.as_ref().unwrap().json).unwrap();
+ assert_eq!(
+ parameters,
+ serde_json::json!({
+ "type": "object",
+ "properties": {
+ "request": {
+ "type": "object",
+ "properties": {
+ "location": {
+ "type": "object",
+ "properties": coordinate_properties,
+ "required": ["latitude", "longitude"],
+ // The field doc overrides the struct-level description of `Coordinate`.
+ "description": "Location to forecast."
+ },
+ "days": { "type": "integer", "description": "Number of days ahead." }
+ },
+ "required": ["location", "days"],
+ "description": "A weather forecast request."
+ }
+ },
+ "required": ["request"]
+ })
+ );
+
+ let return_definition: serde_json::Value =
+ serde_json::from_str(&definition.return_definition.as_ref().unwrap().json).unwrap();
+ assert_eq!(
+ return_definition,
+ serde_json::json!({
+ "type": "object",
+ "properties": {
+ "location": {
+ "type": "object",
+ "properties": coordinate_properties,
+ "required": ["latitude", "longitude"],
+ // No field doc, so the struct-level description of `Coordinate` is kept.
+ "description": "A geographic coordinate."
+ },
+ "days": {
+ "type": "array",
+ "items": {
+ "type": "object",
+ "properties": {
+ "temperature": { "type": "number", "description": "Temperature in degrees Celsius." },
+ "rain_probability": { "type": "number", "description": "Chance of rain from 0 to 1." }
+ },
+ "required": ["temperature", "rain_probability"],
+ "description": "The forecast for a single day."
+ },
+ "description": "Forecasts per day."
+ }
+ },
+ "required": ["location", "days"],
+ "description": "A forecast for the requested location."
+ })
+ );
+
+ let result = definition
+ .execute(
+ serde_json::json!({
+ "request": {
+ "location": { "latitude": 59.94, "longitude": 30.31 },
+ "days": 2
+ }
+ })
+ .into(),
+ )
+ .await
+ .unwrap();
+ let result: serde_json::Value = result.try_into().unwrap();
+ assert_eq!(
+ result,
+ serde_json::json!({
+ "location": { "latitude": 59.94, "longitude": 30.31 },
+ "days": [
+ { "temperature": 20.0, "rain_probability": 0.25 },
+ { "temperature": 21.0, "rain_probability": 0.25 }
+ ]
+ })
+ );
+
+ let error = definition
+ .execute(serde_json::json!({ "request": { "location": { "latitude": 59.94 }, "days": 2 } }).into())
+ .await
+ .unwrap_err();
+ assert!(error.to_string().contains("request"), "unexpected error: {error}");
+}
+
+#[derive(Serialize, Deserialize, UzuToolSchema)]
+#[serde(rename_all = "camelCase")]
+struct UserLookup {
+ #[serde(rename = "id")]
+ user_id: String,
+ display_name: String,
+ #[serde(default)]
+ max_results: u32,
+ #[serde(skip)]
+ internal_note: String,
+}
+
+#[uzu_tool_function]
+fn lookup_user(request: UserLookup) -> String {
+ let _ = &request.internal_note;
+ format!("{}:{}:{}", request.user_id, request.display_name, request.max_results)
+}
+
+#[tokio::test]
+async fn serde_field_attributes_match_generated_schema() {
+ let definition: ToolDescriptor = lookup_user.into();
+ let parameters: serde_json::Value = serde_json::from_str(&definition.parameters.as_ref().unwrap().json).unwrap();
+
+ assert_eq!(
+ parameters,
+ serde_json::json!({
+ "type": "object",
+ "properties": {
+ "request": {
+ "type": "object",
+ "properties": {
+ "id": { "type": "string" },
+ "displayName": { "type": "string" },
+ "maxResults": { "type": "integer" }
+ },
+ "required": ["id", "displayName"]
+ }
+ },
+ "required": ["request"]
+ })
+ );
+
+ let result: serde_json::Value = definition
+ .execute(
+ serde_json::json!({
+ "request": {
+ "id": "00123",
+ "displayName": "Ada"
+ }
+ })
+ .into(),
+ )
+ .await
+ .unwrap()
+ .try_into()
+ .unwrap();
+ assert_eq!(result, serde_json::json!("00123:Ada:0"));
+}
+
+#[derive(Deserialize, UzuToolSchema)]
+#[serde(deny_unknown_fields)]
+struct StrictRequest {
+ query: String,
+}
+
+#[uzu_tool_function]
+fn strict_lookup(request: StrictRequest) -> String {
+ request.query
+}
+
+#[tokio::test]
+async fn deny_unknown_fields_disallows_additional_schema_properties() {
+ let definition: ToolDescriptor = strict_lookup.into();
+ let parameters: serde_json::Value = serde_json::from_str(&definition.parameters.as_ref().unwrap().json).unwrap();
+
+ assert_eq!(
+ parameters,
+ serde_json::json!({
+ "type": "object",
+ "properties": {
+ "request": {
+ "type": "object",
+ "properties": {
+ "query": { "type": "string" }
+ },
+ "required": ["query"],
+ "additionalProperties": false
+ }
+ },
+ "required": ["request"]
+ })
+ );
+
+ let error = definition
+ .execute(serde_json::json!({ "request": { "query": "Ada", "unexpected": true } }).into())
+ .await
+ .unwrap_err();
+ assert!(error.to_string().contains("unknown field"), "unexpected error: {error}");
+}
+
+/// Add two integers.
+#[uzu_tool_function]
+fn add(
+ /// The first addend.
+ a: i64,
+ /// The second addend.
+ b: i64,
+) -> i64 {
+ a + b
+}
+
+#[tokio::test]
+async fn primitive_input_and_output() {
+ let definition: ToolDescriptor = add.into();
+ assert_eq!(definition.name, "add");
+ assert_eq!(definition.description, "Add two integers.");
+
+ let parameters: serde_json::Value = serde_json::from_str(&definition.parameters.as_ref().unwrap().json).unwrap();
+ assert_eq!(
+ parameters,
+ serde_json::json!({
+ "type": "object",
+ "properties": {
+ "a": { "type": "integer", "description": "The first addend." },
+ "b": { "type": "integer", "description": "The second addend." }
+ },
+ "required": ["a", "b"]
+ })
+ );
+
+ let return_definition: serde_json::Value =
+ serde_json::from_str(&definition.return_definition.as_ref().unwrap().json).unwrap();
+ assert_eq!(return_definition, serde_json::json!({ "type": "integer" }));
+
+ let result: serde_json::Value =
+ definition.execute(serde_json::json!({ "a": 40, "b": 2 }).into()).await.unwrap().try_into().unwrap();
+ assert_eq!(result, serde_json::json!(42));
+}
+
+/// Reset the session state.
+#[uzu_tool_function]
+fn reset() {}
+
+#[tokio::test]
+async fn void_input_and_output() {
+ let definition: ToolDescriptor = reset.into();
+ assert_eq!(definition.name, "reset");
+ assert_eq!(definition.description, "Reset the session state.");
+ assert!(definition.parameters.is_none());
+ assert!(definition.return_definition.is_none());
+
+ let result: serde_json::Value = definition.execute(serde_json::json!({}).into()).await.unwrap().try_into().unwrap();
+ assert_eq!(result, serde_json::Value::Null);
+}
+
+/// Clear all stored data.
+#[uzu_tool_function]
+fn clear_data(
+ /// Must be true to confirm the deletion.
+ confirm: bool
+) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
+ if confirm {
+ Ok(())
+ } else {
+ Err("confirmation required".into())
+ }
+}
+
+#[tokio::test]
+async fn void_result_and_error_propagation() {
+ let definition: ToolDescriptor = clear_data.into();
+ assert_eq!(definition.name, "clear_data");
+
+ let parameters: serde_json::Value = serde_json::from_str(&definition.parameters.as_ref().unwrap().json).unwrap();
+ assert_eq!(
+ parameters,
+ serde_json::json!({
+ "type": "object",
+ "properties": {
+ "confirm": { "type": "boolean", "description": "Must be true to confirm the deletion." }
+ },
+ "required": ["confirm"]
+ })
+ );
+ assert!(definition.return_definition.is_none());
+
+ let result: serde_json::Value =
+ definition.execute(serde_json::json!({ "confirm": true }).into()).await.unwrap().try_into().unwrap();
+ assert_eq!(result, serde_json::Value::Null);
+
+ let error = definition.execute(serde_json::json!({ "confirm": false }).into()).await.unwrap_err();
+ assert_eq!(error.to_string(), "confirmation required");
+}
diff --git a/crates/uzu/tests/tool/mod.rs b/crates/uzu/tests/tool/mod.rs
new file mode 100644
index 000000000..a39af35c9
--- /dev/null
+++ b/crates/uzu/tests/tool/mod.rs
@@ -0,0 +1,3 @@
+mod calls;
+mod macro_closure;
+mod macro_function;