A high-performance inference engine for AI models
Execute tool calls concurrently (#682)
uuuvne4d9580bb3eb
diff --git a/crates/nagare/src/chat/mod.rs b/crates/nagare/src/chat/mod.rsindex c58761699..8a169e037 100644--- a/crates/nagare/src/chat/mod.rs+++ b/crates/nagare/src/chat/mod.rs@@ -455,8 +455,7 @@ impl ChatSession {return vec![];};- let mut tool_messages: Vec<ChatMessage> = Vec::with_capacity(tool_calls.len());- for call in tool_calls {+ let executions = tool_calls.into_iter().map(|call| async move {let func = {let registry_guard = registry.lock().await;registry_guard.get_function(&call.name).cloned()@@ -479,14 +478,14 @@ impl ChatSession {};// chat templates expect tool results as plain text, one message per call- let tool_message = ChatMessage::tool().with_block(ChatContentBlock::ToolCallResult {+ ChatMessage::tool().with_block(ChatContentBlock::ToolCallResult {identifier: call.identifier.clone(),name: Some(call.name.clone()),value,- });- tool_messages.push(tool_message);- }- tool_messages+ })+ });+ // results stay in call order even though the calls run concurrently+ futures::future::join_all(executions).await}/// Atomically switches state from `from` to `to`.diff --git a/crates/nagare/src/tool/func_def.rs b/crates/nagare/src/tool/func_def.rsindex 9126303fa..93f87f694 100644--- a/crates/nagare/src/tool/func_def.rs+++ b/crates/nagare/src/tool/func_def.rs@@ -82,6 +82,12 @@ pub fn null_result() -> Value {Value::from(serde_json::Value::Null)}+/// Runs a synchronous tool on the blocking thread pool so heavy work+/// doesn't stall the async runtime.+pub async fn run_blocking<T: Send + 'static>(func: impl FnOnce() -> T + Send + 'static) -> Result<T, ErrorFuture> {+ tokio::task::spawn_blocking(func).await.map_err(|error| -> ErrorFuture { error.into() })+}+fn coerce_to_schema(value: serde_json::Value,schema: &serde_json::Value,diff --git a/crates/proc-macros/src/uzu_tool/closure.rs b/crates/proc-macros/src/uzu_tool/closure.rsindex c04a37e5e..5c563c899 100644--- a/crates/proc-macros/src/uzu_tool/closure.rs+++ b/crates/proc-macros/src/uzu_tool/closure.rs@@ -119,10 +119,11 @@ fn expand_tool_closure(tool: ToolClosure) -> syn::Result<TokenStream2> {};let arg_idents = params.iter().map(|param| ¶m.ident);- let mut call = quote!(__uzu_tool_func(#(#arg_idents),*));- if is_async {- call = quote!(#call.await);- }+ let call = if is_async {+ quote!(__uzu_tool_func(#(#arg_idents),*).await)+ } else {+ quote!(#nagare::tool::func_def::run_blocking(move || __uzu_tool_func(#(#arg_idents),*)).await?)+ };let result_stmt = match &return_type {Some(ty) => quote!(let __uzu_result: #ty = #call;),None => quote!(let __uzu_result = #call;),diff --git a/crates/proc-macros/src/uzu_tool/function.rs b/crates/proc-macros/src/uzu_tool/function.rsindex 09d24d7c0..d679fa951 100644--- a/crates/proc-macros/src/uzu_tool/function.rs+++ b/crates/proc-macros/src/uzu_tool/function.rs@@ -73,10 +73,11 @@ fn expand_tool_function(mut func: ItemFn) -> syn::Result<TokenStream2> {let arg_parsing = arg_parsing_tokens(¶ms, &name_str, &nagare);let arg_idents = params.iter().map(|param| ¶m.ident);- let mut call = quote!(Self::call(#(#arg_idents),*));- if is_async {- call = quote!(#call.await);- }+ let call = if is_async {+ quote!(Self::call(#(#arg_idents),*).await)+ } else {+ quote!(#nagare::tool::func_def::run_blocking(move || Self::call(#(#arg_idents),*)).await?)+ };let map_err = quote!(.map_err(|error| -> #nagare::tool::func_def::ErrorFuture { error.into() })?);let call_stmt = match (&ok_type, is_result) {(Some(_), true) => quote!(let result = #call #map_err;),