Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

Execute tool calls concurrently (#682)

uuuvne4d9580bb3eb
diff --git a/crates/nagare/src/chat/mod.rs b/crates/nagare/src/chat/mod.rs
index c58761699..8a169e037 100644
--- a/crates/nagare/src/chat/mod.rs
+++ b/crates/nagare/src/chat/mod.rs
@@ -455,8 +455,7 @@ impl ChatSession {
return vec![];
};
- let mut tool_messages: Vec<ChatMessage> = Vec::with_capacity(tool_calls.len());
- for call in tool_calls {
+ let executions = tool_calls.into_iter().map(|call| async move {
let func = {
let registry_guard = registry.lock().await;
registry_guard.get_function(&call.name).cloned()
@@ -479,14 +478,14 @@ impl ChatSession {
};
// chat templates expect tool results as plain text, one message per call
- let tool_message = ChatMessage::tool().with_block(ChatContentBlock::ToolCallResult {
+ ChatMessage::tool().with_block(ChatContentBlock::ToolCallResult {
identifier: call.identifier.clone(),
name: Some(call.name.clone()),
value,
- });
- tool_messages.push(tool_message);
- }
- tool_messages
+ })
+ });
+ // results stay in call order even though the calls run concurrently
+ futures::future::join_all(executions).await
}
/// Atomically switches state from `from` to `to`.
diff --git a/crates/nagare/src/tool/func_def.rs b/crates/nagare/src/tool/func_def.rs
index 9126303fa..93f87f694 100644
--- a/crates/nagare/src/tool/func_def.rs
+++ b/crates/nagare/src/tool/func_def.rs
@@ -82,6 +82,12 @@ pub fn null_result() -> Value {
Value::from(serde_json::Value::Null)
}
+/// Runs a synchronous tool on the blocking thread pool so heavy work
+/// doesn't stall the async runtime.
+pub async fn run_blocking<T: Send + 'static>(func: impl FnOnce() -> T + Send + 'static) -> Result<T, ErrorFuture> {
+ tokio::task::spawn_blocking(func).await.map_err(|error| -> ErrorFuture { error.into() })
+}
+
fn coerce_to_schema(
value: serde_json::Value,
schema: &serde_json::Value,
diff --git a/crates/proc-macros/src/uzu_tool/closure.rs b/crates/proc-macros/src/uzu_tool/closure.rs
index c04a37e5e..5c563c899 100644
--- a/crates/proc-macros/src/uzu_tool/closure.rs
+++ b/crates/proc-macros/src/uzu_tool/closure.rs
@@ -119,10 +119,11 @@ fn expand_tool_closure(tool: ToolClosure) -> syn::Result<TokenStream2> {
};
let arg_idents = params.iter().map(|param| &param.ident);
- let mut call = quote!(__uzu_tool_func(#(#arg_idents),*));
- if is_async {
- call = quote!(#call.await);
- }
+ let call = if is_async {
+ quote!(__uzu_tool_func(#(#arg_idents),*).await)
+ } else {
+ quote!(#nagare::tool::func_def::run_blocking(move || __uzu_tool_func(#(#arg_idents),*)).await?)
+ };
let result_stmt = match &return_type {
Some(ty) => quote!(let __uzu_result: #ty = #call;),
None => quote!(let __uzu_result = #call;),
diff --git a/crates/proc-macros/src/uzu_tool/function.rs b/crates/proc-macros/src/uzu_tool/function.rs
index 09d24d7c0..d679fa951 100644
--- a/crates/proc-macros/src/uzu_tool/function.rs
+++ b/crates/proc-macros/src/uzu_tool/function.rs
@@ -73,10 +73,11 @@ fn expand_tool_function(mut func: ItemFn) -> syn::Result<TokenStream2> {
let arg_parsing = arg_parsing_tokens(&params, &name_str, &nagare);
let arg_idents = params.iter().map(|param| &param.ident);
- let mut call = quote!(Self::call(#(#arg_idents),*));
- if is_async {
- call = quote!(#call.await);
- }
+ let call = if is_async {
+ quote!(Self::call(#(#arg_idents),*).await)
+ } else {
+ quote!(#nagare::tool::func_def::run_blocking(move || Self::call(#(#arg_idents),*)).await?)
+ };
let map_err = quote!(.map_err(|error| -> #nagare::tool::func_def::ErrorFuture { error.into() })?);
let call_stmt = match (&ok_type, is_result) {
(Some(_), true) => quote!(let result = #call #map_err;),