Skip to content

armgit / macro

Public Imported from GitHub
0

Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.

fix(agent): stop counting tool auto-loads against the invalid-call budget (#9164)

Wolf Mermelstein5f996b193d3e
diff --git a/crates/agent/src/hook.rs b/crates/agent/src/hook.rs
index 059479513f..bc42089488 100644
--- a/crates/agent/src/hook.rs
+++ b/crates/agent/src/hook.rs
@@ -13,6 +13,7 @@ use rig_agent::tool::ToolOutput;
use std::collections::HashMap;
use std::future::Future;
use std::pin::Pin;
+use std::sync::atomic::{AtomicUsize, Ordering};
use std::sync::{Arc, Mutex};
use tokio::sync::mpsc;
use tokio_util::sync::CancellationToken;
@@ -177,14 +178,18 @@ const PENDING_USER_EXECUTION: &str = "PendingUserExecution";
static CANCELLED_REASON: &str = "user cancelled";
-/// Retry budget for invalid tool calls recovered via
-/// [`StreamBridge::on_invalid_tool_call`]. Lets the common one-shot recovery —
-/// the model calls a searchable tool it never loaded, the bridge loads it, the
-/// call is retried — succeed, while bounding a model that keeps emitting names
-/// that don't exist anywhere. Past the budget the stream fails with
-/// `UnknownToolCall`, as it did before recovery existed. Retries also consume
-/// multi-turn depth.
-pub(crate) const MAX_INVALID_TOOL_CALL_RETRIES: usize = 2;
+/// Retries per run for tool calls to names that exist nowhere, not even the
+/// searchable catalog (see [`StreamBridge::handle_invalid_tool_call`]). Bounds
+/// a model that keeps hallucinating names; past it the run fails with
+/// `UnknownToolCall`, as it did before recovery existed. Calls that auto-load
+/// a catalog tool don't count: they are bounded by multi-turn depth, which
+/// every retry consumes.
+pub(crate) const MAX_UNKNOWN_TOOL_CALL_RETRIES: usize = 2;
+
+/// The retry budget handed to rig. rig counts every retry against one budget
+/// for the whole run, auto-loads included, so the bridge owns the budget (see
+/// [`MAX_UNKNOWN_TOOL_CALL_RETRIES`]) and rig's must never run out first.
+pub(crate) const RIG_INVALID_TOOL_CALL_RETRIES: usize = usize::MAX;
/// Sends [`StreamPart`] items through an unbounded channel as the RIG agentic
/// loop produces events.
@@ -203,6 +208,9 @@ pub struct StreamBridge {
/// [`Self::on_invalid_tool_call`] to recover calls to tools the model
/// discovered but never loaded.
searchable_catalog: Arc<Vec<SearchableTool>>,
+ /// Retries spent this run on names not in the catalog; capped at
+ /// [`MAX_UNKNOWN_TOOL_CALL_RETRIES`].
+ unknown_tool_call_retries: Arc<AtomicUsize>,
/// Finishes a user tool's pending answer before the model reads it, on
/// hosts that can reach the user mid-turn (see [`UserToolFinisher`]).
user_tool_finisher: Option<UserToolFinisher>,
@@ -254,6 +262,7 @@ impl StreamBridge {
register_loaded,
tool_loads,
searchable_catalog,
+ unknown_tool_call_retries: Arc::default(),
user_tool_finisher,
streamed: stream_tool_calls.then(Arc::default),
result_limit,
@@ -310,7 +319,8 @@ impl StreamBridge {
/// never loaded — load it now and ask the model to retry; the next request
/// is rebuilt from the live tool server, so the retried call is valid. For
/// names that exist nowhere, the retry feedback points the model at
- /// `SearchTools` instead of failing the stream on a hallucinated name.
+ /// `SearchTools` instead of failing the stream on a hallucinated name,
+ /// until [`MAX_UNKNOWN_TOOL_CALL_RETRIES`] is spent.
pub(crate) async fn handle_invalid_tool_call(
&self,
tool_name: &str,
@@ -339,11 +349,19 @@ impl StreamBridge {
again, checking your arguments against them."
)))
}
- None => Some(InvalidToolCallAction::retry(format!(
- "Unknown tool `{tool_name}`: no tool with that name exists in this session \
- or its connected integrations. Use `SearchTools` to find the \
- right tool, or continue without it."
- ))),
+ None => {
+ let spent = self
+ .unknown_tool_call_retries
+ .fetch_add(1, Ordering::Relaxed);
+ if spent >= MAX_UNKNOWN_TOOL_CALL_RETRIES {
+ return Some(InvalidToolCallAction::fail());
+ }
+ Some(InvalidToolCallAction::retry(format!(
+ "Unknown tool `{tool_name}`: no tool with that name exists in this session \
+ or its connected integrations. Use `SearchTools` to find the \
+ right tool, or continue without it."
+ )))
+ }
}
}
diff --git a/crates/agent/src/model/router.rs b/crates/agent/src/model/router.rs
index dd2858a7cf..cff037fb39 100644
--- a/crates/agent/src/model/router.rs
+++ b/crates/agent/src/model/router.rs
@@ -699,7 +699,7 @@ where
.stream_prompt(prompt)
.history(history)
.max_turns(max_turns)
- .max_invalid_tool_call_retries(crate::hook::MAX_INVALID_TOOL_CALL_RETRIES)
+ .max_invalid_tool_call_retries(crate::hook::RIG_INVALID_TOOL_CALL_RETRIES)
.add_hook(bridge)
.add_hook(ChatSpanHook(telemetry.clone()))
.await;
diff --git a/crates/agent/src/test/agent_loop/test_tool_search.rs b/crates/agent/src/test/agent_loop/test_tool_search.rs
index 16c6880b49..2bcdaa8031 100644
--- a/crates/agent/src/test/agent_loop/test_tool_search.rs
+++ b/crates/agent/src/test/agent_loop/test_tool_search.rs
@@ -6,6 +6,7 @@
//! loads tools from both servers and then calls one tool from each.
use super::util;
+use crate::AgentError;
use crate::stream::ToolResponse;
use ai_toolset::{
AsyncTool, AsyncToolCollection, RequestContext, RequestSchema, SearchableTool, ServiceContext,
@@ -13,6 +14,8 @@ use ai_toolset::{
};
use ai_toolset::{ToolAnnotated, ToolAnnotations};
use async_trait::async_trait;
+use rig_agent::agent::StreamingError;
+use rig_agent::completion::PromptError;
use rig_core::test_utils::{MockCompletionModel, MockStreamEvent};
use schemars::{JsonSchema, Schema};
use serde::Deserialize;
@@ -102,7 +105,11 @@ impl FakeCombined {
.add_tool::<SearchToolsTest, ()>()
.add_tool::<LoadToolsTest, ()>(),
),
- mcp_tools: vec!["mcp__a__t1".to_owned(), "mcp__b__t2".to_owned()],
+ mcp_tools: vec![
+ "mcp__a__t1".to_owned(),
+ "mcp__b__t2".to_owned(),
+ "mcp__a__t3".to_owned(),
+ ],
}
}
}
@@ -301,6 +308,92 @@ async fn hallucinated_tool_name_gets_feedback_and_the_turn_recovers() {
assert!(result.content().contains("recovered without the tool"));
}
+/// Auto-loads are recoveries, not mistakes, so they don't spend the
+/// invalid-call budget: a turn that calls three unloaded catalog tools one
+/// after another still finishes.
+#[tokio::test]
+async fn three_unloaded_searchable_tools_in_one_turn_all_self_heal() {
+ let model = MockCompletionModel::from_stream_turns([
+ vec![
+ MockStreamEvent::tool_call("call-a", "mcp__a__t1", serde_json::json!({})),
+ MockStreamEvent::final_response_with_default_usage(),
+ ],
+ vec![
+ MockStreamEvent::tool_call("call-a-retry", "mcp__a__t1", serde_json::json!({})),
+ MockStreamEvent::final_response_with_default_usage(),
+ ],
+ vec![
+ MockStreamEvent::tool_call("call-b", "mcp__b__t2", serde_json::json!({})),
+ MockStreamEvent::final_response_with_default_usage(),
+ ],
+ vec![
+ MockStreamEvent::tool_call("call-b-retry", "mcp__b__t2", serde_json::json!({})),
+ MockStreamEvent::final_response_with_default_usage(),
+ ],
+ vec![
+ MockStreamEvent::tool_call("call-c", "mcp__a__t3", serde_json::json!({})),
+ MockStreamEvent::final_response_with_default_usage(),
+ ],
+ vec![
+ MockStreamEvent::tool_call("call-c-retry", "mcp__a__t3", serde_json::json!({})),
+ MockStreamEvent::final_response_with_default_usage(),
+ ],
+ vec![MockStreamEvent::final_response_with_default_usage()],
+ ]);
+
+ let toolset: Arc<dyn AiToolSet<()> + Send + Sync> = Arc::new(FakeCombined::new());
+ let mut session = util::session(toolset, Arc::new(()), model).await;
+ let result = util::drive(&mut session, "use three integration tools without loading").await;
+
+ assert!(
+ result.error.is_none(),
+ "stream ended with error: {:?}",
+ result.error
+ );
+ expect_json_ok(&result, "mcp__a__t1");
+ expect_json_ok(&result, "mcp__b__t2");
+ expect_json_ok(&result, "mcp__a__t3");
+}
+
+/// A model that keeps calling a name that exists nowhere still fails the turn
+/// with `UnknownToolCall` once the invalid-call budget is spent.
+#[tokio::test]
+async fn repeated_hallucinated_tool_name_fails_after_the_budget() {
+ let model = MockCompletionModel::from_stream_turns([
+ vec![
+ MockStreamEvent::tool_call("call-bad-1", "mcp__zz__nope", serde_json::json!({})),
+ MockStreamEvent::final_response_with_default_usage(),
+ ],
+ vec![
+ MockStreamEvent::tool_call("call-bad-2", "mcp__zz__nope", serde_json::json!({})),
+ MockStreamEvent::final_response_with_default_usage(),
+ ],
+ vec![
+ MockStreamEvent::tool_call("call-bad-3", "mcp__zz__nope", serde_json::json!({})),
+ MockStreamEvent::final_response_with_default_usage(),
+ ],
+ vec![
+ MockStreamEvent::text("should never be reached"),
+ MockStreamEvent::final_response_with_default_usage(),
+ ],
+ ]);
+
+ let toolset: Arc<dyn AiToolSet<()> + Send + Sync> = Arc::new(FakeCombined::new());
+ let mut session = util::session(toolset, Arc::new(()), model).await;
+ let result = util::drive(&mut session, "keep calling a tool that does not exist").await;
+
+ match &result.error {
+ Some(AgentError::Streaming(StreamingError::Prompt(error))) => match error.as_ref() {
+ PromptError::UnknownToolCall { tool_name, .. } => {
+ assert_eq!(tool_name, "mcp__zz__nope");
+ }
+ other => panic!("expected UnknownToolCall, got {other:?}"),
+ },
+ other => panic!("expected UnknownToolCall, got {other:?}"),
+ }
+ assert!(!result.content().contains("should never be reached"));
+}
+
/// Load + call server a, then load + call server b, all in one execution.
#[tokio::test]
async fn mcp_calls_from_two_providers_loaded_sequentially() {