Skip to content

Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.

Rust

fix(vgr): skip checks that can't change the result after a tool error (#1315)

Adam Haile2b239def8108
diff --git a/CHANGELOG.md b/CHANGELOG.md
index f3c5bebe0..60517ca0c 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -39,6 +39,8 @@ adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
- The standalone `switchyard-server` now binds to `127.0.0.1` by default.
Existing deployments that accept remote clients must add `--host 0.0.0.0`.
The provided Docker image and `dev-server/switchyard.service` set this explicitly.
+- **VGR escalates sooner after a tool error** - chat steps skip verification,
+ and steps typed as chat skip the efficient attempt (`chat_after_tool_error`).
- **TypeScript composite stage profiles** - `CompositeRouter` options take
`stage.efficient_first` and `stage.capable_first` profiles in place of
`efficient_first_threshold`, `capable_first_threshold`, and
diff --git a/crates/libsy/src/algorithms/vgr/decide.rs b/crates/libsy/src/algorithms/vgr/decide.rs
index 186084a61..108c44525 100644
--- a/crates/libsy/src/algorithms/vgr/decide.rs
+++ b/crates/libsy/src/algorithms/vgr/decide.rs
@@ -9,7 +9,7 @@
use super::{Branch, Capabilities};
/// Version of these rules and dials, carried on every decision record.
-pub(super) const POLICY_VERSION: &str = "1.2";
+pub(super) const POLICY_VERSION: &str = "1.3";
const READOUT: f64 = 0.9;
const DELIBERATION: f64 = 0.5;
diff --git a/crates/libsy/src/algorithms/vgr/runtime.rs b/crates/libsy/src/algorithms/vgr/runtime.rs
index 9fe171375..e9f5d3be8 100644
--- a/crates/libsy/src/algorithms/vgr/runtime.rs
+++ b/crates/libsy/src/algorithms/vgr/runtime.rs
@@ -171,6 +171,23 @@ impl Classifier for VgrClassifier {
let Some(trial) = self.breaker.attempt() else {
return Ok(self.escalate(driver, "breaker_open", None, request, state, None));
};
+ let typed = if text::ToolRecord::from_request(request).errors > 0 {
+ let context = RequestContext {
+ driver,
+ request,
+ started,
+ };
+ Some(self.type_task(&context).await)
+ } else {
+ None
+ };
+ if typed == Some(Some(TaskType::Chat)) {
+ let reason = "chat_after_tool_error";
+ return Ok(self.escalate(driver, reason, Some(Branch::Chat), request, state, None));
+ }
+ if self.config.bounded_efficient && self.remaining(started).is_none() {
+ return Ok(self.escalate(driver, "efficient_timed_out", None, request, state, None));
+ }
let attempted = self
.within_budget(started, async {
let response = driver
@@ -255,7 +272,10 @@ impl Classifier for VgrClassifier {
}
let attempt = rungs::response_text(&buffered.agg).unwrap_or_default();
- let task_type = self.type_task(&context).await;
+ let task_type = match typed {
+ Some(task_type) => task_type,
+ None => self.type_task(&context).await,
+ };
let caps = derive_capabilities(
request,
&attempt,
@@ -451,7 +471,10 @@ impl VgrClassifier {
let run = (branch == Branch::Agentic).then(|| decide::agentic_run(caps, confirmed));
let (Some(judged), false) = (
caps.transcript.as_deref(),
- branch == Branch::Unknown || run == Some(AgenticRun::Vetoed),
+ branch == Branch::Unknown
+ || run == Some(AgenticRun::Vetoed)
+ || (matches!(branch, Branch::Chat | Branch::DefaultVerified)
+ && caps.tools.is_some_and(|tools| tools.errors > 0)),
) else {
return signals;
};
diff --git a/crates/libsy/src/algorithms/vgr/runtime/tests.rs b/crates/libsy/src/algorithms/vgr/runtime/tests.rs
index fe9599559..767b2d8fa 100644
--- a/crates/libsy/src/algorithms/vgr/runtime/tests.rs
+++ b/crates/libsy/src/algorithms/vgr/runtime/tests.rs
@@ -472,7 +472,7 @@ async fn decisions_attach_the_reason_and_exclude_efficient_capable_fallbacks() -
assert_eq!(evidence["source"], "vgr");
assert_eq!(evidence["reason_code"], reason);
assert_eq!(evidence["verdict"], served);
- assert_eq!(evidence["policy_version"], "1.2");
+ assert_eq!(evidence["policy_version"], "1.3");
if reason == "decided" {
assert_eq!(evidence["branch"], "DefaultVerified");
}
@@ -1376,3 +1376,33 @@ async fn a_failed_capable_verifier_still_offers_the_efficient_answer() -> Result
assert_eq!(fallback.llm_request.messages, task().llm_request.messages);
Ok(())
}
+
+#[tokio::test]
+async fn a_step_that_cannot_commit_over_a_tool_error_skips_local_checks() -> Result<()> {
+ let failed = session(vec![
+ Message::text(Role::User, "What is happening today?"),
+ call("c1"),
+ result("c1", r#"{"error": "search-only backend"}"#),
+ ]);
+ // A typed answer with tool use in its turn is judged as chat, which needs a clean record.
+ for (typed, expected) in [
+ ("Answer", vec!["judge", "efficient", "capable"]),
+ ("Chat", vec!["judge", "capable"]),
+ ] {
+ let calls = Arc::new(parking_lot::Mutex::new(Vec::new()));
+ let seen = Arc::clone(&calls);
+ let (selected, _) = test_drive_with_models(
+ route(|config| config.task_typing = true)?,
+ failed.clone(),
+ category_models(Category::Any, &["efficient", "capable"]),
+ move |target: ModelId, _| {
+ seen.lock().push(target.to_string());
+ std::future::ready(Ok(reply(typed)))
+ },
+ )
+ .await?;
+ assert_eq!(selected.as_str(), "capable");
+ assert_eq!(*calls.lock(), expected, "typed {typed}");
+ }
+ Ok(())
+}
diff --git a/docs/routing_algorithms/vgr_routing.md b/docs/routing_algorithms/vgr_routing.md
index 34abc9f92..8c196c418 100644
--- a/docs/routing_algorithms/vgr_routing.md
+++ b/docs/routing_algorithms/vgr_routing.md
@@ -190,10 +190,13 @@ headers take precedence. Hosts that omit reporting retain message-shape detectio
## Verification and capable-tier outages
-Policy version `1.2` allows the default verification branch to accept a clean
+Policy version `1.3` allows the default verification branch to accept a clean
efficient attempt when the capable judge approves both the evidence and the
answer. A negative or unavailable evidence verdict skips the answer check.
+After a failed tool result, chat steps escalate without verification, and steps
+typed as chat escalate before any efficient attempt.
+
Typed factual answers are verified on the answer itself, including
later user turns without tool work. A readout of at least 0.9 can serve the
efficient answer; lower scores require capable-tier verification. Scored verdicts