Skip to content

Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.

Rust

feat(vgr): hand off escalated agentic runs (#881)

GDRSdf27594b1eb8
diff --git a/crates/libsy/src/algorithms/vgr/config.rs b/crates/libsy/src/algorithms/vgr/config.rs
index 1b6471a52..fc7965e6a 100644
--- a/crates/libsy/src/algorithms/vgr/config.rs
+++ b/crates/libsy/src/algorithms/vgr/config.rs
@@ -61,6 +61,12 @@ pub struct VgrConfig {
/// Lets a long agentic run with earlier tool errors commit on a capable-tier
/// confirmation once it ends in this many consecutive clean tool results.
pub confirmed_recovery_min_clean_tail: Option<u32>,
+ /// Tells the capable tier, once per user turn, that it inherits unverified
+ /// tool-using work.
+ pub agentic_handoff: bool,
+ /// Wall-clock budget for the local tier within one user turn, including the
+ /// client's tool execution.
+ pub local_turn_budget: Option<Duration>,
}
impl VgrConfig {
@@ -80,6 +86,8 @@ impl VgrConfig {
task_typing: true,
local_supports_images: false,
confirmed_recovery_min_clean_tail: None,
+ agentic_handoff: false,
+ local_turn_budget: None,
}
}
diff --git a/crates/libsy/src/algorithms/vgr/runtime.rs b/crates/libsy/src/algorithms/vgr/runtime.rs
index 706af49d1..51af4121f 100644
--- a/crates/libsy/src/algorithms/vgr/runtime.rs
+++ b/crates/libsy/src/algorithms/vgr/runtime.rs
@@ -9,7 +9,7 @@
//! by what is left of the turn's deadline, and a call that fails or times out
//! is evidence never gathered, which does not commit.
-use std::time::{Duration, Instant};
+use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
use async_trait::async_trait;
use switchyard_protocol::{AggLlmResponse, ContentBlock, ModelId, Request, Response, StopReason};
@@ -19,8 +19,10 @@ use super::decide::{self, AgenticRun, Route, Signals, Tri};
use super::rungs::{self, Question};
use super::safety::{CircuitBreaker, endpoint_failure, fallback_eligible};
use super::{Branch, Capabilities, TaskType, derive_capabilities, readout, text};
+use crate::algorithms::util::affinity::has_new_user_turn;
use crate::algorithms::util::buffered_response::buffer_response;
use crate::algorithms::util::decisive;
+use crate::algorithms::util::prompts::append_note;
use crate::core::algorithm::Driver;
use crate::core::classifier::{Classification, Classifier};
use crate::{LibsyError, Result};
@@ -29,6 +31,9 @@ use crate::{LibsyError, Result};
pub(super) struct VgrState {
pub(super) turn_streak: u32,
pub(super) turn_latched: bool,
+ pub(super) turn_started: Option<u32>,
+ pub(super) handoff_sent: bool,
+ pub(super) local_served: bool,
}
struct RequestContext<'a> {
@@ -58,6 +63,11 @@ impl Classifier<VgrState> for VgrClassifier {
) -> Result<Scored> {
let started = Instant::now();
let local = &self.config.targets.local;
+ if has_new_user_turn(&request.llm_request.messages) {
+ state.turn_started = None;
+ state.handoff_sent = false;
+ }
+ let turn_budget_spent = self.turn_budget_spent(state);
let short_circuit = if self.config.mode == ServingMode::Off {
Some("mode_off")
} else if self
@@ -69,20 +79,22 @@ impl Classifier<VgrState> for VgrClassifier {
Some("kill_switch")
} else if state.turn_latched {
Some("turn_verification_latched")
+ } else if turn_budget_spent {
+ Some("local_turn_budget")
} else if !self.config.local_supports_images && request_has_image(request) {
Some("local_image_unsupported")
} else {
None
};
if let Some(reason) = short_circuit {
- return Ok(self.escalate(driver, reason, None));
+ return Ok(self.escalate(driver, reason, None, request, state, None));
}
let Some(budget) = self.remaining(started) else {
- return Ok(self.escalate(driver, "local_timed_out", None));
+ return Ok(self.escalate(driver, "local_timed_out", None, request, state, None));
};
let Some(trial) = self.breaker.attempt() else {
- return Ok(self.escalate(driver, "breaker_open", None));
+ return Ok(self.escalate(driver, "breaker_open", None, request, state, None));
};
let attempted = tokio::time::timeout(budget, async {
let response = driver
@@ -103,11 +115,11 @@ impl Classifier<VgrState> for VgrClassifier {
if !fallback_eligible(&error) {
return Err(error);
}
- return Ok(self.escalate(driver, "local_unavailable", None));
+ return Ok(self.escalate(driver, "local_unavailable", None, request, state, None));
}
Err(_) => {
trial.failure();
- return Ok(self.escalate(driver, "local_timed_out", None));
+ return Ok(self.escalate(driver, "local_timed_out", None, request, state, None));
}
};
@@ -118,15 +130,37 @@ impl Classifier<VgrState> for VgrClassifier {
};
if rungs::has_tool_call(&buffered.agg) {
if !tool_use_is_complete(&buffered.agg) {
- return Ok(self.escalate(driver, "malformed_tool_call", None));
+ return Ok(self.escalate(
+ driver,
+ "malformed_tool_call",
+ None,
+ request,
+ state,
+ None,
+ ));
}
if self.verify_turn(&context, &buffered.agg, state).await {
- return Ok(self.escalate(driver, "turn_verification_escalated", None));
+ return Ok(self.escalate(
+ driver,
+ "turn_verification_escalated",
+ None,
+ request,
+ state,
+ None,
+ ));
}
if self.config.mode == ServingMode::Shadow {
- return Ok(self.escalate(driver, "turn_verification_complete", None));
+ return Ok(self.escalate(
+ driver,
+ "turn_verification_complete",
+ None,
+ request,
+ state,
+ None,
+ ));
}
log(driver, "turn_verification_complete", None, Route::Local);
+ state.local_served = true;
return Ok((decisive(local), Some(buffered.into_response())));
}
@@ -136,19 +170,74 @@ impl Classifier<VgrState> for VgrClassifier {
let signals = self.gather(&context, &caps).await;
let route = decide::decide(&caps, &signals, self.config.confirmed_min_clean_tail());
if route == Route::Cloud || self.config.mode == ServingMode::Shadow {
- return Ok(self.escalate(driver, "decided", Some(caps.branch)));
+ return Ok(self.escalate(
+ driver,
+ "decided",
+ Some(caps.branch),
+ request,
+ state,
+ Some(&attempt),
+ ));
}
log(driver, "decided", Some(caps.branch), Route::Local);
+ state.local_served = true;
Ok((decisive(local), Some(buffered.into_response())))
}
}
impl VgrClassifier {
- fn escalate(&self, driver: &Driver, reason: &'static str, branch: Option<Branch>) -> Scored {
+ fn escalate(
+ &self,
+ driver: &Driver,
+ reason: &'static str,
+ branch: Option<Branch>,
+ request: &mut Request,
+ state: &mut VgrState,
+ unverified_final: Option<&str>,
+ ) -> Scored {
+ if reason != "mode_off" {
+ self.hand_off(request, state, unverified_final);
+ }
log(driver, reason, branch, Route::Cloud);
(decisive(&self.config.targets.cloud), None)
}
+ /// Whether the current user turn has used up its local wall-clock budget.
+ ///
+ /// The clock starts at the first request of the user turn and includes the
+ /// client's tool execution, since that is the time the client's own limit
+ /// is spending.
+ fn turn_budget_spent(&self, state: &mut VgrState) -> bool {
+ let Some(budget) = self.config.local_turn_budget else {
+ return false;
+ };
+ let now = unix_seconds();
+ let started = *state.turn_started.get_or_insert(now);
+ u64::from(now.saturating_sub(started)) >= budget.as_secs()
+ }
+
+ /// Tells the capable tier it is inheriting unverified tool-using work.
+ ///
+ /// Sent at most once per user turn, on its first escalation.
+ fn hand_off(
+ &self,
+ request: &mut Request,
+ state: &mut VgrState,
+ unverified_final: Option<&str>,
+ ) {
+ if !self.config.agentic_handoff
+ || state.handoff_sent
+ || !state.local_served
+ || !text::has_tool_trajectory(request)
+ {
+ return;
+ }
+ state.handoff_sent = true;
+ state.local_served = false;
+ let note = handoff_note(unverified_final);
+ append_note(request, &note);
+ }
+
/// Judges one proposed tool call; `true` escalates the rest of the session.
async fn verify_turn(
&self,
@@ -332,6 +421,43 @@ impl VgrClassifier {
}
}
+fn unix_seconds() -> u32 {
+ SystemTime::now()
+ .duration_since(UNIX_EPOCH)
+ .map_or(0, |elapsed| {
+ u32::try_from(elapsed.as_secs()).unwrap_or(u32::MAX)
+ })
+}
+
+/// The note a capable tier receives when it takes over a tool-using session.
+fn handoff_note(unverified_final: Option<&str>) -> String {
+ let mut note = String::from(
+ "\n\n[Routing notice from the serving infrastructure, not from the user] Until now the \
+ work in this session was carried out by a smaller, faster model. That work has NOT been \
+ verified: it may be incomplete or incorrect, and it may have left the environment in a \
+ broken state. You are taking over from here. Do not assume that any earlier step \
+ succeeded or that the task is finished. Inspect the actual current state of the \
+ environment against every requirement of the original task, fix whatever is missing or \
+ wrong, and give your final answer only after you have confirmed that the requirements \
+ are met.",
+ );
+ if let Some(claim) = unverified_final.filter(|claim| !claim.trim().is_empty()) {
+ const CLIPPING_MARKER_RESERVE: usize = 64;
+ let claim = text::clip_mid(
+ claim,
+ super::render::ATTEMPT_BUDGET.saturating_sub(CLIPPING_MARKER_RESERVE),
+ 1.0 / 3.0,
+ );
+ note.push_str(
+ "\n\nThe smaller model was about to finish with the following unchecked message:\n\
+ <unverified_final_message>\n",
+ );
+ note.push_str(&claim);
+ note.push_str("\n</unverified_final_message>");
+ }
+ note
+}
+
fn log(driver: &Driver, reason: &'static str, branch: Option<Branch>, served: Route) {
driver.set_evidence(serde_json::json!({
"source": "vgr",
diff --git a/crates/libsy/src/algorithms/vgr/runtime/tests.rs b/crates/libsy/src/algorithms/vgr/runtime/tests.rs
index 9a5a45cfb..ea2f6fdb6 100644
--- a/crates/libsy/src/algorithms/vgr/runtime/tests.rs
+++ b/crates/libsy/src/algorithms/vgr/runtime/tests.rs
@@ -398,3 +398,226 @@ async fn decisions_attach_the_reason_and_exclude_local_cloud_fallbacks() -> Resu
}
Ok(())
}
+
+#[tokio::test]
+async fn cloud_only_work_does_not_get_a_local_handoff_notice() -> Result<()> {
+ for scenario in ["shadow", "kill_switch", "local_turn_budget"] {
+ let route = route(|config| {
+ config.agentic_handoff = true;
+ match scenario {
+ "shadow" => config.mode = ServingMode::Shadow,
+ "kill_switch" => {
+ let switch = super::super::safety::KillSwitch::new();
+ switch.engage();
+ config.kill_switch = Some(switch);
+ }
+ _ => config.local_turn_budget = Some(std::time::Duration::ZERO),
+ }
+ })?;
+ let task = session(vec![
+ Message::text(Role::User, "fix the build"),
+ call("c1"),
+ result("c1", "ok"),
+ ]);
+ let (selected, _) = test_drive_with_models(
+ route,
+ task,
+ category_models(Category::Any, &["local", "cloud"]),
+ move |target: ModelId, request: Request| async move {
+ Ok(match target.as_str() {
+ "local" => complete_call(),
+ "judge" => scored(0.0),
+ _ => {
+ let text = serde_json::to_string(&request.llm_request.messages)
+ .unwrap_or_default();
+ assert!(!text.contains("Routing notice"), "{scenario}");
+ reply("cloud answer")
+ }
+ })
+ },
+ )
+ .await?;
+ assert_eq!(selected, "cloud");
+ }
+ Ok(())
+}
+
+#[tokio::test]
+async fn latched_sessions_do_not_repeat_the_handoff_on_later_user_turns() -> Result<()> {
+ let route = route(|config| config.agentic_handoff = true)?;
+ assert_eq!(
+ drive(
+ &route,
+ session(vec![Message::text(Role::User, "fix the build")]),
+ Arc::new(Calls::default()),
+ complete_call,
+ 0.9,
+ )
+ .await?,
+ "local"
+ );
+ let sent = Arc::new(parking_lot::Mutex::new(Vec::new()));
+ for new_user_turn in [false, true] {
+ let mut messages = vec![
+ Message::text(Role::User, "fix the build"),
+ call("c1"),
+ result("c1", "ok"),
+ ];
+ if new_user_turn {
+ messages.push(Message::text(Role::User, "now run the tests"));
+ }
+ let sent = Arc::clone(&sent);
+ let (selected, _) = test_drive_with_models(
+ Arc::clone(&route),
+ session(messages),
+ category_models(Category::Any, &["local", "cloud"]),
+ move |target: ModelId, request: Request| {
+ let sent = Arc::clone(&sent);
+ async move {
+ Ok(match target.as_str() {
+ "local" => complete_call(),
+ "judge" => scored(0.9),
+ _ => {
+ sent.lock().push(
+ serde_json::to_string(&request.llm_request.messages)
+ .unwrap_or_default(),
+ );
+ reply("cloud answer")
+ }
+ })
+ }
+ },
+ )
+ .await?;
+ assert_eq!(selected, "cloud");
+ }
+ let sent = sent.lock();
+ assert_eq!(sent.len(), 2);
+ assert!(sent[0].contains("Routing notice"));
+ assert!(!sent[1].contains("Routing notice"));
+ Ok(())
+}
+
+#[tokio::test]
+async fn an_escalated_agentic_run_hands_off_once_with_the_unverified_claim() -> Result<()> {
+ let route = route(|config| config.agentic_handoff = true)?;
+ assert_eq!(
+ drive(
+ &route,
+ session(vec![Message::text(Role::User, "fix the build")]),
+ Arc::new(Calls::default()),
+ complete_call,
+ 0.0,
+ )
+ .await?,
+ "local"
+ );
+ let sent = Arc::new(parking_lot::Mutex::new(Vec::new()));
+ let task = || {
+ session(vec![
+ Message::text(Role::User, "fix the build"),
+ call("c1"),
+ result("c1", "ok"),
+ ])
+ };
+ for _ in 0..2 {
+ let sent = Arc::clone(&sent);
+ test_drive_with_models(
+ Arc::clone(&route),
+ task(),
+ category_models(Category::Any, &["local", "cloud"]),
+ move |target: ModelId, request: Request| {
+ let sent = Arc::clone(&sent);
+ async move {
+ Ok(match target.as_str() {
+ "local" => reply("the build passes"),
+ "judge" => scored(0.0),
+ "cloud" => {
+ let text = serde_json::to_string(&request.llm_request.messages)
+ .unwrap_or_default();
+ sent.lock().push(text);
+ reply("cloud answer")
+ }
+ _ => reply("no"),
+ })
+ }
+ },
+ )
+ .await?;
+ }
+ let sent = sent.lock();
+ assert_eq!(sent.len(), 2);
+ assert!(sent[0].contains("Routing notice") && sent[0].contains("the build passes"));
+ assert!(
+ !sent[1].contains("Routing notice"),
+ "sent once per user turn"
+ );
+ Ok(())
+}
+
+#[tokio::test]
+async fn a_spent_local_turn_budget_skips_the_local_attempt() -> Result<()> {
+ let route = route(|config| config.local_turn_budget = Some(std::time::Duration::ZERO))?;
+ let calls = Arc::new(Calls::default());
+ let task = session(vec![Message::text(Role::User, "fix the build")]);
+ assert_eq!(
+ drive(&route, task, calls.clone(), complete_call, 0.0).await?,
+ "cloud"
+ );
+ assert_eq!(calls.0.load(Ordering::Relaxed), 0);
+ Ok(())
+}
+
+#[test]
+fn handoff_preserves_existing_messages_and_non_text_content() {
+ use switchyard_protocol::ImageSource;
+
+ let mut classifier = recovering_classifier();
+ classifier.config.agentic_handoff = true;
+ let mut task = request(vec![
+ Message::text(Role::User, "finish the task"),
+ Message::text(Role::User, "keep the original files"),
+ call("c1"),
+ result("c1", "the command failed"),
+ Message::text(Role::User, "preserve this requirement"),
+ Message::text(Role::User, "continue"),
+ Message::text(Role::User, "then report the result"),
+ ]);
+ if let ContentBlock::ToolResult(result) = &mut task.llm_request.messages[3].content[0] {
+ result.content.push(ContentBlock::Image {
+ source: ImageSource::Url {
+ url: "https://example.test/result.png".into(),
+ detail: None,
+ },
+ });
+ }
+ let original = task.llm_request.messages.clone();
+ let mut state = super::VgrState {
+ local_served: true,
+ ..Default::default()
+ };
+ classifier.hand_off(&mut task, &mut state, Some("unchecked result"));
+ // A notice may be appended, but all original content stays unchanged.
+ let messages = &task.llm_request.messages;
+ assert_eq!(messages.len(), original.len());
+ assert_eq!(
+ &messages[..original.len() - 1],
+ &original[..original.len() - 1]
+ );
+ let last = messages.last().expect("last user message");
+ assert_eq!(
+ &last.content[..original
+ .last()
+ .expect("original last message")
+ .content
+ .len()],
+ &original.last().expect("original last message").content
+ );
+ assert!(
+ last.text_content("")
+ .is_some_and(|text| text.contains("unchecked result"))
+ );
+ let after_first = task.llm_request.messages.clone();
+ classifier.hand_off(&mut task, &mut state, Some("duplicate notice"));
+ assert_eq!(task.llm_request.messages, after_first);
+}
diff --git a/crates/switchyard-runner/src/algorithm.rs b/crates/switchyard-runner/src/algorithm.rs
index edf4cc287..a59eb2e3c 100644
--- a/crates/switchyard-runner/src/algorithm.rs
+++ b/crates/switchyard-runner/src/algorithm.rs
@@ -546,6 +546,12 @@ pub struct VgrRouteConfig {
/// Clean trailing tool results that let a long run recover on the cloud judge.
#[serde(default)]
pub confirmed_recovery_min_clean_tail: Option<u32>,
+ /// Tells the capable tier it inherits unverified tool-using work.
+ #[serde(default)]
+ pub agentic_handoff: bool,
+ /// Wall-clock budget for the local tier within one user turn.
+ #[serde(default)]
+ pub local_turn_budget_seconds: Option<f64>,
}
#[derive(Clone, Debug, Default, Deserialize)]
@@ -1798,6 +1804,11 @@ fn build_vgr(
runtime.deadline = duration(route_name, "deadline_seconds", config.deadline_seconds)?;
runtime.task_typing = config.task_typing;
runtime.confirmed_recovery_min_clean_tail = config.confirmed_recovery_min_clean_tail;
+ runtime.agentic_handoff = config.agentic_handoff;
+ runtime.local_turn_budget = config
+ .local_turn_budget_seconds
+ .map(|seconds| duration(route_name, "local_turn_budget_seconds", seconds))
+ .transpose()?;
runtime.breaker = BreakerConfig {
threshold: config.breaker_threshold,
cooldown: duration(
diff --git a/crates/switchyard-runner/src/config.rs b/crates/switchyard-runner/src/config.rs
index 288e4586f..82010f6c3 100644
--- a/crates/switchyard-runner/src/config.rs
+++ b/crates/switchyard-runner/src/config.rs
@@ -1175,6 +1175,9 @@ task_typing = false
let runner = runner_from_toml(&vgr_config())?;
assert!(runner.route("switchyard/vgr").is_some());
+ let compact = format!("{}\ncompact_handoff = true\n", vgr_config());
+ assert!(error_message(&compact).contains("unknown field `compact_handoff`"));
+
let identical_ids =
vgr_config().replace("local_target = \"weak\"", "local_target = \"strong\"");
assert!(error_message(&identical_ids).contains("distinct local and cloud model IDs"));
diff --git a/docs/routing_algorithms/vgr_routing.md b/docs/routing_algorithms/vgr_routing.md
index ba0e1dde8..21cc9d4f9 100644
--- a/docs/routing_algorithms/vgr_routing.md
+++ b/docs/routing_algorithms/vgr_routing.md
@@ -38,6 +38,22 @@ route sets `context_window`, `/v1/models` advertises that configured limit.
run that recovered from tool errors commit locally once that many trailing tool
results are clean and the cloud judge confirms the evidence.
+## Agentic handoff
+
+These settings are off by default:
+
+```toml
+agentic_handoff = true
+local_turn_budget_seconds = 600
+```
+
+`agentic_handoff` tells the cloud tier when it takes over tool-using work that
+the local tier previously served. The notice can include the local tier's
+unchecked final message. It is sent at most once per user turn, and only after
+new local work has been served since the previous handoff. The handoff preserves
+the conversation's existing messages and content. `local_turn_budget_seconds`
+escalates a user turn once the local tier has spent that much wall-clock time on it.
+
## Serving modes
- `off` skips candidate generation and serves cloud. This is the default.