Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.
Rust
feat(vgr): hand off escalated agentic runs (#881)
GDRSdf27594b1eb8
diff --git a/crates/libsy/src/algorithms/vgr/config.rs b/crates/libsy/src/algorithms/vgr/config.rsindex 1b6471a52..fc7965e6a 100644--- a/crates/libsy/src/algorithms/vgr/config.rs+++ b/crates/libsy/src/algorithms/vgr/config.rs@@ -61,6 +61,12 @@ pub struct VgrConfig {/// Lets a long agentic run with earlier tool errors commit on a capable-tier/// confirmation once it ends in this many consecutive clean tool results.pub confirmed_recovery_min_clean_tail: Option<u32>,+ /// Tells the capable tier, once per user turn, that it inherits unverified+ /// tool-using work.+ pub agentic_handoff: bool,+ /// Wall-clock budget for the local tier within one user turn, including the+ /// client's tool execution.+ pub local_turn_budget: Option<Duration>,}impl VgrConfig {@@ -80,6 +86,8 @@ impl VgrConfig {task_typing: true,local_supports_images: false,confirmed_recovery_min_clean_tail: None,+ agentic_handoff: false,+ local_turn_budget: None,}}diff --git a/crates/libsy/src/algorithms/vgr/runtime.rs b/crates/libsy/src/algorithms/vgr/runtime.rsindex 706af49d1..51af4121f 100644--- a/crates/libsy/src/algorithms/vgr/runtime.rs+++ b/crates/libsy/src/algorithms/vgr/runtime.rs@@ -9,7 +9,7 @@//! by what is left of the turn's deadline, and a call that fails or times out//! is evidence never gathered, which does not commit.-use std::time::{Duration, Instant};+use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};use async_trait::async_trait;use switchyard_protocol::{AggLlmResponse, ContentBlock, ModelId, Request, Response, StopReason};@@ -19,8 +19,10 @@ use super::decide::{self, AgenticRun, Route, Signals, Tri};use super::rungs::{self, Question};use super::safety::{CircuitBreaker, endpoint_failure, fallback_eligible};use super::{Branch, Capabilities, TaskType, derive_capabilities, readout, text};+use crate::algorithms::util::affinity::has_new_user_turn;use crate::algorithms::util::buffered_response::buffer_response;use crate::algorithms::util::decisive;+use crate::algorithms::util::prompts::append_note;use crate::core::algorithm::Driver;use crate::core::classifier::{Classification, Classifier};use crate::{LibsyError, Result};@@ -29,6 +31,9 @@ use crate::{LibsyError, Result};pub(super) struct VgrState {pub(super) turn_streak: u32,pub(super) turn_latched: bool,+ pub(super) turn_started: Option<u32>,+ pub(super) handoff_sent: bool,+ pub(super) local_served: bool,}struct RequestContext<'a> {@@ -58,6 +63,11 @@ impl Classifier<VgrState> for VgrClassifier {) -> Result<Scored> {let started = Instant::now();let local = &self.config.targets.local;+ if has_new_user_turn(&request.llm_request.messages) {+ state.turn_started = None;+ state.handoff_sent = false;+ }+ let turn_budget_spent = self.turn_budget_spent(state);let short_circuit = if self.config.mode == ServingMode::Off {Some("mode_off")} else if self@@ -69,20 +79,22 @@ impl Classifier<VgrState> for VgrClassifier {Some("kill_switch")} else if state.turn_latched {Some("turn_verification_latched")+ } else if turn_budget_spent {+ Some("local_turn_budget")} else if !self.config.local_supports_images && request_has_image(request) {Some("local_image_unsupported")} else {None};if let Some(reason) = short_circuit {- return Ok(self.escalate(driver, reason, None));+ return Ok(self.escalate(driver, reason, None, request, state, None));}let Some(budget) = self.remaining(started) else {- return Ok(self.escalate(driver, "local_timed_out", None));+ return Ok(self.escalate(driver, "local_timed_out", None, request, state, None));};let Some(trial) = self.breaker.attempt() else {- return Ok(self.escalate(driver, "breaker_open", None));+ return Ok(self.escalate(driver, "breaker_open", None, request, state, None));};let attempted = tokio::time::timeout(budget, async {let response = driver@@ -103,11 +115,11 @@ impl Classifier<VgrState> for VgrClassifier {if !fallback_eligible(&error) {return Err(error);}- return Ok(self.escalate(driver, "local_unavailable", None));+ return Ok(self.escalate(driver, "local_unavailable", None, request, state, None));}Err(_) => {trial.failure();- return Ok(self.escalate(driver, "local_timed_out", None));+ return Ok(self.escalate(driver, "local_timed_out", None, request, state, None));}};@@ -118,15 +130,37 @@ impl Classifier<VgrState> for VgrClassifier {};if rungs::has_tool_call(&buffered.agg) {if !tool_use_is_complete(&buffered.agg) {- return Ok(self.escalate(driver, "malformed_tool_call", None));+ return Ok(self.escalate(+ driver,+ "malformed_tool_call",+ None,+ request,+ state,+ None,+ ));}if self.verify_turn(&context, &buffered.agg, state).await {- return Ok(self.escalate(driver, "turn_verification_escalated", None));+ return Ok(self.escalate(+ driver,+ "turn_verification_escalated",+ None,+ request,+ state,+ None,+ ));}if self.config.mode == ServingMode::Shadow {- return Ok(self.escalate(driver, "turn_verification_complete", None));+ return Ok(self.escalate(+ driver,+ "turn_verification_complete",+ None,+ request,+ state,+ None,+ ));}log(driver, "turn_verification_complete", None, Route::Local);+ state.local_served = true;return Ok((decisive(local), Some(buffered.into_response())));}@@ -136,19 +170,74 @@ impl Classifier<VgrState> for VgrClassifier {let signals = self.gather(&context, &caps).await;let route = decide::decide(&caps, &signals, self.config.confirmed_min_clean_tail());if route == Route::Cloud || self.config.mode == ServingMode::Shadow {- return Ok(self.escalate(driver, "decided", Some(caps.branch)));+ return Ok(self.escalate(+ driver,+ "decided",+ Some(caps.branch),+ request,+ state,+ Some(&attempt),+ ));}log(driver, "decided", Some(caps.branch), Route::Local);+ state.local_served = true;Ok((decisive(local), Some(buffered.into_response())))}}impl VgrClassifier {- fn escalate(&self, driver: &Driver, reason: &'static str, branch: Option<Branch>) -> Scored {+ fn escalate(+ &self,+ driver: &Driver,+ reason: &'static str,+ branch: Option<Branch>,+ request: &mut Request,+ state: &mut VgrState,+ unverified_final: Option<&str>,+ ) -> Scored {+ if reason != "mode_off" {+ self.hand_off(request, state, unverified_final);+ }log(driver, reason, branch, Route::Cloud);(decisive(&self.config.targets.cloud), None)}+ /// Whether the current user turn has used up its local wall-clock budget.+ ///+ /// The clock starts at the first request of the user turn and includes the+ /// client's tool execution, since that is the time the client's own limit+ /// is spending.+ fn turn_budget_spent(&self, state: &mut VgrState) -> bool {+ let Some(budget) = self.config.local_turn_budget else {+ return false;+ };+ let now = unix_seconds();+ let started = *state.turn_started.get_or_insert(now);+ u64::from(now.saturating_sub(started)) >= budget.as_secs()+ }++ /// Tells the capable tier it is inheriting unverified tool-using work.+ ///+ /// Sent at most once per user turn, on its first escalation.+ fn hand_off(+ &self,+ request: &mut Request,+ state: &mut VgrState,+ unverified_final: Option<&str>,+ ) {+ if !self.config.agentic_handoff+ || state.handoff_sent+ || !state.local_served+ || !text::has_tool_trajectory(request)+ {+ return;+ }+ state.handoff_sent = true;+ state.local_served = false;+ let note = handoff_note(unverified_final);+ append_note(request, ¬e);+ }+/// Judges one proposed tool call; `true` escalates the rest of the session.async fn verify_turn(&self,@@ -332,6 +421,43 @@ impl VgrClassifier {}}+fn unix_seconds() -> u32 {+ SystemTime::now()+ .duration_since(UNIX_EPOCH)+ .map_or(0, |elapsed| {+ u32::try_from(elapsed.as_secs()).unwrap_or(u32::MAX)+ })+}++/// The note a capable tier receives when it takes over a tool-using session.+fn handoff_note(unverified_final: Option<&str>) -> String {+ let mut note = String::from(+ "\n\n[Routing notice from the serving infrastructure, not from the user] Until now the \+ work in this session was carried out by a smaller, faster model. That work has NOT been \+ verified: it may be incomplete or incorrect, and it may have left the environment in a \+ broken state. You are taking over from here. Do not assume that any earlier step \+ succeeded or that the task is finished. Inspect the actual current state of the \+ environment against every requirement of the original task, fix whatever is missing or \+ wrong, and give your final answer only after you have confirmed that the requirements \+ are met.",+ );+ if let Some(claim) = unverified_final.filter(|claim| !claim.trim().is_empty()) {+ const CLIPPING_MARKER_RESERVE: usize = 64;+ let claim = text::clip_mid(+ claim,+ super::render::ATTEMPT_BUDGET.saturating_sub(CLIPPING_MARKER_RESERVE),+ 1.0 / 3.0,+ );+ note.push_str(+ "\n\nThe smaller model was about to finish with the following unchecked message:\n\+ <unverified_final_message>\n",+ );+ note.push_str(&claim);+ note.push_str("\n</unverified_final_message>");+ }+ note+}+fn log(driver: &Driver, reason: &'static str, branch: Option<Branch>, served: Route) {driver.set_evidence(serde_json::json!({"source": "vgr",diff --git a/crates/libsy/src/algorithms/vgr/runtime/tests.rs b/crates/libsy/src/algorithms/vgr/runtime/tests.rsindex 9a5a45cfb..ea2f6fdb6 100644--- a/crates/libsy/src/algorithms/vgr/runtime/tests.rs+++ b/crates/libsy/src/algorithms/vgr/runtime/tests.rs@@ -398,3 +398,226 @@ async fn decisions_attach_the_reason_and_exclude_local_cloud_fallbacks() -> Resu}Ok(())}++#[tokio::test]+async fn cloud_only_work_does_not_get_a_local_handoff_notice() -> Result<()> {+ for scenario in ["shadow", "kill_switch", "local_turn_budget"] {+ let route = route(|config| {+ config.agentic_handoff = true;+ match scenario {+ "shadow" => config.mode = ServingMode::Shadow,+ "kill_switch" => {+ let switch = super::super::safety::KillSwitch::new();+ switch.engage();+ config.kill_switch = Some(switch);+ }+ _ => config.local_turn_budget = Some(std::time::Duration::ZERO),+ }+ })?;+ let task = session(vec![+ Message::text(Role::User, "fix the build"),+ call("c1"),+ result("c1", "ok"),+ ]);+ let (selected, _) = test_drive_with_models(+ route,+ task,+ category_models(Category::Any, &["local", "cloud"]),+ move |target: ModelId, request: Request| async move {+ Ok(match target.as_str() {+ "local" => complete_call(),+ "judge" => scored(0.0),+ _ => {+ let text = serde_json::to_string(&request.llm_request.messages)+ .unwrap_or_default();+ assert!(!text.contains("Routing notice"), "{scenario}");+ reply("cloud answer")+ }+ })+ },+ )+ .await?;+ assert_eq!(selected, "cloud");+ }+ Ok(())+}++#[tokio::test]+async fn latched_sessions_do_not_repeat_the_handoff_on_later_user_turns() -> Result<()> {+ let route = route(|config| config.agentic_handoff = true)?;+ assert_eq!(+ drive(+ &route,+ session(vec![Message::text(Role::User, "fix the build")]),+ Arc::new(Calls::default()),+ complete_call,+ 0.9,+ )+ .await?,+ "local"+ );+ let sent = Arc::new(parking_lot::Mutex::new(Vec::new()));+ for new_user_turn in [false, true] {+ let mut messages = vec![+ Message::text(Role::User, "fix the build"),+ call("c1"),+ result("c1", "ok"),+ ];+ if new_user_turn {+ messages.push(Message::text(Role::User, "now run the tests"));+ }+ let sent = Arc::clone(&sent);+ let (selected, _) = test_drive_with_models(+ Arc::clone(&route),+ session(messages),+ category_models(Category::Any, &["local", "cloud"]),+ move |target: ModelId, request: Request| {+ let sent = Arc::clone(&sent);+ async move {+ Ok(match target.as_str() {+ "local" => complete_call(),+ "judge" => scored(0.9),+ _ => {+ sent.lock().push(+ serde_json::to_string(&request.llm_request.messages)+ .unwrap_or_default(),+ );+ reply("cloud answer")+ }+ })+ }+ },+ )+ .await?;+ assert_eq!(selected, "cloud");+ }+ let sent = sent.lock();+ assert_eq!(sent.len(), 2);+ assert!(sent[0].contains("Routing notice"));+ assert!(!sent[1].contains("Routing notice"));+ Ok(())+}++#[tokio::test]+async fn an_escalated_agentic_run_hands_off_once_with_the_unverified_claim() -> Result<()> {+ let route = route(|config| config.agentic_handoff = true)?;+ assert_eq!(+ drive(+ &route,+ session(vec![Message::text(Role::User, "fix the build")]),+ Arc::new(Calls::default()),+ complete_call,+ 0.0,+ )+ .await?,+ "local"+ );+ let sent = Arc::new(parking_lot::Mutex::new(Vec::new()));+ let task = || {+ session(vec![+ Message::text(Role::User, "fix the build"),+ call("c1"),+ result("c1", "ok"),+ ])+ };+ for _ in 0..2 {+ let sent = Arc::clone(&sent);+ test_drive_with_models(+ Arc::clone(&route),+ task(),+ category_models(Category::Any, &["local", "cloud"]),+ move |target: ModelId, request: Request| {+ let sent = Arc::clone(&sent);+ async move {+ Ok(match target.as_str() {+ "local" => reply("the build passes"),+ "judge" => scored(0.0),+ "cloud" => {+ let text = serde_json::to_string(&request.llm_request.messages)+ .unwrap_or_default();+ sent.lock().push(text);+ reply("cloud answer")+ }+ _ => reply("no"),+ })+ }+ },+ )+ .await?;+ }+ let sent = sent.lock();+ assert_eq!(sent.len(), 2);+ assert!(sent[0].contains("Routing notice") && sent[0].contains("the build passes"));+ assert!(+ !sent[1].contains("Routing notice"),+ "sent once per user turn"+ );+ Ok(())+}++#[tokio::test]+async fn a_spent_local_turn_budget_skips_the_local_attempt() -> Result<()> {+ let route = route(|config| config.local_turn_budget = Some(std::time::Duration::ZERO))?;+ let calls = Arc::new(Calls::default());+ let task = session(vec![Message::text(Role::User, "fix the build")]);+ assert_eq!(+ drive(&route, task, calls.clone(), complete_call, 0.0).await?,+ "cloud"+ );+ assert_eq!(calls.0.load(Ordering::Relaxed), 0);+ Ok(())+}++#[test]+fn handoff_preserves_existing_messages_and_non_text_content() {+ use switchyard_protocol::ImageSource;++ let mut classifier = recovering_classifier();+ classifier.config.agentic_handoff = true;+ let mut task = request(vec![+ Message::text(Role::User, "finish the task"),+ Message::text(Role::User, "keep the original files"),+ call("c1"),+ result("c1", "the command failed"),+ Message::text(Role::User, "preserve this requirement"),+ Message::text(Role::User, "continue"),+ Message::text(Role::User, "then report the result"),+ ]);+ if let ContentBlock::ToolResult(result) = &mut task.llm_request.messages[3].content[0] {+ result.content.push(ContentBlock::Image {+ source: ImageSource::Url {+ url: "https://example.test/result.png".into(),+ detail: None,+ },+ });+ }+ let original = task.llm_request.messages.clone();+ let mut state = super::VgrState {+ local_served: true,+ ..Default::default()+ };+ classifier.hand_off(&mut task, &mut state, Some("unchecked result"));+ // A notice may be appended, but all original content stays unchanged.+ let messages = &task.llm_request.messages;+ assert_eq!(messages.len(), original.len());+ assert_eq!(+ &messages[..original.len() - 1],+ &original[..original.len() - 1]+ );+ let last = messages.last().expect("last user message");+ assert_eq!(+ &last.content[..original+ .last()+ .expect("original last message")+ .content+ .len()],+ &original.last().expect("original last message").content+ );+ assert!(+ last.text_content("")+ .is_some_and(|text| text.contains("unchecked result"))+ );+ let after_first = task.llm_request.messages.clone();+ classifier.hand_off(&mut task, &mut state, Some("duplicate notice"));+ assert_eq!(task.llm_request.messages, after_first);+}diff --git a/crates/switchyard-runner/src/algorithm.rs b/crates/switchyard-runner/src/algorithm.rsindex edf4cc287..a59eb2e3c 100644--- a/crates/switchyard-runner/src/algorithm.rs+++ b/crates/switchyard-runner/src/algorithm.rs@@ -546,6 +546,12 @@ pub struct VgrRouteConfig {/// Clean trailing tool results that let a long run recover on the cloud judge.#[serde(default)]pub confirmed_recovery_min_clean_tail: Option<u32>,+ /// Tells the capable tier it inherits unverified tool-using work.+ #[serde(default)]+ pub agentic_handoff: bool,+ /// Wall-clock budget for the local tier within one user turn.+ #[serde(default)]+ pub local_turn_budget_seconds: Option<f64>,}#[derive(Clone, Debug, Default, Deserialize)]@@ -1798,6 +1804,11 @@ fn build_vgr(runtime.deadline = duration(route_name, "deadline_seconds", config.deadline_seconds)?;runtime.task_typing = config.task_typing;runtime.confirmed_recovery_min_clean_tail = config.confirmed_recovery_min_clean_tail;+ runtime.agentic_handoff = config.agentic_handoff;+ runtime.local_turn_budget = config+ .local_turn_budget_seconds+ .map(|seconds| duration(route_name, "local_turn_budget_seconds", seconds))+ .transpose()?;runtime.breaker = BreakerConfig {threshold: config.breaker_threshold,cooldown: duration(diff --git a/crates/switchyard-runner/src/config.rs b/crates/switchyard-runner/src/config.rsindex 288e4586f..82010f6c3 100644--- a/crates/switchyard-runner/src/config.rs+++ b/crates/switchyard-runner/src/config.rs@@ -1175,6 +1175,9 @@ task_typing = falselet runner = runner_from_toml(&vgr_config())?;assert!(runner.route("switchyard/vgr").is_some());+ let compact = format!("{}\ncompact_handoff = true\n", vgr_config());+ assert!(error_message(&compact).contains("unknown field `compact_handoff`"));+let identical_ids =vgr_config().replace("local_target = \"weak\"", "local_target = \"strong\"");assert!(error_message(&identical_ids).contains("distinct local and cloud model IDs"));diff --git a/docs/routing_algorithms/vgr_routing.md b/docs/routing_algorithms/vgr_routing.mdindex ba0e1dde8..21cc9d4f9 100644--- a/docs/routing_algorithms/vgr_routing.md+++ b/docs/routing_algorithms/vgr_routing.md@@ -38,6 +38,22 @@ route sets `context_window`, `/v1/models` advertises that configured limit.run that recovered from tool errors commit locally once that many trailing toolresults are clean and the cloud judge confirms the evidence.+## Agentic handoff++These settings are off by default:++```toml+agentic_handoff = true+local_turn_budget_seconds = 600+```++`agentic_handoff` tells the cloud tier when it takes over tool-using work that+the local tier previously served. The notice can include the local tier's+unchecked final message. It is sent at most once per user turn, and only after+new local work has been served since the previous handoff. The handoff preserves+the conversation's existing messages and content. `local_turn_budget_seconds`+escalates a user turn once the local tier has spent that much wall-clock time on it.+## Serving modes- `off` skips candidate generation and serves cloud. This is the default.