Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.
Rust
fix(vgr): repair user-turn holds and session handoffs (#1041)
GDRSe06530206423
diff --git a/crates/libsy/src/algorithms/util/affinity.rs b/crates/libsy/src/algorithms/util/affinity.rsindex 48e00ecb6..8b3840fa1 100644--- a/crates/libsy/src/algorithms/util/affinity.rs+++ b/crates/libsy/src/algorithms/util/affinity.rs@@ -50,7 +50,7 @@ pub enum ClassifyTrigger {/// Anthropic carries tool results as a `Role::User` message, so role alone cannot tell a/// human turn from a tool continuation.-fn is_user_turn(message: &Message) -> bool {+pub(crate) fn is_user_turn(message: &Message) -> bool {message.role == Role::User&& !message.content@@ -62,6 +62,37 @@ pub(crate) fn has_new_user_turn(messages: &[Message]) -> bool {messages.last().is_some_and(is_user_turn)}+/// Who started the request, `user` or `agent` (GitHub Copilot's convention).+const INITIATOR_HEADER: &str = "x-initiator";++/// The client's `x-initiator` header: `Some(true)` when a person wrote the+/// request, `Some(false)` for the agent's own follow-up calls, `None` when the+/// header is absent or unrecognized and the message shape decides.+pub(crate) fn initiated_by_user(request: &Request) -> Option<bool> {+ let value = request+ .metadata+ .as_ref()?+ .http_headers+ .as_ref()?+ .get(INITIATOR_HEADER)?+ .to_str()+ .ok()?+ .trim();+ if value.eq_ignore_ascii_case("user") {+ Some(true)+ } else if value.eq_ignore_ascii_case("agent") {+ Some(false)+ } else {+ None+ }+}++/// Whether `request` opens a user turn: the `x-initiator` header says so, or else+/// its newest message is the user's.+fn starts_user_turn(request: &Request) -> bool {+ initiated_by_user(request).unwrap_or_else(|| has_new_user_turn(&request.llm_request.messages))+}+/// Retains a model per request identity and forces it on later matching requests.////// Register the same instance as both a processor and a classifier; the two roles share@@ -185,6 +216,10 @@ impl Processor for AffinityRouter {if self.should_latch(selected_model_id) && writable {evict_if_full(&mut assignments);assignments.insert(key, (selected_model_id.clone(), category));+ } else if self.release_on_user_turn && starts_user_turn(request) {+ // The new turn was decided for a model that is not retained, so the+ // previous turn's hold must not reach this turn's tool calls.+ assignments.remove(&key);}}Ok(())@@ -236,7 +271,7 @@ impl Classifier for AffinityRouter {};// Abstaining rather than clearing keeps this read path free of writes, so the judge// decides the turn whatever order concurrent latches land in.- if self.release_on_user_turn && has_new_user_turn(&request.llm_request.messages) {+ if self.release_on_user_turn && starts_user_turn(request) {return Ok((Classification::Scores(Vec::new()), None));}// An empty `any` group carries no information about which models are still@@ -876,6 +911,42 @@ mod tests {Ok(())}+ #[tokio::test]+ async fn a_user_turn_decided_for_an_unlatched_model_ends_the_previous_hold()+ -> Result<(), BoxErr> {+ let router = AffinityRouter::new()+ .with_release_on_user_turn()+ .with_latch_only(["strong"]);+ let mut state = State::default();+ let mut escalated =+ task_request(Some(session("session-1", "agent-a")), "add caching", None);+ retain(&router, &mut state, &mut escalated, "strong").await?;++ let mut spoke = task_request(+ Some(session("session-1", "agent-a")),+ "add caching",+ Some("now add a test"),+ );+ retain(&router, &mut state, &mut spoke, "weak").await?;++ // The new turn's tool continuation is not held on the earlier turn's model.+ let mut continued = spoke.clone();+ continued.llm_request.messages.push(Message {+ role: Role::User,+ content: vec![ContentBlock::ToolResult(ToolResult {+ tool_call_id: "call-1".to_string(),+ content: Vec::new(),+ is_error: None,+ })],+ });+ assert!(+ scores(&router, &mut state, &mut continued)+ .await?+ .is_empty()+ );+ Ok(())+ }+#[tokio::test]async fn latch_only_retains_matching_models() -> Result<(), BoxErr> {let router = AffinityRouter::new().with_latch_only(["strong"]);diff --git a/crates/libsy/src/algorithms/vgr.rs b/crates/libsy/src/algorithms/vgr.rsindex 3509dc2f2..fc99ada46 100644--- a/crates/libsy/src/algorithms/vgr.rs+++ b/crates/libsy/src/algorithms/vgr.rs@@ -43,22 +43,22 @@ impl Vgr {/// Validates and constructs a verification-gated route.pub fn new(config: VgrConfig) -> Result<Self> {config.validate()?;- let cloud = config.targets.cloud.clone();- // Every local turn re-enters verification; an escalation holds until the- // next user turn.- let turn_affinity = Arc::new(- AffinityRouter::new()- .with_release_on_user_turn()- .with_latch_only([cloud.clone()]),- );+ let mut affinity = AffinityRouter::new().with_latch_only([config.targets.cloud.clone()]);+ let reason = if config.latch_escalation && config.mode != ServingMode::Shadow {+ "session_latched"+ } else {+ affinity = affinity.with_release_on_user_turn();+ "turn_verification_latched"+ };+ let affinity = Arc::new(affinity);let classifier = Arc::new(runtime::VgrClassifier {breaker: safety::CircuitBreaker::new(config.breaker),config,});let route = FallThrough::new_with_state().with_name("vgr")- .with_processor(turn_affinity.clone())- .with_classifier(turn_affinity)+ .with_processor(affinity.clone())+ .with_classifier(Arc::new(runtime::Held(affinity, reason))).with_classifier(classifier);Ok(Self { route })}diff --git a/crates/libsy/src/algorithms/vgr/config.rs b/crates/libsy/src/algorithms/vgr/config.rsindex fc7965e6a..987318c03 100644--- a/crates/libsy/src/algorithms/vgr/config.rs+++ b/crates/libsy/src/algorithms/vgr/config.rs@@ -67,6 +67,9 @@ pub struct VgrConfig {/// Wall-clock budget for the local tier within one user turn, including the/// client's tool execution.pub local_turn_budget: Option<Duration>,+ /// Holds an escalated session on the capable tier past later user turns.+ /// Shadow mode retains only the current turn's hold.+ pub latch_escalation: bool,}impl VgrConfig {@@ -88,6 +91,7 @@ impl VgrConfig {confirmed_recovery_min_clean_tail: None,agentic_handoff: false,local_turn_budget: None,+ latch_escalation: false,}}diff --git a/crates/libsy/src/algorithms/vgr/decide.rs b/crates/libsy/src/algorithms/vgr/decide.rsindex 417c34035..b68e275c0 100644--- a/crates/libsy/src/algorithms/vgr/decide.rs+++ b/crates/libsy/src/algorithms/vgr/decide.rs@@ -8,6 +8,9 @@use super::{Branch, Capabilities};+/// Version of these rules and dials, carried on every decision record.+pub(super) const POLICY_VERSION: &str = "1.0";+const READOUT: f64 = 0.9;/// Most tool results a run may contain and still recover without the capable tier.const SHORT_RUN_MAX_TOOL_RESULTS: i32 = 100;diff --git a/crates/libsy/src/algorithms/vgr/render.rs b/crates/libsy/src/algorithms/vgr/render.rsindex 560ae3de3..81c24c10c 100644--- a/crates/libsy/src/algorithms/vgr/render.rs+++ b/crates/libsy/src/algorithms/vgr/render.rs@@ -11,6 +11,8 @@ use switchyard_protocol::{ContentBlock, Request, Role};use super::text::{Turn, clip_mid, content_text, redact, tool_result_failed};const ASSISTANT_TURN_BUDGET: usize = 1200;+// Agent frameworks send system prompts of many thousand tokens; each local rung would re-read them.+const SYSTEM_TURN_BUDGET: usize = 1000;pub(super) const ATTEMPT_BUDGET: usize = 2200;const AGENTIC_TRAJECTORY_BUDGET: usize = 12_000;const AGENTIC_EVENT_BUDGET: usize = 1400;@@ -25,10 +27,12 @@ pub(super) fn render_session(turns: &[Turn], attempt: &str) -> String {if turn.text.trim().is_empty() || Some(index) == latest_user {continue;}- let mut text = redact(&turn.text);- if matches!(turn.role, Role::Assistant | Role::Tool) {- text = clip_mid(&text, ASSISTANT_TURN_BUDGET, 0.5);- }+ let text = redact(&turn.text);+ let text = match turn.role {+ Role::User => text,+ Role::System | Role::Developer => clip_mid(&text, SYSTEM_TURN_BUDGET, 0.5),+ Role::Assistant | Role::Tool => clip_mid(&text, ASSISTANT_TURN_BUDGET, 0.5),+ };lines.push(format!("[{}] {text}", turn.role));}if let Some(index) = latest_user {@@ -112,3 +116,30 @@ fn agentic_trajectory(request: &Request) -> Vec<String> {}events}++#[cfg(test)]+mod tests {+ use super::*;++ #[test]+ fn a_session_view_clips_the_system_prompt_but_not_the_user() {+ let system = "framework rules ".repeat(500);+ let request = "please keep every word of this ".repeat(100);+ let turns = [+ Turn {+ role: Role::System,+ text: system.clone(),+ },+ Turn {+ role: Role::User,+ text: request.clone(),+ },+ ];++ let view = render_session(&turns, "done");++ assert!(view.contains("chars omitted"));+ assert!(view.chars().count() < SYSTEM_TURN_BUDGET + request.len() + 200);+ assert!(view.contains(&request));+ }+}diff --git a/crates/libsy/src/algorithms/vgr/runtime.rs b/crates/libsy/src/algorithms/vgr/runtime.rsindex bd248cfcf..acd05c237 100644--- a/crates/libsy/src/algorithms/vgr/runtime.rs+++ b/crates/libsy/src/algorithms/vgr/runtime.rs@@ -9,17 +9,20 @@//! by what is left of the turn's deadline, and a call that fails or times out//! is evidence never gathered, which does not commit.+use std::sync::Arc;use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};use async_trait::async_trait;-use switchyard_protocol::{AggLlmResponse, ContentBlock, ModelId, Request, Response, StopReason};+use switchyard_protocol::{+ AggLlmResponse, ContentBlock, Message, ModelId, Request, Response, StopReason,+};use super::config::{ServingMode, VgrConfig};use super::decide::{self, AgenticRun, Route, Signals, Tri};use super::rungs::{self, Question};use super::safety::{CircuitBreaker, endpoint_failure, fallback_eligible};use super::{Branch, Capabilities, TaskType, derive_capabilities, readout, text};-use crate::algorithms::util::affinity::has_new_user_turn;+use crate::algorithms::util::affinity::{AffinityRouter, has_new_user_turn, initiated_by_user};use crate::algorithms::util::buffered_response::buffer_response;use crate::algorithms::util::decisive;use crate::algorithms::util::prompts::append_note;@@ -28,8 +31,8 @@ use crate::core::classifier::{Classification, Classifier};use crate::core::state::{State, StateValue};use crate::{LibsyError, Result};-/// Consecutive escalation votes. Reaching [`TURN_CONFIRMATIONS`] latches the session to the-/// capable tier; once latched, verification is skipped, so the count never resets.+/// Whether turn verification latched the capable tier; cleared when the user responds.+const TURN_LATCHED_KEY: &str = "vgr_turn_latched";pub(super) const TURN_STREAK_KEY: &str = "vgr_turn_streak";/// Unix seconds at the first request of the current user turn.const TURN_STARTED_KEY: &str = "vgr_turn_started";@@ -80,6 +83,25 @@ pub(super) struct VgrClassifier {pub(super) breaker: CircuitBreaker,}+/// An escalation hold, recorded as a VGR decision with its reason whenever it serves a call.+pub(super) struct Held(pub(super) Arc<AffinityRouter>, pub(super) &'static str);++#[async_trait]+impl Classifier for Held {+ async fn score(+ &self,+ state: &mut State,+ request: &mut Request,+ driver: &Driver,+ ) -> Result<Scored> {+ let scored = self.0.score(state, request, driver).await?;+ if matches!(&scored.0, Classification::Scores(scores) if !scores.is_empty()) {+ log(driver, self.1, None, Route::Cloud);+ }+ Ok(scored)+ }+}+#[async_trait]impl Classifier for VgrClassifier {async fn score(@@ -90,10 +112,19 @@ impl Classifier for VgrClassifier {) -> Result<Scored> {let started = Instant::now();let local = &self.config.targets.local;- if has_new_user_turn(&request.llm_request.messages) {+ let messages = &request.llm_request.messages;+ let (new_turn, answered) = match initiated_by_user(request) {+ Some(by_user) => (by_user, by_user),+ None => (has_new_user_turn(messages), answers_final_reply(messages)),+ };+ if new_turn {state.extra.remove(TURN_STARTED_KEY);set_flag(state, HANDOFF_SENT_KEY, false);}+ if answered {+ set_flag(state, TURN_LATCHED_KEY, false);+ set_count(state, TURN_STREAK_KEY, 0);+ }let turn_budget_spent = self.turn_budget_spent(state);let short_circuit = if self.config.mode == ServingMode::Off {Some("mode_off")@@ -104,12 +135,14 @@ impl Classifier for VgrClassifier {.is_some_and(|switch| switch.is_engaged()){Some("kill_switch")- } else if turn_streak(state) >= TURN_CONFIRMATIONS {+ } else if flag(state, TURN_LATCHED_KEY) || turn_streak(state) >= TURN_CONFIRMATIONS {Some("turn_verification_latched")} else if turn_budget_spent {Some("local_turn_budget")} else if !self.config.local_supports_images && request_has_image(request) {Some("local_image_unsupported")+ } else if text::newest_prompt_is_blank(request) {+ Some("empty_prompt")} else {None};@@ -125,7 +158,7 @@ impl Classifier for VgrClassifier {};let attempted = tokio::time::timeout(budget, async {let response = driver- .call_model(request.clone(), vec![local.clone()])+ .call_model_with_error_recovery(request.clone(), vec![local.clone()], true).await?;buffer_response(local.as_str(), response).await})@@ -251,7 +284,7 @@ impl VgrClassifier {if !self.config.agentic_handoff|| flag(state, HANDOFF_SENT_KEY)|| !flag(state, LOCAL_SERVED_KEY)- || !text::has_tool_trajectory(request)+ || !text::turn_has_tool_trajectory(request){return;}@@ -283,7 +316,7 @@ impl VgrClassifier {Some(Err(error)) => {if *judge == self.config.targets.local && endpoint_failure(&error) {self.breaker.failure();- set_count(state, TURN_STREAK_KEY, TURN_CONFIRMATIONS);+ set_flag(state, TURN_LATCHED_KEY, true);return true;}return false;@@ -298,6 +331,9 @@ impl VgrClassifier {}let streak = turn_streak(state).saturating_add(1);set_count(state, TURN_STREAK_KEY, streak);+ if streak >= TURN_CONFIRMATIONS {+ set_flag(state, TURN_LATCHED_KEY, true);+ }streak >= TURN_CONFIRMATIONS}@@ -453,13 +489,17 @@ fn unix_seconds() -> u32 {fn handoff_note(unverified_final: Option<&str>) -> String {let mut note = String::from("\n\n[Routing notice from the serving infrastructure, not from the user] Until now the \- work in this session was carried out by a smaller, faster model. That work has NOT been \- verified: it may be incomplete or incorrect, and it may have left the environment in a \- broken state. You are taking over from here. Do not assume that any earlier step \- succeeded or that the task is finished. Inspect the actual current state of the \- environment against every requirement of the original task, fix whatever is missing or \- wrong, and give your final answer only after you have confirmed that the requirements \- are met.",+ work on the user's current request was carried out by a smaller, faster model. That work \+ has NOT been verified: it may be incomplete or incorrect, and it may have left the \+ environment in a broken state. You are taking over from here. Do not assume that any \+ step taken on that request succeeded or that the task is finished. Inspect the actual \+ current state of the environment against every requirement of the request, fix \+ whatever is missing or wrong, and give your final answer only after you have confirmed \+ that the requirements are met. Earlier steps may already have done part of the work. \+ Keep what they did and build on it: confirm a step's effect by inspecting the \+ environment rather than by running the step again, and repeat or reverse a step only \+ when the request cannot be met otherwise. Replies to the user's earlier requests are \+ not in question; do not revisit or correct them unless the user asks.",);if let Some(claim) = unverified_final.filter(|claim| !claim.trim().is_empty()) {const CLIPPING_MARKER_RESERVE: usize = 64;@@ -484,8 +524,24 @@ fn log(driver: &Driver, reason: &'static str, branch: Option<Branch>, served: Ro"reason_code": reason,"verdict": if served == Route::Local { "local" } else { "cloud" },"branch": branch.map(|branch| format!("{branch:?}")),+ "policy_version": decide::POLICY_VERSION,}));- tracing::info!(target: "libsy", reason, branch = ?branch, served = ?served, "vgr decision");+ tracing::info!(+ target: "libsy",+ reason,+ branch = ?branch,+ served = ?served,+ policy_version = decide::POLICY_VERSION,+ "vgr decision"+ );+}++/// Whether the user is answering a finished assistant reply.+fn answers_final_reply(messages: &[Message]) -> bool {+ messages+ .len()+ .checked_sub(1)+ .is_some_and(|last| text::starts_user_turn(messages, last))}fn tool_use_is_complete(response: &AggLlmResponse) -> bool {diff --git a/crates/libsy/src/algorithms/vgr/runtime/tests.rs b/crates/libsy/src/algorithms/vgr/runtime/tests.rsindex f037dcff5..95b079c57 100644--- a/crates/libsy/src/algorithms/vgr/runtime/tests.rs+++ b/crates/libsy/src/algorithms/vgr/runtime/tests.rs@@ -398,6 +398,7 @@ async fn decisions_attach_the_reason_and_exclude_local_cloud_fallbacks() -> Resuassert_eq!(evidence["source"], "vgr");assert_eq!(evidence["reason_code"], reason);assert_eq!(evidence["verdict"], served);+ assert_eq!(evidence["policy_version"], "1.0");if reason == "decided" {assert_eq!(evidence["branch"], "DefaultVerified");}@@ -625,3 +626,241 @@ fn handoff_preserves_existing_messages_and_non_text_content() {classifier.hand_off(&mut task, &mut state, Some("duplicate notice"));assert_eq!(task.llm_request.messages, after_first);}++#[tokio::test]+async fn a_blank_prompt_escalates_without_a_local_attempt() -> Result<()> {+ let continuing = route(|_| {})?;+ let route = route(|_| {})?;+ let calls = Arc::new(Calls::default());+ let blank_follow_up = vec![+ Message::text(Role::User, "what is 2 + 2?"),+ Message::text(Role::Assistant, "4"),+ Message::text(Role::User, " "),+ ];+ for messages in [+ vec![Message::text(Role::User, " \n ")],+ vec![Message::text(Role::User, "\u{200B}\u{FEFF}")],+ blank_follow_up,+ ] {+ assert_eq!(+ drive(&route, session(messages), calls.clone(), complete_call, 0.0).await?,+ "cloud"+ );+ }+ assert_eq!(calls.0.load(Ordering::Relaxed), 0);++ // Tool results sent as a user message continue the task; they are not a blank prompt.+ let mut results = result("a", "file.txt");+ results.role = Role::User;+ let continuation = session(vec![+ Message::text(Role::User, "list files"),+ call("a"),+ results,+ ]);+ drive(&continuing, continuation, calls.clone(), complete_call, 0.0).await?;+ assert_eq!(calls.0.load(Ordering::Relaxed), 1);+ Ok(())+}++#[tokio::test]+async fn an_agent_marked_nudge_keeps_a_single_escalation_on_cloud() -> Result<()> {+ let route = route(|_| {})?;+ let calls = Arc::new(Calls::default());+ let truncated = || tool_call(json!(r#"{"command":"l"#), StopReason::MaxTokens);+ let task = session(vec![Message::text(Role::User, "fix the build")]);+ drive(&route, task, calls.clone(), truncated, 0.0).await?;+ let nudge = vec![+ Message::text(Role::User, "fix the build"),+ Message::text(Role::Assistant, "fixed"),+ Message::text(Role::User, "Verify the change before you stop."),+ ];+ let nudge = initiated_by(session(nudge), "agent");+ assert_eq!(+ drive(&route, nudge, calls.clone(), complete_call, 0.0).await?,+ "cloud"+ );+ assert_eq!(calls.0.load(Ordering::Relaxed), 1);+ Ok(())+}++fn initiated_by(mut request: Request, initiator: &str) -> Request {+ let metadata = request.metadata.get_or_insert_with(Metadata::default);+ metadata+ .http_headers+ .get_or_insert_with(http::HeaderMap::new)+ .insert("x-initiator", initiator.parse().unwrap());+ request+}++#[tokio::test]+async fn a_held_follow_up_records_the_latch_as_its_reason() -> Result<()> {+ use crate::core::algorithm::Call;+ use crate::core::testing::serve_decision;++ let algorithm = route(|config| config.mode = ServingMode::Shadow)?;+ let models = Arc::new(category_models(Category::Any, &["local", "cloud"]).into());+ let serve = |call| async move {+ match call {+ Call::Model(call) => call.respond(std::future::ready(Ok(reply("done")))).await,+ Call::Decision(call) => serve_decision(*call).await,+ }+ };+ let escalated = crate::drive(+ Arc::clone(&algorithm),+ session(vec![Message::text(Role::User, "fix the build")]),+ Arc::clone(&models),+ serve,+ )+ .await?;+ assert_eq!(escalated.selected_model_ids, vec![ModelId::new("cloud")]);++ let held = crate::drive(+ algorithm,+ session(vec![+ Message::text(Role::User, "fix the build"),+ call("c1"),+ result("c1", "ok"),+ ]),+ models,+ serve,+ )+ .await?;+ assert_eq!(held.selected_model_ids, vec![ModelId::new("cloud")]);+ let evidence = held+ .metadata+ .and_then(|metadata| metadata.evidence)+ .unwrap_or_default();+ assert_eq!(evidence["source"], "vgr");+ assert_eq!(evidence["reason_code"], "turn_verification_latched");+ assert_eq!(evidence["verdict"], "cloud");+ Ok(())+}++#[tokio::test]+async fn shadow_rechecks_new_user_turns_with_session_latching_enabled() -> Result<()> {+ let route = route(|config| {+ config.mode = ServingMode::Shadow;+ config.latch_escalation = true;+ })?;+ let calls = Arc::new(Calls::default());+ for messages in [+ vec![Message::text(Role::User, "fix the build")],+ vec![+ Message::text(Role::User, "fix the build"),+ Message::text(Role::Assistant, "fixed"),+ Message::text(Role::User, "now add a test"),+ ],+ ] {+ assert_eq!(+ drive(&route, session(messages), calls.clone(), complete_call, 0.0).await?,+ "cloud"+ );+ }+ assert_eq!(calls.0.load(Ordering::Relaxed), 2);+ Ok(())+}++#[tokio::test]+async fn an_escalation_latch_holds_the_session_past_new_questions() -> Result<()> {+ let route = route(|config| config.latch_escalation = true)?;+ let calls = Arc::new(Calls::default());+ let truncated = || tool_call(json!(r#"{"command":"l"#), StopReason::MaxTokens);+ let task = session(vec![Message::text(Role::User, "fix the build")]);+ assert_eq!(+ drive(&route, task, calls.clone(), truncated, 0.0).await?,+ "cloud"+ );+ let follow_up = session(vec![+ Message::text(Role::User, "fix the build"),+ Message::text(Role::Assistant, "fixed"),+ Message::text(Role::User, "now add a test"),+ ]);+ assert_eq!(+ drive(&route, follow_up, calls.clone(), complete_call, 0.0).await?,+ "cloud"+ );+ assert_eq!(calls.0.load(Ordering::Relaxed), 1);+ Ok(())+}++#[tokio::test]+async fn an_initiator_header_decides_when_the_turn_latch_releases() -> Result<()> {+ let route = route(|_| {})?;+ let calls = Arc::new(Calls::default());+ let task = || session(vec![Message::text(Role::User, "fix the build")]);+ drive(&route, task(), calls.clone(), complete_call, 0.9).await?;+ drive(&route, task(), calls.clone(), complete_call, 0.9).await?;+ assert_eq!(calls.0.load(Ordering::Relaxed), 2);++ // A client nudge after a final reply looks like a new question; marked agent, it keeps the latch.+ let nudge = vec![+ Message::text(Role::User, "fix the build"),+ Message::text(Role::Assistant, "fixed"),+ Message::text(Role::User, "Verify the change before you stop."),+ ];+ assert_eq!(+ drive(+ &route,+ initiated_by(session(nudge), "agent"),+ calls.clone(),+ complete_call,+ 0.0+ )+ .await?,+ "cloud"+ );+ assert_eq!(calls.0.load(Ordering::Relaxed), 2);++ // A person's correction after tool results looks like a continuation; marked user, it is verified afresh.+ let steer = vec![+ Message::text(Role::User, "fix the build"),+ call("c1"),+ result("c1", "ok"),+ Message::text(Role::User, "use the staging config instead"),+ ];+ assert_eq!(+ drive(+ &route,+ initiated_by(session(steer), "user"),+ calls.clone(),+ complete_call,+ 0.0+ )+ .await?,+ "local"+ );+ assert_eq!(calls.0.load(Ordering::Relaxed), 3);+ Ok(())+}++#[tokio::test]+async fn tool_work_from_an_earlier_user_turn_is_not_handed_off() -> Result<()> {+ let route = route(|config| config.agentic_handoff = true)?;+ let (selected, _) = test_drive_with_models(+ route,+ session(vec![+ Message::text(Role::User, "fix the build"),+ call("c1"),+ result("c1", "ok"),+ Message::text(Role::Assistant, "the build passes"),+ Message::text(Role::User, "What is 41 plus 42?"),+ ]),+ category_models(Category::Any, &["local", "cloud"]),+ |target: ModelId, request: Request| async move {+ Ok(match target.as_str() {+ "local" => reply("83"),+ "cloud" => {+ let text =+ serde_json::to_string(&request.llm_request.messages).unwrap_or_default();+ assert!(text.contains("41 plus 42"));+ assert!(!text.contains("Routing notice"));+ reply("83")+ }+ _ => scored(0.0),+ })+ },+ )+ .await?;+ assert_eq!(selected, "cloud");+ Ok(())+}diff --git a/crates/libsy/src/algorithms/vgr/safety.rs b/crates/libsy/src/algorithms/vgr/safety.rsindex f14f0b362..eac01fc36 100644--- a/crates/libsy/src/algorithms/vgr/safety.rs+++ b/crates/libsy/src/algorithms/vgr/safety.rs@@ -141,7 +141,8 @@ pub(super) fn fallback_eligible(error: &LibsyError) -> bool {LibsyError::ClientCall { source, .. } if match source {LlmClientError::ContextWindowExceeded { .. }| LlmClientError::Transport { .. }- | LlmClientError::Timeout { .. } => true,+ | LlmClientError::Timeout { .. }+ | LlmClientError::TemporarilyUnavailable => true,LlmClientError::UpstreamHttp { status, .. } =>matches!(*status,@@ -159,7 +160,9 @@ pub(super) fn endpoint_failure(error: &LibsyError) -> bool {matches!(error,LibsyError::ClientCall { source, .. } if match source {- LlmClientError::Transport { .. } | LlmClientError::Timeout { .. } => true,+ LlmClientError::Transport { .. }+ | LlmClientError::Timeout { .. }+ | LlmClientError::TemporarilyUnavailable => true,LlmClientError::UpstreamHttp { status, .. } => matches!(*status,StatusCode::BAD_GATEWAYdiff --git a/crates/libsy/src/algorithms/vgr/text.rs b/crates/libsy/src/algorithms/vgr/text.rsindex 981b9649b..b011f058f 100644--- a/crates/libsy/src/algorithms/vgr/text.rs+++ b/crates/libsy/src/algorithms/vgr/text.rs@@ -11,6 +11,7 @@ use regex::Regex;use serde_json::Value;use switchyard_protocol::{ContentBlock, InstructionBlock, Message, Request, Role, ToolResult};+use crate::algorithms::util::affinity::is_user_turn;use crate::algorithms::util::tool_signals::classify_text;/// One flattened turn: instructions first, then messages.@@ -93,21 +94,74 @@ pub(super) fn user_task_text(turns: &[Turn]) -> &str {.unwrap_or_default()}+/// Whether the newest user message is empty, ignoring whitespace and zero-width characters.+///+/// Any non-text block, such as an image or a tool result, is content.+pub(super) fn newest_prompt_is_blank(request: &Request) -> bool {+ request+ .llm_request+ .messages+ .iter()+ .rev()+ .find(|message| message.role == Role::User)+ .is_none_or(|message| {+ message.content.iter().all(|block| match block {+ ContentBlock::Text { text } => text.chars().all(|c| {+ c.is_whitespace()+ || matches!(c, '\u{200B}'..='\u{200D}' | '\u{2060}' | '\u{FEFF}')+ }),+ _ => false,+ })+ })+}+pub(super) fn has_assistant_turn(turns: &[Turn]) -> bool {turns.iter().any(|turn| turn.role == Role::Assistant)}/// Whether the conversation contains an executed or proposed tool step.pub(super) fn has_tool_trajectory(request: &Request) -> bool {- request.llm_request.messages.iter().any(|message| {- message.role == Role::Tool- || message.content.iter().any(|block| {- matches!(- block,- ContentBlock::ToolCall(_) | ContentBlock::ToolResult(_)- )- })- })+ request.llm_request.messages.iter().any(is_tool_step)+}++/// Whether the user message at `index` answers a finished assistant reply.+///+/// Agent clients also inject user-role continuations (after context compaction,+/// at an iteration limit) that follow tool results, not a final reply; those+/// continue the current user turn.+pub(super) fn starts_user_turn(messages: &[Message], index: usize) -> bool {+ let (Some(message), Some(previous)) = (+ messages.get(index),+ index.checked_sub(1).and_then(|i| messages.get(i)),+ ) else {+ return false;+ };+ is_user_turn(message)+ && previous.role == Role::Assistant+ && !previous+ .content+ .iter()+ .any(|block| matches!(block, ContentBlock::ToolCall(_)))+}++/// Whether the current user turn contains an executed or proposed tool step.+pub(super) fn turn_has_tool_trajectory(request: &Request) -> bool {+ let messages = &request.llm_request.messages;+ let start = (0..messages.len())+ .rev()+ .find(|&index| starts_user_turn(messages, index))+ .unwrap_or(0);+ messages[start..].iter().any(is_tool_step)+}++fn is_tool_step(message: &Message) -> bool {+ message.role == Role::Tool+ || message.content.iter().any(|block| {+ matches!(+ block,+ ContentBlock::ToolCall(_) | ContentBlock::ToolResult(_)+ )+ })}/// Idempotent secret and PII scrub applied before text reaches a verifier.diff --git a/crates/switchyard-runner/src/algorithm.rs b/crates/switchyard-runner/src/algorithm.rsindex 9729bcde3..11220076e 100644--- a/crates/switchyard-runner/src/algorithm.rs+++ b/crates/switchyard-runner/src/algorithm.rs@@ -552,6 +552,9 @@ pub struct VgrRouteConfig {/// Wall-clock budget for the local tier within one user turn.#[serde(default)]pub local_turn_budget_seconds: Option<f64>,+ /// Holds escalations across later user turns.+ #[serde(default)]+ pub latch_escalation: bool,}#[derive(Clone, Debug, Default, Deserialize)]@@ -1808,17 +1811,18 @@ fn build_vgr(approval: config.active_approval.clone().unwrap_or_default(),},};- runtime.deadline = duration(route_name, "deadline_seconds", config.deadline_seconds)?;+ runtime.deadline = vgr_duration(route_name, "deadline_seconds", config.deadline_seconds)?;runtime.task_typing = config.task_typing;runtime.confirmed_recovery_min_clean_tail = config.confirmed_recovery_min_clean_tail;runtime.agentic_handoff = config.agentic_handoff;+ runtime.latch_escalation = config.latch_escalation;runtime.local_turn_budget = config.local_turn_budget_seconds- .map(|seconds| duration(route_name, "local_turn_budget_seconds", seconds))+ .map(|seconds| vgr_duration(route_name, "local_turn_budget_seconds", seconds)).transpose()?;runtime.breaker = BreakerConfig {threshold: config.breaker_threshold,- cooldown: duration(+ cooldown: vgr_duration(route_name,"breaker_cooldown_seconds",config.breaker_cooldown_seconds,@@ -1831,13 +1835,14 @@ fn build_vgr(})}-fn duration(route: &str, field: &str, seconds: f64) -> AlgorithmResult<std::time::Duration> {- std::time::Duration::try_from_secs_f64(seconds).map_err(|error| {- AlgorithmConfigError::with_source(- format!("vgr route {route}: {field} must be a non-negative number"),- error,- )- })+fn vgr_duration(route: &str, field: &str, seconds: f64) -> AlgorithmResult<std::time::Duration> {+ let message = || format!("vgr route {route}: {field} must be a positive number");+ let duration = std::time::Duration::try_from_secs_f64(seconds)+ .map_err(|error| AlgorithmConfigError::with_source(message(), error))?;+ if duration.is_zero() {+ return Err(AlgorithmConfigError::new(message()));+ }+ Ok(duration)}const fn default_max_reviews() -> u32 {diff --git a/crates/switchyard-runner/src/config.rs b/crates/switchyard-runner/src/config.rsindex 7e3999c43..096af0136 100644--- a/crates/switchyard-runner/src/config.rs+++ b/crates/switchyard-runner/src/config.rs@@ -1184,6 +1184,11 @@ task_typing = falselet missing_target =vgr_config().replace("local_target = \"weak\"", "local_target = \"missing\"");assert!(error_message(&missing_target).contains("unknown target missing"));++ let zero_cooldown = format!("{}breaker_cooldown_seconds = 0\n", vgr_config());+ assert!(+ error_message(&zero_cooldown).contains("breaker_cooldown_seconds must be a positive")+ );Ok(())}diff --git a/crates/switchyard-server/tests/server.rs b/crates/switchyard-server/tests/server.rsindex d0fad52ab..bb0dbf32e 100644--- a/crates/switchyard-server/tests/server.rs+++ b/crates/switchyard-server/tests/server.rs@@ -1853,6 +1853,125 @@ task_typing = falseOk(())}+#[tokio::test]+async fn vgr_escalates_while_the_local_endpoint_is_down() -> TestResult {+ let upstream = MockUpstream::start().await?;+ let closed = std::net::TcpListener::bind("127.0.0.1:0")?.local_addr()?;+ let app = build_switchyard_router(load_test_config(&format!(+ r#"+schema_version = 1++[llm_clients.primary]+format = "openai_chat"+base_url = "{}"++[llm_clients.down]+format = "openai_chat"+base_url = "http://{closed}/v1"++[targets.local]+id = "model/vgr-local"+llm_client = "down"++[targets.cloud]+id = "model/vgr-cloud"+llm_client = "primary"++[routes.vgr]+id = "switchyard/vgr"+type = "vgr"+local_target = "local"+cloud_target = "cloud"+mode = "active"+active_approval = "prospective-validation-and-canary-approved"+task_typing = false+"#,+ upstream.base_url+ ))?);+ // The second request meets the failed endpoint's cooldown rather than its socket.+ for _ in 0..2 {+ let response = send(+ &app,+ "POST",+ "/v1/chat/completions",+ Some(json!({+ "model": "switchyard/vgr",+ "messages": [{"role": "user", "content": "answer the question"}]+ })),+ )+ .await?;+ assert_eq!(response.status, StatusCode::OK, "{}", response.text()?);+ assert_eq!(+ response.headers.get("x-model-router-selected-model"),+ Some(&HeaderValue::from_static("model/vgr-cloud"))+ );+ }+ Ok(())+}++#[tokio::test]+async fn vgr_keeps_agent_nudges_in_their_session_and_releases_user_turns() -> TestResult {+ let upstream = MockUpstream::start().await?;+ for session_header in ["x-switchyard-session-id", "x-session-id"] {+ let app = build_switchyard_router(load_test_config(&format!(+ r#"+schema_version = 1+[llm_clients.primary]+format = "openai_chat"+base_url = "{}"+[targets]+local = {{ id = "model/vgr-local", llm_client = "primary" }}+cloud = {{ id = "model/vgr-cloud", llm_client = "primary" }}+judge = {{ id = "model/vgr-judge", llm_client = "primary" }}+[routes.vgr]+id = "switchyard/vgr"+type = "vgr"+local_target = "local"+cloud_target = "cloud"+judge_target = "judge"+mode = "evaluate"+task_typing = false+"#,+ upstream.base_url+ ))?);+ let initial = json!([{"role": "user", "content": " "}]);+ let nudge = json!([+ {"role": "user", "content": " "},+ {"role": "assistant", "content": "4"},+ {"role": "user", "content": "Verify the answer before stopping."}+ ]);+ for (session, initiator, messages, served) in [+ ("a", Some("user"), &initial, "model/vgr-cloud"),+ ("a", Some("agent"), &nudge, "model/vgr-cloud"),+ ("b", Some("agent"), &nudge, "model/vgr-local"),+ ("a", Some("user"), &nudge, "model/vgr-local"),+ ("a", Some("agent"), &nudge, "model/vgr-local"),+ ("c", None, &initial, "model/vgr-cloud"),+ ("c", None, &nudge, "model/vgr-local"),+ ] {+ let mut headers = vec![(session_header, session)];+ if let Some(initiator) = initiator {+ headers.push(("x-initiator", initiator));+ }+ let response = send_with_headers(+ &app,+ "POST",+ "/v1/chat/completions",+ Some(json!({"model": "switchyard/vgr", "messages": messages})),+ &headers,+ )+ .await?;+ assert_eq!(response.status, StatusCode::OK, "{}", response.text()?);+ assert_eq!(+ response.headers.get("x-model-router-selected-model"),+ Some(&HeaderValue::from_str(served)?),+ "{session_header}: {session} {initiator:?}"+ );+ }+ }+ Ok(())+}+fn metric_line<'a>(metrics: &'a str, name: &str, labels: &[(&str, &str)]) -> Option<&'a str> {metrics.lines().find(|line| {line.starts_with(name)diff --git a/docs/routing_algorithms/vgr_routing.md b/docs/routing_algorithms/vgr_routing.mdindex 9378703a9..b8ca77fa6 100644--- a/docs/routing_algorithms/vgr_routing.md+++ b/docs/routing_algorithms/vgr_routing.md@@ -72,6 +72,37 @@ new local work has been served since the previous handoff. The handoff preservesthe conversation's existing messages and content. `local_turn_budget_seconds`escalates a user turn once the local tier has spent that much wall-clock time on it.+The notice covers the current request and asks the cloud tier to preserve earlier+work, inspect its effects, and repeat or undo steps only when needed.++Escalations release when a new user turn is decided locally. Set+`latch_escalation = true` to retain the cloud tier across later user turns instead.+Shadow mode always releases the hold at the next user turn.+All configured VGR durations must be positive.++## Hermes session headers++Use a Hermes build that supports both provider options below. The session header+must stay the same throughout a conversation; each conversation needs its own ID.++```yaml+model:+ default: assistant+ provider: custom+ base_url: http://localhost:4000/v1+providers:+ switchyard:+ api: http://localhost:4000/v1+ session_affinity_header: x-switchyard-session-id+ initiator_header: true+```++`x-initiator: user` releases the turn hold for a person's new request or steer.+`x-initiator: agent` keeps the hold through tool calls, recovery nudges, and+automatic continuations. Retries retain the original call's tag. Without a+recognized tag, Switchyard uses the message shape to identify a user turn.+`x-session-id` is also accepted as a session-header alias.+## Serving modes- `off` skips candidate generation and serves cloud. This is the default.