Skip to content

Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.

Rust

fix(vgr): repair user-turn holds and session handoffs (#1041)

GDRSe06530206423
diff --git a/crates/libsy/src/algorithms/util/affinity.rs b/crates/libsy/src/algorithms/util/affinity.rs
index 48e00ecb6..8b3840fa1 100644
--- a/crates/libsy/src/algorithms/util/affinity.rs
+++ b/crates/libsy/src/algorithms/util/affinity.rs
@@ -50,7 +50,7 @@ pub enum ClassifyTrigger {
/// Anthropic carries tool results as a `Role::User` message, so role alone cannot tell a
/// human turn from a tool continuation.
-fn is_user_turn(message: &Message) -> bool {
+pub(crate) fn is_user_turn(message: &Message) -> bool {
message.role == Role::User
&& !message
.content
@@ -62,6 +62,37 @@ pub(crate) fn has_new_user_turn(messages: &[Message]) -> bool {
messages.last().is_some_and(is_user_turn)
}
+/// Who started the request, `user` or `agent` (GitHub Copilot's convention).
+const INITIATOR_HEADER: &str = "x-initiator";
+
+/// The client's `x-initiator` header: `Some(true)` when a person wrote the
+/// request, `Some(false)` for the agent's own follow-up calls, `None` when the
+/// header is absent or unrecognized and the message shape decides.
+pub(crate) fn initiated_by_user(request: &Request) -> Option<bool> {
+ let value = request
+ .metadata
+ .as_ref()?
+ .http_headers
+ .as_ref()?
+ .get(INITIATOR_HEADER)?
+ .to_str()
+ .ok()?
+ .trim();
+ if value.eq_ignore_ascii_case("user") {
+ Some(true)
+ } else if value.eq_ignore_ascii_case("agent") {
+ Some(false)
+ } else {
+ None
+ }
+}
+
+/// Whether `request` opens a user turn: the `x-initiator` header says so, or else
+/// its newest message is the user's.
+fn starts_user_turn(request: &Request) -> bool {
+ initiated_by_user(request).unwrap_or_else(|| has_new_user_turn(&request.llm_request.messages))
+}
+
/// Retains a model per request identity and forces it on later matching requests.
///
/// Register the same instance as both a processor and a classifier; the two roles share
@@ -185,6 +216,10 @@ impl Processor for AffinityRouter {
if self.should_latch(selected_model_id) && writable {
evict_if_full(&mut assignments);
assignments.insert(key, (selected_model_id.clone(), category));
+ } else if self.release_on_user_turn && starts_user_turn(request) {
+ // The new turn was decided for a model that is not retained, so the
+ // previous turn's hold must not reach this turn's tool calls.
+ assignments.remove(&key);
}
}
Ok(())
@@ -236,7 +271,7 @@ impl Classifier for AffinityRouter {
};
// Abstaining rather than clearing keeps this read path free of writes, so the judge
// decides the turn whatever order concurrent latches land in.
- if self.release_on_user_turn && has_new_user_turn(&request.llm_request.messages) {
+ if self.release_on_user_turn && starts_user_turn(request) {
return Ok((Classification::Scores(Vec::new()), None));
}
// An empty `any` group carries no information about which models are still
@@ -876,6 +911,42 @@ mod tests {
Ok(())
}
+ #[tokio::test]
+ async fn a_user_turn_decided_for_an_unlatched_model_ends_the_previous_hold()
+ -> Result<(), BoxErr> {
+ let router = AffinityRouter::new()
+ .with_release_on_user_turn()
+ .with_latch_only(["strong"]);
+ let mut state = State::default();
+ let mut escalated =
+ task_request(Some(session("session-1", "agent-a")), "add caching", None);
+ retain(&router, &mut state, &mut escalated, "strong").await?;
+
+ let mut spoke = task_request(
+ Some(session("session-1", "agent-a")),
+ "add caching",
+ Some("now add a test"),
+ );
+ retain(&router, &mut state, &mut spoke, "weak").await?;
+
+ // The new turn's tool continuation is not held on the earlier turn's model.
+ let mut continued = spoke.clone();
+ continued.llm_request.messages.push(Message {
+ role: Role::User,
+ content: vec![ContentBlock::ToolResult(ToolResult {
+ tool_call_id: "call-1".to_string(),
+ content: Vec::new(),
+ is_error: None,
+ })],
+ });
+ assert!(
+ scores(&router, &mut state, &mut continued)
+ .await?
+ .is_empty()
+ );
+ Ok(())
+ }
+
#[tokio::test]
async fn latch_only_retains_matching_models() -> Result<(), BoxErr> {
let router = AffinityRouter::new().with_latch_only(["strong"]);
diff --git a/crates/libsy/src/algorithms/vgr.rs b/crates/libsy/src/algorithms/vgr.rs
index 3509dc2f2..fc99ada46 100644
--- a/crates/libsy/src/algorithms/vgr.rs
+++ b/crates/libsy/src/algorithms/vgr.rs
@@ -43,22 +43,22 @@ impl Vgr {
/// Validates and constructs a verification-gated route.
pub fn new(config: VgrConfig) -> Result<Self> {
config.validate()?;
- let cloud = config.targets.cloud.clone();
- // Every local turn re-enters verification; an escalation holds until the
- // next user turn.
- let turn_affinity = Arc::new(
- AffinityRouter::new()
- .with_release_on_user_turn()
- .with_latch_only([cloud.clone()]),
- );
+ let mut affinity = AffinityRouter::new().with_latch_only([config.targets.cloud.clone()]);
+ let reason = if config.latch_escalation && config.mode != ServingMode::Shadow {
+ "session_latched"
+ } else {
+ affinity = affinity.with_release_on_user_turn();
+ "turn_verification_latched"
+ };
+ let affinity = Arc::new(affinity);
let classifier = Arc::new(runtime::VgrClassifier {
breaker: safety::CircuitBreaker::new(config.breaker),
config,
});
let route = FallThrough::new_with_state()
.with_name("vgr")
- .with_processor(turn_affinity.clone())
- .with_classifier(turn_affinity)
+ .with_processor(affinity.clone())
+ .with_classifier(Arc::new(runtime::Held(affinity, reason)))
.with_classifier(classifier);
Ok(Self { route })
}
diff --git a/crates/libsy/src/algorithms/vgr/config.rs b/crates/libsy/src/algorithms/vgr/config.rs
index fc7965e6a..987318c03 100644
--- a/crates/libsy/src/algorithms/vgr/config.rs
+++ b/crates/libsy/src/algorithms/vgr/config.rs
@@ -67,6 +67,9 @@ pub struct VgrConfig {
/// Wall-clock budget for the local tier within one user turn, including the
/// client's tool execution.
pub local_turn_budget: Option<Duration>,
+ /// Holds an escalated session on the capable tier past later user turns.
+ /// Shadow mode retains only the current turn's hold.
+ pub latch_escalation: bool,
}
impl VgrConfig {
@@ -88,6 +91,7 @@ impl VgrConfig {
confirmed_recovery_min_clean_tail: None,
agentic_handoff: false,
local_turn_budget: None,
+ latch_escalation: false,
}
}
diff --git a/crates/libsy/src/algorithms/vgr/decide.rs b/crates/libsy/src/algorithms/vgr/decide.rs
index 417c34035..b68e275c0 100644
--- a/crates/libsy/src/algorithms/vgr/decide.rs
+++ b/crates/libsy/src/algorithms/vgr/decide.rs
@@ -8,6 +8,9 @@
use super::{Branch, Capabilities};
+/// Version of these rules and dials, carried on every decision record.
+pub(super) const POLICY_VERSION: &str = "1.0";
+
const READOUT: f64 = 0.9;
/// Most tool results a run may contain and still recover without the capable tier.
const SHORT_RUN_MAX_TOOL_RESULTS: i32 = 100;
diff --git a/crates/libsy/src/algorithms/vgr/render.rs b/crates/libsy/src/algorithms/vgr/render.rs
index 560ae3de3..81c24c10c 100644
--- a/crates/libsy/src/algorithms/vgr/render.rs
+++ b/crates/libsy/src/algorithms/vgr/render.rs
@@ -11,6 +11,8 @@ use switchyard_protocol::{ContentBlock, Request, Role};
use super::text::{Turn, clip_mid, content_text, redact, tool_result_failed};
const ASSISTANT_TURN_BUDGET: usize = 1200;
+// Agent frameworks send system prompts of many thousand tokens; each local rung would re-read them.
+const SYSTEM_TURN_BUDGET: usize = 1000;
pub(super) const ATTEMPT_BUDGET: usize = 2200;
const AGENTIC_TRAJECTORY_BUDGET: usize = 12_000;
const AGENTIC_EVENT_BUDGET: usize = 1400;
@@ -25,10 +27,12 @@ pub(super) fn render_session(turns: &[Turn], attempt: &str) -> String {
if turn.text.trim().is_empty() || Some(index) == latest_user {
continue;
}
- let mut text = redact(&turn.text);
- if matches!(turn.role, Role::Assistant | Role::Tool) {
- text = clip_mid(&text, ASSISTANT_TURN_BUDGET, 0.5);
- }
+ let text = redact(&turn.text);
+ let text = match turn.role {
+ Role::User => text,
+ Role::System | Role::Developer => clip_mid(&text, SYSTEM_TURN_BUDGET, 0.5),
+ Role::Assistant | Role::Tool => clip_mid(&text, ASSISTANT_TURN_BUDGET, 0.5),
+ };
lines.push(format!("[{}] {text}", turn.role));
}
if let Some(index) = latest_user {
@@ -112,3 +116,30 @@ fn agentic_trajectory(request: &Request) -> Vec<String> {
}
events
}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn a_session_view_clips_the_system_prompt_but_not_the_user() {
+ let system = "framework rules ".repeat(500);
+ let request = "please keep every word of this ".repeat(100);
+ let turns = [
+ Turn {
+ role: Role::System,
+ text: system.clone(),
+ },
+ Turn {
+ role: Role::User,
+ text: request.clone(),
+ },
+ ];
+
+ let view = render_session(&turns, "done");
+
+ assert!(view.contains("chars omitted"));
+ assert!(view.chars().count() < SYSTEM_TURN_BUDGET + request.len() + 200);
+ assert!(view.contains(&request));
+ }
+}
diff --git a/crates/libsy/src/algorithms/vgr/runtime.rs b/crates/libsy/src/algorithms/vgr/runtime.rs
index bd248cfcf..acd05c237 100644
--- a/crates/libsy/src/algorithms/vgr/runtime.rs
+++ b/crates/libsy/src/algorithms/vgr/runtime.rs
@@ -9,17 +9,20 @@
//! by what is left of the turn's deadline, and a call that fails or times out
//! is evidence never gathered, which does not commit.
+use std::sync::Arc;
use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
use async_trait::async_trait;
-use switchyard_protocol::{AggLlmResponse, ContentBlock, ModelId, Request, Response, StopReason};
+use switchyard_protocol::{
+ AggLlmResponse, ContentBlock, Message, ModelId, Request, Response, StopReason,
+};
use super::config::{ServingMode, VgrConfig};
use super::decide::{self, AgenticRun, Route, Signals, Tri};
use super::rungs::{self, Question};
use super::safety::{CircuitBreaker, endpoint_failure, fallback_eligible};
use super::{Branch, Capabilities, TaskType, derive_capabilities, readout, text};
-use crate::algorithms::util::affinity::has_new_user_turn;
+use crate::algorithms::util::affinity::{AffinityRouter, has_new_user_turn, initiated_by_user};
use crate::algorithms::util::buffered_response::buffer_response;
use crate::algorithms::util::decisive;
use crate::algorithms::util::prompts::append_note;
@@ -28,8 +31,8 @@ use crate::core::classifier::{Classification, Classifier};
use crate::core::state::{State, StateValue};
use crate::{LibsyError, Result};
-/// Consecutive escalation votes. Reaching [`TURN_CONFIRMATIONS`] latches the session to the
-/// capable tier; once latched, verification is skipped, so the count never resets.
+/// Whether turn verification latched the capable tier; cleared when the user responds.
+const TURN_LATCHED_KEY: &str = "vgr_turn_latched";
pub(super) const TURN_STREAK_KEY: &str = "vgr_turn_streak";
/// Unix seconds at the first request of the current user turn.
const TURN_STARTED_KEY: &str = "vgr_turn_started";
@@ -80,6 +83,25 @@ pub(super) struct VgrClassifier {
pub(super) breaker: CircuitBreaker,
}
+/// An escalation hold, recorded as a VGR decision with its reason whenever it serves a call.
+pub(super) struct Held(pub(super) Arc<AffinityRouter>, pub(super) &'static str);
+
+#[async_trait]
+impl Classifier for Held {
+ async fn score(
+ &self,
+ state: &mut State,
+ request: &mut Request,
+ driver: &Driver,
+ ) -> Result<Scored> {
+ let scored = self.0.score(state, request, driver).await?;
+ if matches!(&scored.0, Classification::Scores(scores) if !scores.is_empty()) {
+ log(driver, self.1, None, Route::Cloud);
+ }
+ Ok(scored)
+ }
+}
+
#[async_trait]
impl Classifier for VgrClassifier {
async fn score(
@@ -90,10 +112,19 @@ impl Classifier for VgrClassifier {
) -> Result<Scored> {
let started = Instant::now();
let local = &self.config.targets.local;
- if has_new_user_turn(&request.llm_request.messages) {
+ let messages = &request.llm_request.messages;
+ let (new_turn, answered) = match initiated_by_user(request) {
+ Some(by_user) => (by_user, by_user),
+ None => (has_new_user_turn(messages), answers_final_reply(messages)),
+ };
+ if new_turn {
state.extra.remove(TURN_STARTED_KEY);
set_flag(state, HANDOFF_SENT_KEY, false);
}
+ if answered {
+ set_flag(state, TURN_LATCHED_KEY, false);
+ set_count(state, TURN_STREAK_KEY, 0);
+ }
let turn_budget_spent = self.turn_budget_spent(state);
let short_circuit = if self.config.mode == ServingMode::Off {
Some("mode_off")
@@ -104,12 +135,14 @@ impl Classifier for VgrClassifier {
.is_some_and(|switch| switch.is_engaged())
{
Some("kill_switch")
- } else if turn_streak(state) >= TURN_CONFIRMATIONS {
+ } else if flag(state, TURN_LATCHED_KEY) || turn_streak(state) >= TURN_CONFIRMATIONS {
Some("turn_verification_latched")
} else if turn_budget_spent {
Some("local_turn_budget")
} else if !self.config.local_supports_images && request_has_image(request) {
Some("local_image_unsupported")
+ } else if text::newest_prompt_is_blank(request) {
+ Some("empty_prompt")
} else {
None
};
@@ -125,7 +158,7 @@ impl Classifier for VgrClassifier {
};
let attempted = tokio::time::timeout(budget, async {
let response = driver
- .call_model(request.clone(), vec![local.clone()])
+ .call_model_with_error_recovery(request.clone(), vec![local.clone()], true)
.await?;
buffer_response(local.as_str(), response).await
})
@@ -251,7 +284,7 @@ impl VgrClassifier {
if !self.config.agentic_handoff
|| flag(state, HANDOFF_SENT_KEY)
|| !flag(state, LOCAL_SERVED_KEY)
- || !text::has_tool_trajectory(request)
+ || !text::turn_has_tool_trajectory(request)
{
return;
}
@@ -283,7 +316,7 @@ impl VgrClassifier {
Some(Err(error)) => {
if *judge == self.config.targets.local && endpoint_failure(&error) {
self.breaker.failure();
- set_count(state, TURN_STREAK_KEY, TURN_CONFIRMATIONS);
+ set_flag(state, TURN_LATCHED_KEY, true);
return true;
}
return false;
@@ -298,6 +331,9 @@ impl VgrClassifier {
}
let streak = turn_streak(state).saturating_add(1);
set_count(state, TURN_STREAK_KEY, streak);
+ if streak >= TURN_CONFIRMATIONS {
+ set_flag(state, TURN_LATCHED_KEY, true);
+ }
streak >= TURN_CONFIRMATIONS
}
@@ -453,13 +489,17 @@ fn unix_seconds() -> u32 {
fn handoff_note(unverified_final: Option<&str>) -> String {
let mut note = String::from(
"\n\n[Routing notice from the serving infrastructure, not from the user] Until now the \
- work in this session was carried out by a smaller, faster model. That work has NOT been \
- verified: it may be incomplete or incorrect, and it may have left the environment in a \
- broken state. You are taking over from here. Do not assume that any earlier step \
- succeeded or that the task is finished. Inspect the actual current state of the \
- environment against every requirement of the original task, fix whatever is missing or \
- wrong, and give your final answer only after you have confirmed that the requirements \
- are met.",
+ work on the user's current request was carried out by a smaller, faster model. That work \
+ has NOT been verified: it may be incomplete or incorrect, and it may have left the \
+ environment in a broken state. You are taking over from here. Do not assume that any \
+ step taken on that request succeeded or that the task is finished. Inspect the actual \
+ current state of the environment against every requirement of the request, fix \
+ whatever is missing or wrong, and give your final answer only after you have confirmed \
+ that the requirements are met. Earlier steps may already have done part of the work. \
+ Keep what they did and build on it: confirm a step's effect by inspecting the \
+ environment rather than by running the step again, and repeat or reverse a step only \
+ when the request cannot be met otherwise. Replies to the user's earlier requests are \
+ not in question; do not revisit or correct them unless the user asks.",
);
if let Some(claim) = unverified_final.filter(|claim| !claim.trim().is_empty()) {
const CLIPPING_MARKER_RESERVE: usize = 64;
@@ -484,8 +524,24 @@ fn log(driver: &Driver, reason: &'static str, branch: Option<Branch>, served: Ro
"reason_code": reason,
"verdict": if served == Route::Local { "local" } else { "cloud" },
"branch": branch.map(|branch| format!("{branch:?}")),
+ "policy_version": decide::POLICY_VERSION,
}));
- tracing::info!(target: "libsy", reason, branch = ?branch, served = ?served, "vgr decision");
+ tracing::info!(
+ target: "libsy",
+ reason,
+ branch = ?branch,
+ served = ?served,
+ policy_version = decide::POLICY_VERSION,
+ "vgr decision"
+ );
+}
+
+/// Whether the user is answering a finished assistant reply.
+fn answers_final_reply(messages: &[Message]) -> bool {
+ messages
+ .len()
+ .checked_sub(1)
+ .is_some_and(|last| text::starts_user_turn(messages, last))
}
fn tool_use_is_complete(response: &AggLlmResponse) -> bool {
diff --git a/crates/libsy/src/algorithms/vgr/runtime/tests.rs b/crates/libsy/src/algorithms/vgr/runtime/tests.rs
index f037dcff5..95b079c57 100644
--- a/crates/libsy/src/algorithms/vgr/runtime/tests.rs
+++ b/crates/libsy/src/algorithms/vgr/runtime/tests.rs
@@ -398,6 +398,7 @@ async fn decisions_attach_the_reason_and_exclude_local_cloud_fallbacks() -> Resu
assert_eq!(evidence["source"], "vgr");
assert_eq!(evidence["reason_code"], reason);
assert_eq!(evidence["verdict"], served);
+ assert_eq!(evidence["policy_version"], "1.0");
if reason == "decided" {
assert_eq!(evidence["branch"], "DefaultVerified");
}
@@ -625,3 +626,241 @@ fn handoff_preserves_existing_messages_and_non_text_content() {
classifier.hand_off(&mut task, &mut state, Some("duplicate notice"));
assert_eq!(task.llm_request.messages, after_first);
}
+
+#[tokio::test]
+async fn a_blank_prompt_escalates_without_a_local_attempt() -> Result<()> {
+ let continuing = route(|_| {})?;
+ let route = route(|_| {})?;
+ let calls = Arc::new(Calls::default());
+ let blank_follow_up = vec![
+ Message::text(Role::User, "what is 2 + 2?"),
+ Message::text(Role::Assistant, "4"),
+ Message::text(Role::User, " "),
+ ];
+ for messages in [
+ vec![Message::text(Role::User, " \n ")],
+ vec![Message::text(Role::User, "\u{200B}\u{FEFF}")],
+ blank_follow_up,
+ ] {
+ assert_eq!(
+ drive(&route, session(messages), calls.clone(), complete_call, 0.0).await?,
+ "cloud"
+ );
+ }
+ assert_eq!(calls.0.load(Ordering::Relaxed), 0);
+
+ // Tool results sent as a user message continue the task; they are not a blank prompt.
+ let mut results = result("a", "file.txt");
+ results.role = Role::User;
+ let continuation = session(vec![
+ Message::text(Role::User, "list files"),
+ call("a"),
+ results,
+ ]);
+ drive(&continuing, continuation, calls.clone(), complete_call, 0.0).await?;
+ assert_eq!(calls.0.load(Ordering::Relaxed), 1);
+ Ok(())
+}
+
+#[tokio::test]
+async fn an_agent_marked_nudge_keeps_a_single_escalation_on_cloud() -> Result<()> {
+ let route = route(|_| {})?;
+ let calls = Arc::new(Calls::default());
+ let truncated = || tool_call(json!(r#"{"command":"l"#), StopReason::MaxTokens);
+ let task = session(vec![Message::text(Role::User, "fix the build")]);
+ drive(&route, task, calls.clone(), truncated, 0.0).await?;
+ let nudge = vec![
+ Message::text(Role::User, "fix the build"),
+ Message::text(Role::Assistant, "fixed"),
+ Message::text(Role::User, "Verify the change before you stop."),
+ ];
+ let nudge = initiated_by(session(nudge), "agent");
+ assert_eq!(
+ drive(&route, nudge, calls.clone(), complete_call, 0.0).await?,
+ "cloud"
+ );
+ assert_eq!(calls.0.load(Ordering::Relaxed), 1);
+ Ok(())
+}
+
+fn initiated_by(mut request: Request, initiator: &str) -> Request {
+ let metadata = request.metadata.get_or_insert_with(Metadata::default);
+ metadata
+ .http_headers
+ .get_or_insert_with(http::HeaderMap::new)
+ .insert("x-initiator", initiator.parse().unwrap());
+ request
+}
+
+#[tokio::test]
+async fn a_held_follow_up_records_the_latch_as_its_reason() -> Result<()> {
+ use crate::core::algorithm::Call;
+ use crate::core::testing::serve_decision;
+
+ let algorithm = route(|config| config.mode = ServingMode::Shadow)?;
+ let models = Arc::new(category_models(Category::Any, &["local", "cloud"]).into());
+ let serve = |call| async move {
+ match call {
+ Call::Model(call) => call.respond(std::future::ready(Ok(reply("done")))).await,
+ Call::Decision(call) => serve_decision(*call).await,
+ }
+ };
+ let escalated = crate::drive(
+ Arc::clone(&algorithm),
+ session(vec![Message::text(Role::User, "fix the build")]),
+ Arc::clone(&models),
+ serve,
+ )
+ .await?;
+ assert_eq!(escalated.selected_model_ids, vec![ModelId::new("cloud")]);
+
+ let held = crate::drive(
+ algorithm,
+ session(vec![
+ Message::text(Role::User, "fix the build"),
+ call("c1"),
+ result("c1", "ok"),
+ ]),
+ models,
+ serve,
+ )
+ .await?;
+ assert_eq!(held.selected_model_ids, vec![ModelId::new("cloud")]);
+ let evidence = held
+ .metadata
+ .and_then(|metadata| metadata.evidence)
+ .unwrap_or_default();
+ assert_eq!(evidence["source"], "vgr");
+ assert_eq!(evidence["reason_code"], "turn_verification_latched");
+ assert_eq!(evidence["verdict"], "cloud");
+ Ok(())
+}
+
+#[tokio::test]
+async fn shadow_rechecks_new_user_turns_with_session_latching_enabled() -> Result<()> {
+ let route = route(|config| {
+ config.mode = ServingMode::Shadow;
+ config.latch_escalation = true;
+ })?;
+ let calls = Arc::new(Calls::default());
+ for messages in [
+ vec![Message::text(Role::User, "fix the build")],
+ vec![
+ Message::text(Role::User, "fix the build"),
+ Message::text(Role::Assistant, "fixed"),
+ Message::text(Role::User, "now add a test"),
+ ],
+ ] {
+ assert_eq!(
+ drive(&route, session(messages), calls.clone(), complete_call, 0.0).await?,
+ "cloud"
+ );
+ }
+ assert_eq!(calls.0.load(Ordering::Relaxed), 2);
+ Ok(())
+}
+
+#[tokio::test]
+async fn an_escalation_latch_holds_the_session_past_new_questions() -> Result<()> {
+ let route = route(|config| config.latch_escalation = true)?;
+ let calls = Arc::new(Calls::default());
+ let truncated = || tool_call(json!(r#"{"command":"l"#), StopReason::MaxTokens);
+ let task = session(vec![Message::text(Role::User, "fix the build")]);
+ assert_eq!(
+ drive(&route, task, calls.clone(), truncated, 0.0).await?,
+ "cloud"
+ );
+ let follow_up = session(vec![
+ Message::text(Role::User, "fix the build"),
+ Message::text(Role::Assistant, "fixed"),
+ Message::text(Role::User, "now add a test"),
+ ]);
+ assert_eq!(
+ drive(&route, follow_up, calls.clone(), complete_call, 0.0).await?,
+ "cloud"
+ );
+ assert_eq!(calls.0.load(Ordering::Relaxed), 1);
+ Ok(())
+}
+
+#[tokio::test]
+async fn an_initiator_header_decides_when_the_turn_latch_releases() -> Result<()> {
+ let route = route(|_| {})?;
+ let calls = Arc::new(Calls::default());
+ let task = || session(vec![Message::text(Role::User, "fix the build")]);
+ drive(&route, task(), calls.clone(), complete_call, 0.9).await?;
+ drive(&route, task(), calls.clone(), complete_call, 0.9).await?;
+ assert_eq!(calls.0.load(Ordering::Relaxed), 2);
+
+ // A client nudge after a final reply looks like a new question; marked agent, it keeps the latch.
+ let nudge = vec![
+ Message::text(Role::User, "fix the build"),
+ Message::text(Role::Assistant, "fixed"),
+ Message::text(Role::User, "Verify the change before you stop."),
+ ];
+ assert_eq!(
+ drive(
+ &route,
+ initiated_by(session(nudge), "agent"),
+ calls.clone(),
+ complete_call,
+ 0.0
+ )
+ .await?,
+ "cloud"
+ );
+ assert_eq!(calls.0.load(Ordering::Relaxed), 2);
+
+ // A person's correction after tool results looks like a continuation; marked user, it is verified afresh.
+ let steer = vec![
+ Message::text(Role::User, "fix the build"),
+ call("c1"),
+ result("c1", "ok"),
+ Message::text(Role::User, "use the staging config instead"),
+ ];
+ assert_eq!(
+ drive(
+ &route,
+ initiated_by(session(steer), "user"),
+ calls.clone(),
+ complete_call,
+ 0.0
+ )
+ .await?,
+ "local"
+ );
+ assert_eq!(calls.0.load(Ordering::Relaxed), 3);
+ Ok(())
+}
+
+#[tokio::test]
+async fn tool_work_from_an_earlier_user_turn_is_not_handed_off() -> Result<()> {
+ let route = route(|config| config.agentic_handoff = true)?;
+ let (selected, _) = test_drive_with_models(
+ route,
+ session(vec![
+ Message::text(Role::User, "fix the build"),
+ call("c1"),
+ result("c1", "ok"),
+ Message::text(Role::Assistant, "the build passes"),
+ Message::text(Role::User, "What is 41 plus 42?"),
+ ]),
+ category_models(Category::Any, &["local", "cloud"]),
+ |target: ModelId, request: Request| async move {
+ Ok(match target.as_str() {
+ "local" => reply("83"),
+ "cloud" => {
+ let text =
+ serde_json::to_string(&request.llm_request.messages).unwrap_or_default();
+ assert!(text.contains("41 plus 42"));
+ assert!(!text.contains("Routing notice"));
+ reply("83")
+ }
+ _ => scored(0.0),
+ })
+ },
+ )
+ .await?;
+ assert_eq!(selected, "cloud");
+ Ok(())
+}
diff --git a/crates/libsy/src/algorithms/vgr/safety.rs b/crates/libsy/src/algorithms/vgr/safety.rs
index f14f0b362..eac01fc36 100644
--- a/crates/libsy/src/algorithms/vgr/safety.rs
+++ b/crates/libsy/src/algorithms/vgr/safety.rs
@@ -141,7 +141,8 @@ pub(super) fn fallback_eligible(error: &LibsyError) -> bool {
LibsyError::ClientCall { source, .. } if match source {
LlmClientError::ContextWindowExceeded { .. }
| LlmClientError::Transport { .. }
- | LlmClientError::Timeout { .. } => true,
+ | LlmClientError::Timeout { .. }
+ | LlmClientError::TemporarilyUnavailable => true,
LlmClientError::UpstreamHttp { status, .. } =>
matches!(
*status,
@@ -159,7 +160,9 @@ pub(super) fn endpoint_failure(error: &LibsyError) -> bool {
matches!(
error,
LibsyError::ClientCall { source, .. } if match source {
- LlmClientError::Transport { .. } | LlmClientError::Timeout { .. } => true,
+ LlmClientError::Transport { .. }
+ | LlmClientError::Timeout { .. }
+ | LlmClientError::TemporarilyUnavailable => true,
LlmClientError::UpstreamHttp { status, .. } => matches!(
*status,
StatusCode::BAD_GATEWAY
diff --git a/crates/libsy/src/algorithms/vgr/text.rs b/crates/libsy/src/algorithms/vgr/text.rs
index 981b9649b..b011f058f 100644
--- a/crates/libsy/src/algorithms/vgr/text.rs
+++ b/crates/libsy/src/algorithms/vgr/text.rs
@@ -11,6 +11,7 @@ use regex::Regex;
use serde_json::Value;
use switchyard_protocol::{ContentBlock, InstructionBlock, Message, Request, Role, ToolResult};
+use crate::algorithms::util::affinity::is_user_turn;
use crate::algorithms::util::tool_signals::classify_text;
/// One flattened turn: instructions first, then messages.
@@ -93,21 +94,74 @@ pub(super) fn user_task_text(turns: &[Turn]) -> &str {
.unwrap_or_default()
}
+/// Whether the newest user message is empty, ignoring whitespace and zero-width characters.
+///
+/// Any non-text block, such as an image or a tool result, is content.
+pub(super) fn newest_prompt_is_blank(request: &Request) -> bool {
+ request
+ .llm_request
+ .messages
+ .iter()
+ .rev()
+ .find(|message| message.role == Role::User)
+ .is_none_or(|message| {
+ message.content.iter().all(|block| match block {
+ ContentBlock::Text { text } => text.chars().all(|c| {
+ c.is_whitespace()
+ || matches!(c, '\u{200B}'..='\u{200D}' | '\u{2060}' | '\u{FEFF}')
+ }),
+ _ => false,
+ })
+ })
+}
+
pub(super) fn has_assistant_turn(turns: &[Turn]) -> bool {
turns.iter().any(|turn| turn.role == Role::Assistant)
}
/// Whether the conversation contains an executed or proposed tool step.
pub(super) fn has_tool_trajectory(request: &Request) -> bool {
- request.llm_request.messages.iter().any(|message| {
- message.role == Role::Tool
- || message.content.iter().any(|block| {
- matches!(
- block,
- ContentBlock::ToolCall(_) | ContentBlock::ToolResult(_)
- )
- })
- })
+ request.llm_request.messages.iter().any(is_tool_step)
+}
+
+/// Whether the user message at `index` answers a finished assistant reply.
+///
+/// Agent clients also inject user-role continuations (after context compaction,
+/// at an iteration limit) that follow tool results, not a final reply; those
+/// continue the current user turn.
+pub(super) fn starts_user_turn(messages: &[Message], index: usize) -> bool {
+ let (Some(message), Some(previous)) = (
+ messages.get(index),
+ index.checked_sub(1).and_then(|i| messages.get(i)),
+ ) else {
+ return false;
+ };
+ is_user_turn(message)
+ && previous.role == Role::Assistant
+ && !previous
+ .content
+ .iter()
+ .any(|block| matches!(block, ContentBlock::ToolCall(_)))
+}
+
+/// Whether the current user turn contains an executed or proposed tool step.
+pub(super) fn turn_has_tool_trajectory(request: &Request) -> bool {
+ let messages = &request.llm_request.messages;
+ let start = (0..messages.len())
+ .rev()
+ .find(|&index| starts_user_turn(messages, index))
+ .unwrap_or(0);
+ messages[start..].iter().any(is_tool_step)
+}
+
+fn is_tool_step(message: &Message) -> bool {
+ message.role == Role::Tool
+ || message.content.iter().any(|block| {
+ matches!(
+ block,
+ ContentBlock::ToolCall(_) | ContentBlock::ToolResult(_)
+ )
+ })
}
/// Idempotent secret and PII scrub applied before text reaches a verifier.
diff --git a/crates/switchyard-runner/src/algorithm.rs b/crates/switchyard-runner/src/algorithm.rs
index 9729bcde3..11220076e 100644
--- a/crates/switchyard-runner/src/algorithm.rs
+++ b/crates/switchyard-runner/src/algorithm.rs
@@ -552,6 +552,9 @@ pub struct VgrRouteConfig {
/// Wall-clock budget for the local tier within one user turn.
#[serde(default)]
pub local_turn_budget_seconds: Option<f64>,
+ /// Holds escalations across later user turns.
+ #[serde(default)]
+ pub latch_escalation: bool,
}
#[derive(Clone, Debug, Default, Deserialize)]
@@ -1808,17 +1811,18 @@ fn build_vgr(
approval: config.active_approval.clone().unwrap_or_default(),
},
};
- runtime.deadline = duration(route_name, "deadline_seconds", config.deadline_seconds)?;
+ runtime.deadline = vgr_duration(route_name, "deadline_seconds", config.deadline_seconds)?;
runtime.task_typing = config.task_typing;
runtime.confirmed_recovery_min_clean_tail = config.confirmed_recovery_min_clean_tail;
runtime.agentic_handoff = config.agentic_handoff;
+ runtime.latch_escalation = config.latch_escalation;
runtime.local_turn_budget = config
.local_turn_budget_seconds
- .map(|seconds| duration(route_name, "local_turn_budget_seconds", seconds))
+ .map(|seconds| vgr_duration(route_name, "local_turn_budget_seconds", seconds))
.transpose()?;
runtime.breaker = BreakerConfig {
threshold: config.breaker_threshold,
- cooldown: duration(
+ cooldown: vgr_duration(
route_name,
"breaker_cooldown_seconds",
config.breaker_cooldown_seconds,
@@ -1831,13 +1835,14 @@ fn build_vgr(
})
}
-fn duration(route: &str, field: &str, seconds: f64) -> AlgorithmResult<std::time::Duration> {
- std::time::Duration::try_from_secs_f64(seconds).map_err(|error| {
- AlgorithmConfigError::with_source(
- format!("vgr route {route}: {field} must be a non-negative number"),
- error,
- )
- })
+fn vgr_duration(route: &str, field: &str, seconds: f64) -> AlgorithmResult<std::time::Duration> {
+ let message = || format!("vgr route {route}: {field} must be a positive number");
+ let duration = std::time::Duration::try_from_secs_f64(seconds)
+ .map_err(|error| AlgorithmConfigError::with_source(message(), error))?;
+ if duration.is_zero() {
+ return Err(AlgorithmConfigError::new(message()));
+ }
+ Ok(duration)
}
const fn default_max_reviews() -> u32 {
diff --git a/crates/switchyard-runner/src/config.rs b/crates/switchyard-runner/src/config.rs
index 7e3999c43..096af0136 100644
--- a/crates/switchyard-runner/src/config.rs
+++ b/crates/switchyard-runner/src/config.rs
@@ -1184,6 +1184,11 @@ task_typing = false
let missing_target =
vgr_config().replace("local_target = \"weak\"", "local_target = \"missing\"");
assert!(error_message(&missing_target).contains("unknown target missing"));
+
+ let zero_cooldown = format!("{}breaker_cooldown_seconds = 0\n", vgr_config());
+ assert!(
+ error_message(&zero_cooldown).contains("breaker_cooldown_seconds must be a positive")
+ );
Ok(())
}
diff --git a/crates/switchyard-server/tests/server.rs b/crates/switchyard-server/tests/server.rs
index d0fad52ab..bb0dbf32e 100644
--- a/crates/switchyard-server/tests/server.rs
+++ b/crates/switchyard-server/tests/server.rs
@@ -1853,6 +1853,125 @@ task_typing = false
Ok(())
}
+#[tokio::test]
+async fn vgr_escalates_while_the_local_endpoint_is_down() -> TestResult {
+ let upstream = MockUpstream::start().await?;
+ let closed = std::net::TcpListener::bind("127.0.0.1:0")?.local_addr()?;
+ let app = build_switchyard_router(load_test_config(&format!(
+ r#"
+schema_version = 1
+
+[llm_clients.primary]
+format = "openai_chat"
+base_url = "{}"
+
+[llm_clients.down]
+format = "openai_chat"
+base_url = "http://{closed}/v1"
+
+[targets.local]
+id = "model/vgr-local"
+llm_client = "down"
+
+[targets.cloud]
+id = "model/vgr-cloud"
+llm_client = "primary"
+
+[routes.vgr]
+id = "switchyard/vgr"
+type = "vgr"
+local_target = "local"
+cloud_target = "cloud"
+mode = "active"
+active_approval = "prospective-validation-and-canary-approved"
+task_typing = false
+"#,
+ upstream.base_url
+ ))?);
+ // The second request meets the failed endpoint's cooldown rather than its socket.
+ for _ in 0..2 {
+ let response = send(
+ &app,
+ "POST",
+ "/v1/chat/completions",
+ Some(json!({
+ "model": "switchyard/vgr",
+ "messages": [{"role": "user", "content": "answer the question"}]
+ })),
+ )
+ .await?;
+ assert_eq!(response.status, StatusCode::OK, "{}", response.text()?);
+ assert_eq!(
+ response.headers.get("x-model-router-selected-model"),
+ Some(&HeaderValue::from_static("model/vgr-cloud"))
+ );
+ }
+ Ok(())
+}
+
+#[tokio::test]
+async fn vgr_keeps_agent_nudges_in_their_session_and_releases_user_turns() -> TestResult {
+ let upstream = MockUpstream::start().await?;
+ for session_header in ["x-switchyard-session-id", "x-session-id"] {
+ let app = build_switchyard_router(load_test_config(&format!(
+ r#"
+schema_version = 1
+[llm_clients.primary]
+format = "openai_chat"
+base_url = "{}"
+[targets]
+local = {{ id = "model/vgr-local", llm_client = "primary" }}
+cloud = {{ id = "model/vgr-cloud", llm_client = "primary" }}
+judge = {{ id = "model/vgr-judge", llm_client = "primary" }}
+[routes.vgr]
+id = "switchyard/vgr"
+type = "vgr"
+local_target = "local"
+cloud_target = "cloud"
+judge_target = "judge"
+mode = "evaluate"
+task_typing = false
+"#,
+ upstream.base_url
+ ))?);
+ let initial = json!([{"role": "user", "content": " "}]);
+ let nudge = json!([
+ {"role": "user", "content": " "},
+ {"role": "assistant", "content": "4"},
+ {"role": "user", "content": "Verify the answer before stopping."}
+ ]);
+ for (session, initiator, messages, served) in [
+ ("a", Some("user"), &initial, "model/vgr-cloud"),
+ ("a", Some("agent"), &nudge, "model/vgr-cloud"),
+ ("b", Some("agent"), &nudge, "model/vgr-local"),
+ ("a", Some("user"), &nudge, "model/vgr-local"),
+ ("a", Some("agent"), &nudge, "model/vgr-local"),
+ ("c", None, &initial, "model/vgr-cloud"),
+ ("c", None, &nudge, "model/vgr-local"),
+ ] {
+ let mut headers = vec![(session_header, session)];
+ if let Some(initiator) = initiator {
+ headers.push(("x-initiator", initiator));
+ }
+ let response = send_with_headers(
+ &app,
+ "POST",
+ "/v1/chat/completions",
+ Some(json!({"model": "switchyard/vgr", "messages": messages})),
+ &headers,
+ )
+ .await?;
+ assert_eq!(response.status, StatusCode::OK, "{}", response.text()?);
+ assert_eq!(
+ response.headers.get("x-model-router-selected-model"),
+ Some(&HeaderValue::from_str(served)?),
+ "{session_header}: {session} {initiator:?}"
+ );
+ }
+ }
+ Ok(())
+}
+
fn metric_line<'a>(metrics: &'a str, name: &str, labels: &[(&str, &str)]) -> Option<&'a str> {
metrics.lines().find(|line| {
line.starts_with(name)
diff --git a/docs/routing_algorithms/vgr_routing.md b/docs/routing_algorithms/vgr_routing.md
index 9378703a9..b8ca77fa6 100644
--- a/docs/routing_algorithms/vgr_routing.md
+++ b/docs/routing_algorithms/vgr_routing.md
@@ -72,6 +72,37 @@ new local work has been served since the previous handoff. The handoff preserves
the conversation's existing messages and content. `local_turn_budget_seconds`
escalates a user turn once the local tier has spent that much wall-clock time on it.
+The notice covers the current request and asks the cloud tier to preserve earlier
+work, inspect its effects, and repeat or undo steps only when needed.
+
+Escalations release when a new user turn is decided locally. Set
+`latch_escalation = true` to retain the cloud tier across later user turns instead.
+Shadow mode always releases the hold at the next user turn.
+All configured VGR durations must be positive.
+
+## Hermes session headers
+
+Use a Hermes build that supports both provider options below. The session header
+must stay the same throughout a conversation; each conversation needs its own ID.
+
+```yaml
+model:
+ default: assistant
+ provider: custom
+ base_url: http://localhost:4000/v1
+providers:
+ switchyard:
+ api: http://localhost:4000/v1
+ session_affinity_header: x-switchyard-session-id
+ initiator_header: true
+```
+
+`x-initiator: user` releases the turn hold for a person's new request or steer.
+`x-initiator: agent` keeps the hold through tool calls, recovery nudges, and
+automatic continuations. Retries retain the original call's tag. Without a
+recognized tag, Switchyard uses the message shape to identify a user turn.
+`x-session-id` is also accepted as a session-header alias.
+
## Serving modes
- `off` skips candidate generation and serves cloud. This is the default.