Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.
Rust
fix(vgr): restore QA verification and cloud outage fallback (#1042)
GDRS3a48a933de0c
diff --git a/crates/libsy-llm-client/src/run.rs b/crates/libsy-llm-client/src/run.rsindex 661e05c6b..fd9c6a2fd 100644--- a/crates/libsy-llm-client/src/run.rs+++ b/crates/libsy-llm-client/src/run.rs@@ -111,11 +111,12 @@ pub async fn run(&clients,&algorithm_name,&outcome.request,+ outcome.fallback_request.as_ref(),&outcome.selected_model_ids,&observe,).await- .and_then(|response| clients.remember_state_owner(&outcome.request, response))+ .and_then(|(request, response)| clients.remember_state_owner(request, response))};if let Some(observer) = &observer {observer(RunObservation::RoutingOverhead(overhead));@@ -245,19 +246,24 @@ async fn serve(}/// Try candidates in order until one succeeds or a failure stops fallback.-async fn call_first_available(+async fn call_first_available<'a>(clients: &ClientRouter,algorithm: &str,- request: &Request,+ request: &'a Request,+ fallback_request: Option<&'a Request>,models: &[ModelId],observe: &(dyn Fn(ModelCallObservation) + Send + Sync),-) -> Result<Response> {+) -> Result<(&'a Request, Response)> {for (index, target) in models.iter().enumerate() {- let request = clients.prepare_completion_request(request.clone(), target);+ let request = match fallback_request {+ Some(fallback) if index > 0 => fallback,+ _ => request,+ };+ let prepared = clients.prepare_completion_request(request.clone(), target);match call_one(clients,target,- request,+ prepared,algorithm,observe,index,@@ -266,7 +272,7 @@ async fn call_first_available().await{- Ok(response) => return Ok(response),+ Ok(response) => return Ok((request, response)),Err(error) if index + 1 == models.len() => return Err(error),Err(error)if clients.rate_limit_fallback_disabled.contains(target)@@ -276,7 +282,7 @@ async fn call_first_available({return Err(error);}- Err(error) => match fallback_reason(&error) {+ Err(error) => match fallback_reason(&error, fallback_request.is_none()) {Some(reason) => tracing::info!(from = %target,to = %models[index + 1],@@ -442,7 +448,10 @@ async fn buffer_routing_stream(}/// Whether a failed candidate is worth routing around.-fn fallback_reason(error: &LibsyError) -> Option<RoutingFallbackReason> {+fn fallback_reason(+ error: &LibsyError,+ allow_policy_fallback: bool,+) -> Option<RoutingFallbackReason> {let LibsyError::ClientCall { source, .. } = error else {return None;};@@ -451,23 +460,22 @@ fn fallback_reason(error: &LibsyError) -> Option<RoutingFallbackReason> {LlmClientError::Transport { .. } | LlmClientError::TemporarilyUnavailable => {Some(RoutingFallbackReason::Unavailable)}- // A policy denial can be specific to one provider. Preserve its HTTP- // error, but allow another candidate to serve the request.- LlmClientError::UpstreamHttp { status, body }- if *status == StatusCode::BAD_REQUEST- && serde_json::from_str::<serde_json::Value>(body).is_ok_and(|value| {- value["error"]["code"].as_str() == Some("content_policy_violation")- }) =>- {- Some(RoutingFallbackReason::Unavailable)- }- LlmClientError::UpstreamHttp { status, .. }- if matches!(- *status,- StatusCode::FORBIDDEN | StatusCode::REQUEST_TIMEOUT | StatusCode::TOO_MANY_REQUESTS- ) || status.is_server_error() =>- {- Some(RoutingFallbackReason::Unavailable)+ LlmClientError::UpstreamHttp { status, body } => {+ let policy_denial = serde_json::from_str::<Value>(body).is_ok_and(|value| {+ value["error"]["code"].as_str() == Some("content_policy_violation")+ });+ if policy_denial && !allow_policy_fallback {+ return None;+ }+ ((*status == StatusCode::BAD_REQUEST && policy_denial)+ || matches!(+ *status,+ StatusCode::FORBIDDEN+ | StatusCode::REQUEST_TIMEOUT+ | StatusCode::TOO_MANY_REQUESTS+ )+ || status.is_server_error())+ .then_some(RoutingFallbackReason::Unavailable)}_ => None,}@@ -1101,10 +1109,36 @@ mod tests {}}+ /// Rewrites the request for its selected model only.+ struct RewritingAlgorithm {}++ #[async_trait]+ impl Algorithm for RewritingAlgorithm {+ fn name(&self) -> &str {+ "rewriting_test"+ }++ async fn route(+ self: Arc<Self>,+ _driver: Driver,+ request: Request,+ ) -> Result<RoutingOutcome> {+ let mut rewritten = request.clone();+ rewritten+ .llm_request+ .messages+ .push(Message::text(Role::User, "for weak only"));+ let mut outcome =+ RoutingOutcome::route_to("weak".into(), vec!["strong".into()], rewritten);+ outcome.fallback_request = Some(request);+ Ok(outcome)+ }+ }+#[derive(Clone, Copy)]enum FirstOutcome {ContextWindow,- ContentPolicy,+ ContentPolicy(StatusCode),Unauthorized,StreamSuccess,MidStreamError,@@ -1128,8 +1162,8 @@ mod tests {model,message: "too long".to_string(),}),- FirstOutcome::ContentPolicy => Err(LlmClientError::UpstreamHttp {- status: StatusCode::BAD_REQUEST,+ FirstOutcome::ContentPolicy(status) => Err(LlmClientError::UpstreamHttp {+ status,body: r#"{"error":{"code":"content_policy_violation","message":"request blocked by content policy","type":"invalid_request_error"},"metadata":{"documentation_section":"context window"}}"#.to_string(),}),FirstOutcome::Unauthorized => Err(LlmClientError::UpstreamHttp {@@ -1157,7 +1191,10 @@ mod tests {};}Ok(Response {- llm_response: LlmResponse::Agg(text_response(Some(model.to_string()), model)),+ llm_response: LlmResponse::Agg(AggLlmResponse {+ id: Some("candidate_response".into()),+ ..text_response(Some(model.to_string()), model)+ }),metadata: None,upstream_headers: http::HeaderMap::new(),})@@ -2256,10 +2293,13 @@ mod tests {fn fallback_only_accepts_context_policy_and_unavailable_failures() {let error = |source| LibsyError::client_call("target", source);assert_eq!(- fallback_reason(&error(LlmClientError::ContextWindowExceeded {- model: "target".into(),- message: "too long".to_string(),- })),+ fallback_reason(+ &error(LlmClientError::ContextWindowExceeded {+ model: "target".into(),+ message: "too long".to_string(),+ }),+ true+ ),Some(RoutingFallbackReason::ContextWindow));for status in [@@ -2270,10 +2310,13 @@ mod tests {StatusCode::from_u16(599).expect("599 is a valid status"),] {assert_eq!(- fallback_reason(&error(LlmClientError::UpstreamHttp {- status,- body: "failed".to_string(),- })),+ fallback_reason(+ &error(LlmClientError::UpstreamHttp {+ status,+ body: "failed".to_string(),+ }),+ true+ ),Some(RoutingFallbackReason::Unavailable));}@@ -2290,10 +2333,13 @@ mod tests {("content_policy_violation", None),] {assert_eq!(- fallback_reason(&error(LlmClientError::UpstreamHttp {- status: StatusCode::BAD_REQUEST,- body: body.to_string(),- })),+ fallback_reason(+ &error(LlmClientError::UpstreamHttp {+ status: StatusCode::BAD_REQUEST,+ body: body.to_string(),+ }),+ true+ ),expected,"{body}");@@ -2307,10 +2353,13 @@ mod tests {StatusCode::from_u16(600).expect("600 is a valid status"),] {assert_eq!(- fallback_reason(&error(LlmClientError::UpstreamHttp {- status,- body: "failed".to_string(),- })),+ fallback_reason(+ &error(LlmClientError::UpstreamHttp {+ status,+ body: "failed".to_string(),+ }),+ true+ ),None);}@@ -2335,7 +2384,8 @@ mod tests {assert_eq!(response.served_model().map(ModelId::as_str), Some("strong"));// A provider-specific policy denial also permits another candidate.- let (client, result) = run_candidates(FirstOutcome::ContentPolicy).await;+ let (client, result) =+ run_candidates(FirstOutcome::ContentPolicy(StatusCode::BAD_REQUEST)).await;let (_, response) = result?;assert_eq!(&*client.calls.lock(),@@ -2359,6 +2409,78 @@ mod tests {Ok(())}+ #[tokio::test]+ async fn fallback_candidates_receive_the_fallback_request() -> Result<()> {+ let client = Arc::new(CandidateClient {+ calls: Mutex::new(Vec::new()),+ requests: Mutex::new(Vec::new()),+ first: FirstOutcome::ContextWindow,+ });+ let clients = ClientRouter::single(client.clone());+ let mut seed = request();+ seed.llm_request+ .preservation+ .requests+ .insert(WireFormat::OpenAiResponses.into(), json!({}));+ run(+ Arc::new(RewritingAlgorithm {}),+ clients.clone(),+ seed,+ to_category_map(&["weak", "strong"]),+ None,+ )+ .await?;+ let sent: Vec<_> = client+ .requests+ .lock()+ .iter()+ .map(|request| request.llm_request.messages.clone())+ .collect();+ assert_eq!(sent.len(), 2);+ assert_eq!(sent[0].len(), request().llm_request.messages.len() + 1);+ assert_eq!(sent[1], request().llm_request.messages);+ let mut follow = request();+ follow+ .llm_request+ .extensions+ .fields+ .insert("previous_response_id".into(), json!("candidate_response"));+ follow.llm_request.messages = vec![Message::text(Role::User, "continue")];+ let owner = clients+ .stored_state_owner(&follow)+ .expect("fallback Responses state");+ let continued = continue_on(owner, "test", follow);+ let mut expected = sent[1].clone();+ expected.push(Message::text(Role::Assistant, "strong"));+ expected.push(Message::text(Role::User, "continue"));+ assert_eq!(continued.request.llm_request.messages, expected);+ Ok(())+ }++ #[tokio::test]+ async fn fallback_request_does_not_bypass_content_policy() -> Result<()> {+ for status in [StatusCode::BAD_REQUEST, StatusCode::FORBIDDEN] {+ let client = Arc::new(CandidateClient {+ calls: Mutex::new(Vec::new()),+ requests: Mutex::new(Vec::new()),+ first: FirstOutcome::ContentPolicy(status),+ });+ let result = run(+ Arc::new(RewritingAlgorithm {}),+ ClientRouter::single(client.clone()),+ request(),+ to_category_map(&["weak", "strong"]),+ None,+ )+ .await;+ assert!(+ matches!(result, Err(LibsyError::ClientCall { source: LlmClientError::UpstreamHttp { status: actual, .. }, .. }) if actual == status)+ );+ assert_eq!(&*client.calls.lock(), &[ModelId::from("weak")]);+ }+ Ok(())+ }+#[tokio::test]async fn retry_exhaustion_falls_back_and_optional_cooldown_skips_failed_backend() -> Result<()>{diff --git a/crates/libsy/src/algorithms/vgr.rs b/crates/libsy/src/algorithms/vgr.rsindex fc99ada46..7a94338f4 100644--- a/crates/libsy/src/algorithms/vgr.rs+++ b/crates/libsy/src/algorithms/vgr.rs@@ -37,6 +37,7 @@ mod tests;/// A verification-gated route between a local and a capable tier.pub struct Vgr {route: FallThrough,+ classifier: Arc<runtime::VgrClassifier>,}impl Vgr {@@ -59,8 +60,8 @@ impl Vgr {.with_name("vgr").with_processor(affinity.clone()).with_classifier(Arc::new(runtime::Held(affinity, reason)))- .with_classifier(classifier);- Ok(Self { route })+ .with_classifier(classifier.clone());+ Ok(Self { route, classifier })}}@@ -71,9 +72,27 @@ impl Algorithm for Vgr {}async fn route(self: Arc<Self>, driver: Driver, request: Request) -> Result<RoutingOutcome> {+ // The handoff note is written for the cloud tier, so a local stand-in gets the request+ // as it arrived.+ let arrived = self+ .classifier+ .local_may_stand_in(&request)+ .then(|| request.clone());let mut outcome = self.route.execute(driver, request).await?;- // Local answers are already buffered; cloud decisions cannot fall back to local.+ // Local answers are already buffered and have no fallback.outcome.selected_model_ids.truncate(1);+ let config = &self.classifier.config;+ if let Some(arrived) = arrived+ && outcome.response.is_none()+ && outcome.selected_model_ids.first() == Some(&config.targets.cloud)+ // The local tier may have failed while this request was routed.+ && self.classifier.local_may_stand_in(&arrived)+ {+ outcome+ .selected_model_ids+ .push(config.targets.local.clone());+ outcome.fallback_request = Some(arrived);+ }Ok(outcome)}}@@ -150,10 +169,10 @@ fn derive_capabilities(Some(render::render_agentic_view(request, &turns, attempt)),);}+ let transcript = Some(render::render_session(&turns, attempt));if text::observed_hardening(attempt) || task_type == Some(TaskType::Coding) {- return caps(Branch::Coding, None);+ return caps(Branch::Coding, transcript);}- let transcript = Some(render::render_session(&turns, attempt));if text::observed_tool_activity(attempt) {caps(Branch::Agentic, transcript)} else if matches!(task_type, Some(TaskType::Chat | TaskType::Answer)) {diff --git a/crates/libsy/src/algorithms/vgr/decide.rs b/crates/libsy/src/algorithms/vgr/decide.rsindex b68e275c0..11c05f22d 100644--- a/crates/libsy/src/algorithms/vgr/decide.rs+++ b/crates/libsy/src/algorithms/vgr/decide.rs@@ -12,6 +12,9 @@ use super::{Branch, Capabilities};pub(super) const POLICY_VERSION: &str = "1.0";const READOUT: f64 = 0.9;+const DELIBERATION: f64 = 0.5;+/// Readout at which clean coding commits without the capable tier's affirmation.+const CODING_CONFIDENT: f64 = 0.6;/// Most tool results a run may contain and still recover without the capable tier.const SHORT_RUN_MAX_TOOL_RESULTS: i32 = 100;@@ -33,8 +36,8 @@ pub(super) enum Tri {pub(super) struct Signals {/// Local logprob score that the attempt is correct.pub(super) readout: Option<f64>,- /// Local deliberating verdict.- pub(super) deliberation: Option<Tri>,+ /// Local logprob score that the attempt is correct, read after a deliberation.+ pub(super) deliberation: Option<f64>,/// Capable-tier evidence verdict.pub(super) cloud_judge: Option<Tri>,/// Capable-tier answer verdict after `cloud_judge` affirms.@@ -79,8 +82,8 @@ fn dial(branch: Branch) -> Option<f64> {match branch {Branch::Answer | Branch::DefaultVerified => Some(0.7),Branch::Chat => Some(0.3),- Branch::Agentic => Some(0.2),- Branch::Coding | Branch::Unknown => None,+ Branch::Agentic | Branch::Coding => Some(0.2),+ Branch::Unknown => None,}}@@ -95,12 +98,13 @@ fn affirms(verdict: Option<Tri>) -> bool {/// The local rungs affirm: the readout at the branch dial, or deliberation.pub(super) fn locally_verified(branch: Branch, signals: &Signals) -> bool {let bar = dial(branch).map_or(READOUT, |dial| dial.min(READOUT));- clears(signals.readout, bar) || affirms(signals.deliberation)+ clears(signals.readout, bar) || clears(signals.deliberation, DELIBERATION)}/// Commits the local attempt or escalates.///-/// Coding has no sandboxed checker here, so it never commits.+/// Coding has no sandboxed checker here, so below a confident readout it commits+/// only on the capable tier's affirmation.pub(super) fn decide(caps: &Capabilities,signals: &Signals,@@ -109,7 +113,11 @@ pub(super) fn decide(let branch = caps.branch;let clean = caps.tools.is_none_or(|tools| tools.errors == 0);let commit = match branch {- Branch::Coding | Branch::Unknown => false,+ Branch::Unknown => false,+ Branch::Coding => {+ (clean && clears(signals.readout, CODING_CONFIDENT))+ || (locally_verified(branch, signals) && affirms(signals.cloud_judge))+ }Branch::Answer => {affirms(signals.answer_verifier)|| affirms(signals.evidence_verifier)@@ -117,7 +125,7 @@ pub(super) fn decide(}Branch::Chat => {clean- && (affirms(signals.deliberation)+ && (clears(signals.deliberation, DELIBERATION)|| clears(signals.readout, 0.3)|| (affirms(signals.cloud_judge) && affirms(signals.evidence_confirm)))}diff --git a/crates/libsy/src/algorithms/vgr/rungs.rs b/crates/libsy/src/algorithms/vgr/rungs.rsindex 3c75a8ec8..e6a7aaff6 100644--- a/crates/libsy/src/algorithms/vgr/rungs.rs+++ b/crates/libsy/src/algorithms/vgr/rungs.rs@@ -17,12 +17,14 @@ use super::text::{Turn, char_len, clip_mid, content_text, redact};use crate::algorithms::util::transcript::message_text;pub(super) const DELIBERATION_MAX_OUTPUT_TOKENS: u64 = 512;-pub(super) const LOCAL_DELIBERATION_MAX_OUTPUT_TOKENS: u64 = 768;+const LOCAL_DELIBERATION_MAX_OUTPUT_TOKENS: u64 = 768;const TYPING_MAX_OUTPUT_TOKENS: u64 = 8;const TYPING_TASK_BUDGET: usize = 4000;// The condensed trajectory a turn judge sees.const TURN_RECENT_MESSAGES: usize = 28;+// Advance in blocks to reuse the cached prefix.+const TURN_WINDOW_STEP: usize = 8;const TURN_MESSAGE_CHARS: usize = 500;const TURN_SYSTEM_CHARS: usize = 1_000;const TURN_FIRST_USER_CHARS: usize = 2_000;@@ -34,10 +36,9 @@ const ANSWER_VERIFIER: &str = include_str!("../../prompts/vgr/answer-verifier.mdconst EVIDENCE_VERIFIER_ROLE: &str = "You are a strict verifier.";const EVIDENCE_VERIFIER_TASK: &str = include_str!("../../prompts/vgr/evidence-verifier-task.md");const DIRECT_VERDICT: &str = " Reply with exactly one word: yes or no.";-const BOUNDED_DELIBERATION: &str = " Think briefly: use no more than three short sentences of \-private analysis, then give the verdict.";-const COMPLETE_DELIBERATION: &str = " You must finish within the available token budget. Your \-visible reply must be exactly one word: yes or no.";+const DELIBERATION: &str = " Reason carefully about whether the evidence supports the result.";+const GIVEN_ANALYSIS: &str = " You are also given a verifier's analysis of the record, which may \+be cut off before its conclusion; weigh it, but judge the record yourself.";const TYPING_SYSTEM: &str = include_str!("../../prompts/vgr/typing-system.md");const TURN_JUDGE_SYSTEM: &str = include_str!("../../prompts/vgr/turn-judge-system.md");@@ -47,8 +48,10 @@ pub(super) enum Question {Answer,/// Does this record show the work was completed?Evidence,- /// The evidence question, briefly reasoned.+ /// The evidence question, reasoned through without a verdict.Deliberation,+ /// The evidence question, answered given that reasoning.+ Verdict,Typing,/// Is this proposed tool call part of a persistently bad trajectory?TurnTrajectory,@@ -63,7 +66,11 @@ impl Question {EVIDENCE_VERIFIER_TASK.trim_end()),Self::Deliberation => format!(- "{EVIDENCE_VERIFIER_ROLE}{BOUNDED_DELIBERATION}{}{COMPLETE_DELIBERATION}{INJECTION_GUARD}",+ "{EVIDENCE_VERIFIER_ROLE}{}{DELIBERATION}{INJECTION_GUARD}",+ EVIDENCE_VERIFIER_TASK.trim_end()+ ),+ Self::Verdict => format!(+ "{EVIDENCE_VERIFIER_ROLE}{}{GIVEN_ANALYSIS}{DIRECT_VERDICT}{INJECTION_GUARD}",EVIDENCE_VERIFIER_TASK.trim_end()),Self::Typing => TYPING_SYSTEM.trim_end().to_string(),@@ -118,8 +125,52 @@ pub(super) fn build_typing_request(task_text: &str, metadata: Option<Metadata>)request}-/// The system and task anchors, the recent turns, and the proposed turn.-pub(super) fn turn_view(request: &Request, proposed: &AggLlmResponse) -> String {+pub(super) fn build_deliberation_request(judged_text: &str, metadata: Option<Metadata>) -> Request {+ build_request(+ Question::Deliberation,+ judged_text,+ LOCAL_DELIBERATION_MAX_OUTPUT_TOKENS,+ metadata,+ )+}++/// Asks for a scored verdict on the record given a deliberation, however it stopped.+pub(super) fn build_verdict_request(+ judged_text: &str,+ deliberation: &AggLlmResponse,+ metadata: Option<Metadata>,+) -> Request {+ let output = deliberation.first_output();+ let analysis = output+ .into_iter()+ .flat_map(|output| &output.content)+ .filter_map(|block| match block {+ ContentBlock::Reasoning { text, .. } | ContentBlock::Text { text } => Some(text.trim()),+ _ => None,+ })+ .collect::<Vec<_>>()+ .join("\n");+ let cut_off = output+ .and_then(|output| output.stop_reason)+ .is_some_and(|reason| reason != StopReason::EndTurn);+ let note = if cut_off {+ "\n[analysis cut off at its token budget]"+ } else {+ ""+ };+ let view = format!("{judged_text}\n\n[verifier analysis]\n{analysis}{note}");+ let mut request = build_request(+ Question::Verdict,+ &view,+ super::readout::MAX_OUTPUT_TOKENS,+ metadata,+ );+ super::readout::request_logprobs(&mut request);+ request+}++/// Stable history messages let llama.cpp reuse its cache before the proposed turn.+pub(super) fn turn_view(request: &Request, proposed: &AggLlmResponse) -> Vec<Message> {let turns: Vec<Turn> = request.llm_request.instructions@@ -144,16 +195,52 @@ pub(super) fn turn_view(request: &Request, proposed: &AggLlmResponse) -> Stringlet task = clip_mid(&redact(&turns[index].text), TURN_FIRST_USER_CHARS, 0.5);anchors.push(format!("[task] {task}"));}- let mut window: Vec<String> = turns+ let recent: Vec<&Turn> = turns.iter().enumerate().filter(|(index, _)| Some(*index) != system_index && Some(*index) != first_user_index)- .rev()- .take(TURN_RECENT_MESSAGES)- .map(|(_, turn)| turn_line(turn, TURN_MESSAGE_CHARS))+ .map(|(_, turn)| turn).collect();- window.reverse();+ // Retain the prefix until the next window step.+ let overflow = recent.len().saturating_sub(TURN_RECENT_MESSAGES);+ let mut window: Vec<(Role, String)> = recent[overflow / TURN_WINDOW_STEP * TURN_WINDOW_STEP..]+ .iter()+ .map(|turn| {+ let role = if turn.role == Role::Assistant {+ Role::Assistant+ } else {+ Role::User+ };+ (role, turn_line(turn, TURN_MESSAGE_CHARS))+ })+ .collect();++ let proposed = proposed_line(proposed);+ let anchor_text = anchors.join("\n");+ let total = |window: &[(Role, String)]| {+ char_len(&anchor_text)+ + window+ .iter()+ .map(|(_, line)| char_len(line) + 1)+ .sum::<usize>()+ + char_len(&proposed)+ };+ while !window.is_empty() && total(&window) > TURN_MAX_CHARS {+ window.drain(..TURN_WINDOW_STEP.min(window.len()));+ }+ let mut messages = Vec::with_capacity(window.len() + 2);+ messages.push(Message::text(Role::User, anchor_text));+ for (role, line) in window {+ messages.push(Message::text(role, line));+ }+ // Only this trailing message changes on each call.+ messages.push(Message::text(Role::User, proposed));+ messages+}++/// The proposed next turn, as the judge sees it.+fn proposed_line(proposed: &AggLlmResponse) -> String {let mut fragments = Vec::new();for output in proposed.outputs@@ -174,24 +261,21 @@ pub(super) fn turn_view(request: &Request, proposed: &AggLlmResponse) -> String}}}- let proposed = format!(+ format!("[proposed next turn] [assistant] {}",clip_mid(&fragments.join(" "), TURN_MESSAGE_CHARS, 0.5)- );- let joined = |window: &[String]| {- anchors- .iter()- .chain(window)- .map(|line| char_len(line) + 1)- .sum::<usize>()- + char_len(&proposed)- };- while !window.is_empty() && joined(&window) > TURN_MAX_CHARS {- window.remove(0);- }- anchors.extend(window);- anchors.push(proposed);- anchors.join("\n").chars().take(TURN_MAX_CHARS).collect()+ )+}++/// A turn-judge call built from a message list.+pub(super) fn build_turn_request(+ messages: Vec<Message>,+ max_output_tokens: u64,+ metadata: Option<Metadata>,+) -> Request {+ let mut request = build_request(Question::TurnTrajectory, "", max_output_tokens, metadata);+ request.llm_request.messages = messages;+ request}fn turn_line(turn: &Turn, budget: usize) -> String {@@ -317,6 +401,27 @@ mod tests {assert_eq!(request.llm_request.messages.len(), 1);assert!(request.llm_request.tools.is_empty());assert_eq!(request.llm_request.sampling.temperature, None);+ let mut deliberation = replied("the dates check out", Some(StopReason::MaxTokens));+ deliberation.outputs[0].content.insert(+ 0,+ ContentBlock::Reasoning {+ text: "Two stories are in the window.".into(),+ signature: None,+ details: Vec::new(),+ },+ );+ let verdict = build_verdict_request("record", &deliberation, None);+ assert_eq!(+ verdict.llm_request.messages[0].text_content("").as_deref(),+ Some(+ "record\n\n[verifier analysis]\nTwo stories are in the window.\nthe dates check \+ out\n[analysis cut off at its token budget]"+ )+ );+ assert_eq!(+ verdict.llm_request.reasoning.effort.as_deref(),+ Some("none")+ );let typing = build_typing_request(&"x".repeat(5000), None);assert_eq!(typing.llm_request.sampling.temperature, Some(0.0));assert_eq!(typing.llm_request.reasoning.effort.as_deref(), Some("none"));@@ -328,6 +433,13 @@ mod tests {);}+ fn flatten(view: &[Message]) -> String {+ view.iter()+ .filter_map(|message| message.text_content(""))+ .collect::<Vec<_>>()+ .join("\n")+ }+#[test]fn turn_view_keeps_anchors_and_recent_turns_in_order() {let mut messages = vec![@@ -338,13 +450,14 @@ mod tests {.extend((0..30).map(|index| Message::text(Role::Assistant, format!("turn {index}"))));let request = super::super::tests::request(messages);let view = turn_view(&request, &replied("next", None));- let mut expected = vec![- "[system] system rule".to_string(),- "[task] task".to_string(),- ];- expected.extend((2..30).map(|index| format!("[assistant] turn {index}")));+ assert_eq!(view.len(), 32);+ assert_eq!(view[1].role, Role::Assistant);+ assert_eq!(view[31].role, Role::User);+ let mut expected = vec!["[system] system rule\n[task] task".to_string()];+ // Two past the recent window, but its start moves only in whole steps.+ expected.extend((0..30).map(|index| format!("[assistant] turn {index}")));expected.push("[proposed next turn] [assistant] next".to_string());- assert_eq!(view, expected.join("\n"));+ assert_eq!(flatten(&view), expected.join("\n"));}#[test]@@ -373,7 +486,7 @@ mod tests {}],..Default::default()};- let view = turn_view(&request, &proposed);+ let view = flatten(&turn_view(&request, &proposed));assert!(view.contains(r#"[assistant] tool_call bash({"command":"echo [REDACTED]"})"#));assert!(view.contains("[tool] failed; contact [REDACTED]"));assert!(view.contains(@@ -389,7 +502,7 @@ mod tests {Message::text(Role::User, "task"),super::super::tests::result("history", &format!("BEGIN{}END", "é".repeat(1000))),]);- let view = turn_view(&request, &replied("next", None));+ let view = flatten(&turn_view(&request, &replied("next", None)));assert!(view.contains("[tool] BEGIN"));assert!(view.contains("chars omitted]..."));assert!(view.contains("END\n[proposed next turn]"));diff --git a/crates/libsy/src/algorithms/vgr/runtime.rs b/crates/libsy/src/algorithms/vgr/runtime.rsindex acd05c237..b00647ce4 100644--- a/crates/libsy/src/algorithms/vgr/runtime.rs+++ b/crates/libsy/src/algorithms/vgr/runtime.rs@@ -246,6 +246,20 @@ impl Classifier for VgrClassifier {}impl VgrClassifier {+ /// Whether the local tier may answer `request` when the cloud tier fails it.+ ///+ /// That answer is unverified, so it is offered only while local attempts are allowed.+ pub(super) fn local_may_stand_in(&self, request: &Request) -> bool {+ matches!(self.config.mode, ServingMode::Active { .. })+ && !self+ .config+ .kill_switch+ .as_ref()+ .is_some_and(|switch| switch.is_engaged())+ && !self.breaker.is_open()+ && (self.config.local_supports_images || !request_has_image(request))+ }+fn escalate(&self,driver: &Driver,@@ -301,9 +315,8 @@ impl VgrClassifier {proposed: &AggLlmResponse,state: &mut State,) -> bool {- let mut call = rungs::build_request(- Question::TurnTrajectory,- &rungs::turn_view(context.request, proposed),+ let mut call = rungs::build_turn_request(+ rungs::turn_view(context.request, proposed),readout::MAX_OUTPUT_TOKENS,context.request.metadata.clone(),);@@ -360,7 +373,7 @@ impl VgrClassifier {let run = (branch == Branch::Agentic).then(|| decide::agentic_run(caps, confirmed));let (Some(judged), false) = (caps.transcript.as_deref(),- matches!(branch, Branch::Coding | Branch::Unknown) || run == Some(AgenticRun::Vetoed),+ branch == Branch::Unknown || run == Some(AgenticRun::Vetoed),) else {return signals;};@@ -371,19 +384,33 @@ impl VgrClassifier {self.ask(context, target, question, judged, max_output_tokens)};- signals.readout = self.readout(context, judged).await;+ // Coding here ran no tools, so its record holds no evidence: judge the code itself.+ let asked = match branch {+ Branch::Coding => Question::Answer,+ _ => Question::Evidence,+ };+ signals.readout = self.readout(context, judged, asked).await;+ if branch == Branch::Coding {+ if let Some(cloud) = self.config.targets.cloud_judge.as_ref()+ && decide::locally_verified(branch, &signals)+ && !settled(&signals)+ {+ let tokens = rungs::DELIBERATION_MAX_OUTPUT_TOKENS;+ signals.cloud_judge = Some(ask(cloud, asked, tokens).await);+ }+ return signals;+ }if settled(&signals) {return signals;}if !(cloud_confirms && decide::locally_verified(branch, &signals)) {- signals.deliberation = Some(- ask(- local,- Question::Deliberation,- rungs::LOCAL_DELIBERATION_MAX_OUTPUT_TOKENS,- )- .await,- );+ let metadata = || context.request.metadata.clone();+ let call = rungs::build_deliberation_request(judged, metadata());+ if let Some(Ok(deliberation)) = self.call(context, call, local).await {+ let call = rungs::build_verdict_request(judged, &deliberation, metadata());+ let verdict = self.call(context, call, local).await.and_then(Result::ok);+ signals.deliberation = verdict.and_then(|agg| readout::p_yes(&agg));+ }if settled(&signals) {return signals;}@@ -417,9 +444,14 @@ impl VgrClassifier {signals}- async fn readout(&self, context: &RequestContext<'_>, judged: &str) -> Option<f64> {+ async fn readout(+ &self,+ context: &RequestContext<'_>,+ judged: &str,+ question: Question,+ ) -> Option<f64> {let mut call = rungs::build_request(- Question::Evidence,+ question,judged,readout::MAX_OUTPUT_TOKENS,context.request.metadata.clone(),diff --git a/crates/libsy/src/algorithms/vgr/runtime/tests.rs b/crates/libsy/src/algorithms/vgr/runtime/tests.rsindex 95b079c57..f8018166e 100644--- a/crates/libsy/src/algorithms/vgr/runtime/tests.rs+++ b/crates/libsy/src/algorithms/vgr/runtime/tests.rs@@ -146,8 +146,8 @@ async fn a_truncated_local_attempt_is_judged_before_it_escalates() -> Result<()>};let selected = drive(&route, task, calls.clone(), cut_off, 0.0).await?;assert_eq!(selected, "cloud");- // Attempt, readout and deliberation; read as absent, only the attempt runs.- assert_eq!(calls.0.load(Ordering::Relaxed), 3);+ // Attempt, readout, deliberation and its verdict; read as absent, only the attempt runs.+ assert_eq!(calls.0.load(Ordering::Relaxed), 4);Ok(())}@@ -864,3 +864,71 @@ async fn tool_work_from_an_earlier_user_turn_is_not_handed_off() -> Result<()> {assert_eq!(selected, "cloud");Ok(())}++#[tokio::test]+async fn a_cloud_decision_falls_back_to_local_without_the_handoff_note() -> Result<()> {+ use crate::core::algorithm::Call;+ use crate::core::testing::serve_decision;++ let task = || {+ session(vec![+ Message::text(Role::User, "fix the build"),+ call("c1"),+ result("c1", "ok"),+ ])+ };+ let noted = |request: &Request| {+ serde_json::to_string(&request.llm_request.messages)+ .is_ok_and(|text| text.contains("Routing notice"))+ };+ for killed in [false, true] {+ let switch = super::super::KillSwitch::new();+ let algorithm = route(|config| {+ config.agentic_handoff = true;+ config.kill_switch = Some(switch.clone());+ })?;+ drive(+ &algorithm,+ session(vec![Message::text(Role::User, "fix the build")]),+ Arc::new(Calls::default()),+ complete_call,+ 0.0,+ )+ .await?;+ if killed {+ switch.engage();+ }+ let outcome = crate::drive(+ algorithm,+ task(),+ Arc::new(category_models(Category::Any, &["local", "cloud"]).into()),+ |call| async move {+ match call {+ Call::Model(call) => {+ let response = match call.models[0].as_str() {+ "local" => reply("the build passes"),+ "judge" => scored(0.0),+ _ => reply("no"),+ };+ call.respond(std::future::ready(Ok(response))).await+ }+ Call::Decision(call) => serve_decision(*call).await,+ }+ },+ )+ .await?;+ assert!(noted(&outcome.request));+ if killed {+ assert_eq!(outcome.selected_model_ids, vec![ModelId::new("cloud")]);+ assert!(outcome.fallback_request.is_none());+ } else {+ assert_eq!(+ outcome.selected_model_ids,+ vec![ModelId::new("cloud"), ModelId::new("local")]+ );+ let fallback = outcome.fallback_request.expect("a local fallback request");+ assert_eq!(fallback.llm_request.messages, task().llm_request.messages);+ }+ }+ Ok(())+}diff --git a/crates/libsy/src/algorithms/vgr/safety.rs b/crates/libsy/src/algorithms/vgr/safety.rsindex eac01fc36..91a67d550 100644--- a/crates/libsy/src/algorithms/vgr/safety.rs+++ b/crates/libsy/src/algorithms/vgr/safety.rs@@ -97,6 +97,11 @@ impl CircuitBreaker {*self.state.lock() = BreakerState::default();}+ /// Whether repeated failures have opened the breaker; it stays open until a trial succeeds.+ pub(super) fn is_open(&self) -> bool {+ self.state.lock().opened_at.is_some()+ }+pub(super) fn failure(&self) {let mut state = self.state.lock();state.failures = state.failures.saturating_add(1);diff --git a/crates/libsy/src/algorithms/vgr/tests.rs b/crates/libsy/src/algorithms/vgr/tests.rsindex cbae6959e..6760420df 100644--- a/crates/libsy/src/algorithms/vgr/tests.rs+++ b/crates/libsy/src/algorithms/vgr/tests.rs@@ -329,14 +329,13 @@ mod decide {}#[test]- fn deliberation_commits_only_on_an_affirmative_verdict() {+ fn deliberation_commits_only_on_an_affirmative_score() {for branch in [Branch::Chat, Branch::Agentic, Branch::DefaultVerified] {let mut caps = agentic(0, 1, 1);caps.branch = branch;for (verdict, expected) in [- (Some(Tri::Yes), Route::Local),- (Some(Tri::No), Route::Cloud),- (Some(Tri::Unknown), Route::Cloud),+ (Some(0.5), Route::Local),+ (Some(0.49), Route::Cloud),(None, Route::Cloud),] {let signals = Signals {@@ -351,4 +350,20 @@ mod decide {}}}++ #[test]+ fn coding_commits_on_a_confident_readout_or_a_capable_tier_affirmation() {+ let mut coding = agentic(0, 0, 0);+ coding.branch = Branch::Coding;+ let mut signals = readout(0.2);+ assert_eq!(decide(&coding, &signals, None), Route::Cloud);+ signals.cloud_judge = Some(Tri::Unknown);+ assert_eq!(decide(&coding, &signals, None), Route::Cloud);+ signals.cloud_judge = Some(Tri::Yes);+ assert_eq!(decide(&coding, &signals, None), Route::Local);+ signals.readout = Some(0.19);+ assert_eq!(decide(&coding, &signals, None), Route::Cloud);+ assert_eq!(decide(&coding, &readout(0.6), None), Route::Local);+ assert_eq!(decide(&coding, &readout(0.59), None), Route::Cloud);+ }}diff --git a/crates/libsy/src/core/algorithm.rs b/crates/libsy/src/core/algorithm.rsindex a43a9edda..da6fd7cbd 100644--- a/crates/libsy/src/core/algorithm.rs+++ b/crates/libsy/src/core/algorithm.rs@@ -258,6 +258,10 @@ pub struct RoutingOutcome {pub selected_model_ids: Vec<ModelId>,/// The request after all routing-time rewrites, stamped with the selected model.pub request: Request,+ /// The request the fallback candidates receive instead of `request`, when routing+ /// rewrote `request` for the selected model only.+ /// Content-policy denials do not advance to these fallback candidates.+ pub fallback_request: Option<Request>,/// A response produced while routing, or `None` when the client must make the answer call.pub response: Option<Response>,/// Outcome identity and optional algorithm evidence.@@ -291,6 +295,7 @@ impl RoutingOutcome {Self {selected_model_ids,request,+ fallback_request: None,response: None,metadata: None,state: State::default(),@@ -304,6 +309,7 @@ impl RoutingOutcome {Self {selected_model_ids: vec![selected_model_id],request,+ fallback_request: None,response: Some(response),metadata: None,state: State::default(),diff --git a/crates/switchyard-py/src/libsy_bindings.rs b/crates/switchyard-py/src/libsy_bindings.rsindex 83de01ff1..057cd3401 100644--- a/crates/switchyard-py/src/libsy_bindings.rs+++ b/crates/switchyard-py/src/libsy_bindings.rs@@ -754,6 +754,7 @@ impl PyOutcomeMetadata {struct PyRoutingOutcome {selected_model_ids: Vec<String>,request: Py<PyAny>,+ fallback_request: Option<Py<PyAny>>,response: Option<Py<PyAny>>,#[pyo3(get)]metadata: Option<Py<PyOutcomeMetadata>>,@@ -773,6 +774,14 @@ impl PyRoutingOutcome {self.request.clone_ref(py)}+ /// The request for the fallback candidates, when it differs from `request`.+ #[getter]+ fn fallback_request(&self, py: Python<'_>) -> Option<Py<PyAny>> {+ self.fallback_request+ .as_ref()+ .map(|request| request.clone_ref(py))+ }+/// An aggregate or streamed answer produced while routing, when one already exists.#[getter]fn response(&self, py: Python<'_>) -> Option<Py<PyAny>> {@@ -939,6 +948,7 @@ fn step_to_python(step: RustStep) -> PyResult<PyStep> {let RoutingOutcome {selected_model_ids,request,+ fallback_request,response,metadata,state: _,@@ -956,6 +966,9 @@ fn step_to_python(step: RustStep) -> PyResult<PyStep> {.map(ToString::to_string).collect(),request: to_python(py, &request.llm_request)?,+ fallback_request: fallback_request+ .map(|request| to_python(py, &request.llm_request))+ .transpose()?,response: response.map(|response| response_to_python(py, response.llm_response)).transpose()?,diff --git a/crates/switchyard-server/tests/server.rs b/crates/switchyard-server/tests/server.rsindex c9d668766..445fde093 100644--- a/crates/switchyard-server/tests/server.rs+++ b/crates/switchyard-server/tests/server.rs@@ -456,6 +456,13 @@ async fn upstream_chat(} else {"ok".to_string()};+ let logprobs = (body["logprobs"] == true).then(|| {+ let yes = if content == "yes" { 0.9_f64 } else { 0.1_f64 };+ json!({"content": [{"token": content, "logprob": 0.0, "top_logprobs": [+ {"token": "yes", "logprob": yes.ln()},+ {"token": "no", "logprob": (1.0 - yes).ln()}+ ]}]})+ });Json(json!({"id": "chatcmpl-test","object": "chat.completion",@@ -463,6 +470,7 @@ async fn upstream_chat("choices": [{"index": 0,"message": {"role": "assistant", "content": content},+ "logprobs": logprobs,"finish_reason": "stop"}],"usage": {diff --git a/docs/routing_algorithms/vgr_routing.md b/docs/routing_algorithms/vgr_routing.mdindex b8ca77fa6..f8f7953f7 100644--- a/docs/routing_algorithms/vgr_routing.md+++ b/docs/routing_algorithms/vgr_routing.md@@ -103,6 +103,17 @@ automatic continuations. Retries retain the original call's tag. Without arecognized tag, Switchyard uses the message shape to identify a user turn.`x-session-id` is also accepted as a session-header alias.+## Verification and cloud outages++Coding without tool history can commit locally on a clean readout of at least+0.6, or a readout of at least 0.2 plus cloud-judge affirmation. Other final-answer+branches can use a separate scored verdict after local deliberation.++In `active` mode, an unavailable cloud tier can fall back to an unverified local+answer. That call receives the original request without the cloud handoff note.+Fallback is disabled by the kill switch, an open local circuit breaker, or an+image request the local tier cannot serve.+## Serving modes- `off` skips candidate generation and serves cloud. This is the default.diff --git a/switchyard_rust/libsy.py b/switchyard_rust/libsy.pyindex 15ff7fc49..9d944257e 100644--- a/switchyard_rust/libsy.py+++ b/switchyard_rust/libsy.py@@ -173,6 +173,9 @@ if TYPE_CHECKING:@propertydef request(self) -> dict[str, object]: ...+ @property+ def fallback_request(self) -> dict[str, object] | None: ...+@propertydef response(self) -> LlmResponse.Agg | LlmResponse.Stream | None: ...