Skip to content

Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.

Rust

fix(vgr): restore QA verification and cloud outage fallback (#1042)

GDRS3a48a933de0c
diff --git a/crates/libsy-llm-client/src/run.rs b/crates/libsy-llm-client/src/run.rs
index 661e05c6b..fd9c6a2fd 100644
--- a/crates/libsy-llm-client/src/run.rs
+++ b/crates/libsy-llm-client/src/run.rs
@@ -111,11 +111,12 @@ pub async fn run(
&clients,
&algorithm_name,
&outcome.request,
+ outcome.fallback_request.as_ref(),
&outcome.selected_model_ids,
&observe,
)
.await
- .and_then(|response| clients.remember_state_owner(&outcome.request, response))
+ .and_then(|(request, response)| clients.remember_state_owner(request, response))
};
if let Some(observer) = &observer {
observer(RunObservation::RoutingOverhead(overhead));
@@ -245,19 +246,24 @@ async fn serve(
}
/// Try candidates in order until one succeeds or a failure stops fallback.
-async fn call_first_available(
+async fn call_first_available<'a>(
clients: &ClientRouter,
algorithm: &str,
- request: &Request,
+ request: &'a Request,
+ fallback_request: Option<&'a Request>,
models: &[ModelId],
observe: &(dyn Fn(ModelCallObservation) + Send + Sync),
-) -> Result<Response> {
+) -> Result<(&'a Request, Response)> {
for (index, target) in models.iter().enumerate() {
- let request = clients.prepare_completion_request(request.clone(), target);
+ let request = match fallback_request {
+ Some(fallback) if index > 0 => fallback,
+ _ => request,
+ };
+ let prepared = clients.prepare_completion_request(request.clone(), target);
match call_one(
clients,
target,
- request,
+ prepared,
algorithm,
observe,
index,
@@ -266,7 +272,7 @@ async fn call_first_available(
)
.await
{
- Ok(response) => return Ok(response),
+ Ok(response) => return Ok((request, response)),
Err(error) if index + 1 == models.len() => return Err(error),
Err(error)
if clients.rate_limit_fallback_disabled.contains(target)
@@ -276,7 +282,7 @@ async fn call_first_available(
{
return Err(error);
}
- Err(error) => match fallback_reason(&error) {
+ Err(error) => match fallback_reason(&error, fallback_request.is_none()) {
Some(reason) => tracing::info!(
from = %target,
to = %models[index + 1],
@@ -442,7 +448,10 @@ async fn buffer_routing_stream(
}
/// Whether a failed candidate is worth routing around.
-fn fallback_reason(error: &LibsyError) -> Option<RoutingFallbackReason> {
+fn fallback_reason(
+ error: &LibsyError,
+ allow_policy_fallback: bool,
+) -> Option<RoutingFallbackReason> {
let LibsyError::ClientCall { source, .. } = error else {
return None;
};
@@ -451,23 +460,22 @@ fn fallback_reason(error: &LibsyError) -> Option<RoutingFallbackReason> {
LlmClientError::Transport { .. } | LlmClientError::TemporarilyUnavailable => {
Some(RoutingFallbackReason::Unavailable)
}
- // A policy denial can be specific to one provider. Preserve its HTTP
- // error, but allow another candidate to serve the request.
- LlmClientError::UpstreamHttp { status, body }
- if *status == StatusCode::BAD_REQUEST
- && serde_json::from_str::<serde_json::Value>(body).is_ok_and(|value| {
- value["error"]["code"].as_str() == Some("content_policy_violation")
- }) =>
- {
- Some(RoutingFallbackReason::Unavailable)
- }
- LlmClientError::UpstreamHttp { status, .. }
- if matches!(
- *status,
- StatusCode::FORBIDDEN | StatusCode::REQUEST_TIMEOUT | StatusCode::TOO_MANY_REQUESTS
- ) || status.is_server_error() =>
- {
- Some(RoutingFallbackReason::Unavailable)
+ LlmClientError::UpstreamHttp { status, body } => {
+ let policy_denial = serde_json::from_str::<Value>(body).is_ok_and(|value| {
+ value["error"]["code"].as_str() == Some("content_policy_violation")
+ });
+ if policy_denial && !allow_policy_fallback {
+ return None;
+ }
+ ((*status == StatusCode::BAD_REQUEST && policy_denial)
+ || matches!(
+ *status,
+ StatusCode::FORBIDDEN
+ | StatusCode::REQUEST_TIMEOUT
+ | StatusCode::TOO_MANY_REQUESTS
+ )
+ || status.is_server_error())
+ .then_some(RoutingFallbackReason::Unavailable)
}
_ => None,
}
@@ -1101,10 +1109,36 @@ mod tests {
}
}
+ /// Rewrites the request for its selected model only.
+ struct RewritingAlgorithm {}
+
+ #[async_trait]
+ impl Algorithm for RewritingAlgorithm {
+ fn name(&self) -> &str {
+ "rewriting_test"
+ }
+
+ async fn route(
+ self: Arc<Self>,
+ _driver: Driver,
+ request: Request,
+ ) -> Result<RoutingOutcome> {
+ let mut rewritten = request.clone();
+ rewritten
+ .llm_request
+ .messages
+ .push(Message::text(Role::User, "for weak only"));
+ let mut outcome =
+ RoutingOutcome::route_to("weak".into(), vec!["strong".into()], rewritten);
+ outcome.fallback_request = Some(request);
+ Ok(outcome)
+ }
+ }
+
#[derive(Clone, Copy)]
enum FirstOutcome {
ContextWindow,
- ContentPolicy,
+ ContentPolicy(StatusCode),
Unauthorized,
StreamSuccess,
MidStreamError,
@@ -1128,8 +1162,8 @@ mod tests {
model,
message: "too long".to_string(),
}),
- FirstOutcome::ContentPolicy => Err(LlmClientError::UpstreamHttp {
- status: StatusCode::BAD_REQUEST,
+ FirstOutcome::ContentPolicy(status) => Err(LlmClientError::UpstreamHttp {
+ status,
body: r#"{"error":{"code":"content_policy_violation","message":"request blocked by content policy","type":"invalid_request_error"},"metadata":{"documentation_section":"context window"}}"#.to_string(),
}),
FirstOutcome::Unauthorized => Err(LlmClientError::UpstreamHttp {
@@ -1157,7 +1191,10 @@ mod tests {
};
}
Ok(Response {
- llm_response: LlmResponse::Agg(text_response(Some(model.to_string()), model)),
+ llm_response: LlmResponse::Agg(AggLlmResponse {
+ id: Some("candidate_response".into()),
+ ..text_response(Some(model.to_string()), model)
+ }),
metadata: None,
upstream_headers: http::HeaderMap::new(),
})
@@ -2256,10 +2293,13 @@ mod tests {
fn fallback_only_accepts_context_policy_and_unavailable_failures() {
let error = |source| LibsyError::client_call("target", source);
assert_eq!(
- fallback_reason(&error(LlmClientError::ContextWindowExceeded {
- model: "target".into(),
- message: "too long".to_string(),
- })),
+ fallback_reason(
+ &error(LlmClientError::ContextWindowExceeded {
+ model: "target".into(),
+ message: "too long".to_string(),
+ }),
+ true
+ ),
Some(RoutingFallbackReason::ContextWindow)
);
for status in [
@@ -2270,10 +2310,13 @@ mod tests {
StatusCode::from_u16(599).expect("599 is a valid status"),
] {
assert_eq!(
- fallback_reason(&error(LlmClientError::UpstreamHttp {
- status,
- body: "failed".to_string(),
- })),
+ fallback_reason(
+ &error(LlmClientError::UpstreamHttp {
+ status,
+ body: "failed".to_string(),
+ }),
+ true
+ ),
Some(RoutingFallbackReason::Unavailable)
);
}
@@ -2290,10 +2333,13 @@ mod tests {
("content_policy_violation", None),
] {
assert_eq!(
- fallback_reason(&error(LlmClientError::UpstreamHttp {
- status: StatusCode::BAD_REQUEST,
- body: body.to_string(),
- })),
+ fallback_reason(
+ &error(LlmClientError::UpstreamHttp {
+ status: StatusCode::BAD_REQUEST,
+ body: body.to_string(),
+ }),
+ true
+ ),
expected,
"{body}"
);
@@ -2307,10 +2353,13 @@ mod tests {
StatusCode::from_u16(600).expect("600 is a valid status"),
] {
assert_eq!(
- fallback_reason(&error(LlmClientError::UpstreamHttp {
- status,
- body: "failed".to_string(),
- })),
+ fallback_reason(
+ &error(LlmClientError::UpstreamHttp {
+ status,
+ body: "failed".to_string(),
+ }),
+ true
+ ),
None
);
}
@@ -2335,7 +2384,8 @@ mod tests {
assert_eq!(response.served_model().map(ModelId::as_str), Some("strong"));
// A provider-specific policy denial also permits another candidate.
- let (client, result) = run_candidates(FirstOutcome::ContentPolicy).await;
+ let (client, result) =
+ run_candidates(FirstOutcome::ContentPolicy(StatusCode::BAD_REQUEST)).await;
let (_, response) = result?;
assert_eq!(
&*client.calls.lock(),
@@ -2359,6 +2409,78 @@ mod tests {
Ok(())
}
+ #[tokio::test]
+ async fn fallback_candidates_receive_the_fallback_request() -> Result<()> {
+ let client = Arc::new(CandidateClient {
+ calls: Mutex::new(Vec::new()),
+ requests: Mutex::new(Vec::new()),
+ first: FirstOutcome::ContextWindow,
+ });
+ let clients = ClientRouter::single(client.clone());
+ let mut seed = request();
+ seed.llm_request
+ .preservation
+ .requests
+ .insert(WireFormat::OpenAiResponses.into(), json!({}));
+ run(
+ Arc::new(RewritingAlgorithm {}),
+ clients.clone(),
+ seed,
+ to_category_map(&["weak", "strong"]),
+ None,
+ )
+ .await?;
+ let sent: Vec<_> = client
+ .requests
+ .lock()
+ .iter()
+ .map(|request| request.llm_request.messages.clone())
+ .collect();
+ assert_eq!(sent.len(), 2);
+ assert_eq!(sent[0].len(), request().llm_request.messages.len() + 1);
+ assert_eq!(sent[1], request().llm_request.messages);
+ let mut follow = request();
+ follow
+ .llm_request
+ .extensions
+ .fields
+ .insert("previous_response_id".into(), json!("candidate_response"));
+ follow.llm_request.messages = vec![Message::text(Role::User, "continue")];
+ let owner = clients
+ .stored_state_owner(&follow)
+ .expect("fallback Responses state");
+ let continued = continue_on(owner, "test", follow);
+ let mut expected = sent[1].clone();
+ expected.push(Message::text(Role::Assistant, "strong"));
+ expected.push(Message::text(Role::User, "continue"));
+ assert_eq!(continued.request.llm_request.messages, expected);
+ Ok(())
+ }
+
+ #[tokio::test]
+ async fn fallback_request_does_not_bypass_content_policy() -> Result<()> {
+ for status in [StatusCode::BAD_REQUEST, StatusCode::FORBIDDEN] {
+ let client = Arc::new(CandidateClient {
+ calls: Mutex::new(Vec::new()),
+ requests: Mutex::new(Vec::new()),
+ first: FirstOutcome::ContentPolicy(status),
+ });
+ let result = run(
+ Arc::new(RewritingAlgorithm {}),
+ ClientRouter::single(client.clone()),
+ request(),
+ to_category_map(&["weak", "strong"]),
+ None,
+ )
+ .await;
+ assert!(
+ matches!(result, Err(LibsyError::ClientCall { source: LlmClientError::UpstreamHttp { status: actual, .. }, .. }) if actual == status)
+ );
+ assert_eq!(&*client.calls.lock(), &[ModelId::from("weak")]);
+ }
+ Ok(())
+ }
+
#[tokio::test]
async fn retry_exhaustion_falls_back_and_optional_cooldown_skips_failed_backend() -> Result<()>
{
diff --git a/crates/libsy/src/algorithms/vgr.rs b/crates/libsy/src/algorithms/vgr.rs
index fc99ada46..7a94338f4 100644
--- a/crates/libsy/src/algorithms/vgr.rs
+++ b/crates/libsy/src/algorithms/vgr.rs
@@ -37,6 +37,7 @@ mod tests;
/// A verification-gated route between a local and a capable tier.
pub struct Vgr {
route: FallThrough,
+ classifier: Arc<runtime::VgrClassifier>,
}
impl Vgr {
@@ -59,8 +60,8 @@ impl Vgr {
.with_name("vgr")
.with_processor(affinity.clone())
.with_classifier(Arc::new(runtime::Held(affinity, reason)))
- .with_classifier(classifier);
- Ok(Self { route })
+ .with_classifier(classifier.clone());
+ Ok(Self { route, classifier })
}
}
@@ -71,9 +72,27 @@ impl Algorithm for Vgr {
}
async fn route(self: Arc<Self>, driver: Driver, request: Request) -> Result<RoutingOutcome> {
+ // The handoff note is written for the cloud tier, so a local stand-in gets the request
+ // as it arrived.
+ let arrived = self
+ .classifier
+ .local_may_stand_in(&request)
+ .then(|| request.clone());
let mut outcome = self.route.execute(driver, request).await?;
- // Local answers are already buffered; cloud decisions cannot fall back to local.
+ // Local answers are already buffered and have no fallback.
outcome.selected_model_ids.truncate(1);
+ let config = &self.classifier.config;
+ if let Some(arrived) = arrived
+ && outcome.response.is_none()
+ && outcome.selected_model_ids.first() == Some(&config.targets.cloud)
+ // The local tier may have failed while this request was routed.
+ && self.classifier.local_may_stand_in(&arrived)
+ {
+ outcome
+ .selected_model_ids
+ .push(config.targets.local.clone());
+ outcome.fallback_request = Some(arrived);
+ }
Ok(outcome)
}
}
@@ -150,10 +169,10 @@ fn derive_capabilities(
Some(render::render_agentic_view(request, &turns, attempt)),
);
}
+ let transcript = Some(render::render_session(&turns, attempt));
if text::observed_hardening(attempt) || task_type == Some(TaskType::Coding) {
- return caps(Branch::Coding, None);
+ return caps(Branch::Coding, transcript);
}
- let transcript = Some(render::render_session(&turns, attempt));
if text::observed_tool_activity(attempt) {
caps(Branch::Agentic, transcript)
} else if matches!(task_type, Some(TaskType::Chat | TaskType::Answer)) {
diff --git a/crates/libsy/src/algorithms/vgr/decide.rs b/crates/libsy/src/algorithms/vgr/decide.rs
index b68e275c0..11c05f22d 100644
--- a/crates/libsy/src/algorithms/vgr/decide.rs
+++ b/crates/libsy/src/algorithms/vgr/decide.rs
@@ -12,6 +12,9 @@ use super::{Branch, Capabilities};
pub(super) const POLICY_VERSION: &str = "1.0";
const READOUT: f64 = 0.9;
+const DELIBERATION: f64 = 0.5;
+/// Readout at which clean coding commits without the capable tier's affirmation.
+const CODING_CONFIDENT: f64 = 0.6;
/// Most tool results a run may contain and still recover without the capable tier.
const SHORT_RUN_MAX_TOOL_RESULTS: i32 = 100;
@@ -33,8 +36,8 @@ pub(super) enum Tri {
pub(super) struct Signals {
/// Local logprob score that the attempt is correct.
pub(super) readout: Option<f64>,
- /// Local deliberating verdict.
- pub(super) deliberation: Option<Tri>,
+ /// Local logprob score that the attempt is correct, read after a deliberation.
+ pub(super) deliberation: Option<f64>,
/// Capable-tier evidence verdict.
pub(super) cloud_judge: Option<Tri>,
/// Capable-tier answer verdict after `cloud_judge` affirms.
@@ -79,8 +82,8 @@ fn dial(branch: Branch) -> Option<f64> {
match branch {
Branch::Answer | Branch::DefaultVerified => Some(0.7),
Branch::Chat => Some(0.3),
- Branch::Agentic => Some(0.2),
- Branch::Coding | Branch::Unknown => None,
+ Branch::Agentic | Branch::Coding => Some(0.2),
+ Branch::Unknown => None,
}
}
@@ -95,12 +98,13 @@ fn affirms(verdict: Option<Tri>) -> bool {
/// The local rungs affirm: the readout at the branch dial, or deliberation.
pub(super) fn locally_verified(branch: Branch, signals: &Signals) -> bool {
let bar = dial(branch).map_or(READOUT, |dial| dial.min(READOUT));
- clears(signals.readout, bar) || affirms(signals.deliberation)
+ clears(signals.readout, bar) || clears(signals.deliberation, DELIBERATION)
}
/// Commits the local attempt or escalates.
///
-/// Coding has no sandboxed checker here, so it never commits.
+/// Coding has no sandboxed checker here, so below a confident readout it commits
+/// only on the capable tier's affirmation.
pub(super) fn decide(
caps: &Capabilities,
signals: &Signals,
@@ -109,7 +113,11 @@ pub(super) fn decide(
let branch = caps.branch;
let clean = caps.tools.is_none_or(|tools| tools.errors == 0);
let commit = match branch {
- Branch::Coding | Branch::Unknown => false,
+ Branch::Unknown => false,
+ Branch::Coding => {
+ (clean && clears(signals.readout, CODING_CONFIDENT))
+ || (locally_verified(branch, signals) && affirms(signals.cloud_judge))
+ }
Branch::Answer => {
affirms(signals.answer_verifier)
|| affirms(signals.evidence_verifier)
@@ -117,7 +125,7 @@ pub(super) fn decide(
}
Branch::Chat => {
clean
- && (affirms(signals.deliberation)
+ && (clears(signals.deliberation, DELIBERATION)
|| clears(signals.readout, 0.3)
|| (affirms(signals.cloud_judge) && affirms(signals.evidence_confirm)))
}
diff --git a/crates/libsy/src/algorithms/vgr/rungs.rs b/crates/libsy/src/algorithms/vgr/rungs.rs
index 3c75a8ec8..e6a7aaff6 100644
--- a/crates/libsy/src/algorithms/vgr/rungs.rs
+++ b/crates/libsy/src/algorithms/vgr/rungs.rs
@@ -17,12 +17,14 @@ use super::text::{Turn, char_len, clip_mid, content_text, redact};
use crate::algorithms::util::transcript::message_text;
pub(super) const DELIBERATION_MAX_OUTPUT_TOKENS: u64 = 512;
-pub(super) const LOCAL_DELIBERATION_MAX_OUTPUT_TOKENS: u64 = 768;
+const LOCAL_DELIBERATION_MAX_OUTPUT_TOKENS: u64 = 768;
const TYPING_MAX_OUTPUT_TOKENS: u64 = 8;
const TYPING_TASK_BUDGET: usize = 4000;
// The condensed trajectory a turn judge sees.
const TURN_RECENT_MESSAGES: usize = 28;
+// Advance in blocks to reuse the cached prefix.
+const TURN_WINDOW_STEP: usize = 8;
const TURN_MESSAGE_CHARS: usize = 500;
const TURN_SYSTEM_CHARS: usize = 1_000;
const TURN_FIRST_USER_CHARS: usize = 2_000;
@@ -34,10 +36,9 @@ const ANSWER_VERIFIER: &str = include_str!("../../prompts/vgr/answer-verifier.md
const EVIDENCE_VERIFIER_ROLE: &str = "You are a strict verifier.";
const EVIDENCE_VERIFIER_TASK: &str = include_str!("../../prompts/vgr/evidence-verifier-task.md");
const DIRECT_VERDICT: &str = " Reply with exactly one word: yes or no.";
-const BOUNDED_DELIBERATION: &str = " Think briefly: use no more than three short sentences of \
-private analysis, then give the verdict.";
-const COMPLETE_DELIBERATION: &str = " You must finish within the available token budget. Your \
-visible reply must be exactly one word: yes or no.";
+const DELIBERATION: &str = " Reason carefully about whether the evidence supports the result.";
+const GIVEN_ANALYSIS: &str = " You are also given a verifier's analysis of the record, which may \
+be cut off before its conclusion; weigh it, but judge the record yourself.";
const TYPING_SYSTEM: &str = include_str!("../../prompts/vgr/typing-system.md");
const TURN_JUDGE_SYSTEM: &str = include_str!("../../prompts/vgr/turn-judge-system.md");
@@ -47,8 +48,10 @@ pub(super) enum Question {
Answer,
/// Does this record show the work was completed?
Evidence,
- /// The evidence question, briefly reasoned.
+ /// The evidence question, reasoned through without a verdict.
Deliberation,
+ /// The evidence question, answered given that reasoning.
+ Verdict,
Typing,
/// Is this proposed tool call part of a persistently bad trajectory?
TurnTrajectory,
@@ -63,7 +66,11 @@ impl Question {
EVIDENCE_VERIFIER_TASK.trim_end()
),
Self::Deliberation => format!(
- "{EVIDENCE_VERIFIER_ROLE}{BOUNDED_DELIBERATION}{}{COMPLETE_DELIBERATION}{INJECTION_GUARD}",
+ "{EVIDENCE_VERIFIER_ROLE}{}{DELIBERATION}{INJECTION_GUARD}",
+ EVIDENCE_VERIFIER_TASK.trim_end()
+ ),
+ Self::Verdict => format!(
+ "{EVIDENCE_VERIFIER_ROLE}{}{GIVEN_ANALYSIS}{DIRECT_VERDICT}{INJECTION_GUARD}",
EVIDENCE_VERIFIER_TASK.trim_end()
),
Self::Typing => TYPING_SYSTEM.trim_end().to_string(),
@@ -118,8 +125,52 @@ pub(super) fn build_typing_request(task_text: &str, metadata: Option<Metadata>)
request
}
-/// The system and task anchors, the recent turns, and the proposed turn.
-pub(super) fn turn_view(request: &Request, proposed: &AggLlmResponse) -> String {
+pub(super) fn build_deliberation_request(judged_text: &str, metadata: Option<Metadata>) -> Request {
+ build_request(
+ Question::Deliberation,
+ judged_text,
+ LOCAL_DELIBERATION_MAX_OUTPUT_TOKENS,
+ metadata,
+ )
+}
+
+/// Asks for a scored verdict on the record given a deliberation, however it stopped.
+pub(super) fn build_verdict_request(
+ judged_text: &str,
+ deliberation: &AggLlmResponse,
+ metadata: Option<Metadata>,
+) -> Request {
+ let output = deliberation.first_output();
+ let analysis = output
+ .into_iter()
+ .flat_map(|output| &output.content)
+ .filter_map(|block| match block {
+ ContentBlock::Reasoning { text, .. } | ContentBlock::Text { text } => Some(text.trim()),
+ _ => None,
+ })
+ .collect::<Vec<_>>()
+ .join("\n");
+ let cut_off = output
+ .and_then(|output| output.stop_reason)
+ .is_some_and(|reason| reason != StopReason::EndTurn);
+ let note = if cut_off {
+ "\n[analysis cut off at its token budget]"
+ } else {
+ ""
+ };
+ let view = format!("{judged_text}\n\n[verifier analysis]\n{analysis}{note}");
+ let mut request = build_request(
+ Question::Verdict,
+ &view,
+ super::readout::MAX_OUTPUT_TOKENS,
+ metadata,
+ );
+ super::readout::request_logprobs(&mut request);
+ request
+}
+
+/// Stable history messages let llama.cpp reuse its cache before the proposed turn.
+pub(super) fn turn_view(request: &Request, proposed: &AggLlmResponse) -> Vec<Message> {
let turns: Vec<Turn> = request
.llm_request
.instructions
@@ -144,16 +195,52 @@ pub(super) fn turn_view(request: &Request, proposed: &AggLlmResponse) -> String
let task = clip_mid(&redact(&turns[index].text), TURN_FIRST_USER_CHARS, 0.5);
anchors.push(format!("[task] {task}"));
}
- let mut window: Vec<String> = turns
+ let recent: Vec<&Turn> = turns
.iter()
.enumerate()
.filter(|(index, _)| Some(*index) != system_index && Some(*index) != first_user_index)
- .rev()
- .take(TURN_RECENT_MESSAGES)
- .map(|(_, turn)| turn_line(turn, TURN_MESSAGE_CHARS))
+ .map(|(_, turn)| turn)
.collect();
- window.reverse();
+ // Retain the prefix until the next window step.
+ let overflow = recent.len().saturating_sub(TURN_RECENT_MESSAGES);
+ let mut window: Vec<(Role, String)> = recent[overflow / TURN_WINDOW_STEP * TURN_WINDOW_STEP..]
+ .iter()
+ .map(|turn| {
+ let role = if turn.role == Role::Assistant {
+ Role::Assistant
+ } else {
+ Role::User
+ };
+ (role, turn_line(turn, TURN_MESSAGE_CHARS))
+ })
+ .collect();
+
+ let proposed = proposed_line(proposed);
+ let anchor_text = anchors.join("\n");
+ let total = |window: &[(Role, String)]| {
+ char_len(&anchor_text)
+ + window
+ .iter()
+ .map(|(_, line)| char_len(line) + 1)
+ .sum::<usize>()
+ + char_len(&proposed)
+ };
+ while !window.is_empty() && total(&window) > TURN_MAX_CHARS {
+ window.drain(..TURN_WINDOW_STEP.min(window.len()));
+ }
+ let mut messages = Vec::with_capacity(window.len() + 2);
+ messages.push(Message::text(Role::User, anchor_text));
+ for (role, line) in window {
+ messages.push(Message::text(role, line));
+ }
+ // Only this trailing message changes on each call.
+ messages.push(Message::text(Role::User, proposed));
+ messages
+}
+
+/// The proposed next turn, as the judge sees it.
+fn proposed_line(proposed: &AggLlmResponse) -> String {
let mut fragments = Vec::new();
for output in proposed
.outputs
@@ -174,24 +261,21 @@ pub(super) fn turn_view(request: &Request, proposed: &AggLlmResponse) -> String
}
}
}
- let proposed = format!(
+ format!(
"[proposed next turn] [assistant] {}",
clip_mid(&fragments.join(" "), TURN_MESSAGE_CHARS, 0.5)
- );
- let joined = |window: &[String]| {
- anchors
- .iter()
- .chain(window)
- .map(|line| char_len(line) + 1)
- .sum::<usize>()
- + char_len(&proposed)
- };
- while !window.is_empty() && joined(&window) > TURN_MAX_CHARS {
- window.remove(0);
- }
- anchors.extend(window);
- anchors.push(proposed);
- anchors.join("\n").chars().take(TURN_MAX_CHARS).collect()
+ )
+}
+
+/// A turn-judge call built from a message list.
+pub(super) fn build_turn_request(
+ messages: Vec<Message>,
+ max_output_tokens: u64,
+ metadata: Option<Metadata>,
+) -> Request {
+ let mut request = build_request(Question::TurnTrajectory, "", max_output_tokens, metadata);
+ request.llm_request.messages = messages;
+ request
}
fn turn_line(turn: &Turn, budget: usize) -> String {
@@ -317,6 +401,27 @@ mod tests {
assert_eq!(request.llm_request.messages.len(), 1);
assert!(request.llm_request.tools.is_empty());
assert_eq!(request.llm_request.sampling.temperature, None);
+ let mut deliberation = replied("the dates check out", Some(StopReason::MaxTokens));
+ deliberation.outputs[0].content.insert(
+ 0,
+ ContentBlock::Reasoning {
+ text: "Two stories are in the window.".into(),
+ signature: None,
+ details: Vec::new(),
+ },
+ );
+ let verdict = build_verdict_request("record", &deliberation, None);
+ assert_eq!(
+ verdict.llm_request.messages[0].text_content("").as_deref(),
+ Some(
+ "record\n\n[verifier analysis]\nTwo stories are in the window.\nthe dates check \
+ out\n[analysis cut off at its token budget]"
+ )
+ );
+ assert_eq!(
+ verdict.llm_request.reasoning.effort.as_deref(),
+ Some("none")
+ );
let typing = build_typing_request(&"x".repeat(5000), None);
assert_eq!(typing.llm_request.sampling.temperature, Some(0.0));
assert_eq!(typing.llm_request.reasoning.effort.as_deref(), Some("none"));
@@ -328,6 +433,13 @@ mod tests {
);
}
+ fn flatten(view: &[Message]) -> String {
+ view.iter()
+ .filter_map(|message| message.text_content(""))
+ .collect::<Vec<_>>()
+ .join("\n")
+ }
+
#[test]
fn turn_view_keeps_anchors_and_recent_turns_in_order() {
let mut messages = vec![
@@ -338,13 +450,14 @@ mod tests {
.extend((0..30).map(|index| Message::text(Role::Assistant, format!("turn {index}"))));
let request = super::super::tests::request(messages);
let view = turn_view(&request, &replied("next", None));
- let mut expected = vec![
- "[system] system rule".to_string(),
- "[task] task".to_string(),
- ];
- expected.extend((2..30).map(|index| format!("[assistant] turn {index}")));
+ assert_eq!(view.len(), 32);
+ assert_eq!(view[1].role, Role::Assistant);
+ assert_eq!(view[31].role, Role::User);
+ let mut expected = vec!["[system] system rule\n[task] task".to_string()];
+ // Two past the recent window, but its start moves only in whole steps.
+ expected.extend((0..30).map(|index| format!("[assistant] turn {index}")));
expected.push("[proposed next turn] [assistant] next".to_string());
- assert_eq!(view, expected.join("\n"));
+ assert_eq!(flatten(&view), expected.join("\n"));
}
#[test]
@@ -373,7 +486,7 @@ mod tests {
}],
..Default::default()
};
- let view = turn_view(&request, &proposed);
+ let view = flatten(&turn_view(&request, &proposed));
assert!(view.contains(r#"[assistant] tool_call bash({"command":"echo [REDACTED]"})"#));
assert!(view.contains("[tool] failed; contact [REDACTED]"));
assert!(view.contains(
@@ -389,7 +502,7 @@ mod tests {
Message::text(Role::User, "task"),
super::super::tests::result("history", &format!("BEGIN{}END", "é".repeat(1000))),
]);
- let view = turn_view(&request, &replied("next", None));
+ let view = flatten(&turn_view(&request, &replied("next", None)));
assert!(view.contains("[tool] BEGIN"));
assert!(view.contains("chars omitted]..."));
assert!(view.contains("END\n[proposed next turn]"));
diff --git a/crates/libsy/src/algorithms/vgr/runtime.rs b/crates/libsy/src/algorithms/vgr/runtime.rs
index acd05c237..b00647ce4 100644
--- a/crates/libsy/src/algorithms/vgr/runtime.rs
+++ b/crates/libsy/src/algorithms/vgr/runtime.rs
@@ -246,6 +246,20 @@ impl Classifier for VgrClassifier {
}
impl VgrClassifier {
+ /// Whether the local tier may answer `request` when the cloud tier fails it.
+ ///
+ /// That answer is unverified, so it is offered only while local attempts are allowed.
+ pub(super) fn local_may_stand_in(&self, request: &Request) -> bool {
+ matches!(self.config.mode, ServingMode::Active { .. })
+ && !self
+ .config
+ .kill_switch
+ .as_ref()
+ .is_some_and(|switch| switch.is_engaged())
+ && !self.breaker.is_open()
+ && (self.config.local_supports_images || !request_has_image(request))
+ }
+
fn escalate(
&self,
driver: &Driver,
@@ -301,9 +315,8 @@ impl VgrClassifier {
proposed: &AggLlmResponse,
state: &mut State,
) -> bool {
- let mut call = rungs::build_request(
- Question::TurnTrajectory,
- &rungs::turn_view(context.request, proposed),
+ let mut call = rungs::build_turn_request(
+ rungs::turn_view(context.request, proposed),
readout::MAX_OUTPUT_TOKENS,
context.request.metadata.clone(),
);
@@ -360,7 +373,7 @@ impl VgrClassifier {
let run = (branch == Branch::Agentic).then(|| decide::agentic_run(caps, confirmed));
let (Some(judged), false) = (
caps.transcript.as_deref(),
- matches!(branch, Branch::Coding | Branch::Unknown) || run == Some(AgenticRun::Vetoed),
+ branch == Branch::Unknown || run == Some(AgenticRun::Vetoed),
) else {
return signals;
};
@@ -371,19 +384,33 @@ impl VgrClassifier {
self.ask(context, target, question, judged, max_output_tokens)
};
- signals.readout = self.readout(context, judged).await;
+ // Coding here ran no tools, so its record holds no evidence: judge the code itself.
+ let asked = match branch {
+ Branch::Coding => Question::Answer,
+ _ => Question::Evidence,
+ };
+ signals.readout = self.readout(context, judged, asked).await;
+ if branch == Branch::Coding {
+ if let Some(cloud) = self.config.targets.cloud_judge.as_ref()
+ && decide::locally_verified(branch, &signals)
+ && !settled(&signals)
+ {
+ let tokens = rungs::DELIBERATION_MAX_OUTPUT_TOKENS;
+ signals.cloud_judge = Some(ask(cloud, asked, tokens).await);
+ }
+ return signals;
+ }
if settled(&signals) {
return signals;
}
if !(cloud_confirms && decide::locally_verified(branch, &signals)) {
- signals.deliberation = Some(
- ask(
- local,
- Question::Deliberation,
- rungs::LOCAL_DELIBERATION_MAX_OUTPUT_TOKENS,
- )
- .await,
- );
+ let metadata = || context.request.metadata.clone();
+ let call = rungs::build_deliberation_request(judged, metadata());
+ if let Some(Ok(deliberation)) = self.call(context, call, local).await {
+ let call = rungs::build_verdict_request(judged, &deliberation, metadata());
+ let verdict = self.call(context, call, local).await.and_then(Result::ok);
+ signals.deliberation = verdict.and_then(|agg| readout::p_yes(&agg));
+ }
if settled(&signals) {
return signals;
}
@@ -417,9 +444,14 @@ impl VgrClassifier {
signals
}
- async fn readout(&self, context: &RequestContext<'_>, judged: &str) -> Option<f64> {
+ async fn readout(
+ &self,
+ context: &RequestContext<'_>,
+ judged: &str,
+ question: Question,
+ ) -> Option<f64> {
let mut call = rungs::build_request(
- Question::Evidence,
+ question,
judged,
readout::MAX_OUTPUT_TOKENS,
context.request.metadata.clone(),
diff --git a/crates/libsy/src/algorithms/vgr/runtime/tests.rs b/crates/libsy/src/algorithms/vgr/runtime/tests.rs
index 95b079c57..f8018166e 100644
--- a/crates/libsy/src/algorithms/vgr/runtime/tests.rs
+++ b/crates/libsy/src/algorithms/vgr/runtime/tests.rs
@@ -146,8 +146,8 @@ async fn a_truncated_local_attempt_is_judged_before_it_escalates() -> Result<()>
};
let selected = drive(&route, task, calls.clone(), cut_off, 0.0).await?;
assert_eq!(selected, "cloud");
- // Attempt, readout and deliberation; read as absent, only the attempt runs.
- assert_eq!(calls.0.load(Ordering::Relaxed), 3);
+ // Attempt, readout, deliberation and its verdict; read as absent, only the attempt runs.
+ assert_eq!(calls.0.load(Ordering::Relaxed), 4);
Ok(())
}
@@ -864,3 +864,71 @@ async fn tool_work_from_an_earlier_user_turn_is_not_handed_off() -> Result<()> {
assert_eq!(selected, "cloud");
Ok(())
}
+
+#[tokio::test]
+async fn a_cloud_decision_falls_back_to_local_without_the_handoff_note() -> Result<()> {
+ use crate::core::algorithm::Call;
+ use crate::core::testing::serve_decision;
+
+ let task = || {
+ session(vec![
+ Message::text(Role::User, "fix the build"),
+ call("c1"),
+ result("c1", "ok"),
+ ])
+ };
+ let noted = |request: &Request| {
+ serde_json::to_string(&request.llm_request.messages)
+ .is_ok_and(|text| text.contains("Routing notice"))
+ };
+ for killed in [false, true] {
+ let switch = super::super::KillSwitch::new();
+ let algorithm = route(|config| {
+ config.agentic_handoff = true;
+ config.kill_switch = Some(switch.clone());
+ })?;
+ drive(
+ &algorithm,
+ session(vec![Message::text(Role::User, "fix the build")]),
+ Arc::new(Calls::default()),
+ complete_call,
+ 0.0,
+ )
+ .await?;
+ if killed {
+ switch.engage();
+ }
+ let outcome = crate::drive(
+ algorithm,
+ task(),
+ Arc::new(category_models(Category::Any, &["local", "cloud"]).into()),
+ |call| async move {
+ match call {
+ Call::Model(call) => {
+ let response = match call.models[0].as_str() {
+ "local" => reply("the build passes"),
+ "judge" => scored(0.0),
+ _ => reply("no"),
+ };
+ call.respond(std::future::ready(Ok(response))).await
+ }
+ Call::Decision(call) => serve_decision(*call).await,
+ }
+ },
+ )
+ .await?;
+ assert!(noted(&outcome.request));
+ if killed {
+ assert_eq!(outcome.selected_model_ids, vec![ModelId::new("cloud")]);
+ assert!(outcome.fallback_request.is_none());
+ } else {
+ assert_eq!(
+ outcome.selected_model_ids,
+ vec![ModelId::new("cloud"), ModelId::new("local")]
+ );
+ let fallback = outcome.fallback_request.expect("a local fallback request");
+ assert_eq!(fallback.llm_request.messages, task().llm_request.messages);
+ }
+ }
+ Ok(())
+}
diff --git a/crates/libsy/src/algorithms/vgr/safety.rs b/crates/libsy/src/algorithms/vgr/safety.rs
index eac01fc36..91a67d550 100644
--- a/crates/libsy/src/algorithms/vgr/safety.rs
+++ b/crates/libsy/src/algorithms/vgr/safety.rs
@@ -97,6 +97,11 @@ impl CircuitBreaker {
*self.state.lock() = BreakerState::default();
}
+ /// Whether repeated failures have opened the breaker; it stays open until a trial succeeds.
+ pub(super) fn is_open(&self) -> bool {
+ self.state.lock().opened_at.is_some()
+ }
+
pub(super) fn failure(&self) {
let mut state = self.state.lock();
state.failures = state.failures.saturating_add(1);
diff --git a/crates/libsy/src/algorithms/vgr/tests.rs b/crates/libsy/src/algorithms/vgr/tests.rs
index cbae6959e..6760420df 100644
--- a/crates/libsy/src/algorithms/vgr/tests.rs
+++ b/crates/libsy/src/algorithms/vgr/tests.rs
@@ -329,14 +329,13 @@ mod decide {
}
#[test]
- fn deliberation_commits_only_on_an_affirmative_verdict() {
+ fn deliberation_commits_only_on_an_affirmative_score() {
for branch in [Branch::Chat, Branch::Agentic, Branch::DefaultVerified] {
let mut caps = agentic(0, 1, 1);
caps.branch = branch;
for (verdict, expected) in [
- (Some(Tri::Yes), Route::Local),
- (Some(Tri::No), Route::Cloud),
- (Some(Tri::Unknown), Route::Cloud),
+ (Some(0.5), Route::Local),
+ (Some(0.49), Route::Cloud),
(None, Route::Cloud),
] {
let signals = Signals {
@@ -351,4 +350,20 @@ mod decide {
}
}
}
+
+ #[test]
+ fn coding_commits_on_a_confident_readout_or_a_capable_tier_affirmation() {
+ let mut coding = agentic(0, 0, 0);
+ coding.branch = Branch::Coding;
+ let mut signals = readout(0.2);
+ assert_eq!(decide(&coding, &signals, None), Route::Cloud);
+ signals.cloud_judge = Some(Tri::Unknown);
+ assert_eq!(decide(&coding, &signals, None), Route::Cloud);
+ signals.cloud_judge = Some(Tri::Yes);
+ assert_eq!(decide(&coding, &signals, None), Route::Local);
+ signals.readout = Some(0.19);
+ assert_eq!(decide(&coding, &signals, None), Route::Cloud);
+ assert_eq!(decide(&coding, &readout(0.6), None), Route::Local);
+ assert_eq!(decide(&coding, &readout(0.59), None), Route::Cloud);
+ }
}
diff --git a/crates/libsy/src/core/algorithm.rs b/crates/libsy/src/core/algorithm.rs
index a43a9edda..da6fd7cbd 100644
--- a/crates/libsy/src/core/algorithm.rs
+++ b/crates/libsy/src/core/algorithm.rs
@@ -258,6 +258,10 @@ pub struct RoutingOutcome {
pub selected_model_ids: Vec<ModelId>,
/// The request after all routing-time rewrites, stamped with the selected model.
pub request: Request,
+ /// The request the fallback candidates receive instead of `request`, when routing
+ /// rewrote `request` for the selected model only.
+ /// Content-policy denials do not advance to these fallback candidates.
+ pub fallback_request: Option<Request>,
/// A response produced while routing, or `None` when the client must make the answer call.
pub response: Option<Response>,
/// Outcome identity and optional algorithm evidence.
@@ -291,6 +295,7 @@ impl RoutingOutcome {
Self {
selected_model_ids,
request,
+ fallback_request: None,
response: None,
metadata: None,
state: State::default(),
@@ -304,6 +309,7 @@ impl RoutingOutcome {
Self {
selected_model_ids: vec![selected_model_id],
request,
+ fallback_request: None,
response: Some(response),
metadata: None,
state: State::default(),
diff --git a/crates/switchyard-py/src/libsy_bindings.rs b/crates/switchyard-py/src/libsy_bindings.rs
index 83de01ff1..057cd3401 100644
--- a/crates/switchyard-py/src/libsy_bindings.rs
+++ b/crates/switchyard-py/src/libsy_bindings.rs
@@ -754,6 +754,7 @@ impl PyOutcomeMetadata {
struct PyRoutingOutcome {
selected_model_ids: Vec<String>,
request: Py<PyAny>,
+ fallback_request: Option<Py<PyAny>>,
response: Option<Py<PyAny>>,
#[pyo3(get)]
metadata: Option<Py<PyOutcomeMetadata>>,
@@ -773,6 +774,14 @@ impl PyRoutingOutcome {
self.request.clone_ref(py)
}
+ /// The request for the fallback candidates, when it differs from `request`.
+ #[getter]
+ fn fallback_request(&self, py: Python<'_>) -> Option<Py<PyAny>> {
+ self.fallback_request
+ .as_ref()
+ .map(|request| request.clone_ref(py))
+ }
+
/// An aggregate or streamed answer produced while routing, when one already exists.
#[getter]
fn response(&self, py: Python<'_>) -> Option<Py<PyAny>> {
@@ -939,6 +948,7 @@ fn step_to_python(step: RustStep) -> PyResult<PyStep> {
let RoutingOutcome {
selected_model_ids,
request,
+ fallback_request,
response,
metadata,
state: _,
@@ -956,6 +966,9 @@ fn step_to_python(step: RustStep) -> PyResult<PyStep> {
.map(ToString::to_string)
.collect(),
request: to_python(py, &request.llm_request)?,
+ fallback_request: fallback_request
+ .map(|request| to_python(py, &request.llm_request))
+ .transpose()?,
response: response
.map(|response| response_to_python(py, response.llm_response))
.transpose()?,
diff --git a/crates/switchyard-server/tests/server.rs b/crates/switchyard-server/tests/server.rs
index c9d668766..445fde093 100644
--- a/crates/switchyard-server/tests/server.rs
+++ b/crates/switchyard-server/tests/server.rs
@@ -456,6 +456,13 @@ async fn upstream_chat(
} else {
"ok".to_string()
};
+ let logprobs = (body["logprobs"] == true).then(|| {
+ let yes = if content == "yes" { 0.9_f64 } else { 0.1_f64 };
+ json!({"content": [{"token": content, "logprob": 0.0, "top_logprobs": [
+ {"token": "yes", "logprob": yes.ln()},
+ {"token": "no", "logprob": (1.0 - yes).ln()}
+ ]}]})
+ });
Json(json!({
"id": "chatcmpl-test",
"object": "chat.completion",
@@ -463,6 +470,7 @@ async fn upstream_chat(
"choices": [{
"index": 0,
"message": {"role": "assistant", "content": content},
+ "logprobs": logprobs,
"finish_reason": "stop"
}],
"usage": {
diff --git a/docs/routing_algorithms/vgr_routing.md b/docs/routing_algorithms/vgr_routing.md
index b8ca77fa6..f8f7953f7 100644
--- a/docs/routing_algorithms/vgr_routing.md
+++ b/docs/routing_algorithms/vgr_routing.md
@@ -103,6 +103,17 @@ automatic continuations. Retries retain the original call's tag. Without a
recognized tag, Switchyard uses the message shape to identify a user turn.
`x-session-id` is also accepted as a session-header alias.
+## Verification and cloud outages
+
+Coding without tool history can commit locally on a clean readout of at least
+0.6, or a readout of at least 0.2 plus cloud-judge affirmation. Other final-answer
+branches can use a separate scored verdict after local deliberation.
+
+In `active` mode, an unavailable cloud tier can fall back to an unverified local
+answer. That call receives the original request without the cloud handoff note.
+Fallback is disabled by the kill switch, an open local circuit breaker, or an
+image request the local tier cannot serve.
+
## Serving modes
- `off` skips candidate generation and serves cloud. This is the default.
diff --git a/switchyard_rust/libsy.py b/switchyard_rust/libsy.py
index 15ff7fc49..9d944257e 100644
--- a/switchyard_rust/libsy.py
+++ b/switchyard_rust/libsy.py
@@ -173,6 +173,9 @@ if TYPE_CHECKING:
@property
def request(self) -> dict[str, object]: ...
+ @property
+ def fallback_request(self) -> dict[str, object] | None: ...
+
@property
def response(self) -> LlmResponse.Agg | LlmResponse.Stream | None: ...