Skip to content

Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.

Rust

fix(vgr): confirm default answers and preserve shadow evaluation (#1170)

GDRS3d02c9fb7522
diff --git a/crates/libsy/src/algorithms/vgr.rs b/crates/libsy/src/algorithms/vgr.rs
index 4a7d64664..2786c2150 100644
--- a/crates/libsy/src/algorithms/vgr.rs
+++ b/crates/libsy/src/algorithms/vgr.rs
@@ -61,10 +61,13 @@ impl Vgr {
breaker: safety::CircuitBreaker::new(config.breaker),
config,
});
- let route = route
- .with_processor(affinity.clone())
- .with_classifier(Arc::new(runtime::Held(affinity, reason)))
- .with_classifier(classifier.clone());
+ // Shadow always serves the capable tier; only a VGR escalation should hold it there.
+ if classifier.config.mode != ServingMode::Shadow {
+ route = route
+ .with_processor(affinity.clone())
+ .with_classifier(Arc::new(runtime::Held(affinity, reason)));
+ }
+ let route = route.with_classifier(classifier.clone());
Ok(Self { route, classifier })
}
}
diff --git a/crates/libsy/src/algorithms/vgr/decide.rs b/crates/libsy/src/algorithms/vgr/decide.rs
index 57f636f39..186084a61 100644
--- a/crates/libsy/src/algorithms/vgr/decide.rs
+++ b/crates/libsy/src/algorithms/vgr/decide.rs
@@ -9,7 +9,7 @@
use super::{Branch, Capabilities};
/// Version of these rules and dials, carried on every decision record.
-pub(super) const POLICY_VERSION: &str = "1.1";
+pub(super) const POLICY_VERSION: &str = "1.2";
const READOUT: f64 = 0.9;
const DELIBERATION: f64 = 0.5;
@@ -140,7 +140,11 @@ pub(super) fn decide(
AgenticRun::Vetoed => false,
}
}
- Branch::DefaultVerified => clean && efficient_verified(branch, signals),
+ Branch::DefaultVerified => {
+ clean
+ && (efficient_verified(branch, signals)
+ || (affirms(signals.capable_judge) && affirms(signals.evidence_confirm)))
+ }
};
if commit {
Route::Efficient
diff --git a/crates/libsy/src/algorithms/vgr/runtime.rs b/crates/libsy/src/algorithms/vgr/runtime.rs
index 26c2878fd..ecfb1b500 100644
--- a/crates/libsy/src/algorithms/vgr/runtime.rs
+++ b/crates/libsy/src/algorithms/vgr/runtime.rs
@@ -33,6 +33,8 @@ use crate::{LibsyError, Result};
/// Whether turn verification latched the capable tier; cleared when the user responds.
const TURN_LATCHED_KEY: &str = "vgr_turn_latched";
+/// Holds a shadow turn only after a VGR escalation.
+const SHADOW_HELD_KEY: &str = "vgr_shadow_held";
pub(super) const TURN_STREAK_KEY: &str = "vgr_turn_streak";
/// Unix seconds at the first request of the current user turn.
const TURN_STARTED_KEY: &str = "vgr_turn_started";
@@ -119,6 +121,7 @@ impl Classifier for VgrClassifier {
};
if new_turn {
state.extra.remove(TURN_STARTED_KEY);
+ state.extra.remove(SHADOW_HELD_KEY);
set_flag(state, HANDOFF_SENT_KEY, false);
}
if answered {
@@ -126,7 +129,9 @@ impl Classifier for VgrClassifier {
set_count(state, TURN_STREAK_KEY, 0);
}
let turn_budget_spent = self.turn_budget_spent(state);
- let short_circuit = if self.config.mode == ServingMode::Off {
+ let short_circuit = if flag(state, SHADOW_HELD_KEY) {
+ Some("turn_verification_latched")
+ } else if self.config.mode == ServingMode::Off {
Some("mode_off")
} else if self
.config
@@ -224,7 +229,7 @@ impl Classifier for VgrClassifier {
));
}
if self.config.mode == ServingMode::Shadow {
- return Ok(self.escalate(
+ return Ok(self.serve_capable(
driver,
"turn_verification_complete",
None,
@@ -243,7 +248,7 @@ impl Classifier for VgrClassifier {
let caps = derive_capabilities(request, &attempt, task_type);
let signals = self.gather(&context, &caps).await;
let route = decide::decide(&caps, &signals, self.config.confirmed_min_clean_tail());
- if route == Route::Capable || self.config.mode == ServingMode::Shadow {
+ if route == Route::Capable {
return Ok(self.escalate(
driver,
"decided",
@@ -253,6 +258,16 @@ impl Classifier for VgrClassifier {
Some(&attempt),
));
}
+ if self.config.mode == ServingMode::Shadow {
+ return Ok(self.serve_capable(
+ driver,
+ "decided",
+ Some(caps.branch),
+ request,
+ state,
+ Some(&attempt),
+ ));
+ }
log(driver, "decided", Some(caps.branch), Route::Efficient);
set_flag(state, EFFICIENT_SERVED_KEY, true);
Ok((decisive(efficient), Some(buffered.into_response())))
@@ -282,6 +297,21 @@ impl VgrClassifier {
request: &mut Request,
state: &mut State,
unverified_final: Option<&str>,
+ ) -> Scored {
+ if self.config.mode == ServingMode::Shadow {
+ set_flag(state, SHADOW_HELD_KEY, true);
+ }
+ self.serve_capable(driver, reason, branch, request, state, unverified_final)
+ }
+
+ fn serve_capable(
+ &self,
+ driver: &Driver,
+ reason: &'static str,
+ branch: Option<Branch>,
+ request: &mut Request,
+ state: &mut State,
+ unverified_final: Option<&str>,
) -> Scored {
if reason != "mode_off" {
self.hand_off(request, state, unverified_final);
@@ -456,7 +486,7 @@ impl VgrClassifier {
signals.evidence_verifier = Some(capable_ask(Question::Evidence).await);
}
}
- Branch::Chat => {
+ Branch::Chat | Branch::DefaultVerified => {
signals.capable_judge = Some(capable_ask(Question::Evidence).await);
// Asked only once the first affirms, so a refutation costs one call.
if signals.capable_judge == Some(Tri::Yes) {
diff --git a/crates/libsy/src/algorithms/vgr/runtime/tests.rs b/crates/libsy/src/algorithms/vgr/runtime/tests.rs
index 82aaa9225..a882b3353 100644
--- a/crates/libsy/src/algorithms/vgr/runtime/tests.rs
+++ b/crates/libsy/src/algorithms/vgr/runtime/tests.rs
@@ -134,7 +134,10 @@ async fn malformed_tool_calls_escalate_and_complete_ones_are_judged() -> Result<
#[tokio::test]
async fn a_truncated_efficient_attempt_is_judged_before_it_escalates() -> Result<()> {
- let route = route(|config| config.targets.judge = None)?;
+ let route = route(|config| {
+ config.targets.judge = None;
+ config.targets.capable_judge = None;
+ })?;
let calls = Arc::new(Calls::default());
let task = session(vec![Message::text(Role::User, "explain quicksort")]);
let cut_off = || {
@@ -468,7 +471,7 @@ async fn decisions_attach_the_reason_and_exclude_efficient_capable_fallbacks() -
assert_eq!(evidence["source"], "vgr");
assert_eq!(evidence["reason_code"], reason);
assert_eq!(evidence["verdict"], served);
- assert_eq!(evidence["policy_version"], "1.1");
+ assert_eq!(evidence["policy_version"], "1.2");
if reason == "decided" {
assert_eq!(evidence["branch"], "DefaultVerified");
}
@@ -1114,3 +1117,139 @@ async fn a_capable_decision_falls_back_to_efficient_without_the_handoff_note() -
}
Ok(())
}
+
+#[tokio::test]
+async fn default_verification_asks_capable_evidence_then_answer() -> Result<()> {
+ for (evidence, answer, expected) in [
+ ("yes", "yes", "efficient"),
+ ("yes", "no", "capable"),
+ ("yes", "unclear", "capable"),
+ ("no", "yes", "capable"),
+ ("unclear", "yes", "capable"),
+ ] {
+ let route = route(|_| {})?;
+ let calls = Arc::new(parking_lot::Mutex::new(Vec::new()));
+ let seen = Arc::clone(&calls);
+ let (selected, _) = test_drive_with_models(
+ route,
+ session(vec![Message::text(Role::User, "explain quicksort")]),
+ category_models(Category::Any, &["efficient", "capable"]),
+ move |target: ModelId, request: Request| {
+ let seen = Arc::clone(&seen);
+ async move {
+ Ok(match target.as_str() {
+ "efficient" => reply("Choose a pivot and partition the input."),
+ "judge" => scored(0.1),
+ "capable-judge" => {
+ let is_answer = request
+ .llm_request
+ .instructions
+ .iter()
+ .flat_map(|block| &block.content)
+ .any(|block| {
+ matches!(block, ContentBlock::Text { text }
+ if text.starts_with("You are a strict answer verifier"))
+ });
+ seen.lock().push(is_answer);
+ reply(if is_answer { answer } else { evidence })
+ }
+ _ => reply("capable answer"),
+ })
+ }
+ },
+ )
+ .await?;
+ assert_eq!(selected.as_str(), expected, "{evidence}, {answer}");
+ assert_eq!(
+ *calls.lock(),
+ if evidence == "yes" {
+ vec![false, true]
+ } else {
+ vec![false]
+ }
+ );
+ }
+ Ok(())
+}
+
+#[tokio::test]
+async fn shadow_keeps_judging_a_turn_until_vgr_escalates_it() -> Result<()> {
+ fn truncated() -> Response {
+ tool_call(json!(r#"{"command":"l"#), StopReason::MaxTokens)
+ }
+ for latch_escalation in [false, true] {
+ let route = route(|config| {
+ config.mode = ServingMode::Shadow;
+ config.latch_escalation = latch_escalation;
+ })?;
+ let calls = Arc::new(Calls::default());
+ let mut messages = vec![Message::text(Role::User, "fix the build")];
+ for (step, efficient) in [complete_call, complete_call, truncated, complete_call]
+ .into_iter()
+ .enumerate()
+ {
+ assert_eq!(
+ drive(
+ &route,
+ session(messages.clone()),
+ calls.clone(),
+ efficient,
+ 0.1
+ )
+ .await?,
+ "capable"
+ );
+ let id = format!("c{step}");
+ messages.extend([call(&id), result(&id, "ok")]);
+ }
+ assert_eq!(calls.0.load(Ordering::Relaxed), 3);
+ messages.push(Message::text(Role::User, "keep going"));
+ drive(
+ &route,
+ initiated_by(session(messages.clone()), "agent"),
+ calls.clone(),
+ complete_call,
+ 0.1,
+ )
+ .await?;
+ assert_eq!(calls.0.load(Ordering::Relaxed), 3);
+ drive(
+ &route,
+ initiated_by(session(messages), "user"),
+ calls.clone(),
+ complete_call,
+ 0.1,
+ )
+ .await?;
+ assert_eq!(calls.0.load(Ordering::Relaxed), 4);
+ }
+ Ok(())
+}
+
+#[tokio::test]
+async fn shadow_final_answer_commits_do_not_latch_but_rejections_do() -> Result<()> {
+ let route = route(|config| {
+ config.mode = ServingMode::Shadow;
+ config.targets.capable_judge = None;
+ })?;
+ let calls = Arc::new(Calls::default());
+ let mut messages = vec![Message::text(Role::User, "explain quicksort")];
+ for probability in [0.9, 0.9, 0.1, 0.9] {
+ let task = initiated_by(session(messages.clone()), "agent");
+ assert_eq!(
+ drive(
+ &route,
+ task,
+ calls.clone(),
+ || reply("Partition around a pivot."),
+ probability
+ )
+ .await?,
+ "capable"
+ );
+ messages.push(Message::text(Role::Assistant, "Partition around a pivot."));
+ messages.push(Message::text(Role::User, "continue"));
+ }
+ assert_eq!(calls.0.load(Ordering::Relaxed), 3);
+ Ok(())
+}
diff --git a/crates/libsy/src/algorithms/vgr/tests.rs b/crates/libsy/src/algorithms/vgr/tests.rs
index b70260bcb..741eaf732 100644
--- a/crates/libsy/src/algorithms/vgr/tests.rs
+++ b/crates/libsy/src/algorithms/vgr/tests.rs
@@ -384,6 +384,30 @@ mod decide {
assert_eq!(decide(&chat, &readout(1.0), None), Route::Capable);
}
+ #[test]
+ fn default_verification_requires_both_capable_verdicts_and_a_clean_record() {
+ let mut caps = agentic(0, 0, 0);
+ caps.branch = Branch::DefaultVerified;
+ for first in [None, Some(Tri::Unknown), Some(Tri::No), Some(Tri::Yes)] {
+ for second in [None, Some(Tri::Unknown), Some(Tri::No), Some(Tri::Yes)] {
+ let signals = Signals {
+ capable_judge: first,
+ evidence_confirm: second,
+ ..Signals::default()
+ };
+ let expected = if first == Some(Tri::Yes) && second == Some(Tri::Yes) {
+ Route::Efficient
+ } else {
+ Route::Capable
+ };
+ assert_eq!(decide(&caps, &signals, None), expected);
+ caps.tools.as_mut().unwrap().errors = 1;
+ assert_eq!(decide(&caps, &signals, None), Route::Capable);
+ caps.tools.as_mut().unwrap().errors = 0;
+ }
+ }
+ }
+
#[test]
fn deliberation_commits_only_on_an_affirmative_score() {
for branch in [Branch::Chat, Branch::Agentic, Branch::DefaultVerified] {
diff --git a/docs/routing_algorithms/vgr_routing.md b/docs/routing_algorithms/vgr_routing.md
index 213328095..d1d89f5de 100644
--- a/docs/routing_algorithms/vgr_routing.md
+++ b/docs/routing_algorithms/vgr_routing.md
@@ -132,7 +132,11 @@ recognized tag, Switchyard uses the message shape to identify a user turn.
## Verification and capable-tier outages
-Policy version `1.1` verifies typed factual answers on the answer itself, including
+Policy version `1.2` allows the default verification branch to accept a clean
+efficient attempt when the capable judge approves both the evidence and the
+answer. A negative or unavailable evidence verdict skips the answer check.
+
+Typed factual answers are verified on the answer itself, including
later user turns without tool work. A readout of at least 0.9 can serve the
efficient answer; lower scores require capable-tier verification. Scored verdicts
require yes and no together to hold at least half the first token's probability.
@@ -149,7 +153,10 @@ image request the efficient tier cannot serve.
## Serving modes
- `off` skips candidate generation and serves the capable tier. This is the default.
-- `shadow` computes a decision but serves the capable tier.
+- `shadow` computes a decision but serves the capable tier. It continues evaluating
+ subsequent steps until VGR escalates, then holds the capable tier for the rest
+ of the user turn. A new user turn resumes evaluation, including when
+ `latch_escalation = true`.
- `evaluate` serves the policy decision for isolated evaluation.
- `active` serves the policy decision in production and requires
`active_approval = "prospective-validation-and-canary-approved"`.