Skip to content

Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.

Rust

fix(vgr): restore QA verifier and Responses compatibility (#1163)

GDRS9ed588b4d0a9
diff --git a/crates/libsy/src/algorithms/vgr.rs b/crates/libsy/src/algorithms/vgr.rs
index 199546100..4a7d64664 100644
--- a/crates/libsy/src/algorithms/vgr.rs
+++ b/crates/libsy/src/algorithms/vgr.rs
@@ -154,7 +154,9 @@ fn derive_capabilities(
tools: Some(ToolRecord::from_request(request)),
};
- if task_type == Some(TaskType::Answer) && !text::has_assistant_turn(&turns) {
+ // Typing abstains on a request that leans on earlier context, so a typed answer stands
+ // on its own; only tool use in the current user turn takes it off this branch.
+ if task_type == Some(TaskType::Answer) && !text::turn_has_tool_trajectory(request) {
return Capabilities {
tools: None,
..caps(
diff --git a/crates/libsy/src/algorithms/vgr/decide.rs b/crates/libsy/src/algorithms/vgr/decide.rs
index 9f36a1012..57f636f39 100644
--- a/crates/libsy/src/algorithms/vgr/decide.rs
+++ b/crates/libsy/src/algorithms/vgr/decide.rs
@@ -9,12 +9,14 @@
use super::{Branch, Capabilities};
/// Version of these rules and dials, carried on every decision record.
-pub(super) const POLICY_VERSION: &str = "1.0";
+pub(super) const POLICY_VERSION: &str = "1.1";
const READOUT: f64 = 0.9;
const DELIBERATION: f64 = 0.5;
/// Readout at which clean coding commits without the capable tier's affirmation.
const CODING_CONFIDENT: f64 = 0.6;
+/// Readout required to serve a typed answer without capable-tier verification.
+const ANSWER_CONFIDENT: f64 = READOUT;
/// Most tool results a run may contain and still recover without the capable tier.
const SHORT_RUN_MAX_TOOL_RESULTS: i32 = 100;
@@ -80,7 +82,8 @@ pub(super) fn agentic_run(
/// The readout bar that licenses a commit on each branch.
fn dial(branch: Branch) -> Option<f64> {
match branch {
- Branch::Answer | Branch::DefaultVerified => Some(0.7),
+ Branch::Answer => Some(ANSWER_CONFIDENT),
+ Branch::DefaultVerified => Some(0.7),
Branch::Chat => Some(0.3),
Branch::Agentic | Branch::Coding => Some(0.2),
Branch::Unknown => None,
@@ -121,7 +124,7 @@ pub(super) fn decide(
Branch::Answer => {
affirms(signals.answer_verifier)
|| affirms(signals.evidence_verifier)
- || clears(signals.readout, 0.7)
+ || clears(signals.readout, ANSWER_CONFIDENT)
}
Branch::Chat => {
clean
diff --git a/crates/libsy/src/algorithms/vgr/readout.rs b/crates/libsy/src/algorithms/vgr/readout.rs
index 62533af93..c9d1da1e6 100644
--- a/crates/libsy/src/algorithms/vgr/readout.rs
+++ b/crates/libsy/src/algorithms/vgr/readout.rs
@@ -12,9 +12,21 @@ use switchyard_protocol::{AggLlmResponse, FormatId, Request, WireFormat};
pub(super) const MAX_OUTPUT_TOKENS: u64 = 4;
+/// Minimum combined yes/no probability for the first token to count as a verdict.
+const MIN_VERDICT_MASS: f64 = 0.5;
+
+/// Disables reasoning for one verifier call, including llama.cpp thinking templates.
+pub(super) fn disable_thinking(request: &mut Request) {
+ request.llm_request.reasoning.effort = Some("none".to_string());
+ request.llm_request.extensions.fields.insert(
+ "chat_template_kwargs".to_string(),
+ json!({"enable_thinking": false}),
+ );
+}
+
/// Asks for a direct verdict with its alternatives; reasoning stays request-local.
pub(super) fn request_logprobs(request: &mut Request) {
- request.llm_request.reasoning.effort = Some("none".to_string());
+ disable_thinking(request);
let extensions = &mut request.llm_request.extensions.fields;
extensions.insert("logprobs".to_string(), json!(true));
extensions.insert("top_logprobs".to_string(), json!(8));
@@ -46,7 +58,7 @@ pub(super) fn p_yes(response: &AggLlmResponse) -> Option<f64> {
*slot.get_or_insert(0.0) += probability.exp();
}
match (yes, no) {
- (Some(yes), Some(no)) if yes + no > 0.0 => Some(yes / (yes + no)),
+ (Some(yes), Some(no)) if yes + no >= MIN_VERDICT_MASS => Some(yes / (yes + no)),
_ => None,
}
}
@@ -79,6 +91,16 @@ mod tests {
assert_eq!(p_yes(&AggLlmResponse::default()), None);
}
+ #[test]
+ fn a_first_token_spent_elsewhere_is_no_verdict() {
+ let stray = scored(json!([
+ {"token": "Okay", "logprob": -0.05},
+ {"token": "yes", "logprob": 0.05_f64.ln()},
+ {"token": "no", "logprob": 0.001_f64.ln()}
+ ]));
+ assert_eq!(stray, None);
+ }
+
#[test]
fn unavailable_chat_logprobs_produce_no_score() {
let mut response = AggLlmResponse::default();
diff --git a/crates/libsy/src/algorithms/vgr/rungs.rs b/crates/libsy/src/algorithms/vgr/rungs.rs
index 917cce98e..ec677856f 100644
--- a/crates/libsy/src/algorithms/vgr/rungs.rs
+++ b/crates/libsy/src/algorithms/vgr/rungs.rs
@@ -121,7 +121,7 @@ pub(super) fn build_typing_request(task_text: &str, metadata: Option<Metadata>)
TYPING_MAX_OUTPUT_TOKENS,
metadata,
);
- request.llm_request.reasoning.effort = Some("none".to_string());
+ super::readout::disable_thinking(&mut request);
request
}
@@ -418,13 +418,31 @@ mod tests {
out\n[analysis cut off at its token budget]"
)
);
- assert_eq!(
- verdict.llm_request.reasoning.effort.as_deref(),
- Some("none")
- );
let typing = build_typing_request(&"x".repeat(5000), None);
assert_eq!(typing.llm_request.sampling.temperature, Some(0.0));
- assert_eq!(typing.llm_request.reasoning.effort.as_deref(), Some("none"));
+ for request in [&verdict, &typing] {
+ assert_eq!(
+ request.llm_request.reasoning.effort.as_deref(),
+ Some("none")
+ );
+ assert_eq!(
+ request
+ .llm_request
+ .extensions
+ .fields
+ .get("chat_template_kwargs"),
+ Some(&serde_json::json!({"enable_thinking": false}))
+ );
+ }
+ let deliberation = build_deliberation_request("record", None);
+ assert_eq!(deliberation.llm_request.reasoning.effort, None);
+ assert!(
+ !deliberation
+ .llm_request
+ .extensions
+ .fields
+ .contains_key("chat_template_kwargs")
+ );
assert_eq!(
typing.llm_request.messages[0]
.text_content("")
diff --git a/crates/libsy/src/algorithms/vgr/runtime.rs b/crates/libsy/src/algorithms/vgr/runtime.rs
index c5cdd6baf..26c2878fd 100644
--- a/crates/libsy/src/algorithms/vgr/runtime.rs
+++ b/crates/libsy/src/algorithms/vgr/runtime.rs
@@ -401,9 +401,10 @@ impl VgrClassifier {
self.ask(context, target, question, judged, max_output_tokens)
};
- // Coding here ran no tools, so its record holds no evidence: judge the code itself.
+ // Coding here ran no tools, and a typed answer cites none, so neither record holds
+ // evidence: judge the answer itself.
let asked = match branch {
- Branch::Coding => Question::Answer,
+ Branch::Coding | Branch::Answer => Question::Answer,
_ => Question::Evidence,
};
signals.readout = self.readout(context, judged, asked).await;
@@ -420,7 +421,10 @@ impl VgrClassifier {
if settled(&signals) {
return signals;
}
- if !(capable_confirms && decide::efficient_verified(branch, &signals)) {
+ // A deliberation never commits a typed answer, so the answer branch goes to the capable tier.
+ if branch != Branch::Answer
+ && !(capable_confirms && decide::efficient_verified(branch, &signals))
+ {
let metadata = || context.request.metadata.clone();
let call = rungs::build_deliberation_request(judged, metadata());
if let Some(Ok(deliberation)) = self.call(context, call, efficient).await {
diff --git a/crates/libsy/src/algorithms/vgr/runtime/tests.rs b/crates/libsy/src/algorithms/vgr/runtime/tests.rs
index 83b63654d..82aaa9225 100644
--- a/crates/libsy/src/algorithms/vgr/runtime/tests.rs
+++ b/crates/libsy/src/algorithms/vgr/runtime/tests.rs
@@ -217,6 +217,76 @@ async fn final_answers_commit_only_on_a_verified_clean_record() -> Result<()> {
Ok(())
}
+#[tokio::test]
+async fn a_typed_follow_up_answer_is_judged_on_the_answer_itself() -> Result<()> {
+ let route = route(|config| config.task_typing = true)?;
+ let conversation = session(vec![
+ Message::text(Role::User, "fix the build"),
+ call("c1"),
+ result("c1", "ok"),
+ Message::text(Role::Assistant, "The build passes."),
+ Message::text(Role::User, "What is 1+1?"),
+ ]);
+ // A confident readout serves on the efficient tier with no capable-tier call; a weak one asks the
+ // capable tier's answer verifier, with no efficient deliberation first.
+ for (readout, expected_calls) in [
+ (0.95, vec!["judge: typing", "judge: answer"]),
+ (
+ 0.5,
+ vec!["judge: typing", "judge: answer", "capable-judge: answer"],
+ ),
+ ] {
+ let calls = Arc::new(parking_lot::Mutex::new(Vec::new()));
+ let seen = Arc::clone(&calls);
+ let (selected, _) = test_drive_with_models(
+ Arc::clone(&route),
+ conversation.clone(),
+ category_models(Category::Any, &["efficient", "capable"]),
+ move |target: ModelId, request: Request| {
+ let seen = Arc::clone(&seen);
+ async move {
+ let system = request
+ .llm_request
+ .instructions
+ .first()
+ .and_then(|block| block.content.first())
+ .and_then(|block| match block {
+ ContentBlock::Text { text } => Some(text.clone()),
+ _ => None,
+ })
+ .unwrap_or_default();
+ let question = if system.starts_with("You are a strict answer verifier") {
+ "answer"
+ } else if request
+ .llm_request
+ .extensions
+ .fields
+ .contains_key("logprobs")
+ {
+ "evidence"
+ } else {
+ "typing"
+ };
+ if target.as_str().contains("judge") {
+ seen.lock().push(format!("{}: {question}", target.as_str()));
+ }
+ Ok(match target.as_str() {
+ "efficient" => reply("2"),
+ "judge" if question == "typing" => reply("Answer"),
+ "judge" => scored(readout),
+ "capable-judge" => reply("yes"),
+ _ => reply("capable answer"),
+ })
+ }
+ },
+ )
+ .await?;
+ assert_eq!(selected.as_str(), "efficient", "readout {readout}");
+ assert_eq!(*calls.lock(), expected_calls, "readout {readout}");
+ }
+ Ok(())
+}
+
#[tokio::test]
async fn an_unavailable_efficient_tier_escalates_and_other_failures_surface() -> Result<()> {
let route = route(|_| {})?;
@@ -398,7 +468,7 @@ async fn decisions_attach_the_reason_and_exclude_efficient_capable_fallbacks() -
assert_eq!(evidence["source"], "vgr");
assert_eq!(evidence["reason_code"], reason);
assert_eq!(evidence["verdict"], served);
- assert_eq!(evidence["policy_version"], "1.0");
+ assert_eq!(evidence["policy_version"], "1.1");
if reason == "decided" {
assert_eq!(evidence["branch"], "DefaultVerified");
}
diff --git a/crates/libsy/src/algorithms/vgr/tests.rs b/crates/libsy/src/algorithms/vgr/tests.rs
index f2fe35956..b70260bcb 100644
--- a/crates/libsy/src/algorithms/vgr/tests.rs
+++ b/crates/libsy/src/algorithms/vgr/tests.rs
@@ -107,6 +107,22 @@ fn router_typing_selects_the_verification_regime() {
);
assert_eq!(branch(&session, "done", Some(TaskType::Chat)), Branch::Chat);
assert_eq!(branch(&single, "```\nx\n```", None), Branch::Coding);
+
+ // A typed answer keeps its branch after earlier turns, unless this turn used tools.
+ let mut follow_up = session.llm_request.messages.clone();
+ follow_up.extend([
+ Message::text(Role::Assistant, "The build passes."),
+ Message::text(Role::User, "What is 1+1?"),
+ ]);
+ let follow_up = request(follow_up);
+ assert_eq!(
+ branch(&follow_up, "2", Some(TaskType::Answer)),
+ Branch::Answer
+ );
+ assert_eq!(
+ branch(&session, "done", Some(TaskType::Answer)),
+ Branch::Chat
+ );
}
#[test]
@@ -406,4 +422,18 @@ mod decide {
assert_eq!(decide(&coding, &readout(0.6), None), Route::Efficient);
assert_eq!(decide(&coding, &readout(0.59), None), Route::Capable);
}
+
+ #[test]
+ fn answers_commit_on_a_confident_readout_or_a_capable_tier_verdict() {
+ let answer = Capabilities {
+ branch: Branch::Answer,
+ transcript: Some("view".into()),
+ tools: None,
+ };
+ assert_eq!(decide(&answer, &readout(0.9), None), Route::Efficient);
+ let mut signals = readout(0.89);
+ assert_eq!(decide(&answer, &signals, None), Route::Capable);
+ signals.answer_verifier = Some(Tri::Yes);
+ assert_eq!(decide(&answer, &signals, None), Route::Efficient);
+ }
}
diff --git a/crates/libsy/src/algorithms/vgr/text.rs b/crates/libsy/src/algorithms/vgr/text.rs
index 1638fff0e..887093da7 100644
--- a/crates/libsy/src/algorithms/vgr/text.rs
+++ b/crates/libsy/src/algorithms/vgr/text.rs
@@ -115,10 +115,6 @@ pub(super) fn newest_prompt_is_blank(request: &Request) -> bool {
})
}
-pub(super) fn has_assistant_turn(turns: &[Turn]) -> bool {
- turns.iter().any(|turn| turn.role == Role::Assistant)
-}
-
/// Whether the conversation contains an executed or proposed tool step.
pub(super) fn has_tool_trajectory(request: &Request) -> bool {
request.llm_request.messages.iter().any(is_tool_step)
diff --git a/crates/switchyard-translation/src/codecs/openai_chat/buffered.rs b/crates/switchyard-translation/src/codecs/openai_chat/buffered.rs
index 0dc064162..970fbce44 100644
--- a/crates/switchyard-translation/src/codecs/openai_chat/buffered.rs
+++ b/crates/switchyard-translation/src/codecs/openai_chat/buffered.rs
@@ -1031,6 +1031,8 @@ fn copy_openai_chat_request_extensions(
extensions: &Map<String, Value>,
) {
for field in [
+ // Template switches for llama.cpp, vLLM and SGLang, such as `enable_thinking`.
+ "chat_template_kwargs",
"metadata",
"parallel_tool_calls",
"prompt_cache_key",
diff --git a/crates/switchyard-translation/src/codecs/responses/buffered.rs b/crates/switchyard-translation/src/codecs/responses/buffered.rs
index 294e59740..b5ce2a8b1 100644
--- a/crates/switchyard-translation/src/codecs/responses/buffered.rs
+++ b/crates/switchyard-translation/src/codecs/responses/buffered.rs
@@ -222,9 +222,7 @@ impl FormatCodec for OpenAiResponsesCodec {
);
if let Some(additional) = crate::codex_custom_tools::additional_tools(&request.extensions) {
// A Responses-lite request carried its tools inside `input`; give them back the same
- // way, verbatim, and leave top-level `tools` absent as the client did. A request that
- // reduced to a single user text encodes `input` as a plain string, which has no
- // room for an item, so it is widened to the equivalent one-message array first.
+ // way, verbatim, and leave top-level `tools` absent as the client did.
let item = json!({
"type": "additional_tools",
"role": "developer",
@@ -232,10 +230,6 @@ impl FormatCodec for OpenAiResponsesCodec {
});
match body.get_mut("input") {
Some(Value::Array(input)) => input.insert(0, item),
- Some(input @ Value::String(_)) => {
- let text = input.take();
- *input = Value::Array(vec![item, json!({"role": "user", "content": text})]);
- }
_ => body
.insert("input".to_string(), Value::Array(vec![item]))
.map_or((), |_| ()),
@@ -1278,14 +1272,8 @@ fn encode_responses_input(
// custom outputs. A tool result is typed by the call it answers, and Responses history
// always lists the call before its output, so recording ids as calls are encoded is enough
// to type the outputs that follow.
- if messages.len() == 1
- && matches!(messages[0].role, Role::User)
- && messages[0].content.len() == 1
- && matches!(messages[0].content[0], ContentBlock::Text { .. })
- && let ContentBlock::Text { text } = &messages[0].content[0]
- {
- return Ok(Value::String(text.clone()));
- }
+ //
+ // A lone user text is still sent as an item list: the Codex backend rejects a string `input`.
let mut encoded = Vec::new();
// Some upstream translators (Kimi K3) resolve a tool output by an explicit
// `name`, so pair every output with the name of the call it answers.
diff --git a/crates/switchyard-translation/tests/request_translation.rs b/crates/switchyard-translation/tests/request_translation.rs
index 18e1ce4f3..b2c23a97b 100644
--- a/crates/switchyard-translation/tests/request_translation.rs
+++ b/crates/switchyard-translation/tests/request_translation.rs
@@ -38,6 +38,24 @@ fn normalized_chat_requests_forward_logprobs_with_top_logprobs() -> TestResult {
Ok(())
}
+#[test]
+fn normalized_chat_requests_forward_chat_template_kwargs() -> TestResult {
+ let engine = TranslationEngine::default();
+ let mut request = switchyard_translation::LlmRequest::default();
+ request.extensions.fields.insert(
+ "chat_template_kwargs".into(),
+ json!({"enable_thinking": false}),
+ );
+ let body = engine
+ .encode_request(WireFormat::OpenAiChat, &request, &normalized_policy())?
+ .body;
+ assert_eq!(
+ body["chat_template_kwargs"],
+ json!({"enable_thinking": false})
+ );
+ Ok(())
+}
+
#[test]
fn openai_chat_rejects_mcp_history_before_text_only_returns() {
use switchyard_translation::{LlmRequest, McpContent, Message, Role};
@@ -2296,6 +2314,35 @@ fn chat_compatible_extensions_survive_to_responses() -> TestResult {
Ok(())
}
+// Verifies a lone user text reaches Responses as an item list, which the Codex backend requires.
+#[test]
+fn chat_single_user_text_encodes_responses_input_as_a_list() -> TestResult {
+ let engine = TranslationEngine::default();
+ let body = json!({
+ "model": "gpt-4",
+ "messages": [
+ {"role": "system", "content": "Be terse."},
+ {"role": "user", "content": "What is 17*23?"}
+ ]
+ });
+
+ let output = engine
+ .translate_request(
+ WireFormat::OpenAiChat,
+ WireFormat::OpenAiResponses,
+ &body,
+ &TranslationPolicy::default(),
+ )?
+ .body;
+
+ assert_eq!(output["instructions"], "Be terse.");
+ assert_eq!(
+ output["input"],
+ json!([{"type": "message", "role": "user", "content": "What is 17*23?"}])
+ );
+ Ok(())
+}
+
#[test]
fn responses_chat_compatible_extensions_survive_to_openai_chat() -> TestResult {
let engine = TranslationEngine::default();
diff --git a/docs/routing_algorithms/vgr_routing.md b/docs/routing_algorithms/vgr_routing.md
index 2dc9f792c..213328095 100644
--- a/docs/routing_algorithms/vgr_routing.md
+++ b/docs/routing_algorithms/vgr_routing.md
@@ -48,6 +48,13 @@ for an active llama.cpp configuration and commented Ollama/vLLM alternatives.
For Ollama and vLLM, set `context_window` to match the running service; automatic
discovery for those servers is not implemented.
+The judge reads its verdicts from the first token's logprobs, so its scored calls
+turn thinking off: they send `reasoning_effort = "none"` and
+`chat_template_kwargs = {"enable_thinking": false}` on `openai_chat` targets.
+Candidates and the judge's deliberation keep the target's own setting. A backend
+that rejects `chat_template_kwargs` can drop it with the target's
+`omit_body_fields`.
+
The server caches the model catalog, including unavailable discovery results,
for one minute. Discovery probes run concurrently with a two-second total budget
per refresh. Completed discoveries are retained when that budget expires.
@@ -125,6 +132,11 @@ recognized tag, Switchyard uses the message shape to identify a user turn.
## Verification and capable-tier outages
+Policy version `1.1` verifies typed factual answers on the answer itself, including
+later user turns without tool work. A readout of at least 0.9 can serve the
+efficient answer; lower scores require capable-tier verification. Scored verdicts
+require yes and no together to hold at least half the first token's probability.
+
Coding without tool history can serve the efficient attempt with a clean readout of at least
0.6, or a readout of at least 0.2 plus capable-judge affirmation. Other final-answer
branches can use a separate scored verdict after efficient deliberation.