Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.
Rust
fix(vgr): restore QA verifier and Responses compatibility (#1163)
GDRS9ed588b4d0a9
diff --git a/crates/libsy/src/algorithms/vgr.rs b/crates/libsy/src/algorithms/vgr.rsindex 199546100..4a7d64664 100644--- a/crates/libsy/src/algorithms/vgr.rs+++ b/crates/libsy/src/algorithms/vgr.rs@@ -154,7 +154,9 @@ fn derive_capabilities(tools: Some(ToolRecord::from_request(request)),};- if task_type == Some(TaskType::Answer) && !text::has_assistant_turn(&turns) {+ // Typing abstains on a request that leans on earlier context, so a typed answer stands+ // on its own; only tool use in the current user turn takes it off this branch.+ if task_type == Some(TaskType::Answer) && !text::turn_has_tool_trajectory(request) {return Capabilities {tools: None,..caps(diff --git a/crates/libsy/src/algorithms/vgr/decide.rs b/crates/libsy/src/algorithms/vgr/decide.rsindex 9f36a1012..57f636f39 100644--- a/crates/libsy/src/algorithms/vgr/decide.rs+++ b/crates/libsy/src/algorithms/vgr/decide.rs@@ -9,12 +9,14 @@use super::{Branch, Capabilities};/// Version of these rules and dials, carried on every decision record.-pub(super) const POLICY_VERSION: &str = "1.0";+pub(super) const POLICY_VERSION: &str = "1.1";const READOUT: f64 = 0.9;const DELIBERATION: f64 = 0.5;/// Readout at which clean coding commits without the capable tier's affirmation.const CODING_CONFIDENT: f64 = 0.6;+/// Readout required to serve a typed answer without capable-tier verification.+const ANSWER_CONFIDENT: f64 = READOUT;/// Most tool results a run may contain and still recover without the capable tier.const SHORT_RUN_MAX_TOOL_RESULTS: i32 = 100;@@ -80,7 +82,8 @@ pub(super) fn agentic_run(/// The readout bar that licenses a commit on each branch.fn dial(branch: Branch) -> Option<f64> {match branch {- Branch::Answer | Branch::DefaultVerified => Some(0.7),+ Branch::Answer => Some(ANSWER_CONFIDENT),+ Branch::DefaultVerified => Some(0.7),Branch::Chat => Some(0.3),Branch::Agentic | Branch::Coding => Some(0.2),Branch::Unknown => None,@@ -121,7 +124,7 @@ pub(super) fn decide(Branch::Answer => {affirms(signals.answer_verifier)|| affirms(signals.evidence_verifier)- || clears(signals.readout, 0.7)+ || clears(signals.readout, ANSWER_CONFIDENT)}Branch::Chat => {cleandiff --git a/crates/libsy/src/algorithms/vgr/readout.rs b/crates/libsy/src/algorithms/vgr/readout.rsindex 62533af93..c9d1da1e6 100644--- a/crates/libsy/src/algorithms/vgr/readout.rs+++ b/crates/libsy/src/algorithms/vgr/readout.rs@@ -12,9 +12,21 @@ use switchyard_protocol::{AggLlmResponse, FormatId, Request, WireFormat};pub(super) const MAX_OUTPUT_TOKENS: u64 = 4;+/// Minimum combined yes/no probability for the first token to count as a verdict.+const MIN_VERDICT_MASS: f64 = 0.5;++/// Disables reasoning for one verifier call, including llama.cpp thinking templates.+pub(super) fn disable_thinking(request: &mut Request) {+ request.llm_request.reasoning.effort = Some("none".to_string());+ request.llm_request.extensions.fields.insert(+ "chat_template_kwargs".to_string(),+ json!({"enable_thinking": false}),+ );+}+/// Asks for a direct verdict with its alternatives; reasoning stays request-local.pub(super) fn request_logprobs(request: &mut Request) {- request.llm_request.reasoning.effort = Some("none".to_string());+ disable_thinking(request);let extensions = &mut request.llm_request.extensions.fields;extensions.insert("logprobs".to_string(), json!(true));extensions.insert("top_logprobs".to_string(), json!(8));@@ -46,7 +58,7 @@ pub(super) fn p_yes(response: &AggLlmResponse) -> Option<f64> {*slot.get_or_insert(0.0) += probability.exp();}match (yes, no) {- (Some(yes), Some(no)) if yes + no > 0.0 => Some(yes / (yes + no)),+ (Some(yes), Some(no)) if yes + no >= MIN_VERDICT_MASS => Some(yes / (yes + no)),_ => None,}}@@ -79,6 +91,16 @@ mod tests {assert_eq!(p_yes(&AggLlmResponse::default()), None);}+ #[test]+ fn a_first_token_spent_elsewhere_is_no_verdict() {+ let stray = scored(json!([+ {"token": "Okay", "logprob": -0.05},+ {"token": "yes", "logprob": 0.05_f64.ln()},+ {"token": "no", "logprob": 0.001_f64.ln()}+ ]));+ assert_eq!(stray, None);+ }+#[test]fn unavailable_chat_logprobs_produce_no_score() {let mut response = AggLlmResponse::default();diff --git a/crates/libsy/src/algorithms/vgr/rungs.rs b/crates/libsy/src/algorithms/vgr/rungs.rsindex 917cce98e..ec677856f 100644--- a/crates/libsy/src/algorithms/vgr/rungs.rs+++ b/crates/libsy/src/algorithms/vgr/rungs.rs@@ -121,7 +121,7 @@ pub(super) fn build_typing_request(task_text: &str, metadata: Option<Metadata>)TYPING_MAX_OUTPUT_TOKENS,metadata,);- request.llm_request.reasoning.effort = Some("none".to_string());+ super::readout::disable_thinking(&mut request);request}@@ -418,13 +418,31 @@ mod tests {out\n[analysis cut off at its token budget]"));- assert_eq!(- verdict.llm_request.reasoning.effort.as_deref(),- Some("none")- );let typing = build_typing_request(&"x".repeat(5000), None);assert_eq!(typing.llm_request.sampling.temperature, Some(0.0));- assert_eq!(typing.llm_request.reasoning.effort.as_deref(), Some("none"));+ for request in [&verdict, &typing] {+ assert_eq!(+ request.llm_request.reasoning.effort.as_deref(),+ Some("none")+ );+ assert_eq!(+ request+ .llm_request+ .extensions+ .fields+ .get("chat_template_kwargs"),+ Some(&serde_json::json!({"enable_thinking": false}))+ );+ }+ let deliberation = build_deliberation_request("record", None);+ assert_eq!(deliberation.llm_request.reasoning.effort, None);+ assert!(+ !deliberation+ .llm_request+ .extensions+ .fields+ .contains_key("chat_template_kwargs")+ );assert_eq!(typing.llm_request.messages[0].text_content("")diff --git a/crates/libsy/src/algorithms/vgr/runtime.rs b/crates/libsy/src/algorithms/vgr/runtime.rsindex c5cdd6baf..26c2878fd 100644--- a/crates/libsy/src/algorithms/vgr/runtime.rs+++ b/crates/libsy/src/algorithms/vgr/runtime.rs@@ -401,9 +401,10 @@ impl VgrClassifier {self.ask(context, target, question, judged, max_output_tokens)};- // Coding here ran no tools, so its record holds no evidence: judge the code itself.+ // Coding here ran no tools, and a typed answer cites none, so neither record holds+ // evidence: judge the answer itself.let asked = match branch {- Branch::Coding => Question::Answer,+ Branch::Coding | Branch::Answer => Question::Answer,_ => Question::Evidence,};signals.readout = self.readout(context, judged, asked).await;@@ -420,7 +421,10 @@ impl VgrClassifier {if settled(&signals) {return signals;}- if !(capable_confirms && decide::efficient_verified(branch, &signals)) {+ // A deliberation never commits a typed answer, so the answer branch goes to the capable tier.+ if branch != Branch::Answer+ && !(capable_confirms && decide::efficient_verified(branch, &signals))+ {let metadata = || context.request.metadata.clone();let call = rungs::build_deliberation_request(judged, metadata());if let Some(Ok(deliberation)) = self.call(context, call, efficient).await {diff --git a/crates/libsy/src/algorithms/vgr/runtime/tests.rs b/crates/libsy/src/algorithms/vgr/runtime/tests.rsindex 83b63654d..82aaa9225 100644--- a/crates/libsy/src/algorithms/vgr/runtime/tests.rs+++ b/crates/libsy/src/algorithms/vgr/runtime/tests.rs@@ -217,6 +217,76 @@ async fn final_answers_commit_only_on_a_verified_clean_record() -> Result<()> {Ok(())}+#[tokio::test]+async fn a_typed_follow_up_answer_is_judged_on_the_answer_itself() -> Result<()> {+ let route = route(|config| config.task_typing = true)?;+ let conversation = session(vec![+ Message::text(Role::User, "fix the build"),+ call("c1"),+ result("c1", "ok"),+ Message::text(Role::Assistant, "The build passes."),+ Message::text(Role::User, "What is 1+1?"),+ ]);+ // A confident readout serves on the efficient tier with no capable-tier call; a weak one asks the+ // capable tier's answer verifier, with no efficient deliberation first.+ for (readout, expected_calls) in [+ (0.95, vec!["judge: typing", "judge: answer"]),+ (+ 0.5,+ vec!["judge: typing", "judge: answer", "capable-judge: answer"],+ ),+ ] {+ let calls = Arc::new(parking_lot::Mutex::new(Vec::new()));+ let seen = Arc::clone(&calls);+ let (selected, _) = test_drive_with_models(+ Arc::clone(&route),+ conversation.clone(),+ category_models(Category::Any, &["efficient", "capable"]),+ move |target: ModelId, request: Request| {+ let seen = Arc::clone(&seen);+ async move {+ let system = request+ .llm_request+ .instructions+ .first()+ .and_then(|block| block.content.first())+ .and_then(|block| match block {+ ContentBlock::Text { text } => Some(text.clone()),+ _ => None,+ })+ .unwrap_or_default();+ let question = if system.starts_with("You are a strict answer verifier") {+ "answer"+ } else if request+ .llm_request+ .extensions+ .fields+ .contains_key("logprobs")+ {+ "evidence"+ } else {+ "typing"+ };+ if target.as_str().contains("judge") {+ seen.lock().push(format!("{}: {question}", target.as_str()));+ }+ Ok(match target.as_str() {+ "efficient" => reply("2"),+ "judge" if question == "typing" => reply("Answer"),+ "judge" => scored(readout),+ "capable-judge" => reply("yes"),+ _ => reply("capable answer"),+ })+ }+ },+ )+ .await?;+ assert_eq!(selected.as_str(), "efficient", "readout {readout}");+ assert_eq!(*calls.lock(), expected_calls, "readout {readout}");+ }+ Ok(())+}+#[tokio::test]async fn an_unavailable_efficient_tier_escalates_and_other_failures_surface() -> Result<()> {let route = route(|_| {})?;@@ -398,7 +468,7 @@ async fn decisions_attach_the_reason_and_exclude_efficient_capable_fallbacks() -assert_eq!(evidence["source"], "vgr");assert_eq!(evidence["reason_code"], reason);assert_eq!(evidence["verdict"], served);- assert_eq!(evidence["policy_version"], "1.0");+ assert_eq!(evidence["policy_version"], "1.1");if reason == "decided" {assert_eq!(evidence["branch"], "DefaultVerified");}diff --git a/crates/libsy/src/algorithms/vgr/tests.rs b/crates/libsy/src/algorithms/vgr/tests.rsindex f2fe35956..b70260bcb 100644--- a/crates/libsy/src/algorithms/vgr/tests.rs+++ b/crates/libsy/src/algorithms/vgr/tests.rs@@ -107,6 +107,22 @@ fn router_typing_selects_the_verification_regime() {);assert_eq!(branch(&session, "done", Some(TaskType::Chat)), Branch::Chat);assert_eq!(branch(&single, "```\nx\n```", None), Branch::Coding);++ // A typed answer keeps its branch after earlier turns, unless this turn used tools.+ let mut follow_up = session.llm_request.messages.clone();+ follow_up.extend([+ Message::text(Role::Assistant, "The build passes."),+ Message::text(Role::User, "What is 1+1?"),+ ]);+ let follow_up = request(follow_up);+ assert_eq!(+ branch(&follow_up, "2", Some(TaskType::Answer)),+ Branch::Answer+ );+ assert_eq!(+ branch(&session, "done", Some(TaskType::Answer)),+ Branch::Chat+ );}#[test]@@ -406,4 +422,18 @@ mod decide {assert_eq!(decide(&coding, &readout(0.6), None), Route::Efficient);assert_eq!(decide(&coding, &readout(0.59), None), Route::Capable);}++ #[test]+ fn answers_commit_on_a_confident_readout_or_a_capable_tier_verdict() {+ let answer = Capabilities {+ branch: Branch::Answer,+ transcript: Some("view".into()),+ tools: None,+ };+ assert_eq!(decide(&answer, &readout(0.9), None), Route::Efficient);+ let mut signals = readout(0.89);+ assert_eq!(decide(&answer, &signals, None), Route::Capable);+ signals.answer_verifier = Some(Tri::Yes);+ assert_eq!(decide(&answer, &signals, None), Route::Efficient);+ }}diff --git a/crates/libsy/src/algorithms/vgr/text.rs b/crates/libsy/src/algorithms/vgr/text.rsindex 1638fff0e..887093da7 100644--- a/crates/libsy/src/algorithms/vgr/text.rs+++ b/crates/libsy/src/algorithms/vgr/text.rs@@ -115,10 +115,6 @@ pub(super) fn newest_prompt_is_blank(request: &Request) -> bool {})}-pub(super) fn has_assistant_turn(turns: &[Turn]) -> bool {- turns.iter().any(|turn| turn.role == Role::Assistant)-}-/// Whether the conversation contains an executed or proposed tool step.pub(super) fn has_tool_trajectory(request: &Request) -> bool {request.llm_request.messages.iter().any(is_tool_step)diff --git a/crates/switchyard-translation/src/codecs/openai_chat/buffered.rs b/crates/switchyard-translation/src/codecs/openai_chat/buffered.rsindex 0dc064162..970fbce44 100644--- a/crates/switchyard-translation/src/codecs/openai_chat/buffered.rs+++ b/crates/switchyard-translation/src/codecs/openai_chat/buffered.rs@@ -1031,6 +1031,8 @@ fn copy_openai_chat_request_extensions(extensions: &Map<String, Value>,) {for field in [+ // Template switches for llama.cpp, vLLM and SGLang, such as `enable_thinking`.+ "chat_template_kwargs","metadata","parallel_tool_calls","prompt_cache_key",diff --git a/crates/switchyard-translation/src/codecs/responses/buffered.rs b/crates/switchyard-translation/src/codecs/responses/buffered.rsindex 294e59740..b5ce2a8b1 100644--- a/crates/switchyard-translation/src/codecs/responses/buffered.rs+++ b/crates/switchyard-translation/src/codecs/responses/buffered.rs@@ -222,9 +222,7 @@ impl FormatCodec for OpenAiResponsesCodec {);if let Some(additional) = crate::codex_custom_tools::additional_tools(&request.extensions) {// A Responses-lite request carried its tools inside `input`; give them back the same- // way, verbatim, and leave top-level `tools` absent as the client did. A request that- // reduced to a single user text encodes `input` as a plain string, which has no- // room for an item, so it is widened to the equivalent one-message array first.+ // way, verbatim, and leave top-level `tools` absent as the client did.let item = json!({"type": "additional_tools","role": "developer",@@ -232,10 +230,6 @@ impl FormatCodec for OpenAiResponsesCodec {});match body.get_mut("input") {Some(Value::Array(input)) => input.insert(0, item),- Some(input @ Value::String(_)) => {- let text = input.take();- *input = Value::Array(vec![item, json!({"role": "user", "content": text})]);- }_ => body.insert("input".to_string(), Value::Array(vec![item])).map_or((), |_| ()),@@ -1278,14 +1272,8 @@ fn encode_responses_input(// custom outputs. A tool result is typed by the call it answers, and Responses history// always lists the call before its output, so recording ids as calls are encoded is enough// to type the outputs that follow.- if messages.len() == 1- && matches!(messages[0].role, Role::User)- && messages[0].content.len() == 1- && matches!(messages[0].content[0], ContentBlock::Text { .. })- && let ContentBlock::Text { text } = &messages[0].content[0]- {- return Ok(Value::String(text.clone()));- }+ //+ // A lone user text is still sent as an item list: the Codex backend rejects a string `input`.let mut encoded = Vec::new();// Some upstream translators (Kimi K3) resolve a tool output by an explicit// `name`, so pair every output with the name of the call it answers.diff --git a/crates/switchyard-translation/tests/request_translation.rs b/crates/switchyard-translation/tests/request_translation.rsindex 18e1ce4f3..b2c23a97b 100644--- a/crates/switchyard-translation/tests/request_translation.rs+++ b/crates/switchyard-translation/tests/request_translation.rs@@ -38,6 +38,24 @@ fn normalized_chat_requests_forward_logprobs_with_top_logprobs() -> TestResult {Ok(())}+#[test]+fn normalized_chat_requests_forward_chat_template_kwargs() -> TestResult {+ let engine = TranslationEngine::default();+ let mut request = switchyard_translation::LlmRequest::default();+ request.extensions.fields.insert(+ "chat_template_kwargs".into(),+ json!({"enable_thinking": false}),+ );+ let body = engine+ .encode_request(WireFormat::OpenAiChat, &request, &normalized_policy())?+ .body;+ assert_eq!(+ body["chat_template_kwargs"],+ json!({"enable_thinking": false})+ );+ Ok(())+}+#[test]fn openai_chat_rejects_mcp_history_before_text_only_returns() {use switchyard_translation::{LlmRequest, McpContent, Message, Role};@@ -2296,6 +2314,35 @@ fn chat_compatible_extensions_survive_to_responses() -> TestResult {Ok(())}+// Verifies a lone user text reaches Responses as an item list, which the Codex backend requires.+#[test]+fn chat_single_user_text_encodes_responses_input_as_a_list() -> TestResult {+ let engine = TranslationEngine::default();+ let body = json!({+ "model": "gpt-4",+ "messages": [+ {"role": "system", "content": "Be terse."},+ {"role": "user", "content": "What is 17*23?"}+ ]+ });++ let output = engine+ .translate_request(+ WireFormat::OpenAiChat,+ WireFormat::OpenAiResponses,+ &body,+ &TranslationPolicy::default(),+ )?+ .body;++ assert_eq!(output["instructions"], "Be terse.");+ assert_eq!(+ output["input"],+ json!([{"type": "message", "role": "user", "content": "What is 17*23?"}])+ );+ Ok(())+}+#[test]fn responses_chat_compatible_extensions_survive_to_openai_chat() -> TestResult {let engine = TranslationEngine::default();diff --git a/docs/routing_algorithms/vgr_routing.md b/docs/routing_algorithms/vgr_routing.mdindex 2dc9f792c..213328095 100644--- a/docs/routing_algorithms/vgr_routing.md+++ b/docs/routing_algorithms/vgr_routing.md@@ -48,6 +48,13 @@ for an active llama.cpp configuration and commented Ollama/vLLM alternatives.For Ollama and vLLM, set `context_window` to match the running service; automaticdiscovery for those servers is not implemented.+The judge reads its verdicts from the first token's logprobs, so its scored calls+turn thinking off: they send `reasoning_effort = "none"` and+`chat_template_kwargs = {"enable_thinking": false}` on `openai_chat` targets.+Candidates and the judge's deliberation keep the target's own setting. A backend+that rejects `chat_template_kwargs` can drop it with the target's+`omit_body_fields`.+The server caches the model catalog, including unavailable discovery results,for one minute. Discovery probes run concurrently with a two-second total budgetper refresh. Completed discoveries are retained when that budget expires.@@ -125,6 +132,11 @@ recognized tag, Switchyard uses the message shape to identify a user turn.## Verification and capable-tier outages+Policy version `1.1` verifies typed factual answers on the answer itself, including+later user turns without tool work. A readout of at least 0.9 can serve the+efficient answer; lower scores require capable-tier verification. Scored verdicts+require yes and no together to hold at least half the first token's probability.+Coding without tool history can serve the efficient attempt with a clean readout of at least0.6, or a readout of at least 0.2 plus capable-judge affirmation. Other final-answerbranches can use a separate scored verdict after efficient deliberation.