Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.
Rust
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0
//! The cheap readout: the probability a verifier put on "yes" at its first token.
//!
//! Normalized responses carry no logprobs, so this reads the preserved Chat
//! body. Anything that cannot be scored is `None`, which never commits.
//! The readout supports OpenAI Chat-compatible verifiers only.
use serde_json::{Value, json};
use switchyard_protocol::{AggLlmResponse, FormatId, Request, WireFormat};
pub(super) const MAX_OUTPUT_TOKENS: u64 = 4;
/// Minimum combined yes/no probability for the first token to count as a verdict.
const MIN_VERDICT_MASS: f64 = 0.5;
/// Disables reasoning for one verifier call, including llama.cpp thinking templates.
pub(super) fn disable_thinking(request: &mut Request) {
request.llm_request.reasoning.effort = Some("none".to_string());
request.llm_request.extensions.fields.insert(
"chat_template_kwargs".to_string(),
json!({"enable_thinking": false}),
);
}
/// Asks for a direct verdict with its alternatives; reasoning stays request-local.
pub(super) fn request_logprobs(request: &mut Request) {
disable_thinking(request);
let extensions = &mut request.llm_request.extensions.fields;
extensions.insert("logprobs".to_string(), json!(true));
extensions.insert("top_logprobs".to_string(), json!(8));
}
pub(super) fn p_yes(response: &AggLlmResponse) -> Option<f64> {
let alternatives = response
.preservation
.responses
.get(&FormatId::from(WireFormat::OpenAiChat))?
.pointer("/choices/0/logprobs/content/0/top_logprobs")?
.as_array()?;
let (mut yes, mut no) = (None, None);
for entry in alternatives {
let Some(probability) = entry.get("logprob").and_then(Value::as_f64) else {
continue;
};
let token = entry.get("token")?.as_str()?;
let slot = match token
.trim()
.trim_matches(['"', '\'', '.', ','])
.to_lowercase()
.as_str()
{
"yes" | "y" | "true" => &mut yes,
"no" | "n" | "false" => &mut no,
_ => continue,
};
*slot.get_or_insert(0.0) += probability.exp();
}
match (yes, no) {
(Some(yes), Some(no)) if yes + no >= MIN_VERDICT_MASS => Some(yes / (yes + no)),
_ => None,
}
}
#[cfg(test)]
mod tests {
use super::*;
fn scored(alternatives: Value) -> Option<f64> {
let mut response = AggLlmResponse::default();
response.preservation.responses.insert(
FormatId::from(WireFormat::OpenAiChat),
json!({"choices": [{"logprobs": {"content": [{"top_logprobs": alternatives}]}}]}),
);
p_yes(&response)
}
#[test]
fn verdict_mass_is_scored_against_the_pair() {
let half = 0.5_f64.ln();
let score = scored(json!([
{"token": " Yes", "logprob": half},
{"token": "no", "logprob": half},
{"token": ".", "logprob": -9.0}
]));
assert!(score.is_some_and(|score| (score - 0.5).abs() < 1e-9));
assert_eq!(scored(json!([{"token": "yes", "logprob": -3.0}])), None);
assert_eq!(scored(json!([{"token": "no", "logprob": -3.0}])), None);
assert_eq!(scored(json!([{"token": "maybe", "logprob": 0.0}])), None);
assert_eq!(p_yes(&AggLlmResponse::default()), None);
}
#[test]
fn a_first_token_spent_elsewhere_is_no_verdict() {
let stray = scored(json!([
{"token": "Okay", "logprob": -0.05},
{"token": "yes", "logprob": 0.05_f64.ln()},
{"token": "no", "logprob": 0.001_f64.ln()}
]));
assert_eq!(stray, None);
}
#[test]
fn unavailable_chat_logprobs_produce_no_score() {
let mut response = AggLlmResponse::default();
for format in [WireFormat::OpenAiResponses, WireFormat::AnthropicMessages] {
response.preservation.responses.insert(
FormatId::from(format),
json!({"choices": [{"logprobs": {"content": [{"top_logprobs": [
{"token": "yes", "logprob": 0.0}
]}]}}]}),
);
assert_eq!(p_yes(&response), None);
}
for body in [
json!({"choices": []}),
json!({"choices": [{"logprobs": null}]}),
json!({"choices": [{"logprobs": {"content": []}}]}),
] {
response
.preservation
.responses
.insert(FormatId::from(WireFormat::OpenAiChat), body);
assert_eq!(p_yes(&response), None);
}
}
}