Skip to content

Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.

Rust
4.6 KB
Raw Blame
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0

//! The cheap readout: the probability a verifier put on "yes" at its first token.
//!
//! Normalized responses carry no logprobs, so this reads the preserved Chat
//! body. Anything that cannot be scored is `None`, which never commits.
//! The readout supports OpenAI Chat-compatible verifiers only.

use serde_json::{Value, json};
use switchyard_protocol::{AggLlmResponse, FormatId, Request, WireFormat};

pub(super) const MAX_OUTPUT_TOKENS: u64 = 4;

/// Minimum combined yes/no probability for the first token to count as a verdict.
const MIN_VERDICT_MASS: f64 = 0.5;

/// Disables reasoning for one verifier call, including llama.cpp thinking templates.
pub(super) fn disable_thinking(request: &mut Request) {
    request.llm_request.reasoning.effort = Some("none".to_string());
    request.llm_request.extensions.fields.insert(
        "chat_template_kwargs".to_string(),
        json!({"enable_thinking": false}),
    );
}

/// Asks for a direct verdict with its alternatives; reasoning stays request-local.
pub(super) fn request_logprobs(request: &mut Request) {
    disable_thinking(request);
    let extensions = &mut request.llm_request.extensions.fields;
    extensions.insert("logprobs".to_string(), json!(true));
    extensions.insert("top_logprobs".to_string(), json!(8));
}

pub(super) fn p_yes(response: &AggLlmResponse) -> Option<f64> {
    let alternatives = response
        .preservation
        .responses
        .get(&FormatId::from(WireFormat::OpenAiChat))?
        .pointer("/choices/0/logprobs/content/0/top_logprobs")?
        .as_array()?;
    let (mut yes, mut no) = (None, None);
    for entry in alternatives {
        let Some(probability) = entry.get("logprob").and_then(Value::as_f64) else {
            continue;
        };
        let token = entry.get("token")?.as_str()?;
        let slot = match token
            .trim()
            .trim_matches(['"', '\'', '.', ','])
            .to_lowercase()
            .as_str()
        {
            "yes" | "y" | "true" => &mut yes,
            "no" | "n" | "false" => &mut no,
            _ => continue,
        };
        *slot.get_or_insert(0.0) += probability.exp();
    }
    match (yes, no) {
        (Some(yes), Some(no)) if yes + no >= MIN_VERDICT_MASS => Some(yes / (yes + no)),
        _ => None,
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    fn scored(alternatives: Value) -> Option<f64> {
        let mut response = AggLlmResponse::default();
        response.preservation.responses.insert(
            FormatId::from(WireFormat::OpenAiChat),
            json!({"choices": [{"logprobs": {"content": [{"top_logprobs": alternatives}]}}]}),
        );
        p_yes(&response)
    }

    #[test]
    fn verdict_mass_is_scored_against_the_pair() {
        let half = 0.5_f64.ln();
        let score = scored(json!([
            {"token": " Yes", "logprob": half},
            {"token": "no", "logprob": half},
            {"token": ".", "logprob": -9.0}
        ]));
        assert!(score.is_some_and(|score| (score - 0.5).abs() < 1e-9));
        assert_eq!(scored(json!([{"token": "yes", "logprob": -3.0}])), None);
        assert_eq!(scored(json!([{"token": "no", "logprob": -3.0}])), None);
        assert_eq!(scored(json!([{"token": "maybe", "logprob": 0.0}])), None);
        assert_eq!(p_yes(&AggLlmResponse::default()), None);
    }

    #[test]
    fn a_first_token_spent_elsewhere_is_no_verdict() {
        let stray = scored(json!([
            {"token": "Okay", "logprob": -0.05},
            {"token": "yes", "logprob": 0.05_f64.ln()},
            {"token": "no", "logprob": 0.001_f64.ln()}
        ]));
        assert_eq!(stray, None);
    }

    #[test]
    fn unavailable_chat_logprobs_produce_no_score() {
        let mut response = AggLlmResponse::default();
        for format in [WireFormat::OpenAiResponses, WireFormat::AnthropicMessages] {
            response.preservation.responses.insert(
                FormatId::from(format),
                json!({"choices": [{"logprobs": {"content": [{"top_logprobs": [
                    {"token": "yes", "logprob": 0.0}
                ]}]}}]}),
            );
            assert_eq!(p_yes(&response), None);
        }
        for body in [
            json!({"choices": []}),
            json!({"choices": [{"logprobs": null}]}),
            json!({"choices": [{"logprobs": {"content": []}}]}),
        ] {
            response
                .preservation
                .responses
                .insert(FormatId::from(WireFormat::OpenAiChat), body);
            assert_eq!(p_yes(&response), None);
        }
    }
}