Skip to content

Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.

Rust
26.6 KB
Raw Blame
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0

//! The classifier that runs one verification-gated turn.
//!
//! A proposed tool call is judged against the session's trajectory; a final
//! answer is typed, judged by rungs that run cheapest-first and stop once the
//! decision is settled, and then committed or escalated. Bounded mode shares one
//! deadline across calls. A failed or timed-out call supplies no evidence.

use std::future::Future;
use std::sync::Arc;
use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};

use async_trait::async_trait;
use switchyard_protocol::{
    AggLlmResponse, ContentBlock, Message, ModelId, Request, Response, StopReason,
};

use super::config::{ServingMode, VgrConfig};
use super::decide::{self, AgenticRun, Route, Signals, Tri};
use super::rungs::{self, Question};
use super::safety::{CircuitBreaker, endpoint_failure, fallback_eligible};
use super::{Branch, Capabilities, TaskType, derive_capabilities, readout, text};
use crate::algorithms::util::affinity::{AffinityRouter, has_new_user_turn, initiated_by_user};
use crate::algorithms::util::buffered_response::buffer_response;
use crate::algorithms::util::decisive;
use crate::algorithms::util::prompts::append_note;
use crate::core::algorithm::{AnswerSummary, Driver};
use crate::core::classifier::{Classification, Classifier};
use crate::core::state::{State, StateValue};
use crate::{LibsyError, Result};

/// Whether turn verification latched the capable tier; cleared when the user responds.
const TURN_LATCHED_KEY: &str = "vgr_turn_latched";
/// Holds a shadow turn only after a VGR escalation.
const SHADOW_HELD_KEY: &str = "vgr_shadow_held";
const LAST_ANSWER_FINAL_KEY: &str = "vgr_last_answer_final";
const FOLLOW_UP_QUIET_SECS: u32 = 2;
pub(super) const TURN_STREAK_KEY: &str = "vgr_turn_streak";
/// Unix seconds at the first request of the current user turn.
const TURN_STARTED_KEY: &str = "vgr_turn_started";
/// Set once the handoff note was sent in the current user turn.
const HANDOFF_SENT_KEY: &str = "vgr_handoff_sent";
/// Set once the efficient tier served a response since the last handoff.
pub(super) const EFFICIENT_SERVED_KEY: &str = "vgr_local_served";

fn count(state: &State, key: &str) -> Option<u32> {
    match state.extra.get(key) {
        Some(StateValue::Count(n)) => Some(*n),
        _ => None,
    }
}

fn set_count(state: &mut State, key: &str, value: u32) {
    state
        .extra
        .insert(key.to_string(), StateValue::Count(value));
}

pub(super) fn set_flag(state: &mut State, key: &str, on: bool) {
    state.extra.insert(key.to_string(), StateValue::Bool(on));
}

fn flag(state: &State, key: &str) -> bool {
    matches!(state.extra.get(key), Some(StateValue::Bool(true)))
}

pub(super) fn turn_streak(state: &State) -> u32 {
    count(state, TURN_STREAK_KEY).unwrap_or(0)
}

struct RequestContext<'a> {
    driver: &'a Driver,
    request: &'a Request,
    started: Instant,
}

/// Consecutive escalation votes that latch a session to the capable tier.
pub(super) const TURN_CONFIRMATIONS: u32 = 2;
const TURN_ESCALATE_AT: f64 = 0.5;

type Scored = (Classification, Option<Response>);

pub(super) struct VgrClassifier {
    pub(super) config: VgrConfig,
    pub(super) breaker: CircuitBreaker,
}

/// An escalation hold, recorded as a VGR decision with its reason whenever it serves a call.
pub(super) struct Held(pub(super) Arc<AffinityRouter>, pub(super) &'static str);

#[async_trait]
impl Classifier for Held {
    async fn score(
        &self,
        state: &mut State,
        request: &mut Request,
        driver: &Driver,
    ) -> Result<Scored> {
        let scored = self.0.score(state, request, driver).await?;
        if matches!(&scored.0, Classification::Scores(scores) if !scores.is_empty()) {
            log(driver, self.1, None, Route::Capable);
        }
        Ok(scored)
    }
}

#[async_trait]
impl Classifier for VgrClassifier {
    async fn score(
        &self,
        state: &mut State,
        request: &mut Request,
        driver: &Driver,
    ) -> Result<Scored> {
        let started = Instant::now();
        let efficient = &self.config.targets.efficient;
        let messages = &request.llm_request.messages;
        let (new_turn, answered) = match initiated_by_user(request) {
            Some(by_user) => (by_user, by_user),
            None => {
                let answered = answers_final_reply(state, messages);
                let new_turn = if state.extra.contains_key(LAST_ANSWER_FINAL_KEY) {
                    answered
                } else {
                    has_new_user_turn(messages)
                };
                (new_turn, answered)
            }
        };
        if new_turn {
            state.extra.remove(TURN_STARTED_KEY);
            state.extra.remove(SHADOW_HELD_KEY);
            set_flag(state, HANDOFF_SENT_KEY, false);
        }
        if answered {
            set_flag(state, TURN_LATCHED_KEY, false);
            set_count(state, TURN_STREAK_KEY, 0);
        }
        let turn_budget_spent = self.turn_budget_spent(state);
        let short_circuit = if flag(state, SHADOW_HELD_KEY) {
            Some("turn_verification_latched")
        } else if self.config.mode == ServingMode::Off {
            Some("mode_off")
        } else if self
            .config
            .kill_switch
            .as_ref()
            .is_some_and(|switch| switch.is_engaged())
        {
            Some("kill_switch")
        } else if flag(state, TURN_LATCHED_KEY) || turn_streak(state) >= TURN_CONFIRMATIONS {
            Some("turn_verification_latched")
        } else if turn_budget_spent {
            Some("efficient_turn_budget")
        } else if !self.config.efficient_supports_attachments && request_has_image(request) {
            Some("efficient_image_unsupported")
        } else if text::newest_prompt_is_blank(request) {
            Some("empty_prompt")
        } else {
            None
        };
        if let Some(reason) = short_circuit {
            return Ok(self.escalate(driver, reason, None, request, state, None));
        }

        if self.config.bounded_efficient && self.remaining(started).is_none() {
            return Ok(self.escalate(driver, "efficient_timed_out", None, request, state, None));
        }
        let Some(trial) = self.breaker.attempt() else {
            return Ok(self.escalate(driver, "breaker_open", None, request, state, None));
        };
        let typed = if text::ToolRecord::from_request(request).errors > 0 {
            let context = RequestContext {
                driver,
                request,
                started,
            };
            Some(self.type_task(&context).await)
        } else {
            None
        };
        if typed == Some(Some(TaskType::Chat)) {
            let reason = "chat_after_tool_error";
            return Ok(self.escalate(driver, reason, Some(Branch::Chat), request, state, None));
        }
        if self.config.bounded_efficient && self.remaining(started).is_none() {
            return Ok(self.escalate(driver, "efficient_timed_out", None, request, state, None));
        }
        let attempted = self
            .within_budget(started, async {
                let response = driver
                    .call_model_with_error_recovery(request.clone(), vec![efficient.clone()], true)
                    .await?;
                buffer_response(efficient.as_str(), response).await
            })
            .await;
        let buffered = match attempted {
            Some(Ok(buffered)) => {
                trial.success();
                buffered
            }
            Some(Err(error)) => {
                if endpoint_failure(&error) {
                    trial.failure();
                }
                if !fallback_eligible(&error) {
                    return Err(error);
                }
                return Ok(self.escalate(
                    driver,
                    "efficient_unavailable",
                    None,
                    request,
                    state,
                    None,
                ));
            }
            None => {
                trial.failure();
                return Ok(self.escalate(
                    driver,
                    "efficient_timed_out",
                    None,
                    request,
                    state,
                    None,
                ));
            }
        };

        let context = RequestContext {
            driver,
            request,
            started,
        };
        if rungs::has_tool_call(&buffered.agg) {
            if !tool_use_is_complete(&buffered.agg) {
                return Ok(self.escalate(
                    driver,
                    "malformed_tool_call",
                    None,
                    request,
                    state,
                    None,
                ));
            }
            if self.verify_turn(&context, &buffered.agg, state).await {
                return Ok(self.escalate(
                    driver,
                    "turn_verification_escalated",
                    None,
                    request,
                    state,
                    None,
                ));
            }
            if self.config.mode == ServingMode::Shadow {
                return Ok(self.serve_capable(
                    driver,
                    "turn_verification_complete",
                    None,
                    request,
                    state,
                    None,
                ));
            }
            log(driver, "turn_verification_complete", None, Route::Efficient);
            set_flag(state, EFFICIENT_SERVED_KEY, true);
            return Ok((decisive(efficient), Some(buffered.into_response())));
        }

        let attempt = rungs::response_text(&buffered.agg).unwrap_or_default();
        let task_type = match typed {
            Some(task_type) => task_type,
            None => self.type_task(&context).await,
        };
        let caps = derive_capabilities(
            request,
            &attempt,
            task_type,
            self.config.efficient_supports_attachments,
        );
        let signals = self.gather(&context, &caps).await;
        let route = decide::decide(&caps, &signals, self.config.confirmed_min_clean_tail());
        if route == Route::Capable {
            return Ok(self.escalate(
                driver,
                "decided",
                Some(caps.branch),
                request,
                state,
                Some(&attempt),
            ));
        }
        if self.config.mode == ServingMode::Shadow {
            return Ok(self.serve_capable(
                driver,
                "decided",
                Some(caps.branch),
                request,
                state,
                Some(&attempt),
            ));
        }
        log(driver, "decided", Some(caps.branch), Route::Efficient);
        set_flag(state, EFFICIENT_SERVED_KEY, true);
        Ok((decisive(efficient), Some(buffered.into_response())))
    }

    fn tracks_answers(&self) -> bool {
        true
    }

    fn observe_answer(&self, state: &mut State, answer: AnswerSummary) {
        let finished = if answer.final_reply {
            unix_seconds().max(1)
        } else {
            0
        };
        set_count(state, LAST_ANSWER_FINAL_KEY, finished);
    }
}

impl VgrClassifier {
    /// Whether the efficient tier may answer `request` when the capable tier fails it.
    ///
    /// That answer is unverified, so it is offered only while efficient attempts are allowed.
    pub(super) fn efficient_may_stand_in(&self, request: &Request) -> bool {
        matches!(self.config.mode, ServingMode::Active { .. })
            && !self
                .config
                .kill_switch
                .as_ref()
                .is_some_and(|switch| switch.is_engaged())
            && !self.breaker.is_open()
            && (self.config.efficient_supports_attachments || !request_has_image(request))
    }

    fn escalate(
        &self,
        driver: &Driver,
        reason: &'static str,
        branch: Option<Branch>,
        request: &mut Request,
        state: &mut State,
        unverified_final: Option<&str>,
    ) -> Scored {
        if self.config.mode == ServingMode::Shadow {
            set_flag(state, SHADOW_HELD_KEY, true);
        }
        self.serve_capable(driver, reason, branch, request, state, unverified_final)
    }

    fn serve_capable(
        &self,
        driver: &Driver,
        reason: &'static str,
        branch: Option<Branch>,
        request: &mut Request,
        state: &mut State,
        unverified_final: Option<&str>,
    ) -> Scored {
        if reason != "mode_off" {
            self.hand_off(request, state, unverified_final);
        }
        log(driver, reason, branch, Route::Capable);
        (decisive(&self.config.targets.capable), None)
    }

    /// Whether the current user turn has used up its efficient wall-clock budget.
    ///
    /// The clock starts at the first request of the user turn and includes the
    /// client's tool execution, since that is the time the client's own limit
    /// is spending.
    fn turn_budget_spent(&self, state: &mut State) -> bool {
        if !self.config.bounded_efficient {
            return false;
        }
        let Some(budget) = self.config.efficient_turn_budget else {
            return false;
        };
        let now = unix_seconds();
        let started = count(state, TURN_STARTED_KEY).unwrap_or(now);
        set_count(state, TURN_STARTED_KEY, started);
        u64::from(now.saturating_sub(started)) >= budget.as_secs()
    }

    /// Tells the capable tier it is inheriting unverified tool-using work.
    ///
    /// Sent at most once per user turn, on its first escalation.
    fn hand_off(&self, request: &mut Request, state: &mut State, unverified_final: Option<&str>) {
        if !self.config.agentic_handoff
            || flag(state, HANDOFF_SENT_KEY)
            || !flag(state, EFFICIENT_SERVED_KEY)
            || !text::turn_has_tool_trajectory(request)
        {
            return;
        }
        set_flag(state, HANDOFF_SENT_KEY, true);
        set_flag(state, EFFICIENT_SERVED_KEY, false);
        let note = handoff_note(unverified_final);
        if !(self.config.compact_handoff && super::compaction::compact(request, state, &note)) {
            append_note(request, &note);
        }
    }

    /// Judges one proposed tool call; `true` escalates the rest of the session.
    async fn verify_turn(
        &self,
        context: &RequestContext<'_>,
        proposed: &AggLlmResponse,
        state: &mut State,
    ) -> bool {
        let mut call = rungs::build_turn_request(
            rungs::turn_view(context.request, proposed),
            readout::MAX_OUTPUT_TOKENS,
            context.request.metadata.clone(),
        );
        readout::request_logprobs(&mut call);
        let judge = self.config.judge();
        let probability = match self.call(context, call, judge).await {
            None => return false,
            Some(Ok(agg)) => readout::p_yes(&agg),
            // Verification fails open, unless the efficient endpoint itself is down.
            Some(Err(error)) => {
                if *judge == self.config.targets.efficient && endpoint_failure(&error) {
                    self.breaker.failure();
                    set_flag(state, TURN_LATCHED_KEY, true);
                    return true;
                }
                return false;
            }
        };
        let Some(probability) = probability else {
            return false;
        };
        if probability < TURN_ESCALATE_AT {
            set_count(state, TURN_STREAK_KEY, 0);
            return false;
        }
        let streak = turn_streak(state).saturating_add(1);
        set_count(state, TURN_STREAK_KEY, streak);
        if streak >= TURN_CONFIRMATIONS {
            set_flag(state, TURN_LATCHED_KEY, true);
        }
        streak >= TURN_CONFIRMATIONS
    }

    /// Types the request; abstains, selecting the default regime, on any failure.
    async fn type_task(&self, context: &RequestContext<'_>) -> Option<TaskType> {
        if !self.config.task_typing {
            return None;
        }
        let (turns, _) = text::turns(context.request);
        let task = text::user_task_text(&turns);
        if task.trim().is_empty() {
            return None;
        }
        let call = rungs::build_typing_request(task, context.request.metadata.clone());
        let agg = self.call(context, call, self.config.judge()).await?.ok()?;
        rungs::parse_task_type(&agg)
    }

    /// Gathers evidence cheapest-first, stopping once the decision commits.
    async fn gather(&self, context: &RequestContext<'_>, caps: &Capabilities) -> Signals {
        let mut signals = Signals::default();
        let branch = caps.branch;
        let confirmed = self.config.confirmed_min_clean_tail();
        let run = (branch == Branch::Agentic).then(|| decide::agentic_run(caps, confirmed));
        let (Some(judged), false) = (
            caps.transcript.as_deref(),
            branch == Branch::Unknown
                || run == Some(AgenticRun::Vetoed)
                || (matches!(branch, Branch::Chat | Branch::DefaultVerified)
                    && caps.tools.is_some_and(|tools| tools.errors > 0)),
        ) else {
            return signals;
        };
        let settled =
            |signals: &Signals| decide::decide(caps, signals, confirmed) == Route::Efficient;
        let capable_confirms = run == Some(AgenticRun::ConfirmedRecovery);
        let efficient = self.config.judge();
        let ask = |target, question, max_output_tokens| {
            self.ask(context, target, question, judged, max_output_tokens)
        };

        // Coding here ran no tools, and a typed answer cites none, so neither record holds
        // evidence: judge the answer itself.
        let asked = match branch {
            Branch::Coding | Branch::Answer => Question::Answer,
            _ => Question::Evidence,
        };
        signals.readout = self.readout(context, judged, asked).await;
        if branch == Branch::Coding {
            if let Some(capable) = self.config.targets.capable_judge.as_ref()
                && decide::efficient_verified(branch, &signals)
                && !settled(&signals)
            {
                let tokens = rungs::DELIBERATION_MAX_OUTPUT_TOKENS;
                signals.capable_judge = Some(ask(capable, asked, tokens).await);
            }
            return signals;
        }
        if settled(&signals) {
            return signals;
        }
        // A deliberation never commits a typed answer, so the answer branch goes to the capable tier.
        if branch != Branch::Answer
            && !(capable_confirms && decide::efficient_verified(branch, &signals))
        {
            let metadata = || context.request.metadata.clone();
            let call = rungs::build_deliberation_request(judged, metadata());
            if let Some(Ok(deliberation)) = self.call(context, call, efficient).await {
                let call = rungs::build_verdict_request(judged, &deliberation, metadata());
                let verdict = self
                    .call(context, call, efficient)
                    .await
                    .and_then(Result::ok);
                signals.deliberation = verdict.and_then(|agg| readout::p_yes(&agg));
            }
            if settled(&signals) {
                return signals;
            }
        }
        let Some(capable) = self.config.targets.capable_judge.as_ref() else {
            return signals;
        };
        let capable_ask = |question| ask(capable, question, rungs::DELIBERATION_MAX_OUTPUT_TOKENS);
        if capable_confirms {
            if decide::efficient_verified(branch, &signals) {
                signals.capable_judge = Some(capable_ask(Question::Evidence).await);
            }
            return signals;
        }
        match branch {
            Branch::Answer => {
                signals.answer_verifier = Some(capable_ask(Question::Answer).await);
                if !settled(&signals) {
                    signals.evidence_verifier = Some(capable_ask(Question::Evidence).await);
                }
            }
            Branch::Chat | Branch::DefaultVerified => {
                signals.capable_judge = Some(capable_ask(Question::Evidence).await);
                // Asked only once the first affirms, so a refutation costs one call.
                if signals.capable_judge == Some(Tri::Yes) {
                    signals.evidence_confirm = Some(capable_ask(Question::Answer).await);
                }
            }
            _ => {}
        }
        signals
    }

    async fn readout(
        &self,
        context: &RequestContext<'_>,
        judged: &str,
        question: Question,
    ) -> Option<f64> {
        let mut call = rungs::build_request(
            question,
            judged,
            readout::MAX_OUTPUT_TOKENS,
            context.request.metadata.clone(),
        );
        readout::request_logprobs(&mut call);
        let agg = self.call(context, call, self.config.judge()).await?.ok()?;
        readout::p_yes(&agg)
    }

    async fn ask(
        &self,
        context: &RequestContext<'_>,
        target: &ModelId,
        question: Question,
        judged: &str,
        max_output_tokens: u64,
    ) -> Tri {
        let call = rungs::build_request(
            question,
            judged,
            max_output_tokens,
            context.request.metadata.clone(),
        );
        match self.call(context, call, target).await {
            Some(Ok(agg)) => rungs::parse_verdict(&agg),
            _ => Tri::Unknown,
        }
    }

    /// One verifier call inside the remaining budget; `None` once it is spent.
    async fn call(
        &self,
        context: &RequestContext<'_>,
        call: Request,
        target: &ModelId,
    ) -> Option<Result<AggLlmResponse>> {
        self.within_budget(context.started, async {
            let response = context
                .driver
                .call_model_with_error_recovery(call, vec![target.clone()], true)
                .await?;
            response
                .llm_response
                .into_agg()
                .await
                .map_err(|source| LibsyError::client_call(target.clone(), source))
        })
        .await
    }

    /// Waits for a candidate or verifier, enforcing a shared deadline only in bounded mode.
    async fn within_budget<T>(
        &self,
        started: Instant,
        operation: impl Future<Output = T>,
    ) -> Option<T> {
        if !self.config.bounded_efficient {
            return Some(operation.await);
        }
        tokio::time::timeout(self.remaining(started)?, operation)
            .await
            .ok()
    }

    fn remaining(&self, started: Instant) -> Option<Duration> {
        self.config.deadline.checked_sub(started.elapsed())
    }
}

fn unix_seconds() -> u32 {
    SystemTime::now()
        .duration_since(UNIX_EPOCH)
        .map_or(0, |elapsed| {
            u32::try_from(elapsed.as_secs()).unwrap_or(u32::MAX)
        })
}

/// The note a capable tier receives when it takes over a tool-using session.
fn handoff_note(unverified_final: Option<&str>) -> String {
    let mut note = String::from(
        "\n\n[Routing notice from the serving infrastructure, not from the user] Until now the \
         work on the user's current request was carried out by a smaller, faster model. That work \
         has NOT been verified: it may be incomplete or incorrect, and it may have left the \
         environment in a broken state. You are taking over from here. Do not assume that any \
         step taken on that request succeeded or that the task is finished. Inspect the actual \
         current state of the environment against every requirement of the request, fix \
         whatever is missing or wrong, and give your final answer only after you have confirmed \
         that the requirements are met. Earlier steps may already have done part of the work. \
         Keep what they did and build on it: confirm a step's effect by inspecting the \
         environment rather than by running the step again, and repeat or reverse a step only \
         when the request cannot be met otherwise. Replies to the user's earlier requests are \
         not in question; do not revisit or correct them unless the user asks.",
    );
    if let Some(claim) = unverified_final.filter(|claim| !claim.trim().is_empty()) {
        const CLIPPING_MARKER_RESERVE: usize = 64;
        let claim = text::clip_mid(
            claim,
            super::render::ATTEMPT_BUDGET.saturating_sub(CLIPPING_MARKER_RESERVE),
            1.0 / 3.0,
        );
        note.push_str(
            "\n\nThe smaller model was about to finish with the following unchecked message:\n\
             <unverified_final_message>\n",
        );
        note.push_str(&claim);
        note.push_str("\n</unverified_final_message>");
    }
    note
}

fn log(driver: &Driver, reason: &'static str, branch: Option<Branch>, served: Route) {
    driver.set_evidence(serde_json::json!({
        "source": "vgr",
        "reason_code": reason,
        "verdict": if served == Route::Efficient { "efficient" } else { "capable" },
        "branch": branch.map(|branch| format!("{branch:?}")),
        "policy_version": decide::POLICY_VERSION,
    }));
    tracing::info!(
        target: "libsy",
        reason,
        branch = ?branch,
        served = ?served,
        policy_version = decide::POLICY_VERSION,
        "vgr decision"
    );
}

/// Whether the user is answering a finished assistant reply.
fn answers_final_reply(state: &State, messages: &[Message]) -> bool {
    if let Some(finished) = count(state, LAST_ANSWER_FINAL_KEY)
        && (finished == 0 || unix_seconds().saturating_sub(finished) < FOLLOW_UP_QUIET_SECS)
    {
        return false;
    }
    messages
        .len()
        .checked_sub(1)
        .is_some_and(|last| text::starts_user_turn(messages, last))
}

fn tool_use_is_complete(response: &AggLlmResponse) -> bool {
    response.outputs.iter().all(|output| {
        output.stop_reason != Some(StopReason::MaxTokens)
            && output.content.iter().all(|block| {
                !matches!(block, ContentBlock::ToolCall(call) if call.arguments.is_string())
            })
    })
}

/// Whether a request carries image content directly or inside a tool result.
fn request_has_image(request: &Request) -> bool {
    fn has_image(block: &ContentBlock) -> bool {
        match block {
            ContentBlock::Image { .. } => true,
            ContentBlock::ToolResult(result) => result.content.iter().any(has_image),
            _ => false,
        }
    }
    let llm = &request.llm_request;
    llm.instructions
        .iter()
        .flat_map(|instruction| &instruction.content)
        .chain(llm.messages.iter().flat_map(|message| &message.content))
        .any(has_image)
}

#[cfg(test)]
mod tests;