Skip to content

Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.

Rust
7.0 KB
Raw Blame
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0

//! Verification-gated routing: fail-closed capability derivation.
//!
//! Capabilities are never client-declared. They come from the router's own
//! typing of the request, the attempt it produced on the efficient tier, and its own reading
//! of the tool results in the conversation. No client-reachable input selects
//! a weaker regime than the default one; unsupported content and missing
//! evidence select [`Branch::Unknown`], which never commits.

use std::sync::Arc;

use switchyard_protocol::Request;

use self::text::ToolRecord;
use crate::Result;
use crate::algorithms::fall_through::FallThrough;
use crate::algorithms::util::affinity::AffinityRouter;
use crate::core::algorithm::{Algorithm, AnswerSummary, Driver, RoutingOutcome};

mod compaction;
mod config;
mod decide;
mod readout;
mod render;
mod rungs;
mod runtime;
mod safety;
mod text;

pub use config::{ACTIVE_APPROVAL, ServingMode, Targets, VgrConfig};
pub use safety::{BreakerConfig, KillSwitch};

#[cfg(test)]
mod tests;

/// A verification-gated route between an efficient and a capable tier.
pub struct Vgr {
    route: FallThrough,
    classifier: Arc<runtime::VgrClassifier>,
}

impl Vgr {
    /// Validates and constructs a verification-gated route.
    pub fn new(config: VgrConfig) -> Result<Self> {
        config.validate()?;
        let mut affinity = AffinityRouter::new().with_latch_only([config.targets.capable.clone()]);
        let reason = if config.latch_escalation && config.mode != ServingMode::Shadow {
            "session_latched"
        } else {
            affinity = affinity.with_release_on_user_turn();
            "turn_verification_latched"
        };
        let affinity = Arc::new(affinity);
        let mut route = FallThrough::new_with_state().with_name("vgr");
        if config.compact_handoff {
            route = route.with_classifier(Arc::new(compaction::Compactor));
        }
        let classifier = Arc::new(runtime::VgrClassifier {
            breaker: safety::CircuitBreaker::new(config.breaker),
            config,
        });
        // Shadow always serves the capable tier; only a VGR escalation should hold it there.
        if classifier.config.mode != ServingMode::Shadow {
            route = route
                .with_processor(affinity.clone())
                .with_classifier(Arc::new(runtime::Held(affinity, reason)));
        }
        let route = route.with_classifier(classifier.clone());
        Ok(Self { route, classifier })
    }
}

#[async_trait::async_trait]
impl Algorithm for Vgr {
    fn name(&self) -> &str {
        "vgr"
    }

    async fn route(self: Arc<Self>, driver: Driver, request: Request) -> Result<RoutingOutcome> {
        // The handoff note is written for the capable tier, so an efficient stand-in gets the request
        // as it arrived.
        let arrived = self
            .classifier
            .efficient_may_stand_in(&request)
            .then(|| request.clone());
        let mut outcome = self.route.execute(driver, request).await?;
        // Efficient answers are already buffered and have no fallback.
        outcome.selected_model_ids.truncate(1);
        let config = &self.classifier.config;
        if let Some(arrived) = arrived
            && outcome.response.is_none()
            && outcome.selected_model_ids.first() == Some(&config.targets.capable)
            // The efficient tier may have failed while this request was routed.
            && self.classifier.efficient_may_stand_in(&arrived)
        {
            outcome
                .selected_model_ids
                .push(config.targets.efficient.clone());
            outcome.fallback_request = Some(arrived);
        }
        Ok(outcome)
    }

    fn tracks_answers(&self) -> bool {
        self.route.tracks_answers()
    }

    async fn observe_answer(&self, session_id: &str, answer: AnswerSummary) {
        self.route.observe_answer(session_id, answer).await;
    }
}

/// The verification regime a request's capabilities license.
#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)]
enum Branch {
    /// Code or test activity, with no sandboxed checker to appeal to.
    Coding,
    /// A single-turn request typed as answer-seeking.
    Answer,
    /// Conversational traffic.
    Chat,
    /// A tool-using session verified from its trajectory.
    Agentic,
    /// Anything else with an attempt to verify.
    DefaultVerified,
    /// Nothing to verify.
    #[default]
    Unknown,
}

/// A task type produced by the router's own typing call; `None` abstains.
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
enum TaskType {
    Coding,
    Agentic,
    Answer,
    Chat,
}

/// The complete input the decision core sees.
#[derive(Default)]
struct Capabilities {
    branch: Branch,
    /// The judged view of the request and attempt.
    transcript: Option<String>,
    /// The router's own tool-result record; unused by the answer regime.
    tools: Option<ToolRecord>,
}

/// Derives capabilities from the router's own evidence, failing closed.
///
/// Media blocks fail closed unless `media_judged_as_text` lets the judge read the text around them.
fn derive_capabilities(
    request: &Request,
    attempt: &str,
    task_type: Option<TaskType>,
    media_judged_as_text: bool,
) -> Capabilities {
    let (turns, unsupported) = text::turns(request);
    if (unsupported && !media_judged_as_text)
        || text::user_task_text(&turns).trim().is_empty()
        || attempt.trim().is_empty()
    {
        return Capabilities::default();
    }
    let caps = |branch, transcript| Capabilities {
        branch,
        transcript,
        tools: Some(ToolRecord::from_request(request)),
    };

    // Typing abstains on a request that leans on earlier context, so a typed answer stands
    // on its own; only tool use in the current user turn takes it off this branch.
    if task_type == Some(TaskType::Answer) && !text::turn_has_tool_trajectory(request) {
        return Capabilities {
            tools: None,
            ..caps(
                Branch::Answer,
                Some(render::render_session(&turns, attempt)),
            )
        };
    }
    // A tool trajectory is judged as agentic work unless typed conversational.
    if task_type == Some(TaskType::Agentic)
        || (text::has_tool_trajectory(request)
            && !matches!(task_type, Some(TaskType::Answer | TaskType::Chat)))
    {
        return caps(
            Branch::Agentic,
            Some(render::render_agentic_view(request, &turns, attempt)),
        );
    }
    let transcript = Some(render::render_session(&turns, attempt));
    if text::observed_hardening(attempt) || task_type == Some(TaskType::Coding) {
        return caps(Branch::Coding, transcript);
    }
    if text::observed_tool_activity(attempt) {
        caps(Branch::Agentic, transcript)
    } else if matches!(task_type, Some(TaskType::Chat | TaskType::Answer)) {
        caps(Branch::Chat, transcript)
    } else {
        caps(Branch::DefaultVerified, transcript)
    }
}