Skip to content

Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.

Rust
5.5 KB
Raw Blame
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0

//! Fake clients for libsy's own tests.
//!
//! libsy offloads every model call, so a test needs something to answer them. [`drive`] is
//! [`crate::drive`] with the promise handling filled in: a test supplies a [`Serve`] closure
//! standing in for the client a real host would use, and gets back the same
//! `(selected model, response)` pair a host does — the same path
//! `switchyard-llm-client`'s `run`
//! takes over HTTP.
//!
//! The closure is async so a fake can block on a barrier, wait on a notify, or never
//! resolve, which is what the concurrency, hedging, and fan-out tests need.

use std::collections::HashMap;
use std::future::Future;
use std::sync::Arc;

use futures::future::BoxFuture;
use switchyard_protocol::{
    Category, DecisionResponse, LlmClientError, LlmResponse, ModelId, Request, Response,
    text_response,
};

use crate::core::algorithm::{Algorithm, Call, CallDecision, Driver, RuntimeModels};
use crate::{LibsyError, Result, RoutingOutcome, State};

/// Builds one runtime model category for a test.
pub(crate) fn category_models(
    category: Category,
    names: &[&str],
) -> HashMap<Category, Vec<ModelId>> {
    [(
        category,
        names.iter().map(|name| ModelId::from(*name)).collect(),
    )]
    .into()
}

pub(crate) fn empty_driver() -> Driver {
    Driver::new("test", Arc::new(RuntimeModels::default())).0
}

/// The result a fake client hands back for one offloaded call.
pub(crate) type ServeResult = std::result::Result<Response, LlmClientError>;

/// Answers offloaded model calls. Returning `Err` propagates a failed *model* call back into
/// the algorithm, which may route around it.
pub(crate) trait Serve: Send + Sync + 'static {
    fn serve(&self, target: ModelId, request: Request) -> BoxFuture<'static, ServeResult>;
}

impl<F, Fut> Serve for F
where
    F: Fn(ModelId, Request) -> Fut + Send + Sync + 'static,
    Fut: Future<Output = ServeResult> + Send + 'static,
{
    fn serve(&self, target: ModelId, request: Request) -> BoxFuture<'static, ServeResult> {
        Box::pin(self(target, request))
    }
}

/// Run `algorithm` to completion, serving each offloaded call with `serve`.
pub(crate) async fn test_drive(
    algorithm: Arc<dyn Algorithm>,
    request: Request,
    serve: impl Serve,
) -> Result<(ModelId, Response)> {
    test_drive_with_models(algorithm, request, RuntimeModels::default(), serve).await
}

/// Drive one request with an explicit runtime model set.
pub(crate) async fn test_drive_with_models(
    algorithm: Arc<dyn Algorithm>,
    request: Request,
    models: impl Into<RuntimeModels>,
    serve: impl Serve,
) -> Result<(ModelId, Response)> {
    let serve = Arc::new(serve);
    let routing_serve = Arc::clone(&serve);
    let outcome = crate::drive(algorithm, request, Arc::new(models.into()), move |call| {
        fulfill(Arc::clone(&routing_serve), call)
    })
    .await?;
    let selected_model = outcome.selected_model_id()?.clone();
    let response = match outcome.response {
        Some(response) => response,
        None => serve
            .serve(selected_model.clone(), outcome.request)
            .await
            .map_err(|source| LibsyError::client_call(selected_model.clone(), source))?,
    };
    Ok((selected_model, response))
}

/// Drive one request from caller `state` and return the routing outcome, which carries the
/// updated state.
pub(crate) async fn test_drive_with_state(
    algorithm: Arc<dyn Algorithm>,
    request: Request,
    models: impl Into<RuntimeModels>,
    state: State,
    serve: impl Serve,
) -> Result<RoutingOutcome> {
    let serve = Arc::new(serve);
    crate::core::algorithm::drive_with_state(
        algorithm,
        request,
        Arc::new(models.into()),
        state,
        move |call| fulfill(Arc::clone(&serve), call),
    )
    .await
}

pub(crate) async fn serve_decision(call: CallDecision) -> Result<()> {
    let response = DecisionResponse {
        id: None,
        model: Some(call.model.clone()),
        answers: Default::default(),
        usage: Default::default(),
    };
    call.respond(std::future::ready(Ok(response))).await
}

/// Serve one call and fulfill its promise, mapping failures the way a host does so
/// error-shape assertions match production.
async fn fulfill(serve: Arc<impl Serve>, call: Call) -> Result<()> {
    let mut call = match call {
        Call::Model(call) => *call,
        Call::Decision(call) => return serve_decision(*call).await,
    };
    let request = call.request.clone();
    let target = call.models.first().cloned().ok_or(LibsyError::NoTargets)?;
    // Test clients return failures to routing policies, including policies under test.
    call.recover_errors = true;
    call.respond(async {
        serve
            .serve(target.clone(), request)
            .await
            .map_err(|source| LibsyError::client_call(target, source))
    })
    .await
}

/// Answers with the selected model name as the completion — what most routing tests need,
/// since they assert on *which* target was called.
pub(crate) fn echo() -> impl Serve {
    |target: ModelId, _request: Request| async move { Ok(reply(target)) }
}

/// A buffered response whose completion text is `completion`.
pub(crate) fn reply(completion: impl Into<String>) -> Response {
    Response {
        llm_response: LlmResponse::Agg(text_response(None, completion.into())),
        metadata: None,
        upstream_headers: http::HeaderMap::new(),
    }
}