Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.
Rust
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0
use std::collections::BTreeMap;
use std::sync::atomic::{AtomicUsize, Ordering};
use std::sync::{Arc, Mutex};
use async_trait::async_trait;
use futures_util::StreamExt;
use switchyard_llm_client::{ClientRouter, RunObservation};
use switchyard_protocol::{
LlmClientError, LlmResponse, ModelId, Request, Response, RoutedLlmClient, text_request,
text_response,
};
use switchyard_runner::{AlgorithmSpec, ModelCapabilities, Route};
struct StubClient;
#[async_trait]
impl RoutedLlmClient for StubClient {
async fn call(&self, request: Request) -> Result<Response, LlmClientError> {
Ok(Response {
llm_response: LlmResponse::Agg(text_response(
request.llm_request.model.clone(),
"plugin response",
)),
metadata: None,
upstream_headers: Default::default(),
})
}
}
fn plugin_route(client: Arc<dyn RoutedLlmClient>) -> Route {
let spec = AlgorithmSpec::Passthrough {
target: "semantic-target".to_string(),
subagents: None,
};
let targets = BTreeMap::from([(
"semantic-target".to_string(),
ModelId::from("semantic-target"),
)]);
let (algorithm, models) = spec
.build_with_runtime_models("switchyard", &targets)
.expect("identity target map should build");
let clients = ClientRouter::new(
BTreeMap::from([(ModelId::from("semantic-target"), client)])
.into_iter()
.collect(),
);
Route::new(
algorithm,
clients,
None,
ModelCapabilities::default(),
None,
None,
Vec::new(),
models,
)
}
#[test]
fn runtime_model_builder_rejects_an_unknown_target() {
// Embedding hosts do not get DeploymentConfig's target prevalidation.
let spec = AlgorithmSpec::Passthrough {
target: "missing".to_string(),
subagents: None,
};
let error = match spec.build_with_runtime_models("embedded", &BTreeMap::new()) {
Ok(_) => panic!("unknown target should fail"),
Err(error) => error,
};
assert_eq!(
error.to_string(),
"route embedded references unknown target missing"
);
}
#[tokio::test]
async fn plugin_shaped_route_executes_without_runner_model_or_toml() {
let route = plugin_route(Arc::new(StubClient));
let observations = Arc::new(Mutex::new(Vec::new()));
let observer = {
let observations = Arc::clone(&observations);
Arc::new(move |observation| observations.lock().unwrap().push(observation))
};
let request = Request {
llm_request: text_request(Some("arbitrary-upstream-model".to_string()), "hello"),
..Request::default()
};
let output = route
.execute(request, Some(observer))
.await
.expect("route should execute");
assert_eq!(output.selected_model, "semantic-target");
assert_eq!(
output
.response
.llm_response
.as_agg()
.unwrap()
.model
.as_deref(),
Some("semantic-target")
);
assert!(
observations
.lock()
.unwrap()
.iter()
.any(|observation| { matches!(observation, RunObservation::AnswerCall(_)) })
);
assert!(
observations
.lock()
.unwrap()
.iter()
.any(|observation| { matches!(observation, RunObservation::RoutingOverhead(_)) })
);
}
struct LazyStreamClient {
polls: Arc<AtomicUsize>,
}
#[async_trait]
impl RoutedLlmClient for LazyStreamClient {
async fn call(&self, _request: Request) -> Result<Response, LlmClientError> {
let polls = Arc::clone(&self.polls);
let stream = futures_util::stream::poll_fn(move |_| {
polls.fetch_add(1, Ordering::SeqCst);
std::task::Poll::Ready(None)
})
.boxed();
Ok(Response {
llm_response: LlmResponse::Stream(stream),
metadata: None,
upstream_headers: Default::default(),
})
}
}
#[tokio::test]
async fn route_returns_stream_without_polling_it() {
let polls = Arc::new(AtomicUsize::new(0));
let route = plugin_route(Arc::new(LazyStreamClient {
polls: Arc::clone(&polls),
}));
let request = Request {
llm_request: text_request(None, "hello"),
..Request::default()
};
let output = route
.execute(request, None)
.await
.expect("stream handle should be returned");
assert!(matches!(
output.response.llm_response,
LlmResponse::Stream(_)
));
assert_eq!(polls.load(Ordering::SeqCst), 0);
}
// Preserve checkpoint target order and explicit TOML overrides.
#[test]
fn prefill_router_config_preserves_target_order_and_overrides() {
let spec: AlgorithmSpec = toml::from_str(
r#"
type = "prefill_router"
targets = ["fast", "strong"]
checkpoint = "/models/router.pt"
device = "cuda:1"
max_length = 4096
batch_size = 8
"#,
)
.expect("prefill router config should parse");
let AlgorithmSpec::PrefillRouter {
targets,
checkpoint,
device,
cache_dir,
max_length,
batch_size,
} = spec
else {
panic!("expected prefill router config");
};
assert_eq!(targets, ["fast", "strong"]);
assert_eq!(checkpoint.to_string_lossy(), "/models/router.pt");
assert_eq!(device.as_deref(), Some("cuda:1"));
assert_eq!(cache_dir, None);
assert_eq!(max_length, Some(4096));
assert_eq!(batch_size, Some(8));
}