Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.
Rust
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0
//! Trusted requirements constrain routing roles, provider calls and retained tasks.
use serde_json::{Value, json};
use switchyard_llm_client::AuxiliaryOperation;
use switchyard_protocol::{LlmClientError, ModelId, Request, text_request};
use switchyard_runner::{Runner, RunnerError, require_target_labels};
use wiremock::matchers::{body_partial_json, method, path};
use wiremock::{Mock, MockServer, ResponseTemplate};
const STAGE: &str = r#"
type = "stage_router"
efficient_target = "both_fast"
capable_target = "both_capable"
picker = "efficient_first"
confidence_threshold = 0.6
"#;
const ALTERNATIVES: &str = r#"
target_alternatives = { fast = ["private_fast", "both_fast"], capable = ["private_capable", "both_capable"] }
"#;
fn config(endpoint: &str, algorithm: &str, policy: &str) -> String {
let targets = [
("fast", "cloud/fast", &["public"] as &[&str]),
("capable", "cloud/capable", &[]),
("private_fast", "private/fast", &["private"]),
("private_capable", "private/capable", &["private"]),
("both_fast", "approved/fast", &["private", "customer_data"]),
(
"both_capable",
"approved/capable",
&["private", "customer_data"],
),
("judge", "cloud/judge", &[]),
(
"approved_judge",
"approved/judge",
&["private", "customer_data"],
),
]
.into_iter()
.map(|(name, id, labels)| {
format!(
"[targets.{name}]\nid = {id:?}\nllm_client = \"primary\"\nlabels = {}\n",
json!(labels)
)
})
.collect::<String>();
format!(
r#"
schema_version = 1
[llm_clients.primary]
format = "openai_chat"
base_url = "{endpoint}/v1"
max_retries = 0
failure_cooldown_ms = 0
{targets}
[routes.chat]
id = "chat"
{algorithm}
[routes.chat.eligibility]
{policy}
"#
)
}
fn request(task: &str, labels: Option<&[&str]>) -> Request {
let mut request = Request {
llm_request: text_request(None, "Summarize these synthetic notes."),
..Request::default()
};
request.metadata.get_or_insert_default().task_id = Some(task.into());
if let Some(labels) = labels {
require_target_labels(&mut request, labels.iter().copied()).expect("trusted requirements");
}
request
}
fn completion(model: &str, content: &str) -> ResponseTemplate {
ResponseTemplate::new(200).set_body_json(json!({
"id": "fixture", "object": "chat.completion", "model": model,
"choices": [{"index": 0, "message": {"role": "assistant", "content": content},
"finish_reason": "stop"}]
}))
}
async fn respond(server: &MockServer, model: &str, response: ResponseTemplate, count: u64) {
Mock::given(method("POST"))
.and(path("/v1/chat/completions"))
.and(body_partial_json(json!({"model": model})))
.respond_with(response)
.expect(count)
.mount(server)
.await;
}
async fn models_called(server: &MockServer) -> Vec<String> {
server
.received_requests()
.await
.expect("recorded requests")
.iter()
.map(|request| {
let body: Value = serde_json::from_slice(&request.body).expect("provider JSON");
body["model"].as_str().expect("provider model").to_owned()
})
.collect()
}
#[tokio::test]
async fn configured_stage_keeps_answer_fallbacks_approved() {
let server = MockServer::start().await;
let runner = Runner::from_toml(&config(&server.uri(), STAGE, "")).expect("approved Stage");
let route = runner.route("chat").expect("route");
let normal = route
.decide(request("normal", None))
.await
.expect("normal routing");
assert_eq!(normal.selected_model_ids, ["both_fast", "both_capable"]);
respond(&server, "approved/fast", ResponseTemplate::new(503), 1).await;
respond(
&server,
"approved/capable",
completion("approved/capable", "approved answer"),
1,
)
.await;
let outcome = route
.execute(
request("private", Some(&["private", "customer_data"])),
None,
)
.await
.expect("approved fallback");
assert_eq!(outcome.selected_model, "both_fast");
assert_eq!(
outcome.response.served_model().map(|model| model.as_str()),
Some("both_capable")
);
assert_eq!(
models_called(&server).await,
["approved/fast", "approved/capable"]
);
server.verify().await;
}
#[tokio::test]
async fn ordered_alternatives_rebind_clients_settings_prompts_and_fallbacks() {
let primary = MockServer::start().await;
let approved = MockServer::start().await;
let algorithm = STAGE
.replace("both_fast", "fast")
.replace("both_capable", "capable");
let mut source = config(&primary.uri(), &algorithm, ALTERNATIVES);
let cases = [
(
&primary,
"fast",
"cloud/fast",
"minimal",
"PUBLIC PROMPT",
200,
),
(
&approved,
"private_fast",
"private/fast",
"low",
"PRIVATE FAST PROMPT",
503,
),
(
&approved,
"private_capable",
"private/capable",
"high",
"PRIVATE CAPABLE PROMPT",
200,
),
(
&approved,
"both_fast",
"approved/fast",
"medium",
"APPROVED PROMPT",
200,
),
];
for (server, target, model, effort, prompt, status) in cases {
let client = if target == "fast" {
"primary"
} else {
"approved"
};
source = source.replace(
&format!("[targets.{target}]\nid = {model:?}\nllm_client = \"primary\""),
&format!(
"[targets.{target}]\nid = {model:?}\nllm_client = {client:?}\nreasoning_effort = {effort:?}\nsystem_prompt = {prompt:?}"
),
);
Mock::given(method("POST"))
.and(path("/v1/chat/completions"))
.and(body_partial_json(json!({
"model": model,
"reasoning_effort": effort,
"messages": [
{"role": "system", "content": prompt},
{"role": "user", "content": "Summarize these synthetic notes."}
]
})))
.respond_with(if status == 200 {
completion(model, "answer")
} else {
ResponseTemplate::new(status)
})
.expect(1)
.mount(server)
.await;
}
source.push_str(&format!(
"\n[llm_clients.approved]\nformat = 'openai_chat'\nbase_url = '{}/v1'\nmax_retries = 0\nfailure_cooldown_ms = 0\n",
approved.uri()
));
let runner = Runner::from_toml(&source).expect("ordered alternatives");
let route = runner.route("chat").expect("route");
for (task, labels, selected, served) in [
("normal", None, "fast", "fast"),
(
"private",
Some(&["private"] as &[&str]),
"private_fast",
"private_capable",
),
(
"pair",
Some(&["private", "customer_data"] as &[&str]),
"both_fast",
"both_fast",
),
] {
let outcome = route
.execute(request(task, labels), None)
.await
.expect("resolved answer and fallback");
assert_eq!(outcome.selected_model, selected);
assert_eq!(
outcome.response.served_model().map(|model| model.as_str()),
Some(served)
);
}
let alternative = ModelId::from("private_fast");
assert_eq!(
route
.target(&alternative)
.expect("alternative metadata")
.reasoning_effort
.as_deref(),
Some("low")
);
assert_eq!(route.provider_model(&alternative), "private/fast");
assert_eq!(models_called(&primary).await, ["cloud/fast"]);
assert_eq!(
models_called(&approved).await,
["private/fast", "private/capable", "approved/fast"]
);
primary.verify().await;
approved.verify().await;
}
#[tokio::test]
async fn filtered_targets_sharing_a_model_keep_their_own_settings() {
let server = MockServer::start().await;
for (target, effort) in [("both_fast", "low"), ("both_capable", "high")] {
let algorithm = format!("type = 'random'\ntargets = ['fast', '{target}']");
let source = config(&server.uri(), &algorithm, "")
.replace("id = \"approved/capable\"", "id = \"approved/fast\"")
.replace(
"[targets.both_fast]",
"[targets.both_fast]\nreasoning_effort = 'low'",
)
.replace(
"[targets.both_capable]",
"[targets.both_capable]\nreasoning_effort = 'high'",
);
let runner = Runner::from_toml(&source).expect("shared provider model");
let route = runner.route("chat").expect("route");
Mock::given(method("POST"))
.and(path("/v1/chat/completions"))
.and(body_partial_json(
json!({"model": "approved/fast", "reasoning_effort": effort}),
))
.respond_with(completion("approved/fast", "approved answer"))
.expect(1)
.mount(&server)
.await;
let outcome = route
.execute(request("shared", Some(&["private", "customer_data"])), None)
.await
.expect("approved target settings");
assert_eq!(outcome.selected_model, target);
assert_eq!(
outcome.response.served_model().map(|id| id.as_str()),
Some(target)
);
assert_eq!(
route.provider_model(&outcome.selected_model),
"approved/fast"
);
let metadata = route
.target(&outcome.selected_model)
.expect("target metadata");
assert_eq!(metadata.reasoning_effort.as_deref(), Some(effort));
server.verify().await;
server.reset().await;
}
}
#[tokio::test]
async fn every_label_is_required_and_subsets_need_no_catalogue() {
let server = MockServer::start().await;
let algorithm = "type = 'passthrough'\ntarget = 'both_fast'";
let runner = Runner::from_toml(&config(&server.uri(), algorithm, "")).expect("route");
let route = runner.route("chat").expect("route");
for labels in [
&[] as &[&str],
&["private"],
&["customer_data"],
&["customer_data", "private", "private"],
] {
let outcome = route
.decide(request("independent", Some(labels)))
.await
.expect("every required label is approved");
assert_eq!(outcome.selected_model_ids, ["both_fast"]);
}
assert!(
route
.decide(request("unknown", Some(&["unapproved"])))
.await
.is_err()
);
let runner = Runner::from_toml(&config(
&server.uri(),
"type = 'passthrough'\ntarget = 'private_fast'",
"",
))
.expect("single-label target");
assert!(
runner
.route("chat")
.expect("route")
.decide(request("pair", Some(&["private", "customer_data"])))
.await
.is_err()
);
assert!(models_called(&server).await.is_empty());
}
#[tokio::test]
async fn filtering_random_targets_preserves_weight_positions_and_zero_weight_exclusions() {
let server = MockServer::start().await;
for (weights, labels, expected) in [
(
"[0.0, 0.0, 1.0]",
&["private", "customer_data"] as &[&str],
"both_fast",
),
("[0.0, 1.0, 0.0]", &["private"], "private_fast"),
] {
let algorithm = format!(
r#"
type = "random"
targets = ["fast", "private_fast", "both_fast"]
weights = {weights}
seed = 7
"#
);
let runner =
Runner::from_toml(&config(&server.uri(), &algorithm, "")).expect("weighted route");
let outcome = runner
.route("chat")
.expect("route")
.decide(request("weights", Some(labels)))
.await
.expect("filtered weighted route");
assert_eq!(outcome.selected_model_ids, [expected]);
}
assert!(models_called(&server).await.is_empty());
}
#[tokio::test]
async fn explicit_requirements_apply_only_to_the_current_request() {
let server = MockServer::start().await;
let algorithm = "type = 'random'\ntargets = ['fast', 'private_fast']\nseed = 7";
let runner = Runner::from_toml(&config(
&server.uri(),
algorithm,
"require_explicit_requirements = true",
))
.expect("request-scoped route");
let route = runner.route("chat").expect("route");
let error = route
.decide(request("shared", None))
.await
.err()
.expect("missing requirements");
assert!(error.to_string().contains("explicit"), "{error}");
let unrestricted = route
.decide(request("shared", Some(&[])))
.await
.expect("explicit empty requirements");
assert_eq!(unrestricted.selected_model_ids.len(), 2);
for (labels, expected) in [
(&["private"] as &[&str], "private_fast"),
(&["public"], "fast"),
] {
let outcome = route
.decide(request("shared", Some(labels)))
.await
.expect("independent requirements on the same task");
assert_eq!(outcome.selected_model_ids, [expected]);
}
route
.decide(request("shared", Some(&[])))
.await
.expect("empty requirements remain valid");
assert!(route.decide(request("shared", None)).await.is_err());
assert!(models_called(&server).await.is_empty());
}
#[tokio::test]
async fn explicit_empty_differs_from_missing_and_task_requirements_only_accumulate() {
let server = MockServer::start().await;
let algorithm = STAGE
.replace("both_fast", "fast")
.replace("both_capable", "capable");
let policy = format!("require_explicit_requirements = true\nscope = 'task'\n{ALTERNATIVES}");
let source = config(&server.uri(), &algorithm, &policy);
let runner = Runner::from_toml(&source).expect("task-scoped route");
let route = runner.route("chat").expect("route");
assert!(route.decide(request("shared", None)).await.is_err());
for (task, labels, expected) in [
("shared", &[] as &[&str], ["fast", "capable"]),
("shared", &["private"], ["private_fast", "private_capable"]),
("shared", &["customer_data"], ["both_fast", "both_capable"]),
("shared", &[], ["both_fast", "both_capable"]),
("other", &[], ["fast", "capable"]),
] {
let outcome = route
.decide(request(task, Some(labels)))
.await
.expect("retained task requirements");
assert_eq!(outcome.selected_model_ids, expected);
}
assert!(route.decide(request("shared", None)).await.is_err());
let unavailable = source.replace("[\"private\",\"customer_data\"]", "[\"private\"]");
let runner =
Runner::from_toml(&unavailable).expect("valid config with unavailable intersection");
let route = runner.route("chat").expect("route");
route
.decide(request("blocked", Some(&["private"])))
.await
.expect("initial private task");
for labels in [&["customer_data"] as &[&str], &["private"], &[]] {
assert!(
route
.decide(request("blocked", Some(labels)))
.await
.is_err()
);
}
let other = route
.decide(request("other", Some(&["private"])))
.await
.expect("independent task requirements");
assert_eq!(
other.selected_model_ids,
["private_fast", "private_capable"]
);
assert!(models_called(&server).await.is_empty());
}
#[tokio::test]
async fn unapproved_primary_fallback_or_judge_fails_before_provider_io() {
let server = MockServer::start().await;
let judge = r#"
type = "llm_classifier"
classifier_target = "judge"
strong_target = "both_capable"
weak_target = "both_fast"
base_threshold = 0.5
fail_open = false
"#;
for algorithm in [
STAGE.replace("both_fast", "fast"),
STAGE.replace("both_capable", "capable"),
judge.to_owned(),
] {
let runner = Runner::from_toml(&config(&server.uri(), &algorithm, ""))
.expect("valid fixed-target configuration");
let error = match runner
.route("chat")
.expect("route")
.execute(
request("blocked", Some(&["private", "customer_data"])),
None,
)
.await
{
Ok(_) => panic!("unapproved model was admitted"),
Err(error) => error,
};
assert!(
matches!(
error,
RunnerError::Client(LlmClientError::InvalidRequest { .. })
),
"{error}"
);
}
assert!(models_called(&server).await.is_empty());
}
#[tokio::test]
async fn alternative_judges_and_auxiliary_calls_require_the_same_labels() {
let server = MockServer::start().await;
let algorithm = r#"
type = "llm_classifier"
classifier_target = "judge"
strong_target = "capable"
weak_target = "fast"
base_threshold = 0.5
fail_open = false
"#;
let policy = r#"target_alternatives = { fast = ["both_fast"], capable = ["both_capable"], judge = ["approved_judge"] }"#;
let runner = Runner::from_toml(&config(&server.uri(), algorithm, policy))
.expect("approved judge alternatives");
respond(&server, "approved/judge", completion("approved/judge", &json!({
"crux": "bounded task", "primary_rule": "SUP-1", "capability_boundary": "supported", "p_solve": 0.9
}).to_string()), 1).await;
respond(
&server,
"approved/fast",
completion("approved/fast", "approved answer"),
1,
)
.await;
let outcome = runner
.route("chat")
.expect("route")
.execute(request("judge", Some(&["private", "customer_data"])), None)
.await
.expect("approved judge and answer");
assert_eq!(outcome.selected_model, "both_fast");
assert_eq!(
models_called(&server).await,
["approved/judge", "approved/fast"]
);
server.verify().await;
server.reset().await;
let stage = STAGE
.replace("both_fast", "fast")
.replace("both_capable", "capable");
for (format, endpoint, operation) in [
(
"openai_responses",
"/v1/responses/input_tokens",
AuxiliaryOperation::ResponsesInputTokens,
),
(
"anthropic_messages",
"/v1/messages/count_tokens",
AuxiliaryOperation::AnthropicCountTokens,
),
] {
let source = config(&server.uri(), &stage, ALTERNATIVES)
.replace("format = \"openai_chat\"", &format!("format = {format:?}"));
let runner = Runner::from_toml(&source).expect("auxiliary route");
let route = runner.route("chat").expect("route");
Mock::given(method("POST"))
.and(path(endpoint))
.and(body_partial_json(json!({"model": "approved/capable"})))
.respond_with(ResponseTemplate::new(200).set_body_json(json!({"input_tokens": 42})))
.expect(1)
.mount(&server)
.await;
let result = route
.call_auxiliary(
request("aux", Some(&["private", "customer_data"])),
operation,
)
.await
.expect("approved auxiliary call");
assert_eq!(result["input_tokens"], 42);
assert!(
route
.call_auxiliary(request("blocked", Some(&["unapproved"])), operation)
.await
.is_err()
);
assert_eq!(models_called(&server).await, ["approved/capable"]);
server.verify().await;
server.reset().await;
}
}
#[tokio::test]
async fn typed_decision_alternatives_rebind_the_judge_and_candidate_targets() {
let primary = MockServer::start().await;
let approved = MockServer::start().await;
let algorithm = r#"
type = "llm_classifier"
classifier_target = "standard"
strong_target = "capable"
weak_target = "fast"
fail_open = false
"#;
let policy = r#"target_alternatives = { fast = ["both_fast"], capable = ["both_capable"], standard = ["approved_decision"] }"#;
let source = format!(
r#"{}
[routes.chat.decision]
cutoff = 0.5
[decision_clients.standard]
format = "openai_decisions"
endpoint = "{}/v1/decisions"
api_key_env = "PATH"
timeout_ms = 2000
[decision_clients.approved]
format = "openai_decisions"
endpoint = "{}/v1/decisions"
api_key_env = "PATH"
timeout_ms = 2000
[decision_targets.standard]
id = "cloud/decision"
decision_client = "standard"
[decision_targets.approved_decision]
id = "approved/decision"
decision_client = "approved"
labels = ["private", "customer_data"]
"#,
config(&primary.uri(), algorithm, policy),
primary.uri(),
approved.uri()
);
let runner = Runner::from_toml(&source).expect("typed judge alternatives");
Mock::given(method("POST"))
.and(path("/v1/decisions"))
.and(body_partial_json(json!({"model": "approved/decision"})))
.respond_with(ResponseTemplate::new(200).set_body_json(json!({
"model": "approved/decision",
"answers": [{
"type": "choice", "name": "route", "choice": "efficient",
"probabilities": [
{"value": "capable", "probability": 0.0},
{"value": "efficient", "probability": 1.0}
]
}],
"usage": {
"input_tokens": 1, "output_tokens": 1, "total_tokens": 2,
"input_tokens_details": {"cached_tokens": 0, "cache_write_tokens": 0},
"output_tokens_details": {"reasoning_tokens": 0}
}
})))
.expect(1)
.mount(&approved)
.await;
respond(
&primary,
"approved/fast",
completion("approved/fast", "approved answer"),
1,
)
.await;
let route = runner.route("chat").expect("route");
let outcome = route
.execute(request("typed", Some(&["private", "customer_data"])), None)
.await
.expect("approved typed judge and candidate");
assert_eq!(outcome.selected_model, "both_fast");
assert_eq!(models_called(&primary).await, ["approved/fast"]);
assert_eq!(models_called(&approved).await, ["approved/decision"]);
primary.verify().await;
approved.verify().await;
}