Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.
Rust
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0
//! Integration tests for the libsy Rust server.
use std::collections::{BTreeMap, BTreeSet, HashSet};
use std::convert::Infallible;
use std::error::Error;
use std::io::Write;
use std::sync::Arc;
use axum::body::{Body, Bytes};
use axum::extract::{DefaultBodyLimit, Path, State};
use axum::http::{HeaderMap, HeaderValue, Request as HttpRequest, StatusCode, Uri};
use axum::response::sse::{Event, Sse};
use axum::response::{IntoResponse, Response as HttpResponse};
use axum::routing::{get, post};
use axum::{Json, Router};
use http_body_util::BodyExt;
use libsy::{Algorithm, Random};
use serde_json::{Value, json};
use switchyard_llm_client::{
Backend, ClientRouter, HttpBackendConfig, ModelConfig, TranslatingLlmClient,
};
use switchyard_protocol::RoutedLlmClient;
use switchyard_protocol::{Category, ModelId, WireFormat};
use switchyard_runner::{DecisionTarget, ModelCapabilities, Route, Runner, RuntimeModels};
use switchyard_server::config::load_server_state;
use switchyard_server::{
DEFAULT_MAX_REQUEST_BODY_BYTES, ServerState, build_llm_router, build_switchyard_router,
};
use tokio::net::TcpListener;
use tokio::sync::Mutex;
use tokio::task::JoinHandle;
use tower::ServiceExt;
type TestError = Box<dyn Error + Send + Sync>;
type TestResult<T = ()> = Result<T, TestError>;
const ROUTE_MODEL: &str = "switchyard/random";
const VERSION: &str = env!("CARGO_PKG_VERSION");
struct MockUpstream {
base_url: String,
calls: Arc<Mutex<Vec<Value>>>,
task: JoinHandle<()>,
}
impl MockUpstream {
async fn start() -> TestResult<Self> {
let calls = Arc::new(Mutex::new(Vec::new()));
let app = Router::new()
.route("/props", get(upstream_props))
.route("/v1/chat/completions", post(upstream_chat))
.route("/buffered/responses", post(upstream_buffered_responses))
.route(
"/v1/messages",
post(upstream_messages_requires_forwarded_oauth),
)
.route(
"/v1/responses",
post(upstream_responses_requires_forwarded_auth),
)
.route("/silo/{silo}/responses", post(upstream_responses_silo))
.route("/capture", post(upstream_redirect_capture))
.route("/v1/messages/count_tokens", post(upstream_count_tokens))
.route(
"/v1/responses/input_tokens",
post(upstream_responses_auxiliary),
)
.route("/v1/responses/compact", post(upstream_responses_auxiliary))
.route("/future/provider/endpoint", post(upstream_fallback))
.layer(DefaultBodyLimit::disable())
.with_state(Arc::clone(&calls));
let listener = TcpListener::bind("127.0.0.1:0").await?;
let addr = listener.local_addr()?;
let task = tokio::spawn(async move {
if let Err(error) = axum::serve(listener, app).await {
tracing::error!(error = %error, "mock upstream stopped");
}
});
Ok(Self {
base_url: format!("http://{addr}/v1"),
calls,
task,
})
}
/// The upstream model id of every request this upstream received, in order.
async fn models(&self) -> Vec<String> {
self.calls
.lock()
.await
.iter()
.filter_map(|call| call.get("model")?.as_str().map(str::to_string))
.collect()
}
}
impl Drop for MockUpstream {
fn drop(&mut self) {
self.task.abort();
}
}
fn user_prompt(body: &Value) -> &str {
body["messages"]
.as_array()
.and_then(|messages| messages.iter().find(|message| message["role"] == "user"))
.and_then(|message| message["content"].as_str())
.unwrap_or_default()
}
fn has_system_prompt(call: &Value, expected: &str) -> bool {
call["messages"].as_array().is_some_and(|messages| {
messages.iter().any(|message| {
message["role"] == "system" && message["content"].as_str() == Some(expected)
})
})
}
async fn upstream_props(State(calls): State<Arc<Mutex<Vec<Value>>>>) -> Json<Value> {
calls.lock().await.push(json!({"path": "/props"}));
Json(json!({"default_generation_settings": {"n_ctx": 98_304}}))
}
async fn upstream_chat(
State(calls): State<Arc<Mutex<Vec<Value>>>>,
Json(body): Json<Value>,
) -> HttpResponse {
calls.lock().await.push(body.clone());
let prompt = user_prompt(&body);
if prompt == "fail" {
return (
StatusCode::IM_A_TEAPOT,
Json(json!({"error": {"message": "upstream rejected request"}})),
)
.into_response();
}
if prompt == "auth-fail" {
return (
StatusCode::UNAUTHORIZED,
Json(json!({"error": {"message": "upstream authentication failed"}})),
)
.into_response();
}
let model = body["model"].as_str().unwrap_or("unknown").to_string();
if model == "model/rate-limited" {
return (
if prompt == "unavailable" {
StatusCode::SERVICE_UNAVAILABLE
} else {
StatusCode::TOO_MANY_REQUESTS
},
[("retry-after", "0")],
Json(json!({"error": {"message": "quota exhausted", "code": "usage_limit"}})),
)
.into_response();
}
if model == "model/classifier" {
let mut pending = HashSet::new();
for message in body["messages"].as_array().into_iter().flatten() {
for call in message["tool_calls"].as_array().into_iter().flatten() {
if let Some(id) = call["id"].as_str() {
pending.insert(id);
}
}
if message["role"] == "tool"
&& !pending.remove(message["tool_call_id"].as_str().unwrap_or_default())
{
return (
StatusCode::BAD_REQUEST,
Json(json!({"error": {
"message": "tool result has no preceding assistant tool call"
}})),
)
.into_response();
}
}
}
if prompt == "retry-once"
&& calls
.lock()
.await
.iter()
.filter(|call| user_prompt(call) == "retry-once")
.count()
== 1
{
return (
StatusCode::SERVICE_UNAVAILABLE,
[("retry-after", "0")],
Json(json!({"error": {"message": "upstream is temporarily unavailable"}})),
)
.into_response();
}
if (model == "model/weak" && prompt == "unavailable") || prompt == "all-unavailable" {
return (
StatusCode::SERVICE_UNAVAILABLE,
Json(json!({"error": {"message": "upstream is unavailable"}})),
)
.into_response();
}
if model == "model/weak" && prompt == "overflow" {
return (
StatusCode::BAD_REQUEST,
Json(json!({
"error": {
"code": "context_length_exceeded",
"message": "request exceeds this model's context window"
}
})),
)
.into_response();
}
if prompt == "upstream-headers" {
// Both the buffered and the streamed reply echo the same set, so the two
// capture paths are compared against one expectation.
const UPSTREAM_HEADER_ECHO: [(&str, &str); 5] = [
("x-upstream-trace", "trace-123"),
("x-request-id", "req-42"),
("request-id", "req_anthropic_42"),
("x-model-router-selected-model", "model/upstream-echo"),
("x-switchyard-session-id", "spoofed-by-upstream"),
];
// Streaming captures the headers off the response head, before any body
// arrives, so the streamed variant exercises a different capture branch.
let mut response = if body["stream"].as_bool() == Some(true) {
let events = [
json!({"id": "chatcmpl-headers", "model": model, "choices": [{"index": 0, "delta": {"role": "assistant"}}]}).to_string(),
json!({"id": "chatcmpl-headers", "model": model, "choices": [{"index": 0, "delta": {"content": "ok"}}]}).to_string(),
json!({"id": "chatcmpl-headers", "model": model, "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}], "usage": {"prompt_tokens": 10, "completion_tokens": 2, "total_tokens": 12}}).to_string(),
"[DONE]".to_string(),
];
let stream = futures_util::stream::iter(
events
.into_iter()
.map(|data| Ok::<Event, Infallible>(Event::default().data(data))),
);
let mut response = Sse::new(stream).into_response();
let headers = response.headers_mut();
for (name, value) in UPSTREAM_HEADER_ECHO {
headers.append(name, HeaderValue::from_static(value));
}
response
} else {
(
UPSTREAM_HEADER_ECHO,
Json(json!({
"id": "chatcmpl-test",
"object": "chat.completion",
"model": model,
"choices": [{
"index": 0,
"message": {"role": "assistant", "content": "ok"},
"finish_reason": "stop"
}],
"usage": {"prompt_tokens": 10, "completion_tokens": 2, "total_tokens": 12}
})),
)
.into_response()
};
let headers = response.headers_mut();
headers.append("x-upstream-trace", HeaderValue::from_static("trace-456"));
headers.append(
"link",
HeaderValue::from_static("<https://example.test/next>; rel=next"),
);
headers.append(
"link",
HeaderValue::from_static("<https://example.test/prev>; rel=prev"),
);
headers.append(
"set-cookie",
HeaderValue::from_static("session=upstream; HttpOnly"),
);
return response;
}
if body["stream"].as_bool() == Some(true) {
// Streamed tool call, for the namespace-on-every-event assertions. The
// model calls a tool by the name it was given, so echo that name back.
if prompt == "mcp-tool-call" {
let called = body["tool_choice"]["function"]["name"]
.as_str()
.or_else(|| body["tools"][0]["function"]["name"].as_str())
.unwrap_or("search")
.to_string();
let events = [
json!({"id": "chatcmpl-mcp", "model": model, "choices": [{"index": 0, "delta": {"role": "assistant", "tool_calls": [{"index": 0, "id": "call_1", "type": "function", "function": {"name": called, "arguments": ""}}]}}]}).to_string(),
json!({"id": "chatcmpl-mcp", "model": model, "choices": [{"index": 0, "delta": {"tool_calls": [{"index": 0, "function": {"arguments": "{\"q\":\"rust\"}"}}]}}]}).to_string(),
json!({"id": "chatcmpl-mcp", "model": model, "choices": [{"index": 0, "delta": {}, "finish_reason": "tool_calls"}], "usage": {"prompt_tokens": 4, "completion_tokens": 3, "total_tokens": 7}}).to_string(),
"[DONE]".to_string(),
];
let stream = futures_util::stream::iter(
events
.into_iter()
.map(|data| Ok::<Event, Infallible>(Event::default().data(data))),
);
return Sse::new(stream).into_response();
}
if prompt == "stream-error" {
let events = [
json!({"id": "chatcmpl-stream-error", "model": model, "choices": [{"index": 0, "delta": {"role": "assistant"}}]}).to_string(),
json!({"id": "chatcmpl-stream-error", "model": model, "choices": [{"index": 0, "delta": {"content": "before"}}]}).to_string(),
json!({"id": "chatcmpl-stream-error", "model": model, "choices": [{"index": 0, "delta": {"content": "still here"}}], "usage": {"prompt_tokens": 6, "completion_tokens": 2, "total_tokens": 8}}).to_string(),
json!({"error": {"message": "upstream stream failed", "type": "server_error"}}).to_string(),
];
let stream = futures_util::stream::iter(
events
.into_iter()
.map(|data| Ok::<Event, Infallible>(Event::default().data(data))),
);
return Sse::new(stream).into_response();
}
let events = [
json!({"id": "chatcmpl-stream", "model": model, "choices": [{"index": 0, "delta": {"role": "assistant"}}]}).to_string(),
json!({"id": "chatcmpl-stream", "model": model, "choices": [{"index": 0, "delta": {"content": "hello"}}]}).to_string(),
json!({"id": "chatcmpl-stream", "model": model, "choices": [{"index": 0, "delta": {"content": "-partial"}}], "usage": {"prompt_tokens": 5, "completion_tokens": 1, "total_tokens": 6, "prompt_tokens_details": {"cached_tokens": 2, "cache_creation_tokens": 1}}}).to_string(),
json!({"id": "chatcmpl-stream", "model": model, "choices": [{"index": 0, "delta": {"content": "-final"}}], "usage": {"prompt_tokens": 12, "completion_tokens": 5, "total_tokens": 17, "prompt_tokens_details": {"cached_tokens": 7, "cache_creation_tokens": 2}, "completion_tokens_details": {"reasoning_tokens": 3}}}).to_string(),
json!({"id": "chatcmpl-stream", "model": model, "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}]}).to_string(),
"[DONE]".to_string(),
];
let stream = futures_util::stream::iter(
events
.into_iter()
.map(|data| Ok::<Event, Infallible>(Event::default().data(data))),
);
return Sse::new(stream).into_response();
}
if model == "model/advisor" {
// The review consult carries the serialized transcript in its user
// message, so the original prompt text rides inside it: tests script
// the verdict (or an outage) from the prompt they send.
let haystack = body["messages"].to_string();
if haystack.contains("advisor-down") {
return (
StatusCode::SERVICE_UNAVAILABLE,
Json(json!({"error": {"message": "advisor is unavailable"}})),
)
.into_response();
}
let verdict = if haystack.contains("please-redo") {
"REDO run the tests"
} else {
"APPROVE"
};
return Json(json!({
"id": "chatcmpl-advisor",
"object": "chat.completion",
"model": model,
"choices": [{
"index": 0,
"message": {"role": "assistant", "content": verdict},
"finish_reason": "stop"
}],
"usage": {"prompt_tokens": 40, "completion_tokens": 4, "total_tokens": 44}
}))
.into_response();
}
// A generation that did not complete (`abort` from vLLM or SGLang, `error` from OpenRouter,
// `repetition` from vLLM).
if let Some(reason @ ("abort" | "error" | "repetition")) = model.strip_prefix("model/finish-") {
return Json(json!({
"id": "chatcmpl-finish",
"object": "chat.completion",
"model": model,
"choices": [{
"index": 0,
"message": {"role": "assistant", "content": "partial"},
"finish_reason": reason
}],
"usage": {"prompt_tokens": 3, "completion_tokens": 1, "total_tokens": 4}
}))
.into_response();
}
// Buffered tool call, the non-streaming counterpart of the branch above.
if prompt == "mcp-tool-call" {
let called = body["tool_choice"]["function"]["name"]
.as_str()
.or_else(|| body["tools"][0]["function"]["name"].as_str())
.unwrap_or("search")
.to_string();
return Json(json!({
"id": "chatcmpl-mcp",
"object": "chat.completion",
"model": model,
"choices": [{
"index": 0,
"message": {
"role": "assistant",
"content": null,
"tool_calls": [{
"id": "call_1",
"type": "function",
"function": {"name": called, "arguments": "{\"q\":\"rust\"}"}
}]
},
"finish_reason": "tool_calls"
}],
"usage": {"prompt_tokens": 4, "completion_tokens": 3, "total_tokens": 7}
}))
.into_response();
}
let custom_target_schema = body
.pointer("/response_format/json_schema/schema/properties/decision/properties/target")
.is_some();
let requests_invalid_verdict = body["messages"].as_array().is_some_and(|messages| {
messages.iter().any(|message| {
message["content"]
.as_str()
.is_some_and(|content| content.contains("invalid verdict"))
})
});
let requests_schema_invalid_verdict = body["messages"].as_array().is_some_and(|messages| {
messages.iter().any(|message| {
message["content"]
.as_str()
.is_some_and(|content| content.contains("schema-invalid verdict"))
})
});
// A custom-mode task may name the group it wants the judge to pick, so one
// config can be driven through each of its groups in turn.
let requested_group = body["messages"].as_array().and_then(|messages| {
messages.iter().find_map(|message| {
message["content"]
.as_str()?
.split_once("route to ")
.map(|(_, group)| group.trim().to_string())
})
});
let content = if model == "model/classifier" && custom_target_schema {
if requests_invalid_verdict {
r#"{"decision":{"target":"unknown"}}"#.to_string()
} else {
let group = requested_group.unwrap_or_else(|| "efficient".to_string());
format!(r#"{{"decision":{{"target":"{group}"}}}}"#)
}
} else if body
.pointer("/response_format/json_schema/schema/properties/escalate")
.is_some()
{
r#"{"escalate":false,"category":"none","new_evidence":false,"reason":"making progress"}"#
.to_string()
} else if model == "model/classifier" && requests_schema_invalid_verdict {
r#"{"crux":"bounded task","primary_rule":"SUP-1","capability_boundary":"supported","p_solve":0.1,"unexpected":true}"#.to_string()
} else if model == "model/classifier" {
r#"{"crux":"bounded task","primary_rule":"SUP-1","capability_boundary":"supported","p_solve":0.9}"#.to_string()
} else if model == "model/vgr-judge" {
"yes".to_string()
} else if model == "model/vgr-reject" {
"no".to_string()
} else {
"ok".to_string()
};
let logprobs = (body["logprobs"] == true).then(|| {
let yes = if content == "yes" { 0.9_f64 } else { 0.1_f64 };
json!({"content": [{"token": content, "logprob": 0.0, "top_logprobs": [
{"token": "yes", "logprob": yes.ln()},
{"token": "no", "logprob": (1.0 - yes).ln()}
]}]})
});
Json(json!({
"id": "chatcmpl-test",
"object": "chat.completion",
"model": model,
"choices": [{
"index": 0,
"message": {"role": "assistant", "content": content},
"logprobs": logprobs,
"finish_reason": "stop"
}],
"usage": {
"prompt_tokens": 10,
"completion_tokens": 2,
"total_tokens": 12,
"prompt_tokens_details": {"cached_tokens": 7}
}
}))
.into_response()
}
async fn upstream_messages_requires_forwarded_oauth(
State(calls): State<Arc<Mutex<Vec<Value>>>>,
headers: HeaderMap,
Json(body): Json<Value>,
) -> HttpResponse {
calls.lock().await.push(body.clone());
// The diagnostics fixture intentionally has no OAuth credentials.
let has_expected_headers = body["model"] == "model/anthropic-diagnostics"
|| (headers
.get("authorization")
.and_then(|value| value.to_str().ok())
== Some("Bearer claude-oauth-token")
&& headers
.get("anthropic-beta")
.and_then(|value| value.to_str().ok())
== Some("oauth-2025-04-20")
&& headers
.get("anthropic-version")
.and_then(|value| value.to_str().ok())
== Some("2023-06-01")
&& headers
.get("chatgpt-account-id")
.and_then(|value| value.to_str().ok())
== Some("account-123")
&& headers
.get("x-openai-fedramp")
.and_then(|value| value.to_str().ok())
== Some("true"));
if !has_expected_headers {
return (
StatusCode::UNAUTHORIZED,
Json(json!({"error": {"message": "missing forwarded Anthropic OAuth headers"}})),
)
.into_response();
}
Json(json!({
"id": "msg_test",
"type": "message",
"role": "assistant",
"model": body["model"],
"content": [{"type": "text", "text": "ok"}],
"stop_reason": "end_turn",
"stop_sequence": null,
"usage": {"input_tokens": 1, "output_tokens": 1}
}))
.into_response()
}
async fn upstream_responses_requires_forwarded_auth(
State(calls): State<Arc<Mutex<Vec<Value>>>>,
headers: HeaderMap,
Json(body): Json<Value>,
) -> HttpResponse {
calls.lock().await.push(body.clone());
if headers.contains_key("x-test-redirect") {
return (StatusCode::TEMPORARY_REDIRECT, [("location", "/capture")]).into_response();
}
if headers.contains_key("x-test-echo-auth") {
let authorization = headers
.get("authorization")
.and_then(|value| value.to_str().ok())
.unwrap_or_default();
return (
StatusCode::UNAUTHORIZED,
Json(json!({"error": {"message": authorization}})),
)
.into_response();
}
let has_expected_headers = headers
.get("authorization")
.and_then(|value| value.to_str().ok())
== Some("Bearer codex-login-token")
&& headers
.get("chatgpt-account-id")
.and_then(|value| value.to_str().ok())
== Some("account-123")
&& headers
.get("x-openai-fedramp")
.and_then(|value| value.to_str().ok())
== Some("true")
&& headers
.get("x-api-key")
.and_then(|value| value.to_str().ok())
== Some("provider-api-key")
&& headers
.get("anthropic-beta")
.and_then(|value| value.to_str().ok())
== Some("provider-beta");
if !has_expected_headers {
return (
StatusCode::UNAUTHORIZED,
Json(json!({"error": {"message": "missing forwarded OpenAI login"}})),
)
.into_response();
}
Json(json!({
"id": "resp_test",
"object": "response",
"model": body["model"],
"status": "completed",
"output": [{
"type": "message",
"role": "assistant",
"content": [{"type": "output_text", "text": "ok"}]
}],
"usage": {"input_tokens": 1, "output_tokens": 1, "total_tokens": 2}
}))
.into_response()
}
async fn upstream_buffered_responses(
State(calls): State<Arc<Mutex<Vec<Value>>>>,
headers: HeaderMap,
Json(body): Json<Value>,
) -> Json<Value> {
calls.lock().await.push(body.clone());
let model = body["model"].as_str().unwrap_or_default();
let mut response = json!({
"id": "resp_buffered", "object": "response", "model": model,
"status": "failed", "output": [], "usage": null,
"error": {"code": "server_error", "message": "deterministic upstream failure"}
});
match model {
"model/missing-error" => response = json!({"status": "failed"}),
"model/invalid-error" => response["error"] = json!("invalid error details"),
"model/invalid-code" => response["error"]["code"] = json!(42),
"model/empty-code" => response["error"]["code"] = json!(""),
"model/fallback" | "model/efficient" | "openai/gpt-5" => {
response["status"] = json!("completed");
response["error"] = Value::Null;
response["output"] = json!([{
"type": "message", "role": "assistant",
"content": [{"type": "output_text", "text": "ok"}]
}]);
}
_ => {}
}
if let Some(secret) = headers.get("x-private-token") {
response["error"]["code"] = json!(secret.to_str().unwrap_or_default());
response["error"]["message"] = json!(format!(
"provider rejected {}",
secret.to_str().unwrap_or_default()
));
}
Json(response)
}
/// A local Responses provider whose IDs normally contain its name. It returns `state_not_found`
/// for another provider's ID, except `resp_shared_conflict`, which tests duplicate ownership.
/// The judge selects the strong tier when the input contains `ROUTE_B`.
async fn upstream_responses_silo(
State(calls): State<Arc<Mutex<Vec<Value>>>>,
Path(silo): Path<String>,
Json(body): Json<Value>,
) -> HttpResponse {
let mut calls = calls.lock().await;
calls.push(body.clone());
let model = body["model"].as_str().unwrap_or_default().to_string();
if model == "model/judge" {
let strong = body.to_string().contains("ROUTE_B");
let verdict = if body["text"]["format"]["schema"]["properties"]
.get("escalate")
.is_some()
{
json!({
"escalate": strong,
"category": if strong { "capability_gap" } else { "none" },
"new_evidence": strong,
"reason": "state probe",
})
} else {
json!({
"crux": "state probe", "primary_rule": if strong { "LIM-1" } else { "SUP-1" },
"capability_boundary": if strong { "unsupported" } else { "supported" },
"p_solve": if strong { 0.1 } else { 0.9 },
})
};
return Json(responses_body("resp_judge", &model, &verdict.to_string())).into_response();
}
let minted_prefix = format!("resp_{silo}_");
if silo == "A" && body.to_string().contains("owner-unavailable") {
return (
StatusCode::SERVICE_UNAVAILABLE,
Json(json!({"error": {"message": "provider A is unavailable"}})),
)
.into_response();
}
if let Some(previous) = body["previous_response_id"].as_str()
&& !previous.starts_with(&minted_prefix)
&& previous != "resp_shared_conflict"
{
return (
StatusCode::BAD_REQUEST,
Json(
json!({"error": {"code": "state_not_found", "message": format!(
"previous_response_id {previous} is not present in provider {silo}"
)}}),
),
)
.into_response();
}
let id = if body.to_string().contains("state-id-conflict") {
"resp_shared_conflict".to_string()
} else {
format!("{minted_prefix}{}", calls.len())
};
let text = format!("served-by:{silo}");
let mut response = responses_body(&id, &model, &text);
response["store"] = body.get("store").cloned().unwrap_or(json!(true));
response["conversation"] = body.get("conversation").cloned().unwrap_or(Value::Null);
if body["stream"] == true {
let mut created = response.clone();
created["status"] = json!("in_progress");
created["output"] = json!([]);
let events = [
json!({"type": "response.created", "response": created}),
json!({"type": "response.output_text.delta", "output_index": 0, "delta": text}),
json!({"type": "response.completed", "response": response}),
];
let stream = futures_util::stream::iter(events.into_iter().map(|event| {
Ok::<Event, Infallible>(
Event::default()
.event(event["type"].as_str().unwrap_or_default())
.data(event.to_string()),
)
}));
return Sse::new(stream).into_response();
}
Json(response).into_response()
}
fn responses_body(id: &str, model: &str, text: &str) -> Value {
json!({
"id": id, "object": "response", "created_at": 1, "model": model, "status": "completed",
"output": [{"id": "msg_1", "type": "message", "status": "completed", "role": "assistant",
"content": [{"type": "output_text", "text": text, "annotations": []}]}],
"usage": {"input_tokens": 1, "output_tokens": 1, "total_tokens": 2,
"output_tokens_details": {"reasoning_tokens": 0}}
})
}
async fn upstream_redirect_capture(
State(calls): State<Arc<Mutex<Vec<Value>>>>,
headers: HeaderMap,
) -> HttpResponse {
calls.lock().await.push(json!({
"redirected": true,
"has_authorization": headers.contains_key("authorization")
}));
StatusCode::OK.into_response()
}
async fn upstream_count_tokens(
State(calls): State<Arc<Mutex<Vec<Value>>>>,
Json(body): Json<Value>,
) -> HttpResponse {
calls.lock().await.push(body.clone());
Json(json!({"input_tokens": 7})).into_response()
}
async fn upstream_responses_auxiliary(
State(calls): State<Arc<Mutex<Vec<Value>>>>,
uri: Uri,
headers: HeaderMap,
Json(body): Json<Value>,
) -> HttpResponse {
calls.lock().await.push(json!({
"path": uri.path(),
"body": body,
"configured_header": headers.get("x-configured-client").and_then(|value| value.to_str().ok())
}));
if uri.path().ends_with("/input_tokens") {
Json(json!({"input_tokens": 11})).into_response()
} else {
Json(json!({"id": "resp_compacted", "object": "response", "output": []})).into_response()
}
}
async fn upstream_fallback(
State(calls): State<Arc<Mutex<Vec<Value>>>>,
headers: HeaderMap,
Json(body): Json<Value>,
) -> HttpResponse {
calls.lock().await.push(json!({
"body": body,
"authorization": headers.get("authorization").and_then(|value| value.to_str().ok()),
"end_to_end": headers.get("x-end-to-end").and_then(|value| value.to_str().ok()),
"configured_secret": headers.contains_key("x-configured-secret"),
"connection": headers.contains_key("connection"),
"connection_nominated": headers.contains_key("x-remove-me")
}));
let mut response = Json(json!({"input_tokens": 7})).into_response();
response
.headers_mut()
.insert("connection", HeaderValue::from_static("x-upstream-hop"));
response
.headers_mut()
.insert("x-upstream-hop", HeaderValue::from_static("remove"));
response.headers_mut().insert(
"x-end-to-end-response",
HeaderValue::from_static("preserve"),
);
response
}
fn random_state(base_url: &str, routes: &[(&str, &[&str])]) -> TestResult<ServerState> {
random_state_with_retries(base_url, routes, 0)
}
fn random_state_with_retries(
base_url: &str,
routes: &[(&str, &[&str])],
max_retries: u32,
) -> TestResult<ServerState> {
let backend = Backend::OpenAiChat(HttpBackendConfig {
base_url: base_url.to_string(),
api_key: Some("test-key".to_string()),
forward_auth: false,
extra_headers: BTreeMap::new(),
extra_body: BTreeMap::new(),
omit_body_fields: BTreeSet::new(),
reasoning_effort: None,
max_retries,
failure_cooldown: std::time::Duration::ZERO,
timeout: None,
});
let target_models = routes
.iter()
.flat_map(|(_, targets)| targets.iter().copied())
.collect::<HashSet<_>>();
let model_configs = target_models
.into_iter()
.map(|model| ModelConfig::new(model, backend.clone(), None))
.collect::<Vec<_>>();
let client: Arc<dyn RoutedLlmClient> = Arc::new(TranslatingLlmClient::new(&model_configs)?);
let entries = routes
.iter()
.map(|(route_model, targets)| {
let algorithm: Arc<dyn Algorithm> = Arc::new(Random::new(None, None)?);
let decision_targets = targets
.iter()
.map(|model| DecisionTarget {
target: (*model).to_string(),
model: ModelId::from(*model),
format: WireFormat::OpenAiChat,
base_url: base_url.to_string(),
extra_body: BTreeMap::new(),
reasoning_effort: None,
})
.collect();
Ok((
ModelId::from(*route_model),
Route::new(
algorithm,
ClientRouter::single(Arc::clone(&client)),
None,
ModelCapabilities::default(),
None,
None,
decision_targets,
RuntimeModels::new(
[(
Category::Any,
targets.iter().map(|model| ModelId::from(*model)).collect(),
)]
.into(),
),
),
))
})
.collect::<TestResult<Vec<_>>>()?;
ServerState::from_runner(Runner::new(entries)).map_err(Into::into)
}
async fn test_app(routes: &[(&str, &[&str])]) -> TestResult<(MockUpstream, Router)> {
let upstream = MockUpstream::start().await?;
let app = build_switchyard_router(random_state(&upstream.base_url, routes)?);
Ok((upstream, app))
}
// Embedders expose only the three primary inference endpoints and own every other route.
#[tokio::test]
async fn llm_router_exposes_only_primary_llm_endpoints() -> TestResult {
let state = random_state("http://127.0.0.1:1/v1", &[(ROUTE_MODEL, &["model/weak"])])?;
let app = build_llm_router(state);
for path in ["/v1/chat/completions", "/v1/messages", "/v1/responses"] {
assert_eq!(
send(&app, "GET", path, None).await?.status,
StatusCode::METHOD_NOT_ALLOWED,
"{path} should be registered"
);
}
for path in [
"/v1/decision",
"/v1/messages/count_tokens",
"/v1/responses/input_tokens",
"/v1/responses/compact",
"/v1/models",
"/v1/stats",
"/v1/stats/reset",
"/v1/routing/session-stats",
"/metrics",
"/health",
"/future/provider/endpoint",
] {
assert_eq!(
send(&app, "POST", path, None).await?.status,
StatusCode::NOT_FOUND,
"{path} should not be registered"
);
}
Ok(())
}
fn empty_token_totals() -> Value {
json!({
"prompt": 0,
"completion": 0,
"cached": 0,
"cache_creation": 0,
"reasoning": 0,
"total": 0
})
}
#[tokio::test]
async fn stats_exposes_the_exact_empty_schema_and_no_legacy_alias() -> TestResult {
let (_upstream, app) = test_app(&[(ROUTE_MODEL, &["model/a"])]).await?;
let response = send(&app, "GET", "/v1/stats", None).await?;
assert_eq!(response.status, StatusCode::OK);
assert_eq!(
response.json()?,
json!({
"total_requests": 0,
"total_errors": 0,
"total_tokens": empty_token_totals(),
"models": {},
"targets": {},
"routing_overhead": {
"count": 0,
"total_ms": 0.0,
"min_ms": 0.0,
"max_ms": 0.0,
"avg_ms": 0.0,
"p50_ms": 0.0,
"p99_ms": 0.0
},
"routing_fallbacks": {
"context_window": 0,
"unavailable": 0
},
"classifier": {
"total_requests": 0,
"total_errors": 0,
"total_tokens": empty_token_totals(),
"models": {},
"targets": {},
},
"algorithm_stats": {},
})
);
assert_eq!(
send(&app, "GET", "/v1/routing/stats", None).await?.status,
StatusCode::NOT_FOUND
);
Ok(())
}
#[tokio::test]
async fn stats_accumulates_buffered_success_error_and_shared_routes() -> TestResult {
let (_upstream, app) = test_app(&[
("switchyard/one", &["gemini-3.5-flash"]),
("switchyard/two", &["model/unknown"]),
])
.await?;
for route in ["switchyard/one", "switchyard/two"] {
assert_eq!(
send(
&app,
"POST",
"/v1/chat/completions",
Some(json!({
"model": route,
"messages": [{"role": "user", "content": "hello"}]
})),
)
.await?
.status,
StatusCode::OK
);
}
assert_eq!(
send(
&app,
"POST",
"/v1/chat/completions",
Some(json!({
"model": "switchyard/one",
"messages": [{"role": "user", "content": "fail"}]
})),
)
.await?
.status,
StatusCode::IM_A_TEAPOT
);
let stats = send(&app, "GET", "/v1/stats", None).await?.json()?;
assert_eq!(stats["total_requests"], 3);
assert_eq!(stats["total_errors"], 1);
assert_eq!(
stats["total_tokens"],
json!({
"prompt": 20,
"completion": 4,
"cached": 14,
"cache_creation": 0,
"reasoning": 0,
"total": 24
})
);
assert_eq!(stats["models"]["gemini-3.5-flash"]["calls"], 1);
assert_eq!(stats["models"]["gemini-3.5-flash"]["errors"], 1);
assert_eq!(stats["models"]["model/unknown"]["calls"], 1);
assert_eq!(stats["routing_overhead"]["count"], 3);
Ok(())
}
fn buffered_responses_app(
upstream: &MockUpstream,
model: &str,
fallback: bool,
forward_auth: bool,
) -> TestResult<Router> {
let route = if fallback {
"type = \"random\"\ntargets = [\"first\", \"second\"]\nweights = [1000, 1]\nseed = 17"
} else {
"type = \"passthrough\"\ntarget = \"first\""
};
let state = load_test_config(&format!(
r#"
schema_version = 1
[llm_clients.mock]
format = "openai_responses"
base_url = "{base_url}/buffered"
forward_auth = {forward_auth}
max_retries = 0
[targets]
first = {{ id = "{model}", llm_client = "mock" }}
second = {{ id = "model/fallback", llm_client = "mock" }}
[routes.response]
id = "{ROUTE_MODEL}"
{route}
"#,
base_url = upstream.base_url.trim_end_matches("/v1"),
))?;
Ok(build_switchyard_router(state))
}
#[tokio::test]
async fn caller_metadata_cannot_replace_upstream_request() -> TestResult {
let upstream = MockUpstream::start().await?;
let app = buffered_responses_app(&upstream, "model/fallback", false, false)?;
for (path, mut body) in [
(
"/v1/chat/completions",
json!({
"model": ROUTE_MODEL,
"messages": [{"role": "user", "content": "visible request"}],
"tool_choice": "none", "max_tokens": 16
}),
),
(
"/v1/messages",
json!({
"model": ROUTE_MODEL,
"messages": [{"role": "user", "content": "visible request"}],
"max_tokens": 16
}),
),
(
"/v1/responses",
json!({
"model": ROUTE_MODEL, "input": "visible request",
"tool_choice": "none", "max_output_tokens": 16
}),
),
] {
body["metadata"] = json!({"audit_label": "caller metadata"});
let clean = send(&app, "POST", path, Some(body.clone())).await?;
assert_eq!(clean.status, StatusCode::OK, "{path}");
body["metadata"]["_switchyard_translation"] = json!({
"requests": {
"openai_responses": {
"model": "attacker/model",
"input": "hidden request",
"instructions": "attacker instructions",
"max_output_tokens": 4096,
"tools": [{
"type": "function", "name": "dangerous_action",
"parameters": {"type": "object", "properties": {}}
}],
"tool_choice": {"type": "function", "name": "dangerous_action"},
"store": true
}
},
"responses": {}
});
let injected = send(&app, "POST", path, Some(body)).await?;
assert_eq!(injected.status, StatusCode::OK, "{path}");
let mut calls = upstream.calls.lock().await;
assert_eq!(calls.len(), 2, "{path}");
assert_eq!(calls[0]["model"], "model/fallback");
assert_eq!(calls[0]["max_output_tokens"], 16);
assert!(calls[0]["input"].to_string().contains("visible request"));
assert_eq!(calls[0]["metadata"]["audit_label"], "caller metadata");
assert_eq!(
calls[1], calls[0],
"caller metadata changed upstream body: {path}"
);
calls.clear();
}
Ok(())
}
#[tokio::test]
async fn failed_responses_return_errors_and_try_fallback_across_endpoints() -> TestResult {
let upstream = MockUpstream::start().await?;
let requests = [
(
"/v1/chat/completions",
json!({
"model": ROUTE_MODEL, "messages": [{"role": "user", "content": "hello"}]
}),
),
(
"/v1/messages",
json!({
"model": ROUTE_MODEL, "max_tokens": 16,
"messages": [{"role": "user", "content": "hello"}]
}),
),
(
"/v1/responses",
json!({"model": ROUTE_MODEL, "input": "hello"}),
),
];
let failure = "deterministic upstream failure";
let missing = "provider reported status \"failed\" without error details";
for (model, message, code) in [
("model/failed", failure, "server_error"),
("model/missing-error", missing, "upstream_error"),
("model/invalid-error", missing, "upstream_error"),
("model/invalid-code", failure, "upstream_error"),
("model/empty-code", failure, "upstream_error"),
] {
let app = buffered_responses_app(&upstream, model, false, false)?;
for (path, body) in &requests {
let response = send(&app, "POST", path, Some(body.clone())).await?;
assert_eq!(response.status, StatusCode::BAD_GATEWAY, "{model}: {path}");
let expected = if *path == "/v1/messages" {
json!({"type": "error", "error": {"type": "api_error", "message": message}})
} else {
json!({"error": {"type": "upstream_error", "code": code, "message": message}})
};
assert_eq!(response.json()?, expected, "{model}: {path}");
}
let stats = send(&app, "GET", "/v1/stats", None).await?.json()?;
assert_eq!(stats["total_requests"], requests.len(), "{model}");
assert_eq!(stats["total_errors"], requests.len(), "{model}");
assert_eq!(stats["models"][model]["errors"], requests.len(), "{model}");
}
let app = buffered_responses_app(&upstream, "model/failed", false, true)?;
let (path, body) = &requests[0];
let response = send_with_headers(
&app,
"POST",
path,
Some(body.clone()),
&[("x-private-token", "test-private-credential")],
)
.await?;
assert_eq!(response.status, StatusCode::BAD_GATEWAY);
assert_eq!(
response.json()?["error"]["message"],
"provider rejected [REDACTED]"
);
assert_eq!(response.json()?["error"]["code"], "[REDACTED]");
let app = buffered_responses_app(&upstream, "model/failed", true, false)?;
for (path, body) in &requests {
let previous_calls = upstream.models().await.len();
let response = send(&app, "POST", path, Some(body.clone())).await?;
assert_eq!(response.status, StatusCode::OK, "{path}");
assert_eq!(response.json()?["model"], "model/fallback", "{path}");
assert_eq!(
&upstream.models().await[previous_calls..],
["model/failed", "model/fallback"],
"{path}"
);
}
let stats = send(&app, "GET", "/v1/stats", None).await?.json()?;
assert_eq!(stats["models"]["model/failed"]["errors"], requests.len());
assert_eq!(stats["targets"]["first"]["errors"], requests.len());
assert_eq!(stats["models"]["model/fallback"]["calls"], requests.len());
assert_eq!(stats["targets"]["second"]["calls"], requests.len());
Ok(())
}
// A Chat backend that ends a generation with `abort`, `error` or `repetition` did
// not complete it: every client path gets a 502, and a fallback route moves on.
#[tokio::test]
async fn chat_failed_finish_reasons_return_errors_and_try_fallback() -> TestResult {
let upstream = MockUpstream::start().await?;
let requests = [
(
"/v1/chat/completions",
json!({
"model": ROUTE_MODEL, "messages": [{"role": "user", "content": "hello"}]
}),
),
(
"/v1/messages",
json!({
"model": ROUTE_MODEL, "max_tokens": 16,
"messages": [{"role": "user", "content": "hello"}]
}),
),
(
"/v1/responses",
json!({"model": ROUTE_MODEL, "input": "hello"}),
),
];
for reason in ["abort", "error", "repetition"] {
let model = format!("model/finish-{reason}");
let message = format!("provider finished with finish_reason \"{reason}\"");
for fallback in [false, true] {
let route = if fallback {
"type = \"random\"\ntargets = [\"first\", \"second\"]\nweights = [1000, 1]\nseed = 17"
} else {
"type = \"passthrough\"\ntarget = \"first\""
};
let app = build_switchyard_router(load_test_config(&format!(
r#"
schema_version = 1
[llm_clients.mock]
format = "openai_chat"
base_url = "{base_url}"
max_retries = 0
[targets]
first = {{ id = "{model}", llm_client = "mock" }}
second = {{ id = "model/fallback", llm_client = "mock" }}
[routes.chat]
id = "{ROUTE_MODEL}"
{route}
"#,
base_url = upstream.base_url,
))?);
for (path, body) in &requests {
let previous_calls = upstream.models().await.len();
let response = send(&app, "POST", path, Some(body.clone())).await?;
let calls = upstream.models().await[previous_calls..].to_vec();
if fallback {
assert_eq!(response.status, StatusCode::OK, "{model}: {path}");
assert_eq!(
response.json()?["model"],
"model/fallback",
"{model}: {path}"
);
assert_eq!(calls, [model.as_str(), "model/fallback"], "{model}: {path}");
continue;
}
assert_eq!(response.status, StatusCode::BAD_GATEWAY, "{model}: {path}");
let expected = if *path == "/v1/messages" {
json!({"type": "error", "error": {"type": "api_error", "message": message}})
} else {
json!({"error": {
"type": "upstream_error", "code": "upstream_error", "message": message
}})
};
assert_eq!(response.json()?, expected, "{model}: {path}");
assert_eq!(calls, [model.as_str()], "{model}: {path}");
}
}
}
Ok(())
}
#[tokio::test]
async fn stats_reset_returns_confirmation_and_clears_all_stats() -> TestResult {
let (_upstream, app) = test_app(&[(ROUTE_MODEL, &["model/a"])]).await?;
assert_eq!(
send(
&app,
"POST",
"/v1/chat/completions",
Some(json!({
"model": ROUTE_MODEL,
"messages": [{"role": "user", "content": "hello"}]
})),
)
.await?
.status,
StatusCode::OK
);
let reset = send(&app, "POST", "/v1/stats/reset", None).await?;
assert_eq!(reset.status, StatusCode::OK);
assert_eq!(reset.json()?, json!({"status": "reset"}));
let stats = send(&app, "GET", "/v1/stats", None).await?.json()?;
assert_eq!(stats["total_requests"], 0);
assert_eq!(stats["total_errors"], 0);
assert_eq!(stats["total_tokens"], empty_token_totals());
assert_eq!(stats["models"], json!({}));
assert_eq!(stats["routing_overhead"]["count"], 0);
assert_eq!(stats["classifier"]["total_requests"], 0);
assert_eq!(stats["classifier"]["models"], json!({}));
Ok(())
}
#[tokio::test]
async fn metrics_exposes_switchyard_otel_instruments() -> TestResult {
const MODEL: &str = "model/metrics-buffered";
let upstream = MockUpstream::start().await?;
let app = build_switchyard_router(random_state_with_retries(
&upstream.base_url,
&[(ROUTE_MODEL, &[MODEL])],
1,
)?);
let before = send(&app, "GET", "/metrics", None).await?;
assert_eq!(before.status, StatusCode::OK);
assert_eq!(
before
.headers
.get("content-type")
.and_then(|value| value.to_str().ok()),
Some("text/plain; version=0.0.4; charset=utf-8")
);
let seeded = before.text()?;
for expected in [
"# TYPE switchyard_client_responses_total counter",
"switchyard_client_responses_total{outcome=\"ok\",",
"switchyard_client_responses_total{outcome=\"retryable_error\",",
"switchyard_client_responses_total{outcome=\"other_error\",",
"# TYPE switchyard_upstream_attempts_total counter",
"switchyard_upstream_attempts_total{code=\"200\",outcome=\"ok\",",
"switchyard_upstream_attempts_total{code=\"429\",outcome=\"retryable_error\",",
"switchyard_upstream_attempts_total{code=\"500\",outcome=\"retryable_error\",",
"switchyard_upstream_attempts_total{code=\"504\",outcome=\"retryable_error\",",
"switchyard_upstream_attempts_total{code=\"none\",outcome=\"retryable_error\",",
"# TYPE switchyard_router_retry_recovered_total counter",
"switchyard_router_retry_recovered_total{otel_scope_name=\"switchyard\"} 0",
] {
assert!(
seeded.contains(expected),
"missing seeded {expected:?} in metrics:\n{seeded}"
);
}
let response = send(
&app,
"POST",
"/v1/chat/completions",
Some(json!({
"model": ROUTE_MODEL,
"messages": [{"role": "user", "content": "retry-once"}]
})),
)
.await?;
assert_eq!(response.status, StatusCode::OK);
let after = send(&app, "GET", "/metrics", None).await?;
let metrics = after.text()?;
assert_eq!(
metric_delta(
seeded,
metrics,
"switchyard_router_retry_recovered_total",
&[]
),
Some(1.0)
);
for expected in [
"# TYPE switchyard_build_info gauge",
&format!("switchyard_build_info{{version=\"{VERSION}\""),
"# TYPE switchyard_total_requests gauge",
"# TYPE switchyard_total_errors gauge",
"# TYPE switchyard_requests_total counter",
"# TYPE switchyard_model_call_latency_ms histogram",
"switchyard_client_responses_total{outcome=\"ok\",",
"switchyard_upstream_attempts_total{code=\"200\",outcome=\"ok\",",
"# TYPE switchyard_runs_total counter",
"# TYPE switchyard_run_duration_ms histogram",
"# TYPE switchyard_prompt_tokens_total counter",
"# TYPE switchyard_completion_tokens_total counter",
"# TYPE switchyard_cached_tokens_total counter",
"# TYPE switchyard_total_latency_ms histogram",
"# TYPE switchyard_routing_overhead_ms histogram",
"algorithm=\"random\"",
&format!("selected_model=\"{MODEL}\""),
] {
assert!(
metrics.contains(expected),
"missing {expected:?} in metrics:\n{metrics}"
);
}
for (name, expected_delta) in [
("switchyard_prompt_tokens_total", 10.0),
("switchyard_completion_tokens_total", 2.0),
("switchyard_cached_tokens_total", 7.0),
("switchyard_total_latency_ms_count", 1.0),
] {
assert_eq!(
metric_delta(seeded, metrics, name, &[("model", MODEL)]),
Some(expected_delta),
"unexpected delta for {name}"
);
}
// A sub-millisecond boundary exists only because of the server's bucket view.
assert!(
metric_line(
metrics,
"switchyard_routing_overhead_ms_bucket",
&[("algorithm", "random"), ("le", "0.1")]
)
.is_some()
);
for metric in [
"switchyard_model_call_latency_ms_bucket",
"switchyard_total_latency_ms_bucket",
] {
assert!(
metric_line(metrics, metric, &[("model", MODEL), ("le", "300000")]).is_some(),
"missing five-minute bucket for {metric}"
);
}
assert!(
metric_line(
metrics,
"switchyard_cache_creation_tokens_total",
&[("model", MODEL)]
)
.is_none()
);
assert!(
metric_line(
metrics,
"switchyard_reasoning_tokens_total",
&[("model", MODEL)]
)
.is_none()
);
for metric in [
"switchyard_prompt_tokens_total",
"switchyard_completion_tokens_total",
"switchyard_cached_tokens_total",
"switchyard_total_latency_ms_count",
] {
let line = metric_line(metrics, metric, &[("model", MODEL)])
.ok_or_else(|| format!("missing {metric} series for {MODEL}"))?;
assert!(!line.contains("tier="), "unexpected tier label in {line}");
}
Ok(())
}
// A successful cross-provider request must expose any contract weakening in runtime telemetry.
#[tokio::test]
async fn metrics_exposes_lossy_outbound_request_translation() -> TestResult {
let upstream = MockUpstream::start().await?;
let state = load_test_config(&format!(
r#"
schema_version = 1
[llm_clients.anthropic]
format = "anthropic_messages"
base_url = "{base_url}"
max_retries = 0
[targets.anthropic]
id = "model/anthropic-diagnostics"
llm_client = "anthropic"
[routes.diagnostics]
id = "switchyard/diagnostics"
type = "passthrough"
target = "anthropic"
"#,
base_url = upstream.base_url
))?;
let app = build_switchyard_router(state);
let diagnostic_labels = [
("code", "lossy_conversion"),
("format", "anthropic_messages"),
("operation", "request_encode"),
("severity", "warning"),
];
let request = |min_length| {
let answer = match min_length {
Some(min_length) => json!({"type": "string", "minLength": min_length}),
None => json!({"type": "string"}),
};
json!({
"model": "switchyard/diagnostics",
"messages": [{"role": "user", "content": "Return JSON"}],
"response_format": {
"type": "json_schema",
"json_schema": {
"name": "answer",
"strict": true,
"schema": {
"type": "object",
"properties": {"answer": answer},
"required": ["answer"]
}
}
}
})
};
let mut metrics = send(&app, "GET", "/metrics", None)
.await?
.text()?
.to_string();
for (min_length, expected_delta) in [(None, 0.0), (Some(5), 1.0)] {
let response = send(
&app,
"POST",
"/v1/chat/completions",
Some(request(min_length)),
)
.await?;
assert_eq!(response.status, StatusCode::OK);
assert_eq!(response.json()?["choices"][0]["message"]["content"], "ok");
let after = send(&app, "GET", "/metrics", None)
.await?
.text()?
.to_string();
assert_eq!(
metric_delta(
&metrics,
&after,
"switchyard_translation_diagnostics_total",
&diagnostic_labels,
)
.unwrap_or_default(),
expected_delta
);
metrics = after;
}
let calls = upstream.calls.lock().await;
assert_eq!(calls.len(), 2);
assert_eq!(
calls[0].pointer("/output_config/format/schema/properties/answer"),
Some(&json!({"type": "string"}))
);
assert!(
calls[1]
.pointer("/output_config/format/schema/properties/answer/minLength")
.is_none()
);
let diagnostic_line = metric_line(
&metrics,
"switchyard_translation_diagnostics_total",
&diagnostic_labels,
)
.ok_or("missing translation diagnostic metric")?;
for forbidden_label in ["diagnostic=", "path="] {
assert!(
!diagnostic_line.contains(forbidden_label),
"unexpected high-cardinality label in {diagnostic_line}"
);
}
Ok(())
}
#[tokio::test]
async fn accepts_requests_larger_than_the_axum_default_body_limit() -> TestResult {
let (_upstream, app) = test_app(&[(ROUTE_MODEL, &["model/a"])]).await?;
let content = "x".repeat(2 * 1024 * 1024);
let response = send(
&app,
"POST",
"/v1/chat/completions",
Some(json!({
"model": ROUTE_MODEL,
"messages": [{"role": "user", "content": content}]
})),
)
.await?;
assert_eq!(response.status, StatusCode::OK);
Ok(())
}
fn load_test_config(toml: &str) -> TestResult<ServerState> {
let mut config = tempfile::Builder::new()
.prefix("switchyard-server-config-")
.suffix(".toml")
.tempfile()?;
config.write_all(toml.as_bytes())?;
config.flush()?;
Ok(load_server_state(config.path())?)
}
fn weighted_random_state(base_url: &str, weights: [u32; 2]) -> TestResult<ServerState> {
load_test_config(&format!(
r#"
schema_version = 1
[llm_clients.mock]
format = "openai_chat"
base_url = "{base_url}"
max_retries = 0
# Exercise upstream fallback on every request.
failure_cooldown_ms = 0
[targets.first]
id = "{first}"
llm_client = "mock"
system_prompt = "weak answer prompt"
[targets.second]
id = "{second}"
llm_client = "mock"
system_prompt = "strong answer prompt"
[routes.random]
id = "{ROUTE_MODEL}"
type = "random"
targets = ["first", "second"]
weights = {weights:?}
seed = 17
"#,
first = "model/weak",
second = "model/strong",
))
}
async fn send(app: &Router, method: &str, path: &str, body: Option<Value>) -> TestResult<Response> {
send_with_headers(app, method, path, body, &[]).await
}
async fn send_with_headers(
app: &Router,
method: &str,
path: &str,
body: Option<Value>,
headers: &[(&str, &str)],
) -> TestResult<Response> {
let mut builder = HttpRequest::builder().method(method).uri(path);
for (name, value) in headers {
builder = builder.header(*name, *value);
}
let request_body = if let Some(body) = body {
builder = builder.header("content-type", "application/json");
Body::from(serde_json::to_vec(&body)?)
} else {
Body::empty()
};
let response = app.clone().oneshot(builder.body(request_body)?).await?;
let status = response.status();
let headers = response.headers().clone();
let bytes = response.into_body().collect().await?.to_bytes();
Ok(Response {
status,
headers,
bytes,
})
}
async fn send_raw_json(
app: &Router,
path: &str,
body: Vec<u8>,
content_type: Option<&str>,
) -> TestResult<Response> {
let mut builder = HttpRequest::builder().method("POST").uri(path);
if let Some(content_type) = content_type {
builder = builder.header("content-type", content_type);
}
let response = app.clone().oneshot(builder.body(Body::from(body))?).await?;
let status = response.status();
let headers = response.headers().clone();
let bytes = response.into_body().collect().await?.to_bytes();
Ok(Response {
status,
headers,
bytes,
})
}
struct Response {
status: StatusCode,
headers: axum::http::HeaderMap,
bytes: Bytes,
}
impl Response {
fn json(&self) -> TestResult<Value> {
Ok(serde_json::from_slice(&self.bytes)?)
}
fn text(&self) -> TestResult<&str> {
Ok(std::str::from_utf8(&self.bytes)?)
}
}
#[tokio::test]
async fn models_probes_routes_concurrently_and_caches_failed_discovery() -> TestResult {
for stalled in [false, true] {
let calls = Arc::new(Mutex::new(Vec::new()));
let barrier = Arc::new(tokio::sync::Barrier::new(4));
let probe_barrier = Arc::clone(&barrier);
let captured = Arc::clone(&calls);
let upstream_app = Router::new().route(
"/props",
get(move || {
let barrier = Arc::clone(&barrier);
let calls = Arc::clone(&captured);
async move {
calls.lock().await.push(json!({"path": "/props"}));
barrier.wait().await;
if stalled {
std::future::pending::<()>().await;
}
StatusCode::SERVICE_UNAVAILABLE
}
}),
);
let listener = TcpListener::bind("127.0.0.1:0").await?;
let addr = listener.local_addr()?;
let upstream = MockUpstream {
base_url: format!("http://{addr}/v1"),
calls,
task: tokio::spawn(async move {
axum::serve(listener, upstream_app).await.unwrap();
}),
};
let routes: String = (0..3)
.map(|index| {
format!(
r#"
[routes.vgr{index}]
id = "switchyard/vgr{index}"
type = "vgr"
efficient_target = "efficient"
capable_target = "capable"
"#
)
})
.collect();
let app = build_switchyard_router(load_test_config(&format!(
r#"
schema_version = 1
[llm_clients.efficient]
format = "openai_chat"
base_url = "{}"
context_discovery = "llama_cpp"
[targets.efficient]
id = "efficient"
llm_client = "efficient"
[targets.capable]
id = "capable"
llm_client = "efficient"
{routes}
"#,
upstream.base_url
))?);
let (first, concurrent, ()) =
tokio::time::timeout(std::time::Duration::from_secs(3), async {
tokio::join!(
send(&app, "GET", "/v1/models", None),
send(&app, "GET", "/v1/models", None),
async {
probe_barrier.wait().await;
if stalled {
// All probes have reached the server and started their deadlines.
tokio::time::pause();
tokio::time::advance(std::time::Duration::from_secs(2)).await;
tokio::time::resume();
}
}
)
})
.await?;
let models = first?.json()?;
assert_eq!(concurrent?.json()?, models);
assert_eq!(models["data"].as_array().unwrap().len(), 3);
assert!(
models["data"]
.as_array()
.unwrap()
.iter()
.all(|model| model["context_length"].is_null())
);
assert_eq!(send(&app, "GET", "/v1/models", None).await?.json()?, models);
assert_eq!(upstream.calls.lock().await.len(), 3);
}
Ok(())
}
#[tokio::test]
async fn vgr_serves_verified_attempts_on_efficient_tier_and_escalates_rejected_ones() -> TestResult
{
let upstream = MockUpstream::start().await?;
for (discovery, configured, expected_context, probes) in [
("", "context_window = 32768", Some(32_768), false),
("", "", None, false),
("context_discovery = \"llama_cpp\"", "", Some(98_304), true),
(
"context_discovery = \"llama_cpp\"",
"context_window = 32768",
Some(32_768),
false,
),
] {
for (judge, served) in [
("model/vgr-judge", "model/vgr-efficient"),
("model/vgr-reject", "model/vgr-capable"),
] {
upstream.calls.lock().await.clear();
let app = build_switchyard_router(load_test_config(&format!(
r#"
schema_version = 1
[llm_clients.primary]
format = "openai_chat"
base_url = "{}"
{discovery}
[targets.efficient]
id = "model/vgr-efficient"
llm_client = "primary"
[targets.capable]
id = "model/vgr-capable"
llm_client = "primary"
[targets.judge]
id = "{judge}"
llm_client = "primary"
[routes.vgr]
id = "switchyard/vgr"
type = "vgr"
{configured}
efficient_target = "efficient"
capable_target = "capable"
judge_target = "judge"
mode = "active"
active_approval = "prospective-validation-and-canary-approved"
task_typing = false
"#,
upstream.base_url
))?);
let response = send(
&app,
"POST",
"/v1/chat/completions",
Some(json!({
"model": "switchyard/vgr",
"messages": [{"role": "user", "content": "answer the question"}]
})),
)
.await?;
assert_eq!(response.status, StatusCode::OK);
assert_eq!(
response.headers.get("x-model-router-selected-model"),
Some(&HeaderValue::from_str(served)?)
);
let models = send(&app, "GET", "/v1/models", None).await?.json()?;
assert_eq!(models["data"][0]["context_length"], json!(expected_context));
assert_eq!(send(&app, "GET", "/v1/models", None).await?.json()?, models);
assert_eq!(
upstream
.calls
.lock()
.await
.iter()
.filter(|call| call["path"] == "/props")
.count(),
usize::from(probes)
);
}
}
Ok(())
}
#[tokio::test]
async fn vgr_escalates_while_the_efficient_endpoint_is_down() -> TestResult {
let upstream = MockUpstream::start().await?;
let closed = std::net::TcpListener::bind("127.0.0.1:0")?.local_addr()?;
let app = build_switchyard_router(load_test_config(&format!(
r#"
schema_version = 1
[llm_clients.primary]
format = "openai_chat"
base_url = "{}"
[llm_clients.down]
format = "openai_chat"
base_url = "http://{closed}/v1"
[targets.efficient]
id = "model/vgr-efficient"
llm_client = "down"
[targets.capable]
id = "model/vgr-capable"
llm_client = "primary"
[routes.vgr]
id = "switchyard/vgr"
type = "vgr"
efficient_target = "efficient"
capable_target = "capable"
mode = "active"
active_approval = "prospective-validation-and-canary-approved"
task_typing = false
"#,
upstream.base_url
))?);
// The second request meets the failed endpoint's cooldown rather than its socket.
for _ in 0..2 {
let response = send(
&app,
"POST",
"/v1/chat/completions",
Some(json!({
"model": "switchyard/vgr",
"messages": [{"role": "user", "content": "answer the question"}]
})),
)
.await?;
assert_eq!(response.status, StatusCode::OK, "{}", response.text()?);
assert_eq!(
response.headers.get("x-model-router-selected-model"),
Some(&HeaderValue::from_static("model/vgr-capable"))
);
}
Ok(())
}
#[tokio::test]
async fn vgr_keeps_agent_nudges_in_their_session_and_releases_user_turns() -> TestResult {
let upstream = MockUpstream::start().await?;
for session_header in ["x-switchyard-session-id", "x-session-id"] {
let app = build_switchyard_router(load_test_config(&format!(
r#"
schema_version = 1
[llm_clients.primary]
format = "openai_chat"
base_url = "{}"
[targets]
efficient = {{ id = "model/vgr-efficient", llm_client = "primary" }}
capable = {{ id = "model/vgr-capable", llm_client = "primary" }}
judge = {{ id = "model/vgr-judge", llm_client = "primary" }}
[routes.vgr]
id = "switchyard/vgr"
type = "vgr"
efficient_target = "efficient"
capable_target = "capable"
judge_target = "judge"
mode = "evaluate"
task_typing = false
"#,
upstream.base_url
))?);
let initial = json!([{"role": "user", "content": " "}]);
let nudge = json!([
{"role": "user", "content": " "},
{"role": "assistant", "content": "4"},
{"role": "user", "content": "Verify the answer before stopping."}
]);
for (session, initiator, messages, served) in [
("a", Some("user"), &initial, "model/vgr-capable"),
("a", Some("agent"), &nudge, "model/vgr-capable"),
("b", Some("agent"), &nudge, "model/vgr-efficient"),
("a", Some("user"), &nudge, "model/vgr-efficient"),
("a", Some("agent"), &nudge, "model/vgr-efficient"),
("c", None, &initial, "model/vgr-capable"),
("c", None, &nudge, "model/vgr-efficient"),
] {
let mut headers = vec![(session_header, session)];
if let Some(initiator) = initiator {
headers.push(("x-initiator", initiator));
}
let response = send_with_headers(
&app,
"POST",
"/v1/chat/completions",
Some(json!({"model": "switchyard/vgr", "messages": messages})),
&headers,
)
.await?;
assert_eq!(response.status, StatusCode::OK, "{}", response.text()?);
assert_eq!(
response.headers.get("x-model-router-selected-model"),
Some(&HeaderValue::from_str(served)?),
"{session_header}: {session} {initiator:?}"
);
}
}
Ok(())
}
fn metric_line<'a>(metrics: &'a str, name: &str, labels: &[(&str, &str)]) -> Option<&'a str> {
metrics.lines().find(|line| {
line.starts_with(name)
&& labels
.iter()
.all(|(key, value)| line.contains(&format!("{key}=\"{value}\"")))
})
}
fn metric_value(metrics: &str, name: &str, labels: &[(&str, &str)]) -> Option<f64> {
metric_line(metrics, name, labels)?
.split_whitespace()
.last()?
.parse()
.ok()
}
fn metric_delta(before: &str, after: &str, name: &str, labels: &[(&str, &str)]) -> Option<f64> {
metric_value(after, name, labels)
.map(|after| after - metric_value(before, name, labels).unwrap_or_default())
}
fn assert_in_order(haystack: &str, needles: &[&str]) {
let mut remainder = haystack;
for needle in needles {
let offset = remainder
.find(needle)
.unwrap_or_else(|| panic!("missing {needle:?} after prior events in:\n{haystack}"));
remainder = &remainder[offset + needle.len()..];
}
}
/// A route is a synthetic model (`switchyard/classify`) with no upstream of its own; its
/// algorithm picks real targets, and *those* name the client. One request can emit several
/// `Step::CallModel` for different targets, and two targets may sit on different
/// `[llm_clients.*]` sections — here the judge is on one provider and the serving models on
/// another. Pin that each call reaches its own target's upstream, rather than one client
/// chosen per route serving all of them.
#[tokio::test]
async fn each_target_in_one_request_is_served_by_its_own_client() -> TestResult {
let judge_upstream = MockUpstream::start().await?;
let model_upstream = MockUpstream::start().await?;
let state = load_test_config(&format!(
r#"
schema_version = 1
[llm_clients.judge_provider]
format = "openai_chat"
base_url = "{judge_url}"
[llm_clients.model_provider]
format = "openai_chat"
base_url = "{model_url}"
[targets.judge]
id = "model/judge"
llm_client = "judge_provider"
[targets.strong]
id = "model/strong"
llm_client = "model_provider"
[targets.weak]
id = "model/weak"
llm_client = "model_provider"
[routes.classify]
id = "switchyard/classify"
type = "llm_classifier"
classifier_target = "judge"
strong_target = "strong"
weak_target = "weak"
base_threshold = 0.5
"#,
judge_url = judge_upstream.base_url,
model_url = model_upstream.base_url,
))?;
let app = build_switchyard_router(state);
let response = send(
&app,
"POST",
"/v1/chat/completions",
Some(json!({
"model": "switchyard/classify",
"messages": [{"role": "user", "content": "hi"}]
})),
)
.await?;
assert_eq!(response.status, StatusCode::OK);
// The judge call went to the judge's provider and nowhere else; the serving call went to
// the models' provider. A single per-route client would have sent both to one upstream.
assert_eq!(
judge_upstream.models().await,
vec!["model/judge".to_string()]
);
let served = model_upstream.models().await;
assert_eq!(
served.len(),
1,
"expected exactly one routed call: {served:?}"
);
assert!(
served[0] == "model/weak" || served[0] == "model/strong",
"routed call went to {served:?}"
);
Ok(())
}
#[tokio::test]
async fn decision_removes_api_keys_from_selected_and_fallback_urls() -> TestResult {
let judge_upstream = MockUpstream::start().await?;
let state = load_test_config(&format!(
r#"
schema_version = 1
[llm_clients.judge]
format = "openai_chat"
base_url = "{judge_url}"
[llm_clients.model]
format = "openai_chat"
base_url = "https://example.test/v1?key=secret&api_version=2026&api_key=other-secret"
[targets.judge]
id = "model/classifier"
llm_client = "judge"
[targets.quality]
id = "model/strong"
llm_client = "model"
[targets.economy]
id = "model/weak"
llm_client = "model"
[routes.classify]
id = "switchyard/classify"
type = "llm_classifier"
classifier_target = "judge"
strong_target = "quality"
weak_target = "economy"
base_threshold = 0.5
"#,
judge_url = judge_upstream.base_url,
))?;
let app = build_switchyard_router(state);
let response = send(
&app,
"POST",
"/v1/decision",
Some(json!({
"input_format": "openai_chat",
"request": {
"model": "switchyard/classify",
"messages": [{"role": "user", "content": "bounded task"}]
}
})),
)
.await?;
assert_eq!(response.status, StatusCode::OK);
let response = response.json()?;
assert_eq!(response["fallbacks"].as_array().map(Vec::len), Some(1));
for target in [&response["selected"], &response["fallbacks"][0]] {
assert_eq!(
target["llm_client"]["base_url"],
"https://example.test/v1?api_version=2026"
);
}
Ok(())
}
/// Configure classifier and escalation routes over two Responses providers.
/// `route/shared` uses one answer client; the other routes must return known continuations
/// to the model that served their state, even when the judge would select another provider.
fn state_silo_config(base_url: &str) -> String {
let root = base_url.trim_end_matches("/v1");
format!(
r#"
schema_version = 1
[llm_clients.judge]
format = "openai_responses"
base_url = "{root}/silo/judge"
[llm_clients.a]
format = "openai_responses"
base_url = "{root}/silo/A"
max_retries = 0
[llm_clients.b]
format = "openai_responses"
base_url = "{root}/silo/B"
[targets.judge]
id = "model/judge"
llm_client = "judge"
[targets.a]
id = "model/provider-a"
llm_client = "a"
[targets.b]
id = "model/provider-b"
llm_client = "b"
[targets.shared_b]
id = "model/shared-b"
llm_client = "a"
[routes.shared]
id = "route/shared"
type = "llm_classifier"
classifier_target = "judge"
weak_target = "a"
strong_target = "shared_b"
base_threshold = 0.5
classify_trigger = "every_request"
[routes.escalation]
id = "route/escalation"
type = "llm_classifier"
mode = "escalation"
classifier_target = "judge"
weak_target = "a"
strong_target = "b"
escalation = {{ confirmations = 1 }}
[routes.dynamic]
id = "route/dynamic"
type = "llm_classifier"
classifier_target = "judge"
weak_target = "a"
strong_target = "b"
base_threshold = 0.5
classify_trigger = "every_request"
"#
)
}
#[tokio::test]
async fn responses_continuations_preserve_state_ownership() -> TestResult {
for (route, stream, endpoint) in [
("route/dynamic", false, "/v1/responses"),
("route/dynamic", true, "/v1/responses"),
("route/escalation", false, "/v1/decision"),
("route/escalation", true, "/v1/decision"),
("route/shared", false, "/v1/responses"),
] {
let upstream = MockUpstream::start().await?;
let app =
build_switchyard_router(load_test_config(&state_silo_config(&upstream.base_url))?);
let mut request =
json!({"model": route, "input": "ROUTE_A remember mango", "stream": stream});
if endpoint == "/v1/decision" {
request = json!({"input_format": "openai_responses", "request": request});
}
let seed = send(&app, "POST", endpoint, Some(request)).await?;
assert_eq!(seed.status, StatusCode::OK, "{}", seed.text()?);
let body = if endpoint == "/v1/decision" {
seed.json()?["response"].clone()
} else if stream {
sse_events(seed.text()?)
.into_iter()
.find(|event| event["type"] == "response.completed")
.ok_or("stream ended without response.completed")?["response"]
.clone()
} else {
seed.json()?
};
assert!(
body["id"]
.as_str()
.is_some_and(|id| id.starts_with("resp_A_"))
);
let mut follow_request = json!({"model": route, "input": "ROUTE_B recall mango", "previous_response_id": body["id"]});
let before = upstream.models().await.len();
let follow = send(&app, "POST", "/v1/responses", Some(follow_request.clone())).await?;
assert_eq!(follow.status, StatusCode::OK, "{}", follow.text()?);
let expected = if route == "route/shared" {
vec!["model/judge", "model/shared-b"]
} else {
vec!["model/provider-a"]
};
assert_eq!(&upstream.models().await[before..], expected);
assert_eq!(
follow.headers["x-model-router-selected-model"],
*expected.last().ok_or("missing target")?
);
if route == "route/dynamic" {
follow_request["previous_response_id"] = follow.json()?["id"].clone();
let chained = send(&app, "POST", "/v1/responses", Some(follow_request.clone())).await?;
assert_eq!(chained.status, StatusCode::OK);
assert_eq!(
chained.headers["x-model-router-selected-model"],
"model/provider-a"
);
let before = upstream.models().await.len();
let fresh = send(
&app,
"POST",
"/v1/responses",
Some(json!({"model": route, "input": "ROUTE_B a new task"})),
)
.await?;
assert_eq!(fresh.status, StatusCode::OK);
assert_eq!(
fresh.headers["x-model-router-selected-model"],
"model/provider-b"
);
assert_eq!(
&upstream.models().await[before..],
["model/judge", "model/provider-b"]
);
follow_request["input"] = json!("ROUTE_B owner-unavailable");
let before = upstream.models().await.len();
let unavailable = send(&app, "POST", "/v1/responses", Some(follow_request)).await?;
assert_eq!(unavailable.status, StatusCode::SERVICE_UNAVAILABLE);
assert_eq!(&upstream.models().await[before..], ["model/provider-a"]);
}
}
for (route, stream, endpoint) in [
("route/dynamic", false, "/v1/responses"),
("route/dynamic", true, "/v1/responses"),
("route/escalation", false, "/v1/decision"),
("route/escalation", true, "/v1/decision"),
] {
let upstream = MockUpstream::start().await?;
let app =
build_switchyard_router(load_test_config(&state_silo_config(&upstream.base_url))?);
let decision = endpoint == "/v1/decision";
let input = if decision {
"ROUTE_B state-id-conflict"
} else {
"ROUTE_A state-id-conflict"
};
let seed = send(
&app,
"POST",
"/v1/responses",
Some(json!({"model": route, "input": input})),
)
.await?;
assert_eq!(seed.status, StatusCode::OK);
let owner = seed.headers["x-model-router-selected-model"].clone();
let input = if decision {
"ROUTE_A state-id-conflict"
} else {
"ROUTE_B state-id-conflict"
};
let mut request = json!({"model": route, "input": input, "stream": stream});
if decision {
request = json!({"input_format": "openai_responses", "request": request});
}
let before = upstream.models().await.len();
let rejected = send(&app, "POST", endpoint, Some(request)).await?;
if stream && !decision {
assert_eq!(rejected.status, StatusCode::OK);
let events = sse_events(rejected.text()?);
assert_eq!(events.len(), 1);
assert_eq!(events[0]["type"], "error");
} else {
assert_eq!(
rejected.status,
StatusCode::CONFLICT,
"{}",
rejected.text()?
);
assert_eq!(rejected.json()?["error"]["code"], "response_state_conflict");
}
assert_eq!(
&upstream.models().await[before..],
if decision {
["model/provider-a", "model/judge"]
} else {
["model/judge", "model/provider-b"]
}
);
if !decision {
let stats = send(&app, "GET", "/v1/stats", None).await?.json()?;
assert_eq!(stats["total_requests"], 2);
assert_eq!(stats["total_errors"], 1);
}
let follow = send(&app, "POST", "/v1/responses", Some(json!({"model": route, "input": "ROUTE_B recall", "previous_response_id": seed.json()?["id"], "store": false}))).await?;
assert_eq!(follow.status, StatusCode::OK);
assert_eq!(follow.headers["x-model-router-selected-model"], owner);
}
Ok(())
}
/// Decision-only routing returns callable metadata and preserves any answer produced while routing.
#[tokio::test]
async fn decision_returns_callable_target_and_routing_answer() -> TestResult {
let judge_upstream = MockUpstream::start().await?;
let model_upstream = MockUpstream::start().await?;
let state = load_test_config(&format!(
r#"
schema_version = 1
[llm_clients.judge_provider]
format = "openai_chat"
base_url = "{judge_url}"
[llm_clients.model_provider]
format = "openai_chat"
base_url = "{model_url}"
[targets.judge]
id = "model/classifier"
llm_client = "judge_provider"
system_prompt = "judge target prompt"
[targets.quality]
id = "model/strong"
llm_client = "model_provider"
[targets.economy]
id = "model/weak"
llm_client = "model_provider"
extra_body = {{ service_tier = "priority" }}
system_prompt = "economy answer prompt"
[routes.classify]
id = "switchyard/classify"
type = "llm_classifier"
classifier_target = "judge"
strong_target = "quality"
weak_target = "economy"
base_threshold = 0.5
[routes.escalation]
id = "switchyard/escalation"
type = "llm_classifier"
mode = "escalation"
classifier_target = "judge"
strong_target = "quality"
weak_target = "economy"
escalation = {{ confirmations = 1 }}
"#,
judge_url = judge_upstream.base_url,
model_url = model_upstream.base_url,
))?;
let app = build_switchyard_router(state);
let response = send(
&app,
"POST",
"/v1/decision",
Some(json!({
"input_format": "openai_chat",
"request": {
"model": "switchyard/classify",
"messages": [{"role": "user", "content": "bounded task"}]
}
})),
)
.await?;
assert_eq!(response.status, StatusCode::OK);
let mut body = response.json()?;
let outcome_id = body["metadata"]
.as_object_mut()
.and_then(|metadata| metadata.remove("outcome_id"));
assert!(outcome_id.is_some_and(|id| id.as_str().is_some_and(|id| !id.is_empty())));
assert_eq!(
body,
json!({
"selected": {
"target": "economy",
"model": "model/weak",
"llm_client": {
"format": "openai_chat",
"base_url": model_upstream.base_url,
},
"extra_body": {"service_tier": "priority"},
},
"fallbacks": [{
"target": "quality",
"model": "model/strong",
"llm_client": {
"format": "openai_chat",
"base_url": model_upstream.base_url,
},
"extra_body": {},
}],
"metadata": {
"algorithm": "llm_task_classifier",
"evidence": {"source": "llm-classifier", "score": 0.9, "threshold": 0.5},
"judgments": [{
"target": "economy",
"category": "efficient",
"evidence": {"source": "llm-classifier", "score": 0.9, "threshold": 0.5},
}],
},
})
);
assert_eq!(
judge_upstream.models().await,
vec!["model/classifier".to_string()]
);
assert!(model_upstream.models().await.is_empty());
judge_upstream.calls.lock().await.clear();
model_upstream.calls.lock().await.clear();
let response = send(
&app,
"POST",
"/v1/decision",
Some(json!({
"input_format": "openai_chat",
"request": {
"model": "switchyard/escalation",
"messages": [{"role": "user", "content": "bounded task"}]
}
})),
)
.await?;
assert_eq!(response.status, StatusCode::OK);
let response = response.json()?;
assert_eq!(response["selected"]["target"], "economy");
assert_eq!(response["fallbacks"], json!([]));
assert_eq!(response["response"]["model"], "model/weak");
assert_eq!(
response["response"]["choices"][0]["message"]["content"],
"ok"
);
assert_eq!(model_upstream.models().await, ["model/weak"]);
assert_eq!(judge_upstream.models().await, ["model/classifier"]);
assert!(has_system_prompt(
&model_upstream.calls.lock().await[0],
"economy answer prompt"
));
assert!(!has_system_prompt(
&judge_upstream.calls.lock().await[0],
"judge target prompt"
));
Ok(())
}
/// A critical tool error must reach the stage router's signal scorer, which reads
/// the decoded conversation. The endpoint records no inbound wire format, so a
/// scorer that parsed the raw body instead would find nothing and route every turn
/// as if the conversation had no signals at all.
#[tokio::test]
async fn stage_route_escalates_on_a_signal_in_the_conversation() -> TestResult {
let upstream = MockUpstream::start().await?;
let state = load_test_config(&format!(
r#"
schema_version = 1
[llm_clients.upstream]
format = "openai_chat"
base_url = "{base_url}"
[targets.strong]
id = "model/stats-strong"
llm_client = "upstream"
[targets.weak]
id = "model/stats-weak"
llm_client = "upstream"
[routes.stage]
id = "switchyard/stage"
type = "stage_router"
capable_target = "strong"
efficient_target = "weak"
picker = "efficient_first"
confidence_threshold = 0.5
"#,
base_url = upstream.base_url
))?;
let app = build_switchyard_router(state);
let response = send(
&app,
"POST",
"/v1/chat/completions",
Some(json!({
"model": "switchyard/stage",
"messages": [
{"role": "user", "content": "fix the build"},
{"role": "assistant", "tool_calls": [{
"id": "call_1",
"type": "function",
"function": {"name": "Bash", "arguments": "{\"command\": \"cargo test\"}"}
}]},
{"role": "tool", "tool_call_id": "call_1", "content": "fatal runtime error: out of memory"},
]
})),
)
.await?;
assert_eq!(response.status, StatusCode::OK);
assert_eq!(
response
.headers
.get("x-model-router-selected-model")
.and_then(|value| value.to_str().ok()),
Some("model/stats-strong"),
"a critical error should escalate on the signals alone"
);
let stats = send(&app, "GET", "/v1/stats", None).await?.json()?;
assert_eq!(
stats["algorithm_stats"]["stage_router"]["routing_decisions"]["override"]["targets"]["model/stats-strong"],
1
);
Ok(())
}
/// Anthropic failure flags must affect routing before translating to Responses.
#[tokio::test]
async fn stage_route_honors_anthropic_tool_result_errors() -> TestResult {
let upstream = MockUpstream::start().await?;
let app = build_switchyard_router(load_test_config(&format!(
r#"
schema_version = 1
[llm_clients.upstream]
format = "openai_responses"
base_url = "{}/buffered"
[targets.capable]
id = "model/fallback"
llm_client = "upstream"
[targets.efficient]
id = "model/efficient"
llm_client = "upstream"
[routes.stage]
id = "switchyard/stage-structured-failure"
type = "stage_router"
capable_target = "capable"
efficient_target = "efficient"
picker = "efficient_first"
confidence_threshold = 0.5
capable_hold_turns = 0
recent_turn_window = 3
context_window = 131072
tool_calling = true
reasoning = true
"#,
upstream.base_url.trim_end_matches("/v1")
))?);
let neutral = "Synthetic dependency unavailable; retry with the recovery path.";
for (is_error, text, expected) in [
(false, neutral, "model/efficient"),
(true, neutral, "model/fallback"),
(
false,
"fatal runtime error: out of memory",
"model/fallback",
),
] {
let mut messages = vec![json!({
"role": "user", "content": "Run the synthetic dependency probe."
})];
for id in ["toolu_probe_1", "toolu_probe_2"] {
messages.push(json!({"role": "assistant", "content": [{
"type": "tool_use", "id": id,
"name": "synthetic_dependency_probe", "input": {}
}]}));
messages.push(json!({"role": "user", "content": [{
"type": "tool_result", "tool_use_id": id,
"content": text, "is_error": is_error
}]}));
}
let body = json!({
"model": "switchyard/stage-structured-failure",
"max_tokens": 16, "messages": messages
});
for _ in 0..3 {
let response = send(&app, "POST", "/v1/messages", Some(body.clone())).await?;
assert_eq!(response.status, StatusCode::OK);
assert_eq!(
response.json()?["model"],
expected,
"is_error={is_error}, text={text}"
);
assert_eq!(
upstream.models().await.last().map(String::as_str),
Some(expected)
);
}
}
Ok(())
}
#[tokio::test]
async fn targets_sharing_a_model_keep_their_own_settings() -> TestResult {
let upstream = MockUpstream::start().await?;
let state = load_test_config(&format!(
r#"
schema_version = 1
[llm_clients.upstream]
format = "openai_chat"
base_url = "{base_url}"
[targets.efficient]
id = "model/shared"
llm_client = "upstream"
reasoning_effort = "medium"
system_prompt = "efficient prompt"
[targets.capable]
id = "model/shared"
llm_client = "upstream"
reasoning_effort = "max"
system_prompt = "capable prompt"
[routes.random]
id = "switchyard/random"
type = "random"
targets = ["efficient", "capable"]
seed = 17
"#,
base_url = upstream.base_url
))?;
let log_dir = tempfile::tempdir()?;
let log_path = log_dir.path().join("routing.jsonl");
let app = build_switchyard_router(state.with_routing_log(&log_path)?);
let requests = 16;
for index in 0..requests {
let streaming = index % 2 == 0;
let response = send_with_headers(
&app,
"POST",
"/v1/chat/completions",
Some(json!({
"model": "switchyard/random",
"messages": [{"role": "user", "content": "hello"}],
"reasoning_effort": "high",
"stream": streaming
})),
&[("x-switchyard-session-id", "shared-efforts")],
)
.await?;
assert_eq!(response.status, StatusCode::OK);
assert_eq!(
response.headers["x-model-router-selected-model"],
"model/shared"
);
if streaming {
assert!(response.text()?.contains("model/shared"));
} else {
assert_eq!(response.json()?["model"], "model/shared");
}
}
let calls = upstream.calls.lock().await;
assert_eq!(calls.len(), requests);
for call in calls.iter() {
assert_eq!(call["model"], "model/shared");
let effort = call["reasoning_effort"].as_str().unwrap_or_default();
let expected_prompt = match effort {
"medium" => "efficient prompt",
"max" => "capable prompt",
other => panic!("unexpected effort {other}"),
};
assert!(has_system_prompt(call, expected_prompt), "{call}");
}
drop(calls);
let decision = send(
&app,
"POST",
"/v1/decision",
Some(json!({
"input_format": "openai_chat",
"request": {
"model": "switchyard/random",
"messages": [{"role": "user", "content": "hello"}]
}
})),
)
.await?
.json()?;
let selected = &decision["selected"];
assert_eq!(selected["model"], "model/shared");
let expected_effort = match selected["target"].as_str() {
Some("capable") => "max",
Some("efficient") => "medium",
other => panic!("unexpected target {other:?}"),
};
assert_eq!(selected["reasoning_effort"], expected_effort);
let stats = send(&app, "GET", "/v1/stats", None).await?.json()?;
let efficient = stats["targets"]["efficient"]["calls"].as_u64().unwrap_or(0);
let capable = stats["targets"]["capable"]["calls"].as_u64().unwrap_or(0);
assert!(efficient > 0 && capable > 0, "{}", stats["models"]);
assert_eq!(efficient + capable, requests as u64);
assert_eq!(stats["models"].as_object().unwrap().len(), 1);
assert_eq!(stats["models"]["model/shared"]["calls"], requests);
for (target, effort) in [("capable", "max"), ("efficient", "medium")] {
assert_eq!(stats["targets"][target]["model"], "model/shared");
assert_eq!(stats["targets"][target]["reasoning_effort"], effort);
}
assert_eq!(
stats["models"]["model/shared"]["prompt_tokens"],
stats["total_tokens"]["prompt"]
);
let session = send(
&app,
"GET",
"/v1/routing/session-stats?session_id=shared-efforts",
None,
)
.await?
.json()?;
assert_eq!(session["total_calls"], requests);
assert_eq!(session["models"]["model/shared"]["calls"], requests);
for target in ["capable", "efficient"] {
assert_eq!(
session["targets"][target]["calls"],
stats["targets"][target]["calls"]
);
assert_eq!(
session["targets"][target]["reasoning_effort"],
stats["targets"][target]["reasoning_effort"]
);
assert_eq!(
session["targets"][target]["prompt_tokens"],
stats["targets"][target]["prompt_tokens"]
);
}
for line in std::fs::read_to_string(log_path)?.lines() {
let record: Value = serde_json::from_str(line)?;
assert_eq!(record["model"], "model/shared");
let target = record["target"].as_str().unwrap();
assert_eq!(
record["reasoning_effort"],
stats["targets"][target]["reasoning_effort"]
);
}
send(&app, "POST", "/v1/stats/reset", None).await?;
let reset = send(&app, "GET", "/v1/stats", None).await?.json()?;
assert_eq!(reset["total_requests"], 0);
assert_eq!(reset["models"], json!({}));
assert_eq!(reset["targets"], json!({}));
Ok(())
}
#[tokio::test]
async fn toml_config_constructs_and_serves_multiple_algorithms() -> TestResult {
let upstream = MockUpstream::start().await?;
let state = load_test_config(&format!(
r#"
schema_version = 1
[llm_clients.upstream]
format = "openai_chat"
base_url = "{base_url}"
[targets.classifier]
id = "model/classifier"
llm_client = "upstream"
[targets.strong]
id = "model/strong"
llm_client = "upstream"
[targets.weak]
id = "model/weak"
llm_client = "upstream"
[routes.random]
id = "switchyard/random"
type = "random"
targets = ["weak"]
[routes.classifier]
id = "switchyard/classifier"
type = "llm_classifier"
classifier_target = "classifier"
strong_target = "strong"
weak_target = "weak"
base_threshold = 0.5
[routes.passthrough]
id = "switchyard/passthrough"
type = "passthrough"
target = "weak"
[routes.stage]
id = "switchyard/stage"
type = "stage_router"
capable_target = "strong"
efficient_target = "weak"
picker = "efficient_first"
confidence_threshold = 0.5
recent_turn_window = 3
[routes.stage.handoff_notes]
escalation_note = "the previous model was stalling"
[routes.stage.classifier]
target = "classifier"
base_threshold = 0.5
"#,
base_url = upstream.base_url
))?;
let app = build_switchyard_router(state);
for (route, selected) in [
("switchyard/random", "model/weak"),
("switchyard/classifier", "model/weak"),
("switchyard/passthrough", "model/weak"),
] {
let response = send(
&app,
"POST",
"/v1/chat/completions",
Some(json!({
"model": route,
"messages": [{"role": "user", "content": "hi"}]
})),
)
.await?;
assert_eq!(response.status, StatusCode::OK);
assert_eq!(
response
.headers
.get("x-model-router-selected-model")
.and_then(|value| value.to_str().ok()),
Some(selected)
);
}
let calls = upstream.calls.lock().await;
assert_eq!(calls.len(), 4);
assert_eq!(calls[0]["model"], "model/weak");
assert_eq!(calls[1]["model"], "model/classifier");
assert_eq!(calls[2]["model"], "model/weak");
assert_eq!(calls[3]["model"], "model/weak");
drop(calls);
let stats = send(&app, "GET", "/v1/stats", None).await?.json()?;
assert_eq!(stats["total_requests"], 3);
assert_eq!(stats["models"]["model/weak"]["calls"], 3);
assert_eq!(stats["classifier"]["total_requests"], 1);
assert_eq!(
stats["classifier"]["models"]["model/classifier"]["calls"],
1
);
assert_eq!(stats["classifier"]["total_tokens"]["prompt"], 10);
Ok(())
}
// A configured mutation must select the efficient tier through the HTTP configuration path.
#[tokio::test]
async fn stage_router_uses_configured_tool_semantics() -> TestResult {
let upstream = MockUpstream::start().await?;
let state = load_test_config(&format!(
r#"
schema_version = 1
[llm_clients.upstream]
format = "openai_chat"
base_url = "{base_url}"
[targets.strong]
id = "model/strong"
llm_client = "upstream"
[targets.weak]
id = "model/weak"
llm_client = "upstream"
[routes.stage]
id = "switchyard/stage"
type = "stage_router"
capable_target = "strong"
efficient_target = "weak"
picker = "capable_first"
confidence_threshold = 0.3
[routes.stage.tool_semantics]
mutate = ["send_payment_request"]
"#,
base_url = upstream.base_url
))?;
let app = build_switchyard_router(state);
let response = send(
&app,
"POST",
"/v1/chat/completions",
Some(json!({
"model": "switchyard/stage",
"messages": [
{"role": "user", "content": "pay the balance"},
{"role": "assistant", "tool_calls": [{
"id": "call_1",
"type": "function",
"function": {
"name": "send_payment_request",
"arguments": "{}"
}
}]},
{"role": "tool", "tool_call_id": "call_1", "content": "payment sent"}
]
})),
)
.await?;
assert_eq!(response.status, StatusCode::OK);
assert_eq!(
response
.headers
.get("x-model-router-selected-model")
.and_then(|value| value.to_str().ok()),
Some("model/weak")
);
Ok(())
}
/// The executor receives the plan and write result, but not the planner's prompt.
#[tokio::test]
async fn stage_plan_execute_handoff_preserves_history_and_prompts() -> TestResult {
let upstream = MockUpstream::start().await?;
// Stage metrics are process-wide, so unique model IDs keep this test's counts separate.
let app = build_switchyard_router(load_test_config(&format!(
r#"
schema_version = 1
[llm_clients.local]
format = "openai_chat"
base_url = "{}"
[targets]
planner = {{ id = "model/plan-execute-planner", llm_client = "local" }}
executor = {{ id = "model/plan-execute-executor", llm_client = "local" }}
[routes.plan_execute]
id = "plan-execute"
type = "stage_router"
capable_target = "planner"
efficient_target = "executor"
picker = "capable_first"
confidence_threshold = 0.0
capable_hold_turns = 0
efficient_hold_turns = -1
capable_system_prompt = "Inspect the task and plan before modifying files."
"#,
upstream.base_url,
))?);
let planning_history = json!([{"role": "user", "content": "implement the task"}]);
let execution_history = json!([
{"role": "user", "content": "implement the task"},
{"role": "assistant", "content": "The plan is to update the file.",
"tool_calls": [{"id": "call", "type": "function",
"function": {"name": "write_file", "arguments": "{}"}}]},
{"role": "tool", "tool_call_id": "call", "content": "file written"}
]);
for (messages, expected) in [
(&planning_history, "model/plan-execute-planner"),
(&execution_history, "model/plan-execute-executor"),
] {
let response = send_with_headers(
&app,
"POST",
"/v1/chat/completions",
Some(json!({"model": "plan-execute", "messages": messages, "stream": false})),
&[("x-switchyard-session-id", "plan-execute-session")],
)
.await?;
assert_eq!(response.status, StatusCode::OK);
assert_eq!(
response
.headers
.get("x-model-router-selected-model")
.and_then(|value| value.to_str().ok()),
Some(expected),
);
}
{
let calls = upstream.calls.lock().await;
assert_eq!(calls.len(), 2);
assert_eq!(calls[0]["model"], "model/plan-execute-planner");
assert!(has_system_prompt(
&calls[0],
"Inspect the task and plan before modifying files.",
));
assert_eq!(calls[1]["model"], "model/plan-execute-executor");
assert_eq!(calls[1]["messages"], execution_history);
assert!(!has_system_prompt(
&calls[1],
"Inspect the task and plan before modifying files.",
));
}
let stats = send(&app, "GET", "/v1/stats", None).await?.json()?;
assert_eq!(stats["classifier"]["total_requests"], 0);
assert_eq!(
stats["algorithm_stats"]["stage_router"]["routing_decisions"]["dimensions"]["targets"]["model/plan-execute-executor"],
1
);
Ok(())
}
// Composite TOML must pass custom stage semantics through to the nested stage router.
#[tokio::test]
async fn composite_router_uses_configured_tool_semantics() -> TestResult {
let upstream = MockUpstream::start().await?;
let state = load_test_config(&format!(
r#"
schema_version = 1
[llm_clients.upstream]
format = "openai_chat"
base_url = "{base_url}"
[targets.classifier]
id = "model/classifier"
llm_client = "upstream"
[targets.strong]
id = "model/strong"
llm_client = "upstream"
[targets.weak]
id = "model/weak"
llm_client = "upstream"
[routes.composite]
id = "switchyard/composite"
type = "composite"
[routes.composite.classifier]
target = "classifier"
base_threshold = 0.5
classify_trigger = "user_turn"
[routes.composite.stage]
capable_target = "strong"
efficient_target = "weak"
confidence_threshold = 0.3
[routes.composite.stage.tool_semantics]
new = ["send_message_to_user"]
"#,
base_url = upstream.base_url
))?;
let app = build_switchyard_router(state);
let response = send(
&app,
"POST",
"/v1/chat/completions",
Some(json!({
"model": "switchyard/composite",
"messages": [
{"role": "user", "content": "help the customer"},
{"role": "assistant", "content": "working"},
{"role": "user", "content": "continue"},
{"role": "assistant", "content": "working"},
{"role": "user", "content": "continue"},
{"role": "assistant", "content": "working"},
{"role": "assistant", "tool_calls": [{
"id": "call_1",
"type": "function",
"function": {
"name": "send_message_to_user",
"arguments": "{}"
}
}]},
{"role": "tool", "tool_call_id": "call_1", "content": "message sent"}
]
})),
)
.await?;
assert_eq!(response.status, StatusCode::OK);
assert_eq!(
response
.headers
.get("x-model-router-selected-model")
.and_then(|value| value.to_str().ok()),
Some("model/weak")
);
assert_eq!(
upstream.models().await,
["model/weak"],
"configured new activity must suppress the deep-turn stall without consulting the judge"
);
Ok(())
}
#[tokio::test]
async fn custom_classifier_uses_categories_and_falls_back_on_an_invalid_verdict() -> TestResult {
let upstream = MockUpstream::start().await?;
let state = load_test_config(&format!(
r#"
schema_version = 1
[llm_clients.upstream]
format = "openai_chat"
base_url = "{base_url}"
[targets.classifier]
id = "model/classifier"
llm_client = "upstream"
[targets.strong]
id = "model/strong"
llm_client = "upstream"
[targets.middle]
id = "model/middle"
llm_client = "upstream"
[targets.premium]
id = "model/premium"
llm_client = "upstream"
[targets.weak]
id = "model/weak"
llm_client = "upstream"
[routes.custom]
id = "switchyard/custom"
type = "llm_classifier"
mode = "custom"
models = {{ judge = ["classifier"], fast = ["weak"], balanced = ["middle"], reasoning = ["strong"], premium = ["premium"], any = ["weak", "middle", "strong", "premium"] }}
default_target = "premium"
prompt = "CUSTOM MULTI TARGET"
response_schema = '''
{{
"type": "object",
"properties": {{
"decision": {{
"type": "object",
"properties": {{
"target": {{"type": "string", "enum": ["fast", "balanced", "reasoning", "premium"]}}
}},
"required": ["target"],
"additionalProperties": false
}}
}},
"required": ["decision"],
"additionalProperties": false
}}
'''
[routes.custom.policy]
type = "target_selector"
selector = "/decision/target"
"#,
base_url = upstream.base_url
))?;
let app = build_switchyard_router(state);
// Each named group resolves to its own model, so the policy picks between
// four of them rather than between the two tier categories.
for (task, selected) in [
("route to fast", "model/weak"),
("route to balanced", "model/middle"),
("route to reasoning", "model/strong"),
("route to premium", "model/premium"),
("return an invalid verdict", "model/premium"),
] {
let response = send(
&app,
"POST",
"/v1/chat/completions",
Some(json!({
"model": "switchyard/custom",
"messages": [{"role": "user", "content": task}]
})),
)
.await?;
assert_eq!(response.status, StatusCode::OK);
assert_eq!(
response
.headers
.get("x-model-router-selected-model")
.and_then(|value| value.to_str().ok()),
Some(selected)
);
}
let calls = upstream.calls.lock().await;
let judge_call = calls
.iter()
.find(|call| call["model"] == "model/classifier")
.ok_or("custom classifier target was not called")?;
let prompt = judge_call["messages"][0]["content"]
.as_str()
.ok_or("custom classifier prompt was not text")?;
assert_eq!(prompt, "CUSTOM MULTI TARGET");
assert_eq!(judge_call["response_format"]["type"], "json_schema");
assert_eq!(
judge_call["response_format"]["json_schema"]["name"],
"switchyard_classifier_response"
);
assert_eq!(judge_call["response_format"]["json_schema"]["strict"], true);
assert_eq!(
judge_call["response_format"]["json_schema"]["schema"]["properties"]["decision"]["properties"]
["target"]["enum"],
json!(["fast", "balanced", "reasoning", "premium"])
);
Ok(())
}
#[tokio::test]
async fn classifier_contract_overrides_reach_every_server_mode() -> TestResult {
let upstream = MockUpstream::start().await?;
let state = load_test_config(&format!(
r#"
schema_version = 1
[llm_clients.upstream]
format = "openai_chat"
base_url = "{base_url}"
[targets.classifier]
id = "model/classifier"
llm_client = "upstream"
[targets.strong]
id = "model/strong"
llm_client = "upstream"
[targets.weak]
id = "model/weak"
llm_client = "upstream"
[routes.capability]
id = "switchyard/capability"
type = "llm_classifier"
mode = "capability"
classifier_target = "classifier"
strong_target = "strong"
weak_target = "weak"
base_threshold = 0.5
prompt = "CUSTOM CAPABILITY"
response_format_type = "json_object"
[routes.escalation]
id = "switchyard/escalation"
type = "llm_classifier"
mode = "escalation"
classifier_target = "classifier"
strong_target = "strong"
weak_target = "weak"
prompt = "CUSTOM ESCALATION"
response_format_type = "json_object"
escalation = {{ confirmations = 1 }}
[routes.stage]
id = "switchyard/stage"
type = "stage_router"
capable_target = "strong"
efficient_target = "weak"
picker = "efficient_first"
confidence_threshold = 1.0
[routes.stage.classifier]
target = "classifier"
base_threshold = 0.5
prompt = "CUSTOM STAGE"
"#,
base_url = upstream.base_url
))?;
let app = build_switchyard_router(state);
for (route, prompt_prefix, schema_field, json_object) in [
(
"switchyard/capability",
"CUSTOM CAPABILITY",
"p_solve",
true,
),
(
"switchyard/escalation",
"CUSTOM ESCALATION",
"escalate",
true,
),
("switchyard/stage", "CUSTOM STAGE", "p_solve", false),
] {
upstream.calls.lock().await.clear();
let response = send(
&app,
"POST",
"/v1/chat/completions",
Some(json!({
"model": route,
"messages": [{"role": "user", "content": "bounded task"}]
})),
)
.await?;
assert_eq!(response.status, StatusCode::OK);
let calls = upstream.calls.lock().await;
let judge_call = calls
.iter()
.find(|call| call["model"] == "model/classifier")
.ok_or("classifier target was not called")?;
let prompt = judge_call["messages"][0]["content"]
.as_str()
.ok_or("classifier prompt was not text")?;
assert!(prompt.starts_with(prompt_prefix), "{route}: {prompt}");
if json_object {
assert_eq!(
judge_call["response_format"],
json!({"type": "json_object"}),
"{route}: {judge_call}"
);
assert!(prompt.contains("JSON Schema"), "{route}: {prompt}");
assert!(
prompt.contains(&format!("\"{schema_field}\"")),
"{route}: missing {schema_field} in {prompt}"
);
} else {
assert!(
judge_call["response_format"]["json_schema"]["schema"]["properties"]
.get(schema_field)
.is_some(),
"{route}: missing {schema_field} in {judge_call}"
);
}
}
Ok(())
}
#[tokio::test]
async fn classifier_task_input_excludes_tool_results_across_request_formats() -> TestResult {
let upstream = MockUpstream::start().await?;
let app = build_switchyard_router(load_test_config(&format!(
r#"
schema_version = 1
[llm_clients.upstream]
format = "openai_chat"
base_url = "{base_url}"
max_retries = 0
[targets.judge]
id = "model/classifier"
llm_client = "upstream"
[targets.strong]
id = "model/strong"
llm_client = "upstream"
[targets.weak]
id = "model/weak"
llm_client = "upstream"
[routes.task]
id = "task"
type = "llm_classifier"
classifier_target = "judge"
strong_target = "strong"
weak_target = "weak"
base_threshold = 0.5
classify_trigger = "every_request"
[routes.window]
id = "window"
type = "llm_classifier"
classifier_target = "judge"
strong_target = "strong"
weak_target = "weak"
base_threshold = 0.5
classify_trigger = "every_request"
recent_turn_window = 1
"#,
base_url = upstream.base_url
))?);
let chat = json!({"messages": [
{"role": "user", "content": "Read the file."},
{"role": "assistant", "content": null, "tool_calls": [{
"id": "call_read", "type": "function", "function": {"name": "read_file", "arguments": "{}"}
}]},
{"role": "tool", "tool_call_id": "call_read", "content": "File contents."}
]});
for (route, endpoint, mut body) in [
("task", "/v1/chat/completions", chat.clone()),
(
"task",
"/v1/messages",
json!({"messages": [
{"role": "user", "content": "Read the file."},
{"role": "assistant", "content": [{"type": "tool_use", "id": "call_read", "name": "read_file", "input": {}}]},
{"role": "user", "content": [{"type": "tool_result", "tool_use_id": "call_read", "content": "File contents."}]}
]}),
),
(
"task",
"/v1/responses",
json!({"input": [
{"role": "user", "content": "Read the file."},
{"type": "function_call", "call_id": "call_read", "name": "read_file", "arguments": "{}"},
{"type": "function_call_output", "call_id": "call_read", "output": "File contents."}
]}),
),
("window", "/v1/chat/completions", chat),
] {
upstream.calls.lock().await.clear();
body["model"] = json!(route);
let response = send(&app, "POST", endpoint, Some(body)).await?;
assert_eq!(response.status, StatusCode::OK, "{}", response.text()?);
assert_eq!(
response.headers["x-model-router-selected-model"], "model/weak",
"{endpoint}"
);
let calls = upstream.calls.lock().await;
assert_eq!(calls.len(), 2);
let judge_roles: Vec<_> = calls[0]["messages"]
.as_array()
.ok_or("missing judge messages")?
.iter()
.map(|message| message["role"].clone())
.collect();
assert_eq!(
judge_roles,
if route == "task" {
json!(["system", "user"])
} else {
json!(["system", "user", "assistant", "tool", "user"])
}
.as_array()
.ok_or("missing expected roles")?
.clone()
);
assert_eq!(calls[1]["messages"][1]["tool_calls"][0]["id"], "call_read");
assert_eq!(calls[1]["messages"][2]["tool_call_id"], "call_read");
}
let stats = send(&app, "GET", "/v1/stats", None).await?.json()?;
assert_eq!(stats["classifier"]["total_requests"], 4);
assert_eq!(
stats["classifier"]["models"]["model/classifier"]["errors"],
0
);
Ok(())
}
#[tokio::test]
async fn accepted_escalation_response_is_logged_once_as_the_final_answer() -> TestResult {
let upstream = MockUpstream::start().await?;
let temp_dir = tempfile::tempdir()?;
let state = load_test_config(&format!(
r#"
schema_version = 1
[llm_clients.upstream]
format = "openai_chat"
base_url = "{base_url}"
[targets.classifier]
id = "model/strong"
llm_client = "upstream"
[targets.strong]
id = "model/strong"
llm_client = "upstream"
system_prompt = "strong answer prompt"
[targets.weak]
id = "model/weak"
llm_client = "upstream"
system_prompt = "weak answer prompt"
[routes.escalation]
id = "switchyard/escalation"
type = "llm_classifier"
mode = "escalation"
classifier_target = "classifier"
strong_target = "strong"
weak_target = "weak"
escalation = {{ confirmations = 1 }}
"#,
base_url = upstream.base_url
))?
.with_routing_log(temp_dir.path().join("routing.jsonl"))?;
let app = build_switchyard_router(state);
let response = send_with_headers(
&app,
"POST",
"/v1/chat/completions",
Some(json!({
"model": "switchyard/escalation",
"messages": [{"role": "user", "content": "bounded task"}]
})),
&[("x-switchyard-session-id", "accepted-escalation")],
)
.await?;
assert_eq!(response.status, StatusCode::OK);
assert_eq!(upstream.models().await, ["model/weak", "model/strong"]);
let calls = upstream.calls.lock().await;
assert!(has_system_prompt(&calls[0], "weak answer prompt"));
assert!(!has_system_prompt(&calls[1], "strong answer prompt"));
drop(calls);
let stats = send(
&app,
"GET",
"/v1/routing/session-stats?session_id=accepted-escalation",
None,
)
.await?;
assert_eq!(stats.status, StatusCode::OK);
let stats = stats.json()?;
assert_eq!(stats["total_calls"], 2);
assert_eq!(stats["total_prompt_tokens"], 20);
assert_eq!(stats["total_completion_tokens"], 4);
// Session stats come from the routing log, which prices by provider model.
assert_eq!(stats["models"]["model/weak"]["calls"], 1);
assert_eq!(stats["models"]["model/strong"]["calls"], 1);
let process_stats = send(&app, "GET", "/v1/stats", None).await?.json()?;
assert_eq!(process_stats["total_requests"], 1);
assert_eq!(process_stats["models"]["model/weak"]["calls"], 1);
assert_eq!(
process_stats["models"]["model/weak"]["model_call_latency"]["count"],
1
);
Ok(())
}
#[tokio::test]
async fn stage_classifier_can_request_json_object_output() -> TestResult {
let upstream = MockUpstream::start().await?;
let state = load_test_config(&format!(
r#"
schema_version = 1
[llm_clients.upstream]
format = "openai_chat"
base_url = "{base_url}"
[targets.classifier]
id = "model/classifier"
llm_client = "upstream"
[targets.strong]
id = "model/strong"
llm_client = "upstream"
[targets.weak]
id = "model/weak"
llm_client = "upstream"
[routes.stage]
id = "switchyard/stage"
type = "stage_router"
capable_target = "strong"
efficient_target = "weak"
picker = "efficient_first"
confidence_threshold = 1.0
[routes.stage.classifier]
target = "classifier"
base_threshold = 0.5
response_format_type = "json_object"
"#,
base_url = upstream.base_url
))?;
let app = build_switchyard_router(state);
let response = send(
&app,
"POST",
"/v1/chat/completions",
Some(json!({
"model": "switchyard/stage",
"messages": [{"role": "user", "content": "bounded task"}]
})),
)
.await?;
assert_eq!(response.status, StatusCode::OK);
let calls = upstream.calls.lock().await;
let judge_call = calls
.iter()
.find(|call| call["model"] == "model/classifier")
.ok_or("classifier target was not called")?;
assert_eq!(
judge_call["response_format"],
json!({"type": "json_object"})
);
let prompt = judge_call["messages"][0]["content"]
.as_str()
.ok_or("classifier prompt was not text")?;
assert!(prompt.contains("JSON Schema"), "{prompt}");
assert!(prompt.contains("\"p_solve\""), "{prompt}");
drop(calls);
let invalid_response = send(
&app,
"POST",
"/v1/chat/completions",
Some(json!({
"model": "switchyard/stage",
"messages": [{"role": "user", "content": "return a schema-invalid verdict"}]
})),
)
.await?;
assert_eq!(invalid_response.status, StatusCode::OK);
assert_eq!(
invalid_response
.headers
.get("x-model-router-selected-model")
.and_then(|value| value.to_str().ok()),
Some("model/weak")
);
Ok(())
}
#[tokio::test]
async fn model_bearing_auxiliary_endpoints_use_configured_route_targets() -> TestResult {
let upstream = MockUpstream::start().await?;
let state = load_test_config(&format!(
r#"
schema_version = 1
[llm_clients.claude]
format = "anthropic_messages"
base_url = "{base_url}"
[llm_clients.responses]
format = "openai_responses"
base_url = "{base_url}"
extra_headers = {{ "x-configured-client" = "responses" }}
[targets.responses]
id = "real/responses-model"
llm_client = "responses"
[targets.strong]
id = "real/opus"
llm_client = "claude"
system_prompt = "completion instructions"
[targets.other]
id = "real/sonnet"
llm_client = "claude"
[routes.random]
id = "switchyard/random"
type = "random"
targets = ["responses", "other", "strong"]
"#,
base_url = upstream.base_url
))?;
let app = build_switchyard_router(state);
let count_tokens = send(
&app,
"POST",
"/v1/messages/count_tokens",
Some(json!({
"model": "switchyard/random",
"messages": [{"role": "user", "content": "hi"}]
})),
)
.await?;
assert_eq!(count_tokens.status, StatusCode::OK);
assert_eq!(count_tokens.json()?["input_tokens"], 7);
let input_tokens = send(
&app,
"POST",
"/v1/responses/input_tokens",
Some(json!({"model": "switchyard/random", "input": "count me"})),
)
.await?;
assert_eq!(input_tokens.status, StatusCode::OK);
assert_eq!(input_tokens.json()?["input_tokens"], 11);
let compact = send(
&app,
"POST",
"/v1/responses/compact",
Some(json!({"model": "switchyard/random", "input": "compact me"})),
)
.await?;
assert_eq!(compact.status, StatusCode::OK);
assert_eq!(compact.json()?["id"], "resp_compacted");
let calls = upstream.calls.lock().await;
assert_eq!(calls.len(), 3);
assert_eq!(calls[0]["model"], "real/opus");
assert!(calls[0].get("system").is_none());
assert_eq!(
calls[1],
json!({
"path": "/v1/responses/input_tokens",
"body": {"model": "real/responses-model", "input": "count me"},
"configured_header": "responses"
})
);
assert_eq!(
calls[2],
json!({
"path": "/v1/responses/compact",
"body": {"model": "real/responses-model", "input": "compact me"},
"configured_header": "responses"
})
);
drop(calls);
let unsupported = random_state(&upstream.base_url, &[(ROUTE_MODEL, &["model/weak"])])?;
let unsupported = build_switchyard_router(unsupported);
for (path, body) in [
(
"/v1/messages/count_tokens",
json!({"model": ROUTE_MODEL, "messages": [{"role": "user", "content": "hi"}]}),
),
(
"/v1/responses/input_tokens",
json!({"model": ROUTE_MODEL, "input": "count me"}),
),
(
"/v1/responses/compact",
json!({"model": ROUTE_MODEL, "input": "compact me"}),
),
] {
let response = send(&unsupported, "POST", path, Some(body)).await?;
assert_eq!(response.status, StatusCode::BAD_REQUEST, "{path}");
}
Ok(())
}
#[tokio::test]
async fn fallback_client_forwards_unmatched_requests_and_is_optional() -> TestResult {
let upstream = MockUpstream::start().await?;
let state = load_test_config(&format!(
r#"
schema_version = 1
fallback_client = "fallback"
[llm_clients.fallback]
format = "openai_responses"
base_url = "{base_url}"
extra_headers = {{ "x-configured-secret" = "must-not-forward" }}
[llm_clients.routed]
format = "openai_chat"
base_url = "{base_url}"
[targets.weak]
id = "model/weak"
llm_client = "routed"
[routes.random]
id = "switchyard/random"
type = "passthrough"
target = "weak"
"#,
base_url = upstream.base_url
))?;
let app = build_switchyard_router(state);
let response = send_with_headers(
&app,
"POST",
"/future/provider/endpoint?mode=raw",
Some(json!({
"model": "provider/model",
"provider_field": {"nested": true}
})),
&[
("authorization", "Bearer caller-key"),
("connection", "keep-alive, x-remove-me"),
("keep-alive", "timeout=5"),
("x-remove-me", "remove"),
("x-end-to-end", "preserve"),
],
)
.await?;
assert_eq!(response.status, StatusCode::OK);
assert_eq!(response.json()?["input_tokens"], 7);
assert!(!response.headers.contains_key("connection"));
assert!(!response.headers.contains_key("x-upstream-hop"));
assert_eq!(
response
.headers
.get("x-end-to-end-response")
.and_then(|value| value.to_str().ok()),
Some("preserve")
);
let calls = upstream.calls.lock().await;
assert_eq!(
calls.as_slice(),
&[json!({
"body": {
"model": "provider/model",
"provider_field": {"nested": true}
},
"authorization": "Bearer caller-key",
"end_to_end": "preserve",
"configured_secret": false,
"connection": false,
"connection_nominated": false
})]
);
drop(calls);
let state = random_state(&upstream.base_url, &[(ROUTE_MODEL, &["model/weak"])])?;
let app = build_switchyard_router(state);
let response = send(&app, "POST", "/future/provider/endpoint", None).await?;
assert_eq!(response.status, StatusCode::NOT_FOUND);
Ok(())
}
#[tokio::test]
async fn transport_errors_hide_credential_bearing_upstream_urls() -> TestResult {
const CANARY: &str = "CANARY_ADMIN_QUERY_KEY";
let listener = TcpListener::bind("127.0.0.1:0").await?;
let base_url = format!("http://{}/v1?key={CANARY}", listener.local_addr()?);
drop(listener);
let routed = build_switchyard_router(random_state(&base_url, &[(ROUTE_MODEL, &["model/a"])])?);
let routed_response = send(
&routed,
"POST",
"/v1/chat/completions",
Some(json!({
"model": ROUTE_MODEL,
"messages": [{"role": "user", "content": "hello"}]
})),
)
.await?;
let fallback = load_test_config(&format!(
r#"
schema_version = 1
fallback_client = "upstream"
[llm_clients.upstream]
format = "openai_chat"
base_url = "{base_url}"
[targets.model]
id = "model/a"
llm_client = "upstream"
[routes.model]
id = "switchyard/model"
type = "passthrough"
target = "model"
"#
))?;
let fallback = build_switchyard_router(fallback);
let fallback_response = send(&fallback, "POST", "/unmatched", None).await?;
for response in [routed_response, fallback_response] {
assert_eq!(response.status, StatusCode::BAD_GATEWAY);
let body = response.json()?;
let message = body["error"]["message"].as_str().unwrap_or_default();
assert!(
!message.contains(CANARY),
"credential leaked in {message:?}"
);
}
Ok(())
}
#[tokio::test]
async fn anthropic_client_forwards_oauth_when_configured() -> TestResult {
let upstream = MockUpstream::start().await?;
let state = load_test_config(&format!(
r#"
schema_version = 1
[llm_clients.claude]
format = "anthropic_messages"
base_url = "{base_url}"
forward_auth = true
max_retries = 0
[targets.claude]
id = "claude-opus"
llm_client = "claude"
[routes.claude]
id = "switchyard/claude"
type = "passthrough"
target = "claude"
"#,
base_url = upstream.base_url
))?;
let app = build_switchyard_router(state);
let response = send_with_headers(
&app,
"POST",
"/v1/messages",
Some(json!({
"model": "switchyard/claude",
"max_tokens": 16,
"messages": [{"role": "user", "content": "hello"}]
})),
&[
("authorization", "Bearer claude-oauth-token"),
("anthropic-beta", "oauth-2025-04-20,unsupported-beta"),
("chatgpt-account-id", "account-123"),
("x-openai-fedramp", "true"),
],
)
.await?;
assert_eq!(response.status, StatusCode::OK);
let wrong_api = send_with_headers(
&app,
"POST",
"/v1/responses",
Some(json!({"model": "switchyard/claude", "input": "hello"})),
&[("authorization", "Bearer codex-login-token")],
)
.await?;
assert_eq!(wrong_api.status, StatusCode::BAD_REQUEST);
assert_eq!(upstream.calls.lock().await.len(), 1);
Ok(())
}
#[tokio::test]
async fn responses_client_forwards_openai_login_when_configured() -> TestResult {
let upstream = MockUpstream::start().await?;
let state = load_test_config(&format!(
r#"
schema_version = 1
[llm_clients.openai]
format = "openai_responses"
base_url = "{base_url}"
forward_auth = true
max_retries = 0
[targets.openai]
id = "gpt-codex"
llm_client = "openai"
[routes.openai]
id = "switchyard/codex"
type = "passthrough"
target = "openai"
"#,
base_url = upstream.base_url
))?;
let app = build_switchyard_router(state);
let response = send_with_headers(
&app,
"POST",
"/v1/responses",
Some(json!({"model": "switchyard/codex", "input": "hello"})),
&[
("authorization", "Bearer codex-login-token"),
("chatgpt-account-id", "account-123"),
("x-openai-fedramp", "true"),
("x-api-key", "provider-api-key"),
("anthropic-beta", "provider-beta"),
],
)
.await?;
assert_eq!(response.status, StatusCode::OK);
let redirect = send_with_headers(
&app,
"POST",
"/v1/responses",
Some(json!({"model": "switchyard/codex", "input": "hello"})),
&[
("authorization", "Bearer codex-login-token"),
("chatgpt-account-id", "account-123"),
("x-openai-fedramp", "true"),
("x-test-redirect", "1"),
],
)
.await?;
assert_eq!(redirect.status, StatusCode::TEMPORARY_REDIRECT);
assert_eq!(upstream.calls.lock().await.len(), 2);
let echoed_auth = send_with_headers(
&app,
"POST",
"/v1/responses",
Some(json!({"model": "switchyard/codex", "input": "hello"})),
&[
("authorization", "Bearer codex-login-token"),
("x-test-echo-auth", "1"),
],
)
.await?;
assert_eq!(echoed_auth.status, StatusCode::UNAUTHORIZED);
let error = echoed_auth.text()?;
assert!(error.contains("[REDACTED]"));
assert!(!error.contains("codex-login-token"));
Ok(())
}
/// Serves `/v1/responses` and `/v1/messages` for a stub gateway on one host and records the
/// path and `authorization` values of each call. `/v1/responses` calls get a judge verdict.
async fn upstream_gateway_records_auth(
State(calls): State<Arc<Mutex<Vec<Value>>>>,
uri: Uri,
headers: HeaderMap,
Json(body): Json<Value>,
) -> HttpResponse {
let authorization: Vec<_> = headers
.get_all("authorization")
.iter()
.filter_map(|value| value.to_str().ok())
.collect();
calls
.lock()
.await
.push(json!({"path": uri.path(), "authorization": authorization}));
let model = body["model"].as_str().unwrap_or_default();
if uri.path() == "/v1/responses" {
let verdict = json!({
"crux": "bounded task", "primary_rule": "SUP-1",
"capability_boundary": "supported", "p_solve": 0.9,
});
return Json(responses_body("resp_judge", model, &verdict.to_string())).into_response();
}
Json(json!({
"id": "msg_gateway", "type": "message", "role": "assistant", "model": model,
"content": [{"type": "text", "text": "ok"}],
"stop_reason": "end_turn", "stop_sequence": null,
"usage": {"input_tokens": 1, "output_tokens": 1}
}))
.into_response()
}
/// A route that mixes formats on one gateway forwards the caller's bearer token to both of the
/// gateway's APIs: `/v1/responses` for the judge and `/v1/messages` for the answer.
#[tokio::test]
async fn route_on_one_host_forwards_the_bearer_token_to_responses_and_messages() -> TestResult {
let calls = Arc::new(Mutex::new(Vec::new()));
let gateway = Router::new()
.route("/v1/responses", post(upstream_gateway_records_auth))
.route("/v1/messages", post(upstream_gateway_records_auth))
.with_state(Arc::clone(&calls));
let listener = TcpListener::bind("127.0.0.1:0").await?;
let host = format!("http://{}", listener.local_addr()?);
tokio::spawn(async move { axum::serve(listener, gateway).await });
let state = load_test_config(&format!(
r#"
schema_version = 1
[llm_clients.gateway_responses]
format = "openai_responses"
base_url = "{host}/v1"
forward_auth = true
max_retries = 0
[llm_clients.gateway_messages]
format = "anthropic_messages"
base_url = "{host}"
forward_auth = true
max_retries = 0
[targets]
judge = {{ id = "model/judge", llm_client = "gateway_responses" }}
capable = {{ id = "model/capable", llm_client = "gateway_messages" }}
efficient = {{ id = "model/efficient", llm_client = "gateway_messages" }}
[routes.agent]
id = "switchyard/agent"
type = "composite"
classifier = {{ target = "judge", base_threshold = 0.5, classify_trigger = "user_turn" }}
stage = {{ capable_target = "capable", efficient_target = "efficient", confidence_threshold = 0.5 }}
"#
))?;
let response = send_with_headers(
&build_switchyard_router(state),
"POST",
"/v1/responses",
Some(json!({"model": "switchyard/agent", "input": "hi there"})),
&[("authorization", "Bearer gateway-key")],
)
.await?;
assert_eq!(response.status, StatusCode::OK, "{}", response.text()?);
// The judge call and the answer call each carry the caller's bearer token once, unchanged.
assert_eq!(
*calls.lock().await,
[
json!({"path": "/v1/responses", "authorization": ["Bearer gateway-key"]}),
json!({"path": "/v1/messages", "authorization": ["Bearer gateway-key"]}),
]
);
Ok(())
}
#[tokio::test]
async fn routes_dispatch_and_discovery_endpoints_are_stable() -> TestResult {
let (upstream, app) = test_app(&[
("switchyard/coding", &["model/code"]),
("switchyard/general", &["model/general"]),
])
.await?;
let health = send(&app, "GET", "/health", None).await?;
assert_eq!(health.status, StatusCode::OK);
assert_eq!(health.json()?, json!({"status": "ok"}));
let models = send(&app, "GET", "/v1/models", None).await?;
assert_eq!(models.status, StatusCode::OK);
assert_eq!(
models.json()?["model_pool"],
json!(["switchyard/coding", "switchyard/general"])
);
let missing = send(&app, "GET", "/missing", None).await?;
assert_eq!(missing.status, StatusCode::NOT_FOUND);
assert_eq!(missing.json()?["error"]["code"], "endpoint_not_found");
for (route_model, target_model) in [
("switchyard/general", "model/general"),
("switchyard/coding", "model/code"),
] {
let response = send(
&app,
"POST",
"/v1/chat/completions",
Some(json!({
"model": route_model,
"messages": [{"role": "user", "content": "hi"}]
})),
)
.await?;
assert_eq!(response.status, StatusCode::OK);
assert_eq!(
response
.headers
.get("x-model-router-selected-model")
.and_then(|value| value.to_str().ok()),
Some(target_model)
);
}
let calls = upstream.calls.lock().await;
assert_eq!(calls[0]["model"], "model/general");
assert_eq!(calls[1]["model"], "model/code");
Ok(())
}
#[tokio::test]
async fn json_extractor_statuses_keep_api_specific_error_envelopes() -> TestResult {
let (_upstream, app) = test_app(&[(ROUTE_MODEL, &["model/a"])]).await?;
for (content_type, expected_status) in [
(Some("application/json"), StatusCode::BAD_REQUEST),
(None, StatusCode::UNSUPPORTED_MEDIA_TYPE),
(Some("text/plain"), StatusCode::UNSUPPORTED_MEDIA_TYPE),
] {
let body = if expected_status == StatusCode::BAD_REQUEST {
br#"{"model":"broken""#.to_vec()
} else {
br#"{"model":"valid-json"}"#.to_vec()
};
let response = send_raw_json(&app, "/v1/chat/completions", body, content_type).await?;
assert_eq!(response.status, expected_status);
let body = response.json()?;
assert_eq!(body["error"]["type"], "invalid_request_error");
assert_eq!(body["error"]["code"], "invalid_body");
}
let response = send_raw_json(
&app,
"/v1/chat/completions",
vec![b' '; DEFAULT_MAX_REQUEST_BODY_BYTES + 1],
Some("application/json"),
)
.await?;
assert_eq!(response.status, StatusCode::PAYLOAD_TOO_LARGE);
assert_eq!(response.json()?["error"]["code"], "invalid_body");
for (body, content_type, expected_status, expected_type) in [
(
br#"{"model":"broken""#.as_slice(),
Some("application/json"),
StatusCode::BAD_REQUEST,
"invalid_request_error",
),
(
br#"{"model":"valid-json"}"#.as_slice(),
None,
StatusCode::UNSUPPORTED_MEDIA_TYPE,
"api_error",
),
] {
let response = send_raw_json(&app, "/v1/messages", body.to_vec(), content_type).await?;
assert_eq!(response.status, expected_status);
let body = response.json()?;
assert_eq!(body["type"], "error");
assert_eq!(body["error"]["type"], expected_type);
}
let response = send_raw_json(
&app,
"/v1/messages",
vec![b' '; DEFAULT_MAX_REQUEST_BODY_BYTES + 1],
Some("application/json"),
)
.await?;
assert_eq!(response.status, StatusCode::PAYLOAD_TOO_LARGE);
let body = response.json()?;
assert_eq!(body["type"], "error");
assert_eq!(body["error"]["type"], "request_too_large");
Ok(())
}
#[tokio::test]
async fn models_endpoint_reports_declared_route_capabilities_and_null_when_undeclared() -> TestResult
{
const CONFIG: &str = r#"
schema_version = 1
[llm_clients.primary]
format = "openai_chat"
base_url = "https://example.test/v1"
[targets.shared]
id = "nvidia/deepseek-ai/deepseek-v4-pro"
llm_client = "primary"
[routes.declared]
id = "declared"
type = "passthrough"
target = "shared"
context_window = 1000000
tool_calling = true
vision = true
[routes.restricted]
id = "restricted"
type = "passthrough"
target = "shared"
context_window = 262000
tool_calling = false
vision = false
[routes.undeclared]
id = "undeclared"
type = "passthrough"
target = "shared"
"#;
let app = build_switchyard_router(load_test_config(CONFIG)?);
let models = send(&app, "GET", "/v1/models?client_version=0.152.0", None).await?;
assert_eq!(models.status, StatusCode::OK);
let body = models.json()?;
let data = body["data"].as_array().cloned().unwrap_or_default();
let entries = data
.iter()
.filter_map(|entry| entry["id"].as_str().map(|id| (id, entry)))
.collect::<BTreeMap<_, _>>();
let capabilities = |id: &str| &entries[id]["capabilities"];
assert_eq!(entries["declared"]["context_length"], json!(1_000_000));
assert_eq!(capabilities("declared")["tool_calling"], json!(true));
assert_eq!(entries["restricted"]["context_length"], json!(262_000));
assert_eq!(capabilities("restricted")["tool_calling"], json!(false));
assert_eq!(entries["undeclared"]["context_length"], json!(null));
assert_eq!(capabilities("undeclared")["tool_calling"], json!(null));
assert_eq!(capabilities("declared")["vision"], json!(true));
assert_eq!(capabilities("restricted")["vision"], json!(false));
assert_eq!(capabilities("undeclared")["vision"], json!(null));
assert_eq!(body["models"], json!([]));
Ok(())
}
// Build routes through the TOML loader to test disabled, enabled, and unset capabilities.
fn capability_app(base_url: &str, format: &str) -> TestResult<Router> {
Ok(build_switchyard_router(load_test_config(&format!(
r#"
schema_version = 1
[llm_clients.upstream]
format = "{format}"
base_url = "{base_url}"
[targets]
shared = {{ id = "model/efficient", llm_client = "upstream" }}
[routes]
restricted = {{ id = "restricted", type = "passthrough", target = "shared", vision = false, reasoning = false, tool_calling = false }}
enabled = {{ id = "enabled", type = "passthrough", target = "shared", vision = true, reasoning = true, tool_calling = true }}
undeclared = {{ id = "undeclared", type = "passthrough", target = "shared" }}
"#,
))?))
}
// Chat Completions, Responses, and Messages reject disabled inputs before dispatch.
// Allowed Responses input retains instructions and options after translation
// to Chat Completions.
#[tokio::test]
async fn disabled_route_capabilities_reject_requests_before_calling_upstream() -> TestResult {
let upstream = MockUpstream::start().await?;
let app = capability_app(&upstream.base_url, "openai_chat")?;
let cases = [
(
"/v1/chat/completions",
"vision",
json!({"messages": [{"role": "user", "content": [{"type": "image_url", "image_url": {"url": "https://example.test/image.png"}}]}]}),
),
(
"/v1/messages",
"vision",
json!({"messages": [{"role": "user", "content": [{"type": "image", "source": {"type": "url", "url": "https://example.test/image.png"}}]}]}),
),
(
"/v1/responses",
"vision",
json!({"input": [{"role": "user", "content": [{"type": "input_image", "image_url": "https://example.test/image.png"}]}]}),
),
(
"/v1/chat/completions",
"reasoning",
json!({"messages": [{"role": "user", "content": "hello"}], "reasoning_effort": "high"}),
),
(
"/v1/messages",
"reasoning",
json!({"messages": [{"role": "user", "content": "hello"}], "thinking": {"type": "enabled", "budget_tokens": 1024}}),
),
(
"/v1/responses",
"reasoning",
json!({"input": "hello", "reasoning": {"summary": "auto"}}),
),
(
"/v1/chat/completions",
"tool_calling",
json!({"messages": [{"role": "user", "content": "hello"}], "tools": [{"type": "function", "function": {"name": "exec_command", "parameters": {"type": "object"}}}], "tool_choice": "none", "parallel_tool_calls": false}),
),
(
"/v1/chat/completions",
"tool_calling",
json!({"messages": [{"role": "user", "content": "hello"}], "function_call": {"name": "exec_command", "type": "none"}}),
),
(
"/v1/chat/completions",
"tool_calling",
json!({"messages": [
{"role": "assistant", "tool_calls": [{"id": "call_1", "type": "function", "function": {"name": "exec_command", "arguments": "{}"}}]},
{"role": "tool", "tool_call_id": "call_1", "content": "done"}
], "tool_choice": "none"}),
),
(
"/v1/messages",
"tool_calling",
json!({"messages": [{"role": "user", "content": "hello"}], "tools": [{"name": "exec_command", "input_schema": {"type": "object"}}]}),
),
(
"/v1/responses",
"tool_calling",
json!({"input": [{"type": "additional_tools", "tools": [{"type": "function", "name": "exec_command", "parameters": {"type": "object"}}]}, {"role": "user", "content": "hello"}]}),
),
];
for (endpoint, capability, mut body) in cases {
body["model"] = json!("restricted");
let response = send(&app, "POST", endpoint, Some(body)).await?;
assert_eq!(
response.status,
StatusCode::BAD_REQUEST,
"{endpoint}: {capability}"
);
let error = response.json()?;
assert_eq!(error["error"]["type"], "invalid_request_error");
if endpoint == "/v1/messages" {
assert_eq!(error["type"], "error");
} else {
assert_eq!(error["error"]["code"], "unsupported_capability");
}
assert!(
error["error"]["message"]
.as_str()
.unwrap_or_default()
.contains(&format!("{capability} = false"))
);
}
assert!(upstream.calls.lock().await.is_empty());
for (endpoint, mut body) in [
(
"/v1/chat/completions",
json!({"messages": [{"role": "system", "content": "Keep the caller's instructions."}, {"role": "user", "content": "hello"}], "tool_choice": "none", "parallel_tool_calls": false}),
),
(
"/v1/chat/completions",
json!({"messages": [{"role": "system", "content": "Keep the caller's instructions."}, {"role": "user", "content": "hello"}], "function_call": "none"}),
),
(
"/v1/responses",
json!({"instructions": "Keep the caller's instructions.", "input": "hello", "tool_choice": "none", "parallel_tool_calls": false}),
),
(
"/v1/messages",
json!({"system": "Keep the caller's instructions.", "messages": [{"role": "user", "content": "hello"}], "tool_choice": {"type": "none"}}),
),
] {
body["model"] = json!("restricted");
let response = send(&app, "POST", endpoint, Some(body)).await?;
assert_eq!(response.status, StatusCode::OK, "{endpoint}");
}
let calls = upstream.calls.lock().await;
assert_eq!(calls.len(), 4);
assert!(
calls
.iter()
.all(|call| call["model"] == "model/efficient" && call.get("tools").is_none())
);
drop(calls);
let body = json!({
"model": "restricted", "instructions": "Keep the caller's instructions.", "input": "hello"
});
let response = send(&app, "POST", "/v1/responses", Some(body)).await?;
assert_eq!(response.status, StatusCode::OK);
for model in ["enabled", "undeclared"] {
let response = send(&app, "POST", "/v1/responses", Some(json!({
"model": model,
"instructions": "Keep the caller's instructions.",
"input": [{"role": "user", "content": [{"type": "input_image", "image_url": "https://example.test/image.png"}]}],
"reasoning": {"effort": "high"},
"tools": [{"type": "function", "name": "exec_command", "parameters": {"type": "object"}}]
}))).await?;
assert_eq!(response.status, StatusCode::OK);
}
let calls = upstream.calls.lock().await;
assert_eq!(calls.len(), 7);
assert!(
calls
.iter()
.all(|call| has_system_prompt(call, "Keep the caller's instructions."))
);
for call in &calls[5..] {
assert_eq!(call["reasoning_effort"], "high");
assert_eq!(call["tools"][0]["function"]["name"], "exec_command");
assert!(call["messages"].as_array().is_some_and(|messages| {
messages
.iter()
.any(|message| message["content"][0]["type"] == "image_url")
}));
}
Ok(())
}
// Approval replies can resume tool use without a tools field or a decoded tool call.
#[tokio::test]
async fn tool_approval_replies_follow_route_capabilities() -> TestResult {
let upstream = MockUpstream::start().await?;
let base_url = format!("{}/buffered", upstream.base_url.trim_end_matches("/v1"));
let app = capability_app(&base_url, "openai_responses")?;
let input = json!([
{"type": "mcp_approval_response", "approval_request_id": "approval_1", "approve": true}
]);
for model in ["restricted", "enabled", "undeclared"] {
let body = json!({"model": model, "input": input});
let response = send(&app, "POST", "/v1/responses", Some(body)).await?;
if model == "restricted" {
assert_eq!(response.status, StatusCode::BAD_REQUEST);
let error = response.json()?;
assert_eq!(error["error"]["code"], "unsupported_capability");
assert!(upstream.calls.lock().await.is_empty());
} else {
assert_eq!(response.status, StatusCode::OK);
}
}
let calls = upstream.calls.lock().await;
assert_eq!(calls.len(), 2);
for call in calls.iter() {
assert_eq!(call["input"], input);
}
Ok(())
}
// Tool results and blank user messages must not replace the classifier's task text.
#[tokio::test]
async fn subagent_tool_continuations_are_classified_on_every_request_across_apis() -> TestResult {
let upstream = MockUpstream::start().await?;
let app = build_switchyard_router(load_test_config(&format!(
r#"
schema_version = 1
[llm_clients.upstream]
format = "openai_chat"
base_url = "{base_url}"
[targets]
classifier = {{ id = "model/classifier", llm_client = "upstream" }}
strong = {{ id = "model/strong", llm_client = "upstream" }}
weak = {{ id = "model/weak", llm_client = "upstream" }}
[routes.agent]
id = "agent"
type = "passthrough"
target = "weak"
[routes.agent.subagents]
type = "llm_classifier"
mode = "custom"
classify_trigger = "every_request"
models = {{ judge = ["classifier"], capable = ["strong"], efficient = ["weak"], any = ["strong", "weak"] }}
default_target = "efficient"
prompt = "classify the delegated task"
response_schema = '''{{"type":"object","properties":{{"decision":{{"type":"object","properties":{{"target":{{"type":"string","enum":["capable","efficient"]}}}},"required":["target"],"additionalProperties":false}}}},"required":["decision"],"additionalProperties":false}}'''
[routes.agent.subagents.policy]
type = "target_selector"
selector = "/decision/target"
"#,
base_url = upstream.base_url
))?);
let headers = [
("x-claude-code-session-id", "root-session"),
("x-claude-code-agent-id", "child-agent"),
];
for (path, key, mut body, tool_turn) in [
(
"/v1/chat/completions",
"messages",
json!({"model":"agent","messages":[
{"role":"system","content":"child system instructions"},
{"role":"user","content":"harness context"},
{"role":"user","content":[{"type":"text","text":"injected context"},{"type":"text","text":"opening task: route to capable"}]}
]}),
vec![
json!({"role":"assistant","tool_calls":[{"id":"call_1","type":"function","function":{"name":"read_file","arguments":"{}"}}]}),
json!({"role":"tool","tool_call_id":"call_1","content":"tool output"}),
],
),
(
"/v1/messages",
"messages",
json!({"model":"agent","max_tokens":16,"system":"child system instructions","messages":[
{"role":"user","content":"harness context"},
{"role":"user","content":[{"type":"text","text":"injected context"},{"type":"text","text":"opening task: route to capable"}]}
]}),
vec![
json!({"role":"assistant","content":[{"type":"tool_use","id":"call_1","name":"read_file","input":{}}]}),
json!({"role":"user","content":[{"type":"tool_result","tool_use_id":"call_1","content":"tool output"}]}),
],
),
(
"/v1/responses",
"input",
json!({"model":"agent","instructions":"child system instructions","input":[
{"role":"user","content":"harness context"},
{"role":"user","content":[{"type":"input_text","text":"injected context"},{"type":"input_text","text":"opening task: route to capable"}]}
]}),
vec![
json!({"type":"function_call","call_id":"call_1","name":"read_file","arguments":"{}"}),
json!({"type":"function_call_output","call_id":"call_1","output":"tool output"}),
],
),
] {
for (step, (additions, expected_prompt)) in [
(vec![], "opening task: route to capable"),
(tool_turn.clone(), "opening task: route to capable"),
(
vec![json!({"role":"user","content":" \n\t"})],
"opening task: route to capable",
),
(
vec![json!({"role":"user","content":"continue task: route to capable"})],
"continue task: route to capable",
),
]
.into_iter()
.enumerate()
{
body[key]
.as_array_mut()
.ok_or("missing request messages")?
.extend(additions);
let response =
send_with_headers(&app, "POST", path, Some(body.clone()), &headers).await?;
assert_eq!(response.status, StatusCode::OK);
assert_eq!(
response.headers["x-model-router-selected-model"],
"model/strong"
);
let calls = upstream.calls.lock().await;
let judge = &calls[calls.len() - 2];
assert_eq!(judge["model"], "model/classifier");
assert_eq!(
judge["messages"],
json!([
{"role":"system","content":"classify the delegated task"},
{"role":"user","content":expected_prompt}
])
);
let answer = calls.last().ok_or("missing answer request")?.to_string();
for preserved in [
"child system instructions",
"harness context",
"injected context",
"opening task: route to capable",
expected_prompt,
] {
assert!(answer.contains(preserved));
}
if step > 0 {
assert!(answer.contains("tool output"));
assert!(answer.contains("call_1"));
}
}
let mut text_free = body;
text_free[key] = json!(tool_turn);
let response = send_with_headers(&app, "POST", path, Some(text_free), &headers).await?;
assert_eq!(response.status, StatusCode::OK);
assert_eq!(
response.headers["x-model-router-selected-model"],
"model/weak"
);
}
let stats = send(&app, "GET", "/v1/stats", None).await?.json()?;
assert_eq!(stats["classifier"]["total_requests"], 12);
Ok(())
}
// Codex's structured kind takes precedence over the flat `collab_spawn` header:
// maintenance uses the parent classifier; delegated work uses the subagent route.
#[tokio::test]
async fn codex_maintenance_uses_the_parent_classifier_across_apis() -> TestResult {
let upstream = MockUpstream::start().await?;
let app = build_switchyard_router(load_test_config(&format!(
r#"
schema_version = 1
[llm_clients.upstream]
format = "openai_chat"
base_url = "{base_url}"
[targets]
classifier = {{ id = "model/classifier", llm_client = "upstream" }}
strong = {{ id = "model/strong", llm_client = "upstream" }}
weak = {{ id = "model/weak", llm_client = "upstream" }}
[routes.agent]
id = "agent"
type = "composite"
classifier = {{ target = "classifier", base_threshold = 0.5, classify_trigger = "user_turn" }}
stage = {{ capable_target = "strong", efficient_target = "weak", confidence_threshold = 0.3 }}
subagents = {{ type = "passthrough", target = "strong" }}
"#,
base_url = upstream.base_url
))?);
let cases = [
(Some("compact"), None, "model/weak"),
(Some("review"), None, "model/strong"),
(Some("thread_spawn"), None, "model/strong"),
(Some("collab_spawn"), None, "model/strong"),
(Some("unknown"), None, "model/weak"),
(Some("memory_consolidation"), None, "model/weak"),
(None, None, "model/strong"),
(Some("review"), Some("false"), "model/weak"),
(Some("compact"), Some("true"), "model/weak"),
];
for (path, body, cases) in [
(
"/v1/chat/completions",
json!({"model":"agent","messages":[{"role":"user","content":"hi"}]}),
&cases[..],
),
(
"/v1/messages",
json!({"model":"agent","max_tokens":16,"messages":[{"role":"user","content":"hi"}]}),
&cases[..3],
),
(
"/v1/responses",
json!({"model":"agent","input":"hi"}),
&cases[..3],
),
] {
for (kind, explicit, expected) in cases {
let mut headers = Vec::new();
let mut metadata = json!({
"thread_id": "child",
"parent_thread_id": "root",
"thread_source": "subagent",
});
if let Some(kind) = kind {
metadata["subagent_kind"] = json!(kind);
headers.push(("x-openai-subagent", "collab_spawn"));
}
let metadata = metadata.to_string();
headers.push(("x-codex-turn-metadata", metadata.as_str()));
if let Some(explicit) = explicit {
headers.push(("x-switchyard-is-subagent", explicit));
}
let response =
send_with_headers(&app, "POST", path, Some(body.clone()), &headers).await?;
assert_eq!(response.status, StatusCode::OK);
assert_eq!(response.headers["x-model-router-selected-model"], *expected);
}
}
let models = upstream.models().await;
for (model, expected) in [
("model/classifier", 7),
("model/weak", 7),
("model/strong", 8),
] {
assert_eq!(
models.iter().filter(|actual| *actual == model).count(),
expected
);
}
let stats = send(&app, "GET", "/v1/stats", None).await?.json()?;
assert_eq!(stats["classifier"]["total_requests"], 7);
Ok(())
}
#[tokio::test]
async fn all_inbound_formats_run_libsy_and_return_the_caller_format() -> TestResult {
let (upstream, app) = test_app(&[(ROUTE_MODEL, &["model/a"])]).await?;
let cases = [
(
"/v1/chat/completions",
json!({
"model": ROUTE_MODEL,
"messages": [{"role": "user", "content": "hi"}]
}),
),
(
"/v1/messages",
json!({
"model": ROUTE_MODEL,
"max_tokens": 16,
"messages": [{"role": "user", "content": "hi"}]
}),
),
(
"/v1/responses",
json!({"model": ROUTE_MODEL, "input": "hi"}),
),
];
let mut responses = Vec::new();
for (path, body) in cases {
responses.push(send(&app, "POST", path, Some(body)).await?);
}
assert!(
responses
.iter()
.all(|response| response.status == StatusCode::OK)
);
assert_eq!(
responses[0].json()?["choices"][0]["message"]["content"],
"ok"
);
assert_eq!(responses[1].json()?["content"][0]["text"], "ok");
assert_eq!(
responses[2].json()?["output"][0]["content"][0]["text"],
"ok"
);
assert_eq!(responses[0].json()?["usage"]["prompt_tokens"], 10);
assert_eq!(
responses[0].json()?["usage"]["prompt_tokens_details"]["cached_tokens"],
7
);
assert_eq!(responses[1].json()?["usage"]["input_tokens"], 3);
assert_eq!(responses[1].json()?["usage"]["cache_read_input_tokens"], 7);
assert_eq!(responses[2].json()?["usage"]["input_tokens"], 10);
assert_eq!(
responses[2].json()?["usage"]["input_tokens_details"]["cached_tokens"],
7
);
for response in &responses {
assert_eq!(
response
.headers
.get("x-model-router-selected-model")
.and_then(|value| value.to_str().ok()),
Some("model/a")
);
// The body names the model that answered, not the route id the caller
// addressed, so it agrees with the routing header above.
assert_eq!(response.json()?["model"], "model/a");
}
let calls = upstream.calls.lock().await;
assert_eq!(calls.len(), 3);
assert!(calls.iter().all(|call| call["model"] == "model/a"));
Ok(())
}
// Normalized metadata is authoritative when both ID forms are present;
// legacy-only callers remain supported for backward compatibility.
#[tokio::test]
async fn routing_log_prefers_canonical_and_preserves_legacy_fallback() -> TestResult {
let upstream = MockUpstream::start().await?;
let temp_dir = tempfile::tempdir()?;
let log_path = temp_dir.path().join("routing.jsonl");
let state = random_state(
&upstream.base_url,
&[(ROUTE_MODEL, &["anthropic/claude-haiku-4.5"])],
)?
.with_routing_log(&log_path)?;
let app = build_switchyard_router(state);
let request = HttpRequest::builder()
.method("POST")
.uri("/v1/chat/completions")
.header("content-type", "application/json")
.header("x-switchyard-session-id", "canonical-session")
.header("x-switchyard-turn-id", "canonical-turn")
.header("proxy_x_session_id", "legacy-session")
.header("x-switchyard-origin", r#"custom-agent/"quoted"\path"#)
.body(Body::from(serde_json::to_vec(&json!({
"model": ROUTE_MODEL,
"messages": [{"role": "user", "content": "hello"}]
}))?))?;
let response = app.clone().oneshot(request).await?;
assert_eq!(response.status(), StatusCode::OK);
let stats = send(
&app,
"GET",
"/v1/routing/session-stats?session_id=canonical-session",
None,
)
.await?;
assert_eq!(stats.status, StatusCode::OK);
let stats = stats.json()?;
assert_eq!(stats["total_calls"], 1);
assert_eq!(stats["total_prompt_tokens"], 10);
assert_eq!(stats["total_cached_tokens"], 7);
assert_eq!(
stats["models"]["anthropic/claude-haiku-4.5"]["completion_tokens"],
2
);
let legacy = send(
&app,
"GET",
"/v1/routing/session-stats?session_id=legacy-session",
None,
)
.await?;
assert_eq!(legacy.status, StatusCode::NOT_FOUND);
let legacy_only = send_with_headers(
&app,
"POST",
"/v1/chat/completions",
Some(json!({
"model": ROUTE_MODEL,
"messages": [{"role": "user", "content": "hello"}]
})),
&[("proxy_x_session_id", "legacy-only-session")],
)
.await?;
assert_eq!(legacy_only.status, StatusCode::OK);
let legacy_stats = send(
&app,
"GET",
"/v1/routing/session-stats?session_id=legacy-only-session",
None,
)
.await?;
assert_eq!(legacy_stats.status, StatusCode::OK);
assert_eq!(legacy_stats.json()?["total_calls"], 1);
let records = std::fs::read_to_string(log_path)?;
let first: Value =
serde_json::from_str(records.lines().next().ok_or("routing log was empty")?)?;
assert_eq!(first["session_id"], "canonical-session");
assert_eq!(first["model"], "anthropic/claude-haiku-4.5");
assert_eq!(first["cost"]["source"], "estimated");
assert!((first["cost"]["usd"].as_f64().ok_or("missing cost")? - 0.0000137).abs() < 1e-12);
assert_eq!(first["turn_id"], "canonical-turn");
assert_eq!(first["origin"], r#"custom-agent/"quoted"\path"#);
let second: Value = serde_json::from_str(records.lines().nth(1).ok_or("missing record")?)?;
assert_eq!(second["turn_id"], Value::Null);
assert_eq!(second.get("origin"), Some(&Value::Null));
assert!(
first["ts"]
.as_str()
.is_some_and(|value| value.ends_with('Z'))
);
Ok(())
}
/// Two routes serving one model must remain distinguishable in the durable accounting record.
#[tokio::test]
async fn routing_log_attributes_shared_models_to_the_requested_route() -> TestResult {
let upstream = MockUpstream::start().await?;
let temp_dir = tempfile::tempdir()?;
let log_path = temp_dir.path().join("routing.jsonl");
let state = load_test_config(&format!(
r#"
schema_version = 1
[llm_clients.upstream]
format = "openai_chat"
base_url = "{base_url}"
[targets.shared]
id = "model/route-sanity"
llm_client = "upstream"
reasoning_effort = "medium"
[targets.capable]
id = "model/route-sanity"
llm_client = "upstream"
reasoning_effort = "max"
[routes.passthrough]
id = "route/passthrough"
type = "passthrough"
target = "shared"
[routes.stage]
id = "route/stage"
type = "stage_router"
capable_target = "capable"
efficient_target = "shared"
picker = "efficient_first"
confidence_threshold = 1.0
"#,
base_url = upstream.base_url
))?
.with_routing_log(&log_path)?;
let app = build_switchyard_router(state);
let before = send(&app, "GET", "/metrics", None).await?;
let normal = json!([{"role": "user", "content": "hello"}]);
let critical = json!([
{"role": "user", "content": "fix the build"},
{"role": "assistant", "tool_calls": [{
"id": "call_1", "type": "function",
"function": {"name": "Bash", "arguments": "{\"command\": \"cargo test\"}"}
}]},
{"role": "tool", "tool_call_id": "call_1", "content": "fatal runtime error: out of memory"}
]);
for (route_id, messages) in [
("route/passthrough", &normal),
("route/stage", &normal),
("route/stage", &critical),
] {
for streaming in [false, true] {
let response = send_with_headers(
&app,
"POST",
"/v1/chat/completions",
Some(json!({"model": route_id, "messages": messages, "stream": streaming})),
&[("x-switchyard-session-id", "route-sanity")],
)
.await?;
assert_eq!(response.status, StatusCode::OK, "{route_id}");
assert_eq!(
response.headers["x-model-router-selected-model"],
"model/route-sanity"
);
if streaming {
assert!(response.text()?.contains("model/route-sanity"));
} else {
assert_eq!(response.json()?["model"], "model/route-sanity");
}
}
}
let records = std::fs::read_to_string(&log_path)?
.lines()
.map(serde_json::from_str::<Value>)
.collect::<Result<Vec<_>, _>>()?;
assert_eq!(records.len(), 6);
for (index, record) in records.iter().enumerate() {
let (route, algorithm) = if index < 2 {
("route/passthrough", "passthrough")
} else {
("route/stage", "stage_router")
};
let (target, effort) = if index < 4 {
("shared", "medium")
} else {
("capable", "max")
};
assert_eq!(record["route_id"], route);
assert_eq!(record["algorithm"], algorithm);
assert_eq!(record["model"], "model/route-sanity");
assert_eq!(record["target"], target);
assert_eq!(record["reasoning_effort"], effort);
assert_eq!(record["session_id"], "route-sanity");
}
let calls = upstream.calls.lock().await;
assert_eq!(calls.len(), 6);
for (index, call) in calls.iter().enumerate() {
assert_eq!(call["model"], "model/route-sanity");
assert_eq!(
call["reasoning_effort"],
if index < 4 { "medium" } else { "max" }
);
}
drop(calls);
let stats = send(&app, "GET", "/v1/stats", None).await?.json()?;
let session = send(
&app,
"GET",
"/v1/routing/session-stats?session_id=route-sanity",
None,
)
.await?
.json()?;
assert_eq!(stats["total_requests"], 6);
assert_eq!(stats["total_errors"], 0);
assert_eq!(session["total_calls"], 6);
for snapshot in [&stats, &session] {
assert_eq!(snapshot["models"].as_object().unwrap().len(), 1);
assert_eq!(snapshot["models"]["model/route-sanity"]["calls"], 6);
for (target, effort, calls) in [("shared", "medium", 4), ("capable", "max", 2)] {
assert_eq!(snapshot["targets"][target]["model"], "model/route-sanity");
assert_eq!(snapshot["targets"][target]["reasoning_effort"], effort);
assert_eq!(snapshot["targets"][target]["calls"], calls);
}
}
assert_eq!(
stats["total_tokens"]["prompt"],
session["total_prompt_tokens"]
);
assert_eq!(
stats["total_tokens"]["completion"],
session["total_completion_tokens"]
);
for source in ["override", "capable_hold"] {
assert_eq!(
stats["algorithm_stats"]["stage_router"]["routing_decisions"][source]["targets"]["model/route-sanity"],
1
);
}
let after = send(&app, "GET", "/metrics", None).await?;
for (metric, label, expected) in [
("switchyard_requests_total", "model", 6.0),
(
"switchyard_prompt_tokens_total",
"model",
session["total_prompt_tokens"].as_f64().unwrap(),
),
(
"switchyard_completion_tokens_total",
"model",
session["total_completion_tokens"].as_f64().unwrap(),
),
] {
assert_eq!(
metric_delta(
before.text()?,
after.text()?,
metric,
&[(label, "model/route-sanity")]
),
Some(expected),
"{metric}"
);
for target in ["shared", "capable"] {
assert!(
metric_line(after.text()?, metric, &[(label, target)]).is_none(),
"{metric}"
);
}
}
for (algorithm, calls) in [("passthrough", 2.0), ("stage_router", 4.0)] {
assert_eq!(
metric_delta(
before.text()?,
after.text()?,
"switchyard_decisions_total",
&[
("algorithm", algorithm),
("selected_model", "model/route-sanity")
],
),
Some(calls),
);
}
Ok(())
}
#[tokio::test]
async fn routing_log_keeps_the_canonical_session_id_until_a_stream_drains() -> TestResult {
let upstream = MockUpstream::start().await?;
let temp_dir = tempfile::tempdir()?;
let log_path = temp_dir.path().join("routing.jsonl");
let state = random_state(
&upstream.base_url,
&[(ROUTE_MODEL, &["anthropic/claude-haiku-4.5"])],
)?
.with_routing_log(&log_path)?;
let app = build_switchyard_router(state);
// `send_with_headers` collects the response body, so the stream wrapper reaches
// its terminal usage record before the stats query runs.
let response = send_with_headers(
&app,
"POST",
"/v1/chat/completions",
Some(json!({
"model": ROUTE_MODEL,
"messages": [{"role": "user", "content": "hello"}],
"stream": true
})),
&[
("x-switchyard-session-id", "streaming-session"),
("x-switchyard-turn-id", "streaming-turn"),
("x-switchyard-origin", "codex-cli"),
],
)
.await?;
assert_eq!(response.status, StatusCode::OK);
assert!(response.text()?.contains("data: [DONE]"));
let stats = send(
&app,
"GET",
"/v1/routing/session-stats?session_id=streaming-session",
None,
)
.await?;
assert_eq!(stats.status, StatusCode::OK);
let stats = stats.json()?;
assert_eq!(stats["total_calls"], 1);
assert_eq!(stats["total_prompt_tokens"], 12);
assert_eq!(stats["total_cached_tokens"], 7);
assert_eq!(stats["total_cache_creation_tokens"], 2);
assert_eq!(stats["total_completion_tokens"], 5);
let record: Value = serde_json::from_str(&std::fs::read_to_string(log_path)?)?;
assert_eq!(record["route_id"], ROUTE_MODEL);
assert_eq!(record["model"], "anthropic/claude-haiku-4.5");
assert_eq!(record["cost"]["source"], "estimated");
assert!((record["cost"]["usd"].as_f64().ok_or("missing cost")? - 0.0000312).abs() < 1e-12);
assert_eq!(record["turn_id"], "streaming-turn");
assert_eq!(record["algorithm"], "random");
assert_eq!(record["origin"], "codex-cli");
Ok(())
}
#[tokio::test]
async fn random_zero_weight_target_is_not_advertised_or_called() -> TestResult {
let upstream = MockUpstream::start().await?;
let app = build_switchyard_router(weighted_random_state(&upstream.base_url, [1, 0])?);
let response = send(
&app,
"POST",
"/v1/decision",
Some(json!({
"input_format": "openai_chat",
"request": {
"model": ROUTE_MODEL,
"messages": [{"role": "user", "content": "unavailable"}]
}
})),
)
.await?;
assert_eq!(response.status, StatusCode::OK);
let decision = response.json()?;
assert_eq!(decision["selected"]["target"], "first");
assert_eq!(decision["fallbacks"], json!([]));
assert!(upstream.models().await.is_empty());
for (path, body) in [
(
"/v1/chat/completions",
json!({
"model": ROUTE_MODEL,
"messages": [{"role": "user", "content": "unavailable"}]
}),
),
(
"/v1/messages",
json!({
"model": ROUTE_MODEL,
"max_tokens": 16,
"messages": [{"role": "user", "content": "unavailable"}]
}),
),
(
"/v1/responses",
json!({"model": ROUTE_MODEL, "input": "unavailable"}),
),
] {
upstream.calls.lock().await.clear();
let response = send(&app, "POST", path, Some(body)).await?;
assert_eq!(response.status, StatusCode::SERVICE_UNAVAILABLE);
assert_eq!(upstream.models().await, ["model/weak"]);
}
Ok(())
}
#[tokio::test]
async fn unavailable_target_fails_over_across_endpoints_and_stops_when_exhausted() -> TestResult {
let upstream = MockUpstream::start().await?;
let temp_dir = tempfile::tempdir()?;
let log_path = temp_dir.path().join("routing.jsonl");
// The fixed seed selects `first` for these requests while keeping `second` enabled.
let state =
weighted_random_state(&upstream.base_url, [1000, 1])?.with_routing_log(&log_path)?;
let app = build_switchyard_router(state);
let cases = [
(
"/v1/chat/completions",
json!({
"model": ROUTE_MODEL,
"messages": [{"role": "user", "content": "unavailable"}]
}),
),
(
"/v1/messages",
json!({
"model": ROUTE_MODEL,
"max_tokens": 16,
"messages": [{"role": "user", "content": "unavailable"}]
}),
),
(
"/v1/responses",
json!({"model": ROUTE_MODEL, "input": "unavailable"}),
),
];
for (path, body) in cases {
let previous_call_count = upstream.calls.lock().await.len();
let response = send(&app, "POST", path, Some(body)).await?;
assert_eq!(response.status, StatusCode::OK);
assert_eq!(
response
.headers
.get("x-model-router-selected-model")
.and_then(|value| value.to_str().ok()),
Some("model/strong")
);
assert_eq!(response.json()?["model"], "model/strong");
let calls = upstream.calls.lock().await;
let candidate_calls = &calls[previous_call_count..];
assert_eq!(
candidate_calls
.iter()
.map(|call| call["model"].as_str().unwrap_or(""))
.collect::<Vec<_>>(),
["model/weak", "model/strong"]
);
assert!(has_system_prompt(&candidate_calls[0], "weak answer prompt"));
assert!(has_system_prompt(
&candidate_calls[1],
"strong answer prompt"
));
assert!(!has_system_prompt(
&candidate_calls[1],
"weak answer prompt"
));
}
let stats = send(&app, "GET", "/v1/stats", None).await?.json()?;
// Fallback causes are logged rather than accumulated in the legacy stats counters.
assert_eq!(stats["routing_fallbacks"]["unavailable"], 0);
assert_eq!(stats["routing_fallbacks"]["context_window"], 0);
assert_eq!(stats["models"]["model/strong"]["calls"], 3);
assert_eq!(stats["models"]["model/weak"]["errors"], 3);
let records = std::fs::read_to_string(&log_path)?;
let records = records
.lines()
.map(serde_json::from_str::<Value>)
.collect::<Result<Vec<_>, _>>()?;
assert_eq!(records.len(), 6);
assert_eq!(
records
.iter()
.filter(|record| {
record["model"] == "model/strong"
&& record["target"] == "second"
&& record.get("fallback_reason").is_none()
})
.count(),
3
);
let previous_call_count = upstream.calls.lock().await.len();
let response = send(
&app,
"POST",
"/v1/chat/completions",
Some(json!({
"model": ROUTE_MODEL,
"messages": [{"role": "user", "content": "all-unavailable"}]
})),
)
.await?;
assert_eq!(response.status, StatusCode::SERVICE_UNAVAILABLE);
let error = response.json()?;
assert_eq!(error["error"]["type"], "upstream_error");
assert_eq!(error["error"]["code"], "upstream_error");
let calls = upstream.calls.lock().await;
assert_eq!(
calls[previous_call_count..]
.iter()
.map(|call| call["model"].as_str().unwrap_or(""))
.collect::<Vec<_>>(),
["model/weak", "model/strong"]
);
Ok(())
}
#[tokio::test]
async fn streaming_response_is_framed_for_the_inbound_api() -> TestResult {
let (_upstream, app) = test_app(&[(ROUTE_MODEL, &["model/a"])]).await?;
let response = send(
&app,
"POST",
"/v1/chat/completions",
Some(json!({
"model": ROUTE_MODEL,
"messages": [{"role": "user", "content": "hi"}],
"stream": true
})),
)
.await?;
assert_eq!(response.status, StatusCode::OK);
assert!(response.text()?.contains("hello"));
assert!(response.text()?.contains("data: [DONE]"));
Ok(())
}
// SWITCH-922: every streaming codec must report the routed target, not the route
// id the caller addressed — the route id is meaningless to anything reading the
// trajectory (a Bench UI, a spend log, the client's own display).
#[tokio::test]
async fn streamed_response_model_names_the_served_model_not_the_route() -> TestResult {
let (_upstream, app) = test_app(&[(ROUTE_MODEL, &["model/a"])]).await?;
// Each case names the JSON pointer to the model on that format's first event.
let cases = [
(
"/v1/chat/completions",
json!({
"model": ROUTE_MODEL,
"messages": [{"role": "user", "content": "hi"}],
"stream": true
}),
vec!["model"],
),
(
"/v1/messages",
json!({
"model": ROUTE_MODEL,
"max_tokens": 16,
"messages": [{"role": "user", "content": "hi"}],
"stream": true
}),
vec!["message", "model"],
),
(
"/v1/responses",
json!({"model": ROUTE_MODEL, "input": "hi", "stream": true}),
vec!["response", "model"],
),
];
for (path, body, pointer) in cases {
let response = send(&app, "POST", path, Some(body)).await?;
assert_eq!(response.status, StatusCode::OK, "{path}");
let first = first_sse_event(response.text()?)
.ok_or_else(|| format!("{path} produced no SSE data frames"))?;
let model = pointer
.iter()
.try_fold(&first, |value, key| value.get(key))
.and_then(Value::as_str);
assert_eq!(model, Some("model/a"), "{path}");
}
Ok(())
}
// Returns the first `data:` frame of an SSE body as JSON, skipping `[DONE]`.
fn first_sse_event(body: &str) -> Option<Value> {
body.lines()
.filter_map(|line| line.strip_prefix("data: "))
.filter(|data| *data != "[DONE]")
.find_map(|data| serde_json::from_str(data).ok())
}
#[tokio::test]
async fn streaming_success_records_only_final_usage_and_one_latency() -> TestResult {
const MODEL: &str = "model/stream-success";
let (_upstream, app) = test_app(&[(ROUTE_MODEL, &[MODEL])]).await?;
let before = send(&app, "GET", "/metrics", None).await?;
let before = before.text()?;
let response = send(
&app,
"POST",
"/v1/chat/completions",
Some(json!({
"model": ROUTE_MODEL,
"messages": [{"role": "user", "content": "stream-success"}],
"stream": true
})),
)
.await?;
assert_eq!(response.status, StatusCode::OK);
assert_in_order(
response.text()?,
&[
"hello",
"-partial",
"-final",
"\"finish_reason\":\"stop\"",
"[DONE]",
],
);
let after = send(&app, "GET", "/metrics", None).await?;
let after = after.text()?;
for (name, expected_delta) in [
("switchyard_prompt_tokens_total", 12.0),
("switchyard_completion_tokens_total", 5.0),
("switchyard_cached_tokens_total", 7.0),
("switchyard_cache_creation_tokens_total", 2.0),
("switchyard_reasoning_tokens_total", 3.0),
("switchyard_total_latency_ms_count", 1.0),
] {
assert_eq!(
metric_delta(before, after, name, &[("model", MODEL)]),
Some(expected_delta),
"unexpected delta for {name}"
);
}
let stats = send(&app, "GET", "/v1/stats", None).await?.json()?;
assert_eq!(stats["total_requests"], 1);
assert_eq!(
stats["total_tokens"],
json!({
"prompt": 12, "completion": 5, "cached": 7,
"cache_creation": 2, "reasoning": 3, "total": 17
})
);
assert_eq!(stats["models"][MODEL]["model_call_latency"]["count"], 1);
assert_eq!(stats["models"][MODEL]["total_latency"]["count"], 1);
Ok(())
}
#[tokio::test]
// A terminal stream failure records errors without usage or terminal latency.
async fn streaming_error_records_error_without_usage_or_latency() -> TestResult {
const MODEL: &str = "model/stream-error";
let (_upstream, app) = test_app(&[(ROUTE_MODEL, &[MODEL])]).await?;
let before = send(&app, "GET", "/metrics", None).await?;
let before = before.text()?;
let response = send(
&app,
"POST",
"/v1/chat/completions",
Some(json!({
"model": ROUTE_MODEL,
"messages": [{"role": "user", "content": "stream-error"}],
"stream": true
})),
)
.await?;
assert_eq!(response.status, StatusCode::OK);
assert_in_order(
response.text()?,
&["before", "still here", "upstream stream failed"],
);
let after = send(&app, "GET", "/metrics", None).await?;
let after = after.text()?;
for name in [
"switchyard_requests_total",
"switchyard_model_call_latency_ms_count",
"switchyard_prompt_tokens_total",
"switchyard_completion_tokens_total",
"switchyard_cached_tokens_total",
"switchyard_cache_creation_tokens_total",
"switchyard_reasoning_tokens_total",
"switchyard_total_latency_ms_count",
] {
assert_eq!(
metric_value(after, name, &[("model", MODEL)]),
metric_value(before, name, &[("model", MODEL)]),
"{name} changed after a failed stream"
);
}
assert_eq!(
metric_delta(
before,
after,
"switchyard_errors_total",
&[("model", MODEL)]
),
Some(1.0)
);
assert!(metric_delta(before, after, "switchyard_total_errors", &[]).unwrap_or_default() >= 1.0);
let stats = send(&app, "GET", "/v1/stats", None).await?.json()?;
assert_eq!(stats["total_requests"], 1);
assert_eq!(stats["total_errors"], 1);
assert_eq!(stats["total_tokens"], empty_token_totals());
assert_eq!(stats["models"][MODEL]["calls"], 1);
assert_eq!(stats["models"][MODEL]["errors"], 1);
assert_eq!(stats["models"][MODEL]["total_latency"]["count"], 0);
assert_eq!(stats["routing_overhead"]["count"], 1);
Ok(())
}
#[tokio::test]
async fn responses_stream_error_does_not_emit_success_terminal_events() -> TestResult {
// A distinct target keeps this test's error-counter increments off the shared
// model/stream-error metric that streaming_error_records_... asserts an exact delta on.
let (_upstream, app) = test_app(&[(ROUTE_MODEL, &["model/responses-stream-error"])]).await?;
let response = send(
&app,
"POST",
"/v1/responses",
Some(json!({
"model": ROUTE_MODEL,
"input": "stream-error",
"stream": true
})),
)
.await?;
assert_eq!(response.status, StatusCode::OK);
let body = response.text()?;
assert_in_order(body, &["before", "upstream stream failed"]);
for event_type in [
"response.output_text.done",
"response.content_part.done",
"response.output_item.done",
"response.completed",
] {
assert!(
!body.contains(event_type),
"{event_type} followed an upstream stream error"
);
}
Ok(())
}
#[tokio::test]
async fn chat_stream_error_does_not_emit_success_terminal_chunk() -> TestResult {
// A distinct target keeps this test's error-counter increments off the shared
// model/stream-error metric that streaming_error_records_... asserts an exact delta on.
let (_upstream, app) = test_app(&[(ROUTE_MODEL, &["model/chat-stream-error"])]).await?;
let response = send(
&app,
"POST",
"/v1/chat/completions",
Some(json!({
"model": ROUTE_MODEL,
"messages": [{"role": "user", "content": "stream-error"}],
"stream": true
})),
)
.await?;
assert_eq!(response.status, StatusCode::OK);
let body = response.text()?;
assert_in_order(body, &["before", "still here", "upstream stream failed"]);
// The finalizer must not synthesize a `finish_reason: stop` completion chunk after the error.
let after_error = body
.split_once("upstream stream failed")
.map(|(_, rest)| rest)
.unwrap_or_default();
assert!(
!after_error.contains(r#""finish_reason":"stop""#),
"a finish_reason=stop chunk followed an upstream stream error:\n{body}"
);
// `[DONE]` is the Chat success sentinel: an SDK client stops there and keeps the
// truncated turn as a completed answer, so a failed stream must not emit it.
assert!(
!after_error.contains("[DONE]"),
"a [DONE] success sentinel followed an upstream stream error:\n{body}"
);
Ok(())
}
#[tokio::test]
async fn anthropic_stream_error_does_not_emit_success_terminal_events() -> TestResult {
// A distinct target keeps this test's error-counter increments off the shared
// model/stream-error metric that streaming_error_records_... asserts an exact delta on.
let (_upstream, app) = test_app(&[(ROUTE_MODEL, &["model/anthropic-stream-error"])]).await?;
let response = send(
&app,
"POST",
"/v1/messages",
Some(json!({
"model": ROUTE_MODEL,
"messages": [{"role": "user", "content": "stream-error"}],
"max_tokens": 16,
"stream": true
})),
)
.await?;
assert_eq!(response.status, StatusCode::OK);
let body = response.text()?;
assert_in_order(body, &["before", "upstream stream failed"]);
// The finalizer must not close the turn with message_delta/message_stop after the error.
let after_error = body
.split_once("upstream stream failed")
.map(|(_, rest)| rest)
.unwrap_or_default();
for event_type in ["message_delta", "message_stop"] {
assert!(
!after_error.contains(event_type),
"{event_type} followed an upstream stream error:\n{body}"
);
}
Ok(())
}
#[tokio::test]
async fn request_and_upstream_errors_use_the_inbound_wire_format() -> TestResult {
let (_upstream, app) = test_app(&[(ROUTE_MODEL, &["model/a"])]).await?;
let unknown = send(
&app,
"POST",
"/v1/chat/completions",
Some(json!({
"model": "other",
"messages": [{"role": "user", "content": "hi"}]
})),
)
.await?;
assert_eq!(unknown.status, StatusCode::NOT_FOUND);
assert_eq!(unknown.json()?["error"]["code"], "model_not_found");
let missing_model = send(
&app,
"POST",
"/v1/chat/completions",
Some(json!({"messages": [{"role": "user", "content": "hi"}]})),
)
.await?;
assert_eq!(missing_model.status, StatusCode::BAD_REQUEST);
assert_eq!(
missing_model.json()?["error"]["code"],
"invalid_request_error"
);
let upstream_cases = [
(
"/v1/chat/completions",
json!({
"model": ROUTE_MODEL,
"messages": [{"role": "user", "content": "auth-fail"}]
}),
json!({
"error": {
"message": "upstream authentication failed",
"type": "upstream_error",
"code": "upstream_error"
}
}),
),
(
"/v1/responses",
json!({"model": ROUTE_MODEL, "input": "auth-fail"}),
json!({
"error": {
"message": "upstream authentication failed",
"type": "upstream_error",
"code": "upstream_error"
}
}),
),
(
"/v1/messages",
json!({
"model": ROUTE_MODEL,
"max_tokens": 64,
"messages": [{"role": "user", "content": "auth-fail"}]
}),
json!({
"type": "error",
"error": {
"type": "authentication_error",
"message": "upstream authentication failed"
}
}),
),
];
for (path, body, expected) in upstream_cases {
let response = send(&app, "POST", path, Some(body)).await?;
assert_eq!(response.status, StatusCode::UNAUTHORIZED, "{path}");
assert_eq!(response.json()?, expected, "{path}");
}
let anthropic_unknown = send(
&app,
"POST",
"/v1/messages",
Some(json!({
"model": "other",
"max_tokens": 64,
"messages": [{"role": "user", "content": "hi"}]
})),
)
.await?;
assert_eq!(anthropic_unknown.status, StatusCode::NOT_FOUND);
assert_eq!(
anthropic_unknown.json()?,
json!({
"type": "error",
"error": {
"type": "not_found_error",
"message": "No route registered for model other"
}
})
);
Ok(())
}
/// A `type = "advisor"` deployment: gated executor + reviewer on one mock upstream.
fn advisor_state(base_url: &str) -> TestResult<ServerState> {
load_test_config(&format!(
r#"
schema_version = 1
[llm_clients.upstream]
format = "openai_chat"
base_url = "{base_url}"
[targets.executor]
id = "model/executor"
llm_client = "upstream"
system_prompt = "executor answer prompt"
[targets.advisor]
id = "model/advisor"
llm_client = "upstream"
system_prompt = "advisor target prompt"
[routes.gated]
id = "switchyard/advisor"
type = "advisor"
executor_target = "executor"
advisor_target = "advisor"
"#,
))
}
fn advisor_chat_body(prompt: &str) -> Value {
json!({
"model": "switchyard/advisor",
"messages": [{"role": "user", "content": prompt}]
})
}
#[tokio::test]
async fn advisor_route_approve_flow_and_stats() -> TestResult {
let upstream = MockUpstream::start().await?;
let app = build_switchyard_router(advisor_state(&upstream.base_url)?);
let response = send(
&app,
"POST",
"/v1/chat/completions",
Some(advisor_chat_body("hi")),
)
.await?;
assert_eq!(response.status, StatusCode::OK);
assert_eq!(response.json()?["choices"][0]["message"]["content"], "ok");
assert_eq!(
response
.headers
.get("x-model-router-selected-model")
.and_then(|value| value.to_str().ok()),
Some("model/executor")
);
// Executor turn first, then the review consult.
assert_eq!(upstream.models().await, ["model/executor", "model/advisor"]);
let calls = upstream.calls.lock().await;
assert!(has_system_prompt(&calls[0], "executor answer prompt"));
assert!(!has_system_prompt(&calls[1], "advisor target prompt"));
drop(calls);
let stats = send(&app, "GET", "/v1/stats", None).await?.json()?;
assert_eq!(stats["models"]["model/executor"]["calls"], 1);
// The consult lands in the classifier bucket with its usage.
assert_eq!(stats["classifier"]["models"]["model/advisor"]["calls"], 1);
assert_eq!(stats["classifier"]["total_tokens"]["prompt"], 40);
Ok(())
}
#[tokio::test]
async fn advisor_route_budget_scoped_by_proxy_header() -> TestResult {
let upstream = MockUpstream::start().await?;
let app = build_switchyard_router(advisor_state(&upstream.base_url)?);
for (session, expected_consults) in [("eval-a", 1), ("eval-a", 1), ("eval-b", 2)] {
let response = send_with_headers(
&app,
"POST",
"/v1/chat/completions",
Some(advisor_chat_body("hi")),
&[("proxy_x_session_id", session)],
)
.await?;
assert_eq!(response.status, StatusCode::OK);
let consults = upstream
.models()
.await
.iter()
.filter(|model| *model == "model/advisor")
.count();
assert_eq!(consults, expected_consults, "session {session}");
}
Ok(())
}
#[tokio::test]
async fn advisor_route_streaming_approval_replays_provider_events() -> TestResult {
let upstream = MockUpstream::start().await?;
let app = build_switchyard_router(advisor_state(&upstream.base_url)?);
let mut body = advisor_chat_body("hi");
body["stream"] = json!(true);
let response = send(&app, "POST", "/v1/chat/completions", Some(body)).await?;
assert_eq!(response.status, StatusCode::OK);
// The gate buffered the executor stream for the review, then replayed the
// provider events verbatim.
assert_eq!(upstream.models().await, ["model/executor", "model/advisor"]);
let text = response.text()?;
let events: Vec<Value> = text
.lines()
.filter_map(|line| line.strip_prefix("data: "))
.filter(|data| *data != "[DONE]")
.map(serde_json::from_str)
.collect::<Result<_, _>>()?;
assert_eq!(events.len(), 5);
assert_eq!(events[1]["choices"][0]["delta"]["content"], "hello");
assert_eq!(events[2]["choices"][0]["delta"]["content"], "-partial");
assert_eq!(events[3]["choices"][0]["delta"]["content"], "-final");
// Provider-specific usage detail rides through untouched.
assert_eq!(
events[3]["usage"]["prompt_tokens_details"]["cache_creation_tokens"],
2
);
assert_eq!(events[4]["choices"][0]["finish_reason"], "stop");
assert!(text.trim_end().ends_with("data: [DONE]"));
Ok(())
}
#[tokio::test]
async fn advisor_route_routing_log_records_classifier_tier() -> TestResult {
let upstream = MockUpstream::start().await?;
let temp_dir = tempfile::tempdir()?;
let log_path = temp_dir.path().join("routing.jsonl");
let state = advisor_state(&upstream.base_url)?.with_routing_log(&log_path)?;
let app = build_switchyard_router(state);
let response = send_with_headers(
&app,
"POST",
"/v1/chat/completions",
Some(advisor_chat_body("hi")),
&[
("proxy_x_session_id", "session-1"),
("x-switchyard-origin", "custom-agent"),
],
)
.await?;
assert_eq!(response.status, StatusCode::OK);
let records: Vec<Value> = std::fs::read_to_string(&log_path)?
.lines()
.map(serde_json::from_str)
.collect::<Result<_, _>>()?;
// The consult is appended under the shared judge tier; the served turn is
// the terminal answer row. The discarded-turn row does not exist in v1 —
// its tokens live in the advisor_gate stats block instead.
assert_eq!(records.len(), 2);
assert!(
records
.iter()
.all(|record| record["origin"] == "custom-agent")
);
let consult = records
.iter()
.find(|record| record["model"] == "model/advisor")
.ok_or("consult row present")?;
assert_eq!(consult["target"], "advisor");
assert_eq!(consult["tier"], "classifier");
assert_eq!(consult["route_id"], "switchyard/advisor");
assert_eq!(consult["algorithm"], "advisor_gate");
assert_eq!(consult["session_id"], "session-1");
assert_eq!(consult["prompt_tokens"], 40);
Ok(())
}
/// An advisor deployment whose reviewer client never retries, so a down
/// advisor hits fail-open after a single attempt (the documented deployment
/// posture for the advisor tier).
fn advisor_state_no_retry(base_url: &str) -> TestResult<ServerState> {
load_test_config(&format!(
r#"
schema_version = 1
[llm_clients.upstream]
format = "openai_chat"
base_url = "{base_url}"
[llm_clients.reviewer]
format = "openai_chat"
base_url = "{base_url}"
max_retries = 0
[targets.executor]
id = "model/executor"
llm_client = "upstream"
[targets.advisor]
id = "model/advisor"
llm_client = "reviewer"
[routes.gated]
id = "switchyard/advisor"
type = "advisor"
executor_target = "executor"
advisor_target = "advisor"
"#,
))
}
fn gate_count(stats: &Value, path: &[&str]) -> u64 {
let mut value = &stats["algorithm_stats"]["advisor_gate"];
for key in path {
value = &value[*key];
}
value.as_u64().unwrap_or(0)
}
// REDO mechanics, fail-open, and the /v1/stats advisor_gate projection in one
// sequential test: the OpenTelemetry meter behind algorithm_stats is
// process-global, so this is the only test that emits redo / consult-failure
// metrics and the only one that may assert their exact counts.
#[tokio::test]
async fn advisor_route_redo_client_error_and_stats_projection() -> TestResult {
let upstream = MockUpstream::start().await?;
let app = build_switchyard_router(advisor_state_no_retry(&upstream.base_url)?);
let before = send(&app, "GET", "/v1/stats", None).await?.json()?;
// REDO: the gated turn is discarded, the advisor plan is fed back, and
// the executor continues. Each flow gets its own budget scope so the
// second one is still reviewable.
let response = send_with_headers(
&app,
"POST",
"/v1/chat/completions",
Some(advisor_chat_body("please-redo")),
&[("proxy_x_session_id", "redo-flow")],
)
.await?;
assert_eq!(response.status, StatusCode::OK);
assert_eq!(response.json()?["choices"][0]["message"]["content"], "ok");
assert_eq!(
upstream.models().await,
["model/executor", "model/advisor", "model/executor"]
);
let calls = upstream.calls.lock().await;
let redo_messages = calls[2]["messages"]
.as_array()
.ok_or("redo call has messages")?
.clone();
drop(calls);
assert_eq!(redo_messages.len(), 3);
assert_eq!(redo_messages[1]["role"], "assistant");
assert_eq!(redo_messages[1]["content"], "ok");
assert_eq!(redo_messages[2]["role"], "user");
let feedback = redo_messages[2]["content"]
.as_str()
.ok_or("feedback is text")?;
assert!(feedback.starts_with("A senior reviewer examined your work"));
assert!(feedback.ends_with("run the tests"));
// Fail-open returns the buffered executor turn when the HTTP advisor call fails.
let response = send_with_headers(
&app,
"POST",
"/v1/chat/completions",
Some(advisor_chat_body("advisor-down")),
&[("proxy_x_session_id", "fail-flow")],
)
.await?;
assert_eq!(response.status, StatusCode::OK);
assert_eq!(response.json()?["choices"][0]["message"]["content"], "ok");
assert_eq!(
upstream.models().await,
[
"model/executor",
"model/advisor",
"model/executor",
"model/executor",
"model/advisor",
]
);
let stats = send(&app, "GET", "/v1/stats", None).await?.json()?;
// Both requests return an executor answer.
assert_eq!(stats["models"]["model/executor"]["calls"], 2);
assert_eq!(stats["classifier"]["total_errors"], 1);
// Projection deltas for the metrics only this test emits.
let redo = gate_count(&stats, &["reviews", "redo", "total"])
- gate_count(&before, &["reviews", "redo", "total"]);
assert_eq!(redo, 1);
assert_eq!(
gate_count(&stats, &["reviews", "redo", "by_trigger", "no_tool_call"]),
gate_count(&before, &["reviews", "redo", "by_trigger", "no_tool_call"]) + 1
);
assert_eq!(
gate_count(&stats, &["discarded", "turns"]),
gate_count(&before, &["discarded", "turns"]) + 1
);
// Mock usage: prompt 10 with 7 cached -> 3 non-cached input, 2 output.
assert_eq!(
gate_count(&stats, &["discarded", "tokens", "input"]),
gate_count(&before, &["discarded", "tokens", "input"]) + 3
);
assert_eq!(
gate_count(&stats, &["discarded", "tokens", "cached"]),
gate_count(&before, &["discarded", "tokens", "cached"]) + 7
);
assert_eq!(
gate_count(&stats, &["discarded", "tokens", "output"]),
gate_count(&before, &["discarded", "tokens", "output"]) + 2
);
assert_eq!(
gate_count(&stats, &["consult_failures", "upstream_5xx"]),
gate_count(&before, &["consult_failures", "upstream_5xx"]) + 1
);
// Reset re-baselines the projection: the redo/discard counts this test
// produced disappear from the next snapshot.
let reset = send(&app, "POST", "/v1/stats/reset", None).await?;
assert_eq!(reset.status, StatusCode::OK);
let stats = send(&app, "GET", "/v1/stats", None).await?.json()?;
assert_eq!(gate_count(&stats, &["reviews", "redo", "total"]), 0);
assert_eq!(gate_count(&stats, &["discarded", "turns"]), 0);
Ok(())
}
// Returns every `data:` frame of an SSE body as JSON, skipping `[DONE]`.
fn sse_events(body: &str) -> Vec<Value> {
body.lines()
.filter_map(|line| line.strip_prefix("data: "))
.filter(|data| *data != "[DONE]")
.filter_map(|data| serde_json::from_str(data).ok())
.collect()
}
// Two MCP servers expose `search`, so the upstream needs a qualified name for
// each tool. Tool definitions, history, and the forced tool choice must use the
// same qualified names. Response items and argument completions must contain
// the tool name and namespace that Codex uses to select the tool.
#[tokio::test]
async fn responses_round_trips_codex_tool_namespaces() -> TestResult {
const MODEL: &str = "model/mcp-namespaces";
let (upstream, app) = test_app(&[(ROUTE_MODEL, &[MODEL])]).await?;
let response = send(
&app,
"POST",
"/v1/responses",
Some(json!({
"model": ROUTE_MODEL,
"stream": true,
"input": [
{"type": "message", "role": "user",
"content": [{"type": "input_text", "text": "mcp-tool-call"}]},
{"type": "function_call", "call_id": "call_prior", "name": "search",
"namespace": "mcp__b", "arguments": "{}"},
{"type": "function_call_output", "call_id": "call_prior", "output": "earlier"}
],
"tool_choice": {"type": "function", "name": "search", "namespace": "mcp__b"},
"tools": [
{"type": "namespace", "name": "mcp__a", "tools": [{
"type": "function", "name": "search",
"parameters": {"type": "object", "properties": {"q": {"type": "string"}}}
}]},
{"type": "namespace", "name": "mcp__b", "tools": [{
"type": "function", "name": "search",
"parameters": {"type": "object", "properties": {"q": {"type": "string"}}}
}]}
]
})),
)
.await?;
assert_eq!(response.status, StatusCode::OK);
// The upstream sees two distinct tools, and every reference to the forced
// one uses the qualified spelling.
let calls = upstream.calls.lock().await;
let sent = &calls[0];
let offered = sent["tools"]
.as_array()
.map(|tools| {
tools
.iter()
.filter_map(|tool| tool["function"]["name"].as_str())
.collect::<Vec<_>>()
})
.unwrap_or_default();
assert_eq!(offered, vec!["mcp__a__search", "mcp__b__search"]);
assert_eq!(sent["tool_choice"]["function"]["name"], "mcp__b__search");
let recorded = sent["messages"]
.as_array()
.and_then(|messages| {
messages
.iter()
.find_map(|message| message["tool_calls"][0]["function"]["name"].as_str())
})
.ok_or("no recorded tool call reached the upstream")?;
assert_eq!(recorded, "mcp__b__search");
drop(calls);
// The upstream answers with the qualified name; Codex must receive the tool
// name and the namespace it dispatches on, on every event that names a call.
let events = sse_events(response.text()?);
for event_type in ["response.output_item.added", "response.output_item.done"] {
let item = events
.iter()
.find(|event| event["type"] == event_type)
.map(|event| event["item"].clone())
.ok_or(format!("stream produced no {event_type}"))?;
assert_eq!(item["name"], "search", "{event_type}");
assert_eq!(item["namespace"], "mcp__b", "{event_type}");
}
let arguments_done = events
.iter()
.find(|event| event["type"] == "response.function_call_arguments.done");
assert_eq!(
arguments_done.map(|event| &event["name"]),
Some(&json!("search"))
);
assert_eq!(
arguments_done.map(|event| &event["namespace"]),
Some(&json!("mcp__b"))
);
let completed = events
.iter()
.find(|event| event["type"] == "response.completed")
.ok_or("stream produced no response.completed event")?;
assert_eq!(completed["response"]["output"][0]["name"], "search");
assert_eq!(completed["response"]["output"][0]["namespace"], "mcp__b");
Ok(())
}
/// Allowed upstream response headers ride through to the client, while body, cookie,
/// and Switchyard-owned headers do not; a header this server writes always beats an
/// upstream echo of the same name.
#[tokio::test]
async fn upstream_headers_forward_but_switchyard_writes_win() -> TestResult {
let (_upstream, app) = test_app(&[(ROUTE_MODEL, &["model/a"])]).await?;
let body = json!({
"model": ROUTE_MODEL,
"messages": [{"role": "user", "content": "upstream-headers"}]
});
let response = send(&app, "POST", "/v1/chat/completions", Some(body)).await?;
assert_eq!(response.status, StatusCode::OK);
// Observability headers survive the proxy hop.
let traces = response
.headers
.get_all("x-upstream-trace")
.iter()
.map(|value| value.to_str())
.collect::<Result<Vec<_>, _>>()?;
assert_eq!(traces, ["trace-123", "trace-456"]);
assert_eq!(
response
.headers
.get("x-request-id")
.and_then(|value| value.to_str().ok()),
Some("req-42")
);
// Anthropic spells its correlation id without the `x-` prefix.
assert_eq!(
response
.headers
.get("request-id")
.and_then(|value| value.to_str().ok()),
Some("req_anthropic_42")
);
assert!(!response.headers.contains_key("link"));
// Upstream cookies must never become Switchyard-origin cookies.
assert!(!response.headers.contains_key("set-cookie"));
// Switchyard's own namespace never forwards from upstream.
assert!(!response.headers.contains_key("x-switchyard-session-id"));
// …and Switchyard's routing write beats the upstream echo.
assert_eq!(
response
.headers
.get("x-model-router-selected-model")
.and_then(|value| value.to_str().ok()),
Some("model/a")
);
Ok(())
}
/// A streamed reply captures its headers off the response head, on a branch the
/// buffered path never touches, so the same contract is asserted there too.
#[tokio::test]
async fn upstream_headers_forward_on_streaming_responses() -> TestResult {
let (_upstream, app) = test_app(&[(ROUTE_MODEL, &["model/a"])]).await?;
let body = json!({
"model": ROUTE_MODEL,
"stream": true,
"messages": [{"role": "user", "content": "upstream-headers"}]
});
let response = send(&app, "POST", "/v1/chat/completions", Some(body)).await?;
assert_eq!(response.status, StatusCode::OK);
let traces = response
.headers
.get_all("x-upstream-trace")
.iter()
.map(|value| value.to_str())
.collect::<Result<Vec<_>, _>>()?;
assert_eq!(traces, ["trace-123", "trace-456"]);
assert_eq!(
response
.headers
.get("x-request-id")
.and_then(|value| value.to_str().ok()),
Some("req-42")
);
assert_eq!(
response
.headers
.get("request-id")
.and_then(|value| value.to_str().ok()),
Some("req_anthropic_42")
);
assert!(!response.headers.contains_key("link"));
assert!(!response.headers.contains_key("set-cookie"));
assert!(!response.headers.contains_key("x-switchyard-session-id"));
assert_eq!(
response
.headers
.get("x-model-router-selected-model")
.and_then(|value| value.to_str().ok()),
Some("model/a")
);
Ok(())
}
#[tokio::test]
async fn configured_responses_reasoning_policy_reaches_upstream() -> TestResult {
let upstream = MockUpstream::start().await?;
let base_url = upstream.base_url.replace("/v1", "/buffered");
let plaintext = json!({"type": "reasoning", "id": "rs_1", "summary": [{"type": "summary_text", "text": "plan"}], "encrypted_content": null});
let foreign =
json!({"type": "reasoning", "id": "rs_2", "summary": [], "encrypted_content": "opaque"});
let mut kept_foreign = foreign.clone();
kept_foreign["content"] = json!([]);
for (provider_model_id, setting, expected_reasoning) in [
("openai/gpt-5", "", vec![kept_foreign.clone()]),
("model/efficient", "", vec![plaintext.clone()]),
(
"model/efficient",
"responses_reasoning = \"preserve_encrypted\"",
vec![kept_foreign],
),
("model/efficient", "responses_reasoning = \"drop\"", vec![]),
] {
let config = format!(
r#"
schema_version = 1
[llm_clients.upstream]
format = "openai_responses"
base_url = "{base_url}"
{setting}
[targets.primary]
id = "{provider_model_id}"
llm_client = "upstream"
extra_body = {{ reasoning = {{ effort = "high" }} }}
[routes.switchyard]
id = "switchyard"
type = "passthrough"
target = "primary"
"#
);
let app = build_switchyard_router(load_test_config(&config)?);
let response = send(
&app,
"POST",
"/v1/responses",
Some(json!({
"model": "switchyard", "store": false, "input": [
{"type": "message", "role": "user", "content": "continue"},
plaintext,
foreign
]
})),
)
.await?;
assert_eq!(response.status, StatusCode::OK, "{}", response.json()?);
let calls = upstream.calls.lock().await;
let body = calls.last().expect("upstream request");
let reasoning: Vec<_> = body["input"]
.as_array()
.expect("input array")
.iter()
.filter(|item| item["type"] == "reasoning")
.cloned()
.collect();
assert_eq!(body["model"], json!(provider_model_id));
assert_eq!(
reasoning, expected_reasoning,
"{provider_model_id}, {setting}"
);
assert_eq!(body["reasoning"], json!({"effort": "high"}));
}
Ok(())
}
#[tokio::test]
async fn rate_limit_fallback_policy() -> TestResult {
for (policy, forward_auth, prompt, is_terminal) in [
("", false, "hello", false),
("", true, "hello", false),
("fallback_on_rate_limit = true", false, "hello", false),
("fallback_on_rate_limit = false", false, "hello", true),
("fallback_on_rate_limit = false", true, "hello", true),
(
"fallback_on_rate_limit = false",
false,
"unavailable",
false,
),
] {
let upstream = MockUpstream::start().await?;
let app = build_switchyard_router(load_test_config(&format!(
r#"
schema_version = 1
[llm_clients.primary]
format = "openai_chat"
base_url = "{base_url}"
forward_auth = {forward_auth}
{policy}
max_retries = 1
failure_cooldown_ms = 60000
[llm_clients.secondary]
format = "openai_chat"
base_url = "{base_url}"
[targets]
first = {{ id = "model/rate-limited", llm_client = "primary" }}
second = {{ id = "model/fallback", llm_client = "secondary" }}
[routes.test]
id = "{ROUTE_MODEL}"
type = "random"
targets = ["first", "second"]
weights = [1000, 1]
seed = 17
"#,
base_url = upstream.base_url,
))?);
let mut expected_calls = Vec::new();
// The second request exercises cooldown as well as streaming HTTP errors.
for stream in [false, true] {
let response = send_with_headers(
&app,
"POST",
"/v1/chat/completions",
Some(json!({
"model": ROUTE_MODEL, "stream": stream,
"messages": [{"role": "user", "content": prompt}]
})),
&[("authorization", "Bearer test-caller")],
)
.await?;
if !stream || is_terminal || forward_auth {
expected_calls.extend(["model/rate-limited"; 2]);
}
if is_terminal {
assert_eq!(response.status, StatusCode::TOO_MANY_REQUESTS);
assert_eq!(response.json()?["error"]["code"], "usage_limit");
} else {
assert_eq!(response.status, StatusCode::OK);
expected_calls.push("model/fallback");
}
assert_eq!(
upstream.models().await,
expected_calls,
"{policy}, forward_auth={forward_auth}, {prompt}, stream={stream}"
);
}
}
Ok(())
}
#[tokio::test]
async fn vgr_time_limits_can_be_disabled_from_toml() -> TestResult {
let upstream = MockUpstream::start().await?;
for limit in ["deadline_seconds", "efficient_turn_budget_seconds"] {
for (configured, served) in [
("", "model/vgr-capable"),
("bounded_efficient = false", "model/vgr-efficient"),
] {
let app = build_switchyard_router(load_test_config(&format!(
r#"
schema_version = 1
[llm_clients.primary]
format = "openai_chat"
base_url = "{}"
[targets]
efficient = {{ id = "model/vgr-efficient", llm_client = "primary" }}
capable = {{ id = "model/vgr-capable", llm_client = "primary" }}
judge = {{ id = "model/vgr-judge", llm_client = "primary" }}
[routes.vgr]
id = "switchyard/vgr"
type = "vgr"
efficient_target = "efficient"
capable_target = "capable"
judge_target = "judge"
mode = "evaluate"
task_typing = false
{configured}
{limit} = 0.000000001
"#,
upstream.base_url
))?);
let response = send(
&app,
"POST",
"/v1/chat/completions",
Some(json!({
"model": "switchyard/vgr",
"messages": [{"role": "user", "content": "answer the question"}]
})),
)
.await?;
assert_eq!(response.status, StatusCode::OK, "{}", response.text()?);
assert_eq!(
response.headers.get("x-model-router-selected-model"),
Some(&HeaderValue::from_str(served)?),
"{limit}, {configured}"
);
}
}
Ok(())
}
#[tokio::test]
async fn vgr_image_requests_stay_on_the_efficient_tier_when_configured() -> TestResult {
let upstream = MockUpstream::start().await?;
for (configured, served) in [
("", "model/vgr-capable"),
(
"efficient_supports_attachments = true",
"model/vgr-efficient",
),
] {
let app = build_switchyard_router(load_test_config(&format!(
r#"
schema_version = 1
[llm_clients.primary]
format = "openai_chat"
base_url = "{}"
[targets]
efficient = {{ id = "model/vgr-efficient", llm_client = "primary" }}
capable = {{ id = "model/vgr-capable", llm_client = "primary" }}
judge = {{ id = "model/vgr-judge", llm_client = "primary" }}
[routes.vgr]
id = "switchyard/vgr"
type = "vgr"
efficient_target = "efficient"
capable_target = "capable"
judge_target = "judge"
mode = "evaluate"
task_typing = false
bounded_efficient = false
{configured}
"#,
upstream.base_url
))?);
let response = send(
&app,
"POST",
"/v1/chat/completions",
Some(json!({
"model": "switchyard/vgr",
"messages": [{"role": "user", "content": [
{"type": "text", "text": "answer the question"},
{"type": "image_url", "image_url": {"url": "https://example.test/image.png"}}
]}]
})),
)
.await?;
assert_eq!(response.status, StatusCode::OK, "{}", response.text()?);
assert_eq!(
response.headers.get("x-model-router-selected-model"),
Some(&HeaderValue::from_str(served)?),
"{configured}"
);
}
Ok(())
}