Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.
Rust
feat(server): proxy unmatched requests through fallback client (#547)
nachiketb-nvidiac597bfd5ee77
diff --git a/Cargo.lock b/Cargo.lockindex fd4d9b6bf..89b712d9d 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -2392,6 +2392,7 @@ dependencies = ["opentelemetry_sdk","parking_lot","prometheus",+ "reqwest","rustls","serde","serde_json",diff --git a/crates/libsy-llm-client/src/backend.rs b/crates/libsy-llm-client/src/backend.rsindex fab446095..a526712fc 100644--- a/crates/libsy-llm-client/src/backend.rs+++ b/crates/libsy-llm-client/src/backend.rs@@ -391,34 +391,6 @@ mod tests {);}- #[test]- fn count_tokens_url_joins_every_base_url_shape() {- assert_eq!(- Backend::Anthropic(config("https://host")).count_tokens_url(),- "https://host/v1/messages/count_tokens"- );- assert_eq!(- Backend::Anthropic(config("https://host/v1")).count_tokens_url(),- "https://host/v1/messages/count_tokens"- );- assert_eq!(- Backend::Anthropic(config("https://host/v1/messages")).count_tokens_url(),- "https://host/v1/messages/count_tokens"- );- // Trailing slash is trimmed before the join.- assert_eq!(- Backend::Anthropic(config("https://host/v1/")).count_tokens_url(),- "https://host/v1/messages/count_tokens"- );- }-- #[test]- fn only_anthropic_backend_is_anthropic() {- assert!(Backend::Anthropic(config("x")).is_anthropic());- assert!(!Backend::OpenAiChat(config("x")).is_anthropic());- assert!(!Backend::OpenAiResponses(config("x")).is_anthropic());- }-#[test]fn wire_format_matches_variant() {assert_eq!(diff --git a/crates/libsy-llm-client/src/client.rs b/crates/libsy-llm-client/src/client.rsindex 4a517936f..9f47828aa 100644--- a/crates/libsy-llm-client/src/client.rs+++ b/crates/libsy-llm-client/src/client.rs@@ -82,6 +82,34 @@ impl ModelConfig {}}+/// A model-bearing provider operation outside the normal completion endpoint.+#[derive(Clone, Copy, Debug, Eq, PartialEq)]+pub enum AuxiliaryOperation {+ /// Anthropic Messages input-token counting.+ AnthropicCountTokens,+ /// OpenAI Responses input-token counting.+ ResponsesInputTokens,+ /// OpenAI Responses compaction.+ ResponsesCompact,+}++impl AuxiliaryOperation {+ const fn wire_format(self) -> WireFormat {+ match self {+ Self::AnthropicCountTokens => WireFormat::AnthropicMessages,+ Self::ResponsesInputTokens | Self::ResponsesCompact => WireFormat::OpenAiResponses,+ }+ }++ fn url(self, backend: &Backend) -> String {+ match self {+ Self::AnthropicCountTokens => backend.count_tokens_url(),+ Self::ResponsesInputTokens => format!("{}/input_tokens", backend.url()),+ Self::ResponsesCompact => format!("{}/compact", backend.url()),+ }+ }+}+/// A client that dispatches neutral-IR requests to per-model HTTP backends.////// Construct it with a list of [`ModelConfig`]s — one per model, each naming a@@ -146,22 +174,27 @@ impl TranslatingLlmClient {})}- /// Whether `model` has an Anthropic backend that supports token counting.- pub fn supports_count_tokens(&self, model: &ModelId) -> bool {- self.backend_for(model, WireFormat::AnthropicMessages)- .is_some()+ /// Whether `model` has a backend for `operation`.+ pub fn supports_auxiliary(&self, model: &ModelId, operation: AuxiliaryOperation) -> bool {+ self.backend_for(model, operation.wire_format()).is_some()}- /// Counts input tokens with `model`'s Anthropic backend.+ /// Calls a model-bearing auxiliary provider operation.///- /// Returns an error when the model has no Anthropic backend or the upstream+ /// Returns an error when the model has no compatible backend or the upstream/// request fails or returns invalid JSON.- pub async fn count_tokens(&self, model: &ModelId, request: Request) -> Result<Value> {- let backend = self- .backend_for(model, WireFormat::AnthropicMessages)- .ok_or_else(|| LlmClientError::Configuration {- message: format!("model {model} has no Anthropic backend for count_tokens"),- })?;+ pub async fn call_auxiliary(+ &self,+ model: &ModelId,+ request: Request,+ operation: AuxiliaryOperation,+ ) -> Result<Value> {+ let wire_format = operation.wire_format();+ let backend =+ self.backend_for(model, wire_format)+ .ok_or_else(|| LlmClientError::Configuration {+ message: format!("model {model} has no backend for {operation:?}"),+ })?;let Request {mut llm_request,metadata,@@ -171,20 +204,17 @@ impl TranslatingLlmClient {let http_response = self.send_encoded(backend,- WireFormat::AnthropicMessages,+ wire_format,llm_request,metadata.as_ref(),model,- UpstreamEndpoint::CountTokens,+ UpstreamEndpoint::Auxiliary(operation),).await?;- let body = match http_response {- EncodedResponse::Buffered { body, .. } => body,- EncodedResponse::Streaming(_) => {- return Err(LlmClientError::InvalidRequest {- message: "count_tokens does not support streaming requests".to_string(),- });- }+ let EncodedResponse::Buffered { body, .. } = http_response else {+ return Err(LlmClientError::InvalidRequest {+ message: "auxiliary endpoints do not support streaming".to_string(),+ });};serde_json::from_slice(&body).map_err(|error| LlmClientError::InvalidResponse {source: Box::new(error),@@ -200,8 +230,7 @@ impl TranslatingLlmClient {/// overflow via the backend's provider rules. Shared by/// [`call_rewrite_model`](Self::call_rewrite_model) (which POSTs to the/// backend's completion URL and decodes a response) and- /// [`count_tokens`](Self::count_tokens) (which POSTs to the `count_tokens`- /// URL and returns the raw JSON).+ /// the model-bearing auxiliary operations, which return raw JSON.async fn send_encoded(&self,backend: &Backend,@@ -562,14 +591,14 @@ impl RoutedLlmClient for TranslatingLlmClient {#[derive(Clone, Copy)]enum UpstreamEndpoint {Completion,- CountTokens,+ Auxiliary(AuxiliaryOperation),}impl UpstreamEndpoint {fn url(self, backend: &Backend) -> String {match self {UpstreamEndpoint::Completion => backend.url(),- UpstreamEndpoint::CountTokens => backend.count_tokens_url(),+ UpstreamEndpoint::Auxiliary(operation) => operation.url(backend),}}diff --git a/crates/libsy-llm-client/src/lib.rs b/crates/libsy-llm-client/src/lib.rsindex adce273b8..d5f579c7d 100644--- a/crates/libsy-llm-client/src/lib.rs+++ b/crates/libsy-llm-client/src/lib.rs@@ -26,7 +26,7 @@ pub mod raw;pub mod run;pub use backend::{Backend, DEFAULT_MAX_RETRIES, HttpBackendConfig};-pub use client::{ModelConfig, TranslatingLlmClient};+pub use client::{AuxiliaryOperation, ModelConfig, TranslatingLlmClient};pub use error::{LlmClientError, Result};pub use observation::{LlmCallObservation, RunObservation, RunObserver};pub use raw::RawResponse;diff --git a/crates/switchyard-runner/src/config.rs b/crates/switchyard-runner/src/config.rsindex d4887fab1..4b211ae81 100644--- a/crates/switchyard-runner/src/config.rs+++ b/crates/switchyard-runner/src/config.rs@@ -12,13 +12,13 @@ use serde::de::DeserializeOwned;use serde::{Deserialize, Deserializer};use serde_json::Value;use switchyard_llm_client::{- Backend, ClientRouter, DEFAULT_MAX_RETRIES, HttpBackendConfig, ModelConfig,+ AuxiliaryOperation, Backend, ClientRouter, DEFAULT_MAX_RETRIES, HttpBackendConfig, ModelConfig,TranslatingLlmClient,};use switchyard_protocol::{ModelId, RoutedLlmClient, WireFormat};use crate::{- AlgorithmSpec, CallerAuthKind, CountTokensTarget, DecisionTarget, ModelCapabilities, Route,+ AlgorithmSpec, AuxiliaryTarget, CallerAuthKind, DecisionTarget, ModelCapabilities, Route,Runner, RunnerError,};@@ -54,6 +54,7 @@ pub(crate) fn runner_from_toml(source: &str) -> RunnerResult<Runner> {#[serde(deny_unknown_fields)]pub(crate) struct DeploymentConfig {schema_version: u32,+ fallback_client: Option<String>,#[serde(default)]llm_clients: BTreeMap<String, LlmClientConfig>,targets: BTreeMap<String, TargetConfig>,@@ -165,6 +166,7 @@ impl DeploymentConfig {let clients = self.build_clients()?;let targets = self.build_targets();+ let fallback_base_url = self.fallback_base_url()?;let mut routes = Vec::with_capacity(self.routes.len());for (route_name, config) in &self.routes {validate_value("route name", route_name)?;@@ -188,7 +190,10 @@ impl DeploymentConfig {.map_err(|error| RunnerError::configuration_source(error.to_string(), error))?;let (route_clients, caller_auth) =self.build_route_clients(route_name, config, &clients)?;- let count_tokens_target = self.build_count_tokens_target(config, &clients);+ let anthropic_auxiliary_target =+ self.build_anthropic_auxiliary_target(config, &clients);+ let responses_auxiliary_target =+ self.build_responses_auxiliary_target(config, &clients);let decision_targets = config.routing_target_names().into_iter()@@ -199,12 +204,14 @@ impl DeploymentConfig {route_clients,caller_auth,capabilities,- count_tokens_target,+ anthropic_auxiliary_target,+ responses_auxiliary_target,decision_targets,);routes.push((config.id.clone(), route));}- Ok(Runner::new(routes))+ let runner = Runner::new(routes).with_fallback_url(fallback_base_url);+ Ok(runner)}fn build_clients(&self) -> RunnerResult<BTreeMap<String, Arc<TranslatingLlmClient>>> {@@ -291,27 +298,60 @@ impl DeploymentConfig {Ok((ClientRouter::new(by_model), caller_auth))}- fn build_count_tokens_target(+ fn fallback_base_url(&self) -> RunnerResult<Option<String>> {+ let Some(name) = &self.fallback_client else {+ return Ok(None);+ };+ let config = self.llm_clients.get(name).ok_or_else(|| {+ RunnerError::configuration(format!(+ "fallback_client references unknown llm client {name}"+ ))+ })?;+ Ok(Some(config.base_url.as_str().to_string()))+ }++ fn build_anthropic_auxiliary_target(&self,route: &RouteConfig,clients: &BTreeMap<String, Arc<TranslatingLlmClient>>,- ) -> Option<CountTokensTarget> {+ ) -> Option<AuxiliaryTarget> {route.routing_target_names().into_iter().enumerate().filter_map(|(index, name)| {- let target = self.targets.get(name)?;- let client = clients.get(&target.llm_client)?;- client.supports_count_tokens(&target.id).then_some((- count_tokens_priority(name, &target.id),- index,- target,- client,- ))+ let target = self.build_auxiliary_target(+ name,+ clients,+ AuxiliaryOperation::AnthropicCountTokens,+ )?;+ Some((count_tokens_priority(name, &target.model), index, target))})- .min_by_key(|(priority, index, _, _)| (*priority, *index))- .map(|(_, _, target, client)| CountTokensTarget {+ .min_by_key(|(priority, index, _)| (*priority, *index))+ .map(|(_, _, target)| target)+ }++ fn build_responses_auxiliary_target(+ &self,+ route: &RouteConfig,+ clients: &BTreeMap<String, Arc<TranslatingLlmClient>>,+ ) -> Option<AuxiliaryTarget> {+ route.routing_target_names().into_iter().find_map(|name| {+ self.build_auxiliary_target(name, clients, AuxiliaryOperation::ResponsesInputTokens)+ })+ }++ fn build_auxiliary_target(+ &self,+ name: &str,+ clients: &BTreeMap<String, Arc<TranslatingLlmClient>>,+ operation: AuxiliaryOperation,+ ) -> Option<AuxiliaryTarget> {+ let target = self.targets.get(name)?;+ let client = clients.get(&target.llm_client)?;+ client+ .supports_auxiliary(&target.id, operation)+ .then(|| AuxiliaryTarget {model: target.id.clone(),client: client.clone(),})diff --git a/crates/switchyard-runner/src/failure.rs b/crates/switchyard-runner/src/failure.rsindex 4e2901b7a..8113ef25b 100644--- a/crates/switchyard-runner/src/failure.rs+++ b/crates/switchyard-runner/src/failure.rs@@ -100,7 +100,7 @@ impl RunnerError {),Self::UnknownRouteModel(_)| Self::IncompatibleCallerFormat(_)- | Self::CountTokensUnsupported => summary(+ | Self::AuxiliaryUnsupported => summary(RouteErrorKind::InvalidRequest,RouteErrorPhase::BeforeResponse,None,@@ -329,7 +329,7 @@ mod tests {RouteErrorKind::Configuration));- let unsupported = RunnerError::CountTokensUnsupported;+ let unsupported = RunnerError::AuxiliaryUnsupported;assert!(matches!(unsupported.execution_error_summary().kind,RouteErrorKind::InvalidRequestdiff --git a/crates/switchyard-runner/src/lib.rs b/crates/switchyard-runner/src/lib.rsindex 75543c2fe..efb0441e4 100644--- a/crates/switchyard-runner/src/lib.rs+++ b/crates/switchyard-runner/src/lib.rs@@ -15,6 +15,6 @@ pub use algorithm::{};pub use failure::{RouteErrorKind, RouteErrorPhase, RouteErrorSummary, stream_error_summary};pub use route::{- CallerAuthKind, CountTokensTarget, ModelCapabilities, Route, RunOutput, RunnerError,+ AuxiliaryTarget, CallerAuthKind, ModelCapabilities, Route, RunOutput, RunnerError,};pub use runner::{DecisionDescription, DecisionTarget, ModelInfo, Runner};diff --git a/crates/switchyard-runner/src/route.rs b/crates/switchyard-runner/src/route.rsindex 673a36bc4..857d51dfd 100644--- a/crates/switchyard-runner/src/route.rs+++ b/crates/switchyard-runner/src/route.rs@@ -8,7 +8,7 @@ use std::sync::Arc;use libsy::{Algorithm, LibsyError, RoutingOutcome};use serde_json::Value;-use switchyard_llm_client::{ClientRouter, RunObserver, TranslatingLlmClient};+use switchyard_llm_client::{AuxiliaryOperation, ClientRouter, RunObserver, TranslatingLlmClient};use switchyard_protocol::{LlmClientError, ModelId, Request, Response, WireFormat};use thiserror::Error;@@ -56,9 +56,9 @@ impl CallerAuthKind {}}-/// Exact upstream model used for Anthropic token counting.+/// Exact upstream model and client used for an auxiliary provider operation.#[derive(Clone)]-pub struct CountTokensTarget {+pub struct AuxiliaryTarget {pub model: ModelId,pub client: Arc<TranslatingLlmClient>,}@@ -76,8 +76,8 @@ pub enum RunnerError {UnknownRouteModel(String),#[error("caller format is incompatible with {} credentials", .0.as_str())]IncompatibleCallerFormat(CallerAuthKind),- #[error("route has no Anthropic target for token counting")]- CountTokensUnsupported,+ #[error("route has no compatible target for the auxiliary operation")]+ AuxiliaryUnsupported,#[error(transparent)]Algorithm(#[from] LibsyError),#[error(transparent)]@@ -112,7 +112,8 @@ pub struct Route {clients: ClientRouter,caller_auth: Option<CallerAuthKind>,capabilities: ModelCapabilities,- count_tokens_target: Option<CountTokensTarget>,+ anthropic_auxiliary_target: Option<AuxiliaryTarget>,+ responses_auxiliary_target: Option<AuxiliaryTarget>,decision_targets: Vec<DecisionTarget>,}@@ -129,7 +130,8 @@ impl Route {clients: ClientRouter,caller_auth: Option<CallerAuthKind>,capabilities: ModelCapabilities,- count_tokens_target: Option<CountTokensTarget>,+ anthropic_auxiliary_target: Option<AuxiliaryTarget>,+ responses_auxiliary_target: Option<AuxiliaryTarget>,decision_targets: Vec<DecisionTarget>,) -> Self {Self {@@ -137,7 +139,8 @@ impl Route {clients,caller_auth,capabilities,- count_tokens_target,+ anthropic_auxiliary_target,+ responses_auxiliary_target,decision_targets,}}@@ -201,15 +204,23 @@ impl Route {.map_err(Into::into)}- /// Counts tokens using the configured Anthropic-capable target.- pub async fn count_tokens(&self, request: Request) -> Result<Value, RunnerError> {- let target = self- .count_tokens_target- .as_ref()- .ok_or(RunnerError::CountTokensUnsupported)?;+ /// Executes a model-bearing provider operation through a compatible target.+ pub async fn call_auxiliary(+ &self,+ request: Request,+ operation: AuxiliaryOperation,+ ) -> Result<Value, RunnerError> {+ let target = match operation {+ AuxiliaryOperation::AnthropicCountTokens => &self.anthropic_auxiliary_target,+ AuxiliaryOperation::ResponsesInputTokens | AuxiliaryOperation::ResponsesCompact => {+ &self.responses_auxiliary_target+ }+ }+ .as_ref()+ .ok_or(RunnerError::AuxiliaryUnsupported)?;target.client- .count_tokens(&target.model, request)+ .call_auxiliary(&target.model, request, operation).await.map_err(Into::into)}diff --git a/crates/switchyard-runner/src/runner.rs b/crates/switchyard-runner/src/runner.rsindex 93dce5e76..bcd175344 100644--- a/crates/switchyard-runner/src/runner.rs+++ b/crates/switchyard-runner/src/runner.rs@@ -16,6 +16,7 @@ use crate::{ModelCapabilities, Route, RunnerError};/// Immutable named route table.pub struct Runner {routes: Vec<(ModelId, Route)>,+ fallback_base_url: Option<String>,}/// Borrowed model metadata returned while listing routes.@@ -57,7 +58,15 @@ impl Runner {/// Builds a runner from named routes in caller-provided order./// Pre-condition: There must be at least one route.pub fn new(routes: Vec<(ModelId, Route)>) -> Self {- Self { routes }+ Self {+ routes,+ fallback_base_url: None,+ }+ }++ pub(crate) fn with_fallback_url(mut self, fallback_base_url: Option<String>) -> Self {+ self.fallback_base_url = fallback_base_url;+ self}/// Returns the route registered for a model.@@ -77,6 +86,11 @@ impl Runner {})}+ /// Returns the validated API root used for unmatched HTTP requests.+ pub fn fallback_base_url(&self) -> Option<&str> {+ self.fallback_base_url.as_deref()+ }+/// Resolves an outcome to configured target names and non-secret client settings.pub fn describe_decision(&self,diff --git a/crates/switchyard-runner/tests/route.rs b/crates/switchyard-runner/tests/route.rsindex 71ce57c7e..b6b1d1827 100644--- a/crates/switchyard-runner/tests/route.rs+++ b/crates/switchyard-runner/tests/route.rs@@ -52,6 +52,7 @@ fn plugin_route(client: Arc<dyn RoutedLlmClient>) -> Route {None,ModelCapabilities::default(),None,+ None,Vec::new(),)}diff --git a/crates/switchyard-server/Cargo.toml b/crates/switchyard-server/Cargo.tomlindex 0c4b47e24..29cea432f 100644--- a/crates/switchyard-server/Cargo.toml+++ b/crates/switchyard-server/Cargo.toml@@ -32,6 +32,7 @@ opentelemetry-prometheus = "0.32"opentelemetry_sdk = { version = "0.32", default-features = false, features = ["metrics", "trace"] }parking_lot.workspace = trueprometheus = "0.14"+reqwest.workspace = trueserde.workspace = trueswitchyard-llm-client.workspace = trueswitchyard-protocol.workspace = truediff --git a/crates/switchyard-server/README.md b/crates/switchyard-server/README.mdindex 2ff1a7a5a..3f59fd0af 100644--- a/crates/switchyard-server/README.md+++ b/crates/switchyard-server/README.md@@ -130,6 +130,9 @@ documented in [Stage-Router Routing](../../docs/routing_algorithms/stage_router_| `POST` | `/v1/responses` | OpenAI Responses || `POST` | `/v1/decision` | Resolve selected and fallback targets without a post-routing answer call || `POST` | `/v1/messages/count_tokens` | Token count from a route's Anthropic target |+| `POST` | `/v1/responses/input_tokens` | Token count from a route's OpenAI Responses target |+| `POST` | `/v1/responses/compact` | Compaction through a route's OpenAI Responses target |+| `ANY` | Any unmatched path | Raw forward through the optional `fallback_client` || `GET` | `/v1/models` | Routes served by this deployment || `GET` | `/v1/stats` | Per-model usage plus curated algorithm stats || `POST` | `/v1/stats/reset` | Clear accumulated stats |@@ -140,6 +143,16 @@ Requests name a route by its `id`, so `POST /v1/chat/completions` with `"model":routes through the `[routes.general]` entry above. Any of the three request formats can address anyroute, and the server translates between them.+Set the top-level `fallback_client` to an entry under `[llm_clients]` to proxy any otherwise+unmatched method and path through that client. The fallback client does not need a target. Requests+and responses are forwarded without translation, including paths, query strings, bodies, and model+identifiers. The caller's end-to-end headers, including authorization, are forwarded; hop-by-hop+headers are removed, and the client's configured API key, extra headers, format, and retry policy+are not applied. Without `fallback_client`, unmatched paths return `404`.++The three model-bearing auxiliary endpoints above remain routed operations: they resolve the route+name to a compatible target, rewrite the upstream model ID, and use that target's configured client.+`POST /v1/decision` accepts `{"input_format": "openai_chat", "request": {...}}`, where thenested request names the route in `model`. It executes required classifier or judge calls, thenreturns the selected target and ordered fallbacks with their model, format, base URL, anddiff --git a/crates/switchyard-server/src/lib.rs b/crates/switchyard-server/src/lib.rsindex 77150399c..eea1c7a2e 100644--- a/crates/switchyard-server/src/lib.rs+++ b/crates/switchyard-server/src/lib.rs@@ -23,6 +23,7 @@ use std::path::PathBuf;use std::sync::Arc;use std::time::{Duration, Instant};+use axum::body::Body;use axum::extract::rejection::{JsonRejection, QueryRejection};use axum::extract::{DefaultBodyLimit, Query, Request as HttpRequest, State};use axum::http::header::CONTENT_TYPE;@@ -36,7 +37,7 @@ use libsy::{Algorithm, LibsyError, RoutingOutcome};use parking_lot::Mutex;use serde::{Deserialize, Serialize};use serde_json::{Value, json};-use switchyard_llm_client::{ClientRouter, RunObservation, RunObserver};+use switchyard_llm_client::{AuxiliaryOperation, ClientRouter, RunObservation, RunObserver};use switchyard_protocol::{LlmClientError, Metadata, ModelId, Request, Usage};use switchyard_runner::{CallerAuthKind, DecisionTarget, ModelCapabilities, Route, RunOutput, Runner, RunnerError,@@ -136,6 +137,7 @@ struct DecisionLlmClientResponse {#[derive(Clone)]pub struct ServerState {runner: Arc<Runner>,+ fallback_http: reqwest::Client,metrics: prometheus::Registry,stats: StatsAccumulator,routing_log: Option<SharedRoutingLog>,@@ -190,6 +192,7 @@ impl ServerState {None,ModelCapabilities::default(),None,+ None,Vec::new(),),)@@ -202,12 +205,17 @@ impl ServerState {/// Creates HTTP-server state around an already configured runner.pub fn from_runner(runner: Runner) -> ServerResult<Self> {let metrics = metrics::registry().map_err(ServerError::new)?;+ let fallback_http = reqwest::Client::builder()+ .redirect(reqwest::redirect::Policy::none())+ .build()+ .map_err(|error| ServerError::new(error.to_string()))?;let stats = StatsAccumulator::new(metrics.clone(),runner.models().map(|model| model.algorithm),);Ok(Self {runner: Arc::new(runner),+ fallback_http,metrics,stats,routing_log: None,@@ -479,6 +487,11 @@ pub fn build_switchyard_router(state: ServerState) -> Router {.route("/v1/responses", post(openai_responses)).route("/v1/decision", post(decision)).route("/v1/messages/count_tokens", post(anthropic_count_tokens))+ .route(+ "/v1/responses/input_tokens",+ post(openai_responses_input_tokens),+ )+ .route("/v1/responses/compact", post(openai_responses_compact)).route("/v1/models", get(models)).route("/v1/stats", get(get_stats)).route("/v1/stats/reset", post(reset_stats))@@ -488,7 +501,7 @@ pub fn build_switchyard_router(state: ServerState) -> Router {router = router.route("/v1/routing/session-stats", get(get_session_stats));}router- .fallback(not_found)+ .fallback(proxy_unmatched).layer(DefaultBodyLimit::max(DEFAULT_MAX_REQUEST_BODY_BYTES))// `layer` only wraps routes registered before it, so this stays last..layer(axum::middleware::from_fn(stamp_request_start))@@ -539,6 +552,119 @@ async fn openai_responses(handle_endpoint(state, started, headers, body, WireFormat::OpenAiResponses).await}+// Forwards unmatched requests unchanged to the configured API root.+async fn proxy_unmatched(State(state): State<ServerState>, request: HttpRequest) -> Response {+ let Some(base_url) = state.runner.fallback_base_url() else {+ return not_found().await;+ };+ let (mut parts, body) = request.into_parts();+ let path_and_query = parts+ .uri+ .path_and_query()+ .map_or_else(|| parts.uri.path().to_string(), ToString::to_string);+ strip_hop_by_hop_headers(&mut parts.headers);+ parts.headers.remove(axum::http::header::HOST);+ let url = match fallback_url(base_url, &path_and_query) {+ Ok(url) => url,+ Err(error) => {+ return error_response(+ StatusCode::BAD_GATEWAY,+ error,+ "upstream_error",+ "upstream_error",+ );+ }+ };+ let body = reqwest::Body::wrap_stream(body.into_data_stream());+ match state+ .fallback_http+ .request(parts.method, url)+ .headers(parts.headers)+ .body(body)+ .send()+ .await+ {+ Ok(response) => {+ let response: http::Response<reqwest::Body> = response.into();+ let (mut parts, body) = response.into_parts();+ strip_hop_by_hop_headers(&mut parts.headers);+ Response::from_parts(parts, Body::new(body))+ }+ Err(error) => error_response(+ StatusCode::BAD_GATEWAY,+ error.to_string(),+ "upstream_error",+ "upstream_error",+ ),+ }+}++fn fallback_url(base_url: &str, path_and_query: &str) -> Result<reqwest::Url, String> {+ let mut url = reqwest::Url::parse(base_url.trim())+ .map_err(|error| format!("invalid fallback base URL: {error}"))?;+ let base_query = url.query().map(str::to_owned);+ url.set_query(None);+ url.set_fragment(None);++ let base_path = url.path().trim_end_matches('/');+ let root_path = [+ "/v1/chat/completions",+ "/v1/responses",+ "/v1/messages",+ "/v1",+ ]+ .iter()+ .find_map(|suffix| base_path.strip_suffix(suffix))+ .unwrap_or(base_path);+ let (request_path, request_query) = path_and_query+ .split_once('?')+ .map_or((path_and_query, None), |(path, query)| (path, Some(query)));+ let request_path = request_path.strip_prefix('/').unwrap_or(request_path);+ let target_path = if root_path.is_empty() {+ format!("/{request_path}")+ } else if request_path.is_empty() {+ format!("{root_path}/")+ } else {+ format!("{root_path}/{request_path}")+ };+ url.set_path(&target_path);++ let query = match (base_query, request_query) {+ (Some(base_query), Some(request_query)) => Some(format!("{base_query}&{request_query}")),+ (Some(base_query), None) => Some(base_query),+ (None, Some(request_query)) => Some(request_query.to_owned()),+ (None, None) => None,+ };+ url.set_query(query.as_deref());+ Ok(url)+}++fn strip_hop_by_hop_headers(headers: &mut HeaderMap) {+ let connection_headers = headers+ .get_all(axum::http::header::CONNECTION)+ .iter()+ .filter_map(|value| value.to_str().ok())+ .flat_map(|value| value.split(','))+ .filter_map(|name| HeaderName::from_bytes(name.trim().as_bytes()).ok())+ .collect::<Vec<_>>();+ for name in connection_headers {+ headers.remove(name);+ }+ for name in [+ "connection",+ "keep-alive",+ "proxy-authenticate",+ "proxy-authorization",+ "proxy-connection",+ "te",+ "trailer",+ "transfer-encoding",+ "upgrade",+ ] {+ headers.remove(name);+ }+}+/// One provider request submitted for a routing decision without an answer-model call.#[derive(Deserialize)]#[serde(deny_unknown_fields)]@@ -634,22 +760,86 @@ async fn anthropic_count_tokens(Ok(resolved) => resolved,Err(response) => return anthropic_error_response(response),};- anthropic_error_response(match route.count_tokens(request).await {- Ok(payload) => (StatusCode::OK, Json(payload)).into_response(),- Err(RunnerError::CountTokensUnsupported) => error_response(- StatusCode::BAD_REQUEST,- "route has no Anthropic target for token counting",- "invalid_request_error",- "count_tokens_unsupported",- ),- Err(RunnerError::Client(error)) => count_tokens_error(error),- Err(error) => server_error(error.to_string()),- })+ anthropic_error_response(+ match route+ .call_auxiliary(request, AuxiliaryOperation::AnthropicCountTokens)+ .await+ {+ Ok(payload) => (StatusCode::OK, Json(payload)).into_response(),+ Err(RunnerError::AuxiliaryUnsupported) => error_response(+ StatusCode::BAD_REQUEST,+ "route has no Anthropic target for token counting",+ "invalid_request_error",+ "count_tokens_unsupported",+ ),+ Err(RunnerError::Client(error)) => client_error(&error),+ Err(error) => server_error(error.to_string()),+ },+ )+}++async fn openai_responses_input_tokens(+ State(state): State<ServerState>,+ headers: HeaderMap,+ body: std::result::Result<Json<Value>, JsonRejection>,+) -> Response {+ openai_responses_auxiliary(+ state,+ headers,+ body,+ AuxiliaryOperation::ResponsesInputTokens,+ )+ .await}-/// Maps a token-count client failure with the same policy as a routed client call.-fn count_tokens_error(error: LlmClientError) -> Response {- client_error(&error)+async fn openai_responses_compact(+ State(state): State<ServerState>,+ headers: HeaderMap,+ body: std::result::Result<Json<Value>, JsonRejection>,+) -> Response {+ openai_responses_auxiliary(state, headers, body, AuxiliaryOperation::ResponsesCompact).await+}++// Resolves route aliases and configured credentials before calling a model-bearing Responses API.+async fn openai_responses_auxiliary(+ state: ServerState,+ headers: HeaderMap,+ body: std::result::Result<Json<Value>, JsonRejection>,+ operation: AuxiliaryOperation,+) -> Response {+ let body = match llm_json_body(body) {+ Ok(body) => body,+ Err((status, message)) => {+ return render_error_response(+ invalid_body_error(status, message),+ WireFormat::OpenAiResponses,+ );+ }+ };+ let (route, request) = match resolve_route(+ &state,+ metadata_from_headers(headers),+ body,+ WireFormat::OpenAiResponses,+ ) {+ Ok(resolved) => resolved,+ Err(response) => return render_error_response(response, WireFormat::OpenAiResponses),+ };+ let result = route.call_auxiliary(request, operation).await;+ render_error_response(+ match result {+ Ok(payload) => (StatusCode::OK, Json(payload)).into_response(),+ Err(RunnerError::AuxiliaryUnsupported) => error_response(+ StatusCode::BAD_REQUEST,+ "route has no OpenAI Responses target",+ "invalid_request_error",+ "responses_auxiliary_unsupported",+ ),+ Err(RunnerError::Client(error)) => client_error(&error),+ Err(error) => server_error(error.to_string()),+ },+ WireFormat::OpenAiResponses,+ )}async fn handle_endpoint(@@ -1450,6 +1640,9 @@ fn endpoint_listing(has_routing_log: bool) -> String {" POST /v1/messages Anthropic Messages"," POST /v1/responses OpenAI Responses"," POST /v1/messages/count_tokens",+ " POST /v1/responses/input_tokens",+ " POST /v1/responses/compact",+ " ANY unmatched paths optional fallback client"," GET /v1/models configured routes"," GET /v1/stats routing stats"," POST /v1/stats/reset",diff --git a/crates/switchyard-server/tests/server.rs b/crates/switchyard-server/tests/server.rsindex 704be1865..4fca26f02 100644--- a/crates/switchyard-server/tests/server.rs+++ b/crates/switchyard-server/tests/server.rs@@ -11,7 +11,7 @@ use std::sync::Arc;use axum::body::{Body, Bytes};use axum::extract::{DefaultBodyLimit, State};-use axum::http::{HeaderMap, Request as HttpRequest, StatusCode};+use axum::http::{HeaderMap, HeaderValue, Request as HttpRequest, StatusCode, Uri};use axum::response::sse::{Event, Sse};use axum::response::{IntoResponse, Response as HttpResponse};use axum::routing::post;@@ -58,6 +58,12 @@ impl MockUpstream {).route("/capture", post(upstream_redirect_capture)).route("/v1/messages/count_tokens", post(upstream_count_tokens))+ .route(+ "/v1/responses/input_tokens",+ post(upstream_responses_auxiliary),+ )+ .route("/v1/responses/compact", post(upstream_responses_auxiliary))+ .route("/future/provider/endpoint", post(upstream_fallback)).layer(DefaultBodyLimit::disable()).with_state(Arc::clone(&calls));let listener = TcpListener::bind("127.0.0.1:0").await?;@@ -430,6 +436,51 @@ async fn upstream_count_tokens(Json(json!({"input_tokens": 7})).into_response()}+async fn upstream_responses_auxiliary(+ State(calls): State<Arc<Mutex<Vec<Value>>>>,+ uri: Uri,+ headers: HeaderMap,+ Json(body): Json<Value>,+) -> HttpResponse {+ calls.lock().await.push(json!({+ "path": uri.path(),+ "body": body,+ "configured_header": headers.get("x-configured-client").and_then(|value| value.to_str().ok())+ }));+ if uri.path().ends_with("/input_tokens") {+ Json(json!({"input_tokens": 11})).into_response()+ } else {+ Json(json!({"id": "resp_compacted", "object": "response", "output": []})).into_response()+ }+}++async fn upstream_fallback(+ State(calls): State<Arc<Mutex<Vec<Value>>>>,+ headers: HeaderMap,+ Json(body): Json<Value>,+) -> HttpResponse {+ calls.lock().await.push(json!({+ "body": body,+ "authorization": headers.get("authorization").and_then(|value| value.to_str().ok()),+ "end_to_end": headers.get("x-end-to-end").and_then(|value| value.to_str().ok()),+ "configured_secret": headers.contains_key("x-configured-secret"),+ "connection": headers.contains_key("connection"),+ "connection_nominated": headers.contains_key("x-remove-me")+ }));+ let mut response = Json(json!({"input_tokens": 7})).into_response();+ response+ .headers_mut()+ .insert("connection", HeaderValue::from_static("x-upstream-hop"));+ response+ .headers_mut()+ .insert("x-upstream-hop", HeaderValue::from_static("remove"));+ response.headers_mut().insert(+ "x-end-to-end-response",+ HeaderValue::from_static("preserve"),+ );+ response+}+fn random_state(base_url: &str, routes: &[(&str, &[&str])]) -> TestResult<ServerState> {random_state_with_retries(base_url, routes, 0)}@@ -1713,7 +1764,7 @@ response_format_type = "json_object"}#[tokio::test]-async fn count_tokens_forwards_to_configured_anthropic_target() -> TestResult {+async fn model_bearing_auxiliary_endpoints_use_configured_route_targets() -> TestResult {let upstream = MockUpstream::start().await?;let state = load_test_config(&format!(r#"@@ -1723,6 +1774,15 @@ schema_version = 1format = "anthropic_messages"base_url = "{base_url}"+[llm_clients.responses]+format = "openai_responses"+base_url = "{base_url}"+extra_headers = {{ "x-configured-client" = "responses" }}++[targets.responses]+id = "real/responses-model"+llm_client = "responses"+[targets.strong]id = "real/opus"llm_client = "claude"@@ -1734,13 +1794,13 @@ llm_client = "claude"[routes.random]id = "switchyard/random"type = "random"-targets = ["other", "strong"]+targets = ["responses", "other", "strong"]"#,base_url = upstream.base_url))?;let app = build_switchyard_router(state);- let response = send(+ let count_tokens = send(&app,"POST","/v1/messages/count_tokens",@@ -1750,13 +1810,151 @@ targets = ["other", "strong"]})),).await?;- assert_eq!(response.status, StatusCode::OK);- assert_eq!(response.json()?["input_tokens"], 7);+ assert_eq!(count_tokens.status, StatusCode::OK);+ assert_eq!(count_tokens.json()?["input_tokens"], 7);++ let input_tokens = send(+ &app,+ "POST",+ "/v1/responses/input_tokens",+ Some(json!({"model": "switchyard/random", "input": "count me"})),+ )+ .await?;+ assert_eq!(input_tokens.status, StatusCode::OK);+ assert_eq!(input_tokens.json()?["input_tokens"], 11);++ let compact = send(+ &app,+ "POST",+ "/v1/responses/compact",+ Some(json!({"model": "switchyard/random", "input": "compact me"})),+ )+ .await?;+ assert_eq!(compact.status, StatusCode::OK);+ assert_eq!(compact.json()?["id"], "resp_compacted");let calls = upstream.calls.lock().await;- assert_eq!(calls.len(), 1);- // The inbound route name is rewritten to the real upstream model.+ assert_eq!(calls.len(), 3);assert_eq!(calls[0]["model"], "real/opus");+ assert_eq!(+ calls[1],+ json!({+ "path": "/v1/responses/input_tokens",+ "body": {"model": "real/responses-model", "input": "count me"},+ "configured_header": "responses"+ })+ );+ assert_eq!(+ calls[2],+ json!({+ "path": "/v1/responses/compact",+ "body": {"model": "real/responses-model", "input": "compact me"},+ "configured_header": "responses"+ })+ );+ drop(calls);++ let unsupported = random_state(&upstream.base_url, &[(ROUTE_MODEL, &["model/weak"])])?;+ let unsupported = build_switchyard_router(unsupported);+ for (path, body) in [+ (+ "/v1/messages/count_tokens",+ json!({"model": ROUTE_MODEL, "messages": [{"role": "user", "content": "hi"}]}),+ ),+ (+ "/v1/responses/input_tokens",+ json!({"model": ROUTE_MODEL, "input": "count me"}),+ ),+ (+ "/v1/responses/compact",+ json!({"model": ROUTE_MODEL, "input": "compact me"}),+ ),+ ] {+ let response = send(&unsupported, "POST", path, Some(body)).await?;+ assert_eq!(response.status, StatusCode::BAD_REQUEST, "{path}");+ }+ Ok(())+}++#[tokio::test]+async fn fallback_client_forwards_unmatched_requests_and_is_optional() -> TestResult {+ let upstream = MockUpstream::start().await?;+ let state = load_test_config(&format!(+ r#"+schema_version = 1+fallback_client = "fallback"++[llm_clients.fallback]+format = "openai_responses"+base_url = "{base_url}"+extra_headers = {{ "x-configured-secret" = "must-not-forward" }}++[llm_clients.routed]+format = "openai_chat"+base_url = "{base_url}"++[targets.weak]+id = "model/weak"+llm_client = "routed"++[routes.random]+id = "switchyard/random"+type = "passthrough"+target = "weak"+"#,+ base_url = upstream.base_url+ ))?;+ let app = build_switchyard_router(state);++ let response = send_with_headers(+ &app,+ "POST",+ "/future/provider/endpoint?mode=raw",+ Some(json!({+ "model": "provider/model",+ "provider_field": {"nested": true}+ })),+ &[+ ("authorization", "Bearer caller-key"),+ ("connection", "keep-alive, x-remove-me"),+ ("keep-alive", "timeout=5"),+ ("x-remove-me", "remove"),+ ("x-end-to-end", "preserve"),+ ],+ )+ .await?;+ assert_eq!(response.status, StatusCode::OK);+ assert_eq!(response.json()?["input_tokens"], 7);+ assert!(!response.headers.contains_key("connection"));+ assert!(!response.headers.contains_key("x-upstream-hop"));+ assert_eq!(+ response+ .headers+ .get("x-end-to-end-response")+ .and_then(|value| value.to_str().ok()),+ Some("preserve")+ );+ let calls = upstream.calls.lock().await;+ assert_eq!(+ calls.as_slice(),+ &[json!({+ "body": {+ "model": "provider/model",+ "provider_field": {"nested": true}+ },+ "authorization": "Bearer caller-key",+ "end_to_end": "preserve",+ "configured_secret": false,+ "connection": false,+ "connection_nominated": false+ })]+ );+ drop(calls);++ let state = random_state(&upstream.base_url, &[(ROUTE_MODEL, &["model/weak"])])?;+ let app = build_switchyard_router(state);+ let response = send(&app, "POST", "/future/provider/endpoint", None).await?;+ assert_eq!(response.status, StatusCode::NOT_FOUND);Ok(())}@@ -1896,55 +2094,6 @@ target = "openai"Ok(())}-#[tokio::test]-async fn count_tokens_without_anthropic_target_returns_bad_request() -> TestResult {- let upstream = MockUpstream::start().await?;- let state = load_test_config(&format!(- r#"-schema_version = 1--[llm_clients.upstream]-format = "openai_chat"-base_url = "{base_url}"--[targets.weak]-id = "model/weak"-llm_client = "upstream"--[routes.random]-id = "switchyard/random"-type = "random"-targets = ["weak"]-"#,- base_url = upstream.base_url- ))?;- let app = build_switchyard_router(state);-- let response = send(- &app,- "POST",- "/v1/messages/count_tokens",- Some(json!({- "model": "switchyard/random",- "messages": [{"role": "user", "content": "hi"}]- })),- )- .await?;- assert_eq!(response.status, StatusCode::BAD_REQUEST);- // This route has no Anthropic-format target.- assert_eq!(- response.json()?,- json!({- "type": "error",- "error": {- "type": "invalid_request_error",- "message": "route has no Anthropic target for token counting"- }- })- );- Ok(())-}-#[tokio::test]async fn routes_dispatch_and_discovery_endpoints_are_stable() -> TestResult {let (upstream, app) = test_app(&[@@ -3075,55 +3224,6 @@ async fn advisor_route_routing_log_records_classifier_tier() -> TestResult {Ok(())}-#[tokio::test]-async fn advisor_route_count_tokens_uses_executor() -> TestResult {- let upstream = MockUpstream::start().await?;- let state = load_test_config(&format!(- r#"-schema_version = 1--[llm_clients.claude]-format = "anthropic_messages"-base_url = "{base_url}"--[targets.executor]-id = "model/executor"-llm_client = "claude"--[targets.advisor]-id = "model/advisor"-llm_client = "claude"--[routes.gated]-id = "switchyard/advisor"-type = "advisor"-executor_target = "executor"-advisor_target = "advisor"-"#,- base_url = upstream.base_url- ))?;- let app = build_switchyard_router(state);-- let response = send(- &app,- "POST",- "/v1/messages/count_tokens",- Some(json!({- "model": "switchyard/advisor",- "messages": [{"role": "user", "content": "hi"}]- })),- )- .await?;- assert_eq!(response.status, StatusCode::OK);- assert_eq!(response.json()?["input_tokens"], 7);- // The executor is the route's only completion target, so it backs- // count_tokens; the judge-only advisor never does.- let calls = upstream.calls.lock().await;- assert_eq!(calls.len(), 1);- assert_eq!(calls[0]["model"], "model/executor");- Ok(())-}-/// An advisor deployment whose reviewer client never retries, so a down/// advisor hits fail-open after a single attempt (the documented deployment/// posture for the advisor tier).diff --git a/docs/reference/toml_schema.md b/docs/reference/toml_schema.mdindex f4f2f7742..966edd978 100644--- a/docs/reference/toml_schema.md+++ b/docs/reference/toml_schema.md@@ -32,6 +32,13 @@ target = "strong"`schema_version` must be `1`. Table names under `llm_clients`, `targets`, and`routes` are local references; clients send the route's `id` as the model name.+The optional top-level `fallback_client` names an entry under `[llm_clients]`. When set, any HTTP+method and path not implemented by Switchyard is forwarded through that client without translating+the path, query string, body, response, or model identifier. The fallback client does not need a+target. Only its `base_url` is used: caller end-to-end headers are forwarded, hop-by-hop headers are+removed, and configured API keys, extra headers, format, and retries are not applied. When omitted,+unmatched paths return `404`.+`schema_version`, `[targets]`, and `[routes]` must all be present, even when aroute reaches no upstream. A file without a `[targets]` table is rejected with`missing field targets`; an empty `[targets]` table satisfies it.