Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.
Rust
feat(server): add decision endpoint (#456)
nachiketb-nvidia210766446028
diff --git a/crates/switchyard-server/README.md b/crates/switchyard-server/README.mdindex 65c5083b6..75d1b1fd1 100644--- a/crates/switchyard-server/README.md+++ b/crates/switchyard-server/README.md@@ -128,6 +128,7 @@ documented in [Stage-Router Routing](../../docs/routing_algorithms/stage_router_| `POST` | `/v1/chat/completions` | OpenAI Chat Completions || `POST` | `/v1/messages` | Anthropic Messages || `POST` | `/v1/responses` | OpenAI Responses |+| `POST` | `/v1/decision` | Resolve selected and fallback targets without a post-routing answer call || `POST` | `/v1/messages/count_tokens` | Token count from a route's Anthropic target || `GET` | `/v1/models` | Routes served by this deployment || `GET` | `/v1/stats` | Per-model usage plus curated algorithm stats |@@ -139,6 +140,14 @@ Requests name a route by its `id`, so `POST /v1/chat/completions` with `"model":routes through the `[routes.general]` entry above. Any of the three request formats can address anyroute, and the server translates between them.+`POST /v1/decision` accepts `{"input_format": "openai_chat", "request": {...}}`, where the+nested request names the route in `model`. It executes required classifier or judge calls, then+returns the selected target and ordered fallbacks with their model, format, base URL, and+`extra_body`. It does not make a post-routing answer call. Routing-time calls still execute, and+response-dependent algorithms such as escalation and advisor routing may produce an answer while+deciding. When they do, the endpoint includes the buffered answer as `response`, encoded in+`input_format`; otherwise the field is omitted.+For `stage_router`, `algorithm_stats.stage_router` groups routing decisions by source and semantictarget and summarizes its score, confidence, and input-dimension histograms. These values resetwith `/v1/stats/reset`; the process-lifetime counters on `/metrics` remain cumulative.diff --git a/crates/switchyard-server/src/config.rs b/crates/switchyard-server/src/config.rsindex a78695cd5..420ac5e2b 100644--- a/crates/switchyard-server/src/config.rs+++ b/crates/switchyard-server/src/config.rs@@ -20,7 +20,7 @@ use switchyard_llm_client::{Backend, ClientRouter, DEFAULT_MAX_RETRIES, HttpBackendConfig, ModelConfig,TranslatingLlmClient,};-use switchyard_protocol::{ModelId, RoutedLlmClient};+use switchyard_protocol::{ModelId, RoutedLlmClient, WireFormat};use crate::{CallerAuthKind, CountTokensTarget, ModelCapabilities, ServerError, ServerResult, ServerState,@@ -44,22 +44,31 @@ pub fn load_server_state(path: impl AsRef<Path>) -> ServerResult<ServerState> {}fn server_state_from_toml(toml: &str) -> ServerResult<ServerState> {- let config: ServerConfig = toml::from_str(toml)- .map_err(|error| ServerError::new(format!("failed to parse TOML: {error}")))?;- config.build()+ let config: Arc<ServerConfig> = Arc::new(+ toml::from_str(toml)+ .map_err(|error| ServerError::new(format!("failed to parse TOML: {error}")))?,+ );+ let state = config.build()?;+ Ok(state.with_config(config))}#[derive(Debug, Deserialize)]#[serde(deny_unknown_fields)]-struct ServerConfig {+pub(crate) struct ServerConfig {schema_version: u32,#[serde(default)]- llm_clients: BTreeMap<String, LlmClientConfig>,- targets: BTreeMap<String, TargetConfig>,+ pub(crate) llm_clients: BTreeMap<String, LlmClientConfig>,+ pub(crate) targets: BTreeMap<String, TargetConfig>,routes: BTreeMap<String, RouteConfig>,}impl ServerConfig {+ pub(crate) fn routing_target_names(&self, route_name: &str) -> Option<Vec<&str>> {+ self.routes+ .get(route_name)+ .map(RouteConfig::routing_target_names)+ }+fn build(&self) -> ServerResult<ServerState> {if self.schema_version != SUPPORTED_SCHEMA_VERSION {return Err(ServerError::new(format!(@@ -94,6 +103,11 @@ impl ServerConfig {for (route_name, config) in &self.routes {validate_value("route name", route_name)?;validate_value(&format!("route {route_name} id"), config.id())?;+ for target_name in config.callable_target_names() {+ self.targets.get(target_name).ok_or_else(|| {+ ServerError::new(format!("route references unknown target {target_name}"))+ })?;+ }let capabilities = config.capabilities();if capabilities.context_window == Some(0) {return Err(ServerError::new(format!(@@ -110,6 +124,7 @@ impl ServerConfig {caller_auth,capabilities,count_tokens_target,+ Some(route_name.clone()),));}ServerState::new_with_capabilities(routes)@@ -246,9 +261,9 @@ fn count_tokens_priority(target_name: &str, model_id: &ModelId) -> usize {#[derive(Debug, Deserialize)]#[serde(deny_unknown_fields)]-struct LlmClientConfig {- format: ClientFormat,- base_url: String,+pub(crate) struct LlmClientConfig {+ pub(crate) format: ClientFormat,+ pub(crate) base_url: String,api_key_env: Option<String>,#[serde(default)]forward_auth: bool,@@ -260,15 +275,15 @@ struct LlmClientConfig {#[derive(Debug, Deserialize)]#[serde(deny_unknown_fields)]-struct TargetConfig {- id: ModelId,- llm_client: String,+pub(crate) struct TargetConfig {+ pub(crate) id: ModelId,+ pub(crate) llm_client: String,#[serde(default)]- extra_body: BTreeMap<String, Value>,+ pub(crate) extra_body: BTreeMap<String, Value>,}#[derive(Clone, Copy, Debug, Deserialize)]-enum ClientFormat {+pub(crate) enum ClientFormat {#[serde(rename = "openai_chat")]OpenAiChat,#[serde(rename = "openai_responses")]@@ -278,6 +293,14 @@ enum ClientFormat {}impl ClientFormat {+ pub(crate) const fn wire_format(self) -> WireFormat {+ match self {+ Self::OpenAiChat => WireFormat::OpenAiChat,+ Self::OpenAiResponses => WireFormat::OpenAiResponses,+ Self::AnthropicMessages => WireFormat::AnthropicMessages,+ }+ }+const fn caller_auth_kind(self) -> CallerAuthKind {match self {Self::AnthropicMessages => CallerAuthKind::Anthropic,@@ -1493,9 +1516,10 @@ classifier_magic = true];for (toml, expected) in cases {+ let error = error_message(&toml);assert!(- error_message(&toml).contains(expected),- "expected error containing {expected}"+ error.contains(expected),+ "expected error containing {expected}, got {error}");}}diff --git a/crates/switchyard-server/src/lib.rs b/crates/switchyard-server/src/lib.rsindex e302e29dd..483c1f6a6 100644--- a/crates/switchyard-server/src/lib.rs+++ b/crates/switchyard-server/src/lib.rs@@ -32,9 +32,9 @@ use axum::response::{IntoResponse, Response};use axum::routing::{get, post};use axum::{Extension, Json, Router};use axum_server::tls_rustls::RustlsConfig;-use libsy::{Algorithm, LibsyError};+use libsy::{Algorithm, CallModel, LibsyError, RoutingOutcome, drive};use parking_lot::Mutex;-use serde::Deserialize;+use serde::{Deserialize, Serialize};use serde_json::{Value, json};use switchyard_llm_client::{ClientRouter, RunObservation, RunObserver, TranslatingLlmClient};use switchyard_protocol::{LlmClientError, Metadata, ModelId, Request, Usage};@@ -42,8 +42,9 @@ use tokio::net::{TcpListener, TcpSocket};use tokio::task;use tracing::{Instrument, Level};-use switchyard_translation::{WireFormat, decode_request};+use switchyard_translation::{WireFormat, decode_request, encode_aggregated_response};+use crate::config::ServerConfig;use crate::response::into_http_response;use crate::stats::{StatsAccumulator, StatsSnapshot, prefix_probe, tracking_enabled_from_env};@@ -112,6 +113,32 @@ struct RouteEntry {caller_auth: Option<CallerAuthKind>,capabilities: ModelCapabilities,count_tokens_target: Option<CountTokensTarget>,+ config_name: Option<String>,+}++/// Decision result using the same selected/fallback order as libsy.+#[derive(Serialize)]+struct DecisionResponse<'a> {+ selected: DecisionTargetResponse<'a>,+ fallbacks: Vec<DecisionTargetResponse<'a>>,+ #[serde(skip_serializing_if = "Option::is_none")]+ response: Option<Value>,+}++/// Borrowed response view over one target's existing configuration.+#[derive(Serialize)]+struct DecisionTargetResponse<'a> {+ target: &'a str,+ model: &'a ModelId,+ llm_client: DecisionLlmClientResponse<'a>,+ extra_body: &'a BTreeMap<String, Value>,+}++/// Non-secret client settings needed to call a selected model.+#[derive(Serialize)]+struct DecisionLlmClientResponse<'a> {+ format: WireFormat,+ base_url: &'a str,}/// Caller credential family required by forwarded-auth backends.@@ -158,6 +185,7 @@ impl CountTokensTarget {#[derive(Clone)]pub struct ServerState {routes: Arc<BTreeMap<ModelId, RouteEntry>>,+ config: Option<Arc<ServerConfig>>,metrics: prometheus::Registry,stats: StatsAccumulator,routing_log: Option<SharedRoutingLog>,@@ -212,6 +240,7 @@ impl ServerState {None,ModelCapabilities::default(),None,+ None,)}))}@@ -225,12 +254,20 @@ impl ServerState {Option<CallerAuthKind>,ModelCapabilities,Option<CountTokensTarget>,+ Option<String>,),>,) -> ServerResult<Self> {let mut entries = BTreeMap::new();- for (model, algorithm, target_clients, caller_auth, capabilities, count_tokens_target) in- routes+ for (+ model,+ algorithm,+ target_clients,+ caller_auth,+ capabilities,+ count_tokens_target,+ config_name,+ ) in routes{let model = ModelId::from(model.trim());if model.is_empty() {@@ -242,6 +279,7 @@ impl ServerState {caller_auth,capabilities,count_tokens_target,+ config_name,};if entries.insert(model.clone(), entry).is_some() {return Err(ServerError::new(format!("duplicate route model {model}")));@@ -257,6 +295,7 @@ impl ServerState {);Ok(Self {routes: Arc::new(entries),+ config: None,metrics,stats,routing_log: None,@@ -264,6 +303,12 @@ impl ServerState {})}+ /// Retains the validated configuration used to describe decision results.+ fn with_config(mut self, config: Arc<ServerConfig>) -> Self {+ self.config = Some(config);+ self+ }+/// Enables durable per-request routing records at `path`.pub fn with_routing_log(mut self, path: impl Into<PathBuf>) -> ServerResult<Self> {self.routing_log = Some(SharedRoutingLog::new(path.into())?);@@ -285,6 +330,42 @@ impl ServerState {fn route_for_model(&self, model: &str) -> Option<&RouteEntry> {self.routes.get(model)}++ /// Resolves the routing outcome through the target names configured for its route.+ fn decision_response<'a>(+ &'a self,+ route: &'a RouteEntry,+ outcome: &RoutingOutcome,+ response: Option<Value>,+ ) -> Option<DecisionResponse<'a>> {+ let config = self.config.as_ref()?;+ let target_names = config.routing_target_names(route.config_name.as_ref()?)?;+ let resolve = |model: &ModelId| {+ let (target_name, target) = target_names+ .iter()+ .filter_map(|name| config.targets.get_key_value(*name))+ .find(|(_, target)| target.id == *model)?;+ let client = config.llm_clients.get(&target.llm_client)?;+ Some(DecisionTargetResponse {+ target: target_name,+ model: &target.id,+ llm_client: DecisionLlmClientResponse {+ format: client.format.wire_format(),+ base_url: &client.base_url,+ },+ extra_body: &target.extra_body,+ })+ };+ Some(DecisionResponse {+ selected: resolve(&outcome.selected_model_id)?,+ fallbacks: outcome+ .fallback_models+ .iter()+ .map(resolve)+ .collect::<Option<Vec<_>>>()?,+ response,+ })+ }}/// Runtime options shared by server entry points.@@ -503,6 +584,7 @@ pub fn build_switchyard_router(state: ServerState) -> Router {.route("/v1/chat/completions", post(openai_chat_completions)).route("/v1/messages", post(anthropic_messages)).route("/v1/responses", post(openai_responses))+ .route("/v1/decision", post(decision)).route("/v1/messages/count_tokens", post(anthropic_count_tokens)).route("/v1/models", get(models)).route("/v1/stats", get(get_stats))@@ -564,6 +646,92 @@ async fn openai_responses(handle_endpoint(state, started, headers, body, WireFormat::OpenAiResponses).await}+/// One provider request submitted for a routing decision without an answer-model call.+#[derive(Deserialize)]+#[serde(deny_unknown_fields)]+struct DecisionEndpointRequest {+ input_format: WireFormat,+ request: Value,+}++/// Selects a target while still allowing the algorithm's classifier and judge calls.+async fn decision(+ State(state): State<ServerState>,+ headers: HeaderMap,+ body: std::result::Result<Json<DecisionEndpointRequest>, JsonRejection>,+) -> Response {+ let body = match body {+ Ok(Json(body)) if body.request.is_object() => body,+ Ok(_) => {+ return invalid_body_error(StatusCode::BAD_REQUEST, "request must be a JSON object");+ }+ Err(error) => {+ return invalid_body_error(+ error.status(),+ format!("Request body must be valid JSON: {error}"),+ );+ }+ };+ let input_format = body.input_format;+ let (route, request) = match resolve_route(+ &state,+ metadata_from_headers(headers),+ body.request,+ input_format,+ ) {+ Ok(resolved) => resolved,+ Err(response) => return response,+ };+ let mut outcome = match run_decision_only(route, request).await {+ Ok(outcome) => outcome,+ Err(error) => return algorithm_error(error),+ };+ let response = match outcome.response.take() {+ Some(response) => {+ let aggregate = match response.llm_response.into_agg().await {+ Ok(aggregate) => aggregate,+ Err(error) => return client_error(&error),+ };+ match encode_aggregated_response(+ &aggregate,+ input_format,+ Some(outcome.selected_model_id.as_str()),+ ) {+ Ok(response) => Some(response),+ Err(error) => return server_error(error.to_string()),+ }+ }+ None => None,+ };+ match state.decision_response(route, &outcome, response) {+ Some(response) => Json(response).into_response(),+ None => {+ server_error("routing outcome contains a model with no callable target configuration")+ }+ }+}++/// Completes routing-time calls and returns the outcome without serving its answer target.+async fn run_decision_only(route: &RouteEntry, request: Request) -> libsy::Result<RoutingOutcome> {+ drive(Arc::clone(&route.algorithm), request, |call| {+ serve_decision_dependency(route, call)+ })+ .await+}++/// Serves a classifier or judge call that the decision depends on.+async fn serve_decision_dependency(route: &RouteEntry, call: CallModel) -> libsy::Result<()> {+ let model = call.models.first().cloned().ok_or(LibsyError::NoTargets)?;+ let result = match route.target_clients.route(&model) {+ Ok(client) => client+ .call(call.request.clone())+ .await+ .map_err(|source| LibsyError::client_call(model, source)),+ Err(source) => Err(LibsyError::client_call(model, source)),+ };+ call.respond(result)+}+/// Anthropic token counting against the route's explicitly configured target.async fn anthropic_count_tokens(State(state): State<ServerState>,diff --git a/crates/switchyard-server/tests/server.rs b/crates/switchyard-server/tests/server.rsindex 0a97f90e3..21b1e46da 100644--- a/crates/switchyard-server/tests/server.rs+++ b/crates/switchyard-server/tests/server.rs@@ -956,6 +956,132 @@ base_threshold = 0.5Ok(())}+/// Decision-only routing returns callable metadata and preserves any answer produced while routing.+#[tokio::test]+async fn decision_returns_callable_target_and_routing_answer() -> TestResult {+ let judge_upstream = MockUpstream::start().await?;+ let model_upstream = MockUpstream::start().await?;+ let state = load_test_config(&format!(+ r#"+schema_version = 1++[llm_clients.judge_provider]+format = "openai_chat"+base_url = "{judge_url}"++[llm_clients.model_provider]+format = "openai_chat"+base_url = "{model_url}"++[targets.judge]+id = "model/classifier"+llm_client = "judge_provider"++[targets.quality]+id = "model/strong"+llm_client = "model_provider"++[targets.economy]+id = "model/weak"+llm_client = "model_provider"+extra_body = {{ service_tier = "priority" }}++[routes.classify]+id = "switchyard/classify"+type = "llm_classifier"+classifier_target = "judge"+strong_target = "quality"+weak_target = "economy"+base_threshold = 0.5++[routes.escalation]+id = "switchyard/escalation"+type = "llm_classifier"+mode = "escalation"+classifier_target = "judge"+strong_target = "quality"+weak_target = "economy"+escalation = {{ confirmations = 1 }}+"#,+ judge_url = judge_upstream.base_url,+ model_url = model_upstream.base_url,+ ))?;+ let app = build_switchyard_router(state);++ let response = send(+ &app,+ "POST",+ "/v1/decision",+ Some(json!({+ "input_format": "openai_chat",+ "request": {+ "model": "switchyard/classify",+ "messages": [{"role": "user", "content": "bounded task"}]+ }+ })),+ )+ .await?;++ assert_eq!(response.status, StatusCode::OK);+ assert_eq!(+ response.json()?,+ json!({+ "selected": {+ "target": "economy",+ "model": "model/weak",+ "llm_client": {+ "format": "openai_chat",+ "base_url": model_upstream.base_url,+ },+ "extra_body": {"service_tier": "priority"},+ },+ "fallbacks": [{+ "target": "quality",+ "model": "model/strong",+ "llm_client": {+ "format": "openai_chat",+ "base_url": model_upstream.base_url,+ },+ "extra_body": {},+ }],+ })+ );+ assert_eq!(+ judge_upstream.models().await,+ vec!["model/classifier".to_string()]+ );+ assert!(model_upstream.models().await.is_empty());++ judge_upstream.calls.lock().await.clear();+ model_upstream.calls.lock().await.clear();+ let response = send(+ &app,+ "POST",+ "/v1/decision",+ Some(json!({+ "input_format": "openai_chat",+ "request": {+ "model": "switchyard/escalation",+ "messages": [{"role": "user", "content": "bounded task"}]+ }+ })),+ )+ .await?;++ assert_eq!(response.status, StatusCode::OK);+ let response = response.json()?;+ assert_eq!(response["selected"]["target"], "economy");+ assert_eq!(response["fallbacks"], json!([]));+ assert_eq!(response["response"]["model"], "model/weak");+ assert_eq!(+ response["response"]["choices"][0]["message"]["content"],+ "ok"+ );+ assert_eq!(model_upstream.models().await, ["model/weak"]);+ assert_eq!(judge_upstream.models().await, ["model/classifier"]);+ Ok(())+}+/// A critical tool error must reach the stage router's signal scorer, which reads/// the decoded conversation. The endpoint records no inbound wire format, so a/// scorer that parsed the raw body instead would find nothing and route every turn